petermolnar/nasg
1
0
Fork 0
Code Issues Pull requests 5 Projects Releases Packages Wiki Activity Actions

fixing missing active page indicator in header; moved db classes to shared; fixed webmention parse error if content is missing; is_uptodate of singular based on last comment time, so singular gets re-rendered, but category not

Browse source
This commit is contained in:
Peter Molnar 2017-10-30 09:24:46 +00:00
parent e5518ba4a1
commit 0fc792fe00
6 changed files with 273 additions and 288 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
archive.py
View file

@ -14,7 +14,6 @@ from requests_oauthlib import OAuth2Session
from requests_oauthlib import oauth2_session
from oauthlib.oauth2 import BackendApplicationClient
import db
import shared
class Favs(object):
@ -734,7 +733,7 @@ class Oauth1Flow(object):
self.service = service
self.key = shared.config.get("api_%s" % service, 'api_key')
self.secret = shared.config.get("api_%s" % service, 'api_secret')
self.tokendb = db.TokenDB()
self.tokendb = shared.TokenDB()
self.t = self.tokendb.get_service(self.service)
self.oauth_init()

249
db.py
View file

@ -1,249 +0,0 @@
import os
import json
import sqlite3
import glob
import shared
import logging
class TokenDB(object):
def __init__(self, uuid='tokens'):
self.db = shared.config.get('var', 'tokendb')
self.tokens = {}
self.refresh()
def refresh(self):
self.tokens = {}
if os.path.isfile(self.db):
with open(self.db, 'rt') as f:
self.tokens = json.loads(f.read())
def save(self):
with open(self.db, 'wt') as f:
f.write(json.dumps(
self.tokens, indent=4, sort_keys=True
))
def get_token(self, token):
return self.tokens.get(token, None)
def get_service(self, service):
token = self.tokens.get(service, None)
return token
def set_service(self, service, tokenid):
self.tokens.update({
service: tokenid
})
self.save()
def update_token(self,
token,
oauth_token_secret=None,
access_token=None,
access_token_secret=None,
verifier=None):
t = self.tokens.get(token, {})
if oauth_token_secret:
t.update({
'oauth_token_secret': oauth_token_secret
})
if access_token:
t.update({
'access_token': access_token
})
if access_token_secret:
t.update({
'access_token_secret': access_token_secret
})
if verifier:
t.update({
'verifier': verifier
})
self.tokens.update({
token: t
})
self.save()
def clear(self):
self.tokens = {}
self.save()
def clear_service(self, service):
t = self.tokens.get(service)
if t:
del(self.tokens[t])
del(self.tokens[service])
self.save()
class SearchDB(object):
tmplfile = 'Search.html'
def __init__(self):
self.db = sqlite3.connect(
"%s" % shared.config.get('var', 'searchdb')
)
cursor = self.db.cursor()
cursor.execute('''CREATE VIRTUAL TABLE IF NOT EXISTS data USING FTS5(
id,
corpus,
mtime,
url,
category,
title
)''')
self.db.commit()
def __exit__(self):
self.finish()
def finish(self):
self.db.close()
def append(self, id, corpus, mtime, url, category, title):
mtime = int(mtime)
logging.debug("adding %s to searchdb", id)
cursor = self.db.cursor()
cursor.execute('''DELETE FROM data WHERE id=?''', (id,))
cursor.execute('''INSERT OR IGNORE INTO data (id, corpus, mtime, url, category, title) VALUES (?,?,?,?,?,?);''', (
id,
corpus,
mtime,
url,
category,
title
))
self.db.commit()
def is_uptodate(self, fname, mtime):
mtime = int(mtime)
ret = {}
cursor = self.db.cursor()
cursor.execute('''SELECT mtime
FROM data
WHERE id = ? AND mtime = ?''',
(fname,mtime)
)
rows = cursor.fetchall()
if len(rows):
logging.debug("%s is up to date in searchdb", fname)
return True
logging.debug("%s is out of date in searchdb", fname)
return False
def search_by_query(self, query):
ret = {}
cursor = self.db.cursor()
cursor.execute('''SELECT
id, category, url, title, highlight(data, 0, '<strong>', '</strong>') corpus
FROM data
WHERE data MATCH ?
ORDER BY category, rank;''', (query,))
rows = cursor.fetchall()
for r in rows:
r = {
'id': r[0],
'category': r[1],
'url': r[2],
'title': r[3],
'txt': r[4],
}
category = r.get('category')
if category not in ret:
ret.update({category: {}})
maybe_fpath = os.path.join(
shared.config.get('dirs', 'content'),
category,
"%s.*" % r.get('id')
)
#fpath = glob.glob(maybe_fpath).pop()
ret.get(category).update({
r.get('id'): {
#'fpath': fpath,
'url': r.get('url'),
'title': r.get('title'),
'txt': r.get('txt')
}
})
return ret
def cli(self, query):
results = self.search_by_query(query)
for c, items in sorted(results.items()):
print("%s:" % c)
for fname, data in sorted(items.items()):
print(" %s" % data.get('fpath'))
print(" %s" % data.get('url'))
print("")
def html(self, query):
tmplvars = {
'results': self.search_by_query(query),
'term': query
}
return shared.j2.get_template(self.tmplfile).render(tmplvars)
class WebmentionQueue(object):
def __init__(self):
self.db = sqlite3.connect(
"%s" % shared.config.get('var', 'webmentiondb')
)
cursor = self.db.cursor()
cursor.execute('''
CREATE TABLE IF NOT EXISTS `queue` (
`id` INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE,
`timestamp` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
`source` TEXT NOT NULL,
`target` TEXT NOT NULL,
`status` INTEGER NOT NULL DEFAULT 0,
`mtime` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
''')
self.db.commit()
def __exit__(self):
self.finish()
def finish(self):
self.db.close()
def queue(self, source, target):
cursor = self.db.cursor()
cursor.execute(
'''INSERT INTO queue (source,target) VALUES (?,?);''', (
source,
target
)
)
self.db.commit()
def get_queued(self, fname=None):
logging.debug('getting queued webmentions for %s', fname)
ret = []
cursor = self.db.cursor()
cursor.execute('''SELECT * FROM queue WHERE target LIKE ? AND status = 0''', ('%'+fname+'%',))
rows = cursor.fetchall()
for r in rows:
ret.append({
'id': r[0],
'dt': r[1],
'source': r[2],
'target': r[3],
})
return ret
def entry_done(self, id):
logging.debug('setting %s webmention to done', id)
cursor = self.db.cursor()
cursor.execute("UPDATE queue SET status = 1 where ID=?", (id,))
self.db.commit()

42
nasg.py
View file

@ -22,7 +22,6 @@ import wand.image
from emoji import UNICODE_EMOJI
import shared
import db
from pprint import pprint
@ -206,22 +205,6 @@ class Category(NoDupeContainer):
url = '/'
return url
#def url_paged(self, page=1, feed=False):
#x = '/'
#if self.name:
#x = "%s%s/%s" % (
#x,
#self.taxonomy,
#self.name,
#)
#if page == 1 and feed:
#x = "%s/%s/" % (x, self.feeddir)
#else:
#x = "%s/%s/%s/" % (x, self.pagedir, "%s" % page)
#return x
def path_paged(self, page=1, feed=False):
x = shared.config.get('common', 'build')
@ -309,6 +292,7 @@ class Singular(object):
logging.debug("initiating singular object from %s", fpath)
self.fpath = fpath
self.mtime = os.path.getmtime(self.fpath)
self.stime = self.mtime
self.fname, self.fext = os.path.splitext(os.path.basename(self.fpath))
self.category = os.path.basename(os.path.dirname(self.fpath))
self._images = NoDupeContainer()
@ -333,7 +317,7 @@ class Singular(object):
# TODO this should be async
def process_webmentions(self):
wdb = db.WebmentionQueue()
wdb = shared.WebmentionQueue()
queued = wdb.get_queued(self.url)
for incoming in queued:
wm = Webmention(
@ -358,7 +342,7 @@ class Singular(object):
if not os.path.isfile(self.htmlfile):
return False
mtime = os.path.getmtime(self.htmlfile)
if mtime == self.mtime:
if mtime >= self.stime:
return True
return False
@ -397,8 +381,8 @@ class Singular(object):
cfiles = [*cfiles, *maybe]
for cpath in cfiles:
cmtime = os.path.getmtime(cpath)
if cmtime > self.mtime:
self.mtime = cmtime
if cmtime > self.stime:
self.stime = cmtime
c = Comment(cpath)
comments.append(c.mtime, c)
@ -1089,6 +1073,8 @@ class Webmention(object):
def run(self):
self._fetch()
if 'data' not in self._source:
return
self._save()
@property
@ -1115,9 +1101,15 @@ class Webmention(object):
@property
def content(self):
return shared.Pandoc('html').convert(
self._source.get('data').get('content').get('html')
)
if 'content' not in self._source.get('data'):
return ''
elif 'html' in self._source.get('data').get('content'):
what = self._source.get('data').get('content').get('html')
elif 'text' in self._source.get('data').get('content'):
what = self._source.get('data').get('content').get('text')
else:
return ''
return shared.Pandoc('html').convert(what)
@property
def fname(self):
@ -1214,7 +1206,7 @@ def build():
loop = asyncio.get_event_loop()
tasks = []
content = Content()
sdb = db.SearchDB()
sdb = shared.SearchDB()
magic = MagicPHP()
collector_front = Category()

8
router.py
View file

@ -5,11 +5,9 @@
from sanic import Sanic
import sanic.response
import logging
import db
import shared
import validators
import urllib.parse
import requests
import shared
if __name__ == '__main__':
logging_format = "[%(asctime)s] %(process)d-%(levelname)s "
@ -26,7 +24,7 @@ if __name__ == '__main__':
# since I'm running this from systemctl it already goes into syslog
app = Sanic('router', log_config=None)
# this is ok to be read-only
sdb = db.SearchDB()
sdb = shared.SearchDB()
@app.route("/oauth1", methods=["GET"])
@ -78,7 +76,7 @@ if __name__ == '__main__':
# it is unfortunate that I need to init this every time, but
# otherwise it'll become read-only for reasons I'm yet to grasp
# the actual parsing will be done at site generation time
wdb = db.WebmentionQueue()
wdb = shared.WebmentionQueue()
wdb.queue(source,target)
# telegram notification, if set

251
shared.py
View file

@ -218,6 +218,251 @@ class ExifTool(CMDLine):
return exif
class TokenDB(object):
def __init__(self, uuid='tokens'):
self.db = config.get('var', 'tokendb')
self.tokens = {}
self.refresh()
def refresh(self):
self.tokens = {}
if os.path.isfile(self.db):
with open(self.db, 'rt') as f:
self.tokens = json.loads(f.read())
def save(self):
with open(self.db, 'wt') as f:
f.write(json.dumps(
self.tokens, indent=4, sort_keys=True
))
def get_token(self, token):
return self.tokens.get(token, None)
def get_service(self, service):
token = self.tokens.get(service, None)
return token
def set_service(self, service, tokenid):
self.tokens.update({
service: tokenid
})
self.save()
def update_token(self,
token,
oauth_token_secret=None,
access_token=None,
access_token_secret=None,
verifier=None):
t = self.tokens.get(token, {})
if oauth_token_secret:
t.update({
'oauth_token_secret': oauth_token_secret
})
if access_token:
t.update({
'access_token': access_token
})
if access_token_secret:
t.update({
'access_token_secret': access_token_secret
})
if verifier:
t.update({
'verifier': verifier
})
self.tokens.update({
token: t
})
self.save()
def clear(self):
self.tokens = {}
self.save()
def clear_service(self, service):
t = self.tokens.get(service)
if t:
del(self.tokens[t])
del(self.tokens[service])
self.save()
class SearchDB(object):
tmplfile = 'Search.html'
def __init__(self):
self.db = sqlite3.connect(
"%s" % config.get('var', 'searchdb')
)
cursor = self.db.cursor()
cursor.execute('''CREATE VIRTUAL TABLE IF NOT EXISTS data USING FTS5(
id,
corpus,
mtime,
url,
category,
title
)''')
self.db.commit()
def __exit__(self):
self.finish()
def finish(self):
self.db.close()
def append(self, id, corpus, mtime, url, category, title):
mtime = int(mtime)
logging.debug("adding %s to searchdb", id)
cursor = self.db.cursor()
cursor.execute('''DELETE FROM data WHERE id=?''', (id,))
cursor.execute('''INSERT OR IGNORE INTO data (id, corpus, mtime, url, category, title) VALUES (?,?,?,?,?,?);''', (
id,
corpus,
mtime,
url,
category,
title
))
self.db.commit()
def is_uptodate(self, fname, mtime):
mtime = int(mtime)
ret = {}
cursor = self.db.cursor()
cursor.execute('''SELECT mtime
FROM data
WHERE id = ? AND mtime = ?''',
(fname,mtime)
)
rows = cursor.fetchall()
if len(rows):
logging.debug("%s is up to date in searchdb", fname)
return True
logging.debug("%s is out of date in searchdb", fname)
return False
def search_by_query(self, query):
ret = {}
cursor = self.db.cursor()
cursor.execute('''SELECT
id, category, url, title, snippet(data, 1, '', '', '[...]', 24)
FROM data
WHERE data MATCH ?
ORDER BY category, rank;''', (query,))
rows = cursor.fetchall()
for r in rows:
r = {
'id': r[0],
'category': r[1],
'url': r[2],
'title': r[3],
'txt': r[4],
}
category = r.get('category')
if category not in ret:
ret.update({category: {}})
maybe_fpath = os.path.join(
config.get('dirs', 'content'),
category,
"%s.*" % r.get('id')
)
#fpath = glob.glob(maybe_fpath).pop()
ret.get(category).update({
r.get('id'): {
#'fpath': fpath,
'url': r.get('url'),
'title': r.get('title'),
'txt': r.get('txt')
}
})
return ret
def cli(self, query):
results = self.search_by_query(query)
for c, items in sorted(results.items()):
print("%s:" % c)
for fname, data in sorted(items.items()):
print(" %s" % data.get('fpath'))
print(" %s" % data.get('url'))
print("")
def html(self, query):
tmplvars = {
'results': self.search_by_query(query),
'term': query
}
return j2.get_template(self.tmplfile).render(tmplvars)
class WebmentionQueue(object):
def __init__(self):
self.db = sqlite3.connect(
"%s" % config.get('var', 'webmentiondb')
)
cursor = self.db.cursor()
cursor.execute('''
CREATE TABLE IF NOT EXISTS `queue` (
`id` INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE,
`timestamp` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
`source` TEXT NOT NULL,
`target` TEXT NOT NULL,
`status` INTEGER NOT NULL DEFAULT 0,
`mtime` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
''')
self.db.commit()
def __exit__(self):
self.finish()
def finish(self):
self.db.close()
def queue(self, source, target):
cursor = self.db.cursor()
cursor.execute(
'''INSERT INTO queue (source,target) VALUES (?,?);''', (
source,
target
)
)
self.db.commit()
def get_queued(self, fname=None):
logging.debug('getting queued webmentions for %s', fname)
ret = []
cursor = self.db.cursor()
cursor.execute('''SELECT * FROM queue WHERE target LIKE ? AND status = 0''', ('%'+fname+'%',))
rows = cursor.fetchall()
for r in rows:
ret.append({
'id': r[0],
'dt': r[1],
'source': r[2],
'target': r[3],
})
return ret
def entry_done(self, id):
logging.debug('setting %s webmention to done', id)
cursor = self.db.cursor()
cursor.execute("UPDATE queue SET status = 1 where ID=?", (id,))
self.db.commit()
def __expandconfig():
c = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation(),
@ -275,14 +520,14 @@ def __setup_sitevars():
def notify(msg):
# telegram notification, if set
if not shared.config.has_section('api_telegram'):
if not config.has_section('api_telegram'):
return
url = "https://api.telegram.org/bot%s/sendMessage" % (
shared.config.get('api_telegram', 'api_token')
config.get('api_telegram', 'api_token')
)
data = {
'chat_id': shared.config.get('api_telegram', 'chat_id'),
'chat_id': config.get('api_telegram', 'chat_id'),
'text': msg
}
# fire and forget

8
templates/block_header_close.html
View file

@ -19,7 +19,7 @@
</a>
</li>
{% set cssclass = '' %}
{% if (post is defined and post.category == 'photo' ) or ( taxonomy is defined and taxonomy.slug == 'photo' ) %}
{% if (post is defined and post.category == 'photo' ) or ( taxonomy is defined and taxonomy.name == 'photo' ) %}
{% set cssclass = 'active' %}
{% endif %}
<li>
@ -29,7 +29,7 @@
</a>
</li>
{% set cssclass = '' %}
{% if (post is defined and post.category == 'journal' ) or ( taxonomy is defined and taxonomy.slug == 'journal' ) %}
{% if (post is defined and post.category == 'journal' ) or ( taxonomy is defined and taxonomy.name == 'journal' ) %}
{% set cssclass = 'active' %}
{% endif %}
<li>
@ -39,7 +39,7 @@
</a>
</li>
{% set cssclass = '' %}
{% if (post is defined and post.category == 'article' ) or ( taxonomy is defined and taxonomy.slug == 'article' ) %}
{% if (post is defined and post.category == 'article' ) or ( taxonomy is defined and taxonomy.name == 'article' ) %}
{% set cssclass = 'active' %}
{% endif %}
<li>
@ -49,7 +49,7 @@
</a>
</li>
{% set cssclass = '' %}
{% if (post is defined and post.category == 'note' ) or ( taxonomy is defined and taxonomy.slug == 'note' ) %}
{% if (post is defined and post.category == 'note' ) or ( taxonomy is defined and taxonomy.name == 'note' ) %}
{% set cssclass = 'active' %}
{% endif %}
<li>