From 0fc792fe0050a7702d00d13a04ef975c88f1ea61 Mon Sep 17 00:00:00 2001 From: Peter Molnar Date: Mon, 30 Oct 2017 09:24:46 +0000 Subject: [PATCH] fixing missing active page indicator in header; moved db classes to shared; fixed webmention parse error if content is missing; is_uptodate of singular based on last comment time, so singular gets re-rendered, but category not --- archive.py | 3 +- db.py | 249 ----------------------------- nasg.py | 42 ++--- router.py | 8 +- shared.py | 251 +++++++++++++++++++++++++++++- templates/block_header_close.html | 8 +- 6 files changed, 273 insertions(+), 288 deletions(-) delete mode 100644 db.py diff --git a/archive.py b/archive.py index 9be0dda..8d7784d 100644 --- a/archive.py +++ b/archive.py @@ -14,7 +14,6 @@ from requests_oauthlib import OAuth2Session from requests_oauthlib import oauth2_session from oauthlib.oauth2 import BackendApplicationClient -import db import shared class Favs(object): @@ -734,7 +733,7 @@ class Oauth1Flow(object): self.service = service self.key = shared.config.get("api_%s" % service, 'api_key') self.secret = shared.config.get("api_%s" % service, 'api_secret') - self.tokendb = db.TokenDB() + self.tokendb = shared.TokenDB() self.t = self.tokendb.get_service(self.service) self.oauth_init() diff --git a/db.py b/db.py deleted file mode 100644 index fdf35be..0000000 --- a/db.py +++ /dev/null @@ -1,249 +0,0 @@ -import os -import json -import sqlite3 -import glob -import shared -import logging - -class TokenDB(object): - def __init__(self, uuid='tokens'): - self.db = shared.config.get('var', 'tokendb') - self.tokens = {} - self.refresh() - - def refresh(self): - self.tokens = {} - if os.path.isfile(self.db): - with open(self.db, 'rt') as f: - self.tokens = json.loads(f.read()) - - def save(self): - with open(self.db, 'wt') as f: - f.write(json.dumps( - self.tokens, indent=4, sort_keys=True - )) - - def get_token(self, token): - return self.tokens.get(token, None) - - def get_service(self, service): - token = self.tokens.get(service, None) - return token - - def set_service(self, service, tokenid): - self.tokens.update({ - service: tokenid - }) - self.save() - - def update_token(self, - token, - oauth_token_secret=None, - access_token=None, - access_token_secret=None, - verifier=None): - - t = self.tokens.get(token, {}) - if oauth_token_secret: - t.update({ - 'oauth_token_secret': oauth_token_secret - }) - if access_token: - t.update({ - 'access_token': access_token - }) - if access_token_secret: - t.update({ - 'access_token_secret': access_token_secret - }) - if verifier: - t.update({ - 'verifier': verifier - }) - - self.tokens.update({ - token: t - }) - self.save() - - def clear(self): - self.tokens = {} - self.save() - - def clear_service(self, service): - t = self.tokens.get(service) - if t: - del(self.tokens[t]) - del(self.tokens[service]) - self.save() - -class SearchDB(object): - tmplfile = 'Search.html' - - def __init__(self): - self.db = sqlite3.connect( - "%s" % shared.config.get('var', 'searchdb') - ) - - cursor = self.db.cursor() - cursor.execute('''CREATE VIRTUAL TABLE IF NOT EXISTS data USING FTS5( - id, - corpus, - mtime, - url, - category, - title - )''') - self.db.commit() - - def __exit__(self): - self.finish() - - def finish(self): - self.db.close() - - def append(self, id, corpus, mtime, url, category, title): - mtime = int(mtime) - logging.debug("adding %s to searchdb", id) - cursor = self.db.cursor() - cursor.execute('''DELETE FROM data WHERE id=?''', (id,)) - cursor.execute('''INSERT OR IGNORE INTO data (id, corpus, mtime, url, category, title) VALUES (?,?,?,?,?,?);''', ( - id, - corpus, - mtime, - url, - category, - title - )) - self.db.commit() - - def is_uptodate(self, fname, mtime): - mtime = int(mtime) - ret = {} - cursor = self.db.cursor() - cursor.execute('''SELECT mtime - FROM data - WHERE id = ? AND mtime = ?''', - (fname,mtime) - ) - rows = cursor.fetchall() - - if len(rows): - logging.debug("%s is up to date in searchdb", fname) - return True - - logging.debug("%s is out of date in searchdb", fname) - return False - - def search_by_query(self, query): - ret = {} - cursor = self.db.cursor() - cursor.execute('''SELECT - id, category, url, title, highlight(data, 0, '', '') corpus - FROM data - WHERE data MATCH ? - ORDER BY category, rank;''', (query,)) - rows = cursor.fetchall() - for r in rows: - r = { - 'id': r[0], - 'category': r[1], - 'url': r[2], - 'title': r[3], - 'txt': r[4], - } - - category = r.get('category') - if category not in ret: - ret.update({category: {}}) - - - maybe_fpath = os.path.join( - shared.config.get('dirs', 'content'), - category, - "%s.*" % r.get('id') - ) - #fpath = glob.glob(maybe_fpath).pop() - ret.get(category).update({ - r.get('id'): { - #'fpath': fpath, - 'url': r.get('url'), - 'title': r.get('title'), - 'txt': r.get('txt') - } - }) - return ret - - - def cli(self, query): - results = self.search_by_query(query) - for c, items in sorted(results.items()): - print("%s:" % c) - for fname, data in sorted(items.items()): - print(" %s" % data.get('fpath')) - print(" %s" % data.get('url')) - print("") - - def html(self, query): - tmplvars = { - 'results': self.search_by_query(query), - 'term': query - } - return shared.j2.get_template(self.tmplfile).render(tmplvars) - - -class WebmentionQueue(object): - def __init__(self): - self.db = sqlite3.connect( - "%s" % shared.config.get('var', 'webmentiondb') - ) - - cursor = self.db.cursor() - cursor.execute(''' - CREATE TABLE IF NOT EXISTS `queue` ( - `id` INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE, - `timestamp` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - `source` TEXT NOT NULL, - `target` TEXT NOT NULL, - `status` INTEGER NOT NULL DEFAULT 0, - `mtime` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ); - ''') - self.db.commit() - - def __exit__(self): - self.finish() - - def finish(self): - self.db.close() - - def queue(self, source, target): - cursor = self.db.cursor() - cursor.execute( - '''INSERT INTO queue (source,target) VALUES (?,?);''', ( - source, - target - ) - ) - self.db.commit() - - def get_queued(self, fname=None): - logging.debug('getting queued webmentions for %s', fname) - ret = [] - cursor = self.db.cursor() - cursor.execute('''SELECT * FROM queue WHERE target LIKE ? AND status = 0''', ('%'+fname+'%',)) - rows = cursor.fetchall() - for r in rows: - ret.append({ - 'id': r[0], - 'dt': r[1], - 'source': r[2], - 'target': r[3], - }) - return ret - - def entry_done(self, id): - logging.debug('setting %s webmention to done', id) - cursor = self.db.cursor() - cursor.execute("UPDATE queue SET status = 1 where ID=?", (id,)) - self.db.commit() diff --git a/nasg.py b/nasg.py index 4cb85d8..684cd06 100644 --- a/nasg.py +++ b/nasg.py @@ -22,7 +22,6 @@ import wand.image from emoji import UNICODE_EMOJI import shared -import db from pprint import pprint @@ -206,22 +205,6 @@ class Category(NoDupeContainer): url = '/' return url - - #def url_paged(self, page=1, feed=False): - #x = '/' - #if self.name: - #x = "%s%s/%s" % ( - #x, - #self.taxonomy, - #self.name, - #) - - #if page == 1 and feed: - #x = "%s/%s/" % (x, self.feeddir) - #else: - #x = "%s/%s/%s/" % (x, self.pagedir, "%s" % page) - #return x - def path_paged(self, page=1, feed=False): x = shared.config.get('common', 'build') @@ -309,6 +292,7 @@ class Singular(object): logging.debug("initiating singular object from %s", fpath) self.fpath = fpath self.mtime = os.path.getmtime(self.fpath) + self.stime = self.mtime self.fname, self.fext = os.path.splitext(os.path.basename(self.fpath)) self.category = os.path.basename(os.path.dirname(self.fpath)) self._images = NoDupeContainer() @@ -333,7 +317,7 @@ class Singular(object): # TODO this should be async def process_webmentions(self): - wdb = db.WebmentionQueue() + wdb = shared.WebmentionQueue() queued = wdb.get_queued(self.url) for incoming in queued: wm = Webmention( @@ -358,7 +342,7 @@ class Singular(object): if not os.path.isfile(self.htmlfile): return False mtime = os.path.getmtime(self.htmlfile) - if mtime == self.mtime: + if mtime >= self.stime: return True return False @@ -397,8 +381,8 @@ class Singular(object): cfiles = [*cfiles, *maybe] for cpath in cfiles: cmtime = os.path.getmtime(cpath) - if cmtime > self.mtime: - self.mtime = cmtime + if cmtime > self.stime: + self.stime = cmtime c = Comment(cpath) comments.append(c.mtime, c) @@ -1089,6 +1073,8 @@ class Webmention(object): def run(self): self._fetch() + if 'data' not in self._source: + return self._save() @property @@ -1115,9 +1101,15 @@ class Webmention(object): @property def content(self): - return shared.Pandoc('html').convert( - self._source.get('data').get('content').get('html') - ) + if 'content' not in self._source.get('data'): + return '' + elif 'html' in self._source.get('data').get('content'): + what = self._source.get('data').get('content').get('html') + elif 'text' in self._source.get('data').get('content'): + what = self._source.get('data').get('content').get('text') + else: + return '' + return shared.Pandoc('html').convert(what) @property def fname(self): @@ -1214,7 +1206,7 @@ def build(): loop = asyncio.get_event_loop() tasks = [] content = Content() - sdb = db.SearchDB() + sdb = shared.SearchDB() magic = MagicPHP() collector_front = Category() diff --git a/router.py b/router.py index 4a6e24b..057d60d 100644 --- a/router.py +++ b/router.py @@ -5,11 +5,9 @@ from sanic import Sanic import sanic.response import logging -import db -import shared import validators import urllib.parse -import requests +import shared if __name__ == '__main__': logging_format = "[%(asctime)s] %(process)d-%(levelname)s " @@ -26,7 +24,7 @@ if __name__ == '__main__': # since I'm running this from systemctl it already goes into syslog app = Sanic('router', log_config=None) # this is ok to be read-only - sdb = db.SearchDB() + sdb = shared.SearchDB() @app.route("/oauth1", methods=["GET"]) @@ -78,7 +76,7 @@ if __name__ == '__main__': # it is unfortunate that I need to init this every time, but # otherwise it'll become read-only for reasons I'm yet to grasp # the actual parsing will be done at site generation time - wdb = db.WebmentionQueue() + wdb = shared.WebmentionQueue() wdb.queue(source,target) # telegram notification, if set diff --git a/shared.py b/shared.py index e660763..e9878eb 100644 --- a/shared.py +++ b/shared.py @@ -218,6 +218,251 @@ class ExifTool(CMDLine): return exif + +class TokenDB(object): + def __init__(self, uuid='tokens'): + self.db = config.get('var', 'tokendb') + self.tokens = {} + self.refresh() + + def refresh(self): + self.tokens = {} + if os.path.isfile(self.db): + with open(self.db, 'rt') as f: + self.tokens = json.loads(f.read()) + + def save(self): + with open(self.db, 'wt') as f: + f.write(json.dumps( + self.tokens, indent=4, sort_keys=True + )) + + def get_token(self, token): + return self.tokens.get(token, None) + + def get_service(self, service): + token = self.tokens.get(service, None) + return token + + def set_service(self, service, tokenid): + self.tokens.update({ + service: tokenid + }) + self.save() + + def update_token(self, + token, + oauth_token_secret=None, + access_token=None, + access_token_secret=None, + verifier=None): + + t = self.tokens.get(token, {}) + if oauth_token_secret: + t.update({ + 'oauth_token_secret': oauth_token_secret + }) + if access_token: + t.update({ + 'access_token': access_token + }) + if access_token_secret: + t.update({ + 'access_token_secret': access_token_secret + }) + if verifier: + t.update({ + 'verifier': verifier + }) + + self.tokens.update({ + token: t + }) + self.save() + + def clear(self): + self.tokens = {} + self.save() + + def clear_service(self, service): + t = self.tokens.get(service) + if t: + del(self.tokens[t]) + del(self.tokens[service]) + self.save() + +class SearchDB(object): + tmplfile = 'Search.html' + + def __init__(self): + self.db = sqlite3.connect( + "%s" % config.get('var', 'searchdb') + ) + + cursor = self.db.cursor() + cursor.execute('''CREATE VIRTUAL TABLE IF NOT EXISTS data USING FTS5( + id, + corpus, + mtime, + url, + category, + title + )''') + self.db.commit() + + def __exit__(self): + self.finish() + + def finish(self): + self.db.close() + + def append(self, id, corpus, mtime, url, category, title): + mtime = int(mtime) + logging.debug("adding %s to searchdb", id) + cursor = self.db.cursor() + cursor.execute('''DELETE FROM data WHERE id=?''', (id,)) + cursor.execute('''INSERT OR IGNORE INTO data (id, corpus, mtime, url, category, title) VALUES (?,?,?,?,?,?);''', ( + id, + corpus, + mtime, + url, + category, + title + )) + self.db.commit() + + def is_uptodate(self, fname, mtime): + mtime = int(mtime) + ret = {} + cursor = self.db.cursor() + cursor.execute('''SELECT mtime + FROM data + WHERE id = ? AND mtime = ?''', + (fname,mtime) + ) + rows = cursor.fetchall() + + if len(rows): + logging.debug("%s is up to date in searchdb", fname) + return True + + logging.debug("%s is out of date in searchdb", fname) + return False + + def search_by_query(self, query): + ret = {} + cursor = self.db.cursor() + cursor.execute('''SELECT + id, category, url, title, snippet(data, 1, '', '', '[...]', 24) + FROM data + WHERE data MATCH ? + ORDER BY category, rank;''', (query,)) + rows = cursor.fetchall() + for r in rows: + r = { + 'id': r[0], + 'category': r[1], + 'url': r[2], + 'title': r[3], + 'txt': r[4], + } + + category = r.get('category') + if category not in ret: + ret.update({category: {}}) + + + maybe_fpath = os.path.join( + config.get('dirs', 'content'), + category, + "%s.*" % r.get('id') + ) + #fpath = glob.glob(maybe_fpath).pop() + ret.get(category).update({ + r.get('id'): { + #'fpath': fpath, + 'url': r.get('url'), + 'title': r.get('title'), + 'txt': r.get('txt') + } + }) + return ret + + + def cli(self, query): + results = self.search_by_query(query) + for c, items in sorted(results.items()): + print("%s:" % c) + for fname, data in sorted(items.items()): + print(" %s" % data.get('fpath')) + print(" %s" % data.get('url')) + print("") + + def html(self, query): + tmplvars = { + 'results': self.search_by_query(query), + 'term': query + } + return j2.get_template(self.tmplfile).render(tmplvars) + + +class WebmentionQueue(object): + def __init__(self): + self.db = sqlite3.connect( + "%s" % config.get('var', 'webmentiondb') + ) + + cursor = self.db.cursor() + cursor.execute(''' + CREATE TABLE IF NOT EXISTS `queue` ( + `id` INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE, + `timestamp` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + `source` TEXT NOT NULL, + `target` TEXT NOT NULL, + `status` INTEGER NOT NULL DEFAULT 0, + `mtime` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + ''') + self.db.commit() + + def __exit__(self): + self.finish() + + def finish(self): + self.db.close() + + def queue(self, source, target): + cursor = self.db.cursor() + cursor.execute( + '''INSERT INTO queue (source,target) VALUES (?,?);''', ( + source, + target + ) + ) + self.db.commit() + + def get_queued(self, fname=None): + logging.debug('getting queued webmentions for %s', fname) + ret = [] + cursor = self.db.cursor() + cursor.execute('''SELECT * FROM queue WHERE target LIKE ? AND status = 0''', ('%'+fname+'%',)) + rows = cursor.fetchall() + for r in rows: + ret.append({ + 'id': r[0], + 'dt': r[1], + 'source': r[2], + 'target': r[3], + }) + return ret + + def entry_done(self, id): + logging.debug('setting %s webmention to done', id) + cursor = self.db.cursor() + cursor.execute("UPDATE queue SET status = 1 where ID=?", (id,)) + self.db.commit() + + def __expandconfig(): c = configparser.ConfigParser( interpolation=configparser.ExtendedInterpolation(), @@ -275,14 +520,14 @@ def __setup_sitevars(): def notify(msg): # telegram notification, if set - if not shared.config.has_section('api_telegram'): + if not config.has_section('api_telegram'): return url = "https://api.telegram.org/bot%s/sendMessage" % ( - shared.config.get('api_telegram', 'api_token') + config.get('api_telegram', 'api_token') ) data = { - 'chat_id': shared.config.get('api_telegram', 'chat_id'), + 'chat_id': config.get('api_telegram', 'chat_id'), 'text': msg } # fire and forget diff --git a/templates/block_header_close.html b/templates/block_header_close.html index f9225aa..ba6f49b 100644 --- a/templates/block_header_close.html +++ b/templates/block_header_close.html @@ -19,7 +19,7 @@ {% set cssclass = '' %} -{% if (post is defined and post.category == 'photo' ) or ( taxonomy is defined and taxonomy.slug == 'photo' ) %} +{% if (post is defined and post.category == 'photo' ) or ( taxonomy is defined and taxonomy.name == 'photo' ) %} {% set cssclass = 'active' %} {% endif %}
  • @@ -29,7 +29,7 @@
    • {% set cssclass = '' %} -{% if (post is defined and post.category == 'journal' ) or ( taxonomy is defined and taxonomy.slug == 'journal' ) %} +{% if (post is defined and post.category == 'journal' ) or ( taxonomy is defined and taxonomy.name == 'journal' ) %} {% set cssclass = 'active' %} {% endif %}
  • @@ -39,7 +39,7 @@
    • {% set cssclass = '' %} -{% if (post is defined and post.category == 'article' ) or ( taxonomy is defined and taxonomy.slug == 'article' ) %} +{% if (post is defined and post.category == 'article' ) or ( taxonomy is defined and taxonomy.name == 'article' ) %} {% set cssclass = 'active' %} {% endif %}
  • @@ -49,7 +49,7 @@
    • {% set cssclass = '' %} -{% if (post is defined and post.category == 'note' ) or ( taxonomy is defined and taxonomy.slug == 'note' ) %} +{% if (post is defined and post.category == 'note' ) or ( taxonomy is defined and taxonomy.name == 'note' ) %} {% set cssclass = 'active' %} {% endif %}