fixing missing active page indicator in header; moved db classes to shared; fixed webmention parse error if content is missing; is_uptodate of singular based on last comment time, so singular gets re-rendered, but category not

archive.py

@@ -14,7 +14,6 @@ from requests_oauthlib import OAuth2Session
 from requests_oauthlib import oauth2_session
 from oauthlib.oauth2 import BackendApplicationClient
 
-import db
 import shared
 
 class Favs(object):
@@ -734,7 +733,7 @@ class Oauth1Flow(object):
         self.service = service
         self.key = shared.config.get("api_%s" % service, 'api_key')
         self.secret = shared.config.get("api_%s" % service, 'api_secret')
-        self.tokendb = db.TokenDB()
+        self.tokendb = shared.TokenDB()
         self.t = self.tokendb.get_service(self.service)
         self.oauth_init()
 

db.py

@@ -1,249 +0,0 @@
-import os
-import json
-import sqlite3
-import glob
-import shared
-import logging
-
-class TokenDB(object):
-    def __init__(self, uuid='tokens'):
-        self.db = shared.config.get('var', 'tokendb')
-        self.tokens = {}
-        self.refresh()
-
-    def refresh(self):
-        self.tokens = {}
-        if os.path.isfile(self.db):
-            with open(self.db, 'rt') as f:
-                self.tokens = json.loads(f.read())
-
-    def save(self):
-        with open(self.db, 'wt') as f:
-            f.write(json.dumps(
-                self.tokens, indent=4, sort_keys=True
-            ))
-
-    def get_token(self, token):
-        return self.tokens.get(token, None)
-
-    def get_service(self, service):
-        token = self.tokens.get(service, None)
-        return token
-
-    def set_service(self, service, tokenid):
-        self.tokens.update({
-            service: tokenid
-        })
-        self.save()
-
-    def update_token(self,
-        token,
-        oauth_token_secret=None,
-        access_token=None,
-        access_token_secret=None,
-        verifier=None):
-
-        t = self.tokens.get(token, {})
-        if oauth_token_secret:
-            t.update({
-                'oauth_token_secret': oauth_token_secret
-            })
-        if access_token:
-            t.update({
-                'access_token': access_token
-            })
-        if access_token_secret:
-            t.update({
-                'access_token_secret': access_token_secret
-            })
-        if verifier:
-            t.update({
-                'verifier': verifier
-            })
-
-        self.tokens.update({
-            token: t
-        })
-        self.save()
-
-    def clear(self):
-        self.tokens = {}
-        self.save()
-
-    def clear_service(self, service):
-        t = self.tokens.get(service)
-        if t:
-            del(self.tokens[t])
-        del(self.tokens[service])
-        self.save()
-
-class SearchDB(object):
-    tmplfile = 'Search.html'
-
-    def __init__(self):
-        self.db = sqlite3.connect(
-            "%s" % shared.config.get('var', 'searchdb')
-        )
-
-        cursor = self.db.cursor()
-        cursor.execute('''CREATE VIRTUAL TABLE IF NOT EXISTS data USING FTS5(
-                id,
-                corpus,
-                mtime,
-                url,
-                category,
-                title
-            )''')
-        self.db.commit()
-
-    def __exit__(self):
-        self.finish()
-
-    def finish(self):
-        self.db.close()
-
-    def append(self, id, corpus, mtime, url, category, title):
-        mtime = int(mtime)
-        logging.debug("adding %s to searchdb", id)
-        cursor = self.db.cursor()
-        cursor.execute('''DELETE FROM data WHERE id=?''', (id,))
-        cursor.execute('''INSERT OR IGNORE INTO data (id, corpus, mtime, url, category, title) VALUES (?,?,?,?,?,?);''', (
-            id,
-            corpus,
-            mtime,
-            url,
-            category,
-            title
-        ))
-        self.db.commit()
-
-    def is_uptodate(self, fname, mtime):
-        mtime = int(mtime)
-        ret = {}
-        cursor = self.db.cursor()
-        cursor.execute('''SELECT mtime
-            FROM data
-            WHERE id = ? AND mtime = ?''',
-            (fname,mtime)
-        )
-        rows = cursor.fetchall()
-
-        if len(rows):
-            logging.debug("%s is up to date in searchdb", fname)
-            return True
-
-        logging.debug("%s is out of  date in searchdb", fname)
-        return False
-
-    def search_by_query(self, query):
-        ret = {}
-        cursor = self.db.cursor()
-        cursor.execute('''SELECT
-            id, category, url, title, highlight(data, 0, '<strong>', '</strong>') corpus
-            FROM data
-            WHERE data MATCH ?
-            ORDER BY category, rank;''', (query,))
-        rows = cursor.fetchall()
-        for r in rows:
-            r = {
-                'id': r[0],
-                'category': r[1],
-                'url': r[2],
-                'title': r[3],
-                'txt': r[4],
-            }
-
-            category = r.get('category')
-            if category not in ret:
-                ret.update({category: {}})
-
-
-            maybe_fpath = os.path.join(
-                shared.config.get('dirs', 'content'),
-                category,
-                "%s.*" % r.get('id')
-            )
-            #fpath = glob.glob(maybe_fpath).pop()
-            ret.get(category).update({
-                r.get('id'): {
-                    #'fpath': fpath,
-                    'url': r.get('url'),
-                    'title': r.get('title'),
-                    'txt': r.get('txt')
-                }
-            })
-        return ret
-
-
-    def cli(self, query):
-        results = self.search_by_query(query)
-        for c, items in sorted(results.items()):
-            print("%s:" % c)
-            for fname, data in sorted(items.items()):
-                print("  %s" % data.get('fpath'))
-                print("  %s" % data.get('url'))
-                print("")
-
-    def html(self, query):
-        tmplvars = {
-            'results': self.search_by_query(query),
-            'term': query
-        }
-        return shared.j2.get_template(self.tmplfile).render(tmplvars)
-
-
-class WebmentionQueue(object):
-    def __init__(self):
-        self.db = sqlite3.connect(
-            "%s" % shared.config.get('var', 'webmentiondb')
-        )
-
-        cursor = self.db.cursor()
-        cursor.execute('''
-            CREATE TABLE IF NOT EXISTS  `queue` (
-                `id` INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE,
-                `timestamp` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-                `source` TEXT NOT NULL,
-                `target` TEXT NOT NULL,
-                `status` INTEGER NOT NULL DEFAULT 0,
-                `mtime` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
-            );
-        ''')
-        self.db.commit()
-
-    def __exit__(self):
-        self.finish()
-
-    def finish(self):
-        self.db.close()
-
-    def queue(self, source, target):
-        cursor = self.db.cursor()
-        cursor.execute(
-            '''INSERT INTO queue (source,target) VALUES (?,?);''', (
-                source,
-                target
-            )
-        )
-        self.db.commit()
-
-    def get_queued(self, fname=None):
-        logging.debug('getting queued webmentions for %s', fname)
-        ret = []
-        cursor = self.db.cursor()
-        cursor.execute('''SELECT * FROM queue WHERE target LIKE ? AND status = 0''', ('%'+fname+'%',))
-        rows = cursor.fetchall()
-        for r in rows:
-            ret.append({
-                'id': r[0],
-                'dt': r[1],
-                'source': r[2],
-                'target': r[3],
-            })
-        return ret
-
-    def entry_done(self, id):
-        logging.debug('setting %s webmention to done', id)
-        cursor = self.db.cursor()
-        cursor.execute("UPDATE queue SET status = 1 where ID=?", (id,))
-        self.db.commit()

nasg.py

@@ -22,7 +22,6 @@ import wand.image
 from emoji import UNICODE_EMOJI
 
 import shared
-import db
 
 from pprint import pprint
 
@@ -206,22 +205,6 @@ class Category(NoDupeContainer):
             url = '/'
         return url
 
-
-    #def url_paged(self, page=1, feed=False):
-        #x = '/'
-        #if self.name:
-            #x = "%s%s/%s" % (
-                #x,
-                #self.taxonomy,
-                #self.name,
-            #)
-
-        #if page == 1 and feed:
-            #x = "%s/%s/" % (x, self.feeddir)
-        #else:
-            #x = "%s/%s/%s/" % (x, self.pagedir, "%s" % page)
-        #return x
-
     def path_paged(self, page=1, feed=False):
         x = shared.config.get('common', 'build')
 
@@ -309,6 +292,7 @@ class Singular(object):
         logging.debug("initiating singular object from %s", fpath)
         self.fpath = fpath
         self.mtime = os.path.getmtime(self.fpath)
+        self.stime = self.mtime
         self.fname, self.fext = os.path.splitext(os.path.basename(self.fpath))
         self.category = os.path.basename(os.path.dirname(self.fpath))
         self._images = NoDupeContainer()
@@ -333,7 +317,7 @@ class Singular(object):
 
     # TODO this should be async
     def process_webmentions(self):
-        wdb = db.WebmentionQueue()
+        wdb = shared.WebmentionQueue()
         queued = wdb.get_queued(self.url)
         for incoming in queued:
             wm = Webmention(
@@ -358,7 +342,7 @@ class Singular(object):
         if not os.path.isfile(self.htmlfile):
             return False
         mtime = os.path.getmtime(self.htmlfile)
-        if mtime == self.mtime:
+        if mtime >= self.stime:
             return True
         return False
 
@@ -397,8 +381,8 @@ class Singular(object):
             cfiles = [*cfiles, *maybe]
         for cpath in cfiles:
             cmtime = os.path.getmtime(cpath)
-            if cmtime > self.mtime:
+            if cmtime > self.stime:
-                self.mtime = cmtime
+                self.stime = cmtime
 
             c = Comment(cpath)
             comments.append(c.mtime, c)
@@ -1089,6 +1073,8 @@ class Webmention(object):
 
     def run(self):
         self._fetch()
+        if 'data' not in self._source:
+            return
         self._save()
 
     @property
@@ -1115,9 +1101,15 @@ class Webmention(object):
 
     @property
     def content(self):
-        return  shared.Pandoc('html').convert(
+        if 'content' not in self._source.get('data'):
-            self._source.get('data').get('content').get('html')
+            return ''
-        )
+        elif 'html' in self._source.get('data').get('content'):
+            what = self._source.get('data').get('content').get('html')
+        elif 'text' in self._source.get('data').get('content'):
+            what = self._source.get('data').get('content').get('text')
+        else:
+            return ''
+        return  shared.Pandoc('html').convert(what)
 
     @property
     def fname(self):
@@ -1214,7 +1206,7 @@ def build():
     loop = asyncio.get_event_loop()
     tasks = []
     content = Content()
-    sdb = db.SearchDB()
+    sdb = shared.SearchDB()
     magic = MagicPHP()
 
     collector_front = Category()

router.py

@@ -5,11 +5,9 @@
 from sanic import Sanic
 import sanic.response
 import logging
-import db
-import shared
 import validators
 import urllib.parse
-import requests
+import shared
 
 if __name__ == '__main__':
     logging_format = "[%(asctime)s] %(process)d-%(levelname)s "
@@ -26,7 +24,7 @@ if __name__ == '__main__':
     # since I'm running this from systemctl it already goes into syslog
     app = Sanic('router', log_config=None)
     # this is ok to be read-only
-    sdb = db.SearchDB()
+    sdb = shared.SearchDB()
 
 
     @app.route("/oauth1", methods=["GET"])
@@ -78,7 +76,7 @@ if __name__ == '__main__':
         # it is unfortunate that I need to init this every time, but
         # otherwise it'll become read-only for reasons I'm yet to grasp
         # the actual parsing will be done at site generation time
-        wdb = db.WebmentionQueue()
+        wdb = shared.WebmentionQueue()
         wdb.queue(source,target)
 
         # telegram notification, if set

shared.py

@@ -218,6 +218,251 @@ class ExifTool(CMDLine):
 
         return exif
 
+
+class TokenDB(object):
+    def __init__(self, uuid='tokens'):
+        self.db = config.get('var', 'tokendb')
+        self.tokens = {}
+        self.refresh()
+
+    def refresh(self):
+        self.tokens = {}
+        if os.path.isfile(self.db):
+            with open(self.db, 'rt') as f:
+                self.tokens = json.loads(f.read())
+
+    def save(self):
+        with open(self.db, 'wt') as f:
+            f.write(json.dumps(
+                self.tokens, indent=4, sort_keys=True
+            ))
+
+    def get_token(self, token):
+        return self.tokens.get(token, None)
+
+    def get_service(self, service):
+        token = self.tokens.get(service, None)
+        return token
+
+    def set_service(self, service, tokenid):
+        self.tokens.update({
+            service: tokenid
+        })
+        self.save()
+
+    def update_token(self,
+        token,
+        oauth_token_secret=None,
+        access_token=None,
+        access_token_secret=None,
+        verifier=None):
+
+        t = self.tokens.get(token, {})
+        if oauth_token_secret:
+            t.update({
+                'oauth_token_secret': oauth_token_secret
+            })
+        if access_token:
+            t.update({
+                'access_token': access_token
+            })
+        if access_token_secret:
+            t.update({
+                'access_token_secret': access_token_secret
+            })
+        if verifier:
+            t.update({
+                'verifier': verifier
+            })
+
+        self.tokens.update({
+            token: t
+        })
+        self.save()
+
+    def clear(self):
+        self.tokens = {}
+        self.save()
+
+    def clear_service(self, service):
+        t = self.tokens.get(service)
+        if t:
+            del(self.tokens[t])
+        del(self.tokens[service])
+        self.save()
+
+class SearchDB(object):
+    tmplfile = 'Search.html'
+
+    def __init__(self):
+        self.db = sqlite3.connect(
+            "%s" % config.get('var', 'searchdb')
+        )
+
+        cursor = self.db.cursor()
+        cursor.execute('''CREATE VIRTUAL TABLE IF NOT EXISTS data USING FTS5(
+                id,
+                corpus,
+                mtime,
+                url,
+                category,
+                title
+            )''')
+        self.db.commit()
+
+    def __exit__(self):
+        self.finish()
+
+    def finish(self):
+        self.db.close()
+
+    def append(self, id, corpus, mtime, url, category, title):
+        mtime = int(mtime)
+        logging.debug("adding %s to searchdb", id)
+        cursor = self.db.cursor()
+        cursor.execute('''DELETE FROM data WHERE id=?''', (id,))
+        cursor.execute('''INSERT OR IGNORE INTO data (id, corpus, mtime, url, category, title) VALUES (?,?,?,?,?,?);''', (
+            id,
+            corpus,
+            mtime,
+            url,
+            category,
+            title
+        ))
+        self.db.commit()
+
+    def is_uptodate(self, fname, mtime):
+        mtime = int(mtime)
+        ret = {}
+        cursor = self.db.cursor()
+        cursor.execute('''SELECT mtime
+            FROM data
+            WHERE id = ? AND mtime = ?''',
+            (fname,mtime)
+        )
+        rows = cursor.fetchall()
+
+        if len(rows):
+            logging.debug("%s is up to date in searchdb", fname)
+            return True
+
+        logging.debug("%s is out of  date in searchdb", fname)
+        return False
+
+    def search_by_query(self, query):
+        ret = {}
+        cursor = self.db.cursor()
+        cursor.execute('''SELECT
+            id, category, url, title, snippet(data, 1, '', '', '[...]', 24)
+            FROM data
+            WHERE data MATCH ?
+            ORDER BY category, rank;''', (query,))
+        rows = cursor.fetchall()
+        for r in rows:
+            r = {
+                'id': r[0],
+                'category': r[1],
+                'url': r[2],
+                'title': r[3],
+                'txt': r[4],
+            }
+
+            category = r.get('category')
+            if category not in ret:
+                ret.update({category: {}})
+
+
+            maybe_fpath = os.path.join(
+                config.get('dirs', 'content'),
+                category,
+                "%s.*" % r.get('id')
+            )
+            #fpath = glob.glob(maybe_fpath).pop()
+            ret.get(category).update({
+                r.get('id'): {
+                    #'fpath': fpath,
+                    'url': r.get('url'),
+                    'title': r.get('title'),
+                    'txt': r.get('txt')
+                }
+            })
+        return ret
+
+
+    def cli(self, query):
+        results = self.search_by_query(query)
+        for c, items in sorted(results.items()):
+            print("%s:" % c)
+            for fname, data in sorted(items.items()):
+                print("  %s" % data.get('fpath'))
+                print("  %s" % data.get('url'))
+                print("")
+
+    def html(self, query):
+        tmplvars = {
+            'results': self.search_by_query(query),
+            'term': query
+        }
+        return j2.get_template(self.tmplfile).render(tmplvars)
+
+
+class WebmentionQueue(object):
+    def __init__(self):
+        self.db = sqlite3.connect(
+            "%s" % config.get('var', 'webmentiondb')
+        )
+
+        cursor = self.db.cursor()
+        cursor.execute('''
+            CREATE TABLE IF NOT EXISTS  `queue` (
+                `id` INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE,
+                `timestamp` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+                `source` TEXT NOT NULL,
+                `target` TEXT NOT NULL,
+                `status` INTEGER NOT NULL DEFAULT 0,
+                `mtime` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+            );
+        ''')
+        self.db.commit()
+
+    def __exit__(self):
+        self.finish()
+
+    def finish(self):
+        self.db.close()
+
+    def queue(self, source, target):
+        cursor = self.db.cursor()
+        cursor.execute(
+            '''INSERT INTO queue (source,target) VALUES (?,?);''', (
+                source,
+                target
+            )
+        )
+        self.db.commit()
+
+    def get_queued(self, fname=None):
+        logging.debug('getting queued webmentions for %s', fname)
+        ret = []
+        cursor = self.db.cursor()
+        cursor.execute('''SELECT * FROM queue WHERE target LIKE ? AND status = 0''', ('%'+fname+'%',))
+        rows = cursor.fetchall()
+        for r in rows:
+            ret.append({
+                'id': r[0],
+                'dt': r[1],
+                'source': r[2],
+                'target': r[3],
+            })
+        return ret
+
+    def entry_done(self, id):
+        logging.debug('setting %s webmention to done', id)
+        cursor = self.db.cursor()
+        cursor.execute("UPDATE queue SET status = 1 where ID=?", (id,))
+        self.db.commit()
+
+
 def __expandconfig():
     c = configparser.ConfigParser(
         interpolation=configparser.ExtendedInterpolation(),
@@ -275,14 +520,14 @@ def __setup_sitevars():
 
 def notify(msg):
     # telegram notification, if set
-    if not shared.config.has_section('api_telegram'):
+    if not config.has_section('api_telegram'):
         return
 
     url = "https://api.telegram.org/bot%s/sendMessage" % (
-        shared.config.get('api_telegram', 'api_token')
+        config.get('api_telegram', 'api_token')
     )
     data = {
-        'chat_id': shared.config.get('api_telegram', 'chat_id'),
+        'chat_id': config.get('api_telegram', 'chat_id'),
         'text': msg
     }
     # fire and forget

templates/block_header_close.html

@@ -19,7 +19,7 @@
 				</a>
 			</li>
 {% set cssclass = '' %}
-{% if (post is defined and post.category ==  'photo' ) or ( taxonomy is defined and taxonomy.slug == 'photo' ) %}
+{% if (post is defined and post.category ==  'photo' ) or ( taxonomy is defined and taxonomy.name == 'photo' ) %}
 	{% set cssclass = 'active' %}
 {% endif %}
 			<li>
@@ -29,7 +29,7 @@
 				</a>
 			</li>
 {% set cssclass = '' %}
-{% if (post is defined and post.category ==  'journal' ) or ( taxonomy is defined and taxonomy.slug == 'journal' ) %}
+{% if (post is defined and post.category ==  'journal' ) or ( taxonomy is defined and taxonomy.name == 'journal' ) %}
 	{% set cssclass = 'active' %}
 {% endif %}
 			<li>
@@ -39,7 +39,7 @@
 				</a>
 			</li>
 {% set cssclass = '' %}
-{% if (post is defined and post.category ==  'article' ) or ( taxonomy is defined and taxonomy.slug == 'article' ) %}
+{% if (post is defined and post.category ==  'article' ) or ( taxonomy is defined and taxonomy.name == 'article' ) %}
 	{% set cssclass = 'active' %}
 {% endif %}
 			<li>
@@ -49,7 +49,7 @@
 				</a>
 			</li>
 {% set cssclass = '' %}
-{% if (post is defined and post.category ==  'note' ) or ( taxonomy is defined and taxonomy.slug == 'note' ) %}
+{% if (post is defined and post.category ==  'note' ) or ( taxonomy is defined and taxonomy.name == 'note' ) %}
 	{% set cssclass = 'active' %}
 {% endif %}
 			<li>