diff --git a/.gitignore b/.gitignore index 1c81da7..e4470d5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ __pycache__ _scratch keys.py -nasg.proj .idea +.venv diff --git a/assets/icomoon-2018-08-13.zip b/assets/icomoon-2018-08-13.zip new file mode 100644 index 0000000..492fbaa Binary files /dev/null and b/assets/icomoon-2018-08-13.zip differ diff --git a/nasg.py b/nasg.py index c7f7b0e..4dd5bf7 100644 --- a/nasg.py +++ b/nasg.py @@ -41,6 +41,8 @@ MarkdownImage = namedtuple( ['match', 'alt', 'fname', 'title', 'css'] ) +REPLY_TYPES = ['webmention', 'in-reply-to', 'reply'] + J2 = jinja2.Environment( loader=jinja2.FileSystemLoader(searchpath=settings.paths.get('tmpl')), lstrip_blocks=True, @@ -78,6 +80,58 @@ class cached_property(object): setattr(inst, self.name, result) return result +class Webmention(object): + def __init__(self, source, target, stime): + self.source = source + self.target = target + self.stime = stime + + @property + def fpath(self): + return os.path.join( + settings.paths.get('webmentions'), + '%s => %s.txt' % ( + url2slug(self.source, 100), + url2slug(self.target, 100) + ) + ) + + @property + def exists(self): + if not os.path.isfile(self.fpath): + return False + elif os.path.getmtime(self.fpath) > self.stime: + return True + else: + return False + + async def save(self, content): + d = os.path.dirname(self.fpath) + if not os.path.isdir(d): + os.makedirs(d) + with open(self.fpath, 'wt') as f: + f.write(content) + + async def send(self): + if self.exists: + return + telegraph_url = 'https://telegraph.p3k.io/webmention' + telegraph_params = { + 'token': '%s' % (keys.telegraph.get('token')), + 'source': '%s' % (self.source), + 'target': '%s' % (self.target) + } + r = requests.post(telegraph_url, data=telegraph_params) + settings.logger.info( + "sent webmention to telegraph from %s to %s", + self.source, + self.target + ) + if r.status_code not in [200, 201, 202]: + settings.logger.error('sending failed: %s %s', r.status_code, r.text) + else: + await self.save(r.text) + class MarkdownDoc(object): @cached_property @@ -95,12 +149,19 @@ class MarkdownDoc(object): def content(self): return self._parsed[1] + @property + def has_mainimg(self): + if hasattr(self, 'images') and len(self.images) == 1: + return True + else: + return False + @cached_property def html_content(self): c = "%s" % (self.content) if hasattr(self, 'images') and len(self.images): for match, img in self.images.items(): - c = c.replace(match, str(img)) + c = c.replace(match, img.mkstring(self.has_mainimg)) # return MD.reset().convert(c) c = Pandoc(c) c = RE_PRECODE.sub('
', c)
@@ -337,8 +398,6 @@ class Singular(MarkdownDoc):
     @property
     def syndicate(self):
         urls = self.meta.get('syndicate', [])
-        if "https://brid.gy/publish/twitter" not in urls:
-            urls.append("https://brid.gy/publish/twitter")
         if self.is_photo:
             urls.append("https://brid.gy/publish/flickr")
         return urls
@@ -375,6 +434,14 @@ class Singular(MarkdownDoc):
             return True
         return False
 
+    @property
+    def to_ping(self):
+        urls = []
+        if self.is_reply:
+            w = Webmention(self.url, self.is_reply, self.mtime)
+            urls.append(w)
+        return urls
+
     @property
     def licence(self):
         if self.category in settings.licence:
@@ -404,15 +471,16 @@ class Singular(MarkdownDoc):
     def replies(self):
         r = OrderedDict()
         for mtime, c in self.comments.items():
-            if c.type in ['webmention', 'in-reply-to']:
-                r[mtime] = c.tmplvars
+            if c.type not in REPLY_TYPES:
+                continue
+            r[mtime] = c.tmplvars
         return r
 
     @property
     def reactions(self):
         r = OrderedDict()
         for mtime, c in self.comments.items():
-            if c.type in ['webmention', 'in-reply-to']:
+            if c.type in REPLY_TYPES:
                 continue
             t = "%s" % (c.type)
             if t not in r:
@@ -450,7 +518,10 @@ class Singular(MarkdownDoc):
             'has_code': self.has_code,
         }
         if (self.enclosure):
-            v.update({'enclosure': self.enclosure})
+            v.update({
+                'enclosure': self.enclosure,
+                'has_mainimg': self.has_mainimg
+            })
         return v
 
     @property
@@ -488,6 +559,14 @@ class Singular(MarkdownDoc):
             self.content,
         ])
 
+    #async def update(self):
+        #fm = frontmatter.loads('')
+        #fm.metadata = self.meta
+        #fm.content = self.content
+        #with open(fpath, 'wt') as f:
+            #settings.logger.info("updating %s", fpath)
+            #f.write(frontmatter.dumps(fm))
+
     async def copyfiles(self):
         copystatics( os.path.dirname(self.fpath))
 
@@ -501,7 +580,6 @@ class Singular(MarkdownDoc):
             'meta': settings.meta,
             'licence': settings.licence,
             'tips': settings.tips,
-            'labels': settings.labels
         })
         if not os.path.isdir(self.renderdir):
             settings.logger.info("creating directory: %s", self.renderdir)
@@ -530,7 +608,7 @@ class WebImage(object):
                 self.Resized(self, max(self.width, self.height))
             ))
 
-    def __str__(self):
+    def mkstring(self, is_mainimg=False):
         if len(self.mdimg.css):
             return self.mdimg.match
         tmpl = J2.get_template("%s.j2.html" % (self.__class__.__name__))
@@ -543,6 +621,7 @@ class WebImage(object):
             'caption': self.caption,
             'exif': self.exif,
             'is_photo': self.is_photo,
+            'is_mainimg': is_mainimg
         })
 
     @cached_property
@@ -847,18 +926,143 @@ class AsyncWorker(object):
     def run(self):
         self._loop.run_until_complete(asyncio.wait(self._tasks))
 
-
-class IndexPHP(object):
-    def __init__(self):
-        self.gone = {}
-        self.redirect = {}
+class PHPFile(object):
+    @property
+    def exists(self):
+        if settings.args.get('force'):
+            return False
+        if not os.path.exists(self.renderfile):
+            return False
+        if self.mtime > os.path.getmtime(self.renderfile):
+            return False
+        return True
 
     @property
     def mtime(self):
-        r = 0
-        if os.path.exists(self.renderfile):
-            r = os.path.getmtime(self.renderfile)
-        return r
+        return os.path.getmtime(
+            os.path.join(
+                settings.paths.get('tmpl'),
+                self.templatefile
+            )
+        )
+
+    @property
+    def renderfile(self):
+        raise ValueError('Not implemented')
+
+    @property
+    def templatefile(self):
+        raise ValueError('Not implemented')
+
+    async def render(self):
+        if self.exists:
+            return
+        await self._render()
+
+
+class Search(PHPFile):
+    def __init__(self):
+        self.fpath = os.path.join(
+            settings.paths.get('build'),
+            'search.sqlite'
+        )
+        self.db = sqlite3.connect(self.fpath)
+        self.db.execute('PRAGMA auto_vacuum = INCREMENTAL;')
+        self.db.execute('PRAGMA journal_mode = MEMORY;')
+        self.db.execute('PRAGMA temp_store = MEMORY;')
+        self.db.execute('PRAGMA locking_mode = NORMAL;')
+        self.db.execute('PRAGMA synchronous = FULL;')
+        self.db.execute('PRAGMA encoding = "UTF-8";')
+        self.db.execute('''
+            CREATE VIRTUAL TABLE IF NOT EXISTS data USING fts4(
+                url,
+                mtime,
+                name,
+                title,
+                category,
+                content,
+                notindexed=category,
+                notindexed=url,
+                notindexed=mtime,
+                tokenize=porter
+            )'''
+        )
+
+    def __exit__(self):
+        self.db.commit()
+        self.db.execute('PRAGMA auto_vacuum;')
+        self.db.close()
+
+    def check(self, name):
+        ret = 0
+        maybe = self.db.execute('''
+            SELECT
+                mtime
+            FROM
+                data
+            WHERE
+                name = ?
+        ''', (name,)).fetchone()
+        if maybe:
+            ret = int(maybe[0])
+        return ret
+
+    def append(self, url, mtime, name, title, category, content):
+        mtime = int(mtime)
+        check = self.check(name)
+        if (check and check < mtime):
+            self.db.execute('''
+            DELETE
+            FROM
+                data
+            WHERE
+                name=?''', (name,))
+            check = False
+        if not check:
+            self.db.execute('''
+                INSERT INTO
+                    data
+                    (url, mtime, name, title, category, content)
+                VALUES
+                    (?,?,?,?,?,?);
+            ''', (
+                url,
+                mtime,
+                name,
+                title,
+                category,
+                content
+            ))
+
+    @property
+    def renderfile(self):
+        return os.path.join(
+            settings.paths.get('build'),
+            'search.php'
+        )
+
+    @property
+    def templatefile(self):
+        return 'Search.j2.php'
+
+    async def _render(self):
+        r = J2.get_template(self.templatefile).render({
+            'post': {},
+            'site': settings.site,
+            'author': settings.author,
+            'meta': settings.meta,
+            'licence': settings.licence,
+            'tips': settings.tips,
+        })
+        with open(self.renderfile, 'wt') as f:
+            settings.logger.info("rendering to %s", self.renderfile)
+            f.write(r)
+
+
+class IndexPHP(PHPFile):
+    def __init__(self):
+        self.gone = {}
+        self.redirect = {}
 
     def add_gone(self, uri):
         self.gone[uri] = True
@@ -878,8 +1082,12 @@ class IndexPHP(object):
             'index.php'
         )
 
-    async def render(self):
-        r = J2.get_template('Index.j2.php').render({
+    @property
+    def templatefile(self):
+        return 'Index.j2.php'
+
+    async def _render(self):
+        r = J2.get_template(self.templatefile).render({
             'post': {},
             'site': settings.site,
             'gones': self.gone,
@@ -890,7 +1098,7 @@ class IndexPHP(object):
             f.write(r)
 
 
-class WebhookPHP(object):
+class WebhookPHP(PHPFile):
     @property
     def renderfile(self):
         return os.path.join(
@@ -898,8 +1106,12 @@ class WebhookPHP(object):
             'webhook.php'
         )
 
-    async def render(self):
-        r = J2.get_template('Webhook.j2.php').render({
+    @property
+    def templatefile(self):
+        return 'Webhook.j2.php'
+
+    async def _render(self):
+        r = J2.get_template(self.templatefile).render({
             'author': settings.author,
             'callback_secret': keys.webmentionio.get('callback_secret'),
         })
@@ -1084,7 +1296,6 @@ class Category(dict):
             'meta': settings.meta,
             'licence': settings.licence,
             'tips': settings.tips,
-            'labels': settings.labels,
             'category': self.tmplvars,
             'pages': {
                 'current': pagenum,
@@ -1119,103 +1330,6 @@ class Category(dict):
         await self.render_feed()
 
 
-class Search(object):
-    def __init__(self):
-        self.changed = False
-        self.fpath = os.path.join(
-            settings.paths.get('build'),
-            'search.sqlite'
-        )
-        self.db = sqlite3.connect(self.fpath)
-        self.db.execute('PRAGMA auto_vacuum = INCREMENTAL;')
-        self.db.execute('PRAGMA journal_mode = MEMORY;')
-        self.db.execute('PRAGMA temp_store = MEMORY;')
-        self.db.execute('PRAGMA locking_mode = NORMAL;')
-        self.db.execute('PRAGMA synchronous = FULL;')
-        self.db.execute('PRAGMA encoding = "UTF-8";')
-        self.db.execute('''
-            CREATE VIRTUAL TABLE IF NOT EXISTS data USING fts4(
-                url,
-                mtime,
-                name,
-                title,
-                category,
-                content,
-                notindexed=category,
-                notindexed=url,
-                notindexed=mtime,
-                tokenize=porter
-            )'''
-        )
-
-    def __exit__(self):
-        if (self.changed):
-            self.db.commit()
-            self.db.execute('PRAGMA auto_vacuum;')
-            self.db.close()
-
-    def exists(self, name):
-        ret = 0
-        maybe = self.db.execute('''
-            SELECT
-                mtime
-            FROM
-                data
-            WHERE
-                name = ?
-        ''', (name,)).fetchone()
-        if maybe:
-            ret = int(maybe[0])
-        return ret
-
-    def append(self, url, mtime, name, title, category, content):
-        mtime = int(mtime)
-        exists = self.exists(name)
-        if (exists and exists < mtime):
-            self.db.execute('''
-            DELETE
-            FROM
-                data
-            WHERE
-                name=?''', (name,))
-            exists = False
-        if not exists:
-            self.db.execute('''
-                INSERT INTO
-                    data
-                    (url, mtime, name, title, category, content)
-                VALUES
-                    (?,?,?,?,?,?);
-            ''', (
-                url,
-                mtime,
-                name,
-                title,
-                category,
-                content
-            ))
-            self.changed = True
-
-    async def render(self):
-        target = os.path.join(
-            settings.paths.get('build'),
-            'search.php'
-        )
-        if os.path.exists(target):
-            return
-        r = J2.get_template('Search.j2.php').render({
-            'post': {},
-            'site': settings.site,
-            'author': settings.author,
-            'meta': settings.meta,
-            'licence': settings.licence,
-            'tips': settings.tips,
-            'labels': settings.labels
-        })
-        with open(target, 'wt') as f:
-            settings.logger.info("rendering to %s", target)
-            f.write(r)
-
 class Sitemap(dict):
     @property
     def mtime(self):
@@ -1280,11 +1394,7 @@ def mkcomment(webmention):
         fdir,
         "%d-%s.md" % (
             dt.timestamp,
-            slugify(
-                re.sub(r"^https?://(?:www)?", "", webmention.get('source')),
-                only_ascii=True,
-                lower=True
-            )[:200]
+            url2slug(webmention.get('source'))
         )
     )
 
@@ -1336,6 +1446,14 @@ def makecomments():
         pass
 
 
+def url2slug(url, limit=200):
+    return slugify(
+        re.sub(r"^https?://(?:www)?", "", url),
+        only_ascii=True,
+        lower=True
+    )[:limit]
+
+
 def make():
     start = int(round(time.time() * 1000))
     last = 0
@@ -1344,8 +1462,9 @@ def make():
 
     content = settings.paths.get('content')
     worker = AsyncWorker()
-    rules = IndexPHP()
+    webmentions = AsyncWorker()
 
+    rules = IndexPHP()
     for e in glob.glob(os.path.join(content, '*', '*.ptr')):
         post = Gone(e)
         if post.mtime > last:
@@ -1356,16 +1475,11 @@ def make():
         if post.mtime > last:
             last = post.mtime
         rules.add_redirect(post.source, post.target)
-
-    if rules.mtime < last or settings.args.get('force'):
-        worker.add(rules.render())
+    worker.add(rules.render())
 
     webhook = WebhookPHP()
     worker.add(webhook.render())
 
-    if rules.mtime < last or settings.args.get('force'):
-        worker.add(rules.render())
-
     sitemap = Sitemap()
     search = Search()
     categories = {}
@@ -1375,6 +1489,9 @@ def make():
         post = Singular(e)
         for i in post.images.values():
             worker.add(i.downsize())
+        for i in post.to_ping:
+            webmentions.add(i.send())
+
         worker.add(post.render())
         worker.add(post.copyfiles())
         if post.is_future:
@@ -1400,11 +1517,10 @@ def make():
 
     search.__exit__()
     worker.add(search.render())
+    worker.add(sitemap.render())
     for category in categories.values():
         worker.add(category.render())
 
-    worker.add(sitemap.render())
-
     worker.run()
     settings.logger.info('worker finished')
 
@@ -1427,7 +1543,17 @@ def make():
 
     end = int(round(time.time() * 1000))
     settings.logger.info('process took %d ms' % (end - start))
-
+    settings.logger.info('starting syncing')
+    os.system(
+        "rsync -avuhH --delete-after %s/ %s/" % (
+            settings.paths.get('build'),
+            settings.syncserver
+        )
+    )
+    settings.logger.info('syncing finished')
+    settings.logger.info('sending webmentions')
+    webmentions.run()
+    settings.logger.info('sending webmentions finished')
 
 if __name__ == '__main__':
     make()
diff --git a/requirements.txt b/requirements.txt
index 2ba1fb5..a59b465 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,23 +1,10 @@
 arrow==0.12.1
 bleach==2.1.3
-certifi==2018.4.16
-chardet==3.0.4
 emoji==0.5.0
 feedgen==0.7.0
-html5lib==1.0.1
-idna==2.7
 Jinja2==2.10
 langdetect==1.0.7
-lxml==4.2.4
-MarkupSafe==1.0
-pkg-resources==0.0.0
-python-dateutil==2.7.3
 python-frontmatter==0.4.2
-PyYAML==3.13
 requests==2.19.1
-six==1.11.0
 unicode-slugify==0.1.3
-Unidecode==1.0.22
-urllib3==1.23
 Wand==0.4.4
-webencodings==0.5.1
diff --git a/run b/run
index 3d2b4db..6176617 100755
--- a/run
+++ b/run
@@ -3,5 +3,5 @@
 set -euo pipefail
 IFS=$'\n\t'
 
-source ./.venv/bin/activate
+#source ./.venv/bin/activate
 python3 nasg.py "$@"
diff --git a/settings.py b/settings.py
index 9aa6a6b..ff7bd30 100644
--- a/settings.py
+++ b/settings.py
@@ -4,6 +4,7 @@ import argparse
 import logging
 
 base = os.path.abspath(os.path.expanduser('~/Projects/petermolnar.net'))
+syncserver = 'liveserver:/web/petermolnar.net/web'
 
 site = {
     'title': 'Peter Molnar',
@@ -16,10 +17,6 @@ site = {
         'journal'
     ],
     'licence': 'by-nc-nd',
-    'piwik': {
-        'domain': 'stats.petermolnar.net',
-        'id': 1
-    }
 }
 
 categorydisplay = {
@@ -33,25 +30,12 @@ licence = {
     'journal': 'by-nc',
 }
 
-labels = {
-    'tiptext': {
-        'photo':
-            "Did you like this photo?
" - "Leave a tip! If you're interested in prints, please get in touch.", - 'article': - "Did you find this article useful?
" - "Support me, so I can write more like this.
" - "If you want my help for your project, get in touch.", - 'journal': - "Did you like this entry?
" - "Encourage me to write more of them.", - } -} - meta = { 'webmention': 'https://webmention.io/petermolnar.net/webmention', 'pingback': 'https://webmention.io/petermolnar.net/xmlrpc', - 'hub': 'https://petermolnar.superfeedr.com/' + 'hub': 'https://petermolnar.superfeedr.com/', + 'authorization_endpoint': 'https://indieauth.com/auth', + 'token_endpoint': 'https://tokens.indieauth.com/token', } author = { @@ -61,14 +45,18 @@ author = { 'avatar': 'https://petermolnar.net/molnar_peter_avatar.jpg', 'gpg': 'https://petermolnar.net/pgp.asc', 'cv': 'https://petermolnar.net/about.html', - 'xmpp': 'mail@petermolnar.net', - 'flickr': 'petermolnareu', - 'github': 'petermolnar', - 'twitter': 'petermolnar' + 'contact': { + 'xmpp': 'xmpp:mail@petermolnar.net', + 'tumblr': 'https://petermolnarnet.tumblr.com/', + 'wordpress': 'https://petermolnareu.wordpress.com/', + 'flickr': 'https://flickr.com/people/petermolnareu', + 'github': 'https://github.com/petermolnar', + } } paths = { 'content': os.path.join(base, 'content'), + 'webmentions': os.path.join(base, 'content', 'webmentions'), 'tmpl': os.path.join(base, 'nasg', 'templates'), 'watermark': os.path.join(base, 'nasg', 'templates', 'watermark.png'), 'build': os.path.join(base, 'www'), @@ -85,20 +73,10 @@ photo = { }, } -tips = [ - { - 'name': 'paypal', - 'label': 'PayPal', - 'value': '£3', - 'url': 'https://paypal.me/petermolnar/3GBP', - }, - { - 'name': 'monzo', - 'label': 'Monzo (UK)', - 'value': '£3', - 'url': 'https://monzo.me/petermolnar/3', - }, -] +tips = { + 'paypal': 'https://paypal.me/petermolnar/3GBP', + 'monzo': 'https://monzo.me/petermolnar/3', +} dateformat = { 'iso': 'YYYY-MM-DDTHH:mm:ssZZ', diff --git a/sync b/sync deleted file mode 100755 index 3da4833..0000000 --- a/sync +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -IFS=$'\n\t' - -rsync -avuhH --delete-after ../www/ liveserver:/web/petermolnar.net/web/ diff --git a/templates/Category.j2.html b/templates/Category.j2.html index a6ca6b0..f9a0bb5 100644 --- a/templates/Category.j2.html +++ b/templates/Category.j2.html @@ -19,26 +19,26 @@ {% endif %} {% set _ = year.append(post.year)%} -
+
{% if category.display == 'flat' %} -

+

{% else %} -

+

{% endif %} {% if post.is_reply %} - + RE: {{ post.is_reply }} {% else %} - - {{ post.title }} + + {{ post.title }} {% endif %} {% if category.display == 'flat' %} @@ -49,7 +49,7 @@

{% if post.summary %} -
+
{{ post.html_summary }}

@@ -58,7 +58,7 @@

{% else %} -
+
{{ post.html_content }}
{% endif %} diff --git a/templates/WebImage.j2.html b/templates/WebImage.j2.html index 345d907..989e9e6 100644 --- a/templates/WebImage.j2.html +++ b/templates/WebImage.j2.html @@ -2,7 +2,7 @@ {% if href != src %}
{% endif %} - + {% if href != src %} {% endif %} diff --git a/templates/Webhook.j2.php b/templates/Webhook.j2.php index d12fbb2..7427cd1 100644 --- a/templates/Webhook.j2.php +++ b/templates/Webhook.j2.php @@ -13,5 +13,21 @@ if(! isset($payload['secret']) || $payload['secret'] != '{{ callback_secret }}' die('Bad Request'); } -mail("{{ author.email }}", "[webmention] {$payload['source']}", $raw); +$msg = sprintf(' +Type: %s +Source: %s +Target: %s +From: %s + +%s +', +$payload['post']['wm-property'], +$payload['source'], +$payload['target'], +$payload['post']['author']['name'], +$payload['post']['content']['text'] +); + + +mail("{{ author.email }}", "[webmention] {$payload['source']}", $msg); header('HTTP/1.1 202 Accepted'); diff --git a/templates/base.j2.html b/templates/base.j2.html index 7936f5a..815cbe4 100644 --- a/templates/base.j2.html +++ b/templates/base.j2.html @@ -36,7 +36,7 @@ } - + {% macro activemenu(name) %}{% if (post is defined and post.category == name ) or ( category is defined and category.name == name ) %}active{% endif %}{% endmacro %} @@ -103,9 +103,9 @@ {% block content %}
-
+
-

+

{% if post.is_reply %} @@ -128,35 +128,32 @@ {% if post.review %}
-
+
Review summary of: {{ post.review.title }}

By - at + + {{ author.name }} at

- - - {{ post.review.rating }} + + {{ post.review.rating }} out of - 5 + 5

-

{{ post.review.summary }}

+

{{ post.review.summary }}


{% endif %} {% if post.summary %} -
+
{{ post.html_summary }}
{% endif %} -
+
{{ post.html_content }}
@@ -167,35 +164,27 @@
Author
-

Photo of {{ author.name }} + /> - + >{{ author.name }} + <>

Entry URL
- {% if not post.has_mainimg %} - - {% endif %} - + {{ post.url }}
@@ -203,7 +192,7 @@
License
{% if post.licence == 'by' %} - CC BY 4.0 + CC BY 4.0
  • you can share it
  • you can republish it
  • @@ -212,7 +201,7 @@
  • you always need to make a link back here
{% elif post.licence.text == 'by-nc' %} - CC BY-NC 4.0 + CC BY-NC 4.0
  • you can share it
  • you can republish it
  • @@ -222,7 +211,7 @@
For commercial use, please contact me. {% else %} - CC BY-NC-ND 4.0 + CC BY-NC-ND 4.0