removed enclosure from rss; re-added initial steps for sending webmentions for replies; minor cleanups

README.md

@@ -1,17 +1,36 @@
-# NASG (Not Another Static Generator)
+# NASG (Not Another Static Generator...)
 
 This is a tiny static site generator, written in Python, to scratch my own itches.
-It is most probably not suitable for anyone else.
+It is most probably not suitable for anyone else, but feel free to use it for ideas. Keep in mind that the project is licenced under GPL.
 
 ## Why not [insert static generator here]?
 
-- DRY -Don't Repeat Yourself - is good, so instead of sidefiles for images, I'm using XMP metadata, which most of the ones availabe don't handle well;
+- DRY - Don't Repeat Yourself - is good, so instead of sidefiles for images, I'm using XMP metadata, which most of the ones availabe don't handle well;
-- writing a proper plugin to existing generators - Pelican, Nicola, etc - might have taken longer and I wanted to extend my Python knowledge
+- writing plugins to existing generators - Pelican, Nicola, etc - might have taken longer and I wanted to extend my Python knowledge
 - I wanted to use the best available utilities for some tasks, like `Pandoc` and  `exiftool` instead of Python libraries trying to achive the same
 - I needed to handle webmentions and comments
 
 Don't expect anything fancy: my Python Fu has much to learn.
 
+## Install
+
+### External dependencies
+
+PHP is in order to use [XRay](https://github.com/aaronpk/XRay/)
+
+```
+apt-get install pandoc exiftool php7.0-bcmath php7.0-bz2 php7.0-cli php7.0-common php7.0-curl php7.0-gd php7.0-imap php7.0-intl php7.0-json php7.0-mbstring php7.0-mcrypt php7.0-mysql php7.0-odbc php7.0-opcache php7.0-readline php7.0-sqlite3 php7.0-xml php7.0-zip python3 python3-pip python3-dev
+```
+
+Get XRay:
+```
+mkdir /usr/local/lib/php
+cd /usr/local/lib/php
+wget https://github.com/aaronpk/XRay/releases/download/v1.3.1/xray-app.zip
+unzip xray-app.zip
+rm xray-app.zip
+```
+
 ## How content is organized
 
 The directory structure of the "source" is something like this:

__init__.py

@@ -1,3 +1 @@
 #!/usr/bin/env python3
-
-__version__ = '2.0.0'

nasg.py

@@ -13,7 +13,6 @@ import asyncio
 from math import ceil
 import csv
 import sqlite3
-import magic
 
 import frontmatter
 import arrow
@@ -191,7 +190,6 @@ class Category(NoDupeContainer):
 
     @property
     def title(self):
-        # TODO proper title
         return self.name
 
     @property
@@ -311,26 +309,37 @@ class Singular(object):
 
 
     def init_extras(self):
-        self.process_webmentions()
+        self.receive_webmentions()
         c = self.comments
 
-
+    # note: due to SQLite locking, this will not be async for now
-    # TODO this should be async
+    def receive_webmentions(self):
-    def process_webmentions(self):
         wdb = shared.WebmentionQueue()
         queued = wdb.get_queued(self.url)
         for incoming in queued:
             wm = Webmention(
-                incoming.get('id'),
                 incoming.get('source'),
                 incoming.get('target'),
                 incoming.get('dt')
             )
-            wm.run()
+            wm.receive()
-
             wdb.entry_done(incoming.get('id'))
         wdb.finish()
 
+    # note: due to SQLite locking, this will not be async for now
+    def send_webmentions(self):
+        if not self.is_reply:
+            return
+        wdb = shared.WebmentionQueue()
+        id = wdb.queue(self.url, self.is_reply)
+        wm = Webmention(
+            self.url,
+            self.is_reply
+        )
+        wm.send()
+        wdb.entry_done(id)
+        wdb.finish()
+
     @property
     def redirects(self):
         r = self.meta.get('redirect', [])
@@ -523,9 +532,6 @@ class Singular(object):
             im.title = title
             im.cssclass = css
             body = body.replace(shortcode, str(im))
-
-        # TODO if multiple meta images, inline all except the first
-        # which will be added at the HTML stage or as enclosure to the feed
         return body
 
     @property
@@ -558,16 +564,6 @@ class Singular(object):
     def shortslug(self):
         return shared.baseN(self.pubtime)
 
-    @property
-    def enclosure(self):
-        if not self.photo:
-            return {}
-        return {
-            'length': os.path.getsize(self.photo.fpath),
-            'url': self.photo.href,
-            'mime': magic.Magic(mime=True).from_file(self.photo.fpath),
-        }
-
     @property
     def tmplvars(self):
         # very simple caching because we might use this 4 times:
@@ -584,13 +580,11 @@ class Singular(object):
                 'slug': self.fname,
                 'shortslug': self.shortslug,
                 'licence': self.licence,
-                #'sourceurl': self.sourceurl,
                 'is_reply': self.is_reply,
                 'age': int(self.published.format('YYYY')) - int(arrow.utcnow().format('YYYY')),
                 'summary': self.summary,
                 'replies': self.replies,
                 'reactions': self.reactions,
-                'enclosure': self.enclosure,
             }
         return self._tmplvars
 
@@ -1049,10 +1043,9 @@ class Comment(object):
 
 
 class Webmention(object):
-    def __init__ (self, id, source, target, dt):
+    def __init__ (self, source, target, dt=arrow.utcnow().timestamp):
         self.source = source
         self.target = target
-        self.id = id
         self.dt = arrow.get(dt).to('utc')
         logging.info(
             "processing webmention %s => %s",
@@ -1071,7 +1064,26 @@ class Webmention(object):
             f.write(frontmatter.dumps(fm))
         return
 
-    def run(self):
+    def send(self):
+        rels = shared.XRay(self.source).set_discover().parse()
+        endpoint = False
+        if 'rels' not in rels:
+            return
+        for k in rels.get('rels').keys():
+            if 'webmention' in k:
+                endpoint = rels.get('rels').get(k)
+                break
+        if not endpoint:
+            return
+        requests.post(
+            self.target,
+            data = {
+                'source': self.source,
+                'target': self.target
+            }
+        )
+
+    def receive(self):
         self._fetch()
         if 'data' not in self._source:
             return
@@ -1131,7 +1143,6 @@ class Webmention(object):
             self.fname
         )
 
-
 def setup():
     """ parse input parameters and add them as params section to config """
     parser = argparse.ArgumentParser(description='Parameters for NASG')
@@ -1281,8 +1292,6 @@ def build():
     tasks.append(task)
 
     # TODO: send webmentions to any url
-    # TODO: comments
-    # TODO: ping websub?
 
     # do all the things!
     w = asyncio.wait(tasks)

setup.py

@@ -1,16 +1,25 @@
 from setuptools import setup, find_packages
-from . import __version__
 
 setup(
-    version=__version__,
+    version='2.0.0',
     name="nasg",
     author="Peter Molnar",
     author_email="hello@petermolnar.eu",
     description="Not Another Static Generator - a static generator",
     long_description=open('README.md').read(),
     packages=['nasg'],
-    install_requires=['arrow', 'Jinja2', 'langdetect', 'requests', 'requests-oauthlib', 'sanic', 'unicode-slugify', 'Wand', 'emoji', 'html5lib', 'BeautifulSoup'],
+    install_requires=[
+        'arrow',
+        'Jinja2',
+        'langdetect',
+        'requests',
+        'requests-oauthlib',
+        'sanic',
+        'unicode-slugify',
+        'Wand',
+        'emoji',
+    ],
     url='https://github.com/petermolnar/nasg',
-    license=open('LICENCE').read(),
+    license=open('./LICENSE').read(),
     include_package_data=True,
 )

shared.py

@@ -7,7 +7,6 @@ import subprocess
 import json
 import sqlite3
 import requests
-
 from slugify import slugify
 import jinja2
 
@@ -28,21 +27,48 @@ class CMDLine(object):
 
 
 class XRay(CMDLine):
-    xraypath = '/usr/local/lib/php/xray'
+    cmd_prefix = 'chdir("/usr/local/lib/php/xray"); include("vendor/autoload.php"); $xray = new p3k\XRay();'
 
     def __init__(self, url):
         super().__init__('php')
         self.url = url
-
+        self.target = ''
-    def parse(self):
+        self.cmd = (
-        cmd = (
             self.executable,
             '-r',
-            '''chdir("%s"); include("vendor/autoload.php"); $xray = new p3k\XRay(); echo(json_encode($xray->parse("%s")));''' % (self.xraypath, self.url)
+            '%s; echo(json_encode($xray->parse("%s")));' % (
+                self.cmd_prefix,
+                self.url
             )
+        )
+
+    def set_receive(self, target):
+        self.cmd = (
+            self.executable,
+            '-r',
+            '%s; echo(json_encode($xray->parse("%s")));' % (
+                self.cmd_prefix,
+                self.url,
+                target
+            )
+        )
+        return self
+
+    def set_discover(self):
+        self.cmd = (
+            self.executable,
+            '-r',
+            '%s; echo(json_encode($xray->rels("%s")));' % (
+                self.cmd_prefix,
+                self.url,
+            )
+        )
+        return self
+
+    def parse(self):
         logging.debug('pulling %s with XRay', self.url)
         p = subprocess.Popen(
-            cmd,
+            self.cmd,
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
@@ -54,7 +80,6 @@ class XRay(CMDLine):
 
         return json.loads(stdout.decode('utf-8').strip())
 
-
 class Pandoc(CMDLine):
     """ Pandoc command line call with piped in- and output """
 
@@ -439,7 +464,9 @@ class WebmentionQueue(object):
                 target
             )
         )
+        r = cursor.lastrowid
         self.db.commit()
+        return r
 
     def get_queued(self, fname=None):
         logging.debug('getting queued webmentions for %s', fname)

templates/Category_feed.html

@@ -19,13 +19,6 @@
 {%- if post.tags %}{% for tname in post.tags %}
 			<category>{{ tname }}></category>
 {% endfor %}{% endif -%}
-{%- if post.rssenclosure %}
-		<enclosure
-			url="{{ post.enclosure.url }}"
-			type="{{ post.enclosure.mime }}"
-			length="{{ post.enclosure.size }}"
-		/>
-{% endif -%}
 	</item>
 {% endfor %}
 </channel>