From 3bef504be152149be1f331528aee9b8763b84221 Mon Sep 17 00:00:00 2001 From: Peter Molnar Date: Mon, 30 Oct 2017 10:47:08 +0000 Subject: [PATCH] removed enclosure from rss; re-added initial steps for sending webmentions for replies; minor cleanups --- README.md | 27 +++++++++++--- __init__.py | 2 -- nasg.py | 69 ++++++++++++++++++++---------------- setup.py | 17 ++++++--- shared.py | 43 +++++++++++++++++----- templates/Category_feed.html | 7 ---- 6 files changed, 110 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index d728b97..ee5e6a6 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,36 @@ -# NASG (Not Another Static Generator) +# NASG (Not Another Static Generator...) This is a tiny static site generator, written in Python, to scratch my own itches. -It is most probably not suitable for anyone else. +It is most probably not suitable for anyone else, but feel free to use it for ideas. Keep in mind that the project is licenced under GPL. ## Why not [insert static generator here]? -- DRY -Don't Repeat Yourself - is good, so instead of sidefiles for images, I'm using XMP metadata, which most of the ones availabe don't handle well; -- writing a proper plugin to existing generators - Pelican, Nicola, etc - might have taken longer and I wanted to extend my Python knowledge +- DRY - Don't Repeat Yourself - is good, so instead of sidefiles for images, I'm using XMP metadata, which most of the ones availabe don't handle well; +- writing plugins to existing generators - Pelican, Nicola, etc - might have taken longer and I wanted to extend my Python knowledge - I wanted to use the best available utilities for some tasks, like `Pandoc` and `exiftool` instead of Python libraries trying to achive the same - I needed to handle webmentions and comments Don't expect anything fancy: my Python Fu has much to learn. +## Install + +### External dependencies + +PHP is in order to use [XRay](https://github.com/aaronpk/XRay/) + +``` +apt-get install pandoc exiftool php7.0-bcmath php7.0-bz2 php7.0-cli php7.0-common php7.0-curl php7.0-gd php7.0-imap php7.0-intl php7.0-json php7.0-mbstring php7.0-mcrypt php7.0-mysql php7.0-odbc php7.0-opcache php7.0-readline php7.0-sqlite3 php7.0-xml php7.0-zip python3 python3-pip python3-dev +``` + +Get XRay: +``` +mkdir /usr/local/lib/php +cd /usr/local/lib/php +wget https://github.com/aaronpk/XRay/releases/download/v1.3.1/xray-app.zip +unzip xray-app.zip +rm xray-app.zip +``` + ## How content is organized The directory structure of the "source" is something like this: diff --git a/__init__.py b/__init__.py index efad603..e5a0d9b 100644 --- a/__init__.py +++ b/__init__.py @@ -1,3 +1 @@ #!/usr/bin/env python3 - -__version__ = '2.0.0' diff --git a/nasg.py b/nasg.py index 684cd06..3766e04 100644 --- a/nasg.py +++ b/nasg.py @@ -13,7 +13,6 @@ import asyncio from math import ceil import csv import sqlite3 -import magic import frontmatter import arrow @@ -191,7 +190,6 @@ class Category(NoDupeContainer): @property def title(self): - # TODO proper title return self.name @property @@ -311,26 +309,37 @@ class Singular(object): def init_extras(self): - self.process_webmentions() + self.receive_webmentions() c = self.comments - - # TODO this should be async - def process_webmentions(self): + # note: due to SQLite locking, this will not be async for now + def receive_webmentions(self): wdb = shared.WebmentionQueue() queued = wdb.get_queued(self.url) for incoming in queued: wm = Webmention( - incoming.get('id'), incoming.get('source'), incoming.get('target'), incoming.get('dt') ) - wm.run() - + wm.receive() wdb.entry_done(incoming.get('id')) wdb.finish() + # note: due to SQLite locking, this will not be async for now + def send_webmentions(self): + if not self.is_reply: + return + wdb = shared.WebmentionQueue() + id = wdb.queue(self.url, self.is_reply) + wm = Webmention( + self.url, + self.is_reply + ) + wm.send() + wdb.entry_done(id) + wdb.finish() + @property def redirects(self): r = self.meta.get('redirect', []) @@ -523,9 +532,6 @@ class Singular(object): im.title = title im.cssclass = css body = body.replace(shortcode, str(im)) - - # TODO if multiple meta images, inline all except the first - # which will be added at the HTML stage or as enclosure to the feed return body @property @@ -558,16 +564,6 @@ class Singular(object): def shortslug(self): return shared.baseN(self.pubtime) - @property - def enclosure(self): - if not self.photo: - return {} - return { - 'length': os.path.getsize(self.photo.fpath), - 'url': self.photo.href, - 'mime': magic.Magic(mime=True).from_file(self.photo.fpath), - } - @property def tmplvars(self): # very simple caching because we might use this 4 times: @@ -584,13 +580,11 @@ class Singular(object): 'slug': self.fname, 'shortslug': self.shortslug, 'licence': self.licence, - #'sourceurl': self.sourceurl, 'is_reply': self.is_reply, 'age': int(self.published.format('YYYY')) - int(arrow.utcnow().format('YYYY')), 'summary': self.summary, 'replies': self.replies, 'reactions': self.reactions, - 'enclosure': self.enclosure, } return self._tmplvars @@ -1049,10 +1043,9 @@ class Comment(object): class Webmention(object): - def __init__ (self, id, source, target, dt): + def __init__ (self, source, target, dt=arrow.utcnow().timestamp): self.source = source self.target = target - self.id = id self.dt = arrow.get(dt).to('utc') logging.info( "processing webmention %s => %s", @@ -1071,7 +1064,26 @@ class Webmention(object): f.write(frontmatter.dumps(fm)) return - def run(self): + def send(self): + rels = shared.XRay(self.source).set_discover().parse() + endpoint = False + if 'rels' not in rels: + return + for k in rels.get('rels').keys(): + if 'webmention' in k: + endpoint = rels.get('rels').get(k) + break + if not endpoint: + return + requests.post( + self.target, + data = { + 'source': self.source, + 'target': self.target + } + ) + + def receive(self): self._fetch() if 'data' not in self._source: return @@ -1131,7 +1143,6 @@ class Webmention(object): self.fname ) - def setup(): """ parse input parameters and add them as params section to config """ parser = argparse.ArgumentParser(description='Parameters for NASG') @@ -1281,8 +1292,6 @@ def build(): tasks.append(task) # TODO: send webmentions to any url - # TODO: comments - # TODO: ping websub? # do all the things! w = asyncio.wait(tasks) diff --git a/setup.py b/setup.py index f5faa46..e3e648c 100644 --- a/setup.py +++ b/setup.py @@ -1,16 +1,25 @@ from setuptools import setup, find_packages -from . import __version__ setup( - version=__version__, + version='2.0.0', name="nasg", author="Peter Molnar", author_email="hello@petermolnar.eu", description="Not Another Static Generator - a static generator", long_description=open('README.md').read(), packages=['nasg'], - install_requires=['arrow', 'Jinja2', 'langdetect', 'requests', 'requests-oauthlib', 'sanic', 'unicode-slugify', 'Wand', 'emoji', 'html5lib', 'BeautifulSoup'], + install_requires=[ + 'arrow', + 'Jinja2', + 'langdetect', + 'requests', + 'requests-oauthlib', + 'sanic', + 'unicode-slugify', + 'Wand', + 'emoji', + ], url='https://github.com/petermolnar/nasg', - license=open('LICENCE').read(), + license=open('./LICENSE').read(), include_package_data=True, ) diff --git a/shared.py b/shared.py index e9878eb..8b116ba 100644 --- a/shared.py +++ b/shared.py @@ -7,7 +7,6 @@ import subprocess import json import sqlite3 import requests - from slugify import slugify import jinja2 @@ -28,21 +27,48 @@ class CMDLine(object): class XRay(CMDLine): - xraypath = '/usr/local/lib/php/xray' + cmd_prefix = 'chdir("/usr/local/lib/php/xray"); include("vendor/autoload.php"); $xray = new p3k\XRay();' def __init__(self, url): super().__init__('php') self.url = url - - def parse(self): - cmd = ( + self.target = '' + self.cmd = ( self.executable, '-r', - '''chdir("%s"); include("vendor/autoload.php"); $xray = new p3k\XRay(); echo(json_encode($xray->parse("%s")));''' % (self.xraypath, self.url) + '%s; echo(json_encode($xray->parse("%s")));' % ( + self.cmd_prefix, + self.url + ) ) + + def set_receive(self, target): + self.cmd = ( + self.executable, + '-r', + '%s; echo(json_encode($xray->parse("%s")));' % ( + self.cmd_prefix, + self.url, + target + ) + ) + return self + + def set_discover(self): + self.cmd = ( + self.executable, + '-r', + '%s; echo(json_encode($xray->rels("%s")));' % ( + self.cmd_prefix, + self.url, + ) + ) + return self + + def parse(self): logging.debug('pulling %s with XRay', self.url) p = subprocess.Popen( - cmd, + self.cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -54,7 +80,6 @@ class XRay(CMDLine): return json.loads(stdout.decode('utf-8').strip()) - class Pandoc(CMDLine): """ Pandoc command line call with piped in- and output """ @@ -439,7 +464,9 @@ class WebmentionQueue(object): target ) ) + r = cursor.lastrowid self.db.commit() + return r def get_queued(self, fname=None): logging.debug('getting queued webmentions for %s', fname) diff --git a/templates/Category_feed.html b/templates/Category_feed.html index cd29a3a..5c5a92a 100644 --- a/templates/Category_feed.html +++ b/templates/Category_feed.html @@ -19,13 +19,6 @@ {%- if post.tags %}{% for tname in post.tags %} {{ tname }}> {% endfor %}{% endif -%} -{%- if post.rssenclosure %} - -{% endif -%} {% endfor %}