jsonfeed and gopher fixes
Peter Molnar hello@petermolnar.eu
Fri, 22 Mar 2019 15:49:24 +0000
9 files changed,
252 insertions(+),
253 deletions(-)
M
meta.py
→
meta.py
@@ -18,7 +18,6 @@ r'^(?P<year>[0-9]{4}):(?P<month>[0-9]{2}):(?P<day>[0-9]{2})\s+'
r'(?P<time>[0-9]{2}:[0-9]{2}:[0-9]{2})$' ) - class CachedMeta(dict): def __init__(self, fpath): self.fpath = fpath@@ -30,7 +29,7 @@ if fname == 'index.md':
fname = os.path.basename(os.path.dirname(self.fpath)) return os.path.join( - settings.paths.get('tmp', 'tmp'), + settings.tmpdir, "%s.%s.json" % ( fname, self.__class__.__name__,
M
nasg.py
→
nasg.py
@@ -34,7 +34,7 @@ from slugify import slugify
import requests import lxml.etree as etree -from pandoc import PandocMarkdown, PandocTXT +from pandoc import PandocMD2HTML, PandocMD2TXT, PandocHTML2TXT from meta import Exif import settings from settings import struct@@ -47,6 +47,9 @@ MDFILE = 'index.md'
TXTFILE = 'index.txt' HTMLFILE = 'index.html' GOPHERFILE = 'gophermap' +ATOMFILE = 'atom.xml' +RSSFILE = 'index.xml' +JSONFEEDFILE = 'index.json' MarkdownImage = namedtuple( 'MarkdownImage',@@ -110,6 +113,14 @@ return "%s" % (t)
J2.filters['printdate'] = rfc3339todt + + +def extractlicense(url): + """ extract license name """ + n, e = os.path.splitext(os.path.basename(url)) + return n.upper() + +J2.filters['extractlicense'] = extractlicense RE_MYURL = re.compile( r'(^(%s[^"]+)$|"(%s[^"]+)")' % (@@ -283,28 +294,29 @@ logger.debug('parsing YAML+MD file %s', self.fpath)
meta, txt = frontmatter.parse(f.read()) return(meta, txt) - @property + @cached_property def meta(self): return self._parsed[0] - @property + @cached_property def content(self): return self._parsed[1] - def pandoc(self, c): - if c and len(c): - c = str(PandocMarkdown(c)) - c = RE_PRECODE.sub( - '<pre><code lang="\g<1>" class="language-\g<1>">', c) - return c - @cached_property def html_content(self): c = "%s" % (self.content) + if not len(c): + return c + if hasattr(self, 'images') and len(self.images): for match, img in self.images.items(): c = c.replace(match, str(img)) - return self.pandoc(c) + c = str(PandocMD2HTML(c)) + c = RE_PRECODE.sub( + '<pre><code lang="\g<1>" class="language-\g<1>">', + c + ) + return c class Comment(MarkdownDoc):@@ -543,10 +555,16 @@ return self.meta.get('summary', '')
@cached_property def html_summary(self): - c = self.summary - if c and len(c): - c = self.pandoc(self.summary) - return c + c = "%s" % (self.summary) + return PandocMD2HTML(c) + + @cached_property + def txt_summary(self): + return PandocMD2TXT(self.summary) + + @cached_property + def txt_content(self): + return PandocMD2TXT(self.content) @property def title(self):@@ -930,8 +948,8 @@ )
g = { 'post': self.jsonld, - 'summary': PandocTXT(self.summary), - 'content': PandocTXT(self.content) + 'summary': self.txt_summary, + 'content': self.txt_content } writepath( self.gopherfile,@@ -992,9 +1010,9 @@ settings.site.name
) lines.append(line) lines.append('') - lines.append('') - lines = lines + list(map(lambda x: ("%s" % x), settings.bye.split('\n'))) - lines.append('') + #lines.append('') + #lines = lines + list(settings.bye.split('\n')) + #lines.append('') writepath(self.renderfile.replace(HTMLFILE,GOPHERFILE), "\r\n".join(lines)) async def render(self):@@ -1683,7 +1701,7 @@ def rssfeedfpath(self):
return os.path.join( self.dpath, 'feed', - 'index.xml' + RSSFILE ) @property@@ -1691,7 +1709,15 @@ def atomfeedfpath(self):
return os.path.join( self.dpath, 'feed', - 'atom.xml' + ATOMFILE + ) + + @property + def jsonfeedfpath(self): + return os.path.join( + self.dpath, + 'feed', + JSONFEEDFILE ) def get_posts(self, start=0, end=-1):@@ -1836,6 +1862,50 @@ fg.link(href=self.feedurl, rel='self')
fg.link(href=settings.meta.get('hub'), rel='hub') writepath(self.atomfeedfpath, fg.atom_str(pretty=True)) + async def render_json(self): + logger.info( + 'rendering category "%s" JSON feed', + self.name, + ) + start = 0 + end = int(settings.pagination) + + js = { + "version": "https://jsonfeed.org/version/1", + "title": self.title, + "home_page_url": settings.site.url, + "feed_url": "%s%s" % (self.url, JSONFEEDFILE), + "author": { + "name": settings.author.name, + "url": settings.author.url, + "avatar": settings.author.image, + }, + "items": [] + } + + for k in reversed(self.sortedkeys[start:end]): + post = self[k] + pjs = { + "id": post.url, + "content_text": post.txt_content, + "content_html": post.html_content, + "url": post.url, + "date_published": str(post.published), + } + if len(post.summary): + pjs.update({"summary": post.txt_summary}) + if post.is_photo: + pjs.update({"attachment": { + "url": post.photo.href, + "mime_type": post.photo.mime_type, + "size_in_bytes": "%d" % post.photo.mime_size + }}) + js["items"].append(pjs) + writepath( + self.jsonfeedfpath, + json.dumps(js, indent=4, ensure_ascii=False) + ) + async def render_flat(self): r = J2.get_template(self.template).render( self.tmplvars(self.get_posts())@@ -1856,6 +1926,9 @@ TXTFILE,
settings.site.name ) lines.append(line) + #lines.append(post.datePublished) + if (len(post.description)): + lines.extend(str(PandocHTML2TXT(post.description)).split("\n")) if isinstance(post['image'], list): for img in post['image']: line = "I%s\t/%s/%s\t%s\t70" % (@@ -1929,6 +2002,19 @@ '%s ATOM feed up to date',
self.name ) + if not self.is_uptodate(self.jsonfeedfpath, self.newest()): + logger.info( + '%s JSON feed outdated, generating new', + self.name + ) + await self.render_json() + else: + logger.info( + '%s JSON feed up to date', + self.name + ) + + async def render(self): await self.render_feeds() if not self.is_uptodate(self.indexfpath(), self.newest()):@@ -2081,6 +2167,31 @@ self.makecomment(webmention)
except ValueError as e: logger.error('failed to query webmention.io: %s', e) pass + + +# class GranaryIO(dict): + # granary = 'https://granary.io/url' + # convert_to = ['as2', 'mf2-json', 'jsonfeed'] + + # def __init__(self, source): + # self.source = source + + # def run(self): + # for c in self.convert_to: + # p = { + # 'url': self.source, + # 'input': html, + # 'output': c + # } + # r = requests.get(self.granary, params=p) + # logger.info("queried granary.io for %s for url: %s", c, self.source) + # if r.status_code != requests.codes.ok: + # continue + # try: + # self[c] = webmentions.text + # except ValueError as e: + # logger.error('failed to query granary.io: %s', e) + # pass def make():
M
pandoc.py
→
pandoc.py
@@ -6,17 +6,44 @@ __email__ = "mail@petermolnar.net"
import subprocess import logging +from tempfile import gettempdir +import hashlib +import os +import settings - -class PandocBase(str): +class Pandoc(str): in_format = 'html' in_options = [] out_format = 'plain' out_options = [] columns = None + @property + def hash(self): + return str(hashlib.sha1(self.source.encode()).hexdigest()) + + @property + def cachefile(self): + return os.path.join( + settings.tmpdir, + "%s_%s.pandoc" % ( + self.__class__.__name__, + self.hash + ) + ) + + @property + def cache(self): + if not os.path.exists(self.cachefile): + return False + with open(self.cachefile, 'rt') as f: + self.result = f.read() + return True + def __init__(self, text): self.source = text + if self.cache: + return conv_to = '--to=%s' % (self.out_format) if (len(self.out_options)): conv_to = '%s+%s' % (@@ -58,6 +85,8 @@ stderr
) r = stdout.decode('utf-8').strip() self.result = r + with open(self.cachefile, 'wt') as f: + f.write(self.result) def __str__(self): return str(self.result)@@ -66,7 +95,7 @@ def __repr__(self):
return str(self.result) -class PandocMarkdown(PandocBase): +class PandocMD2HTML(Pandoc): in_format = 'markdown' in_options = [ 'footnotes',@@ -86,7 +115,7 @@ out_format = 'html5'
out_options = [] -class PandocHTML(PandocBase): +class PandocHTML2MD(Pandoc): in_format = 'html' in_options = [] out_format = 'markdown'@@ -94,8 +123,6 @@ out_options = [
'footnotes', 'pipe_tables', 'strikeout', - # 'superscript', - # 'subscript', 'raw_html', 'definition_lists', 'backtick_code_blocks',@@ -106,14 +133,12 @@ 'autolink_bare_uris',
] -class PandocTXT(PandocBase): +class PandocMD2TXT(Pandoc): in_format = 'markdown' in_options = [ 'footnotes', 'pipe_tables', 'strikeout', - # 'superscript', - # 'subscript', 'raw_html', 'definition_lists', 'backtick_code_blocks',@@ -124,91 +149,12 @@ 'autolink_bare_uris',
] out_format = 'plain' out_options = [] - columns = '--columns=72' - - -#class PandocMarkdown(str): - #def __new__(cls, text): - #""" Pandoc command line call with piped in- and output """ - #cmd = ( - #'pandoc', - #'-o-', - #'--from=markdown+%s' % ( - #'+'.join([ - #'footnotes', - #'pipe_tables', - #'strikeout', - ## 'superscript', - ## 'subscript', - #'raw_html', - #'definition_lists', - #'backtick_code_blocks', - #'fenced_code_attributes', - #'shortcut_reference_links', - #'lists_without_preceding_blankline', - #'autolink_bare_uris', - #]) - #), - #'--to=html5', - #'--quiet', - #'--no-highlight' - #) - #p = subprocess.Popen( - #cmd, - #stdin=subprocess.PIPE, - #stdout=subprocess.PIPE, - #stderr=subprocess.PIPE, - #) - - #stdout, stderr = p.communicate(input=text.encode()) - #if stderr: - #logging.warning( - #"Error during pandoc covert:\n\t%s\n\t%s", - #cmd, - #stderr - #) - #r = stdout.decode('utf-8').strip() - #return str.__new__(cls, r) + columns = '--columns=80' -#class PandocHTML(str): - #def __new__(cls, text): - #""" Pandoc command line call with piped in- and output """ - #cmd = ( - #'pandoc', - #'-o-', - #'--to=markdown+%s' % ( - #'+'.join([ - #'footnotes', - #'pipe_tables', - #'strikeout', - ## 'superscript', - ## 'subscript', - #'raw_html', - #'definition_lists', - #'backtick_code_blocks', - #'fenced_code_attributes', - #'shortcut_reference_links', - #'lists_without_preceding_blankline', - #'autolink_bare_uris', - #]) - #), - #'--from=html', - #'--quiet', - #) - #p = subprocess.Popen( - #cmd, - #stdin=subprocess.PIPE, - #stdout=subprocess.PIPE, - #stderr=subprocess.PIPE, - #) - - #stdout, stderr = p.communicate(input=text.encode()) - #if stderr: - #logging.warning( - #"Error during pandoc covert:\n\t%s\n\t%s", - #cmd, - #stderr - #) - #r = stdout.decode('utf-8').strip() - #return str.__new__(cls, r) +class PandocHTML2TXT(Pandoc): + in_format = 'html' + in_options = [] + out_format = 'plain' + out_options = [] + columns = '--columns=80'
M
settings.py
→
settings.py
@@ -8,6 +8,7 @@ import os
import re import argparse import logging +from tempfile import gettempdir class struct(dict):@@ -156,7 +157,6 @@ 'queue': os.path.join(base, 'queue'),
'remotewww': 'web', 'remotequeue': 'queue', 'micropub': os.path.join(base, 'content', 'note'), - 'tmp': os.path.join(base, 'tmp'), 'home': os.path.join(base, 'content', 'home', 'index.md'), })@@ -171,28 +171,9 @@ 1280: '_b',
}, }) -bye = """ -███████╗███████╗███████╗ ██╗ ██╗ ██████╗ ██╗ ██╗ -██╔════╝██╔════╝██╔════╝ ╚██╗ ██╔╝██╔═══██╗██║ ██║ -███████╗█████╗ █████╗ ╚████╔╝ ██║ ██║██║ ██║ -╚════██║██╔══╝ ██╔══╝ ╚██╔╝ ██║ ██║██║ ██║ -███████║███████╗███████╗ ██║ ╚██████╔╝╚██████╔╝ -╚══════╝╚══════╝╚══════╝ ╚═╝ ╚═════╝ ╚═════╝ - -███████╗██████╗ █████╗ ██████╗███████╗ -██╔════╝██╔══██╗██╔══██╗██╔════╝██╔════╝ -███████╗██████╔╝███████║██║ █████╗ -╚════██║██╔═══╝ ██╔══██║██║ ██╔══╝ -███████║██║ ██║ ██║╚██████╗███████╗ -╚══════╝╚═╝ ╚═╝ ╚═╝ ╚═════╝╚══════╝ - - ██████╗ ██████╗ ██╗ ██╗██████╗ ██████╗ ██╗ ██╗ -██╔════╝██╔═══██╗██║ ██║██╔══██╗██╔═══██╗╚██╗ ██╔╝ -██║ ██║ ██║██║ █╗ ██║██████╔╝██║ ██║ ╚████╔╝ -██║ ██║ ██║██║███╗██║██╔══██╗██║ ██║ ╚██╔╝ -╚██████╗╚██████╔╝╚███╔███╔╝██████╔╝╚██████╔╝ ██║ - ╚═════╝ ╚═════╝ ╚══╝╚══╝ ╚═════╝ ╚═════╝ ╚═╝ - """ +tmpdir = os.path.join(gettempdir(),'nasg') +if not os.path.isdir(tmpdir): + os.makedirs(tmpdir) _parser = argparse.ArgumentParser(description='Parameters for NASG') _booleanparams = {
M
templates/Category.j2.html
→
templates/Category.j2.html
@@ -5,6 +5,7 @@ {% block title %}{{ category.title }}{% endblock %}
{% block meta %} <link rel="alternate" type="application/rss+xml" title="{{ category.title }} RSS feed" href="{{ category.feed }}" /> <link rel="alternate" type="application/atom+xml" title="{{ category.title }} ATOM feed" href="{{ category.feed }}atom.xml" /> + <link rel="alternate" type="application/json" title="{{ category.title }} JSON feed" href="{{ category.feed }}index.json" /> <link rel="feed" title="{{ category.title}} feed" href="{{ category.url }}" /> {% endblock %}
M
templates/Singular.j2.html
→
templates/Singular.j2.html
@@ -9,7 +9,6 @@ <meta name="description" content="{{ post.description|striptags|e }}" />
<link rel="canonical" href="{{ post.url }}" /> <link rel="alternate" type="application/json" href="{{ post.url }}index.json" /> <link rel="alternate" type="application/ld+json" href="{{ post.url }}index.json" /> - <link rel="alternate" type="application/mf2+json" href="https://pin13.net/mf2/?url={{ post.url|urlencode }}" /> <link rel="alternate" type="text/plain" href="{{ post.url }}index.txt" /> <meta property="og:title" content="{{ post.headline }}" /> <meta property="og:type" content="article" />@@ -75,6 +74,58 @@ {{ post.headline }}
</a> {% endif %} </h1> + + <p> + <a rel="license" href="{{ post.license }}" class="u-license"> + {{ post.license | extractlicense }} + </a> + by + <span class="p-author h-card vcard"> + <img class="u-photo photo" src="{{ post.author.image|relurl(baseurl) }}" alt="" /> + <a class="p-name u-url fn url" href="{{ post.author.url }}">{{ post.author.name }}</a> + (<a class="u-email email" href="mailto:{{ post.author.email }}">{{ post.author.email }}</a>) + </span> + at + <time datetime="{{ post.datePublished }}" class="dt-published published">{{ post.datePublished|printdate }}</time> + <time datetime="{{ post.dateModified }}" class="dt-updated updated"></time> + <br /> + <a class="u-url u-uuid bookmark" href="{{ post.url }}">{{ post.url }}</a> +<!-- +{% if post.sameAs|length %} + <br /> + Syndicated to: + <ul> + {% for url in post.sameAs %} + <li> + <a class="u-syndication" href="{{ url }}"> + {{ url }} + </a> + </li> + {% endfor %} + </ul> +{% endif %} +--> + </p> + +{% if post.subjectOf %} + <p class="h-event vevent"> + <span class="summary"> + Journey from + <time class="dt-start dtstart" datetime="{{ post.subjectOf.startDate }}"> + {{ post.subjectOf.startDate|printdate }} + </time> + to + <time class="dt-end dtend" datetime="{{ post.subjectOf.endDate }}"> + {{ post.subjectOf.endDate|printdate }} + </time>, in + <span class="p-location location"> + {{ post.subjectOf.location.name }} + </span> + </span> + <a class="u-url url" href="{{ post.url }}"></a> + </p> +{% endif %} + </header> {% if post.review %}@@ -107,106 +158,6 @@
<div class="e-content entry-content"> {{ post.text|relurl(baseurl) }} </div> - - <footer> - <dl> - {% if post.subjectOf %} - <dt>Trip details</dt> - <dd class="h-event vevent"> - <span class="summary"> - From - <time class="dt-start dtstart" datetime="{{ post.subjectOf.startDate }}"> - {{ post.subjectOf.startDate|printdate }} - </time> - to - <time class="dt-end dtend" datetime="{{ post.subjectOf.endDate }}"> - {{ post.subjectOf.endDate|printdate }} - </time>, in - <span class="p-location location"> - {{ post.subjectOf.location.name }} - </span> - </span> - <a class="u-url url" href="{{ post.url }}"></a> - </dd> - {% endif %} - - <dt>Published</dt> - <dd> - <time datetime="{{ post.datePublished }}" class="dt-published published">{{ post.datePublished|printdate }}</time> - <time datetime="{{ post.dateModified }}" class="dt-updated updated"></time> - </dd> - - <dt>License</dt> - <dd class="license"> - {% if 'CC-BY-4.0' in post.license %} - <a rel="license" href="{{ post.license }}" class="u-license"> - CC-BY-4.0 - </a> - <ul> - <li>you can share it</li> - <li>you can republish it</li> - <li>you can modify it, but you need to indicate the modifications</li> - <li>you can use it for commercial purposes</li> - <li>you always need to make a link back here</li> - </ul> - {% elif 'CC-BY-NC-4.0' in post.license %} - <a rel="license" href="{{ post.license }}" class="u-license"> - CC-BY-NC-4.0 - </a> - <ul> - <li>you can share it</li> - <li>you can republish it</li> - <li>you can modify it, but you need to indicate the modifications</li> - <li>you can't use it for commercial purposes</li> - <li>you always need to make a link back here</li> - </ul> - For commercial use, please contact me. - {% elif 'CC-BY-NC-ND-4.0' in post.license %} - <a rel="license" href="{{ post.license }}" class="u-license"> - CC-BY-NC-ND-4.0 - </a> - <ul> - <li>you can share it</li> - <li>you can't modify it</li> - <li>you can't republish it</li> - <li>you can't use it for commercial purposes</li> - <li>you always need to make a link back here</li> - </ul> - For commercial use, please contact me. - {% endif %} - </dd> - - <dt>Author</dt> - <dd class="p-author h-card vcard"> - <img class="u-photo photo" src="{{ post.author.image|relurl(baseurl) }}" alt="" /> - <a class="p-name u-url fn url" href="{{ post.author.url }}">{{ post.author.name }}</a> - <a class="u-email email" href="mailto:{{ post.author.email }}">{{ post.author.email }}</a> - </dd> - - <dt>Entry URL</dt> - <dd> - <a class="u-url u-uuid bookmark" href="{{ post.url }}"> - {{ post.url }} - </a> - </dd> - - {% if post.sameAs|length %} - <dt>Also on</dt> - <dd> - <ul> - {% for url in post.sameAs %} - <li> - <a class="u-syndication" href="{{ url }}"> - {{ url }} - </a> - </li> - {% endfor %} - </ul> - </dd> - {% endif %} - - </dl> - </footer> {% if 'WebPage' != post['@type'] %} <section class="syndication">
M
templates/Singular.j2.txt
→
templates/Singular.j2.txt
@@ -1,9 +1,7 @@
---- -Title: {{ post.headline }} -Author: {{ post.author.name }} <{{ post.author.email}}> -URL: {{ post.url }} -Published: {{ post.datePublished|printdate }} ---- +{{ post.headline|center(width=80) }} + +by {{ post.author.name }} <{{ post.author.email}}> +{{ post.datePublished|printdate }} {{ summary }}
M
templates/base.j2.html
→
templates/base.j2.html
@@ -137,9 +137,9 @@ {% endfor %}
</ul> </nav> <nav> - <a href="https://xn--sr8hvo.ws/🇻🇮📢/previous">←</a> + <a href="https://xn--sr8hvo.ws/%F0%9F%87%BB%F0%9F%87%AE%F0%9F%93%A2/previous">←</a> Member of <a href="https://xn--sr8hvo.ws">IndieWeb Webring</a> - <a href="https://xn--sr8hvo.ws/🇻🇮📢/next">→</a> + <a href="https://xn--sr8hvo.ws/%F0%9F%87%BB%F0%9F%87%AE%F0%9F%93%A2/next">→</a> </nav> </section> <section>
M
templates/style.css
→
templates/style.css
@@ -49,6 +49,8 @@
h1 { border-bottom: 4px double #999; text-transform:uppercase; + text-align: center; + padding-bottom: 1em; } article > footer > dl > dt,@@ -215,7 +217,7 @@ figcaption > dl dd {
margin: 0 0.3em; } -footer img { +.vcard img { height: 1em; }@@ -276,6 +278,16 @@ main > header > p {
text-align: center; } +article > header { + border-bottom: 4px double #999; + margin-bottom: 2em; +} + +.h-feed article > header { + border: none; + margin: 0; +} + main ul { margin-left: 2em; }@@ -343,4 +355,4 @@ bottom: 0;
right: 0; width: 10em; height: auto; -} +}