alpha3, webmentions still not put back
@@ -3,8 +3,7 @@ import json
import sqlite3 import glob import shared - -# TODO sqlite3 cache instead of filesystem ? +import logging class TokenDB(object): def __init__(self, uuid='tokens'):@@ -105,15 +104,9 @@ self.db.close()
def append(self, id, corpus, mtime, url, category, title): mtime = int(mtime) + logging.debug("adding %s to searchdb", id) cursor = self.db.cursor() - cursor.execute('''UPDATE data SET corpus=?, mtime=?, url=?, category=?, title=? WHERE id=?;''', ( - corpus, - mtime, - url, - category, - title, - id - )) + cursor.execute('''DELETE FROM data WHERE id=?''', (id,)) cursor.execute('''INSERT OR IGNORE INTO data (id, corpus, mtime, url, category, title) VALUES (?,?,?,?,?,?);''', ( id, corpus,@@ -125,6 +118,7 @@ ))
self.db.commit() def is_uptodate(self, fname, mtime): + mtime = int(mtime) ret = {} cursor = self.db.cursor() cursor.execute('''SELECT mtime@@ -133,8 +127,12 @@ WHERE id = ? AND mtime = ?''',
(fname,mtime) ) rows = cursor.fetchall() + if len(rows): + logging.debug("%s is up to date in searchdb", fname) return True + + logging.debug("%s is out of date in searchdb", fname) return False def search_by_query(self, query):
@@ -14,6 +14,7 @@ import asyncio
from math import ceil import csv import sqlite3 +import magic import frontmatter import arrow@@ -167,7 +168,7 @@
class Category(NoDupeContainer): """ A Category which holds pubtime (int) => Singular data """ indexfile = 'index.html' - feedfile = 'index.atom' + feedfile = 'index.xml' feeddir = 'feed' pagedir = 'page' taxonomy = 'category'@@ -198,20 +199,32 @@ def title(self):
# TODO proper title return self.name - def url_paged(self, page=1, feed=False): - x = '/' + @property + def url(self): if self.name: - x = "%s%s/%s" % ( - x, + url = "/%s/%s/" % ( self.taxonomy, self.name, ) - - if page == 1 and feed: - x = "%s/%s/" % (x, self.feeddir) else: - x = "%s/%s/%s/" % (x, self.pagedir, "%s" % page) - return x + url = '/' + return url + + + #def url_paged(self, page=1, feed=False): + #x = '/' + #if self.name: + #x = "%s%s/%s" % ( + #x, + #self.taxonomy, + #self.name, + #) + + #if page == 1 and feed: + #x = "%s/%s/" % (x, self.feeddir) + #else: + #x = "%s/%s/%s/" % (x, self.pagedir, "%s" % page) + #return x def path_paged(self, page=1, feed=False): x = shared.config.get('common', 'build')@@ -242,9 +255,6 @@ os.utime(path, (self.mtime, self.mtime))
async def render(self): - if self.is_uptodate: - return - pagination = shared.config.getint('display', 'pagination') pages = ceil(len(self.data) / pagination) page = 1@@ -267,9 +277,12 @@ 'name': self.name,
'page': page, 'total': pages, 'perpage': pagination, - 'lastmod': arrow.get(self.mtime).format(shared.ARROWFORMAT['iso']), - 'feed': self.url_paged(page=1, feed=True), - 'url': self.url_paged(page), + 'lastmod': arrow.get(self.mtime).format(shared.ARROWFORMAT['rcf']), + 'url': self.url, + 'feed': "%s/%s/" % ( + self.url, + shared.config.get('site', 'feed') + ), }, 'site': shared.site, 'posts': posttmpls,@@ -464,19 +477,23 @@ pass
return lang def _find_image(self, fname): + fname = os.path.basename(fname) pattern = os.path.join( shared.config.get('dirs', 'files'), - '*', + '**', fname ) logging.debug('trying to locate image %s in %s', fname, pattern) maybe = glob.glob(pattern) if not maybe: + logging.error('image not found: %s', fname) return None + maybe = maybe.pop() + logging.debug('image found: %s', maybe) if fname not in self._images: - im = WebImage(maybe.pop()) + im = WebImage(maybe) self._images.append(fname,im) return self._images[fname]@@ -493,7 +510,7 @@ def body(self):
body = "%s" % (self.content) # get inline images, downsize them and convert them to figures for shortcode, alt, fname, title, css in self.inline_images: - fname = os.path.basename(fname) + #fname = os.path.basename(fname) im = self._find_image(fname) if not im: continue@@ -538,6 +555,16 @@ def shortslug(self):
return shared.baseN(self.pubtime) @property + def enclosure(self): + if not self.photo: + return {} + return { + 'length': os.path.getsize(self.photo.fpath), + 'url': self.photo.href, + 'mime': magic.Magic(mime=True).from_file(self.photo.fpath), + } + + @property def tmplvars(self): # very simple caching because we might use this 4 times: # post HTML, category, front posts and atom feed@@ -546,6 +573,7 @@ self._tmplvars = {
'title': self.title, 'pubtime': self.published.format(shared.ARROWFORMAT['iso']), 'pubdate': self.published.format(shared.ARROWFORMAT['display']), + 'pubrfc': self.published.format(shared.ARROWFORMAT['rcf']), 'category': self.category, 'html': self.html, 'lang': self.lang,@@ -558,15 +586,13 @@ 'age': int(self.published.format('YYYY')) - int(arrow.utcnow().format('YYYY')),
'summary': self.summary, 'replies': self.replies, 'reactions': self.reactions, + 'enclosure': self.enclosure, } return self._tmplvars async def render(self): logging.info('rendering %s' % (self.fname)) o = self.htmlfile - if self.is_uptodate: - logging.debug('%s is up to date' % (o)) - return tmplfile = "%s.html" % (__class__.__name__) r = shared.j2.get_template(tmplfile).render({@@ -1092,8 +1118,38 @@ level=shared.LLEVEL[shared.config.get('params', 'loglevel')],
format='%(asctime)s - %(levelname)s - %(message)s' ) +def youngest_mtime(root): + youngest = 0 + files = glob.glob(os.path.join(root, '**'), recursive=True) + for f in files: + mtime = os.path.getmtime(f) + if mtime > youngest: + youngest = mtime + return youngest + def build(): setup() + + #if not shared.config.getboolean('params', 'force'): + #last_run = youngest_mtime(shared.config.get('common', 'build')) + #lookin = [ + #shared.config.get('dirs', 'content'), + #shared.config.get('dirs', 'comment'), + #] + #youngest = 0 + #for root in lookin: + #maybe = youngest_mtime(root) + #if maybe > youngest: + #youngest = maybe + + #if last_run > youngest: + #logging.info("last build happened at %s which is later than the last modification at %s; exiting silently", + #arrow.get(last_run), + #arrow.get(youngest) + #) + #return + + loop = asyncio.get_event_loop() tasks = [] content = Content()@@ -1122,7 +1178,7 @@ post.title
) # add render task, if needed - if not post.is_uptodate or shared.config.get('params', 'force'): + if not post.is_uptodate or shared.config.getboolean('params', 'force'): task = loop.create_task(post.render()) tasks.append(task)@@ -1131,6 +1187,10 @@ for fname, im in post.images:
task = loop.create_task(im.downsize()) tasks.append(task) + # skip adding future posts to any category + if post.is_future: + continue + # skip categories starting with _ if post.category.startswith('_'): continue@@ -1152,13 +1212,15 @@ # write search db
sdb.finish() # render front - task = loop.create_task(collector_front.render()) - tasks.append(task) + if not collector_front.is_uptodate or shared.config.getboolean('params', 'force'): + task = loop.create_task(collector_front.render()) + tasks.append(task) # render categories for name, c in collector_categories: - task = loop.create_task(c.render()) - tasks.append(task) + if not c.is_uptodate or shared.config.getboolean('params', 'force'): + task = loop.create_task(c.render()) + tasks.append(task) # add magic.php rendering task = loop.create_task(magic.render())
@@ -8,7 +8,7 @@ <section class="content-body h-feed">
<aside class="follow"> <p> <svg class="icon"><use xlink:href="#icon-rss" /></svg> - <a title="follow {{ taxonomy.title }}" href="{{ taxonomy.feed }}">RSS/Atom feed</a> + <a title="follow {{ taxonomy.title }}" href="{{ site.url }}{{ taxonomy.feed }}">RSS/Atom feed</a> </p> </aside>
@@ -1,33 +1,32 @@
-<?xml version="1.0" encoding="utf-8"?> -<feed xmlns="http://www.w3.org/2005/Atom"> +<?xml version="1.0" encoding="UTF-8"?> + <rss version="2.0" + xmlns:content="http://purl.org/rss/1.0/modules/content/" + xmlns:dc="http://purl.org/dc/elements/1.1/" + > +<channel> <title>{{ taxonomy.title }}</title> - <link href="{{ taxonomy.feed }}" rel="self" /> - <id>{{ taxonomy.feed }}</id> - <updated>{{ taxonomy.lastmod }}</updated> - + <link>{{ site.url }}{{ taxonomy.feed }}</link> + <lastBuildDate>{{ taxonomy.lastmod }}</lastBuildDate> + <description>{{ site.title }}: {{ taxonomy.name }}</description> {% for post in posts %} - <entry> + <item> <title>{{ post.title }}</title> - <link href="{{ site.url }}/{{ post.slug }}/" /> - <id>{{ site.url }}/{{ post.slug }}</id> - <updated>{{ post.pubtime }}</updated> - {% if post.summary|length > 0 %} - <summary type="xhtml"> - <div xmlns="http://www.w3.org/1999/xhtml"> - {{ post.summary }}> - </div> - </summary> - {% endif %} - <content type="xhtml"> - <div xmlns="http://www.w3.org/1999/xhtml"> - {{ post.html }} - </div> - </content> - <author> - <name>{{ site.author.name }}</name> - <email>{{ site.author.email }}</email> - </author> - <!-- TODO: add enclosure ! --> - </entry> + <link>{{ site.url }}/{{ post.slug }}/</link> + <content:encoded><![CDATA[{{ post.html }}]]></content:encoded> + <pubDate>{{ post.pubrfc }}</pubDate> + <guid>{{ site.url }}/{{ post.slug }}</guid> + <dc:language>{{ post.lang }}</dc:language> +{%- if post.tags %}{% for tname in post.tags %} + <category>{{ tname }}></category> +{% endfor %}{% endif -%} +{%- if post.rssenclosure %} + <enclosure + url="{{ post.enclosure.url }}" + type="{{ post.enclosure.mime }}" + length="{{ post.enclosure.size }}" + /> +{% endif -%} + </item> {% endfor %} -</feed> +</channel> +</rss>
@@ -60,6 +60,7 @@ {% endif %}
</p> </footer> +{% if post.replies|length %} <section class="replies"> <h6><a name="replies"></a>Replies</h6> <ol>@@ -68,7 +69,8 @@ {% include 'Comment.html' %}
{% endfor %} </ol> </section> - +{% endif %} +{% if post.reactions|length %} <section class="reactions"> <h6><a name="reactions"></a>Reactions</h6> <dl>@@ -84,9 +86,8 @@ </dd>
{% endfor %} </dl> </section> - +{% endif %} </article> - </section>
@@ -145,7 +145,7 @@
code, pre { font-family: "Inconsolata", "Courier New", monospace; - font-size: 0.7rem; + font-size: 0.9rem; color: limegreen; direction: ltr; text-align: left;@@ -154,7 +154,7 @@ word-spacing: normal;
word-break: normal; tab-size: 2; hyphens: none; - border: 1px solid #666; + outline: 1px solid #666; background-color: #222; }@@ -166,7 +166,7 @@ }
code { display: inline-block; - padding: 0 0.3rem; + padding: 0 0.3rem 0.1rem 0.3rem; line-height: 0.9rem; }