#!/usr/bin/env python3 import os import re import configparser import argparse import shutil import logging import json import glob import tempfile import atexit import re import hashlib import math import asyncio import csv import getpass import quopri import base64 import mimetypes import magic import arrow import wand.image import similar_text import frontmatter from slugify import slugify import langdetect import requests from whoosh import index from whoosh import qparser import jinja2 import urllib.parse from webmentiontools.send import WebmentionSend import bleach from emoji import UNICODE_EMOJI from bs4 import BeautifulSoup from readability.readability import Document import shared def splitpath(path): parts = [] (path, tail) = os.path.split(path) while path and tail: parts.insert(0,tail) (path,tail) = os.path.split(path) return parts class BaseIter(object): def __init__(self): self.data = {} def append(self, key, value): if key in self.data: logging.warning("duplicate key: %s, using existing instead", key) existing = self.data.get(key) if hasattr(value, 'fname') and hasattr(existing, 'fname'): logging.warning( "%s collides with existing %s", value.fname, existing.fname ) return self.data[key] = value def __getitem__(self, key): return self.data.get(key, {}) def __repr__(self): return json.dumps(list(self.data.values())) def __next__(self): try: r = self.data.next() except: raise StopIteration() return r def __iter__(self): for k, v in self.data.items(): yield (k, v) return class BaseRenderable(object): def __init__(self): return def writerendered(self, content, mtime=None): mtime = mtime or self.mtime d = os.path.dirname(self.target) if not os.path.isdir(d): os.mkdir(d) with open(self.target, "w") as html: logging.debug('writing %s', self.target) html.write(content) html.close() os.utime(self.target, (mtime, mtime)) class Indexer(object): def __init__(self): self.target = os.path.abspath(os.path.join( shared.config.get('target', 'builddir'), shared.config.get('var', 'searchdb') )) if not os.path.isdir(self.target): os.mkdir(self.target) self.mtime = 0 if index.exists_in(self.target): self.ix = index.open_dir(self.target) tocfiles = glob.glob(os.path.join( self.target, "_MAIN_*.toc" )) if len(tocfiles): self.mtime = int(os.path.getmtime(tocfiles[0])) else: self.ix = index.create_in(self.target, shared.schema) self.writer = self.ix.writer() self.qp = qparser.QueryParser("url", schema=shared.schema) async def append(self, singular): if singular.isfuture: return logging.debug("searching for existing index for %s", singular.fname) if self.mtime >= singular.mtime: logging.debug("search index is newer than post mtime (%d vs %d), skipping post", self.mtime, singular.mtime) return exists = False q = self.qp.parse(singular.url) r = self.ix.searcher().search(q, limit=1) if r: r = r[0] # nothing to do, the entry is present and is up to date ixtime = r['mtime'] if int(ixtime) == int(singular.mtime): logging.info("search index is up to date for %s", singular.fname) return else: logging.info("search index is out of date: %d (indexed) vs %d", ixtime, singular.mtime) exists = True reactions = [] for t, v in singular.reactions.items(): if isinstance(v, list) and len(v): reactions = [*reactions, *v] content = "\n\n".join([ "\n\n".join([ bleach.clean(c, tags=[], strip_comments=True, strip=True) for c in [singular.sumhtml, singular.html] ]), "\n".join(["%s" % c for c in singular.tags]), "\n\n".join(reactions) ]) weight = 1 if singular.isbookmark or singular.isfav: weight = 10 if singular.ispage: weight = 100 if not len(singular.title): title = singular.fname else: title = singular.title if singular.photo: img = shared.Pandoc().convert("%s" % singular.photo) else: img = '' args = { 'url': singular.url, 'category': singular.category, 'date': singular.published.datetime, 'title': title, 'weight': weight, 'img': img, 'content': content, 'fuzzy': content, 'mtime': singular.mtime } if exists: logging.info("updating search index with %s", singular.fname) self.writer.add_document(**args) else: logging.info("appending search index with %s", singular.fname) self.writer.update_document(**args) def finish(self): self.writer.commit() class Renderer(object): def __init__(self): self.sitevars = dict(shared.config.items('site')) self.sitevars['author'] = dict(shared.config.items('author')) self.sitevars['author']['socials'] = dict(shared.config.items('socials')) self.sitevars['author']['qr'] = dict(shared.config.items('qr')) self.jinjaldr = jinja2.FileSystemLoader( searchpath=shared.config.get('source', 'templatesdir') ) self.j2 = jinja2.Environment(loader=self.jinjaldr) self.j2.filters['date'] = Renderer.jinja_filter_date self.j2.filters['search'] = Renderer.jinja_filter_search self.j2.filters['slugify'] = Renderer.jinja_filter_slugify @staticmethod def jinja_filter_date(d, form='%Y-%m-%d %H:%m:%S'): if d == 'now': d = arrow.now().datetime if form == 'c': return d.isoformat() #form = '%Y-%m-%dT%H:%M:%S%z' return d.strftime(form) @staticmethod def jinja_filter_slugify(s): return slugify(s, only_ascii=True, lower=True) @staticmethod def jinja_filter_search(s, r): if r in s: return True return False # based on http://stackoverflow.com/a/10075210 class ExifTool(shared.CMDLine): """ Handles calling external binary `exiftool` in an efficient way """ sentinel = "{ready}\n" def __init__(self): super().__init__('exiftool') def run(self, *filenames): return json.loads(self.execute( '-sort', '-json', '-MIMEType', '-FileType', '-FileName', '-ModifyDate', '-CreateDate', '-DateTimeOriginal', '-ImageHeight', '-ImageWidth', '-Aperture', '-FOV', '-ISO', '-FocalLength', '-FNumber', '-FocalLengthIn35mmFormat', '-ExposureTime', '-Copyright', '-Artist', '-Model', '-GPSLongitude#', '-GPSLatitude#', '-LensID', *filenames)) class Comment(BaseRenderable): def __init__(self, path): logging.debug("initiating comment object from %s", path) self.path = path self.fname, self.ext = os.path.splitext(os.path.basename(self.path)) self.mtime = int(os.path.getmtime(self.path)) self.meta = {} self.content = '' self.tmplfile = 'comment.html' self.__parse() def __repr__(self): return "%s" % (self.path) def __parse(self): with open(self.path, mode='rt') as f: self.meta, self.content = frontmatter.parse(f.read()) @property def reacji(self): if hasattr(self, '_reacji'): return self._reacji self._reacji = '' maybe = bleach.clean( self.content, tags=[], strip_comments=True, strip=True, ).strip() if not len(maybe): self._reacji = '★' elif maybe in UNICODE_EMOJI: self._reacji = maybe #t = self.meta.get('type', 'webmention') #typemap = { #'like-of': '👍', #'bookmark-of': '🔖', #'favorite': '★', #} #if t in typemap.keys(): #self._reacji = typemap[t] #else: return self._reacji @property def html(self): if hasattr(self, '_html'): return self._html tmp = shared.Pandoc().convert(self.content) self._html = bleach.clean(tmp, strip=True) return self._html @property def tmplvars(self): if hasattr(self, '_tmplvars'): return self._tmplvars self._tmplvars = { 'published': self.published.datetime, 'author': self.meta.get('author', {}), #'content': self.content, #'html': self.html, 'source': self.source, 'target': self.targeturl, 'type': self.meta.get('type', 'webmention'), 'reacji': self.reacji, 'fname': self.fname } return self._tmplvars @property def published(self): if hasattr(self, '_published'): return self._published self._published = arrow.get(self.meta.get('date', self.mtime)) return self._published @property def pubtime(self): return int(self.published.timestamp) @property def source(self): if hasattr(self, '_source'): return self._source s = self.meta.get('source', '') domains = shared.config.get('site', 'domains').split(' ') self._source = s for d in domains: if d in s: self._source = '' return self._source @property def targeturl(self): if hasattr(self, '_targeturl'): return self._targeturl t = self.meta.get('target', shared.config.get('site', 'url')) self._targeturl = '{p.path}'.format(p=urllib.parse.urlparse(t)).strip('/') return self._targeturl @property def target(self): if hasattr(self, '_target'): return self._target targetdir = os.path.abspath(os.path.join( shared.config.get('target', 'builddir'), shared.config.get('site', 'commentspath'), self.fname )) self._target = os.path.join(targetdir, 'index.html') return self._target async def render(self, renderer): logging.info("rendering and saving comment %s", self.fname) if not shared.config.getboolean('params', 'force') and os.path.isfile(self.target): ttime = int(os.path.getmtime(self.target)) logging.debug('ttime is %d mtime is %d', ttime, self.mtime) if ttime == self.mtime: logging.debug( '%s exists and up-to-date (lastmod: %d)', self.target, ttime ) return tmplvars = { 'reply': self.tmplvars, 'site': renderer.sitevars, 'taxonomy': {}, } r = renderer.j2.get_template(self.tmplfile).render(tmplvars) self.writerendered(r) class Comments(object): def __init__(self): self.files = glob.glob(os.path.join( shared.config.get('source', 'commentsdir'), "*.md" )) self.bytarget = {} def __getitem__(self, key): return self.bytarget.get(key, BaseIter()) def populate(self): for fpath in self.files: item = Comment(fpath) t = item.targeturl if not self.bytarget.get(t): self.bytarget[t] = BaseIter() self.bytarget[t].append(item.pubtime, item) class Images(BaseIter): def __init__(self, extensions=['jpg', 'gif', 'png']): super(Images, self).__init__() logging.info( "initiating images with extensions: %s", extensions ) self.files = [] self.data = {} # if anyone knows how to do this in a more pythonic way, please tell me paths = [ shared.config.get('source', 'filesdir'), shared.config.get('source', 'photosdir') ] for p in paths: for ext in extensions: self.files += glob.glob(os.path.join(p, "*.%s" % ext)) def populate(self): with ExifTool() as e: _meta = e.run(*self.files) # parsing the returned meta into a dict of [filename]={meta} for e in _meta: if 'FileName' not in e: logging.error("missing 'FileName' in element %s", e) continue fname = os.path.basename(e['FileName']) del(e['FileName']) # duplicate files are going to be a problem, so don't send it # away with a simple error log entry if fname in self.data: raise ValueError('filename collision: %s', fname) # convert dates for k, v in e.items(): e[k] = self.exifdate(v) self.data[fname] = WebImage(fname, e) def exifdate(self, value): """ converts and EXIF date string to ISO 8601 format :param value: EXIF date (2016:05:01 00:08:24) :type arg1: str :return: ISO 8601 string with UTC timezone 2016-05-01T00:08:24+0000 :rtype: str """ if not isinstance(value, str): return value match = shared.EXIFREXEG.match(value) if not match: return value return "%s-%s-%sT%s+0000" % ( match.group('year'), match.group('month'), match.group('day'), match.group('time') ) class WebImage(object): def __init__(self, fname, meta): logging.info( "parsing image: %s", fname ) self.meta = meta self.fpath = os.path.abspath(meta.get('SourceFile', fname)) self.fname, self.ext = os.path.splitext(fname) self.alttext = '' self.sizes = [] self.fallbacksize = int(shared.config.get('common','fallbackimg', fallback='720')) self.cl = '' self.singleimage = False for size in shared.config.options('downsize'): sizeext = shared.config.get('downsize', size) fname = "%s_%s%s" % (self.fname, sizeext, self.ext) self.sizes.append(( int(size), { 'fpath': os.path.join( shared.config.get('target', 'filesdir'), fname ), 'url': "/%s/%s" % ( #'url': "%s/%s/%s" % ( #shared.config.get('site', 'url'), shared.config.get('source', 'files'), fname ), 'crop': shared.config.getboolean('crop', size, fallback=False), } )) self.sizes = sorted(self.sizes, reverse=False) self.target = False if self.is_downsizeable: self.fallback = [e for e in self.sizes if e[0] == self.fallbacksize][0][1]['url'] self.small = [e for e in self.sizes if e[1]['crop'] == False][0][1]['url'] self.target = self.sizes[-1][1]['url'] else: self.small = self.fallback = "/%s/%s" % ( #self.small = self.fallback = "%s/%s/%s" % ( #shared.config.get('site', 'url'), shared.config.get('source', 'files'), "%s%s" % (self.fname, self.ext) ) def _setupvars(self): if self.is_downsizeable: if self.singleimage and not self.cl: self.cl = '.u-photo' elif self.singleimage: self.cl = '.u-photo %s' % self.cl else: if not self.cl: self.cl = '.aligncenter' @property def tmplvars(self): self._setupvars() if hasattr(self, '_tmplvars'): return self._tmplvars self._tmplvars = { 'alttext': self.alttext, 'fallback': self.fallback, 'small': self.small, 'title': "%s%s" % (self.fname, self.ext), 'target': self.target, 'cl': " ".join([s[1:] for s in self.cl.split()]), 'orientation': self.orientation } return self._tmplvars def __str__(self): self._setupvars() if self.is_downsizeable: #if self.singleimage and not self.cl: #self.cl = '.u-photo' #elif self.singleimage: #self.cl = '.u-photo %s' % self.cl return '[![%s](%s "%s%s"){.adaptimg}](%s){.adaptive %s}' % ( self.alttext, self.fallback, self.fname, self.ext, self.target, self.cl ) else: #if not self.cl: #self.cl = '.aligncenter' return '![%s](%s "%s%s"){%s}' % ( self.alttext, self.fallback, self.fname, self.ext, self.cl ) @property def exif(self): if not self.is_photo: return {} if hasattr(self, '_exif'): return self._exif exif = {} mapping = { 'camera': [ 'Model' ], 'aperture': [ 'FNumber', 'Aperture' ], 'shutter_speed': [ 'ExposureTime' ], 'focallength35mm': [ 'FocalLengthIn35mmFormat', ], 'focallength': [ 'FocalLength', ], 'iso': [ 'ISO' ], 'lens': [ 'LensID', ], 'date': [ 'CreateDate', 'DateTimeOriginal', ], 'geo_latitude': [ 'GPSLatitude' ], 'geo_longitude': [ 'GPSLongitude' ], } for ekey, candidates in mapping.items(): for candidate in candidates: maybe = self.meta.get(candidate, None) if maybe: if 'geo_' in ekey: exif[ekey] = round(float(maybe), 5) else: exif[ekey] = maybe break self._exif = exif return self._exif @property def orientation(self): width = int(self.meta.get('ImageWidth', 0)) height = int(self.meta.get('ImageHeight', 0)) if width >= height: return 'horizontal' return 'vertical' @property def rssenclosure(self): """ Returns the largest available image for RSS to add as attachment """ if hasattr(self, '_rssenclosure'): return self._rssenclosure target = self.sizes[-1][1] self._rssenclosure = { 'mime': magic.Magic(mime=True).from_file(target['fpath']), 'url': target['url'], 'size': os.path.getsize(target['fpath']), 'fname': self.fname } return self._rssenclosure @property def is_photo(self): if hasattr(self, '_is_photo'): return self._is_photo self._is_photo = False #if not pattern or not isinstance(pattern, str): # return False pattern = re.compile(shared.config.get('photo', 'regex')) cpr = self.meta.get('Copyright', '') art = self.meta.get('Artist', '') if not cpr and not art: return False if cpr and art: if pattern.search(cpr) or pattern.search(art): self._is_photo = True return self._is_photo @property def is_downsizeable(self): if hasattr(self, '_is_downsizeable'): return self._is_downsizeable self._is_downsizeable = False """ Check if the image is large enough and jpeg or png in order to downsize it """ fb = self.sizes[-1][0] ftype = self.meta.get('FileType', None) if not ftype: return self._is_downsizeable if ftype.lower() == 'jpeg' or ftype.lower() == 'png': width = int(self.meta.get('ImageWidth', 0)) height = int(self.meta.get('ImageHeight', 0)) if width > fb or height > fb: self._is_downsizeable = True return self._is_downsizeable def _copy(self): target = os.path.join( shared.config.get('target', 'filesdir'), "%s%s" % (self.fname, self.ext) ) if not os.path.isfile(target): logging.debug("can't downsize %s, copying instead" % self.fname) shutil.copy(self.fpath, target) def _watermark(self, img): """ Composite image by adding watermark file over it """ wmarkfile = os.path.join( shared.config.get('common', 'basedir'), shared.config.get('common', 'watermark') ) if not os.path.isfile(wmarkfile): return img with wand.image.Image(filename=wmarkfile) as wmark: if img.width > img.height: w = img.width * 0.16 h = wmark.height * (w / wmark.width) x = img.width - w - (img.width * 0.01) y = img.height - h - (img.height * 0.01) else: w = img.height * 0.16 h = wmark.height * (w / wmark.width) x = img.width - h - (img.width * 0.01) y = img.height - w - (img.height * 0.01) w = round(w) h = round(h) x = round(x) y = round(y) wmark.resize(w, h) if img.width <= img.height: wmark.rotate(-90) img.composite(image=wmark, left=x, top=y) return img def _intermediate_dimensions(self, size, width, height, crop = False): size = int(size) w = width h = height if (width > height and not crop) \ or (width < height and crop): w = size h = int(float(size / width) * height) else: h = size w = int(float(size / height) * width) return (w, h) def _intermediate(self, img, size, meta, existing = []): if img.width <= size and img.height <= size: return False crop = meta.get('crop', False) with img.clone() as thumb: width, height = self._intermediate_dimensions( size, img.width, img.height, crop ) thumb.resize(width, height) if crop: thumb.liquid_rescale(size, size, 1, 1) if self.meta.get('FileType', 'jpeg').lower() == 'jpeg': thumb.compression_quality = 86 thumb.unsharp_mask( radius=0, sigma=0.5, amount=1, threshold=0.03 ) thumb.format = 'pjpeg' # this is to make sure pjpeg happens with open(meta['fpath'], 'wb') as f: thumb.save(file=f) return True async def downsize(self, existing = []): if not self.is_downsizeable: self._copy() return logging.info("checking downsizing for %s", self.fname) needed = shared.config.getboolean('params', 'regenerate', fallback=False) if not needed: for (size, meta) in self.sizes: if meta['fpath'] not in existing: needed = True if not needed: logging.debug("downsizing not needed for %s", self.fname) return with wand.image.Image(filename=self.fpath) as img: img.auto_orient() if self.is_photo: logging.info("%s is a photo", self.fpath) img = self._watermark(img) for (size, meta) in self.sizes: self._intermediate(img, size, meta, existing) class Taxonomy(BaseIter): def __init__(self, name = None, taxonomy = None, slug = None): super(Taxonomy, self).__init__() self.name = name if name and not slug: self.slug = slugify(name, only_ascii=True, lower=True) else: self.slug = slug self.taxonomy = taxonomy #@property #def pages(self): #if hasattr(self, '_pages'): #return self._pages #self._pages = math.ceil(len(self.data) / shared.config.getint('common', 'pagination')) #return self._pages def __repr__(self): return "taxonomy %s with %d items" % (self.taxonomy, len(self.data)) @property def basep(self): p = shared.config.get('target', 'builddir') if self.taxonomy: p = os.path.join(p, self.taxonomy) return p @property def myp(self): p = self.basep if self.slug: return os.path.join(p,self.slug) return p @property def feedp(self): return os.path.join(self.myp, 'feed') @property def pagep(self): return os.path.join(self.myp, 'page') @property def baseurl(self): if self.taxonomy and self.slug: return "/%s/%s/" % (self.taxonomy, self.slug) else: return '/' @property def mtime(self): if hasattr(self, '_mtime'): return self._mtime self._mtime = int(list(sorted(self.data.keys(), reverse=True))[0]) return self._mtime def __mkdirs(self): check = [self.basep, self.myp, self.feedp] for p in check: if not os.path.isdir(p): logging.debug("creating dir %s", p) os.mkdir(p) def tpath(self, page): if page == 1: p = "%s" % (self.myp) else: p = os.path.join(self.pagep, "%d" % page) if not os.path.isdir(p): logging.debug("creating dir %s", p) os.mkdir(p) return os.path.join(p, "index.html") @property def is_singlepage(self): spcats = shared.config.get('common', 'onepagecategories').split(',') if self.name in spcats and 'category' == self.taxonomy: return True return False def posttmpls(self, order='time', start=0, end=None): end = end or len(self.data) if 'all' == order: return [ i.tmplvars for k, i in list(sorted( self.data.items(), key=lambda value: value[1].title.lower() )) ] return [ self.data[k].tmplvars for k in list(sorted( self.data.keys(), reverse=True ))[start:end] ] async def render(self, renderer): #if not self.slug or self.slug is 'None': #return self.__mkdirs() page = 1 testpath = self.tpath(page) if not shared.config.getboolean('params', 'force') and os.path.isfile(testpath): ttime = int(os.path.getmtime(testpath)) mtime = self.mtime if ttime == mtime: logging.info('taxonomy index for "%s" exists and up-to-date (lastmod: %d)', self.slug, ttime) return else: logging.info('taxonomy update needed: %s timestamp is %d, last post timestamp is %d (%s)', testpath, ttime, mtime, self.data[mtime].fname ) if self.is_singlepage: pagination = len(self.data) else: pagination = shared.config.getint('common', 'pagination') pages = math.ceil(len(self.data) / pagination) while page <= pages: self.render_page(renderer, page, pagination, pages) page = page+1 self.render_feeds(renderer) self.ping_websub def render_feeds(self, renderer): pagination = shared.config.getint('common', 'pagination') start = 0 end = int(start + pagination) posttmpls = self.posttmpls('time', start, end) tmplvars = { 'taxonomy': { 'url': self.baseurl, 'name': self.name, 'slug': self.slug, 'taxonomy': self.taxonomy, 'lastmod': arrow.get(self.mtime).datetime }, 'site': renderer.sitevars, 'posts': posttmpls, } target = os.path.join(self.feedp, 'index.atom') logging.info("rendering Atom feed to %s", target) r = renderer.j2.get_template('atom.html').render(tmplvars) with open(target, "wt") as html: html.write(r) os.utime(target, (self.mtime, self.mtime)) def render_page(self, renderer, page, pagination, pages): if self.is_singlepage: posttmpls = self.posttmpls('all') else: start = int((page-1) * pagination) end = int(start + pagination) posttmpls = self.posttmpls('time', start, end) target = self.tpath(page) tdir = os.path.dirname(target) if not os.path.isdir(tdir): logging.debug("creating dir %s", tdir) os.mkdir(tdir) logging.info("rendering taxonomy page %d to %s", page, target) tmplvars = { 'taxonomy': { 'url': self.baseurl, 'name': self.name, 'slug': self.slug, 'taxonomy': self.taxonomy, 'paged': page, 'total': pages, 'perpage': pagination, 'lastmod': arrow.get(self.mtime).datetime }, 'site': renderer.sitevars, 'posts': posttmpls, } r = renderer.j2.get_template('archive.html').render(tmplvars) with open(target, "wt") as html: html.write(r) os.utime(target, (self.mtime, self.mtime)) def ping_websub(self): if not self.taxonomy or self.taxonomy == 'category': t = shared.config.get('site', 'websuburl') data = { 'hub.mode': 'publish', 'hub.url': "%s%s" % ( shared.config.get('site', 'url'), self.baseurl ) } logging.info("pinging %s with data %s", t, data) requests.post(t, data=data) #def renderpage(self, renderer, page): #pagination = int(shared.config.get('common', 'pagination')) #start = int((page-1) * pagination) #end = int(start + pagination) #posttmpls = [self.data[k].tmplvars for k in list(sorted( #self.data.keys(), reverse=True))[start:end]] #target = self.tpath(page) #logging.info("rendering taxonomy page %d to %s", page, target) #tmplvars = { #'taxonomy': { #'url': self.baseurl, #'name': self.name, #'slug': self.slug, #'taxonomy': self.taxonomy, #'paged': page, #'total': self.pages, #'perpage': pagination, #'lastmod': arrow.get(self.mtime).datetime #}, #'site': renderer.sitevars, #'posts': posttmpls, #} #r = renderer.j2.get_template('archive.html').render(tmplvars) #with open(target, "wt") as html: #html.write(r) #os.utime(target, (self.mtime, self.mtime)) #if 1 == page: ##target = os.path.join(self.feedp, 'index.rss') ##logging.info("rendering RSS feed to %s", target) ##r = renderer.j2.get_template('rss.html').render(tmplvars) ##with open(target, "wt") as html: ##html.write(r) ##os.utime(target, (self.mtime, self.mtime)) #target = os.path.join(self.feedp, 'index.atom') #logging.info("rendering Atom feed to %s", target) #r = renderer.j2.get_template('atom.html').render(tmplvars) #with open(target, "wt") as html: #html.write(r) #os.utime(target, (self.mtime, self.mtime)) ## --- ## this is a joke ## see http://indieweb.org/YAMLFeed ## don't do YAMLFeeds. #if 1 == page: #fm = frontmatter.loads('') #fm.metadata = { #'site': { #'author': renderer.sitevars['author'], #'url': renderer.sitevars['url'], #'title': renderer.sitevars['title'], #}, #'items': [], #} #for p in posttmpls: #fm.metadata['items'].append({ #'title': p['title'], #'url': "%s/%s/" % ( renderer.sitevars['url'], p['slug']), #'content': p['content'], #'summary': p['summary'], #'published': p['published'], #'updated': p['updated'], #}) #target = os.path.join(self.feedp, 'index.yml') #logging.info("rendering YAML feed to %s", target) #with open(target, "wt") as html: #html.write(frontmatter.dumps(fm)) #os.utime(target, (self.mtime, self.mtime)) ## --- #if 1 == page: #if not self.taxonomy or self.taxonomy == 'category': #t = shared.config.get('site', 'websuburl') #data = { #'hub.mode': 'publish', #'hub.url': "%s%s" % ( #shared.config.get('site', 'url'), self.baseurl #) #} #logging.info("pinging %s with data %s", t, data) #requests.post(t, data=data) class Content(BaseIter): def __init__(self, images, comments, extensions=['md']): super(Content, self).__init__() self.images = images self.comments = comments basepath = shared.config.get('source', 'contentdir') self.files = [] for ext in extensions: self.files += glob.glob(os.path.join(basepath, "*", "*.%s" % ext)) self.tags = {} self.categories = {} self.front = Taxonomy() self.shortslugmap = {} def populate(self): now = arrow.utcnow().timestamp for fpath in self.files: item = Singular(fpath, self.images, self.comments) self.append(item.pubtime, item) #self.shortslugmap[item.shortslug] = item.fname if item.isfuture: logging.warning("skipping future post %s", item.fname) continue if item.isonfront: self.front.append(item.pubtime, item) if item.iscategorised: if item.category not in self.categories: self.categories[item.category] = Taxonomy(item.category, 'category') self.categories[item.category].append(item.pubtime, item) for tag in item.tags: tslug = slugify(tag, only_ascii=True, lower=True) if tslug not in self.tags: self.tags[tslug] = Taxonomy(tag, 'tag', tslug) self.tags[tslug].append(item.pubtime, item) #self.symlinktag(tslug, item.path) #def symlinktag(self, tslug, fpath): #fdir, fname = os.path.split(fpath) #tagpath = os.path.join(shared.config.get('source', 'tagsdir'), tslug) #if not os.path.isdir(tagpath): #os.mkdir(tagpath) #sympath = os.path.relpath(fdir, tagpath) #dst = os.path.join(tagpath, fname) #src = os.path.join(sympath, fname) #if not os.path.islink(dst): #os.symlink(src, dst) def sitemap(self): target = os.path.join( shared.config.get('target', 'builddir'), 'sitemap.txt' ) urls = [] for item in self.data.values(): urls.append( "%s/%s/" % ( shared.config.get('site', 'url'), item.fname )) with open(target, "wt") as f: logging.info("writing sitemap to %s" % (target)) f.write("\n".join(urls)) def magicphp(self, renderer): redirects = [] gones = [] rfile = os.path.join( shared.config.get('common', 'basedir'), shared.config.get('common', 'redirects') ) if os.path.isfile(rfile): with open(rfile, newline='') as csvfile: r = csv.reader(csvfile, delimiter=' ') for row in r: redirects.append((row[0], row[1])) for item in self.data.values(): redirects.append((item.shortslug, item.fname)) rfile = os.path.join( shared.config.get('common', 'basedir'), shared.config.get('common', 'gone') ) if os.path.isfile(rfile): with open(rfile, newline='') as csvfile: r = csv.reader(csvfile, delimiter=' ') for row in r: gones.append(row[0]) tmplvars = { 'site': renderer.sitevars, 'redirects': redirects, 'gones': gones } r = renderer.j2.get_template("magic.php").render(tmplvars) target = os.path.abspath(os.path.join( shared.config.get('target', 'builddir'), 'magic.php' )) with open(target, "w") as html: logging.debug('writing %s', target) html.write(r) html.close() class Singular(BaseRenderable): def __init__(self, path, images = None, comments = None): logging.debug("initiating singular object from %s", path) self.path = path self.images = images or Images() self.allcomments = comments or Comments() self.category = splitpath(path)[-2] self.mtime = int(os.path.getmtime(self.path)) self.fname, self.ext = os.path.splitext(os.path.basename(self.path)) self.meta = {} self.content = '' self.photo = self.images.data.get("%s.jpg" % self.fname, None) if self.photo: self.photo.singleimage = True self.__parse() def __repr__(self): return "%s (lastmod: %s)" % (self.fname, self.published) def __parse(self): with open(self.path, mode='rt') as f: self.meta, self.content = frontmatter.parse(f.read()) #self.__filter_syndication() self.__filter_favs() self.__filter_images() #if self.isphoto: #self.content = "%s\n\n%s" % ( #self.photo, #self.content, #) #if shared.config.getboolean('params', 'nooffline'): #return #trigger = self.offlinecopies #def __filter_syndication(self): #syndications = self.meta.get('syndicate', None) #if not syndications: #return #s = "\n\n".join(['' % s for s in syndications]) #self.content = "%s\n\n%s" % ( #s, #self.content #) def __filter_favs(self): url = self.meta.get('favorite-of', self.meta.get('like-of', self.meta.get('bookmark-of', False ) ) ) if not url: return img = self.meta.get('image', False) imgs = self.meta.get('images', []) if img: imgs.append(img) if not imgs or not len(imgs): return c = '' for i in imgs: c = '%s\n[![%s](/%s/%s)](%s){.favurl}' % ( c, self.title, shared.config.get('source', 'files'), i, url ) if self.isbookmark: c = "%s\n\n%s" % (c, self.content) self.content = c def __filter_images(self): linkto = False isrepost = None if len(self.reactions.keys()): isrepost = list(self.reactions.keys())[0] if isrepost and \ len(self.reactions[isrepost]) == 1: linkto = self.reactions[isrepost][0] m = shared.MDIMGREGEX.findall(self.content) if not m: logging.debug("no images found") return for shortcode, alt, fname, title, cl in m: image = self.images.data.get(fname, None) if not image: logging.debug("%s not found in images", fname) continue if cl: image.cl = cl logging.debug( "replacing %s in content with %s", shortcode, "%s" % image ) self.content = self.content.replace( shortcode, "%s" % image ) @property def comments(self): if hasattr(self, '_comments'): return self._comments # the comments could point to both the "real" url and the shortslug # so I need to get both c = {} for by in [self.fname, self.shortslug]: c = {**c, **self.allcomments[by].data} #self._comments = [c[k].tmplvars for k in list(sorted(c.keys(), reverse=True))] self._comments = [c[k] for k in list(sorted(c.keys(), reverse=False))] return self._comments @property def replies(self): if hasattr(self, '_replies'): return self._replies self._replies = [c.tmplvars for c in self.comments if not len(c.reacji)] return self._replies @property def reacjis(self): if hasattr(self, '_reacjis'): return self._reacjis reacjis = {} for c in self.comments: rj = c.reacji if not len(rj): continue if not reacjis.get(rj, False): reacjis[rj] = [] reacjis[rj].append(c.tmplvars) self._reacjis = reacjis return self._reacjis @property def reactions(self): if hasattr(self, '_reactions'): return self._reactions # getting rid of '-' to avoid css trouble and similar convert = { 'bookmark-of': 'bookmark', 'repost-of': 'repost', 'in-reply-to': 'reply', 'favorite-of': 'fav', 'like-of': 'like', } reactions = {} for k, v in convert.items(): x = self.meta.get(k, None) if not x: continue if isinstance(x, str): x = [x] reactions[v] = x self._reactions = reactions return self._reactions @property def author(self): return dict(shared.config.items('author')) @property def license(self): if hasattr(self, '_licence'): return self._licence if 'article' == self.category: l = { 'url': 'https://creativecommons.org/licenses/by/4.0/', 'text': 'CC BY 4.0', 'description': 'Licensed under Creative Commons Attribution 4.0 International. You are free to share or republish, even if modified, if you link back here and indicate the modifications, even for commercial use.' } elif 'journal' == self.category: l = { 'url': 'https://creativecommons.org/licenses/by-nc/4.0/', 'text': 'CC BY-NC 4.0', 'description': 'Licensed under Creative Commons Attribution-NonCommercial 4.0 International. You are free to share or republish, even if modified, if you link back here and indicate the modifications, for non commercial use. For commercial use please contact the author.' } else: l = { 'url': 'https://creativecommons.org/licenses/by-nc-nd/4.0/', 'text': 'CC BY-NC-ND 4.0', 'description': 'Licensed under Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International. You are free to share if you link back here for non commercial use, but you can\'t publish any altered versions of it. For commercial use please contact the author.' } self._licence = l return self._licence @property def syndicate(self): return self.meta.get('syndicate', []) @property def urls(self): if hasattr(self, '_urls'): return self._urls #urls = shared.URLREGEX.findall(self.content) #urls = [*urls, *self.syndicate] urls = list(self.syndicate) for reactionurls in self.reactions.values(): urls = [*urls, *reactionurls] #r = [] #logging.debug('searching for urls for %s', self.fname) #for link in urls: #purl = urllib.parse.urlparse(link) #if purl.netloc in shared.config.get('site', 'domains'): #logging.debug('excluding url %s - %s matches %s', link, purl.netloc, shared.config.get('site', 'domains')) #continue #if link in r: #continue #r.append(link) self._urls = urls logging.debug('urls for %s: %s', self.fname, urls) return self._urls @property def lang(self): if hasattr(self, '_lang'): return self._lang lang = 'en' try: lang = langdetect.detect("\n".join([ self.title, self.content ])) except: pass self._lang = lang return self._lang @property def tags(self): return list(self.meta.get('tags', [])) @property def published(self): if hasattr(self, '_published'): return self._published self._published = arrow.get( self.meta.get('published', self.mtime) ) return self._published @property def updated(self): if hasattr(self, '_updated'): return self._updated self._updated = arrow.get( self.meta.get('updated', self.meta.get('published', self.mtime) ) ) return self._updated @property def pubtime(self): return int(self.published.timestamp) @property def isphoto(self): if not self.photo: return False return self.photo.is_photo @property def isbookmark(self): return self.meta.get('bookmark-of', False) @property def isreply(self): return self.meta.get('in-reply-to', False) @property def isfuture(self): now = arrow.utcnow().timestamp if self.pubtime > now: return True return False # TODO #@property #def isrvsp(self): # r'([^<]+)' @property def isfav(self): r = False for maybe in ['like-of', 'favorite-of']: maybe = self.meta.get(maybe, False) if maybe: r = maybe break return r @property def ispage(self): if not self.meta: return True return False @property def isonfront(self): if self.ispage: return False if self.isbookmark: return False if self.isfav: return False return True @property def iscategorised(self): if self.ispage: return False return True @property def summary(self): return self.meta.get('summary', '') @property def title(self): if hasattr(self, '_title'): return self._title self._title = '' for maybe in ['title', 'bookmark-of', 'in-reply-to', 'repost-of']: maybe = self.meta.get(maybe, False) if maybe: if isinstance(maybe, list): maybe = maybe.pop() self._title = maybe.replace('\n', ' ').replace('\r', '') break return self._title @property def url(self): return "%s/%s/" % (shared.config.get('site', 'url'), self.fname) @property def tmplfile(self): if self.ispage: return 'page.html' else: return 'singular.html' @property def html(self): if hasattr(self, '_html'): return self._html self._html = shared.Pandoc().convert(self.content) return self._html @property def sumhtml(self): if hasattr(self, '_sumhtml'): return self._sumhtml self._sumhtml = self.meta.get('summary', '') if len(self._sumhtml): self._sumhtml = shared.Pandoc().convert(self.summary) return self._sumhtml #@property #def offlinecopies(self): ## stupidly simple property caching #if hasattr(self, 'copies'): #return self.copies #copies = {} #for maybe in ['bookmark-of', 'in-reply-to', 'repost-of', 'favorite-of']: #maybe = self.meta.get(maybe, False) #if not maybe: #continue #if not isinstance(maybe, list): #maybe = [maybe] #for url in maybe: #arch = OfflineArchive(url) #arch.run() ##copies[url] = arch.read() ##self.copies = copies ##return copies @property def exif(self): if not self.isphoto: return {} return self.photo.exif #@property #def rssenclosure(self): #if not self.isphoto: #return {} #return self.photo.rssenclosure @property def tmplvars(self): if hasattr(self, '_tmplvars'): return self._tmplvars self._tmplvars = { 'title': self.title, 'published': self.published.datetime, 'tags': self.tags, 'author': self.author, 'content': self.content, 'html': self.html, 'category': self.category, 'reactions': self.reactions, 'updated': self.updated.datetime, 'summary': self.summary, 'sumhtml': self.sumhtml, 'exif': self.exif, 'lang': self.lang, 'syndicate': self.syndicate, 'slug': self.fname, 'shortslug': self.shortslug, 'comments': self.comments, 'replies': self.replies, 'reacjis': self.reacjis, 'photo': {}, 'rssenclosure': {}, 'license': self.license, } if self.isphoto: self._tmplvars.update({ 'photo': self.photo.tmplvars, 'rssenclosure': self.photo.rssenclosure }) return self._tmplvars @property def shortslug(self): if hasattr(self, '_shortslug'): return self._shortslug self._shortslug = shared.baseN(self.pubtime) return self._shortslug @property def target(self): targetdir = os.path.abspath(os.path.join( shared.config.get('target', 'builddir'), self.fname )) return os.path.join(targetdir, 'index.html') async def rendercomments(self, renderer): for comment in self.comments: await comment.render(renderer) async def render(self, renderer = None): renderer = renderer or Renderer() # this is only when I want salmentions and I want to include all of the comments as well # otherwise it affects both webmentions sending and search indexing #if len(self.comments): #lctime = self.comments[0].mtime #if lctime > self.mtime: #self.mtime = lctime #await self.rendercomments(renderer) mtime = self.mtime if len(self.comments): lctime = self.comments[0].mtime if lctime > self.mtime: mtime = lctime logging.info("rendering and saving %s", self.fname) if not shared.config.getboolean('params', 'force') and os.path.isfile(self.target): ttime = int(os.path.getmtime(self.target)) logging.debug('ttime is %d mtime is %d', ttime, mtime) if ttime == mtime: logging.debug( '%s exists and up-to-date (lastmod: %d)', self.target, ttime ) return tmplvars = { 'post': self.tmplvars, 'site': renderer.sitevars, 'taxonomy': {}, } r = renderer.j2.get_template(self.tmplfile).render(tmplvars) self.writerendered(r, mtime) async def ping(self, pinger): if self.isfuture: return logging.debug('urls in %s: %s', self.fname, self.urls) for target in self.urls: record = { 'mtime': self.mtime, 'source': self.url, 'target': target } h = json.dumps(record, sort_keys=True) h = hashlib.sha1(h.encode('utf-8')).hexdigest() if pinger.db.get(h, False): logging.debug( "%s is already pinged from %s @ %d, skipping", target, self.url, self.mtime ) continue logging.info("sending webmention from %s to %s", self.url, target) ws = WebmentionSend(self.url, target) try: ws.send(allow_redirects=True, timeout=30) except Exception as e: logging.error('ping failed to %s', target) pinger.db[h] = record class Webmentioner(object): def __init__(self): self.dbpath = os.path.abspath(os.path.join( shared.config.get('target', 'builddir'), shared.config.get('var', 'webmentions') )) if os.path.isfile(self.dbpath): with open(self.dbpath, 'rt') as f: self.db = json.loads(f.read()) else: self.db = {} def finish(self): with open(self.dbpath, 'wt') as f: f.write(json.dumps(self.db, sort_keys=True, indent=4)) class NASG(object): lockfile = os.path.join(tempfile.gettempdir(), 'nasg_%s.lock' % getpass.getuser()) def __init__(self): # --- set params parser = argparse.ArgumentParser(description='Parameters for NASG') parser.add_argument( '--clear', action='store_true', default=False, help='clear build directory in advance' ) parser.add_argument( '--regenerate', action='store_true', default=False, help='force downsizing images' ) parser.add_argument( '--force', action='store_true', default=False, help='force rendering HTML' ) parser.add_argument( '--loglevel', default='error', help='change loglevel' ) parser.add_argument( '--nooffline', action='store_true', default=False, help='skip offline copie checks' ) parser.add_argument( '--nodownsize', action='store_true', default=False, help='skip image downsizing' ) parser.add_argument( '--norender', action='store_true', default=False, help='skip rendering' ) parser.add_argument( '--refetch', action='store_true', default=False, help='force re-fetching offline archives' ) params = vars(parser.parse_args()) shared.config.add_section('params') for k, v in params.items(): shared.config.set('params', k, str(v)) # remove the rest of the potential loggers while len(logging.root.handlers) > 0: logging.root.removeHandler(logging.root.handlers[-1]) # --- set loglevel logging.basicConfig( level=shared.LLEVEL[shared.config.get('params', 'loglevel')], format='%(asctime)s - %(levelname)s - %(message)s' ) async def __adownsize(self): for fname, img in self.images: await img.downsize(self.existing) async def __acrender(self): for (pubtime, singular) in self.content: await singular.render(self.renderer) async def __atrender(self): for e in [self.content.categories, self.content.tags]: for name, t in e.items(): await t.render(self.renderer) async def __afrender(self): await self.content.front.render(self.renderer) async def __aindex(self): for (pubtime, singular) in self.content: await self.searchdb.append(singular) async def __aping(self): for (pubtime, singular) in self.content: await singular.ping(self.pinger) def __atindexrender(self): m = { 'category': self.content.categories, 'tag': self.content.tags } for p, taxonomy in m.items(): target = os.path.abspath(os.path.join( shared.config.get('target', 'builddir'), p, 'index.html' )) tmplvars = { 'site': self.renderer.sitevars, 'taxonomy': {}, 'taxonomies': {} } for name, t in taxonomy.items(): logging.debug('adding %s to taxonomyindex' % name) tmplvars['taxonomies'][name] = [{'title': t.data[k].title, 'url': t.data[k].url} for k in list(sorted( t.data.keys(), reverse=True))] tmplvars['taxonomies'] = sorted(tmplvars['taxonomies'].items()) logging.debug('rendering taxonomy index to %s', target) r = self.renderer.j2.get_template('archiveindex.html').render(tmplvars) with open(target, "wt") as html: html.write(r) @property def images(self): if hasattr(self, '_images'): return self._images logging.info("discovering images") images = Images() images.populate() self._images = images return self._images @property def content(self): if hasattr(self, '_content'): return self._content logging.info("discovering content") content = Content(self.images, self.comments) content.populate() self._content = content return self._content @property def comments(self): if hasattr(self, '_comments'): return self._comments logging.info("discovering comments") comments = Comments() comments.populate() self._comments = comments return self._comments @property def existing(self): if hasattr(self, '_existing'): return self._existing existing = glob.glob(os.path.join( shared.config.get('target', 'filesdir'), "*" )) self._existing = existing return self._existing @property def renderer(self): if hasattr(self, '_renderer'): return self._renderer self._renderer = Renderer() return self._renderer @property def searchdb(self): if hasattr(self, '_searchdb'): return self._searchdb self._searchdb = Indexer() return self._searchdb @property def pinger(self): if hasattr(self, '_pinger'): return self._pinger self._pinger = Webmentioner() return self._pinger def run(self): if os.path.isfile(self.lockfile): raise ValueError( "Lockfile is present at %s; another instance is running." % ( self.lockfile ) ) else: atexit.register(os.remove, self.lockfile) with open(self.lockfile, "wt") as f: f.write(arrow.utcnow().format()) if shared.config.getboolean('params', 'clear'): input('about to clear build directory, press enter to continue') shutil.rmtree(os.path.abspath( shared.config.get('target', 'builddir') )) loop = asyncio.get_event_loop() for d in shared.config.options('target'): if 'dir' in d and not os.path.isdir(shared.config.get('target', d)): os.mkdir(shared.config.get('target', d)) if not shared.config.getboolean('params', 'nodownsize'): logging.info("downsizing images") loop.run_until_complete(self.__adownsize()) if not shared.config.getboolean('params', 'norender'): logging.info("rendering content") loop.run_until_complete(self.__acrender()) logging.info("rendering categories and tags") loop.run_until_complete(self.__atrender()) logging.info("rendering the front page elements") loop.run_until_complete(self.__afrender()) logging.info("rendering taxonomy indexes") self.__atindexrender() logging.info("rendering sitemap") self.content.sitemap() logging.info("render magic.php") self.content.magicphp(self.renderer) logging.info("copy the static bits") src = shared.config.get('source', 'staticdir') for item in os.listdir(src): s = os.path.join(src, item) d = os.path.join(shared.config.get('target', 'builddir'), item) logging.debug("copying %s to %s", s, d) shutil.copy2(s, d) logging.info("pouplating searchdb") loop.run_until_complete(self.__aindex()) self.searchdb.finish() logging.info("webmentioning urls") loop.run_until_complete(self.__aping()) self.pinger.finish() loop.close() if __name__ == '__main__': worker = NASG() worker.run()