v2.1.0
- changing version numbering - automated webmentions sending
This commit is contained in:
parent
12e6866e0f
commit
f84088f311
4 changed files with 88 additions and 56 deletions
|
@ -5,7 +5,7 @@ It is most probably not suitable for anyone else, but feel free to use it for id
|
||||||
|
|
||||||
## Why not [insert static generator here]?
|
## Why not [insert static generator here]?
|
||||||
|
|
||||||
- DRY - Don't Repeat Yourself - is good, so instead of sidefiles for images, I'm using XMP metadata, which most of the ones availabe don't handle well;
|
- I'm using embedded XMP metadata in photos, which most of the ones availabe don't handle well;
|
||||||
- writing plugins to existing generators - Pelican, Nicola, etc - might have taken longer and I wanted to extend my Python knowledge
|
- writing plugins to existing generators - Pelican, Nicola, etc - might have taken longer and I wanted to extend my Python knowledge
|
||||||
- I wanted to use the best available utilities for some tasks, like `Pandoc` and `exiftool` instead of Python libraries trying to achive the same
|
- I wanted to use the best available utilities for some tasks, like `Pandoc` and `exiftool` instead of Python libraries trying to achive the same
|
||||||
- I needed to handle webmentions and comments
|
- I needed to handle webmentions and comments
|
||||||
|
|
89
nasg.py
89
nasg.py
|
@ -3,16 +3,16 @@
|
||||||
# vim: set fileencoding=utf-8 :
|
# vim: set fileencoding=utf-8 :
|
||||||
|
|
||||||
__author__ = "Peter Molnar"
|
__author__ = "Peter Molnar"
|
||||||
__copyright__ = "Copyright 2017, Peter Molnar"
|
__copyright__ = "Copyright 2017-2018, Peter Molnar"
|
||||||
__license__ = "GPLv3"
|
__license__ = "GPLv3"
|
||||||
__version__ = "2.0"
|
__version__ = "2.1.0"
|
||||||
__maintainer__ = "Peter Molnar"
|
__maintainer__ = "Peter Molnar"
|
||||||
__email__ = "hello@petermolnar.eu"
|
__email__ = "hello@petermolnar.eu"
|
||||||
__status__ = "Production"
|
__status__ = "Production"
|
||||||
|
|
||||||
"""
|
"""
|
||||||
silo archiver module of NASG
|
silo archiver module of NASG
|
||||||
Copyright (C) 2017 Peter Molnar
|
Copyright (C) 2017-2018 Peter Molnar
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
This program is free software; you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
|
@ -242,7 +242,6 @@ class Category(NoDupeContainer):
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def url(self):
|
def url(self):
|
||||||
if self.name:
|
if self.name:
|
||||||
|
@ -280,7 +279,6 @@ class Category(NoDupeContainer):
|
||||||
out.write(content)
|
out.write(content)
|
||||||
os.utime(path, (self.mtime, self.mtime))
|
os.utime(path, (self.mtime, self.mtime))
|
||||||
|
|
||||||
|
|
||||||
async def render(self):
|
async def render(self):
|
||||||
if self.is_altrender:
|
if self.is_altrender:
|
||||||
self.render_onepage()
|
self.render_onepage()
|
||||||
|
@ -490,20 +488,32 @@ class Singular(object):
|
||||||
wdb.entry_done(incoming.get('id'))
|
wdb.entry_done(incoming.get('id'))
|
||||||
wdb.finish()
|
wdb.finish()
|
||||||
|
|
||||||
# note: due to SQLite locking, this will not be async for now
|
def queue_webmentions(self):
|
||||||
def send_webmentions(self):
|
|
||||||
if not self.is_reply:
|
|
||||||
return
|
|
||||||
wdb = shared.WebmentionQueue()
|
wdb = shared.WebmentionQueue()
|
||||||
wid = wdb.queue(self.url, self.is_reply)
|
for target in self.urls_to_ping:
|
||||||
wm = Webmention(
|
if not wdb.exists(self.url, target, self.published):
|
||||||
self.url,
|
wdb.queue(self.url, target)
|
||||||
self.is_reply
|
else:
|
||||||
)
|
logging.debug("not queueing - webmention already queued from %s to %s", self.url, target)
|
||||||
wm.send()
|
|
||||||
wdb.entry_done(wid)
|
|
||||||
wdb.finish()
|
wdb.finish()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def urls_to_ping(self):
|
||||||
|
urls = [x.strip() for x in shared.REGEX.get('urls').findall(self.content)]
|
||||||
|
if self.is_reply:
|
||||||
|
urls.append(self.is_reply)
|
||||||
|
for url in self.syndicate:
|
||||||
|
urls.append(url)
|
||||||
|
r = {}
|
||||||
|
for link in urls:
|
||||||
|
parsed = urlparse(link)
|
||||||
|
if parsed.netloc in shared.config.get('site', 'domains'):
|
||||||
|
continue
|
||||||
|
if link in r:
|
||||||
|
continue
|
||||||
|
r.update({link: True})
|
||||||
|
return r.keys()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def redirects(self):
|
def redirects(self):
|
||||||
r = self.meta.get('redirect', [])
|
r = self.meta.get('redirect', [])
|
||||||
|
@ -684,7 +694,7 @@ class Singular(object):
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def url(self):
|
def url(self):
|
||||||
return "%s/%s" % (shared.config.get('site', 'url'), self.fname)
|
return "%s/%s/" % (shared.config.get('site', 'url'), self.fname)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def body(self):
|
def body(self):
|
||||||
|
@ -732,7 +742,7 @@ class Singular(object):
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def syndicate(self):
|
def syndicate(self):
|
||||||
urls = []
|
urls = self.meta.get('syndicate', [])
|
||||||
if self.photo and self.photo.is_photo:
|
if self.photo and self.photo.is_photo:
|
||||||
urls.append("https://brid.gy/publish/flickr")
|
urls.append("https://brid.gy/publish/flickr")
|
||||||
return urls
|
return urls
|
||||||
|
@ -1300,27 +1310,42 @@ class Webmention(object):
|
||||||
fm.content = self.content
|
fm.content = self.content
|
||||||
fm.metadata = self.meta
|
fm.metadata = self.meta
|
||||||
with open(self.fpath, 'wt') as f:
|
with open(self.fpath, 'wt') as f:
|
||||||
|
logging.info("Saving webmention to %s", self.fpath)
|
||||||
f.write(frontmatter.dumps(fm))
|
f.write(frontmatter.dumps(fm))
|
||||||
return
|
return
|
||||||
|
|
||||||
def send(self):
|
def send(self):
|
||||||
rels = shared.XRay(self.source).set_discover().parse()
|
rels = shared.XRay(self.target).set_discover().parse()
|
||||||
endpoint = False
|
endpoint = False
|
||||||
if 'rels' not in rels:
|
if 'rels' not in rels:
|
||||||
return
|
logging.debug("no rel found for %s", self.target)
|
||||||
|
return True
|
||||||
for k in rels.get('rels').keys():
|
for k in rels.get('rels').keys():
|
||||||
if 'webmention' in k:
|
if 'webmention' in k:
|
||||||
endpoint = rels.get('rels').get(k)
|
endpoint = rels.get('rels').get(k).pop()
|
||||||
break
|
break
|
||||||
if not endpoint:
|
if not endpoint:
|
||||||
return
|
logging.debug("no endpoint found for %s", self.target)
|
||||||
requests.post(
|
return True
|
||||||
|
logging.info(
|
||||||
|
"Sending webmention to endpoint: %s, source: %s, target: %s",
|
||||||
|
endpoint,
|
||||||
|
self.source,
|
||||||
self.target,
|
self.target,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
p = requests.post(
|
||||||
|
endpoint,
|
||||||
data={
|
data={
|
||||||
'source': self.source,
|
'source': self.source,
|
||||||
'target': self.target
|
'target': self.target
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
if p.status_code == requests.codes.ok:
|
||||||
|
return True
|
||||||
|
except Exception as e:
|
||||||
|
logging.error("sending webmention failed: %s", e)
|
||||||
|
return False
|
||||||
|
|
||||||
def receive(self):
|
def receive(self):
|
||||||
self._fetch()
|
self._fetch()
|
||||||
|
@ -1464,6 +1489,7 @@ def build():
|
||||||
for f, post in content:
|
for f, post in content:
|
||||||
logging.info("PARSING %s", f)
|
logging.info("PARSING %s", f)
|
||||||
post.init_extras()
|
post.init_extras()
|
||||||
|
post.queue_webmentions()
|
||||||
|
|
||||||
# add to sitemap
|
# add to sitemap
|
||||||
sitemap.update({ post.url: post.mtime })
|
sitemap.update({ post.url: post.mtime })
|
||||||
|
@ -1525,17 +1551,24 @@ def build():
|
||||||
if not c.is_uptodate or shared.config.getboolean('params', 'force'):
|
if not c.is_uptodate or shared.config.getboolean('params', 'force'):
|
||||||
worker.append(c.render())
|
worker.append(c.render())
|
||||||
|
|
||||||
# TODO move ping to separate function and add it as a task
|
|
||||||
# TODO separate an aiohttpworker?
|
|
||||||
|
|
||||||
# add magic.php rendering
|
# add magic.php rendering
|
||||||
worker.append(magic.render())
|
worker.append(magic.render())
|
||||||
|
|
||||||
# TODO: send webmentions
|
|
||||||
|
|
||||||
# do all the things!
|
# do all the things!
|
||||||
worker.run()
|
worker.run()
|
||||||
|
|
||||||
|
# send webmentions - this is synchronous due to the SQLite locking
|
||||||
|
wdb = shared.WebmentionQueue()
|
||||||
|
for out in wdb.get_outbox():
|
||||||
|
wm = Webmention(
|
||||||
|
out.get('source'),
|
||||||
|
out.get('target'),
|
||||||
|
out.get('dt')
|
||||||
|
)
|
||||||
|
if wm.send():
|
||||||
|
wdb.entry_done(out.get('id'))
|
||||||
|
wdb.finish()
|
||||||
|
|
||||||
# copy static
|
# copy static
|
||||||
logging.info('copying static files')
|
logging.info('copying static files')
|
||||||
src = shared.config.get('dirs', 'static')
|
src = shared.config.get('dirs', 'static')
|
||||||
|
|
19
router.py
19
router.py
|
@ -3,16 +3,16 @@
|
||||||
# vim: set fileencoding=utf-8 :
|
# vim: set fileencoding=utf-8 :
|
||||||
|
|
||||||
__author__ = "Peter Molnar"
|
__author__ = "Peter Molnar"
|
||||||
__copyright__ = "Copyright 2017, Peter Molnar"
|
__copyright__ = "Copyright 2017-2018, Peter Molnar"
|
||||||
__license__ = "GPLv3"
|
__license__ = "GPLv3"
|
||||||
__version__ = "2.0"
|
__version__ = "2.1.0"
|
||||||
__maintainer__ = "Peter Molnar"
|
__maintainer__ = "Peter Molnar"
|
||||||
__email__ = "hello@petermolnar.eu"
|
__email__ = "hello@petermolnar.eu"
|
||||||
__status__ = "Production"
|
__status__ = "Production"
|
||||||
|
|
||||||
"""
|
"""
|
||||||
silo archiver module of NASG
|
silo archiver module of NASG
|
||||||
Copyright (C) 2017 Peter Molnar
|
Copyright (C) 2017-2018 Peter Molnar
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
This program is free software; you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
|
@ -39,21 +39,10 @@ import envelope
|
||||||
import socket
|
import socket
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
#logging_format = "[%(asctime)s] %(process)d-%(levelname)s "
|
|
||||||
#logging_format += "%(module)s::%(funcName)s():l%(lineno)d: "
|
|
||||||
#logging_format += "%(message)s"
|
|
||||||
|
|
||||||
#logging.basicConfig(
|
|
||||||
#format=logging_format,
|
|
||||||
#level=logging.DEBUG
|
|
||||||
#)
|
|
||||||
#log = logging.getLogger()
|
|
||||||
|
|
||||||
# log_config=None prevents creation of access_log and error_log files
|
# log_config=None prevents creation of access_log and error_log files
|
||||||
# since I'm running this from systemctl it already goes into syslog
|
# since I'm running this from systemctl it already goes into syslog
|
||||||
app = Sanic('router')
|
app = Sanic('router')
|
||||||
#app = Sanic('router', log_config=None)
|
# this is read only this way!
|
||||||
# this is ok to be read-only
|
|
||||||
sdb = shared.SearchDB()
|
sdb = shared.SearchDB()
|
||||||
|
|
||||||
@app.route("/oauth1", methods=["GET"])
|
@app.route("/oauth1", methods=["GET"])
|
||||||
|
|
26
shared.py
26
shared.py
|
@ -3,16 +3,16 @@
|
||||||
# vim: set fileencoding=utf-8 :
|
# vim: set fileencoding=utf-8 :
|
||||||
|
|
||||||
__author__ = "Peter Molnar"
|
__author__ = "Peter Molnar"
|
||||||
__copyright__ = "Copyright 2017, Peter Molnar"
|
__copyright__ = "Copyright 2017-2018, Peter Molnar"
|
||||||
__license__ = "GPLv3"
|
__license__ = "GPLv3"
|
||||||
__version__ = "2.0"
|
__version__ = "2.1.0"
|
||||||
__maintainer__ = "Peter Molnar"
|
__maintainer__ = "Peter Molnar"
|
||||||
__email__ = "hello@petermolnar.eu"
|
__email__ = "hello@petermolnar.eu"
|
||||||
__status__ = "Production"
|
__status__ = "Production"
|
||||||
|
|
||||||
"""
|
"""
|
||||||
silo archiver module of NASG
|
silo archiver module of NASG
|
||||||
Copyright (C) 2017 Peter Molnar
|
Copyright (C) 2017-2018 Peter Molnar
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
This program is free software; you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
|
@ -42,6 +42,7 @@ from slugify import slugify
|
||||||
import jinja2
|
import jinja2
|
||||||
from inspect import getsourcefile
|
from inspect import getsourcefile
|
||||||
import sys
|
import sys
|
||||||
|
import arrow
|
||||||
|
|
||||||
class CMDLine(object):
|
class CMDLine(object):
|
||||||
def __init__(self, executable):
|
def __init__(self, executable):
|
||||||
|
@ -485,6 +486,8 @@ class SearchDB(BaseDB):
|
||||||
|
|
||||||
|
|
||||||
class WebmentionQueue(BaseDB):
|
class WebmentionQueue(BaseDB):
|
||||||
|
tsform = 'YYYY-MM-DD HH:mm:ss'
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.fpath = "%s" % config.get('var', 'webmentiondb')
|
self.fpath = "%s" % config.get('var', 'webmentiondb')
|
||||||
super().__init__(self.fpath)
|
super().__init__(self.fpath)
|
||||||
|
@ -507,7 +510,7 @@ class WebmentionQueue(BaseDB):
|
||||||
def finish(self):
|
def finish(self):
|
||||||
self.db.close()
|
self.db.close()
|
||||||
|
|
||||||
def exists(self, source, target):
|
def exists(self, source, target, dt=arrow.now()):
|
||||||
logging.debug(
|
logging.debug(
|
||||||
'checking webmention existence for source: %s ; target: %s',
|
'checking webmention existence for source: %s ; target: %s',
|
||||||
source,
|
source,
|
||||||
|
@ -515,15 +518,22 @@ class WebmentionQueue(BaseDB):
|
||||||
)
|
)
|
||||||
cursor = self.db.cursor()
|
cursor = self.db.cursor()
|
||||||
cursor.execute(
|
cursor.execute(
|
||||||
'''SELECT id FROM queue WHERE source=? AND target=? LIMIT 1''',
|
'''SELECT id,timestamp FROM queue WHERE source=? AND target=? ORDER BY timestamp DESC LIMIT 1''',
|
||||||
(source,target)
|
(source,target)
|
||||||
)
|
)
|
||||||
|
|
||||||
rows = cursor.fetchall()
|
rows = cursor.fetchall()
|
||||||
if not rows:
|
if not rows:
|
||||||
return False
|
return False
|
||||||
return int(rows.pop()[0])
|
|
||||||
|
row = rows.pop()
|
||||||
|
if arrow.get(row[1], self.tsform).timestamp >= dt.timestamp:
|
||||||
|
return int(row[0])
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
def queue(self, source, target):
|
def queue(self, source, target):
|
||||||
|
logging.debug("Queueing webmention: %s to %s", source, target)
|
||||||
cursor = self.db.cursor()
|
cursor = self.db.cursor()
|
||||||
cursor.execute(
|
cursor.execute(
|
||||||
'''INSERT INTO queue (source,target) VALUES (?,?);''', (
|
'''INSERT INTO queue (source,target) VALUES (?,?);''', (
|
||||||
|
@ -585,14 +595,14 @@ class WebmentionQueue(BaseDB):
|
||||||
cursor = self.db.cursor()
|
cursor = self.db.cursor()
|
||||||
ret = []
|
ret = []
|
||||||
cursor.execute(
|
cursor.execute(
|
||||||
'''SELECT * FROM queue WHERE source LIKE ? AND status = 0''',
|
'''SELECT id,timestamp,source,target FROM queue WHERE source LIKE ? AND status = 0''',
|
||||||
('%' + config.get('common', 'domain') + '%',)
|
('%' + config.get('common', 'domain') + '%',)
|
||||||
)
|
)
|
||||||
rows = cursor.fetchall()
|
rows = cursor.fetchall()
|
||||||
for r in rows:
|
for r in rows:
|
||||||
ret.append({
|
ret.append({
|
||||||
'id': r[0],
|
'id': r[0],
|
||||||
'dt': r[1],
|
'dt': arrow.get(r[1], self.tsform).timestamp,
|
||||||
'source': r[2],
|
'source': r[2],
|
||||||
'target': r[3],
|
'target': r[3],
|
||||||
})
|
})
|
||||||
|
|
Loading…
Reference in a new issue