petermolnar/nasg
1
0
Fork 0
Code Issues Pull requests 5 Projects Releases Packages Wiki Activity Actions

bridgy publish + webmention.io backfill fix, thx to @snarfed for https://github.com/snarfed/bridgy/issues/84

Browse source
This commit is contained in:
Peter Molnar 2019-08-28 08:53:09 +01:00
parent c5ec1e95fa
commit 36a8e1d7a0
1 changed files with 48 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

89
nasg.py
View file

@ -1246,7 +1246,7 @@ class Singular(MarkdownDoc):
return return
if self.is_future: if self.is_future:
return return
if (self.published.timestamp + 7200) > arrow.utcnow().timestamp: if (self.published.timestamp + 86400) > arrow.utcnow().timestamp:
return return
logger.info("archive.org .copy is missing for %s", self.name) logger.info("archive.org .copy is missing for %s", self.name)
if len(self.category) and not ( if len(self.category) and not (
@ -2104,6 +2104,18 @@ class Sitemap(dict):
f.write("\n".join(sorted(self.keys()))) f.write("\n".join(sorted(self.keys())))
#def json_decode(string):
#r = {}
#try:
#r = json.loads(string)
#for k, v in j.items():
#if isinstance(v, str):
#r[k] = json_decode(v)
#except Exception as e:
##logger.error("failed to recursive parse JSON portion: %s", e)
#pass
#return r
class Webmention(object): class Webmention(object):
""" outgoing webmention class """ """ outgoing webmention class """
@ -2172,56 +2184,51 @@ class Webmention(object):
txt = f.read() txt = f.read()
try: try:
maybe = json.loads(txt) data = json.loads(txt)
except Exception as e: except Exception as e:
# if it's not a JSON, it's a manually placed file, ignore it """ if it's not a JSON, it's a manually placed file, ignore it """
logger.debug("not a JSON webmention at %s", self.fpath)
return return
if "status" in maybe and "error" == maybe["status"]: # unprocessed webmention
logger.error( if "http_body" not in data and "location" in data:
"errored webmention found at %s: %s", self.dpath, maybe logger.debug(
"fetching webmention.io respose from %s",
data["location"]
) )
# maybe["location"] = maybe[""] wio = requests.get(data["location"])
# TODO finish cleanup and re-fetching with from 'original' in JSON if wio.status_code != requests.codes.ok:
return logger.debug("fetching %s failed", data["location"])
try:
if "location" not in maybe:
return return
if "http_body" not in maybe:
logger.debug( try:
"trying to re-fetch %s for %s", wio_json = json.loads(wio.text)
maybe["location"], logger.debug("got response %s", wio_json)
self.fpath, if "http_body" in wio_json and isinstance(wio_json["http_body"], str):
) wio_json.update({"http_body": json.loads("".join(wio_json["http_body"]))})
wio = requests.get(maybe["location"]) if "original" in wio_json["http_body"].keys():
if wio.status_code != requests.codes.ok: wio_json.update({"http_body": wio_json["http_body"]["original"]})
return data = {**data, **wio_json}
maybe = wio.json() except Exception as e:
logger.debug("response: %s", maybe) logger.error("failed to JSON load webmention.io response %s because: %s", wio.text, e)
with open(self.fpath, "wt") as update: return
update.write(
json.dumps(maybe, sort_keys=True, indent=4) logger.debug("saving updated webmention.io data %s to %s", data, self.fpath)
) with open(self.fpath, "wt") as update:
if "url" in maybe["http_body"]: update.write(json.dumps(data, sort_keys=True, indent=4))
data = json.loads(maybe["http_body"])
url = data["url"] if "http_body" in data.keys():
# healthy and processed webmention
if isinstance(data["http_body"], dict) and "url" in data["http_body"].keys():
url = data["http_body"]["url"]
sp = os.path.join(self.dpath, "%s.copy" % url2slug(url)) sp = os.path.join(self.dpath, "%s.copy" % url2slug(url))
if os.path.exists(sp): if os.path.exists(sp):
logger.debug("syndication already exists for %s", url)
return return
with open(sp, "wt") as f: with open(sp, "wt") as f:
logger.info( logger.info("writing syndication copy %s to %s", url, sp)
"writing syndication copy %s to %s", url, sp
)
f.write(url) f.write(url)
except Exception as e: return
logger.error(
"failed to fetch syndication URL for %s: %s",
self.dpath,
e,
)
pass
class WebmentionIO(object): class WebmentionIO(object):
def __init__(self): def __init__(self):