572 lines
17 KiB
Python
572 lines
17 KiB
Python
import os
|
|
import glob
|
|
import imghdr
|
|
import re
|
|
import logging
|
|
import shutil
|
|
import subprocess
|
|
import json
|
|
from io import BytesIO
|
|
import lxml.etree as etree
|
|
from slugify import slugify
|
|
import requests
|
|
from requests.auth import HTTPBasicAuth
|
|
import arrow
|
|
import settings
|
|
import keys
|
|
import yaml
|
|
from pprint import pprint
|
|
import feedparser
|
|
|
|
TMPFEXT = ".xyz"
|
|
MDFEXT = ".md"
|
|
|
|
TMPSUBDIR = "nasg"
|
|
SHM = "/dev/shm"
|
|
|
|
if os.path.isdir(SHM) and os.access(SHM, os.W_OK):
|
|
TMPDIR = f"{SHM}/{TMPSUBDIR}"
|
|
else:
|
|
TMPDIR = os.path.join(gettempdir(), TMPSUBDIR)
|
|
|
|
if not os.path.isdir(TMPDIR):
|
|
os.makedirs(TMPDIR)
|
|
|
|
|
|
def utfyamldump(data):
|
|
""" dump YAML with actual UTF-8 chars """
|
|
return yaml.dump(
|
|
data, default_flow_style=False, indent=4, allow_unicode=True
|
|
)
|
|
|
|
|
|
def url2slug(url):
|
|
return slugify(
|
|
re.sub(r"^https?://(?:www)?", "", url),
|
|
only_ascii=True,
|
|
lower=True,
|
|
)[:200]
|
|
|
|
|
|
class cached_property(object):
|
|
""" extermely simple cached_property decorator:
|
|
whenever something is called as @cached_property, on first run, the
|
|
result is calculated, then the class method is overwritten to be
|
|
a property, contaning the result from the method
|
|
"""
|
|
|
|
def __init__(self, method, name=None):
|
|
self.method = method
|
|
self.name = name or method.__name__
|
|
|
|
def __get__(self, inst, cls):
|
|
if inst is None:
|
|
return self
|
|
result = self.method(inst)
|
|
setattr(inst, self.name, result)
|
|
return result
|
|
|
|
|
|
class Aperture(object):
|
|
def __init__(self):
|
|
self.session = requests.Session()
|
|
self.session.headers.update(
|
|
{
|
|
"Authorization": "Bearer %s"
|
|
% (keys.aperture["access_token"])
|
|
}
|
|
)
|
|
self.url = keys.aperture["url"]
|
|
|
|
@cached_property
|
|
def channels(self):
|
|
channels = self.session.get(f"{self.url}?action=channels")
|
|
if channels.status_code != requests.codes.ok:
|
|
logging.error(
|
|
"failed to get channels from aperture: ", channels.text
|
|
)
|
|
return None
|
|
try:
|
|
channels = channels.json()
|
|
except ValueError as e:
|
|
logging.error("failed to parse channels from aperture: ", e)
|
|
return None
|
|
|
|
if "channels" not in channels:
|
|
logging.error("no channels found in aperture: ")
|
|
return None
|
|
|
|
return channels["channels"]
|
|
|
|
def channelid(self, channelname):
|
|
for channel in self.channels:
|
|
if channel["name"].lower() == channelname.lower():
|
|
return channel["uid"]
|
|
return None
|
|
|
|
def feedmeta(self, url):
|
|
cfile = os.path.join(
|
|
TMPDIR,
|
|
"%s.%s.json" % (url2slug(url), self.__class__.__name__)
|
|
)
|
|
if os.path.exists(cfile):
|
|
with open(cfile, 'rt') as cache:
|
|
return json.loads(cache.read())
|
|
r = {
|
|
'title': url,
|
|
'feed': url,
|
|
'link': url,
|
|
'type': 'rss'
|
|
}
|
|
try:
|
|
feed = feedparser.parse(url)
|
|
if 'feed' in feed:
|
|
for maybe in ['title', 'link']:
|
|
if maybe in feed['feed']:
|
|
r[maybe] = feed['feed'][maybe]
|
|
except Exception as e:
|
|
logging.error("feedparser failed on %s: %s" %(url, e))
|
|
r['type']: 'hfeed'
|
|
pass
|
|
|
|
with open(cfile, 'wt') as cache:
|
|
cache.write(json.dumps(r))
|
|
|
|
return r
|
|
|
|
|
|
def channelfollows(self, channelid):
|
|
follows = self.session.get(
|
|
f"{self.url}?action=follow&channel={channelid}"
|
|
)
|
|
if follows.status_code != requests.codes.ok:
|
|
logging.error(
|
|
"failed to get follows from aperture: ", follows.text
|
|
)
|
|
return
|
|
try:
|
|
follows = follows.json()
|
|
except ValueError as e:
|
|
logging.error("failed to parse follows from aperture: ", e)
|
|
return
|
|
|
|
if "items" not in follows:
|
|
logging.error(
|
|
f"no follows found in aperture for channel {channelid}"
|
|
)
|
|
return
|
|
|
|
existing = {}
|
|
for follow in follows["items"]:
|
|
meta = self.feedmeta(follow["url"])
|
|
existing.update({follow["url"]: meta})
|
|
return existing
|
|
|
|
@cached_property
|
|
def follows(self):
|
|
follows = {}
|
|
for channel in self.channels:
|
|
follows[channel["name"]] = self.channelfollows(
|
|
channel["uid"]
|
|
)
|
|
return follows
|
|
|
|
def export(self):
|
|
opml = etree.Element("opml", version="1.0")
|
|
xmldoc = etree.ElementTree(opml)
|
|
opml.addprevious(
|
|
etree.ProcessingInstruction(
|
|
"xml-stylesheet",
|
|
'type="text/xsl" href="%s"'
|
|
% (settings.opml.get("xsl")),
|
|
)
|
|
)
|
|
|
|
head = etree.SubElement(opml, "head")
|
|
title = etree.SubElement(
|
|
head, "title"
|
|
).text = settings.opml.get("title")
|
|
dt = etree.SubElement(
|
|
head, "dateCreated"
|
|
).text = arrow.utcnow().format("ddd, DD MMM YYYY HH:mm:ss UTC")
|
|
owner = etree.SubElement(
|
|
head, "ownerName"
|
|
).text = settings.opml.get("owner")
|
|
email = etree.SubElement(
|
|
head, "ownerEmail"
|
|
).text = settings.opml.get("email")
|
|
|
|
body = etree.SubElement(opml, "body")
|
|
groups = {}
|
|
for group, feeds in self.follows.items():
|
|
if (
|
|
"private" in group.lower()
|
|
or "nsfw" in group.lower()
|
|
):
|
|
continue
|
|
|
|
if group not in groups.keys():
|
|
groups[group] = etree.SubElement(
|
|
body, "outline", text=group
|
|
)
|
|
for url, meta in feeds.items():
|
|
entry = etree.SubElement(
|
|
groups[group],
|
|
"outline",
|
|
type="rss",
|
|
text=meta['title'],
|
|
xmlUrl=meta['feed'],
|
|
htmlUrl=meta['link']
|
|
)
|
|
etree.tostring(
|
|
xmldoc,
|
|
encoding="utf-8",
|
|
xml_declaration=True,
|
|
pretty_print=True,
|
|
)
|
|
opmlfile = os.path.join(
|
|
settings.paths.get("content"), "following.opml"
|
|
)
|
|
with open(opmlfile, "wb") as f:
|
|
f.write(
|
|
etree.tostring(
|
|
xmldoc,
|
|
encoding="utf-8",
|
|
xml_declaration=True,
|
|
pretty_print=True,
|
|
)
|
|
)
|
|
|
|
|
|
class MinifluxFollows(dict):
|
|
def __init__(self):
|
|
self.auth = HTTPBasicAuth(
|
|
keys.miniflux.get("username"), keys.miniflux.get("token")
|
|
)
|
|
|
|
@property
|
|
def subscriptions(self):
|
|
feeds = []
|
|
params = {
|
|
"jsonrpc": "2.0",
|
|
"method": "getFeeds",
|
|
"id": keys.miniflux.get("id"),
|
|
}
|
|
r = requests.post(
|
|
keys.miniflux.get("url"),
|
|
data=json.dumps(params),
|
|
auth=self.auth,
|
|
)
|
|
return r.json().get("result", [])
|
|
|
|
def sync(self):
|
|
current = []
|
|
for feed in self.subscriptions:
|
|
try:
|
|
current.append(feed["feed_url"])
|
|
except Exception as e:
|
|
logging.error("problem with feed entry: %s", feed)
|
|
for silo, feeds in self.items():
|
|
for feed in feeds:
|
|
xmlurl = feed.get("xmlUrl")
|
|
if len(xmlurl) and xmlurl not in current:
|
|
logging.info("creating subscription for: %s", feed)
|
|
params = {
|
|
"jsonrpc": "2.0",
|
|
"method": "createFeed",
|
|
"id": keys.miniflux.get("id"),
|
|
"params": {"url": xmlurl, "group_name": silo},
|
|
}
|
|
r = requests.post(
|
|
keys.miniflux.get("url"),
|
|
data=json.dumps(params),
|
|
auth=self.auth,
|
|
)
|
|
|
|
def export(self):
|
|
opml = etree.Element("opml", version="1.0")
|
|
xmldoc = etree.ElementTree(opml)
|
|
opml.addprevious(
|
|
etree.ProcessingInstruction(
|
|
"xml-stylesheet",
|
|
'type="text/xsl" href="%s"'
|
|
% (settings.opml.get("xsl")),
|
|
)
|
|
)
|
|
head = etree.SubElement(opml, "head")
|
|
title = etree.SubElement(
|
|
head, "title"
|
|
).text = settings.opml.get("title")
|
|
dt = etree.SubElement(
|
|
head, "dateCreated"
|
|
).text = arrow.utcnow().format("ddd, DD MMM YYYY HH:mm:ss UTC")
|
|
owner = etree.SubElement(
|
|
head, "ownerName"
|
|
).text = settings.opml.get("owner")
|
|
email = etree.SubElement(
|
|
head, "ownerEmail"
|
|
).text = settings.opml.get("email")
|
|
|
|
body = etree.SubElement(opml, "body")
|
|
groups = {}
|
|
for feed in self.subscriptions:
|
|
# contains sensitive data, skip it
|
|
if "sessionid" in feed.get(
|
|
"feed_url"
|
|
) or "sessionid" in feed.get("site_url"):
|
|
continue
|
|
|
|
fgroup = feed.get("groups", None)
|
|
if not fgroup:
|
|
fgroup = [{"title": "Unknown", "id": -1}]
|
|
fgroup = fgroup.pop()
|
|
# some groups need to be skipped
|
|
if fgroup["title"].lower() in ["private"]:
|
|
continue
|
|
if fgroup["title"] not in groups.keys():
|
|
groups[fgroup["title"]] = etree.SubElement(
|
|
body, "outline", text=fgroup["title"]
|
|
)
|
|
entry = etree.SubElement(
|
|
groups[fgroup["title"]],
|
|
"outline",
|
|
type="rss",
|
|
text=feed.get("title"),
|
|
xmlUrl=feed.get("feed_url"),
|
|
htmlUrl=feed.get("site_url"),
|
|
)
|
|
|
|
opmlfile = os.path.join(
|
|
settings.paths.get("content"), "following.opml"
|
|
)
|
|
|
|
with open(opmlfile, "wb") as f:
|
|
f.write(
|
|
etree.tostring(
|
|
xmldoc,
|
|
encoding="utf-8",
|
|
xml_declaration=True,
|
|
pretty_print=True,
|
|
)
|
|
)
|
|
|
|
|
|
class Favs(object):
|
|
def __init__(self, silo):
|
|
self.silo = silo
|
|
self.aperture_auth = {
|
|
"Authorization": "Bearer %s"
|
|
% (keys.aperture["access_token"])
|
|
}
|
|
self.aperture_chid = 0
|
|
|
|
@property
|
|
def feeds(self):
|
|
return []
|
|
|
|
@property
|
|
def since(self):
|
|
d = os.path.join(
|
|
settings.paths.get("archive"), "favorite", "%s*" % self.silo
|
|
)
|
|
files = glob.glob(d)
|
|
if len(files):
|
|
mtime = max([int(os.path.getmtime(f)) for f in files])
|
|
else:
|
|
mtime = 0
|
|
return mtime
|
|
|
|
def sync_with_aperture(self):
|
|
channels = requests.get(
|
|
"%s?action=channels" % (keys.aperture["url"]),
|
|
headers=self.aperture_auth,
|
|
)
|
|
if channels.status_code != requests.codes.ok:
|
|
logging.error(
|
|
"failed to get channels from aperture: ", channels.text
|
|
)
|
|
return
|
|
try:
|
|
channels = channels.json()
|
|
except ValueError as e:
|
|
logging.error("failed to parse channels from aperture: ", e)
|
|
return
|
|
|
|
if "channels" not in channels:
|
|
logging.error("no channels found in aperture: ")
|
|
return
|
|
|
|
for channel in channels["channels"]:
|
|
if channel["name"].lower() == self.silo.lower():
|
|
self.aperture_chid = channel["uid"]
|
|
break
|
|
|
|
if not self.aperture_chid:
|
|
logging.error("no channels found for silo ", self.silo)
|
|
return
|
|
|
|
follows = requests.get(
|
|
"%s?action=follow&channel=%s"
|
|
% (keys.aperture["url"], self.aperture_chid),
|
|
headers=self.aperture_auth,
|
|
)
|
|
if follows.status_code != requests.codes.ok:
|
|
logging.error(
|
|
"failed to get follows from aperture: ", follows.text
|
|
)
|
|
return
|
|
try:
|
|
follows = follows.json()
|
|
except ValueError as e:
|
|
logging.error("failed to parse follows from aperture: ", e)
|
|
return
|
|
|
|
if "items" not in follows:
|
|
logging.error(
|
|
"no follows found in aperture for channel %s (%s)"
|
|
% (self.silo, self.aperture_chid)
|
|
)
|
|
return
|
|
|
|
existing = []
|
|
for follow in follows["items"]:
|
|
existing.append(follow["url"])
|
|
existing = list(set(existing))
|
|
|
|
for feed in self.feeds:
|
|
if feed["xmlUrl"] not in existing:
|
|
subscribe_to = {
|
|
"action": "follow",
|
|
"channel": self.aperture_chid,
|
|
"url": feed["xmlUrl"],
|
|
}
|
|
logging.info(
|
|
"subscribing to %s into %s (%s)"
|
|
% (feed, self.silo, self.aperture_chid)
|
|
)
|
|
subscribe = requests.post(
|
|
keys.aperture["url"],
|
|
headers=self.aperture_auth,
|
|
data=subscribe_to,
|
|
)
|
|
logging.debug(subscribe.text)
|
|
|
|
|
|
class ImgFav(object):
|
|
def __init__(self):
|
|
return
|
|
|
|
def run(self):
|
|
if not self.exists:
|
|
self.fetch_images()
|
|
self.save_txt()
|
|
|
|
@property
|
|
def exists(self):
|
|
maybe = glob.glob("%s*" % self.targetprefix)
|
|
if len(maybe):
|
|
return True
|
|
return False
|
|
|
|
def save_txt(self):
|
|
attachments = [
|
|
os.path.basename(fn)
|
|
for fn in glob.glob("%s*" % self.targetprefix)
|
|
if not os.path.basename(fn).endswith(".md")
|
|
]
|
|
meta = {
|
|
"title": self.title,
|
|
"favorite-of": self.url,
|
|
"date": str(self.published),
|
|
"sources": list(self.images.values()),
|
|
"attachments": attachments,
|
|
"author": self.author,
|
|
}
|
|
r = "---\n%s\n---\n\n" % (utfyamldump(meta))
|
|
with open("%s%s" % (self.targetprefix, MDFEXT), "wt") as fpath:
|
|
fpath.write(r)
|
|
|
|
def fetch_images(self):
|
|
for fpath, url in self.images.items():
|
|
self.fetch_image(fpath, url)
|
|
|
|
def fetch_image(self, fpath, url):
|
|
logging.info("pulling image %s to %s", url, fpath)
|
|
r = requests.get(url, stream=True)
|
|
if r.status_code == 200:
|
|
with open(fpath, "wb") as f:
|
|
r.raw.decode_content = True
|
|
shutil.copyfileobj(r.raw, f)
|
|
|
|
imgtype = imghdr.what(fpath)
|
|
if not imgtype:
|
|
os.remove(fpath)
|
|
return
|
|
if imgtype in ["jpg", "jpeg", "png"]:
|
|
self.write_exif(fpath)
|
|
os.rename(fpath, fpath.replace(TMPFEXT, ".%s" % (imgtype)))
|
|
|
|
def write_exif(self, fpath):
|
|
logging.info("populating EXIF data of %s" % fpath)
|
|
|
|
geo_lat = False
|
|
geo_lon = False
|
|
|
|
if hasattr(self, "geo") and self.geo != None:
|
|
lat, lon = self.geo
|
|
if lat and lon and "null" != lat and "null" != lon:
|
|
geo_lat = lat
|
|
geo_lon = lon
|
|
|
|
params = [
|
|
"exiftool",
|
|
"-overwrite_original",
|
|
"-XMP:Copyright=Copyright %s %s (%s)"
|
|
% (
|
|
self.published.to("utc").format("YYYY"),
|
|
self.author.get("name"),
|
|
self.author.get("url"),
|
|
),
|
|
"-XMP:Source=%s" % self.url,
|
|
"-XMP:ReleaseDate=%s"
|
|
% self.published.to("utc").format("YYYY:MM:DD HH:mm:ss"),
|
|
"-XMP:Headline=%s" % self.title,
|
|
"-XMP:Description=%s" % self.content,
|
|
]
|
|
|
|
for t in self.tags:
|
|
params.append("-XMP:HierarchicalSubject+=%s" % t)
|
|
params.append("-XMP:Subject+=%s" % t)
|
|
|
|
if geo_lat and geo_lon:
|
|
geo_lat = round(float(geo_lat), 6)
|
|
geo_lon = round(float(geo_lon), 6)
|
|
|
|
if geo_lat < 0:
|
|
GPSLatitudeRef = "S"
|
|
else:
|
|
GPSLatitudeRef = "N"
|
|
|
|
if geo_lon < 0:
|
|
GPSLongitudeRef = "W"
|
|
else:
|
|
GPSLongitudeRef = "E"
|
|
|
|
params.append("-GPSLongitude=%s" % abs(geo_lon))
|
|
params.append("-GPSLatitude=%s" % abs(geo_lat))
|
|
params.append("-GPSLongitudeRef=%s" % GPSLongitudeRef)
|
|
params.append("-GPSLatitudeRef=%s" % GPSLatitudeRef)
|
|
|
|
params.append(fpath)
|
|
|
|
p = subprocess.Popen(
|
|
params,
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
stdout, stderr = p.communicate()
|
|
_original = "%s_original" % fpath
|
|
if os.path.exists(_original):
|
|
os.unlink(_original)
|