petermolnar/silo.pasta
1
0
Fork 0
Code Issues Pull requests 2 Projects Releases Packages Wiki Activity Actions

I lost track of changes, this is a commit that has to be followed by a large cleanup

Browse source
This commit is contained in:
Peter Molnar 2020-05-06 13:27:49 +01:00
parent 6a3b127245
commit d5665d15e7
11 changed files with 401 additions and 342 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.gitignore vendored
View file

@ -1,7 +1,6 @@
keys.py
.venv
__pycache
keys.py
__pycache__
lib
__pycache
__pycache__
_scratch
.venv

59
Artstation.py
View file

@ -7,34 +7,51 @@ import requests
import keys
import common
import settings
from time import sleep
from math import ceil
import random
from pprint import pprint
class ASFavs(common.Favs):
headers = {
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Accept-Encoding": "gzip, deflate, br",
"DNT": "1",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"Pragma": "no-cache",
"Cache-Control": "no-cache",
}
def __init__(self):
super().__init__("artstation")
self.user = keys.artstation.get("username")
self.session = requests.Session()
self.session.headers.update({
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
#"DNT": "1",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"Pragma": "no-cache",
"Cache-Control": "max-age=0, no-cache",
})
session.headers.update({
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"DNT": "1",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"Pragma": "no-cache",
"Cache-Control": "max-age=0, no-cache",
})
def paged_likes(self, page=1):
url = "https://www.artstation.com/users/%s/likes.json?page=%s" % (
self.user,
page,
)
js = self.session.get(url, headers=self.headers)
js = self.session.get(url)
while js.status_code != requests.codes.ok:
# FU cloudflare
pprint(self.session.cookies)
sleep(round(random.uniform(0.7,3.5), 2))
js = self.session.get(url)
try:
js = js.json()
if "data" not in js:
@ -46,18 +63,13 @@ class ASFavs(common.Favs):
@property
def likes(self):
# init Session and it's cookies before doing anything
# FU cloudflare, I'm trying to access my own likes and followings!
url = "https://www.artstation.com/"
self.session.get(url, headers=self.headers)
# now do the real work
js = self.paged_likes()
js = self.paged_likes(1)
if not js:
return []
likes = js.get("data", [])
pages = ceil(js.get("total_count", 1) / 50)
while pages > 1:
extras = self.paged_likes()
extras = self.paged_likes(pages)
if not extras:
continue
likes = likes + extras.get("data", [])
@ -88,6 +100,7 @@ class ASFavs(common.Favs):
return feeds
def run(self):
# FU cloudflare
for like in self.likes:
like = ASLike(like, self.session, self.headers)
like.run()
@ -138,14 +151,14 @@ class ASLike(common.ImgFav):
if not len(title):
title = self.like.get("slug")
if not len(title):
title = common.slugfname(self.url)
title = common.url2slug(self.url)
return title
@property
def slug(self):
maybe = self.like.get("slug")
if not len(maybe):
maybe = common.slugfname(self.url)
maybe = common.url2slug(self.url)
return maybe
@property
@ -155,7 +168,7 @@ class ASLike(common.ImgFav):
"favorite",
"artstation_%s_%s_%s"
% (
common.slugfname("%s" % self.like.get("user").get("username")),
common.url2slug("%s" % self.like.get("user").get("username")),
self.like.get("hash_id"),
self.slug,
),

8
DeviantArt.py
View file

@ -122,7 +122,7 @@ class DAFav(common.ImgFav):
def title(self):
title = self.deviation.title
if not len(title):
title = common.slugfname(self.url)
title = common.url2slug(self.url)
return clean(title.strip())
@property
@ -132,15 +132,15 @@ class DAFav(common.ImgFav):
"favorite",
"deviantart_%s_%s_%s"
% (
common.slugfname("%s" % self.deviation.author),
common.url2slug("%s" % self.deviation.author),
self.id,
common.slugfname("%s" % self.title),
common.url2slug("%s" % self.title),
),
)
@property
def published(self):
return arrow.get(self.deviation.published_time)
return arrow.get(int(self.deviation.published_time))
@property
def tags(self):

11
Flickr.py
View file

@ -10,9 +10,6 @@ import settings
from pprint import pprint
import logging
# class FlickrFollows(common.Follows):
class FlickrFavs(common.Favs):
def __init__(self):
super().__init__("flickr")
@ -109,12 +106,16 @@ class FlickrFav(common.ImgFav):
return os.path.join(
settings.paths.get("archive"),
"favorite",
"flickr_%s_%s" % (common.slugfname("%s" % self.owner.id), self.id),
"flickr_%s_%s" % (common.url2slug("%s" % self.owner.id), self.id),
)
@property
def published(self):
return arrow.get(self.info.get("dateuploaded"))
x = self.info.get("dateuploaded")
if x.isnumeric():
return arrow.get(int(x))
else:
return arrow.get(x)
@property
def tags(self):

51
LastFM.py
View file

@ -10,31 +10,12 @@ import settings
import keys
from pprint import pprint
from math import floor
from common import cached_property
Track = namedtuple(
"Track", ["timestamp", "artist", "album", "title", "artistid", "albumid", "img"]
)
class cached_property(object):
""" extermely simple cached_property decorator:
whenever something is called as @cached_property, on first run, the
result is calculated, then the class method is overwritten to be
a property, contaning the result from the method
"""
def __init__(self, method, name=None):
self.method = method
self.name = name or method.__name__
def __get__(self, inst, cls):
if inst is None:
return self
result = self.method(inst)
setattr(inst, self.name, result)
return result
class LastFM(object):
url = "http://ws.audioscrobbler.com/2.0/"
@ -46,9 +27,9 @@ class LastFM(object):
"format": "json",
"limit": "200",
}
if os.path.isfile(self.target):
mtime = os.path.getmtime(self.target)
self.params.update({"from": mtime})
# if os.path.isfile(self.target):
# mtime = os.path.getmtime(self.target)
# self.params.update({"from": mtime})
@property
def target(self):
@ -57,14 +38,15 @@ class LastFM(object):
@cached_property
def existing(self):
timestamps = []
with open(self.target, "r") as f:
r = csv.reader(f)
for row in r:
try:
timestamps.append(arrow.get(row[0]).timestamp)
except Exception as e:
logging.error("arrow failed on row %s", row)
continue
if os.path.isfile(self.target):
with open(self.target, "r") as f:
r = csv.reader(f)
for row in r:
try:
timestamps.append(arrow.get(row[0]).timestamp)
except Exception as e:
logging.error("arrow failed on row %s", row)
continue
return timestamps
@property
@ -98,8 +80,11 @@ class LastFM(object):
return json.loads(r.text).get("recenttracks")
def run(self):
startpage = floor(len(self.existing) / int(self.params.get("limit")))
self.params.update({"page": startpage})
if len(self.existing):
self.params.update({"from": sorted(self.existing)[-1]})
#startpage = max(1, floor(len(self.existing) / int(self.params.get("limit"))))
#startpage = 1
self.params.update({"page": 1})
try:
data = self.fetch()
tracks = self.extracttracks(data)

20
Pipfile
View file

@ -1,20 +0,0 @@
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
[packages]
requests = "*"
arrow = "*"
unicode-slugify = "*"
lxml = "*"
bleach = "*"
deviantart = "*"
flickr-api = "*"
pytumblr = "*"
pyyaml = "*"
[requires]
python_version = "3.7"

219
Pipfile.lock generated
View file

@ -1,219 +0,0 @@
{
"_meta": {
"hash": {
"sha256": "654f2f42d6d9e3dd3aaf13b371369e3943573472fc93786661eff68d965dcb8b"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.7"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"arrow": {
"hashes": [
"sha256:03404b624e89ac5e4fc19c52045fa0f3203419fd4dd64f6e8958c522580a574a",
"sha256:41be7ea4c53c2cf57bf30f2d614f60c411160133f7a0a8c49111c30fb7e725b5"
],
"index": "pypi",
"version": "==0.14.2"
},
"bleach": {
"hashes": [
"sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16",
"sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa"
],
"index": "pypi",
"version": "==3.1.0"
},
"certifi": {
"hashes": [
"sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939",
"sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695"
],
"version": "==2019.6.16"
},
"chardet": {
"hashes": [
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
],
"version": "==3.0.4"
},
"deviantart": {
"hashes": [
"sha256:6100cc73f162e8c945f8304109d72a8eb94c6df8348d7319b3c86ea1bcc511b6"
],
"index": "pypi",
"version": "==0.1.5"
},
"flickr-api": {
"hashes": [
"sha256:2ff036ce4ca6f9be71a90310be80916b44feaeb95df5c1a9e5f57d49b64032c9",
"sha256:b9782c06315946b395d7f1b1e051fa2ff6aab4b21c5e82b1d95c04d7295f5f24"
],
"index": "pypi",
"version": "==0.7.3"
},
"future": {
"hashes": [
"sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8"
],
"version": "==0.17.1"
},
"httplib2": {
"hashes": [
"sha256:158fbd0ffbba536829d664bf3f32c4f45df41f8f791663665162dfaf21ffd075",
"sha256:d1146939d270f1f1eb8cbf8f5aa72ff37d897faccca448582bb1e180aeb4c6b2"
],
"version": "==0.13.0"
},
"idna": {
"hashes": [
"sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
"sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
],
"version": "==2.8"
},
"lxml": {
"hashes": [
"sha256:06c7616601430aa140a69f97e3116308fffe0848f543b639a5ec2e8920ae72fd",
"sha256:177202792f9842374a8077735c69c41a4282183f7851443d2beb8ee310720819",
"sha256:19317ad721ceb9e39847d11131903931e2794e447d4751ebb0d9236f1b349ff2",
"sha256:36d206e62f3e5dbaafd4ec692b67157e271f5da7fd925fda8515da675eace50d",
"sha256:387115b066c797c85f9861a9613abf50046a15aac16759bc92d04f94acfad082",
"sha256:3ce1c49d4b4a7bc75fb12acb3a6247bb7a91fe420542e6d671ba9187d12a12c2",
"sha256:4d2a5a7d6b0dbb8c37dab66a8ce09a8761409c044017721c21718659fa3365a1",
"sha256:58d0a1b33364d1253a88d18df6c0b2676a1746d27c969dc9e32d143a3701dda5",
"sha256:62a651c618b846b88fdcae0533ec23f185bb322d6c1845733f3123e8980c1d1b",
"sha256:69ff21064e7debc9b1b1e2eee8c2d686d042d4257186d70b338206a80c5bc5ea",
"sha256:7060453eba9ba59d821625c6af6a266bd68277dce6577f754d1eb9116c094266",
"sha256:7d26b36a9c4bce53b9cfe42e67849ae3c5c23558bc08363e53ffd6d94f4ff4d2",
"sha256:83b427ad2bfa0b9705e02a83d8d607d2c2f01889eb138168e462a3a052c42368",
"sha256:923d03c84534078386cf50193057aae98fa94cace8ea7580b74754493fda73ad",
"sha256:b773715609649a1a180025213f67ffdeb5a4878c784293ada300ee95a1f3257b",
"sha256:baff149c174e9108d4a2fee192c496711be85534eab63adb122f93e70aa35431",
"sha256:bca9d118b1014b4c2d19319b10a3ebed508ff649396ce1855e1c96528d9b2fa9",
"sha256:ce580c28845581535dc6000fc7c35fdadf8bea7ccb57d6321b044508e9ba0685",
"sha256:d34923a569e70224d88e6682490e24c842907ba2c948c5fd26185413cbe0cd96",
"sha256:dd9f0e531a049d8b35ec5e6c68a37f1ba6ec3a591415e6804cbdf652793d15d7",
"sha256:ecb805cbfe9102f3fd3d2ef16dfe5ae9e2d7a7dfbba92f4ff1e16ac9784dbfb0",
"sha256:ede9aad2197a0202caff35d417b671f5f91a3631477441076082a17c94edd846",
"sha256:ef2d1fc370400e0aa755aab0b20cf4f1d0e934e7fd5244f3dd4869078e4942b9",
"sha256:f2fec194a49bfaef42a548ee657362af5c7a640da757f6f452a35da7dd9f923c"
],
"index": "pypi",
"version": "==4.3.4"
},
"oauth2": {
"hashes": [
"sha256:15b5c42301f46dd63113f1214b0d81a8b16254f65a86d3c32a1b52297f3266e6",
"sha256:c006a85e7c60107c7cc6da1b184b5c719f6dd7202098196dfa6e55df669b59bf"
],
"version": "==1.9.0.post1"
},
"oauthlib": {
"hashes": [
"sha256:40a63637707e9163eda62d0f5345120c65e001a790480b8256448543c1f78f66",
"sha256:b4d99ae8ccfb7d33ba9591b59355c64eef5241534aa3da2e4c0435346b84bc8e"
],
"version": "==3.0.2"
},
"python-dateutil": {
"hashes": [
"sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb",
"sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e"
],
"version": "==2.8.0"
},
"pytumblr": {
"hashes": [
"sha256:a3774d3978bcff2db98f36a2e5d17bb8496ac21157b1b518089adad86d0dca72",
"sha256:eaa4d98217df7ab6392fa5d8801f4a2bdcba35bf0fd49328aa3c98e3b231b6f2"
],
"index": "pypi",
"version": "==0.1.0"
},
"pyyaml": {
"hashes": [
"sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3",
"sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043",
"sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7",
"sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265",
"sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391",
"sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778",
"sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225",
"sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955",
"sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e",
"sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190",
"sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd"
],
"index": "pypi",
"version": "==5.1.1"
},
"requests": {
"hashes": [
"sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
"sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
],
"index": "pypi",
"version": "==2.22.0"
},
"requests-oauthlib": {
"hashes": [
"sha256:bd6533330e8748e94bf0b214775fed487d309b8b8fe823dc45641ebcd9a32f57",
"sha256:d3ed0c8f2e3bbc6b344fa63d6f933745ab394469da38db16bdddb461c7e25140",
"sha256:dd5a0499abfefd087c6dd96693cbd5bfd28aa009719a7f85ab3fabe3956ef19a"
],
"version": "==1.2.0"
},
"sanction": {
"hashes": [
"sha256:3e41b24e28590a0dfed68eddd10e44fa01feb81812ffb49085ca764e51aea9fe"
],
"version": "==0.4.1"
},
"six": {
"hashes": [
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
],
"version": "==1.12.0"
},
"unicode-slugify": {
"hashes": [
"sha256:34cf3afefa6480efe705a4fc0eaeeaf7f49754aec322ba3e8b2f27dc1cbcf650"
],
"index": "pypi",
"version": "==0.1.3"
},
"unidecode": {
"hashes": [
"sha256:1d7a042116536098d05d599ef2b8616759f02985c85b4fef50c78a5aaf10822a",
"sha256:2b6aab710c2a1647e928e36d69c21e76b453cd455f4e2621000e54b2a9b8cce8"
],
"version": "==1.1.1"
},
"urllib3": {
"hashes": [
"sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1",
"sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232"
],
"version": "==1.25.3"
},
"webencodings": {
"hashes": [
"sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78",
"sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"
],
"version": "==0.5.1"
}
},
"develop": {}
}

5
Tumblr.py
View file

@ -39,7 +39,8 @@ class TumblrFavs(common.Favs):
feeds.append(
{
"text": u.get("name"),
"xmlUrl": "%srss" % u.get("url"),
"xmlUrl": "https://cloud.petermolnar.net/rss-bridge/index.php?action=display&bridge=Tumblr&searchUsername=%s&format=Atom" % u.get("name"),
#"xmlUrl": "%srss" % u.get("url"),
"htmlUrl": u.get("url"),
}
)
@ -95,7 +96,7 @@ class TumblrFav(common.ImgFav):
if not len(title):
title = self.data.get("slug", "")
if not len(title):
title = common.slugfname(self.url)
title = common.url2slug(self.url)
return clean(title.strip())
@property

336
common.py
View file

@ -16,20 +16,36 @@ import settings
import keys
import yaml
from pprint import pprint
import feedparser
TMPFEXT = ".xyz"
MDFEXT = ".md"
TMPSUBDIR = "nasg"
SHM = "/dev/shm"
if os.path.isdir(SHM) and os.access(SHM, os.W_OK):
TMPDIR = f"{SHM}/{TMPSUBDIR}"
else:
TMPDIR = os.path.join(gettempdir(), TMPSUBDIR)
if not os.path.isdir(TMPDIR):
os.makedirs(TMPDIR)
def utfyamldump(data):
""" dump YAML with actual UTF-8 chars """
return yaml.dump(data, default_flow_style=False, indent=4, allow_unicode=True)
return yaml.dump(
data, default_flow_style=False, indent=4, allow_unicode=True
)
def slugfname(url):
return slugify(re.sub(r"^https?://(?:www)?", "", url), only_ascii=True, lower=True)[
:200
]
def url2slug(url):
return slugify(
re.sub(r"^https?://(?:www)?", "", url),
only_ascii=True,
lower=True,
)[:200]
class cached_property(object):
@ -51,7 +67,178 @@ class cached_property(object):
return result
class Follows(dict):
class Aperture(object):
def __init__(self):
self.session = requests.Session()
self.session.headers.update(
{
"Authorization": "Bearer %s"
% (keys.aperture["access_token"])
}
)
self.url = keys.aperture["url"]
@cached_property
def channels(self):
channels = self.session.get(f"{self.url}?action=channels")
if channels.status_code != requests.codes.ok:
logging.error(
"failed to get channels from aperture: ", channels.text
)
return None
try:
channels = channels.json()
except ValueError as e:
logging.error("failed to parse channels from aperture: ", e)
return None
if "channels" not in channels:
logging.error("no channels found in aperture: ")
return None
return channels["channels"]
def channelid(self, channelname):
for channel in self.channels:
if channel["name"].lower() == channelname.lower():
return channel["uid"]
return None
def feedmeta(self, url):
cfile = os.path.join(
TMPDIR,
"%s.%s.json" % (url2slug(url), self.__class__.__name__)
)
if os.path.exists(cfile):
with open(cfile, 'rt') as cache:
return json.loads(cache.read())
r = {
'title': url,
'feed': url,
'link': url,
'type': 'rss'
}
try:
feed = feedparser.parse(url)
if 'feed' in feed:
for maybe in ['title', 'link']:
if maybe in feed['feed']:
r[maybe] = feed['feed'][maybe]
except Exception as e:
logging.error("feedparser failed on %s: %s" %(url, e))
r['type']: 'hfeed'
pass
with open(cfile, 'wt') as cache:
cache.write(json.dumps(r))
return r
def channelfollows(self, channelid):
follows = self.session.get(
f"{self.url}?action=follow&channel={channelid}"
)
if follows.status_code != requests.codes.ok:
logging.error(
"failed to get follows from aperture: ", follows.text
)
return
try:
follows = follows.json()
except ValueError as e:
logging.error("failed to parse follows from aperture: ", e)
return
if "items" not in follows:
logging.error(
f"no follows found in aperture for channel {channelid}"
)
return
existing = {}
for follow in follows["items"]:
meta = self.feedmeta(follow["url"])
existing.update({follow["url"]: meta})
return existing
@cached_property
def follows(self):
follows = {}
for channel in self.channels:
follows[channel["name"]] = self.channelfollows(
channel["uid"]
)
return follows
def export(self):
opml = etree.Element("opml", version="1.0")
xmldoc = etree.ElementTree(opml)
opml.addprevious(
etree.ProcessingInstruction(
"xml-stylesheet",
'type="text/xsl" href="%s"'
% (settings.opml.get("xsl")),
)
)
head = etree.SubElement(opml, "head")
title = etree.SubElement(
head, "title"
).text = settings.opml.get("title")
dt = etree.SubElement(
head, "dateCreated"
).text = arrow.utcnow().format("ddd, DD MMM YYYY HH:mm:ss UTC")
owner = etree.SubElement(
head, "ownerName"
).text = settings.opml.get("owner")
email = etree.SubElement(
head, "ownerEmail"
).text = settings.opml.get("email")
body = etree.SubElement(opml, "body")
groups = {}
for group, feeds in self.follows.items():
if (
"private" in group.lower()
or "nsfw" in group.lower()
):
continue
if group not in groups.keys():
groups[group] = etree.SubElement(
body, "outline", text=group
)
for url, meta in feeds.items():
entry = etree.SubElement(
groups[group],
"outline",
type="rss",
text=meta['title'],
xmlUrl=meta['feed'],
htmlUrl=meta['link']
)
etree.tostring(
xmldoc,
encoding="utf-8",
xml_declaration=True,
pretty_print=True,
)
opmlfile = os.path.join(
settings.paths.get("content"), "following.opml"
)
with open(opmlfile, "wb") as f:
f.write(
etree.tostring(
xmldoc,
encoding="utf-8",
xml_declaration=True,
pretty_print=True,
)
)
class MinifluxFollows(dict):
def __init__(self):
self.auth = HTTPBasicAuth(
keys.miniflux.get("username"), keys.miniflux.get("token")
@ -60,9 +247,15 @@ class Follows(dict):
@property
def subscriptions(self):
feeds = []
params = {"jsonrpc": "2.0", "method": "getFeeds", "id": keys.miniflux.get("id")}
params = {
"jsonrpc": "2.0",
"method": "getFeeds",
"id": keys.miniflux.get("id"),
}
r = requests.post(
keys.miniflux.get("url"), data=json.dumps(params), auth=self.auth
keys.miniflux.get("url"),
data=json.dumps(params),
auth=self.auth,
)
return r.json().get("result", [])
@ -96,24 +289,31 @@ class Follows(dict):
opml.addprevious(
etree.ProcessingInstruction(
"xml-stylesheet",
'type="text/xsl" href="%s"' % (settings.opml.get("xsl")),
'type="text/xsl" href="%s"'
% (settings.opml.get("xsl")),
)
)
head = etree.SubElement(opml, "head")
title = etree.SubElement(head, "title").text = settings.opml.get("title")
dt = etree.SubElement(head, "dateCreated").text = arrow.utcnow().format(
"ddd, DD MMM YYYY HH:mm:ss UTC"
)
owner = etree.SubElement(head, "ownerName").text = settings.opml.get("owner")
email = etree.SubElement(head, "ownerEmail").text = settings.opml.get("email")
title = etree.SubElement(
head, "title"
).text = settings.opml.get("title")
dt = etree.SubElement(
head, "dateCreated"
).text = arrow.utcnow().format("ddd, DD MMM YYYY HH:mm:ss UTC")
owner = etree.SubElement(
head, "ownerName"
).text = settings.opml.get("owner")
email = etree.SubElement(
head, "ownerEmail"
).text = settings.opml.get("email")
body = etree.SubElement(opml, "body")
groups = {}
for feed in self.subscriptions:
# contains sensitive data, skip it
if "sessionid" in feed.get("feed_url") or "sessionid" in feed.get(
"site_url"
):
if "sessionid" in feed.get(
"feed_url"
) or "sessionid" in feed.get("site_url"):
continue
fgroup = feed.get("groups", None)
@ -136,12 +336,17 @@ class Follows(dict):
htmlUrl=feed.get("site_url"),
)
opmlfile = os.path.join(settings.paths.get("content"), "following.opml")
opmlfile = os.path.join(
settings.paths.get("content"), "following.opml"
)
with open(opmlfile, "wb") as f:
f.write(
etree.tostring(
xmldoc, encoding="utf-8", xml_declaration=True, pretty_print=True
xmldoc,
encoding="utf-8",
xml_declaration=True,
pretty_print=True,
)
)
@ -149,6 +354,11 @@ class Follows(dict):
class Favs(object):
def __init__(self, silo):
self.silo = silo
self.aperture_auth = {
"Authorization": "Bearer %s"
% (keys.aperture["access_token"])
}
self.aperture_chid = 0
@property
def feeds(self):
@ -156,7 +366,9 @@ class Favs(object):
@property
def since(self):
d = os.path.join(settings.paths.get("archive"), "favorite", "%s*" % self.silo)
d = os.path.join(
settings.paths.get("archive"), "favorite", "%s*" % self.silo
)
files = glob.glob(d)
if len(files):
mtime = max([int(os.path.getmtime(f)) for f in files])
@ -164,6 +376,81 @@ class Favs(object):
mtime = 0
return mtime
def sync_with_aperture(self):
channels = requests.get(
"%s?action=channels" % (keys.aperture["url"]),
headers=self.aperture_auth,
)
if channels.status_code != requests.codes.ok:
logging.error(
"failed to get channels from aperture: ", channels.text
)
return
try:
channels = channels.json()
except ValueError as e:
logging.error("failed to parse channels from aperture: ", e)
return
if "channels" not in channels:
logging.error("no channels found in aperture: ")
return
for channel in channels["channels"]:
if channel["name"].lower() == self.silo.lower():
self.aperture_chid = channel["uid"]
break
if not self.aperture_chid:
logging.error("no channels found for silo ", self.silo)
return
follows = requests.get(
"%s?action=follow&channel=%s"
% (keys.aperture["url"], self.aperture_chid),
headers=self.aperture_auth,
)
if follows.status_code != requests.codes.ok:
logging.error(
"failed to get follows from aperture: ", follows.text
)
return
try:
follows = follows.json()
except ValueError as e:
logging.error("failed to parse follows from aperture: ", e)
return
if "items" not in follows:
logging.error(
"no follows found in aperture for channel %s (%s)"
% (self.silo, self.aperture_chid)
)
return
existing = []
for follow in follows["items"]:
existing.append(follow["url"])
existing = list(set(existing))
for feed in self.feeds:
if feed["xmlUrl"] not in existing:
subscribe_to = {
"action": "follow",
"channel": self.aperture_chid,
"url": feed["xmlUrl"],
}
logging.info(
"subscribing to %s into %s (%s)"
% (feed, self.silo, self.aperture_chid)
)
subscribe = requests.post(
keys.aperture["url"],
headers=self.aperture_auth,
data=subscribe_to,
)
logging.debug(subscribe.text)
class ImgFav(object):
def __init__(self):
@ -182,8 +469,11 @@ class ImgFav(object):
return False
def save_txt(self):
attachments = [os.path.basename(fn) for fn in glob.glob("%s*" % self.targetprefix)
if not os.path.basename(fn).endswith('.md')]
attachments = [
os.path.basename(fn)
for fn in glob.glob("%s*" % self.targetprefix)
if not os.path.basename(fn).endswith(".md")
]
meta = {
"title": self.title,
"favorite-of": self.url,

13
run.py
View file

@ -4,24 +4,27 @@ import Tumblr
import LastFM
import DeviantArt
import Flickr
import Artstation
#import Artstation
from pprint import pprint
lfm = LastFM.LastFM()
lfm.run()
opml = common.Follows()
#opml = common.Follows()
silos = [
DeviantArt.DAFavs(),
Flickr.FlickrFavs(),
Tumblr.TumblrFavs(),
Artstation.ASFavs(),
# Artstation.ASFavs(),
]
for silo in silos:
silo.run()
opml.update({silo.silo: silo.feeds})
#silo.sync_with_aperture()
#opml.update({silo.silo: silo.feeds})
opml.sync()
#opml.sync()
#opml.export()
opml = common.Aperture()
opml.export()

14
settings.py
View file

@ -3,19 +3,25 @@ import re
import argparse
import logging
class nameddict(dict):
__getattr__ = dict.get
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__
base = os.path.abspath(os.path.expanduser("~/Projects/petermolnar.net"))
opml = {
opml = nameddict({
"owner": "Peter Molnar",
"email": "mail@petermolnar.net",
"title": "feeds followed by petermolnar.net",
"xsl": "https://petermolnar.net/following.xsl",
}
})
paths = {
paths = nameddict({
"archive": os.path.join(base, "archive"),
"content": os.path.join(base, "content"),
}
"bookmarks": os.path.join(base, "archive", "bookmarks")
})
loglevels = {"critical": 50, "error": 40, "warning": 30, "info": 20, "debug": 10}