diff --git a/Artstation.py b/Artstation.py new file mode 100644 index 0000000..06bda0d --- /dev/null +++ b/Artstation.py @@ -0,0 +1,184 @@ +import os +import glob +import json +import logging +import arrow +import requests +import keys +import common +import settings +from math import ceil +from pprint import pprint + +class ASFavs(common.Favs): + def __init__(self): + super().__init__('artstation') + self.user = keys.artstation.get('username') + + def paged_likes(self, page=1): + url = "https://www.artstation.com/users/%s/likes.json?page=%s" % ( + self.user, + page + ) + js = requests.get(url) + try: + js = js.json() + if 'data' not in js: + return None + return js + except Exception as e: + logging.error('fetching artstation failed: %s', e) + return None + + @property + def likes(self): + js = self.paged_likes() + if not js: + return [] + likes = js.get('data', []) + pages = ceil(js.get('total_count', 1) / 50) + while pages > 1: + extras = self.paged_likes() + if not extras: + continue + likes = likes + extras.get('data', []) + pages = pages - 1 + return likes + + @property + def feeds(self): + feeds = [] + js = requests.get( + "https://www.artstation.com/users/%s/following.json" % self.user + ) + try: + js = js.json() + if 'data' not in js: + logging.error('fetching artstation follows failed: missing data') + return feeds + for f in js.get('data'): + feeds.append({ + 'text': f.get('username'), + 'xmlUrl': "https://www.artstation.com/%s.rss" % f.get('subdomain'), + 'htmlUrl': "https://www.artstation.com/%s" % f.get('subdomain'), + }) + except Exception as e: + logging.error('parsing artstation follows failed: %s', e) + return feeds + + def run(self): + for like in self.likes: + like = ASLike(like) + like.run() + + +class ASLike(common.ImgFav): + def __init__(self, like ): + self.like = like + + def __str__(self): + return "like-of %s" % (self.url) + + @property + def url(self): + return self.like.get('permalink') + + @property + def data(self): + purl = "%s.json" % (self.url.replace('artwork', 'projects')) + data = requests.get(purl) + try: + data = data.json() + except Exception as e: + logging.error( + 'fetching artstation project %s failed: %s', + self.url, + e + ) + return None + return data + + @property + def author(self): + return { + 'name': self.like.get('user').get('username'), + 'url': self.like.get('user').get('permalink'), + } + + @property + def id(self): + return self.like.get('id') + + @property + def content(self): + return '%s' % self.data.get('description_html', '') + + @property + def title(self): + title = self.like.get('title') + if not len(title): + title = self.like.get('slug') + if not len(title): + title = common.slugfname(self.url) + return title + + @property + def slug(self): + maybe = self.like.get('slug') + if not len(maybe): + maybe = common.slugfname(self.url) + return maybe + + @property + def targetprefix(self): + return os.path.join( + settings.paths.get('archive'), + 'favorite', + "artstation_%s_%s_%s" % ( + common.slugfname('%s' % self.like.get('user').get('username')), + self.like.get('hash_id'), + self.slug + ) + ) + + @property + def exists(self): + maybe = glob.glob("%s*" % self.targetprefix) + if len(maybe): + return True + return False + + @property + def published(self): + return arrow.get(self.like.get('published_at')) + + @property + def tags(self): + t = [] + for c in self.data.get('categories'): + t.append(c.get('name')) + return t + + @property + def images(self): + r = {} + cntr = 0 + for img in self.data.get('assets'): + if img.get('asset_type') != 'image': + logging.debug('skipping asset: %s' % img) + continue + + f = "%s_%d%s" % (self.targetprefix, cntr, common.TMPFEXT) + r.update({ + f: img.get('image_url') + }) + cntr = cntr + 1 + return r + + def run(self): + if not self.exists: + self.fetch_images() + +if __name__ == '__main__': + t = ASFavs() + t.run() diff --git a/Pipfile b/Pipfile index 51fafd7..5b2f9d1 100644 --- a/Pipfile +++ b/Pipfile @@ -6,11 +6,14 @@ verify_ssl = true [dev-packages] [packages] -deviantart = "==0.1.5" -arrow = "==0.12.1" -requests = "==2.19.1" -flickr_api = "==0.6.1" -PyTumblr = "==0.0.8" +requests = "*" +arrow = "*" +unicode-slugify = "*" +lxml = "*" +bleach = "*" +deviantart = "*" +flickr-api = "*" +pytumblr = "*" [requires] python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock index f216834..483d9e9 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "df68630bf7b4a7d867f577a19e75d45abb741270e6c8155a99ff2a39aa80f755" + "sha256": "8b6f21b4f6848f8c116032411e7508ea0392c799dcbaa14ddfbf68ee24e2cc59" }, "pipfile-spec": 6, "requires": { @@ -23,6 +23,14 @@ "index": "pypi", "version": "==0.12.1" }, + "bleach": { + "hashes": [ + "sha256:48d39675b80a75f6d1c3bdbffec791cf0bbbab665cf01e20da701c77de278718", + "sha256:73d26f018af5d5adcdabf5c1c974add4361a9c76af215fe32fdec8a6fc5fb9b9" + ], + "index": "pypi", + "version": "==3.0.2" + }, "certifi": { "hashes": [ "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", @@ -65,10 +73,46 @@ }, "idna": { "hashes": [ - "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", - "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" ], - "version": "==2.7" + "version": "==2.8" + }, + "lxml": { + "hashes": [ + "sha256:02bc220d61f46e9b9d5a53c361ef95e9f5e1d27171cd461dddb17677ae2289a5", + "sha256:22f253b542a342755f6cfc047fe4d3a296515cf9b542bc6e261af45a80b8caf6", + "sha256:2f31145c7ff665b330919bfa44aacd3a0211a76ca7e7b441039d2a0b0451e415", + "sha256:36720698c29e7a9626a0dc802ef8885f8f0239bfd1689628ecd459a061f2807f", + "sha256:438a1b0203545521f6616132bfe0f4bca86f8a401364008b30e2b26ec408ce85", + "sha256:4815892904c336bbaf73dafd54f45f69f4021c22b5bad7332176bbf4fb830568", + "sha256:5be031b0f15ad63910d8e5038b489d95a79929513b3634ad4babf77100602588", + "sha256:5c93ae37c3c588e829b037fdfbd64a6e40c901d3f93f7beed6d724c44829a3ad", + "sha256:60842230678674cdac4a1cf0f707ef12d75b9a4fc4a565add4f710b5fcf185d5", + "sha256:62939a8bb6758d1bf923aa1c13f0bcfa9bf5b2fc0f5fa917a6e25db5fe0cfa4e", + "sha256:75830c06a62fe7b8fe3bbb5f269f0b308f19f3949ac81cfd40062f47c1455faf", + "sha256:81992565b74332c7c1aff6a913a3e906771aa81c9d0c68c68113cffcae45bc53", + "sha256:8c892fb0ee52c594d9a7751c7d7356056a9682674b92cc1c4dc968ff0f30c52f", + "sha256:9d862e3cf4fc1f2837dedce9c42269c8c76d027e49820a548ac89fdcee1e361f", + "sha256:a623965c086a6e91bb703d4da62dabe59fe88888e82c4117d544e11fd74835d6", + "sha256:a7783ab7f6a508b0510490cef9f857b763d796ba7476d9703f89722928d1e113", + "sha256:aab09fbe8abfa3b9ce62aaf45aca2d28726b1b9ee44871dbe644050a2fff4940", + "sha256:abf181934ac3ef193832fb973fd7f6149b5c531903c2ec0f1220941d73eee601", + "sha256:ae07fa0c115733fce1e9da96a3ac3fa24801742ca17e917e0c79d63a01eeb843", + "sha256:b9c78242219f674ab645ec571c9a95d70f381319a23911941cd2358a8e0521cf", + "sha256:bccb267678b870d9782c3b44d0cefe3ba0e329f9af8c946d32bf3778e7a4f271", + "sha256:c4df4d27f4c93b2cef74579f00b1d3a31a929c7d8023f870c4b476f03a274db4", + "sha256:caf0e50b546bb60dfa99bb18dfa6748458a83131ecdceaf5c071d74907e7e78a", + "sha256:d3266bd3ac59ac4edcd5fa75165dee80b94a3e5c91049df5f7c057ccf097551c", + "sha256:db0d213987bcd4e6d41710fb4532b22315b0d8fb439ff901782234456556aed1", + "sha256:dbbd5cf7690a40a9f0a9325ab480d0fccf46d16b378eefc08e195d84299bfae1", + "sha256:e16e07a0ec3a75b5ee61f2b1003c35696738f937dc8148fbda9fe2147ccb6e61", + "sha256:e175a006725c7faadbe69e791877d09936c0ef2cf49d01b60a6c1efcb0e8be6f", + "sha256:edd9c13a97f6550f9da2236126bb51c092b3b1ce6187f2bd966533ad794bbb5e", + "sha256:fa39ea60d527fbdd94215b5e5552f1c6a912624521093f1384a491a8ad89ad8b" + ], + "index": "pypi", + "version": "==4.2.5" }, "oauth2": { "hashes": [ @@ -101,16 +145,17 @@ }, "requests": { "hashes": [ - "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", - "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" ], "index": "pypi", - "version": "==2.19.1" + "version": "==2.21.0" }, "requests-oauthlib": { "hashes": [ "sha256:8886bfec5ad7afb391ed5443b1f697c6f4ae98d0e5620839d8b4499c032ada3f", - "sha256:e21232e2465808c0e892e0e4dbb8c2faafec16ac6dc067dd546e9b466f3deac8" + "sha256:e21232e2465808c0e892e0e4dbb8c2faafec16ac6dc067dd546e9b466f3deac8", + "sha256:fe3282f48fb134ee0035712159f5429215459407f6d5484013343031ff1a400d" ], "version": "==1.0.0" }, @@ -122,17 +167,38 @@ }, "six": { "hashes": [ - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], - "version": "==1.11.0" + "version": "==1.12.0" + }, + "unicode-slugify": { + "hashes": [ + "sha256:34cf3afefa6480efe705a4fc0eaeeaf7f49754aec322ba3e8b2f27dc1cbcf650" + ], + "index": "pypi", + "version": "==0.1.3" + }, + "unidecode": { + "hashes": [ + "sha256:092cdf7ad9d1052c50313426a625b717dab52f7ac58f859e09ea020953b1ad8f", + "sha256:8b85354be8fd0c0e10adbf0675f6dc2310e56fda43fa8fe049123b6c475e52fb" + ], + "version": "==1.0.23" }, "urllib3": { "hashes": [ - "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", - "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" + "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", + "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" ], - "version": "==1.23" + "version": "==1.24.1" + }, + "webencodings": { + "hashes": [ + "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", + "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" + ], + "version": "==0.5.1" } }, "develop": {} diff --git a/common.py b/common.py index 5c4dbc6..e1622e4 100644 --- a/common.py +++ b/common.py @@ -6,6 +6,7 @@ import logging import shutil import subprocess import json +from io import BytesIO import lxml.etree as etree from slugify import slugify import requests @@ -41,16 +42,15 @@ class cached_property(object): setattr(inst, self.name, result) return result -class Follows(object): +class Follows(dict): def __init__(self): - self.feeds = {} self.auth = HTTPBasicAuth( keys.miniflux.get('username'), keys.miniflux.get('token') ) - @cached_property - def active_subscriptions(self): + @property + def subscriptions(self): feeds = [] params = { 'jsonrpc': '2.0', @@ -60,26 +60,29 @@ class Follows(object): r = requests.post( keys.miniflux.get('url'), data=json.dumps(params), - auth=self.auth, + auth=self.auth ) - for feed in r.json().get('result', []): + return r.json().get('result', []) + + + def sync(self): + current = [] + for feed in self.subscriptions: try: - feeds.append(feed['feed_url']) + current.append(feed['feed_url']) except Exception as e: logging.error('problem with feed entry: %s', feed) - return feeds - - def syncminiflux(self): - for silo, feeds in self.feeds.items(): - for f in feeds: - feed = f.get('xmlUrl') - if feed not in self.active_subscriptions: + for silo, feeds in self.items(): + for feed in feeds: + xmlurl = feed.get('xmlUrl') + if len(xmlurl) and xmlurl not in current: + logging.info('creating subscription for: %s', feed) params = { 'jsonrpc': '2.0', 'method': 'createFeed', 'id': keys.miniflux.get('id'), 'params': { - 'url': feed, + 'url': xmlurl, 'group_name': silo } } @@ -89,35 +92,68 @@ class Follows(object): auth=self.auth, ) - def append(self, silo, feeds): - self.feeds.update({silo: feeds}) - def export(self): - opml = etree.Element("opml") - + opml = etree.Element("opml", version="1.0") + xmldoc = etree.ElementTree(opml) + opml.addprevious( + etree.ProcessingInstruction( + "xml-stylesheet", + 'type="text/xsl" href="%s"' % (settings.opml.get('xsl')) + ) + ) head = etree.SubElement(opml, "head") - title = etree.SubElement(head, "title").text = "Social media RSS feeds" + title = etree.SubElement(head, "title").text = settings.opml.get('title') + dt = etree.SubElement(head, "dateCreated").text = arrow.utcnow().format('ddd, DD MMM YYYY HH:mm:ss UTC') + owner = etree.SubElement(head, "ownerName").text = settings.opml.get('owner') + email = etree.SubElement(head, "ownerEmail").text = settings.opml.get('email') body = etree.SubElement(opml, "body") - for silo, feeds in self.feeds.items(): - s = etree.SubElement(body, "outline", text=silo) - for f in feeds: - entry = etree.SubElement( - s, + groups = {} + for feed in self.subscriptions: + # contains sensitive data, skip it + if 'sessionid' in feed.get('feed_url') or 'sessionid' in feed.get('site_url'): + continue + + fgroup = feed.get('groups',None) + if not fgroup: + fgroup = [{ + 'title': 'Unknown', + 'id': -1 + }] + fgroup = fgroup.pop() + # some groups need to be skipped + if fgroup['title'].lower() in ['nsfw', '_self']: + continue + if fgroup['title'] not in groups.keys(): + groups[fgroup['title']] = etree.SubElement( + body, "outline", - type="rss", - text=f.get('text'), - xmlUrl=f.get('xmlUrl'), - htmlUrl=f.get('htmlUrl') - ) + text=fgroup['title'] + ) + entry = etree.SubElement( + groups[fgroup['title']], + "outline", + type="rss", + text=feed.get('title'), + xmlUrl=feed.get('feed_url'), + htmlUrl=feed.get('site_url') + ) opmlfile = os.path.join( - settings.paths.get('archive'), - 'feeds.opml' + settings.paths.get('content'), + 'following.opml' ) with open(opmlfile, 'wb') as f: - f.write(etree.tostring(opml, pretty_print=True)) + f.write( + etree.tostring( + xmldoc, + encoding='utf-8', + xml_declaration=True, + pretty_print=True + ) + ) + class Favs(object): def __init__(self, silo): diff --git a/run.py b/run.py index e612d4b..c66b3b8 100644 --- a/run.py +++ b/run.py @@ -4,6 +4,7 @@ import Tumblr import LastFM import DeviantArt import Flickr +import Artstation from pprint import pprint lfm = LastFM.LastFM() @@ -14,12 +15,14 @@ opml = common.Follows() silos = [ DeviantArt.DAFavs(), Flickr.FlickrFavs(), - Tumblr.TumblrFavs() + Tumblr.TumblrFavs(), + Artstation.ASFavs() ] for silo in silos: silo.run() - opml.append(silo.silo, silo.feeds) + opml.update({silo.silo: silo.feeds}) -opml.syncminiflux() + +opml.sync() opml.export() diff --git a/settings.py b/settings.py index 9dcaa9c..ad8b243 100644 --- a/settings.py +++ b/settings.py @@ -5,8 +5,16 @@ import logging base = os.path.abspath(os.path.expanduser('~/Projects/petermolnar.net')) +opml = { + 'owner': 'Peter Molnar', + 'email': 'mail@petermolnar.net', + 'title': 'feeds followed by petermolnar.net', + 'xsl': 'https://petermolnar.net/following.xsl' +} + paths = { 'archive': os.path.join(base, 'archive'), + 'content': os.path.join(base, 'content'), } loglevels = { @@ -20,7 +28,7 @@ loglevels = { _parser = argparse.ArgumentParser(description='Parameters for silo.pasta') _parser.add_argument( '--loglevel', - default='debug', + default='info', help='change loglevel' )