From d5665d15e71aaa871e97f58c5d47b2fd2a4cb018 Mon Sep 17 00:00:00 2001
From: Peter Molnar <hello@petermolnar.eu>
Date: Wed, 6 May 2020 13:27:49 +0100
Subject: [PATCH] I lost track of changes, this is a commit that has to be
 followed by a large cleanup

---
 .gitignore    |   7 +-
 Artstation.py |  59 +++++----
 DeviantArt.py |   8 +-
 Flickr.py     |  11 +-
 LastFM.py     |  51 +++-----
 Pipfile       |  20 ---
 Pipfile.lock  | 219 --------------------------------
 Tumblr.py     |   5 +-
 common.py     | 336 ++++++++++++++++++++++++++++++++++++++++++++++----
 run.py        |  13 +-
 settings.py   |  14 ++-
 11 files changed, 401 insertions(+), 342 deletions(-)
 delete mode 100644 Pipfile
 delete mode 100644 Pipfile.lock

diff --git a/.gitignore b/.gitignore
index 8714296..cdc6c4d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,6 @@
 keys.py
-.venv
-__pycache
-keys.py
-__pycache__
 lib
+__pycache
+__pycache__
 _scratch
+.venv
diff --git a/Artstation.py b/Artstation.py
index 5846e06..49e21e2 100644
--- a/Artstation.py
+++ b/Artstation.py
@@ -7,34 +7,51 @@ import requests
 import keys
 import common
 import settings
+from time import sleep
 from math import ceil
+import random
 from pprint import pprint
 
 
 class ASFavs(common.Favs):
-    headers = {
-        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0",
-        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-        "Accept-Language": "en-US,en;q=0.5",
-        "Accept-Encoding": "gzip, deflate, br",
-        "DNT": "1",
-        "Connection": "keep-alive",
-        "Upgrade-Insecure-Requests": "1",
-        "Pragma": "no-cache",
-        "Cache-Control": "no-cache",
-    }
-
     def __init__(self):
         super().__init__("artstation")
         self.user = keys.artstation.get("username")
         self.session = requests.Session()
+        self.session.headers.update({
+            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+            #"DNT": "1",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+            "Pragma": "no-cache",
+            "Cache-Control": "max-age=0, no-cache",
+        })
+
+
+session.headers.update({
+    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+    "DNT": "1",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+    "Pragma": "no-cache",
+    "Cache-Control": "max-age=0, no-cache",
+})
 
     def paged_likes(self, page=1):
         url = "https://www.artstation.com/users/%s/likes.json?page=%s" % (
             self.user,
             page,
         )
-        js = self.session.get(url, headers=self.headers)
+        js = self.session.get(url)
+        while js.status_code != requests.codes.ok:
+            # FU cloudflare
+            pprint(self.session.cookies)
+            sleep(round(random.uniform(0.7,3.5), 2))
+            js = self.session.get(url)
         try:
             js = js.json()
             if "data" not in js:
@@ -46,18 +63,13 @@ class ASFavs(common.Favs):
 
     @property
     def likes(self):
-        # init Session and it's cookies before doing anything
-        # FU cloudflare, I'm trying to access my own likes and followings!
-        url = "https://www.artstation.com/"
-        self.session.get(url, headers=self.headers)
-        # now do the real work
-        js = self.paged_likes()
+        js = self.paged_likes(1)
         if not js:
             return []
         likes = js.get("data", [])
         pages = ceil(js.get("total_count", 1) / 50)
         while pages > 1:
-            extras = self.paged_likes()
+            extras = self.paged_likes(pages)
             if not extras:
                 continue
             likes = likes + extras.get("data", [])
@@ -88,6 +100,7 @@ class ASFavs(common.Favs):
         return feeds
 
     def run(self):
+        # FU cloudflare
         for like in self.likes:
             like = ASLike(like, self.session, self.headers)
             like.run()
@@ -138,14 +151,14 @@ class ASLike(common.ImgFav):
         if not len(title):
             title = self.like.get("slug")
         if not len(title):
-            title = common.slugfname(self.url)
+            title = common.url2slug(self.url)
         return title
 
     @property
     def slug(self):
         maybe = self.like.get("slug")
         if not len(maybe):
-            maybe = common.slugfname(self.url)
+            maybe = common.url2slug(self.url)
         return maybe
 
     @property
@@ -155,7 +168,7 @@ class ASLike(common.ImgFav):
             "favorite",
             "artstation_%s_%s_%s"
             % (
-                common.slugfname("%s" % self.like.get("user").get("username")),
+                common.url2slug("%s" % self.like.get("user").get("username")),
                 self.like.get("hash_id"),
                 self.slug,
             ),
diff --git a/DeviantArt.py b/DeviantArt.py
index da962ae..a70d4ea 100644
--- a/DeviantArt.py
+++ b/DeviantArt.py
@@ -122,7 +122,7 @@ class DAFav(common.ImgFav):
     def title(self):
         title = self.deviation.title
         if not len(title):
-            title = common.slugfname(self.url)
+            title = common.url2slug(self.url)
         return clean(title.strip())
 
     @property
@@ -132,15 +132,15 @@ class DAFav(common.ImgFav):
             "favorite",
             "deviantart_%s_%s_%s"
             % (
-                common.slugfname("%s" % self.deviation.author),
+                common.url2slug("%s" % self.deviation.author),
                 self.id,
-                common.slugfname("%s" % self.title),
+                common.url2slug("%s" % self.title),
             ),
         )
 
     @property
     def published(self):
-        return arrow.get(self.deviation.published_time)
+        return arrow.get(int(self.deviation.published_time))
 
     @property
     def tags(self):
diff --git a/Flickr.py b/Flickr.py
index a7ce08a..663bcaf 100644
--- a/Flickr.py
+++ b/Flickr.py
@@ -10,9 +10,6 @@ import settings
 from pprint import pprint
 import logging
 
-# class FlickrFollows(common.Follows):
-
-
 class FlickrFavs(common.Favs):
     def __init__(self):
         super().__init__("flickr")
@@ -109,12 +106,16 @@ class FlickrFav(common.ImgFav):
         return os.path.join(
             settings.paths.get("archive"),
             "favorite",
-            "flickr_%s_%s" % (common.slugfname("%s" % self.owner.id), self.id),
+            "flickr_%s_%s" % (common.url2slug("%s" % self.owner.id), self.id),
         )
 
     @property
     def published(self):
-        return arrow.get(self.info.get("dateuploaded"))
+        x = self.info.get("dateuploaded")
+        if x.isnumeric():
+            return arrow.get(int(x))
+        else:
+            return arrow.get(x)
 
     @property
     def tags(self):
diff --git a/LastFM.py b/LastFM.py
index b3a21c9..f8c37a2 100644
--- a/LastFM.py
+++ b/LastFM.py
@@ -10,31 +10,12 @@ import settings
 import keys
 from pprint import pprint
 from math import floor
+from common import cached_property
 
 Track = namedtuple(
     "Track", ["timestamp", "artist", "album", "title", "artistid", "albumid", "img"]
 )
 
-
-class cached_property(object):
-    """ extermely simple cached_property decorator:
-    whenever something is called as @cached_property, on first run, the
-    result is calculated, then the class method is overwritten to be
-    a property, contaning the result from the method
-    """
-
-    def __init__(self, method, name=None):
-        self.method = method
-        self.name = name or method.__name__
-
-    def __get__(self, inst, cls):
-        if inst is None:
-            return self
-        result = self.method(inst)
-        setattr(inst, self.name, result)
-        return result
-
-
 class LastFM(object):
     url = "http://ws.audioscrobbler.com/2.0/"
 
@@ -46,9 +27,9 @@ class LastFM(object):
             "format": "json",
             "limit": "200",
         }
-        if os.path.isfile(self.target):
-            mtime = os.path.getmtime(self.target)
-            self.params.update({"from": mtime})
+        # if os.path.isfile(self.target):
+            # mtime = os.path.getmtime(self.target)
+            # self.params.update({"from": mtime})
 
     @property
     def target(self):
@@ -57,14 +38,15 @@ class LastFM(object):
     @cached_property
     def existing(self):
         timestamps = []
-        with open(self.target, "r") as f:
-            r = csv.reader(f)
-            for row in r:
-                try:
-                    timestamps.append(arrow.get(row[0]).timestamp)
-                except Exception as e:
-                    logging.error("arrow failed on row %s", row)
-                    continue
+        if os.path.isfile(self.target):
+            with open(self.target, "r") as f:
+                r = csv.reader(f)
+                for row in r:
+                    try:
+                        timestamps.append(arrow.get(row[0]).timestamp)
+                    except Exception as e:
+                        logging.error("arrow failed on row %s", row)
+                        continue
         return timestamps
 
     @property
@@ -98,8 +80,11 @@ class LastFM(object):
         return json.loads(r.text).get("recenttracks")
 
     def run(self):
-        startpage = floor(len(self.existing) / int(self.params.get("limit")))
-        self.params.update({"page": startpage})
+        if len(self.existing):
+            self.params.update({"from": sorted(self.existing)[-1]})
+        #startpage = max(1, floor(len(self.existing) / int(self.params.get("limit"))))
+        #startpage = 1
+        self.params.update({"page": 1})
         try:
             data = self.fetch()
             tracks = self.extracttracks(data)
diff --git a/Pipfile b/Pipfile
deleted file mode 100644
index 24321c9..0000000
--- a/Pipfile
+++ /dev/null
@@ -1,20 +0,0 @@
-[[source]]
-name = "pypi"
-url = "https://pypi.org/simple"
-verify_ssl = true
-
-[dev-packages]
-
-[packages]
-requests = "*"
-arrow = "*"
-unicode-slugify = "*"
-lxml = "*"
-bleach = "*"
-deviantart = "*"
-flickr-api = "*"
-pytumblr = "*"
-pyyaml = "*"
-
-[requires]
-python_version = "3.7"
diff --git a/Pipfile.lock b/Pipfile.lock
deleted file mode 100644
index 748df1c..0000000
--- a/Pipfile.lock
+++ /dev/null
@@ -1,219 +0,0 @@
-{
-    "_meta": {
-        "hash": {
-            "sha256": "654f2f42d6d9e3dd3aaf13b371369e3943573472fc93786661eff68d965dcb8b"
-        },
-        "pipfile-spec": 6,
-        "requires": {
-            "python_version": "3.7"
-        },
-        "sources": [
-            {
-                "name": "pypi",
-                "url": "https://pypi.org/simple",
-                "verify_ssl": true
-            }
-        ]
-    },
-    "default": {
-        "arrow": {
-            "hashes": [
-                "sha256:03404b624e89ac5e4fc19c52045fa0f3203419fd4dd64f6e8958c522580a574a",
-                "sha256:41be7ea4c53c2cf57bf30f2d614f60c411160133f7a0a8c49111c30fb7e725b5"
-            ],
-            "index": "pypi",
-            "version": "==0.14.2"
-        },
-        "bleach": {
-            "hashes": [
-                "sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16",
-                "sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa"
-            ],
-            "index": "pypi",
-            "version": "==3.1.0"
-        },
-        "certifi": {
-            "hashes": [
-                "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939",
-                "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695"
-            ],
-            "version": "==2019.6.16"
-        },
-        "chardet": {
-            "hashes": [
-                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
-                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
-            ],
-            "version": "==3.0.4"
-        },
-        "deviantart": {
-            "hashes": [
-                "sha256:6100cc73f162e8c945f8304109d72a8eb94c6df8348d7319b3c86ea1bcc511b6"
-            ],
-            "index": "pypi",
-            "version": "==0.1.5"
-        },
-        "flickr-api": {
-            "hashes": [
-                "sha256:2ff036ce4ca6f9be71a90310be80916b44feaeb95df5c1a9e5f57d49b64032c9",
-                "sha256:b9782c06315946b395d7f1b1e051fa2ff6aab4b21c5e82b1d95c04d7295f5f24"
-            ],
-            "index": "pypi",
-            "version": "==0.7.3"
-        },
-        "future": {
-            "hashes": [
-                "sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8"
-            ],
-            "version": "==0.17.1"
-        },
-        "httplib2": {
-            "hashes": [
-                "sha256:158fbd0ffbba536829d664bf3f32c4f45df41f8f791663665162dfaf21ffd075",
-                "sha256:d1146939d270f1f1eb8cbf8f5aa72ff37d897faccca448582bb1e180aeb4c6b2"
-            ],
-            "version": "==0.13.0"
-        },
-        "idna": {
-            "hashes": [
-                "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
-                "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
-            ],
-            "version": "==2.8"
-        },
-        "lxml": {
-            "hashes": [
-                "sha256:06c7616601430aa140a69f97e3116308fffe0848f543b639a5ec2e8920ae72fd",
-                "sha256:177202792f9842374a8077735c69c41a4282183f7851443d2beb8ee310720819",
-                "sha256:19317ad721ceb9e39847d11131903931e2794e447d4751ebb0d9236f1b349ff2",
-                "sha256:36d206e62f3e5dbaafd4ec692b67157e271f5da7fd925fda8515da675eace50d",
-                "sha256:387115b066c797c85f9861a9613abf50046a15aac16759bc92d04f94acfad082",
-                "sha256:3ce1c49d4b4a7bc75fb12acb3a6247bb7a91fe420542e6d671ba9187d12a12c2",
-                "sha256:4d2a5a7d6b0dbb8c37dab66a8ce09a8761409c044017721c21718659fa3365a1",
-                "sha256:58d0a1b33364d1253a88d18df6c0b2676a1746d27c969dc9e32d143a3701dda5",
-                "sha256:62a651c618b846b88fdcae0533ec23f185bb322d6c1845733f3123e8980c1d1b",
-                "sha256:69ff21064e7debc9b1b1e2eee8c2d686d042d4257186d70b338206a80c5bc5ea",
-                "sha256:7060453eba9ba59d821625c6af6a266bd68277dce6577f754d1eb9116c094266",
-                "sha256:7d26b36a9c4bce53b9cfe42e67849ae3c5c23558bc08363e53ffd6d94f4ff4d2",
-                "sha256:83b427ad2bfa0b9705e02a83d8d607d2c2f01889eb138168e462a3a052c42368",
-                "sha256:923d03c84534078386cf50193057aae98fa94cace8ea7580b74754493fda73ad",
-                "sha256:b773715609649a1a180025213f67ffdeb5a4878c784293ada300ee95a1f3257b",
-                "sha256:baff149c174e9108d4a2fee192c496711be85534eab63adb122f93e70aa35431",
-                "sha256:bca9d118b1014b4c2d19319b10a3ebed508ff649396ce1855e1c96528d9b2fa9",
-                "sha256:ce580c28845581535dc6000fc7c35fdadf8bea7ccb57d6321b044508e9ba0685",
-                "sha256:d34923a569e70224d88e6682490e24c842907ba2c948c5fd26185413cbe0cd96",
-                "sha256:dd9f0e531a049d8b35ec5e6c68a37f1ba6ec3a591415e6804cbdf652793d15d7",
-                "sha256:ecb805cbfe9102f3fd3d2ef16dfe5ae9e2d7a7dfbba92f4ff1e16ac9784dbfb0",
-                "sha256:ede9aad2197a0202caff35d417b671f5f91a3631477441076082a17c94edd846",
-                "sha256:ef2d1fc370400e0aa755aab0b20cf4f1d0e934e7fd5244f3dd4869078e4942b9",
-                "sha256:f2fec194a49bfaef42a548ee657362af5c7a640da757f6f452a35da7dd9f923c"
-            ],
-            "index": "pypi",
-            "version": "==4.3.4"
-        },
-        "oauth2": {
-            "hashes": [
-                "sha256:15b5c42301f46dd63113f1214b0d81a8b16254f65a86d3c32a1b52297f3266e6",
-                "sha256:c006a85e7c60107c7cc6da1b184b5c719f6dd7202098196dfa6e55df669b59bf"
-            ],
-            "version": "==1.9.0.post1"
-        },
-        "oauthlib": {
-            "hashes": [
-                "sha256:40a63637707e9163eda62d0f5345120c65e001a790480b8256448543c1f78f66",
-                "sha256:b4d99ae8ccfb7d33ba9591b59355c64eef5241534aa3da2e4c0435346b84bc8e"
-            ],
-            "version": "==3.0.2"
-        },
-        "python-dateutil": {
-            "hashes": [
-                "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb",
-                "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e"
-            ],
-            "version": "==2.8.0"
-        },
-        "pytumblr": {
-            "hashes": [
-                "sha256:a3774d3978bcff2db98f36a2e5d17bb8496ac21157b1b518089adad86d0dca72",
-                "sha256:eaa4d98217df7ab6392fa5d8801f4a2bdcba35bf0fd49328aa3c98e3b231b6f2"
-            ],
-            "index": "pypi",
-            "version": "==0.1.0"
-        },
-        "pyyaml": {
-            "hashes": [
-                "sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3",
-                "sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043",
-                "sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7",
-                "sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265",
-                "sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391",
-                "sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778",
-                "sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225",
-                "sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955",
-                "sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e",
-                "sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190",
-                "sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd"
-            ],
-            "index": "pypi",
-            "version": "==5.1.1"
-        },
-        "requests": {
-            "hashes": [
-                "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
-                "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
-            ],
-            "index": "pypi",
-            "version": "==2.22.0"
-        },
-        "requests-oauthlib": {
-            "hashes": [
-                "sha256:bd6533330e8748e94bf0b214775fed487d309b8b8fe823dc45641ebcd9a32f57",
-                "sha256:d3ed0c8f2e3bbc6b344fa63d6f933745ab394469da38db16bdddb461c7e25140",
-                "sha256:dd5a0499abfefd087c6dd96693cbd5bfd28aa009719a7f85ab3fabe3956ef19a"
-            ],
-            "version": "==1.2.0"
-        },
-        "sanction": {
-            "hashes": [
-                "sha256:3e41b24e28590a0dfed68eddd10e44fa01feb81812ffb49085ca764e51aea9fe"
-            ],
-            "version": "==0.4.1"
-        },
-        "six": {
-            "hashes": [
-                "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
-                "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
-            ],
-            "version": "==1.12.0"
-        },
-        "unicode-slugify": {
-            "hashes": [
-                "sha256:34cf3afefa6480efe705a4fc0eaeeaf7f49754aec322ba3e8b2f27dc1cbcf650"
-            ],
-            "index": "pypi",
-            "version": "==0.1.3"
-        },
-        "unidecode": {
-            "hashes": [
-                "sha256:1d7a042116536098d05d599ef2b8616759f02985c85b4fef50c78a5aaf10822a",
-                "sha256:2b6aab710c2a1647e928e36d69c21e76b453cd455f4e2621000e54b2a9b8cce8"
-            ],
-            "version": "==1.1.1"
-        },
-        "urllib3": {
-            "hashes": [
-                "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1",
-                "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232"
-            ],
-            "version": "==1.25.3"
-        },
-        "webencodings": {
-            "hashes": [
-                "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78",
-                "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"
-            ],
-            "version": "==0.5.1"
-        }
-    },
-    "develop": {}
-}
diff --git a/Tumblr.py b/Tumblr.py
index a07141e..0da0dd8 100644
--- a/Tumblr.py
+++ b/Tumblr.py
@@ -39,7 +39,8 @@ class TumblrFavs(common.Favs):
                 feeds.append(
                     {
                         "text": u.get("name"),
-                        "xmlUrl": "%srss" % u.get("url"),
+                        "xmlUrl": "https://cloud.petermolnar.net/rss-bridge/index.php?action=display&bridge=Tumblr&searchUsername=%s&format=Atom" % u.get("name"),
+                        #"xmlUrl": "%srss" % u.get("url"),
                         "htmlUrl": u.get("url"),
                     }
                 )
@@ -95,7 +96,7 @@ class TumblrFav(common.ImgFav):
         if not len(title):
             title = self.data.get("slug", "")
         if not len(title):
-            title = common.slugfname(self.url)
+            title = common.url2slug(self.url)
         return clean(title.strip())
 
     @property
diff --git a/common.py b/common.py
index aaeadf6..02ca59f 100644
--- a/common.py
+++ b/common.py
@@ -16,20 +16,36 @@ import settings
 import keys
 import yaml
 from pprint import pprint
+import feedparser
 
 TMPFEXT = ".xyz"
 MDFEXT = ".md"
 
+TMPSUBDIR = "nasg"
+SHM = "/dev/shm"
+
+if os.path.isdir(SHM) and os.access(SHM, os.W_OK):
+    TMPDIR = f"{SHM}/{TMPSUBDIR}"
+else:
+    TMPDIR = os.path.join(gettempdir(), TMPSUBDIR)
+
+if not os.path.isdir(TMPDIR):
+    os.makedirs(TMPDIR)
+
 
 def utfyamldump(data):
     """ dump YAML with actual UTF-8 chars """
-    return yaml.dump(data, default_flow_style=False, indent=4, allow_unicode=True)
+    return yaml.dump(
+        data, default_flow_style=False, indent=4, allow_unicode=True
+    )
 
 
-def slugfname(url):
-    return slugify(re.sub(r"^https?://(?:www)?", "", url), only_ascii=True, lower=True)[
-        :200
-    ]
+def url2slug(url):
+    return slugify(
+        re.sub(r"^https?://(?:www)?", "", url),
+        only_ascii=True,
+        lower=True,
+    )[:200]
 
 
 class cached_property(object):
@@ -51,7 +67,178 @@ class cached_property(object):
         return result
 
 
-class Follows(dict):
+class Aperture(object):
+    def __init__(self):
+        self.session = requests.Session()
+        self.session.headers.update(
+            {
+                "Authorization": "Bearer %s"
+                % (keys.aperture["access_token"])
+            }
+        )
+        self.url = keys.aperture["url"]
+
+    @cached_property
+    def channels(self):
+        channels = self.session.get(f"{self.url}?action=channels")
+        if channels.status_code != requests.codes.ok:
+            logging.error(
+                "failed to get channels from aperture: ", channels.text
+            )
+            return None
+        try:
+            channels = channels.json()
+        except ValueError as e:
+            logging.error("failed to parse channels from aperture: ", e)
+            return None
+
+        if "channels" not in channels:
+            logging.error("no channels found in aperture: ")
+            return None
+
+        return channels["channels"]
+
+    def channelid(self, channelname):
+        for channel in self.channels:
+            if channel["name"].lower() == channelname.lower():
+                return channel["uid"]
+        return None
+
+    def feedmeta(self, url):
+        cfile = os.path.join(
+            TMPDIR,
+            "%s.%s.json" % (url2slug(url), self.__class__.__name__)
+        )
+        if os.path.exists(cfile):
+            with open(cfile, 'rt') as cache:
+                return json.loads(cache.read())
+        r = {
+            'title': url,
+            'feed': url,
+            'link': url,
+            'type': 'rss'
+        }
+        try:
+            feed = feedparser.parse(url)
+            if 'feed' in feed:
+                for maybe in ['title', 'link']:
+                    if maybe in feed['feed']:
+                        r[maybe] = feed['feed'][maybe]
+        except Exception as e:
+            logging.error("feedparser failed on %s: %s" %(url, e))
+            r['type']: 'hfeed'
+            pass
+
+        with open(cfile, 'wt') as cache:
+            cache.write(json.dumps(r))
+
+        return r
+
+
+    def channelfollows(self, channelid):
+        follows = self.session.get(
+            f"{self.url}?action=follow&channel={channelid}"
+        )
+        if follows.status_code != requests.codes.ok:
+            logging.error(
+                "failed to get follows from aperture: ", follows.text
+            )
+            return
+        try:
+            follows = follows.json()
+        except ValueError as e:
+            logging.error("failed to parse follows from aperture: ", e)
+            return
+
+        if "items" not in follows:
+            logging.error(
+                f"no follows found in aperture for channel {channelid}"
+            )
+            return
+
+        existing = {}
+        for follow in follows["items"]:
+            meta = self.feedmeta(follow["url"])
+            existing.update({follow["url"]: meta})
+        return existing
+
+    @cached_property
+    def follows(self):
+        follows = {}
+        for channel in self.channels:
+            follows[channel["name"]] = self.channelfollows(
+                channel["uid"]
+            )
+        return follows
+
+    def export(self):
+        opml = etree.Element("opml", version="1.0")
+        xmldoc = etree.ElementTree(opml)
+        opml.addprevious(
+            etree.ProcessingInstruction(
+                "xml-stylesheet",
+                'type="text/xsl" href="%s"'
+                % (settings.opml.get("xsl")),
+            )
+        )
+
+        head = etree.SubElement(opml, "head")
+        title = etree.SubElement(
+            head, "title"
+        ).text = settings.opml.get("title")
+        dt = etree.SubElement(
+            head, "dateCreated"
+        ).text = arrow.utcnow().format("ddd, DD MMM YYYY HH:mm:ss UTC")
+        owner = etree.SubElement(
+            head, "ownerName"
+        ).text = settings.opml.get("owner")
+        email = etree.SubElement(
+            head, "ownerEmail"
+        ).text = settings.opml.get("email")
+
+        body = etree.SubElement(opml, "body")
+        groups = {}
+        for group, feeds in self.follows.items():
+            if (
+                "private" in group.lower()
+                or "nsfw" in group.lower()
+            ):
+                continue
+
+            if group not in groups.keys():
+                groups[group] = etree.SubElement(
+                    body, "outline", text=group
+                )
+            for url, meta in feeds.items():
+                entry = etree.SubElement(
+                    groups[group],
+                    "outline",
+                    type="rss",
+                    text=meta['title'],
+                    xmlUrl=meta['feed'],
+                    htmlUrl=meta['link']
+                )
+                etree.tostring(
+                    xmldoc,
+                    encoding="utf-8",
+                    xml_declaration=True,
+                    pretty_print=True,
+                )
+        opmlfile = os.path.join(
+            settings.paths.get("content"), "following.opml"
+        )
+        with open(opmlfile, "wb") as f:
+            f.write(
+                etree.tostring(
+                    xmldoc,
+                    encoding="utf-8",
+                    xml_declaration=True,
+                    pretty_print=True,
+                )
+            )
+
+
+class MinifluxFollows(dict):
     def __init__(self):
         self.auth = HTTPBasicAuth(
             keys.miniflux.get("username"), keys.miniflux.get("token")
@@ -60,9 +247,15 @@ class Follows(dict):
     @property
     def subscriptions(self):
         feeds = []
-        params = {"jsonrpc": "2.0", "method": "getFeeds", "id": keys.miniflux.get("id")}
+        params = {
+            "jsonrpc": "2.0",
+            "method": "getFeeds",
+            "id": keys.miniflux.get("id"),
+        }
         r = requests.post(
-            keys.miniflux.get("url"), data=json.dumps(params), auth=self.auth
+            keys.miniflux.get("url"),
+            data=json.dumps(params),
+            auth=self.auth,
         )
         return r.json().get("result", [])
 
@@ -96,24 +289,31 @@ class Follows(dict):
         opml.addprevious(
             etree.ProcessingInstruction(
                 "xml-stylesheet",
-                'type="text/xsl" href="%s"' % (settings.opml.get("xsl")),
+                'type="text/xsl" href="%s"'
+                % (settings.opml.get("xsl")),
             )
         )
         head = etree.SubElement(opml, "head")
-        title = etree.SubElement(head, "title").text = settings.opml.get("title")
-        dt = etree.SubElement(head, "dateCreated").text = arrow.utcnow().format(
-            "ddd, DD MMM YYYY HH:mm:ss UTC"
-        )
-        owner = etree.SubElement(head, "ownerName").text = settings.opml.get("owner")
-        email = etree.SubElement(head, "ownerEmail").text = settings.opml.get("email")
+        title = etree.SubElement(
+            head, "title"
+        ).text = settings.opml.get("title")
+        dt = etree.SubElement(
+            head, "dateCreated"
+        ).text = arrow.utcnow().format("ddd, DD MMM YYYY HH:mm:ss UTC")
+        owner = etree.SubElement(
+            head, "ownerName"
+        ).text = settings.opml.get("owner")
+        email = etree.SubElement(
+            head, "ownerEmail"
+        ).text = settings.opml.get("email")
 
         body = etree.SubElement(opml, "body")
         groups = {}
         for feed in self.subscriptions:
             # contains sensitive data, skip it
-            if "sessionid" in feed.get("feed_url") or "sessionid" in feed.get(
-                "site_url"
-            ):
+            if "sessionid" in feed.get(
+                "feed_url"
+            ) or "sessionid" in feed.get("site_url"):
                 continue
 
             fgroup = feed.get("groups", None)
@@ -136,12 +336,17 @@ class Follows(dict):
                 htmlUrl=feed.get("site_url"),
             )
 
-        opmlfile = os.path.join(settings.paths.get("content"), "following.opml")
+        opmlfile = os.path.join(
+            settings.paths.get("content"), "following.opml"
+        )
 
         with open(opmlfile, "wb") as f:
             f.write(
                 etree.tostring(
-                    xmldoc, encoding="utf-8", xml_declaration=True, pretty_print=True
+                    xmldoc,
+                    encoding="utf-8",
+                    xml_declaration=True,
+                    pretty_print=True,
                 )
             )
 
@@ -149,6 +354,11 @@ class Follows(dict):
 class Favs(object):
     def __init__(self, silo):
         self.silo = silo
+        self.aperture_auth = {
+            "Authorization": "Bearer %s"
+            % (keys.aperture["access_token"])
+        }
+        self.aperture_chid = 0
 
     @property
     def feeds(self):
@@ -156,7 +366,9 @@ class Favs(object):
 
     @property
     def since(self):
-        d = os.path.join(settings.paths.get("archive"), "favorite", "%s*" % self.silo)
+        d = os.path.join(
+            settings.paths.get("archive"), "favorite", "%s*" % self.silo
+        )
         files = glob.glob(d)
         if len(files):
             mtime = max([int(os.path.getmtime(f)) for f in files])
@@ -164,6 +376,81 @@ class Favs(object):
             mtime = 0
         return mtime
 
+    def sync_with_aperture(self):
+        channels = requests.get(
+            "%s?action=channels" % (keys.aperture["url"]),
+            headers=self.aperture_auth,
+        )
+        if channels.status_code != requests.codes.ok:
+            logging.error(
+                "failed to get channels from aperture: ", channels.text
+            )
+            return
+        try:
+            channels = channels.json()
+        except ValueError as e:
+            logging.error("failed to parse channels from aperture: ", e)
+            return
+
+        if "channels" not in channels:
+            logging.error("no channels found in aperture: ")
+            return
+
+        for channel in channels["channels"]:
+            if channel["name"].lower() == self.silo.lower():
+                self.aperture_chid = channel["uid"]
+                break
+
+        if not self.aperture_chid:
+            logging.error("no channels found for silo ", self.silo)
+            return
+
+        follows = requests.get(
+            "%s?action=follow&channel=%s"
+            % (keys.aperture["url"], self.aperture_chid),
+            headers=self.aperture_auth,
+        )
+        if follows.status_code != requests.codes.ok:
+            logging.error(
+                "failed to get follows from aperture: ", follows.text
+            )
+            return
+        try:
+            follows = follows.json()
+        except ValueError as e:
+            logging.error("failed to parse follows from aperture: ", e)
+            return
+
+        if "items" not in follows:
+            logging.error(
+                "no follows found in aperture for channel %s (%s)"
+                % (self.silo, self.aperture_chid)
+            )
+            return
+
+        existing = []
+        for follow in follows["items"]:
+            existing.append(follow["url"])
+        existing = list(set(existing))
+
+        for feed in self.feeds:
+            if feed["xmlUrl"] not in existing:
+                subscribe_to = {
+                    "action": "follow",
+                    "channel": self.aperture_chid,
+                    "url": feed["xmlUrl"],
+                }
+                logging.info(
+                    "subscribing to %s into %s (%s)"
+                    % (feed, self.silo, self.aperture_chid)
+                )
+                subscribe = requests.post(
+                    keys.aperture["url"],
+                    headers=self.aperture_auth,
+                    data=subscribe_to,
+                )
+                logging.debug(subscribe.text)
+
 
 class ImgFav(object):
     def __init__(self):
@@ -182,8 +469,11 @@ class ImgFav(object):
         return False
 
     def save_txt(self):
-        attachments = [os.path.basename(fn) for fn in glob.glob("%s*" % self.targetprefix)
-            if not os.path.basename(fn).endswith('.md')]
+        attachments = [
+            os.path.basename(fn)
+            for fn in glob.glob("%s*" % self.targetprefix)
+            if not os.path.basename(fn).endswith(".md")
+        ]
         meta = {
             "title": self.title,
             "favorite-of": self.url,
diff --git a/run.py b/run.py
index b9dbffa..9f0ce14 100644
--- a/run.py
+++ b/run.py
@@ -4,24 +4,27 @@ import Tumblr
 import LastFM
 import DeviantArt
 import Flickr
-import Artstation
+#import Artstation
 from pprint import pprint
 
 lfm = LastFM.LastFM()
 lfm.run()
 
-opml = common.Follows()
+#opml = common.Follows()
 
 silos = [
     DeviantArt.DAFavs(),
     Flickr.FlickrFavs(),
     Tumblr.TumblrFavs(),
-    Artstation.ASFavs(),
+#    Artstation.ASFavs(),
 ]
 
 for silo in silos:
     silo.run()
-    opml.update({silo.silo: silo.feeds})
+    #silo.sync_with_aperture()
+    #opml.update({silo.silo: silo.feeds})
 
-opml.sync()
+#opml.sync()
+#opml.export()
+opml = common.Aperture()
 opml.export()
diff --git a/settings.py b/settings.py
index a52d2e8..957b717 100644
--- a/settings.py
+++ b/settings.py
@@ -3,19 +3,25 @@ import re
 import argparse
 import logging
 
+class nameddict(dict):
+    __getattr__ = dict.get
+    __setattr__ = dict.__setitem__
+    __delattr__ = dict.__delitem__
+
 base = os.path.abspath(os.path.expanduser("~/Projects/petermolnar.net"))
 
-opml = {
+opml = nameddict({
     "owner": "Peter Molnar",
     "email": "mail@petermolnar.net",
     "title": "feeds followed by petermolnar.net",
     "xsl": "https://petermolnar.net/following.xsl",
-}
+})
 
-paths = {
+paths = nameddict({
     "archive": os.path.join(base, "archive"),
     "content": os.path.join(base, "content"),
-}
+    "bookmarks": os.path.join(base, "archive", "bookmarks")
+})
 
 loglevels = {"critical": 50, "error": 40, "warning": 30, "info": 20, "debug": 10}