From e121a01bc2befab228baa68a707cd31deb1090b5 Mon Sep 17 00:00:00 2001
From: Peter Molnar <hello@petermolnar.eu>
Date: Tue, 16 Oct 2018 08:59:52 +0100
Subject: [PATCH] re-adding since epoch to most silos, raising offset for
 deviantart, moving Flickr extra photo info into a property to avoid premature
 fetching

---
 DeviantArt.py |  8 ++++++--
 Flickr.py     | 15 ++++++++++++---
 Tumblr.py     |  2 +-
 common.py     | 31 +++++++++++++++++++++----------
 4 files changed, 40 insertions(+), 16 deletions(-)

diff --git a/DeviantArt.py b/DeviantArt.py
index a736a73..d49c117 100644
--- a/DeviantArt.py
+++ b/DeviantArt.py
@@ -30,7 +30,8 @@ class DAFavs(common.Favs):
             try:
                 following = self.client.get_friends(
                     username=keys.deviantart.get('username'),
-                    offset=offset
+                    offset=offset,
+                    limit=24
                 )
                 offset = following.get('next_offset')
                 for follow in following.get('results'):
@@ -53,7 +54,8 @@ class DAFavs(common.Favs):
             try:
                 folders = self.client.get_collections(
                     username=keys.deviantart.get('username'),
-                    offset=offset
+                    offset=offset,
+                    limit=24
                 )
                 offset = folders.get('next_offset')
                 for r in folders.get('results'):
@@ -73,6 +75,8 @@ class DAFavs(common.Favs):
                     self.favfolder,
                     username=keys.deviantart.get('username'),
                     offset=offset,
+                    limit=24,
+                    #mature_content=True
                 )
                 for r in fetched.get('results'):
                     fav = DAFav(r)
diff --git a/Flickr.py b/Flickr.py
index 3da11e3..be6ece7 100644
--- a/Flickr.py
+++ b/Flickr.py
@@ -50,7 +50,8 @@ class FlickrFavs(common.Favs):
             logging.info('fetching for Flickr: page %d' % page)
             fetched = self.user.getFavorites(
                 user_id=self.user.id,
-                page=page
+                page=page,
+                min_fave_date=self.since
             )
             for p in fetched:
                 photo = FlickrFav(p)
@@ -62,12 +63,20 @@ class FlickrFavs(common.Favs):
 class FlickrFav(common.ImgFav):
     def __init__(self, flickrphoto):
         self.flickrphoto = flickrphoto
-        self.info = flickrphoto.getInfo()
-        self.owner = self.info.get('owner')
 
     def __str__(self):
         return "fav-of %s" % (self.url)
 
+    @property
+    @common.cached_property
+    def owner(self):
+        return self.info.get('owner')
+
+    @property
+    @common.cached_property
+    def info(self):
+        return flickrphoto.getInfo()
+
     @property
     def author(self):
         return {
diff --git a/Tumblr.py b/Tumblr.py
index 8eff394..5abbc6e 100644
--- a/Tumblr.py
+++ b/Tumblr.py
@@ -43,7 +43,7 @@ class TumblrFavs(common.Favs):
 
     def run(self):
         has_more = True
-        after = 0
+        after = self.since
         while has_more:
             logging.info('fetching for Tumblr: after %d' % after)
             fetched = self.client.likes(after=after)
diff --git a/common.py b/common.py
index a735a58..4b099ea 100644
--- a/common.py
+++ b/common.py
@@ -21,6 +21,22 @@ def slugfname(url):
         lower=True
     )[:200]
 
+class cached_property(object):
+    """ extermely simple cached_property decorator:
+    whenever something is called as @cached_property, on first run, the
+    result is calculated, then the class method is overwritten to be
+    a property, contaning the result from the method
+    """
+    def __init__(self, method, name=None):
+        self.method = method
+        self.name = name or method.__name__
+    def __get__(self, inst, cls):
+        if inst is None:
+            return self
+        result = self.method(inst)
+        setattr(inst, self.name, result)
+        return result
+
 
 class Follows(object):
     def __init__(self):
@@ -66,21 +82,16 @@ class Favs(object):
 
     @property
     def since(self):
-        mtime = 0
         d = os.path.join(
             settings.paths.get('archive'),
             'favorite',
-            "%s-*" % self.silo
+            "%s*" % self.silo
         )
         files = glob.glob(d)
-
-        if (len(files)):
-            for f in files:
-                ftime = int(os.path.getmtime(f))
-                if ftime > mtime:
-                    mtime = ftime
-        # TODO why is this here?
-        mtime = mtime + 1
+        if len(files):
+            mtime = max([int(os.path.getmtime(f)) for f in files])
+        else:
+            mtime = 0
         return mtime