re-adding since epoch to most silos, raising offset for deviantart, moving Flickr extra photo info into a property to avoid premature fetching

DeviantArt.py

@@ -30,7 +30,8 @@ class DAFavs(common.Favs):
             try:
                 following = self.client.get_friends(
                     username=keys.deviantart.get('username'),
-                    offset=offset
+                    offset=offset,
+                    limit=24
                 )
                 offset = following.get('next_offset')
                 for follow in following.get('results'):
@@ -53,7 +54,8 @@ class DAFavs(common.Favs):
             try:
                 folders = self.client.get_collections(
                     username=keys.deviantart.get('username'),
-                    offset=offset
+                    offset=offset,
+                    limit=24
                 )
                 offset = folders.get('next_offset')
                 for r in folders.get('results'):
@@ -73,6 +75,8 @@ class DAFavs(common.Favs):
                     self.favfolder,
                     username=keys.deviantart.get('username'),
                     offset=offset,
+                    limit=24,
+                    #mature_content=True
                 )
                 for r in fetched.get('results'):
                     fav = DAFav(r)

Flickr.py

@@ -50,7 +50,8 @@ class FlickrFavs(common.Favs):
             logging.info('fetching for Flickr: page %d' % page)
             fetched = self.user.getFavorites(
                 user_id=self.user.id,
-                page=page
+                page=page,
+                min_fave_date=self.since
             )
             for p in fetched:
                 photo = FlickrFav(p)
@@ -62,12 +63,20 @@ class FlickrFavs(common.Favs):
 class FlickrFav(common.ImgFav):
     def __init__(self, flickrphoto):
         self.flickrphoto = flickrphoto
-        self.info = flickrphoto.getInfo()
-        self.owner = self.info.get('owner')
 
     def __str__(self):
         return "fav-of %s" % (self.url)
 
+    @property
+    @common.cached_property
+    def owner(self):
+        return self.info.get('owner')
+
+    @property
+    @common.cached_property
+    def info(self):
+        return flickrphoto.getInfo()
+
     @property
     def author(self):
         return {

Tumblr.py

@@ -43,7 +43,7 @@ class TumblrFavs(common.Favs):
 
     def run(self):
         has_more = True
-        after = 0
+        after = self.since
         while has_more:
             logging.info('fetching for Tumblr: after %d' % after)
             fetched = self.client.likes(after=after)

common.py

@@ -21,6 +21,22 @@ def slugfname(url):
         lower=True
     )[:200]
 
+class cached_property(object):
+    """ extermely simple cached_property decorator:
+    whenever something is called as @cached_property, on first run, the
+    result is calculated, then the class method is overwritten to be
+    a property, contaning the result from the method
+    """
+    def __init__(self, method, name=None):
+        self.method = method
+        self.name = name or method.__name__
+    def __get__(self, inst, cls):
+        if inst is None:
+            return self
+        result = self.method(inst)
+        setattr(inst, self.name, result)
+        return result
+
 
 class Follows(object):
     def __init__(self):
@@ -66,21 +82,16 @@ class Favs(object):
 
     @property
     def since(self):
-        mtime = 0
         d = os.path.join(
             settings.paths.get('archive'),
             'favorite',
-            "%s-*" % self.silo
+            "%s*" % self.silo
         )
         files = glob.glob(d)
-
-        if (len(files)):
-            for f in files:
-                ftime = int(os.path.getmtime(f))
-                if ftime > mtime:
-                    mtime = ftime
-        # TODO why is this here?
-        mtime = mtime + 1
+        if len(files):
+            mtime = max([int(os.path.getmtime(f)) for f in files])
+        else:
+            mtime = 0
         return mtime