pagination handling added to flickr and 500px

pesos.py

@@ -16,15 +16,6 @@ from slugify import slugify
 import oauth
 import argparse
 
-""" TODO
-
-- followings?
-
-- favs from:
-    - wordpress.com
-    - twitter
-
-"""
 
 class Bookmark(object):
     def __init__(self, title, url, fname=None):
@@ -378,19 +369,46 @@ class FlickrFavs(Favs):
             'api_key': shared.config.get('flickr', 'api_key'),
             'user_id': shared.config.get('flickr', 'user_id'),
             'extras': 'description,geo,tags,url_z,url_b,owner_name,date_upload',
-            'per_page': 500,
+            'per_page': 500, # maximim
             'format': 'json',
             'nojsoncallback': '1',
             'min_fave_date': self.lastpulled
         }
 
+    def getpaged(self, offset):
+        logging.info('requesting page #%d of paginated results', offset)
+        self.params.update({
+            'page': offset
+        })
+        r = requests.get(
+            self.url,
+            params=self.params
+        )
+        parsed = json.loads(r.text)
+        return parsed.get('photos', {}).get('photo', [])
+
     def run(self):
         r = requests.get(self.url,params=self.params)
         js = json.loads(r.text)
-        for photo in js.get('photos', {}).get('photo', []):
+        js = js.get('photos', {})
+
+        photos = js.get('photo', [])
+
+        total = int(js.get('pages', 1))
+        current = int(js.get('page', 1))
+        cntr = total - current
+
+        while cntr > 0:
+            current = current + 1
+            paged = self.getpaged(current)
+            photos = photos + paged
+            cntr = total - current
+
+        for photo in photos:
             fav = FlickrFav(photo)
-            fav.run()
+            if not fav.exists:
-            fav.write()
+                fav.run()
+                fav.write()
 
 
 class FivehpxFavs(Favs):
@@ -398,16 +416,42 @@ class FivehpxFavs(Favs):
         super(FivehpxFavs, self).__init__('500px')
         self.params = {
             'consumer_key': shared.config.get('500px', 'api_key'),
-            'rpp': 100,
+            'rpp': 100, # maximum
             'image_size': 4,
             'include_tags': 1,
-            'include_geo': 1
+            'include_geo': 1,
+            'sort': 'created_at',
+            'sort_direction': 'desc'
         }
 
+    def getpaged(self, offset):
+        logging.info('requesting page #%d of paginated results', offset)
+        self.params.update({
+            'page': offset
+        })
+        r = requests.get(
+            self.url,
+            params=self.params
+        )
+        parsed = json.loads(r.text)
+        return parsed.get('photos')
+
     def run(self):
         r = requests.get(self.url,params=self.params)
         js = json.loads(r.text)
-        for photo in js.get('photos', []):
+        photos = js.get('photos')
+
+        total = int(js.get('total_pages', 1))
+        current = int(js.get('current_page', 1))
+        cntr = total - current
+
+        while cntr > 0:
+            current = current + 1
+            paged = self.getpaged(current)
+            photos = photos + paged
+            cntr = total - current
+
+        for photo in photos:
             fav = FivehpxFav(photo)
             if not fav.exists:
                 fav.run()
@@ -438,7 +482,6 @@ class TumblrFavs(Favs):
 
         js = json.loads(r.text)
         total = int(js.get('response', {}).get('liked_count', 20))
-        print('total: %d' % total)
         offset = 20
         cntr = total - offset
         likes = js.get('response', {}).get('liked_posts', [])
@@ -507,9 +550,9 @@ class DAFavs(Favs):
         )
 
         js = json.loads(r.text)
+        favs = js.get('results', [])
         has_more = js.get('has_more')
         offset = js.get('next_offset')
-        favs = js.get('results', [])
         while True == has_more:
             logging.debug('iterating over DA results with offset %d', offset)
             paged = self.getpaged(offset)
@@ -529,197 +572,6 @@ class DAFavs(Favs):
             f.run()
             f.write()
 
-
-
-#class WPFavs(Favs):
-    #def __init__(self):
-        #from pprint import pprint
-        #super(DAFavs, self).__init__('wordpress')
-        #self.oauth = oauth.DAOauth()
-        #self.params = {
-            #'limit': 24,
-            #'mature_content': 'true',
-            #'username': shared.config.get('deviantart', 'username')
-        #}
-        #self.likes = []
-
-    #def getpaged(self, offset):
-        #self.params.update({'offset': offset})
-        #r = self.oauth.request(
-            #self.url,
-            #self.params
-        #)
-        #return json.loads(r.text)
-
-    #def getsinglemeta(self, daid):
-        #r = self.oauth.request(
-            #'https://www.deviantart.com/api/v1/oauth2/deviation/metadata',
-            #params={
-                #'deviationids[]': daid,
-                #'ext_submission': False,
-                #'ext_camera': False,
-                #'ext_stats': False,
-                #'ext_collection': False,
-                #'mature_content': True,
-            #}
-        #)
-        #meta = {}
-        #try:
-            #meta = json.loads(r.text)
-            #return meta.get('metadata', []).pop()
-        #except:
-            #return meta
-
-    #def has_more(self, q):
-        #if 'True' == q or 'true' == q:
-            #return True
-        #return False
-
-    #def run(self):
-        #r = self.oauth.request(
-            #self.url,
-            #self.params
-        #)
-
-        #js = json.loads(r.text)
-        #has_more = js.get('has_more')
-        #offset = js.get('next_offset')
-        #favs = js.get('results', [])
-        #while True == has_more:
-            #logging.debug('iterating over DA results with offset %d', offset)
-            #paged = self.getpaged(offset)
-            #favs = favs + paged.get('results', [])
-            #has_more = paged.get('has_more')
-            #n = paged.get('next_offset')
-            #if n:
-                #offset = offset + n
-
-        #self.favs = favs
-        #for fav in self.favs:
-            #f = DAFav(fav)
-            #if f.exists:
-                #continue
-
-            #f.fav.update({'meta': self.getsinglemeta(fav.get('deviationid'))})
-            #f.run()
-            #f.write()
-
-#class Following(object):
-    #def __init__(self, confgroup):
-        #self.confgroup = confgroup
-        #self.url = shared.config.get(confgroup, 'following_api')
-        #self.followings = []
-
-
-#class FlickrFollowing(Following):
-    #def __init__(self):
-        #super(FlickrFollowing, self).__init__('flickr')
-        #self.oauth = oauth.FlickrOauth()
-
-
-    #def run(self):
-        #r = self.oauth.request(self.url, params={
-            #'method': 'flickr.contacts.getList',
-            #'format': 'json',
-            #'nojsoncallback': 1,
-            #'api_key': shared.config.get(self.confgroup, 'api_key')
-        #})
-
-        #try:
-            #contacts = json.loads(r.text)
-            #for c in contacts.get('contacts', {}).get('contact', []):
-                #self.followings.append({
-                    #'url': "https://www.flickr.com/people/%s/" % c.get('nsid'),
-                    #'name': c.get('realname'),
-                    #'username': c.get('username'),
-                    #'userid': c.get('nsid')
-                #})
-
-        #except Exception as e:
-            #logging.error('getting following from flickr failed: %s', e)
-
-
-#class TumblrFollowing(Following):
-    #def __init__(self):
-        #super(TumblrFollowing, self).__init__('tumblr')
-        #self.oauth = oauth.FlickrOauth()
-
-
-    #def run(self):
-        #r = self.oauth.request(self.url, params={
-            #'method': 'flickr.contacts.getList',
-            #'format': 'json',
-            #'nojsoncallback': 1,
-            #'api_key': shared.config.get(self.confgroup, 'api_key')
-        #})
-
-        #try:
-            #contacts = json.loads(r.text)
-            #for c in contacts.get('contacts', {}).get('contact', []):
-                #self.followings.append({
-                    #'url': "https://www.flickr.com/people/%s/" % c.get('nsid'),
-                    #'name': c.get('realname'),
-                    #'username': c.get('username'),
-                    #'userid': c.get('nsid')
-                #})
-
-        #except Exception as e:
-            #logging.error('getting following from flickr failed: %s', e)
-
-
-#class FlickrFollowing(object):
-    #def __init__(self):
-        #super(FlickrFollowing, self).__init__('flickr')
-        #self.params = {
-            #'consumer_key': shared.config.get('500px', 'api_key'),
-            #'rpp': 100,
-            #'image_size': 4,
-            #'include_tags': 1,
-            #'include_geo': 1
-        #}
-
-    #def run(self):
-        #r = requests.get(self.url,params=self.params)
-        #js = json.loads(r.text)
-        #for photo in js.get('photos', []):
-            #fav = FivehpxFav(photo)
-            #if not fav.exists:
-                #fav.run()
-                #fav.write()
-
-
-    #def run(self):
-
-
-
-#https://api.flickr.com/services/rest/?method=flickr.contacts.getList&api_key=27d8a5bf7dabf882ff1c710894041f64&format=json&nojsoncallback=1&auth_token=72157682938907284-9c5f21debeec9833&api_sig=8ac87b900f44debea06a3765ed223680
-
-
-#class Following(object):
-    #def __init__(self, confgroup):
-        #self.confgroup = confgroup
-        #self.url = shared.config.get(confgroup, 'fav_api')
-
-
-#class FlickrFollowing(Following):
-    #def __init__(self):
-        #super(FlickrFollowing, self).__init__('flickr')
-        #self.params = {
-            #'method': 'flickr.contacts.getList',
-            #'api_key': shared.config.get('flickr', 'api_key'),
-            #'format': 'json',
-            #'nojsoncallback': '1',
-        #}
-
-    #def run(self):
-        #r = requests.get(self.url,params=self.params)
-        #js = json.loads(r.text)
-        #pprint(js)
-        #for contact in js.get('contacts', {}).get('contact', []):
-            #pprint(contact)
-
-
-
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser(description='Parameters for NASG')