pagination handling added to flickr and 500px
Peter Molnar hello@petermolnar.eu
Wed, 05 Jul 2017 22:36:44 +0000
1 files changed,
61 insertions(+),
209 deletions(-)
jump to
M
pesos.py
→
pesos.py
@@ -16,15 +16,6 @@ from slugify import slugify
import oauth import argparse -""" TODO - -- followings? - -- favs from: - - wordpress.com - - twitter - -""" class Bookmark(object): def __init__(self, title, url, fname=None):@@ -378,19 +369,46 @@ 'method': 'flickr.favorites.getList',
'api_key': shared.config.get('flickr', 'api_key'), 'user_id': shared.config.get('flickr', 'user_id'), 'extras': 'description,geo,tags,url_z,url_b,owner_name,date_upload', - 'per_page': 500, + 'per_page': 500, # maximim 'format': 'json', 'nojsoncallback': '1', 'min_fave_date': self.lastpulled } + def getpaged(self, offset): + logging.info('requesting page #%d of paginated results', offset) + self.params.update({ + 'page': offset + }) + r = requests.get( + self.url, + params=self.params + ) + parsed = json.loads(r.text) + return parsed.get('photos', {}).get('photo', []) + def run(self): r = requests.get(self.url,params=self.params) js = json.loads(r.text) - for photo in js.get('photos', {}).get('photo', []): + js = js.get('photos', {}) + + photos = js.get('photo', []) + + total = int(js.get('pages', 1)) + current = int(js.get('page', 1)) + cntr = total - current + + while cntr > 0: + current = current + 1 + paged = self.getpaged(current) + photos = photos + paged + cntr = total - current + + for photo in photos: fav = FlickrFav(photo) - fav.run() - fav.write() + if not fav.exists: + fav.run() + fav.write() class FivehpxFavs(Favs):@@ -398,16 +416,42 @@ def __init__(self):
super(FivehpxFavs, self).__init__('500px') self.params = { 'consumer_key': shared.config.get('500px', 'api_key'), - 'rpp': 100, + 'rpp': 100, # maximum 'image_size': 4, 'include_tags': 1, - 'include_geo': 1 + 'include_geo': 1, + 'sort': 'created_at', + 'sort_direction': 'desc' } + def getpaged(self, offset): + logging.info('requesting page #%d of paginated results', offset) + self.params.update({ + 'page': offset + }) + r = requests.get( + self.url, + params=self.params + ) + parsed = json.loads(r.text) + return parsed.get('photos') + def run(self): r = requests.get(self.url,params=self.params) js = json.loads(r.text) - for photo in js.get('photos', []): + photos = js.get('photos') + + total = int(js.get('total_pages', 1)) + current = int(js.get('current_page', 1)) + cntr = total - current + + while cntr > 0: + current = current + 1 + paged = self.getpaged(current) + photos = photos + paged + cntr = total - current + + for photo in photos: fav = FivehpxFav(photo) if not fav.exists: fav.run()@@ -438,7 +482,6 @@ )
js = json.loads(r.text) total = int(js.get('response', {}).get('liked_count', 20)) - print('total: %d' % total) offset = 20 cntr = total - offset likes = js.get('response', {}).get('liked_posts', [])@@ -507,9 +550,9 @@ self.params
) js = json.loads(r.text) + favs = js.get('results', []) has_more = js.get('has_more') offset = js.get('next_offset') - favs = js.get('results', []) while True == has_more: logging.debug('iterating over DA results with offset %d', offset) paged = self.getpaged(offset)@@ -528,197 +571,6 @@
f.fav.update({'meta': self.getsinglemeta(fav.get('deviationid'))}) f.run() f.write() - - - -#class WPFavs(Favs): - #def __init__(self): - #from pprint import pprint - #super(DAFavs, self).__init__('wordpress') - #self.oauth = oauth.DAOauth() - #self.params = { - #'limit': 24, - #'mature_content': 'true', - #'username': shared.config.get('deviantart', 'username') - #} - #self.likes = [] - - #def getpaged(self, offset): - #self.params.update({'offset': offset}) - #r = self.oauth.request( - #self.url, - #self.params - #) - #return json.loads(r.text) - - #def getsinglemeta(self, daid): - #r = self.oauth.request( - #'https://www.deviantart.com/api/v1/oauth2/deviation/metadata', - #params={ - #'deviationids[]': daid, - #'ext_submission': False, - #'ext_camera': False, - #'ext_stats': False, - #'ext_collection': False, - #'mature_content': True, - #} - #) - #meta = {} - #try: - #meta = json.loads(r.text) - #return meta.get('metadata', []).pop() - #except: - #return meta - - #def has_more(self, q): - #if 'True' == q or 'true' == q: - #return True - #return False - - #def run(self): - #r = self.oauth.request( - #self.url, - #self.params - #) - - #js = json.loads(r.text) - #has_more = js.get('has_more') - #offset = js.get('next_offset') - #favs = js.get('results', []) - #while True == has_more: - #logging.debug('iterating over DA results with offset %d', offset) - #paged = self.getpaged(offset) - #favs = favs + paged.get('results', []) - #has_more = paged.get('has_more') - #n = paged.get('next_offset') - #if n: - #offset = offset + n - - #self.favs = favs - #for fav in self.favs: - #f = DAFav(fav) - #if f.exists: - #continue - - #f.fav.update({'meta': self.getsinglemeta(fav.get('deviationid'))}) - #f.run() - #f.write() - -#class Following(object): - #def __init__(self, confgroup): - #self.confgroup = confgroup - #self.url = shared.config.get(confgroup, 'following_api') - #self.followings = [] - - -#class FlickrFollowing(Following): - #def __init__(self): - #super(FlickrFollowing, self).__init__('flickr') - #self.oauth = oauth.FlickrOauth() - - - #def run(self): - #r = self.oauth.request(self.url, params={ - #'method': 'flickr.contacts.getList', - #'format': 'json', - #'nojsoncallback': 1, - #'api_key': shared.config.get(self.confgroup, 'api_key') - #}) - - #try: - #contacts = json.loads(r.text) - #for c in contacts.get('contacts', {}).get('contact', []): - #self.followings.append({ - #'url': "https://www.flickr.com/people/%s/" % c.get('nsid'), - #'name': c.get('realname'), - #'username': c.get('username'), - #'userid': c.get('nsid') - #}) - - #except Exception as e: - #logging.error('getting following from flickr failed: %s', e) - - -#class TumblrFollowing(Following): - #def __init__(self): - #super(TumblrFollowing, self).__init__('tumblr') - #self.oauth = oauth.FlickrOauth() - - - #def run(self): - #r = self.oauth.request(self.url, params={ - #'method': 'flickr.contacts.getList', - #'format': 'json', - #'nojsoncallback': 1, - #'api_key': shared.config.get(self.confgroup, 'api_key') - #}) - - #try: - #contacts = json.loads(r.text) - #for c in contacts.get('contacts', {}).get('contact', []): - #self.followings.append({ - #'url': "https://www.flickr.com/people/%s/" % c.get('nsid'), - #'name': c.get('realname'), - #'username': c.get('username'), - #'userid': c.get('nsid') - #}) - - #except Exception as e: - #logging.error('getting following from flickr failed: %s', e) - - -#class FlickrFollowing(object): - #def __init__(self): - #super(FlickrFollowing, self).__init__('flickr') - #self.params = { - #'consumer_key': shared.config.get('500px', 'api_key'), - #'rpp': 100, - #'image_size': 4, - #'include_tags': 1, - #'include_geo': 1 - #} - - #def run(self): - #r = requests.get(self.url,params=self.params) - #js = json.loads(r.text) - #for photo in js.get('photos', []): - #fav = FivehpxFav(photo) - #if not fav.exists: - #fav.run() - #fav.write() - - - #def run(self): - - - -#https://api.flickr.com/services/rest/?method=flickr.contacts.getList&api_key=27d8a5bf7dabf882ff1c710894041f64&format=json&nojsoncallback=1&auth_token=72157682938907284-9c5f21debeec9833&api_sig=8ac87b900f44debea06a3765ed223680 - - -#class Following(object): - #def __init__(self, confgroup): - #self.confgroup = confgroup - #self.url = shared.config.get(confgroup, 'fav_api') - - -#class FlickrFollowing(Following): - #def __init__(self): - #super(FlickrFollowing, self).__init__('flickr') - #self.params = { - #'method': 'flickr.contacts.getList', - #'api_key': shared.config.get('flickr', 'api_key'), - #'format': 'json', - #'nojsoncallback': '1', - #} - - #def run(self): - #r = requests.get(self.url,params=self.params) - #js = json.loads(r.text) - #pprint(js) - #for contact in js.get('contacts', {}).get('contact', []): - #pprint(contact) - - if __name__ == '__main__':