pagination handling added to flickr and 500px
This commit is contained in:
Peter Molnar
parent
ba08942632
commit
529fabc1ba
1 changed files with 61 additions and 209 deletions
266
pesos.py
266
pesos.py
|
|
@ -16,15 +16,6 @@ from slugify import slugify
|
|||
import oauth
|
||||
import argparse
|
||||
|
||||
""" TODO
|
||||
|
||||
- followings?
|
||||
|
||||
- favs from:
|
||||
- wordpress.com
|
||||
- twitter
|
||||
|
||||
"""
|
||||
|
||||
class Bookmark(object):
|
||||
def __init__(self, title, url, fname=None):
|
||||
|
|
@ -378,17 +369,44 @@ class FlickrFavs(Favs):
|
|||
'api_key': shared.config.get('flickr', 'api_key'),
|
||||
'user_id': shared.config.get('flickr', 'user_id'),
|
||||
'extras': 'description,geo,tags,url_z,url_b,owner_name,date_upload',
|
||||
'per_page': 500,
|
||||
'per_page': 500, # maximim
|
||||
'format': 'json',
|
||||
'nojsoncallback': '1',
|
||||
'min_fave_date': self.lastpulled
|
||||
}
|
||||
|
||||
def getpaged(self, offset):
|
||||
logging.info('requesting page #%d of paginated results', offset)
|
||||
self.params.update({
|
||||
'page': offset
|
||||
})
|
||||
r = requests.get(
|
||||
self.url,
|
||||
params=self.params
|
||||
)
|
||||
parsed = json.loads(r.text)
|
||||
return parsed.get('photos', {}).get('photo', [])
|
||||
|
||||
def run(self):
|
||||
r = requests.get(self.url,params=self.params)
|
||||
js = json.loads(r.text)
|
||||
for photo in js.get('photos', {}).get('photo', []):
|
||||
js = js.get('photos', {})
|
||||
|
||||
photos = js.get('photo', [])
|
||||
|
||||
total = int(js.get('pages', 1))
|
||||
current = int(js.get('page', 1))
|
||||
cntr = total - current
|
||||
|
||||
while cntr > 0:
|
||||
current = current + 1
|
||||
paged = self.getpaged(current)
|
||||
photos = photos + paged
|
||||
cntr = total - current
|
||||
|
||||
for photo in photos:
|
||||
fav = FlickrFav(photo)
|
||||
if not fav.exists:
|
||||
fav.run()
|
||||
fav.write()
|
||||
|
||||
|
|
@ -398,16 +416,42 @@ class FivehpxFavs(Favs):
|
|||
super(FivehpxFavs, self).__init__('500px')
|
||||
self.params = {
|
||||
'consumer_key': shared.config.get('500px', 'api_key'),
|
||||
'rpp': 100,
|
||||
'rpp': 100, # maximum
|
||||
'image_size': 4,
|
||||
'include_tags': 1,
|
||||
'include_geo': 1
|
||||
'include_geo': 1,
|
||||
'sort': 'created_at',
|
||||
'sort_direction': 'desc'
|
||||
}
|
||||
|
||||
def getpaged(self, offset):
|
||||
logging.info('requesting page #%d of paginated results', offset)
|
||||
self.params.update({
|
||||
'page': offset
|
||||
})
|
||||
r = requests.get(
|
||||
self.url,
|
||||
params=self.params
|
||||
)
|
||||
parsed = json.loads(r.text)
|
||||
return parsed.get('photos')
|
||||
|
||||
def run(self):
|
||||
r = requests.get(self.url,params=self.params)
|
||||
js = json.loads(r.text)
|
||||
for photo in js.get('photos', []):
|
||||
photos = js.get('photos')
|
||||
|
||||
total = int(js.get('total_pages', 1))
|
||||
current = int(js.get('current_page', 1))
|
||||
cntr = total - current
|
||||
|
||||
while cntr > 0:
|
||||
current = current + 1
|
||||
paged = self.getpaged(current)
|
||||
photos = photos + paged
|
||||
cntr = total - current
|
||||
|
||||
for photo in photos:
|
||||
fav = FivehpxFav(photo)
|
||||
if not fav.exists:
|
||||
fav.run()
|
||||
|
|
@ -438,7 +482,6 @@ class TumblrFavs(Favs):
|
|||
|
||||
js = json.loads(r.text)
|
||||
total = int(js.get('response', {}).get('liked_count', 20))
|
||||
print('total: %d' % total)
|
||||
offset = 20
|
||||
cntr = total - offset
|
||||
likes = js.get('response', {}).get('liked_posts', [])
|
||||
|
|
@ -507,9 +550,9 @@ class DAFavs(Favs):
|
|||
)
|
||||
|
||||
js = json.loads(r.text)
|
||||
favs = js.get('results', [])
|
||||
has_more = js.get('has_more')
|
||||
offset = js.get('next_offset')
|
||||
favs = js.get('results', [])
|
||||
while True == has_more:
|
||||
logging.debug('iterating over DA results with offset %d', offset)
|
||||
paged = self.getpaged(offset)
|
||||
|
|
@ -529,197 +572,6 @@ class DAFavs(Favs):
|
|||
f.run()
|
||||
f.write()
|
||||
|
||||
|
||||
|
||||
#class WPFavs(Favs):
|
||||
#def __init__(self):
|
||||
#from pprint import pprint
|
||||
#super(DAFavs, self).__init__('wordpress')
|
||||
#self.oauth = oauth.DAOauth()
|
||||
#self.params = {
|
||||
#'limit': 24,
|
||||
#'mature_content': 'true',
|
||||
#'username': shared.config.get('deviantart', 'username')
|
||||
#}
|
||||
#self.likes = []
|
||||
|
||||
#def getpaged(self, offset):
|
||||
#self.params.update({'offset': offset})
|
||||
#r = self.oauth.request(
|
||||
#self.url,
|
||||
#self.params
|
||||
#)
|
||||
#return json.loads(r.text)
|
||||
|
||||
#def getsinglemeta(self, daid):
|
||||
#r = self.oauth.request(
|
||||
#'https://www.deviantart.com/api/v1/oauth2/deviation/metadata',
|
||||
#params={
|
||||
#'deviationids[]': daid,
|
||||
#'ext_submission': False,
|
||||
#'ext_camera': False,
|
||||
#'ext_stats': False,
|
||||
#'ext_collection': False,
|
||||
#'mature_content': True,
|
||||
#}
|
||||
#)
|
||||
#meta = {}
|
||||
#try:
|
||||
#meta = json.loads(r.text)
|
||||
#return meta.get('metadata', []).pop()
|
||||
#except:
|
||||
#return meta
|
||||
|
||||
#def has_more(self, q):
|
||||
#if 'True' == q or 'true' == q:
|
||||
#return True
|
||||
#return False
|
||||
|
||||
#def run(self):
|
||||
#r = self.oauth.request(
|
||||
#self.url,
|
||||
#self.params
|
||||
#)
|
||||
|
||||
#js = json.loads(r.text)
|
||||
#has_more = js.get('has_more')
|
||||
#offset = js.get('next_offset')
|
||||
#favs = js.get('results', [])
|
||||
#while True == has_more:
|
||||
#logging.debug('iterating over DA results with offset %d', offset)
|
||||
#paged = self.getpaged(offset)
|
||||
#favs = favs + paged.get('results', [])
|
||||
#has_more = paged.get('has_more')
|
||||
#n = paged.get('next_offset')
|
||||
#if n:
|
||||
#offset = offset + n
|
||||
|
||||
#self.favs = favs
|
||||
#for fav in self.favs:
|
||||
#f = DAFav(fav)
|
||||
#if f.exists:
|
||||
#continue
|
||||
|
||||
#f.fav.update({'meta': self.getsinglemeta(fav.get('deviationid'))})
|
||||
#f.run()
|
||||
#f.write()
|
||||
|
||||
#class Following(object):
|
||||
#def __init__(self, confgroup):
|
||||
#self.confgroup = confgroup
|
||||
#self.url = shared.config.get(confgroup, 'following_api')
|
||||
#self.followings = []
|
||||
|
||||
|
||||
#class FlickrFollowing(Following):
|
||||
#def __init__(self):
|
||||
#super(FlickrFollowing, self).__init__('flickr')
|
||||
#self.oauth = oauth.FlickrOauth()
|
||||
|
||||
|
||||
#def run(self):
|
||||
#r = self.oauth.request(self.url, params={
|
||||
#'method': 'flickr.contacts.getList',
|
||||
#'format': 'json',
|
||||
#'nojsoncallback': 1,
|
||||
#'api_key': shared.config.get(self.confgroup, 'api_key')
|
||||
#})
|
||||
|
||||
#try:
|
||||
#contacts = json.loads(r.text)
|
||||
#for c in contacts.get('contacts', {}).get('contact', []):
|
||||
#self.followings.append({
|
||||
#'url': "https://www.flickr.com/people/%s/" % c.get('nsid'),
|
||||
#'name': c.get('realname'),
|
||||
#'username': c.get('username'),
|
||||
#'userid': c.get('nsid')
|
||||
#})
|
||||
|
||||
#except Exception as e:
|
||||
#logging.error('getting following from flickr failed: %s', e)
|
||||
|
||||
|
||||
#class TumblrFollowing(Following):
|
||||
#def __init__(self):
|
||||
#super(TumblrFollowing, self).__init__('tumblr')
|
||||
#self.oauth = oauth.FlickrOauth()
|
||||
|
||||
|
||||
#def run(self):
|
||||
#r = self.oauth.request(self.url, params={
|
||||
#'method': 'flickr.contacts.getList',
|
||||
#'format': 'json',
|
||||
#'nojsoncallback': 1,
|
||||
#'api_key': shared.config.get(self.confgroup, 'api_key')
|
||||
#})
|
||||
|
||||
#try:
|
||||
#contacts = json.loads(r.text)
|
||||
#for c in contacts.get('contacts', {}).get('contact', []):
|
||||
#self.followings.append({
|
||||
#'url': "https://www.flickr.com/people/%s/" % c.get('nsid'),
|
||||
#'name': c.get('realname'),
|
||||
#'username': c.get('username'),
|
||||
#'userid': c.get('nsid')
|
||||
#})
|
||||
|
||||
#except Exception as e:
|
||||
#logging.error('getting following from flickr failed: %s', e)
|
||||
|
||||
|
||||
#class FlickrFollowing(object):
|
||||
#def __init__(self):
|
||||
#super(FlickrFollowing, self).__init__('flickr')
|
||||
#self.params = {
|
||||
#'consumer_key': shared.config.get('500px', 'api_key'),
|
||||
#'rpp': 100,
|
||||
#'image_size': 4,
|
||||
#'include_tags': 1,
|
||||
#'include_geo': 1
|
||||
#}
|
||||
|
||||
#def run(self):
|
||||
#r = requests.get(self.url,params=self.params)
|
||||
#js = json.loads(r.text)
|
||||
#for photo in js.get('photos', []):
|
||||
#fav = FivehpxFav(photo)
|
||||
#if not fav.exists:
|
||||
#fav.run()
|
||||
#fav.write()
|
||||
|
||||
|
||||
#def run(self):
|
||||
|
||||
|
||||
|
||||
#https://api.flickr.com/services/rest/?method=flickr.contacts.getList&api_key=27d8a5bf7dabf882ff1c710894041f64&format=json&nojsoncallback=1&auth_token=72157682938907284-9c5f21debeec9833&api_sig=8ac87b900f44debea06a3765ed223680
|
||||
|
||||
|
||||
#class Following(object):
|
||||
#def __init__(self, confgroup):
|
||||
#self.confgroup = confgroup
|
||||
#self.url = shared.config.get(confgroup, 'fav_api')
|
||||
|
||||
|
||||
#class FlickrFollowing(Following):
|
||||
#def __init__(self):
|
||||
#super(FlickrFollowing, self).__init__('flickr')
|
||||
#self.params = {
|
||||
#'method': 'flickr.contacts.getList',
|
||||
#'api_key': shared.config.get('flickr', 'api_key'),
|
||||
#'format': 'json',
|
||||
#'nojsoncallback': '1',
|
||||
#}
|
||||
|
||||
#def run(self):
|
||||
#r = requests.get(self.url,params=self.params)
|
||||
#js = json.loads(r.text)
|
||||
#pprint(js)
|
||||
#for contact in js.get('contacts', {}).get('contact', []):
|
||||
#pprint(contact)
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
parser = argparse.ArgumentParser(description='Parameters for NASG')
|
||||
|
|
|
|||
Loading…
Reference in a new issue