diff --git a/nasg.py b/nasg.py index 6f2b804..d3f96a7 100755 --- a/nasg.py +++ b/nasg.py @@ -157,6 +157,7 @@ class Indexer(object): singular.fname, singular.summary, singular.content, + singular.reactions.values() ] content_remote = [] @@ -164,7 +165,7 @@ class Indexer(object): #content_remote.append("%s" % offlinecopy) weight = 1 - if singular.isbookmark: + if singular.isbookmark or singular.isfav: weight = 10 if singular.ispage: weight = 100 @@ -250,6 +251,24 @@ class OfflineArchive(object): self.exists = os.path.isfile(self.target) + #def read(self): + #if not self.exists: + #return '' + + #with open(self.target, 'rt') as f: + #self.fm = frontmatter.loads(f.read()) + + #readable = '' + #try: + #readable = Document(self.fm.content) + #readable = shared.Pandoc(False).convert(readable.summary()) + #readable = shared.Pandoc().convert(readable) + #except Exception as e: + #logging.error('Failed to readable %s', self.target) + + #return readable + + def _getimage(self, src): imgname, imgext = os.path.splitext(os.path.basename(src)) imgtarget = os.path.join( @@ -396,11 +415,11 @@ class OfflineArchive(object): return None - def read(): - if os.path.isfile(self.target): - with open(self.target) as f: - self.fm = frontmatter.loads(f.read()) - return + #def read(): + #if os.path.isfile(self.target): + #with open(self.target) as f: + #self.fm = frontmatter.loads(f.read()) + #return def run(self): @@ -1394,7 +1413,6 @@ class Singular(BaseRenderable): self.content, self.photo ) - # REMOVE THIS trigger = self.offlinecopies @@ -1406,18 +1424,27 @@ class Singular(BaseRenderable): ) ) ) - img = self.meta.get('image', False) - if not img: - return + if not url: return - c = '[![%s](/%s/%s)](%s){.favurl}' % ( - self.title, - shared.config.get('source', 'files'), - img, - url - ) + img = self.meta.get('image', False) + imgs = self.meta.get('images', []) + if img: + imgs.append(img) + + if not imgs or not len(imgs): + return + + c = '' + for i in imgs: + c = '%s\n[![%s](/%s/%s)](%s){.favurl}' % ( + c, + self.title, + shared.config.get('source', 'files'), + i, + url + ) if self.isbookmark: c = "%s\n\n%s" % (c, self.content) @@ -1674,7 +1701,9 @@ class Singular(BaseRenderable): for maybe in ['title', 'bookmark-of', 'in-reply-to', 'repost-of']: maybe = self.meta.get(maybe, False) if maybe: - self._title = maybe + if isinstance(maybe, list): + maybe = maybe.pop() + self._title = maybe.replace('\n', ' ').replace('\r', '') break return self._title @@ -1717,18 +1746,19 @@ class Singular(BaseRenderable): return self.copies copies = {} - for maybe in ['bookmark-of', 'in-reply-to', 'repost-of']: + for maybe in ['bookmark-of', 'in-reply-to', 'repost-of', 'favorite-of']: maybe = self.meta.get(maybe, False) if not maybe: continue if not isinstance(maybe, list): maybe = [maybe] for url in maybe: - copies[url] = OfflineArchive(url) - copies[url].run() + arch = OfflineArchive(url) + arch.run() + #copies[url] = arch.read() - self.copies = copies - return copies + #self.copies = copies + #return copies @property @@ -1768,8 +1798,8 @@ class Singular(BaseRenderable): 'slug': self.fname, 'shortslug': self.shortslug, 'rssenclosure': self.rssenclosure, - #'copies': self.offlinecopies, - 'copies': [], + #'offlinecopies': self.offlinecopies, + #'copies': [], 'comments': self.comments, 'replies': self.replies, 'reacjis': self.reacjis, diff --git a/oauth.py b/oauth.py new file mode 100755 index 0000000..7a32f30 --- /dev/null +++ b/oauth.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 + +import asyncio +import uvloop +import os +import json +from sanic import Sanic +import sanic.response +from sanic.log import log as logging +import shared +import requests +from requests_oauthlib import OAuth1Session, oauth1_session, OAuth2Session, oauth2_session +from oauthlib.oauth2 import BackendApplicationClient +import json +import tempfile + +from pprint import pprint + +class TokenDB(object): + def __init__(self, uuid='tokens'): + self.db = os.path.abspath(os.path.join( + tempfile.gettempdir(), + "%s.json" % uuid + )) + self.tokens = {} + self.refresh() + + def refresh(self): + self.tokens = {} + if os.path.isfile(self.db): + with open(self.db, 'rt') as f: + self.tokens = json.loads(f.read()) + + def save(self): + with open(self.db, 'wt') as f: + f.write(json.dumps( + self.tokens, indent=4, sort_keys=True + )) + self.refresh() + + def get_token(self, token): + return self.tokens.get(token, None) + + def get_service(self, service): + token = self.tokens.get(service, None) + #if token: + #token = self.get_token(token) + return token + + def set_service(self, service, tokenid): + self.tokens.update({ + service: tokenid + }) + self.save() + + def update_token(self, + token, + oauth_token_secret=None, + access_token=None, + access_token_secret=None, + verifier=None): + + t = self.tokens.get(token, {}) + if oauth_token_secret: + t.update({ + 'oauth_token_secret': oauth_token_secret + }) + if access_token: + t.update({ + 'access_token': access_token + }) + if access_token_secret: + t.update({ + 'access_token_secret': access_token_secret + }) + if verifier: + t.update({ + 'verifier': verifier + }) + + self.tokens.update({ + token: t + }) + self.save() + + def clear(self): + self.tokens = {} + self.save() + + def clear_service(self, service): + t = self.tokens.get(service) + if t: + del(self.tokens[t]) + del(self.tokens[service]) + self.save() + + +class Oauth2Flow(object): + token_url = '' + + def __init__(self, service): + self.service = service + self.key = shared.config.get(service, 'api_key') + self.secret = shared.config.get(service, 'api_secret') + client = BackendApplicationClient( + client_id=self.key + ) + client.prepare_request_body(scope=['browse']) + oauth = OAuth2Session(client=client) + token = oauth.fetch_token( + token_url=self.token_url, + client_id=self.key, + client_secret=self.secret + ) + self.client = OAuth2Session( + self.key, + token=token + ) + + def request(self, url, params={}): + return self.client.get(url, params=params) + + + +class DAOauth(Oauth2Flow): + token_url = 'https://www.deviantart.com/oauth2/token' + + def __init__(self): + super(DAOauth, self).__init__('deviantart') + + +class Oauth1Flow(object): + request_token_url = '' + access_token_url = '' + authorize_url = '' + + def __init__(self, service): + self.service = service + self.key = shared.config.get(service, 'api_key') + self.secret = shared.config.get(service, 'api_secret') + self.tokendb = TokenDB() + self.t = self.tokendb.get_service(self.service) + self.oauth_init() + + def oauth_init(self): + if not self.t: + self.request_oauth_token() + + t = self.tokendb.get_token(self.t) + if not t.get('access_token', None) or not t.get('access_token_secret', None): + self.request_access_token() + + def request_oauth_token(self): + client = OAuth1Session( + self.key, + client_secret=self.secret, + callback_uri="%s/oauth1/" % shared.config.get('site', 'url') + ) + r = client.fetch_request_token(self.request_token_url) + logging.debug('setting token to %s', r.get('oauth_token')) + self.t = r.get('oauth_token') + logging.debug('updating secret to %s', r.get('oauth_token_secret')) + self.tokendb.update_token( + self.t, + oauth_token_secret=r.get('oauth_token_secret') + ) + self.tokendb.set_service( + self.service, + self.t + ) + + existing = self.tokendb.get_token(self.t) + verified = existing.get('verifier', None) + while not verified: + logging.debug('verifier missing for %s', self.t) + self.auth_url(existing) + self.tokendb.refresh() + existing = self.tokendb.get_token(self.t) + verified = existing.get('verifier', None) + + def auth_url(self, existing): + t = self.tokendb.get_token(self.t) + client = OAuth1Session( + self.key, + client_secret=self.secret, + resource_owner_key=self.t, + resource_owner_secret=t.get('oauth_token_secret'), + callback_uri="%s/oauth1/" % shared.config.get('site', 'url') + ) + input('Visit: %s and press any key after' % ( + client.authorization_url(self.authorize_url) + )) + + def request_access_token(self): + try: + t = self.tokendb.get_token(self.t) + client = OAuth1Session( + self.key, + client_secret=self.secret, + callback_uri="%s/oauth1/" % shared.config.get('site', 'url'), + resource_owner_key=self.t, + resource_owner_secret=t.get('oauth_token_secret'), + verifier=t.get('verifier') + ) + r = client.fetch_access_token(self.access_token_url) + self.tokendb.update_token( + self.t, + access_token=r.get('oauth_token'), + access_token_secret=r.get('oauth_token_secret') + ) + except oauth1_session.TokenRequestDenied as e: + logging.error('getting access token was denied, clearing former oauth tokens and re-running everyting') + self.tokendb.clear_service(self.service) + self.oauth_init() + + + def request(self, url, params): + t = self.tokendb.get_token(self.t) + client = OAuth1Session( + self.key, + client_secret=self.secret, + resource_owner_key=t.get('access_token'), + resource_owner_secret=t.get('access_token_secret') + ) + return client.get(url, params=params) + + + + + + +class FlickrOauth(Oauth1Flow): + request_token_url = 'https://www.flickr.com/services/oauth/request_token' + access_token_url = 'https://www.flickr.com/services/oauth/access_token' + authorize_url = 'https://www.flickr.com/services/oauth/authorize' + + def __init__(self): + super(FlickrOauth, self).__init__('flickr') + +class TumblrOauth(Oauth1Flow): + request_token_url = 'https://www.tumblr.com/oauth/request_token' + access_token_url = 'https://www.tumblr.com/oauth/access_token' + authorize_url = 'https://www.tumblr.com/oauth/authorize' + + def __init__(self): + super(TumblrOauth, self).__init__('tumblr') + +#class WPOauth(Oauth1Flow): + #request_token_url = 'https://public-api.wordpress.com/oauth2/token' + #access_token_url = 'https://public-api.wordpress.com/oauth2/authenticate' + #authorize_url = 'https://public-api.wordpress.com/oauth2/authorize' + + #def __init__(self): + #super(WPOauth, self).__init__('wordpress.com') + + +if __name__ == '__main__': + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + app = Sanic() + + @app.route("/oauth1", methods=["GET"]) + async def oa(request): + token = request.args.get('oauth_token') + verifier = request.args.get('oauth_verifier') + tokendb = TokenDB() + tokendb.update_token( + token, + verifier=verifier + ) + return sanic.response.text( + "OK", + status=200 + ) + + #@app.route("/oauth2", methods=["GET"]) + #async def oa2(request): + ##token = request.args.get('oauth_token') + ##verifier = request.args.get('oauth_verifier') + ##tokendb = TokenDB() + ##tokendb.update_token( + ##token, + ##verifier=verifier + ##) + #return sanic.response.text( + #json.dumps(request.args), + #status=200 + #) + + app.run(host="127.0.0.1", port=8006, debug=True) diff --git a/pesos.py b/pesos.py index bff3338..9f61205 100644 --- a/pesos.py +++ b/pesos.py @@ -11,25 +11,20 @@ import shutil import arrow import bs4 from slugify import slugify +import oauth from pprint import pprint """ TODO -- following from: - - tumblr - - deviantart - - flickr +- followings? + +- favs from: - wordpress.com - twitter - - 500px - """ - - - class Bookmark(object): def __init__(self, title, url, fname=None): self.fm = frontmatter.loads('') @@ -122,8 +117,8 @@ class Fav(object): self.imgname ) - def saveimg(self, url): - target = self.imgtarget + def saveimg(self, url, target=None): + target = target or self.imgtarget if os.path.isfile(target): logging.error("%s already exists, refusing to overwrite", target) return @@ -259,6 +254,97 @@ class FivehpxFav(Fav): content = '' self.fm.content = content + +class TumblrFav(Fav): + def __init__(self, like): + super(TumblrFav, self).__init__() + self.like = like + self.blogname = like.get('blog_name') + self.postid = like.get('id') + self.fname = "tumblr-%s-%s.md" % (self.blogname, self.postid) + self.url = like.get('post_url') + self.images = [] + + def run(self): + icntr = 0 + for p in self.like.get('photos', []): + i = p.get('original_size').get('url') + logging.debug('parsing image %s', i) + n = self.fname.replace('.md', '_%d.jpg' % icntr) + self.images.append(n) + nt = os.path.join( + shared.config.get('source', 'filesdir'), + n + ) + self.saveimg(i, nt) + icntr = icntr + 1 + + self.arrow = arrow.get( + self.like.get('liked_timestamp', + self.like.get('date', + arrow.utcnow().timestamp + ) + ) + ) + + self.fm.content = self.like.get('caption', '') + + title = self.like.get('summary', '').strip() + if not len(title): + title = self.like.get('slug', '').strip() + if not len(title): + title = shared.slugfname(self.like.get('post_url')) + + self.fm.metadata = { + 'published': self.arrow.format(shared.ARROWISO), + 'title': title, + 'favorite-of': self.url, + 'tumblr_tags': self.like.get('tags'), + 'author': { + 'name': self.like.get('blog_name'), + 'url': 'http://%s.tumblr.com' % self.like.get('blog_name') + }, + 'images': self.images + } + + +class DAFav(Fav): + def __init__(self, fav): + super(DAFav, self).__init__() + self.fav = fav + self.deviationid = fav.get('deviationid') + self.url = fav.get('url') + self.title = fav.get('title', False) or self.deviationid + self.author = self.fav.get('author').get('username') + self.fname = "deviantart-%s-by-%s.md" % ( + slugify(self.title), slugify(self.author) + ) + self.image = fav.get('content', {}).get('src') + + def run(self): + self.saveimg(self.image) + + self.arrow = arrow.get( + self.fav.get('published_time', arrow.utcnow().timestamp) + ) + + self.fm.metadata = { + 'published': self.arrow.format(shared.ARROWISO), + 'title': '%s' % self.title, + 'favorite-of': self.url, + 'da_tags': [t.get('tag_name') for t in self.fav.get('meta', {}).get('tags', [])], + 'author': { + 'name': self.author, + 'url': 'https://%s.deviantart.com' % (self.author), + }, + 'image': self.imgname + } + + content = self.fav.get('meta', {}).get('description', '') + content = shared.Pandoc(False).convert(content) + self.fm.content = content + + class Favs(object): def __init__(self, confgroup): self.confgroup = confgroup @@ -327,6 +413,287 @@ class FivehpxFavs(Favs): fav.write() +class TumblrFavs(Favs): + def __init__(self): + super(TumblrFavs, self).__init__('tumblr') + self.oauth = oauth.TumblrOauth() + self.params = { + 'after': self.lastpulled + } + self.likes = [] + + def getpaged(self, offset): + r = self.oauth.request( + self.url, + params={'offset': offset} + ) + return json.loads(r.text) + + def run(self): + r = self.oauth.request( + self.url, + params=self.params + ) + + js = json.loads(r.text) + total = int(js.get('response', {}).get('liked_count', 20)) + print('total: %d' % total) + offset = 20 + cntr = total - offset + likes = js.get('response', {}).get('liked_posts', []) + while cntr > 0: + paged = self.getpaged(offset) + likes = likes + paged.get('response', {}).get('liked_posts', []) + offset = offset + 20 + cntr = total - offset + + self.likes = likes + for like in self.likes: + fav = TumblrFav(like) + if not fav.exists: + fav.run() + fav.write() + + +class DAFavs(Favs): + def __init__(self): + from pprint import pprint + super(DAFavs, self).__init__('deviantart') + self.oauth = oauth.DAOauth() + self.params = { + 'limit': 24, + 'mature_content': 'true', + 'username': shared.config.get('deviantart', 'username') + } + self.likes = [] + + def getpaged(self, offset): + self.params.update({'offset': offset}) + r = self.oauth.request( + self.url, + self.params + ) + return json.loads(r.text) + + def getsinglemeta(self, daid): + r = self.oauth.request( + 'https://www.deviantart.com/api/v1/oauth2/deviation/metadata', + params={ + 'deviationids[]': daid, + 'ext_submission': False, + 'ext_camera': False, + 'ext_stats': False, + 'ext_collection': False, + 'mature_content': True, + } + ) + meta = {} + try: + meta = json.loads(r.text) + return meta.get('metadata', []).pop() + except: + return meta + + def has_more(self, q): + if 'True' == q or 'true' == q: + return True + return False + + def run(self): + r = self.oauth.request( + self.url, + self.params + ) + + js = json.loads(r.text) + has_more = js.get('has_more') + offset = js.get('next_offset') + favs = js.get('results', []) + while True == has_more: + logging.debug('iterating over DA results with offset %d', offset) + paged = self.getpaged(offset) + favs = favs + paged.get('results', []) + has_more = paged.get('has_more') + n = paged.get('next_offset') + if n: + offset = offset + n + + self.favs = favs + for fav in self.favs: + f = DAFav(fav) + if f.exists: + continue + + f.fav.update({'meta': self.getsinglemeta(fav.get('deviationid'))}) + f.run() + f.write() + + + +#class WPFavs(Favs): + #def __init__(self): + #from pprint import pprint + #super(DAFavs, self).__init__('wordpress') + #self.oauth = oauth.DAOauth() + #self.params = { + #'limit': 24, + #'mature_content': 'true', + #'username': shared.config.get('deviantart', 'username') + #} + #self.likes = [] + + #def getpaged(self, offset): + #self.params.update({'offset': offset}) + #r = self.oauth.request( + #self.url, + #self.params + #) + #return json.loads(r.text) + + #def getsinglemeta(self, daid): + #r = self.oauth.request( + #'https://www.deviantart.com/api/v1/oauth2/deviation/metadata', + #params={ + #'deviationids[]': daid, + #'ext_submission': False, + #'ext_camera': False, + #'ext_stats': False, + #'ext_collection': False, + #'mature_content': True, + #} + #) + #meta = {} + #try: + #meta = json.loads(r.text) + #return meta.get('metadata', []).pop() + #except: + #return meta + + #def has_more(self, q): + #if 'True' == q or 'true' == q: + #return True + #return False + + #def run(self): + #r = self.oauth.request( + #self.url, + #self.params + #) + + #js = json.loads(r.text) + #has_more = js.get('has_more') + #offset = js.get('next_offset') + #favs = js.get('results', []) + #while True == has_more: + #logging.debug('iterating over DA results with offset %d', offset) + #paged = self.getpaged(offset) + #favs = favs + paged.get('results', []) + #has_more = paged.get('has_more') + #n = paged.get('next_offset') + #if n: + #offset = offset + n + + #self.favs = favs + #for fav in self.favs: + #f = DAFav(fav) + #if f.exists: + #continue + + #f.fav.update({'meta': self.getsinglemeta(fav.get('deviationid'))}) + #f.run() + #f.write() + +#class Following(object): + #def __init__(self, confgroup): + #self.confgroup = confgroup + #self.url = shared.config.get(confgroup, 'following_api') + #self.followings = [] + + +#class FlickrFollowing(Following): + #def __init__(self): + #super(FlickrFollowing, self).__init__('flickr') + #self.oauth = oauth.FlickrOauth() + + + #def run(self): + #r = self.oauth.request(self.url, params={ + #'method': 'flickr.contacts.getList', + #'format': 'json', + #'nojsoncallback': 1, + #'api_key': shared.config.get(self.confgroup, 'api_key') + #}) + + #try: + #contacts = json.loads(r.text) + #for c in contacts.get('contacts', {}).get('contact', []): + #self.followings.append({ + #'url': "https://www.flickr.com/people/%s/" % c.get('nsid'), + #'name': c.get('realname'), + #'username': c.get('username'), + #'userid': c.get('nsid') + #}) + + #except Exception as e: + #logging.error('getting following from flickr failed: %s', e) + + +#class TumblrFollowing(Following): + #def __init__(self): + #super(TumblrFollowing, self).__init__('tumblr') + #self.oauth = oauth.FlickrOauth() + + + #def run(self): + #r = self.oauth.request(self.url, params={ + #'method': 'flickr.contacts.getList', + #'format': 'json', + #'nojsoncallback': 1, + #'api_key': shared.config.get(self.confgroup, 'api_key') + #}) + + #try: + #contacts = json.loads(r.text) + #for c in contacts.get('contacts', {}).get('contact', []): + #self.followings.append({ + #'url': "https://www.flickr.com/people/%s/" % c.get('nsid'), + #'name': c.get('realname'), + #'username': c.get('username'), + #'userid': c.get('nsid') + #}) + + #except Exception as e: + #logging.error('getting following from flickr failed: %s', e) + + +#class FlickrFollowing(object): + #def __init__(self): + #super(FlickrFollowing, self).__init__('flickr') + #self.params = { + #'consumer_key': shared.config.get('500px', 'api_key'), + #'rpp': 100, + #'image_size': 4, + #'include_tags': 1, + #'include_geo': 1 + #} + + #def run(self): + #r = requests.get(self.url,params=self.params) + #js = json.loads(r.text) + #for photo in js.get('photos', []): + #fav = FivehpxFav(photo) + #if not fav.exists: + #fav.run() + #fav.write() + + + #def run(self): + + + +#https://api.flickr.com/services/rest/?method=flickr.contacts.getList&api_key=27d8a5bf7dabf882ff1c710894041f64&format=json&nojsoncallback=1&auth_token=72157682938907284-9c5f21debeec9833&api_sig=8ac87b900f44debea06a3765ed223680 + + #class Following(object): #def __init__(self, confgroup): #self.confgroup = confgroup @@ -357,7 +724,7 @@ if __name__ == '__main__': logging.root.removeHandler(logging.root.handlers[-1]) logging.basicConfig( - level=20, + level=10, format='%(asctime)s - %(levelname)s - %(message)s' ) @@ -370,5 +737,12 @@ if __name__ == '__main__': fivehpx = FivehpxFavs() fivehpx.run() + tumblr = TumblrFavs() + tumblr.run() + + da = DAFavs() + da.run() + + #flickrfollow = FlickrFollowing() #flickrfollow.run() diff --git a/shared.py b/shared.py index 6cea144..dcdff34 100644 --- a/shared.py +++ b/shared.py @@ -107,66 +107,6 @@ config = configparser.ConfigParser( config.read('config.ini') config = __expandconfig(config) - -class TokenDB(object): - def __init__(self): - self.db = os.path.abspath(os.path.join( - config.get('common', 'basedir'), - 'tokens.json' - )) - self.tokens = {} - self.refresh() - - def refresh(self): - if os.path.isfile(self.db): - with open(self.db, 'rt') as f: - self.tokens = json.loads(f.read()) - - def save(self): - with open(self.db, 'wt') as f: - f.write( - json.dumps( - self.tokens, indent=4, sort_keys=True - ) - ) - self.refresh() - - def get_token(self, token): - return self.tokens.get(token, None) - - def get_service(self, service): - s = self.tokens.get(service, None) - if s: - s = self.get_token(s) - return s - - def set_service(self, service, token): - self.tokens.update({ - service: token - }) - #self.save() - - def set_token(self, token, secret): - self.tokens.update({ - token: { - 'oauth_token': token, - 'oauth_token_secret': secret - } - }) - #self.save() - - def set_verifier(self, token, verifier): - t = self.tokens.get(token) - t.update({ - 'verifier': verifier - }) - self.tokens.update({ - token: t - }) - #self.save() - -tokendb = TokenDB() - class CMDLine(object): def __init__(self, executable): self.executable = self._which(executable)