diff --git a/archive.py b/archive.py index c1d40e1..0f9fc23 100644 --- a/archive.py +++ b/archive.py @@ -17,8 +17,10 @@ from oauthlib.oauth2 import BackendApplicationClient import shared + class LastFM(object): url = 'http://ws.audioscrobbler.com/2.0/' + def __init__(self): self.service = 'lastfm' self.target = shared.config.get("api_%s" % self.service, 'logfile') @@ -59,7 +61,7 @@ class LastFM(object): return parsed def run(self): - r = requests.get(self.url,params=self.params) + r = requests.get(self.url, params=self.params) js = json.loads(r.text) js = js.get('recenttracks', {}) unordered = js.get('track', []) @@ -129,7 +131,7 @@ class FlickrFavs(Favs): 'method': 'flickr.favorites.getList', 'api_key': shared.config.get('api_flickr', 'api_key'), 'user_id': self.uid, - 'extras': ','.join([ + 'extras': ','.join([ 'description', 'geo', 'tags', @@ -142,7 +144,7 @@ class FlickrFavs(Favs): 'url_c', 'url_z', ]), - 'per_page': 500, # maximim + 'per_page': 500, # maximim 'format': 'json', 'nojsoncallback': '1', 'min_fave_date': self.lastpulled @@ -163,7 +165,6 @@ class FlickrFavs(Favs): parsed = json.loads(r.text) self.uid = parsed.get('user', {}).get('id') - def getpaged(self, offset): logging.info('requesting page #%d of paginated results', offset) self.params.update({ @@ -177,7 +178,7 @@ class FlickrFavs(Favs): return parsed.get('photos', {}).get('photo', []) def run(self): - r = requests.get(self.url,params=self.params) + r = requests.get(self.url, params=self.params) js = json.loads(r.text) js = js.get('photos', {}) @@ -197,14 +198,15 @@ class FlickrFavs(Favs): fav = FlickrFav(photo) if not fav.exists: fav.run() - #fav.fix_extension() + # fav.fix_extension() + class FivehpxFavs(Favs): def __init__(self): super(FivehpxFavs, self).__init__('500px') self.params = { 'consumer_key': shared.config.get('api_500px', 'api_key'), - 'rpp': 100, # maximum + 'rpp': 100, # maximum 'image_size': 4, 'include_tags': 1, 'include_geo': 1, @@ -227,14 +229,13 @@ class FivehpxFavs(Favs): r = self.oauth.request( 'https://api.500px.com/v1/users/%s/galleries' % (self.uid), params={ - 'kinds': 5 # see https://github.com/500px/api-documentation/blob/master/basics/formats_and_terms.md#gallery-kinds + 'kinds': 5 # see https://github.com/500px/api-documentation/blob/master/basics/formats_and_terms.md#gallery-kinds } ) js = json.loads(r.text) g = js.get('galleries', []).pop() self.galid = g.get('id') - @property def url(self): return 'https://api.500px.com/v1/users/%s/galleries/%s/items' % ( @@ -258,7 +259,7 @@ class FivehpxFavs(Favs): self.get_uid() self.get_favgalid() - r = requests.get(self.url,params=self.params) + r = requests.get(self.url, params=self.params) js = json.loads(r.text) photos = js.get('photos') @@ -276,7 +277,7 @@ class FivehpxFavs(Favs): fav = FivehpxFav(photo) if not fav.exists: fav.run() - #fav.fix_extension() + # fav.fix_extension() class TumblrFavs(Favs): @@ -330,7 +331,7 @@ class DAFavs(Favs): self.likes = [] self.galid = None self.params = { - 'limit': 24, # this is the max as far as I can tell + 'limit': 24, # this is the max as far as I can tell 'mature_content': 'true', 'username': self.username } @@ -353,8 +354,8 @@ class DAFavs(Favs): @property def url(self): - return 'https://www.deviantart.com/api/v1/oauth2/collections/%s' % (self.galid) - + return 'https://www.deviantart.com/api/v1/oauth2/collections/%s' % ( + self.galid) def getpaged(self, offset): self.params.update({'offset': offset}) @@ -381,7 +382,7 @@ class DAFavs(Favs): try: meta = json.loads(r.text) return meta.get('metadata', []).pop() - except: + except BaseException: return meta def has_more(self, q): @@ -421,9 +422,10 @@ class DAFavs(Favs): for fav in self.favs: f = DAFav(fav) if not f.exists: - f.fav.update({'meta': self.getsinglemeta(fav.get('deviationid'))}) + f.fav.update( + {'meta': self.getsinglemeta(fav.get('deviationid'))}) f.run() - #f.fix_extension() + # f.fix_extension() class ImgFav(object): @@ -475,7 +477,7 @@ class ImgFav(object): return logging.info('populating EXIF data of %s' % self.target) - tags = list(set(self.meta.get('tags',[]))) + tags = list(set(self.meta.get('tags', []))) dt = self.meta.get('dt').to('utc') geo_lat = False @@ -510,13 +512,13 @@ class ImgFav(object): '-XMP:ReleaseDate=%s' % dt.format('YYYY:MM:DD HH:mm:ss'), '-XMP:Headline=%s' % self.meta.get('title'), '-XMP:Description=%s' % self.content, - ]; + ] for t in tags: params.append('-XMP:HierarchicalSubject+=%s' % t) params.append('-XMP:Subject+=%s' % t) if geo_lat and geo_lon: - geo_lat = round(float(geo_lat),6) - geo_lon = round(float(geo_lon),6) + geo_lat = round(float(geo_lat), 6) + geo_lon = round(float(geo_lon), 6) if geo_lat < 0: GPSLatitudeRef = 'S' @@ -532,7 +534,7 @@ class ImgFav(object): params.append('-GPSLatitude=%s' % abs(geo_lat)) params.append('-GPSLongitudeRef=%s' % GPSLongitudeRef) params.append('-GPSLatitudeRef=%s' % GPSLatitudeRef) - params.append(self.target); + params.append(self.target) p = subprocess.Popen( params, @@ -553,7 +555,8 @@ class FlickrFav(ImgFav): self.photo = photo self.ownerid = photo.get('owner') self.photoid = photo.get('id') - self.url = "https://www.flickr.com/photos/%s/%s" % (self.ownerid, self.photoid) + self.url = "https://www.flickr.com/photos/%s/%s" % ( + self.ownerid, self.photoid) self.target = os.path.join( shared.config.get('archive', 'favorite'), "flickr-%s-%s.jpg" % (self.ownerid, self.photoid) @@ -567,20 +570,11 @@ class FlickrFav(ImgFav): # the bigger the better, see # https://www.flickr.com/services/api/misc.urls.html - img = self.photo.get( - 'url_o', - self.photo.get('url_k', - self.photo.get('url_h', - self.photo.get('url_b', - self.photo.get('url_c', - self.photo.get('url_z', - False - ) - ) - ) - ) - ) - ) + img = False + for x in ['url_o', 'url_k', 'url_h', 'url_b', 'url_c', 'url_z']: + if x in self.photo: + img = self.photo.get(x) + break if not img: logging.error("image url was empty for %s, skipping fav", self.url) @@ -590,8 +584,8 @@ class FlickrFav(ImgFav): self.meta = { 'dt': arrow.get( self.photo.get('date_faved', - arrow.utcnow().timestamp - ) + arrow.utcnow().timestamp + ) ), 'title': '%s' % shared.Pandoc('plain').convert( self.photo.get('title', '') @@ -617,6 +611,7 @@ class FlickrFav(ImgFav): self.fix_extension() self.write_exif() + class FivehpxFav(ImgFav): def __init__(self, photo): self.photo = photo @@ -639,8 +634,8 @@ class FivehpxFav(ImgFav): self.meta = { 'dt': arrow.get( self.photo.get('created_at', - arrow.utcnow().timestamp - ) + arrow.utcnow().timestamp + ) ), 'title': '%s' % shared.Pandoc('plain').convert( self.photo.get('name', '') @@ -663,6 +658,7 @@ class FivehpxFav(ImgFav): self.fix_extension() self.write_exif() + class DAFav(ImgFav): def __init__(self, fav): self.fav = fav @@ -690,7 +686,9 @@ class DAFav(ImgFav): def run(self): if not self.imgurl: - logging.error('imgurl is empty for deviantart %s', self.deviationid) + logging.error( + 'imgurl is empty for deviantart %s', + self.deviationid) return self.pull_image() @@ -698,8 +696,8 @@ class DAFav(ImgFav): self.meta = { 'dt': arrow.get( self.fav.get('published_time', - arrow.utcnow().timestamp - ) + arrow.utcnow().timestamp + ) ), 'title': '%s' % shared.Pandoc('plain').convert(self.title).rstrip(), 'favorite-of': self.url, @@ -745,10 +743,10 @@ class TumblrFav(object): meta = { 'dt': arrow.get( self.like.get('liked_timestamp', - self.like.get('date', - arrow.utcnow().timestamp - ) - ) + self.like.get('date', + arrow.utcnow().timestamp + ) + ) ), 'title': title, 'favorite-of': self.url, @@ -823,7 +821,8 @@ class Oauth1Flow(object): self.request_oauth_token() t = self.tokendb.get_token(self.t) - if not t.get('access_token', None) or not t.get('access_token_secret', None): + if not t.get('access_token', None) or not t.get( + 'access_token_secret', None): self.request_access_token() def request_oauth_token(self): @@ -885,11 +884,11 @@ class Oauth1Flow(object): access_token_secret=r.get('oauth_token_secret') ) except oauth1_session.TokenRequestDenied as e: - logging.error('getting access token was denied, clearing former oauth tokens and re-running everyting') + logging.error( + 'getting access token was denied, clearing former oauth tokens and re-running everyting') self.tokendb.clear_service(self.service) self.oauth_init() - def request(self, url, params): t = self.tokendb.get_token(self.t) client = OAuth1Session( diff --git a/nasg.py b/nasg.py index 1b09b2a..2fee0f2 100644 --- a/nasg.py +++ b/nasg.py @@ -64,6 +64,7 @@ class MagicPHP(object): class NoDupeContainer(object): ''' Base class to hold keys => data dicts with errors on dupes ''' + def __init__(self): self.data = {} self.default = None @@ -110,7 +111,7 @@ class NoDupeContainer(object): def __next__(self): try: r = self.data.next() - except: + except BaseException: raise StopIteration() return r @@ -123,6 +124,7 @@ class NoDupeContainer(object): class FContainer(NoDupeContainer): """ This is a container that holds a lists of files based on Container so it errors on duplicate slugs and is popolated with recorsive glob """ + def __init__(self, dirs, extensions=['*']): super().__init__() files = [] @@ -143,6 +145,7 @@ class Content(FContainer): """ This is a container that holds markdown files that are parsed when the container is populated on the fly; based on FContainer which is a Container """ + def __init__(self): dirs = [os.path.join(shared.config.get('dirs', 'content'), "**")] extensions = ['md', 'jpg'] @@ -227,7 +230,7 @@ class Category(NoDupeContainer): page = 1 while page <= pages: # list relevant post templates - start = int((page-1) * pagination) + start = int((page - 1) * pagination) end = int(start + pagination) posttmpls = [ self.data[k].tmplvars @@ -285,7 +288,7 @@ class Category(NoDupeContainer): r = shared.j2.get_template(tmplfile).render(tmplvars) self.write_html(o, r) # inc. page counter - page = page+1 + page = page + 1 class Singular(object): @@ -406,7 +409,7 @@ class Singular(object): r = {} for mtime, c in self.comments: if c.type == 'webmention': - r.update({mtime:c.tmplvars}) + r.update({mtime: c.tmplvars}) return sorted(r.items()) @property @@ -417,7 +420,7 @@ class Singular(object): continue if c.type not in r: r[c.type] = {} - r[c.type].update({mtime:c.tmplvars}) + r[c.type].update({mtime: c.tmplvars}) for icon, comments in r.items(): r[icon] = sorted(comments.items()) @@ -492,7 +495,7 @@ class Singular(object): self.meta.get('title', ''), self.content ])) - except: + except BaseException: pass return lang @@ -682,10 +685,10 @@ class WebImage(object): if self.is_downsizeable: try: src = [ - e for e in self.sizes \ + e for e in self.sizes if e[0] == shared.config.getint('photo', 'default') ][0][1]['url'] - except: + except BaseException: pass return src @@ -743,15 +746,15 @@ class WebImage(object): return exif mapping = { - 'camera': ['Model'], - 'aperture': ['FNumber', 'Aperture'], - 'shutter_speed': ['ExposureTime'], + 'camera': ['Model'], + 'aperture': ['FNumber', 'Aperture'], + 'shutter_speed': ['ExposureTime'], #'focallength': ['FocalLengthIn35mmFormat', 'FocalLength'], - 'focallength': ['FocalLength'], - 'iso': ['ISO'], - 'lens': ['LensID', 'LensSpec', 'Lens'], - 'geo_latitude': ['GPSLatitude'], - 'geo_longitude': ['GPSLongitude'], + 'focallength': ['FocalLength'], + 'iso': ['ISO'], + 'lens': ['LensID', 'LensSpec', 'Lens'], + 'geo_latitude': ['GPSLatitude'], + 'geo_longitude': ['GPSLongitude'], } for ekey, candidates in mapping.items(): @@ -891,7 +894,7 @@ class WebImage(object): """ Calculate intermediate resize dimension and return a tuple of width, height """ size = int(size) if (width > height and not crop) \ - or (width < height and crop): + or (width < height and crop): w = size h = int(float(size / width) * height) else: @@ -953,7 +956,8 @@ class WebImage(object): if not self.is_downsizeable: return self._copy() - if not self.needs_downsize and not shared.config.getboolean('params', 'regenerate'): + if not self.needs_downsize and not shared.config.getboolean( + 'params', 'regenerate'): return build_files = os.path.join( @@ -1309,7 +1313,7 @@ def build(): # render front if not collector_front.is_uptodate or \ - shared.config.getboolean('params', 'force'): + shared.config.getboolean('params', 'force'): worker.append(collector_front.render()) # render categories @@ -1335,5 +1339,6 @@ def build(): logging.debug("copying static file %s to %s", s, d) shutil.copy2(s, d) + if __name__ == '__main__': build() diff --git a/shared.py b/shared.py index a7abaad..ddebe41 100644 --- a/shared.py +++ b/shared.py @@ -10,6 +10,7 @@ import requests from slugify import slugify import jinja2 + class CMDLine(object): def __init__(self, executable): self.executable = self._which(executable) @@ -80,6 +81,7 @@ class XRay(CMDLine): return json.loads(stdout.decode('utf-8').strip()) + class Pandoc(CMDLine): """ Pandoc command line call with piped in- and output """ @@ -233,7 +235,8 @@ class ExifTool(CMDLine): exif = json.loads(stdout.decode('utf-8').strip()).pop() if 'ReleaseDate' in exif and 'ReleaseTime' in exif: - exif['DateTimeRelease'] = "%s %s" % (exif.get('ReleaseDate'), exif.get('ReleaseTime')[:8]) + exif['DateTimeRelease'] = "%s %s" % ( + exif.get('ReleaseDate'), exif.get('ReleaseTime')[:8]) del(exif['ReleaseDate']) del(exif['ReleaseTime']) @@ -242,6 +245,7 @@ class ExifTool(CMDLine): return exif + class BaseDB(object): def __init__(self, fpath): self.db = sqlite3.connect(fpath) @@ -260,24 +264,23 @@ class BaseDB(object): cursor.execute('PRAGMA auto_vacuum;') self.db.close() -#class TokenDBng(BaseDB): - #def __init__(self): +# class TokenDBng(BaseDB): + # def __init__(self): #self.fpath = config.get('var', 'tokendb') - #super().__init__(self.fpath) + # super().__init__(self.fpath) #cursor = self.db.cursor() - #cursor.execute(''' - #CREATE TABLE IF NOT EXISTS `tokens` ( - #`service` TEXT PRIMARY KEY NOT NULL UNIQUE, - #`timestamp` TIMESTAMP NOT NULL DEFAULT (strftime('%s', 'now')) - #`oauth_token` TEXT NOT NULL, - #`oauth_token_secret` TEXT NOT NULL, - #`access_token` TEXT NOT NULL, - #`access_token_secret` TEXT NOT NULL, - #`verifier` TEXT NOT NULL - #); + # cursor.execute(''' + # CREATE TABLE IF NOT EXISTS `tokens` ( + #`service` TEXT PRIMARY KEY NOT NULL UNIQUE, + #`timestamp` TIMESTAMP NOT NULL DEFAULT (strftime('%s', 'now')) + #`oauth_token` TEXT NOT NULL, + #`oauth_token_secret` TEXT NOT NULL, + #`access_token` TEXT NOT NULL, + #`access_token_secret` TEXT NOT NULL, + #`verifier` TEXT NOT NULL + #); #''') - #self.db.commit() - + # self.db.commit() # TODO class SearchDBng(object): @@ -315,11 +318,11 @@ class TokenDB(object): self.save() def update_token(self, - token, - oauth_token_secret=None, - access_token=None, - access_token_secret=None, - verifier=None): + token, + oauth_token_secret=None, + access_token=None, + access_token_secret=None, + verifier=None): t = self.tokens.get(token, {}) if oauth_token_secret: @@ -355,8 +358,10 @@ class TokenDB(object): del(self.tokens[service]) self.save() + class SearchDB(BaseDB): tmplfile = 'Search.html' + def __init__(self): self.fpath = "%s" % config.get('var', 'searchdb') super().__init__(self.fpath) @@ -402,8 +407,8 @@ class SearchDB(BaseDB): cursor.execute('''SELECT mtime FROM data WHERE id = ? AND mtime = ?''', - (fname,mtime) - ) + (fname, mtime) + ) rows = cursor.fetchall() if len(rows): @@ -435,7 +440,6 @@ class SearchDB(BaseDB): if category not in ret: ret.update({category: {}}) - maybe_fpath = os.path.join( config.get('dirs', 'content'), category, @@ -452,7 +456,6 @@ class SearchDB(BaseDB): }) return ret - def cli(self, query): results = self.search_by_query(query) for c, items in sorted(results.items()): @@ -509,7 +512,12 @@ class WebmentionQueue(BaseDB): logging.debug('getting queued webmentions for %s', fname) ret = [] cursor = self.db.cursor() - cursor.execute('''SELECT * FROM queue WHERE target LIKE ? AND status = 0''', ('%'+fname+'%',)) + cursor.execute( + '''SELECT * FROM queue WHERE target LIKE ? AND status = 0''', + ('%' + + fname + + '%', + )) rows = cursor.fetchall() for r in rows: ret.append({ @@ -541,6 +549,7 @@ def __expandconfig(): c.set(s, o, os.path.expanduser(curr)) return c + def baseN(num, b=36, numerals="0123456789abcdefghijklmnopqrstuvwxyz"): """ Used to create short, lowercase slug for a number (an epoch) passed """ num = int(num) @@ -560,6 +569,7 @@ def slugfname(url): lower=True )[:200] + def __setup_sitevars(): SiteVars = {} section = 'site' @@ -584,7 +594,6 @@ def __setup_sitevars(): for o in config.options(section): SiteVars[section].update({o: config.get(section, o)}) - # push the whole thing into cache return SiteVars @@ -604,7 +613,7 @@ def notify(msg): # fire and forget try: requests.post(url, data=data) - except: + except BaseException: pass