petermolnar/nasg
1
0
Fork 0
Code Issues Pull requests 5 Projects Releases Packages Wiki Activity Actions

autopep8

Browse source
This commit is contained in:
Peter Molnar 2017-11-10 16:04:05 +00:00
parent 07830313e4
commit aaaf494063
3 changed files with 111 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

101
archive.py
View file

@ -17,8 +17,10 @@ from oauthlib.oauth2 import BackendApplicationClient
import shared
class LastFM(object):
url = 'http://ws.audioscrobbler.com/2.0/'
def __init__(self):
self.service = 'lastfm'
self.target = shared.config.get("api_%s" % self.service, 'logfile')
@ -59,7 +61,7 @@ class LastFM(object):
return parsed
def run(self):
r = requests.get(self.url,params=self.params)
r = requests.get(self.url, params=self.params)
js = json.loads(r.text)
js = js.get('recenttracks', {})
unordered = js.get('track', [])
@ -129,7 +131,7 @@ class FlickrFavs(Favs):
'method': 'flickr.favorites.getList',
'api_key': shared.config.get('api_flickr', 'api_key'),
'user_id': self.uid,
'extras': ','.join([
'extras': ','.join([
'description',
'geo',
'tags',
@ -142,7 +144,7 @@ class FlickrFavs(Favs):
'url_c',
'url_z',
]),
'per_page': 500, # maximim
'per_page': 500, # maximim
'format': 'json',
'nojsoncallback': '1',
'min_fave_date': self.lastpulled
@ -163,7 +165,6 @@ class FlickrFavs(Favs):
parsed = json.loads(r.text)
self.uid = parsed.get('user', {}).get('id')
def getpaged(self, offset):
logging.info('requesting page #%d of paginated results', offset)
self.params.update({
@ -177,7 +178,7 @@ class FlickrFavs(Favs):
return parsed.get('photos', {}).get('photo', [])
def run(self):
r = requests.get(self.url,params=self.params)
r = requests.get(self.url, params=self.params)
js = json.loads(r.text)
js = js.get('photos', {})
@ -197,14 +198,15 @@ class FlickrFavs(Favs):
fav = FlickrFav(photo)
if not fav.exists:
fav.run()
#fav.fix_extension()
# fav.fix_extension()
class FivehpxFavs(Favs):
def __init__(self):
super(FivehpxFavs, self).__init__('500px')
self.params = {
'consumer_key': shared.config.get('api_500px', 'api_key'),
'rpp': 100, # maximum
'rpp': 100, # maximum
'image_size': 4,
'include_tags': 1,
'include_geo': 1,
@ -227,14 +229,13 @@ class FivehpxFavs(Favs):
r = self.oauth.request(
'https://api.500px.com/v1/users/%s/galleries' % (self.uid),
params={
'kinds': 5 # see https://github.com/500px/api-documentation/blob/master/basics/formats_and_terms.md#gallery-kinds
'kinds': 5 # see https://github.com/500px/api-documentation/blob/master/basics/formats_and_terms.md#gallery-kinds
}
)
js = json.loads(r.text)
g = js.get('galleries', []).pop()
self.galid = g.get('id')
@property
def url(self):
return 'https://api.500px.com/v1/users/%s/galleries/%s/items' % (
@ -258,7 +259,7 @@ class FivehpxFavs(Favs):
self.get_uid()
self.get_favgalid()
r = requests.get(self.url,params=self.params)
r = requests.get(self.url, params=self.params)
js = json.loads(r.text)
photos = js.get('photos')
@ -276,7 +277,7 @@ class FivehpxFavs(Favs):
fav = FivehpxFav(photo)
if not fav.exists:
fav.run()
#fav.fix_extension()
# fav.fix_extension()
class TumblrFavs(Favs):
@ -330,7 +331,7 @@ class DAFavs(Favs):
self.likes = []
self.galid = None
self.params = {
'limit': 24, # this is the max as far as I can tell
'limit': 24, # this is the max as far as I can tell
'mature_content': 'true',
'username': self.username
}
@ -353,8 +354,8 @@ class DAFavs(Favs):
@property
def url(self):
return 'https://www.deviantart.com/api/v1/oauth2/collections/%s' % (self.galid)
return 'https://www.deviantart.com/api/v1/oauth2/collections/%s' % (
self.galid)
def getpaged(self, offset):
self.params.update({'offset': offset})
@ -381,7 +382,7 @@ class DAFavs(Favs):
try:
meta = json.loads(r.text)
return meta.get('metadata', []).pop()
except:
except BaseException:
return meta
def has_more(self, q):
@ -421,9 +422,10 @@ class DAFavs(Favs):
for fav in self.favs:
f = DAFav(fav)
if not f.exists:
f.fav.update({'meta': self.getsinglemeta(fav.get('deviationid'))})
f.fav.update(
{'meta': self.getsinglemeta(fav.get('deviationid'))})
f.run()
#f.fix_extension()
# f.fix_extension()
class ImgFav(object):
@ -475,7 +477,7 @@ class ImgFav(object):
return
logging.info('populating EXIF data of %s' % self.target)
tags = list(set(self.meta.get('tags',[])))
tags = list(set(self.meta.get('tags', [])))
dt = self.meta.get('dt').to('utc')
geo_lat = False
@ -510,13 +512,13 @@ class ImgFav(object):
'-XMP:ReleaseDate=%s' % dt.format('YYYY:MM:DD HH:mm:ss'),
'-XMP:Headline=%s' % self.meta.get('title'),
'-XMP:Description=%s' % self.content,
];
]
for t in tags:
params.append('-XMP:HierarchicalSubject+=%s' % t)
params.append('-XMP:Subject+=%s' % t)
if geo_lat and geo_lon:
geo_lat = round(float(geo_lat),6)
geo_lon = round(float(geo_lon),6)
geo_lat = round(float(geo_lat), 6)
geo_lon = round(float(geo_lon), 6)
if geo_lat < 0:
GPSLatitudeRef = 'S'
@ -532,7 +534,7 @@ class ImgFav(object):
params.append('-GPSLatitude=%s' % abs(geo_lat))
params.append('-GPSLongitudeRef=%s' % GPSLongitudeRef)
params.append('-GPSLatitudeRef=%s' % GPSLatitudeRef)
params.append(self.target);
params.append(self.target)
p = subprocess.Popen(
params,
@ -553,7 +555,8 @@ class FlickrFav(ImgFav):
self.photo = photo
self.ownerid = photo.get('owner')
self.photoid = photo.get('id')
self.url = "https://www.flickr.com/photos/%s/%s" % (self.ownerid, self.photoid)
self.url = "https://www.flickr.com/photos/%s/%s" % (
self.ownerid, self.photoid)
self.target = os.path.join(
shared.config.get('archive', 'favorite'),
"flickr-%s-%s.jpg" % (self.ownerid, self.photoid)
@ -567,20 +570,11 @@ class FlickrFav(ImgFav):
# the bigger the better, see
# https://www.flickr.com/services/api/misc.urls.html
img = self.photo.get(
'url_o',
self.photo.get('url_k',
self.photo.get('url_h',
self.photo.get('url_b',
self.photo.get('url_c',
self.photo.get('url_z',
False
)
)
)
)
)
)
img = False
for x in ['url_o', 'url_k', 'url_h', 'url_b', 'url_c', 'url_z']:
if x in self.photo:
img = self.photo.get(x)
break
if not img:
logging.error("image url was empty for %s, skipping fav", self.url)
@ -590,8 +584,8 @@ class FlickrFav(ImgFav):
self.meta = {
'dt': arrow.get(
self.photo.get('date_faved',
arrow.utcnow().timestamp
)
arrow.utcnow().timestamp
)
),
'title': '%s' % shared.Pandoc('plain').convert(
self.photo.get('title', '')
@ -617,6 +611,7 @@ class FlickrFav(ImgFav):
self.fix_extension()
self.write_exif()
class FivehpxFav(ImgFav):
def __init__(self, photo):
self.photo = photo
@ -639,8 +634,8 @@ class FivehpxFav(ImgFav):
self.meta = {
'dt': arrow.get(
self.photo.get('created_at',
arrow.utcnow().timestamp
)
arrow.utcnow().timestamp
)
),
'title': '%s' % shared.Pandoc('plain').convert(
self.photo.get('name', '')
@ -663,6 +658,7 @@ class FivehpxFav(ImgFav):
self.fix_extension()
self.write_exif()
class DAFav(ImgFav):
def __init__(self, fav):
self.fav = fav
@ -690,7 +686,9 @@ class DAFav(ImgFav):
def run(self):
if not self.imgurl:
logging.error('imgurl is empty for deviantart %s', self.deviationid)
logging.error(
'imgurl is empty for deviantart %s',
self.deviationid)
return
self.pull_image()
@ -698,8 +696,8 @@ class DAFav(ImgFav):
self.meta = {
'dt': arrow.get(
self.fav.get('published_time',
arrow.utcnow().timestamp
)
arrow.utcnow().timestamp
)
),
'title': '%s' % shared.Pandoc('plain').convert(self.title).rstrip(),
'favorite-of': self.url,
@ -745,10 +743,10 @@ class TumblrFav(object):
meta = {
'dt': arrow.get(
self.like.get('liked_timestamp',
self.like.get('date',
arrow.utcnow().timestamp
)
)
self.like.get('date',
arrow.utcnow().timestamp
)
)
),
'title': title,
'favorite-of': self.url,
@ -823,7 +821,8 @@ class Oauth1Flow(object):
self.request_oauth_token()
t = self.tokendb.get_token(self.t)
if not t.get('access_token', None) or not t.get('access_token_secret', None):
if not t.get('access_token', None) or not t.get(
'access_token_secret', None):
self.request_access_token()
def request_oauth_token(self):
@ -885,11 +884,11 @@ class Oauth1Flow(object):
access_token_secret=r.get('oauth_token_secret')
)
except oauth1_session.TokenRequestDenied as e:
logging.error('getting access token was denied, clearing former oauth tokens and re-running everyting')
logging.error(
'getting access token was denied, clearing former oauth tokens and re-running everyting')
self.tokendb.clear_service(self.service)
self.oauth_init()
def request(self, url, params):
t = self.tokendb.get_token(self.t)
client = OAuth1Session(

43
nasg.py
View file

@ -64,6 +64,7 @@ class MagicPHP(object):
class NoDupeContainer(object):
''' Base class to hold keys => data dicts with errors on dupes '''
def __init__(self):
self.data = {}
self.default = None
@ -110,7 +111,7 @@ class NoDupeContainer(object):
def __next__(self):
try:
r = self.data.next()
except:
except BaseException:
raise StopIteration()
return r
@ -123,6 +124,7 @@ class NoDupeContainer(object):
class FContainer(NoDupeContainer):
""" This is a container that holds a lists of files based on Container so
it errors on duplicate slugs and is popolated with recorsive glob """
def __init__(self, dirs, extensions=['*']):
super().__init__()
files = []
@ -143,6 +145,7 @@ class Content(FContainer):
""" This is a container that holds markdown files that are parsed when the
container is populated on the fly; based on FContainer which is a Container
"""
def __init__(self):
dirs = [os.path.join(shared.config.get('dirs', 'content'), "**")]
extensions = ['md', 'jpg']
@ -227,7 +230,7 @@ class Category(NoDupeContainer):
page = 1
while page <= pages:
# list relevant post templates
start = int((page-1) * pagination)
start = int((page - 1) * pagination)
end = int(start + pagination)
posttmpls = [
self.data[k].tmplvars
@ -285,7 +288,7 @@ class Category(NoDupeContainer):
r = shared.j2.get_template(tmplfile).render(tmplvars)
self.write_html(o, r)
# inc. page counter
page = page+1
page = page + 1
class Singular(object):
@ -406,7 +409,7 @@ class Singular(object):
r = {}
for mtime, c in self.comments:
if c.type == 'webmention':
r.update({mtime:c.tmplvars})
r.update({mtime: c.tmplvars})
return sorted(r.items())
@property
@ -417,7 +420,7 @@ class Singular(object):
continue
if c.type not in r:
r[c.type] = {}
r[c.type].update({mtime:c.tmplvars})
r[c.type].update({mtime: c.tmplvars})
for icon, comments in r.items():
r[icon] = sorted(comments.items())
@ -492,7 +495,7 @@ class Singular(object):
self.meta.get('title', ''),
self.content
]))
except:
except BaseException:
pass
return lang
@ -682,10 +685,10 @@ class WebImage(object):
if self.is_downsizeable:
try:
src = [
e for e in self.sizes \
e for e in self.sizes
if e[0] == shared.config.getint('photo', 'default')
][0][1]['url']
except:
except BaseException:
pass
return src
@ -743,15 +746,15 @@ class WebImage(object):
return exif
mapping = {
'camera': ['Model'],
'aperture': ['FNumber', 'Aperture'],
'shutter_speed': ['ExposureTime'],
'camera': ['Model'],
'aperture': ['FNumber', 'Aperture'],
'shutter_speed': ['ExposureTime'],
#'focallength': ['FocalLengthIn35mmFormat', 'FocalLength'],
'focallength': ['FocalLength'],
'iso': ['ISO'],
'lens': ['LensID', 'LensSpec', 'Lens'],
'geo_latitude': ['GPSLatitude'],
'geo_longitude': ['GPSLongitude'],
'focallength': ['FocalLength'],
'iso': ['ISO'],
'lens': ['LensID', 'LensSpec', 'Lens'],
'geo_latitude': ['GPSLatitude'],
'geo_longitude': ['GPSLongitude'],
}
for ekey, candidates in mapping.items():
@ -891,7 +894,7 @@ class WebImage(object):
""" Calculate intermediate resize dimension and return a tuple of width, height """
size = int(size)
if (width > height and not crop) \
or (width < height and crop):
or (width < height and crop):
w = size
h = int(float(size / width) * height)
else:
@ -953,7 +956,8 @@ class WebImage(object):
if not self.is_downsizeable:
return self._copy()
if not self.needs_downsize and not shared.config.getboolean('params', 'regenerate'):
if not self.needs_downsize and not shared.config.getboolean(
'params', 'regenerate'):
return
build_files = os.path.join(
@ -1309,7 +1313,7 @@ def build():
# render front
if not collector_front.is_uptodate or \
shared.config.getboolean('params', 'force'):
shared.config.getboolean('params', 'force'):
worker.append(collector_front.render())
# render categories
@ -1335,5 +1339,6 @@ def build():
logging.debug("copying static file %s to %s", s, d)
shutil.copy2(s, d)
if __name__ == '__main__':
build()

65
shared.py
View file

@ -10,6 +10,7 @@ import requests
from slugify import slugify
import jinja2
class CMDLine(object):
def __init__(self, executable):
self.executable = self._which(executable)
@ -80,6 +81,7 @@ class XRay(CMDLine):
return json.loads(stdout.decode('utf-8').strip())
class Pandoc(CMDLine):
""" Pandoc command line call with piped in- and output """
@ -233,7 +235,8 @@ class ExifTool(CMDLine):
exif = json.loads(stdout.decode('utf-8').strip()).pop()
if 'ReleaseDate' in exif and 'ReleaseTime' in exif:
exif['DateTimeRelease'] = "%s %s" % (exif.get('ReleaseDate'), exif.get('ReleaseTime')[:8])
exif['DateTimeRelease'] = "%s %s" % (
exif.get('ReleaseDate'), exif.get('ReleaseTime')[:8])
del(exif['ReleaseDate'])
del(exif['ReleaseTime'])
@ -242,6 +245,7 @@ class ExifTool(CMDLine):
return exif
class BaseDB(object):
def __init__(self, fpath):
self.db = sqlite3.connect(fpath)
@ -260,24 +264,23 @@ class BaseDB(object):
cursor.execute('PRAGMA auto_vacuum;')
self.db.close()
#class TokenDBng(BaseDB):
#def __init__(self):
# class TokenDBng(BaseDB):
# def __init__(self):
#self.fpath = config.get('var', 'tokendb')
#super().__init__(self.fpath)
# super().__init__(self.fpath)
#cursor = self.db.cursor()
#cursor.execute('''
#CREATE TABLE IF NOT EXISTS `tokens` (
#`service` TEXT PRIMARY KEY NOT NULL UNIQUE,
#`timestamp` TIMESTAMP NOT NULL DEFAULT (strftime('%s', 'now'))
#`oauth_token` TEXT NOT NULL,
#`oauth_token_secret` TEXT NOT NULL,
#`access_token` TEXT NOT NULL,
#`access_token_secret` TEXT NOT NULL,
#`verifier` TEXT NOT NULL
#);
# cursor.execute('''
# CREATE TABLE IF NOT EXISTS `tokens` (
#`service` TEXT PRIMARY KEY NOT NULL UNIQUE,
#`timestamp` TIMESTAMP NOT NULL DEFAULT (strftime('%s', 'now'))
#`oauth_token` TEXT NOT NULL,
#`oauth_token_secret` TEXT NOT NULL,
#`access_token` TEXT NOT NULL,
#`access_token_secret` TEXT NOT NULL,
#`verifier` TEXT NOT NULL
#);
#''')
#self.db.commit()
# self.db.commit()
# TODO class SearchDBng(object):
@ -315,11 +318,11 @@ class TokenDB(object):
self.save()
def update_token(self,
token,
oauth_token_secret=None,
access_token=None,
access_token_secret=None,
verifier=None):
token,
oauth_token_secret=None,
access_token=None,
access_token_secret=None,
verifier=None):
t = self.tokens.get(token, {})
if oauth_token_secret:
@ -355,8 +358,10 @@ class TokenDB(object):
del(self.tokens[service])
self.save()
class SearchDB(BaseDB):
tmplfile = 'Search.html'
def __init__(self):
self.fpath = "%s" % config.get('var', 'searchdb')
super().__init__(self.fpath)
@ -402,8 +407,8 @@ class SearchDB(BaseDB):
cursor.execute('''SELECT mtime
FROM data
WHERE id = ? AND mtime = ?''',
(fname,mtime)
)
(fname, mtime)
)
rows = cursor.fetchall()
if len(rows):
@ -435,7 +440,6 @@ class SearchDB(BaseDB):
if category not in ret:
ret.update({category: {}})
maybe_fpath = os.path.join(
config.get('dirs', 'content'),
category,
@ -452,7 +456,6 @@ class SearchDB(BaseDB):
})
return ret
def cli(self, query):
results = self.search_by_query(query)
for c, items in sorted(results.items()):
@ -509,7 +512,12 @@ class WebmentionQueue(BaseDB):
logging.debug('getting queued webmentions for %s', fname)
ret = []
cursor = self.db.cursor()
cursor.execute('''SELECT * FROM queue WHERE target LIKE ? AND status = 0''', ('%'+fname+'%',))
cursor.execute(
'''SELECT * FROM queue WHERE target LIKE ? AND status = 0''',
('%' +
fname +
'%',
))
rows = cursor.fetchall()
for r in rows:
ret.append({
@ -541,6 +549,7 @@ def __expandconfig():
c.set(s, o, os.path.expanduser(curr))
return c
def baseN(num, b=36, numerals="0123456789abcdefghijklmnopqrstuvwxyz"):
""" Used to create short, lowercase slug for a number (an epoch) passed """
num = int(num)
@ -560,6 +569,7 @@ def slugfname(url):
lower=True
)[:200]
def __setup_sitevars():
SiteVars = {}
section = 'site'
@ -584,7 +594,6 @@ def __setup_sitevars():
for o in config.options(section):
SiteVars[section].update({o: config.get(section, o)})
# push the whole thing into cache
return SiteVars
@ -604,7 +613,7 @@ def notify(msg):
# fire and forget
try:
requests.post(url, data=data)
except:
except BaseException:
pass