nasg/pesos.py

#!/usr/bin/env python3

import json
import os
import hashlib
import glob
import frontmatter
import requests
import shared
import logging
import re
import shutil
import arrow
import bs4
from slugify import slugify
import oauth
import argparse

""" TODO

- followings?

- favs from:
    - wordpress.com
    - twitter

"""

class Bookmark(object):
    def __init__(self, title, url, fname=None):
        self.fm = frontmatter.loads('')
        fname = fname or slugify(title)
        self.fname = "%s.md" % fname
        self.target = os.path.join(
            shared.config.get('source', 'contentdir'),
            shared.config.get('source', 'bookmarks'),
            self.fname
        )
        self.fm.metadata = {
            'published': arrow.utcnow().format(shared.ARROWISO),
            'title': title,
            'bookmark-of': url,
        }

    def write(self):
        logging.info('saving bookmark to %s', self.target)
        with open(self.target, 'wt') as t:
            t.write(frontmatter.dumps(self.fm))

class HNBookmarks(object):
    prefix = 'hn-'
    def __init__(self):
        self.url = 'https://news.ycombinator.com/favorites?id=%s' % (
            shared.config.get('hackernews', 'user_id')
        )

    @property
    def existing(self):
        if hasattr(self, '_existing'):
            return self._existing

        d = os.path.join(
            shared.config.get('source', 'contentdir'),
            "*",
            "%s*.md" % self.prefix
        )
        files = reversed(sorted(glob.glob(d)))
        self._existing = [
            os.path.basename(f.replace(self.prefix, '').replace('.md', ''))
            for f in files
        ]

        return self._existing

    def run(self):
        r = requests.get(self.url)
        soup = bs4.BeautifulSoup(r.text, "html5lib")
        rows = soup.find_all('tr', attrs={'class':'athing' })
        for row in rows:
            rid = row.get('id')
            if rid in self.existing:
                continue

            link = row.find('a', attrs={'class':'storylink' })
            url = link.get('href')
            title = " ".join(link.contents)
            fname = "%s%s" % (self.prefix, rid)

            bookmark = Bookmark(title, url, fname)
            bookmark.write()

class Fav(object):
    def __init__(self):
        self.arrow = arrow.utcnow()
        self.fm = frontmatter.loads('')

    @property
    def target(self):
        return os.path.join(
            shared.config.get('source', 'contentdir'),
            shared.config.get('source', 'favs'),
            self.fname
        )

    @property
    def exists(self):
        return os.path.isfile(self.target)

    @property
    def imgname(self):
        # the _ is to differentiate between my photos, where the md and jpg name is the same, and favs
        return self.fname.replace('.md', '_.jpg')

    @property
    def imgtarget(self):
        return os.path.join(
            shared.config.get('source', 'filesdir'),
            self.imgname
        )

    def saveimg(self, url, target=None):
        target = target or self.imgtarget
        if os.path.isfile(target):
            logging.error("%s already exists, refusing to overwrite", target)
            return

        logging.info("pulling image %s to files", url)
        r = requests.get(url, stream=True)
        if r.status_code == 200:
            with open(target, 'wb') as f:
                r.raw.decode_content = True
                shutil.copyfileobj(r.raw, f)

    def write(self):
        logging.info('saving fav to %s', self.target)
        with open(self.target, 'wt') as t:
            t.write(frontmatter.dumps(self.fm))
        os.utime(self.target, (self.arrow.timestamp, self.arrow.timestamp))


class PinterestFav(Fav):
    def __init__(self, url):
        super(PinterestFav, self).__init__()
        self.url = url
        self.fname = "pinterest-%s.md" % (list(filter(None, url.split('/')))[-1])

    def run(self):
        try:
            r = requests.get(self.url)
            soup = bs4.BeautifulSoup(r.text, 'lxml')
            ld = json.loads(soup.find('script', type='application/ld+json').text)
            imgurl = ld.get('image')
            self.saveimg(imgurl)

            self.fm.metadata = {
                'published': arrow.get(
                    ld.get('datePublished', arrow.utcnow().timestamp)
                ).format(shared.ARROWISO),
                'title': ld.get('headline', self.url),
                'favorite-of': self.url,
                'image': self.imgname
            }
            content = ld.get('articleBody', '')
            content = shared.Pandoc(False).convert(content)
            self.fm.content = content

        except Exception as e:
            logging.error('saving pinterest fav %s failed: %s', self.url, e)
            return


class FlickrFav(Fav):
    def __init__(self, photo):
        super(FlickrFav, self).__init__()
        self.photo = photo
        self.ownerid = photo.get('owner')
        self.photoid = photo.get('id')
        self.fname = "flickr-%s-%s.md" % (self.ownerid, self.photoid)
        self.url = "https://www.flickr.com/photos/%s/%s" % (self.ownerid, self.photoid)

    def run(self):
        img = self.photo.get('url_b', self.photo.get('url_z', False))
        if not img:
            logging.error("image url was empty for %s, skipping fav", self.url)
            return

        self.saveimg(img)
        self.arrow = arrow.get(
            self.photo.get('date_faved', arrow.utcnow().timestamp)
        )
        self.fm.metadata = {
            'published': self.arrow.format(shared.ARROWISO),
            'title': '%s' % self.photo.get('title', self.fname),
            'favorite-of': self.url,
            'flickr_tags': self.photo.get('tags', '').split(' '),
            'geo': {
                'latitude': self.photo.get('latitude', ''),
                'longitude': self.photo.get('longitude', ''),
            },
            'author': {
                'name': self.photo.get('owner_name'),
                'url': 'https://www.flickr.com/people/%s' % (
                    self.photo.get('owner')
                ),
            },
            'image': self.imgname
        }

        content = self.photo.get('description', {}).get('_content', '')
        content = shared.Pandoc(False).convert(content)
        self.fm.content = content


class FivehpxFav(Fav):
    def __init__(self, photo):
        super(FivehpxFav, self).__init__()
        self.photo = photo
        self.ownerid = photo.get('user_id')
        self.photoid = photo.get('id')
        self.fname = "500px-%s-%s.md" % (self.ownerid, self.photoid)
        self.url = "https://www.500px.com%s" % (photo.get('url'))

    def run(self):
        img = self.photo.get('images')[0].get('url')
        if not img:
            logging.error("image url was empty for %s, skipping fav", self.url)
            return

        self.saveimg(img)
        self.arrow = arrow.get(
            self.photo.get('created_at', arrow.utcnow().timestamp)
        )
        self.fm.metadata = {
            'published': self.arrow.format(shared.ARROWISO),
            'title': '%s' % self.photo.get('name', self.fname),
            'favorite-of': self.url,
            'fivehpx_tags': self.photo.get('tags', []),
            'geo': {
                'latitude': self.photo.get('latitude', ''),
                'longitude': self.photo.get('longitude', ''),
            },
            'author': {
                'name': self.photo.get('user').get('fullname', self.ownerid),
                'url': 'https://www.500px.com/%s' % (
                    self.photo.get('user').get('username', self.ownerid)
                ),
            },
            'image': self.imgname
        }

        content = self.photo.get('description', '')
        if content:
            content = shared.Pandoc(False).convert(content)
        else:
            content = ''
        self.fm.content = content


class TumblrFav(Fav):
    def __init__(self, like):
        super(TumblrFav, self).__init__()
        self.like = like
        self.blogname = like.get('blog_name')
        self.postid = like.get('id')
        self.fname = "tumblr-%s-%s.md" % (self.blogname, self.postid)
        self.url = like.get('post_url')
        self.images = []

    def run(self):
        icntr = 0
        for p in self.like.get('photos', []):
            i = p.get('original_size').get('url')
            logging.debug('parsing image %s', i)
            n = self.fname.replace('.md', '_%d.jpg' % icntr)
            self.images.append(n)
            nt = os.path.join(
                shared.config.get('source', 'filesdir'),
                n
            )
            self.saveimg(i, nt)
            icntr = icntr + 1

        self.arrow = arrow.get(
            self.like.get('liked_timestamp',
                self.like.get('date',
                    arrow.utcnow().timestamp
                )
            )
        )

        self.fm.content = self.like.get('caption', '')

        title = self.like.get('summary', '').strip()
        if not len(title):
            title = self.like.get('slug', '').strip()
        if not len(title):
            title = shared.slugfname(self.like.get('post_url'))

        self.fm.metadata = {
            'published': self.arrow.format(shared.ARROWISO),
            'title': title,
            'favorite-of': self.url,
            'tumblr_tags': self.like.get('tags'),
            'author': {
                'name': self.like.get('blog_name'),
                'url': 'http://%s.tumblr.com' % self.like.get('blog_name')
            },
            'images': self.images
        }


class DAFav(Fav):
    def __init__(self, fav):
        super(DAFav, self).__init__()
        self.fav = fav
        self.deviationid = fav.get('deviationid')
        self.url = fav.get('url')
        self.title = fav.get('title', False) or self.deviationid
        self.author = self.fav.get('author').get('username')
        self.fname = "deviantart-%s-by-%s.md" % (
            slugify(self.title), slugify(self.author)
        )
        self.image = fav.get('content', {}).get('src')

    def run(self):
        self.saveimg(self.image)

        self.arrow = arrow.get(
            self.fav.get('published_time', arrow.utcnow().timestamp)
        )

        self.fm.metadata = {
            'published': self.arrow.format(shared.ARROWISO),
            'title': '%s' % self.title,
            'favorite-of': self.url,
            'da_tags': [t.get('tag_name') for t in self.fav.get('meta', {}).get('tags', [])],
            'author': {
                'name': self.author,
                'url': 'https://%s.deviantart.com' % (self.author),
            },
            'image': self.imgname
        }

        content = self.fav.get('meta', {}).get('description', '')
        content = shared.Pandoc(False).convert(content)
        self.fm.content = content


class Favs(object):
    def __init__(self, confgroup):
        self.confgroup = confgroup
        self.url = shared.config.get(confgroup, 'fav_api')

    @property
    def lastpulled(self):
        mtime = 0
        d = os.path.join(
            shared.config.get('source', 'contentdir'),
            shared.config.get('source', 'favs'),
            "%s-*.md" % self.confgroup
        )
        files = glob.glob(d)
        for f in files:
            ftime = int(os.path.getmtime(f))
            if ftime > mtime:
                mtime = ftime

        mtime = mtime + 1
        logging.debug("last flickr fav timestamp: %s", mtime)
        return mtime


class FlickrFavs(Favs):
    def __init__(self):
        super(FlickrFavs, self).__init__('flickr')
        self.params = {
            'method': 'flickr.favorites.getList',
            'api_key': shared.config.get('flickr', 'api_key'),
            'user_id': shared.config.get('flickr', 'user_id'),
            'extras': 'description,geo,tags,url_z,url_b,owner_name,date_upload',
            'per_page': 500,
            'format': 'json',
            'nojsoncallback': '1',
            'min_fave_date': self.lastpulled
        }

    def run(self):
        r = requests.get(self.url,params=self.params)
        js = json.loads(r.text)
        for photo in js.get('photos', {}).get('photo', []):
            fav = FlickrFav(photo)
            fav.run()
            fav.write()


class FivehpxFavs(Favs):
    def __init__(self):
        super(FivehpxFavs, self).__init__('500px')
        self.params = {
            'consumer_key': shared.config.get('500px', 'api_key'),
            'rpp': 100,
            'image_size': 4,
            'include_tags': 1,
            'include_geo': 1
        }

    def run(self):
        r = requests.get(self.url,params=self.params)
        js = json.loads(r.text)
        for photo in js.get('photos', []):
            fav = FivehpxFav(photo)
            if not fav.exists:
                fav.run()
                fav.write()


class TumblrFavs(Favs):
    def __init__(self):
        super(TumblrFavs, self).__init__('tumblr')
        self.oauth = oauth.TumblrOauth()
        self.params = {
            'after': self.lastpulled
        }
        self.likes = []

    def getpaged(self, offset):
        r = self.oauth.request(
            self.url,
            params={'offset': offset}
        )
        return json.loads(r.text)

    def run(self):
        r = self.oauth.request(
            self.url,
            params=self.params
        )

        js = json.loads(r.text)
        total = int(js.get('response', {}).get('liked_count', 20))
        print('total: %d' % total)
        offset = 20
        cntr = total - offset
        likes = js.get('response', {}).get('liked_posts', [])
        while cntr > 0:
            paged = self.getpaged(offset)
            likes = likes + paged.get('response', {}).get('liked_posts', [])
            offset = offset + 20
            cntr = total - offset

        self.likes = likes
        for like in self.likes:
            fav = TumblrFav(like)
            if not fav.exists:
                fav.run()
                fav.write()


class DAFavs(Favs):
    def __init__(self):
        from pprint import pprint
        super(DAFavs, self).__init__('deviantart')
        self.oauth = oauth.DAOauth()
        self.params = {
            'limit': 24,
            'mature_content': 'true',
            'username': shared.config.get('deviantart', 'username')
        }
        self.likes = []

    def getpaged(self, offset):
        self.params.update({'offset': offset})
        r = self.oauth.request(
            self.url,
            self.params
        )
        return json.loads(r.text)

    def getsinglemeta(self, daid):
        r = self.oauth.request(
            'https://www.deviantart.com/api/v1/oauth2/deviation/metadata',
            params={
                'deviationids[]': daid,
                'ext_submission': False,
                'ext_camera': False,
                'ext_stats': False,
                'ext_collection': False,
                'mature_content': True,
            }
        )
        meta = {}
        try:
            meta = json.loads(r.text)
            return meta.get('metadata', []).pop()
        except:
            return meta

    def has_more(self, q):
        if 'True' == q or 'true' == q:
            return True
        return False

    def run(self):
        r = self.oauth.request(
            self.url,
            self.params
        )

        js = json.loads(r.text)
        has_more = js.get('has_more')
        offset = js.get('next_offset')
        favs = js.get('results', [])
        while True == has_more:
            logging.debug('iterating over DA results with offset %d', offset)
            paged = self.getpaged(offset)
            favs = favs + paged.get('results', [])
            has_more = paged.get('has_more')
            n = paged.get('next_offset')
            if n:
                offset = offset + n

        self.favs = favs
        for fav in self.favs:
            f = DAFav(fav)
            if f.exists:
                continue

            f.fav.update({'meta': self.getsinglemeta(fav.get('deviationid'))})
            f.run()
            f.write()



#class WPFavs(Favs):
    #def __init__(self):
        #from pprint import pprint
        #super(DAFavs, self).__init__('wordpress')
        #self.oauth = oauth.DAOauth()
        #self.params = {
            #'limit': 24,
            #'mature_content': 'true',
            #'username': shared.config.get('deviantart', 'username')
        #}
        #self.likes = []

    #def getpaged(self, offset):
        #self.params.update({'offset': offset})
        #r = self.oauth.request(
            #self.url,
            #self.params
        #)
        #return json.loads(r.text)

    #def getsinglemeta(self, daid):
        #r = self.oauth.request(
            #'https://www.deviantart.com/api/v1/oauth2/deviation/metadata',
            #params={
                #'deviationids[]': daid,
                #'ext_submission': False,
                #'ext_camera': False,
                #'ext_stats': False,
                #'ext_collection': False,
                #'mature_content': True,
            #}
        #)
        #meta = {}
        #try:
            #meta = json.loads(r.text)
            #return meta.get('metadata', []).pop()
        #except:
            #return meta

    #def has_more(self, q):
        #if 'True' == q or 'true' == q:
            #return True
        #return False

    #def run(self):
        #r = self.oauth.request(
            #self.url,
            #self.params
        #)

        #js = json.loads(r.text)
        #has_more = js.get('has_more')
        #offset = js.get('next_offset')
        #favs = js.get('results', [])
        #while True == has_more:
            #logging.debug('iterating over DA results with offset %d', offset)
            #paged = self.getpaged(offset)
            #favs = favs + paged.get('results', [])
            #has_more = paged.get('has_more')
            #n = paged.get('next_offset')
            #if n:
                #offset = offset + n

        #self.favs = favs
        #for fav in self.favs:
            #f = DAFav(fav)
            #if f.exists:
                #continue

            #f.fav.update({'meta': self.getsinglemeta(fav.get('deviationid'))})
            #f.run()
            #f.write()

#class Following(object):
    #def __init__(self, confgroup):
        #self.confgroup = confgroup
        #self.url = shared.config.get(confgroup, 'following_api')
        #self.followings = []


#class FlickrFollowing(Following):
    #def __init__(self):
        #super(FlickrFollowing, self).__init__('flickr')
        #self.oauth = oauth.FlickrOauth()


    #def run(self):
        #r = self.oauth.request(self.url, params={
            #'method': 'flickr.contacts.getList',
            #'format': 'json',
            #'nojsoncallback': 1,
            #'api_key': shared.config.get(self.confgroup, 'api_key')
        #})

        #try:
            #contacts = json.loads(r.text)
            #for c in contacts.get('contacts', {}).get('contact', []):
                #self.followings.append({
                    #'url': "https://www.flickr.com/people/%s/" % c.get('nsid'),
                    #'name': c.get('realname'),
                    #'username': c.get('username'),
                    #'userid': c.get('nsid')
                #})

        #except Exception as e:
            #logging.error('getting following from flickr failed: %s', e)


#class TumblrFollowing(Following):
    #def __init__(self):
        #super(TumblrFollowing, self).__init__('tumblr')
        #self.oauth = oauth.FlickrOauth()


    #def run(self):
        #r = self.oauth.request(self.url, params={
            #'method': 'flickr.contacts.getList',
            #'format': 'json',
            #'nojsoncallback': 1,
            #'api_key': shared.config.get(self.confgroup, 'api_key')
        #})

        #try:
            #contacts = json.loads(r.text)
            #for c in contacts.get('contacts', {}).get('contact', []):
                #self.followings.append({
                    #'url': "https://www.flickr.com/people/%s/" % c.get('nsid'),
                    #'name': c.get('realname'),
                    #'username': c.get('username'),
                    #'userid': c.get('nsid')
                #})

        #except Exception as e:
            #logging.error('getting following from flickr failed: %s', e)


#class FlickrFollowing(object):
    #def __init__(self):
        #super(FlickrFollowing, self).__init__('flickr')
        #self.params = {
            #'consumer_key': shared.config.get('500px', 'api_key'),
            #'rpp': 100,
            #'image_size': 4,
            #'include_tags': 1,
            #'include_geo': 1
        #}

    #def run(self):
        #r = requests.get(self.url,params=self.params)
        #js = json.loads(r.text)
        #for photo in js.get('photos', []):
            #fav = FivehpxFav(photo)
            #if not fav.exists:
                #fav.run()
                #fav.write()


    #def run(self):



#https://api.flickr.com/services/rest/?method=flickr.contacts.getList&api_key=27d8a5bf7dabf882ff1c710894041f64&format=json&nojsoncallback=1&auth_token=72157682938907284-9c5f21debeec9833&api_sig=8ac87b900f44debea06a3765ed223680


#class Following(object):
    #def __init__(self, confgroup):
        #self.confgroup = confgroup
        #self.url = shared.config.get(confgroup, 'fav_api')


#class FlickrFollowing(Following):
    #def __init__(self):
        #super(FlickrFollowing, self).__init__('flickr')
        #self.params = {
            #'method': 'flickr.contacts.getList',
            #'api_key': shared.config.get('flickr', 'api_key'),
            #'format': 'json',
            #'nojsoncallback': '1',
        #}

    #def run(self):
        #r = requests.get(self.url,params=self.params)
        #js = json.loads(r.text)
        #pprint(js)
        #for contact in js.get('contacts', {}).get('contact', []):
            #pprint(contact)



if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Parameters for NASG')
    parser.add_argument(
        '--loglevel',
        default='error',
        help='change loglevel'
    )

    params = vars(parser.parse_args())

    while len(logging.root.handlers) > 0:
        logging.root.removeHandler(logging.root.handlers[-1])

    logging.basicConfig(
        level=shared.LLEVEL[params.get('loglevel')],
        format='%(asctime)s - %(levelname)s - %(message)s'
    )

    flickr = FlickrFavs()
    flickr.run()

    hn = HNBookmarks()
    hn.run()

    fivehpx = FivehpxFavs()
    fivehpx.run()

    tumblr = TumblrFavs()
    tumblr.run()

    da = DAFavs()
    da.run()