import os import glob import json import logging import arrow import requests import keys import common import settings from time import sleep from math import ceil import random from pprint import pprint class ASFavs(common.Favs): def __init__(self): super().__init__("artstation") self.user = keys.artstation.get("username") self.session = requests.Session() self.session.headers.update({ "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", #"DNT": "1", "Connection": "keep-alive", "Upgrade-Insecure-Requests": "1", "Pragma": "no-cache", "Cache-Control": "max-age=0, no-cache", }) session.headers.update({ "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", "DNT": "1", "Connection": "keep-alive", "Upgrade-Insecure-Requests": "1", "Pragma": "no-cache", "Cache-Control": "max-age=0, no-cache", }) def paged_likes(self, page=1): url = "https://www.artstation.com/users/%s/likes.json?page=%s" % ( self.user, page, ) js = self.session.get(url) while js.status_code != requests.codes.ok: # FU cloudflare pprint(self.session.cookies) sleep(round(random.uniform(0.7,3.5), 2)) js = self.session.get(url) try: js = js.json() if "data" not in js: return None return js except Exception as e: logging.error("fetching artstation failed: %s, response: %s", e, js.text) return None @property def likes(self): js = self.paged_likes(1) if not js: return [] likes = js.get("data", []) pages = ceil(js.get("total_count", 1) / 50) while pages > 1: extras = self.paged_likes(pages) if not extras: continue likes = likes + extras.get("data", []) pages = pages - 1 return likes @property def feeds(self): feeds = [] url = "https://www.artstation.com/users/%s/following.json" % (self.user) js = self.session.get(url, headers=self.headers) try: js = js.json() if "data" not in js: logging.error("fetching artstation follows failed: missing data") return feeds for f in js.get("data"): feeds.append( { "text": f.get("username"), "xmlUrl": "https://www.artstation.com/%s.rss" % f.get("subdomain"), "htmlUrl": "https://www.artstation.com/%s" % f.get("subdomain"), } ) except Exception as e: logging.error("parsing artstation follows failed: %s", e) return feeds def run(self): # FU cloudflare for like in self.likes: like = ASLike(like, self.session, self.headers) like.run() class ASLike(common.ImgFav): def __init__(self, like, session, headers): self.like = like self.session = session self.headers = headers def __str__(self): return "like-of %s" % (self.url) @property def url(self): return self.like.get("permalink") @property def data(self): purl = "%s.json" % (self.url.replace("artwork", "projects")) data = self.session.get(purl, headers=self.headers) try: data = data.json() except Exception as e: logging.error("fetching artstation project %s failed: %s", self.url, e) return None return data @property def author(self): return { "name": self.like.get("user").get("username"), "url": self.like.get("user").get("permalink"), } @property def id(self): return self.like.get("id") @property def content(self): return "%s" % self.data.get("description_html", "") @property def title(self): title = self.like.get("title") if not len(title): title = self.like.get("slug") if not len(title): title = common.url2slug(self.url) return title @property def slug(self): maybe = self.like.get("slug") if not len(maybe): maybe = common.url2slug(self.url) return maybe @property def targetprefix(self): return os.path.join( settings.paths.get("archive"), "favorite", "artstation_%s_%s_%s" % ( common.url2slug("%s" % self.like.get("user").get("username")), self.like.get("hash_id"), self.slug, ), ) @property def published(self): return arrow.get(self.like.get("published_at")) @property def tags(self): t = [] for c in self.data.get("categories"): t.append(c.get("name")) return t @property def images(self): r = {} cntr = 0 for img in self.data.get("assets"): if img.get("asset_type") != "image": logging.debug("skipping asset: %s" % img) continue f = "%s_%d%s" % (self.targetprefix, cntr, common.TMPFEXT) r.update({f: img.get("image_url")}) cntr = cntr + 1 return r if __name__ == "__main__": t = ASFavs() t.run()