import argparse import logging import os import re import arrow import atexit from concurrent.futures import ProcessPoolExecutor from multiprocessing import cpu_count from slugify import slugify import nasg.config as config import nasg.singular as singular import nasg.searchindex as searchindex import nasg.taxonomy as taxonomy from pprint import pprint parser = argparse.ArgumentParser(description='Parameters for NASG') parser.add_argument( '--regenerate', '-f', dest='regenerate', action='store_true', default=False, help='force regeneration of all HTML outputs' ) parser.add_argument( '--downsize', '-c', action='store_true', dest='downsize', default=False, help='force re-downsizing of all suitable images' ) parser.add_argument( '--debug', '-d', action='store_true', dest='debug', default=False, help='turn on debug log' ) class Engine(object): def __init__(self): self._initdirs() self._lock() atexit.register(self._lock, action='clear') self.files = [] self.categories = {} self.tags = {} self.allposts = taxonomy.TaxonomyHandler('') self.frontposts = taxonomy.TaxonomyHandler('') self.allowedpattern = re.compile(config.accept_sourcefiles) self.counter = {} def _parse_results(self, futures): for future in futures: try: future.result() except Exception as e: logging.error("processing failed: %s", e) def collect(self): self._setup_categories() self._setup_singulars() def render(self): self._render_singulars() #self._render_taxonomy() def _render_singulars(self): logging.warning("rendering singulars") pprint(self.allposts) #futures = [] #with ProcessPoolExecutor(max_workers=cpu_count()) as executor: for p in self.allposts: #futures.append(executor.submit(p.write)) p.write() #for future in futures: #try: #future.result() #except Exception as e: #logging.error("processing failed: %s", e) def _render_taxonomy(self): futures = [] with ProcessPoolExecutor(max_workers=cpu_count()) as executor: for tslug, t in self.tags.items(): #t.write() futures.append(executor.submit(t.write)) for cslug, c in self.categories.items(): #c.write() futures.append(executor.submit(c.write)) #self.frontposts.write() futures.append(executor.submit(self.frontposts.write)) self._parse_results(futures) def _setup_categories(self): for cat, meta in config.categories.items(): cpath = os.path.join(config.CONTENT, cat) if not os.path.isdir(cpath): logging.error("category %s not found at: %s", cat, cpath) continue self.categories[cat] = taxonomy.TaxonomyHandler( meta.get('name', cat), taxonomy=meta.get('type', 'category'), slug=cat, render=meta.get('render', True) ) def _setup_singulars(self): futures = [] with ProcessPoolExecutor(max_workers=cpu_count()) as executor: for slug, tax in self.categories.items(): cpath = os.path.join(config.CONTENT, slug) for f in os.listdir(cpath): fpath = os.path.join(cpath,f) if not self.allowedpattern.fullmatch(f): logging.warning("unexpected file at: %s" % fpath) continue #self._posttype(fpath, slug) futures.append(executor.submit(self._posttype, fpath, slug)) self._parse_results(futures) def _posttype(self, fpath, cat): c = self.categories[cat] if re.match('.*\.jpg', fpath): p = singular.PhotoHandler(fpath) elif 'page' == c.taxonomy: p = singular.PageHandler(fpath) else: p = singular.ArticleHandler(fpath) c.append(p) self.allposts.append(p) front = config.categories[cat].get('front', True) if front: self.frontposts.append(p) ptags = p.vars.get('tags', []) for tag in ptags: tslug = slugify(tag, only_ascii=True, lower=True) if tslug not in self.tags: self.tags[tslug] = taxonomy.TaxonomyHandler( tag, taxonomy='tag', slug=tslug ) self.tags[tslug].append(p) def _initdirs(self): for d in [ config.TARGET, config.TTHEME, config.TFILES, config.VAR, config.SEARCHDB, config.TSDB, config.LOGDIR ]: if not os.path.exists(d): os.mkdir(d) def _lock(self, action='set'): if 'set' == action: if os.path.exists(config.LOCKFILE): raise ValueError("lockfile %s present" % config.LOCKFILE) with open(config.LOCKFILE, "wt") as l: l.write("%s" % arrow.utcnow()) l.close() elif 'clear' == action: if os.path.exists(config.LOCKFILE): os.unlink(config.LOCKFILE) else: return os.path.exists(config.LOCKFILE) if __name__ == '__main__': config.options.update(vars(parser.parse_args())) loglevel = 30 if config.options['debug']: loglevel = 10 while len(logging.root.handlers) > 0: logging.root.removeHandler(logging.root.handlers[-1]) logging.basicConfig( level=loglevel, format='%(asctime)s - %(levelname)s - %(message)s' ) engine = Engine() engine.collect() engine.render()