re-adding index.php magic instead of nginx rules; given that I need to use php

for the search functionality, it's better to keep this contained instead of
adding one more black magic (eg. nginx rules)

Note: I've sort of exhausted client-side JS search options; the smallest index
I was able to build was still a few megabytes, way to large for my taste, hence
the SQLite FTS4 python + PHP solution. Yes, it's ugly, but it works, and PHP
is available nearly everywhere. Nearly.

Without something that can set headers properly, it's impossible to do real
HTTP 410 Gone, and even HTTP 301 or 302, so it's very likely I'm stuck with the
minimal PHP solution.

.gitignore

@@ -4,3 +4,4 @@ _scratch
 .env
 
 keys.py
+nasg.proj

nasg.py

@@ -65,6 +65,7 @@ MD = markdown.Markdown(
     ],
 )
 
+
 class MarkdownDoc(object):
     @property
     @cached()
@@ -160,13 +161,11 @@ class Gone(object):
 
     def __init__(self, fpath):
         self.fpath = fpath
-        self.address, self.fext = os.path.splitext(
-            os.path.basename(self.fpath)
-        )
 
     @property
-    def nginx(self):
+    def source(self):
-        return (self.address, 'return 410')
+        source, fext = os.path.splitext(os.path.basename(self.fpath))
+        return source
 
 
 class Redirect(object):
@@ -176,7 +175,11 @@ class Redirect(object):
 
     def __init__(self, fpath):
         self.fpath = fpath
-        self.source, self.fext = os.path.splitext(os.path.basename(self.fpath))
+
+    @property
+    def source(self):
+        source, fext = os.path.splitext(os.path.basename(self.fpath))
+        return source
 
     @property
     @cached()
@@ -184,20 +187,15 @@ class Redirect(object):
         target = ''
         with open(self.fpath, 'rt') as f:
             target = f.read().strip()
-        if not RE_HTTP.match(target):
-            target = "%s/%s" % (settings.site.get('url'), target)
         return target
 
-    @property
-    def nginx(self):
-        return (self.source, 'return 301 %s' % (self.target))
-
 
 class Singular(MarkdownDoc):
     """
     A Singular object: a complete representation of a post, including
     all it's comments, files, images, etc
     """
+
     def __init__(self, fpath):
         self.fpath = fpath
         n = os.path.dirname(fpath)
@@ -810,20 +808,36 @@ class AsyncWorker(object):
         self._loop.run_until_complete(w)
 
 
-class NginxConf(dict):
+class IndexPHP(object):
-    def __str__(self):
+    def __init__(self):
-        r = ''
+        self.gone = {}
-        for key in self:
+        self.redirect = {}
-            r = "%slocation /%s { %s; }\n" % (r, key, self[key])
-        return r
 
-    def save(self):
+    def add_gone(self, uri):
-        fpath = os.path.join(
+        self.gone[uri] = True
+
+    def add_redirect(self, source, target):
+        if target in self.gone:
+            self.add_gone(source)
+        else:
+            if not RE_HTTP.match(target):
+                target = "%s/%s" % (settings.site.get('url'), target)
+            self.redirect[source] = target
+
+    async def render(self):
+        target = os.path.join(
             settings.paths.get('build'),
-            '.nginx.conf'
+            'index.php'
         )
-        with open(fpath, 'wt') as f:
+        r = J2.get_template('Index.j2.php').render({
-            f.write(str(self))
+            'post': {},
+            'site': settings.site,
+            'gones': self.gone,
+            'redirects': self.redirect
+        })
+        with open(target, 'wt') as f:
+            logging.info("rendering to %s", target)
+            f.write(r)
 
 
 class Category(dict):
@@ -915,7 +929,7 @@ class Category(dict):
     def ping_websub(self):
         return
         # TODO aiohttp?
-        ## ping pubsub
+        # ping pubsub
         # r = requests.post(
         # shared.site.get('websub').get('hub'),
         # data={
@@ -980,7 +994,6 @@ class Category(dict):
             logging.info('writing file: %s', atom)
             f.write(fg.atom_str(pretty=True))
 
-
     def render_page(self, pagenum=1, pages=1):
         if self.display == 'flat':
             start = 1
@@ -1032,6 +1045,7 @@ class Category(dict):
         self.render_feed()
         self.ping_websub()
 
+
 class Search(object):
     def __init__(self):
         self.fpath = os.path.join(
@@ -1065,13 +1079,36 @@ class Search(object):
         self.db.execute('PRAGMA auto_vacuum;')
         self.db.close()
 
+    def exists(self, name, mtime=0):
+        ret = False
+        maybe = self.db.execute('''
+            SELECT
+                mtime
+            FROM
+                data
+            WHERE
+                name = ?
+        ''', (name,)).fetchone()
+        if maybe:
+            ret = int(maybe[0])
+        return ret
+
     def append(self, url, mtime, name, title, category, content):
-        # TODO: delete if mtime differs
         mtime = int(mtime)
+        exists = self.exists(name, mtime)
+        if (exists and exists < mtime):
             self.db.execute('''
-            INSERT OR IGNORE INTO data
+            DELETE
+            FROM
+                data
+            WHERE
+                name=?''', (name,))
+        self.db.execute('''
+            INSERT OR IGNORE INTO
+                data
                 (url, mtime, name, title, category, content)
-            VALUES (?,?,?,?,?,?);
+            VALUES
+                (?,?,?,?,?,?);
         ''', (
             url,
             mtime,
@@ -1082,6 +1119,12 @@ class Search(object):
         ))
 
     async def render(self):
+        target = os.path.join(
+            settings.paths.get('build'),
+            'search.php'
+        )
+        if os.path.exists(target):
+            return
         r = J2.get_template('Search.j2.php').render({
             'post': {},
             'site': settings.site,
@@ -1091,10 +1134,6 @@ class Search(object):
             'tips': settings.tips,
             'labels': settings.labels
         })
-        target = os.path.join(
-            settings.paths.get('build'),
-            'search.php'
-        )
         with open(target, 'wt') as f:
             logging.info("rendering to %s", target)
             f.write(r)
@@ -1103,19 +1142,17 @@ class Search(object):
 def make():
     start = int(round(time.time() * 1000))
     content = settings.paths.get('content')
+    worker = AsyncWorker()
 
-    nginxrules = NginxConf()
+    rules = IndexPHP()
-    for e in glob.glob(os.path.join(content, '*', '*.lnk')):
-        post = Redirect(e)
-        location, rule = post.nginx
-        nginxrules[location] = rule
     for e in glob.glob(os.path.join(content, '*', '*.ptr')):
         post = Gone(e)
-        location, rule = post.nginx
+        rules.add_gone(post.source)
-        nginxrules[location] = rule
+    for e in glob.glob(os.path.join(content, '*', '*.lnk')):
-    nginxrules.save()
+        post = Redirect(e)
+        rules.add_redirect(post.source, post.target)
+    worker.append(rules.render())
 
-    worker = AsyncWorker()
     categories = {}
     categories['/'] = Category()
     sitemap = OrderedDict()
@@ -1166,10 +1203,9 @@ def make():
     with open(t, 'wt') as f:
         f.write("\n".join(sorted(sitemap.keys())))
 
-
-
     end = int(round(time.time() * 1000))
     logging.info('process took %d ms' % (end - start))
 
+
 if __name__ == '__main__':
     make()

templates/Index.j2.php

@@ -0,0 +1,88 @@
+<?php
+
+$redirects = array(
+{% for from, to in redirects.items() %}
+    "{{ from }}" => "{{ to }}",
+{% endfor %}
+);
+
+$gone = array(
+{% for gone in gones %}
+    "{{ gone }}" => true,
+{% endfor %}
+);
+
+
+function redirect_to($uri) {
+    header('HTTP/1.1 301 Moved Permanently');
+    if (preg_match("/^https?/", $uri))
+        $target = $uri;
+    else
+        $target = '{{ site.url }}/'.  trim($uri, '/') . '/';
+    header("Location: ". $target);
+    exit;
+}
+
+function gone($uri) {
+    header('HTTP/1.1 410 Gone');
+    die('<!DOCTYPE html>
+<html lang="en">
+ <head>
+  <meta charset="utf-8"/>
+  <meta content="width=device-width,initial-scale=1,minimum-scale=1" name="viewport"/>
+  <title>Gone</title>
+ </head>
+ <body>
+<h1><center>This content was deleted.</center></h1>
+<hr>
+<p><center>{{ site.domain }}</center></p>
+ </body>
+</html>');
+}
+
+function notfound() {
+    header('HTTP/1.0 404 Not Found');
+    die('<!DOCTYPE html>
+<html lang="en">
+ <head>
+  <meta charset="utf-8"/>
+  <meta content="width=device-width,initial-scale=1,minimum-scale=1" name="viewport"/>
+  <title>Not found</title>
+ </head>
+ <body>
+
+<h1><center>This was not found.</center></h1>
+<h2><center>Please search for it instead.</center></h2>
+<p>
+    <center>
+<form action="/search.php" class="search-form" method="get" role="search">
+  <label for="search">Search</label>
+  <input id="q" name="q" placeholder="search..." title="Search for:" type="search" value=""/>
+  <input type="submit" value="OK"/>
+</form>
+    </center>
+</p>
+ </body>
+</html>');
+}
+
+function maybe_redirect($uri) {
+    if (file_exists("./$uri/index.html")) {
+        redirect_to($uri);
+    }
+}
+
+$uri = filter_var($_SERVER['REQUEST_URI'], FILTER_SANITIZE_URL);
+$uri = str_replace('../', '', $uri);
+$uri = str_replace('/feed/', '', $uri);
+$uri = str_replace('/atom/', '', $uri);
+$uri = trim($uri, '/');
+
+if (isset($gone[$uri]))
+    gone($uri);
+elseif (isset($redirects[$uri]))
+    redirect_to($redirects[$uri]);
+elseif (strstr($uri, '_'))
+    maybe_redirect(str_replace('_', '-', $uri));
+else
+    notfound();