From 80b7dd39308f1da12a426fc11a98ea0d7a23b348 Mon Sep 17 00:00:00 2001 From: Peter Molnar Date: Fri, 23 Mar 2018 10:51:38 +0000 Subject: [PATCH] re-adding envelope for mail creation; replacing telegram notification with mail notification for incoming webmentions --- envelope.py | 193 ++++++++++++++++++++++++++++++++++++++++++++++++++++ router.py | 17 ++++- shared.py | 55 ++++++++++----- 3 files changed, 246 insertions(+), 19 deletions(-) create mode 100644 envelope.py diff --git a/envelope.py b/envelope.py new file mode 100644 index 0000000..bca89f9 --- /dev/null +++ b/envelope.py @@ -0,0 +1,193 @@ +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.mime.image import MIMEImage +from email.header import Header +import email.charset +from email.generator import Generator +from io import StringIO +import mimetypes +from email.mime.base import MIMEBase +from email.encoders import encode_base64 +import email.utils + +import time +import getpass +import socket +import shutil +import requests +import tempfile +import atexit +import os +import re +import smtplib +import logging +from shared import Pandoc + +class Letter(object): + def __init__(self, sender=None, recipient=None, subject='', text=''): + self.sender = sender or (getpass.getuser(), socket.gethostname()) + self.recipient = recipient or self.sender + + self.tmp = tempfile.mkdtemp( + 'envelope_', + dir=tempfile.gettempdir() + ) + atexit.register( + shutil.rmtree, + os.path.abspath(self.tmp) + ) + self.text = text; + self.subject = subject + self.images = [] + self.ready = None + self.time = time.time() + self.headers = {} + + @property + def _html(self): + return Pandoc().convert(self.text) + + @property + def _tmpl(self): + return "%s" % (self._html) + + def __pull_image(self, img): + fname = os.path.basename(img) + i = { + 'url': img, + 'name': fname, + 'tmp': os.path.join(self.tmp, fname), + } + + logging.debug("pulling image %s", i['url']) + r = requests.get(i['url'], stream=True) + if r.status_code == 200: + with open(i['tmp'], 'wb') as f: + logging.debug("writing image %s", i['tmp']) + r.raw.decode_content = True + shutil.copyfileobj(r.raw, f) + if not isinstance(self.images, list): + self.images = [] + self.images.append(i) + + + def __pull_images(self): + mdmatch = re.compile( + r'!\[.*\]\((.*?\.(?:jpe?g|png|gif)(?:\s+[\'\"]?.*?[\'\"]?)?)\)' + r'(?:\{.*?\})?' + ) + [self.__pull_image(img) for img in mdmatch.findall(self.text)] + + + def __attach_images(self): + self.__pull_images() + for i in self.images: + cid = 'cid:%s' % (i['name']) + logging.debug("replacing %s with %s", i['url'], cid) + self.text = self.text.replace(i['url'], cid) + + + def make(self, inline_images=True): + if inline_images: + self.__attach_images() + + + # Python, by default, encodes utf-8 in base64, which makes plain text + # mail painful; this overrides and forces Quoted Printable. + # Quoted Printable is still awful, but better, and we're going to + # force the mail to be 8bit encoded. + # Note: enforcing 8bit breaks compatibility with ancient mail clients. + email.charset.add_charset('utf-8', email.charset.QP, email.charset.QP, 'utf-8') + + mail = MIMEMultipart('alternative') + + # --- setting headers --- + self.headers = { + 'Subject': Header(re.sub(r"\r?\n?$", "", self.subject, 1), 'utf-8').encode(), + 'To': email.utils.formataddr(self.recipient), + 'From': email.utils.formataddr(self.sender), + 'Date': email.utils.formatdate(self.time, localtime=True) + } + + for k, v in self.headers.items(): + mail.add_header(k, "%s" % v) + logging.debug("headers: %s", self.headers) + + # --- adding plain text --- + text = self.text + _text = MIMEText(text, 'text', _charset='utf-8') + # --- + # this is the part where we overwrite the way Python thinks: + # force the text to be the actual, unencoded, utf-8. + # Note:these steps breaks compatibility with ancient mail clients. + _text.replace_header('Content-Transfer-Encoding', '8bit') + _text.replace_header('Content-Type', 'text/plain; charset=utf-8') + _text.set_payload(self.text) + # --- + logging.debug("text: %s", _text) + mail.attach(_text) + + # --- HTML bit --- + # this is where it gets tricky: the HTML part should be a 'related' + # wrapper, in which the text and all the related images are sitting + _envelope = MIMEMultipart('related') + + + html = self._tmpl + _html = MIMEText(html, 'html', _charset='utf-8') + # --- + # see above under 'adding plain text' + _html.replace_header('Content-Transfer-Encoding', '8bit') + _html.replace_header('Content-Type', 'text/html; charset=utf-8') + _html.set_payload(html) + # --- + logging.debug("HTML: %s", _html) + _envelope.attach(_html) + + for i in self.images: + mimetype, encoding = mimetypes.guess_type(i['tmp']) + mimetype = mimetype or 'application/octet-stream' + mimetype = mimetype.split('/', 1) + attachment = MIMEBase(mimetype[0], mimetype[1]) + with open(i['tmp'], 'rb') as img: + attachment.set_payload(img.read()) + img.close() + os.unlink(i['tmp']) + + encode_base64(attachment) + attachment.add_header( + 'Content-Disposition', + 'inline', + filename=i['name'] + ) + attachment.add_header( + 'Content-ID', + '<%s>' % (i['name']) + ) + + _envelope.attach(attachment) + + # add the whole html + image pack to the mail + mail.attach(_envelope) + + str_io = StringIO() + g = Generator(str_io, False) + g.flatten(mail) + + self.ready = str_io.getvalue().encode('utf-8') + + def send(self): + if not self.ready: + logging.error('this mail is not ready') + return + + try: + s = smtplib.SMTP('127.0.0.1', 25) + # unless you do the encode, you'll get: + # File "/usr/local/lib/python3.5/smtplib.py", line 850, in sendmail + # msg = _fix_eols(msg).encode('ascii') + # UnicodeEncodeError: 'ascii' codec can't encode character '\xa0' in position 1073: ordinal not in range(128) + s.sendmail(self.headers['From'], self.headers['To'], self.ready) + s.quit() + except Exception as e: + logging.error('sending mail failed with error: %s', e) diff --git a/router.py b/router.py index eceec34..61db587 100644 --- a/router.py +++ b/router.py @@ -35,6 +35,8 @@ import logging import validators import urllib.parse import shared +import envelope +import socket if __name__ == '__main__': #logging_format = "[%(asctime)s] %(process)d-%(levelname)s " @@ -105,12 +107,23 @@ if __name__ == '__main__': wdb.maybe_queue(source, target) # telegram notification, if set - shared.notify( - 'incoming webmention from %s to %s' % ( + l = envelope.Letter( + sender=( + 'NASG', + 'nasg@%s' % socket.getfqdn() + ), + recipient=( + shared.config.get('author', 'name'), + shared.config.get('author', 'email') + ), + subject="[webmention] from %s" % _source.hostname, + text='incoming webmention from %s to %s' % ( source, target ) ) + l.make() + l.send() response = sanic.response.text("Accepted", status=202) return response diff --git a/shared.py b/shared.py index 2feb656..d1a6dbc 100644 --- a/shared.py +++ b/shared.py @@ -43,7 +43,6 @@ import jinja2 from inspect import getsourcefile import sys - class CMDLine(object): def __init__(self, executable): self.executable = self._which(executable) @@ -673,24 +672,46 @@ def __setup_sitevars(): return SiteVars -def notify(msg): - # telegram notification, if set - if not config.has_section('api_telegram'): - return +#def notify(msg): - url = "https://api.telegram.org/bot%s/sendMessage" % ( - config.get('api_telegram', 'api_token') - ) - data = { - 'chat_id': config.get('api_telegram', 'chat_id'), - 'text': msg - } - # fire and forget - try: - requests.post(url, data=data) - except BaseException: - pass + ### telegram notification, if set + ##if not config.has_section('api_telegram'): + ##return + ##url = "https://api.telegram.org/bot%s/sendMessage" % ( + ##config.get('api_telegram', 'api_token') + ##) + ##data = { + ##'chat_id': config.get('api_telegram', 'chat_id'), + ##'text': msg + ##} + ### fire and forget + ##try: + ##requests.post(url, data=data) + ##except BaseException: + ##pass + + #headers = { + #'Subject': 'notification from NASG', + #'Content-Type': 'text/plain; charset=utf-8', + #'Content-Disposition': 'inline', + #'Content-Transfer-Encoding': '8bit', + #'From': 'nasg@%s' % (socket.getfqdn()), + #'To': config.get('author', 'email'), + #'Date': arrow.now().strftime('%a, %d %b %Y %H:%M:%S %Z'), + #} + + ## create the message + #mail = '' + #for key, value in headers.items(): + #mail += "%s: %s\n" % ( key, value ) + + ## add contents + #mail += "\n%s\n" % ( msg ) + + #s = smtplib.SMTP( '127.0.0.1', 25 ) + #s.sendmail( headers['From'], headers['To'], msg.encode("utf8") ) + #s.quit() ARROWFORMAT = { 'iso': 'YYYY-MM-DDTHH:mm:ssZ',