Adding smtp service

conftest.py

@@ -1,5 +1,4 @@
 from django.conf import settings
-import base64
 import os
 import os.path
 
@@ -57,7 +56,8 @@ def pytest_configure(config):
     settings.INSTALLED_APPS = tuple(settings.INSTALLED_APPS) + (
         'tests',
     )
-    settings.SENTRY_KEY = base64.b64encode(os.urandom(40))
+    # Need a predictable key for tests that involve checking signatures
+    settings.SENTRY_KEY = 'abc123'
     settings.SENTRY_PUBLIC = False
     # This speeds up the tests considerably, pbkdf2 is by design, slow.
     settings.PASSWORD_HASHERS = [
@@ -67,3 +67,4 @@ def pytest_configure(config):
     # enable draft features
     settings.SENTRY_ENABLE_EXPLORE_CODE = True
     settings.SENTRY_ENABLE_EXPLORE_USERS = True
+    settings.SENTRY_ENABLE_EMAIL_REPLIES = True

docs/config/index.rst

@@ -176,6 +176,47 @@ The following settings are available for the built-in UDP API server:
         SENTRY_UDP_PORT = 9001
 
 
+.. _config-smtp-server:
+
+SMTP Server
+~~~~~~~~~~~
+
+The following settings are available for the built-in SMTP mail server:
+
+.. data:: SENTRY_SMTP_HOST
+    :noindex:
+
+    The hostname which the smtp server should bind to.
+
+    Defaults to ``localhost``.
+
+    ::
+
+        SENTRY_SMTP_HOST = '0.0.0.0'  # bind to all addresses
+
+.. data:: SENTRY_SMTP_PORT
+    :noindex:
+
+    The port which the smtp server should listen on.
+
+    Defaults to ``1025``.
+
+    ::
+
+        SENTRY_SMTP_PORT = 1025
+
+.. data:: SENTRY_SMTP_HOSTNAME
+    :noindex:
+
+    The hostname which matches the server's MX record.
+
+    Defaults to ``localhost``.
+
+    ::
+
+        SENTRY_SMTP_HOSTNAME = 'reply.getsentry.com'
+
+
 Data Sampling
 -------------
 

docs/quickstart/nginx.rst

@@ -65,3 +65,40 @@ as well the ``sentry.wsgi`` module:
     ; allow longer headers for raven.js if applicable
     ; default: 4096
     buffer-size = 32768
+
+
+Proxying Incoming Email
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Nginx is recommended for handling incoming emails in front of the Sentry smtp server.
+
+Below is a sample configuration for Nginx:
+
+::
+    http {
+      # Bind an http server to localhost only just for the smtp auth
+      server {
+        listen 127.0.0.1:80;
+
+        # Return back the address and port for the listening
+        # Sentry smtp server. Default is 127.0.0.1:1025.
+        location = /smtp {
+          add_header Auth-Server 127.0.0.1;
+          add_header Auth-Port   1025;
+          return 200;
+        }
+      }
+    }
+
+    mail {
+      auth_http localhost/smtp;
+
+      server {
+        listen 25;
+
+        protocol   smtp;
+        proxy      on;
+        smtp_auth  none;
+        xclient    off;
+      }
+    }

setup.py

@@ -77,6 +77,7 @@ install_requires = [
     'django-social-auth>=0.7.28,<0.8.0',
     'django-static-compiler>=0.3.0,<0.4.0',
     'django-templatetag-sugar>=0.1.0,<0.2.0',
+    'email-reply-parser>=0.2.0,<0.3.0',
     'gunicorn>=0.17.2,<0.18.0',
     'httpagentparser>=1.2.1,<1.3.0',
     'logan>=0.5.8.2,<0.6.0',

src/sentry/conf/server.py

@@ -464,6 +464,12 @@ SENTRY_WEB_OPTIONS = {
 SENTRY_UDP_HOST = 'localhost'
 SENTRY_UDP_PORT = 9001
 
+# SMTP Service
+SENTRY_ENABLE_EMAIL_REPLIES = False
+SENTRY_SMTP_HOSTNAME = 'localhost'
+SENTRY_SMTP_HOST = 'localhost'
+SENTRT_SMTP_PORT = 25
+
 SENTRY_ALLOWED_INTERFACES = set([
     'sentry.interfaces.Exception',
     'sentry.interfaces.Message',

src/sentry/management/commands/start.py

@@ -32,7 +32,7 @@ class Command(BaseCommand):
     )
 
     def handle(self, service_name='http', address=None, upgrade=True, **options):
-        from sentry.services import http, udp
+        from sentry.services import http, udp, smtp
 
         if address:
             if ':' in address:
@@ -47,6 +47,7 @@ class Command(BaseCommand):
         services = {
             'http': http.SentryHTTPServer,
             'udp': udp.SentryUDPServer,
+            'smtp': smtp.SentrySMTPServer,
         }
 
         if service_name == 'worker':

src/sentry/models.py

@@ -1099,7 +1099,7 @@ class Activity(Model):
                 self.event.update(num_comments=F('num_comments') + 1)
 
     def send_notification(self):
-        from sentry.utils.email import MessageBuilder
+        from sentry.utils.email import MessageBuilder, group_id_to_email
 
         if self.type != Activity.NOTE or not self.group:
             return
@@ -1156,11 +1156,16 @@ class Activity(Model):
             'link': self.group.get_absolute_url(),
         }
 
+        headers = {
+            'X-Sentry-Reply-To': group_id_to_email(self.group.pk),
+        }
+
         msg = MessageBuilder(
             subject=subject,
             context=context,
             template='sentry/emails/new_note.txt',
             html_template='sentry/emails/new_note.html',
+            headers=headers,
         )
 
         try:

src/sentry/plugins/sentry_mail/models.py

@@ -9,14 +9,13 @@ import sentry
 
 from django.conf import settings
 from django.core.urlresolvers import reverse
-from django.template.loader import render_to_string
 from django.utils.safestring import mark_safe
 from django.utils.translation import ugettext_lazy as _
 from sentry.models import User, UserOption
 from sentry.plugins import register
 from sentry.plugins.bases.notify import NotificationPlugin
 from sentry.utils.cache import cache
-from sentry.utils.email import MessageBuilder
+from sentry.utils.email import MessageBuilder, group_id_to_email
 from sentry.utils.http import absolute_uri
 
 NOTSET = object()
@@ -33,8 +32,8 @@ class MailPlugin(NotificationPlugin):
     project_conf_form = None
     subject_prefix = settings.EMAIL_SUBJECT_PREFIX
 
-    def _send_mail(self, subject, body, html_body=None, project=None,
-                   headers=None, fail_silently=False):
+    def _send_mail(self, subject, template=None, html_template=None, body=None,
+                   project=None, headers=None, context=None, fail_silently=False):
         send_to = self.get_send_to(project)
         if not send_to:
             return
@@ -43,9 +42,11 @@ class MailPlugin(NotificationPlugin):
 
         msg = MessageBuilder(
             subject='%s%s' % (subject_prefix, subject),
+            template=template,
+            html_template=html_template,
             body=body,
-            html_body=html_body,
             headers=headers,
+            context=context,
         )
         msg.send(send_to, fail_silently=fail_silently)
 
@@ -66,8 +67,14 @@ class MailPlugin(NotificationPlugin):
             project.name.encode('utf-8'),
             alert.message.encode('utf-8'),
         )
-        body = self.get_alert_plaintext_body(alert)
-        html_body = self.get_alert_html_body(alert)
+        template = 'sentry/emails/alert.txt'
+        html_template = 'sentry/emails/alert.html'
+
+        context = {
+            'alert': alert,
+            'link': alert.get_absolute_url(),
+            'settings_link': self.get_notification_settings_url(),
+        }
 
         headers = {
             'X-Sentry-Project': project.name,
@@ -75,26 +82,14 @@ class MailPlugin(NotificationPlugin):
 
         self._send_mail(
             subject=subject,
-            body=body,
-            html_body=html_body,
+            template=template,
+            html_template=html_template,
             project=project,
             fail_silently=False,
             headers=headers,
+            context=context,
         )
 
-    def get_alert_plaintext_body(self, alert):
-        return render_to_string('sentry/emails/alert.txt', {
-            'alert': alert,
-            'link': alert.get_absolute_url(),
-        })
-
-    def get_alert_html_body(self, alert):
-        return render_to_string('sentry/emails/alert.html', {
-            'alert': alert,
-            'link': alert.get_absolute_url(),
-            'settings_link': self.get_notification_settings_url(),
-        })
-
     def get_emails_for_users(self, user_ids, project=None):
         email_list = set()
         user_ids = set(user_ids)
@@ -172,43 +167,35 @@ class MailPlugin(NotificationPlugin):
 
         link = group.get_absolute_url()
 
-        body = self.get_plaintext_body(group, event, link, interface_list)
+        template = 'sentry/emails/error.txt'
+        html_template = 'sentry/emails/error.html'
 
-        html_body = self.get_html_body(group, event, link, interface_list)
+        context = {
+            'group': group,
+            'event': event,
+            'link': link,
+            'interfaces': interface_list,
+            'settings_link': self.get_notification_settings_url(),
+        }
 
         headers = {
             'X-Sentry-Logger': event.logger,
             'X-Sentry-Logger-Level': event.get_level_display(),
             'X-Sentry-Project': project.name,
             'X-Sentry-Server': event.server_name,
+            'X-Sentry-Reply-To': group_id_to_email(group.pk),
         }
 
         self._send_mail(
             subject=subject,
-            body=body,
-            html_body=html_body,
+            template=template,
+            html_template=html_template,
             project=project,
             fail_silently=fail_silently,
             headers=headers,
+            context=context,
         )
 
-    def get_plaintext_body(self, group, event, link, interface_list):
-        return render_to_string('sentry/emails/error.txt', {
-            'group': group,
-            'event': event,
-            'link': link,
-            'interfaces': interface_list,
-        })
-
-    def get_html_body(self, group, event, link, interface_list):
-        return render_to_string('sentry/emails/error.html', {
-            'group': group,
-            'event': event,
-            'link': link,
-            'interfaces': interface_list,
-            'settings_link': self.get_notification_settings_url(),
-        })
-
 
 # Legacy compatibility
 MailProcessor = MailPlugin

src/sentry/services/smtp.py

@@ -0,0 +1,97 @@
+"""
+sentry.services.smtp
+~~~~~~~~~~~~~~~~~~~~
+
+:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
+:license: BSD, see LICENSE for more details.
+"""
+
+import asyncore
+import email
+import logging
+from smtpd import SMTPServer, SMTPChannel
+
+from email_reply_parser import EmailReplyParser
+
+from sentry.services.base import Service
+from sentry.tasks.email import process_inbound_email
+from sentry.utils.email import email_to_group_id
+
+logger = logging.getLogger(__name__)
+
+
+# HACK(mattrobenolt): literally no idea what I'm doing. Mostly made this up.
+# SMTPChannel doesn't support EHLO response, but nginx requires an EHLO.
+# EHLO is available in python 3, so this is backported somewhat
+def smtp_EHLO(self, arg):
+    if not arg:
+        self.push('501 Syntax: EHLO hostname')
+        return
+    if self._SMTPChannel__greeting:
+        self.push('503 Duplicate HELO/EHLO')
+    else:
+        self._SMTPChannel__greeting = arg
+        self.push('250 %s' % self._SMTPChannel__fqdn)
+
+SMTPChannel.smtp_EHLO = smtp_EHLO
+
+
+STATUS = {
+    200: '200 Ok',
+    550: '550 Not found',
+    552: '552 Message too long',
+}
+
+
+class SentrySMTPServer(Service, SMTPServer):
+    name = 'smtp'
+    max_message_length = 20000  # This might be too conservative
+
+    def __init__(self, host=None, port=None, debug=False, workers=None):
+        from django.conf import settings
+
+        self.host = host or getattr(settings, 'SENTRY_SMTP_HOST', '0.0.0.0')
+        self.port = port or getattr(settings, 'SENTRY_SMTP_PORT', 1025)
+
+    def process_message(self, peer, mailfrom, rcpttos, raw_message):
+        if not len(rcpttos):
+            logger.info('Incoming email had no recipients. Ignoring.')
+            return STATUS[550]
+
+        if len(raw_message) > self.max_message_length:
+            logger.info('Inbound email message was too long: %d', len(raw_message))
+            return STATUS[552]
+
+        try:
+            group_id = email_to_group_id(rcpttos[0])
+        except Exception:
+            logger.info('%r is not a valid email address', rcpttos)
+            return STATUS[550]
+
+        message = email.message_from_string(raw_message)
+        payload = None
+        if message.is_multipart():
+            for msg in message.walk():
+                if msg.get_content_type() == 'text/plain':
+                    payload = msg.get_payload()
+                    break
+            if payload is None:
+                # No text/plain part, bailing
+                return STATUS[200]
+        else:
+            payload = message.get_payload()
+
+        payload = EmailReplyParser.parse_reply(payload).strip()
+        if not payload:
+            # If there's no body, we don't need to go any further
+            return STATUS[200]
+
+        process_inbound_email.delay(mailfrom, group_id, payload)
+        return STATUS[200]
+
+    def run(self):
+        SMTPServer.__init__(self, (self.host, self.port), None)
+        try:
+            asyncore.loop()
+        except KeyboardInterrupt:
+            pass

src/sentry/tasks/email.py

@@ -0,0 +1,51 @@
+"""
+sentry.tasks.email
+~~~~~~~~~~~~~~~~~~
+
+:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
+:license: BSD, see LICENSE for more details.
+"""
+import logging
+from celery.task import task
+
+logger = logging.getLogger(__name__)
+
+
+@task(name='sentry.tasks.email.process_inbound_email', queue='email')
+def process_inbound_email(mailfrom, group_id, payload):
+    """
+    """
+    from django.contrib.auth.models import User
+    from sentry.models import Event, Group, Project
+    from sentry.web.forms import NewNoteForm
+
+    try:
+        user = User.objects.get(email__iexact=mailfrom)
+    except User.DoesNotExist:
+        logger.warning('Inbound email from unknown address: %s', mailfrom)
+        return
+    except User.MultipleObjectsReturned:
+        logger.warning('Inbound email address matches multiple accounts: %s', mailfrom)
+        return
+
+    try:
+        group = Group.objects.select_related('project', 'team').get(pk=group_id)
+    except Group.DoesNotExist:
+        logger.warning('Group does not exist: %d', group_id)
+        return
+
+    # Make sure that the user actually has access to this project
+    if group.project not in Project.objects.get_for_user(
+            user, team=group.team, superuser=False):
+        logger.warning('User %r does not have access to group %r', (user, group))
+        return
+
+    event = group.get_latest_event() or Event()
+
+    Event.objects.bind_nodes([event], 'data')
+    event.group = group
+    event.project = group.project
+
+    form = NewNoteForm({'text': payload})
+    if form.is_valid():
+        form.save(event, user)