Relay integration in devservices (#16790)

Adds the possibility to run events through Relay.
At this point it must be explicitly enabled via settings SENTRY_USE_RELAY=True.

MANIFEST.in

@@ -1,4 +1,5 @@
 include setup.py README.md MANIFEST.in LICENSE AUTHORS
 recursive-include ./ requirements*.txt
+recursive-include ./config/relay *
 graft src/sentry
 global-exclude *~

config/relay/config.yml

@@ -0,0 +1,13 @@
+---
+relay:
+  upstream: "http://host.docker.internal:8888/"
+  host: 0.0.0.0
+  port: 3000
+logging:
+  level: INFO
+  enable_backtraces: false
+processing:
+  enabled: true
+  kafka_config:
+    - {name: "bootstrap.servers", value: "sentry_kafka:9093"}
+  redis: redis://sentry_redis:6379

config/relay/credentials.json

@@ -0,0 +1,5 @@
+{
+  "secret_key": "OxE6Du8quMxWj19f7YDCpIxm6XyU9nWGQJkMWFlkchA",
+  "public_key": "SMSesqan65THCV6M4qs4kBzPai60LzuDn-xNsvYpuP8",
+  "id": "88888888-4444-4444-8444-cccccccccccc"
+}

config/reverse_proxy/nginx.conf

@@ -0,0 +1,67 @@
+user  nginx;
+worker_processes  1;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    upstream relay {
+        server sentry_relay.sentry:3000;
+    }
+
+    upstream sentry {
+        server host.docker.internal:8888;
+    }
+
+    server {
+        listen 80;
+        resolver 127.0.0.11 ipv6=off;
+
+        location /api/store/ {
+            proxy_pass http://relay;
+        }
+        location ~ ^/api/\d+/store/$ {
+            proxy_pass http://relay;
+        }
+        location ~ ^/api/\d+/minidump/?$ {
+            proxy_pass http://relay;
+        }
+        location ~ ^/api/\d+/unreal/\w+/$ {
+            proxy_pass http://relay;
+        }
+        location ~ ^/api/\d+/security/$ {
+            proxy_pass http://relay;
+        }
+        location ~ ^/api/\d+/csp-report/$ {
+            proxy_pass http://relay;
+        }
+        location ~ ^/api/\d+/events/[\w-]+/attachments/$ {
+            proxy_pass http://relay;
+        }
+        location / {
+           proxy_pass http://sentry;
+        }
+    }
+}

src/sentry/conf/server.py

@@ -113,6 +113,14 @@ else:
 
 NODE_MODULES_ROOT = os.path.normpath(NODE_MODULES_ROOT)
 
+RELAY_CONFIG_DIR = os.path.normpath(
+    os.path.join(PROJECT_ROOT, os.pardir, os.pardir, "config", "relay")
+)
+
+REVERSE_PROXY_CONFIG = os.path.normpath(
+    os.path.join(PROJECT_ROOT, os.pardir, os.pardir, "config", "reverse_proxy", "nginx.conf")
+)
+
 sys.path.insert(0, os.path.normpath(os.path.join(PROJECT_ROOT, os.pardir)))
 
 DATABASES = {
@@ -147,7 +155,6 @@ if "DATABASE_URL" in os.environ:
     if url.scheme == "postgres":
         DATABASES["default"]["ENGINE"] = "sentry.db.postgres"
 
-
 # This should always be UTC.
 TIME_ZONE = "UTC"
 
@@ -334,7 +341,6 @@ INSTALLED_APPS = (
     "django.contrib.staticfiles",
 )
 
-
 # Silence internal hints from Django's system checks
 SILENCED_SYSTEM_CHECKS = (
     # Django recommends to use OneToOneField over ForeignKey(unique=True)
@@ -1207,7 +1213,10 @@ SENTRY_ROLES = (
     {
         "id": "admin",
         "name": "Admin",
-        "desc": "Admin privileges on any teams of which they're a member. They can create new teams and projects, as well as remove teams and projects which they already hold membership on (or all teams, if open membership is on). Additionally, they can manage memberships of teams that they are members of.",
+        "desc": "Admin privileges on any teams of which they're a member. They can create new teams and projects, "
+        "as well as remove teams and projects which they already hold membership on (or all teams, "
+        "if open membership is on). Additionally, they can manage memberships of teams that they are members "
+        "of.",
         "scopes": set(
             [
                 "event:read",
@@ -1255,7 +1264,8 @@ SENTRY_ROLES = (
     {
         "id": "owner",
         "name": "Owner",
-        "desc": "Unrestricted access to the organization, its data, and its settings. Can add, modify, and delete projects and members, as well as make billing and plan changes.",
+        "desc": "Unrestricted access to the organization, its data, and its settings. Can add, modify, and delete "
+        "projects and members, as well as make billing and plan changes.",
         "is_global": True,
         "scopes": set(
             [
@@ -1320,6 +1330,12 @@ SENTRY_WATCHERS = (
     ),
 )
 
+# Controls whether DEVSERVICES will spin up a Relay and direct store traffic through Relay or not.
+# If Relay is used a reverse proxy server will be run at the 8000 (the port formally used by Sentry) that
+# will split the requests between Relay and Sentry (all store requests will be passed to Relay, and the
+# rest will be forwarded to Sentry)
+SENTRY_USE_RELAY = False
+
 SENTRY_DEVSERVICES = {
     "redis": {
         "image": "redis:5.0-alpine",
@@ -1344,7 +1360,8 @@ SENTRY_DEVSERVICES = {
         "environment": {
             "KAFKA_ZOOKEEPER_CONNECT": "{containers[zookeeper][name]}:2181",
             "KAFKA_LISTENERS": "INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092",
-            "KAFKA_ADVERTISED_LISTENERS": "INTERNAL://{containers[kafka][name]}:9093,EXTERNAL://{containers[kafka][ports][9092/tcp][0]}:{containers[kafka][ports][9092/tcp][1]}",
+            "KAFKA_ADVERTISED_LISTENERS": "INTERNAL://{containers[kafka][name]}:9093,EXTERNAL://{containers[kafka]"
+            "[ports][9092/tcp][0]}:{containers[kafka][ports][9092/tcp][1]}",
             "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP": "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT",
             "KAFKA_INTER_BROKER_LISTENER_NAME": "INTERNAL",
             "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR": "1",
@@ -1383,6 +1400,18 @@ SENTRY_DEVSERVICES = {
         "ports": {"3021/tcp": 3021},
         "command": ["run"],
     },
+    "reverse_proxy": {
+        "image": "nginx:1.16.1",
+        "ports": {"80/tcp": 8000},
+        "volumes": {REVERSE_PROXY_CONFIG: {"bind": "/etc/nginx/nginx.conf"}},
+    },
+    "relay": {
+        "image": "us.gcr.io/sentryio/relay:latest",
+        "pull": True,
+        "ports": {"3000/tcp": 3000},
+        "volumes": {RELAY_CONFIG_DIR: {"bind": "/etc/relay"}},
+        "command": ["run", "--config", "/etc/relay"],
+    },
 }
 
 # Max file size for avatar photo uploads
@@ -1455,7 +1484,6 @@ GITHUB_API_SECRET = DEAD
 
 SUDO_URL = "sentry-sudo"
 
-
 # Endpoint to https://github.com/getsentry/sentry-release-registry, used for
 # alerting the user on outdated SDKs.
 SENTRY_RELEASE_REGISTRY_BASEURL = None
@@ -1610,7 +1638,12 @@ SENTRY_BUILTIN_SOURCES = {
 # Relay
 # List of PKs whitelisted by Sentry.  All relays here are always
 # registered as internal relays.
-SENTRY_RELAY_WHITELIST_PK = []
+SENTRY_RELAY_WHITELIST_PK = [
+    # NOTE (RaduW) This is the relay key for the relay instance used by devservices.
+    # This should NOT be part of any production environment.
+    # This key should match the key in /sentry/config/relay/credentials.json
+    "SMSesqan65THCV6M4qs4kBzPai60LzuDn-xNsvYpuP8"
+]
 
 # When open registration is not permitted then only relays in the
 # whitelist can register.

src/sentry/runner/commands/devserver.py

@@ -27,11 +27,22 @@ from sentry.runner.decorators import configuration, log_options
     default=False,
     help="This enables running sentry with pure separation of the frontend and backend",
 )
-@click.argument("bind", default="127.0.0.1:8000", metavar="ADDRESS", envvar="SENTRY_DEVSERVER_BIND")
+@click.argument(
+    "bind", default=None, metavar="ADDRESS", envvar="SENTRY_DEVSERVER_BIND", required=False
+)
 @log_options()
 @configuration
 def devserver(reload, watchers, workers, experimental_spa, styleguide, prefix, environment, bind):
     "Starts a lightweight web server for development."
+    if bind is None:
+        # default configuration, the dev server address depends on weather we have a reverse proxy
+        # in front that splits the requests between Relay and the dev server or we pass everything
+        # to the dev server
+        from django.conf import settings
+
+        port = 8888 if settings.SENTRY_USE_RELAY else 8000
+        bind = "127.0.0.1:{}".format(port)
+
     if ":" in bind:
         host, port = bind.split(":", 1)
         port = int(port)

src/sentry/runner/commands/devservices.py

@@ -81,19 +81,36 @@ def up(project, exclude):
     if "kafka" in settings.SENTRY_EVENTSTREAM:
         pass
     elif "snuba" in settings.SENTRY_EVENTSTREAM:
-        click.secho(
-            "! Skipping kafka and zookeeper since your eventstream backend does not require it",
-            err=True,
-            fg="cyan",
-        )
-        exclude |= {"kafka", "zookeeper"}
+        if not settings.SENTRY_USE_RELAY:
+            click.secho(
+                "! Skipping kafka and zookeeper since your eventstream backend does not require it",
+                err=True,
+                fg="cyan",
+            )
+            exclude |= {"kafka", "zookeeper"}
     else:
+        if settings.SENTRY_USE_RELAY:
+            click.secho(
+                "! Skipping snuba, and clickhouse since your eventstream backend does not require it",
+                err=True,
+                fg="cyan",
+            )
+            exclude |= {"snuba", "clickhouse"}
+        else:
+            click.secho(
+                "! Skipping kafka, zookeeper, snuba, and clickhouse since your eventstream backend does not require it",
+                err=True,
+                fg="cyan",
+            )
+            exclude |= {"kafka", "zookeeper", "snuba", "clickhouse"}
+
+    if not settings.SENTRY_USE_RELAY:
         click.secho(
-            "! Skipping kafka, zookeeper, snuba, and clickhouse since your eventstream backend does not require it",
+            "! Skipping relay, and reverse_proxy since you are not using Relay.",
             err=True,
             fg="cyan",
         )
-        exclude |= {"kafka", "zookeeper", "snuba", "clickhouse"}
+        exclude |= {"relay", "reverse_proxy"}
 
     if not sentry_options.get("symbolicator.enabled"):
         exclude |= {"symbolicator"}