feat: Implement relay basics (#8162)

This adds the initial support for relays. So far only explicitly whitelisted relays are permitted in production.

.vscode/settings.json

@@ -52,5 +52,6 @@
     "python.unitTest.pyTestEnabled": false,
     "python.unitTest.unittestEnabled": false,
     "python.unitTest.nosetestsEnabled": false,
-    "prettier-eslint.prettierPath": "${env.WORKON_HOME}/node_modules/prettier/bin/prettier.js"
+    "prettier-eslint.prettierPath": "${env.WORKON_HOME}/node_modules/prettier/bin/prettier.js",
+    "editor.tabSize": 4
 }

requirements-base.txt

@@ -46,6 +46,7 @@ raven>=6.0.0,<=6.4.0
 redis>=2.10.3,<2.10.6
 requests[security]>=2.18.4,<2.19.0
 selenium==3.11.0
+semaphore>=0.1.0,<0.2.0
 simplejson>=3.2.0,<3.9.0
 six>=1.10.0,<1.11.0
 setproctitle>=1.1.7,<1.2.0

src/sentry/api/authentication.py

@@ -5,7 +5,10 @@ from rest_framework.authentication import (BasicAuthentication, get_authorizatio
 from rest_framework.exceptions import AuthenticationFailed
 
 from sentry.app import raven
-from sentry.models import ApiKey, ApiToken
+from sentry.models import ApiKey, ApiToken, Relay
+from sentry.relay.utils import get_header_relay_id, get_header_relay_signature
+
+import semaphore
 
 
 class QuietBasicAuthentication(BasicAuthentication):
@@ -13,6 +16,39 @@ class QuietBasicAuthentication(BasicAuthentication):
         return 'xBasic realm="%s"' % self.www_authenticate_realm
 
 
+class RelayAuthentication(BasicAuthentication):
+    def authenticate(self, request):
+        relay_id = get_header_relay_id(request)
+        relay_sig = get_header_relay_signature(request)
+        if not relay_id:
+            raise AuthenticationFailed('Invalid relay ID')
+        if not relay_sig:
+            raise AuthenticationFailed('Missing relay signature')
+        return self.authenticate_credentials(relay_id, relay_sig, request)
+
+    def authenticate_credentials(self, relay_id, relay_sig, request):
+        raven.tags_context({
+            'relay_id': relay_id,
+        })
+
+        try:
+            relay = Relay.objects.get(relay_id=relay_id)
+        except Relay.DoesNotExist:
+            raise AuthenticationFailed('Unknown relay')
+
+        try:
+            data = relay.public_key_object.unpack(request.body, relay_sig,
+                                                  max_age=60 * 5)
+            request.relay = relay
+            request.relay_request_data = data
+        except semaphore.UnpackError:
+            raise AuthenticationFailed('Invalid relay signature')
+
+        # TODO(mitsuhiko): can we return the relay here?  would be nice if we
+        # could find some common interface for it
+        return (AnonymousUser(), None)
+
+
 class ApiKeyAuthentication(QuietBasicAuthentication):
     def authenticate_credentials(self, userid, password):
         if password:

src/sentry/api/endpoints/relay_details.py

@@ -0,0 +1,31 @@
+from __future__ import absolute_import
+
+from rest_framework.response import Response
+
+from sentry.models import Relay
+from sentry.api.base import Endpoint
+from sentry.api.exceptions import ResourceDoesNotExist
+from sentry.api.permissions import SuperuserPermission
+
+
+class RelayDetailsEndpoint(Endpoint):
+    permission_classes = (SuperuserPermission, )
+
+    def delete(self, request, relay_id):
+        """
+        Delete one Relay
+        ````````````````
+        :auth: required
+        """
+        try:
+            relay = Relay.objects.get(
+                id=relay_id,
+            )
+        except Relay.DoesNotExist:
+            raise ResourceDoesNotExist
+
+        # TODO(hazat): Create audit entry?
+
+        relay.delete()
+
+        return Response(status=204)

src/sentry/api/endpoints/relay_heartbeat.py

@@ -0,0 +1,28 @@
+from __future__ import absolute_import
+
+from rest_framework.response import Response
+
+from sentry.api.base import Endpoint
+from sentry.relay import change_set, query
+from sentry.api.permissions import RelayPermission
+from sentry.api.authentication import RelayAuthentication
+
+
+class RelayHeartbeatEndpoint(Endpoint):
+    authentication_classes = (RelayAuthentication, )
+    permission_classes = (RelayPermission, )
+
+    def post(self, request):
+        changesets = request.relay_request_data.get('changesets')
+        if changesets:
+            change_set.execute_changesets(request.relay, changesets)
+
+        queries = request.relay_request_data.get('queries')
+        if queries:
+            query_response = query.execute_queries(request.relay, queries)
+        else:
+            query_response = {}
+
+        return Response({
+            'queryResults': query_response,
+        }, status=200)

src/sentry/api/endpoints/relay_index.py

@@ -0,0 +1,35 @@
+from __future__ import absolute_import
+
+from django.conf import settings
+
+from sentry.models import Relay
+from sentry.api.base import Endpoint
+from sentry.api.serializers import serialize
+from sentry.api.paginator import OffsetPaginator
+from sentry.api.permissions import SuperuserPermission
+
+
+class RelayIndexEndpoint(Endpoint):
+    permission_classes = (SuperuserPermission, )
+
+    def get(self, request):
+        """
+        List your Relays
+        ````````````````
+
+        Return a list of relays know to this Sentry installation available
+        to the authenticated session.
+
+        :auth: required
+        """
+        queryset = Relay.objects.filter(
+            public_key__in=settings.SENTRY_RELAY_WHITELIST_PK,
+        )
+
+        return self.paginate(
+            request=request,
+            queryset=queryset,
+            order_by='relay_id',
+            paginator_cls=OffsetPaginator,
+            on_results=lambda x: serialize(x, request.user),
+        )

src/sentry/api/endpoints/relay_register.py

@@ -0,0 +1,182 @@
+from __future__ import absolute_import
+
+import six
+
+from rest_framework.response import Response
+from rest_framework import serializers, status
+
+from django.conf import settings
+from django.utils import timezone
+from django.core.cache import cache as default_cache
+
+from sentry.utils import json
+from sentry.models import Relay
+from sentry.api.base import Endpoint
+from sentry.api.serializers import serialize
+from sentry.relay.utils import get_header_relay_id, get_header_relay_signature
+
+
+from semaphore import create_register_challenge, validate_register_response, \
+    get_register_response_relay_id, PublicKey
+
+
+class RelayIdSerializer(serializers.Serializer):
+    relay_id = serializers.RegexField(
+        r'^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$',
+        required=True)
+
+
+class RelayRegisterChallengeSerializer(RelayIdSerializer):
+    public_key = serializers.CharField(max_length=64, required=True)
+
+
+class RelayRegisterResponseSerializer(RelayIdSerializer):
+    token = serializers.CharField(required=True)
+
+
+class RelayRegisterChallengeEndpoint(Endpoint):
+    authentication_classes = ()
+    permission_classes = ()
+
+    def check_allowed_relay(self, request, data):
+        """
+        Checks if the relay is allowed to register, otherwise raises an exception
+        """
+        if (settings.DEBUG or
+            request.META.get('REMOTE_ADDR', None) in settings.INTERNAL_IPS or
+                data.get('public_key', None) in settings.SENTRY_RELAY_WHITELIST_PK):
+            return True
+        return False
+
+    def post(self, request):
+        """
+        Requests to Register a Relay
+        ````````````````````````````
+
+        Registers the relay with the sentry installation.  If a relay boots
+        it will always attempt to invoke this endpoint.
+        """
+        try:
+            json_data = json.loads(request.body)
+        except ValueError:
+            return Response({
+                'detail': 'No valid json body',
+            }, status=status.HTTP_400_BAD_REQUEST)
+
+        serializer = RelayRegisterChallengeSerializer(data=json_data)
+
+        if not serializer.is_valid():
+            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+
+        if not self.check_allowed_relay(request, json_data):
+            return Response({
+                'detail': 'Relay is not allowed to register',
+            }, status=status.HTTP_401_UNAUTHORIZED)
+
+        sig = get_header_relay_signature(request)
+        if not sig:
+            return Response({
+                'detail': 'Missing relay signature',
+            }, status=status.HTTP_400_BAD_REQUEST)
+
+        try:
+            challenge = create_register_challenge(request.body, sig)
+        except Exception as exc:
+            return Response({
+                'detail': str(exc).splitlines()[0],
+            }, status=status.HTTP_400_BAD_REQUEST)
+
+        relay_id = six.text_type(challenge['relay_id'])
+        if relay_id != get_header_relay_id(request):
+            return Response({
+                'detail': 'relay_id in payload did not match header',
+            }, status=status.HTTP_400_BAD_REQUEST)
+
+        try:
+            relay = Relay.objects.get(relay_id=relay_id)
+        except Relay.DoesNotExist:
+            pass
+        else:
+            if relay.public_key != six.text_type(challenge['public_key']):
+                # This happens if we have an ID collision or someone copies an existing id
+                return Response({
+                    'detail': 'Attempted to register agent with a different public key',
+                }, status=status.HTTP_400_BAD_REQUEST)
+
+        default_cache.set('relay-auth:%s' % relay_id, {
+            'token': challenge['token'],
+            'public_key': six.text_type(challenge['public_key']),
+        }, 60)
+        return Response(serialize({
+            'relay_id': six.text_type(challenge['relay_id']),
+            'token': challenge['token'],
+        }))
+
+
+class RelayRegisterResponseEndpoint(Endpoint):
+    authentication_classes = ()
+    permission_classes = ()
+
+    def post(self, request):
+        """
+        Registers a Relay
+        `````````````````
+
+        Registers the relay with the sentry installation.  If a relay boots
+        it will always attempt to invoke this endpoint.
+        """
+
+        try:
+            json_data = json.loads(request.body)
+        except ValueError:
+            return Response({
+                'detail': 'No valid json body',
+            }, status=status.HTTP_400_BAD_REQUEST)
+
+        serializer = RelayRegisterResponseSerializer(data=json_data)
+
+        if not serializer.is_valid():
+            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+
+        sig = get_header_relay_signature(request)
+        if not sig:
+            return Response({
+                'detail': 'Missing relay signature',
+            }, status=status.HTTP_400_BAD_REQUEST)
+
+        relay_id = six.text_type(get_register_response_relay_id(request.body))
+        if relay_id != get_header_relay_id(request):
+            return Response({
+                'detail': 'relay_id in payload did not match header',
+            }, status=status.HTTP_400_BAD_REQUEST)
+
+        params = default_cache.get('relay-auth:%s' % relay_id)
+        if params is None:
+            return Response({
+                'detail': 'Challenge expired'
+            }, status=status.HTTP_401_UNAUTHORIZED)
+
+        key = PublicKey.parse(params['public_key'])
+
+        try:
+            validate_register_response(key, request.body, sig)
+        except Exception as exc:
+            return Response({
+                'detail': str(exc).splitlines()[0],
+            }, status=status.HTTP_400_BAD_REQUEST)
+
+        try:
+            relay = Relay.objects.get(relay_id=relay_id)
+        except Relay.DoesNotExist:
+            relay = Relay.objects.create(
+                relay_id=relay_id,
+                public_key=params['public_key'],
+            )
+        else:
+            relay.last_seen = timezone.now()
+            relay.save()
+        default_cache.delete('relay-auth:%s' % relay_id)
+
+        return Response(serialize({
+            'relay_id': relay.relay_id,
+        }))

src/sentry/api/permissions.py

@@ -6,6 +6,11 @@ from sentry.api.exceptions import SuperuserRequired
 from sentry.auth.superuser import is_active_superuser
 
 
+class RelayPermission(permissions.BasePermission):
+    def has_permission(self, request, view):
+        return getattr(request, 'relay', None) is not None
+
+
 class NoPermission(permissions.BasePermission):
     def has_permission(self, request, view):
         return False

src/sentry/api/serializers/models/relay.py

@@ -0,0 +1,18 @@
+from __future__ import absolute_import
+
+import six
+
+from sentry.api.serializers import Serializer, register
+from sentry.models import Relay
+
+
+@register(Relay)
+class RelaySerializer(Serializer):
+    def serialize(self, obj, attrs, user):
+        return {
+            'id': six.text_type(obj.id),
+            'relayId': six.text_type(obj.relay_id),
+            'publicKey': obj.public_key,
+            'firstSeen': obj.first_seen,
+            'lastSeen': obj.last_seen,
+        }

src/sentry/api/urls.py

@@ -3,6 +3,11 @@ from __future__ import absolute_import, print_function
 from django.conf.urls import include, patterns, url
 
 from .endpoints.accept_project_transfer import AcceptProjectTransferEndpoint
+from .endpoints.relay_heartbeat import RelayHeartbeatEndpoint
+from .endpoints.relay_index import RelayIndexEndpoint
+from .endpoints.relay_details import RelayDetailsEndpoint
+from .endpoints.relay_register import RelayRegisterChallengeEndpoint, \
+    RelayRegisterResponseEndpoint
 from .endpoints.api_applications import ApiApplicationsEndpoint
 from .endpoints.api_application_details import ApiApplicationDetailsEndpoint
 from .endpoints.api_authorizations import ApiAuthorizationsEndpoint
@@ -169,6 +174,37 @@ from .endpoints.setup_wizard import SetupWizard
 urlpatterns = patterns(
     '',
 
+    # Relay
+    url(
+        r'^relays/$',
+        RelayIndexEndpoint.as_view(),
+        name='sentry-api-0-relays-index'
+    ),
+
+    url(
+        r'^relays/register/challenge/$',
+        RelayRegisterChallengeEndpoint.as_view(),
+        name='sentry-api-0-relay-register-challenge'
+    ),
+
+    url(
+        r'^relays/register/response/$',
+        RelayRegisterResponseEndpoint.as_view(),
+        name='sentry-api-0-relay-register-response'
+    ),
+
+    url(
+        r'^relays/heartbeat/$',
+        RelayHeartbeatEndpoint.as_view(),
+        name='sentry-api-0-relay-heartbeat'
+    ),
+
+    url(
+        r'^relays/(?P<relay_id>[^\/]+)/$',
+        RelayDetailsEndpoint.as_view(),
+        name='sentry-api-0-relays-details'
+    ),
+
     # Api Data
     url(
         r'^assistant/$',