[quotas] add rate limits to project keys (#5405)

- add per-key rate limits
- add stats for events per key
- fix acceptance tests predictability

.babelrc

@@ -19,7 +19,8 @@
           "Array"
         ]
       }
-    ]
+    ],
+    "idx"
   ],
   "env": {
     "test": {
@@ -29,4 +30,4 @@
       ]
     }
   }
-}
+}

CHANGES

@@ -6,6 +6,8 @@ Version 8.17 (Unreleased)
 - Turned on reprocessing by default
 - Added basics for Data Forwarding integrations.
 - The threads interface now contributes to grouping if it contains a single thread.
+- Added per-key (DSN) rate limits (``project:rate-limits`` feature).
+- Added tsdb statistics for events per-key.
 
 Schema Changes
 ~~~~~~~~~~~~~~

bin/load-mocks

@@ -135,6 +135,8 @@ def create_sample_time_series(event, release=None):
     group = event.group
     project = group.project
 
+    key = project.key_set.all()[0]
+
     now = datetime.utcnow().replace(tzinfo=utc)
 
     environment = Environment.get_or_create(
@@ -167,10 +169,12 @@ def create_sample_time_series(event, release=None):
             (tsdb.models.organization_total_received, project.organization_id),
             (tsdb.models.project_total_forwarded, project.id),
             (tsdb.models.project_total_received, project.id),
+            (tsdb.models.key_total_received, key.id),
         ), now, int(count * 1.1))
         tsdb.incr_multi((
             (tsdb.models.organization_total_rejected, project.organization_id),
             (tsdb.models.project_total_rejected, project.id),
+            (tsdb.models.key_total_rejected, key.id),
         ), now, int(count * 0.1))
 
         frequencies = [
@@ -210,12 +214,14 @@ def create_sample_time_series(event, release=None):
             (tsdb.models.group, group.id),
         ), now, count)
         tsdb.incr_multi((
-            (tsdb.models.organization_total_received, group.project.organization_id),
-            (tsdb.models.project_total_received, group.project.id),
+            (tsdb.models.organization_total_received, project.organization_id),
+            (tsdb.models.project_total_received, project.id),
+            (tsdb.models.key_total_received, key.id),
         ), now, int(count * 1.1))
         tsdb.incr_multi((
-            (tsdb.models.organization_total_rejected, group.project.organization_id),
-            (tsdb.models.project_total_rejected, group.project.id),
+            (tsdb.models.organization_total_rejected, project.organization_id),
+            (tsdb.models.project_total_rejected, project.id),
+            (tsdb.models.key_total_rejected, key.id),
         ), now, int(count * 0.1))
 
         frequencies = [

package.json

@@ -10,6 +10,7 @@
     "babel-gettext-extractor": "^2.0.0",
     "babel-loader": "6.2.10",
     "babel-plugin-add-module-exports": "^0.2.1",
+    "babel-plugin-idx": "^1.5.1",
     "babel-plugin-transform-builtin-extend": "^1.1.0",
     "babel-plugin-transform-object-rest-spread": "^6.20.2",
     "babel-polyfill": "6.20.0",
@@ -24,6 +25,7 @@
     "file-loader": "0.8.4",
     "gettext-parser": "1.1.1",
     "history": "1.13.0",
+    "idx": "^1.5.0",
     "ios-device-list": "^1.1.17",
     "jed": "^1.1.0",
     "jquery": "2.1.4",

src/sentry/api/endpoints/project_key_details.py

@@ -3,6 +3,7 @@ from __future__ import absolute_import
 from rest_framework import serializers, status
 from rest_framework.response import Response
 
+from sentry import features
 from sentry.api.base import DocSection
 from sentry.api.bases.project import ProjectEndpoint
 from sentry.api.exceptions import ResourceDoesNotExist
@@ -34,9 +35,16 @@ def update_key_scenario(runner):
     )
 
 
+class RateLimitSerializer(serializers.Serializer):
+    count = serializers.IntegerField(min_value=0, required=False)
+    window = serializers.IntegerField(min_value=0, max_value=60 * 24,
+                                      required=False)
+
+
 class KeySerializer(serializers.Serializer):
     name = serializers.CharField(max_length=200, required=False)
     isActive = serializers.BooleanField(required=False)
+    rateLimit = RateLimitSerializer(required=False)
 
 
 class ProjectKeyDetailsEndpoint(ProjectEndpoint):
@@ -54,7 +62,6 @@ class ProjectKeyDetailsEndpoint(ProjectEndpoint):
 
         return Response(serialize(key, request.user), status=200)
 
-    @attach_scenarios([update_key_scenario])
     def put(self, request, project, key_id):
         """
         Update a Client Key
@@ -92,6 +99,14 @@ class ProjectKeyDetailsEndpoint(ProjectEndpoint):
             elif result.get('isActive') is False:
                 key.status = ProjectKeyStatus.INACTIVE
 
+            if features.has('projects:rate-limits', project):
+                if result.get('rateLimit', -1) is None:
+                    key.rate_limit_count = None
+                    key.rate_limit_window = None
+                elif result.get('rateLimit'):
+                    key.rate_limit_count = result['rateLimit']['count']
+                    key.rate_limit_window = result['rateLimit']['window']
+
             key.save()
 
             self.create_audit_entry(

src/sentry/api/endpoints/project_key_stats.py

@@ -0,0 +1,51 @@
+from __future__ import absolute_import
+
+import six
+
+from collections import OrderedDict
+from rest_framework.response import Response
+
+from sentry import tsdb
+from sentry.api.base import StatsMixin
+from sentry.api.bases.project import ProjectEndpoint
+from sentry.api.exceptions import ResourceDoesNotExist
+from sentry.models import ProjectKey
+
+
+class ProjectKeyStatsEndpoint(ProjectEndpoint, StatsMixin):
+    def get(self, request, project, key_id):
+        try:
+            key = ProjectKey.objects.get(
+                project=project,
+                public_key=key_id,
+                roles=ProjectKey.roles.store,
+            )
+        except ProjectKey.DoesNotExist:
+            raise ResourceDoesNotExist
+
+        stat_args = self._parse_args(request)
+
+        stats = OrderedDict()
+        for model, name in (
+            (tsdb.models.key_total_received, 'total'),
+            (tsdb.models.key_total_blacklisted, 'filtered'),
+            (tsdb.models.key_total_rejected, 'dropped'),
+        ):
+            result = tsdb.get_range(
+                model=model,
+                keys=[key.id],
+                **stat_args
+            )[key.id]
+            for ts, count in result:
+                stats.setdefault(int(ts), {})[name] = count
+
+        return Response([
+            {
+                'ts': ts,
+                'total': data['total'],
+                'dropped': data['dropped'],
+                'filtered': data['filtered'],
+                'accepted': data['total'] - data['dropped'] - data['filtered'],
+            }
+            for ts, data in six.iteritems(stats)
+        ])

src/sentry/api/serializers/models/project.py

@@ -96,7 +96,7 @@ class ProjectSerializer(Serializer):
         from sentry import features
 
         feature_list = []
-        for feature in ('global-events', 'data-forwarding'):
+        for feature in ('global-events', 'data-forwarding', 'rate-limits'):
             if features.has('projects:' + feature, obj, actor=user):
                 feature_list.append(feature)
 

src/sentry/api/serializers/models/project_key.py

@@ -17,6 +17,10 @@ class ProjectKeySerializer(Serializer):
             'secret': obj.secret_key,
             'projectId': obj.project_id,
             'isActive': obj.is_active,
+            'rateLimit': {
+                'window': obj.rate_limit_window,
+                'count': obj.rate_limit_count,
+            } if (obj.rate_limit_window and obj.rate_limit_count) else None,
             'dsn': {
                 'secret': obj.dsn_private,
                 'public': obj.dsn_public,

src/sentry/api/urls.py

@@ -70,6 +70,7 @@ from .endpoints.project_group_stats import ProjectGroupStatsEndpoint
 from .endpoints.project_index import ProjectIndexEndpoint
 from .endpoints.project_keys import ProjectKeysEndpoint
 from .endpoints.project_key_details import ProjectKeyDetailsEndpoint
+from .endpoints.project_key_stats import ProjectKeyStatsEndpoint
 from .endpoints.project_member_index import ProjectMemberIndexEndpoint
 from .endpoints.project_plugins import ProjectPluginsEndpoint
 from .endpoints.project_plugin_details import ProjectPluginDetailsEndpoint
@@ -320,6 +321,8 @@ urlpatterns = patterns(
     url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/keys/(?P<key_id>[^\/]+)/$',
         ProjectKeyDetailsEndpoint.as_view(),
         name='sentry-api-0-project-key-details'),
+    url(r'^projects/(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/keys/(?P<key_id>[^\/]+)/stats/$',
+        ProjectKeyStatsEndpoint.as_view()),
     url(r'^projects/(?P<organization_slug>[^/]+)/(?P<project_slug>[^/]+)/members/$',
         ProjectMemberIndexEndpoint.as_view(),
         name='sentry-api-0-project-member-index'),

src/sentry/conf/server.py

@@ -751,6 +751,7 @@ SENTRY_FEATURES = {
     'projects:dsym': False,
     'projects:sample-events': True,
     'projects:data-forwarding': True,
+    'projects:rate-limits': True,
 }
 
 # Default time zone for localization in the UI.