Add optimization to look up issue ids in python when possible

apps/event_ingest/process_event.py

@@ -159,6 +159,27 @@ def generate_tags(event: IngestIssueEvent) -> dict[str, str]:
     return {key: value for key, value in tags.items() if value}
 
 
+def check_set_issue_id(
+    processing_events: list[ProcessingEvent],
+    project_id: int,
+    issue_hash: str,
+    issue_id: int,
+):
+    """
+    It's common to receive two duplicate events at the same time,
+    where the issue has never been seen before. This is an optimization
+    that checks if there is a known project/hash. If so, we can infer the
+    issue_id.
+    """
+    for event in processing_events:
+        if (
+            event.issue_id is None
+            and event.event.project_id == project_id
+            and event.issue_hash == issue_hash
+        ):
+            event.issue_id = issue_id
+
+
 def process_issue_events(ingest_events: list[InterchangeIssueEvent]):
     """
     Accepts a list of events to ingest. Events should be:
@@ -297,9 +318,15 @@ def process_issue_events(ingest_events: list[InterchangeIssueEvent]):
                         search_vector=SearchVector(Value(issue_defaults["title"])),
                         **issue_defaults,
                     )
-                    IssueHash.objects.create(
+                    new_issue_hash = IssueHash.objects.create(
                         issue=issue, value=issue_hash, project_id=project_id
                     )
+                    check_set_issue_id(
+                        processing_events,
+                        issue.project_id,
+                        new_issue_hash.value,
+                        issue.id,
+                    )
                 processing_event.issue_id = issue.id
                 processing_event.issue_created = True
             except IntegrityError:

apps/event_ingest/tests/test_process_issue_event.py

@@ -38,7 +38,7 @@ class IssueEventIngestTestCase(EventIngestTestCase):
             events.append(
                 InterchangeIssueEvent(project_id=self.project.id, payload=payload)
             )
-        with self.assertNumQueries(13):
+        with self.assertNumQueries(7):
             process_issue_events(events)
         self.assertEqual(Issue.objects.count(), 1)
         self.assertEqual(IssueHash.objects.count(), 1)

docker-compose.locust.yml

@@ -10,6 +10,8 @@ services:
     environment:
       DJANGO_SETTINGS_MODULE: glitchtip.settings
       SECRET_KEY: nope
+      #GLITCHTIP_ENABLE_NEW_ISSUES: "true"
+      #IS_LOAD_TEST: "true"
     deploy:
       resources:
         limits:

docker-compose.yml

@@ -7,9 +7,12 @@ x-environment: &default-environment
   DEBUG: "true"
   EMAIL_BACKEND: "django.core.mail.backends.console.EmailBackend"
   ENABLE_OBSERVABILITY_API: "true"
-  # GLITCHTIP_ENABLE_NEW_ISSUES: "true"
-  # IS_LOAD_TEST: "true"
-  CELERY_WORKER_CONCURRENCY: 1
+  #GLITCHTIP_ENABLE_NEW_ISSUES: "true"
+  #IS_LOAD_TEST: "true"
+  CELERY_WORKER_CONCURRENCY: 4
+  CELERY_WORKER_PREFETCH_MULTIPLIER: 25
+  CELERY_WORKER_POOL: "threads"
+  CELERY_SKIP_CHECKS: "true"
   PYTHONBREAKPOINT: "ipdb.set_trace"
 
 x-depends_on: &default-depends_on
@@ -28,6 +31,7 @@ services:
     build: .
     user: root # Allows for usage of ipdb, apt, etc in dev
     command: ./manage.py runserver 0.0.0.0:8000
+    #command: python -m gunicorn glitchtip.asgi:application -w 4 -k uvicorn.workers.UvicornWorker -b 0.0.0.0
     depends_on: *default-depends_on
     volumes: *default-volumes
     ports: