feat(event-tags): Implement highlight tags/context backend (#69044)

This PR will allow an organization with the `event-tags-tree-ui` flag to
create project options for highlights.

They are stored as simple lists of strings in the ProjectOptions table,
with `sentry:highlight_context` and `sentry:highlight_tags` keys.

When unset, they will default to a preset depending on the project
platform. When set, the new values will be respected, even if set to
empty (`[]`).

**Update**: it looks like we only want to pick out specific keys from
context items so I'll have to make some edits

**todo**
- [x] Add tests 
- [x] Update with schema validation for specific context keys (rather
than whole context items)

---------

Co-authored-by: Josh Ferge <josh.ferge@sentry.io>

src/sentry/api/endpoints/project_details.py

@@ -34,6 +34,7 @@ from sentry.grouping.enhancer import Enhancements
 from sentry.grouping.enhancer.exceptions import InvalidEnhancerConfig
 from sentry.grouping.fingerprinting import FingerprintingRules, InvalidFingerprintingConfig
 from sentry.ingest.inbound_filters import FilterTypes
+from sentry.issues.highlights import HighlightContextField
 from sentry.lang.native.sources import (
     InvalidSourcesError,
     parse_backfill_sources,
@@ -120,6 +121,8 @@ class ProjectMemberSerializer(serializers.Serializer):
         "performanceIssueCreationRate",
         "performanceIssueCreationThroughPlatform",
         "performanceIssueSendToPlatform",
+        "highlightContext",
+        "highlightTags",
     ]
 )
 class ProjectAdminSerializer(ProjectMemberSerializer):
@@ -179,6 +182,8 @@ class ProjectAdminSerializer(ProjectMemberSerializer):
     dataScrubberDefaults = serializers.BooleanField(required=False)
     sensitiveFields = ListField(child=serializers.CharField(), required=False)
     safeFields = ListField(child=serializers.CharField(), required=False)
+    highlightContext = HighlightContextField(required=False)
+    highlightTags = ListField(child=serializers.CharField(), required=False)
     storeCrashReports = serializers.IntegerField(
         min_value=-1, max_value=STORE_CRASH_REPORTS_MAX, required=False, allow_null=True
     )
@@ -639,6 +644,13 @@ class ProjectDetailsEndpoint(ProjectEndpoint):
         if result.get("safeFields") is not None:
             if project.update_option("sentry:safe_fields", result["safeFields"]):
                 changed_proj_settings["sentry:safe_fields"] = result["safeFields"]
+        if features.has("organizations:event-tags-tree-ui", project.organization):
+            if result.get("highlightContext") is not None:
+                if project.update_option("sentry:highlight_context", result["highlightContext"]):
+                    changed_proj_settings["sentry:highlight_context"] = result["highlightContext"]
+            if result.get("highlightTags") is not None:
+                if project.update_option("sentry:highlight_tags", result["highlightTags"]):
+                    changed_proj_settings["sentry:highlight_tags"] = result["highlightTags"]
         if result.get("storeCrashReports") is not None:
             if project.get_option("sentry:store_crash_reports") != result["storeCrashReports"]:
                 changed_proj_settings["sentry:store_crash_reports"] = result["storeCrashReports"]

src/sentry/api/serializers/models/project.py

@@ -24,6 +24,7 @@ from sentry.digests import backend as digests
 from sentry.eventstore.models import DEFAULT_SUBJECT_TEMPLATE
 from sentry.features.base import ProjectFeature
 from sentry.ingest.inbound_filters import FilterTypes
+from sentry.issues.highlights import get_highlight_preset_for_project
 from sentry.lang.native.sources import parse_sources, redact_source_secrets
 from sentry.lang.native.utils import convert_crashreport_count
 from sentry.models.environment import EnvironmentProject
@@ -906,6 +907,7 @@ class DetailedProjectSerializer(ProjectWithTeamSerializer):
                     "org": orgs[str(item.organization_id)],
                     "options": options_by_project[item.id],
                     "processing_issues": processing_issues_by_project.get(item.id, 0),
+                    "highlight_preset": get_highlight_preset_for_project(item),
                 }
             )
         return attrs
@@ -945,6 +947,14 @@ class DetailedProjectSerializer(ProjectWithTeamSerializer):
                 "verifySSL": bool(attrs["options"].get("sentry:verify_ssl", False)),
                 "scrubIPAddresses": bool(attrs["options"].get("sentry:scrub_ip_address", False)),
                 "scrapeJavaScript": bool(attrs["options"].get("sentry:scrape_javascript", True)),
+                "highlightTags": attrs["options"].get(
+                    "sentry:highlight_tags",
+                    attrs["highlight_preset"].get("tags", []),
+                ),
+                "highlightContext": attrs["options"].get(
+                    "sentry:highlight_context",
+                    attrs["highlight_preset"].get("context", {}),
+                ),
                 "groupingConfig": self.get_value_with_default(attrs, "sentry:grouping_config"),
                 "groupingEnhancements": self.get_value_with_default(
                     attrs, "sentry:grouping_enhancements"

src/sentry/issues/highlights.py

@@ -0,0 +1,65 @@
+import re
+from collections.abc import Mapping
+from typing import TypedDict
+
+from rest_framework import serializers
+
+from sentry.models.project import Project
+from sentry.utils.platform_categories import BACKEND, FRONTEND, MOBILE
+
+
+class HighlightContextField(serializers.Field):
+    def to_internal_value(self, data):
+        if not isinstance(data, dict):
+            raise serializers.ValidationError("Expected a dictionary.")
+
+        for key, value in data.items():
+            if not re.match(r"^.+$", key):
+                raise serializers.ValidationError(f"Key '{key}' is invalid.")
+            if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
+                raise serializers.ValidationError(f"Value for '{key}' must be a list of strings.")
+            # Remove duplicates
+            data[key] = list(set(value))
+
+        return data
+
+    def to_representation(self, value):
+        return value
+
+
+class HighlightPreset(TypedDict):
+    tags: list[str]
+    context: Mapping[str, list[str]]
+
+
+SENTRY_TAGS = ["handled", "level", "release", "environment"]
+
+BACKEND_HIGHLIGHTS: HighlightPreset = {
+    "tags": SENTRY_TAGS + ["url", "transaction", "status_code"],
+    "context": {"trace": ["trace_id"], "runtime": ["name", "version"]},
+}
+FRONTEND_HIGHLIGHTS: HighlightPreset = {
+    "tags": SENTRY_TAGS + ["url", "transaction", "browser", "replayId", "user"],
+    "context": {"browser": ["name"], "user": ["email"]},
+}
+MOBILE_HIGHLIGHTS: HighlightPreset = {
+    "tags": SENTRY_TAGS + ["mobile", "main_thread"],
+    "context": {"profile": ["profile_id"], "app": ["name"], "device": ["family"]},
+}
+
+FALLBACK_HIGHLIGHTS: HighlightPreset = {
+    "tags": SENTRY_TAGS,
+    "context": {"user": ["email"], "trace": ["trace_id"]},
+}
+
+
+def get_highlight_preset_for_project(project: Project) -> HighlightPreset:
+    if not project.platform or project.platform == "other":
+        return FALLBACK_HIGHLIGHTS
+    elif project.platform in FRONTEND:
+        return FRONTEND_HIGHLIGHTS
+    elif project.platform in BACKEND:
+        return BACKEND_HIGHLIGHTS
+    elif project.platform in MOBILE:
+        return MOBILE_HIGHLIGHTS
+    return FALLBACK_HIGHLIGHTS

src/sentry/models/options/project_option.py

@@ -30,6 +30,8 @@ OPTION_KEYS = frozenset(
         "sentry:builtin_symbol_sources",
         "sentry:symbol_sources",
         "sentry:sensitive_fields",
+        "sentry:highlight_tags",
+        "sentry:highlight_context",
         "sentry:csp_ignored_sources_defaults",
         "sentry:csp_ignored_sources",
         "sentry:default_environment",

src/sentry/utils/platform_categories.py

@@ -2,7 +2,7 @@ from django.utils.translation import gettext_lazy as _
 
 # Mirrors `const frontend` in sentry/static/app/data/platformCategories.tsx
 # When changing this file, make sure to keep sentry/static/app/data/platformCategories.tsx in sync.
-FRONTEND = [
+FRONTEND = {
     "dart",
     "javascript",
     "javascript-react",
@@ -18,11 +18,11 @@ FRONTEND = [
     "javascript-sveltekit",
     "javascript-astro",
     "unity",
-]
+}
 
 # Mirrors `const mobile` in sentry/static/app/data/platformCategories.tsx
 # When changing this file, make sure to keep sentry/static/app/data/platformCategories.tsx in sync.
-MOBILE = [
+MOBILE = {
     "android",
     "apple-ios",
     "cordova",
@@ -41,11 +41,11 @@ MOBILE = [
     "java-android",
     "cocoa-objc",
     "cocoa-swift",
-]
+}
 
 # Mirrors `const backend` in sentry/static/app/data/platformCategories.tsx
 # When changing this file, make sure to keep sentry/static/app/data/platformCategories.tsx in sync.
-BACKEND = [
+BACKEND = {
     "bun",
     "deno",
     "dotnet",
@@ -107,11 +107,11 @@ BACKEND = [
     "ruby-rack",
     "ruby-rails",
     "rust",
-]
+}
 
 # Mirrors `const serverless` in sentry/static/app/data/platformCategories.tsx
 # When changing this file, make sure to keep sentry/static/app/data/platformCategories.tsx in sync.
-SERVERLESS = [
+SERVERLESS = {
     "dotnet-awslambda",
     "dotnet-gcpfunctions",
     "node-awslambda",
@@ -121,11 +121,11 @@ SERVERLESS = [
     "python-azurefunctions",
     "python-gcpfunctions",
     "python-serverless",
-]
+}
 
 # Mirrors `const desktop` in sentry/static/app/data/platformCategories.tsx
 # When changing this file, make sure to keep sentry/static/app/data/platformCategories.tsx in sync.
-DESKTOP = [
+DESKTOP = {
     "apple-macos",
     "dotnet",
     "dotnet-maui",
@@ -144,11 +144,11 @@ DESKTOP = [
     "native-qt",
     "unity",
     "unreal",
-]
+}
 
 # TODO: @athena Remove this
 # This is only temporary since we decide the right category. Don't add anything here or your frontend experience will be broken
-TEMPORARY = ["nintendo"]
+TEMPORARY = {"nintendo"}
 
 CATEGORY_LIST = [
     {id: "browser", "name": _("Browser"), "platforms": FRONTEND},

tests/sentry/api/endpoints/test_project_details.py

@@ -13,6 +13,7 @@ from sentry import audit_log
 from sentry.constants import RESERVED_PROJECT_SLUGS, ObjectStatus
 from sentry.dynamic_sampling import DEFAULT_BIASES, RuleType
 from sentry.dynamic_sampling.rules.base import NEW_MODEL_THRESHOLD_IN_MINUTES
+from sentry.issues.highlights import get_highlight_preset_for_project
 from sentry.models.apitoken import ApiToken
 from sentry.models.auditlogentry import AuditLogEntry
 from sentry.models.deletedproject import DeletedProject
@@ -838,6 +839,100 @@ class ProjectUpdateTest(APITestCase):
             'Invalid syntax near "er ror" (line 1),\nDeep wildcard used more than once (line 2)',
         ]
 
+    def test_highlight_tags(self):
+        # Default with or without flag, ignore update attempt
+        highlight_tags = ["bears", "beets", "battlestar_galactica"]
+        resp = self.get_success_response(
+            self.org_slug,
+            self.proj_slug,
+            highlightTags=highlight_tags,
+        )
+        assert self.project.get_option("sentry:highlight_tags") is None
+
+        preset = get_highlight_preset_for_project(self.project)
+        assert resp.data["highlightTags"] == preset["tags"]
+
+        with self.feature("organizations:event-tags-tree-ui"):
+            # Set to custom
+            resp = self.get_success_response(
+                self.org_slug,
+                self.proj_slug,
+                highlightTags=highlight_tags,
+            )
+            assert self.project.get_option("sentry:highlight_tags") == highlight_tags
+            assert resp.data["highlightTags"] == highlight_tags
+
+            # Set to empty
+            resp = self.get_success_response(
+                self.org_slug,
+                self.proj_slug,
+                highlightTags=[],
+            )
+            assert self.project.get_option("sentry:highlight_tags") == []
+            assert resp.data["highlightTags"] == []
+
+    def test_highlight_context(self):
+        # Default with or without flag, ignore update attempt
+        highlight_context_type = "bird-words"
+        highlight_context = {highlight_context_type: ["red", "robin", "blue", "jay", "red", "blue"]}
+        resp = self.get_success_response(
+            self.org_slug,
+            self.proj_slug,
+            highlightContext=highlight_context,
+        )
+        assert self.project.get_option("sentry:highlight_context") is None
+
+        preset = get_highlight_preset_for_project(self.project)
+        assert resp.data["highlightContext"] == preset["context"]
+
+        with self.feature("organizations:event-tags-tree-ui"):
+            # Set to custom
+            resp = self.get_success_response(
+                self.org_slug,
+                self.proj_slug,
+                highlightContext=highlight_context,
+            )
+            option_result = self.project.get_option("sentry:highlight_context")
+            resp_result = resp.data["highlightContext"]
+            for highlight_context_key in highlight_context[highlight_context_type]:
+                assert highlight_context_key in option_result[highlight_context_type]
+                assert highlight_context_key in resp_result[highlight_context_type]
+            # Filters duplicates
+            assert (
+                len(option_result[highlight_context_type])
+                == len(resp_result[highlight_context_type])
+                == 4
+            )
+
+            # Set to empty
+            resp = self.get_success_response(
+                self.org_slug,
+                self.proj_slug,
+                highlightContext={},
+            )
+            assert self.project.get_option("sentry:highlight_context") == {}
+            assert resp.data["highlightContext"] == {}
+
+            # Checking validation
+            resp = self.get_error_response(
+                self.org_slug,
+                self.proj_slug,
+                highlightContext=["bird-words", ["red", "blue"]],
+            )
+            assert "Expected a dictionary" in resp.data["highlightContext"][0]
+            resp = self.get_error_response(
+                self.org_slug,
+                self.proj_slug,
+                highlightContext={"": ["empty", "context", "type"]},
+            )
+            assert "Key '' is invalid" in resp.data["highlightContext"][0]
+            resp = self.get_error_response(
+                self.org_slug,
+                self.proj_slug,
+                highlightContext={"bird-words": ["invalid", 123, "integer"]},
+            )
+            assert "must be a list of strings" in resp.data["highlightContext"][0]
+
     def test_store_crash_reports(self):
         resp = self.get_success_response(self.org_slug, self.proj_slug, storeCrashReports=10)
         assert self.project.get_option("sentry:store_crash_reports") == 10