feat(view-hierarchy): Add Dart plugin to handle deobfuscation (#46129)

Adds a new plugin to handle Flutter deobfuscation. Since we don't currently have
the debug files saved in Sentry, this code is awaiting future changes in
Sentry CLI and the SDK to begin uploading the file, at which point we
can further validate if this needs any updates and roll it out to
customers.

I've made the assumption that the file has no preprocessing applied to
it (i.e. we haven't turned it into a map by the time its stored) and
mocked the file as such in my test cases.

I used the old function in the java plugin as a template to avoid
duplicating code r.e. the attachments cache. As for the other functions,
I've decided to copy and modify them to fit this case because either
they're shorter functions and there isn't a shared location for the
general logic, or the logic is quite different from the Java version in
which case it felt more manageable to treat them as separate
implementations.

Closes #46036

src/sentry/lang/dart/apps.py

@@ -0,0 +1,12 @@
+from django.apps import AppConfig
+
+
+class Config(AppConfig):
+    name = "sentry.lang.dart"
+
+    def ready(self):
+        from sentry.plugins.base import register
+
+        from .plugin import DartPlugin
+
+        register(DartPlugin)

src/sentry/lang/dart/plugin.py

@@ -0,0 +1,32 @@
+from sentry.lang.dart.utils import deobfuscate_view_hierarchy, has_dart_symbols_file
+from sentry.models import Project
+from sentry.plugins.base.v2 import Plugin2
+from sentry.utils.options import sample_modulo
+
+
+class DartPlugin(Plugin2):
+    """
+    This plugin is responsible for Dart specific processing on events or attachments.
+
+    Currently, this plugin is applies deobfuscation for view hierarchies, but
+    since we do not have the proper debug files stored in Sentry, this plugin is
+    disabled. When we are ready to roll out dart deobfuscation, this plugin should
+    be enabled and rolled out through the options system.
+    """
+
+    # TODO: This should be removed and it should not be possible to disable the plugin
+    # when we are ready to roll out dart deobfuscation.
+    enabled = False
+
+    def can_configure_for_project(self, project, **kwargs):
+        return False
+
+    def get_event_preprocessors(self, data):
+        project = Project.objects.get_from_cache(id=data["project"])
+        if not sample_modulo(
+            "processing.view-hierarchies-dart-deobfuscation", project.organization.id
+        ):
+            return []
+
+        if has_dart_symbols_file(data):
+            return [deobfuscate_view_hierarchy]

src/sentry/lang/dart/utils.py

@@ -0,0 +1,114 @@
+import os
+import re
+
+import sentry_sdk
+
+from sentry.eventstore.models import Event
+from sentry.lang.java.utils import deobfuscation_template
+from sentry.models import Project, ProjectDebugFile
+from sentry.utils import json
+from sentry.utils.safe import get_path
+
+# Obfuscated type values are either in the form of "xyz" or "xyz<abc>" where
+# both "xyz" or "abc" need to be deobfuscated. It may also be possible for
+# the values to be more complicated such as "_xyz", so the regex should capture
+# any values other than "<" and ">".
+VIEW_HIERARCHY_TYPE_REGEX = re.compile(r"([^<>]+)(?:<([^<>]+)>)?")
+
+
+def is_valid_image(image):
+    return bool(image) and image.get("type") == "dart_symbols" and image.get("uuid") is not None
+
+
+def has_dart_symbols_file(data):
+    """
+    Checks whether an event contains a dart symbols file
+    """
+    images = get_path(data, "debug_meta", "images", filter=True)
+    return get_path(images, 0, "type") == "dart_symbols"
+
+
+def get_dart_symbols_images(event: Event):
+    return {
+        str(image["uuid"]).lower()
+        for image in get_path(event, "debug_meta", "images", filter=is_valid_image, default=())
+    }
+
+
+def generate_dart_symbols_map(uuid: str, project: Project):
+    """
+    NOTE: This function makes assumptions about the structure of the debug file
+    since we are not currently storing the file. This may need to be updated if we
+    decide to do some pre-processing on the debug file before storing it.
+
+    In its current state, the debug file is expected to be a json file with a list
+    of strings. The strings alternate between deobfuscated and obfuscated names.
+
+    If we preprocess it into a map, we can remove this code and just fetch the file.
+    """
+    obfuscated_to_deobfuscated_name_map = {}
+    with sentry_sdk.start_span(op="dart_symbols.generate_dart_symbols_map") as span:
+        try:
+            dif_paths = ProjectDebugFile.difcache.fetch_difs(project, [uuid], features=["mapping"])
+            debug_file_path = dif_paths.get(uuid)
+            if debug_file_path is None:
+                return
+
+            dart_symbols_file_size_in_mb = os.path.getsize(debug_file_path) / (1024 * 1024.0)
+            span.set_tag("dart_symbols_file_size_in_mb", dart_symbols_file_size_in_mb)
+
+            with open(debug_file_path) as f:
+                debug_array = json.loads(f.read())
+
+            if len(debug_array) % 2 != 0:
+                raise Exception("Debug array contains an odd number of elements")
+
+            # Obfuscated names are the odd indices and deobfuscated names are the even indices
+            obfuscated_to_deobfuscated_name_map = dict(zip(debug_array[1::2], debug_array[::2]))
+        except Exception as err:
+            sentry_sdk.capture_exception(err)
+            return
+
+    return obfuscated_to_deobfuscated_name_map
+
+
+def _deobfuscate_view_hierarchy(event_data: Event, project: Project, view_hierarchy):
+    """
+    Deobfuscates a view hierarchy in-place.
+
+    If we're unable to fetch a dart symbols uuid, then the view hierarchy remains unmodified.
+    """
+    dart_symbols_uuids = get_dart_symbols_images(event_data)
+    if len(dart_symbols_uuids) == 0:
+        return
+
+    with sentry_sdk.start_span(op="dart_symbols.deobfuscate_view_hierarchy_data"):
+        for dart_symbols_uuid in dart_symbols_uuids:
+            map = generate_dart_symbols_map(dart_symbols_uuid, project)
+            if map is None:
+                return
+
+            windows_to_deobfuscate = [*view_hierarchy.get("windows")]
+            while windows_to_deobfuscate:
+                window = windows_to_deobfuscate.pop()
+
+                if window.get("type") is None:
+                    # If there is no type, then skip this window
+                    continue
+
+                matcher = re.match(VIEW_HIERARCHY_TYPE_REGEX, window.get("type"))
+                if not matcher:
+                    continue
+                obfuscated_values = matcher.groups()
+                for obfuscated_value in obfuscated_values:
+                    if obfuscated_value is not None and obfuscated_value in map:
+                        window["type"] = window["type"].replace(
+                            obfuscated_value, map[obfuscated_value]
+                        )
+
+                if children := window.get("children"):
+                    windows_to_deobfuscate.extend(children)
+
+
+def deobfuscate_view_hierarchy(data):
+    return deobfuscation_template(data, "dart_symbols", _deobfuscate_view_hierarchy)

src/sentry/lang/java/utils.py

@@ -5,13 +5,12 @@ from symbolic import ProguardMapper
 
 from sentry.attachments import CachedAttachment, attachment_cache
 from sentry.eventstore.models import Event
+from sentry.ingest.ingest_consumer import CACHE_TIMEOUT
 from sentry.models import Project, ProjectDebugFile
 from sentry.utils import json
 from sentry.utils.cache import cache_key_for_event
 from sentry.utils.safe import get_path
 
-CACHE_TIMEOUT = 3600
-
 
 def is_valid_image(image):
     return bool(image) and image.get("type") == "proguard" and image.get("uuid") is not None
@@ -80,7 +79,12 @@ def _deobfuscate_view_hierarchy(event_data: Event, project: Project, view_hierar
                     windows_to_deobfuscate.extend(children)
 
 
-def deobfuscate_view_hierarchy(data):
+def deobfuscation_template(data, map_type, deobfuscation_fn):
+    """
+    Template for operations involved in deobfuscating view hierarchies.
+
+    The provided deobfuscation function is expected to modify the view hierarchy dict in-place.
+    """
     project = Project.objects.get_from_cache(id=data["project"])
 
     cache_key = cache_key_for_event(data)
@@ -89,12 +93,12 @@ def deobfuscate_view_hierarchy(data):
     if not any(attachment.type == "event.view_hierarchy" for attachment in attachments):
         return
 
-    with sentry_sdk.start_transaction(name="proguard.deobfuscate_view_hierarchy", sampled=True):
+    with sentry_sdk.start_transaction(name=f"{map_type}.deobfuscate_view_hierarchy", sampled=True):
         new_attachments = []
         for attachment in attachments:
             if attachment.type == "event.view_hierarchy":
                 view_hierarchy = json.loads(attachment_cache.get_data(attachment))
-                _deobfuscate_view_hierarchy(data, project, view_hierarchy)
+                deobfuscation_fn(data, project, view_hierarchy)
 
                 # Reupload to cache as a unchunked data
                 new_attachments.append(
@@ -111,3 +115,7 @@ def deobfuscate_view_hierarchy(data):
                 new_attachments.append(attachment)
 
         attachment_cache.set(cache_key, attachments=new_attachments, timeout=CACHE_TIMEOUT)
+
+
+def deobfuscate_view_hierarchy(data):
+    deobfuscation_template(data, "proguard", _deobfuscate_view_hierarchy)

src/sentry/options/defaults.py

@@ -429,6 +429,9 @@ register("store.reprocessing-force-disable", default=False)
 
 register("store.race-free-group-creation-force-disable", default=False)
 
+# Option to enable dart deobfuscation on ingest
+register("processing.view-hierarchies-dart-deobfuscation", default=0.0)
+
 
 # ## sentry.killswitches
 #

tests/sentry/lang/dart/test_utils.py

@@ -0,0 +1,88 @@
+import re
+import tempfile
+from unittest import mock
+
+from sentry.lang.dart.utils import (
+    VIEW_HIERARCHY_TYPE_REGEX,
+    _deobfuscate_view_hierarchy,
+    generate_dart_symbols_map,
+)
+
+MOCK_DEBUG_FILE = b'["","","_NativeInteger","_NativeInteger","SemanticsAction","er","ButtonTheme","mD","_entry","_YMa"]'
+MOCK_DEBUG_MAP = {
+    "": "",
+    "_NativeInteger": "_NativeInteger",
+    "er": "SemanticsAction",
+    "mD": "ButtonTheme",
+    "_YMa": "_entry",
+}
+
+
+def test_view_hierarchy_type_regex():
+    matcher = re.match(VIEW_HIERARCHY_TYPE_REGEX, "abc")
+    assert matcher
+    assert matcher.groups() == ("abc", None)
+
+    matcher = re.match(VIEW_HIERARCHY_TYPE_REGEX, "abc<xyz>")
+    assert matcher
+    assert matcher.groups() == ("abc", "xyz")
+
+    matcher = re.match(VIEW_HIERARCHY_TYPE_REGEX, "_abc<_xyz@1>")
+    assert matcher
+    assert matcher.groups() == ("_abc", "_xyz@1")
+
+
+def test_generate_dart_symbols_map():
+    with tempfile.NamedTemporaryFile() as mocked_debug_file:
+        mocked_debug_file.write(MOCK_DEBUG_FILE)
+        mocked_debug_file.seek(0)
+
+        # Mock the dif file to return a map from the test uuid to the
+        # mocked file location for reading since we don't have the real file
+        with mock.patch(
+            "sentry.models.ProjectDebugFile.difcache.fetch_difs",
+            return_value={"test-uuid": mocked_debug_file.name},
+        ):
+            map = generate_dart_symbols_map("test-uuid", mock.Mock())
+
+            assert map == MOCK_DEBUG_MAP
+
+
+@mock.patch("sentry.lang.dart.utils.generate_dart_symbols_map", return_value=MOCK_DEBUG_MAP)
+@mock.patch("sentry.lang.dart.utils.get_dart_symbols_images", return_value=["test-uuid"])
+def test_view_hierarchy_deobfuscation(mock_images, mock_map):
+    test_view_hierarchy = {
+        "windows": [
+            {
+                "type": "mD",
+                "children": [
+                    {
+                        "type": "er",
+                        "children": [
+                            {"type": "_YMa<er>", "children": [{"type": "_NativeInteger"}]}
+                        ],
+                    },
+                ],
+            }
+        ]
+    }
+    _deobfuscate_view_hierarchy(mock.Mock(), mock.Mock(), test_view_hierarchy)
+
+    assert test_view_hierarchy == {
+        "windows": [
+            {
+                "type": "ButtonTheme",
+                "children": [
+                    {
+                        "type": "SemanticsAction",
+                        "children": [
+                            {
+                                "type": "_entry<SemanticsAction>",
+                                "children": [{"type": "_NativeInteger"}],
+                            }
+                        ],
+                    }
+                ],
+            }
+        ]
+    }