Use symbolicator proguard processing (#66764)

This adds Symbolicator bindings for proguard/JVM processing. All of it
is gated behind the config settings
`"symbolicator.proguard-processing-projects"` and
`"symbolicator.proguard-processing-sample-rate"`.

The implementation passes all the tests in `java/test_plugin`.

Currently, proguard symbolication uses the same Celery queue and
Symbolicator pool as native. This may need to change before we use this
in earnest.

Makefile

@@ -170,6 +170,7 @@ test-symbolicator:
 	@echo "--> Running symbolicator tests"
 	pytest tests/symbolicator -vv --cov . --cov-report="xml:.artifacts/symbolicator.coverage.xml"
 	pytest tests/relay_integration/lang/javascript/ -vv -m symbolicator
+	pytest tests/relay_integration/lang/java/ -vv -m symbolicator
 	@echo ""
 
 test-acceptance: node-version-check

src/sentry/lang/java/plugin.py

@@ -290,6 +290,8 @@ class JavaPlugin(Plugin2):
         return False
 
     def get_stacktrace_processors(self, data, stacktrace_infos, platforms, **kwargs):
+        if data.pop("processed_by_symbolicator", False):
+            return []
         if "java" in platforms:
             return [JavaSourceLookupStacktraceProcessor]
 

src/sentry/lang/java/processing.py

@@ -1,9 +1,21 @@
+import logging
 import re
+from typing import Any
 
+from sentry.lang.java.utils import get_jvm_images, get_proguard_images
+from sentry.lang.native.error import SymbolicationFailed, write_error
+from sentry.lang.native.symbolicator import Symbolicator
+from sentry.models.eventerror import EventError
+from sentry.models.project import Project
+from sentry.models.release import Release
+from sentry.stacktraces.processing import find_stacktraces_in_data
+from sentry.utils import metrics
 from sentry.utils.safe import get_path
 
+logger = logging.getLogger(__name__)
 
-def deobfuscate_exception_value(data):
+
+def deobfuscate_exception_value(data: Any) -> Any:
     # Deobfuscate the exception value by regex replacing
     # Mapping constructed by taking the last lines from the deobfuscated stacktrace and raw stacktrace
     exception = get_path(data, "exception", "values", -1)
@@ -23,3 +35,221 @@ def deobfuscate_exception_value(data):
         )
 
     return data
+
+
+def _merge_frame(new_frame: dict[str, Any], symbolicated: dict[str, Any]):
+    """Merges `symbolicated` into `new_frame`. This updates
+    `new_frame` in place."""
+
+    if symbolicated.get("function"):
+        new_frame["function"] = symbolicated["function"]
+
+    if symbolicated.get("abs_path"):
+        new_frame["abs_path"] = symbolicated["abs_path"]
+
+    # Clear abs_path if Symbolicator unset it
+    elif new_frame.get("abs_path"):
+        del new_frame["abs_path"]
+
+    if symbolicated.get("filename"):
+        new_frame["filename"] = symbolicated["filename"]
+
+    # Clear abs_path if Symbolicator unset it
+    elif new_frame.get("filename"):
+        del new_frame["filename"]
+
+    if symbolicated.get("lineno") is not None:
+        new_frame["lineno"] = symbolicated["lineno"]
+    if symbolicated.get("module"):
+        new_frame["module"] = symbolicated["module"]
+    if symbolicated.get("in_app") is not None:
+        new_frame["in_app"] = symbolicated["in_app"]
+
+    if symbolicated.get("pre_context"):
+        new_frame["pre_context"] = symbolicated["pre_context"]
+    if symbolicated.get("context_line"):
+        new_frame["context_line"] = symbolicated["context_line"]
+    if symbolicated.get("post_context"):
+        new_frame["post_context"] = symbolicated["post_context"]
+
+
+def _handles_frame(frame: dict[str, Any], platform: str) -> bool:
+    "Returns whether the frame should be symbolicated by JVM symbolication."
+
+    return (
+        "function" in frame and "module" in frame and (frame.get("platform") or platform) == "java"
+    )
+
+
+FRAME_FIELDS = ("abs_path", "lineno", "function", "module", "filename", "in_app")
+
+
+def _normalize_frame(raw_frame: Any, index: int) -> dict:
+    "Normalizes the frame into just the fields necessary for symbolication and adds an index."
+
+    frame = {"index": index}
+    for key in FRAME_FIELDS:
+        if (value := raw_frame.get(key)) is not None:
+            frame[key] = value
+
+    return frame
+
+
+def _get_exceptions_for_symbolication(data: Any) -> list[dict[str, Any]]:
+    "Returns the exceptions contained in `data` for symbolication."
+
+    exceptions = []
+
+    for exc in get_path(data, "exception", "values", filter=True, default=()):
+        if exc.get("type") is not None and exc.get("module") is not None:
+            exceptions.append({"type": exc["type"], "module": exc["module"]})
+    return exceptions
+
+
+def _handle_response_status(event_data: Any, response_json: dict[str, Any]) -> bool | None:
+    """Checks the response from Symbolicator and reports errors.
+    Returns `True` on success."""
+
+    if not response_json:
+        error = SymbolicationFailed(type=EventError.NATIVE_INTERNAL_FAILURE)
+    elif response_json["status"] == "completed":
+        return True
+    elif response_json["status"] == "failed":
+        error = SymbolicationFailed(
+            message=response_json.get("message") or None,
+            type=EventError.NATIVE_SYMBOLICATOR_FAILED,
+        )
+    else:
+        logger.error("Unexpected symbolicator status: %s", response_json["status"])
+        error = SymbolicationFailed(type=EventError.NATIVE_INTERNAL_FAILURE)
+
+    write_error(error, event_data)
+    return None
+
+
+def _get_release_package(project: Project, release_name: str | None) -> str | None:
+    """Gets the release package for the given project and release."""
+
+    if not release_name:
+        return None
+
+    release = Release.get(project=project, version=release_name)
+    return release.package if release else None
+
+
+def map_symbolicator_process_jvm_errors(
+    errors: list[dict[str, Any]] | None,
+) -> list[dict[str, Any]]:
+    "Maps processing errors reported by Symbolicator to existing Python errors."
+
+    if errors is None:
+        return []
+
+    mapped_errors = []
+
+    for error in errors:
+        ty = error["type"]
+        uuid = error["uuid"]
+
+        if ty == "missing":
+            mapped_errors.append(
+                {
+                    "symbolicator_type": ty,
+                    "type": EventError.PROGUARD_MISSING_MAPPING,
+                    "mapping_uuid": uuid,
+                }
+            )
+        # according to the `test_error_on_resolving` test, a completely
+        # broken file should result in a `PROGUARD_MISSING_LINENO` error
+        elif ty == "no_line_info" or ty == "invalid":
+            mapped_errors.append(
+                {
+                    "symbolicator_type": ty,
+                    "type": EventError.PROGUARD_MISSING_LINENO,
+                    "mapping_uuid": uuid,
+                }
+            )
+
+    return mapped_errors
+
+
+def process_jvm_stacktraces(symbolicator: Symbolicator, data: Any) -> Any:
+    """Uses Symbolicator to symbolicate a JVM event."""
+
+    modules = []
+    modules.extend([{"uuid": id, "type": "proguard"} for id in get_proguard_images(data)])
+    modules.extend([{"uuid": id, "type": "source"} for id in get_jvm_images(data)])
+
+    stacktrace_infos = find_stacktraces_in_data(data)
+    stacktraces = [
+        {
+            "frames": [
+                _normalize_frame(frame, index)
+                for index, frame in enumerate(sinfo.stacktrace.get("frames") or ())
+                if _handles_frame(frame, data.get("platform", "unknown"))
+            ],
+        }
+        for sinfo in stacktrace_infos
+    ]
+
+    exceptions = _get_exceptions_for_symbolication(data)
+
+    metrics.incr("proguard.symbolicator.events")
+
+    if not any(stacktrace["frames"] for stacktrace in stacktraces) and not exceptions:
+        metrics.incr("proguard.symbolicator.events.skipped")
+        return
+
+    release_package = _get_release_package(symbolicator.project, data.get("release"))
+    metrics.incr("process.java.symbolicate.request")
+    response = symbolicator.process_jvm(
+        exceptions=exceptions,
+        stacktraces=stacktraces,
+        modules=modules,
+        release_package=release_package,
+    )
+
+    if not _handle_response_status(data, response):
+        return
+
+    data["processed_by_symbolicator"] = True
+
+    processing_errors = response.get("errors", [])
+    if len(processing_errors) > 0:
+        data.setdefault("errors", []).extend(map_symbolicator_process_jvm_errors(processing_errors))
+
+    assert len(stacktraces) == len(response["stacktraces"]), (stacktraces, response)
+
+    for sinfo, complete_stacktrace in zip(stacktrace_infos, response["stacktraces"]):
+        raw_frames = sinfo.stacktrace["frames"]
+        complete_frames = complete_stacktrace["frames"]
+        sinfo.stacktrace["frames"] = []
+
+        for index, raw_frame in enumerate(raw_frames):
+            # If symbolicator returned any matching frames for this raw_frame, use them,
+            # otherwise use the raw_frame itself.
+            matching_frames = [frame for frame in complete_frames if frame["index"] == index]
+            if matching_frames:
+                for returned in matching_frames:
+                    new_frame = dict(raw_frame)
+                    _merge_frame(new_frame, returned)
+                    sinfo.stacktrace["frames"].append(new_frame)
+            else:
+                sinfo.stacktrace["frames"].append(raw_frame)
+
+        if sinfo.container is not None:
+            sinfo.container["raw_stacktrace"] = {
+                "frames": raw_frames,
+            }
+
+    assert len(exceptions) == len(response["exceptions"])
+
+    # NOTE: we are using the `data.exception.values` directory here
+    for exception, complete_exception in zip(
+        get_path(data, "exception", "values", filter=True, default=()),
+        response["exceptions"],
+    ):
+        exception["type"] = complete_exception["type"]
+        exception["module"] = complete_exception["module"]
+
+    return data

src/sentry/lang/java/utils.py

@@ -5,7 +5,9 @@ from typing import Any
 
 import sentry_sdk
 
+from sentry import options
 from sentry.attachments import CachedAttachment, attachment_cache
+from sentry.features.rollout import in_rollout_group
 from sentry.ingest.consumer.processors import CACHE_TIMEOUT
 from sentry.lang.java.proguard import open_proguard_mapper
 from sentry.models.debugfile import ProjectDebugFile
@@ -134,3 +136,14 @@ def deobfuscation_template(data, map_type, deobfuscation_fn):
 
 def deobfuscate_view_hierarchy(data):
     deobfuscation_template(data, "proguard", _deobfuscate_view_hierarchy)
+
+
+SYMBOLICATOR_PROGUARD_PROJECTS_OPTION = "symbolicator.proguard-processing-projects"
+SYMBOLICATOR_PROGUARD_SAMPLE_RATE_OPTION = "symbolicator.proguard-processing-sample-rate"
+
+
+def should_use_symbolicator_for_proguard(project_id: int) -> bool:
+    if project_id in options.get(SYMBOLICATOR_PROGUARD_PROJECTS_OPTION, []):
+        return True
+
+    return in_rollout_group(SYMBOLICATOR_PROGUARD_SAMPLE_RATE_OPTION, project_id)

src/sentry/lang/native/symbolicator.py

@@ -29,17 +29,31 @@ MAX_ATTEMPTS = 3
 logger = logging.getLogger(__name__)
 
 
+class SymbolicatorPlatform(Enum):
+    """The platforms for which we want to
+    invoke Symbolicator."""
+
+    jvm = "jvm"
+    js = "js"
+    native = "native"
+
+
 @dataclass(frozen=True)
 class SymbolicatorTaskKind:
-    is_js: bool = False
+    """Bundles information about a symbolication task:
+    the platform, whether it's on the low priority queue, and
+    whether it's an existing event being reprocessed.
+    """
+
+    platform: SymbolicatorPlatform
     is_low_priority: bool = False
     is_reprocessing: bool = False
 
     def with_low_priority(self, is_low_priority: bool) -> SymbolicatorTaskKind:
         return dataclasses.replace(self, is_low_priority=is_low_priority)
 
-    def with_js(self, is_js: bool) -> SymbolicatorTaskKind:
-        return dataclasses.replace(self, is_js=is_js)
+    def with_platform(self, platform: SymbolicatorPlatform) -> SymbolicatorTaskKind:
+        return dataclasses.replace(self, platform=platform)
 
 
 class SymbolicatorPools(Enum):
@@ -59,12 +73,13 @@ class Symbolicator:
     ):
         URLS = settings.SYMBOLICATOR_POOL_URLS
         pool = SymbolicatorPools.default.value
+        # TODO: Add a pool for JVM
         if task_kind.is_low_priority:
-            if task_kind.is_js:
+            if task_kind.platform == SymbolicatorPlatform.js:
                 pool = SymbolicatorPools.lpq_js.value
             else:
                 pool = SymbolicatorPools.lpq.value
-        elif task_kind.is_js:
+        elif task_kind.platform == SymbolicatorPlatform.js:
             pool = SymbolicatorPools.js.value
 
         base_url = (
@@ -228,8 +243,10 @@ class Symbolicator:
 
         :param exceptions: The event's exceptions. These must contain a `type` and a `module`.
         :param stacktraces: The event's stacktraces. Frames must contain a `function` and a `module`.
-        :param modules: ProGuard modules to use for deobfuscation. They must contain a `uuid`.
+        :param modules: ProGuard modules and source bundles. They must contain a `uuid` and have a
+                        `type` of either "proguard" or "source".
         :param release_package: The name of the release's package. This is optional.
+                                Used for determining whether frames are in-app.
         :param apply_source_context: Whether to add source context to frames.
         """
         source = get_internal_source(self.project)

src/sentry/options/defaults.py

@@ -587,6 +587,11 @@ register(
     flags=FLAG_AUTOMATOR_MODIFIABLE,
 )
 
+# Enable use of Symbolicator proguard processing for specific projects.
+register("symbolicator.proguard-processing-projects", type=Sequence, default=[])
+# Enable use of Symbolicator proguard processing for fraction of projects.
+register("symbolicator.proguard-processing-sample-rate", default=0.0)
+
 # Post Process Error Hook Sampling
 register(
     "post-process.use-error-hook-sampling", default=False, flags=FLAG_AUTOMATOR_MODIFIABLE

src/sentry/profiles/task.py

@@ -17,7 +17,7 @@ from sentry.constants import DataCategory
 from sentry.lang.java.proguard import open_proguard_mapper
 from sentry.lang.javascript.processing import _handles_frame as is_valid_javascript_frame
 from sentry.lang.native.processing import _merge_image
-from sentry.lang.native.symbolicator import Symbolicator, SymbolicatorTaskKind
+from sentry.lang.native.symbolicator import Symbolicator, SymbolicatorPlatform, SymbolicatorTaskKind
 from sentry.lang.native.utils import native_images_from_data
 from sentry.models.debugfile import ProjectDebugFile
 from sentry.models.eventerror import EventError
@@ -481,9 +481,12 @@ def run_symbolicate(
         if duration > settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT:
             raise SymbolicationTimeout
 
-    is_js = platform in SHOULD_SYMBOLICATE_JS
+    if platform in SHOULD_SYMBOLICATE_JS:
+        symbolicator_platform = SymbolicatorPlatform.js
+    else:
+        symbolicator_platform = SymbolicatorPlatform.native
     symbolicator = Symbolicator(
-        task_kind=SymbolicatorTaskKind(is_js=is_js),
+        task_kind=SymbolicatorTaskKind(platform=symbolicator_platform),
         on_request=on_symbolicator_request,
         project=project,
         event_id=profile["event_id"],

src/sentry/tasks/store.py

@@ -169,7 +169,7 @@ def _do_preprocess_event(
             "organization", Organization.objects.get_from_cache(id=project.organization_id)
         )
 
-    is_js, symbolication_function = get_symbolication_function(data)
+    platform, symbolication_function = get_symbolication_function(data)
     if symbolication_function:
         symbolication_function_name = getattr(symbolication_function, "__name__", "none")
 
@@ -186,7 +186,9 @@ def _do_preprocess_event(
 
             is_low_priority = should_demote_symbolication(project_id)
             task_kind = SymbolicatorTaskKind(
-                is_js=is_js, is_low_priority=is_low_priority, is_reprocessing=from_reprocessing
+                platform=platform,
+                is_low_priority=is_low_priority,
+                is_reprocessing=from_reprocessing,
             )
             submit_symbolicate(
                 task_kind,

src/sentry/tasks/symbolication.py

@@ -12,7 +12,7 @@ from sentry.eventstore.processing.base import Event
 from sentry.features.rollout import in_random_rollout
 from sentry.killswitches import killswitch_matches_context
 from sentry.lang.javascript.processing import process_js_stacktraces
-from sentry.lang.native.symbolicator import Symbolicator, SymbolicatorTaskKind
+from sentry.lang.native.symbolicator import Symbolicator, SymbolicatorPlatform, SymbolicatorTaskKind
 from sentry.models.organization import Organization
 from sentry.models.project import Project
 from sentry.processing import realtime_metrics
@@ -101,11 +101,16 @@ def get_native_symbolication_function(data: Any) -> Callable[[Symbolicator, Any]
 
 def get_symbolication_function(
     data: Any,
-) -> tuple[bool, Callable[[Symbolicator, Any], Any] | None]:
+) -> tuple[SymbolicatorPlatform, Callable[[Symbolicator, Any], Any] | None]:
+    from sentry.lang.java.processing import process_jvm_stacktraces
+    from sentry.lang.java.utils import should_use_symbolicator_for_proguard
+
     if data["platform"] in ("javascript", "node"):
-        return True, process_js_stacktraces
+        return SymbolicatorPlatform.js, process_js_stacktraces
+    elif data["platform"] == "java" and should_use_symbolicator_for_proguard(data.get("project")):
+        return SymbolicatorPlatform.jvm, process_jvm_stacktraces
     else:
-        return False, get_native_symbolication_function(data)
+        return SymbolicatorPlatform.native, get_native_symbolication_function(data)
 
 
 class SymbolicationTimeout(Exception):
@@ -163,11 +168,11 @@ def _do_symbolicate_event(
         # After JS processing, we check `get_native_symbolication_function`,
         # because maybe we need to feed it to another round of
         # `symbolicate_event`, but for *native* that time.
-        if not was_killswitched and task_kind.is_js:
+        if not was_killswitched and task_kind.platform == SymbolicatorPlatform.js:
             symbolication_function = get_native_symbolication_function(data)
             if symbolication_function:
                 submit_symbolicate(
-                    task_kind=task_kind.with_js(False),
+                    task_kind=task_kind.with_platform(SymbolicatorPlatform.native),
                     cache_key=cache_key,
                     event_id=event_id,
                     start_time=start_time,
@@ -185,10 +190,15 @@ def _do_symbolicate_event(
             has_attachments=has_attachments,
         )
 
-    if not task_kind.is_js:
-        symbolication_function = get_native_symbolication_function(data)
-    else:
+    symbolication_function: Callable[[Symbolicator, Any], Any] | None = None
+    if task_kind.platform == SymbolicatorPlatform.js:
         symbolication_function = process_js_stacktraces
+    elif task_kind.platform == SymbolicatorPlatform.jvm:
+        from sentry.lang.java.processing import process_jvm_stacktraces
+
+        symbolication_function = process_jvm_stacktraces
+    else:
+        symbolication_function = get_native_symbolication_function(data)
 
     symbolication_function_name = getattr(symbolication_function, "__name__", "none")
 
@@ -323,12 +333,18 @@ def get_kind_from_task(task: Any) -> SymbolicatorTaskKind:
         symbolicate_js_event_low_priority,
         symbolicate_event_from_reprocessing_low_priority,
     ]
-    is_js = task in [symbolicate_js_event, symbolicate_js_event_low_priority]
+    if task in [symbolicate_js_event, symbolicate_js_event_low_priority]:
+        platform = SymbolicatorPlatform.js
+    elif task in [symbolicate_jvm_event, symbolicate_jvm_event_low_priority]:
+        platform = SymbolicatorPlatform.jvm
+    else:
+        platform = SymbolicatorPlatform.native
+
     is_reprocessing = task in [
         symbolicate_event_from_reprocessing,
         symbolicate_event_from_reprocessing_low_priority,
     ]
-    return SymbolicatorTaskKind(is_js, is_low_priority, is_reprocessing)
+    return SymbolicatorTaskKind(platform, is_low_priority, is_reprocessing)
 
 
 def submit_symbolicate(
@@ -341,11 +357,16 @@ def submit_symbolicate(
 ) -> None:
     # oh how I miss a real `match` statement...
     task = symbolicate_event
-    if task_kind.is_js:
+    if task_kind.platform == SymbolicatorPlatform.js:
         if task_kind.is_low_priority:
             task = symbolicate_js_event_low_priority
         else:
             task = symbolicate_js_event
+    elif task_kind.platform == SymbolicatorPlatform.jvm:
+        if task_kind.is_low_priority:
+            task = symbolicate_jvm_event_low_priority
+        else:
+            task = symbolicate_jvm_event
     elif task_kind.is_reprocessing:
         if task_kind.is_low_priority:
             task = symbolicate_event_from_reprocessing_low_priority
@@ -433,6 +454,42 @@ def symbolicate_js_event(
     )
 
 
+# NOTE: Intentionally uses the same queue as `symbolicate_event`.
+@instrumented_task(
+    name="sentry.tasks.symbolicate_jvm_event",
+    queue="events.symbolicate_event",
+    time_limit=settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT + 30,
+    soft_time_limit=settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT + 20,
+    acks_late=True,
+    silo_mode=SiloMode.REGION,
+)
+def symbolicate_jvm_event(
+    cache_key: str,
+    start_time: int | None = None,
+    event_id: str | None = None,
+    data: Event | None = None,
+    queue_switches: int = 0,
+    has_attachments: bool = False,
+    **kwargs: Any,
+) -> None:
+    """
+    Handles event symbolication using the external service: symbolicator.
+
+    :param string cache_key: the cache key for the event data
+    :param int start_time: the timestamp when the event was ingested
+    :param string event_id: the event identifier
+    """
+    return _do_symbolicate_event(
+        cache_key=cache_key,
+        start_time=start_time,
+        event_id=event_id,
+        symbolicate_task=symbolicate_jvm_event,
+        data=data,
+        queue_switches=queue_switches,
+        has_attachments=has_attachments,
+    )
+
+
 @instrumented_task(
     name="sentry.tasks.store.symbolicate_event_low_priority",
     queue="events.symbolicate_event_low_priority",
@@ -509,6 +566,45 @@ def symbolicate_js_event_low_priority(
     )
 
 
+# NOTE: Intentionally uses the same queue as `symbolicate_event_low_priority`.
+@instrumented_task(
+    name="sentry.tasks.symbolicate_jvm_event_low_priority",
+    queue="events.symbolicate_event_low_priority",
+    time_limit=settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT + 30,
+    soft_time_limit=settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT + 20,
+    acks_late=True,
+    silo_mode=SiloMode.REGION,
+)
+def symbolicate_jvm_event_low_priority(
+    cache_key: str,
+    start_time: int | None = None,
+    event_id: str | None = None,
+    data: Event | None = None,
+    queue_switches: int = 0,
+    has_attachments: bool = False,
+    **kwargs: Any,
+) -> None:
+    """
+    Handles event symbolication using the external service: symbolicator.
+
+    This puts the task on the low priority queue. Projects whose symbolication
+    events misbehave get sent there to protect the main queue.
+
+    :param string cache_key: the cache key for the event data
+    :param int start_time: the timestamp when the event was ingested
+    :param string event_id: the event identifier
+    """
+    return _do_symbolicate_event(
+        cache_key=cache_key,
+        start_time=start_time,
+        event_id=event_id,
+        symbolicate_task=symbolicate_jvm_event_low_priority,
+        data=data,
+        queue_switches=queue_switches,
+        has_attachments=has_attachments,
+    )
+
+
 @instrumented_task(
     name="sentry.tasks.store.symbolicate_event_from_reprocessing",
     queue="events.reprocessing.symbolicate_event",

tests/relay_integration/lang/java/test_plugin.py

@@ -10,7 +10,9 @@ from sentry.models.debugfile import ProjectDebugFile
 from sentry.models.files.file import File
 from sentry.testutils.cases import TransactionTestCase
 from sentry.testutils.helpers.datetime import before_now, iso_format
+from sentry.testutils.helpers.options import override_options
 from sentry.testutils.relay import RelayStoreHelper
+from sentry.testutils.skips import requires_symbolicator
 from sentry.utils import json
 
 PROGUARD_UUID = "6dc7fdb0-d2fb-4c8e-9d6b-bb1aa98929b1"
@@ -394,6 +396,12 @@ class AnotherClassInSameFile {
 
 @pytest.mark.django_db(transaction=True)
 class BasicResolvingIntegrationTest(RelayStoreHelper, TransactionTestCase):
+    @pytest.fixture(autouse=True)
+    def initialize(self, set_sentry_option, live_server):
+        with set_sentry_option("system.url-prefix", live_server.url):
+            # Run test case
+            yield
+
     def upload_proguard_mapping(self, uuid, mapping_file_content):
         url = reverse(
             "sentry-api-0-dsym-files",
@@ -497,6 +505,12 @@ class BasicResolvingIntegrationTest(RelayStoreHelper, TransactionTestCase):
             "org.slf4j.helpers.Util$ClassContextSecurityManager " "in getExtraClassContext"
         )
 
+    @requires_symbolicator
+    @pytest.mark.symbolicator
+    def test_basic_resolving_symbolicator(self):
+        with override_options({"symbolicator.proguard-processing-sample-rate": 1.0}):
+            self.test_basic_resolving()
+
     def test_resolving_does_not_fail_when_no_value(self):
         self.upload_proguard_mapping(PROGUARD_UUID, PROGUARD_SOURCE)
 
@@ -553,6 +567,12 @@ class BasicResolvingIntegrationTest(RelayStoreHelper, TransactionTestCase):
         metrics = event.data["_metrics"]
         assert not metrics.get("flag.processing.error")
 
+    @requires_symbolicator
+    @pytest.mark.symbolicator
+    def test_resolving_does_not_fail_when_no_value_symbolicator(self):
+        with override_options({"symbolicator.proguard-processing-sample-rate": 1.0}):
+            self.test_resolving_does_not_fail_when_no_value()
+
     def test_resolving_does_not_fail_when_no_module_or_function(self):
         self.upload_proguard_mapping(PROGUARD_UUID, PROGUARD_SOURCE)
 
@@ -621,6 +641,12 @@ class BasicResolvingIntegrationTest(RelayStoreHelper, TransactionTestCase):
         metrics = event.data["_metrics"]
         assert not metrics.get("flag.processing.error")
 
+    @requires_symbolicator
+    @pytest.mark.symbolicator
+    def test_resolving_does_not_fail_when_no_module_or_function_symbolicator(self):
+        with override_options({"symbolicator.proguard-processing-sample-rate": 1.0}):
+            self.test_resolving_does_not_fail_when_no_module_or_function()
+
     def test_sets_inapp_after_resolving(self):
         self.upload_proguard_mapping(PROGUARD_UUID, PROGUARD_SOURCE)
 
@@ -716,6 +742,12 @@ class BasicResolvingIntegrationTest(RelayStoreHelper, TransactionTestCase):
         assert frames[3].in_app is False
         assert frames[4].in_app is True
 
+    @requires_symbolicator
+    @pytest.mark.symbolicator
+    def test_sets_inapp_after_resolving_symbolicator(self):
+        with override_options({"symbolicator.proguard-processing-sample-rate": 1.0}):
+            self.test_sets_inapp_after_resolving()
+
     def test_resolving_inline(self):
         self.upload_proguard_mapping(PROGUARD_INLINE_UUID, PROGUARD_INLINE_SOURCE)
 
@@ -790,6 +822,12 @@ class BasicResolvingIntegrationTest(RelayStoreHelper, TransactionTestCase):
         assert frames[3].filename == "MainActivity.java"
         assert frames[3].module == "io.sentry.sample.MainActivity"
 
+    @requires_symbolicator
+    @pytest.mark.symbolicator
+    def test_resolving_inline_symbolicator(self):
+        with override_options({"symbolicator.proguard-processing-sample-rate": 1.0}):
+            self.test_resolving_inline()
+
     def test_resolving_inline_with_native_frames(self):
         self.upload_proguard_mapping(PROGUARD_INLINE_UUID, PROGUARD_INLINE_SOURCE)
 
@@ -889,6 +927,12 @@ class BasicResolvingIntegrationTest(RelayStoreHelper, TransactionTestCase):
         assert frames[5].function == "__start_thread"
         assert frames[5].package == "/apex/com.android.art/lib64/libart.so"
 
+    @requires_symbolicator
+    @pytest.mark.symbolicator
+    def test_resolving_inline_with_native_frames_symbolicator(self):
+        with override_options({"symbolicator.proguard-processing-sample-rate": 1.0}):
+            self.test_resolving_inline_with_native_frames()
+
     def test_error_on_resolving(self):
         url = reverse(
             "sentry-api-0-dsym-files",
@@ -955,10 +999,15 @@ class BasicResolvingIntegrationTest(RelayStoreHelper, TransactionTestCase):
         event = self.post_and_retrieve_event(event_data)
 
         assert len(event.data["errors"]) == 1
-        assert event.data["errors"][0] == {
-            "mapping_uuid": "071207ac-b491-4a74-957c-2c94fd9594f2",
-            "type": "proguard_missing_lineno",
-        }
+        error = event.data["errors"][0]
+        assert error["mapping_uuid"] == "071207ac-b491-4a74-957c-2c94fd9594f2"
+        assert error["type"] == "proguard_missing_lineno"
+
+    @requires_symbolicator
+    @pytest.mark.symbolicator
+    def test_error_on_resolving_symbolicator(self):
+        with override_options({"symbolicator.proguard-processing-sample-rate": 1.0}):
+            self.test_error_on_resolving()
 
     def upload_jvm_bundle(self, debug_id, source_files):
         files = {}
@@ -973,14 +1022,18 @@ class BasicResolvingIntegrationTest(RelayStoreHelper, TransactionTestCase):
             "files": files,
         }
 
-        file_like = BytesIO()
-        with zipfile.ZipFile(file_like, "w") as zip:
-            for source_file in source_files:
-                zip.writestr(f"files/_/_/{source_file}", source_files[source_file])
+        file_like = BytesIO(b"SYSB")
+        with zipfile.ZipFile(file_like, "a") as zip:
+            for path, contents in source_files.items():
+                zip.writestr(f"files/_/_/{path}", contents)
             zip.writestr("manifest.json", json.dumps(manifest))
         file_like.seek(0)
 
-        file = File.objects.create(name="bundle.zip", type="artifact.bundle")
+        file = File.objects.create(
+            name="bundle.zip",
+            type="sourcebundle",
+            headers={"Content-Type": "application/x-sentry-bundle+zip"},
+        )
         file.putfile(file_like)
 
         ProjectDebugFile.objects.create(
@@ -1190,6 +1243,12 @@ class BasicResolvingIntegrationTest(RelayStoreHelper, TransactionTestCase):
         ]
         assert frames[6].post_context == ["        }", "    }", "}", ""]
 
+    @requires_symbolicator
+    @pytest.mark.symbolicator
+    def test_basic_source_lookup_symbolicator(self):
+        with override_options({"symbolicator.proguard-processing-sample-rate": 1.0}):
+            self.test_basic_source_lookup()
+
     def test_source_lookup_with_proguard(self):
         self.upload_proguard_mapping(PROGUARD_SOURCE_LOOKUP_UUID, PROGUARD_SOURCE_LOOKUP_SOURCE)
         debug_id1 = str(uuid4())
@@ -1495,3 +1554,9 @@ class BasicResolvingIntegrationTest(RelayStoreHelper, TransactionTestCase):
         assert frames[24].context_line is None
         assert frames[24].pre_context is None
         assert frames[24].post_context is None
+
+    @requires_symbolicator
+    @pytest.mark.symbolicator
+    def test_source_lookup_with_proguard_symbolicator(self):
+        with override_options({"symbolicator.proguard-processing-sample-rate": 1.0}):
+            self.test_source_lookup_with_proguard()