feat(appstore): Trigger dSYM download periodically (#27311)

Co-authored-by: Floris Bruynooghe <flub@sentry.io>

src/sentry/conf/server.py

@@ -740,6 +740,11 @@ CELERYBEAT_SCHEDULE = {
         "schedule": timedelta(minutes=5),
         "options": {"expires": 3600},
     },
+    "fetch-appstore-builds": {
+        "task": "sentry.tasks.app_store_connect.refresh_all_builds",
+        "schedule": timedelta(hours=1),
+        "options": {"expires": 3600},
+    },
     "snuba-subscription-checker": {
         "task": "sentry.snuba.tasks.subscription_checker",
         "schedule": timedelta(minutes=20),

src/sentry/lang/native/appconnect.py

@@ -154,6 +154,16 @@ class AppStoreConnectConfig:
         data["itunesCreated"] = dateutil.parser.isoparse(data["itunesCreated"])
         return cls(**data)
 
+    @classmethod
+    def all_for_project(cls, project: Project) -> "List[AppStoreConnectConfig]":
+        sources = []
+        raw = project.get_option(SYMBOL_SOURCES_PROP_NAME, default="[]")
+        all_sources = json.loads(raw)
+        for source in all_sources:
+            if source.get("type") == SYMBOL_SOURCE_TYPE_NAME:
+                sources.append(cls.from_json(source))
+        return sources
+
     @classmethod
     def from_project_config(cls, project: Project, config_id: str) -> "AppStoreConnectConfig":
         """Creates a new instance from the symbol source configured in the project.
@@ -164,7 +174,7 @@ class AppStoreConnectConfig:
         raw = project.get_option(SYMBOL_SOURCES_PROP_NAME, default="[]")
         all_sources = json.loads(raw)
         for source in all_sources:
-            if source.get("type") == SYMBOL_SOURCE_TYPE_NAME and source.get("id") == config_id:
+            if source.get("type") == SYMBOL_SOURCE_TYPE_NAME and (source.get("id") == config_id):
                 return cls.from_json(source)
         else:
             raise KeyError(f"No {SYMBOL_SOURCE_TYPE_NAME} symbol source found with id {config_id}")

src/sentry/tasks/app_store_connect.py

@@ -7,11 +7,12 @@ debug files.  These tasks enable this functionality.
 import logging
 import pathlib
 import tempfile
+from typing import List, Mapping
 
 from sentry.lang.native import appconnect
-from sentry.models import AppConnectBuild, Project, debugfile
+from sentry.models import AppConnectBuild, Project, ProjectOption, debugfile
 from sentry.tasks.base import instrumented_task
-from sentry.utils.sdk import configure_scope
+from sentry.utils import json, sdk
 
 logger = logging.getLogger(__name__)
 
@@ -25,15 +26,12 @@ def dsym_download(project_id: int, config_id: str) -> None:
     inner_dsym_download(project_id=project_id, config_id=config_id)
 
 
-def inner_dsym_download(
-    project_id: int,
-    config_id: str,
-) -> None:
+def inner_dsym_download(project_id: int, config_id: str) -> None:
     """Downloads the dSYMs from App Store Connect and stores them in the Project's debug files."""
     # TODO(flub): we should only run one task ever for a project.  Is
     # sentry.cache.default_cache the right thing to put a "mutex" into?  See how
     # sentry.tasks.assemble uses this.
-    with configure_scope() as scope:
+    with sdk.configure_scope() as scope:
         scope.set_tag("project", project_id)
 
     project = Project.objects.get(pk=project_id)
@@ -80,3 +78,49 @@ def create_difs_from_dsyms_zip(dsyms_zip: str, project: Project) -> None:
         created = debugfile.create_files_from_dif_zip(fp, project, accept_unknown=True)
         for proj_debug_file in created:
             logger.debug("Created %r for project %s", proj_debug_file, project.id)
+
+
+# Untyped decorator would stop type-checking of entire function, split into an inner
+# function instead which can be type checked.
+@instrumented_task(  # type: ignore
+    name="sentry.tasks.app_store_connect.refresh_all_builds", queue="appstoreconnect"
+)
+def refresh_all_builds() -> None:
+    inner_refresh_all_builds()
+
+
+def inner_refresh_all_builds() -> None:
+    """Refreshes all AppStoreConnect builds for all projects.
+
+    This iterates over all the projects configured in Sentry and for any which has an
+    AppStoreConnect symbol source configured will poll the AppStoreConnect API to check if
+    there are new builds.
+    """
+    # We have no way to query for AppStore Connect symbol sources directly, but
+    # getting all of the project options that have custom symbol sources
+    # configured is a reasonable compromise, as the number of those should be
+    # low enough to traverse every hour.
+    # Another alternative would be to get a list of projects that have had a
+    # previous successful import, as indicated by existing `AppConnectBuild`
+    # objects. But that would miss projects that have a valid AppStore Connect
+    # setup, but have not yet published any kind of build to AppStore.
+    options = ProjectOption.objects.filter(key=appconnect.SYMBOL_SOURCES_PROP_NAME)
+    for option in options:
+        with sdk.push_scope() as scope:
+            scope.set_tag("project", option.project_id)
+            try:
+                # We are parsing JSON thus all types are Any, so give the type-checker some
+                # extra help.  We are maybe slightly lying about the type, but the
+                # attributes we do access are all string values.
+                all_sources: List[Mapping[str, str]] = json.loads(option.value)
+                for source in all_sources:
+                    try:
+                        source_id = source["id"]
+                        source_type = source["type"]
+                    except KeyError:
+                        logger.exception("Malformed symbol source")
+                        continue
+                    if source_type == appconnect.SYMBOL_SOURCE_TYPE_NAME:
+                        inner_dsym_download(option.project_id, source_id)
+            except Exception:
+                logger.exception("Failed to refresh AppStoreConnect builds")