ref(native): Improve enumerating AppStoreConnect builds (#27015)

src/sentry/lang/native/appconnect.py

@@ -5,7 +5,6 @@ this.
 """
 
 import dataclasses
-import enum
 import io
 import logging
 import pathlib
@@ -220,13 +219,6 @@ class AppStoreConnectConfig:
         return all_sources
 
 
-@enum.unique
-class BuildKind(enum.Enum):
-    ALL = 1
-    PRE_RELEASE = 2
-    RELEASE = 3
-
-
 @dataclasses.dataclass(frozen=True)
 class BuildInfo:
     """Information about an App Store Connect build.
@@ -235,9 +227,6 @@ class BuildInfo:
     Apple mostly names these differently.
     """
 
-    # The kind of build, either PRE_RELEASE or RELEASE
-    kind: BuildKind
-
     # The app ID
     app_id: str
 
@@ -345,53 +334,21 @@ class AppConnectClient:
         """
         return ITunesClient(itunes_cookie=self._itunes_cookie, itunes_org=self._itunes_org)
 
-    def list_builds(self, kind: BuildKind = BuildKind.ALL) -> List[BuildInfo]:
-        """Returns the available builds, grouped by release.
-
-        :param kind: Whether to only query pre-releases or only releases or all.
-        :param bundle: The bundle ID, e.g. ``io.sentry.sample.iOS-Swift``.
-        """
-        if kind == BuildKind.PRE_RELEASE:
-            ret = appstore_connect.get_pre_release_version_info(
-                self._session, self._api_credentials, self._app_id
-            )
-            all_results = {"pre_releases": ret}
-        elif kind == BuildKind.RELEASE:
-            ret = appstore_connect.get_release_version_info(
-                self._session, self._api_credentials, self._app_id
-            )
-            all_results = {"releases": ret}
-        else:
-            all_results = appstore_connect.get_build_info(
-                self._session, self._api_credentials, self._app_id
-            )
+    def list_builds(self) -> List[BuildInfo]:
+        """Returns the available AppStore builds."""
 
         builds = []
-        for kind_name, results in all_results.items():
-            if kind_name == "pre_releases":
-                kind = BuildKind.PRE_RELEASE
-            else:
-                kind = BuildKind.RELEASE
-
-            for release in results:
-                for build in release["versions"]:
-                    build = BuildInfo(
-                        kind=kind,
-                        app_id=self._app_id,
-                        platform=release["platform"],
-                        version=release["short_version"],
-                        build_number=build["version"],
-                    )
-                    builds.append(build)
-
-        def _try_int(x: str) -> int:
-            try:
-                return int(x)
-            except ValueError:
-                return 0
-
-        # We sort the builds by their "build_number" (version), so that we fetch
-        # newer builds first.
-        builds.sort(key=lambda x: _try_int(x.build_number), reverse=True)
+        all_results = appstore_connect.get_build_info(
+            self._session, self._api_credentials, self._app_id
+        )
+        for build in all_results:
+            builds.append(
+                BuildInfo(
+                    app_id=self._app_id,
+                    platform=build["platform"],
+                    version=build["version"],
+                    build_number=build["build_number"],
+                )
+            )
 
         return builds

src/sentry/utils/appleconnect/appstore_connect.py

@@ -7,6 +7,7 @@ import jwt
 from requests import Session
 
 from sentry.utils import safe
+from sentry.utils.json import JSONData
 
 logger = logging.getLogger(__name__)
 
@@ -79,127 +80,105 @@ def _get_next_page(response_json: Mapping[str, Any]) -> Optional[str]:
     return safe.get_path(response_json, "links", "next")  # type: ignore
 
 
-def _get_appstore_info_paged_data(
+def _get_appstore_info_paged(
     session: Session, credentials: AppConnectCredentials, url: str
 ) -> Generator[Any, None, None]:
     """Iterates through all the pages from a paged response.
 
-    The `data` part of the responses are concatenated.
-
     App Store Connect responses shares the general format:
 
     data:
       - list of elements
+    included:
+      - list of included relations as requested
     links:
       next: link to the next page
     ...
 
     The function iterates through all pages (following the next link) until
-    there is no next page, and returns a generator containing all
-    the data in the arrays from each call
+    there is no next page, and returns a generator containing all pages
 
-    :return: a generator with the contents of all the arrays from each page (flattened).
+    :return: a generator with the pages.
     """
     next_url: Optional[str] = url
     while next_url is not None:
         response = _get_appstore_json(session, credentials, next_url)
-        data = response["data"]
-        yield from data
+        yield response
         next_url = _get_next_page(response)
 
 
-def get_pre_release_version_info(
+def get_build_info(
     session: Session, credentials: AppConnectCredentials, app_id: str
 ) -> List[Dict[str, Any]]:
-    """Get all prerelease builds version information for an application
+    """Returns the build infos for an application.
 
     The release build version information has the following structure:
     platform: str - the platform for the build (e.g. IOS, MAC_OS ...)
-    short_version: str - the short version build info ( e.g. '1.0.1'), also called "train"
+    version: str - the short version build info ( e.g. '1.0.1'), also called "train"
        in starship documentation
-    id: str - the IID of the version
-    versions: vec - a vector with builds
-       version: str - the version of the build (e.g. '101'), looks like the build number
-       id: str - the IID of the build
-
-    NOTE: the pre release version information is identical to the release version information
-    :return: a list of prerelease builds version information (see above)
+    build_number: str - the version of the build (e.g. '101'), looks like the build number
     """
-    url = f"v1/apps/{app_id}/preReleaseVersions"
-    data = _get_appstore_info_paged_data(session, credentials, url)
-    result = []
-    for d in data:
-        versions: List[Dict[str, Any]] = []
-        v = {
-            "platform": safe.get_path(d, "attributes", "platform"),
-            "short_version": safe.get_path(d, "attributes", "version"),
-            "id": safe.get_path(d, "id"),
-            "versions": versions,
-        }
-        builds_url = safe.get_path(d, "relationships", "builds", "links", "related")
-        for build in _get_appstore_info_paged_data(session, credentials, builds_url):
-            b = {
-                "version": safe.get_path(build, "attributes", "version"),
-                "id": safe.get_path(build, "id"),
-            }
-            versions.append(b)
-        result.append(v)
-
-    return result
-
-
-def get_release_version_info(
-    session: Session, credentials: AppConnectCredentials, app_id: str
-) -> List[Dict[str, Any]]:
-    """Get all release builds version information for an application
-
-    The release build version information has the following structure:
-    platform: str - the platform for the build (e.g. IOS, MAC_OS ...)
-    short_version: str - the short version build info ( e.g. '1.0.1'), also called "train"
-       in starship documentation
-    id: str - the IID of the version
-    versions: vec - a vector with builds
-       version: str - the version of the build (e.g. '101'), looks like the build number
-       id: str - the IID of the build
 
-    NOTE: the release version information is identical to the pre release version information
-    :return: a list of release builds version information (see above)
-    """
-    url = f"v1/apps/{app_id}/appStoreVersions"
-    data = _get_appstore_info_paged_data(session, credentials, url)
     result = []
-    for d in data:
-        versions: List[Dict[str, Any]] = []
-        build_url = safe.get_path(d, "relationships", "build", "links", "related")
-        v = {
-            "platform": safe.get_path(d, "attributes", "platform"),
-            "short_version": safe.get_path(d, "attributes", "versionString"),
-            "id": safe.get_path(d, "id"),
-            "versions": versions,
-        }
-
-        build_info = _get_appstore_info_paged_data(session, credentials, build_url)
-        build_info = safe.get_path(build_info, "data")
-        if build_info is not None:
-            # Note RaduW: never seen info in this structure, I assume the same structure as pre release
-            versions.append(
-                {
-                    "id": safe.get_path(build_info, "id"),
-                    "version": safe.get_path(build_info, "attributes", "version"),
-                }
-            )
-        result.append(v)
-    return result
 
+    # https://developer.apple.com/documentation/appstoreconnectapi/list_builds
+    url = (
+        f"v1/builds?filter[app]={app_id}"
+        # we can fetch a maximum of 200 builds at once, so do that
+        "&limit=200"
+        # include related AppStore/PreRelease versions with the response
+        # NOTE: the `iris` web API has related `buildBundles` objects,
+        # which have very useful `includesSymbols` and `dSYMUrl` attributes,
+        # but this is sadly not available in the official API. :-(
+        # Open this in your browser when you are signed into AppStoreConnect:
+        # https://appstoreconnect.apple.com/iris/v1/builds?filter[processingState]=VALID&include=appStoreVersion,preReleaseVersion,buildBundles&limit=1&filter[app]=XYZ
+        "&include=appStoreVersion,preReleaseVersion"
+        # sort newer releases first
+        "&sort=-uploadedDate"
+        # only include valid builds
+        "&filter[processingState]=VALID"
+        # and builds that have not expired yet
+        "&filter[expired]=false"
+    )
+    pages = _get_appstore_info_paged(session, credentials, url)
+
+    for page in pages:
+        included_relations = {}
+        for included in safe.get_path(page, "included", default=[]):
+            type = safe.get_path(included, "type")
+            id = safe.get_path(included, "id")
+            if type is not None and id is not None:
+                included_relations[(type, id)] = included
+
+        def get_related(relation: JSONData) -> JSONData:
+            type = safe.get_path(relation, "data", "type")
+            id = safe.get_path(relation, "data", "id")
+            if type is None or id is None:
+                return None
+            return included_relations.get((type, id))
+
+        for build in safe.get_path(page, "data", default=[]):
+            related_appstore_version = get_related(
+                safe.get_path(build, "relationships", "appStoreVersion")
+            )
+            related_prerelease_version = get_related(
+                safe.get_path(build, "relationships", "preReleaseVersion")
+            )
+            related_version = related_appstore_version or related_prerelease_version
+            if not related_version:
+                logger.error("Missing related version for AppStoreConnect `build`")
+                continue
+            platform = safe.get_path(related_version, "attributes", "platform")
+            version = safe.get_path(related_version, "attributes", "versionString")
+            build_number = safe.get_path(build, "attributes", "version")
+            if platform is not None and version is not None and build_number is not None:
+                result.append(
+                    {"platform": platform, "version": version, "build_number": build_number}
+                )
+            else:
+                logger.error("Malformed AppStoreConnect `builds` data")
 
-def get_build_info(
-    session: Session, credentials: AppConnectCredentials, app_id: str
-) -> Dict[str, List[Dict[str, Any]]]:
-    """Returns the build info for an application."""
-    return {
-        "pre_releases": get_pre_release_version_info(session, credentials, app_id),
-        "releases": get_release_version_info(session, credentials, app_id),
-    }
+    return result
 
 
 AppInfo = namedtuple("AppInfo", ["name", "bundle_id", "app_id"])
@@ -214,19 +193,22 @@ def get_apps(session: Session, credentials: AppConnectCredentials) -> Optional[L
     url = "v1/apps"
     ret_val = []
     try:
-        apps = _get_appstore_info_paged_data(session, credentials, url)
-        for app in apps:
-            app_info = AppInfo(
-                app_id=app.get("id"),
-                bundle_id=safe.get_path(app, "attributes", "bundleId"),
-                name=safe.get_path(app, "attributes", "name"),
-            )
-            if (
-                app_info.app_id is not None
-                and app_info.bundle_id is not None
-                and app_info.name is not None
-            ):
-                ret_val.append(app_info)
+        app_pages = _get_appstore_info_paged(session, credentials, url)
+        for app_page in app_pages:
+            for app in safe.get_path(app_page, "data", default=[]):
+                app_info = AppInfo(
+                    app_id=app.get("id"),
+                    bundle_id=safe.get_path(app, "attributes", "bundleId"),
+                    name=safe.get_path(app, "attributes", "name"),
+                )
+                if (
+                    app_info.app_id is not None
+                    and app_info.bundle_id is not None
+                    and app_info.name is not None
+                ):
+                    ret_val.append(app_info)
+                else:
+                    logger.error("Malformed AppStoreConnect `apps` data")
     except ValueError:
         return None
     return ret_val