feat(server): Add App Store Connect symbols repository (#25546)

requirements-base.txt

@@ -64,7 +64,8 @@ uWSGI==2.0.19.1
 zstandard==0.14.1
 
 msgpack==1.0.0
-
+# for encrypting stored user credentials
+cryptography==3.3.2
 # celery
 billiard==3.6.3
 kombu==4.6.11

src/sentry/api/endpoints/project_app_store_connect_credentials.py

@@ -0,0 +1,532 @@
+from uuid import uuid4
+
+import requests
+from rest_framework import serializers
+from rest_framework.response import Response
+
+from sentry import features
+from sentry.api.bases.project import ProjectEndpoint, StrictProjectPermission
+from sentry.utils import fernet_encrypt as encrypt
+from sentry.utils import json
+from sentry.utils.appleconnect import appstore_connect, itunes_connect
+from sentry.utils.appleconnect.itunes_connect import ITunesHeaders
+from sentry.utils.safe import get_path
+
+
+def credentials_key_name():
+    return "sentry:appleconnect_key"
+
+
+def symbol_sources_prop_name():
+    return "sentry:symbol_sources"
+
+
+def app_store_connect_feature_name():
+    return "organizations:app-store-connect"
+
+
+def get_app_store_credentials(project, credentials_id):
+    sources_config = project.get_option(symbol_sources_prop_name())
+
+    if credentials_id is None:
+        return None
+    try:
+        sources = json.loads(sources_config)
+        for source in sources:
+            if (
+                source.get("type") == "appStoreConnect"
+                and source.get("id") == credentials_id.lower()
+            ):
+                return source
+        return None
+    except BaseException as e:
+        raise ValueError("bad sources") from e
+
+
+class AppStoreConnectCredentialsSerializer(serializers.Serializer):
+    """
+    Serializer for the App Store Connect (Rest) credentials
+    """
+
+    # an IID with the XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX format
+    appconnectIssuer = serializers.CharField(max_length=36, min_length=36, required=True)
+    # about 10 chars
+    appconnectKey = serializers.CharField(max_length=20, min_length=2, required=True)
+    # 512 should fit a private key
+    appconnectPrivateKey = serializers.CharField(max_length=512, required=True)
+
+
+class AppStoreConnectAppsEndpoint(ProjectEndpoint):
+    """
+    This endpoint returns the applications defined for an account
+    It also serves to validate that credentials for App Store connect are valid
+    """
+
+    permission_classes = [StrictProjectPermission]
+
+    def post(self, request, project):
+        if not features.has(
+            app_store_connect_feature_name(), project.organization, actor=request.user
+        ):
+            return Response(status=404)
+
+        serializer = AppStoreConnectCredentialsSerializer(data=request.data)
+
+        if not serializer.is_valid():
+            return Response(serializer.errors, status=400)
+
+        data = serializer.validated_data
+        credentials = appstore_connect.AppConnectCredentials(
+            key_id=data.get("appconnectKey"),
+            key=data.get("appconnectPrivateKey"),
+            issuer_id=data.get("appconnectIssuer"),
+        )
+        session = requests.Session()
+
+        apps = appstore_connect.get_apps(session, credentials)
+
+        if apps is None:
+            return Response("App connect authentication error.", status=401)
+
+        apps = [{"name": app.name, "bundleId": app.bundle_id, "appId": app.app_id} for app in apps]
+        result = {"apps": apps}
+
+        return Response(result, status=200)
+
+
+class AppStoreCreateCredentialsSerializer(serializers.Serializer):
+    """
+    Serializer for the full Apple connect credentials AppStoreConnect + ITunes
+    """
+
+    # an IID with the XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX format
+    appconnectIssuer = serializers.CharField(max_length=36, min_length=36, required=True)
+    # about 10 chars
+    appconnectKey = serializers.CharField(max_length=20, min_length=2, required=True)
+    # 512 should fit a private key
+    appconnectPrivateKey = serializers.CharField(max_length=512, required=True)
+    itunesUser = serializers.CharField(max_length=100, min_length=1, required=True)
+    itunesPassword = serializers.CharField(max_length=512, min_length=1, required=True)
+    appName = serializers.CharField(max_length=512, min_length=1, required=True)
+    appId = serializers.CharField(max_length=512, min_length=1, required=True)
+    sessionContext = serializers.CharField(min_length=1, required=True)
+    # this is the ITunes organization the user is a member of ( known as providers in Itunes terminology)
+    orgId = serializers.IntegerField(required=True)
+    orgName = serializers.CharField(max_length=100, required=True)
+
+
+class AppStoreConnectCreateCredentialsEndpoint(ProjectEndpoint):
+    permission_classes = [StrictProjectPermission]
+
+    def post(self, request, project):
+        if not features.has(
+            app_store_connect_feature_name(), project.organization, actor=request.user
+        ):
+            return Response(status=404)
+
+        serializer = AppStoreCreateCredentialsSerializer(data=request.data)
+
+        if not serializer.is_valid():
+            return Response(serializer.errors, status=400)
+
+        key = project.get_option(credentials_key_name())
+
+        if key is None:
+            # probably stage 1 login was not called
+            return Response(
+                "Invalid state. Must first call appstoreconnect/start/ endpoint.", status=400
+            )
+
+        credentials = serializer.validated_data
+
+        encrypted_context = credentials.pop("sessionContext")
+
+        try:
+            validation_context = encrypt.decrypt_object(encrypted_context, key)
+            itunes_session = validation_context.get("itunes_session")
+            encrypted = {
+                "itunesSession": itunes_session,
+                "itunesPassword": credentials.pop("itunesPassword"),
+                "appconnectPrivateKey": credentials.pop("appconnectPrivateKey"),
+            }
+            credentials["encrypted"] = encrypt.encrypt_object(encrypted, key)
+            credentials["type"] = "appStoreConnect"
+            credentials["id"] = uuid4().hex
+            credentials["name"] = "Apple App Store Connect"
+
+        except ValueError:
+            return Response("Invalid validation context passed.", status=400)
+        return Response(credentials, status=200)
+
+
+class AppStoreUpdateCredentialsSerializer(serializers.Serializer):
+    """
+    Serializer for the full Apple connect credentials AppStoreConnect + ITunes
+    """
+
+    # an IID with the XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX format
+    appconnectIssuer = serializers.CharField(max_length=36, min_length=36, required=False)
+    # about 10 chars
+    appconnectKey = serializers.CharField(max_length=20, min_length=2, required=False)
+    # 512 should fit a private key
+    appconnectPrivateKey = serializers.CharField(max_length=512, required=False)
+    itunesUser = serializers.CharField(max_length=100, min_length=1, required=False)
+    itunesPassword = serializers.CharField(max_length=512, min_length=1, required=False)
+    appName = serializers.CharField(max_length=512, min_length=1, required=False)
+    appId = serializers.CharField(max_length=512, min_length=1, required=False)
+    sessionContext = serializers.CharField(min_length=1, required=False)
+    # this is the ITunes organization the user is a member of ( known as providers in Itunes terminology)
+    orgId = serializers.IntegerField(required=False)
+    orgName = serializers.CharField(max_length=100, required=False)
+
+
+class AppStoreConnectUpdateCredentialsEndpoint(ProjectEndpoint):
+    permission_classes = [StrictProjectPermission]
+
+    def post(self, request, project, credentials_id):
+        if not features.has(
+            app_store_connect_feature_name(), project.organization, actor=request.user
+        ):
+            return Response(status=404)
+
+        serializer = AppStoreUpdateCredentialsSerializer(data=request.data)
+
+        if not serializer.is_valid():
+            return Response(serializer.errors, status=400)
+
+        # get the existing credentials
+        credentials = get_app_store_credentials(project, credentials_id)
+        key = project.get_option(credentials_key_name())
+
+        if key is None or credentials is None:
+            return Response(status=404)
+
+        try:
+            secrets = encrypt.decrypt_object(credentials.pop("encrypted"), key)
+        except ValueError:
+            return Response(status=500)
+
+        # get the new credentials
+        new_credentials = serializer.validated_data
+        encrypted_context = new_credentials.get("sessionContext")
+
+        new_itunes_session = None
+        if encrypted_context is not None:
+            try:
+                validation_context = encrypt.decrypt_object(encrypted_context, key)
+                new_itunes_session = validation_context.get("itunes_session")
+            except ValueError:
+                return Response("Invalid validation context passed.", status=400)
+
+        new_secrets = {}
+
+        if new_itunes_session is not None:
+            new_secrets["itunesSession"] = new_itunes_session
+
+        new_itunes_password = new_credentials.get("itunesPassword")
+        if new_itunes_password is not None:
+            new_secrets["itunesPassword"] = new_itunes_password
+
+        new_appconnect_private_key = new_credentials.get("appconnectPrivateKey")
+        if new_appconnect_private_key is not None:
+            new_secrets["appconnectPrivateKey"] = new_appconnect_private_key
+
+        # merge the new and existing credentials
+
+        try:
+            secrets.update(new_secrets)
+            credentials.update(new_credentials)
+
+            credentials["encrypted"] = encrypt.encrypt_object(secrets, key)
+            credentials["id"] = uuid4().hex
+
+        except ValueError:
+            return Response("Invalid validation context passed.", status=400)
+        return Response(credentials, status=200)
+
+
+class AppStoreConnectCredentialsValidateEndpoint(ProjectEndpoint):
+    permission_classes = [StrictProjectPermission]
+
+    def get_result(self, app_store: bool, itunes: bool):
+        return {
+            "appstoreCredentialsValid": app_store,
+            "itunesSessionValid": itunes,
+        }
+
+    def get(self, request, project, credentials_id):
+        if not features.has(
+            app_store_connect_feature_name(), project.organization, actor=request.user
+        ):
+            return Response(status=404)
+
+        credentials = get_app_store_credentials(project, credentials_id)
+        key = project.get_option(credentials_key_name())
+
+        if key is None or credentials is None:
+            return Response(status=404)
+
+        try:
+            secrets = encrypt.decrypt_object(credentials.get("encrypted"), key)
+        except ValueError:
+            return Response(status=500)
+
+        credentials = appstore_connect.AppConnectCredentials(
+            key_id=credentials.get("appconnectKey"),
+            key=secrets.get("appconnectPrivateKey"),
+            issuer_id=credentials.get("appconnectIssuer"),
+        )
+
+        session = requests.Session()
+        apps = appstore_connect.get_apps(session, credentials)
+
+        appstore_valid = apps is not None
+        itunes_connect.load_session_cookie(session, secrets.get("itunesSession"))
+        itunes_session_info = itunes_connect.get_session_info(session)
+
+        itunes_session_valid = itunes_session_info is not None
+
+        return Response(
+            {
+                "appstoreCredentialsValid": appstore_valid,
+                "itunesSessionValid": itunes_session_valid,
+            },
+            status=200,
+        )
+
+
+class AppStoreConnectStartAuthSerializer(serializers.Serializer):
+    """
+    Serializer for the Itunes start connect operation
+    """
+
+    itunesUser = serializers.CharField(max_length=100, min_length=1, required=False)
+    itunesPassword = serializers.CharField(max_length=512, min_length=1, required=False)
+    id = serializers.CharField(max_length=40, min_length=1, required=False)
+
+
+class AppStoreConnectStartAuthEndpoint(ProjectEndpoint):
+    permission_classes = [StrictProjectPermission]
+
+    def post(self, request, project):
+        if not features.has(
+            app_store_connect_feature_name(), project.organization, actor=request.user
+        ):
+            return Response(status=404)
+
+        serializer = AppStoreConnectStartAuthSerializer(data=request.data)
+        if not serializer.is_valid():
+            return Response(serializer.errors, status=400)
+
+        user_name = serializer.validated_data.get("itunesUser")
+        password = serializer.validated_data.get("itunesPassword")
+        credentials_id = serializer.validated_data.get("id")
+
+        key = project.get_option(credentials_key_name())
+
+        if key is None:
+            # no encryption key for this project, create one
+            key = encrypt.create_key()
+            project.update_option(credentials_key_name(), key)
+        else:
+            # we have an encryption key, see if the credentials were not
+            # supplied and we just want to re validate the session
+            if user_name is None or password is None:
+                # credentials not supplied use saved credentials
+
+                credentials = get_app_store_credentials(project, credentials_id)
+                if key is None or credentials is None:
+                    return Response("No credentials provided.", status=400)
+
+                try:
+                    secrets = encrypt.decrypt_object(credentials.get("encrypted"), key)
+                except ValueError:
+                    return Response("Invalid credentials state.", status=500)
+
+                user_name = credentials.get("itunesUser")
+                password = secrets.get("itunesPassword")
+
+                if user_name is None or password is None:
+                    return Response("Invalid credentials.", status=500)
+
+        session = requests.session()
+
+        auth_key = itunes_connect.get_auth_service_key(session)
+
+        if auth_key is None:
+            return Response("Could not contact itunes store.", status=500)
+
+        if user_name is None:
+            return Response("No user name provided.", status=400)
+        if password is None:
+            return Response("No password provided.", status=400)
+
+        init_login_result = itunes_connect.initiate_login(
+            session, service_key=auth_key, account_name=user_name, password=password
+        )
+        if init_login_result is None:
+            return Response("ITunes login failed.", status=401)
+
+        # send session context to be used in next calls
+        session_context = {
+            "auth_key": auth_key,
+            "session_id": init_login_result.session_id,
+            "scnt": init_login_result.scnt,
+        }
+
+        return Response(
+            {"sessionContext": encrypt.encrypt_object(session_context, key)}, status=200
+        )
+
+
+class AppStoreConnectRequestSmsSerializer(serializers.Serializer):
+    sessionContext = serializers.CharField(min_length=1, required=True)
+
+
+class AppStoreConnectRequestSmsEndpoint(ProjectEndpoint):
+    permission_classes = [StrictProjectPermission]
+
+    def post(self, request, project):
+        if not features.has(
+            app_store_connect_feature_name(), project.organization, actor=request.user
+        ):
+            return Response(status=404)
+
+        serializer = AppStoreConnectRequestSmsSerializer(data=request.data)
+
+        if not serializer.is_valid():
+            return Response(serializer.errors, status=400)
+
+        session = requests.Session()
+
+        encrypted_context = serializer.validated_data.get("sessionContext")
+        key = project.get_option(credentials_key_name())
+
+        if key is None:
+            return Response(
+                "Invalid state. Must first call appstoreconnect/start/ endpoint.", status=400
+            )
+
+        try:
+            # recover the headers set in the first step authentication
+            session_context = encrypt.decrypt_object(encrypted_context, key)
+            headers = ITunesHeaders(
+                session_id=session_context.get("session_id"), scnt=session_context.get("scnt")
+            )
+            auth_key = session_context.get("auth_key")
+
+        except ValueError:
+            return Response("Invalid validation context passed.", status=400)
+
+        phone_info = itunes_connect.get_trusted_phone_info(
+            session, service_key=auth_key, headers=headers
+        )
+
+        if phone_info is None:
+            return Response("Could not get phone info", status=400)
+
+        init_phone_login = itunes_connect.initiate_phone_login(
+            session,
+            service_key=auth_key,
+            headers=headers,
+            phone_id=phone_info.id,
+            push_mode=phone_info.push_mode,
+        )
+
+        if init_phone_login is None:
+            return Response("Phone 2fa failed", status=500)
+
+        # success, return the new session context (add phone_id and push mode to the session context)
+        session_context["phone_id"] = phone_info.id
+        session_context["push_mode"] = phone_info.push_mode
+        encrypted_context = encrypt.encrypt_object(session_context, key)
+        return Response({"sessionContext": encrypted_context}, status=200)
+
+
+class AppStoreConnect2FactorAuthSerializer(serializers.Serializer):
+    code = serializers.CharField(max_length=10, required=True)
+    useSms = serializers.BooleanField(required=True)
+    sessionContext = serializers.CharField(min_length=1, required=True)
+
+
+class AppStoreConnect2FactorAuthEndpoint(ProjectEndpoint):
+    permission_classes = [StrictProjectPermission]
+
+    def post(self, request, project):
+        if not features.has(
+            app_store_connect_feature_name(), project.organization, actor=request.user
+        ):
+            return Response(status=404)
+
+        serializer = AppStoreConnect2FactorAuthSerializer(data=request.data)
+        if not serializer.is_valid():
+            return Response(serializer.errors, status=400)
+
+        encrypted_context = serializer.validated_data.get("sessionContext")
+        key = project.get_option(credentials_key_name())
+        use_sms = serializer.validated_data.get("useSms")
+        code = serializer.validated_data.get("code")
+
+        if key is None:
+            # probably stage 1 login was not called
+            return Response(
+                "Invalid state. Must first call appstoreconnect/start/ endpoint.", status=400
+            )
+
+        try:
+            # recover the headers set in the first step authentication
+            session_context = encrypt.decrypt_object(encrypted_context, key)
+            headers = ITunesHeaders(
+                session_id=session_context.get("session_id"), scnt=session_context.get("scnt")
+            )
+            auth_key = session_context.get("auth_key")
+
+            session = requests.Session()
+
+            if use_sms:
+                phone_id = session_context.get("phone_id")
+                push_mode = session_context.get("push_mode")
+                success = itunes_connect.send_phone_authentication_confirmation_code(
+                    session,
+                    service_key=auth_key,
+                    headers=headers,
+                    phone_id=phone_id,
+                    push_mode=push_mode,
+                    security_code=code,
+                )
+            else:
+                success = itunes_connect.send_authentication_confirmation_code(
+                    session, service_key=auth_key, headers=headers, security_code=code
+                )
+
+            if success:
+                session_info = itunes_connect.get_session_info(session)
+
+                if session_info is None:
+                    return Response("session info failed", status=500)
+
+                existing_providers = get_path(session_info, "availableProviders")
+                providers = [
+                    {"name": provider.get("name"), "organizationId": provider.get("providerId")}
+                    for provider in existing_providers
+                ]
+                prs_id = get_path(session_info, "user", "prsId")
+
+                itunes_session = itunes_connect.get_session_cookie(session)
+                session_context = {
+                    "auth_key": auth_key,
+                    "session_id": headers.session_id,
+                    "scnt": headers.scnt,
+                    "itunes_session": itunes_session,
+                    "itunes_person_id": prs_id,
+                }
+                encrypted_context = encrypt.encrypt_object(session_context, key)
+
+                response_body = {"sessionContext": encrypted_context, "organizations": providers}
+
+                return Response(response_body, status=200)
+            else:
+                return Response("2FA failed.", status=401)
+
+        except ValueError:
+            return Response("Invalid validation context passed.", status=400)

src/sentry/api/urls.py

@@ -252,6 +252,15 @@ from .endpoints.organization_user_reports import OrganizationUserReportsEndpoint
 from .endpoints.organization_user_teams import OrganizationUserTeamsEndpoint
 from .endpoints.organization_users import OrganizationUsersEndpoint
 from .endpoints.project_agnostic_rule_conditions import ProjectAgnosticRuleConditionsEndpoint
+from .endpoints.project_app_store_connect_credentials import (
+    AppStoreConnect2FactorAuthEndpoint,
+    AppStoreConnectAppsEndpoint,
+    AppStoreConnectCreateCredentialsEndpoint,
+    AppStoreConnectCredentialsValidateEndpoint,
+    AppStoreConnectRequestSmsEndpoint,
+    AppStoreConnectStartAuthEndpoint,
+    AppStoreConnectUpdateCredentialsEndpoint,
+)
 from .endpoints.project_avatar import ProjectAvatarEndpoint
 from .endpoints.project_codeowners import ProjectCodeOwnersEndpoint
 from .endpoints.project_codeowners_details import ProjectCodeOwnersDetailsEndpoint
@@ -1805,6 +1814,41 @@ urlpatterns = [
                     ProjectRepoPathParsingEndpoint.as_view(),
                     name="sentry-api-0-project-repo-path-parsing",
                 ),
+                url(
+                    r"^(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/appstoreconnect/$",
+                    AppStoreConnectCreateCredentialsEndpoint.as_view(),
+                    name="sentry-api-0-project-appstoreconnect-credentials-create",
+                ),
+                url(
+                    r"^(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/appstoreconnect/apps/$",
+                    AppStoreConnectAppsEndpoint.as_view(),
+                    name="sentry-api-0-project-appstoreconnect-apps",
+                ),
+                url(
+                    r"^(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/appstoreconnect/validate/(?P<credentials_id>[^\/]+)/$",
+                    AppStoreConnectCredentialsValidateEndpoint.as_view(),
+                    name="sentry-api-0-project-appstoreconnect-validate",
+                ),
+                url(
+                    r"^(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/appstoreconnect/start/$",
+                    AppStoreConnectStartAuthEndpoint.as_view(),
+                    name="sentry-api-0-project-appstoreconnect-start",
+                ),
+                url(
+                    r"^(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/appstoreconnect/requestSms/$",
+                    AppStoreConnectRequestSmsEndpoint.as_view(),
+                    name="sentry-api-0-project-appstoreconnect-requestSms",
+                ),
+                url(
+                    r"^(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/appstoreconnect/2fa/$",
+                    AppStoreConnect2FactorAuthEndpoint.as_view(),
+                    name="sentry-api-0-project-appstoreconnect-2fa",
+                ),
+                url(
+                    r"^(?P<organization_slug>[^\/]+)/(?P<project_slug>[^\/]+)/appstoreconnect/(?P<credentials_id>[^\/]+)/$",
+                    AppStoreConnectUpdateCredentialsEndpoint.as_view(),
+                    name="sentry-api-0-project-appstoreconnect-credentials-update",
+                ),
             ]
         ),
     ),

src/sentry/conf/server.py

@@ -848,6 +848,8 @@ SENTRY_FEATURES = {
     "organizations:advanced-search": True,
     # Enable obtaining and using API keys.
     "organizations:api-keys": False,
+    # Enable Apple app-store-connect dsym symbol file collection.
+    "organizations:app-store-connect": False,
     # Enable explicit use of AND and OR in search.
     "organizations:boolean-search": False,
     # Enable unfurling charts using the Chartcuterie service

src/sentry/features/__init__.py

@@ -56,6 +56,7 @@ default_manager.add("organizations:alert-details-redesign", OrganizationFeature,
 default_manager.add("organizations:alert-filters", OrganizationFeature)  # NOQA
 default_manager.add("organizations:alert-wizard", OrganizationFeature, True)  # NOQA
 default_manager.add("organizations:api-keys", OrganizationFeature)  # NOQA
+default_manager.add("organizations:app-store-connect", OrganizationFeature)  # NOQA
 default_manager.add("organizations:boolean-search", OrganizationFeature)  # NOQA
 default_manager.add("organizations:chart-unfurls", OrganizationFeature, True)  # NOQA
 default_manager.add("organizations:custom-event-title", OrganizationFeature)  # NOQA

src/sentry/lang/native/symbolicator.py

@@ -24,7 +24,6 @@ INTERNAL_SOURCE_NAME = "sentry:project"
 
 logger = logging.getLogger(__name__)
 
-
 VALID_LAYOUTS = ("native", "symstore", "symstore_index2", "ssqp", "unified", "debuginfod")
 
 VALID_FILE_TYPES = ("pe", "pdb", "mach_debug", "mach_code", "elf_debug", "elf_code", "breakpad")
@@ -48,6 +47,37 @@ COMMON_SOURCE_PROPERTIES = {
     "filetypes": {"type": "array", "items": {"type": "string", "enum": list(VALID_FILE_TYPES)}},
 }
 
+APP_STORE_CONNECT_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "type": {"type": "string", "enum": ["appStoreConnect"]},
+        "id": {"type": "string", "minLength": 1},
+        "name": {"type": "string"},
+        "appconnectIssuer": {"type": "string", "minLength": 36, "maxLength": 36},
+        "appconnectKey": {"type": "string", "minLength": 2, "maxLength": 20},
+        "itunesUser": {"type": "string", "minLength": 1, "maxLength": 100},
+        "appName": {"type": "string", "minLength": 1, "maxLength": 512},
+        "appId": {"type": "string", "minLength": 1, "maxLength": 512},
+        "orgId": {"type": "integer"},
+        "orgName": {"type": "string", "minLength": 1, "maxLength": 512},
+        "encrypted": {"type": "string"},
+    },
+    "required": [
+        "id",
+        "name",
+        "type",
+        "appconnectIssuer",
+        "appconnectKey",
+        "itunesUser",
+        "appName",
+        "appId",
+        "orgId",
+        "orgName",
+        "encrypted",
+    ],
+    "additionalProperties": False,
+}
+
 HTTP_SOURCE_SCHEMA = {
     "type": "object",
     "properties": dict(
@@ -92,7 +122,9 @@ GCS_SOURCE_SCHEMA = {
 
 SOURCES_SCHEMA = {
     "type": "array",
-    "items": {"oneOf": [HTTP_SOURCE_SCHEMA, S3_SOURCE_SCHEMA, GCS_SOURCE_SCHEMA]},
+    "items": {
+        "oneOf": [HTTP_SOURCE_SCHEMA, S3_SOURCE_SCHEMA, GCS_SOURCE_SCHEMA, APP_STORE_CONNECT_SCHEMA]
+    },
 }
 
 
@@ -278,6 +310,9 @@ def parse_sources(config):
     except jsonschema.ValidationError as e:
         raise InvalidSourcesError(e.message)
 
+    # remove App Store Connect sources (we don't need them in Symbolicator)
+    filter(lambda src: src.get("type") != "AppStoreConnect", sources)
+
     ids = set()
     for source in sources:
         if is_internal_source_id(source["id"]):

src/sentry/utils/appleconnect/appstore_connect.py

@@ -0,0 +1,224 @@
+import logging
+import time
+from collections import namedtuple
+from typing import Any, Generator, List, Mapping, Optional
+
+import jwt
+from requests import Session
+
+from sentry.utils import safe
+
+logger = logging.getLogger(__name__)
+
+AppConnectCredentials = namedtuple("AppConnectCredentials", ["key_id", "key", "issuer_id"])
+
+
+def _get_authorization_header(credentials=AppConnectCredentials, expiry_sec=None) -> str:
+    """
+    Creates a JWT (javascript web token) for use with app store connect API
+
+    All requests to app store connect require an "Authorization" header build as below.
+
+    Note: Setting a very large expiry for the token will cause the authorization to fail,
+    the default is one hour, which should be suitable for most cases.
+
+    :return: the Bearer auth token to be added as the  "Authorization" header
+    """
+    if expiry_sec is None:
+        expiry_sec = 60 * 60  # default one hour
+    token = jwt.encode(
+        {
+            "iss": credentials.issuer_id,
+            "exp": int(time.time()) + expiry_sec,
+            "aud": "appstoreconnect-v1",
+        },
+        credentials.key,
+        algorithm="ES256",
+        headers={"kid": credentials.key_id, "alg": "ES256", "typ": "JWT"},
+    )
+    return f"Bearer {token}"
+
+
+def _get_appstore_info(
+    session: Session, credentials: AppConnectCredentials, url: str
+) -> Optional[Mapping[str, Any]]:
+    """
+    Get info from an appstore url
+
+    It builds the request, and extracts the data
+
+    :return: a dictionary with the requested data or None if the call fails
+    """
+    headers = {"Authorization": _get_authorization_header(credentials)}
+
+    if not url.startswith("https://"):
+        full_url = "https://api.appstoreconnect.apple.com"
+        if url[0] != "/":
+            full_url += "/"
+    else:
+        full_url = ""
+    full_url += url
+    logger.debug(f"GET {full_url}")
+    response = session.get(full_url, headers=headers)
+    if not response.ok:
+        raise ValueError("Request failed", full_url, response.status_code, response.text)
+    try:
+        return response.json()
+    except Exception as e:
+        raise ValueError(
+            "Response body not JSON", full_url, response.status_code, response.text
+        ) from e
+
+
+def _get_next_page(response_json) -> str:
+    """
+    Gets the next page url from a app store connect paged response
+    """
+    return safe.get_path(response_json, "links", "next")
+
+
+def _get_appstore_info_paged_data(
+    session: Session, credentials: AppConnectCredentials, url: str
+) -> Generator[Any, None, None]:
+    """
+    Iterate through all the pages from a paged response and concatenate the `data` part of the response
+
+    App store connect share the general format:
+
+    data:
+      - list of elements
+    links:
+      next: link to the next page
+    ...
+
+    The function iterates through all pages (following the next link) until
+    there is no next page, and returns a generator containing all
+    the data in the arrays from each call
+
+    :return: a generator with the contents of all the arrays from each page (flattened).
+    """
+    while url is not None:
+        response = _get_appstore_info(session, credentials, url)
+        data = response["data"]
+        yield from data
+        url = _get_next_page(response)
+
+
+def get_pre_release_version_info(session: Session, credentials: AppConnectCredentials, app_id: str):
+    """
+    Get all prerelease builds version information for an application
+
+    The release build version information has the following structure:
+    platform: str - the platform for the build (e.g. IOS, MAC_OS ...)
+    short_version: str - the short version build info ( e.g. '1.0.1'), also called "train" in starship documentation
+    id: str - the IID of the version
+    versions: vec - a vector with builds
+        version: str - the version of the build (e.g. '101'), looks like the build number
+        id: str - the IID of the build
+
+    NOTE: the pre release version information is identical to the release version information
+    :return: a list of prerelease builds version information (see above)
+    """
+    url = f"v1/apps/{app_id}/preReleaseVersions"
+    data = _get_appstore_info_paged_data(session, credentials, url)
+    result = []
+    for d in data:
+        versions = []
+        v = {
+            "platform": safe.get_path(d, "attributes", "platform"),
+            "short_version": safe.get_path(d, "attributes", "version"),
+            "id": safe.get_path(d, "id"),
+            "versions": versions,
+        }
+        builds_url = safe.get_path(d, "relationships", "builds", "links", "related")
+        for build in _get_appstore_info_paged_data(session, credentials, builds_url):
+            b = {
+                "version": safe.get_path(build, "attributes", "version"),
+                "id": safe.get_path(build, "id"),
+            }
+            versions.append(b)
+        result.append(v)
+
+    return result
+
+
+def get_release_version_info(session: Session, credentials: AppConnectCredentials, app_id: str):
+    """
+    Get all release builds version information for an application
+
+    The release build version information has the following structure:
+    platform: str - the platform for the build (e.g. IOS, MAC_OS ...)
+    short_version: str - the short version build info ( e.g. '1.0.1'), also called "train" in starship documentation
+    id: str - the IID of the version
+    versions: vec - a vector with builds
+        version: str - the version of the build (e.g. '101'), looks like the build number
+        id: str - the IID of the build
+
+    NOTE: the release version information is identical to the pre release version information
+    :return: a list of release builds version information (see above)
+    """
+    url = f"v1/apps/{app_id}/appStoreVersions"
+    data = _get_appstore_info_paged_data(session, credentials, url)
+    result = []
+    for d in data:
+        versions = []
+        build_url = safe.get_path(d, "relationships", "build", "links", "related")
+        v = {
+            "platform": safe.get_path(d, "attributes", "platform"),
+            "short_version": safe.get_path(d, "attributes", "versionString"),
+            "id": safe.get_path(d, "id"),
+            "versions": versions,
+        }
+
+        build_info = _get_appstore_info_paged_data(session, credentials, build_url)
+        build_info = safe.get_path(build_info, "data")
+        if build_info is not None:
+            # Note RaduW: never seen info in this structure, I assume the same structure as pre release
+            versions.append(
+                {
+                    "id": safe.get_path(build_info, "id"),
+                    "version": safe.get_path(build_info, "attributes", "version"),
+                }
+            )
+        result.append(v)
+    return result
+
+
+def get_build_info(session: Session, credentials: AppConnectCredentials, app_id: str):
+    """
+    Returns the build info for an application
+    """
+    return {
+        "pre_releases": get_pre_release_version_info(session, credentials, app_id),
+        "releases": get_release_version_info(session, credentials, app_id),
+    }
+
+
+AppInfo = namedtuple("AppInfo", ["name", "bundle_id", "app_id"])
+
+
+def get_apps(session: Session, credentials: AppConnectCredentials) -> Optional[List[AppInfo]]:
+    """
+    Returns the available applications from an account
+    :return: a list of available applications or None if the login failed, an empty list
+    means that the login was successful but there were no applications available
+    """
+    url = "v1/apps"
+    ret_val = []
+    try:
+        apps = _get_appstore_info_paged_data(session, credentials, url)
+        for app in apps:
+            app_info = AppInfo(
+                app_id=app.get("id"),
+                bundle_id=safe.get_path(app, "attributes", "bundleId"),
+                name=safe.get_path(app, "attributes", "name"),
+            )
+            if (
+                app_info.app_id is not None
+                and app_info.bundle_id is not None
+                and app_info.name is not None
+            ):
+                ret_val.append(app_info)
+    except ValueError:
+        return None
+    return ret_val

src/sentry/utils/appleconnect/itunes_connect.py

@@ -0,0 +1,319 @@
+"""
+Contains functionality ported from fastlane-starship for getting app store connect information
+using the old (itunes) api
+"""
+
+import logging
+from collections import namedtuple
+from http import HTTPStatus
+from typing import Any, NewType, Optional
+
+from requests import Session
+
+from sentry.utils import safe
+
+logger = logging.getLogger(__name__)
+
+
+def _session_cookie_name() -> str:
+    """Returns the name of the cookie used by itunes API for the session"""
+    return "myacinfo"
+
+
+def load_session_cookie(session: Session, session_cookie_value: str):
+    """
+    Tries to load the Itunes session cookie in the current session.
+
+    If the session is still valid the user will be logged in.
+
+    """
+
+    session.cookies.set(_session_cookie_name(), session_cookie_value)
+
+
+def get_session_cookie(session: Session) -> Optional[str]:
+    """
+    Tries to extract the session cookies
+
+    :return: the session cookie if available
+    """
+    return session.cookies.get(_session_cookie_name())
+
+
+def get_session_info(session: Session) -> Optional[Any]:
+    """
+    Returns the itunes session info (if valid).
+
+    Note: port of fastlane.spaceship.client.Spaceship.Client.fetch_olympus_session
+
+    :return: The session information (a json) if the session is valid and no login is necessary None if we need to
+    login
+    """
+    url = " https://appstoreconnect.apple.com/olympus/v1/session"
+    logger.debug(f"GET {url}")
+    session_response = session.get(url)
+
+    if session_response.ok:
+        try:
+            data = session_response.json()
+        except ValueError:
+            return None
+        return data
+
+    return None
+
+
+ITunesServiceKey = NewType("ITunesServiceKey", str)
+
+
+def get_auth_service_key(session: Session) -> ITunesServiceKey:
+    """
+    Obtains the authentication service key used in X-Apple-Widget-Key header
+    :return: the service key to be used in all future calls X-Apple-Widget-Key headers
+    """
+    logger.debug(
+        "GET https://appstoreconnect.apple.com/olympus/v1/app/config?hostname=itunesconnect.apple.com"
+    )
+    svc_key_response = session.get(
+        "https://appstoreconnect.apple.com/olympus/v1/app/config?hostname=itunesconnect.apple.com"
+    )
+
+    try:
+        data = svc_key_response.json()
+        return ITunesServiceKey(data.get("authServiceKey"))
+    except (ValueError, KeyError) as err:
+        err_msg = "Could not obtain service key"
+        logger.info(err_msg, exc_info=True)
+        raise ValueError(err_msg, "auth-service-key") from err
+
+
+ITunesHeaders = namedtuple("ITunesHeaders", ["session_id", "scnt"])
+
+
+def initiate_login(
+    session: Session, service_key: str, account_name: str, password: str
+) -> Optional[ITunesHeaders]:
+    """
+    Initiate an Itunes login session, and get the header values needed for Itunes API calls.
+
+    This will also initiate a validation call to any validated machine and display to the user
+    a request for authorization (followed by giving a validation id).
+    :return: ITunesHeaders to be used in further calls
+    """
+    logger.debug("POST https://idmsa.apple.com/appleauth/auth/signin")
+
+    start_login = session.post(
+        "https://idmsa.apple.com/appleauth/auth/signin",
+        json={
+            "accountName": account_name,
+            "password": password,
+            "rememberMe": True,
+        },
+        headers={
+            "X-Requested-With": "XMLHttpRequest",
+            "X-Apple-Widget-Key": service_key,
+            "Accept": "application/json, text/javascript",
+        },
+    )
+
+    if start_login.status_code == HTTPStatus.OK:
+        # this is rather strange the user doesn't have 2 factor auth
+        return None
+    if start_login.status_code == HTTPStatus.CONFLICT:
+        return ITunesHeaders(
+            session_id=start_login.headers["x-apple-id-session-id"],
+            scnt=start_login.headers["scnt"],
+        )
+    else:
+        return None
+
+
+TrustedPhoneInfo = namedtuple(
+    "TrustedPhoneInfo", ["id", "push_mode", "number_with_dial_code", "obfuscated_number"]
+)
+
+
+def get_trusted_phone_info(
+    session: Session, service_key: ITunesServiceKey, headers: ITunesHeaders
+) -> Optional[TrustedPhoneInfo]:
+    """
+    Will return the trusted phone info for the account
+    :return: TrustedPhoneInfo if the call was successful
+    """
+    url = "https://idmsa.apple.com/appleauth/auth"
+    logger.debug(f"GET {url}")
+
+    auth_response = session.get(
+        url,
+        headers={
+            "scnt": headers.scnt,
+            "X-Apple-Id-Session-Id": headers.session_id,
+            "Accept": "application/json",
+            "X-Apple-Widget-Key": service_key,
+        },
+    )
+
+    if auth_response.status_code == HTTPStatus.OK:
+        try:
+            info = auth_response.json()["trustedPhoneNumber"]
+            return TrustedPhoneInfo(
+                id=info["id"],
+                push_mode=info["pushMode"],
+                number_with_dial_code=info["numberWithDialCode"],
+                obfuscated_number=info["obfuscatedNumber"],
+            )
+        except:  # NOQA
+            logger.info("Could not obtain trusted phone info", exc_info=True)
+            return None
+
+    return None
+
+
+def initiate_phone_login(
+    session: Session,
+    service_key: ITunesServiceKey,
+    headers: ITunesHeaders,
+    phone_id: int,
+    push_mode: str,
+) -> bool:
+    """
+    Start phone 2 factor authentication by requesting an SMS to be send to the trusted phone
+    """
+    url = "https://idmsa.apple.com/appleauth/auth/verify/phone"
+    logger.debug(f"PUT {url}")
+
+    phone_auth_response = session.put(
+        url,
+        json={"phoneNumber": {"id": phone_id}, "mode": push_mode},
+        headers={
+            "scnt": headers.scnt,
+            "X-Apple-Id-Session-Id": headers.session_id,
+            "Accept": "application/json",
+            "X-Apple-Widget-Key": service_key,
+            "Content-Type": "application/json",
+        },
+    )
+    return phone_auth_response.status_code == HTTPStatus.OK
+
+
+def send_phone_authentication_confirmation_code(
+    session: Session,
+    service_key: ITunesServiceKey,
+    headers: ITunesHeaders,
+    phone_id: int,
+    push_mode: str,
+    security_code: str,
+) -> bool:
+    """
+    Sends the confirmation code received by the trusted phone and completes the two factor authentication
+    :return: True if successful False otherwise
+    """
+    url = "https://idmsa.apple.com/appleauth/auth/verify/phone/securitycode"
+    logger.debug("PUT {url}")
+
+    phone_security_code_response = session.post(
+        url,
+        json={
+            "securityCode": {"code": security_code},
+            "phoneNumber": {"id": phone_id},
+            "mode": push_mode,
+        },
+        headers={
+            "scnt": headers.scnt,
+            "X-Apple-Id-Session-Id": headers.session_id,
+            "Accept": "application/json",
+            "X-Apple-Widget-Key": service_key,
+        },
+    )
+    if phone_security_code_response.status_code == HTTPStatus.OK:
+        return True
+
+    # If we want more info about the failure extract failure info from the response body
+    # response is a JSON with the following interesting fields:
+    # (Extracted from a login attempt with wrong code)
+    # 'restrictedAccount': False,
+    # 'securityCode': {'code': '123123',
+    #                  'securityCodeCooldown': False,
+    #                  'securityCodeLocked': False,
+    #                  'tooManyCodesSent': False,
+    #                  'tooManyCodesValidated': False},
+    # 'serviceErrors': [{'code': '-21669',
+    #                    'message': 'Incorrect verification code.',
+    #                    'suppressDismissal': False,
+    #                    'title': 'Incorrect Verification Code'}],
+
+    return False
+
+
+def send_authentication_confirmation_code(
+    session: Session, service_key: ITunesServiceKey, headers: ITunesHeaders, security_code: str
+) -> bool:
+    """
+    Sends the confirmation code received by the trusted device and completes the two factor authentication
+
+    :return: True if successful False otherwise
+    """
+    url = "https://idmsa.apple.com/appleauth/auth/verify/trusteddevice/securitycode"
+    logger.debug(f"POST {url}")
+
+    response = session.post(
+        url,
+        json={
+            "securityCode": {
+                "code": security_code,
+            }
+        },
+        headers={
+            "scnt": headers.scnt,
+            "X-Apple-Id-Session-Id": headers.session_id,
+            "Accept": "application/json",
+            "X-Apple-Widget-Key": service_key,
+        },
+    )
+
+    return response.ok
+
+
+def set_provider(session: Session, content_provider_id: int, user_id: str):
+    url = "https://appstoreconnect.apple.com//WebObjects/iTunesConnect.woa/ra/v1/session/webSession"
+    logger.debug(f"POST {url}")
+
+    select_provider_response = session.post(
+        url,
+        json={
+            "contentProviderId": content_provider_id,
+            "dsId": user_id,
+        },
+    )
+    return select_provider_response
+
+
+def get_dsym_url(
+    session: Session, app_id: str, bundle_short_version: str, bundle_version: str, platform: str
+) -> Optional[str]:
+    """
+    Returns the url for a dsyms bundle. The session must be logged in.
+    :return: The url to use for downloading the dsyms bundle
+    """
+    details_url = (
+        f"https://appstoreconnect.apple.com/WebObjects/iTunesConnect.woa/ra/apps/"
+        f"{app_id}/platforms/{platform}/trains/{bundle_short_version}/builds/"
+        f"{bundle_version}/details"
+    )
+
+    logger.debug(f" GET {details_url}")
+
+    details_response = session.get(details_url)
+
+    if details_response.status_code == HTTPStatus.OK:
+        try:
+            data = details_response.json()
+            return safe.get_path(data, "data", "dsymurl")
+        except:  # NOQA
+            logger.info(
+                f"Could not obtain dsms info for app id={app_id}, bundle_short={bundle_short_version}, "
+                f"bundle={bundle_version}, platform={platform}",
+                exc_info=True,
+            )
+            return None
+    return None

src/sentry/utils/fernet_encrypt.py

@@ -0,0 +1,60 @@
+"""
+Simple symmetric encryption utilities
+"""
+
+from typing import Any, Dict, List, Union
+
+from cryptography.fernet import Fernet
+
+from sentry.utils import json
+
+
+def create_key():
+    """
+    Generates a Key and converts it to a String for easy handling.
+    """
+    return str(Fernet.generate_key(), "utf-8")
+
+
+# anything that can be converted to and from JSON
+IntoJsonObject = Union[str, int, float, List[Any], Dict[Any, Any], None]
+
+
+def encrypt_object(obj: IntoJsonObject, key: str) -> str:
+    """
+    Encrypts an object using the given key.
+
+    Takes an object that can be deserialized in json
+    converts it to json a json string, than to a byte array.
+    It then encrypts it using the key (transformed from string to
+    bytes), finally the encrypted result ( which is a base64 string
+    as a byte array) is converted into a string (which should leave
+    the content unchanged as it is base64).
+    """
+    key = key.encode("utf-8")
+    try:
+        f = Fernet(key)
+    except Exception as e:  # NOQA
+        raise ValueError("Invalid encryption key") from e
+    json_str = json.dumps(obj)
+    json_bytes = json_str.encode("utf-8")
+    encrypted_bytes = f.encrypt(json_bytes)
+    return str(encrypted_bytes, "utf-8")
+
+
+def decrypt_object(encrypted: str, key: str) -> IntoJsonObject:
+    """
+    Decrypts an object using the given key.
+
+    Takes a string representing an encrypted json object and using
+    a key decrypts it into an object
+    """
+    key = key.encode("utf-8")
+    try:
+        f = Fernet(key)
+    except Exception as e:  # NOQA
+        raise ValueError("Invalid encryption Key") from e
+    encrypted = encrypted.encode("utf-8")
+    decrypted_bytes = f.decrypt(encrypted)
+    json_str = str(decrypted_bytes, "utf-8")
+    return json.loads(json_str)