feat(sourcemaps): Add logic to insert data into new debug ids table (#44885)

This PR aims at implementing the logic for extracting debug_ids
information from the `manifest.json` of the uploaded `.zip` artifact
bundle.

The new logic reads the updated `manifest.json` file and looks for the
`header.debug-id` and `project` fields.

Co-authored-by: getsantry[bot] <66042841+getsantry[bot]@users.noreply.github.com>
Co-authored-by: Armin Ronacher <armin.ronacher@active-4.com>

fixtures/artifact_bundle_debug_ids/files/_/_/bundle1.js

@@ -0,0 +1,4 @@
+/* eslint-disable */
+function helloWorld() {
+    alert('HelloWorld');
+}

fixtures/artifact_bundle_debug_ids/files/_/_/bundle1.min.js

@@ -0,0 +1,2 @@
+/* eslint-disable */
+function helloWorld(){alert("HelloWorld")}

fixtures/artifact_bundle_debug_ids/files/_/_/bundle1.min.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bundle1.js.map","sources":["bundle1.js"],"names":["helloWorld","alert"],"mappings":"AACA,SAASA,aACLC,MAAM,YAAY,CACtB"}

fixtures/artifact_bundle_debug_ids/files/_/_/index.js

@@ -0,0 +1,127 @@
+/* global exports */
+Object.defineProperty(exports, '__esModule', {value: true});
+const tslib_1 = require('tslib');
+const hub_1 = require('@sentry/core');
+/**
+ * This calls a function on the current hub.
+ * @param method function to call on hub.
+ * @param args to pass to function.
+ */
+function callOnHub(method) {
+  const args = [];
+  for (let _i = 1; _i < arguments.length; _i++) {
+    args[_i - 1] = arguments[_i];
+  }
+  const hub = hub_1.getCurrentHub();
+  if (hub && hub[method]) {
+    // tslint:disable-next-line:no-unsafe-any
+    return hub[method].apply(hub, tslib_1.__spread(args));
+  }
+  throw new Error(
+    'No hub defined or ' + method + ' was not found on the hub, please open a bug report.'
+  );
+}
+/**
+ * Captures an exception event and sends it to Sentry.
+ *
+ * @param exception An exception-like object.
+ * @returns The generated eventId.
+ */
+function captureException(exception) {
+  let syntheticException;
+  try {
+    throw new Error('Sentry syntheticException');
+  } catch (error) {
+    syntheticException = error;
+  }
+  return callOnHub('captureException', exception, {
+    originalException: exception,
+    syntheticException,
+  });
+}
+exports.captureException = captureException;
+/**
+ * Captures a message event and sends it to Sentry.
+ *
+ * @param message The message to send to Sentry.
+ * @param level Define the level of the message.
+ * @returns The generated eventId.
+ */
+function captureMessage(message, level) {
+  let syntheticException;
+  try {
+    throw new Error(message);
+  } catch (exception) {
+    syntheticException = exception;
+  }
+  return callOnHub('captureMessage', message, level, {
+    originalException: message,
+    syntheticException,
+  });
+}
+exports.captureMessage = captureMessage;
+/**
+ * Captures a manually created event and sends it to Sentry.
+ *
+ * @param event The event to send to Sentry.
+ * @returns The generated eventId.
+ */
+function captureEvent(event) {
+  return callOnHub('captureEvent', event);
+}
+exports.captureEvent = captureEvent;
+/**
+ * Records a new breadcrumb which will be attached to future events.
+ *
+ * Breadcrumbs will be added to subsequent events to provide more context on
+ * user's actions prior to an error or crash.
+ *
+ * @param breadcrumb The breadcrumb to record.
+ */
+function addBreadcrumb(breadcrumb) {
+  callOnHub('addBreadcrumb', breadcrumb);
+}
+exports.addBreadcrumb = addBreadcrumb;
+/**
+ * Callback to set context information onto the scope.
+ * @param callback Callback function that receives Scope.
+ */
+function configureScope(callback) {
+  callOnHub('configureScope', callback);
+}
+exports.configureScope = configureScope;
+/**
+ * Creates a new scope with and executes the given operation within.
+ * The scope is automatically removed once the operation
+ * finishes or throws.
+ *
+ * This is essentially a convenience function for:
+ *
+ *     pushScope();
+ *     callback();
+ *     popScope();
+ *
+ * @param callback that will be enclosed into push/popScope.
+ */
+function withScope(callback) {
+  callOnHub('withScope', callback);
+}
+exports.withScope = withScope;
+/**
+ * Calls a function on the latest client. Use this with caution, it's meant as
+ * in "internal" helper so we don't need to expose every possible function in
+ * the shim. It is not guaranteed that the client actually implements the
+ * function.
+ *
+ * @param method The method to call on the client/client.
+ * @param args Arguments to pass to the client/fontend.
+ */
+function _callOnClient(method) {
+  const args = [];
+  for (let _i = 1; _i < arguments.length; _i++) {
+    args[_i - 1] = arguments[_i];
+  }
+  callOnHub.apply(void 0, tslib_1.__spread(['_invokeClient', method], args));
+}
+exports._callOnClient = _callOnClient;
+// # sourceMappingURL=index.js.map

fixtures/artifact_bundle_debug_ids/files/_/_/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js.map","sources":["index.js"],"names":["Object","defineProperty","exports","value","tslib_1","require","hub_1","callOnHub","method","args","let","_i","arguments","length","hub","getCurrentHub","apply","__spread","Error","captureException","exception","syntheticException","error","originalException","captureMessage","message","level","captureEvent","event","addBreadcrumb","breadcrumb","configureScope","callback","withScope","_callOnClient"],"mappings":"AACAA,OAAOC,eAAeC,QAAS,aAAc,CAACC,MAAO,IAAI,CAAC,EAC1D,MAAMC,QAAUC,QAAQ,OAAO,EAC/B,MAAMC,MAAQD,QAAQ,cAAc,EAMpC,SAASE,UAAUC,QACjB,MAAMC,KAAO,GACb,IAAKC,IAAIC,GAAK,EAAGA,GAAKC,UAAUC,OAAQF,EAAE,GAAI,CAC5CF,KAAKE,GAAK,GAAKC,UAAUD,GAC3B,CACA,MAAMG,IAAMR,MAAMS,cAAc,EAChC,GAAID,KAAOA,IAAIN,QAAS,CAEtB,OAAOM,IAAIN,QAAQQ,MAAMF,IAAKV,QAAQa,SAASR,IAAI,CAAC,CACtD,CACA,MAAM,IAAIS,MACR,qBAAuBV,OAAS,sDAClC,CACF,CAOA,SAASW,iBAAiBC,WACxBV,IAAIW,mBACJ,IACE,MAAM,IAAIH,MAAM,2BAA2B,CAG7C,CAFE,MAAOI,OACPD,mBAAqBC,KACvB,CACA,OAAOf,UAAU,mBAAoBa,UAAW,CAC9CG,kBAAmBH,UACnBC,mBAAAA,kBACF,CAAC,CACH,CACAnB,QAAQiB,iBAAmBA,iBAQ3B,SAASK,eAAeC,QAASC,OAC/BhB,IAAIW,mBACJ,IACE,MAAM,IAAIH,MAAMO,OAAO,CAGzB,CAFE,MAAOL,WACPC,mBAAqBD,SACvB,CACA,OAAOb,UAAU,iBAAkBkB,QAASC,MAAO,CACjDH,kBAAmBE,QACnBJ,mBAAAA,kBACF,CAAC,CACH,CACAnB,QAAQsB,eAAiBA,eAOzB,SAASG,aAAaC,OACpB,OAAOrB,UAAU,eAAgBqB,KAAK,CACxC,CACA1B,QAAQyB,aAAeA,aASvB,SAASE,cAAcC,YACrBvB,UAAU,gBAAiBuB,UAAU,CACvC,CACA5B,QAAQ2B,cAAgBA,cAKxB,SAASE,eAAeC,UACtBzB,UAAU,iBAAkByB,QAAQ,CACtC,CACA9B,QAAQ6B,eAAiBA,eAczB,SAASE,UAAUD,UACjBzB,UAAU,YAAayB,QAAQ,CACjC,CACA9B,QAAQ+B,UAAYA,UAUpB,SAASC,cAAc1B,QACrB,MAAMC,KAAO,GACb,IAAKC,IAAIC,GAAK,EAAGA,GAAKC,UAAUC,OAAQF,EAAE,GAAI,CAC5CF,KAAKE,GAAK,GAAKC,UAAUD,GAC3B,CACAJ,UAAUS,MAAM,KAAK,EAAGZ,QAAQa,SAAS,CAAC,gBAAiBT,QAASC,IAAI,CAAC,CAC3E,CACAP,QAAQgC,cAAgBA"}

fixtures/artifact_bundle_debug_ids/files/_/_/index.min.js

@@ -0,0 +1,2 @@
+/* eslint-disable */
+Object.defineProperty(exports,"__esModule",{value:true});const tslib_1=require("tslib");const hub_1=require("@sentry/core");function callOnHub(method){const args=[];for(let _i=1;_i<arguments.length;_i++){args[_i-1]=arguments[_i]}const hub=hub_1.getCurrentHub();if(hub&&hub[method]){return hub[method].apply(hub,tslib_1.__spread(args))}throw new Error("No hub defined or "+method+" was not found on the hub, please open a bug report.")}function captureException(exception){let syntheticException;try{throw new Error("Sentry syntheticException")}catch(error){syntheticException=error}return callOnHub("captureException",exception,{originalException:exception,syntheticException:syntheticException})}exports.captureException=captureException;function captureMessage(message,level){let syntheticException;try{throw new Error(message)}catch(exception){syntheticException=exception}return callOnHub("captureMessage",message,level,{originalException:message,syntheticException:syntheticException})}exports.captureMessage=captureMessage;function captureEvent(event){return callOnHub("captureEvent",event)}exports.captureEvent=captureEvent;function addBreadcrumb(breadcrumb){callOnHub("addBreadcrumb",breadcrumb)}exports.addBreadcrumb=addBreadcrumb;function configureScope(callback){callOnHub("configureScope",callback)}exports.configureScope=configureScope;function withScope(callback){callOnHub("withScope",callback)}exports.withScope=withScope;function _callOnClient(method){const args=[];for(let _i=1;_i<arguments.length;_i++){args[_i-1]=arguments[_i]}callOnHub.apply(void 0,tslib_1.__spread(["_invokeClient",method],args))}exports._callOnClient=_callOnClient;

fixtures/artifact_bundle_debug_ids/manifest.json

@@ -0,0 +1,43 @@
+{
+  "org": "__org__",
+  "release": "__release__",
+  "files": {
+    "files/_/_/index.js": {
+      "url": "~/index.js",
+      "type": "source"
+    },
+    "files/_/_/index.min.js": {
+      "url": "~/index.min.js",
+      "type": "minified_source",
+      "headers": {
+        "Debug-id": "Eb6e60f1-65ff-4f6f-Adff-f1bbeDEd627b",
+        "Sourcemap": "index.js.map"
+      }
+    },
+    "files/_/_/index.js.map": {
+      "url": "~/index.js.map",
+      "type": "source_map",
+      "headers": {
+        "debug-id": "Eb6e60f1-65ff-4f6f-Adff-f1bbeDEd627b"
+      }
+    },
+    "files/_/_/bundle1.js": {
+      "url": "~/bundle1.js",
+      "type": "soure"
+    },
+    "files/_/_/bundle1.min.js": {
+      "url": "~/bundle1.js",
+      "type": "minified_source",
+      "headers": {
+        "Debug-id": "Eb6e60f1asd1bbeDEd627b-1232xwzsd",
+        "Sourcemap": "bundle1.js.map"
+      }
+    },
+    "files/_/_/bundle1.map.js": {
+      "url": "~/bundle1.js",
+      "headers": {
+        "Debug-id": "Eb6e60f1asd1bbeDEd627b-1232xwzsd"
+      }
+    }
+  }
+}

src/sentry/api/bases/organization.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any
+from typing import Any, Optional, Set
 
 import sentry_sdk
 from django.core.cache import cache
@@ -441,7 +441,11 @@ class OrganizationReleasesBaseEndpoint(OrganizationEndpoint):
         )
 
     def has_release_permission(
-        self, request: Request, organization: Organization, release: Release
+        self,
+        request: Request,
+        organization: Organization,
+        release: Optional[Release] = None,
+        project_ids: Optional[Set[int]] = None,
     ) -> bool:
         """
         Does the given request have permission to access this release, based
@@ -459,16 +463,28 @@ class OrganizationReleasesBaseEndpoint(OrganizationEndpoint):
         if getattr(request, "auth", None) and request.auth.id:
             actor_id = "apikey:%s" % request.auth.id
         if actor_id is not None:
-            project_ids = sorted(self.get_requested_project_ids_unchecked(request))
+            requested_project_ids = project_ids
+            if requested_project_ids is None:
+                requested_project_ids = self.get_requested_project_ids_unchecked(request)
             key = "release_perms:1:%s" % hash_values(
-                [actor_id, organization.id, release.id] + project_ids
+                [actor_id, organization.id, release.id if release is not None else 0]
+                + sorted(requested_project_ids)
             )
             has_perms = cache.get(key)
         if has_perms is None:
-            has_perms = ReleaseProject.objects.filter(
-                release=release, project__in=self.get_projects(request, organization)
-            ).exists()
+            projects = self.get_projects(request, organization, project_ids=project_ids)
+            # XXX(iambriccardo): The logic here is that you have access to this release if any of your projects
+            # associated with this release you have release permissions to.  This is a bit of
+            # a problem because anyone can add projects to a release, so this check is easy
+            # to defeat.
+            if release is not None:
+                has_perms = ReleaseProject.objects.filter(
+                    release=release, project__in=projects
+                ).exists()
+            else:
+                has_perms = len(projects) > 0
+
             if key is not None and actor_id is not None:
                 cache.set(key, has_perms, 60)
 
-        return has_perms  # type: ignore[no-any-return]
+        return has_perms

src/sentry/api/endpoints/chunk.py

@@ -9,7 +9,7 @@ from rest_framework import status
 from rest_framework.request import Request
 from rest_framework.response import Response
 
-from sentry import options
+from sentry import features, options
 from sentry.api.base import pending_silo_endpoint
 from sentry.api.bases.organization import OrganizationEndpoint, OrganizationReleasePermission
 from sentry.models import FileBlob
@@ -31,6 +31,8 @@ CHUNK_UPLOAD_ACCEPT = (
     "bcsymbolmaps",  # BCSymbolMaps and associated PLists/UuidMaps
     "il2cpp",  # Il2cpp LineMappingJson files
     "portablepdbs",  # Portable PDB debug file
+    # TODO: This is currently turned on by a feature flag
+    # "artifact_bundles",  # Artifact bundles containing source maps.
 )
 
 
@@ -79,6 +81,13 @@ class ChunkUploadEndpoint(OrganizationEndpoint):
             # If user overridden upload url prefix, we want an absolute, versioned endpoint, with user-configured prefix
             url = absolute_uri(relative_url, endpoint)
 
+        # TODO: artifact bundles are still feature flagged.
+        accept = CHUNK_UPLOAD_ACCEPT
+        if features.has(
+            "organizations:artifact-bundles", organization=organization, actor=request.user
+        ):
+            accept += ("artifact_bundles",)
+
         return Response(
             {
                 "url": url,
@@ -89,7 +98,7 @@ class ChunkUploadEndpoint(OrganizationEndpoint):
                 "concurrency": MAX_CONCURRENCY,
                 "hashAlgorithm": HASH_ALGORITHM,
                 "compression": ["gzip"],
-                "accept": CHUNK_UPLOAD_ACCEPT,
+                "accept": accept,
             }
         )
 

src/sentry/api/endpoints/organization_artifactbundle_assemble.py

@@ -0,0 +1,90 @@
+import jsonschema
+from rest_framework.request import Request
+from rest_framework.response import Response
+
+from sentry.api.base import region_silo_endpoint
+from sentry.api.bases.organization import OrganizationReleasesBaseEndpoint
+from sentry.api.exceptions import ResourceDoesNotExist
+from sentry.models import Project, ProjectStatus
+from sentry.tasks.assemble import (
+    AssembleTask,
+    ChunkFileState,
+    get_assemble_status,
+    set_assemble_status,
+)
+from sentry.utils import json
+
+
+@region_silo_endpoint
+class OrganizationArtifactBundleAssembleEndpoint(OrganizationReleasesBaseEndpoint):
+    def post(self, request: Request, organization) -> Response:
+        """
+        Assembles an artifact bundle and stores the debug ids in the database.
+        """
+        schema = {
+            "type": "object",
+            "properties": {
+                "projects": {"type": "array", "items": {"type": "string"}},
+                "checksum": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
+                "chunks": {
+                    "type": "array",
+                    "items": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
+                },
+            },
+            "required": ["checksum", "chunks", "projects"],
+            "additionalProperties": False,
+        }
+
+        try:
+            data = json.loads(request.body)
+            jsonschema.validate(data, schema)
+        except jsonschema.ValidationError as e:
+            return Response({"error": str(e).splitlines()[0]}, status=400)
+        except Exception:
+            return Response({"error": "Invalid json body"}, status=400)
+
+        projects = set(data.get("projects", []))
+        if len(projects) == 0:
+            return Response({"error": "You need to specify at least one project"}, status=400)
+
+        project_ids = Project.objects.filter(
+            organization=organization, status=ProjectStatus.VISIBLE, slug__in=projects
+        ).values_list("id", flat=True)
+        if len(project_ids) != len(projects):
+            return Response({"error": "One or more projects are invalid"}, status=400)
+
+        if not self.has_release_permission(request, organization, project_ids=set(project_ids)):
+            raise ResourceDoesNotExist
+
+        checksum = data.get("checksum")
+        chunks = data.get("chunks", [])
+
+        state, detail = get_assemble_status(AssembleTask.ARTIFACTS, organization.id, checksum)
+        if state == ChunkFileState.OK:
+            return Response({"state": state, "detail": None, "missingChunks": []}, status=200)
+        elif state is not None:
+            return Response({"state": state, "detail": detail, "missingChunks": []})
+
+        # There is neither a known file nor a cached state, so we will
+        # have to create a new file.  Assure that there are checksums.
+        # If not, we assume this is a poll and report NOT_FOUND
+        if not chunks:
+            return Response({"state": ChunkFileState.NOT_FOUND, "missingChunks": []}, status=200)
+
+        set_assemble_status(
+            AssembleTask.ARTIFACTS, organization.id, checksum, ChunkFileState.CREATED
+        )
+
+        from sentry.tasks.assemble import assemble_artifacts
+
+        assemble_artifacts.apply_async(
+            kwargs={
+                "org_id": organization.id,
+                "project_ids": list(project_ids),
+                "version": None,
+                "checksum": checksum,
+                "chunks": chunks,
+            }
+        )
+
+        return Response({"state": ChunkFileState.CREATED, "missingChunks": []}, status=200)