Don't split files into multiple blobs.

difs/tasks.py

@@ -1,17 +1,16 @@
-import tempfile
 import contextlib
 import logging
-from celery import shared_task
-from projects.models import Project
-from files.models import FileBlob, File
+import tempfile
 from hashlib import sha1
-from symbolic import (
-    Archive,
-)
+
+from celery import shared_task
+from symbolic import Archive
+
 from difs.models import DebugInformationFile
-from events.models import Event
 from difs.stacktrace_processor import StacktraceProcessor
-from django.conf import settings
+from events.models import Event
+from files.models import File, FileBlob
+from projects.models import Project
 
 
 def getLogger():
@@ -34,9 +33,7 @@ DIF_STATE_NOT_FOUND = "not_found"
 @shared_task
 def difs_assemble(project_slug, name, checksum, chunks, debug_id):
     try:
-        project = Project.objects.filter(
-            slug=project_slug
-        ).get()
+        project = Project.objects.filter(slug=project_slug).get()
 
         file = difs_get_file_from_chunks(checksum, chunks)
         if file is None:
@@ -45,14 +42,13 @@ def difs_assemble(project_slug, name, checksum, chunks, debug_id):
         difs_create_difs(project, name, file)
 
     except ChecksumMismatched:
-        getLogger().error(f"difs_assemble: Checksum mismatched: {name}")
-    except Exception as e:
-        getLogger().error(f"difs_assemble: {e}")
+        getLogger().error("difs_assemble: Checksum mismatched: %s", name)
+    except Exception as err:
+        getLogger().error("difs_assemble: %s", err)
 
 
 def difs_run_resolve_stacktrace(event_id):
-    if settings.GLITCHTIP_ENABLE_DIFS:
-        difs_resolve_stacktrace.delay(event_id)
+    difs_resolve_stacktrace.delay(event_id)
 
 
 @shared_task
@@ -67,27 +63,26 @@ def difs_resolve_stacktrace(event_id):
 
     project_id = event.issue.project_id
 
-    difs = DebugInformationFile.objects.filter(
-        project_id=project_id).order_by("-created")
+    difs = DebugInformationFile.objects.filter(project_id=project_id).order_by(
+        "-created"
+    )
     resolved_stracktrackes = []
 
     for dif in difs:
         if StacktraceProcessor.is_supported(event_json, dif) is False:
             continue
-        blobs = dif.file.blobs.all()
+        blobs = [dif.file.blob]
         with difs_concat_file_blobs_to_disk(blobs) as symbol_file:
             remapped_stacktrace = StacktraceProcessor.resolve_stacktrace(
-                event_json,
-                symbol_file.name
+                event_json, symbol_file.name
             )
-            if (remapped_stacktrace is not None and
-                    remapped_stacktrace.score > 0):
+            if remapped_stacktrace is not None and remapped_stacktrace.score > 0:
                 resolved_stracktrackes.append(remapped_stacktrace)
     if len(resolved_stracktrackes) > 0:
         best_remapped_stacktrace = max(
-            resolved_stracktrackes, key=lambda item: item.score)
-        StacktraceProcessor.update_frames(
-            event, best_remapped_stacktrace.frames)
+            resolved_stracktrackes, key=lambda item: item.score
+        )
+        StacktraceProcessor.update_frames(event, best_remapped_stacktrace.frames)
         event.save()
 
 
@@ -95,8 +90,8 @@ def difs_get_file_from_chunks(checksum, chunks):
     files = File.objects.filter(checksum=checksum)
 
     for file in files:
-        blobs = file.blobs.all()
-        file_chunks = [blob.checksum for blob in blobs]
+        blob = file.blob
+        file_chunks = [blob.checksum]
         if file_chunks == chunks:
             return file
 
@@ -106,7 +101,7 @@ def difs_get_file_from_chunks(checksum, chunks):
 def difs_create_file_from_chunks(name, checksum, chunks):
     blobs = FileBlob.objects.filter(checksum__in=chunks)
 
-    total_checksum = sha1(b'')
+    total_checksum = sha1(b"")
     size = 0
 
     for blob in blobs:
@@ -120,14 +115,9 @@ def difs_create_file_from_chunks(name, checksum, chunks):
     if checksum != total_checksum:
         raise ChecksumMismatched()
 
-    file = File(
-        name=name,
-        headers={},
-        size=size,
-        checksum=checksum
-    )
+    file = File(name=name, headers={}, size=size, checksum=checksum)
+    file.blob = blobs[0]
     file.save()
-    file.blobs.set(blobs)
     return file
 
 
@@ -148,13 +138,13 @@ def difs_concat_file_blobs_to_disk(blobs):
 
 
 def difs_extract_metadata_from_file(file):
-    with difs_concat_file_blobs_to_disk(file.blobs.all()) as input:
+    with difs_concat_file_blobs_to_disk([file.blob]) as _input:
         # Only one kind of file format is supported now
         try:
-            archive = Archive.open(input.name)
-        except Exception as e:
-            getLogger().error(f"Extract metadata error: {e}")
-            raise UnsupportedFile()
+            archive = Archive.open(_input.name)
+        except Exception as err:
+            getLogger().error("Extract metadata error: %s", err)
+            raise UnsupportedFile() from err
         else:
             return [
                 {
@@ -164,7 +154,7 @@ def difs_extract_metadata_from_file(file):
                     "debug_id": obj.debug_id,
                     "kind": obj.kind,
                     "features": list(obj.features),
-                    "symbol_type": "native"
+                    "symbol_type": "native",
                 }
                 for obj in archive.iter_objects()
             ]
@@ -174,8 +164,7 @@ def difs_create_difs(project, name, file):
     metadatalist = difs_extract_metadata_from_file(file)
     for metadata in metadatalist:
         dif = DebugInformationFile.objects.filter(
-            project_id=project.id,
-            file=file
+            project_id=project.id, file=file
         ).first()
 
         if dif is not None:
@@ -198,7 +187,7 @@ def difs_create_difs(project, name, file):
                 "code_id": code_id,
                 "kind": kind,
                 "features": features,
-                "symbol_type": symbol_type
-            }
+                "symbol_type": symbol_type,
+            },
         )
         dif.save()

difs/tests.py

@@ -1,36 +1,29 @@
+import contextlib
 import tempfile
-from django.core.files import File as DjangoFile
+from hashlib import sha1
+from unittest.mock import MagicMock, patch
+
 from django.conf import settings
-from glitchtip.test_utils.test_case import GlitchTipTestCase
+from django.core.files import File as DjangoFile
+from django.core.files.uploadedfile import SimpleUploadedFile
+from model_bakery import baker
+
 from difs.tasks import (
+    ChecksumMismatched,
     difs_create_file_from_chunks,
     difs_get_file_from_chunks,
-    difs_concat_file_blobs_to_disk,
-    ChecksumMismatched
 )
-from django.core.files.uploadedfile import SimpleUploadedFile
 from files.models import File
-from model_bakery import baker
-from hashlib import sha1
-import contextlib
-from unittest.mock import patch, MagicMock
+from glitchtip.test_utils.test_case import GlitchTipTestCase
 
 
 class DebugInformationFileModelTestCase(GlitchTipTestCase):
-
     def test_is_proguard(self):
-        dif = baker.make(
-            "difs.DebugInformationFile"
-        )
+        dif = baker.make("difs.DebugInformationFile")
 
         self.assertEqual(dif.is_proguard_mapping(), False)
 
-        dif = baker.make(
-            "difs.DebugInformationFile",
-            data={
-                "symbol_type": "proguard"
-            }
-        )
+        dif = baker.make("difs.DebugInformationFile", data={"symbol_type": "proguard"})
         self.assertEqual(dif.is_proguard_mapping(), True)
 
 
@@ -41,86 +34,54 @@ class DifsAssembleAPITestCase(GlitchTipTestCase):
         self.checksum = "0892b6a9469438d9e5ffbf2807759cd689996271"
         self.chunks = [
             "efa73a85c44d64e995ade0cc3286ea47cfc49c36",
-            "966e44663054d6c1f38d04c6ff4af83467659bd7"
+            "966e44663054d6c1f38d04c6ff4af83467659bd7",
         ]
         self.data = {
             self.checksum: {
                 "name": "test",
                 "debug_id": "a959d2e6-e4e5-303e-b508-670eb84b392c",
-                "chunks": self.chunks
+                "chunks": self.chunks,
             }
         }
-        settings.GLITCHTIP_ENABLE_DIFS = True
-
-    def tearDown(self):
-        settings.GLITCHTIP_ENABLE_DIFS = False
 
     def test_difs_assemble_with_dif_existed(self):
-        file = baker.make(
-            "files.File",
-            checksum=self.checksum
-        )
+        file = baker.make("files.File", checksum=self.checksum)
         baker.make(
             "difs.DebugInformationFile",
             project=self.project,
             file=file,
         )
 
-        expected_response = {
-            self.checksum: {
-                "state": "ok",
-                "missingChunks": []
-            }
-        }
+        expected_response = {self.checksum: {"state": "ok", "missingChunks": []}}
 
-        response = self.client.post(self.url,
-                                    self.data,
-                                    format='json'
-                                    )
+        response = self.client.post(self.url, self.data, format="json")
         self.assertEqual(response.data, expected_response)
 
     def test_difs_assemble_with_missing_chunks(self):
-        baker.make(
-            "files.FileBlob",
-            checksum=self.chunks[0]
-        )
+        baker.make("files.FileBlob", checksum=self.chunks[0])
 
         data = {
             self.checksum: {
                 "name": "test",
                 "debug_id": "a959d2e6-e4e5-303e-b508-670eb84b392c",
-                "chunks": self.chunks
+                "chunks": self.chunks,
             }
         }
 
         expected_response = {
-            self.checksum: {
-                "state": "not_found",
-                "missingChunks": [self.chunks[1]]
-            }
+            self.checksum: {"state": "not_found", "missingChunks": [self.chunks[1]]}
         }
 
-        response = self.client.post(self.url,
-                                    data,
-                                    format='json'
-                                    )
+        response = self.client.post(self.url, data, format="json")
         self.assertEqual(response.data, expected_response)
 
     def test_difs_assemble_without_missing_chunks(self):
         for chunk in self.chunks:
             baker.make("files.FileBlob", checksum=chunk)
 
-        expected_response = {
-            self.checksum: {
-                "state": "created",
-                "missingChunks": []
-            }
-        }
+        expected_response = {self.checksum: {"state": "created", "missingChunks": []}}
 
-        response = self.client.post(self.url,
-                                    self.data,
-                                    format='json'
-                                    )
+        response = self.client.post(self.url, self.data, format="json")
         self.assertEqual(response.data, expected_response)
 
 
@@ -130,10 +91,6 @@ class DsymsAPIViewTestCase(GlitchTipTestCase):
         self.url = f"/api/0/projects/{self.organization.slug}/{self.project.slug}/files/dsyms/"  # noqa
         self.uuid = "afb116cf-efec-49af-a7fe-281ac680d8a0"
         self.checksum = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
-        settings.GLITCHTIP_ENABLE_DIFS = True
-
-    def tearDown(self):
-        settings.GLITCHTIP_ENABLE_DIFS = False
 
     @contextlib.contextmanager
     def patch(self):
@@ -141,12 +98,14 @@ class DsymsAPIViewTestCase(GlitchTipTestCase):
         proguard_file.read.return_value = b""
 
         uploaded_zip_file = MagicMock()
-        uploaded_zip_file.namelist.return_value = iter(
-            [f"proguard/{self.uuid}.txt"])
-        uploaded_zip_file.open.return_value.__enter__.return_value = proguard_file  # noqa
+        uploaded_zip_file.namelist.return_value = iter([f"proguard/{self.uuid}.txt"])
+        uploaded_zip_file.open.return_value.__enter__.return_value = (
+            proguard_file  # noqa
+        )
 
-        with patch('zipfile.is_zipfile', return_value=True), \
-                patch('zipfile.ZipFile') as ZipFile:
+        with patch("zipfile.is_zipfile", return_value=True), patch(
+            "zipfile.ZipFile"
+        ) as ZipFile:
 
             ZipFile.return_value.__enter__.return_value = uploaded_zip_file
             yield
@@ -156,31 +115,27 @@ class DsymsAPIViewTestCase(GlitchTipTestCase):
         It should return the expected response
         """
         upload_file = SimpleUploadedFile(
-            "example.zip",
-            b"random_content",
-            content_type="multipart/form-data"
+            "example.zip", b"random_content", content_type="multipart/form-data"
         )
-        data = {
-            "file": upload_file
-        }
+        data = {"file": upload_file}
 
         with self.patch():
             response = self.client.post(self.url, data)
 
-        expected_response = [{
-            "id": response.data[0]["id"],
-            "debugId": self.uuid,
-            "cpuName": "any",
-            'objectName': 'proguard-mapping',
-            'symbolType': 'proguard',
-            'headers': {'Content-Type': 'text/x-proguard+plain'},
-            'size': 0,
-            'sha1': self.checksum,
-            "dateCreated": response.data[0]["dateCreated"],
-            "data": {
-                "features": ["mapping"]
+        expected_response = [
+            {
+                "id": response.data[0]["id"],
+                "debugId": self.uuid,
+                "cpuName": "any",
+                "objectName": "proguard-mapping",
+                "symbolType": "proguard",
+                "headers": {"Content-Type": "text/x-proguard+plain"},
+                "size": 0,
+                "sha1": self.checksum,
+                "dateCreated": response.data[0]["dateCreated"],
+                "data": {"features": ["mapping"]},
             }
-        }]
+        ]
 
         self.assertEqual(response.status_code, 200)
         self.assertEqual(len(response.data), 1)
@@ -191,48 +146,36 @@ class DsymsAPIViewTestCase(GlitchTipTestCase):
         It should success and return the expected response
         """
 
-        baker.make(
-            "files.FileBlob",
-            checksum=self.checksum
-        )
+        baker.make("files.FileBlob", checksum=self.checksum)
 
-        fileobj = baker.make(
-            "files.File",
-            checksum=self.checksum
-        )
+        fileobj = baker.make("files.File", checksum=self.checksum)
 
         dif = baker.make(
-            "difs.DebugInformationFile",
-            file=fileobj,
-            project=self.project
+            "difs.DebugInformationFile", file=fileobj, project=self.project
         )
 
         upload_file = SimpleUploadedFile(
-            "example.zip",
-            b"random_content",
-            content_type="multipart/form-data"
+            "example.zip", b"random_content", content_type="multipart/form-data"
         )
-        data = {
-            "file": upload_file
-        }
+        data = {"file": upload_file}
 
         with self.patch():
             response = self.client.post(self.url, data)
 
-        expected_response = [{
-            "id": dif.id,
-            "debugId": self.uuid,
-            "cpuName": "any",
-            'objectName': 'proguard-mapping',
-            'symbolType': 'proguard',
-            'headers': {'Content-Type': 'text/x-proguard+plain'},
-            'size': 0,
-            'sha1': 'da39a3ee5e6b4b0d3255bfef95601890afd80709',
-            "dateCreated": response.data[0]["dateCreated"],
-            "data": {
-                "features": ["mapping"]
+        expected_response = [
+            {
+                "id": dif.id,
+                "debugId": self.uuid,
+                "cpuName": "any",
+                "objectName": "proguard-mapping",
+                "symbolType": "proguard",
+                "headers": {"Content-Type": "text/x-proguard+plain"},
+                "size": 0,
+                "sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
+                "dateCreated": response.data[0]["dateCreated"],
+                "data": {"features": ["mapping"]},
             }
-        }]
+        ]
 
         self.assertEqual(response.status_code, 200)
         self.assertEqual(len(response.data), 1)
@@ -240,13 +183,9 @@ class DsymsAPIViewTestCase(GlitchTipTestCase):
 
     def test_post_invalid_zip_file(self):
         upload_file = SimpleUploadedFile(
-            "example.zip",
-            b"random_content",
-            content_type="multipart/form-data"
+            "example.zip", b"random_content", content_type="multipart/form-data"
         )
-        data = {
-            "file": upload_file
-        }
+        data = {"file": upload_file}
         response = self.client.post(self.url, data)
 
         expected_response = {"error": "Invalid file type uploaded"}
@@ -260,7 +199,7 @@ class DifsTasksTestCase(GlitchTipTestCase):
         self.create_user_and_project()
 
     def create_file_blob(self, name, content):
-        bin = content.encode('utf-8')
+        bin = content.encode("utf-8")
         tmp = tempfile.NamedTemporaryFile()
         tmp.write(bin)
         tmp.flush()
@@ -298,14 +237,3 @@ class DifsTasksTestCase(GlitchTipTestCase):
         file = difs_get_file_from_chunks(checksum, chunks)
 
         self.assertEqual(file.checksum, checksum)
-
-    def test_difs_concat_file_blobs_to_disk(self):
-        fileblob1 = self.create_file_blob("1", "1")
-        fileblob2 = self.create_file_blob("2", "2")
-        checksum = sha1(b"12").hexdigest()
-        chunks = [fileblob1.checksum, fileblob2.checksum]
-        file = difs_create_file_from_chunks("12", checksum, chunks)
-
-        with difs_concat_file_blobs_to_disk(file.blobs.all()) as fd:
-            content = fd.read()
-            self.assertEqual(content, b"12")

difs/views.py

@@ -1,61 +1,48 @@
 """ Port of sentry.api.endpoints.debug_files.DifAssembleEndpoint """
-from django.shortcuts import get_object_or_404
-from django.conf import settings
-from django.db import transaction
-from rest_framework.response import Response
-from organizations_ext.models import Organization
-from projects.models import Project
-from rest_framework import views, exceptions, status
-from .tasks import (
-    difs_assemble, DIF_STATE_CREATED, DIF_STATE_OK,
-    DIF_STATE_NOT_FOUND
-)
-from .models import DebugInformationFile
-from files.models import FileBlob, File
-from .permissions import (
-    DifsAssemblePermission, ProjectReprocessingPermission,
-    DymsPermission
-)
-from django.core.files import File as DjangoFile
-import zipfile
 import io
 import re
 import tempfile
+import zipfile
 from hashlib import sha1
-from symbolic import ProguardMapper
 
+from django.core.files import File as DjangoFile
+from django.db import transaction
+from django.shortcuts import get_object_or_404
+from rest_framework import exceptions, status, views
+from rest_framework.response import Response
+from symbolic import ProguardMapper
 
-MAX_UPLOAD_BLOB_SIZE = 8 * 1024 * 1024  # 8MB
+from files.models import File, FileBlob
+from organizations_ext.models import Organization
+from projects.models import Project
 
+from .models import DebugInformationFile
+from .permissions import (
+    DifsAssemblePermission,
+    DymsPermission,
+    ProjectReprocessingPermission,
+)
+from .tasks import DIF_STATE_CREATED, DIF_STATE_NOT_FOUND, DIF_STATE_OK, difs_assemble
 
-def check_difs_enabled(func):
-    def wrapper(*args, **kwargs):
-        if settings.GLITCHTIP_ENABLE_DIFS is not True:
-            raise exceptions.PermissionDenied()
-        return func(*args, **kwargs)
-    return wrapper
+MAX_UPLOAD_BLOB_SIZE = 32 * 1024 * 1024  # 32MB
 
 
 class DifsAssembleAPIView(views.APIView):
     permission_classes = [DifsAssemblePermission]
 
-    @check_difs_enabled
     def post(self, request, organization_slug, project_slug):
         organization = get_object_or_404(
-            Organization,
-            slug=organization_slug.lower(),
-            users=self.request.user
+            Organization, slug=organization_slug.lower(), users=self.request.user
         )
 
         self.check_object_permissions(request, organization)
 
-        project = get_object_or_404(
-            Project, slug=project_slug.lower()
-        )
+        project = get_object_or_404(Project, slug=project_slug.lower())
 
         if project.organization.id != organization.id:
             raise exceptions.PermissionDenied(
-                "The project is not under this organization")
+                "The project is not under this organization"
+            )
 
         responses = {}
 
@@ -65,9 +52,13 @@ class DifsAssembleAPIView(views.APIView):
             chunks = file.get("chunks", [])
             name = file.get("name", None)
             debug_id = file.get("debug_id", None)
-            file = DebugInformationFile.objects.filter(
-                project__slug=project_slug, file__checksum=checksum
-            ).select_related("file").first()
+            file = (
+                DebugInformationFile.objects.filter(
+                    project__slug=project_slug, file__checksum=checksum
+                )
+                .select_related("file")
+                .first()
+            )
 
             if file is not None:
                 responses[checksum] = {
@@ -76,23 +67,20 @@ class DifsAssembleAPIView(views.APIView):
                 }
                 continue
 
-            existed_chunks = FileBlob.objects.filter(
-                checksum__in=chunks
-            ).values_list("checksum", flat=True)
+            existed_chunks = FileBlob.objects.filter(checksum__in=chunks).values_list(
+                "checksum", flat=True
+            )
 
             missing_chunks = list(set(chunks) - set(existed_chunks))
 
             if len(missing_chunks) != 0:
                 responses[checksum] = {
                     "state": DIF_STATE_NOT_FOUND,
-                    "missingChunks": missing_chunks
+                    "missingChunks": missing_chunks,
                 }
                 continue
 
-            responses[checksum] = {
-                "state": DIF_STATE_CREATED,
-                "missingChunks": []
-            }
+            responses[checksum] = {"state": DIF_STATE_CREATED, "missingChunks": []}
             difs_assemble.delay(project_slug, name, checksum, chunks, debug_id)
 
         return Response(responses)
@@ -105,13 +93,12 @@ class ProjectReprocessingAPIView(views.APIView):
 
     permission_classes = [ProjectReprocessingPermission]
 
-    @check_difs_enabled
     def post(self, request, organization_slug, project_slug):
         return Response()
 
 
 def extract_proguard_id(name):
-    match = re.search('proguard/([-a-fA-F0-9]+).txt', name)
+    match = re.search("proguard/([-a-fA-F0-9]+).txt", name)
     if match is None:
         return
     return match.group(1)
@@ -121,13 +108,10 @@ def extract_proguard_metadata(proguard_file):
     try:
         mapper = ProguardMapper.open(proguard_file)
 
-        if (mapper is None):
+        if mapper is None:
             return
 
-        metadata = {
-            "arch": "any",
-            "feature": "mapping"
-        }
+        metadata = {"arch": "any", "feature": "mapping"}
 
         return metadata
 
@@ -139,25 +123,22 @@ class DsymsAPIView(views.APIView):
     """
     Implementation of /files/dsyms API View
     """
+
     permission_classes = [DymsPermission]
 
-    @check_difs_enabled
     def post(self, request, organization_slug, project_slug):
         organization = get_object_or_404(
-            Organization,
-            slug=organization_slug.lower(),
-            users=self.request.user
+            Organization, slug=organization_slug.lower(), users=self.request.user
         )
 
         self.check_object_permissions(request, organization)
 
-        project = get_object_or_404(
-            Project, slug=project_slug.lower()
-        )
+        project = get_object_or_404(Project, slug=project_slug.lower())
 
         if project.organization.id != organization.id:
             raise exceptions.PermissionDenied(
-                "The project is not under this organization")
+                "The project is not under this organization"
+            )
 
         if "file" not in request.data:
             return Response(
@@ -196,31 +177,27 @@ class DsymsAPIView(views.APIView):
 
                     with uploaded_zip_file.open(filename) as proguard_file:
                         result = self.create_dif_from_read_only_file(
-                            proguard_file,
-                            project,
-                            proguard_id,
-                            filename)
+                            proguard_file, project, proguard_id, filename
+                        )
                         if result is None:
                             return Response(
-                                {"error": "Invalid proguard mapping file uploaded"},  # noqa
+                                {
+                                    "error": "Invalid proguard mapping file uploaded"
+                                },  # noqa
                                 status=status.HTTP_400_BAD_REQUEST,
                             )
                         results.append(result)
 
             return Response(results)
 
-        except Exception as e:
+        except Exception as err:
             return Response(
-                {"error": str(e)},
+                {"error": str(err)},
                 status=status.HTTP_400_BAD_REQUEST,
             )
 
     def create_dif_from_read_only_file(
-        self,
-        proguard_file,
-        project,
-        proguard_id,
-        filename
+        self, proguard_file, project, proguard_id, filename
     ):
         with tempfile.NamedTemporaryFile("br+") as tmp:
             content = proguard_file.read()
@@ -233,21 +210,14 @@ class DsymsAPIView(views.APIView):
             with transaction.atomic():
                 size = len(content)
 
-                blob = FileBlob.objects.filter(
-                    checksum=checksum
-                ).first()
+                blob = FileBlob.objects.filter(checksum=checksum).first()
 
                 if blob is None:
-                    blob = FileBlob(
-                        checksum=checksum,  # noqa
-                        size=size
-                    )
+                    blob = FileBlob(checksum=checksum, size=size)  # noqa
                     blob.blob.save(filename, DjangoFile(tmp))
                     blob.save()
 
-                fileobj = File.objects.filter(
-                    checksum=checksum
-                ).first()
+                fileobj = File.objects.filter(checksum=checksum).first()
 
                 if fileobj is None:
                     fileobj = File()
@@ -255,13 +225,11 @@ class DsymsAPIView(views.APIView):
                     fileobj.headers = {}
                     fileobj.checksum = checksum
                     fileobj.size = size
+                    fileobj.blob = blob
                     fileobj.save()
 
-                    fileobj.blobs.set([blob])
-
                 dif = DebugInformationFile.objects.filter(
-                    file__checksum=checksum,
-                    project=project
+                    file__checksum=checksum, project=project
                 ).first()
 
                 if dif is None:
@@ -273,7 +241,7 @@ class DsymsAPIView(views.APIView):
                         "arch": metadata["arch"],
                         "debug_id": proguard_id,
                         "symbol_type": "proguard",
-                        "features": ["mapping"]
+                        "features": ["mapping"],
                     }
                     dif.save()
 
@@ -285,12 +253,8 @@ class DsymsAPIView(views.APIView):
                     "symbolType": "proguard",
                     "size": size,
                     "sha1": checksum,
-                    "data": {
-                        "features": ["mapping"]
-                    },
-                    "headers": {
-                        "Content-Type": "text/x-proguard+plain"
-                    },
+                    "data": {"features": ["mapping"]},
+                    "headers": {"Content-Type": "text/x-proguard+plain"},
                     "dateCreated": fileobj.created,
                 }
 

events/event_processors/javascript.py

@@ -3,12 +3,13 @@ import itertools
 import re
 from os.path import splitext
 from urllib.parse import urlsplit
+
 from symbolic import SourceMapView, SourceView
-from sentry.utils.safe import get_path
 
 from files.models import File
-from .base import EventProcessorBase
+from sentry.utils.safe import get_path
 
+from .base import EventProcessorBase
 
 UNKNOWN_MODULE = "<unknown module>"
 CLEAN_MODULE_RE = re.compile(
@@ -42,7 +43,7 @@ def generate_module(src):
     if not src:
         return UNKNOWN_MODULE
 
-    filename, ext = splitext(urlsplit(src).path)
+    filename, _ = splitext(urlsplit(src).path)
     if filename.endswith(".min"):
         filename = filename[:-4]
 
@@ -85,12 +86,10 @@ class JavascriptEventProcessor(EventProcessorBase):
         if not frame.get("abs_path") or not frame.get("lineno"):
             return
 
-        sourcemap_view = SourceMapView.from_json_bytes(
-            map_file.blobs.first().blob.read()
-        )
-        minified_source_view = SourceView.from_bytes(
-            minified_source.blobs.first().blob.read()
-        )
+        minified_source.blob.blob.seek(0)
+        map_file.blob.blob.seek(0)
+        sourcemap_view = SourceMapView.from_json_bytes(map_file.blob.blob.read())
+        minified_source_view = SourceView.from_bytes(minified_source.blob.blob.read())
         token = sourcemap_view.lookup(
             frame["lineno"] - 1,
             frame["colno"] - 1,

events/tests/test_javascript_processor.py

@@ -73,18 +73,22 @@ class JavaScriptProcessorTestCase(APITestCase):
         )
 
     def test_process_sourcemap(self):
-        release_file_bundle = baker.make(
-            "releases.ReleaseFile", release=self.release, file__name="bundle.js"
-        )
-        release_file_bundle_map = baker.make(
-            "releases.ReleaseFile", release=self.release, file__name="bundle.js.map"
-        )
         blob_bundle = baker.make("files.FileBlob", blob="uploads/file_blobs/bundle.js")
         blob_bundle_map = baker.make(
             "files.FileBlob", blob="uploads/file_blobs/bundle.js.map"
         )
-        release_file_bundle.file.blobs.add(blob_bundle)
-        release_file_bundle_map.file.blobs.add(blob_bundle_map)
+        release_file_bundle = baker.make(
+            "releases.ReleaseFile",
+            release=self.release,
+            file__name="bundle.js",
+            file__blob=blob_bundle,
+        )
+        release_file_bundle_map = baker.make(
+            "releases.ReleaseFile",
+            release=self.release,
+            file__name="bundle.js.map",
+            file__blob=blob_bundle_map,
+        )
         shutil.copyfile(
             "./events/tests/test_data/bundle.js", "./uploads/file_blobs/bundle.js"
         )

files/admin.py

@@ -9,7 +9,7 @@ class FileBlobAdmin(admin.ModelAdmin):
 
 class FileAdmin(admin.ModelAdmin):
     search_fields = ("name", "checksum")
-    list_display = ("name", "type", "checksum")
+    list_display = ("name", "type", "checksum", "blob")
     list_filter = ("type", "created")
 
 

files/migrations/0007_remove_file_blobs_file_blob.py

@@ -0,0 +1,41 @@
+# Generated by Django 4.0.4 on 2022-04-15 14:38
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+def forwards_func(apps, schema_editor):
+    File = apps.get_model("files", "File")
+    for file in File.objects.all():
+        file_blob = file.blobs.first()
+        if file_blob:
+            file.blob = file_blob
+            file.save()
+
+
+def reverse_func(apps, schema_editor):
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("files", "0006_alter_file_headers"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="file",
+            name="blob",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                to="files.fileblob",
+            ),
+        ),
+        migrations.RunPython(forwards_func, reverse_func),
+        migrations.RemoveField(
+            model_name="file",
+            name="blobs",
+        ),
+    ]

files/models.py

@@ -83,19 +83,19 @@ class File(CreatedModel):
     name = models.TextField()
     type = models.CharField(max_length=64)
     headers = models.JSONField(blank=True, null=True)
-    blobs = models.ManyToManyField(FileBlob)
+    blob = models.ForeignKey(FileBlob, on_delete=models.CASCADE, null=True)
     size = models.PositiveIntegerField(null=True)
     checksum = models.CharField(max_length=40, null=True, db_index=True)
 
     def put_django_file(self, fileobj):
-        """ Save a Django File object as a File Blob """
+        """Save a Django File object as a File Blob"""
         self.size = fileobj.size
         file_blob = FileBlob.from_file(fileobj)
         self.checksum = file_blob.checksum
         self.save()
 
     def putfile(self, fileobj):
-        """ Save a file-like object as a File Blob """
+        """Save a file-like object as a File Blob"""
         size, checksum = _get_size_and_checksum(fileobj)
         fileobj.seek(0)
         file_blob, _ = FileBlob.objects.get_or_create(
@@ -104,8 +104,8 @@ class File(CreatedModel):
             checksum=checksum,
         )
         self.checksum = checksum
+        self.blob = file_blob
         self.save()
-        self.blobs.add(file_blob)
 
     def _get_chunked_blob(
         self, mode=None, prefetch=False, prefetch_to=None, delete=True

files/views.py

@@ -11,11 +11,11 @@ from organizations_ext.models import Organization
 from .models import FileBlob
 from .permissions import ChunkUploadPermission
 
-
-CHUNK_UPLOAD_BLOB_SIZE = 8 * 1024 * 1024  # 8MB
-MAX_CHUNKS_PER_REQUEST = 64
-MAX_REQUEST_SIZE = 32 * 1024 * 1024
-MAX_CONCURRENCY = settings.DEBUG and 1 or 8
+# Force just one blob
+CHUNK_UPLOAD_BLOB_SIZE = 32 * 1024 * 1024  # 32MB
+MAX_CHUNKS_PER_REQUEST = 1
+MAX_REQUEST_SIZE = CHUNK_UPLOAD_BLOB_SIZE
+MAX_CONCURRENCY = 1
 HASH_ALGORITHM = "sha1"
 
 CHUNK_UPLOAD_ACCEPT = (
@@ -66,6 +66,7 @@ class ChunkUploadAPIView(views.APIView):
 
         files = request.data.getlist("file")
         files += [GzipChunk(chunk) for chunk in request.data.getlist("file_gzip")]
+
         if len(files) == 0:
             # No files uploaded is ok
             logger.info("chunkupload.end", extra={"status": status.HTTP_200_OK})

glitchtip/settings.py

@@ -88,8 +88,6 @@ EVENT_STORE_DEBUG = env.bool("EVENT_STORE_DEBUG", False)
 # Throttle % of all transaction events. Not intended for general use. May change without warning.
 THROTTLE_TRANSACTION_EVENTS = env.float("THROTTLE_TRANSACTION_EVENTS", None)
 
-GLITCHTIP_ENABLE_DIFS = env.bool("GLITCHTIP_ENABLE_DIFS", False)
-
 # GlitchTip can track GlitchTip's own errors.
 # If enabling this, use a different server to avoid infinite loops.
 def before_send(event, hint):