Files CRUD API

files/migrations/0004_auto_20210509_1658.py

@@ -0,0 +1,33 @@
+# Generated by Django 3.2.1 on 2021-05-09 16:58
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('files', '0003_auto_20210507_1549'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='fileblob',
+            name='upload',
+        ),
+        migrations.AddField(
+            model_name='fileblob',
+            name='blob',
+            field=models.FileField(default='', upload_to='uploads/file_blobs'),
+            preserve_default=False,
+        ),
+        migrations.AlterField(
+            model_name='file',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='fileblob',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+    ]

files/models.py

@@ -1,9 +1,14 @@
+from hashlib import sha1
 from django.db import models
 from glitchtip.base_models import CreatedModel
 
 
 class FileBlob(CreatedModel):
-    upload = models.FileField(upload_to="uploads/")
+    """
+    Port of sentry.models.file.FileBlog with some simplifications
+    """
+
+    blob = models.FileField(upload_to="uploads/file_blobs")
     size = models.PositiveIntegerField(null=True)
     checksum = models.CharField(max_length=40, unique=True)
 
@@ -22,13 +27,43 @@ class FileBlob(CreatedModel):
             blob = cls()
             blob_file = file_with_checksum[0]
             blob.checksum = file_with_checksum[1]
-            blob.upload.save(blob_file.name, blob_file)
+            blob.blob.save(blob_file.name, blob_file)
             blob.save()
 
+    @classmethod
+    def from_file(cls, fileobj):
+        """
+        Retrieve a single FileBlob instances for the given file.
+        """
+        checksum = sha1()
+        with fileobj.open("rb") as f:
+            if f.multiple_chunks():
+                for chunk in f.chunks():
+                    checksum.update(chunk)
+            else:
+                checksum.update(f.read())
+            # Significant deviation from OSS Sentry
+            file_blob, _ = cls.objects.get_or_create(
+                checksum=checksum.hexdigest(),
+                defaults={"blob": fileobj, "size": fileobj.size},
+            )
+        return file_blob
+
 
 class File(CreatedModel):
+    """
+    Port of sentry.models.file.File
+    """
+
     name = models.TextField()
     headers = models.JSONField()
     blobs = models.ManyToManyField(FileBlob)
     size = models.PositiveIntegerField(null=True)
     checksum = models.CharField(max_length=40, null=True, db_index=True)
+
+    def putfile(self, fileobj):
+        self.size = fileobj.size
+        file_blob = FileBlob.from_file(fileobj)
+        self.checksum = file_blob.checksum
+        self.save()
+

files/serializers.py

@@ -1,8 +0,0 @@
-from rest_framework import serializers
-from .models import File
-
-
-class FileSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = File
-        fields = ("name",)

files/views.py

@@ -5,11 +5,10 @@ from gzip import GzipFile
 from django.conf import settings
 from django.urls import reverse
 from django.shortcuts import get_object_or_404
-from rest_framework import views, status, viewsets
+from rest_framework import views, status
 from rest_framework.response import Response
 from organizations_ext.models import Organization
-from .models import FileBlob, File
-from .serializers import FileSerializer
+from .models import FileBlob
 from .permissions import ChunkUploadPermission
 
 
@@ -117,8 +116,3 @@ class ChunkUploadAPIView(views.APIView):
 
         logger.info("chunkupload.end", extra={"status": status.HTTP_200_OK})
         return Response(status=status.HTTP_200_OK)
-
-
-class FileViewSet(viewsets.ModelViewSet):
-    queryset = File.objects.all()
-    serializer_class = FileSerializer

glitchtip/exceptions.py

@@ -0,0 +1,9 @@
+from django.utils.translation import gettext_lazy as _
+from rest_framework import exceptions, status
+
+
+class ConflictException(exceptions.APIException):
+    status_code = status.HTTP_409_CONFLICT
+    default_detail = _("Already present!")
+    default_code = "already_present"
+

glitchtip/settings.py

@@ -248,7 +248,7 @@ DEFAULT_FROM_EMAIL = env.str("DEFAULT_FROM_EMAIL", "webmaster@localhost")
 ANYMAIL = {
     "MAILGUN_API_KEY": env.str("MAILGUN_API_KEY", None),
     "MAILGUN_SENDER_DOMAIN": env.str("MAILGUN_SENDER_DOMAIN", None),
-    "MAILGUN_API_URL": env.str("MAILGUN_API_URL", "https://api.mailgun.net/v3")
+    "MAILGUN_API_URL": env.str("MAILGUN_API_URL", "https://api.mailgun.net/v3"),
 }
 
 ACCOUNT_EMAIL_SUBJECT_PREFIX = ""
@@ -334,7 +334,9 @@ AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL")
 AWS_LOCATION = env("AWS_LOCATION")
 
 if AWS_S3_ENDPOINT_URL:
-    MEDIA_URL = env.str("MEDIA_URL", "https://%s/%s/" % (AWS_S3_ENDPOINT_URL, AWS_LOCATION))
+    MEDIA_URL = env.str(
+        "MEDIA_URL", "https://%s/%s/" % (AWS_S3_ENDPOINT_URL, AWS_LOCATION)
+    )
     DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
 else:
     MEDIA_URL = "media/"

projects/urls.py

@@ -2,9 +2,8 @@ from django.urls import path, include
 from rest_framework_nested import routers
 from issues.views import IssueViewSet, EventViewSet
 from alerts.views import ProjectAlertViewSet
-from releases.views import ReleaseViewSet
+from releases.views import ReleaseViewSet, ReleaseFileViewSet
 from environments.views import EnvironmentProjectViewSet
-from files.views import FileViewSet
 from .views import ProjectViewSet, ProjectKeyViewSet, ProjectTeamViewSet
 
 router = routers.SimpleRouter()
@@ -24,7 +23,7 @@ projects_router.register(r"releases", ReleaseViewSet, basename="project-releases
 releases_router = routers.NestedSimpleRouter(
     projects_router, r"releases", lookup="release"
 )
-releases_router.register(r"files", FileViewSet, basename="files")
+releases_router.register(r"files", ReleaseFileViewSet, basename="files")
 
 urlpatterns = [
     path("", include(router.urls)),

releases/migrations/0003_auto_20210509_1658.py

@@ -0,0 +1,34 @@
+# Generated by Django 3.2.1 on 2021-05-09 16:58
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('files', '0004_auto_20210509_1658'),
+        ('releases', '0002_auto_20201227_1518'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ReleaseFile',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True, db_index=True)),
+                ('ident', models.CharField(max_length=40)),
+                ('name', models.TextField()),
+                ('file', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='files.file')),
+                ('release', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='releases.release')),
+            ],
+            options={
+                'unique_together': {('release', 'file'), ('release', 'ident')},
+            },
+        ),
+        migrations.AddField(
+            model_name='release',
+            name='files',
+            field=models.ManyToManyField(through='releases.ReleaseFile', to='files.File'),
+        ),
+    ]

releases/models.py

@@ -1,3 +1,4 @@
+from hashlib import sha1
 from django.db import models
 from glitchtip.base_models import CreatedModel
 
@@ -26,6 +27,7 @@ class Release(CreatedModel):
     # authors - not implemented
     deploy_count = models.PositiveSmallIntegerField(default=0)
     # last_deploy - not implemented
+    files = models.ManyToManyField("files.File", through="ReleaseFile")
 
     class Meta:
         unique_together = ("organization", "version")
@@ -39,3 +41,25 @@ class ReleaseProject(models.Model):
 
     class Meta:
         unique_together = ("project", "release")
+
+
+class ReleaseFile(CreatedModel):
+    release = models.ForeignKey(Release, on_delete=models.CASCADE)
+    file = models.ForeignKey("files.File", on_delete=models.CASCADE)
+    ident = models.CharField(max_length=40)
+    name = models.TextField()
+
+    class Meta:
+        unique_together = (("release", "file"), ("release", "ident"))
+
+    def save(self, *args, **kwargs):
+        if not self.ident:
+            self.ident = type(self).get_ident(self.name)
+        return super().save(*args, **kwargs)
+
+    @classmethod
+    def get_ident(cls, name, dist=None):
+        if dist is not None:
+            dist_name = name + "\x00\x00" + dist
+            return sha1(dist_name.encode()).hexdigest()
+        return sha1(name.encode()).hexdigest()

releases/permissions.py

@@ -11,3 +11,8 @@ class ReleasePermission(ScopedPermission):
 
     def get_user_scopes(self, obj, user):
         return obj.organization.get_user_scopes(user)
+
+
+class ReleaseFilePermission(ReleasePermission):
+    def get_user_scopes(self, obj, user):
+        return obj.release.organization.get_user_scopes(user)