Start issue events

apps/issue_events/admin.py

@@ -0,0 +1,14 @@
+from django.contrib import admin
+
+from .models import IssueEvent
+
+
+class IssueEventAdmin(admin.ModelAdmin):
+    list_display = ("id", "created")
+    list_filter = ("created",)
+    raw_id_fields = ("issue",)
+    search_fields = ("id",)
+    show_full_result_count = False
+
+
+admin.site.register(IssueEvent, IssueEventAdmin)

apps/issue_events/migrations/0001_initial.py

@@ -0,0 +1,232 @@
+# Generated by Django 4.2.6 on 2023-10-20 01:22
+
+from django.conf import settings
+import django.contrib.postgres.indexes
+import django.contrib.postgres.search
+from django.db import migrations, models
+import django.db.models.deletion
+import psqlextra.backend.migrations.operations.add_default_partition
+import psqlextra.backend.migrations.operations.create_partitioned_model
+import psqlextra.manager.manager
+import psqlextra.models.partitioned
+import psqlextra.types
+import uuid
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = [
+        ("issues", "0013_alter_comment_options_alter_issue_unique_together_and_more"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("projects", "0013_merge_20231017_1350"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Comment",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("text", models.TextField(blank=True, null=True)),
+                (
+                    "issue",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="comments",
+                        to="issues.issue",
+                    ),
+                ),
+                (
+                    "user",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="+",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "ordering": ("-created",),
+            },
+        ),
+        migrations.CreateModel(
+            name="Issue",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("culprit", models.CharField(blank=True, max_length=1024, null=True)),
+                ("is_public", models.BooleanField(default=False)),
+                (
+                    "level",
+                    models.PositiveSmallIntegerField(
+                        choices=[
+                            (0, "sample"),
+                            (1, "debug"),
+                            (2, "info"),
+                            (3, "warning"),
+                            (4, "error"),
+                            (5, "fatal"),
+                        ],
+                        default=4,
+                    ),
+                ),
+                ("title", models.CharField(max_length=255)),
+                (
+                    "type",
+                    models.PositiveSmallIntegerField(
+                        choices=[(0, "default"), (1, "error"), (2, "csp")], default=0
+                    ),
+                ),
+                (
+                    "status",
+                    models.PositiveSmallIntegerField(
+                        choices=[(0, "unresolved"), (1, "resolved"), (2, "ignored")],
+                        default=0,
+                    ),
+                ),
+                ("short_id", models.PositiveIntegerField(null=True)),
+                (
+                    "project",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="issues",
+                        to="projects.project",
+                    ),
+                ),
+            ],
+        ),
+        psqlextra.backend.migrations.operations.create_partitioned_model.PostgresCreatePartitionedModel(
+            name="IssueEvent",
+            fields=[
+                (
+                    "id",
+                    models.UUIDField(
+                        default=uuid.uuid4,
+                        editable=False,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+                (
+                    "type",
+                    models.PositiveSmallIntegerField(
+                        choices=[(0, "default"), (1, "error"), (2, "csp")], default=0
+                    ),
+                ),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("data", models.JSONField()),
+                (
+                    "issue",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="issue_events.issue",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+                "base_manager_name": "objects",
+            },
+            partitioning_options={
+                "method": psqlextra.types.PostgresPartitioningMethod["RANGE"],
+                "key": ["created"],
+            },
+            bases=(psqlextra.models.partitioned.PostgresPartitionedModel,),
+            managers=[
+                ("objects", psqlextra.manager.manager.PostgresManager()),
+            ],
+        ),
+        psqlextra.backend.migrations.operations.add_default_partition.PostgresAddDefaultPartition(
+            model_name="IssueEvent",
+            name="default",
+        ),
+        migrations.CreateModel(
+            name="IssueHash",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("value", models.UUIDField(db_index=True)),
+                (
+                    "issue",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="hashes",
+                        to="issues.issue",
+                    ),
+                ),
+                (
+                    "project",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="+",
+                        to="projects.project",
+                    ),
+                ),
+            ],
+        ),
+        migrations.CreateModel(
+            name="IssueStats",
+            fields=[
+                (
+                    "issue",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        primary_key=True,
+                        serialize=False,
+                        to="issue_events.issue",
+                    ),
+                ),
+                (
+                    "search_vector",
+                    django.contrib.postgres.search.SearchVectorField(
+                        editable=False, null=True
+                    ),
+                ),
+                ("search_vector_created", models.DateTimeField(auto_now_add=True)),
+                ("count", models.PositiveIntegerField(default=1, editable=False)),
+                ("last_seen", models.DateTimeField(auto_now_add=True, db_index=True)),
+            ],
+            options={
+                "indexes": [
+                    django.contrib.postgres.indexes.GinIndex(
+                        fields=["search_vector"], name="issue_event_search__0754e8_gin"
+                    )
+                ],
+            },
+        ),
+        migrations.AddConstraint(
+            model_name="issuehash",
+            constraint=models.UniqueConstraint(
+                fields=("project", "value"), name="issue hash project"
+            ),
+        ),
+        migrations.AlterUniqueTogether(
+            name="issue",
+            unique_together={("project", "short_id")},
+        ),
+    ]

apps/issue_events/models.py

@@ -0,0 +1,107 @@
+import uuid
+from django.db import models
+from django.contrib.postgres.search import SearchVectorField
+from django.contrib.postgres.indexes import GinIndex
+from psqlextra.models import PostgresPartitionedModel
+from psqlextra.types import PostgresPartitioningMethod
+
+from glitchtip.model_utils import FromStringIntegerChoices
+
+
+class IssueEventType(models.IntegerChoices):
+    DEFAULT = 0, "default"
+    ERROR = 1, "error"
+    CSP = 2, "csp"
+
+
+class EventStatus(FromStringIntegerChoices):
+    UNRESOLVED = 0, "unresolved"
+    RESOLVED = 1, "resolved"
+    IGNORED = 2, "ignored"
+
+
+class LogLevel(FromStringIntegerChoices):
+    NOTSET = 0, "sample"
+    DEBUG = 1, "debug"
+    INFO = 2, "info"
+    WARNING = 3, "warning"
+    ERROR = 4, "error"
+    FATAL = 5, "fatal"
+
+
+class Issue(models.Model):
+    created = models.DateTimeField(auto_now_add=True)
+    culprit = models.CharField(max_length=1024, blank=True, null=True)
+    is_public = models.BooleanField(default=False)
+    level = models.PositiveSmallIntegerField(
+        choices=LogLevel.choices, default=LogLevel.ERROR
+    )
+    project = models.ForeignKey(
+        "projects.Project", on_delete=models.CASCADE, related_name="issues"
+    )
+    title = models.CharField(max_length=255)
+    type = models.PositiveSmallIntegerField(
+        choices=IssueEventType.choices, default=IssueEventType.DEFAULT
+    )
+    status = models.PositiveSmallIntegerField(
+        choices=EventStatus.choices, default=EventStatus.UNRESOLVED
+    )
+    short_id = models.PositiveIntegerField(null=True)
+
+    class Meta:
+        unique_together = (("project", "short_id"),)
+
+
+class IssueStats(models.Model):
+    issue = models.OneToOneField(Issue, primary_key=True, on_delete=models.CASCADE)
+    search_vector = SearchVectorField(null=True, editable=False)
+    search_vector_created = models.DateTimeField(auto_now_add=True)
+    count = models.PositiveIntegerField(default=1, editable=False)
+    last_seen = models.DateTimeField(auto_now_add=True, db_index=True)
+
+    class Meta:
+        indexes = [GinIndex(fields=["search_vector"])]
+
+
+class IssueHash(models.Model):
+    issue = models.ForeignKey(
+        "issues.Issue", on_delete=models.CASCADE, related_name="hashes"
+    )
+    # Redundant project allows for unique constraint
+    project = models.ForeignKey(
+        "projects.Project", on_delete=models.CASCADE, related_name="+"
+    )
+    value = models.UUIDField(db_index=True)
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=["project", "value"], name="issue hash project"
+            )
+        ]
+
+
+class Comment(models.Model):
+    created = models.DateTimeField(auto_now_add=True)
+    issue = models.ForeignKey(
+        "issues.Issue", on_delete=models.CASCADE, related_name="comments"
+    )
+    user = models.ForeignKey(
+        "users.User", null=True, on_delete=models.SET_NULL, related_name="+"
+    )
+    text = models.TextField(blank=True, null=True)
+
+    class Meta:
+        ordering = ("-created",)
+
+
+class IssueEvent(PostgresPartitionedModel, models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    issue = models.ForeignKey(Issue, on_delete=models.CASCADE)
+    type = models.PositiveSmallIntegerField(default=0, choices=IssueEventType.choices)
+    created = models.DateTimeField(auto_now_add=True)
+    data = models.JSONField()
+
+    class PartitioningMeta:
+        method = PostgresPartitioningMethod.RANGE
+        key = ["created"]

glitchtip/partitioning.py

@@ -0,0 +1,24 @@
+from dateutil.relativedelta import relativedelta
+
+from psqlextra.partitioning import (
+    PostgresPartitioningManager,
+    PostgresCurrentTimePartitioningStrategy,
+    PostgresTimePartitionSize,
+    partition_by_current_time,
+)
+from psqlextra.partitioning.config import PostgresPartitioningConfig
+
+from apps.issue_events.models import IssueEvent
+
+manager = PostgresPartitioningManager(
+    [
+        PostgresPartitioningConfig(
+            model=IssueEvent,
+            strategy=PostgresCurrentTimePartitioningStrategy(
+                size=PostgresTimePartitionSize(weeks=1),
+                count=2,
+                max_age=relativedelta(months=3),
+            ),
+        ),
+    ]
+)

glitchtip/settings.py

@@ -191,6 +191,7 @@ INSTALLED_APPS = [
     "django.contrib.auth",
     "django.contrib.contenttypes",
     "django.contrib.postgres",
+    "psqlextra",
     "django_prometheus",
     "allauth",
     "allauth.account",
@@ -213,6 +214,7 @@ if DEBUG_TOOLBAR:
 INSTALLED_APPS += [
     "dj_rest_auth",
     "dj_rest_auth.registration",
+    "apps.issue_events",
     "import_export",
     "storages",
     "glitchtip",
@@ -402,7 +404,6 @@ DATABASE_HOST = env.str("DATABASE_HOST", None)
 DATABASE_PASSWORD = env.str("DATABASE_PASSWORD", None)
 if DATABASE_HOST and DATABASE_PASSWORD:
     DATABASES["default"] = {
-        "ENGINE": "django.db.backends.postgresql",
         "NAME": env.str("DATABASE_NAME", "postgres"),
         "USER": env.str("DATABASE_USER", "postgres"),
         "PASSWORD": DATABASE_PASSWORD,
@@ -411,6 +412,8 @@ if DATABASE_HOST and DATABASE_PASSWORD:
         "CONN_MAX_AGE": env.int("DATABASE_CONN_MAX_AGE", 0),
         "CONN_HEALTH_CHECKS": env.bool("DATABASE_CONN_HEALTH_CHECKS", False),
     }
+DATABASES["default"]["ENGINE"] = "psqlextra.backend"
+PSQLEXTRA_PARTITIONING_MANAGER = "glitchtip.partitioning.manager"
 
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
 

mypy.ini

@@ -1,6 +0,0 @@
-[mypy]
-plugins =
-    mypy_django_plugin.main
-
-[mypy.plugins.django-stubs]
-django_settings_module = "glitchtip.settings"

poetry.lock

@@ -1200,6 +1200,27 @@ files = [
 Django = ">=3.2.0"
 six = "*"
 
+[[package]]
+name = "django-postgres-extra"
+version = "2.0.8"
+description = "Bringing all of PostgreSQL's awesomeness to Django."
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "django-postgres-extra-2.0.8.tar.gz", hash = "sha256:9efa08c6f18ed34460af41c6f679bb375b93d12544b1105aa348b787a30b46eb"},
+    {file = "django_postgres_extra-2.0.8-py3-none-any.whl", hash = "sha256:447d5a971759943ee63a9d4cef9c6c1fa290e518611ea521a38b6732681d2f3a"},
+]
+
+[package.dependencies]
+Django = ">=2.0,<5.0"
+python-dateutil = ">=2.8.0,<=3.0.0"
+
+[package.extras]
+analysis = ["autoflake (==1.4)", "autopep8 (==1.6.0)", "black (==22.3.0)", "docformatter (==1.4)", "flake8 (==4.0.1)", "isort (==5.10.0)"]
+docs = ["Sphinx (==2.2.0)", "docutils (<0.18)", "sphinx-rtd-theme (==0.4.3)"]
+publish = ["build (==0.7.0)", "twine (==3.7.1)"]
+test = ["coveralls (==3.3.0)", "dj-database-url (==0.5.0)", "freezegun (==1.1.0)", "psycopg2 (>=2.8.4,<3.0.0)", "pytest (==6.2.5)", "pytest-benchmark (==3.4.1)", "pytest-cov (==3.0.0)", "pytest-django (==4.4.0)", "snapshottest (==0.6.0)", "tox (==3.24.4)"]
+
 [[package]]
 name = "django-prometheus"
 version = "2.3.1"
@@ -4606,4 +4627,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9"
-content-hash = "d9690f0573ee6ce6e494d3b59c48294561f3bee62fc68cb2e072546f2a17996d"
+content-hash = "07bdbc804d347e34b9252333b6e30421a57a5310ad9255a467dc0edf94aa0d7c"