GlitchTip/events/migrations/0004_auto_20240312_1344.py

# Generated by Django 5.0.2 on 2024-03-12 13:44
import os
from datetime import timedelta

from django.db import connection, migrations
from django.db.models import Value
from django.conf import settings
from django.utils.timezone import now
from django.contrib.postgres.search import SearchVector

from apps.issue_events.constants import IssueEventType


MIGRATION_LIMIT = os.getenv("ISSUE_EVENT_MIGRATION_LIMIT", 50000)
MIGRATION_LIMIT = int(MIGRATION_LIMIT)

EVENT_LIMIT = os.getenv("EVENT_PER_ISSUE_MIGRATION_LIMIT", 1000)
EVENT_LIMIT = int(EVENT_LIMIT)

def reformat_data(data):
    if "exception" in data and data["exception"]:
        if "values" in data["exception"]:
            data["exception"] = data["exception"]["values"]
    if "breadcrumbs" in data and data["breadcrumbs"]:
        if "values" in data["breadcrumbs"]:
            data["breadcrumbs"] = data["breadcrumbs"]["values"]
    return data


def daterange(start_date, end_date):
    for n in range(int((end_date - start_date).days)):
        yield start_date + timedelta(n)


def migrate_issue_events(apps, schema_editor):
    if settings.TESTING:
        return
    today = now()
    days_ago = timedelta(days=settings.GLITCHTIP_MAX_EVENT_LIFE_DAYS + 1)
    with connection.cursor() as cursor:
        for single_date in daterange(today - days_ago, today):
            str_date = single_date.strftime("%Y_%b_%d").lower()
            from_date = single_date.date()  # strftime("%Y-%m-%d")
            to_date = (single_date + timedelta(days=1)).date()  # strftime("%Y-%m-%d")
            table_name = "issue_events_issueevent_" + str_date
            sql = f"""CREATE TABLE IF NOT EXISTS {table_name} PARTITION OF issue_events_issueevent
FOR VALUES FROM (%s) TO (%s);
COMMENT ON TABLE {table_name} IS 'psqlextra_auto_partitioned';
"""
            cursor.execute(sql, [from_date, to_date])

    OldIssue = apps.get_model("issues", "Issue")
    Event = apps.get_model("events", "Event")
    Issue = apps.get_model("issue_events", "Issue")
    IssueEvent = apps.get_model("issue_events", "IssueEvent")

    oldest_event = Event.objects.order_by("-created")[
        MIGRATION_LIMIT : MIGRATION_LIMIT + 1
    ].first()
    start_migration_date = oldest_event.created if oldest_event else None

    old_issues = OldIssue.objects.all().defer("search_vector", "tags")
    if start_migration_date:
        old_issues = old_issues.filter(last_seen__gt=start_migration_date)

    for old_issue in old_issues:
        issue = Issue.objects.create(
            culprit=old_issue.culprit[:1024],
            level=old_issue.level,
            metadata=old_issue.metadata,
            project=old_issue.project,
            title=old_issue.title,
            type=old_issue.type,
            status=old_issue.status,
            short_id=old_issue.short_id,
            search_vector=SearchVector(Value(old_issue.title)),
            count=old_issue.count,
            first_seen=old_issue.created,
            last_seen=old_issue.last_seen,
        )
        events = old_issue.event_set.order_by("-created")
        if start_migration_date:
            events = events.filter(created__gt=start_migration_date)
        IssueEvent.objects.bulk_create(
            [
                IssueEvent(
                    id=event.event_id,
                    type=IssueEventType.from_string(event.data.get("type", "default")),
                    timestamp=event.timestamp if event.timestamp else event.created,
                    received=event.created,
                    title=event.data.get("title"),
                    transaction=event.data.get("culprit")[:200],
                    level=event.level,
                    data=reformat_data(event.data),
                    tags=event.tags,
                    issue=issue,
                )
                for event in events[:EVENT_LIMIT]
            ]
        )


class Migration(migrations.Migration):
    dependencies = [
        ("events", "0001_squashed_0003_auto_20210219_1951"),
        ("issues", "0013_alter_comment_options_alter_issue_unique_together_and_more"),
        ("issue_events", "0001_initial"),
    ]

    operations = [
        migrations.RunPython(migrate_issue_events, migrations.RunPython.noop),
    ]