GlitchTip/difs/tasks.py

import contextlib
import logging
import tempfile
from hashlib import sha1

from celery import shared_task
from symbolic import Archive

from difs.models import DebugInformationFile
from difs.stacktrace_processor import StacktraceProcessor
from events.models import Event
from files.models import File, FileBlob
from projects.models import Project


def getLogger():
    return logging.getLogger("glitchtip.difs")


class ChecksumMismatched(Exception):
    pass


class UnsupportedFile(Exception):
    pass


DIF_STATE_CREATED = "created"
DIF_STATE_OK = "ok"
DIF_STATE_NOT_FOUND = "not_found"


@shared_task
def difs_assemble(project_slug, name, checksum, chunks, debug_id):
    try:
        project = Project.objects.filter(slug=project_slug).get()

        file = difs_get_file_from_chunks(checksum, chunks)
        if file is None:
            file = difs_create_file_from_chunks(name, checksum, chunks)

        difs_create_difs(project, name, file)

    except ChecksumMismatched:
        getLogger().error("difs_assemble: Checksum mismatched: %s", name)
    except Exception as err:
        getLogger().error("difs_assemble: %s", err)


def difs_run_resolve_stacktrace(event_id):
    difs_resolve_stacktrace.delay(event_id)


@shared_task
def difs_resolve_stacktrace(event_id):
    event = Event.objects.get(event_id=event_id)
    event_json = event.data
    exception = event_json.get("exception")

    if exception is None:
        # It is not a crash report event
        return

    project_id = event.issue.project_id

    difs = DebugInformationFile.objects.filter(project_id=project_id).order_by(
        "-created"
    )
    resolved_stracktrackes = []

    for dif in difs:
        if StacktraceProcessor.is_supported(event_json, dif) is False:
            continue
        blobs = [dif.file.blob]
        with difs_concat_file_blobs_to_disk(blobs) as symbol_file:
            remapped_stacktrace = StacktraceProcessor.resolve_stacktrace(
                event_json, symbol_file.name
            )
            if remapped_stacktrace is not None and remapped_stacktrace.score > 0:
                resolved_stracktrackes.append(remapped_stacktrace)
    if len(resolved_stracktrackes) > 0:
        best_remapped_stacktrace = max(
            resolved_stracktrackes, key=lambda item: item.score
        )
        StacktraceProcessor.update_frames(event, best_remapped_stacktrace.frames)
        event.save()


def difs_get_file_from_chunks(checksum, chunks):
    files = File.objects.filter(checksum=checksum)

    for file in files:
        blob = file.blob
        file_chunks = [blob.checksum]
        if file_chunks == chunks:
            return file

    return None


def difs_create_file_from_chunks(name, checksum, chunks):
    blobs = FileBlob.objects.filter(checksum__in=chunks)

    total_checksum = sha1(b"")
    size = 0

    for blob in blobs:
        size = size + blob.blob.size

        with open(blob.blob.path, "rb") as binary_file:
            content = binary_file.read()
            total_checksum.update(content)

    total_checksum = total_checksum.hexdigest()
    if checksum != total_checksum:
        raise ChecksumMismatched()

    file = File(name=name, headers={}, size=size, checksum=checksum)
    file.blob = blobs[0]
    file.save()
    return file


@contextlib.contextmanager
def difs_concat_file_blobs_to_disk(blobs):
    output = tempfile.NamedTemporaryFile(delete=False)
    for blob in blobs:
        with open(blob.blob.path, "rb") as binary_file:
            content = binary_file.read()
            output.write(content)

    output.flush()
    output.seek(0)
    try:
        yield output
    finally:
        output.close()


def difs_extract_metadata_from_file(file):
    with difs_concat_file_blobs_to_disk([file.blob]) as _input:
        # Only one kind of file format is supported now
        try:
            archive = Archive.open(_input.name)
        except Exception as err:
            getLogger().error("Extract metadata error: %s", err)
            raise UnsupportedFile() from err
        else:
            return [
                {
                    "arch": obj.arch,
                    "file_format": obj.file_format,
                    "code_id": obj.code_id,
                    "debug_id": obj.debug_id,
                    "kind": obj.kind,
                    "features": list(obj.features),
                    "symbol_type": "native",
                }
                for obj in archive.iter_objects()
            ]


def difs_create_difs(project, name, file):
    metadatalist = difs_extract_metadata_from_file(file)
    for metadata in metadatalist:
        dif = DebugInformationFile.objects.filter(
            project_id=project.id, file=file
        ).first()

        if dif is not None:
            continue

        code_id = metadata["code_id"]
        debug_id = metadata["debug_id"]
        arch = metadata["arch"]
        kind = metadata["kind"]
        features = metadata["features"]
        symbol_type = metadata["symbol_type"]

        dif = DebugInformationFile(
            project=project,
            name=name,
            file=file,
            data={
                "arch": arch,
                "debug_id": debug_id,
                "code_id": code_id,
                "kind": kind,
                "features": features,
                "symbol_type": symbol_type,
            },
        )
        dif.save()