feat(minidump): Support CFI for minidumps (#9344)

* ref(minidump): Move minidump logic to its own module

* ref: Add a constant for the minidump attachment type

* feat(minidump): Expand the merge_minidump_event interface

* feat(minidump): Add a util to detect minidump events

* feat: Add an event enhancer hook to Plugin v2

* feat: Optionally pass CFI to minidump processing

* ref: Move hashing helper from stacktraces to utils.hashlib

* feat: Add an optional stackframe trust

* feat: Emit the stackframe trust for minidump events

* fix: Add frame trust to schema

* feat(native): Reprocess minidumps with CFI

* meta: Add more description and comments to CFI processing

* fix: Fix a doc code example

* feat: Show scanned frames in the UI

* feat: Refactor event cache keys into a function

* fix: Correctly detect outdated CFI caches

* fix: Correctly detect unchanged stacktraces

* test: Add tests for CFI reprocessing

* build: Bump symbolic to 5.5.3

* meta: Add suggested comment

requirements-base.txt

@@ -62,7 +62,7 @@ sqlparse>=0.1.16,<0.2.0
 statsd>=3.1.0,<3.2.0
 strict-rfc3339>=0.7
 structlog==16.1.0
-symbolic>=5.5.0,<6.0.0
+symbolic>=5.5.3,<6.0.0
 toronado>=0.0.11,<0.1.0
 ua-parser>=0.6.1,<0.8.0
 # for bitbucket client

src/sentry/coreapi.py

@@ -200,7 +200,7 @@ class ClientApiHelper(object):
             data = dict(data.items())
 
         cache_timeout = 3600
-        cache_key = u'e:{1}:{0}'.format(data['project'], data['event_id'])
+        cache_key = cache_key_for_event(data)
         default_cache.set(cache_key, data, cache_timeout)
 
         # Attachments will be empty or None if the "event-attachments" feature
@@ -257,6 +257,10 @@ class SecurityApiHelper(ClientApiHelper):
         return auth
 
 
+def cache_key_for_event(data):
+    return u'e:{1}:{0}'.format(data['project'], data['event_id'])
+
+
 def decompress_deflate(encoded_data):
     try:
         return zlib.decompress(encoded_data).decode("utf-8")

src/sentry/interfaces/schemas.py

@@ -109,6 +109,7 @@ FRAME_INTERFACE_SCHEMA = {
         'in_app': {'type': 'boolean', 'default': False},
         'instruction_addr': {},
         'instruction_offset': {},
+        'trust': {'type': 'string'},
         'lineno': {'type': ['number', 'string']},
         'module': {
             'type': 'string',

src/sentry/interfaces/stacktrace.py

@@ -372,6 +372,7 @@ class Frame(Interface):
             'symbol': trim(symbol, 256),
             'symbol_addr': to_hex_addr(data.get('symbol_addr')),
             'instruction_addr': to_hex_addr(data.get('instruction_addr')),
+            'trust': trim(data.get('trust'), 16),
             'in_app': in_app,
             'context_line': context_line,
             # TODO(dcramer): trim pre/post_context
@@ -477,6 +478,7 @@ class Frame(Interface):
             'lineNo': self.lineno,
             'colNo': self.colno,
             'inApp': self.in_app,
+            'trust': self.trust,
             'errors': self.errors,
         }
         if not is_public:
@@ -522,6 +524,7 @@ class Frame(Interface):
             'lineNo': meta.get('lineno'),
             'colNo': meta.get('colno'),
             'inApp': meta.get('in_app'),
+            'trust': meta.get('trust'),
             'errors': meta.get('errors'),
         }
 

src/sentry/lang/native/cfi.py

@@ -0,0 +1,270 @@
+from __future__ import absolute_import
+
+import logging
+import six
+
+from symbolic import FrameInfoMap, FrameTrust, ObjectLookup
+
+from sentry.attachments import attachment_cache
+from sentry.coreapi import cache_key_for_event
+from sentry.lang.native.minidump import process_minidump, frames_from_minidump_thread, \
+    MINIDUMP_ATTACHMENT_TYPE
+from sentry.lang.native.utils import rebase_addr
+from sentry.models import Project, ProjectDebugFile
+from sentry.utils.cache import cache
+from sentry.utils.hashlib import hash_values
+
+
+logger = logging.getLogger(__name__)
+
+# Frame trust values achieved through the use of CFI
+CFI_TRUSTS = ('cfi', 'cfi-scan')
+
+# Minimum frame trust value that we require to omit CFI reprocessing
+MIN_TRUST = FrameTrust.fp
+
+# Placeholder used to indicate that no CFI could be used to stackwalk a thread
+NO_CFI_PLACEHOLDER = '__no_cfi__'
+
+
+class ThreadRef(object):
+    """Cacheable and mutable reference to stack frames of an event thread."""
+
+    def __init__(self, frames, modules):
+        self.raw_frames = frames
+        self.modules = modules
+        self.resolved_frames = None
+
+    def _get_frame_key(self, frame):
+        module = self.modules.find_object(frame['instruction_addr'])
+
+        # If we cannot resolve a module for this frame, this means we're dealing
+        # with an absolute address here. Since this address changes with every
+        # crash and would poison our cache, we skip it for the key calculation.
+        if not module:
+            return None
+
+        return (
+            module.id,
+            rebase_addr(frame['instruction_addr'], module)
+        )
+
+    @property
+    def _cache_key(self):
+        values = [self._get_frame_key(f) for f in self.raw_frames]
+        # XXX: The seed is hard coded for a future refactor
+        return 'st:%s' % hash_values(values, seed='MinidumpCfiProcessor')
+
+    def _frame_from_cache(self, entry):
+        debug_id, offset, trust = entry[:3]
+        module = self.modules.get_object(debug_id)
+
+        # The debug_id can be None or refer to a missing module. If the module
+        # was missing, the stored offset was absolute as well. Otherwise, we
+        # have no choice but to assume an absolute address. In practice, the
+        # latter hopefully never happens.
+        addr = module.addr + offset if module else offset
+
+        return module, {
+            'instruction_addr': '0x%x' % addr,
+            'function': '<unknown>',  # Required by interface
+            'module': module.name if module else None,
+            'trust': trust,
+        }
+
+    def load_from_cache(self):
+        """Attempts to load the reprocessed stack trace from the cache. The
+        return value is ``True`` for a cache hit, and ``False`` for a miss.
+        The loaded addresses are rebased to the provided code modules.
+        """
+
+        cached = cache.get(self._cache_key)
+        if cached is None:
+            return False
+
+        if cached == NO_CFI_PLACEHOLDER:
+            self.resolved_frames = NO_CFI_PLACEHOLDER
+        else:
+            self.resolved_frames = [self._frame_from_cache(c) for c in cached]
+
+        return True
+
+    def save_to_cache(self):
+        """Stores the reprocessed stack trace to the cache. For frames with
+        known code modules only relative offsets are stored, otherwise the
+        absolute address as fallback."""
+        if self.resolved_frames is None:
+            raise RuntimeError('save_to_cache called before resolving frames')
+
+        if self.resolved_frames == NO_CFI_PLACEHOLDER:
+            cache.set(self._cache_key, NO_CFI_PLACEHOLDER)
+            return
+
+        values = []
+        for module, frame in self.resolved_frames:
+            module_id = module and module.id
+            addr = frame['instruction_addr']
+            if module:
+                addr = '0x%x' % rebase_addr(addr, module)
+            values.append((module_id, addr, frame['trust']))
+
+        cache.set(self._cache_key, values)
+
+    def load_from_minidump(self, thread):
+        """Loads the stack trace from a minidump process state thread."""
+
+        # Convert the entire thread into frames conforming to the `Frame`
+        # interface. Note that this is done with the same function as the
+        # initial ingestion to avoid normalization conflicts.
+        frames = frames_from_minidump_thread(thread)
+
+        # Filter out stack traces that did not improve during reprocessing. For
+        # these cases we only store a marker. This also prevents us from
+        # destroying absolute addresses when restoring from the cache. Stack
+        # traces containing CFI frames are mapped to their modules and stored.
+        if any(frame['trust'] in CFI_TRUSTS for frame in frames):
+            self.resolved_frames = [(self.modules.find_object(f['instruction_addr']), f)
+                                    for f in frames]
+        else:
+            self.resolved_frames = NO_CFI_PLACEHOLDER
+
+    def apply_to_event(self):
+        """Writes the loaded stack trace back to the event's payload. Returns
+        ``True`` if the payload was changed, otherwise ``False``."""
+        if self.resolved_frames is None:
+            raise RuntimeError('apply_to_event called before resolving frames')
+
+        if self.resolved_frames == NO_CFI_PLACEHOLDER:
+            return False
+
+        self.raw_frames[:] = [frame for module, frame in self.resolved_frames]
+        return True
+
+    @property
+    def needs_cfi(self):
+        """Indicates whether this thread requires reprocessing with CFI due to
+        scanned stack frames."""
+        return any(
+            getattr(FrameTrust, f.get('trust', ''), 0) < MIN_TRUST
+            for f in self.raw_frames
+        )
+
+
+class ThreadProcessingHandle(object):
+    """Helper object for processing all event threads.
+
+    This class offers a view on all threads in the given event payload,
+    including the crashing exception thread. Use ``iter_threads`` to iterate
+    pointers to the original threads' stack traces. Likewise, ``iter_modules``
+    returns references to all modules (images) loaded into the process.
+
+    The handle keeps track of changes to the original data. To signal mutation,
+    call ``indicate_change``. Finally, ``result`` returns the changed data or
+    None if it was not changed.
+    """
+
+    def __init__(self, data):
+        self.data = data
+        self.modules = self._get_modules()
+        self.changed = False
+
+    def _get_modules(self):
+        modules = self.data.get('debug_meta', {}).get('images', [])
+        return ObjectLookup(modules)
+
+    def iter_modules(self):
+        """Returns an iterator over all code modules (images) loaded by the
+        process at the time of the crash. The values are of type ``ObjectRef``.
+        """
+        return self.modules.iter_objects()
+
+    def iter_threads(self):
+        """Returns an iterator over all threads of the process at the time of
+        the crash, including the crashing thread. The values are of type
+        ``ThreadRef``."""
+        for thread in self.data.get('threads', {}).get('values', []):
+            if thread.get('crashed'):
+                # XXX: Assumes that the full list of threads is present in the
+                # original crash report. This is guaranteed by KSCrash and our
+                # minidump utility.
+                exceptions = self.data.get('exception', {}).get('values', [])
+                exception = exceptions[0] if exceptions else {}
+                frames = exception.get('stacktrace', {}).get('frames')
+            else:
+                frames = thread.get('stacktrace', {}).get('frames')
+
+            tid = thread.get('id')
+            if tid and frames:
+                yield tid, ThreadRef(frames, self.modules)
+
+    def indicate_change(self):
+        """Signals mutation of the data."""
+        self.changed = True
+
+    def result(self):
+        """Returns ``data`` if ``indicate_change`` was called, otherwise None.
+        """
+        if self.changed:
+            return self.data
+
+
+def reprocess_minidump_with_cfi(data):
+    """Reprocesses a minidump event if CFI(call frame information) is available
+    and viable. The event is only processed if there are stack traces that
+    contain scanned frames.
+    """
+
+    handle = ThreadProcessingHandle(data)
+
+    # Check stacktrace caches first and skip all that do not need CFI. This is
+    # either if a thread is trusted (i.e. it does not contain scanned frames) or
+    # since it can be fetched from the cache.
+    threads = {}
+    for tid, thread in handle.iter_threads():
+        if not thread.needs_cfi:
+            continue
+
+        if thread.load_from_cache():
+            if thread.apply_to_event():
+                handle.indicate_change()
+            continue
+
+        threads[tid] = thread
+
+    if not threads:
+        return handle.result()
+
+    # Check if we have a minidump to reprocess
+    cache_key = cache_key_for_event(data)
+    attachments = attachment_cache.get(cache_key) or []
+    minidump = next((a for a in attachments if a.type == MINIDUMP_ATTACHMENT_TYPE), None)
+    if not minidump:
+        return handle.result()
+
+    # Determine modules loaded into the process during the crash
+    debug_ids = [module.id for module in handle.iter_modules()]
+    if not debug_ids:
+        return handle.result()
+
+    # Load CFI caches for all loaded modules (even unreferenced ones)
+    project = Project.objects.get_from_cache(id=data['project'])
+    cficaches = ProjectDebugFile.difcache.get_cficaches(project, debug_ids)
+    if not cficaches:
+        return handle.result()
+
+    # Reprocess the minidump with CFI
+    cfi_map = FrameInfoMap.new()
+    for debug_id, cficache in six.iteritems(cficaches):
+        cfi_map.add(debug_id, cficache)
+    state = process_minidump(minidump.data, cfi=cfi_map)
+
+    # Merge existing stack traces with new ones from the minidump
+    for minidump_thread in state.threads():
+        thread = threads.get(minidump_thread.thread_id)
+        if thread:
+            thread.load_from_minidump(minidump_thread)
+            thread.save_to_cache()
+            if thread.apply_to_event():
+                handle.indicate_change()
+
+    return handle.result()

src/sentry/lang/native/minidump.py

@@ -0,0 +1,111 @@
+from __future__ import absolute_import
+
+import six
+from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile
+
+from symbolic import arch_from_breakpad, ProcessState, id_from_breakpad
+
+# Attachment type used for minidump files
+MINIDUMP_ATTACHMENT_TYPE = 'event.minidump'
+
+# Mapping of well-known minidump OS constants to our internal names
+MINIDUMP_OS_TYPES = {
+    'Mac OS X': 'macOS',
+    'Windows NT': 'Windows',
+}
+
+
+def is_minidump_event(data):
+    exceptions = data.get('exception', {}).get('values', [])
+    if not exceptions:
+        return False
+
+    return exceptions[0].get('mechanism', {}).get('type') == 'minidump'
+
+
+def process_minidump(minidump, cfi=None):
+    if isinstance(minidump, InMemoryUploadedFile):
+        minidump.open()  # seek to start
+        return ProcessState.from_minidump_buffer(minidump.read(), cfi)
+    elif isinstance(minidump, TemporaryUploadedFile):
+        return ProcessState.from_minidump(minidump.temporary_file_path(), cfi)
+    elif isinstance(minidump, six.binary_type) and minidump.startswith('MDMP'):
+        return ProcessState.from_minidump_buffer(minidump, cfi)
+    else:
+        return ProcessState.from_minidump(minidump, cfi)
+
+
+def merge_minidump_event(data, minidump, cfi=None):
+    state = process_minidump(minidump, cfi=cfi)
+
+    data['platform'] = 'native'
+    data['level'] = 'fatal' if state.crashed else 'info'
+    data['message'] = 'Assertion Error: %s' % state.assertion if state.assertion \
+        else 'Fatal Error: %s' % state.crash_reason
+
+    if state.timestamp:
+        data['timestamp'] = float(state.timestamp)
+
+    # Extract as much context information as we can.
+    info = state.system_info
+    context = data.setdefault('contexts', {})
+    os = context.setdefault('os', {})
+    device = context.setdefault('device', {})
+    os['type'] = 'os'  # Required by "get_sdk_from_event"
+    os['name'] = MINIDUMP_OS_TYPES.get(info.os_name, info.os_name)
+    os['version'] = info.os_version
+    os['build'] = info.os_build
+    device['arch'] = arch_from_breakpad(info.cpu_family)
+
+    # We can extract stack traces here already but since CFI is not
+    # available yet (without debug symbols), the stackwalker will
+    # resort to stack scanning which yields low-quality results. If
+    # the user provides us with debug symbols, we reprocess this
+    # minidump and add improved stacktraces later.
+    data['threads'] = [{
+        'id': thread.thread_id,
+        'crashed': False,
+        'stacktrace': {
+            'frames': frames_from_minidump_thread(thread),
+            'registers': thread.get_frame(0).registers if thread.frame_count else None,
+        },
+    } for thread in state.threads()]
+
+    # Mark the crashed thread and add its stacktrace to the exception
+    crashed_thread = data['threads'][state.requesting_thread]
+    crashed_thread['crashed'] = True
+
+    # Extract the crash reason and infos
+    data['exception'] = {
+        'value': data['message'],
+        'thread_id': crashed_thread['id'],
+        'type': state.crash_reason,
+        # Move stacktrace here from crashed_thread (mutating!)
+        'stacktrace': crashed_thread.pop('stacktrace'),
+        'mechanism': {
+            'type': 'minidump',
+            'handled': False,
+            # We cannot extract exception codes or signals with the breakpad
+            # extractor just yet. Once these capabilities are added to symbolic,
+            # these values should go in the mechanism here.
+        }
+    }
+
+    # Extract referenced (not all loaded) images
+    images = [{
+        'type': 'symbolic',
+        'id': id_from_breakpad(module.id),
+        'image_addr': '0x%x' % module.addr,
+        'image_size': module.size,
+        'name': module.name,
+    } for module in state.modules()]
+    data.setdefault('debug_meta', {})['images'] = images
+
+
+def frames_from_minidump_thread(thread):
+    return [{
+        'instruction_addr': '0x%x' % frame.return_address,
+        'function': '<unknown>',  # Required by interface
+        'module': frame.module.name if frame.module else None,
+        'trust': frame.trust,
+    } for frame in reversed(list(thread.frames()))]

src/sentry/lang/native/plugin.py

@@ -9,6 +9,8 @@ from symbolic import parse_addr, find_best_instruction, arch_get_ip_reg_name, \
 
 from sentry import options
 from sentry.plugins import Plugin2
+from sentry.lang.native.cfi import reprocess_minidump_with_cfi
+from sentry.lang.native.minidump import is_minidump_event
 from sentry.lang.native.symbolizer import Symbolizer, SymbolicationFailed
 from sentry.lang.native.utils import get_sdk_from_event, cpu_name_from_data, \
     rebase_addr
@@ -274,6 +276,10 @@ class NativeStacktraceProcessor(StacktraceProcessor):
 class NativePlugin(Plugin2):
     can_disable = False
 
+    def get_event_enhancers(self, data):
+        if is_minidump_event(data):
+            return [reprocess_minidump_with_cfi]
+
     def get_stacktrace_processors(self, data, stacktrace_infos, platforms, **kwargs):
         if any(platform in NativeStacktraceProcessor.supported_platforms for platform in platforms):
             return [NativeStacktraceProcessor]

src/sentry/lang/native/utils.py

@@ -5,8 +5,7 @@ import six
 import logging
 
 from collections import namedtuple
-from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile
-from symbolic import parse_addr, arch_from_breakpad, arch_from_macho, arch_is_known, ProcessState, id_from_breakpad
+from symbolic import parse_addr, arch_from_macho, arch_is_known
 
 from sentry.interfaces.contexts import DeviceContextType
 
@@ -18,12 +17,6 @@ VERSION_RE = re.compile(r'(\d+\.\d+\.\d+)\s+(.*)')
 # Regular expression to guess whether we're dealing with Windows or Unix paths
 WINDOWS_PATH_RE = re.compile(r'^[a-z]:\\', re.IGNORECASE)
 
-# Mapping of well-known minidump OS constants to our internal names
-MINIDUMP_OS_TYPES = {
-    'Mac OS X': 'macOS',
-    'Windows NT': 'Windows',
-}
-
 AppInfo = namedtuple('AppInfo', ['id', 'version', 'build', 'name'])
 
 
@@ -136,80 +129,3 @@ def sdk_info_to_sdk_id(sdk_info):
     if build is not None:
         rv = '%s_%s' % (rv, build)
     return rv
-
-
-def merge_minidump_event(data, minidump):
-    if isinstance(minidump, InMemoryUploadedFile):
-        minidump.open()  # seek to start
-        state = ProcessState.from_minidump_buffer(minidump.read())
-    elif isinstance(minidump, TemporaryUploadedFile):
-        state = ProcessState.from_minidump(minidump.temporary_file_path())
-    else:
-        state = ProcessState.from_minidump(minidump)
-
-    data['platform'] = 'native'
-    data['level'] = 'fatal' if state.crashed else 'info'
-    data['message'] = 'Assertion Error: %s' % state.assertion if state.assertion \
-        else 'Fatal Error: %s' % state.crash_reason
-
-    if state.timestamp:
-        data['timestamp'] = float(state.timestamp)
-
-    # Extract as much context information as we can.
-    info = state.system_info
-    context = data.setdefault('contexts', {})
-    os = context.setdefault('os', {})
-    device = context.setdefault('device', {})
-    os['type'] = 'os'  # Required by "get_sdk_from_event"
-    os['name'] = MINIDUMP_OS_TYPES.get(info.os_name, info.os_name)
-    os['version'] = info.os_version
-    os['build'] = info.os_build
-    device['arch'] = arch_from_breakpad(info.cpu_family)
-
-    # We can extract stack traces here already but since CFI is not
-    # available yet (without debug symbols), the stackwalker will
-    # resort to stack scanning which yields low-quality results. If
-    # the user provides us with debug symbols, we could reprocess this
-    # minidump and add improved stacktraces later.
-    data['threads'] = [{
-        'id': thread.thread_id,
-        'crashed': False,
-        'stacktrace': {
-            'frames': [{
-                'instruction_addr': '0x%x' % frame.return_address,
-                'function': '<unknown>',  # Required by interface
-                'package': frame.module.name if frame.module else None,
-            } for frame in reversed(list(thread.frames()))],
-            'registers': thread.get_frame(0).registers if thread.frame_count else None,
-        },
-    } for thread in state.threads()]
-
-    # Mark the crashed thread and add its stacktrace to the exception
-    crashed_thread = data['threads'][state.requesting_thread]
-    crashed_thread['crashed'] = True
-
-    # Extract the crash reason and infos
-    data['exception'] = {
-        'value': data['message'],
-        'thread_id': crashed_thread['id'],
-        'type': state.crash_reason,
-        # Move stacktrace here from crashed_thread (mutating!)
-        'stacktrace': crashed_thread.pop('stacktrace'),
-        'mechanism': {
-            'type': 'minidump',
-            'handled': False,
-            # We cannot extract exception codes or signals with the breakpad
-            # extractor just yet. Once these capabilities are added to symbolic,
-            # these values should go in the mechanism here.
-        }
-    }
-
-    # Extract referenced (not all loaded) images
-    images = [{
-        'type': 'symbolic',
-        'id': id_from_breakpad(module.id),
-        'image_addr': '0x%x' % module.addr,
-        'image_size': module.size,
-        'name': module.name,
-    } for module in state.modules()]
-    data.setdefault('debug_meta', {})['images'] = images

src/sentry/models/debugfile.py

@@ -25,7 +25,8 @@ from django.db.models.fields.related import OneToOneRel
 
 from symbolic import FatObject, SymbolicError, ObjectErrorUnsupportedObject, \
     SYMCACHE_LATEST_VERSION, SymCache, SymCacheErrorMissingDebugInfo, \
-    SymCacheErrorMissingDebugSection, CfiCache, CfiErrorMissingDebugInfo
+    SymCacheErrorMissingDebugSection, CfiCache, CfiErrorMissingDebugInfo, \
+    CFICACHE_LATEST_VERSION
 
 from sentry import options
 from sentry.cache import default_cache
@@ -288,6 +289,12 @@ class ProjectCacheFile(Model):
         """Indicates whether the cache can be computed from the given DIF."""
         return set(cls.required_features) <= debug_file.features
 
+    @property
+    def outdated(self):
+        """Indicates whether this cache is outdated and needs to be recomputed.
+        """
+        raise NotImplemented
+
     def delete(self, *args, **kwargs):
         super(ProjectCacheFile, self).delete(*args, **kwargs)
         self.cache_file.delete()
@@ -318,6 +325,10 @@ class ProjectSymCacheFile(ProjectCacheFile):
             return debug_file.dif_type in ('breakpad', 'macho', 'elf')
         return super(ProjectSymCacheFile, cls).computes_from(debug_file)
 
+    @property
+    def outdated(self):
+        return self.version != SYMCACHE_LATEST_VERSION
+
 
 class ProjectCfiCacheFile(ProjectCacheFile):
     """Cache for stack unwinding information: CfiCache."""
@@ -337,6 +348,10 @@ class ProjectCfiCacheFile(ProjectCacheFile):
     def cache_cls(cls):
         return CfiCache
 
+    @property
+    def outdated(self):
+        return self.version != CFICACHE_LATEST_VERSION
+
 
 def clean_redundant_difs(project, debug_id):
     """Deletes redundant debug files from the database and file storage. A debug
@@ -650,12 +665,12 @@ class DIFCache(object):
         caches = []
         to_update = debug_files.copy()
         for cache_file in existing_caches:
-            if cache_file.version == SYMCACHE_LATEST_VERSION:
+            if cache_file.outdated:
+                cache_file.delete()
+            else:
                 debug_id = cache_file.debug_file.debug_id
                 to_update.pop(debug_id, None)
                 caches.append((debug_id, cache_file, None))
-            else:
-                cache_file.delete()
 
         # If any cache files need to be updated, do that now
         if to_update:

src/sentry/plugins/base/v2.py

@@ -391,6 +391,26 @@ class IPlugin2(local, PluginConfigMixin, PluginStatusMixin):
                     return [CocoaProcessor(data, stacktrace_infos)]
         """
 
+    def get_event_enhancers(self, data, **kwargs):
+        """
+        Return a list of enhancers to apply to the given event.
+
+        An enhancer is a function that takes the normalized data blob as an
+        input and returns modified data as output. If no changes to the data are
+        made it is safe to return ``None``.
+
+        As opposed to event (pre)processors, enhancers run **before** stacktrace
+        processing and can be used to perform more extensive event normalization
+        or add additional data before stackframes are symbolicated.
+
+        Enhancers should not be returned if there is nothing to do with the
+        event data.
+
+        >>> def get_event_enhancers(self, data, **kwargs):
+        >>>     return [lambda x: x]
+        """
+        return []
+
     def get_feature_hooks(self, **kwargs):
         """
         Return a list of callables to check for feature status.