ref(schema): Update debug images schema (#12453)

Extends the schema for debug images to include more fields and allow more types. The symbolicator will use this additional info to determine which symbols to download and use during symbolication. Additionally, the `apple` debug is now treated as an alias to `macho`.

requirements-base.txt

@@ -56,7 +56,7 @@ redis>=2.10.3,<2.10.6
 requests-oauthlib==0.3.3
 requests[security]>=2.20.0,<2.21.0
 selenium==3.141.0
-semaphore>=0.4.20,<0.5.0
+semaphore>=0.4.21,<0.5.0
 sentry-sdk>=0.7.0
 setproctitle>=1.1.7,<1.2.0
 simplejson>=3.2.0,<3.9.0
@@ -65,7 +65,7 @@ sqlparse>=0.1.16,<0.2.0
 statsd>=3.1.0,<3.2.0
 strict-rfc3339>=0.7
 structlog==16.1.0
-symbolic>=6.0.0,<7.0.0
+symbolic>=6.0.4,<7.0.0
 toronado>=0.0.11,<0.1.0
 ua-parser>=0.6.1,<0.8.0
 # for bitbucket client

src/sentry/interfaces/debug_meta.py

@@ -21,58 +21,42 @@ def imagetype(name):
 
 
 def _addr(x):
+    if x is None:
+        return None
     return '0x%x' % parse_addr(x)
 
 
+@imagetype('apple')
+@imagetype('macho')
+@imagetype('elf')
+@imagetype('pe')
 @imagetype('symbolic')
-def process_symbolic_image(image):
+def process_native_image(image):
+    # NOTE that this is dead code as soon as Rust renormalization is fully
+    # enabled. After that, this code should be deleted. There is a difference
+    # TODO(untitaker): Remove with other normalization code.
     try:
-        symbolic_image = {
-            'id': normalize_debug_id(image.get('id')),
+        native_image = {
+            'code_file': image.get('code_file') or image.get('name'),
+            'debug_id': normalize_debug_id(
+                image.get('debug_id') or image.get('id') or image.get('uuid')),
             'image_addr': _addr(image.get('image_addr')),
-            'image_size': parse_addr(image['image_size']),
-            'image_vmaddr': _addr(image.get('image_vmaddr') or 0),
-            'name': image.get('name'),
+            'image_size': _addr(image.get('image_size')),
+            'image_vmaddr': _addr(image.get('image_vmaddr')),
         }
 
         if image.get('arch') is not None:
-            symbolic_image['arch'] = image.get('arch')
+            native_image['arch'] = image.get('arch')
+        if image.get('code_id') is not None:
+            native_image['code_id'] = image.get('code_id')
+        if image.get('debug_file') is not None:
+            native_image['debug_file'] = image.get('debug_file')
 
-        return symbolic_image
+        return native_image
     except KeyError as e:
         raise InterfaceValidationError('Missing value for symbolic image: %s' % e.args[0])
 
 
-@imagetype('apple')
-def process_apple_image(image):
-    try:
-        if image['uuid'] is None:
-            raise KeyError('uuid')
-
-        apple_image = {
-            'uuid': six.text_type(uuid.UUID(image['uuid'])),
-            'cpu_type': image.get('cpu_type'),
-            'cpu_subtype': image.get('cpu_subtype'),
-            'image_addr': _addr(image.get('image_addr')),
-            'image_size': image['image_size'],
-            'image_vmaddr': _addr(image.get('image_vmaddr') or 0),
-            'name': image.get('name'),
-        }
-
-        if image.get('arch') is not None:
-            apple_image['arch'] = image.get('arch')
-        if image.get('major_version') is not None:
-            apple_image['major_version'] = image['major_version']
-        if image.get('minor_version') is not None:
-            apple_image['minor_version'] = image['minor_version']
-        if image.get('revision_version') is not None:
-            apple_image['revision_version'] = image['revision_version']
-
-        return apple_image
-    except KeyError as e:
-        raise InterfaceValidationError('Missing value for apple image: %s' % e.args[0])
-
-
 @imagetype('proguard')
 def process_proguard_image(image):
     try:
@@ -138,11 +122,16 @@ class DebugMeta(Interface):
         ty = image.get('type')
         if not ty:
             raise InterfaceValidationError('Image type not provided')
+        if ty == 'apple':
+            # Legacy alias. The schema is actually slightly different, but
+            # process_native_image can deal with this and convert to a valid
+            # MachO image payload.
+            ty = 'macho'
         func = image_types.get(ty)
         if func is None:
             raise InterfaceValidationError('Unknown image type %r' % image)
         rv = func(image)
-        assert 'uuid' in rv or 'id' in rv, 'debug image normalizer did not produce an identifier'
+        assert 'uuid' in rv or 'debug_id' in rv, 'debug image normalizer did not produce an identifier'
         rv['type'] = ty
         return rv
 

src/sentry/lang/native/plugin.py

@@ -27,7 +27,8 @@ FRAME_CACHE_VERSION = 6
 
 class NativeStacktraceProcessor(StacktraceProcessor):
     supported_platforms = ('cocoa', 'native')
-    supported_images = ('apple', 'symbolic')
+    # TODO(ja): Clean up all uses of image type "apple", "uuid", "id" and "name"
+    supported_images = ('apple', 'symbolic', 'elf', 'macho', 'pe')
 
     def __init__(self, *args, **kwargs):
         StacktraceProcessor.__init__(self, *args, **kwargs)
@@ -47,11 +48,13 @@ class NativeStacktraceProcessor(StacktraceProcessor):
             self.available = False
 
     def _is_valid_image(self, image):
+        # TODO(ja): Deprecate this. The symbolicator should take care of
+        # filtering valid images.
         return bool(image) \
             and image.get('type') in self.supported_images \
             and image.get('image_addr') is not None \
             and image.get('image_size') is not None \
-            and (image.get('id') or image.get('uuid')) is not None
+            and (image.get('debug_id') or image.get('id') or image.get('uuid')) is not None
 
     def close(self):
         StacktraceProcessor.close(self)

src/sentry/static/sentry/app/components/events/interfaces/debugmeta.jsx

@@ -13,59 +13,46 @@ class DebugMetaInterface extends React.Component {
     data: PropTypes.object.isRequired,
   };
 
-  getImageDetail(img, evt) {
-    // in particular proguard images do not have a name, skip them
-    if (img === null || img.name === null || img.type === 'proguard') {
+  getImageDetail(img) {
+    // in particular proguard images do not have a code file, skip them
+    if (img === null || img.code_file === null || img.type === 'proguard') {
       return null;
     }
 
-    const name = img.name.split(/^([a-z]:\\|\\\\)/i.test(img.name) ? '\\' : '/').pop();
-    if (name == 'dyld_sim') return null; // this is only for simulator builds
-
-    let version = null;
-    if (
-      Number.isInteger(img.major_version) &&
-      Number.isInteger(img.minor_version) &&
-      Number.isInteger(img.revision_version)
-    ) {
-      if (img.major_version == 0 && img.minor_version == 0 && img.revision_version == 0) {
-        // we show the version
-        version = (evt.release && evt.release.shortVersion) || 'unknown';
-      } else
-        version = `${img.major_version}.${img.minor_version}.${img.revision_version}`;
-    } else version = img.id || img.uuid || '<none>';
-
-    if (version) return [name, version];
+    const directorySeparator = /^([a-z]:\\|\\\\)/i.test(img.code_file) ? '\\' : '/';
+    const code_file = img.code_file.split(directorySeparator).pop();
+    if (code_file === 'dyld_sim') {
+      // this is only for simulator builds
+      return null;
+    }
 
-    return null;
+    const version = img.debug_id || '<none>';
+    return [code_file, version];
   }
 
   render() {
     const data = this.props.data;
-    const images = data.images
-      .map(img => this.getImageDetail(img, this.props.event))
-      .filter(img => img); // removes null values
-
-    let result = null;
 
-    if (images.length > 0) {
-      result = (
-        <div>
-          <EventDataSection
-            group={this.props.group}
-            event={this.props.event}
-            type="packages"
-            title={t('Images Loaded')}
-          >
-            <ClippedBox>
-              <KeyValueList data={images} isSorted={false} />
-            </ClippedBox>
-          </EventDataSection>
-        </div>
-      );
+    // skip null values indicating invalid debug images
+    const images = data.images.map(img => this.getImageDetail(img)).filter(img => img);
+    if (images.length === 0) {
+      return null;
     }
 
-    return result;
+    return (
+      <div>
+        <EventDataSection
+          group={this.props.group}
+          event={this.props.event}
+          type="packages"
+          title={t('Images Loaded')}
+        >
+          <ClippedBox>
+            <KeyValueList data={images} isSorted={false} />
+          </ClippedBox>
+        </EventDataSection>
+      </div>
+    );
   }
 }
 

tests/sentry/event_manager/interfaces/snapshots/test_debug_meta/test_apple_behavior.pysnap

@@ -1,19 +1,17 @@
 ---
-created: '2019-03-14T17:12:35.226657Z'
+created: '2019-03-19T19:18:09.262739Z'
 creator: sentry
 source: tests/sentry/event_manager/interfaces/test_debug_meta.py
 ---
 errors: null
 to_json:
   images:
-  - cpu_subtype: 0
-    cpu_type: 16777228
+  - code_file: /var/containers/Bundle/Application/B33C37A8-F933-4B6B-9FFA-152282BFDF13/SentryTest.app/SentryTest
+    debug_id: c05b4ddd-69a7-3840-a649-32180d341587
     image_addr: '0x100020000'
-    image_size: 32768
+    image_size: '0x8000'
     image_vmaddr: '0x100000000'
-    name: /var/containers/Bundle/Application/B33C37A8-F933-4B6B-9FFA-152282BFDF13/SentryTest.app/SentryTest
-    type: apple
-    uuid: c05b4ddd-69a7-3840-a649-32180d341587
+    type: macho
   sdk_info:
     build: null
     dsym_type: none

tests/sentry/event_manager/interfaces/snapshots/test_debug_meta/test_apple_behavior_with_arch.pysnap

@@ -1,5 +1,5 @@
 ---
-created: '2019-03-14T17:12:35.237789Z'
+created: '2019-03-19T19:18:09.270927Z'
 creator: sentry
 source: tests/sentry/event_manager/interfaces/test_debug_meta.py
 ---
@@ -7,14 +7,12 @@ errors: null
 to_json:
   images:
   - arch: x86_64
-    cpu_subtype: 0
-    cpu_type: 16777228
+    code_file: /var/containers/Bundle/Application/B33C37A8-F933-4B6B-9FFA-152282BFDF13/SentryTest.app/SentryTest
+    debug_id: c05b4ddd-69a7-3840-a649-32180d341587
     image_addr: '0x100020000'
-    image_size: 32768
+    image_size: '0x8000'
     image_vmaddr: '0x100000000'
-    name: /var/containers/Bundle/Application/B33C37A8-F933-4B6B-9FFA-152282BFDF13/SentryTest.app/SentryTest
-    type: apple
-    uuid: c05b4ddd-69a7-3840-a649-32180d341587
+    type: macho
   sdk_info:
     build: null
     dsym_type: none

tests/sentry/event_manager/interfaces/snapshots/test_debug_meta/test_symbolic_behavior.pysnap

@@ -1,16 +1,16 @@
 ---
-created: '2019-03-14T17:12:35.247803Z'
+created: '2019-03-19T19:18:09.279589Z'
 creator: sentry
 source: tests/sentry/event_manager/interfaces/test_debug_meta.py
 ---
 errors: null
 to_json:
   images:
-  - id: 3249d99d-0c40-4931-8610-f4e4fb0b6936-1
+  - code_file: C:\projects\breakpad-tools\windows\Release\crash.exe
+    debug_id: 3249d99d-0c40-4931-8610-f4e4fb0b6936-1
     image_addr: '0x2a0000'
-    image_size: 36864
-    image_vmaddr: '0x0'
-    name: C:\projects\breakpad-tools\windows\Release\crash.exe
+    image_size: '0x9000'
+    image_vmaddr: null
     type: symbolic
   sdk_info:
     build: null

tests/sentry/event_manager/interfaces/snapshots/test_debug_meta/test_symbolic_behavior_with_arch.pysnap

@@ -1,5 +1,5 @@
 ---
-created: '2019-03-14T17:12:35.257253Z'
+created: '2019-03-19T19:18:09.287387Z'
 creator: sentry
 source: tests/sentry/event_manager/interfaces/test_debug_meta.py
 ---
@@ -7,11 +7,11 @@ errors: null
 to_json:
   images:
   - arch: x86
-    id: 3249d99d-0c40-4931-8610-f4e4fb0b6936-1
+    code_file: C:\projects\breakpad-tools\windows\Release\crash.exe
+    debug_id: 3249d99d-0c40-4931-8610-f4e4fb0b6936-1
     image_addr: '0x2a0000'
-    image_size: 36864
-    image_vmaddr: '0x0'
-    name: C:\projects\breakpad-tools\windows\Release\crash.exe
+    image_size: '0x9000'
+    image_vmaddr: null
     type: symbolic
   sdk_info:
     build: null