fix(symbolicator): Tests are no longer allowed failures (#13566)

Clear global buckets, strip snapshots of unrelated data and make tests mandatory in Travis.

Most of the input data will also be tested by symbolicator in getsentry/symbolicator#81

.travis.yml

@@ -233,12 +233,6 @@ matrix:
     - language: node_js
       env: STORYBOOK_BUILD=1
 
-    # XXX(ja): Should be removed once symbolicator's interface is more stable or
-    # tests have been refactored to be more lenient. At the moment, snapshots
-    # break every time when a new attribute is added or some secondary attribute
-    # is changed.
-    - env: TEST_SUITE=symbolicator
-
 notifications:
   webhooks:
     urls:

tests/symbolicator/__init__.py

@@ -2,6 +2,8 @@ from __future__ import absolute_import
 
 import os
 
+from sentry.utils.safe import get_path
+
 
 def get_fixture_path(name):
     return os.path.join(
@@ -13,11 +15,49 @@ def get_fixture_path(name):
     )
 
 
+def strip_frame(frame):
+    if frame:
+        frame = {
+            'data': {'symbolicator_status': get_path(frame, 'data', 'symbolicator_status')},
+            'function': frame.get('function'),
+            'instruction_addr': frame.get('instruction_addr'),
+            'symbol': frame.get('symbol'),
+            'package': frame.get('package'),
+            'lineno': frame.get('lineno'),
+            'in_app': frame.get('in_app'),
+            'trust': frame.get('trust'),
+        }
+
+    return frame
+
+
+def strip_stacktrace(stacktrace):
+    if stacktrace:
+        stacktrace = dict(stacktrace)
+        stacktrace['frames'] = [strip_frame(x) for x in stacktrace.get('frames') or ()]
+
+    return stacktrace
+
+
+def strip_stacktrace_container(container):
+    if container:
+        container = dict(container)
+        container['stacktrace'] = strip_stacktrace(container.get('stacktrace'))
+        container['raw_stacktrace'] = strip_stacktrace(container.get('raw_stacktrace'))
+
+    return container
+
+
 def insta_snapshot_stacktrace_data(self, event):
+    # limit amount of data going into a snapshot so that they don't break all
+    # the time due to unrelated changes.
     self.insta_snapshot({
-        "stacktrace": event.get('stacktrace'),
-        "exception": event.get('exception'),
-        "threads": event.get('threads'),
+        "stacktrace": strip_stacktrace(event.get('stacktrace')),
+        "exception": {'values': [strip_stacktrace_container(x) for x in get_path(event, 'exception', 'values') or ()]},
+        "threads": {'values': [
+            strip_stacktrace_container(x)
+            for x in get_path(event, 'threads', 'values') or ()
+        ]},
         "debug_meta": event.get('debug_meta'),
         "contexts": event.get('contexts'),
         "errors": event.get('errors'),

tests/symbolicator/snapshots/SymbolicatorMinidumpIntegrationTest/test_full_minidump.pysnap

@@ -1,5 +1,5 @@
 ---
-created: '2019-06-03T14:50:05.902803Z'
+created: '2019-06-11T21:25:58.992806Z'
 creator: sentry
 source: tests/symbolicator/test_minidump_full.py
 ---
@@ -144,30 +144,38 @@ exception:
       handled: false
       synthetic: true
       type: minidump
+    raw_stacktrace: null
     stacktrace:
       frames:
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f44'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f79'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x750662c4'
+        lineno: null
         package: C:\Windows\System32\kernel32.dll
+        symbol: null
         trust: cfi
-      - abs_path: f:\dd\vctools\crt\vcstartup\src\startup\exe_common.inl
-        data:
+      - data:
           symbolicator_status: symbolicated
-        filename: exe_common.inl
         function: __scrt_common_main_seh
         in_app: false
         instruction_addr: '0x2a2d96'
@@ -175,10 +183,8 @@ exception:
         package: C:\projects\breakpad-tools\windows\Release\crash.exe
         symbol: __scrt_common_main_seh
         trust: cfi
-      - abs_path: c:\projects\breakpad-tools\windows\crash\main.cpp
-        data:
+      - data:
           symbolicator_status: symbolicated
-        filename: main.cpp
         function: main
         in_app: false
         instruction_addr: '0x2a2a3d'
@@ -205,33 +211,48 @@ threads:
   values:
   - crashed: true
     id: 1636
+    raw_stacktrace: null
+    stacktrace: null
   - crashed: false
     id: 3580
+    raw_stacktrace: null
     stacktrace:
       frames:
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f44'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f79'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x750662c4'
+        lineno: null
         package: C:\Windows\System32\kernel32.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771e016c'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: context
       registers:
         eax: '0x0'
@@ -246,31 +267,44 @@ threads:
         esp: '0x159f900'
   - crashed: false
     id: 2600
+    raw_stacktrace: null
     stacktrace:
       frames:
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f44'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f79'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x750662c4'
+        lineno: null
         package: C:\Windows\System32\kernel32.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771e016c'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: context
       registers:
         eax: '0x0'
@@ -285,13 +319,17 @@ threads:
         esp: '0x169f9f4'
   - crashed: false
     id: 2920
+    raw_stacktrace: null
     stacktrace:
       frames:
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771df3dc'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: context
       registers:
         eax: '0x0'

tests/symbolicator/snapshots/SymbolicatorMinidumpIntegrationTest/test_missing_dsym.pysnap

@@ -1,5 +1,5 @@
 ---
-created: '2019-06-03T14:50:09.219100Z'
+created: '2019-06-11T21:26:04.163727Z'
 creator: sentry
 source: tests/symbolicator/test_minidump_full.py
 ---
@@ -147,79 +147,116 @@ exception:
       handled: false
       synthetic: true
       type: minidump
+    raw_stacktrace: null
     stacktrace:
       frames:
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f44'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f79'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x750662c4'
+        lineno: null
         package: C:\Windows\System32\kernel32.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x2a2d97'
+        lineno: null
         package: C:\projects\breakpad-tools\windows\Release\crash.exe
+        symbol: null
         trust: scan
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x2a3435'
+        lineno: null
         package: C:\projects\breakpad-tools\windows\Release\crash.exe
+        symbol: null
         trust: scan
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x7584e9c0'
+        lineno: null
         package: C:\Windows\System32\rpcrt4.dll
+        symbol: null
         trust: scan
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x75810000'
+        lineno: null
         package: C:\Windows\System32\rpcrt4.dll
+        symbol: null
         trust: scan
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x70b7ae40'
+        lineno: null
         package: C:\Windows\System32\dbgcore.dll
+        symbol: null
         trust: scan
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x70850000'
+        lineno: null
         package: C:\Windows\System32\dbghelp.dll
+        symbol: null
         trust: scan
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x7584e9c0'
+        lineno: null
         package: C:\Windows\System32\rpcrt4.dll
+        symbol: null
         trust: scan
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x2a28d0'
+        lineno: null
         package: C:\projects\breakpad-tools\windows\Release\crash.exe
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x2a2a3d'
+        lineno: null
         package: C:\projects\breakpad-tools\windows\Release\crash.exe
+        symbol: null
         trust: context
       registers:
         eax: '0x0'
@@ -240,33 +277,48 @@ threads:
   values:
   - crashed: true
     id: 1636
+    raw_stacktrace: null
+    stacktrace: null
   - crashed: false
     id: 3580
+    raw_stacktrace: null
     stacktrace:
       frames:
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f44'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f79'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x750662c4'
+        lineno: null
         package: C:\Windows\System32\kernel32.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771e016c'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: context
       registers:
         eax: '0x0'
@@ -281,31 +333,44 @@ threads:
         esp: '0x159f900'
   - crashed: false
     id: 2600
+    raw_stacktrace: null
     stacktrace:
       frames:
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f44'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f79'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x750662c4'
+        lineno: null
         package: C:\Windows\System32\kernel32.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771e016c'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: context
       registers:
         eax: '0x0'
@@ -320,13 +385,17 @@ threads:
         esp: '0x169f9f4'
   - crashed: false
     id: 2920
+    raw_stacktrace: null
     stacktrace:
       frames:
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771df3dc'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: context
       registers:
         eax: '0x0'

tests/symbolicator/snapshots/SymbolicatorMinidumpIntegrationTest/test_raw_minidump.pysnap

@@ -1,5 +1,5 @@
 ---
-created: '2019-06-03T14:58:29.235537Z'
+created: '2019-06-11T21:26:07.179068Z'
 creator: sentry
 source: tests/symbolicator/test_minidump_full.py
 ---
@@ -144,30 +144,38 @@ exception:
       handled: false
       synthetic: true
       type: minidump
+    raw_stacktrace: null
     stacktrace:
       frames:
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f44'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f79'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x750662c4'
+        lineno: null
         package: C:\Windows\System32\kernel32.dll
+        symbol: null
         trust: cfi
-      - abs_path: f:\dd\vctools\crt\vcstartup\src\startup\exe_common.inl
-        data:
+      - data:
           symbolicator_status: symbolicated
-        filename: exe_common.inl
         function: __scrt_common_main_seh
         in_app: false
         instruction_addr: '0x2a2d96'
@@ -175,10 +183,8 @@ exception:
         package: C:\projects\breakpad-tools\windows\Release\crash.exe
         symbol: __scrt_common_main_seh
         trust: cfi
-      - abs_path: c:\projects\breakpad-tools\windows\crash\main.cpp
-        data:
+      - data:
           symbolicator_status: symbolicated
-        filename: main.cpp
         function: main
         in_app: false
         instruction_addr: '0x2a2a3d'
@@ -205,33 +211,48 @@ threads:
   values:
   - crashed: true
     id: 1636
+    raw_stacktrace: null
+    stacktrace: null
   - crashed: false
     id: 3580
+    raw_stacktrace: null
     stacktrace:
       frames:
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f44'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f79'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x750662c4'
+        lineno: null
         package: C:\Windows\System32\kernel32.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771e016c'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: context
       registers:
         eax: '0x0'
@@ -246,31 +267,44 @@ threads:
         esp: '0x159f900'
   - crashed: false
     id: 2600
+    raw_stacktrace: null
     stacktrace:
       frames:
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f44'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771d0f79'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x750662c4'
+        lineno: null
         package: C:\Windows\System32\kernel32.dll
+        symbol: null
         trust: fp
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771e016c'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: context
       registers:
         eax: '0x0'
@@ -285,13 +319,17 @@ threads:
         esp: '0x169f9f4'
   - crashed: false
     id: 2920
+    raw_stacktrace: null
     stacktrace:
       frames:
       - data:
           symbolicator_status: missing
+        function: null
         in_app: false
         instruction_addr: '0x771df3dc'
+        lineno: null
         package: C:\Windows\System32\ntdll.dll
+        symbol: null
         trust: context
       registers:
         eax: '0x0'

tests/symbolicator/snapshots/SymbolicatorResolvingIntegrationTest/test_debug_id_resolving.pysnap

@@ -1,5 +1,5 @@
 ---
-created: '2019-05-29T12:11:31.817415Z'
+created: '2019-06-11T21:26:10.030669Z'
 creator: sentry
 source: tests/symbolicator/test_payload_full.py
 ---
@@ -25,24 +25,29 @@ exception:
   values:
   - raw_stacktrace:
       frames:
-      - function: <unknown>
+      - data:
+          symbolicator_status: null
+        function: <unknown>
         in_app: false
         instruction_addr: '0x2a2a3d'
+        lineno: null
         package: C:\projects\breakpad-tools\windows\Release\crash.exe
+        symbol: null
+        trust: null
     stacktrace:
       frames:
-      - abs_path: c:\projects\breakpad-tools\windows\crash\main.cpp
-        data:
+      - data:
           symbolicator_status: symbolicated
-        filename: main.cpp
         function: main
         in_app: false
         instruction_addr: '0x2a2a3d'
         lineno: 35
         package: C:\projects\breakpad-tools\windows\Release\crash.exe
         symbol: main
+        trust: null
     thread_id: 1636
     type: EXCEPTION_ACCESS_VIOLATION_WRITE
     value: 'Fatal Error: EXCEPTION_ACCESS_VIOLATION_WRITE'
 stacktrace: null
-threads: null
+threads:
+  values: []

tests/symbolicator/snapshots/SymbolicatorResolvingIntegrationTest/test_missing_debug_images.pysnap

@@ -1,5 +1,5 @@
 ---
-created: '2019-06-03T15:00:16.825909Z'
+created: '2019-06-11T21:26:12.836777Z'
 creator: sentry
 source: tests/symbolicator/test_payload_full.py
 ---
@@ -10,23 +10,46 @@ exception:
   values:
   - raw_stacktrace:
       frames:
-      - function: hi
+      - data:
+          symbolicator_status: null
+        function: hi
         in_app: false
-        platform: foobar
-      - function: unknown
+        instruction_addr: null
+        lineno: null
+        package: null
+        symbol: null
+        trust: null
+      - data:
+          symbolicator_status: null
+        function: unknown
         in_app: false
         instruction_addr: '0x100000fa0'
+        lineno: null
+        package: null
+        symbol: null
+        trust: null
     stacktrace:
       frames:
-      - function: hi
+      - data:
+          symbolicator_status: null
+        function: hi
         in_app: false
-        platform: foobar
+        instruction_addr: null
+        lineno: null
+        package: null
+        symbol: null
+        trust: null
       - data:
           symbolicator_status: unknown_image
         function: unknown
         in_app: false
         instruction_addr: '0x100000fa0'
+        lineno: null
+        package: null
+        symbol: null
+        trust: null
     type: Fail
     value: fail
 stacktrace: null
-threads: null
+threads:
+  values: []

tests/symbolicator/snapshots/SymbolicatorResolvingIntegrationTest/test_missing_dsym.pysnap

@@ -1,5 +1,5 @@
 ---
-created: '2019-06-03T15:00:19.524785Z'
+created: '2019-06-11T21:26:15.800659Z'
 creator: sentry
 source: tests/symbolicator/test_payload_full.py
 ---
@@ -28,25 +28,46 @@ exception:
   values:
   - raw_stacktrace:
       frames:
-      - function: hi
+      - data:
+          symbolicator_status: null
+        function: hi
         in_app: false
-        platform: foobar
-      - function: unknown
+        instruction_addr: null
+        lineno: null
+        package: null
+        symbol: null
+        trust: null
+      - data:
+          symbolicator_status: null
+        function: unknown
         in_app: true
         instruction_addr: '0x100000fa0'
+        lineno: null
         package: Foo.app/Contents/Foo
+        symbol: null
+        trust: null
     stacktrace:
       frames:
-      - function: hi
+      - data:
+          symbolicator_status: null
+        function: hi
         in_app: false
-        platform: foobar
+        instruction_addr: null
+        lineno: null
+        package: null
+        symbol: null
+        trust: null
       - data:
           symbolicator_status: missing
         function: unknown
         in_app: true
         instruction_addr: '0x100000fa0'
+        lineno: null
         package: Foo.app/Contents/Foo
+        symbol: null
+        trust: null
     type: Fail
     value: fail
 stacktrace: null
-threads: null
+threads:
+  values: []

tests/symbolicator/snapshots/SymbolicatorResolvingIntegrationTest/test_real_resolving.pysnap

@@ -1,5 +1,5 @@
 ---
-created: '2019-06-03T15:00:21.958338Z'
+created: '2019-06-11T21:26:18.500120Z'
 creator: sentry
 source: tests/symbolicator/test_payload_full.py
 ---
@@ -23,29 +23,46 @@ exception:
   values:
   - raw_stacktrace:
       frames:
-      - function: hi
+      - data:
+          symbolicator_status: null
+        function: hi
         in_app: false
-        platform: foobar
-      - function: unknown
+        instruction_addr: null
+        lineno: null
+        package: null
+        symbol: null
+        trust: null
+      - data:
+          symbolicator_status: null
+        function: unknown
         in_app: true
         instruction_addr: '0x100000fa0'
+        lineno: null
         package: Foo.app/Contents/Foo
+        symbol: null
+        trust: null
     stacktrace:
       frames:
-      - function: hi
+      - data:
+          symbolicator_status: null
+        function: hi
         in_app: false
-        platform: foobar
-      - abs_path: /tmp/hello.c
-        data:
+        instruction_addr: null
+        lineno: null
+        package: null
+        symbol: null
+        trust: null
+      - data:
           symbolicator_status: symbolicated
-        filename: hello.c
         function: main
         in_app: true
         instruction_addr: '0x100000fa0'
         lineno: 1
         package: Foo.app/Contents/Foo
         symbol: main
+        trust: null
     type: Fail
     value: fail
 stacktrace: null
-threads: null
+threads:
+  values: []

tests/symbolicator/test_minidump_full.py

@@ -21,6 +21,7 @@ class SymbolicatorMinidumpIntegrationTest(TransactionTestCase):
 
     @pytest.fixture(autouse=True)
     def initialize(self, live_server):
+        self.project.update_option('sentry:builtin_symbol_sources', [])
         new_prefix = live_server.url
 
         with patch('sentry.auth.system.is_internal_ip', return_value=True), \