feat(trace): partially implement trace searching (#67987)

Implements parts of the backend grammar parsing which will enable us to
provide a more fine grained and a more accurate search experience in the
future.

Currently, this addresses only simple primitive values and has gaps in
implementation (cannot search nested keys). In terms of the actual
search UX, we will also need to implement validation and hopefully some
simple error recovery. The entire UI code of selecting and editing
tokens is also missing and will need to be added (though I'm hoping I
can just reuse the sentry search bar for that)

jest.config.ts

@@ -253,7 +253,7 @@ const config: Config.InitialOptions = {
       : '/node_modules/',
   ],
 
-  moduleFileExtensions: ['js', 'ts', 'jsx', 'tsx'],
+  moduleFileExtensions: ['js', 'ts', 'jsx', 'tsx', 'pegjs'],
   globals: {},
 
   testResultsProcessor: JEST_TEST_BALANCER

static/app/views/performance/newTraceDetails/index.tsx

@@ -50,8 +50,8 @@ import {
   searchInTraceTree,
   traceSearchReducer,
   type TraceSearchState,
-} from 'sentry/views/performance/newTraceDetails/traceSearch';
-import {TraceSearchInput} from 'sentry/views/performance/newTraceDetails/traceSearchInput';
+} from 'sentry/views/performance/newTraceDetails/traceSearch/traceSearch';
+import {TraceSearchInput} from 'sentry/views/performance/newTraceDetails/traceSearch/traceSearchInput';
 import {
   traceTabsReducer,
   type TraceTabsReducerState,

static/app/views/performance/newTraceDetails/trace.tsx

@@ -28,7 +28,7 @@ import {
 import type {
   TraceSearchAction,
   TraceSearchState,
-} from 'sentry/views/performance/newTraceDetails/traceSearch';
+} from 'sentry/views/performance/newTraceDetails/traceSearch/traceSearch';
 
 import {
   isAutogroupedNode,

static/app/views/performance/newTraceDetails/traceSearch/traceSearch.pegjs

@@ -0,0 +1,82 @@
+{
+    function as_undefined(){
+        return undefined;
+    }
+
+    function as_null(){
+        return null;
+    }
+
+    function as_operator_type(input){
+        switch(input){
+            case ">=":
+                return "ge";
+            case "<=":
+                return "le";
+            case "=":
+                return "eq";
+            case "<":
+                return "lt";
+            case ">":
+                return "gt";
+        }
+    }
+
+    function as_bool(input){
+        switch(input){
+        case "true": return true;
+        case "false": return false;
+        default: {
+            throw new Error("Failed to parse boolean value")
+            }
+        }
+    }
+}
+
+search_grammar
+   = t:token* {
+        return t
+   }
+
+token = search
+
+search = s:search_key separator vo:value_operator? v:value {
+    return {
+        type: "Token",
+        key: s.value,
+        value: v,
+        ...(s.negated ? {negated: true}: {}),
+        ...(vo ? {operator: vo} : {}),
+    }
+}
+
+
+search_key = space n:[!]? k:value space {
+    return {
+        value: k,
+        ...(!!n ? {negated: true} : {}),
+    }
+}
+value = space k:(bool/undefined/null/float/integer/string) space { return k }
+
+value_operator = space k:(">=" / "<=" / "=" / "<" / ">") space {
+    return as_operator_type(text())
+}
+
+// Primitives
+bool = k:("true" / "false") { return as_bool(text()) }
+undefined = "undefined" { return as_undefined() }
+null = "null" { return as_null() }
+float = ([-])? ([0-9]+)? "." ([0-9]+)? !string {
+    return parseFloat(text(), 10)
+}
+integer = ([-])?[0-9]+ !string{
+    return parseInt(text(), 10)
+}
+string = [a-zA-Z0-9\-._!$]+ !bool !undefined !null {
+    return text()
+}
+
+space = " "*
+separator = ":"
+eoi = [\t\n]

static/app/views/performance/newTraceDetails/traceSearchInput.tsx → static/app/views/performance/newTraceDetails/traceSearch/traceSearchInput.tsx

@@ -8,7 +8,7 @@ import {SearchBarTrailingButton} from 'sentry/components/searchBar';
 import {IconChevron, IconClose, IconSearch} from 'sentry/icons';
 import {t} from 'sentry/locale';
 import {space} from 'sentry/styles/space';
-import type {TraceSearchState} from 'sentry/views/performance/newTraceDetails/traceSearch';
+import type {TraceSearchState} from 'sentry/views/performance/newTraceDetails/traceSearch/traceSearch';
 
 interface TraceSearchInputProps {
   onChange: (event: React.ChangeEvent<HTMLInputElement>) => void;

static/app/views/performance/newTraceDetails/traceSearch/traceSearchTokenizer.spec.tsx

@@ -0,0 +1,212 @@
+import type {RawSpanType} from 'sentry/components/events/interfaces/spans/types';
+import {EntryType, type Event} from 'sentry/types';
+import {evaluateTokenForTraceNode} from 'sentry/views/performance/newTraceDetails/traceSearch/traceSearchTokenizer';
+import {
+  type TraceTree,
+  TraceTreeNode,
+} from 'sentry/views/performance/newTraceDetails/traceTree';
+
+import grammar from './traceSearch.pegjs';
+
+const evaluate = evaluateTokenForTraceNode;
+
+const metadata = {
+  project_slug: 'project',
+  event_id: 'event_id',
+};
+
+function makeSpan(overrides: Partial<RawSpanType> = {}): TraceTree.Span {
+  return {
+    op: '',
+    description: '',
+    span_id: '',
+    start_timestamp: 0,
+    timestamp: 10,
+    event: makeEvent(),
+    errors: [],
+    performance_issues: [],
+    childTransaction: undefined,
+    ...overrides,
+  } as TraceTree.Span;
+}
+
+function makeEvent(overrides: Partial<Event> = {}, spans: RawSpanType[] = []): Event {
+  return {
+    entries: [{type: EntryType.SPANS, data: spans}],
+    ...overrides,
+  } as Event;
+}
+
+function makeSpanNode(span: Partial<RawSpanType>): TraceTreeNode<TraceTree.Span> {
+  return new TraceTreeNode(null, makeSpan(span), metadata);
+}
+
+describe('traceSearchTokenizer', () => {
+  it('empty value', () => {
+    expect(grammar.parse('')).toEqual([]);
+  });
+
+  test.each([
+    'key:value',
+    'key :value',
+    'key : value',
+    '  key:  value',
+    'key:  value   ',
+  ])('parses %s', input => {
+    expect(grammar.parse(input)).toEqual([{type: 'Token', key: 'key', value: 'value'}]);
+  });
+
+  describe('grammar', () => {
+    it('alphanumeric', () => {
+      expect(grammar.parse('key:1a_-.!$')[0].value).toBe('1a_-.!$');
+    });
+
+    it('integer', () => {
+      // @TODO scientific notation?
+      // @TODO should we evaluate arithmetic expressions?
+      // Support unit suffies (B, KB, MB, GB), (ms, s, m, h, d, w, y)
+      expect(grammar.parse('key:1')[0].value).toBe(1);
+      expect(grammar.parse('key:10')[0].value).toBe(10);
+      expect(grammar.parse('key:-10')[0].value).toBe(-10);
+    });
+
+    it('float', () => {
+      expect(grammar.parse('key:.5')[0].value).toBe(0.5);
+      expect(grammar.parse('key:-.5')[0].value).toBe(-0.5);
+      expect(grammar.parse('key:1.000')[0].value).toBe(1.0);
+      expect(grammar.parse('key:1.5')[0].value).toBe(1.5);
+      expect(grammar.parse('key:-1.0')[0].value).toBe(-1.0);
+    });
+
+    it('boolean', () => {
+      expect(grammar.parse('key:true')[0].value).toBe(true);
+      expect(grammar.parse('key:false')[0].value).toBe(false);
+    });
+
+    it('undefined', () => {
+      expect(grammar.parse('key:undefined')[0].value).toBe(undefined);
+    });
+
+    it('null', () => {
+      expect(grammar.parse('key:null')[0].value).toBe(null);
+    });
+
+    it('multiple expressions', () => {
+      expect(grammar.parse('key1:value key2:value')).toEqual([
+        {type: 'Token', key: 'key1', value: 'value'},
+        {type: 'Token', key: 'key2', value: 'value'},
+      ]);
+    });
+
+    it('value operator', () => {
+      expect(grammar.parse('key:>value')[0].operator).toBe('gt');
+      expect(grammar.parse('key:>=value')[0].operator).toBe('ge');
+      expect(grammar.parse('key:<value')[0].operator).toBe('lt');
+      expect(grammar.parse('key:<=value')[0].operator).toBe('le');
+      expect(grammar.parse('key:=value')[0].operator).toBe('eq');
+    });
+
+    it('negation', () => {
+      expect(grammar.parse('!key:value')[0].negated).toBe(true);
+    });
+  });
+
+  describe('transaction properties', () => {});
+  describe('autogrouped properties', () => {});
+  describe('missing instrumentation properties', () => {});
+  describe('error properties', () => {});
+  describe('perf issue', () => {});
+
+  describe('tag properties', () => {});
+  describe('measurement properties', () => {});
+  describe('vitals properties', () => {});
+});
+
+describe('lexer', () => {
+  // it.todo('checks for empty key');
+  // it.todo('checks for invalid key');
+  // it.todo('checks for unknown keys');
+  // it.todo('checks for invalid operator');
+  // it.todo('checks for invalid value');
+  // it.todo("supports OR'ing expressions");
+  // it.todo("supports AND'ing expressions");
+  // it.todo('supports operator precedence via ()');
+});
+
+describe('token evaluator', () => {
+  it('negates expression', () => {
+    const node = makeSpanNode({span_id: '1a3'});
+    expect(evaluate(node, grammar.parse('!span_id:1a3')[0])).toBe(false);
+  });
+
+  describe('string', () => {
+    const node = makeSpanNode({span_id: '1a3'});
+    function g(v: string) {
+      return grammar.parse(`span_id:${v}`)[0];
+    }
+
+    it('exact value', () => {
+      expect(evaluate(node, g('1a3'))).toBe(true);
+    });
+    it('using includes', () => {
+      expect(evaluate(node, g('1a'))).toBe(true);
+    });
+  });
+
+  describe('number', () => {
+    const node = makeSpanNode({start_timestamp: 1000});
+    function g(v: string) {
+      return grammar.parse(`start_timestamp:${v}`)[0];
+    }
+
+    it('exact value', () => {
+      expect(evaluate(node, grammar.parse('start_timestamp:1000')[0])).toBe(true);
+    });
+    it('using gt', () => {
+      expect(evaluate(node, g('>999'))).toBe(true);
+    });
+    it('using ge', () => {
+      expect(evaluate(node, g('>=1000'))).toBe(true);
+    });
+    it('using lt', () => {
+      expect(evaluate(node, g('<1001'))).toBe(true);
+    });
+    it('using le', () => {
+      expect(evaluate(node, g('<=1000'))).toBe(true);
+    });
+    it('using eq', () => {
+      expect(evaluate(node, g('=1000'))).toBe(true);
+    });
+
+    describe('comparing float to int', () => {
+      it('query is float, value is int', () => {
+        expect(evaluate(makeSpanNode({start_timestamp: 1000}), g('1000.0'))).toBe(true);
+      });
+      it('query is int, value is float', () => {
+        expect(evaluate(makeSpanNode({start_timestamp: 1000.0}), g('1000'))).toBe(true);
+      });
+    });
+  });
+
+  describe('boolean', () => {
+    it('true', () => {
+      const node = makeSpanNode({same_process_as_parent: true});
+      expect(evaluate(node, grammar.parse('same_process_as_parent:true')[0])).toBe(true);
+    });
+    it('false', () => {
+      const node = makeSpanNode({same_process_as_parent: false});
+      expect(evaluate(node, grammar.parse('same_process_as_parent:false')[0])).toBe(true);
+    });
+  });
+  it('null', () => {
+    // @ts-expect-error force null on type
+    const node = makeSpanNode({same_process_as_parent: null});
+    expect(evaluate(node, grammar.parse('same_process_as_parent:null')[0])).toBe(true);
+  });
+  it('undefined', () => {
+    const node = makeSpanNode({same_process_as_parent: undefined});
+    expect(evaluate(node, grammar.parse('same_process_as_parent:undefined')[0])).toBe(
+      true
+    );
+  });
+});

static/app/views/performance/newTraceDetails/traceSearch/traceSearchTokenizer.tsx

@@ -0,0 +1,124 @@
+import type {
+  NoDataNode,
+  ParentAutogroupNode,
+  SiblingAutogroupNode,
+  TraceTree,
+  TraceTreeNode,
+} from './../traceTree';
+import grammar from './traceSearch.pegjs';
+
+interface SearchToken {
+  key: string;
+  type: 'Token';
+  value: string | number;
+  negated?: boolean;
+  operator?: 'gt' | 'ge' | 'lt' | 'le' | 'eq';
+}
+
+type Token = SearchToken;
+
+// typeof can return one of the following string values - we ignore BigInt, Symbol as there
+// is no practical case for them + they are not supported by JSON.
+// object (special handling for arrays), number, string, boolean, undefined, null
+type Type = 'object' | 'number' | 'string' | 'boolean' | 'undefined' | 'null';
+
+// @ts-expect-error we ignore some keys on purpose, the TS error makes it helpful
+// for seeing exactly which ones we are ignoring for when we want to add support for them
+const SPAN_KEYS: Record<keyof TraceTree.Span, Type> = {
+  hash: 'string',
+  span_id: 'string',
+  start_timestamp: 'number',
+  timestamp: 'number',
+  trace_id: 'string',
+  description: 'string',
+  exclusive_time: 'number',
+
+  op: 'string',
+  origin: 'string',
+  parent_span_id: 'string',
+  same_process_as_parent: 'boolean',
+  // This one will need to be flattened
+  'span.average_time': 'number',
+  status: 'string',
+
+  // These are both records and will need to be handled differently
+  sentry_tags: 'string',
+  tags: 'string',
+};
+
+export function traceSearchTokenizer(input: string): Token[] {
+  return grammar.parse(input);
+}
+
+export function traceSearchLexer(_input: string): string[] {
+  throw new Error('Not implemented');
+}
+
+export function evaluateTokenForTraceNode(
+  node:
+    | TraceTreeNode<TraceTree.NodeValue>
+    | ParentAutogroupNode
+    | SiblingAutogroupNode
+    | NoDataNode,
+  token: Token
+): boolean {
+  const type = SPAN_KEYS[token.key];
+
+  // @ts-expect-error ignore the lookup as the value will be dynamic
+  const value = node.value[token.key];
+
+  let match: undefined | boolean = undefined;
+  if (token.value === undefined) {
+    match = value === undefined;
+  }
+
+  if (token.value === null) {
+    match = value === null;
+  }
+
+  // @TODO check for the distinction between obj and array here as L78
+  // does not guarantee exact same primitive type in this case
+  if (typeof value !== type && token.value !== null && token.value !== undefined) {
+    // The two types are not the same.
+    return false;
+  }
+
+  // prettier-ignore
+  switch (type) {
+    case 'string': {
+      match = value === token.value || value.includes(token.value);
+      break;
+    }
+    case 'number': {
+      if (!token.operator) {
+        match = value === token.value;
+        break;
+      }
+
+      // prettier-ignore
+      switch (token.operator) {
+        case 'gt': match = value > token.value; break;
+        case 'ge': match = value >= token.value; break;
+        case 'lt': match = value < token.value; break;
+        case 'le': match = value <= token.value; break;
+        case 'eq': match = value === token.value; break;
+        default: break;
+      }
+      break;
+    }
+    case 'boolean': {
+      match = value === token.value;
+      break;
+    }
+    case 'object': {
+      return false;
+    }
+    default: break;
+  }
+
+  if (match === undefined) {
+    return false;
+  }
+
+  return token.negated ? !match : match;
+}