feat(suspect-spans): Add new endpoint for suspect spans (#29357)

This introduced a new endpoint for suspect spans.

mypy.ini

@@ -64,6 +64,7 @@ files = src/sentry/api/bases/external_actor.py,
         src/sentry/utils/dates.py,
         src/sentry/utils/jwt.py,
         src/sentry/utils/kvstore,
+        src/sentry/utils/time_window.py,
         src/sentry/web/decorators.py,
         tests/sentry/processing/realtime_metrics/,
         tests/sentry/tasks/test_low_priority_symbolication.py,

src/sentry/api/endpoints/organization_events_spans_performance.py

@@ -0,0 +1,421 @@
+import dataclasses
+from itertools import chain
+from typing import Any, Dict, List, Optional, Tuple
+
+from rest_framework.exceptions import ParseError
+from rest_framework.request import Request
+from rest_framework.response import Response
+from snuba_sdk.column import Column
+from snuba_sdk.conditions import Condition, Op
+from snuba_sdk.function import Function
+from snuba_sdk.orderby import LimitBy
+
+from sentry import eventstore, features
+from sentry.api.bases import NoProjects, OrganizationEventsEndpointBase
+from sentry.api.paginator import GenericOffsetPaginator
+from sentry.models import Organization
+from sentry.search.events.builder import QueryBuilder
+from sentry.search.events.fields import get_function_alias
+from sentry.search.events.types import ParamsType
+from sentry.utils.snuba import Dataset, raw_snql_query
+from sentry.utils.time_window import TimeWindow, remove_time_windows, union_time_windows
+
+
+@dataclasses.dataclass(frozen=True)
+class SpanPerformanceColumn:
+    suspect_op_group_column: str
+    suspect_example_column: str
+
+
+SPAN_PERFORMANCE_COLUMNS: Dict[str, SpanPerformanceColumn] = {
+    "count": SpanPerformanceColumn("count()", "count()"),
+    "sumExclusiveTime": SpanPerformanceColumn(
+        "sumArray(spans_exclusive_time)", "sumArray(spans_exclusive_time)"
+    ),
+    "p50ExclusiveTime": SpanPerformanceColumn(
+        "percentileArray(spans_exclusive_time, 0.50)", "maxArray(spans_exclusive_time)"
+    ),
+    "p75ExclusiveTime": SpanPerformanceColumn(
+        "percentileArray(spans_exclusive_time, 0.75)", "maxArray(spans_exclusive_time)"
+    ),
+    "p95ExclusiveTime": SpanPerformanceColumn(
+        "percentileArray(spans_exclusive_time, 0.95)", "maxArray(spans_exclusive_time)"
+    ),
+    "p99ExclusiveTime": SpanPerformanceColumn(
+        "percentileArray(spans_exclusive_time, 0.99)", "maxArray(spans_exclusive_time)"
+    ),
+}
+
+
+class OrganizationEventsSpansPerformanceEndpoint(OrganizationEventsEndpointBase):  # type: ignore
+    def has_feature(self, request: Request, organization: Organization) -> bool:
+        return bool(
+            features.has(
+                "organizations:performance-suspect-spans-view",
+                organization,
+                actor=request.user,
+            )
+        )
+
+    def get(self, request: Request, organization: Organization) -> Response:
+        if not self.has_feature(request, organization):
+            return Response(status=404)
+
+        try:
+            params = self.get_snuba_params(request, organization)
+        except NoProjects:
+            return Response(status=404)
+
+        query = request.GET.get("query")
+
+        direction, orderby_column = self.get_orderby_column(request)
+
+        def data_fn(offset: int, limit: int) -> Any:
+            alias = get_function_alias(
+                SPAN_PERFORMANCE_COLUMNS[orderby_column].suspect_op_group_column
+            )
+            orderby = direction + alias
+            suspects = query_suspect_span_groups(params, query, orderby, limit, offset)
+
+            alias = get_function_alias(
+                SPAN_PERFORMANCE_COLUMNS[orderby_column].suspect_example_column
+            )
+            orderby = direction + alias
+
+            # Because we want to support pagination, the limit is 1 more than will be
+            # returned and displayed. Since this extra result is only used for
+            # pagination, we do not need to get any example transactions for it.
+            suspects_requiring_examples = suspects[: limit - 1]
+
+            transaction_ids = query_example_transactions(
+                params, query, orderby, suspects_requiring_examples
+            )
+
+            return [
+                SuspectSpanWithExamples(
+                    examples=[
+                        get_example_transaction(
+                            suspect.project_id,
+                            transaction_id,
+                            suspect.op,
+                            suspect.group,
+                        )
+                        for transaction_id in transaction_ids.get((suspect.op, suspect.group), [])
+                    ],
+                    **dataclasses.asdict(suspect),
+                ).serialize()
+                for suspect in suspects
+            ]
+
+        with self.handle_query_errors():
+            return self.paginate(
+                request,
+                paginator=GenericOffsetPaginator(data_fn=data_fn),
+                default_per_page=4,
+                max_per_page=4,
+            )
+
+        return Response(status=200)
+
+    def get_orderby_column(self, request: Request) -> Tuple[str, str]:
+        orderbys = super().get_orderby(request)
+
+        if orderbys is None:
+            direction = "-"
+            orderby = "sumExclusiveTime"
+        elif len(orderbys) != 1:
+            raise ParseError(detail="Can only order by one column.")
+        else:
+            direction = "-" if orderbys[0].startswith("-") else ""
+            orderby = orderbys[0].lstrip("-")
+
+        if orderby not in SPAN_PERFORMANCE_COLUMNS:
+            options = ", ".join(SPAN_PERFORMANCE_COLUMNS.keys())
+            raise ParseError(detail=f"Can only order by one of {options}")
+
+        return direction, orderby
+
+
+@dataclasses.dataclass(frozen=True)
+class ExampleSpan:
+    id: str
+    start_timestamp: float
+    finish_timestamp: float
+    exclusive_time: float
+
+    def serialize(self) -> Any:
+        return {
+            "id": self.id,
+            "startTimestamp": self.start_timestamp,
+            "finishTimestamp": self.finish_timestamp,
+            "exclusiveTime": self.exclusive_time,
+        }
+
+
+@dataclasses.dataclass(frozen=True)
+class ExampleTransaction:
+    id: str
+    description: Optional[str]
+    start_timestamp: float
+    finish_timestamp: float
+    non_overlapping_exclusive_time: float
+    spans: List[ExampleSpan]
+
+    def serialize(self) -> Any:
+        return {
+            "id": self.id,
+            "description": self.description,
+            "startTimestamp": self.start_timestamp,
+            "finishTimestamp": self.finish_timestamp,
+            "nonOverlappingExclusiveTime": self.non_overlapping_exclusive_time,
+            "spans": [span.serialize() for span in self.spans],
+        }
+
+
+@dataclasses.dataclass(frozen=True)
+class SuspectSpan:
+    project_id: int
+    project: str
+    transaction: str
+    op: str
+    group: str
+    frequency: int
+    count: int
+    sum_exclusive_time: float
+    p50_exclusive_time: float
+    p75_exclusive_time: float
+    p95_exclusive_time: float
+    p99_exclusive_time: float
+
+    def serialize(self) -> Any:
+        return {
+            "projectId": self.project_id,
+            "project": self.project,
+            "transaction": self.transaction,
+            "op": self.op,
+            "group": self.group,
+            "frequency": self.frequency,
+            "count": self.count,
+            "sumExclusiveTime": self.sum_exclusive_time,
+            "p50ExclusiveTime": self.p50_exclusive_time,
+            "p75ExclusiveTime": self.p75_exclusive_time,
+            "p95ExclusiveTime": self.p95_exclusive_time,
+            "p99ExclusiveTime": self.p99_exclusive_time,
+        }
+
+
+@dataclasses.dataclass(frozen=True)
+class SuspectSpanWithExamples(SuspectSpan):
+    examples: Optional[List[ExampleTransaction]] = None
+
+    def serialize(self) -> Any:
+        serialized = super().serialize()
+        serialized["examples"] = (
+            [] if self.examples is None else [ex.serialize() for ex in self.examples]
+        )
+        return serialized
+
+
+def query_suspect_span_groups(
+    params: ParamsType,
+    query: Optional[str],
+    order_column: str,
+    limit: int,
+    offset: int,
+) -> List[SuspectSpan]:
+    builder = QueryBuilder(
+        dataset=Dataset.Discover,
+        params=params,
+        selected_columns=[
+            "project.id",
+            "project",
+            "transaction",
+            "array_join(spans_op)",
+            "array_join(spans_group)",
+            "count_unique(id)",
+            *(column.suspect_op_group_column for column in SPAN_PERFORMANCE_COLUMNS.values()),
+        ],
+        query=query,
+        orderby=order_column,
+        auto_aggregations=True,
+        use_aggregate_conditions=True,
+        limit=limit,
+        offset=offset,
+        functions_acl=["array_join", "sumArray", "percentileArray", "maxArray"],
+    )
+
+    snql_query = builder.get_snql_query()
+    results = raw_snql_query(snql_query, "api.organization-events-spans-performance-suspects")
+
+    return [
+        SuspectSpan(
+            project_id=suspect["project.id"],
+            project=suspect["project"],
+            transaction=suspect["transaction"],
+            op=suspect["array_join_spans_op"],
+            group=suspect["array_join_spans_group"],
+            frequency=suspect["count_unique_id"],
+            count=suspect["count"],
+            sum_exclusive_time=suspect["sumArray_spans_exclusive_time"],
+            p50_exclusive_time=suspect.get("percentileArray_spans_exclusive_time_0_50"),
+            p75_exclusive_time=suspect.get("percentileArray_spans_exclusive_time_0_75"),
+            p95_exclusive_time=suspect.get("percentileArray_spans_exclusive_time_0_95"),
+            p99_exclusive_time=suspect.get("percentileArray_spans_exclusive_time_0_99"),
+        )
+        for suspect in results["data"]
+    ]
+
+
+def query_example_transactions(
+    params: ParamsType,
+    query: Optional[str],
+    order_column: str,
+    suspects: List[SuspectSpan],
+    per_suspect: int = 3,
+) -> Dict[Tuple[str, str], List[str]]:
+    # there aren't any suspects, early return to save an empty query
+    if not suspects:
+        return {}
+
+    builder = QueryBuilder(
+        dataset=Dataset.Discover,
+        params=params,
+        selected_columns=[
+            "id",
+            "array_join(spans_op)",
+            "array_join(spans_group)",
+            *(column.suspect_example_column for column in SPAN_PERFORMANCE_COLUMNS.values()),
+        ],
+        query=query,
+        orderby=get_function_alias(order_column),
+        auto_aggregations=True,
+        use_aggregate_conditions=True,
+        # we want only `per_suspect` examples for each suspect
+        limit=len(suspects) * per_suspect,
+        functions_acl=["array_join", "sumArray", "percentileArray", "maxArray"],
+    )
+
+    # we are only interested in the specific op, group pairs from the suspects
+    builder.add_conditions(
+        [
+            Condition(
+                Function(
+                    "tuple",
+                    [
+                        builder.resolve_function("array_join(spans_op)"),
+                        builder.resolve_function("array_join(spans_group)"),
+                    ],
+                ),
+                Op.IN,
+                Function(
+                    "tuple",
+                    [Function("tuple", [suspect.op, suspect.group]) for suspect in suspects],
+                ),
+            ),
+        ]
+    )
+
+    # Hack: the limit by clause only allows columns but here we want to
+    # do a limitby on the two array joins. For the time being, directly
+    # do the limitby on the internal snuba name for the span group column
+    # but this should not be relied upon in production, and if two spans
+    # differ only by the span op, this will result in a incorrect query
+    builder.limitby = LimitBy(Column("_snuba_array_join_spans_group"), per_suspect)
+
+    snql_query = builder.get_snql_query()
+    results = raw_snql_query(snql_query, "api.organization-events-spans-performance-examples")
+
+    examples: Dict[Tuple[str, str], List[str]] = {
+        (suspect.op, suspect.group): [] for suspect in suspects
+    }
+
+    for example in results["data"]:
+        key = example["array_join_spans_op"], example["array_join_spans_group"]
+        examples[key].append(example["id"])
+
+    return examples
+
+
+def get_example_transaction(
+    project_id: int,
+    transaction_id: str,
+    span_op: str,
+    span_group: str,
+) -> ExampleTransaction:
+    event = eventstore.get_event_by_id(project_id, transaction_id)
+    data = event.data
+
+    # the transaction itself is a span as well but we need to reconstruct
+    # it from the event as it's not present in the spans array
+    trace_context = data.get("contexts", {}).get("trace", {})
+    root_span = {
+        "span_id": trace_context["span_id"],
+        "op": trace_context["op"],
+        "hash": trace_context["hash"],
+        "exclusive_time": trace_context["exclusive_time"],
+        "description": data["transaction"],
+        "start_timestamp": data["start_timestamp"],
+        "timestamp": data["timestamp"],
+    }
+
+    matching_spans = [
+        span
+        for span in chain([root_span], data.get("spans", []))
+        if span["op"] == span_op and span["hash"] == span_group
+    ]
+
+    # get the first non-None description
+    # use None if all descriptions are None
+    description = None
+    for span in matching_spans:
+        if span.get("description") is None:
+            continue
+        description = span["description"]
+
+    spans: List[ExampleSpan] = [
+        ExampleSpan(
+            id=span["span_id"],
+            start_timestamp=span["start_timestamp"],
+            finish_timestamp=span["timestamp"],
+            exclusive_time=span["exclusive_time"],
+        )
+        for span in matching_spans
+    ]
+
+    non_overlapping_exclusive_time_windows = union_time_windows(
+        [
+            window
+            for span in spans
+            for window in get_exclusive_time_windows(
+                span,
+                # don't need to check the root span here because its parent
+                # will never be one of the spans in this transaction
+                data.get("spans", []),
+            )
+        ]
+    )
+
+    return ExampleTransaction(
+        id=transaction_id,
+        description=description,
+        start_timestamp=data["start_timestamp"],
+        finish_timestamp=data["timestamp"],
+        non_overlapping_exclusive_time=sum(
+            window.duration_ms for window in non_overlapping_exclusive_time_windows
+        ),
+        spans=spans,
+    )
+
+
+def get_exclusive_time_windows(span: ExampleSpan, spans: List[Any]) -> List[TimeWindow]:
+    non_overlapping_children_time_windows = union_time_windows(
+        [
+            TimeWindow(start=child["start_timestamp"], end=child["timestamp"])
+            for child in spans
+            if child["parent_span_id"] == span.id
+        ]
+    )
+    return remove_time_windows(
+        TimeWindow(start=span.start_timestamp, end=span.finish_timestamp),
+        non_overlapping_children_time_windows,
+    )

src/sentry/api/urls.py

@@ -182,6 +182,9 @@ from .endpoints.organization_events_meta import (
     OrganizationEventsMetaEndpoint,
     OrganizationEventsRelatedIssuesEndpoint,
 )
+from .endpoints.organization_events_spans_performance import (
+    OrganizationEventsSpansPerformanceEndpoint,
+)
 from .endpoints.organization_events_stats import OrganizationEventsStatsEndpoint
 from .endpoints.organization_events_trace import (
     OrganizationEventsTraceEndpoint,
@@ -998,6 +1001,11 @@ urlpatterns = [
                     OrganizationEventsFacetsPerformanceHistogramEndpoint.as_view(),
                     name="sentry-api-0-organization-events-facets-performance-histogram",
                 ),
+                url(
+                    r"^(?P<organization_slug>[^\/]+)/events-spans-performance/$",
+                    OrganizationEventsSpansPerformanceEndpoint.as_view(),
+                    name="sentry-api-0-organization-events-spans-performance",
+                ),
                 url(
                     r"^(?P<organization_slug>[^\/]+)/events-meta/$",
                     OrganizationEventsMetaEndpoint.as_view(),

src/sentry/search/events/builder.py

@@ -1,6 +1,7 @@
 from typing import List, Optional, Tuple
 
 from snuba_sdk.column import Column
+from snuba_sdk.conditions import Condition
 from snuba_sdk.entity import Entity
 from snuba_sdk.expressions import Limit, Offset
 from snuba_sdk.function import CurriedFunction
@@ -115,6 +116,9 @@ class QueryBuilder(QueryFilter):
                     )
                 )
 
+    def add_conditions(self, conditions: List[Condition]) -> None:
+        self.where += conditions
+
     def get_snql_query(self) -> Query:
         self.validate_having_clause()
 

src/sentry/search/events/fields.py

@@ -2453,6 +2453,9 @@ class QueryFields(QueryBase):
                     result_type_fn=reflective_result_type(),
                     default_result_type="duration",
                     redundant_grouping=True,
+                    combinators=[
+                        SnQLArrayCombinator("column", NumericColumn.numeric_array_columns)
+                    ],
                 ),
                 SnQLFunction(
                     "p50",
@@ -2665,6 +2668,9 @@ class QueryFields(QueryBase):
                     result_type_fn=reflective_result_type(),
                     default_result_type="duration",
                     redundant_grouping=True,
+                    combinators=[
+                        SnQLArrayCombinator("column", NumericColumn.numeric_array_columns)
+                    ],
                 ),
                 SnQLFunction(
                     "avg",

src/sentry/utils/samples.py

@@ -115,6 +115,7 @@ def load_data(
     trace=None,
     span_id=None,
     spans=None,
+    trace_context=None,
 ):
     # NOTE: Before editing this data, make sure you understand the context
     # in which its being used. It is NOT only used for local development and
@@ -191,6 +192,8 @@ def load_data(
                 tag[1] = span_id
         data["contexts"]["trace"]["trace_id"] = trace
         data["contexts"]["trace"]["span_id"] = span_id
+        if trace_context is not None:
+            data["contexts"]["trace"].update(trace_context)
         if spans:
             data["spans"] = spans
 

src/sentry/utils/time_window.py

@@ -0,0 +1,75 @@
+from dataclasses import dataclass
+from typing import List, Optional, Tuple
+
+
+@dataclass(frozen=True)
+class TimeWindow:
+    # Timestamps are in seconds
+    start: float
+    end: float
+
+    def as_tuple(self) -> Tuple[float, float]:
+        return (self.start, self.end)
+
+    @property
+    def duration_ms(self) -> float:
+        return (self.end - self.start) * 1000
+
+    def __add__(self, other: "TimeWindow") -> Tuple[Optional["TimeWindow"], "TimeWindow"]:
+        if self.start < other.start:
+            if self.end < other.start:
+                return self, other
+            return None, TimeWindow(start=self.start, end=max(self.end, other.end))
+        else:
+            if self.start > other.end:
+                return other, self
+            return None, TimeWindow(start=other.start, end=max(self.end, other.end))
+
+    def __sub__(self, other: "TimeWindow") -> Tuple[Optional["TimeWindow"], "TimeWindow"]:
+        if self.start < other.start:
+            if self.end > other.end:
+                return (
+                    TimeWindow(start=self.start, end=other.start),
+                    TimeWindow(start=other.end, end=self.end),
+                )
+            return None, TimeWindow(start=self.start, end=min(self.end, other.start))
+        else:
+            if self.end < other.end:
+                return None, TimeWindow(start=self.end, end=self.end)
+            return None, TimeWindow(start=max(self.start, other.end), end=self.end)
+
+
+def union_time_windows(time_windows: List[TimeWindow]) -> List[TimeWindow]:
+    if not time_windows:
+        return []
+
+    previous, *time_windows = sorted(time_windows, key=lambda window: window.as_tuple())
+
+    unioned: List[TimeWindow] = []
+
+    for current in time_windows:
+        window, previous = previous + current
+        if window:
+            unioned.append(window)
+
+    unioned.append(previous)
+
+    return unioned
+
+
+def remove_time_windows(source: TimeWindow, time_windows: List[TimeWindow]) -> List[TimeWindow]:
+    if not time_windows:
+        return [source]
+
+    removed: List[TimeWindow] = []
+
+    for current in time_windows:
+        window, source = source - current
+        if window:
+            removed.append(window)
+
+    removed.append(source)
+
+    # After subtracting time windows, we may end up with 0 width time_windows.
+    # remove them from the results.
+    return [time_window for time_window in removed if time_window.start != time_window.end]

tests/sentry/utils/test_time_window.py

@@ -0,0 +1,99 @@
+import random
+
+import pytest
+
+from sentry.utils.time_window import TimeWindow, remove_time_windows, union_time_windows
+
+
+@pytest.mark.parametrize(
+    "start, end, expected",
+    [
+        pytest.param(0, 0, 0),
+        pytest.param(0, 1, 1.0 * 1000),
+        pytest.param(10.0, 90.0, 80.0 * 1000),
+    ],
+)
+def test_time_window_duration(start, end, expected):
+    time_window = TimeWindow(start, end)
+    assert time_window.duration_ms == expected
+
+
+union_time_windows_test_cases = [
+    pytest.param(
+        [(0, 1), (2, 3), (4, 5), (6, 7), (8, 9)],
+        [(0, 1), (2, 3), (4, 5), (6, 7), (8, 9)],
+        id="non_overlapping",
+    ),
+    pytest.param([(0, 1), (1, 2), (2, 3), (3, 4), (4, 5)], [(0, 5)], id="all_edges_overlapping"),
+    pytest.param(
+        [(0, 2), (1, 3), (2, 4), (3, 5), (4, 6)], [(0, 6)], id="all_intervals_overlapping"
+    ),
+    pytest.param([(0, 1), (1, 2), (3, 4), (4, 5)], [(0, 2), (3, 5)], id="some_edges_overlapping"),
+    pytest.param(
+        [(0, 2), (1, 3), (4, 6), (5, 7)], [(0, 3), (4, 7)], id="some_intervals_overlapping"
+    ),
+    pytest.param(
+        [(0, 1), (1, 2), (3, 5), (4, 6), (6, 7)], [(0, 2), (3, 7)], id="mixed_of_different_overlaps"
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "time_windows, expected",
+    union_time_windows_test_cases
+    + [
+        # the order of the time windows shouldn't matter,
+        # give it a shuffle to generate additional test cases
+        pytest.param(
+            random.sample(test_case.values[0], len(test_case.values[0])),
+            test_case.values[1],
+            id=f"shuffled_{test_case.id}",
+        )
+        for test_case in union_time_windows_test_cases
+    ],
+)
+def test_union_time_windows(time_windows, expected):
+    time_window_objs = [TimeWindow(start, end) for start, end in time_windows]
+    expected_objs = [TimeWindow(start, end) for start, end in expected]
+    assert union_time_windows(time_window_objs) == expected_objs, time_windows
+
+
+remove_time_windows_test_cases = [
+    pytest.param(
+        (4, 5), [(0, 1), (1, 2), (3, 4), (6, 7), (7, 8), (8, 9)], [(4, 5)], id="non_overlapping"
+    ),
+    pytest.param((0, 1), [(0, 1)], [], id="is_source_time_window"),
+    pytest.param((1, 3), [(0, 2), (2, 4)], [], id="covers_source_time_window"),
+    pytest.param((4, 7), [(3, 5), (6, 8)], [(5, 6)], id="leaves_source_time_window_center"),
+    pytest.param((4, 7), [(5, 6)], [(4, 5), (6, 7)], id="leaves_source_time_window_ends"),
+    pytest.param(
+        (2, 7),
+        [(0, 3), (1, 4), (5, 8), (6, 9)],
+        [(4, 5)],
+        id="covers_source_time_window_ends_multiple_times",
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "source_time_window, time_windows, expected",
+    remove_time_windows_test_cases
+    + [
+        # the order of the time windows shouldn't matter,
+        # give it a shuffle to generate additional test cases
+        pytest.param(
+            test_case.values[0],
+            random.sample(test_case.values[1], len(test_case.values[1])),
+            test_case.values[2],
+            id=f"shuffled_{test_case.id}",
+        )
+        for test_case in remove_time_windows_test_cases
+    ],
+)
+def test_remove_time_windows(source_time_window, time_windows, expected):
+    source_time_window_obj = TimeWindow(source_time_window[0], source_time_window[1])
+    time_window_objs = [TimeWindow(start, end) for start, end in time_windows]
+    expected_objs = [TimeWindow(start, end) for start, end in expected]
+    assert (
+        remove_time_windows(source_time_window_obj, time_window_objs) == expected_objs
+    ), time_windows

tests/snuba/api/endpoints/test_organization_events_spans_performance.py

@@ -0,0 +1,435 @@
+import time
+from datetime import timedelta
+
+import pytest
+from django.urls import reverse
+
+from sentry.testutils import APITestCase, SnubaTestCase
+from sentry.testutils.helpers import parse_link_header
+from sentry.testutils.helpers.datetime import before_now, iso_format
+from sentry.utils import json
+from sentry.utils.samples import load_data
+
+
+class OrganizationEventsSpansPerformanceEndpointBase(APITestCase, SnubaTestCase):
+    FEATURES = ["organizations:performance-suspect-spans-view"]
+
+    def setUp(self):
+        super().setUp()
+        self.login_as(user=self.user)
+
+        self.url = reverse(
+            "sentry-api-0-organization-events-spans-performance",
+            kwargs={"organization_slug": self.organization.slug},
+        )
+
+        self.min_ago = before_now(minutes=1).replace(microsecond=0)
+
+    def update_snuba_config_ensure(self, config, poll=60, wait=1):
+        self.snuba_update_config(config)
+
+        for i in range(poll):
+            updated = True
+
+            new_config = json.loads(self.snuba_get_config().decode("utf-8"))
+
+            for k, v in config.items():
+                if new_config.get(k) != v:
+                    updated = False
+                    break
+
+            if updated:
+                return
+
+            time.sleep(wait)
+
+        assert False, "snuba config not updated in time"
+
+    def create_event(self, **kwargs):
+        if "span_id" not in kwargs:
+            kwargs["span_id"] = "a" * 16
+
+        if "start_timestamp" not in kwargs:
+            kwargs["start_timestamp"] = self.min_ago
+
+        if "timestamp" not in kwargs:
+            kwargs["timestamp"] = self.min_ago + timedelta(seconds=8)
+
+        if "trace_context" not in kwargs:
+            # should appear for all of the pXX metrics
+            kwargs["trace_context"] = {
+                "op": "http.server",
+                "hash": "ab" * 8,
+                "exclusive_time": 4.0,
+            }
+
+        if "spans" not in kwargs:
+            kwargs["spans"] = [
+                # should appear for the sum metric
+                {
+                    "same_process_as_parent": True,
+                    "parent_span_id": "a" * 16,
+                    "span_id": x * 16,
+                    "start_timestamp": iso_format(self.min_ago + timedelta(seconds=1)),
+                    "timestamp": iso_format(self.min_ago + timedelta(seconds=4)),
+                    "op": "django.middleware",
+                    "description": "middleware span",
+                    "hash": "cd" * 8,
+                    "exclusive_time": 3.0,
+                }
+                for x in ["b", "c"]
+            ] + [
+                # should appear for the count metric
+                {
+                    "same_process_as_parent": True,
+                    "parent_span_id": "a" * 16,
+                    "span_id": x * 16,
+                    "start_timestamp": iso_format(self.min_ago + timedelta(seconds=4)),
+                    "timestamp": iso_format(self.min_ago + timedelta(seconds=5)),
+                    "op": "django.view",
+                    "description": "view span",
+                    "hash": "ef" * 8,
+                    "exclusive_time": 1.0,
+                }
+                for x in ["d", "e", "f"]
+            ]
+
+        data = load_data("transaction", **kwargs)
+        data["transaction"] = "root transaction"
+
+        return self.store_event(data, project_id=self.project.id)
+
+    def suspect_span_results(self, key, event):
+        if key == "sum":
+            return {
+                "projectId": self.project.id,
+                "project": self.project.slug,
+                "transaction": event.transaction,
+                "op": "django.middleware",
+                "group": "cd" * 8,
+                "frequency": 1,
+                "count": 2,
+                "sumExclusiveTime": 6.0,
+                "p50ExclusiveTime": 3.0,
+                "p75ExclusiveTime": 3.0,
+                "p95ExclusiveTime": 3.0,
+                "p99ExclusiveTime": 3.0,
+                "examples": [
+                    {
+                        "id": event.event_id,
+                        "description": "middleware span",
+                        "startTimestamp": self.min_ago.timestamp(),
+                        "finishTimestamp": (self.min_ago + timedelta(seconds=8)).timestamp(),
+                        "nonOverlappingExclusiveTime": 3000.0,
+                        "spans": [
+                            {
+                                "id": x * 16,
+                                "startTimestamp": (self.min_ago + timedelta(seconds=1)).timestamp(),
+                                "finishTimestamp": (
+                                    self.min_ago + timedelta(seconds=4)
+                                ).timestamp(),
+                                "exclusiveTime": 3.0,
+                            }
+                            for x in ["b", "c"]
+                        ],
+                    },
+                ],
+            }
+
+        if key == "count":
+            return {
+                "projectId": self.project.id,
+                "project": self.project.slug,
+                "transaction": event.transaction,
+                "op": "django.view",
+                "group": "ef" * 8,
+                "frequency": 1,
+                "count": 3,
+                "sumExclusiveTime": 3.0,
+                "p50ExclusiveTime": 1.0,
+                "p75ExclusiveTime": 1.0,
+                "p95ExclusiveTime": 1.0,
+                "p99ExclusiveTime": 1.0,
+                "examples": [
+                    {
+                        "id": event.event_id,
+                        "description": "view span",
+                        "startTimestamp": self.min_ago.timestamp(),
+                        "finishTimestamp": (self.min_ago + timedelta(seconds=8)).timestamp(),
+                        "nonOverlappingExclusiveTime": 1000.0,
+                        "spans": [
+                            {
+                                "id": x * 16,
+                                "startTimestamp": (self.min_ago + timedelta(seconds=4)).timestamp(),
+                                "finishTimestamp": (
+                                    self.min_ago + timedelta(seconds=5)
+                                ).timestamp(),
+                                "exclusiveTime": 1.0,
+                            }
+                            for x in ["d", "e", "f"]
+                        ],
+                    },
+                ],
+            }
+
+        if key == "percentiles":
+            return {
+                "projectId": self.project.id,
+                "project": self.project.slug,
+                "transaction": event.transaction,
+                "op": "http.server",
+                "group": "ab" * 8,
+                "frequency": 1,
+                "count": 1,
+                "sumExclusiveTime": 4.0,
+                "p50ExclusiveTime": 4.0,
+                "p75ExclusiveTime": 4.0,
+                "p95ExclusiveTime": 4.0,
+                "p99ExclusiveTime": 4.0,
+                "examples": [
+                    {
+                        "id": event.event_id,
+                        "description": "root transaction",
+                        "startTimestamp": self.min_ago.timestamp(),
+                        "finishTimestamp": (self.min_ago + timedelta(seconds=8)).timestamp(),
+                        "nonOverlappingExclusiveTime": 4000.0,
+                        "spans": [
+                            {
+                                "id": "a" * 16,
+                                "startTimestamp": self.min_ago.timestamp(),
+                                "finishTimestamp": (
+                                    self.min_ago + timedelta(seconds=8)
+                                ).timestamp(),
+                                "exclusiveTime": 4.0,
+                            }
+                        ],
+                    },
+                ],
+            }
+
+    def assert_suspect_span(self, result, expected_result):
+        assert len(result) == len(expected_result)
+        for suspect, expected_suspect in zip(result, expected_result):
+            for key in [
+                "projectId",
+                "project",
+                "transaction",
+                "op",
+                "group",
+                "frequency",
+                "count",
+                "sumExclusiveTime",
+                "p50ExclusiveTime",
+                "p75ExclusiveTime",
+                "p95ExclusiveTime",
+                "p99ExclusiveTime",
+            ]:
+                assert suspect[key] == expected_suspect[key], key
+
+            assert len(suspect["examples"]) == len(expected_suspect["examples"])
+
+            for example, expected_example in zip(suspect["examples"], expected_suspect["examples"]):
+                for key in [
+                    "id",
+                    "description",
+                    "startTimestamp",
+                    "finishTimestamp",
+                    "nonOverlappingExclusiveTime",
+                ]:
+                    assert example[key] == expected_example[key], key
+
+                assert len(example["spans"]) == len(expected_example["spans"])
+
+                for span, expected_span in zip(example["spans"], expected_example["spans"]):
+                    for key in [
+                        "id",
+                        "startTimestamp",
+                        "finishTimestamp",
+                        "exclusiveTime",
+                    ]:
+                        assert span[key] == expected_span[key], key
+
+    def test_no_feature(self):
+        response = self.client.get(self.url, format="json")
+        assert response.status_code == 404, response.content
+
+    def test_no_projects(self):
+        user = self.create_user()
+        org = self.create_organization(owner=user)
+        self.login_as(user=user)
+
+        url = reverse(
+            "sentry-api-0-organization-events-spans-performance",
+            kwargs={"organization_slug": org.slug},
+        )
+
+        with self.feature(self.FEATURES):
+            response = self.client.get(url, format="json")
+        assert response.status_code == 404, response.content
+
+    def test_bad_sort(self):
+        with self.feature(self.FEATURES):
+            response = self.client.get(
+                self.url,
+                data={
+                    "project": self.project.id,
+                    "sort": "-stuff",
+                },
+                format="json",
+            )
+        assert response.status_code == 400, response.content
+        assert response.data == {
+            "detail": "Can only order by one of count, sumExclusiveTime, p50ExclusiveTime, p75ExclusiveTime, p95ExclusiveTime, p99ExclusiveTime"
+        }
+
+    def test_sort_sum(self):
+        # TODO: remove this and the @pytest.skip once the config
+        # is no longer necessary as this can add ~10s to the test
+        self.update_snuba_config_ensure({"write_span_columns_projects": f"[{self.project.id}]"})
+
+        event = self.create_event()
+
+        with self.feature(self.FEATURES):
+            response = self.client.get(
+                self.url,
+                data={
+                    "project": self.project.id,
+                    "sort": "-sumExclusiveTime",
+                },
+                format="json",
+            )
+
+        assert response.status_code == 200, response.content
+        self.assert_suspect_span(
+            response.data,
+            [
+                self.suspect_span_results("sum", event),
+                self.suspect_span_results("percentiles", event),
+                self.suspect_span_results("count", event),
+            ],
+        )
+
+    @pytest.mark.skip("setting snuba config is too slow")
+    def test_sort_count(self):
+        event = self.create_event()
+
+        with self.feature(self.FEATURES):
+            response = self.client.get(
+                self.url,
+                data={
+                    "project": self.project.id,
+                    "sort": "-count",
+                },
+                format="json",
+            )
+
+        assert response.status_code == 200, response.content
+        self.assert_suspect_span(
+            response.data,
+            [
+                self.suspect_span_results("count", event),
+                self.suspect_span_results("sum", event),
+                self.suspect_span_results("percentiles", event),
+            ],
+        )
+
+    @pytest.mark.skip("setting snuba config is too slow")
+    def test_sort_percentiles(self):
+        event = self.create_event()
+
+        for sort in [
+            "p50ExclusiveTime",
+            "p75ExclusiveTime",
+            "p95ExclusiveTime",
+            "p99ExclusiveTime",
+        ]:
+            with self.feature(self.FEATURES):
+                response = self.client.get(
+                    self.url,
+                    data={
+                        "project": self.project.id,
+                        "sort": f"-{sort}",
+                    },
+                    format="json",
+                )
+
+            assert response.status_code == 200, response.content
+            self.assert_suspect_span(
+                response.data,
+                [
+                    self.suspect_span_results("percentiles", event),
+                    self.suspect_span_results("sum", event),
+                    self.suspect_span_results("count", event),
+                ],
+            )
+
+    @pytest.mark.skip("setting snuba config is too slow")
+    def test_pagination_first_page(self):
+        self.create_event()
+
+        with self.feature(self.FEATURES):
+            response = self.client.get(
+                self.url,
+                data={
+                    "project": self.project.id,
+                    "sort": "-sumExclusiveTime",
+                    "per_page": 1,
+                },
+                format="json",
+            )
+
+        assert response.status_code == 200, response.content
+        links = parse_link_header(response["Link"])
+        for link, info in links.items():
+            assert f"project={self.project.id}" in link
+            assert "sort=-sumExclusiveTime" in link
+            # first page does not have a previous page, only next
+            assert info["results"] == "true" if info["rel"] == "next" else "false"
+
+    @pytest.mark.skip("setting snuba config is too slow")
+    def test_pagination_middle_page(self):
+        self.create_event()
+
+        with self.feature(self.FEATURES):
+            response = self.client.get(
+                self.url,
+                data={
+                    "project": self.project.id,
+                    "sort": "-sumExclusiveTime",
+                    "per_page": 1,
+                    "cursor": "0:1:0",
+                },
+                format="json",
+            )
+
+        assert response.status_code == 200, response.content
+        links = parse_link_header(response["Link"])
+        for link, info in links.items():
+            assert f"project={self.project.id}" in link
+            assert "sort=-sumExclusiveTime" in link
+            # middle page has both a previous and next
+            assert info["results"] == "true"
+
+    @pytest.mark.skip("setting snuba config is too slow")
+    def test_pagination_last_page(self):
+        self.create_event()
+
+        with self.feature(self.FEATURES):
+            response = self.client.get(
+                self.url,
+                data={
+                    "project": self.project.id,
+                    "sort": "-sumExclusiveTime",
+                    "per_page": 1,
+                    "cursor": "0:2:0",
+                },
+                format="json",
+            )
+
+        assert response.status_code == 200, response.content
+        links = parse_link_header(response["Link"])
+        for link, info in links.items():
+            assert f"project={self.project.id}" in link
+            assert "sort=-sumExclusiveTime" in link
+            # last page does not have a next page, only previous
+            assert info["results"] == ("true" if info["rel"] == "previous" else "false")