feat(explore-charts): Displaying topN events in chart and table. (#77566)

Dev-ui link to test it out:
[link](https://sentry.dev.getsentry.net:7999/traces/?field=project&field=id&field=span.op&field=span.description&field=span.duration&field=timestamp&field=organization_slug&field=project.organization.id&groupBy=deviceMemory&interval=1m&mode=aggregate&project=-1&query=&sort=organization_slug&statsPeriod=1h&utc=true&visualize=%7B%22yAxes%22%3A%5B%22avg%28span.duration%29%22%5D%2C%22chartType%22%3A1%7D)
![Screenshot 2024-09-16 at 1 36
15 PM](https://github.com/user-attachments/assets/a2b5558e-497e-4ae1-956b-04b685c91b88)

---------

Co-authored-by: Abdullah Khan <abdullahkhan@PG9Y57YDXQ.local>

static/app/views/explore/charts/index.tsx

@@ -10,14 +10,22 @@ import {dedupeArray} from 'sentry/utils/dedupeArray';
 import {aggregateOutputType} from 'sentry/utils/discover/fields';
 import {MutableSearch} from 'sentry/utils/tokenizeSearch';
 import usePageFilters from 'sentry/utils/usePageFilters';
+import {formatVersion} from 'sentry/utils/versions/formatVersion';
 import {useChartInterval} from 'sentry/views/explore/hooks/useChartInterval';
 import {useDataset} from 'sentry/views/explore/hooks/useDataset';
 import {useVisualizes} from 'sentry/views/explore/hooks/useVisualizes';
 import Chart, {ChartType} from 'sentry/views/insights/common/components/chart';
 import ChartPanel from 'sentry/views/insights/common/components/chartPanel';
 import {useSpanIndexedSeries} from 'sentry/views/insights/common/queries/useDiscoverSeries';
+import {useSortedTopNSeries} from 'sentry/views/insights/common/queries/useSortedTopNSeries';
 import {CHART_HEIGHT} from 'sentry/views/insights/database/settings';
 
+import {useGroupBys} from '../hooks/useGroupBys';
+import {useResultMode} from '../hooks/useResultsMode';
+import {useSorts} from '../hooks/useSorts';
+import {TOP_EVENTS_LIMIT, useTopEvents} from '../hooks/useTopEvents';
+import {formatSort} from '../tables/aggregatesTable';
+
 interface ExploreChartsProps {
   query: string;
 }
@@ -44,6 +52,28 @@ export function ExploreCharts({query}: ExploreChartsProps) {
   const [dataset] = useDataset();
   const [visualizes, setVisualizes] = useVisualizes();
   const [interval, setInterval, intervalOptions] = useChartInterval();
+  const [groupBys] = useGroupBys();
+  const [resultMode] = useResultMode();
+  const topEvents = useTopEvents();
+
+  const fields: string[] = useMemo(() => {
+    if (resultMode === 'samples') {
+      return [];
+    }
+
+    return [...groupBys, ...visualizes.flatMap(visualize => visualize.yAxes)].filter(
+      Boolean
+    );
+  }, [resultMode, groupBys, visualizes]);
+  const [sorts] = useSorts({fields});
+
+  const orderby: string | string[] | undefined = useMemo(() => {
+    if (!sorts.length) {
+      return undefined;
+    }
+
+    return sorts.map(formatSort);
+  }, [sorts]);
 
   const yAxes = useMemo(() => {
     const deduped = dedupeArray(visualizes.flatMap(visualize => visualize.yAxes));
@@ -51,17 +81,42 @@ export function ExploreCharts({query}: ExploreChartsProps) {
     return deduped;
   }, [visualizes]);
 
-  const series = useSpanIndexedSeries(
+  const singleSeriesResult = useSpanIndexedSeries(
     {
       search: new MutableSearch(query ?? ''),
       yAxis: yAxes,
       interval: interval ?? getInterval(pageFilters.selection.datetime, 'metrics'),
-      enabled: true,
+      enabled: topEvents === undefined,
     },
     'api.explorer.stats',
     dataset
   );
 
+  const topNSeriesResult = useSortedTopNSeries(
+    {
+      search: new MutableSearch(query ?? ''),
+      yAxis: yAxes,
+      interval: interval ?? getInterval(pageFilters.selection.datetime, 'metrics'),
+      enabled: topEvents !== undefined,
+      fields,
+      orderby,
+      topEvents,
+    },
+    'api.explorer.stats',
+    dataset
+  );
+
+  const getSeries = useCallback(
+    (dedupedYAxes: string[]) => {
+      if (topEvents !== undefined) {
+        return topNSeriesResult.data;
+      }
+
+      return dedupedYAxes.map(yAxis => singleSeriesResult.data[yAxis]);
+    },
+    [singleSeriesResult, topNSeriesResult, topEvents]
+  );
+
   const handleChartTypeChange = useCallback(
     (chartType: ChartType, index: number) => {
       const newVisualizes = visualizes.slice();
@@ -71,6 +126,11 @@ export function ExploreCharts({query}: ExploreChartsProps) {
     [visualizes, setVisualizes]
   );
 
+  const error =
+    topEvents === undefined ? singleSeriesResult.error : topNSeriesResult.error;
+  const loading =
+    topEvents === undefined ? singleSeriesResult.isPending : topNSeriesResult.isPending;
+
   return (
     <Fragment>
       {visualizes.map((visualize, index) => {
@@ -108,10 +168,12 @@ export function ExploreCharts({query}: ExploreChartsProps) {
                   top: '8px',
                   bottom: '0',
                 }}
-                data={dedupedYAxes.map(yAxis => series.data[yAxis])}
-                error={series.error}
-                loading={series.isPending}
-                chartColors={CHART_PALETTE[2]}
+                legendFormatter={value => formatVersion(value)}
+                data={getSeries(dedupedYAxes)}
+                error={error}
+                loading={loading}
+                // TODO Abdullah: Make chart colors dynamic, with changing topN events count and overlay count.
+                chartColors={CHART_PALETTE[TOP_EVENTS_LIMIT - 1]}
                 type={chartType}
                 // for now, use the first y axis unit
                 aggregateOutputFormat={aggregateOutputType(dedupedYAxes[0])}

static/app/views/explore/hooks/useTopEvents.tsx

@@ -0,0 +1,35 @@
+import {useMemo} from 'react';
+
+import {dedupeArray} from 'sentry/utils/dedupeArray';
+
+import {useGroupBys} from './useGroupBys';
+import {useResultMode} from './useResultsMode';
+import {useVisualizes} from './useVisualizes';
+
+export const TOP_EVENTS_LIMIT: number = 5;
+
+export function useTopEvents(): number | undefined {
+  const [visualizes] = useVisualizes();
+  const [groupBys] = useGroupBys();
+  const [resultMode] = useResultMode();
+
+  const yAxes = useMemo(() => {
+    const deduped = dedupeArray(visualizes.flatMap(visualize => visualize.yAxes));
+    deduped.sort();
+    return deduped;
+  }, [visualizes]);
+
+  const topEvents: number | undefined = useMemo(() => {
+    if (resultMode === 'samples') {
+      return undefined;
+    }
+
+    // We only support top events for a single chart with no overlaps in aggregate mode and
+    // the data must be grouped by at least one field
+    return yAxes.length > 1 || (groupBys.length === 1 && groupBys[0] === '')
+      ? undefined
+      : TOP_EVENTS_LIMIT;
+  }, [yAxes, groupBys, resultMode]);
+
+  return topEvents;
+}

static/app/views/explore/tables/aggregatesTable.tsx

@@ -1,8 +1,10 @@
 import {Fragment, useMemo} from 'react';
+import styled from '@emotion/styled';
 
 import EmptyStateWarning from 'sentry/components/emptyStateWarning';
 import LoadingIndicator from 'sentry/components/loadingIndicator';
 import Pagination from 'sentry/components/pagination';
+import {CHART_PALETTE} from 'sentry/constants/chartPalette';
 import {IconWarning} from 'sentry/icons';
 import {t} from 'sentry/locale';
 import type {NewQuery} from 'sentry/types/organization';
@@ -29,7 +31,9 @@ import {useUserQuery} from 'sentry/views/explore/hooks/useUserQuery';
 import {useVisualizes} from 'sentry/views/explore/hooks/useVisualizes';
 import {useSpansQuery} from 'sentry/views/insights/common/queries/useSpansQuery';
 
-function formatSort(sort: Sort): string {
+import {TOP_EVENTS_LIMIT, useTopEvents} from '../hooks/useTopEvents';
+
+export function formatSort(sort: Sort): string {
   const direction = sort.kind === 'desc' ? '-' : '';
   return `${direction}${getAggregateAlias(sort.field)}`;
 }
@@ -40,6 +44,7 @@ export function AggregatesTable({}: AggregatesTableProps) {
   const location = useLocation();
   const organization = useOrganization();
   const {selection} = usePageFilters();
+  const topEvents = useTopEvents();
 
   const [dataset] = useDataset();
   const [groupBys] = useGroupBys();
@@ -111,6 +116,9 @@ export function AggregatesTable({}: AggregatesTableProps) {
                   const renderer = getFieldRenderer(field, meta.fields, false);
                   return (
                     <TableBodyCell key={j}>
+                      {topEvents && i < topEvents && j === 0 && (
+                        <TopResultsIndicator index={i} />
+                      )}
                       {renderer(row, {
                         location,
                         organization,
@@ -134,3 +142,16 @@ export function AggregatesTable({}: AggregatesTableProps) {
     </Fragment>
   );
 }
+
+const TopResultsIndicator = styled('div')<{index: number}>`
+  position: absolute;
+  left: -1px;
+  margin-top: 4.5px;
+  width: 9px;
+  height: 15px;
+  border-radius: 0 3px 3px 0;
+
+  background-color: ${p => {
+    return CHART_PALETTE[TOP_EVENTS_LIMIT - 1][p.index];
+  }};
+`;

static/app/views/explore/toolbar/index.tsx

@@ -8,7 +8,6 @@ import {useSorts} from 'sentry/views/explore/hooks/useSorts';
 import {useVisualizes} from 'sentry/views/explore/hooks/useVisualizes';
 import {ToolbarDataset} from 'sentry/views/explore/toolbar/toolbarDataset';
 import {ToolbarGroupBy} from 'sentry/views/explore/toolbar/toolbarGroupBy';
-import {ToolbarLimitTo} from 'sentry/views/explore/toolbar/toolbarLimitTo';
 import {ToolbarResults} from 'sentry/views/explore/toolbar/toolbarResults';
 import {ToolbarSortBy} from 'sentry/views/explore/toolbar/toolbarSortBy';
 import {ToolbarVisualize} from 'sentry/views/explore/toolbar/toolbarVisualize';
@@ -48,7 +47,6 @@ export function ExploreToolbar({extras}: ExploreToolbarProps) {
       <ToolbarVisualize />
       <ToolbarGroupBy disabled={resultMode !== 'aggregate'} />
       <ToolbarSortBy fields={fields} sorts={sorts} setSorts={setSorts} />
-      <ToolbarLimitTo />
     </div>
   );
 }

static/app/views/explore/toolbar/toolbarLimitTo.tsx

@@ -1,15 +0,0 @@
-import {t} from 'sentry/locale';
-
-import {ToolbarHeader, ToolbarHeading, ToolbarSection} from './styles';
-
-interface ToolbarLimitToProps {}
-
-export function ToolbarLimitTo({}: ToolbarLimitToProps) {
-  return (
-    <ToolbarSection>
-      <ToolbarHeader>
-        <ToolbarHeading>{t('Limit To')}</ToolbarHeading>
-      </ToolbarHeader>
-    </ToolbarSection>
-  );
-}

static/app/views/insights/common/queries/getSeriesEventView.tsx

@@ -15,7 +15,8 @@ export function getSeriesEventView(
   pageFilters: PageFilters,
   yAxis: string[],
   topEvents?: number,
-  dataset?: DiscoverDatasets
+  dataset?: DiscoverDatasets,
+  orderby?: string | string[]
 ) {
   // Pick the highest possible interval for the given yAxis selection. Find the ideal interval for each function, then choose the largest one. This results in the lowest granularity, but best performance.
   const interval = sortBy(
@@ -43,6 +44,7 @@ export function getSeriesEventView(
       interval,
       topEvents: topEvents?.toString(),
       version: 2,
+      orderby,
     },
     pageFilters
   );

static/app/views/insights/common/queries/useSortedTopNSeries.tsx

@@ -0,0 +1,133 @@
+import type {Series} from 'sentry/types/echarts';
+import type {EventsStats, MultiSeriesEventsStats} from 'sentry/types/organization';
+import {encodeSort} from 'sentry/utils/discover/eventView';
+import {DURATION_UNITS, SIZE_UNITS} from 'sentry/utils/discover/fieldRenderers';
+import {getAggregateAlias} from 'sentry/utils/discover/fields';
+import {
+  type DiscoverQueryProps,
+  useGenericDiscoverQuery,
+} from 'sentry/utils/discover/genericDiscoverQuery';
+import {DiscoverDatasets} from 'sentry/utils/discover/types';
+import type {MutableSearch} from 'sentry/utils/tokenizeSearch';
+import {useLocation} from 'sentry/utils/useLocation';
+import useOrganization from 'sentry/utils/useOrganization';
+import usePageFilters from 'sentry/utils/usePageFilters';
+import {getSeriesEventView} from 'sentry/views/insights/common/queries/getSeriesEventView';
+import type {SpanFunctions, SpanIndexedField} from 'sentry/views/insights/types';
+
+import {getRetryDelay, shouldRetryHandler} from '../utils/retryHandlers';
+
+interface Options<Fields> {
+  enabled?: boolean;
+  fields?: string[];
+  interval?: string;
+  orderby?: string | string[];
+  overriddenRoute?: string;
+  referrer?: string;
+  search?: MutableSearch;
+  topEvents?: number;
+  yAxis?: Fields;
+}
+
+export const useSortedTopNSeries = <
+  Fields extends SpanIndexedField[] | SpanFunctions[] | string[],
+>(
+  options: Options<Fields> = {},
+  referrer: string,
+  dataset?: DiscoverDatasets
+) => {
+  const location = useLocation();
+  const organization = useOrganization();
+  const {
+    search = undefined,
+    yAxis = [],
+    interval = undefined,
+    topEvents = 5,
+    fields,
+    orderby,
+    overriddenRoute,
+    enabled,
+  } = options;
+
+  const pageFilters = usePageFilters();
+
+  const eventView = getSeriesEventView(
+    search,
+    fields,
+    pageFilters.selection,
+    yAxis,
+    topEvents,
+    dataset ?? DiscoverDatasets.SPANS_INDEXED,
+    orderby
+  );
+
+  if (interval) {
+    eventView.interval = interval;
+  }
+
+  const result = useGenericDiscoverQuery<MultiSeriesEventsStats, DiscoverQueryProps>({
+    route: overriddenRoute ?? 'events-stats',
+    eventView,
+    location,
+    orgSlug: organization.slug,
+    getRequestPayload: () => ({
+      ...eventView.getEventsAPIPayload(location),
+      yAxis: eventView.yAxis,
+      topEvents: eventView.topEvents,
+      excludeOther: 0,
+      partial: 1,
+      orderby: eventView.sorts?.[0] ? encodeSort(eventView.sorts?.[0]) : undefined,
+      interval: eventView.interval,
+    }),
+    options: {
+      enabled: enabled && pageFilters.isReady,
+      refetchOnWindowFocus: false,
+      retry: shouldRetryHandler,
+      retryDelay: getRetryDelay,
+      staleTime: Infinity,
+    },
+    referrer,
+  });
+
+  const isFetchingOrLoading = result.isPending || result.isFetching;
+
+  const data: Series[] = isFetchingOrLoading ? [] : transformToSeries(result.data);
+
+  const pageLinks = result.response?.getResponseHeader('Link') ?? undefined;
+
+  return {
+    ...result,
+    pageLinks,
+    data,
+    meta: result.data?.meta,
+  };
+};
+
+function transformToSeries(result: MultiSeriesEventsStats | undefined): Series[] {
+  if (!result) {
+    return [];
+  }
+
+  const processedResults: [number, Series][] = Object.keys(result).map(seriesName => {
+    const seriesData: EventsStats = result[seriesName];
+
+    let scale = 1;
+    if (seriesName) {
+      const unit = seriesData.meta?.units?.[getAggregateAlias(seriesName)];
+      // Scale series values to milliseconds or bytes depending on units from meta
+      scale = (unit && (DURATION_UNITS[unit] ?? SIZE_UNITS[unit])) ?? 1;
+    }
+
+    const processsedData: Series = {
+      seriesName: seriesName || '(empty string)',
+      data: seriesData.data.map(([timestamp, countsForTimestamp]) => ({
+        name: timestamp * 1000,
+        value: countsForTimestamp.reduce((acc, {count}) => acc + count, 0) * scale,
+      })),
+    };
+
+    return [seriesData.order || 0, processsedData];
+  });
+
+  return processedResults.sort(([a], [b]) => a - b).map(([, series]) => series);
+}