feat(ai-monitoring): Allow creating AI-metric alerts & span alerts in general (#70979)

<img width="1423" alt="Screenshot 2024-05-15 at 4 48 01 PM"
src="https://github.com/getsentry/sentry/assets/161344340/a74d93f5-10cc-4f48-a6b8-4b4ffe3a32b8">

static/app/views/alerts/rules/metric/ruleConditionsForm.tsx

@@ -90,6 +90,7 @@ type Props = {
   disableProjectSelector?: boolean;
   isErrorMigration?: boolean;
   isExtrapolatedChartData?: boolean;
+  isForSpanMetric?: boolean;
   isTransactionMigration?: boolean;
   loadingProjects?: boolean;
   monitorType?: number;
@@ -358,6 +359,7 @@ class RuleConditionsForm extends PureComponent<Props, State> {
       onTimeWindowChange,
       project,
       monitorType,
+      isForSpanMetric,
     } = this.props;
 
     return (
@@ -368,22 +370,25 @@ class RuleConditionsForm extends PureComponent<Props, State> {
           </StyledListTitle>
         </StyledListItem>
         <FormRow>
-          <WizardField
-            name="aggregate"
-            help={null}
-            organization={organization}
-            disabled={disabled}
-            project={project}
-            style={{
-              ...this.formElemBaseStyle,
-              flex: 1,
-            }}
-            inline={false}
-            flexibleControlStateSize
-            columnWidth={200}
-            alertType={alertType}
-            required
-          />
+          {isForSpanMetric ? null : (
+            <WizardField
+              name="aggregate"
+              help={null}
+              organization={organization}
+              disabled={disabled}
+              project={project}
+              style={{
+                ...this.formElemBaseStyle,
+                flex: 1,
+              }}
+              inline={false}
+              flexibleControlStateSize
+              columnWidth={200}
+              alertType={alertType}
+              required
+            />
+          )}
+
           {monitorType !== MonitorType.ACTIVATED && (
             <SelectControl
               name="timeWindow"

static/app/views/alerts/rules/metric/ruleForm.tsx

@@ -1161,6 +1161,7 @@ class RuleFormContainer extends DeprecatedAsyncComponent<Props, State> {
               organization={organization}
               isTransactionMigration={isMigration && !showErrorMigrationWarning}
               isErrorMigration={showErrorMigrationWarning}
+              isForSpanMetric={aggregate.includes(':spans/')}
               router={router}
               disabled={formDisabled}
               thresholdChart={wizardBuilderChart}

static/app/views/alerts/wizard/options.tsx

@@ -36,7 +36,9 @@ export type AlertType =
   | 'crash_free_sessions'
   | 'crash_free_users'
   | 'custom_transactions'
-  | 'custom_metrics';
+  | 'custom_metrics'
+  | 'llm_tokens'
+  | 'llm_cost';
 
 export enum MEPAlertsQueryType {
   ERROR = 0,
@@ -75,26 +77,28 @@ export const AlertWizardAlertNames: Record<AlertType, string> = {
   custom_transactions: t('Custom Measurement'),
   crash_free_sessions: t('Crash Free Session Rate'),
   crash_free_users: t('Crash Free User Rate'),
+  llm_cost: t('LLM cost'),
+  llm_tokens: t('LLM token usage'),
 };
 
 type AlertWizardCategory = {
   categoryHeading: string;
   options: AlertType[];
 };
-export const getAlertWizardCategories = (org: Organization): AlertWizardCategory[] => [
-  {
-    categoryHeading: t('Errors'),
-    options: ['issues', 'num_errors', 'users_experiencing_errors'],
-  },
-  ...(org.features.includes('crash-rate-alerts')
-    ? [
-        {
-          categoryHeading: t('Sessions'),
-          options: ['crash_free_sessions', 'crash_free_users'] satisfies AlertType[],
-        },
-      ]
-    : []),
-  {
+export const getAlertWizardCategories = (org: Organization) => {
+  const result: AlertWizardCategory[] = [
+    {
+      categoryHeading: t('Errors'),
+      options: ['issues', 'num_errors', 'users_experiencing_errors'],
+    },
+  ];
+  if (org.features.includes('crash-rate-alerts')) {
+    result.push({
+      categoryHeading: t('Sessions'),
+      options: ['crash_free_sessions', 'crash_free_users'] satisfies AlertType[],
+    });
+  }
+  result.push({
     categoryHeading: t('Performance'),
     options: [
       'throughput',
@@ -106,12 +110,19 @@ export const getAlertWizardCategories = (org: Organization): AlertWizardCategory
       'cls',
       ...(hasCustomMetrics(org) ? (['custom_transactions'] satisfies AlertType[]) : []),
     ],
-  },
-  {
+  });
+  if (org.features.includes('ai-analytics')) {
+    result.push({
+      categoryHeading: t('LLM Monitoring'),
+      options: ['llm_tokens', 'llm_cost'],
+    });
+  }
+  result.push({
     categoryHeading: hasCustomMetrics(org) ? t('Metrics') : t('Custom'),
     options: [hasCustomMetrics(org) ? 'custom_metrics' : 'custom_transactions'],
-  },
-];
+  });
+  return result;
+};
 
 export type WizardRuleTemplate = {
   aggregate: string;
@@ -179,6 +190,16 @@ export const AlertWizardRuleTemplates: Record<
     dataset: Dataset.GENERIC_METRICS,
     eventTypes: EventTypes.TRANSACTION,
   },
+  llm_tokens: {
+    aggregate: 'sum(c:spans/ai.total_tokens.used@none)',
+    dataset: Dataset.GENERIC_METRICS,
+    eventTypes: EventTypes.TRANSACTION,
+  },
+  llm_cost: {
+    aggregate: 'sum(c:spans/ai.total_cost@usd)',
+    dataset: Dataset.GENERIC_METRICS,
+    eventTypes: EventTypes.TRANSACTION,
+  },
   crash_free_sessions: {
     aggregate: SessionsAggregate.CRASH_FREE_SESSIONS,
     // TODO(scttcper): Use Dataset.Metric on GA of alert-crash-free-metrics

static/app/views/alerts/wizard/panelContent.tsx

@@ -142,6 +142,20 @@ export const AlertWizardPanelContent: Record<AlertType, PanelContent> = {
     ],
     illustration: diagramCustomMetrics,
   },
+  llm_tokens: {
+    description: t(
+      'Receive an alert when the total number of tokens used by your LLMs reaches a limit.'
+    ),
+    examples: [t('When there are more than 100,000 tokens used within an hour')],
+    illustration: diagramCustomMetrics,
+  },
+  llm_cost: {
+    description: t(
+      'Receive an alert when the total cost of tokens used by your LLMs reaches a limit.'
+    ),
+    examples: [t('When there are more than $100 used by LLM  within an hour')],
+    illustration: diagramCustomMetrics,
+  },
   crash_free_sessions: {
     description: t(
       'A session begins when a user starts the application and ends when it’s closed or sent to the background. A crash is when a session ends due to an error and this type of alert lets you monitor when those crashed sessions exceed a threshold. This lets you get a better picture of the health of your app.'

static/app/views/alerts/wizard/utils.tsx

@@ -48,7 +48,7 @@ export function getAlertTypeFromAggregateDataset({
 }: Pick<WizardRuleTemplate, 'aggregate' | 'dataset'>): MetricAlertType {
   const {mri: mri} = parseField(aggregate) ?? {};
 
-  if (getUseCaseFromMRI(mri) === 'custom') {
+  if (getUseCaseFromMRI(mri) === 'custom' || getUseCaseFromMRI(mri) === 'spans') {
     return 'custom_metrics';
   }
   const identifierForDataset = alertTypeIdentifiers[dataset];