feat(replay): Fix truncated JSON request/response bodies (#59266)

This implements the changes from the SDK here:
https://github.com/getsentry/sentry-javascript/pull/9437

If it encounters a request/response body with a `MAYBE_TRUNCATED_JSON`
warning, it will try to auto-fix it with the same logic as we did in the
SDK.

static/app/utils/replays/replay.tsx

@@ -2,6 +2,7 @@ type JsonObject = Record<string, unknown>;
 type JsonArray = unknown[];
 
 export type NetworkMetaWarning =
+  | 'MAYBE_JSON_TRUNCATED'
   | 'JSON_TRUNCATED'
   | 'TEXT_TRUNCATED'
   | 'INVALID_JSON'

static/app/views/replays/detail/network/details/components.tsx

@@ -20,14 +20,14 @@ const WarningText = styled('span')`
   color: ${p => p.theme.errorText};
 `;
 
-export function Warning({warnings}: {warnings: undefined | string[]}) {
-  if (warnings?.includes('JSON_TRUNCATED') || warnings?.includes('TEXT_TRUNCATED')) {
+export function Warning({warnings}: {warnings: string[]}) {
+  if (warnings.includes('JSON_TRUNCATED') || warnings.includes('TEXT_TRUNCATED')) {
     return (
       <WarningText>{t('Truncated (~~) due to exceeding 150k characters')}</WarningText>
     );
   }
 
-  if (warnings?.includes('INVALID_JSON')) {
+  if (warnings.includes('INVALID_JSON')) {
     return <WarningText>{t('Invalid JSON')}</WarningText>;
   }
 

static/app/views/replays/detail/network/details/sections.tsx

@@ -8,6 +8,10 @@ import {useReplayContext} from 'sentry/components/replays/replayContext';
 import {t} from 'sentry/locale';
 import {space} from 'sentry/styles/space';
 import {formatBytesBase10} from 'sentry/utils';
+import {
+  NetworkMetaWarning,
+  ReplayNetworkRequestOrResponse,
+} from 'sentry/utils/replays/replay';
 import {
   getFrameMethod,
   getFrameStatus,
@@ -24,6 +28,7 @@ import {
   Warning,
 } from 'sentry/views/replays/detail/network/details/components';
 import {useDismissReqRespBodiesAlert} from 'sentry/views/replays/detail/network/details/onboarding';
+import {fixJson} from 'sentry/views/replays/detail/network/truncateJson/fixJson';
 import TimestampButton from 'sentry/views/replays/detail/timestampButton';
 
 export type SectionProps = {
@@ -39,9 +44,6 @@ export function GeneralSection({item, startTimestampMs}: SectionProps) {
 
   const requestFrame = isRequestFrame(item) ? item : null;
 
-  // TODO[replay]: what about:
-  // `requestFrame?.data?.request?.size` vs. `requestFrame?.data?.requestBodySize`
-
   const data: KeyValueTuple[] = [
     {key: t('URL'), value: item.description},
     {key: t('Type'), value: item.op},
@@ -179,6 +181,8 @@ export function RequestPayloadSection({item}: SectionProps) {
   const {dismiss, isDismissed} = useDismissReqRespBodiesAlert();
 
   const data = useMemo(() => (isRequestFrame(item) ? item.data : {}), [item]);
+  const {warnings, body} = getBodyAndWarnings(data.request);
+
   useEffect(() => {
     if (!isDismissed && 'request' in data) {
       dismiss();
@@ -195,9 +199,9 @@ export function RequestPayloadSection({item}: SectionProps) {
       }
     >
       <Indent>
-        <Warning warnings={data.request?._meta?.warnings} />
+        <Warning warnings={warnings} />
         {'request' in data ? (
-          <ObjectInspector data={data.request?.body} expandLevel={2} showCopyButton />
+          <ObjectInspector data={body} expandLevel={2} showCopyButton />
         ) : (
           t('Request body not found.')
         )}
@@ -210,6 +214,8 @@ export function ResponsePayloadSection({item}: SectionProps) {
   const {dismiss, isDismissed} = useDismissReqRespBodiesAlert();
 
   const data = useMemo(() => (isRequestFrame(item) ? item.data : {}), [item]);
+  const {warnings, body} = getBodyAndWarnings(data.response);
+
   useEffect(() => {
     if (!isDismissed && 'response' in data) {
       dismiss();
@@ -226,9 +232,9 @@ export function ResponsePayloadSection({item}: SectionProps) {
       }
     >
       <Indent>
-        <Warning warnings={data?.response?._meta?.warnings} />
+        <Warning warnings={warnings} />
         {'response' in data ? (
-          <ObjectInspector data={data.response?.body} expandLevel={2} showCopyButton />
+          <ObjectInspector data={body} expandLevel={2} showCopyButton />
         ) : (
           t('Response body not found.')
         )}
@@ -236,3 +242,29 @@ export function ResponsePayloadSection({item}: SectionProps) {
     </SectionItem>
   );
 }
+
+function getBodyAndWarnings(reqOrRes?: ReplayNetworkRequestOrResponse): {
+  body: ReplayNetworkRequestOrResponse['body'];
+  warnings: NetworkMetaWarning[];
+} {
+  if (!reqOrRes) {
+    return {body: undefined, warnings: []};
+  }
+
+  const warnings = reqOrRes._meta?.warnings ?? [];
+  let body = reqOrRes.body;
+
+  if (typeof body === 'string' && warnings.includes('MAYBE_JSON_TRUNCATED')) {
+    try {
+      const json = fixJson(body);
+      body = JSON.parse(json);
+      warnings.push('JSON_TRUNCATED');
+    } catch {
+      // this can fail, in which case we just use the body string
+      warnings.push('INVALID_JSON');
+      warnings.push('TEXT_TRUNCATED');
+    }
+  }
+
+  return {body, warnings};
+}

static/app/views/replays/detail/network/truncateJson/completeJson.ts

@@ -0,0 +1,127 @@
+import type {JsonToken} from './constants';
+import {
+  ARR,
+  ARR_VAL,
+  ARR_VAL_COMPLETED,
+  ARR_VAL_STR,
+  OBJ,
+  OBJ_KEY,
+  OBJ_KEY_STR,
+  OBJ_VAL,
+  OBJ_VAL_COMPLETED,
+  OBJ_VAL_STR,
+} from './constants';
+
+const ALLOWED_PRIMITIVES = ['true', 'false', 'null'];
+
+/**
+ * Complete an incomplete JSON string.
+ * This will ensure that the last element always has a `"~~"` to indicate it was truncated.
+ * For example, `[1,2,` will be completed to `[1,2,"~~"]`
+ * and `{"aa":"b` will be completed to `{"aa":"b~~"}`
+ */
+export function completeJson(incompleteJson: string, stack: JsonToken[]): string {
+  if (!stack.length) {
+    return incompleteJson;
+  }
+
+  let json = incompleteJson;
+
+  // Most checks are only needed for the last step in the stack
+  const lastPos = stack.length - 1;
+  const lastStep = stack[lastPos];
+
+  json = _fixLastStep(json, lastStep);
+
+  // Complete remaining steps - just add closing brackets
+  for (let i = lastPos; i >= 0; i--) {
+    const step = stack[i];
+
+    // eslint-disable-next-line default-case
+    switch (step) {
+      case OBJ:
+        json = `${json}}`;
+        break;
+      case ARR:
+        json = `${json}]`;
+        break;
+    }
+  }
+
+  return json;
+}
+
+function _fixLastStep(json: string, lastStep: JsonToken): string {
+  switch (lastStep) {
+    // Object cases
+    case OBJ:
+      return `${json}"~~":"~~"`;
+    case OBJ_KEY:
+      return `${json}:"~~"`;
+    case OBJ_KEY_STR:
+      return `${json}~~":"~~"`;
+    case OBJ_VAL:
+      return _maybeFixIncompleteObjValue(json);
+    case OBJ_VAL_STR:
+      return `${json}~~"`;
+    case OBJ_VAL_COMPLETED:
+      return `${json},"~~":"~~"`;
+
+    // Array cases
+    case ARR:
+      return `${json}"~~"`;
+    case ARR_VAL:
+      return _maybeFixIncompleteArrValue(json);
+    case ARR_VAL_STR:
+      return `${json}~~"`;
+    case ARR_VAL_COMPLETED:
+      return `${json},"~~"`;
+
+    default:
+      return json;
+  }
+}
+
+function _maybeFixIncompleteArrValue(json: string): string {
+  const pos = _findLastArrayDelimiter(json);
+
+  if (pos > -1) {
+    const part = json.slice(pos + 1);
+
+    if (ALLOWED_PRIMITIVES.includes(part.trim())) {
+      return `${json},"~~"`;
+    }
+
+    // Everything else is replaced with `"~~"`
+    return `${json.slice(0, pos + 1)}"~~"`;
+  }
+
+  // fallback, this shouldn't happen, to be save
+  return json;
+}
+
+function _findLastArrayDelimiter(json: string): number {
+  for (let i = json.length - 1; i >= 0; i--) {
+    const char = json[i];
+
+    if (char === ',' || char === '[') {
+      return i;
+    }
+  }
+
+  return -1;
+}
+
+function _maybeFixIncompleteObjValue(json: string): string {
+  const startPos = json.lastIndexOf(':');
+
+  const part = json.slice(startPos + 1);
+
+  if (ALLOWED_PRIMITIVES.includes(part.trim())) {
+    return `${json},"~~":"~~"`;
+  }
+
+  // Everything else is replaced with `"~~"`
+  // This also means we do not have incomplete numbers, e.g `[1` is replaced with `["~~"]`
+  return `${json.slice(0, startPos + 1)}"~~"`;
+}

static/app/views/replays/detail/network/truncateJson/constants.ts

@@ -0,0 +1,23 @@
+export const OBJ = 10;
+export const OBJ_KEY = 11;
+export const OBJ_KEY_STR = 12;
+export const OBJ_VAL = 13;
+export const OBJ_VAL_STR = 14;
+export const OBJ_VAL_COMPLETED = 15;
+
+export const ARR = 20;
+export const ARR_VAL = 21;
+export const ARR_VAL_STR = 22;
+export const ARR_VAL_COMPLETED = 23;
+
+export type JsonToken =
+  | typeof OBJ
+  | typeof OBJ_KEY
+  | typeof OBJ_KEY_STR
+  | typeof OBJ_VAL
+  | typeof OBJ_VAL_STR
+  | typeof OBJ_VAL_COMPLETED
+  | typeof ARR
+  | typeof ARR_VAL
+  | typeof ARR_VAL_STR
+  | typeof ARR_VAL_COMPLETED;

static/app/views/replays/detail/network/truncateJson/evaluateJson.ts

@@ -0,0 +1,265 @@
+import type {JsonToken} from './constants';
+import {
+  ARR,
+  ARR_VAL,
+  ARR_VAL_COMPLETED,
+  ARR_VAL_STR,
+  OBJ,
+  OBJ_KEY,
+  OBJ_KEY_STR,
+  OBJ_VAL,
+  OBJ_VAL_COMPLETED,
+  OBJ_VAL_STR,
+} from './constants';
+
+/**
+ * Evaluate an (incomplete) JSON string.
+ */
+export function evaluateJson(json: string): JsonToken[] {
+  const stack: JsonToken[] = [];
+
+  for (let pos = 0; pos < json.length; pos++) {
+    _evaluateJsonPos(stack, json, pos);
+  }
+
+  return stack;
+}
+
+function _evaluateJsonPos(stack: JsonToken[], json: string, pos: number): void {
+  const curStep = stack[stack.length - 1];
+
+  const char = json[pos];
+
+  const whitespaceRegex = /\s/;
+
+  if (whitespaceRegex.test(char)) {
+    return;
+  }
+
+  if (char === '"' && !_isEscaped(json, pos)) {
+    _handleQuote(stack, curStep);
+    return;
+  }
+
+  // eslint-disable-next-line default-case
+  switch (char) {
+    case '{':
+      _handleObj(stack, curStep);
+      break;
+    case '[':
+      _handleArr(stack, curStep);
+      break;
+    case ':':
+      _handleColon(stack, curStep);
+      break;
+    case ',':
+      _handleComma(stack, curStep);
+      break;
+    case '}':
+      _handleObjClose(stack, curStep);
+      break;
+    case ']':
+      _handleArrClose(stack, curStep);
+      break;
+  }
+}
+
+function _handleQuote(stack: JsonToken[], curStep: JsonToken): void {
+  // End of obj value
+  if (curStep === OBJ_VAL_STR) {
+    stack.pop();
+    stack.push(OBJ_VAL_COMPLETED);
+    return;
+  }
+
+  // End of arr value
+  if (curStep === ARR_VAL_STR) {
+    stack.pop();
+    stack.push(ARR_VAL_COMPLETED);
+    return;
+  }
+
+  // Start of obj value
+  if (curStep === OBJ_VAL) {
+    stack.push(OBJ_VAL_STR);
+    return;
+  }
+
+  // Start of arr value
+  if (curStep === ARR_VAL) {
+    stack.push(ARR_VAL_STR);
+    return;
+  }
+
+  // Start of obj key
+  if (curStep === OBJ) {
+    stack.push(OBJ_KEY_STR);
+    return;
+  }
+
+  // End of obj key
+  if (curStep === OBJ_KEY_STR) {
+    stack.pop();
+    stack.push(OBJ_KEY);
+    return;
+  }
+}
+
+function _handleObj(stack: JsonToken[], curStep: JsonToken): void {
+  // Initial object
+  if (!curStep) {
+    stack.push(OBJ);
+    return;
+  }
+
+  // New object as obj value
+  if (curStep === OBJ_VAL) {
+    stack.push(OBJ);
+    return;
+  }
+
+  // New object as array element
+  if (curStep === ARR_VAL) {
+    stack.push(OBJ);
+  }
+
+  // New object as first array element
+  if (curStep === ARR) {
+    stack.push(OBJ);
+    return;
+  }
+}
+
+function _handleArr(stack: JsonToken[], curStep: JsonToken): void {
+  // Initial array
+  if (!curStep) {
+    stack.push(ARR);
+    stack.push(ARR_VAL);
+    return;
+  }
+
+  // New array as obj value
+  if (curStep === OBJ_VAL) {
+    stack.push(ARR);
+    stack.push(ARR_VAL);
+    return;
+  }
+
+  // New array as array element
+  if (curStep === ARR_VAL) {
+    stack.push(ARR);
+    stack.push(ARR_VAL);
+  }
+
+  // New array as first array element
+  if (curStep === ARR) {
+    stack.push(ARR);
+    stack.push(ARR_VAL);
+    return;
+  }
+}
+
+function _handleColon(stack: JsonToken[], curStep: JsonToken): void {
+  if (curStep === OBJ_KEY) {
+    stack.pop();
+    stack.push(OBJ_VAL);
+  }
+}
+
+function _handleComma(stack: JsonToken[], curStep: JsonToken): void {
+  // Comma after obj value
+  if (curStep === OBJ_VAL) {
+    stack.pop();
+    return;
+  }
+  if (curStep === OBJ_VAL_COMPLETED) {
+    // Pop OBJ_VAL_COMPLETED & OBJ_VAL
+    stack.pop();
+    stack.pop();
+    return;
+  }
+
+  // Comma after arr value
+  if (curStep === ARR_VAL) {
+    // do nothing - basically we'd pop ARR_VAL but add it right back
+    return;
+  }
+
+  if (curStep === ARR_VAL_COMPLETED) {
+    // Pop ARR_VAL_COMPLETED
+    stack.pop();
+
+    // basically we'd pop ARR_VAL but add it right back
+    return;
+  }
+}
+
+function _handleObjClose(stack: JsonToken[], curStep: JsonToken): void {
+  // Empty object {}
+  if (curStep === OBJ) {
+    stack.pop();
+  }
+
+  // Object with element
+  if (curStep === OBJ_VAL) {
+    // Pop OBJ_VAL, OBJ
+    stack.pop();
+    stack.pop();
+  }
+
+  // Obj with element
+  if (curStep === OBJ_VAL_COMPLETED) {
+    // Pop OBJ_VAL_COMPLETED, OBJ_VAL, OBJ
+    stack.pop();
+    stack.pop();
+    stack.pop();
+  }
+
+  // if was obj value, complete it
+  if (stack[stack.length - 1] === OBJ_VAL) {
+    stack.push(OBJ_VAL_COMPLETED);
+  }
+
+  // if was arr value, complete it
+  if (stack[stack.length - 1] === ARR_VAL) {
+    stack.push(ARR_VAL_COMPLETED);
+  }
+}
+
+function _handleArrClose(stack: JsonToken[], curStep: JsonToken): void {
+  // Empty array []
+  if (curStep === ARR) {
+    stack.pop();
+  }
+
+  // Array with element
+  if (curStep === ARR_VAL) {
+    // Pop ARR_VAL, ARR
+    stack.pop();
+    stack.pop();
+  }
+
+  // Array with element
+  if (curStep === ARR_VAL_COMPLETED) {
+    // Pop ARR_VAL_COMPLETED, ARR_VAL, ARR
+    stack.pop();
+    stack.pop();
+    stack.pop();
+  }
+
+  // if was obj value, complete it
+  if (stack[stack.length - 1] === OBJ_VAL) {
+    stack.push(OBJ_VAL_COMPLETED);
+  }
+
+  // if was arr value, complete it
+  if (stack[stack.length - 1] === ARR_VAL) {
+    stack.push(ARR_VAL_COMPLETED);
+  }
+}
+
+function _isEscaped(str: string, pos: number): boolean {
+  const previousChar = str[pos - 1];
+
+  return previousChar === '\\' && !_isEscaped(str, pos - 1);
+}

static/app/views/replays/detail/network/truncateJson/fixJson.ts

@@ -0,0 +1,12 @@
+import {completeJson} from './completeJson';
+import {evaluateJson} from './evaluateJson';
+
+/**
+ * Takes an incomplete JSON string, and returns a hopefully valid JSON string.
+ * Note that this _can_ fail, so you should check the return value is valid JSON.
+ */
+export function fixJson(incompleteJson: string): string {
+  const stack = evaluateJson(incompleteJson);
+
+  return completeJson(incompleteJson, stack);
+}