Browse Source

chore(logging): logs for investigating projects tab assertion error (#63459)

Adding logs to further investigate the flakey assertion error
(SENTRY-2CH3) on some organizations' projects tab.

`totals` in `release_health.py` is sometimes being found as `None`
causing loading errors on project tab; these logs are trying to pinpoint
where that is happening.
Isabella Enriquez 1 year ago
parent
commit
e715887dce
2 changed files with 35 additions and 2 deletions
  1. 15 1
      src/sentry/release_health/metrics.py
  2. 20 1
      src/sentry/utils/safe.py

+ 15 - 1
src/sentry/release_health/metrics.py

@@ -58,6 +58,7 @@ from sentry.snuba.metrics import (
 from sentry.snuba.metrics.naming_layer.mri import SessionMRI
 from sentry.snuba.sessions import _make_stats, get_rollup_starts_and_buckets
 from sentry.snuba.sessions_v2 import QueryDefinition
+from sentry.utils import json
 from sentry.utils.dates import to_datetime, to_timestamp
 from sentry.utils.safe import get_path
 from sentry.utils.snuba import QueryOutsideRetentionError
@@ -169,7 +170,20 @@ class MetricsReleaseHealthBackend(ReleaseHealthBackend):
         for group in groups:
             project_id = get_path(group, "by", "project_id")
             assert project_id is not None
-            totals = get_path(group, "totals", "rate")
+            totals = get_path(group, "totals", "rate", should_log=True)
+            try:
+                if totals is None:
+                    logger.info(
+                        "sentry.release_health.metrics._get_crash_free_rate_data.totals_is_none",
+                        extra={
+                            "group": json.dumps(group),
+                            "project_id": json.dumps(project_id),
+                            "organization_id": org_id,
+                            "timeseries_for_query": json.dumps(result),
+                        },
+                    )
+            except Exception as e:
+                logger.exception("Unable to log; %s", e)
             assert totals is not None
             ret_val[project_id] = totals * 100
 

+ 20 - 1
src/sentry/utils/safe.py

@@ -104,7 +104,7 @@ def trim(
     return object_hook(result)
 
 
-def get_path(data: PathSearchable, *path, **kwargs):
+def get_path(data: PathSearchable, *path, should_log=False, **kwargs):
     """
     Safely resolves data from a recursive data structure. A value is only
     returned if the full path exists, otherwise ``None`` is returned.
@@ -115,21 +115,40 @@ def get_path(data: PathSearchable, *path, **kwargs):
     filtered with the given callback. Alternatively, pass ``True`` as filter to
     only filter ``None`` values.
     """
+    logger = logging.getLogger(__name__)
     default = kwargs.pop("default", None)
     f: Optional[bool] = kwargs.pop("filter", None)
     for k in kwargs:
         raise TypeError("get_path() got an undefined keyword argument '%s'" % k)
 
+    logger_data = {}
+    if should_log:
+        logger_data = {
+            "path_searchable": json.dumps(data),
+            "path_arg": json.dumps(path),
+        }
+
     for p in path:
         if isinstance(data, Mapping) and p in data:
             data = data[p]
         elif isinstance(data, (list, tuple)) and isinstance(p, int) and -len(data) <= p < len(data):
             data = data[p]
         else:
+            if should_log:
+                logger_data["invalid_path"] = json.dumps(p)
+                logger.info("sentry.safe.get_path.invalid_path_section", extra=logger_data)
             return default
 
+    if should_log:
+        if data is None:
+            logger.info("sentry.safe.get_path.iterated_path_is_none", extra=logger_data)
+        else:
+            logger_data["iterated_path"] = json.dumps(data)
+
     if f and data and isinstance(data, (list, tuple)):
         data = list(filter((lambda x: x is not None) if f is True else f, data))
+        if should_log and len(data) == 0 and "iterated_path" in logger_data:
+            logger.info("sentry.safe.get_path.filtered_path_is_none", extra=logger_data)
 
     return data if data is not None else default