feat(charts): Add ChartRenderer service (feat Chartcuterie) (#24626)

.github/actions/setup-sentry/action.yml

@@ -9,6 +9,10 @@ inputs:
     description: 'Is kafka required?'
     required: false
     default: 'false'
+  chartcuterie:
+    description: 'Is chartcuterie required?'
+    required: false
+    default: 'false'
 
 outputs:
   yarn-cache-dir:
@@ -97,6 +101,7 @@ runs:
       env:
         NEED_KAFKA: ${{ inputs.kafka }}
         NEED_SNUBA: ${{ inputs.snuba }}
+        NEED_CHARTCUTERIE: ${{ inputs.chartcuterie }}
       run: |
         sentry init
 
@@ -131,4 +136,8 @@ runs:
           sentry devservices up clickhouse snuba
         fi
 
+        if [ "$NEED_CHARTCUTERIE" = "true" ]; then
+          sentry devservices up --skip-only-if chartcuterie
+        fi
+
         docker ps -a

.github/workflows/acceptance.yml

@@ -163,3 +163,83 @@ jobs:
 
       - name: Handle artifacts
         uses: ./.github/actions/artifacts
+
+  chartcuterie:
+    name: chartcuterie integration
+    runs-on: ubuntu-20.04
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        instance: [0]
+    env:
+      VISUAL_SNAPSHOT_ENABLE: 1
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          # Avoid codecov error message related to SHA resolution:
+          # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891
+          fetch-depth: '2'
+
+      - name: Set python version output
+        id: python-version
+        run: |
+          echo "::set-output name=python-version::$(cat .python-version)"
+
+      # Until GH composite actions can use `uses`, we need to setup python here
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ steps.python-version.outputs.python-version }}
+
+      - name: Setup pip
+        uses: ./.github/actions/setup-pip
+        id: pip
+
+      - name: pip cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.pip.outputs.pip-cache-dir }}
+          key: |
+            ${{ runner.os }}-py${{ steps.python-version.outputs.python-version }}-pip${{ steps.pip.outputs.pip-version }}-${{ secrets.PIP_CACHE_VERSION }}-${{ hashFiles('requirements-*.txt', '!requirements-pre-commit.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-py${{ steps.python-version.outputs.python-version }}-pip${{ steps.pip.outputs.pip-version }}-${{ secrets.PIP_CACHE_VERSION }}
+
+      - name: Setup sentry env
+        uses: ./.github/actions/setup-sentry
+        id: setup
+        with:
+          chartcuterie: true
+
+      - name: yarn cache
+        uses: actions/cache@v2
+        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+        with:
+          path: ${{ steps.setup.outputs.yarn-cache-dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock', 'api-docs/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Install Javascript Dependencies
+        run: |
+          yarn install --frozen-lockfile
+
+      - name: Build chartcuterie configuration module
+        run: |
+          make build-chartcuterie-config
+
+      - name: Run chartcuterie tests (${{ steps.setup.outputs.matrix-instance-number }} of ${{ strategy.job-total }})
+        run: |
+          mkdir -p ${{ steps.setup.outputs.acceptance-dir }}
+          make test-chartcuterie
+        env:
+          PYTEST_SNAPSHOTS_DIR: ${{ steps.setup.outputs.acceptance-dir }}
+
+      - name: Save snapshots
+        if: always()
+        uses: getsentry/action-visual-snapshot@v2
+        with:
+          save-only: true
+          snapshot-path: .artifacts/visual-snapshots
+
+      - name: Handle artifacts
+        uses: ./.github/actions/artifacts

Makefile

@@ -96,6 +96,10 @@ build-platform-assets:
 	@echo "--> Building platform assets"
 	@echo "from sentry.utils.integrationdocs import sync_docs; sync_docs(quiet=True)" | sentry exec
 
+build-chartcuterie-config:
+	@echo "--> Building chartcuterie config module"
+	yarn build-chartcuterie-config
+
 fetch-release-registry:
 	@echo "--> Fetching release registry"
 	@echo "from sentry.utils.distutils import sync_registry; sync_registry()" | sentry exec
@@ -158,6 +162,11 @@ test-symbolicator:
 	pytest tests/symbolicator -vv --cov . --cov-report="xml:.artifacts/symbolicator.coverage.xml" --junit-xml=".artifacts/symbolicator.junit.xml"
 	@echo ""
 
+test-chartcuterie:
+	@echo "--> Running chartcuterie tests"
+	pytest tests/chartcuterie -vv --cov . --cov-report="xml:.artifacts/chartcuterie.coverage.xml" --junit-xml=".artifacts/chartcuterie.junit.xml"
+	@echo ""
+
 test-acceptance: node-version-check
 	@echo "--> Building static assets"
 	@$(WEBPACK)
@@ -222,6 +231,7 @@ lint-js:
         sync-transifex \
         update-transifex \
         build-platform-assets \
+        build-chartcuterie-config \
         fetch-release-registry \
         run-acceptance \
         test-cli \
@@ -232,6 +242,7 @@ lint-js:
         test-python-ci \
         test-snuba \
         test-symbolicator \
+        test-chartcuterie \
         test-acceptance \
         test-plugins \
         test-relay-integration \

src/sentry/charts/__init__.py

@@ -0,0 +1,18 @@
+from django.conf import settings
+
+from sentry.utils.services import LazyServiceWrapper
+
+from .base import ChartRenderer  # NOQA
+
+# The charts module provides a service to interface with the external
+# Chartcuterie service, which produces charts as images.
+#
+# This module will handle producing and storing the images given some data that
+# you would like to represent as a chart outside of the frontend application.
+
+backend = LazyServiceWrapper(
+    ChartRenderer,
+    settings.SENTRY_CHART_RENDERER,
+    settings.SENTRY_CHART_RENDERER_OPTIONS,
+)
+backend.expose(locals())

src/sentry/charts/base.py

@@ -0,0 +1,38 @@
+import logging
+
+from typing import Any, Union
+
+from sentry import options
+from sentry.utils.services import Service
+
+from .types import ChartType
+
+logger = logging.getLogger("sentry.charts")
+
+
+class ChartRenderer(Service):
+    """
+    The chart rendering service is used to translate arbitrary data into a
+    image representation of that data, usually a chart.
+    """
+
+    __all__ = (
+        "is_enabled",
+        "generate_chart",
+    )
+
+    def __init__(self, **options):
+        pass
+
+    def is_enabled(self):
+        """
+        Checks that the chart rendering service is enabled
+        """
+        return options.get("chart-rendering.enabled", False)
+
+    def generate_chart(self, style: ChartType, data: Any, upload: bool = True) -> Union[str, bytes]:
+        """
+        Produces a chart. You may specify the upload kwarg to have the chart
+        uploaded to storage and recieve a public URL for the chart
+        """
+        raise NotImplementedError

src/sentry/charts/chartcuterie.py

@@ -0,0 +1,81 @@
+from io import BytesIO
+from typing import Any, Union
+from urllib.parse import urljoin
+from uuid import uuid4
+
+import sentry_sdk
+from django.conf import settings
+
+from sentry import options
+from sentry.exceptions import InvalidConfiguration
+from sentry.models.file import get_storage
+from sentry.net.http import Session
+from sentry.utils.http import absolute_uri
+
+from .base import ChartRenderer, logger
+from .types import ChartType
+
+
+class Chartcuterie(ChartRenderer):
+    """
+    The Chartcuterie service is responsible for converting series data into a
+    chart of the data as an image.
+
+    This uses the external Chartcuterie API to produce charts
+    """
+
+    @property
+    def service_url(self) -> str:
+        return options.get("chart-rendering.chartcuterie", {}).get("url")
+
+    def validate(self):
+        if not self.is_enabled():
+            return
+
+        if not self.service_url:
+            raise InvalidConfiguration("`chart-rendering.chartcuterie.url` is not configured")
+
+    def generate_chart(self, style: ChartType, data: Any, upload: bool = True) -> Union[str, bytes]:
+        request_id = uuid4().hex
+
+        data = {
+            "requestId": request_id,
+            "style": style.value,
+            "data": data,
+        }
+
+        with Session() as session:
+            with sentry_sdk.start_span(
+                op="charts.chartcuterie.generate_chart",
+                description=type(self).__name__,
+            ):
+                resp = session.request(
+                    method="POST",
+                    url=urljoin(self.service_url, "render"),
+                    json=data,
+                )
+
+                if resp.status_code == 503 and settings.DEBUG:
+                    logger.info(
+                        "You may need to build the chartcuterie config using `yarn build-chartcuterie-config`"
+                    )
+
+                if resp.status_code != 200:
+                    raise RuntimeError(
+                        f"Chartcuterie responded with {resp.status_code}: {resp.text}"
+                    )
+
+        if not upload:
+            return resp.content
+
+        file_name = f"{request_id}.png"
+
+        with sentry_sdk.start_span(
+            op="charts.chartcuterie.upload",
+            description=type(self).__name__,
+        ):
+            storage = get_storage()
+            storage.save(file_name, BytesIO(resp.content))
+            url = absolute_uri(storage.url(file_name))
+
+        return url

src/sentry/charts/types.py

@@ -0,0 +1,14 @@
+from enum import Enum
+
+
+class ChartType(Enum):
+    """
+    This enum defines the chart styles we can render.
+
+    This directly maps to the chartcuterie configuration [0] in the frontend
+    code. Be sure to keep these in sync when adding or removing types.
+
+    [0]: app/chartcuterieConfig.tsx.
+    """
+
+    SLACK_DISCOVER_TOTAL_PERIOD = "slack:discover.totalPeriod"

src/sentry/conf/server.py

@@ -1261,6 +1261,10 @@ SENTRY_METRICS_SAMPLE_RATE = 1.0
 SENTRY_METRICS_PREFIX = "sentry."
 SENTRY_METRICS_SKIP_INTERNAL_PREFIXES = []  # Order this by most frequent prefixes.
 
+# Render charts on the backend. This uses the Chartcuterie external service.
+SENTRY_CHART_RENDERER = "sentry.charts.chartcuterie.Chartcuterie"
+SENTRY_CHART_RENDERER_OPTIONS = {}
+
 # URI Prefixes for generating DSN URLs
 # (Defaults to URL_PREFIX by default)
 SENTRY_ENDPOINT = None

src/sentry/static/sentry/app/chartcuterieConfig.tsx

@@ -25,6 +25,9 @@ import {lightTheme as theme} from 'app/utils/theme';
 /**
  * Defines the keys which may be passed into the chartcuterie chart rendering
  * service.
+ *
+ * When adding or removing from this list, please also update the
+ * sentry/charts/types.py file
  */
 export enum ChartType {
   SLACK_DISCOVER_TOTAL_PERIOD = 'slack:discover.totalPeriod',