feat(security): move GitHub Secret Scanning from getsentry (#78386)

Merely moving the code from
https://github.com/getsentry/getsentry/pull/14624 (and follow-up fixes)
to the sentry repo.

src/sentry/api/endpoints/secret_scanning/github.py

@@ -0,0 +1,176 @@
+import hashlib
+import logging
+
+import sentry_sdk
+from django.http import HttpResponse
+from django.utils import timezone
+from django.utils.decorators import method_decorator
+from django.views.decorators.csrf import csrf_exempt
+from django.views.generic.base import View
+
+from sentry import options
+from sentry.hybridcloud.models import ApiTokenReplica, OrgAuthTokenReplica
+from sentry.models.apitoken import ApiToken
+from sentry.models.orgauthtoken import OrgAuthToken
+from sentry.organizations.absolute_url import generate_organization_url
+from sentry.organizations.services.organization import organization_service
+from sentry.types.token import AuthTokenType
+from sentry.users.models.user import User
+from sentry.utils import json, metrics
+from sentry.utils.email import MessageBuilder
+from sentry.utils.github import verify_signature
+from sentry.utils.http import absolute_uri
+from sentry.web.frontend.base import control_silo_view
+
+logger = logging.getLogger(__name__)
+
+TOKEN_TYPE_HUMAN_READABLE = {
+    AuthTokenType.USER: "User Auth Token",
+    AuthTokenType.ORG: "Organization Auth Token",
+}
+
+REVOKE_URLS = {
+    AuthTokenType.USER: "/settings/account/api/auth-tokens/",
+    AuthTokenType.ORG: "/settings/auth-tokens/",
+}
+
+
+@control_silo_view
+class SecretScanningGitHubEndpoint(View):
+    @method_decorator(csrf_exempt)
+    def dispatch(self, request, *args, **kwargs):
+        if request.method != "POST":
+            return HttpResponse(status=405)
+
+        response = super().dispatch(request, *args, **kwargs)
+        metrics.incr(
+            "secret-scanning.github.webhooks",
+            1,
+            tags={"status": response.status_code},
+            skip_internal=False,
+        )
+        return response
+
+    def post(self, request):
+        if request.headers.get("Content-Type") != "application/json":
+            return HttpResponse(
+                json.dumps({"details": "invalid content type specified"}), status=400
+            )
+
+        payload = request.body.decode("utf-8")
+        signature = request.headers.get("Github-Public-Key-Signature")
+        key_id = request.headers.get("Github-Public-Key-Identifier")
+
+        try:
+            if options.get("secret-scanning.github.enable-signature-verification"):
+                verify_signature(
+                    payload,
+                    signature,
+                    key_id,
+                    "secret_scanning",
+                )
+        except ValueError as e:
+            sentry_sdk.capture_exception(e)
+            return HttpResponse(json.dumps({"details": "invalid signature"}), status=400)
+
+        secret_alerts = json.loads(payload)
+        response = []
+        for secret_alert in secret_alerts:
+            alerted_token_str = secret_alert["token"]
+            hashed_alerted_token = hashlib.sha256(alerted_token_str.encode()).hexdigest()
+
+            # no prefix tokens could indicate old user auth tokens with no prefixes
+            token_type = AuthTokenType.USER
+            if alerted_token_str.startswith(AuthTokenType.ORG):
+                token_type = AuthTokenType.ORG
+            elif alerted_token_str.startswith((AuthTokenType.USER_APP, AuthTokenType.INTEGRATION)):
+                # TODO: add support for other token types
+                return HttpResponse(
+                    json.dumps({"details": "auth token type is not implemented"}), status=501
+                )
+
+            try:
+                token: ApiToken | OrgAuthToken
+
+                if token_type == AuthTokenType.USER:
+                    token = ApiToken.objects.get(hashed_token=hashed_alerted_token)
+
+                if token_type == AuthTokenType.ORG:
+                    token = OrgAuthToken.objects.get(
+                        token_hashed=hashed_alerted_token, date_deactivated=None
+                    )
+
+                extra = {
+                    "exposed_source": secret_alert["source"],
+                    "exposed_url": secret_alert["url"],
+                    "hashed_token": hashed_alerted_token,
+                    "token_type": token_type,
+                }
+                logger.info("found an exposed auth token", extra=extra)
+
+                # TODO: mark an API token as exposed in the database
+
+                # TODO: expose this option in the UI
+                revoke_action_enabled = False
+                if revoke_action_enabled:
+                    # TODO: revoke token
+                    pass
+
+                # Send an email
+                url_prefix = options.get("system.url-prefix")
+                if isinstance(token, ApiToken):
+                    # for user token, send an alert to the token owner
+                    users = User.objects.filter(id=token.user_id)
+                elif isinstance(token, OrgAuthToken):
+                    # for org token, send an alert to all organization owners
+                    organization = organization_service.get(id=token.organization_id)
+                    if organization is None:
+                        continue
+
+                    owner_members = organization_service.get_organization_owner_members(
+                        organization_id=organization.id
+                    )
+                    user_ids = [om.user_id for om in owner_members]
+                    users = User.objects.filter(id__in=user_ids)
+
+                    url_prefix = generate_organization_url(organization.slug)
+
+                token_type_human_readable = TOKEN_TYPE_HUMAN_READABLE.get(token_type, "Auth Token")
+
+                revoke_url = absolute_uri(REVOKE_URLS.get(token_type, "/"), url_prefix=url_prefix)
+
+                context = {
+                    "datetime": timezone.now(),
+                    "token_name": token.name,
+                    "token_type": token_type_human_readable,
+                    "token_redacted": f"{token_type}...{token.token_last_characters}",
+                    "hashed_token": hashed_alerted_token,
+                    "exposed_source": secret_alert["source"],
+                    "exposed_url": secret_alert["url"],
+                    "revoke_url": revoke_url,
+                }
+
+                subject = f"Action Required: {token_type_human_readable} Exposed"
+                msg = MessageBuilder(
+                    subject="{}{}".format(options.get("mail.subject-prefix"), subject),
+                    template="sentry/emails/secret-scanning/body.txt",
+                    html_template="sentry/emails/secret-scanning/body.html",
+                    type="user.secret-scanning-alert",
+                    context=context,
+                )
+                msg.send_async([u.username for u in users])
+            except (
+                ApiToken.DoesNotExist,
+                ApiTokenReplica.DoesNotExist,
+                OrgAuthToken.DoesNotExist,
+                OrgAuthTokenReplica.DoesNotExist,
+            ):
+                response.append(
+                    {
+                        "token_hash": hashed_alerted_token,
+                        "token_type": secret_alert["type"],
+                        "label": "false_positive",
+                    }
+                )
+
+        return HttpResponse(json.dumps(response), status=200)

src/sentry/api/urls.py

@@ -55,6 +55,7 @@ from sentry.api.endpoints.relocations.public_key import RelocationPublicKeyEndpo
 from sentry.api.endpoints.relocations.recover import RelocationRecoverEndpoint
 from sentry.api.endpoints.relocations.retry import RelocationRetryEndpoint
 from sentry.api.endpoints.relocations.unpause import RelocationUnpauseEndpoint
+from sentry.api.endpoints.secret_scanning.github import SecretScanningGitHubEndpoint
 from sentry.api.endpoints.seer_rpc import SeerRpcServiceEndpoint
 from sentry.api.endpoints.source_map_debug_blue_thunder_edition import (
     SourceMapDebugBlueThunderEditionEndpoint,
@@ -3320,6 +3321,12 @@ urlpatterns = [
         RelocationPublicKeyEndpoint.as_view(),
         name="sentry-api-0-relocations-public-key",
     ),
+    # Secret Scanning
+    re_path(
+        r"^secret-scanning/github/$",
+        SecretScanningGitHubEndpoint.as_view(),
+        name="sentry-api-0-secret-scanning-github",
+    ),
     # Catch all
     re_path(
         r"^$",

src/sentry/options/defaults.py

@@ -2747,3 +2747,11 @@ register(
     default=False,
     flags=FLAG_AUTOMATOR_MODIFIABLE,
 )
+
+# Secret Scanning. Allows to temporarily disable signature verification.
+register(
+    "secret-scanning.github.enable-signature-verification",
+    type=Bool,
+    default=True,
+    flags=FLAG_AUTOMATOR_MODIFIABLE,
+)

src/sentry/templates/sentry/emails/secret-scanning/body.html

@@ -0,0 +1,17 @@
+{% extends "sentry/emails/base.html" %}
+
+{% load i18n %}
+
+{% block main %}
+    <h3>{{ token_type }} exposed</h3>
+    <p>Your Sentry {{ token_type }} was found publicly on the internet. We recommend <a href="{{ revoke_url }}">revoking</a> this token immediately, as exposed tokens pose a security risk to your account.</p>
+    <p><pre style="font-size: 12px">
+Name:   {{ token_name }}
+Token:  {{ token_redacted }}
+SHA256: {{ hashed_token }}
+
+Source: {{ exposed_source }}
+URL:    {{ exposed_url }}
+Date:   {{ datetime|date:"N j, Y, P e" }}</pre></p>
+    <p>Read more about <a href="https://docs.sentry.io/account/auth-tokens/">Sentry Auth Tokens</a>.</p>
+{% endblock %}

src/sentry/templates/sentry/emails/secret-scanning/body.txt

@@ -0,0 +1,15 @@
+{{ token_type }} exposed
+
+Your Sentry {{ token_type }} was found publicly on the internet. We recommend revoking this token immediately, as exposed tokens pose a security risk to your account:
+{{ revoke_url }}
+
+Name:   {{ token_name }}
+Token:  {{ token_redacted }}
+SHA256: {{ hashed_token }}
+
+Source: {{ exposed_source }}
+URL:    {{ exposed_url }}
+Date:   {{ datetime|date:"N j, Y, P e" }}
+
+Read more about Sentry Auth Tokens:
+https://docs.sentry.io/account/auth-tokens/

src/sentry/utils/github.py

@@ -0,0 +1,45 @@
+import base64
+import binascii
+from typing import Any
+
+from cryptography.exceptions import InvalidSignature
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import ec
+from pydantic import BaseModel
+
+from sentry import options
+
+from .github_client import GitHubClient
+
+
+class GitHubKeysPayload(BaseModel):
+    public_keys: list[dict[str, Any]]
+
+
+def verify_signature(payload: str, signature: str, key_id: str, subpath: str) -> None:
+    if not payload or not signature or not key_id:
+        raise ValueError("Invalid payload, signature, or key_id")
+
+    client_id = options.get("github-login.client-id")
+    client_secret = options.get("github-login.client-secret")
+    client = GitHubClient(client_id=client_id, client_secret=client_secret)
+    response = client.get(f"/meta/public_keys/{subpath}")
+    keys = GitHubKeysPayload.parse_obj(response)
+
+    public_key = next((k for k in keys.public_keys if k["key_identifier"] == key_id), None)
+    if not public_key:
+        raise ValueError("No public key found matching key identifier")
+
+    key = serialization.load_pem_public_key(public_key["key"].encode())
+
+    if not isinstance(key, ec.EllipticCurvePublicKey):
+        raise ValueError("Invalid public key type")
+
+    try:
+        # Decode the base64 signature to bytes
+        signature_bytes = base64.b64decode(signature)
+        key.verify(signature_bytes, payload.encode(), ec.ECDSA(hashes.SHA256()))
+    except InvalidSignature:
+        raise ValueError("Signature does not match payload")
+    except binascii.Error:
+        raise ValueError("Invalid signature encoding")

src/sentry/utils/github_client.py

@@ -0,0 +1,80 @@
+from requests.exceptions import HTTPError
+
+from sentry.http import build_session
+from sentry.utils import json
+
+
+class ApiError(Exception):
+    code = None
+    json = None
+    xml = None
+
+    def __init__(self, text, code=None):
+        if code is not None:
+            self.code = code
+        self.text = text
+        # TODO(dcramer): pull in XML support from Jira
+        if text:
+            try:
+                self.json = json.loads(text)
+            except (json.JSONDecodeError, ValueError):
+                self.json = None
+        else:
+            self.json = None
+        super().__init__(text[:128])
+
+    @classmethod
+    def from_response(cls, response):
+        if response.status_code == 401:
+            return ApiUnauthorized(response.text)
+        return cls(response.text, response.status_code)
+
+
+class ApiUnauthorized(ApiError):
+    code = 401
+
+
+class GitHubClient:
+    ApiError = ApiError
+
+    url = "https://api.github.com"
+
+    def __init__(self, url=None, token=None, client_id=None, client_secret=None):
+        if url is not None:
+            self.url = url.rstrip("/")
+        self.token = token
+        self.client_id = client_id
+        self.client_secret = client_secret
+
+    def _request(self, method, path, headers=None, data=None, params=None, auth=None):
+        with build_session() as session:
+            try:
+                resp = getattr(session, method.lower())(
+                    url=f"{self.url}{path}",
+                    headers=headers,
+                    json=data,
+                    params=params,
+                    allow_redirects=True,
+                    auth=auth,
+                )
+                resp.raise_for_status()
+            except HTTPError as e:
+                raise ApiError.from_response(e.response)
+        return resp.json()
+
+    def request(self, method, path, data=None, params=None, auth=None):
+        headers = {"Accept": "application/vnd.github.valkyrie-preview+json"}
+
+        if self.token:
+            headers.setdefault("Authorization", f"token {self.token}")
+
+        elif auth is None and self.client_id and self.client_secret:
+            auth = (self.client_id, self.client_secret)
+
+        return self._request(method, path, headers=headers, data=data, params=params, auth=auth)
+
+    def get(self, *args, **kwargs):
+        return self.request("GET", *args, **kwargs)
+
+    def post(self, *args, **kwargs):
+        return self.request("POST", *args, **kwargs)

static/app/data/controlsiloUrlPatterns.ts

@@ -136,6 +136,7 @@ const patterns: RegExp[] = [
   new RegExp('^api/0/internal/integration-proxy/$'),
   new RegExp('^api/0/internal/rpc/[^/]+/[^/]+/$'),
   new RegExp('^api/0/internal/feature-flags/$'),
+  new RegExp('^api/0/secret-scanning/github/$'),
   new RegExp('^api/hooks/mailgun/inbound/'),
   new RegExp('^oauth/authorize/$'),
   new RegExp('^oauth/token/$'),

tests/sentry/api/endpoints/secret_scanning/test_github.py

@@ -0,0 +1,196 @@
+from unittest.mock import patch
+
+from django.core import mail
+from django.urls import reverse
+from django.utils import timezone
+
+from sentry.models.apitoken import ApiToken
+from sentry.models.orgauthtoken import OrgAuthToken
+from sentry.testutils.cases import TestCase
+from sentry.testutils.helpers import override_options
+from sentry.testutils.silo import control_silo_test
+from sentry.types.token import AuthTokenType
+from sentry.utils import json
+from sentry.utils.security.orgauthtoken_token import generate_token, hash_token
+
+
+@control_silo_test
+class SecretScanningGitHubTest(TestCase):
+    path = reverse("sentry-api-0-secret-scanning-github")
+
+    def test_invalid_content_type(self):
+        response = self.client.post(self.path, content_type="application/x-www-form-urlencoded")
+        assert response.status_code == 400
+        assert response.content == b'{"details":"invalid content type specified"}'
+
+    def test_invalid_signature(self):
+        response = self.client.post(self.path, content_type="application/json")
+        assert response.status_code == 400
+        assert response.content == b'{"details":"invalid signature"}'
+
+    @override_options({"secret-scanning.github.enable-signature-verification": False})
+    def test_false_positive(self):
+        payload = [
+            {
+                "source": "commit",
+                "token": "some_token",
+                "type": "some_type",
+                "url": "https://example.com/base-repo-url/",
+            }
+        ]
+        response = self.client.post(self.path, content_type="application/json", data=payload)
+        assert response.status_code == 200
+        assert (
+            response.content
+            == b'[{"token_hash":"9a45520a1213f15016d2d768b5fb3d904492a44ee274b44d4de8803e00fb536a","token_type":"some_type","label":"false_positive"}]'
+        )
+
+    @override_options({"secret-scanning.github.enable-signature-verification": False})
+    def test_false_positive_deactivated_user_token(self):
+        user = self.create_user()
+        token = ApiToken.objects.create(user=user, name="test user token", scope_list=[])
+
+        # revoke token
+        token.delete()
+
+        payload = [
+            {
+                "source": "commit",
+                "token": str(token),
+                "type": "sentry_user_auth_token",
+                "url": "https://example.com/base-repo-url/",
+            }
+        ]
+
+        with self.tasks():
+            response = self.client.post(self.path, content_type="application/json", data=payload)
+        assert response.status_code == 200
+        expected = [
+            {
+                "token_hash": hash_token(str(token)),
+                "token_type": "sentry_user_auth_token",
+                "label": "false_positive",
+            }
+        ]
+        assert json.loads(response.content.decode("utf-8")) == expected
+
+        assert len(mail.outbox) == 0
+
+    @override_options({"secret-scanning.github.enable-signature-verification": False})
+    def test_false_positive_deactivated_org_token(self):
+        token_str = generate_token("test-org", "https://test-region.sentry.io")
+        hash_digest = hash_token(token_str)
+        token = OrgAuthToken.objects.create(
+            organization_id=self.organization.id,
+            name="test org token",
+            scope_list=["org:ci"],
+            token_hashed=hash_digest,
+        )
+
+        # revoke token
+        token.update(date_deactivated=timezone.now())
+
+        payload = [
+            {
+                "source": "commit",
+                "token": token_str,
+                "type": "sentry_org_auth_token",
+                "url": "https://example.com/base-repo-url/",
+            }
+        ]
+
+        with self.tasks():
+            response = self.client.post(self.path, content_type="application/json", data=payload)
+        assert response.status_code == 200
+        expected = [
+            {
+                "token_hash": hash_digest,
+                "token_type": "sentry_org_auth_token",
+                "label": "false_positive",
+            }
+        ]
+        assert json.loads(response.content.decode("utf-8")) == expected
+
+        assert len(mail.outbox) == 0
+
+    @override_options({"secret-scanning.github.enable-signature-verification": False})
+    @patch("sentry.api.endpoints.secret_scanning.github.logger")
+    def test_true_positive_user_token(self, mock_logger):
+        user = self.create_user()
+        token = ApiToken.objects.create(user=user, name="test user token", scope_list=[])
+
+        payload = [
+            {
+                "source": "commit",
+                "token": str(token),
+                "type": "sentry_user_auth_token",
+                "url": "https://example.com/base-repo-url/",
+            }
+        ]
+
+        with self.tasks():
+            response = self.client.post(self.path, content_type="application/json", data=payload)
+        assert response.status_code == 200
+        assert response.content == b"[]"
+
+        extra = {
+            "exposed_source": "commit",
+            "exposed_url": "https://example.com/base-repo-url/",
+            "hashed_token": token.hashed_token,
+            "token_type": AuthTokenType.USER,
+        }
+        mock_logger.info.assert_called_with("found an exposed auth token", extra=extra)
+
+        assert len(mail.outbox) == 1
+        assert mail.outbox[0].to == [user.username]
+        assert mail.outbox[0].subject == "[Sentry]Action Required: User Auth Token Exposed"
+        assert (
+            "Your Sentry User Auth Token was found publicly on the internet" in mail.outbox[0].body
+        )
+        assert "http://testserver/settings/account/api/auth-tokens" in mail.outbox[0].body
+        assert "test user token" in mail.outbox[0].body
+        assert token.hashed_token in mail.outbox[0].body
+
+    @override_options({"secret-scanning.github.enable-signature-verification": False})
+    @patch("sentry.api.endpoints.secret_scanning.github.logger")
+    def test_true_positive_org_token(self, mock_logger):
+        token_str = generate_token("test-org", "https://test-region.sentry.io")
+        token = OrgAuthToken.objects.create(
+            organization_id=self.organization.id,
+            name="test org token",
+            scope_list=["org:ci"],
+            token_hashed=hash_token(token_str),
+        )
+
+        payload = [
+            {
+                "source": "commit",
+                "token": token_str,
+                "type": "sentry_org_auth_token",
+                "url": "https://example.com/base-repo-url/",
+            }
+        ]
+
+        with self.tasks():
+            response = self.client.post(self.path, content_type="application/json", data=payload)
+        assert response.status_code == 200
+        assert response.content == b"[]"
+
+        extra = {
+            "exposed_source": "commit",
+            "exposed_url": "https://example.com/base-repo-url/",
+            "hashed_token": token.token_hashed,
+            "token_type": AuthTokenType.ORG,
+        }
+        mock_logger.info.assert_called_with("found an exposed auth token", extra=extra)
+
+        assert len(mail.outbox) == 1
+        assert mail.outbox[0].to == [self.user.username]
+        assert mail.outbox[0].subject == "[Sentry]Action Required: Organization Auth Token Exposed"
+        assert (
+            "Your Sentry Organization Auth Token was found publicly on the internet"
+            in mail.outbox[0].body
+        )
+        assert "http://baz.testserver/settings/auth-tokens/" in mail.outbox[0].body
+        assert "test org token" in mail.outbox[0].body
+        assert token.token_hashed in mail.outbox[0].body

tests/sentry/utils/test_github.py

@@ -0,0 +1,68 @@
+from unittest import TestCase
+
+import pytest
+import responses
+
+from sentry.utils.github import verify_signature
+
+GITHUB_META_PUBLIC_KEYS_RESPONSE = {
+    "public_keys": [
+        {
+            "key_identifier": "90a421169f0a406205f1563a953312f0be898d3c7b6c06b681aa86a874555f4a",
+            "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9MJJHnMfn2+H4xL4YaPDA4RpJqUq\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ8qpVIW4clayyef9gWhFbNHWAA==\n-----END PUBLIC KEY-----\n",
+            "is_current": False,
+        },
+        {
+            "key_identifier": "bcb53661c06b4728e59d897fb6165d5c9cda0fd9cdf9d09ead458168deb7518c",
+            "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEYAGMWO8XgCamYKMJS6jc/qgvSlAd\nAjPuDPRcXU22YxgBrz+zoN19MzuRyW87qEt9/AmtoNP5GrobzUvQSyJFVw==\n-----END PUBLIC KEY-----\n",
+            "is_current": True,
+        },
+    ]
+}
+
+
+class TestGitHub(TestCase):
+    def setUp(self):
+        # https://docs.github.com/en/code-security/secret-scanning/secret-scanning-partner-program#implement-signature-verification-in-your-secret-alert-service
+        self.payload = """[{"source":"commit","token":"some_token","type":"some_type","url":"https://example.com/base-repo-url/"}]"""
+        self.signature = "MEQCIQDaMKqrGnE27S0kgMrEK0eYBmyG0LeZismAEz/BgZyt7AIfXt9fErtRS4XaeSt/AO1RtBY66YcAdjxji410VQV4xg=="
+        self.key_id = "bcb53661c06b4728e59d897fb6165d5c9cda0fd9cdf9d09ead458168deb7518c"
+        self.subpath = "secret_scanning"
+
+    @responses.activate
+    def _verify(self):
+        responses.add(
+            responses.GET,
+            "https://api.github.com/meta/public_keys/secret_scanning",
+            json=GITHUB_META_PUBLIC_KEYS_RESPONSE,
+            status=200,
+        )
+
+        verify_signature(self.payload, self.signature, self.key_id, self.subpath)
+
+    def test_verify_signature_success(self):
+        self._verify()
+
+    def test_verify_signature_missing_key(self):
+        self.key_id = ""
+        with pytest.raises(ValueError) as excinfo:
+            self._verify()
+        assert "Invalid payload, signature, or key_id" in str(excinfo.value)
+
+    def test_verify_signature_invalid_key(self):
+        self.key_id = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
+        with pytest.raises(ValueError) as excinfo:
+            self._verify()
+        assert "No public key found matching key identifier" in str(excinfo.value)
+
+    def test_verify_signature_invalid_signature(self):
+        self.payload = "[]"
+        with pytest.raises(ValueError) as excinfo:
+            self._verify()
+        assert "Signature does not match payload" in str(excinfo.value)
+
+    def test_verify_signature_invalid_encoding(self):
+        self.signature = "fakesignature"
+        with pytest.raises(ValueError) as excinfo:
+            self._verify()
+        assert "Invalid signature encoding" in str(excinfo.value)