feat(uptime): Tasks for automatic hostname detection and monitoring (#73258)

This implements most of the second half of automatic hostname detection
and monitoring. It has a few components:
- `schedule_detections` runs every minute. It looks at the last time it
was run, and fires off tasks for each minute bucket.
- `process_detection_bucket` runs for a specific datetime bucket. It
just fetches all projects from this bucket and fires
`process_project_url_ranking` for each of them. Note that each project
will only belong to a single bucket.
- `process_project_url_ranking` runs for a specific project. This is
where the meat of the logic exists. We get the ranked list of urls
collected from this project's event data and run it through various
criteria to see whether we should attempt to monitor the url

There are a fair few todos around that I need to finish up, as well as
building the logic to send subscriptions to the topic for our rust
consumer.

src/sentry/conf/server.py

@@ -806,6 +806,7 @@ CELERY_IMPORTS = (
     "sentry.middleware.integrations.tasks",
     "sentry.replays.usecases.ingest.issue_creation",
     "sentry.integrations.slack.tasks",
+    "sentry.uptime.detectors.tasks",
 )
 
 default_exchange = Exchange("default", type="direct")
@@ -930,6 +931,7 @@ CELERY_QUEUES_REGION = [
     Queue("subscriptions", routing_key="subscriptions"),
     Queue("unmerge", routing_key="unmerge"),
     Queue("update", routing_key="update"),
+    Queue("uptime", routing_key="uptime"),
     Queue("profiles.process", routing_key="profiles.process"),
     Queue("replays.ingest_replay", routing_key="replays.ingest_replay"),
     Queue("replays.delete_replay", routing_key="replays.delete_replay"),
@@ -1237,6 +1239,11 @@ CELERYBEAT_SCHEDULE_REGION = {
         # Run every 5 minutes
         "schedule": crontab(minute="*/5"),
     },
+    "uptime-detection-scheduler": {
+        "task": "sentry.uptime.detectors.tasks.schedule_detections",
+        # Run every 1 minute
+        "schedule": crontab(minute="*/1"),
+    },
 }
 
 # Assign the configuration keys celery uses based on our silo mode.

src/sentry/uptime/detectors/ranking.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import random
-from datetime import timedelta
+from datetime import datetime, timedelta
 from typing import TYPE_CHECKING
 
 from django.conf import settings
@@ -19,6 +19,7 @@ PROJECT_FLUSH_FREQUENCY = timedelta(days=1)
 # How often do we want to run our task to flush the buckets?
 # XXX: This might actually belong in the task when we have that?
 BUCKET_FLUSH_FREQUENCY = timedelta(minutes=1)
+NUMBER_OF_BUCKETS = int(PROJECT_FLUSH_FREQUENCY / BUCKET_FLUSH_FREQUENCY)
 # How often should we trim the ranked list?
 RANKED_TRIM_CHANCE = 0.01
 # How many urls should we trim the ranked list to?
@@ -53,7 +54,7 @@ def add_base_url_to_rank(project: Project, base_url: str):
     bucket_key = get_project_bucket_key(project)
     pipeline = cluster.pipeline()
     pipeline.hincrby(bucket_key, str(project.id), 1)
-    rank_key = get_project_hostname_rank_key(project)
+    rank_key = get_project_base_url_rank_key(project)
     pipeline.zincrby(rank_key, 1, base_url)
     if random.random() < RANKED_TRIM_CHANCE:
         pipeline.zremrangebyrank(rank_key, 0, -(RANKED_MAX_SIZE + 1))
@@ -69,10 +70,59 @@ def add_base_url_to_rank(project: Project, base_url: str):
         pipeline.execute()
 
 
-def get_project_bucket_key(project: Project) -> str:
-    project_bucket = project.id % (PROJECT_FLUSH_FREQUENCY / BUCKET_FLUSH_FREQUENCY)
-    return f"p:{project_bucket}"
+def get_candidate_urls_for_project(project: Project) -> list[tuple[str, int]]:
+    """
+    Gets all the candidate urls for a project. Returns a tuple of (url, times_url_seen). Urls are sorted by
+    `times_url_seen` desc.
+    """
+    key = get_project_base_url_rank_key(project)
+    cluster = _get_cluster()
+    return cluster.zrange(key, 0, -1, desc=True, withscores=True, score_cast_func=int)
 
 
-def get_project_hostname_rank_key(project: Project) -> str:
+def delete_candidate_urls_for_project(project: Project) -> None:
+    """
+    Deletes all current candidate rules for a project.
+    """
+    key = get_project_base_url_rank_key(project)
+    cluster = _get_cluster()
+    cluster.delete(key)
+
+
+def get_project_base_url_rank_key(project: Project) -> str:
     return f"p:r:{project.id}"
+
+
+def build_project_bucket_key(bucket: int):
+    return f"p:{bucket}"
+
+
+def get_project_bucket_key(project: Project) -> str:
+    project_bucket = int(project.id % NUMBER_OF_BUCKETS)
+    return build_project_bucket_key(project_bucket)
+
+
+def get_project_bucket_key_for_datetime(bucket_datetime: datetime) -> str:
+    date_bucket = int(
+        bucket_datetime.replace(second=0, microsecond=0).timestamp() % NUMBER_OF_BUCKETS
+    )
+    return build_project_bucket_key(date_bucket)
+
+
+def get_project_bucket(bucket: datetime) -> dict[int, int]:
+    """
+    Fetch all projects from a specific datetime bucket. Returns a dict keyed by project id with the value as
+    the total count of seen valid urls
+    """
+    key = get_project_bucket_key_for_datetime(bucket)
+    cluster = _get_cluster()
+    return {int(key): int(val) for key, val in cluster.hgetall(key).items()}
+
+
+def delete_project_bucket(bucket: datetime) -> None:
+    """
+    Delete all projects from a specific datetime bucket.
+    """
+    key = get_project_bucket_key_for_datetime(bucket)
+    cluster = _get_cluster()
+    cluster.delete(key)

src/sentry/uptime/detectors/tasks.py

@@ -0,0 +1,182 @@
+import datetime
+from datetime import timedelta
+
+from django.utils import timezone
+
+from sentry.locks import locks
+from sentry.models.project import Project
+from sentry.tasks.base import instrumented_task
+from sentry.uptime.detectors.ranking import (
+    _get_cluster,
+    delete_candidate_urls_for_project,
+    delete_project_bucket,
+    get_candidate_urls_for_project,
+    get_project_bucket,
+)
+from sentry.uptime.models import ProjectUptimeSubscription, UptimeSubscription
+from sentry.utils import metrics
+from sentry.utils.hashlib import md5_text
+from sentry.utils.locking import UnableToAcquireLock
+
+LAST_PROCESSED_KEY = "uptime_detector_last_processed"
+SCHEDULER_LOCK_KEY = "uptime_detector_scheduler_lock"
+FAILED_URL_RETRY_FREQ = timedelta(days=7)
+URL_MIN_TIMES_SEEN = 5
+URL_MIN_PERCENT = 0.05
+
+
+@instrumented_task(
+    name="sentry.uptime.detectors.tasks.schedule_detections",
+    queue="uptime",
+    time_limit=60,
+    soft_time_limit=55,
+)
+def schedule_detections():
+    """
+    Runs regularly and fires off a task for each detection bucket that needs to be run since
+    this task last ran.
+    """
+    lock = locks.get(
+        SCHEDULER_LOCK_KEY,
+        duration=60,
+        name="uptime.detection.schedule_detections",
+    )
+    try:
+        with lock.acquire():
+            cluster = _get_cluster()
+            last_processed = cluster.get(LAST_PROCESSED_KEY)
+            if last_processed is None:
+                last_processed = timezone.now().replace(second=0, microsecond=0)
+            else:
+                last_processed = datetime.datetime.fromtimestamp(
+                    int(last_processed), tz=datetime.UTC
+                )
+
+            minutes_since_last_processed = int(
+                (timezone.now() - last_processed) / timedelta(minutes=1)
+            )
+            for _ in range(minutes_since_last_processed):
+                last_processed = last_processed + timedelta(minutes=1)
+                process_detection_bucket.delay(last_processed)
+
+            cluster.set(LAST_PROCESSED_KEY, int(last_processed.timestamp()), timedelta(hours=1))
+    except UnableToAcquireLock:
+        # If we can't acquire the lock it just means another task is already handling scheduling,
+        # so just exit
+        metrics.incr("uptime.detectors.scheduler.unable_to_acquire_lock")
+
+
+@instrumented_task(
+    name="sentry.uptime.detectors.tasks.process_detection_bucket",
+    queue="uptime",
+)
+def process_detection_bucket(bucket: datetime.datetime):
+    """
+    Schedules url detection for all projects in this time bucket that saw promising urls.
+    """
+    for project_id, count in get_project_bucket(bucket).items():
+        process_project_url_ranking.delay(project_id, count)
+    delete_project_bucket(bucket)
+
+
+@instrumented_task(
+    name="sentry.uptime.detectors.tasks.process_project_url_ranking",
+    queue="uptime",
+)
+def process_project_url_ranking(project_id: int, project_url_count: int):
+    """
+    Looks at candidate urls for a project and determines whether we should start monitoring them
+    """
+    project = Project.objects.get_from_cache(id=project_id)
+    if not should_detect_for_project(project):
+        return
+
+    for url, url_count in get_candidate_urls_for_project(project)[:5]:
+        if process_candidate_url(project, project_url_count, url, url_count):
+            # TODO: On success, we want to mark this project as not needing to be checked for a while
+            break
+    else:
+        # TODO: If we don't find any urls to monitor, we want to increment a counter in redis and check the value.
+        # After a number of failures, we want to stop checking for this project for a while.
+        pass
+
+    delete_candidate_urls_for_project(project)
+
+
+def process_candidate_url(
+    project: Project, project_url_count: int, url: str, url_count: int
+) -> bool:
+    """
+    Takes a candidate url for a project and determines whether we should create an uptime subscription for it.
+    Checks that:
+     - URL has been seen at least `URL_MIN_TIMES_SEEN` times and is seen in at least `URL_MIN_PERCENT` of events with urls
+     - URL hasn't already been checked and failed recently
+     - Whether we already have a subscription for this url in the system - If so, just link this project to that subscription
+     - Whether the url's robots.txt will allow us to monitor this url
+
+    If the url passes, and we don't already have a subscription for it, then create a new remote subscription for the
+    url and delete any existing automated monitors.
+    """
+    # The url has to be seen a minimum number of times, and make up at least
+    # a certain percentage of all urls seen in this project
+    if url_count < URL_MIN_TIMES_SEEN or url_count / project_url_count < URL_MIN_PERCENT:
+        return False
+
+    # Check whether we've recently attempted to monitor this url recently and failed.
+    if is_failed_url(url):
+        return False
+
+    # See if we're monitoring this url at all
+    try:
+        # TODO: We should have a column that lets us filter to detected urls
+        existing_subscription = UptimeSubscription.objects.get(url=url, interval_seconds=300)
+    except UptimeSubscription.DoesNotExist:
+        existing_subscription = None
+
+    if existing_subscription:
+        # Since we already have an existing subscription to this url, we don't need to perform any other checks
+        # The subscription will have already been created in the rust checker, so we can just link to the
+        # subscription here if we aren't already monitoring it in this project.
+        ProjectUptimeSubscription.objects.get_or_create(
+            project=project, uptime_subscription=existing_subscription
+        )
+        return True
+
+    # Check robots.txt to see if it's ok for us to attempt to monitor this url
+    if not check_url_robots_txt(url):
+        set_failed_url(url)
+        return False
+
+    # If we hit this point, then the url looks worth monitoring. Create an uptime subscription in monitor mode.
+    # Also check if there's already an existing auto detected monitor for this project. If so, delete it.
+    # TODO: Implement subscriptions
+    return True
+
+
+def is_failed_url(url: str) -> bool:
+    key = get_failed_url_key(url)
+    return _get_cluster().exists(key) == 1
+
+
+def set_failed_url(url: str) -> None:
+    """
+    If we failed to monitor a url for some reason, skip processing it for FAILED_URL_RETRY_FREQ
+    """
+    key = get_failed_url_key(url)
+    _get_cluster().set(key, 1, ex=FAILED_URL_RETRY_FREQ)
+
+
+def get_failed_url_key(url: str) -> str:
+    return f"f:u:{md5_text(url).hexdigest()}"
+
+
+def check_url_robots_txt(url: str) -> bool:
+    # TODO: Implement this check
+    return True
+
+
+def should_detect_for_project(project: Project) -> bool:
+    # TODO: Check if project has detection disabled
+    # TODO: If we're already running a detected url monitor for this project, we should stop attempting to
+    # detect urls for a while
+    return True

src/sentry/uptime/models.py

@@ -46,6 +46,7 @@ class ProjectUptimeSubscription(DefaultFieldsModel):
     __relocation_scope__ = RelocationScope.Excluded
 
     project = FlexibleForeignKey("sentry.Project")
+    # TODO: Change this to Models.PROTECT
     uptime_subscription = FlexibleForeignKey("uptime.UptimeSubscription")
 
     objects: ClassVar[BaseManager[Self]] = BaseManager(

tests/sentry/uptime/detectors/test_ranking.py

@@ -1,12 +1,18 @@
+from datetime import datetime
 from unittest import mock
 
 from sentry.models.project import Project
 from sentry.testutils.cases import TestCase
 from sentry.uptime.detectors.ranking import (
+    NUMBER_OF_BUCKETS,
     _get_cluster,
     add_base_url_to_rank,
+    delete_candidate_urls_for_project,
+    delete_project_bucket,
+    get_candidate_urls_for_project,
+    get_project_base_url_rank_key,
+    get_project_bucket,
     get_project_bucket_key,
-    get_project_hostname_rank_key,
 )
 
 
@@ -26,7 +32,7 @@ class AddBaseUrlToRankTest(TestCase):
     def assert_url_count(
         self, project: Project, url: str, count: int | None, expiry: int | None
     ) -> int | None:
-        key = get_project_hostname_rank_key(project)
+        key = get_project_base_url_rank_key(project)
         cluster = _get_cluster()
         if count is None:
             assert cluster.zscore(key, url) is None
@@ -80,7 +86,7 @@ class AddBaseUrlToRankTest(TestCase):
         with mock.patch("sentry.uptime.detectors.ranking.RANKED_TRIM_CHANCE", new=1), mock.patch(
             "sentry.uptime.detectors.ranking.RANKED_MAX_SIZE", new=2
         ):
-            key = get_project_hostname_rank_key(self.project)
+            key = get_project_base_url_rank_key(self.project)
             url_1 = "https://sentry.io"
             url_2 = "https://sentry.sentry.io"
             url_3 = "https://santry.sentry.io"
@@ -95,3 +101,47 @@ class AddBaseUrlToRankTest(TestCase):
             # Since we're trimming immediately, this url will be immediately dropped since it's seen one time
             add_base_url_to_rank(self.project, url_3)
             assert cluster.zrange(key, 0, -1) == [url_2, url_1]
+
+
+class GetCandidateUrlsForProjectTest(TestCase):
+    def test(self):
+        assert get_candidate_urls_for_project(self.project) == []
+        url_1 = "https://sentry.io"
+        url_2 = "https://sentry.sentry.io"
+        add_base_url_to_rank(self.project, url_1)
+        assert get_candidate_urls_for_project(self.project) == [(url_1, 1)]
+        add_base_url_to_rank(self.project, url_2)
+        add_base_url_to_rank(self.project, url_2)
+        assert get_candidate_urls_for_project(self.project) == [(url_2, 2), (url_1, 1)]
+
+
+class DeleteCandidateUrlsForProjectTest(TestCase):
+    def test(self):
+        delete_candidate_urls_for_project(self.project)
+        url_1 = "https://sentry.io"
+        add_base_url_to_rank(self.project, url_1)
+        assert get_candidate_urls_for_project(self.project) == [(url_1, 1)]
+        delete_candidate_urls_for_project(self.project)
+        assert get_candidate_urls_for_project(self.project) == []
+
+
+class GetProjectBucketTest(TestCase):
+    def test(self):
+        bucket = datetime.now().replace(second=0, microsecond=0)
+        assert get_project_bucket(bucket) == {}
+        dummy_project_id = int(bucket.timestamp() % NUMBER_OF_BUCKETS)
+        self.project.id = dummy_project_id
+        add_base_url_to_rank(self.project, "https://sentry.io")
+        assert get_project_bucket(bucket) == {self.project.id: 1}
+
+
+class DeleteProjectBucketTest(TestCase):
+    def test(self):
+        bucket = datetime.now().replace(second=0, microsecond=0)
+        delete_project_bucket(bucket)
+        dummy_project_id = int(bucket.timestamp() % NUMBER_OF_BUCKETS)
+        self.project.id = dummy_project_id
+        add_base_url_to_rank(self.project, "https://sentry.io")
+        assert get_project_bucket(bucket) == {self.project.id: 1}
+        delete_project_bucket(bucket)
+        assert get_project_bucket(bucket) == {}

tests/sentry/uptime/detectors/test_tasks.py

@@ -0,0 +1,207 @@
+from datetime import timedelta
+from unittest import mock
+from unittest.mock import call
+
+from django.utils import timezone
+
+from sentry.locks import locks
+from sentry.models.project import Project
+from sentry.testutils.cases import TestCase
+from sentry.testutils.helpers.datetime import freeze_time
+from sentry.uptime.detectors.ranking import (
+    NUMBER_OF_BUCKETS,
+    _get_cluster,
+    add_base_url_to_rank,
+    get_project_bucket,
+)
+from sentry.uptime.detectors.tasks import (
+    LAST_PROCESSED_KEY,
+    SCHEDULER_LOCK_KEY,
+    is_failed_url,
+    process_candidate_url,
+    process_detection_bucket,
+    process_project_url_ranking,
+    schedule_detections,
+    set_failed_url,
+)
+from sentry.uptime.models import ProjectUptimeSubscription
+
+
+@freeze_time()
+class ScheduleDetectionsTest(TestCase):
+    def test_no_last_processed(self):
+        # The first time this runs we don't expect much to happen,
+        # just that it'll update the last processed date in redis
+        cluster = _get_cluster()
+        assert not cluster.get(LAST_PROCESSED_KEY)
+        with mock.patch(
+            "sentry.uptime.detectors.tasks.process_detection_bucket"
+        ) as mock_process_detection_bucket:
+            schedule_detections()
+            mock_process_detection_bucket.delay.assert_not_called()
+        last_processed = cluster.get(LAST_PROCESSED_KEY)
+        assert last_processed is not None
+        assert int(last_processed) == int(
+            timezone.now().replace(second=0, microsecond=0).timestamp()
+        )
+
+    def test_processes(self):
+        cluster = _get_cluster()
+        current_bucket = timezone.now().replace(second=0, microsecond=0)
+        last_processed_bucket = current_bucket - timedelta(minutes=10)
+        cluster.set(LAST_PROCESSED_KEY, int(last_processed_bucket.timestamp()))
+        with mock.patch(
+            "sentry.uptime.detectors.tasks.process_detection_bucket"
+        ) as mock_process_detection_bucket:
+            schedule_detections()
+            mock_process_detection_bucket.delay.assert_has_calls(
+                [call(last_processed_bucket + timedelta(minutes=i)) for i in range(1, 11)]
+            )
+        last_processed = cluster.get(LAST_PROCESSED_KEY)
+        assert last_processed is not None
+        assert int(last_processed) == int(
+            timezone.now().replace(second=0, microsecond=0).timestamp()
+        )
+
+    def test_lock(self):
+        lock = locks.get(
+            SCHEDULER_LOCK_KEY,
+            duration=60,
+            name="uptime.detection.schedule_detections",
+        )
+        with lock.acquire(), mock.patch("sentry.uptime.detectors.tasks.metrics") as metrics:
+            schedule_detections()
+            metrics.incr.assert_called_once_with(
+                "uptime.detectors.scheduler.unable_to_acquire_lock"
+            )
+
+
+@freeze_time()
+class ProcessDetectionBucketTest(TestCase):
+    def test_empty_bucket(self):
+        with mock.patch(
+            "sentry.uptime.detectors.tasks.process_project_url_ranking"
+        ) as mock_process_project_url_ranking:
+            process_detection_bucket(timezone.now().replace(second=0, microsecond=0))
+            mock_process_project_url_ranking.delay.assert_not_called()
+
+    def test_bucket(self):
+        bucket = timezone.now().replace(second=0, microsecond=0)
+        dummy_project_id = int(bucket.timestamp() % NUMBER_OF_BUCKETS)
+        self.project.id = dummy_project_id
+        other_project = Project(dummy_project_id + NUMBER_OF_BUCKETS)
+        add_base_url_to_rank(self.project, "https://sentry.io")
+        add_base_url_to_rank(other_project, "https://sentry.io")
+
+        with mock.patch(
+            "sentry.uptime.detectors.tasks.process_project_url_ranking"
+        ) as mock_process_project_url_ranking:
+            process_detection_bucket(bucket)
+            mock_process_project_url_ranking.delay.assert_has_calls(
+                [call(self.project.id, 1), call(other_project.id, 1)], any_order=True
+            )
+
+        assert get_project_bucket(bucket) == {}
+
+
+@freeze_time()
+class ProcessProjectUrlRankingTest(TestCase):
+    def test(self):
+        # TODO: Better testing for this function when we implement things that happen on success
+        url_1 = "https://sentry.io"
+        url_2 = "https://sentry.sentry.io"
+        add_base_url_to_rank(self.project, url_2)
+        add_base_url_to_rank(self.project, url_1)
+        add_base_url_to_rank(self.project, url_1)
+        with mock.patch(
+            "sentry.uptime.detectors.tasks.process_candidate_url",
+            return_value=False,
+        ) as mock_process_candidate_url:
+            process_project_url_ranking(self.project.id, 5)
+            mock_process_candidate_url.assert_has_calls(
+                [
+                    call(self.project, 5, url_1, 2),
+                    call(self.project, 5, url_2, 1),
+                ]
+            )
+
+    def test_should_not_detect(self):
+        with mock.patch(
+            # TODO: Replace this mock with real tests when we implement this function properly
+            "sentry.uptime.detectors.tasks.should_detect_for_project",
+            return_value=False,
+        ), mock.patch(
+            "sentry.uptime.detectors.tasks.get_candidate_urls_for_project"
+        ) as mock_get_candidate_urls_for_project:
+            process_project_url_ranking(self.project.id, 5)
+            mock_get_candidate_urls_for_project.assert_not_called()
+
+
+@freeze_time()
+class ProcessCandidateUrlTest(TestCase):
+    def test_succeeds_new(self):
+        assert process_candidate_url(self.project, 100, "https://sentry.io", 50)
+
+    def test_succeeds_existing_subscription_other_project(self):
+        other_project = self.create_project()
+        url = "https://sentry.io"
+        uptime_subscription = self.create_uptime_subscription(url=url, interval_seconds=300)
+        self.create_project_uptime_subscription(
+            project=other_project, uptime_subscription=uptime_subscription
+        )
+        assert (
+            ProjectUptimeSubscription.objects.filter(
+                project=self.project, uptime_subscription=uptime_subscription
+            ).count()
+            == 0
+        )
+        assert process_candidate_url(self.project, 100, url, 50)
+        assert (
+            ProjectUptimeSubscription.objects.filter(
+                project=self.project, uptime_subscription=uptime_subscription
+            ).count()
+            == 1
+        )
+
+    def test_succeeds_existing_subscription_this_project(self):
+        url = "https://sentry.io"
+        uptime_subscription = self.create_uptime_subscription(url=url, interval_seconds=300)
+        self.create_project_uptime_subscription(
+            project=self.project, uptime_subscription=uptime_subscription
+        )
+        assert process_candidate_url(self.project, 100, url, 50)
+        assert (
+            ProjectUptimeSubscription.objects.filter(
+                project=self.project, uptime_subscription=uptime_subscription
+            ).count()
+            == 1
+        )
+        # TODO: Check no other subscriptions or anything made once we finish the rest of this func
+
+    def test_below_thresholds(self):
+        assert not process_candidate_url(self.project, 500, "https://sentry.io", 1)
+        assert not process_candidate_url(self.project, 500, "https://sentry.io", 10)
+
+    def test_failed_url(self):
+        url = "https://sentry.io"
+        set_failed_url(url)
+        assert not process_candidate_url(self.project, 100, url, 50)
+
+    def test_failed_robots_txt(self):
+        url = "https://sentry.io"
+        with mock.patch(
+            # TODO: Replace this mock with real tests when we implement this function properly
+            "sentry.uptime.detectors.tasks.check_url_robots_txt",
+            return_value=False,
+        ):
+            assert not process_candidate_url(self.project, 100, url, 50)
+        assert is_failed_url(url)
+
+
+class TestFailedUrl(TestCase):
+    def test(self):
+        url = "https://sentry.io"
+        assert not is_failed_url(url)
+        set_failed_url(url)
+        assert is_failed_url(url)
+        assert not is_failed_url("https://sentry.sentry.io")