fix(hybridcloud) Move async_send_notification out of email queue (#60610)

The email queue is common between control and region roles. This queue
is used by the silo naive MessageBuilder.send_async method to deliver
email messages.

The `async_send_notification` task contains queries for region
resources, and needs to be moved out of the shared queue so that we
don't have silo cross talk once roles are separated.

src/sentry/conf/server.py

@@ -885,6 +885,7 @@ CELERY_QUEUES_REGION = [
     Queue("incident_snapshots", routing_key="incident_snapshots"),
     Queue("incidents", routing_key="incidents"),
     Queue("merge", routing_key="merge"),
+    Queue("notifications", routing_key="notifications"),
     Queue("options", routing_key="options"),
     Queue("post_process_errors", routing_key="post_process_errors"),
     Queue("post_process_issue_platform", routing_key="post_process_issue_platform"),

src/sentry/notifications/utils/tasks.py

@@ -6,12 +6,15 @@ from django.apps import apps
 
 from sentry.db.models import Model
 from sentry.notifications.class_manager import NotificationClassNotSetException, get
+from sentry.services.hybrid_cloud.util import region_silo_function
+from sentry.silo.base import SiloMode
 from sentry.tasks.base import instrumented_task
 
 if TYPE_CHECKING:
     from sentry.notifications.notifications.base import BaseNotification
 
 
+@region_silo_function
 def async_send_notification(
     NotificationClass: type[BaseNotification], *args: Any, **kwargs: Any
 ) -> None:
@@ -61,7 +64,8 @@ def async_send_notification(
 
 @instrumented_task(
     name="src.sentry.notifications.utils.async_send_notification",
-    queue="email",
+    silo_mode=SiloMode.REGION,
+    queue="notifications",
 )
 def _send_notification(notification_class_name: str, arg_list: Iterable[Mapping[str, Any]]) -> None:
     NotificationClass = get(notification_class_name)