feat(backup): Add `RelocationScope.Config` (#56260)

We add a new scope, which roughly corresponds to "all of the stuff you
need to restore the configuration of administration setup of a Sentry
instance".

To support this, we also need to support dynamic relocation scope
deduction. This means that the `__relocation_scope__` class-level
attribute can now be a single scope or a set of them. The latter case
implies that the actual scope of any given instance of that model relies
on information specific to that instance, and so will be deduced on a
per-model basis.

Issue: getsentry/team-ospo#199

bin/model-dependency-graphviz

@@ -76,6 +76,7 @@ class ClusterColor(Enum):
     Purple = "lavenderblush"
     Yellow = "khaki"
     Green = "honeydew"
+    Blue = "lightsteelblue1"
 
 
 @unique
@@ -126,6 +127,12 @@ def print_edge(
     return f""""{src.__name__}":e -> "{dest.__name__}":w {style.value};"""
 
 
+def get_most_permissive_relocation_scope(mr: ModelRelations) -> RelocationScope:
+    if isinstance(mr.relocation_scope, set):
+        return sorted(list(mr.relocation_scope), key=lambda obj: obj.value * -1)[0]
+    return mr.relocation_scope
+
+
 @click.command()
 @click.option("--show-excluded", default=False, is_flag=True, help="Show unexportable models too")
 def main(show_excluded: bool):
@@ -136,17 +143,27 @@ def main(show_excluded: bool):
     if not show_excluded:
         deps = list(filter(lambda m: m.relocation_scope != RelocationScope.Excluded, deps))
 
-    # Group by region scope.
-    user_scoped = filter(lambda m: m.relocation_scope == RelocationScope.User, deps)
-    org_scoped = filter(lambda m: m.relocation_scope == RelocationScope.Organization, deps)
-    global_scoped = filter(lambda m: m.relocation_scope == RelocationScope.Global, deps)
+    # Group by most permissive region scope.
+    user_scoped = filter(
+        lambda m: get_most_permissive_relocation_scope(m) == RelocationScope.User, deps
+    )
+    org_scoped = filter(
+        lambda m: get_most_permissive_relocation_scope(m) == RelocationScope.Organization, deps
+    )
+    config_scoped = filter(
+        lambda m: get_most_permissive_relocation_scope(m) == RelocationScope.Config, deps
+    )
+    global_scoped = filter(
+        lambda m: get_most_permissive_relocation_scope(m) == RelocationScope.Global, deps
+    )
 
     # Print nodes.
     clusters = "".join(
         [
             print_rel_scope_subgraph("User", 1, user_scoped, ClusterColor.Green),
             print_rel_scope_subgraph("Organization", 2, org_scoped, ClusterColor.Purple),
-            print_rel_scope_subgraph("Global", 3, global_scoped, ClusterColor.Yellow),
+            print_rel_scope_subgraph("Config", 3, config_scoped, ClusterColor.Blue),
+            print_rel_scope_subgraph("Global", 4, global_scoped, ClusterColor.Yellow),
         ]
     )
 

fixtures/backup/model_dependencies/detailed.json

@@ -192,7 +192,10 @@
       }
     },
     "model": "sentry.ApiAuthorization",
-    "relocation_scope": "Global",
+    "relocation_scope": [
+      "Config",
+      "Global"
+    ],
     "silos": [
       "Control"
     ]
@@ -239,7 +242,10 @@
       }
     },
     "model": "sentry.ApiToken",
-    "relocation_scope": "Global",
+    "relocation_scope": [
+      "Config",
+      "Global"
+    ],
     "silos": [
       "Control"
     ]
@@ -569,7 +575,7 @@
   "sentry.ControlOption": {
     "foreign_keys": {},
     "model": "sentry.ControlOption",
-    "relocation_scope": "Global",
+    "relocation_scope": "Config",
     "silos": [
       "Control"
     ]
@@ -1915,7 +1921,7 @@
   "sentry.Option": {
     "foreign_keys": {},
     "model": "sentry.Option",
-    "relocation_scope": "Global",
+    "relocation_scope": "Config",
     "silos": [
       "Region"
     ]
@@ -2584,7 +2590,7 @@
   "sentry.Relay": {
     "foreign_keys": {},
     "model": "sentry.Relay",
-    "relocation_scope": "Global",
+    "relocation_scope": "Config",
     "silos": [
       "Region"
     ]
@@ -2592,7 +2598,7 @@
   "sentry.RelayUsage": {
     "foreign_keys": {},
     "model": "sentry.RelayUsage",
-    "relocation_scope": "Global",
+    "relocation_scope": "Config",
     "silos": [
       "Region"
     ]
@@ -3326,7 +3332,7 @@
       }
     },
     "model": "sentry.UserPermission",
-    "relocation_scope": "Global",
+    "relocation_scope": "Config",
     "silos": [
       "Control"
     ]
@@ -3359,7 +3365,7 @@
   "sentry.UserRole": {
     "foreign_keys": {},
     "model": "sentry.UserRole",
-    "relocation_scope": "Global",
+    "relocation_scope": "Config",
     "silos": [
       "Control"
     ]
@@ -3376,7 +3382,7 @@
       }
     },
     "model": "sentry.UserRoleUser",
-    "relocation_scope": "Global",
+    "relocation_scope": "Config",
     "silos": [
       "Control"
     ]

src/sentry/backup/dependencies.py

@@ -51,7 +51,7 @@ class ModelRelations(NamedTuple):
 
     model: Type[models.base.Model]
     foreign_keys: dict[str, ForeignField]
-    relocation_scope: RelocationScope
+    relocation_scope: RelocationScope | set[RelocationScope]
     silos: list[SiloMode]
 
     def flatten(self) -> set[Type[models.base.Model]]:
@@ -75,6 +75,9 @@ class DependenciesJSONEncoder(json.JSONEncoder):
             return obj.name
         if isinstance(obj, RelocationScope):
             return obj.name
+        if isinstance(obj, set) and all(isinstance(rs, RelocationScope) for rs in obj):
+            # Order by enum value, which should correspond to `RelocationScope` breadth.
+            return sorted(list(obj), key=lambda obj: obj.value)
         if isinstance(obj, SiloMode):
             return obj.name.lower().capitalize()
         if isinstance(obj, set):

src/sentry/backup/exports.py

@@ -84,11 +84,14 @@ def _export(
         filters.append(Filter(Email, "email", set(emails)))
 
     def filter_objects(queryset_iterator):
-        # Intercept each value from the queryset iterator and ensure that all of its dependencies
-        # have already been exported. If they have, store it in the `pk_map`, and then yield it
-        # again. If they have not, we know that some upstream model was filtered out, so we ignore
-        # this one as well.
+        # Intercept each value from the queryset iterator, ensure that it has the correct relocation
+        # scope and that all of its dependencies have already been exported. If they have, store it
+        # in the `pk_map`, and then yield it again. If they have not, we know that some upstream
+        # model was filtered out, so we ignore this one as well.
         for item in queryset_iterator:
+            if not item.get_relocation_scope() in allowed_relocation_scopes:
+                continue
+
             model = type(item)
             model_name = normalize_model_name(model)
 
@@ -116,15 +119,10 @@ def _export(
     def yield_objects():
         # Collate the objects to be serialized.
         for model in sorted_dependencies():
-            includable = False
-
-            # TODO(getsentry/team-ospo#186): This won't be sufficient once we're trying to get
-            # relocation scopes that may vary on a per-instance, rather than
-            # per-model-definition, basis. We'll probably need to make use of something like
-            # Django annotations to efficiently filter down models.
-            if getattr(model, "__relocation_scope__", None) in allowed_relocation_scopes:
-                includable = True
-
+            includable = (
+                hasattr(model, "__relocation_scope__")
+                and model.get_possible_relocation_scopes() & allowed_relocation_scopes
+            )
             if not includable or model._meta.proxy:
                 continue
 

src/sentry/backup/scopes.py

@@ -8,18 +8,29 @@ class RelocationScope(Enum):
     # A model that has been purposefully excluded from import/export functionality entirely.
     Excluded = auto()
 
-    # A model related to some global feature of Sentry. `Global` models are best understood via
-    # exclusion: they are all of the exportable `Control`-silo models that are **not** somehow tied
-    # to a specific user.
-    Global = auto()
+    # Any `Control`-silo model that is either a `User*` model, or directly owner by one, is in this
+    # scope. Models that deal with bestowing administration privileges are excluded, and are
+    # included in the `Config` scope instead.
+    User = auto()
 
     # For all models that transitively depend on either `User` or `Organization` root models, and
     # nothing else.
     Organization = auto()
 
-    # Any `Control`-silo model that is either a `User*` model, or directly owner by one, is in this
-    # scope.
-    User = auto()
+    # Models that deal with configuring or administering an entire Sentry instance. Some of these
+    # models transitively rely on `User` models (since their purpose is to mark certain users as
+    # administrators and give them elevated, instance-wide privileges), but otherwise these models
+    # have no dependencies on other scopes.
+    Config = auto()
+
+    # A model that is inextricably tied to a specific Sentry instance. Often, this applies to models
+    # that include the instance domain in their data (ex: OAuth or social login tokens related to
+    # the specific domain), which therefore are completely non-portable.
+    #
+    # In practice, this scope is reserved for models that are only useful when backing up or
+    # restoring an entire Sentry instance, since there is no reasonable way to use them outside of
+    # that specific context.
+    Global = auto()
 
 
 @unique
@@ -32,7 +43,13 @@ class ExportScope(Enum):
 
     User = {RelocationScope.User}
     Organization = {RelocationScope.User, RelocationScope.Organization}
-    Global = {RelocationScope.User, RelocationScope.Organization, RelocationScope.Global}
+    Config = {RelocationScope.User, RelocationScope.Config}
+    Global = {
+        RelocationScope.User,
+        RelocationScope.Organization,
+        RelocationScope.Config,
+        RelocationScope.Global,
+    }
 
 
 @unique
@@ -45,4 +62,10 @@ class ImportScope(Enum):
 
     User = {RelocationScope.User}
     Organization = {RelocationScope.User, RelocationScope.Organization}
-    Global = {RelocationScope.User, RelocationScope.Organization, RelocationScope.Global}
+    Config = {RelocationScope.User, RelocationScope.Config}
+    Global = {
+        RelocationScope.User,
+        RelocationScope.Organization,
+        RelocationScope.Config,
+        RelocationScope.Global,
+    }

src/sentry/db/models/base.py

@@ -46,7 +46,7 @@ class BaseModel(models.Model):
     class Meta:
         abstract = True
 
-    __relocation_scope__: RelocationScope
+    __relocation_scope__: RelocationScope | set[RelocationScope]
 
     objects = BaseManager[M]()  # type: ignore
 
@@ -106,8 +106,25 @@ class BaseModel(models.Model):
         Retrieves the `RelocationScope` for a `Model` subclass. It generally just forwards `__relocation_scope__`, but some models have instance-specific logic for deducing the scope.
         """
 
+        if isinstance(self.__relocation_scope__, set):
+            raise ValueError(
+                "Must define `get_relocation_scope` override if using multiple relocation scopes."
+            )
+
         return self.__relocation_scope__
 
+    @classmethod
+    def get_possible_relocation_scopes(cls) -> RelocationScope:
+        """
+        Retrieves the `RelocationScope` for a `Model` subclass. It always returns a set, to account for models that support multiple scopes on a situational, per-instance basis.
+        """
+
+        return (
+            cls.__relocation_scope__
+            if isinstance(cls.__relocation_scope__, set)
+            else {cls.__relocation_scope__}
+        )
+
     def _normalize_before_relocation_import(
         self, pk_map: PrimaryKeyMap, _s: ImportScope, _f: ImportFlags
     ) -> Optional[int]:
@@ -204,6 +221,15 @@ def __model_class_prepared(sender: Any, **kwargs: Any) -> None:
             f"like Group."
         )
 
+    if (
+        isinstance(getattr(sender, "__relocation_scope__"), set)
+        and RelocationScope.Excluded in sender.get_possible_relocation_scopes()
+    ):
+        raise ValueError(
+            f"{sender!r} model uses a set of __relocation_scope__ values, one of which is "
+            f"`Excluded`, which does not make sense. `Excluded` must always be a standalone value."
+        )
+
     from .outboxes import ReplicatedControlModel, ReplicatedRegionModel
 
     if issubclass(sender, ReplicatedControlModel):

src/sentry/models/apiauthorization.py

@@ -15,7 +15,7 @@ class ApiAuthorization(Model, HasApiScopes):
     overall approved applications (vs individual tokens).
     """
 
-    __relocation_scope__ = RelocationScope.Global
+    __relocation_scope__ = {RelocationScope.Global, RelocationScope.Config}
 
     # users can generate tokens without being application-bound
     application = FlexibleForeignKey("sentry.ApiApplication", null=True)
@@ -28,3 +28,10 @@ class ApiAuthorization(Model, HasApiScopes):
         unique_together = (("user", "application"),)
 
     __repr__ = sane_repr("user_id", "application_id")
+
+    def get_relocation_scope(self) -> RelocationScope:
+        if self.application_id is not None:
+            # TODO(getsentry/team-ospo#188): this should be extension scope once that gets added.
+            return RelocationScope.Global
+
+        return RelocationScope.Config

src/sentry/models/apitoken.py

@@ -31,7 +31,7 @@ def generate_token():
 
 @control_silo_only_model
 class ApiToken(Model, HasApiScopes):
-    __relocation_scope__ = RelocationScope.Global
+    __relocation_scope__ = {RelocationScope.Global, RelocationScope.Config}
 
     # users can generate tokens without being application-bound
     application = FlexibleForeignKey("sentry.ApiApplication", null=True)
@@ -79,6 +79,13 @@ class ApiToken(Model, HasApiScopes):
 
         self.update(token=generate_token(), refresh_token=generate_token(), expires_at=expires_at)
 
+    def get_relocation_scope(self) -> RelocationScope:
+        if self.application_id is not None:
+            # TODO(getsentry/team-ospo#188): this should be extension scope once that gets added.
+            return RelocationScope.Global
+
+        return RelocationScope.Config
+
     @property
     def organization_id(self) -> int | None:
         from sentry.models import SentryAppInstallation, SentryAppInstallationToken

src/sentry/models/options/option.py

@@ -47,7 +47,7 @@ class BaseOption(Model):
 
 @region_silo_only_model
 class Option(BaseOption):
-    __relocation_scope__ = RelocationScope.Global
+    __relocation_scope__ = RelocationScope.Config
 
     class Meta:
         app_label = "sentry"
@@ -58,7 +58,7 @@ class Option(BaseOption):
 
 @control_silo_only_model
 class ControlOption(BaseOption):
-    __relocation_scope__ = RelocationScope.Global
+    __relocation_scope__ = RelocationScope.Config
 
     class Meta:
         app_label = "sentry"

src/sentry/models/relay.py

@@ -9,7 +9,7 @@ from sentry.db.models import Model, region_silo_only_model
 
 @region_silo_only_model
 class RelayUsage(Model):
-    __relocation_scope__ = RelocationScope.Global
+    __relocation_scope__ = RelocationScope.Config
 
     relay_id = models.CharField(max_length=64)
     version = models.CharField(max_length=32, default="0.0.1")
@@ -25,7 +25,7 @@ class RelayUsage(Model):
 
 @region_silo_only_model
 class Relay(Model):
-    __relocation_scope__ = RelocationScope.Global
+    __relocation_scope__ = RelocationScope.Config
 
     relay_id = models.CharField(max_length=64, unique=True)
     public_key = models.CharField(max_length=200)