migration: add columns for hashed values on ApiToken (#65300)

In support of our improved API tokens initiative
(https://github.com/getsentry/rfcs/pull/32), this PR adds two null-able
columns to the `ApiToken` model that will hold the hash values.

Hashed values for token values will be implemented over several PRs to
maintain backwards compatibility and to prevent broken state between
versions.

migrations_lockfile.txt

@@ -9,5 +9,5 @@ feedback: 0004_index_together
 hybridcloud: 0011_add_hybridcloudapitoken_index
 nodestore: 0002_nodestore_no_dictfield
 replays: 0004_index_together
-sentry: 0646_create_notification_message_table
+sentry: 0647_apitoken_add_hashed_columns
 social_auth: 0002_default_auto_field

src/sentry/backup/comparators.py

@@ -760,7 +760,9 @@ def get_default_comparators():
         list,
         {
             "sentry.apitoken": [
-                HashObfuscatingComparator("refresh_token", "token"),
+                HashObfuscatingComparator(
+                    "refresh_token", "token", "hashed_token", "hashed_refresh_token"
+                ),
                 IgnoredComparator("token_last_characters"),
                 UnorderedListComparator("scope_list"),
             ],

src/sentry/migrations/0632_apitoken_backfill_last_chars.py

@@ -7,7 +7,7 @@ from sentry.utils.query import RangeQuerySetWrapperWithProgressBar
 
 
 def backfill_last_token_characters(apps, _):
-    from sentry.models.apitoken import ApiToken
+    ApiToken = apps.get_model("sentry", "ApiToken")
 
     for api_token in RangeQuerySetWrapperWithProgressBar(ApiToken.objects.all()):
         if api_token.token_last_characters is None:

src/sentry/migrations/0647_apitoken_add_hashed_columns.py

@@ -0,0 +1,36 @@
+# Generated by Django 5.0.2 on 2024-02-16 00:23
+
+from django.db import migrations, models
+
+from sentry.new_migrations.migrations import CheckedMigration
+
+
+class Migration(CheckedMigration):
+    # This flag is used to mark that a migration shouldn't be automatically run in production. For
+    # the most part, this should only be used for operations where it's safe to run the migration
+    # after your code has deployed. So this should not be used for most operations that alter the
+    # schema of a table.
+    # Here are some things that make sense to mark as dangerous:
+    # - Large data migrations. Typically we want these to be run manually by ops so that they can
+    #   be monitored and not block the deploy for a long period of time while they run.
+    # - Adding indexes to large tables. Since this can take a long time, we'd generally prefer to
+    #   have ops run this and not block the deploy. Note that while adding an index is a schema
+    #   change, it's completely safe to run the operation after the code has deployed.
+    is_dangerous = False
+
+    dependencies = [
+        ("sentry", "0646_create_notification_message_table"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="apitoken",
+            name="hashed_refresh_token",
+            field=models.CharField(max_length=128, null=True),
+        ),
+        migrations.AddField(
+            model_name="apitoken",
+            name="hashed_token",
+            field=models.CharField(max_length=128, null=True),
+        ),
+    ]

src/sentry/models/apitoken.py

@@ -41,8 +41,10 @@ class ApiToken(ReplicatedControlModel, HasApiScopes):
     user = FlexibleForeignKey("sentry.User")
     name = models.CharField(max_length=255, null=True)
     token = models.CharField(max_length=64, unique=True, default=generate_token)
+    hashed_token = models.CharField(max_length=128, null=True)
     token_last_characters = models.CharField(max_length=4, null=True)
     refresh_token = models.CharField(max_length=64, unique=True, null=True, default=generate_token)
+    hashed_refresh_token = models.CharField(max_length=128, null=True)
     expires_at = models.DateTimeField(null=True, default=default_expiration)
     date_added = models.DateTimeField(default=timezone.now)
 

tests/sentry/migrations/test_0584_apitoken_add_name_and_last_chars.py

@@ -1,32 +0,0 @@
-from django.db import router
-
-from sentry.silo import unguarded_write
-from sentry.testutils.cases import TestMigrations
-from sentry.testutils.silo import no_silo_test
-
-
-@no_silo_test
-class NameLastCharsApiTokenMigrationTest(TestMigrations):
-    migrate_from = "0583_add_early_adopter_to_organization_mapping"
-    migrate_to = "0584_apitoken_add_name_and_last_chars"
-
-    def setUp(self):
-        from sentry.models.apitoken import ApiToken
-
-        with unguarded_write(using=router.db_for_write(ApiToken)):
-            super().setUp()
-
-    def setup_before_migration(self, apps):
-        ApiToken = apps.get_model("sentry", "ApiToken")
-        self.api_token = ApiToken.objects.create(
-            user_id=self.user.id,
-            refresh_token=None,
-        )
-        self.api_token.save()
-
-    def test(self):
-        from sentry.models.apitoken import ApiToken
-
-        api_token = ApiToken.objects.get()
-        assert api_token.name is None
-        assert api_token.token_last_characters is None

tests/sentry/migrations/test_0632_apitoken_backfill_last_chars.py

@@ -31,9 +31,6 @@ class LastCharsApiTokenMigrationTest(TestMigrations):
         assert self.api_token.token_last_characters is None
 
     def test(self):
-        from sentry.models.apitoken import ApiToken
-
-        api_tokens = ApiToken.objects.all()
-        for api_token in api_tokens:
-            assert api_token.name is None
-            assert api_token.token_last_characters == api_token.token[-4:]
+        self.api_token.refresh_from_db()
+        assert self.api_token.name is None
+        assert self.api_token.token_last_characters == self.api_token.token[-4:]