Update django import export to 4.x

Switch importer view to django ninja

apps/importer/api.py

@@ -0,0 +1,27 @@
+from django.shortcuts import aget_object_or_404
+from ninja import Router
+
+from apps.organizations_ext.models import Organization, OrganizationUserRole
+from glitchtip.api.authentication import AuthHttpRequest
+from glitchtip.api.permissions import has_permission
+
+from .importer import GlitchTipImporter
+from .schema import ImportIn
+
+router = Router()
+
+
+@router.post("import/")
+@has_permission(["org:admin"])
+async def importer(request: AuthHttpRequest, payload: ImportIn):
+    organization = await aget_object_or_404(
+        Organization,
+        slug=payload.organization_slug,
+        users=request.auth.user_id,
+        organization_users__role__gte=OrganizationUserRole.ADMIN,
+    )
+    importer = GlitchTipImporter(
+        str(payload.url), payload.auth_token, payload.organization_slug
+    )
+    await importer.check_auth()
+    await importer.run(organization_id=organization.id)

apps/importer/importer.py

@@ -1,5 +1,6 @@
-import requests
+import aiohttp
 import tablib
+from asgiref.sync import sync_to_async
 from django.db.models import Q
 from django.urls import reverse
 
@@ -7,6 +8,7 @@ from apps.organizations_ext.admin import OrganizationResource, OrganizationUserR
 from apps.organizations_ext.models import OrganizationUser, OrganizationUserRole
 from apps.projects.admin import ProjectKeyResource, ProjectResource
 from apps.projects.models import Project
+from apps.shared.types import TypeJson
 from apps.teams.admin import TeamResource
 from apps.users.admin import UserResource
 from apps.users.models import User
@@ -30,8 +32,7 @@ class GlitchTipImporter:
     def __init__(
         self, url: str, auth_token: str, organization_slug: str, create_users=False
     ):
-        self.api_root_url = reverse("api-root-view")
-        self.url = url
+        self.url = url.rstrip("/")
         self.headers = {"Authorization": f"Bearer {auth_token}"}
         self.create_users = create_users
         self.organization_slug = organization_slug
@@ -52,32 +53,34 @@ class GlitchTipImporter:
             kwargs={"organization_slug": self.organization_slug},
         )
 
-    def run(self, organization_id=None):
+    async def run(self, organization_id=None):
         """Set organization_id to None to import (superuser only)"""
         if organization_id is None:
-            self.import_organization()
+            await self.import_organization()
         else:
             self.organization_id = organization_id
-        self.import_organization_users()
-        self.import_projects()
-        self.import_teams()
+        await self.import_organization_users()
+        await self.import_projects()
+        await self.import_teams()
 
-    def get(self, url):
-        return requests.get(url, headers=self.headers)
+    async def get(self, url: str) -> TypeJson:
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url, headers=self.headers) as res:
+                return await res.json()
 
-    def import_organization(self):
+    async def import_organization(self):
         resource = OrganizationResource()
-        res = self.get(self.url + self.organization_url)
-        data = res.json()
+        data = await self.get(self.url + self.organization_url)
         self.organization_id = data["id"]  # TODO unsafe for web usage
         dataset = tablib.Dataset()
         dataset.dict = [data]
-        resource.import_data(dataset, raise_errors=True)
+        await sync_to_async(resource.import_data)(dataset, raise_errors=True)
 
-    def import_organization_users(self):
+    async def import_organization_users(self):
         resource = OrganizationUserResource()
-        res = self.get(self.url + self.organization_users_url)
-        org_users = res.json()
+        org_users = await self.get(self.url + self.organization_users_url)
+        if not org_users:
+            return
         if self.create_users:
             user_resource = UserResource()
             users_list = [
@@ -89,7 +92,7 @@ class GlitchTipImporter:
             ]
             dataset = tablib.Dataset()
             dataset.dict = users
-            user_resource.import_data(dataset, raise_errors=True)
+            await sync_to_async(user_resource.import_data)(dataset, raise_errors=True)
 
         for org_user in org_users:
             org_user["organization"] = self.organization_id
@@ -104,17 +107,16 @@ class GlitchTipImporter:
                 org_user["user"] = None
         dataset = tablib.Dataset()
         dataset.dict = org_users
-        resource.import_data(dataset, raise_errors=True)
+        await sync_to_async(resource.import_data)(dataset, raise_errors=True)
 
-    def import_projects(self):
+    async def import_projects(self):
         project_resource = ProjectResource()
         project_key_resource = ProjectKeyResource()
-        res = self.get(self.url + self.projects_url)
-        projects = res.json()
+        projects = await self.get(self.url + self.projects_url)
         project_keys = []
         for project in projects:
             project["organization"] = self.organization_id
-            keys = self.get(
+            keys = await self.get(
                 self.url
                 + reverse(
                     "project-keys-list",
@@ -122,46 +124,53 @@ class GlitchTipImporter:
                         "project_pk": f"{self.organization_slug}/{project['slug']}",
                     },
                 )
-            ).json()
+            )
             for key in keys:
                 key["project"] = project["id"]
                 key["public_key"] = key["public"]
             project_keys += keys
         dataset = tablib.Dataset()
         dataset.dict = projects
-        project_resource.import_data(dataset, raise_errors=True)
-        owned_project_ids = Project.objects.filter(
-            organization_id=self.organization_id,
-            pk__in=[d["projectId"] for d in project_keys],
-        ).values_list("pk", flat=True)
+        await sync_to_async(project_resource.import_data)(dataset, raise_errors=True)
+        owned_project_ids = [
+            pk
+            async for pk in Project.objects.filter(
+                organization_id=self.organization_id,
+                pk__in=[d["projectId"] for d in project_keys],
+            ).values_list("pk", flat=True)
+        ]
         project_keys = list(
             filter(lambda key: key["projectId"] in owned_project_ids, project_keys)
         )
         dataset.dict = project_keys
-        project_key_resource.import_data(dataset, raise_errors=True)
+        await sync_to_async(project_key_resource.import_data)(
+            dataset, raise_errors=True
+        )
 
-    def import_teams(self):
+    async def import_teams(self):
         resource = TeamResource()
-        res = self.get(self.url + self.teams_url)
-        teams = res.json()
+        teams = await self.get(self.url + self.teams_url)
         for team in teams:
             team["organization"] = self.organization_id
             team["projects"] = ",".join(
                 map(
                     str,
-                    Project.objects.filter(
-                        organization_id=self.organization_id,
-                        pk__in=[int(d["id"]) for d in team["projects"]],
-                    ).values_list("id", flat=True),
+                    [
+                        pk
+                        async for pk in Project.objects.filter(
+                            organization_id=self.organization_id,
+                            pk__in=[int(d["id"]) for d in team["projects"]],
+                        ).values_list("id", flat=True)
+                    ],
                 )
             )
-            team_members = self.get(
+            team_members = await self.get(
                 self.url
                 + reverse(
                     "team-members-list",
                     kwargs={"team_pk": f"{self.organization_slug}/{team['slug']}"},
                 )
-            ).json()
+            )
             team_member_emails = [d["email"] for d in team_members]
             team["members"] = ",".join(
                 [
@@ -178,10 +187,11 @@ class GlitchTipImporter:
             )
         dataset = tablib.Dataset()
         dataset.dict = teams
-        resource.import_data(dataset, raise_errors=True)
+        await sync_to_async(resource.import_data)(dataset, raise_errors=True)
 
-    def check_auth(self):
-        res = requests.get(self.url + self.api_root_url, headers=self.headers)
-        data = res.json()
-        if res.status_code != 200 or not data["user"]:
-            raise ImporterException("Bad auth token")
+    async def check_auth(self):
+        async with aiohttp.ClientSession() as session:
+            async with session.get(self.url + "/api/0/", headers=self.headers) as res:
+                data = await res.json()
+                if res.status != 200 or not data["user"]:
+                    raise ImporterException("Bad auth token")

apps/importer/management/commands/import.py

@@ -1,3 +1,4 @@
+from asgiref.sync import async_to_sync
 from django.core.management.base import BaseCommand
 
 from apps.importer.importer import GlitchTipImporter
@@ -11,12 +12,13 @@ class Command(BaseCommand):
         parser.add_argument("auth_token", type=str)
         parser.add_argument("organization_slug", type=str)
 
-    def handle(self, *args, **options):
+    @async_to_sync
+    async def handle(self, *args, **options):
         url = options["url"].rstrip("/")
         if not url.startswith("http"):
             url = "https://" + url
         importer = GlitchTipImporter(
             url, options["auth_token"], options["organization_slug"], create_users=True
         )
-        importer.check_auth()
-        importer.run()
+        await importer.check_auth()
+        await importer.run()

apps/importer/schema.py

@@ -0,0 +1,18 @@
+from pydantic import HttpUrl
+
+from glitchtip.schema import CamelSchema
+
+
+class ImportIn(CamelSchema):
+    url: HttpUrl
+    auth_token: str
+    organization_slug: str
+
+    # def __init__(self, context, *args, **kwargs):
+    #     if user := context["request"].user:
+    #         self.fields[
+    #             "organizationSlug"
+    #         ].queryset = user.organizations_ext_organization.filter(
+    #             organization_users__role__gte=OrganizationUserRole.ADMIN
+    #         )
+    #     return super().__init__(*args, **kwargs)

apps/importer/serializers.py

@@ -1,20 +0,0 @@
-from rest_framework import serializers
-
-from apps.organizations_ext.models import Organization, OrganizationUserRole
-
-
-class ImportSerializer(serializers.Serializer):
-    url = serializers.URLField()
-    authToken = serializers.CharField()
-    organizationSlug = serializers.SlugRelatedField(
-        slug_field="slug", queryset=Organization.objects.none()
-    )
-
-    def __init__(self, context, *args, **kwargs):
-        if user := context["request"].user:
-            self.fields[
-                "organizationSlug"
-            ].queryset = user.organizations_ext_organization.filter(
-                organization_users__role__gte=OrganizationUserRole.ADMIN
-            )
-        return super().__init__(*args, **kwargs)

apps/importer/tests.py

@@ -1,11 +1,13 @@
-import requests_mock
+from aioresponses import aioresponses
 from django.core.management import call_command
+from django.test import TestCase
 from django.urls import reverse
 from model_bakery import baker
 
 from apps.projects.models import Project
 from apps.teams.models import Team
-from glitchtip.test_utils.test_case import GlitchTipTestCase
+from glitchtip.test_utils import generators  # noqa: F401
+from glitchtip.test_utils.test_case import GlitchTipTestCaseMixin
 
 from .importer import GlitchTipImporter
 
@@ -18,7 +20,7 @@ test_key = {
 }
 
 
-class ImporterTestCase(GlitchTipTestCase):
+class ImporterTestCase(GlitchTipTestCaseMixin, TestCase):
     def setUp(self):
         self.url = "https://example.com"
         self.org_name = "org"
@@ -28,14 +30,14 @@ class ImporterTestCase(GlitchTipTestCase):
         )
 
     def set_mocks(self, m):
-        m.get(self.url + self.importer.api_root_url, json={"user": {"username": "foo"}})
-        m.get(self.url + self.importer.organization_url, json={"id": 1})
-        m.get(self.url + self.importer.organization_users_url, json=[])
-        m.get(self.url + self.importer.projects_url, json=[test_project])
-        m.get(self.url + "/api/0/projects/org/project/keys/", json=[test_key])
+        m.get(self.url + "/api/0/", payload={"user": {"username": "foo"}})
+        m.get(self.url + self.importer.organization_url, payload={"id": 1})
+        m.get(self.url + self.importer.organization_users_url, payload=[])
+        m.get(self.url + self.importer.projects_url, payload=[test_project])
+        m.get(self.url + "/api/0/projects/org/project/keys/", payload=[test_key])
         m.get(
             self.url + self.importer.teams_url,
-            json=[
+            payload=[
                 {
                     "id": "1",
                     "slug": "team",
@@ -43,9 +45,9 @@ class ImporterTestCase(GlitchTipTestCase):
                 }
             ],
         )
-        m.get(self.url + "/api/0/teams/org/team/members/", json=[])
+        m.get(self.url + "/api/0/teams/org/team/members/", payload=[])
 
-    @requests_mock.Mocker()
+    @aioresponses()
     def test_import_command(self, m):
         self.set_mocks(m)
 
@@ -59,26 +61,26 @@ class ImporterTestCase(GlitchTipTestCase):
             ).exists()
         )
 
-    @requests_mock.Mocker()
+    @aioresponses()
     def test_view(self, m):
-        self.create_user_and_project()
+        self.create_logged_in_user()
         self.organization.slug = self.org_name
         self.organization.save()
         self.set_mocks(m)
-        url = reverse("import")
+        url = reverse("api:importer")
         data = {
             "url": self.url,
             "authToken": self.auth_token,
             "organizationSlug": self.org_name,
         }
-        res = self.client.post(url, data)
+        res = self.client.post(url, data, content_type="application/json")
         self.assertEqual(res.status_code, 200)
         self.assertTrue(Team.objects.filter(slug="team").exists())
 
-    @requests_mock.Mocker()
+    @aioresponses()
     def test_invalid_org(self, m):
-        self.create_user_and_project()
-        url = reverse("import")
+        self.create_logged_in_user()
+        url = reverse("api:importer")
         data = {
             "url": self.url,
             "authToken": self.auth_token,
@@ -92,6 +94,6 @@ class ImporterTestCase(GlitchTipTestCase):
         res = self.client.post(url, data)
         self.assertEqual(res.status_code, 400)
         other_org.add_user(self.user)
-        m.get(self.url + self.importer.api_root_url, json={"user": {"username": "foo"}})
+        m.get(self.url + "api/0/", payload={"user": {"username": "foo"}})
         res = self.client.post(url, data)
         self.assertEqual(res.status_code, 400)

apps/importer/urls.py

@@ -1,5 +0,0 @@
-from django.urls import path
-
-from .views import ImportAPIView
-
-urlpatterns = [path("import", ImportAPIView.as_view(), name="import")]

apps/importer/views.py

@@ -1,25 +0,0 @@
-from rest_framework import views
-from rest_framework.response import Response
-
-from .importer import GlitchTipImporter
-from .serializers import ImportSerializer
-
-
-class ImportAPIView(views.APIView):
-    """Import members, projects, and teams for an organization of which you are an Admin of"""
-
-    serializer_class = ImportSerializer
-
-    def post(self, request):
-        serializer = self.serializer_class(
-            data=request.data, context={"request": request}
-        )
-        serializer.is_valid(raise_exception=True)
-        data = serializer.validated_data
-        importer = GlitchTipImporter(
-            data["url"], data["authToken"], data["organizationSlug"].slug
-        )
-        importer.check_auth()
-        importer.run(organization_id=data["organizationSlug"].pk)
-
-        return Response()

apps/shared/types.py

@@ -0,0 +1,7 @@
+from typing import Union
+
+TypeJson = Union[
+    dict[str, Union[str, int, bool, "TypeJson"]],
+    list[Union[str, int, bool, "TypeJson"]],
+]
+"""Python object that is valid JSON (can be serialized to and from)"""

glitchtip/api/api.py

@@ -14,6 +14,7 @@ from sentry_sdk import capture_exception, set_context, set_level
 from apps.api_tokens.api import router as api_tokens_router
 from apps.event_ingest.api import router as event_ingest_router
 from apps.event_ingest.embed_api import router as embed_router
+from apps.importer.api import router as importer_router
 from apps.issue_events.api import router as issue_events_router
 from apps.teams.api import router as teams_router
 from apps.users.utils import ais_user_registration_open
@@ -40,6 +41,7 @@ api = NinjaAPI(
 
 api.add_router("0", api_tokens_router)
 api.add_router("", event_ingest_router)
+api.add_router("0", importer_router)
 api.add_router("0", issue_events_router)
 api.add_router("0", teams_router)
 api.add_router("embed", embed_router)