feat(apidocs): Integrate drf-spectacular library (#30040)

.github/workflows/api-docs-test.yml

@@ -37,11 +37,6 @@ jobs:
           pip-cache-version: ${{ secrets.PIP_CACHE_VERSION }}
           snuba: true
 
-      - name: Install Javascript Dependencies
-        if: steps.changes.outputs.api_docs == 'true'
-        run: |
-          cd api-docs && yarn install --frozen-lockfile
-
       - name: Run API docs tests
         if: steps.changes.outputs.api_docs == 'true'
         run: |

.github/workflows/openapi-diff.yml

@@ -1,42 +1,64 @@
 name: openapi-diff
 on:
   pull_request:
-    paths:
-      - 'api-docs/**'
 
 jobs:
-    check-diff:
-      runs-on: ubuntu-18.04
-      steps:
-        - name: Checkout getsentry/sentry
-          uses: actions/checkout@v2
-          with:
-            path: sentry
-
-        - name: Getsentry Token
-          id: getsentry
-          uses: getsentry/action-github-app-token@v1
-          with:
-            app_id: ${{ secrets.SENTRY_INTERNAL_APP_ID }}
-            private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }}
-
-        - name: Checkout getsentry/sentry-api-schema
-          uses: actions/checkout@v2
-          with:
-            ref: 'main'
-            repository: getsentry/sentry-api-schema
-            path: sentry-api-schema
-            token: ${{ steps.getsentry.outputs.token }}
-
-        - name: Install/setup node
-          uses: volta-cli/action@v1
-
-        - name: Build OpenAPI Derefed JSON
-          run: |
-            cd sentry
-            yarn install --frozen-lockfile
-            yarn run build-derefed-docs api-docs/openapi-derefed.json
-
-        - name: Compare OpenAPI Derefed JSON
-          run: |
-            npx json-diff --color sentry-api-schema/openapi-derefed.json sentry/api-docs/openapi-derefed.json
+  check-diff:
+    name: build api
+    runs-on: ubuntu-20.04
+    timeout-minutes: 90
+    strategy:
+      matrix:
+        python-version: [3.8.12]
+
+    steps:
+      - name: Getsentry Token
+        id: getsentry
+        uses: getsentry/action-github-app-token@v1
+        with:
+          app_id: ${{ secrets.SENTRY_INTERNAL_APP_ID }}
+          private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }}
+
+      - uses: actions/checkout@v2
+        with:
+          # Avoid codecov error message related to SHA resolution:
+          # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891
+          fetch-depth: '2'
+
+      - name: Check for python file changes
+        uses: getsentry/paths-filter@v2
+        id: changes
+        with:
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+      - name: Setup sentry env (python ${{ matrix.python-version }})
+        uses: ./.github/actions/setup-sentry
+        if: steps.changes.outputs.api_docs == 'true'
+        id: setup
+        with:
+          python-version: ${{ matrix.python-version }}
+          pip-cache-version: ${{ secrets.PIP_CACHE_VERSION }}
+
+      - name: Checkout getsentry/sentry-api-schema
+        uses: actions/checkout@v2
+        if: steps.changes.outputs.api_docs == 'true'
+        with:
+          ref: 'main'
+          repository: getsentry/sentry-api-schema
+          path: sentry-api-schema
+          token: ${{ steps.getsentry.outputs.token }}
+
+      - name: Install/setup node
+        if: steps.changes.outputs.api_docs == 'true'
+        uses: volta-cli/action@v1
+
+      - name: Build OpenAPI Derefed JSON
+        if: steps.changes.outputs.api_docs == 'true'
+        run: |
+          make build-api-docs
+
+      - name: Compare OpenAPI Derefed JSON
+        if: steps.changes.outputs.api_docs == 'true'
+        run: |
+          npx json-diff --color sentry-api-schema/openapi-derefed.json tests/apidocs/openapi-derefed.json

.github/workflows/openapi.yml

@@ -1,51 +1,74 @@
 name: openapi
-
 on:
   push:
     branches:
       - master
 
 jobs:
-  deref_json:
-    runs-on: ubuntu-latest
+  build_and_deref_json:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 90
+    strategy:
+      matrix:
+        python-version: [3.8.12]
+
     steps:
-    - name: Getsentry Token
-      id: getsentry
-      uses: getsentry/action-github-app-token@v1
-      with:
-        app_id: ${{ secrets.SENTRY_INTERNAL_APP_ID }}
-        private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }}
-
-    - name: Checkout getsentry/sentry
-      uses: actions/checkout@v2
-      with:
-        path: sentry
-
-    - name: Checkout getsentry/sentry-api-schema
-      uses: actions/checkout@v2
-      with:
-        ref: 'main'
-        repository: getsentry/sentry-api-schema
-        path: sentry-api-schema
-        token: ${{ steps.getsentry.outputs.token }}
-
-    - uses: volta-cli/action@v1
-
-    - name: Build OpenAPI Derefed JSON
-      run: |
-        cd sentry
-        yarn install --frozen-lockfile
-        yarn run build-derefed-docs api-docs/openapi-derefed.json
-
-    - name: Copy artifact into getsentry/sentry-api-schema
-      run: |
-        cp sentry/api-docs/openapi-derefed.json sentry-api-schema
-
-    - name: Git Commit & Push
-      uses: stefanzweifel/git-auto-commit-action@v4
-      with:
-        repository: sentry-api-schema
-        branch: main
-        commit_message: Generated
-        commit_user_email: bot@getsentry.com
-        commit_user_name: openapi-getsentry-bot
+      - name: Getsentry Token
+        id: getsentry
+        uses: getsentry/action-github-app-token@v1
+        with:
+          app_id: ${{ secrets.SENTRY_INTERNAL_APP_ID }}
+          private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }}
+
+      - uses: actions/checkout@v2
+        with:
+          # Avoid codecov error message related to SHA resolution:
+          # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891
+          fetch-depth: '2'
+
+      - name: Check for python file changes
+        uses: getsentry/paths-filter@v2
+        id: changes
+        with:
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+      - name: Setup sentry env (python ${{ matrix.python-version }})
+        uses: ./.github/actions/setup-sentry
+        if: steps.changes.outputs.api_docs == 'true'
+        id: setup
+        with:
+          python-version: ${{ matrix.python-version }}
+          pip-cache-version: ${{ secrets.PIP_CACHE_VERSION }}
+
+      - name: Checkout getsentry/sentry-api-schema
+        if: steps.changes.outputs.api_docs == 'true'
+        uses: actions/checkout@v2
+        with:
+          ref: 'main'
+          repository: getsentry/sentry-api-schema
+          path: sentry-api-schema
+          token: ${{ steps.getsentry.outputs.token }}
+
+      - name: Install/setup node
+        if: steps.changes.outputs.api_docs == 'true'
+        uses: volta-cli/action@v1
+
+      - name: Build OpenAPI Derefed JSON
+        if: steps.changes.outputs.api_docs == 'true'
+        run: |
+          make build-api-docs
+
+      - name: Copy artifact into getsentry/sentry-api-schema
+        if: steps.changes.outputs.api_docs == 'true'
+        run: |
+          cp tests/apidocs/openapi-derefed.json sentry-api-schema
+
+      - name: Git Commit & Push
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          repository: sentry-api-schema
+          branch: main
+          commit_message: Generated
+          commit_user_email: bot@getsentry.com
+          commit_user_name: openapi-getsentry-bot

.gitignore

@@ -34,7 +34,9 @@ node_modules
 /config/chartcuterie/config.js*
 /config/*.pem
 /tests/apidocs/openapi-derefed.json
+/tests/apidocs/openapi-deprecated.json
 /tests/sentry/utils/appleconnect/*.json
+/tests/apidocs/openapi-spectacular.json
 tests/fixtures/integrations/*/data/**
 /wheelhouse
 /test_cli/

Makefile

@@ -32,6 +32,21 @@ build-js-po: node-version-check
 	rm -rf node_modules/.cache/babel-loader
 	SENTRY_EXTRACT_TRANSLATIONS=1 $(WEBPACK)
 
+build-spectacular-docs:
+	@echo "--> Building drf-spectacular openapi spec (combines with deprecated docs)"
+	@OPENAPIGENERATE=1 sentry django spectacular --file tests/apidocs/openapi-spectacular.json --format openapi-json --validate --fail-on-warn
+
+build-deprecated-docs:
+	@echo "--> Building deprecated openapi spec from json files"
+	yarn build-deprecated-docs
+
+build-api-docs: build-deprecated-docs build-spectacular-docs
+	@echo "--> Dereference the json schema for ease of use"
+	yarn deref-api-docs
+
+watch-api-docs:
+	@node api-docs/watch.js
+
 build: locale
 
 merge-locale-catalogs: build-js-po
@@ -145,10 +160,7 @@ test-relay-integration:
 	pytest tests/relay_integration -vv
 	@echo ""
 
-test-api-docs:
-	@echo "--> Generating testing api doc schema"
-	yarn run build-derefed-docs
-	@echo "--> Validating endpoints' examples against schemas"
+test-api-docs: build-api-docs
 	yarn run validate-api-examples
 	pytest tests/apidocs/endpoints
 	@echo ""

api-docs/index.js

@@ -1,5 +1,6 @@
 /* global process */
 /* eslint import/no-nodejs-modules:0 */
+/* eslint import/no-unresolved:0 */
 
 const fs = require('fs');
 const path = require('path');
@@ -18,26 +19,27 @@ function dictToString(dict) {
 function bundle(originalFile) {
   const root = yaml.safeLoad(fs.readFileSync(originalFile, 'utf8'));
   const options = {
-    filter: ['relative', 'remote'],
+    filter: ['relative', 'remote', 'local'],
     resolveCirculars: true,
     location: originalFile,
     loaderOptions: {
-      processContent: function(res, callback) {
+      processContent: function (res, callback) {
         callback(undefined, yaml.safeLoad(res.text));
       },
     },
   };
   JsonRefs.clearCache();
   return JsonRefs.resolveRefs(root, options).then(
-    function(results) {
+    function (results) {
       const resErrors = {};
       for (const [k, v] of Object.entries(results.refs)) {
         if (
           'missing' in v &&
           v.missing === true &&
           (v.type === 'relative' || v.type === 'remote')
-        )
+        ) {
           resErrors[k] = v.error;
+        }
       }
       if (Object.keys(resErrors).length > 0) {
         return Promise.reject(new Error(dictToString(resErrors)));
@@ -45,9 +47,9 @@ function bundle(originalFile) {
 
       return results.resolved;
     },
-    function(e) {
+    function (e) {
       const error = {};
-      Object.getOwnPropertyNames(e).forEach(function(key) {
+      Object.getOwnPropertyNames(e).forEach(function (key) {
         error[key] = e[key];
       });
       return Promise.reject(new Error(dictToString(error)));
@@ -57,10 +59,10 @@ function bundle(originalFile) {
 
 function build(originalFile, _, bundleTo) {
   bundle(originalFile).then(
-    function(bundled) {
+    function (bundled) {
       const bundleString = JSON.stringify(bundled, null, 2);
       if (typeof bundleTo === 'string') {
-        fs.writeFile(bundleTo, bundleString, function(err) {
+        fs.writeFile(bundleTo, bundleString, function (err) {
           if (err) {
             // eslint-disable-next-line no-console
             console.log(err);
@@ -71,7 +73,7 @@ function build(originalFile, _, bundleTo) {
         });
       }
     },
-    function(err) {
+    function (err) {
       // eslint-disable-next-line no-console
       console.log(err);
     }

api-docs/watch.js

@@ -0,0 +1,23 @@
+/* eslint-env node */
+/* eslint import/no-nodejs-modules:0 no-console:0 */
+
+const sane = require('sane');
+const {execSync} = require('child_process');
+
+const watcherPy = sane('src/sentry');
+const watcherJson = sane('api-docs');
+
+const watchers = [watcherPy, watcherJson];
+
+const makeApiDocsCommand = function () {
+  console.log('rebuilding...');
+  output = execSync('make build-api-docs');
+  console.log(output.toString());
+};
+
+for (const w of watchers) {
+  w.on('change', makeApiDocsCommand);
+  w.on('add', makeApiDocsCommand);
+  w.on('delete', makeApiDocsCommand);
+}
+console.log('rebuilding API docs on changes');

package.json

@@ -225,14 +225,14 @@
     "storybook-build": "build-storybook -c .storybook -o docs-ui/.storybook-out --quiet",
     "webpack-profile": "NO_TS_FORK=1 yarn -s webpack --profile --json > stats.json",
     "install-api-docs": "cd api-docs && yarn install",
-    "build-derefed-docs": "yarn install-api-docs && node api-docs/index.js api-docs/openapi.json",
-    "watch-api-docs": "sane 'yarn build-derefed-docs' api-docs",
+    "build-deprecated-docs": "yarn install-api-docs && node api-docs/index.js api-docs/openapi.json tests/apidocs/openapi-deprecated.json",
+    "deref-api-docs": "node api-docs/index.js tests/apidocs/openapi-spectacular.json tests/apidocs/openapi-derefed.json",
     "build-css": "NODE_ENV=production yarn webpack --config=config/webpack.css.config.ts",
     "build-chartcuterie-config": "NODE_ENV=production yarn webpack --config=config/webpack.chartcuterie.config.ts",
     "build-acceptance": "IS_ACCEPTANCE_TEST=1 NODE_ENV=production yarn webpack --mode development",
     "build-production": "NODE_ENV=production yarn webpack --mode production",
     "build": "NODE_OPTIONS=--max-old-space-size=4096 yarn webpack",
-    "validate-api-examples": "yarn install-api-docs && cd api-docs && yarn openapi-examples-validator ./openapi.json --no-additional-properties",
+    "validate-api-examples": "cd api-docs && yarn openapi-examples-validator ./openapi.json --no-additional-properties",
     "mkcert-localhost": "mkcert -key-file config/localhost-key.pem -cert-file config/localhost.pem localhost"
   },
   "browserslist": {

requirements-base.txt

@@ -14,6 +14,7 @@ django-picklefield==2.1.0
 django-pg-zero-downtime-migrations==0.10
 Django==2.2.24
 djangorestframework==3.11.2
+drf-spectacular==0.21.0
 email-reply-parser==0.5.12
 google-api-core==1.25.1
 google-auth==1.24.0