Ship integration docs statically with the build

.gitignore

@@ -26,5 +26,6 @@ example/db.sqlite
 /src/sentry/static/sentry/admin/
 /src/sentry/static/sentry/debug_toolbar/
 /src/sentry/static/sentry/rest_framework/
+/src/sentry/integration-docs
 /wheelhouse
 /test_cli/

Makefile

@@ -44,7 +44,9 @@ build: locale
 
 clean:
 	@echo "--> Cleaning static cache"
-	rm dist/* static/dist/*
+	rm -f dist/* static/dist/*
+	@echo "--> Cleaning integration docs cache"
+	rm -rf src/sentry/integration-docs
 	@echo "--> Cleaning pyc files"
 	find . -name "*.pyc" -delete
 	@echo ""

setup.py

@@ -49,6 +49,10 @@ from subprocess import check_output
 # The version of sentry
 VERSION = '8.0.0rc1'
 
+# Also see sentry.utils.integrationdocs.DOC_FOLDER
+INTEGRATION_DOC_FOLDER = os.path.join(os.path.abspath(
+    os.path.dirname(__file__)), 'src', 'sentry', 'integration-docs')
+
 
 # Hack to prevent stupid "TypeError: 'NoneType' object is not callable" error
 # in multiprocessing/util.py _exit_function when running `python
@@ -214,6 +218,12 @@ class BuildJavascriptCommand(Command):
             except (OSError, IOError):
                 pass
 
+            log.info('cleaning out integration docs folder')
+            try:
+                shutil.rmtree(INTEGRATION_DOC_FOLDER)
+            except (OSError, IOError):
+                pass
+
         # In place means build_lib is src.  We also log this.
         if self.inplace:
             log.info('In-place js building enabled')
@@ -276,34 +286,46 @@ class BuildJavascriptCommand(Command):
             return True
         return False
 
+    def _needs_integration_docs(self):
+        return not os.path.isdir(INTEGRATION_DOC_FOLDER)
+
     def run(self):
+        need_integration_docs = not os.path.isdir(INTEGRATION_DOC_FOLDER)
         version_info = self._get_package_version()
+
         if not (self.force or self._needs_static(version_info)):
             log.info("skipped asset build (version already built)")
-            self.update_manifests()
-            return
+        else:
+            log.info("building assets for Sentry v{} (build {})".format(
+                version_info['version'] or 'UNKNOWN',
+                version_info['build'] or 'UNKNOWN',
+            ))
+            if not version_info['version'] or not version_info['build']:
+                log.fatal('Could not determine sentry version or build')
+                sys.exit(1)
 
-        log.info("building assets for Sentry v{} (build {})".format(
-            version_info['version'] or 'UNKNOWN',
-            version_info['build'] or 'UNKNOWN',
-        ))
-        if not version_info['version'] or not version_info['build']:
-            log.fatal('Could not determine sentry version or build')
-            sys.exit(1)
+            try:
+                self._build_static()
+            except Exception:
+                traceback.print_exc()
+                log.fatal("unable to build Sentry's static assets!\n"
+                          "Hint: You might be running an invalid version of NPM.")
+                sys.exit(1)
+
+            log.info("writing version manifest")
+            manifest = self._write_version_file(version_info)
+            log.info("recorded manifest\n{}".format(
+                json.dumps(manifest, indent=2),
+            ))
+            need_integration_docs = True
+
+        if not need_integration_docs:
+            log.info('skipped integration docs (already downloaded)')
+        else:
+            log.info('downloading integration docs')
+            from sentry.utils.integrationdocs import sync_docs
+            sync_docs()
 
-        try:
-            self._build_static()
-        except Exception:
-            traceback.print_exc()
-            log.fatal("unable to build Sentry's static assets!\n"
-                      "Hint: You might be running an invalid version of NPM.")
-            sys.exit(1)
-
-        log.info("writing version manifest")
-        manifest = self._write_version_file(version_info)
-        log.info("recorded manifest\n{}".format(
-            json.dumps(manifest, indent=2),
-        ))
         self.update_manifests()
 
     def update_manifests(self):
@@ -320,16 +342,17 @@ class BuildJavascriptCommand(Command):
         # Use the underlying file list so that we skip the file-exists
         # check which we do not want here.
         files = sdist.filelist.files
-        root = self.sentry_static_dist_path
+        base = os.path.abspath('.')
 
         # We need to split off the local parts of the files relative to
         # the current folder.  This will chop off the right path for the
         # manifest.
-        base = os.path.abspath('.')
-        for dirname, dirnames, filenames in os.walk(root):
-            for filename in filenames:
-                filename = os.path.join(root, filename)
-                files.append(filename[len(base):].lstrip(os.path.sep))
+        for root in self.sentry_static_dist_path, INTEGRATION_DOC_FOLDER:
+            for dirname, dirnames, filenames in os.walk(root):
+                for filename in filenames:
+                    filename = os.path.join(root, filename)
+                    files.append(filename[len(base):].lstrip(os.path.sep))
+
         files.append('src/sentry/sentry-package.json')
 
     def _build_static(self):

src/sentry/api/endpoints/project_docs.py

@@ -2,14 +2,16 @@ from __future__ import absolute_import
 
 from rest_framework.response import Response
 
-from sentry import options
 from sentry.api.bases.project import ProjectEndpoint
 from sentry.models import ProjectKey
+from sentry.utils.integrationdocs import load_doc
 
 
 class ProjectDocsEndpoint(ProjectEndpoint):
     def get(self, request, project):
-        data = options.get('sentry:docs')
+        data = load_doc('_platforms')
+        if data is None:
+            raise RuntimeError('Docs not built')
         project_key = ProjectKey.get_default(project)
 
         context = {

src/sentry/api/endpoints/project_docs_platform.py

@@ -2,10 +2,10 @@ from __future__ import absolute_import
 
 from rest_framework.response import Response
 
-from sentry import options
 from sentry.api.bases.project import ProjectEndpoint
 from sentry.api.exceptions import ResourceDoesNotExist
 from sentry.models import ProjectKey
+from sentry.utils.integrationdocs import load_doc
 
 
 def replace_keys(html, project_key):
@@ -21,7 +21,7 @@ def replace_keys(html, project_key):
 
 class ProjectDocsPlatformEndpoint(ProjectEndpoint):
     def get(self, request, project, platform):
-        data = options.get('sentry:docs:{}'.format(platform))
+        data = load_doc(platform)
         if not data:
             raise ResourceDoesNotExist
 

src/sentry/tasks/sync_docs.py → src/sentry/utils/integrationdocs.py

@@ -1,29 +1,57 @@
-from __future__ import absolute_import, print_function
+from __future__ import absolute_import
 
+import os
+import json
 import logging
 
-from sentry.tasks.base import instrumented_task
+import sentry
+
 
 BASE_URL = 'https://docs.getsentry.com/hosted/_platforms/{}'
 
+# Also see INTEGRATION_DOC_FOLDER in setup.py
+DOC_FOLDER = os.path.abspath(os.path.join(os.path.dirname(sentry.__file__),
+                                          'integration-docs'))
+
+
 logger = logging.getLogger('sentry')
 
 
+def dump_doc(path, data):
+    fn = os.path.join(DOC_FOLDER, path + '.json')
+    directory = os.path.dirname(fn)
+    try:
+        os.makedirs(directory)
+    except OSError:
+        pass
+    with open(fn, 'wb') as f:
+        json.dump(data, f, indent=2)
+        f.write('\n')
+
+
+def load_doc(path):
+    if '/' in path:
+        return None
+    fn = os.path.join(DOC_FOLDER, path + '.json')
+    try:
+        with open(fn, 'rb') as f:
+            return json.load(f)
+    except IOError:
+        return None
+
+
 def get_integration_id(platform_id, integration_id):
     if integration_id == '_self':
         return platform_id
     return '{}-{}'.format(platform_id, integration_id)
 
 
-@instrumented_task(name='sentry.tasks.sync_docs', queue='update',
-                   time_limit=15,
-                   soft_time_limit=10)
 def sync_docs():
-    from sentry import http, options
+    from requests import Session
 
-    session = http.build_session()
+    session = Session()
 
-    logger.info('Syncing documentation (platform index)')
+    print 'syncing documentation (platform index)'
     data = session.get(BASE_URL.format('_index.json')).json()
     platform_list = []
     for platform_id, integrations in data['platforms'].iteritems():
@@ -45,29 +73,23 @@ def sync_docs():
 
     platform_list.sort(key=lambda x: x['name'])
 
-    options.set('sentry:docs', {'platforms': platform_list})
+    dump_doc('_platforms', {'platforms': platform_list})
 
     for platform_id, platform_data in data['platforms'].iteritems():
         for integration_id, integration in platform_data.iteritems():
-            sync_integration_docs.delay(platform_id, integration_id,
-                                        integration['details'])
-
+            sync_integration_docs(session, platform_id, integration_id,
+                                  integration['details'])
 
-@instrumented_task(name='sentry.tasks.sync_integration_docs', queue='update',
-                   time_limit=15,
-                   soft_time_limit=10)
-def sync_integration_docs(platform_id, integration_id, path):
-    from sentry import http, options
 
-    logger.info('Syncing documentation for %s.%s integration',
-                platform_id, integration_id)
-
-    session = http.build_session()
+def sync_integration_docs(session, platform_id, integration_id, path):
+    print '  syncing documentation for %s.%s integration' % (
+        platform_id, integration_id)
 
     data = session.get(BASE_URL.format(path)).json()
 
     key = get_integration_id(platform_id, integration_id)
-    options.set('sentry:docs:{}'.format(key), {
+
+    dump_doc(key, {
         'id': key,
         'name': data['name'],
         'html': data['body'],

tests/sentry/tasks/test_sync_docs.py

@@ -1,123 +0,0 @@
-from __future__ import absolute_import, print_function
-
-import mock
-import responses
-
-from sentry import options
-from sentry.testutils import TestCase
-from sentry.tasks.sync_docs import sync_docs, sync_integration_docs
-
-INDEX_JSON = """
-{
-  "platforms": {
-    "go": {
-      "_self": {
-        "doc_link": "https://docs.getsentry.com/hosted/clients/go/",
-        "name": "Go",
-        "details": "go.json",
-        "type": "language"
-      },
-      "http": {
-        "doc_link": null,
-        "name": "net/http",
-        "details": "go/http.json",
-        "type": "framework"
-      }
-    }
-  }
-}
-""".strip()
-
-GO_JSON = """
-{
-  "doc_link": "https://docs.getsentry.com/hosted/clients/go/",
-  "name": "Go",
-  "type": "language",
-  "support_level": null,
-  "body": "foo bar"
-}
-""".strip()
-
-GO_HTTP_JSON = """
-{
-  "doc_link": null,
-  "name": "net/http",
-  "type": "framework",
-  "support_level": null,
-  "body": "foo bar"
-}
-""".strip()
-
-
-class SyncDocsTest(TestCase):
-    @responses.activate
-    @mock.patch('sentry.tasks.sync_docs.sync_integration_docs')
-    def test_simple(self, mock_sync_integration_docs):
-        responses.add('GET',
-                      'https://docs.getsentry.com/hosted/_platforms/_index.json',
-                      body=INDEX_JSON)
-
-        sync_docs()
-
-        data = options.get('sentry:docs')
-        assert data == {
-            'platforms': [
-                {
-                    'id': 'go',
-                    'integrations': [
-                        {
-                            'id': 'go',
-                            'link': 'https://docs.getsentry.com/hosted/clients/go/',
-                            'name': 'Go',
-                            'type': 'language'
-                        },
-                        {
-                            'id': 'go-http',
-                            'link': None,
-                            'name': 'net/http',
-                            'type': 'framework'
-                        }
-                    ],
-                    'name': 'Go',
-                }
-            ]
-        }
-
-        assert mock_sync_integration_docs.mock_calls == [
-            mock.call.delay('go', '_self', 'go.json'),
-            mock.call.delay('go', 'http', 'go/http.json'),
-        ]
-
-
-class SyncIntegrationDocsTest(TestCase):
-    @responses.activate
-    def test_platform(self):
-        responses.add('GET',
-                      'https://docs.getsentry.com/hosted/_platforms/go.json',
-                      body=GO_JSON)
-
-        sync_integration_docs('go', '_self', 'go.json')
-
-        data = options.get('sentry:docs:go')
-        assert data == {
-            'id': 'go',
-            'html': 'foo bar',
-            'link': 'https://docs.getsentry.com/hosted/clients/go/',
-            'name': 'Go',
-        }
-
-    @responses.activate
-    def test_integration(self):
-        responses.add('GET',
-                      'https://docs.getsentry.com/hosted/_platforms/go/http.json',
-                      body=GO_HTTP_JSON)
-
-        sync_integration_docs('go', 'http', 'go/http.json')
-
-        data = options.get('sentry:docs:go-http')
-        assert data == {
-            'id': 'go-http',
-            'html': 'foo bar',
-            'link': None,
-            'name': 'net/http',
-        }