feat: Pull latest SDK information from the registry (#10231)

* feat: Pull latest SDK information from the registry

* fix: Release registry fetch

* Always perform a build and ignore registry file

* fix: Do not read the version file on every request

* fix: Override run() for command and fix logging

* meta: Update MANIFEST.in

.gitignore

@@ -31,6 +31,7 @@ example/db.sqlite
 /src/sentry/static/sentry/debug_toolbar/
 /src/sentry/static/sentry/rest_framework/
 /src/sentry/integration-docs
+/src/sentry/loader/_registry.json
 /wheelhouse
 /test_cli/
 Gemfile.lock

MANIFEST.in

@@ -1,4 +1,5 @@
 include setup.py src/sentry/assets.json README.rst MANIFEST.in LICENSE AUTHORS
+include src/sentry/loader/_registry.json
 recursive-include ./ requirements*.txt
 recursive-include src/sentry/templates *
 recursive-include src/sentry/locale *

Makefile

@@ -93,6 +93,11 @@ build-platform-assets:
 	sentry init
 	@echo "from sentry.utils.integrationdocs import sync_docs; sync_docs(quiet=True)" | sentry exec
 
+fetch-release-registry:
+	@echo "--> Fetching release registry"
+	sentry init
+	@echo "from sentry.utils.distutils import sync_registry; sync_registry()" | sentry exec
+
 test-cli:
 	@echo "--> Testing CLI"
 	rm -rf test_cli

setup.py

@@ -44,7 +44,7 @@ ROOT = os.path.realpath(os.path.join(os.path.dirname(
 sys.path.insert(0, os.path.join(ROOT, 'src'))
 
 from sentry.utils.distutils import (
-    BuildAssetsCommand, BuildIntegrationDocsCommand
+    BuildAssetsCommand, BuildIntegrationDocsCommand, BuildJsSdkRegistryCommand
 )
 
 # The version of sentry
@@ -90,7 +90,9 @@ class SentrySDistCommand(SDistCommand):
     # part of our source build pipeline.
     if not IS_LIGHT_BUILD:
         sub_commands = SDistCommand.sub_commands + \
-            [('build_integration_docs', None), ('build_assets', None)]
+            [('build_integration_docs', None),
+             ('build_assets', None),
+             ('build_js_sdk_registry', None)]
 
 
 class SentryBuildCommand(BuildCommand):
@@ -99,6 +101,7 @@ class SentryBuildCommand(BuildCommand):
         if not IS_LIGHT_BUILD:
             self.run_command('build_integration_docs')
             self.run_command('build_assets')
+            self.run_command('build_js_sdk_registry')
 
 
 class SentryDevelopCommand(DevelopCommand):
@@ -107,6 +110,7 @@ class SentryDevelopCommand(DevelopCommand):
         if not IS_LIGHT_BUILD:
             self.run_command('build_integration_docs')
             self.run_command('build_assets')
+            self.run_command('build_js_sdk_registry')
 
 
 cmdclass = {
@@ -115,6 +119,7 @@ cmdclass = {
     'build': SentryBuildCommand,
     'build_assets': BuildAssetsCommand,
     'build_integration_docs': BuildIntegrationDocsCommand,
+    'build_js_sdk_registry': BuildJsSdkRegistryCommand,
 }
 
 

src/sentry/loader/browsersdkversion.py

@@ -1,23 +1,43 @@
 from __future__ import absolute_import
 
+import os
 import re
+import logging
+import json
+
+from functools32 import lru_cache
+
+import sentry
 
 from django.conf import settings
 
+logger = logging.getLogger('sentry')
 
 _version_regexp = re.compile(r'\d+')
+LOADER_FOLDER = os.path.abspath(os.path.join(os.path.dirname(sentry.__file__), 'loader'))
 DEFAULT_VERSION = '4.x'
 
 
+@lru_cache(maxsize=10)
+def load_registry(path):
+    if '/' in path:
+        return None
+    fn = os.path.join(LOADER_FOLDER, path + '.json')
+    try:
+        with open(fn, 'rb') as f:
+            return json.load(f)
+    except IOError:
+        return None
+
+
 def get_highest_browser_sdk_version():
     return max(get_browser_sdk_version_versions(),
-               key=lambda version: int(_version_regexp.match(version).group(
-                   0)) if _version_regexp.search(version) else -1
+               key=lambda version: int(_version_regexp.match(version).group(0))
+               if _version_regexp.search(version) else -1
                )
 
 
 def get_browser_sdk_version_versions():
-    # TODO(hazat): do request here to fetch versions
     return ['latest', DEFAULT_VERSION]
 
 
@@ -28,11 +48,21 @@ def get_browser_sdk_version_choices():
     return tuple(rv)
 
 
+def load_version_from_file():
+    data = load_registry('_registry')
+    return data['version']
+
+
 def get_browser_sdk_version(project_key):
-    # TODO(hazat): Right now we are only returing our conf version
     selected_version = get_selected_browser_sdk_version(project_key)
+
     if selected_version == DEFAULT_VERSION:
-        return settings.JS_SDK_LOADER_SDK_VERSION
+        try:
+            return load_version_from_file()
+        except BaseException:
+            logger.error('error ocurred while trying to read js sdk information from the registry')
+            return settings.JS_SDK_LOADER_SDK_VERSION
+
     return settings.JS_SDK_LOADER_SDK_VERSION
 
 

src/sentry/utils/distutils/__init__.py

@@ -4,3 +4,4 @@ from __future__ import absolute_import
 
 from .commands.build_integration_docs import BuildIntegrationDocsCommand  # NOQA
 from .commands.build_assets import BuildAssetsCommand  # NOQA
+from .commands.build_js_sdk_registry import BuildJsSdkRegistryCommand, sync_registry  # NOQA

src/sentry/utils/distutils/commands/build_js_sdk_registry.py

@@ -0,0 +1,61 @@
+# NOTE: This is run external to sentry as well as part of the setup
+# process.  Thus we do not want to import non stdlib things here.
+from __future__ import absolute_import
+
+import os
+import sys
+import json
+from distutils import log
+
+import sentry
+
+JS_SDK_REGISTRY_URL = 'https://release-registry.services.sentry.io/packages/npm:@sentry/browser/latest'
+LOADER_FOLDER = os.path.abspath(os.path.join(os.path.dirname(sentry.__file__), 'loader'))
+
+# We cannot leverage six here, so we need to vendor
+# bits that we need.
+if sys.version_info[0] == 3:
+
+    def iteritems(d, **kw):
+        return iter(d.items(**kw))
+
+    from urllib.request import urlopen
+
+else:
+
+    def iteritems(d, **kw):
+        return d.iteritems(**kw)  # NOQA
+
+    from urllib2 import urlopen
+
+
+def dump_registry(path, data):
+    fn = os.path.join(LOADER_FOLDER, path + '.json')
+    directory = os.path.dirname(fn)
+    try:
+        os.makedirs(directory)
+    except OSError:
+        pass
+    with open(fn, 'wb') as f:
+        json.dump(data, f, indent=2)
+        f.write('\n')
+
+
+def sync_registry():
+    body = urlopen(JS_SDK_REGISTRY_URL).read().decode('utf-8')
+    data = json.loads(body)
+    dump_registry('_registry', data)
+
+
+from .base import BaseBuildCommand
+
+
+class BuildJsSdkRegistryCommand(BaseBuildCommand):
+    description = 'build js sdk registry'
+
+    def run(self):
+        log.info('downloading js sdk information from the release registry')
+        try:
+            sync_registry()
+        except BaseException:
+            log.error('error ocurred while trying to fetch js sdk information from the registry')