build(yarn): Upgrade yarn to 1.16.0, embed in repo (#13569)

The best way to install yarn in repos like this is to use the single
JS file version from GitHub instead of installing it via npm. An
even better way is to embed this file in the repo and add a yarn-path
directive to the `.yarnrc` file to ensure everyone always uses the same
yarn version for this repo at all times.

This patch does all this, removes all references to npm too.

.travis.yml

@@ -34,7 +34,6 @@ env:
     - DJANGO_VERSION=">=1.8,<1.9"
     # node's version is pinned by .nvmrc and is autodetected by `nvm install`.
     - NODE_DIR="${HOME}/.nvm/versions/node/v$(< .nvmrc)"
-    - YARN_VERSION="1.13.0"
     - NODE_OPTIONS=--max-old-space-size=4096
 
 script:
@@ -55,7 +54,7 @@ after_script:
         pip install codecov
         codecov -e TEST_SUITE
       fi
-  - npm install -g @zeus-ci/cli
+  - ./bin/yarn global add @zeus-ci/cli
   - zeus upload -t "text/xml+xunit" .artifacts/*junit.xml
   - zeus upload -t "text/xml+coverage" .artifacts/*coverage.xml
   - zeus upload -t "text/xml+coverage" .artifacts/coverage/cobertura-coverage.xml
@@ -90,7 +89,6 @@ base_acceptance: &acceptance_default
   before_install:
     - find "$NODE_DIR" -type d -empty -delete
     - nvm install
-    - npm install -g "yarn@${YARN_VERSION}"
     - docker run -d --network host --name clickhouse-server --ulimit nofile=262144:262144 yandex/clickhouse-server:18.14.9
     - docker run -d --network host --name snuba --env SNUBA_SETTINGS=test --env CLICKHOUSE_SERVER=localhost:9000 getsentry/snuba
     - docker ps -a
@@ -120,7 +118,6 @@ matrix:
         - SENTRY_LIGHT_BUILD=1 pip install -e ".[dev,tests,optional]"
         - find "$NODE_DIR" -type d -empty -delete
         - nvm install
-        - npm install -g "yarn@${YARN_VERSION}"
         - yarn install --pure-lockfile
 
     - <<: *postgres_default
@@ -158,7 +155,6 @@ matrix:
       before_install:
         - find "$NODE_DIR" -type d -empty -delete
         - nvm install
-        - npm install -g "yarn@${YARN_VERSION}"
       install:
         - yarn install --pure-lockfile
 
@@ -180,7 +176,6 @@ matrix:
       before_install:
         - find "$NODE_DIR" -type d -empty -delete
         - nvm install
-        - npm install -g "yarn@${YARN_VERSION}"
 
     - <<: *postgres_default
       name: 'Symbolicator Integration'
@@ -225,7 +220,6 @@ matrix:
         - tar -xzf credentials.tar.gz
         # Use the decrypted service account credentials to authenticate the command line tool
         - gcloud auth activate-service-account --key-file client-secret.json
-        - npm install -g "yarn@${YARN_VERSION}"
       install:
         - yarn install --pure-lockfile
         - gcloud version

.travis/deploy-storybook.sh

@@ -10,7 +10,7 @@ BRANCH_PROCESSED=$(echo "${DEPLOY_BRANCH}" | tr '[:upper:]./' '[:lower:]--' | tr
 BUCKET_DIR_NAME="branches/${BRANCH_PROCESSED}"
 echo "Bucket directory: ${BUCKET_DIR_NAME}"
 
-npm run storybook-build
+./bin/yarn run storybook-build
 
 # Upload the files
 gsutil cp .storybook-out/favicon.ico "gs://${GS_BUCKET_NAME}/favicon.ico"

.yarnrc

@@ -0,0 +1,2 @@
+yarn-path "./bin/yarn"
+disable-self-update-check true

Makefile

@@ -8,8 +8,8 @@ ifneq "$(wildcard /usr/local/opt/openssl/lib)" ""
 endif
 
 PIP = LDFLAGS="$(LDFLAGS)" pip
-WEBPACK = NODE_ENV=production ./node_modules/.bin/webpack
-YARN_VERSION = 1.13.0
+WEBPACK = NODE_ENV=production ./bin/yarn webpack
+YARN = ./bin/yarn
 
 bootstrap: install-system-pkgs develop init-config run-dependent-services create-db apply-migrations
 
@@ -76,19 +76,17 @@ node-version-check:
 install-system-pkgs: node-version-check
 	@echo "--> Installing system packages (from Brewfile)"
 	@command -v brew 2>&1 > /dev/null && brew bundle || (echo 'WARNING: homebrew not found or brew bundle failed - skipping system dependencies.')
-	@echo "--> Installing yarn $(YARN_VERSION) (via npm)"
-	@$(volta --version 2>&1 > /dev/null || npm install -g "yarn@$(YARN_VERSION)")
 
 install-yarn-pkgs:
 	@echo "--> Installing Yarn packages (for development)"
-	@command -v yarn 2>&1 > /dev/null || (echo 'yarn not found. Please install it before proceeding.'; exit 1)
+	@command -v $(YARN) 2>&1 > /dev/null || (echo 'yarn not found. Please install it before proceeding.'; exit 1)
 	# Use NODE_ENV=development so that yarn installs both dependencies + devDependencies
-	NODE_ENV=development yarn install --pure-lockfile
+	NODE_ENV=development $(YARN) install --pure-lockfile
 	# A common problem is with node packages not existing in `node_modules` even though `yarn install`
 	# says everything is up to date. Even though `yarn install` is run already, it doesn't take into
 	# account the state of the current filesystem (it only checks .yarn-integrity).
 	# Add an additional check against `node_modules`
-	yarn check --verify-tree || yarn install --check-files
+	$(YARN) check --verify-tree || $(YARN) install --check-files
 
 install-sentry-dev:
 	@echo "--> Installing Sentry (for development)"
@@ -135,13 +133,13 @@ test-js: node-version-check
 	@echo "--> Building static assets"
 	@$(WEBPACK) --profile --json > .artifacts/webpack-stats.json
 	@echo "--> Running JavaScript tests"
-	@npm run test-ci
+	@$(YARN) run test-ci
 	@echo ""
 
 # builds and creates percy snapshots
 test-styleguide:
 	@echo "--> Building and snapshotting styleguide"
-	@npm run snapshot
+	@$(YARN) run snapshot
 	@echo ""
 
 test-python:

package.json

@@ -148,6 +148,6 @@
   },
   "volta": {
     "node": "8.15.1",
-    "yarn": "1.13.0"
+    "yarn": "1.16.0"
   }
 }

src/sentry/lint/engine.py

@@ -249,7 +249,7 @@ def js_test(file_list=None):
 
     has_errors = False
     if js_file_list:
-        status = Popen(['yarn', 'test-precommit'] + js_file_list).wait()
+        status = Popen(['./bin/yarn', 'test-precommit'] + js_file_list).wait()
         has_errors = status != 0
 
     return has_errors

src/sentry/runner/commands/devserver.py

@@ -193,7 +193,7 @@ def devserver(reload, watchers, workers, browser_reload, styleguide, prefix, env
     ]
 
     if styleguide:
-        daemons += [('storybook', ['yarn', 'storybook'])]
+        daemons += [('storybook', ['./bin/yarn', 'storybook'])]
 
     cwd = os.path.realpath(os.path.join(settings.PROJECT_ROOT, os.pardir, os.pardir))
 

src/sentry/utils/distutils/commands/base.py

@@ -119,30 +119,22 @@ class BaseBuildCommand(Command):
             self._run_command(['git', 'submodule', 'init'])
             self._run_command(['git', 'submodule', 'update'])
 
-    def _setup_npm(self):
-        node_version = []
-        for app in 'node', 'npm', 'yarn':
-            try:
-                node_version.append(self._run_command([app, '--version']).rstrip())
-            except OSError:
-                if app == 'yarn':
-                    # yarn is optional
-                    node_version.append(None)
-                else:
-                    log.fatal(
-                        u'Cannot find `{0}` executable. Please install {0}`'
-                        ' and try again.'.format(app)
-                    )
-                    sys.exit(1)
+    def _setup_js_deps(self):
+        node_version = None
+        try:
+            node_version = self._run_command(['node', '--version']).rstrip()
+        except OSError:
+            log.fatal(
+                u'Cannot find node executable. Please install node'
+                ' and try again.'
+            )
+            sys.exit(1)
 
         if node_version[2] is not None:
-            log.info(u'using node ({0}) and yarn ({2})'.format(*node_version))
+            log.info(u'using node ({0}))'.format(node_version))
             self._run_command(
-                ['yarn', 'install', '--production', '--pure-lockfile']
+                ['./bin/yarn', 'install', '--production', '--pure-lockfile', '--quiet']
             )
-        else:
-            log.info(u'using node ({0}) and npm ({1})'.format(*node_version))
-            self._run_command(['npm', 'install', '--production', '--quiet'])
 
     def _run_command(self, cmd, env=None):
         log.debug('running [%s]' % (' '.join(cmd), ))
@@ -183,6 +175,6 @@ class BaseBuildCommand(Command):
     def run(self):
         if self.force or self._needs_built():
             self._setup_git()
-            self._setup_npm()
+            self._setup_js_deps()
             self._build()
             self.update_manifests()

src/sentry/utils/distutils/commands/build_assets.py

@@ -118,8 +118,7 @@ class BuildAssetsCommand(BaseBuildCommand):
         except Exception:
             traceback.print_exc()
             log.fatal(
-                'unable to build Sentry\'s static assets!\n'
-                'Hint: You might be running an invalid version of NPM.'
+                'unable to build Sentry\'s static assets!'
             )
             sys.exit(1)
 
@@ -136,7 +135,7 @@ class BuildAssetsCommand(BaseBuildCommand):
         env = dict(os.environ)
         env['SENTRY_STATIC_DIST_PATH'] = self.sentry_static_dist_path
         env['NODE_ENV'] = 'production'
-        self._run_command(['node_modules/.bin/webpack', '--bail'], env=env)
+        self._run_command(['./bin/yarn', 'webpack', '--bail'], env=env)
 
     def _write_version_file(self, version_info):
         manifest = {