build(docker): Split builder image out for better caching (#18999)

This is an experimental new approach which creates a separate builder image, that is highly cacheable (and allows caching for yarn install step) to generate the Python wheel. It should save us multiple minutes by leveraging our builds' incremental nature much better.

This change cuts the GCB build times by ~20% (more than 2 minutes per build).

This also switches to `volta` for installing `node` and `yarn` (cc @EvanPurkhiser). I think after this we can remove the `.nvmrc` and `bin/yarn` files along with the `yarn-path` directive in the `.yarnrc` file.

.dockerignore

@@ -1,66 +1,8 @@
-**/__pycache__
-.git
-.gitignore
-.gitattributes
-.dockerignore
-**/cloudbuild.yaml
-Gemfile.lock
-.idea/
-*.iml
-.pytest_cache/
-.vscode
-.tx
-.travis
-.github
-.mailmap
-.npmrc
-.travis.yml
-.pre-commit-config.yaml
-.eslint*
-.env
-.envrc
-coverage.xml
-conftest.py
-netlify.toml
-static
-junit.xml
-*.codestyle.xml
-package-lock.json
-.cache/
-.coverage
-.storybook-out/
-.DS_Store
-.venv
-*.egg-info
-*.pyc
-*.log
-*.egg
-*.db
-*.pid
-Brewfile
-Makefile
-MANIFEST
-test.conf
-pip-log.txt
-celerybeat-schedule
-sentry-package.json
-/.artifacts
-/coverage/
-/cover
-/build
-/env
-/tests
-/tmp
-/node_modules/
-/docs-ui/node_modules/
-/scripts
-/src/sentry/assets.json
-/src/sentry/static/version
-/src/sentry/static/sentry/dist/
-/src/sentry/static/sentry/vendor/
-/src/sentry/static/sentry/admin/
-/src/sentry/static/sentry/rest_framework/
-/src/sentry/integration-docs
-/src/sentry/loader/_registry.json
-/wheelhouse
-/test_cli/
+# Ignore everything
+*
+
+!/docker
+!/package.json
+!/yarn.lock
+!/dist/requirements.txt
+!/dist/*.whl

MANIFEST.in

@@ -2,4 +2,4 @@ include setup.py README.md MANIFEST.in LICENSE AUTHORS
 recursive-include ./ requirements*.txt
 recursive-include ./config/relay *
 graft src/sentry
-global-exclude *~
+global-exclude *.pyc

docker/Dockerfile

@@ -1,95 +1,3 @@
-FROM python:2.7.16-slim-buster as sdist
-
-LABEL maintainer="oss@sentry.io"
-LABEL org.opencontainers.image.title="Sentry PyPI Wheel"
-LABEL org.opencontainers.image.description="PyPI Wheel Builder for Sentry"
-LABEL org.opencontainers.image.url="https://sentry.io/"
-LABEL org.opencontainers.image.source="https://github.com/getsentry/sentry"
-LABEL org.opencontainers.image.vendor="Functional Software, Inc."
-LABEL org.opencontainers.image.authors="oss@sentry.io"
-
-# Sane defaults for pip
-ENV PIP_NO_CACHE_DIR=off \
-    PIP_DISABLE_PIP_VERSION_CHECK=1
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    # Needed for GPG
-    dirmngr \
-    gnupg \
-    # Needed for fetching stuff
-    wget \
-    && rm -rf /var/lib/apt/lists/* \
-    # Needed to extract final dependencies from the whl
-    && pip install pkginfo==1.5.0.1
-
-# Fetch trusted keys
-RUN for key in \
-      # gosu
-      B42F6819007F00F88E364FD4036A9C25BF357DD4 \
-      # tini
-      595E85A6B1B4779EA4DAAEC70B588DFF0527A9B7 \
-      # Node - gpg keys listed at https://github.com/nodejs/node
-      94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \
-      FD3A5288F042B6850C66B31F09FE44734EB7990E \
-      71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \
-      DD8F2338BAE7501E3DD5AC78C273792F7D83545D \
-      C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \
-      B9AE9905FFD7803F25714661B63B535A4C206CA9 \
-      77984A986EBC2AA786BC0F66B01FBB92821C587A \
-      8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \
-      4ED778F539E3634C779C87C6D7062848A1AB005C \
-      A48C2BEE680E841632CD4E44F07496B3EB3C1762 \
-      B9E2F5981AA6E0CD28160D9FF13993A75599653C \
-    ; do \
-      # TODO(byk): Replace the keyserver below w/ something owned by Sentry
-      gpg --batch --keyserver hkps://mattrobenolt-keyserver.global.ssl.fastly.net:443 --recv-keys "$key"; \
-    done
-
-# grab gosu for easy step-down from root
-ENV GOSU_VERSION 1.11
-RUN set -x \
-    && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
-    && wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
-    && gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \
-    && rm -r /usr/local/bin/gosu.asc \
-    && chmod +x /usr/local/bin/gosu
-
-# grab tini for signal processing and zombie killing
-ENV TINI_VERSION 0.18.0
-RUN set -x \
-    && wget -O /usr/local/bin/tini "https://github.com/krallin/tini/releases/download/v$TINI_VERSION/tini" \
-    && wget -O /usr/local/bin/tini.asc "https://github.com/krallin/tini/releases/download/v$TINI_VERSION/tini.asc" \
-    && gpg --batch --verify /usr/local/bin/tini.asc /usr/local/bin/tini \
-    && rm /usr/local/bin/tini.asc \
-    && chmod +x /usr/local/bin/tini
-
-# Get and set up Node for front-end asset building
-COPY .nvmrc /usr/src/sentry/
-RUN cd /usr/src/sentry \
-    && export NODE_VERSION="$(cat .nvmrc)" \
-    && wget "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.gz" \
-    && wget "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \
-    && gpg --batch --verify SHASUMS256.txt.asc \
-    && grep " node-v$NODE_VERSION-linux-x64.tar.gz\$" SHASUMS256.txt.asc | sha256sum -c - \
-    && tar -xzf "node-v$NODE_VERSION-linux-x64.tar.gz" -C /usr/local --strip-components=1 \
-    && rm -r "node-v$NODE_VERSION-linux-x64.tar.gz" SHASUMS256.txt.asc
-
-ARG SOURCE_COMMIT
-ENV SENTRY_BUILD=${SOURCE_COMMIT:-unknown}
-LABEL org.opencontainers.image.revision=$SOURCE_COMMIT
-LABEL org.opencontainers.image.licenses="https://github.com/getsentry/sentry/blob/${SOURCE_COMMIT:-master}/LICENSE"
-
-COPY . /usr/src/sentry/
-RUN export YARN_CACHE_FOLDER="$(mktemp -d)" \
-    && cd /usr/src/sentry \
-    && python setup.py bdist_wheel \
-    && rm -r "$YARN_CACHE_FOLDER" \
-    && mv /usr/src/sentry/dist /dist \
-    # Dump the dependencies of our wheel as a separate requirements.txt file
-    # so we can install them first, leveraging Docker's caching when they
-    # don't change across versions.
-    && pkginfo -f requires_dist --single --sequence-delim=! /dist/*.whl | tr ! \\n > /dist/requirements.txt
-
 # This is the image to be run
 FROM python:2.7.16-slim-buster
 
@@ -102,91 +10,123 @@ LABEL org.opencontainers.image.source="https://github.com/getsentry/sentry"
 LABEL org.opencontainers.image.vendor="Functional Software, Inc."
 LABEL org.opencontainers.image.authors="oss@sentry.io"
 
-
 # add our user and group first to make sure their IDs get assigned consistently
 RUN groupadd -r sentry && useradd -r -m -g sentry sentry
 
-COPY --from=sdist /usr/local/bin/gosu /usr/local/bin/tini /usr/local/bin/
+ENV GOSU_VERSION=1.11 \
+  TINI_VERSION=0.18.0
+
+RUN set -x \
+  && buildDeps=" \
+  dirmngr \
+  gnupg \
+  wget \
+  " \
+  && apt-get update && apt-get install -y --no-install-recommends $buildDeps \
+  && rm -rf /var/lib/apt/lists/* \
+  # Fetch trusted keys
+  && for key in \
+  # gosu
+  B42F6819007F00F88E364FD4036A9C25BF357DD4 \
+  # tini
+  595E85A6B1B4779EA4DAAEC70B588DFF0527A9B7 \
+  ; do \
+  # TODO(byk): Replace the keyserver below w/ something owned by Sentry
+  gpg --batch --keyserver hkps://mattrobenolt-keyserver.global.ssl.fastly.net:443 --recv-keys "$key"; \
+  done \
+  # grab gosu for easy step-down from root
+  && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
+  && wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
+  && gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \
+  && rm -r /usr/local/bin/gosu.asc \
+  && chmod +x /usr/local/bin/gosu \
+  # grab tini for signal processing and zombie killing
+  && wget -O /usr/local/bin/tini "https://github.com/krallin/tini/releases/download/v$TINI_VERSION/tini" \
+  && wget -O /usr/local/bin/tini.asc "https://github.com/krallin/tini/releases/download/v$TINI_VERSION/tini.asc" \
+  && gpg --batch --verify /usr/local/bin/tini.asc /usr/local/bin/tini \
+  && rm /usr/local/bin/tini.asc \
+  && chmod +x /usr/local/bin/tini \
+  && apt-get purge -y --auto-remove $buildDeps
 
 # Sane defaults for pip
 ENV PIP_NO_CACHE_DIR=off \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    # Sentry config params
-    SENTRY_CONF=/etc/sentry \
-    # Disable some unused uWSGI features, saving dependencies
-    # Thank to https://stackoverflow.com/a/25260588/90297
-    UWSGI_PROFILE_OVERRIDE=ssl=false;xml=false;routing=false \
-    # UWSGI dogstatsd plugin
-    UWSGI_NEED_PLUGIN=/var/lib/uwsgi/dogstatsd
+  PIP_DISABLE_PIP_VERSION_CHECK=1 \
+  # Sentry config params
+  SENTRY_CONF=/etc/sentry \
+  # Disable some unused uWSGI features, saving dependencies
+  # Thank to https://stackoverflow.com/a/25260588/90297
+  UWSGI_PROFILE_OVERRIDE=ssl=false;xml=false;routing=false \
+  # UWSGI dogstatsd plugin
+  UWSGI_NEED_PLUGIN=/var/lib/uwsgi/dogstatsd
 
 # Copy and install dependencies first to leverage Docker layer caching.
-COPY --from=sdist /dist/requirements.txt /tmp/dist/requirements.txt
+COPY /dist/requirements.txt /tmp/dist/requirements.txt
 RUN set -x \
-    && buildDeps="" \
-    # uwsgi
-    && buildDeps="$buildDeps \
-      gcc \
-      g++ \
-      wget \
-    " \
-    # maxminddb
-    && buildDeps="$buildDeps \
-      libmaxminddb-dev \
-    "\
-    # librabbitmq
-    && buildDeps="$buildDeps \
-      make \
-    " \
-       # xmlsec
-    && buildDeps="$buildDeps \
-      libxmlsec1-dev \
-      pkg-config \
-    " \
-    && apt-get update \
-    && apt-get install -y --no-install-recommends $buildDeps \
-    && pip install -r /tmp/dist/requirements.txt \
-    # Separate these due to https://git.io/fjyz6
-    # Otherwise librabbitmq will install the latest amqp version,
-    # violating kombu's amqp<2.0 constraint.
-    && pip install librabbitmq==1.6.1 \
-    && mkdir /tmp/uwsgi-dogstatsd \
-    && wget -O - https://github.com/eventbrite/uwsgi-dogstatsd/archive/filters-and-tags.tar.gz | \
-       tar -xzf - -C /tmp/uwsgi-dogstatsd --strip-components=1 \
-    && UWSGI_NEED_PLUGIN="" uwsgi --build-plugin /tmp/uwsgi-dogstatsd \
-    && mkdir -p /var/lib/uwsgi \
-    && mv dogstatsd_plugin.so /var/lib/uwsgi/ \
-    && rm -rf /tmp/dist /tmp/uwsgi-dogstatsd .uwsgi_plugins_builder \
-    && apt-get purge -y --auto-remove $buildDeps \
-    # We install run-time dependencies strictly after
-    # build dependencies to prevent accidental collusion.
-    # These are also installed last as they are needed
-    # during container run and can have the same deps w/
-    # build deps such as maxminddb.
-    && apt-get install -y --no-install-recommends \
-      # pillow
-      libjpeg-dev \
-      # rust bindings
-      libffi-dev \
-      # maxminddb bindings
-      libmaxminddb-dev \
-      # SAML needs these run-time
-      libxmlsec1-dev \
-      libxslt-dev \
-      # pyyaml needs this run-time
-      libyaml-dev \
-      # other
-      pkg-config \
-    \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* \
-    && python -c 'import librabbitmq' \
-    # Fully verify that the C extension is correctly installed, it unfortunately
-    # requires a full check into maxminddb.extension.Reader
-    && python -c 'import maxminddb.extension; maxminddb.extension.Reader' \
-    && mkdir -p $SENTRY_CONF
-
-COPY --from=sdist /dist/*.whl /tmp/dist/
-RUN pip install /tmp/dist/*.whl && pip check
+  && buildDeps="" \
+  # uwsgi
+  && buildDeps="$buildDeps \
+  gcc \
+  g++ \
+  wget \
+  " \
+  # maxminddb
+  && buildDeps="$buildDeps \
+  libmaxminddb-dev \
+  "\
+  # librabbitmq
+  && buildDeps="$buildDeps \
+  make \
+  " \
+  # xmlsec
+  && buildDeps="$buildDeps \
+  libxmlsec1-dev \
+  pkg-config \
+  " \
+  && apt-get update \
+  && apt-get install -y --no-install-recommends $buildDeps \
+  && pip install -r /tmp/dist/requirements.txt \
+  # Separate these due to https://git.io/fjyz6
+  # Otherwise librabbitmq will install the latest amqp version,
+  # violating kombu's amqp<2.0 constraint.
+  && pip install librabbitmq==1.6.1 \
+  && mkdir /tmp/uwsgi-dogstatsd \
+  && wget -O - https://github.com/eventbrite/uwsgi-dogstatsd/archive/filters-and-tags.tar.gz | \
+  tar -xzf - -C /tmp/uwsgi-dogstatsd --strip-components=1 \
+  && UWSGI_NEED_PLUGIN="" uwsgi --build-plugin /tmp/uwsgi-dogstatsd \
+  && mkdir -p /var/lib/uwsgi \
+  && mv dogstatsd_plugin.so /var/lib/uwsgi/ \
+  && rm -rf /tmp/dist /tmp/uwsgi-dogstatsd .uwsgi_plugins_builder \
+  && apt-get purge -y --auto-remove $buildDeps \
+  # We install run-time dependencies strictly after
+  # build dependencies to prevent accidental collusion.
+  # These are also installed last as they are needed
+  # during container run and can have the same deps w/
+  # build deps such as maxminddb.
+  && apt-get install -y --no-install-recommends \
+  # pillow
+  libjpeg-dev \
+  # rust bindings
+  libffi-dev \
+  # maxminddb bindings
+  libmaxminddb-dev \
+  # SAML needs these run-time
+  libxmlsec1-dev \
+  libxslt-dev \
+  # pyyaml needs this run-time
+  libyaml-dev \
+  # other
+  pkg-config \
+  \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/* \
+  && python -c 'import librabbitmq' \
+  # Fully verify that the C extension is correctly installed, it unfortunately
+  # requires a full check into maxminddb.extension.Reader
+  && python -c 'import maxminddb.extension; maxminddb.extension.Reader' \
+  && mkdir -p $SENTRY_CONF
+
+COPY /dist/*.whl /tmp/dist/
+RUN pip install /tmp/dist/*.whl && pip check && rm -rf /tmp/dist
 RUN sentry help | sed '1,/Commands:/d' | awk '{print $1}' >  /sentry-commands.txt
 
 COPY ./docker/sentry.conf.py ./docker/config.yml $SENTRY_CONF/

docker/builder.dockerfile

@@ -0,0 +1,51 @@
+FROM python:2.7.16-slim-buster as sdist
+
+LABEL maintainer="oss@sentry.io"
+LABEL org.opencontainers.image.title="Sentry Wheel Builder"
+LABEL org.opencontainers.image.description="Python Wheel Builder for Sentry"
+LABEL org.opencontainers.image.url="https://sentry.io/"
+LABEL org.opencontainers.image.source="https://github.com/getsentry/sentry"
+LABEL org.opencontainers.image.vendor="Functional Software, Inc."
+LABEL org.opencontainers.image.authors="oss@sentry.io"
+
+# Sane defaults for pip
+ENV PIP_NO_CACHE_DIR=off \
+  PIP_DISABLE_PIP_VERSION_CHECK=1
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  # Needed for fetching stuff
+  wget \
+  && rm -rf /var/lib/apt/lists/* \
+  # Needed to extract final dependencies from the whl
+  && pip install pkginfo==1.5.0.1
+
+# Get and set up Node for front-end asset building
+ENV VOLTA_VERSION=0.8.1 \
+  VOLTA_HOME=/.volta \
+  PATH=/.volta/bin:$PATH
+
+RUN wget "https://github.com/volta-cli/volta/releases/download/v$VOLTA_VERSION/volta-$VOLTA_VERSION-linux-openssl-1.1.tar.gz" \
+  && tar -xzf "volta-$VOLTA_VERSION-linux-openssl-1.1.tar.gz" -C /usr/local/bin \
+  # Running `volta -v` triggers setting up the shims in VOLTA_HOME (otherwise node won't work)
+  && volta -v
+
+WORKDIR /js
+
+COPY package.json /js
+# Running `node -v` and `yarn -v` triggers Volta to install the versions set in the project
+RUN node -v && yarn -v
+
+COPY yarn.lock /js
+RUN export YARN_CACHE_FOLDER="$(mktemp -d)" \
+  && yarn install --frozen-lockfile --production --quiet \
+  && rm -r "$YARN_CACHE_FOLDER"
+
+WORKDIR /workspace
+VOLUME ["/workspace/node_modules", "/workspace/build"]
+COPY docker/builder.sh /builder.sh
+ENTRYPOINT [ "/builder.sh" ]
+
+ARG SOURCE_COMMIT
+ENV SENTRY_BUILD=${SOURCE_COMMIT:-unknown}
+LABEL org.opencontainers.image.revision=$SOURCE_COMMIT
+LABEL org.opencontainers.image.licenses="https://github.com/getsentry/sentry/blob/${SOURCE_COMMIT:-master}/LICENSE"

docker/builder.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+if [[ ! -f setup.py ]]; then
+    >&2 echo "Cannot find setup.py, make sure you have mounted your source dir to /workspace"
+    exit 1
+fi
+
+mkdir -p ./node_modules
+echo "Populating node_modules cache..."
+cp -ur /js/node_modules/* ./node_modules/
+
+export YARN_CACHE_FOLDER="$(mktemp -d)"
+python setup.py bdist_wheel
+rm -r "$YARN_CACHE_FOLDER"
+pkginfo -f requires_dist --single --sequence-delim=! dist/*.whl | tr ! \\n > dist/requirements.txt

docker/cloudbuild.yaml

@@ -1,12 +1,25 @@
 steps:
-- name: 'gcr.io/kaniko-project/executor:v0.17.1'
+- name: 'gcr.io/kaniko-project/executor:v0.22.0'
+  args: [
+    '--cache=true',
+    '--build-arg', 'SOURCE_COMMIT=$COMMIT_SHA',
+    '--destination=us.gcr.io/$PROJECT_ID/sentry-builder:$COMMIT_SHA',
+    '-f', './docker/builder.dockerfile'
+  ]
+  timeout: 180s
+- name: 'us.gcr.io/$PROJECT_ID/sentry-builder:$COMMIT_SHA'
+  env: [
+    'SOURCE_COMMIT=$COMMIT_SHA'
+  ]
+  timeout: 360s
+- name: 'gcr.io/kaniko-project/executor:v0.22.0'
   args: [
     '--cache=true',
     '--build-arg', 'SOURCE_COMMIT=$COMMIT_SHA',
     '--destination=us.gcr.io/$PROJECT_ID/sentry:$COMMIT_SHA',
     '-f', './docker/Dockerfile'
   ]
-  timeout: 900s
+  timeout: 300s
 # Smoke tests
 - name: 'gcr.io/$PROJECT_ID/docker-compose'
   entrypoint: 'bash'
@@ -29,7 +42,7 @@ steps:
     docker-compose up -d
     timeout 20 bash -c 'until $(curl -Isf -o /dev/null http://web:9000); do printf "."; sleep 0.5; done' || docker-compose logs web
     ./test.sh
-  timeout: 1200s
+  timeout: 300s
 - name: 'gcr.io/cloud-builders/docker'
   secretEnv: ['DOCKER_PASSWORD']
   entrypoint: 'bash'

src/sentry/utils/distutils/commands/base.py

@@ -123,14 +123,6 @@ class BaseBuildCommand(Command):
                 return True
         return False
 
-    def _setup_git(self):
-        work_path = self.work_path
-
-        if os.path.exists(os.path.join(work_path, ".git")):
-            log.info("initializing git submodules")
-            self._run_command(["git", "submodule", "init"])
-            self._run_command(["git", "submodule", "update"])
-
     def _setup_js_deps(self):
         node_version = None
         try:
@@ -195,7 +187,6 @@ class BaseBuildCommand(Command):
 
     def run(self):
         if self.force or self._needs_built():
-            self._setup_git()
             self._setup_js_deps()
             self._build()
             self.update_manifests()