build(github): Add acceptance tests to GitHub Actions (#18576)

We want to evaluate using GitHub Actions for CI. This brings our acceptance test suite over from Travis. It will not be a required status check for PRs.

.github/workflows/acceptance.yml

@@ -0,0 +1,226 @@
+name: acceptance
+on: pull_request
+
+jobs:
+    percynonce:
+      name: Generate Percy nonce
+      runs-on: ubuntu-latest
+      steps:
+        - name: Generate Percy nonce
+          if: always()
+          run: |
+            echo ${{ github.run_id }}.$(($(date +%s))) > nonce.txt
+
+        - name: Upload nonce as artifact
+          if: always()
+          uses: actions/upload-artifact@v1
+          with:
+            name: percy
+            path: nonce.txt
+
+    acceptance:
+      needs: [percynonce]
+      runs-on: ubuntu-16.04
+      strategy:
+        matrix:
+          instance: [0, 1, 2]
+
+      env:
+        PIP_DISABLE_PIP_VERSION_CHECK: on
+        # PIP_QUIET: 1
+
+        SENTRY_LIGHT_BUILD: 1
+        SENTRY_SKIP_BACKEND_VALIDATION: 1
+        MIGRATIONS_TEST_MIGRATE: 0
+
+        # Use this to override the django version in the requirements file.
+        DJANGO_VERSION: ">=1.11,<1.12"
+
+        # Node configuration
+        # node's version is pinned by .nvmrc and is autodetected by `nvm install`.
+        NODE_OPTIONS: --max-old-space-size=4096
+        NODE_ENV: development
+
+        PYTEST_SENTRY_DSN: https://6fd5cfea2d4d46b182ad214ac7810508@sentry.io/2423079
+        PYTEST_ADDOPTS: "--reruns 5"
+
+        # services configuration
+        SENTRY_KAFKA_HOSTS: kafka:9093
+        SENTRY_ZOOKEEPER_HOSTS: zookeeper:2182
+        SENTRY_REDIS_HOST: redis
+        # The hostname used to communicate with the PostgreSQL from sentry
+        DATABASE_URL: postgresql://postgres:postgres@localhost/sentry
+
+        # Percy config
+        PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}
+        PERCY_PROJECT: ${{ secrets.PERCY_PROJECT }}
+
+        # Number of matrix instances
+        TOTAL_TEST_GROUPS: 3
+
+      services:
+        clickhouse:
+          image: yandex/clickhouse-server:19.11
+          options: --ulimit nofile=262144:262144
+          ports:
+            - 9000:9000
+
+        snuba:
+          image: getsentry/snuba
+          env:
+            SNUBA_SETTINGS: test
+            REDIS_HOST: redis
+            CLICKHOUSE_HOST: clickhouse
+            CLICKHOUSE_PORT: 9000
+          ports:
+            - 1218:1218
+
+        redis:
+          image: redis:5.0-alpine
+          ports:
+            - 6379:6379
+
+        memcached:
+          image: memcached:1.5-alpine
+          ports:
+            - 11211:11211
+
+        postgres:
+          image: postgres:9.6
+          env:
+            POSTGRES_USER: postgres
+            POSTGRES_PASSWORD: postgres
+          ports:
+            # Maps tcp port 5432 on service container to the host
+            - 5432:5432
+          # needed because the postgres container does not provide a healthcheck
+          options: >-
+            --health-cmd pg_isready
+            --health-interval 10s
+            --health-timeout 5s
+            --health-retries 5
+
+        zookeeper:
+          image: confluentinc/cp-zookeeper:4.1.0
+          env:
+            ZOOKEEPER_CLIENT_PORT: 2181
+
+        kafka:
+          image: confluentinc/cp-kafka:5.1.2
+          env:
+            KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+            KAFKA_LISTENERS: INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092
+            KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:9093,EXTERNAL://kafka:9092
+            KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
+            KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
+            KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+
+      steps:
+        # Checkout codebase
+        - uses: actions/checkout@v2
+
+        # See https://github.com/actions/cache/blob/master/examples.md#node---yarn for example
+        - name: Get yarn cache dir
+          id: yarn-cache-dir-path
+          run: echo "::set-output name=dir::$(yarn cache dir)"
+
+        # Use nvmrc so we don't have yet another place node version is defined
+        - name: Get node version from .nvmrc
+          id: nvmrc
+          run: echo "::set-output name=version::$(cat .nvmrc)"
+
+        # yarn cache
+        - uses: actions/cache@v1
+          id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+          with:
+            path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+            key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+            restore-keys: |
+              ${{ runner.os }}-yarn-
+
+        # Setup node
+        - uses: actions/setup-node@v1
+          with:
+            node-version: ${{ steps.nvmrc.outputs.version }}
+
+        # Use `.python-version` to avoid duplication
+        # XXX: can't actually read from .python-version because GitHub Actions
+        # does not support our version (2.7.16)
+        - name: Get python version from `.python-version`
+          id: python-version
+          run: echo "::set-output name=version::2.7.17"
+
+        # setup python
+        - name: Set up Python ${{ steps.python-version.outputs.version }}
+          uses: actions/setup-python@v1
+          with:
+            python-version: ${{ steps.python-version.outputs.version}}
+
+        # setup pip
+        - name: Install pip
+          run: |
+            pip install --no-cache-dir --upgrade "pip>=20.0.2"
+
+        # pip cache
+        - name: Get pip cache dir
+          id: pip-cache
+          run: |
+            echo "::set-output name=dir::$(pip cache dir)"
+
+        - name: pip cache
+          uses: actions/cache@v1
+          with:
+            path: ${{ steps.pip-cache.outputs.dir }}
+            key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements-*.txt') }}
+            restore-keys: |
+              ${{ runner.os }}-pip-
+
+        - name: Install System Dependencies
+          run: |
+            sudo apt-get update
+            sudo apt-get install libxmlsec1-dev libmaxminddb-dev
+
+        - name: Install Javascript Dependencies
+          run: |
+            yarn install --frozen-lockfile
+
+        - name: Install Python Dependencies
+          env:
+            PGPASSWORD: postgres
+          run: |
+            python setup.py install_egg_info
+            pip install wheel # GitHub Actions does not have this installed by default (unlike Travis)
+            pip install -U -e ".[dev]"
+            psql -c 'create database sentry;' -h localhost -U postgres
+
+        - name: Download percy nonce
+          uses: actions/download-artifact@v1
+          with:
+            name: percy
+
+        - name: Set percy nonce
+          id: set_percy_nonce
+          run: |
+            echo "::set-output name=percy_nonce::$(<percy/nonce.txt)"
+
+        - name: Build platform assets
+          run: |
+            sentry init
+            make build-platform-assets
+
+        - name: webpack
+          run: |
+            yarn webpack --display errors-only
+
+        - name: Acceptance tests (#${{ matrix.instance }})
+          if: always()
+          uses: percy/exec-action@v0.3.0
+          with:
+            command: "make run-acceptance"
+          env:
+            USE_SNUBA: 1
+            TEST_GROUP: ${{ matrix.instance }}
+            PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}
+            PERCY_PROJECT: ${{ secrets.PERCY_PROJECT }}
+            PERCY_PARALLEL_TOTAL: ${{ env.TOTAL_TEST_GROUPS }}
+            PERCY_PARALLEL_NONCE: ${{ steps.set_percy_nonce.outputs.percy_nonce }}

Makefile

@@ -122,6 +122,16 @@ fetch-release-registry:
 	@echo "--> Fetching release registry"
 	@echo "from sentry.utils.distutils import sync_registry; sync_registry()" | sentry exec
 
+run-acceptance:
+	@echo "--> Running acceptance tests"
+ifndef TEST_GROUP
+	py.test tests/acceptance --cov . --cov-report="xml:.artifacts/acceptance.coverage.xml" --junit-xml=".artifacts/acceptance.junit.xml" --html=".artifacts/acceptance.pytest.html" --self-contained-html
+else
+	py.test tests/acceptance -m group_$(TEST_GROUP) --cov . --cov-report="xml:.artifacts/acceptance.coverage.xml" --junit-xml=".artifacts/acceptance.junit.xml" --html=".artifacts/acceptance.pytest.html" --self-contained-html
+endif
+
+	@echo ""
+
 test-cli:
 	@echo "--> Testing CLI"
 	rm -rf test_cli
@@ -184,14 +194,7 @@ test-acceptance: node-version-check
 	sentry init
 	@echo "--> Building static assets"
 	@$(WEBPACK) --display errors-only
-	@echo "--> Running acceptance tests"
-ifndef TEST_GROUP
-	py.test tests/acceptance --cov . --cov-report="xml:.artifacts/acceptance.coverage.xml" --junit-xml=".artifacts/acceptance.junit.xml" --html=".artifacts/acceptance.pytest.html" --self-contained-html
-else
-	py.test tests/acceptance -m group_$(TEST_GROUP) --cov . --cov-report="xml:.artifacts/acceptance.coverage.xml" --junit-xml=".artifacts/acceptance.junit.xml" --html=".artifacts/acceptance.pytest.html" --self-contained-html
-endif
-
-	@echo ""
+	make run-acceptance
 
 test-plugins:
 	@echo "--> Building static assets"