build(gha): Separate acceptance tests into 2 workflows... (#21386)

* build(gha): Separate acceptance tests into 2 workflows...

Separate the existing `acceptance` workflow into two workflows:

- acceptance [py2.7] + visual snapshots (should this just be "visual snapshots?")
  - acceptance tests using py2.7
  - frontend/jest tests
  - visual diff action

- acceptance [py3.6]

This will allow us to restart 3.6 tests separate from 2.7

* rename back to acceptance.yml

* going to wait before changing names

* rename again, workflow name uses name and not file name

* change py3 acceptance to only run when backend files change, add timeout-minutes

* Update .github/workflows/acceptance-py3.6.yml

Co-authored-by: josh <josh@jrl.ninja>

* remove visual snapshot env var

* restore USE_SNUBA env var

* update comments re: USE_SNUBA

Co-authored-by: josh <josh@jrl.ninja>

.github/workflows/acceptance-py3.6.yml

@@ -0,0 +1,108 @@
+name: acceptance
+on:
+  push:
+    branches:
+      - master
+      - releases/**
+  pull_request:
+
+jobs:
+  # TODO(billy): Currently, keeping name the same because of `required` status checks
+  py3-acceptance:
+    name: python3.6 acceptance
+    runs-on: ubuntu-16.04
+    timeout-minutes: 20
+    strategy:
+      matrix:
+        instance: [0, 1, 2]
+
+    env:
+      MIGRATIONS_TEST_MIGRATE: 1
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: volta-cli/action@v1
+
+      # If we make these jobs "required" to merge on GH, then on every PR, GitHub automatically
+      # creates a status check in the "pending" state. This means that the workflow needs to run
+      # for every PR in order to update the status checks.
+      #
+      # In order to optimize CI usage, we want the tests to only run when python files change,
+      # since frontend changes should have no effect on these test suites. We cannot use GH workflow
+      # path filters because entire workflow would be skipped vs skipping individual jobs which
+      # would still allow this status check to pass.
+      - name: Check for python file changes
+        uses: getsentry/paths-filter@v2
+        id: changes
+        with:
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+      # XXX: If taking snapshots with this, be sure to remove above and the following `if` conditions!
+
+      - name: Set python version output
+        id: python-version
+        if: steps.changes.outputs.backend == 'true'
+        run: |
+          echo "::set-output name=python-version::$(awk 'FNR == 2' .python-version)"
+
+      # Until GH composite actions can use `uses`, we need to setup python here
+      - uses: actions/setup-python@v2
+        if: steps.changes.outputs.backend == 'true'
+        with:
+          python-version: ${{ steps.python-version.outputs.python-version }}
+
+      - name: Setup pip
+        uses: ./.github/actions/setup-pip
+        id: pip
+        if: steps.changes.outputs.backend == 'true'
+
+      - name: pip cache
+        uses: actions/cache@v2
+        if: steps.changes.outputs.backend == 'true'
+        with:
+          path: ${{ steps.pip.outputs.pip-cache-dir }}
+          key: ${{ runner.os }}-pip-py${{ steps.python-version.outputs.python-version }}-${{ hashFiles('**/requirements-*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ steps.python-version.outputs.python-version }}
+
+      - name: Setup sentry python env
+        uses: ./.github/actions/setup-sentry
+        id: setup
+        if: steps.changes.outputs.backend == 'true'
+        with:
+          python: 3
+          snuba: true
+
+      - name: yarn cache
+        uses: actions/cache@v2
+        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+        if: steps.changes.outputs.backend == 'true'
+        with:
+          path: ${{ steps.setup.outputs.yarn-cache-dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Install Javascript Dependencies
+        if: steps.changes.outputs.backend == 'true'
+        run: |
+          yarn install --frozen-lockfile
+
+      - name: webpack
+        if: steps.changes.outputs.backend == 'true'
+        run: |
+          yarn webpack --display errors-only
+
+      - name: Run acceptance tests (#${{ steps.setup.outputs.matrix-instance-number }} of ${{ strategy.job-total }})
+        if: steps.changes.outputs.backend == 'true'
+        run: |
+          mkdir -p ${{ steps.setup.outputs.acceptance-dir }}
+          mkdir -p ${{ steps.setup.outputs.acceptance-dir }}-mobile
+          mkdir -p ${{ steps.setup.outputs.acceptance-dir }}-tooltips
+          make run-acceptance
+        env:
+          PYTEST_SNAPSHOTS_DIR: ${{ steps.setup.outputs.acceptance-dir }}
+          USE_SNUBA: 1
+
+      # TODO(joshuarli): snapshots, visual-diff needs py3-acceptance.

.github/workflows/backend-test-py3.6.yml

@@ -15,6 +15,7 @@ jobs:
         instance: [0, 1, 2]
 
     env:
+      # Note: `USE_SNUBA` is not used for backend tests because there are a few failing tests with Snuba enabled.
       MIGRATIONS_TEST_MIGRATE: 1
 
     steps:

.github/workflows/snuba-integration-test-py3.6.yml

@@ -15,7 +15,6 @@ jobs:
         instance: [0]
 
     env:
-      # Note: `USE_SNUBA` is only used for the Snuba test suite because we have some failing acceptance tests with Snuba enabled.
       USE_SNUBA: 1
       MIGRATIONS_TEST_MIGRATE: 1
 

.github/workflows/acceptance.yml → .github/workflows/visual-snapshots-and-acceptance-py2.7.yml

@@ -1,3 +1,6 @@
+# TODO(billy): this workflow has not been re-named from `acceptance` because
+# Visual Snapshots compares against artifacts from the same workflow name (on main branch)
+# We should rename this when we have a more finalized naming scheme.
 name: acceptance
 on:
   push:
@@ -7,7 +10,8 @@ on:
   pull_request:
 
 jobs:
-  jest:
+  frontend:
+    name: frontend tests
     runs-on: ubuntu-16.04
     env:
       VISUAL_HTML_ENABLE: 1
@@ -58,7 +62,9 @@ jobs:
 
   acceptance:
     # TODO(joshuarli): Convert to py3 with snapshots. See other TODO as well.
+    name: python2.7 acceptance
     runs-on: ubuntu-16.04
+    timeout-minutes: 20
     strategy:
       matrix:
         instance: [0, 1, 2]
@@ -129,84 +135,9 @@ jobs:
           save-only: true
           snapshot-path: .artifacts/visual-snapshots
 
-  py3-acceptance:
-    name: 'python3.6 acceptance'
-    runs-on: ubuntu-16.04
-    strategy:
-      matrix:
-        instance: [0, 1, 2]
-
-    env:
-      MIGRATIONS_TEST_MIGRATE: 1
-      VISUAL_SNAPSHOT_ENABLE: 1
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - uses: volta-cli/action@v1
-
-      - name: Set python version output
-        id: python-version
-        run: |
-          echo "::set-output name=python-version::$(awk 'FNR == 2' .python-version)"
-
-      # Until GH composite actions can use `uses`, we need to setup python here
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ steps.python-version.outputs.python-version }}
-
-      - name: Setup pip
-        uses: ./.github/actions/setup-pip
-        id: pip
-
-      - name: pip cache
-        uses: actions/cache@v2
-        with:
-          path: ${{ steps.pip.outputs.pip-cache-dir }}
-          key: ${{ runner.os }}-pip-py${{ steps.python-version.outputs.python-version }}-${{ hashFiles('**/requirements-*.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-py${{ steps.python-version.outputs.python-version }}
-
-      - name: Setup sentry python env
-        uses: ./.github/actions/setup-sentry
-        id: setup
-        with:
-          python: 3
-          snuba: true
-
-      - name: yarn cache
-        uses: actions/cache@v2
-        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-        with:
-          path: ${{ steps.setup.outputs.yarn-cache-dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
-
-      - name: Install Javascript Dependencies
-        run: |
-          yarn install --frozen-lockfile
-
-      - name: webpack
-        run: |
-          yarn webpack --display errors-only
-
-      - name: Run acceptance tests (#${{ steps.setup.outputs.matrix-instance-number }} of ${{ strategy.job-total }})
-        if: always()
-        run: |
-          mkdir -p ${{ steps.setup.outputs.acceptance-dir }}
-          mkdir -p ${{ steps.setup.outputs.acceptance-dir }}-mobile
-          mkdir -p ${{ steps.setup.outputs.acceptance-dir }}-tooltips
-          make run-acceptance
-        env:
-          PYTEST_SNAPSHOTS_DIR: ${{ steps.setup.outputs.acceptance-dir }}
-          USE_SNUBA: 1
-
-      # TODO(joshuarli): SENTRY_PYTHON3=1, snapshots, visual-diff needs py3-acceptance.
-
   visual-diff:
     if: ${{ github.ref != 'refs/heads/master' }}
-    needs: [acceptance, jest]
+    needs: [acceptance, frontend]
     runs-on: ubuntu-16.04
 
     steps: