build(github): Save acceptance test snapshots to GitHub Actions (#19557)

This adds an environment variable to save a PNG whenever we call `snapshot()` in acceptance tests. Use GitHub actions to save artifacts for the run as well. Changes acceptance tests to also run on master pushes.

.github/workflows/acceptance.yml

@@ -1,26 +1,14 @@
 name: acceptance
-on: pull_request
+on:
+  push:
+    branches:
+      - master
+  pull_request:
 
 jobs:
-    percynonce:
-      name: Generate Percy ID
-      runs-on: ubuntu-latest
-      steps:
-        - name: Generate Percy nonce
-          if: always()
-          run: |
-            echo ${{ github.run_id }}.$(($(date +%s))) > nonce.txt
-
-        - name: Upload nonce as artifact
-          if: always()
-          uses: actions/upload-artifact@v1
-          with:
-            name: percy
-            path: nonce.txt
-
     acceptance:
-      needs: [percynonce]
       runs-on: ubuntu-16.04
+      continue-on-error: true
       strategy:
         matrix:
           instance: [0, 1, 2]
@@ -50,13 +38,11 @@ jobs:
         # The hostname used to communicate with the PostgreSQL from sentry
         DATABASE_URL: postgresql://postgres:postgres@localhost/sentry
 
-        # Percy config
-        PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}
-        PERCY_PROJECT: ${{ secrets.PERCY_PROJECT }}
-
         # Number of matrix instances
         TOTAL_TEST_GROUPS: ${{ strategy.job-total }}
 
+        VISUAL_SNAPSHOT_ENABLE: 1
+
       services:
         clickhouse:
           image: yandex/clickhouse-server:19.11
@@ -121,11 +107,6 @@ jobs:
         # Install node
         - uses: volta-cli/action@v1
 
-        - name: Download percy nonce
-          uses: actions/download-artifact@v1
-          with:
-            name: percy
-
         # Yarn
         #   - See https://github.com/actions/cache/blob/master/examples.md#node---yarn for example
         # Python
@@ -142,7 +123,6 @@ jobs:
           run: |
             echo "::set-output name=yarn-cache-dir::$(yarn cache dir)"
             echo "::set-output name=python-version::2.7"
-            echo "::set-output name=percy-nonce::$(<percy/nonce.txt)"
             echo "::set-output name=matrix-instance-number::$(($MATRIX_INSTANCE+1))"
 
         # yarn cache
@@ -196,11 +176,7 @@ jobs:
             pip install wheel # GitHub Actions does not have this installed by default (unlike Travis)
             pip install -U -e ".[dev]"
             psql -c 'create database sentry;' -h localhost -U postgres
-
-        - name: Build platform assets
-          run: |
             sentry init
-            make build-platform-assets
 
         - name: webpack
           run: |
@@ -215,13 +191,16 @@ jobs:
 
         - name: Run acceptance tests (#${{ steps.config.outputs.matrix-instance-number }} of ${{ strategy.job-total }})
           if: always()
-          uses: percy/exec-action@v0.3.0
-          with:
-            command: "make run-acceptance"
+          run: |
+            mkdir -p .artifacts/visual-snapshots/acceptance
+            make run-acceptance
           env:
             USE_SNUBA: 1
             TEST_GROUP: ${{ matrix.instance }}
-            PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}
-            PERCY_PROJECT: ${{ secrets.PERCY_PROJECT }}
-            PERCY_PARALLEL_TOTAL: ${{ env.TOTAL_TEST_GROUPS }}
-            PERCY_PARALLEL_NONCE: ${{ steps.config.outputs.percy-nonce }}
+
+        - name: Save snapshots
+          if: always()
+          uses: actions/upload-artifact@v2
+          with:
+            name: visual-snapshots
+            path: .artifacts/visual-snapshots

src/sentry/static/sentry/app/utils/getDynamicText.tsx

@@ -9,5 +9,5 @@ export default function getDynamicText<Value, Fixed = Value>({
   value: Value;
   fixed: Fixed;
 }): Value | Fixed {
-  return process.env.IS_PERCY || process.env.FIXED_DYNAMIC_CONTENT ? fixed : value;
+  return process.env.IS_CI || process.env.FIXED_DYNAMIC_CONTENT ? fixed : value;
 }

src/sentry/static/sentry/app/utils/marked.tsx

@@ -49,7 +49,7 @@ marked.setOptions({
   sanitize: true,
 
   // Silence sanitize deprecation warning in test / travs (Travis sets NODE_NV
-  // to production, but specifies IS_PERCY).
+  // to production, but specifies `CI`).
   //
   // [!!] This has the side effect of causing failed markdown content to render
   //      as a html error, instead of throwing an exception, however none of
@@ -57,7 +57,7 @@ marked.setOptions({
   //      tradeoff to turn off off the deprecation warning.
   //
   // eslint-disable-next-line no-undef
-  silent: !!process.env.IS_PERCY || process.env.NODE_ENV === 'test',
+  silent: !!process.env.IS_CI || process.env.NODE_ENV === 'test',
 });
 
 const sanitizedMarked = (...args: Parameters<typeof marked>) =>

src/sentry/static/sentry/app/utils/testablePose.tsx

@@ -8,7 +8,7 @@ type PoseConfig = {[key: string]: any};
  *
  * This function simply sets delays and durations to 0.
  */
-const testablePose = !process.env.IS_PERCY
+const testablePose = !process.env.IS_CI
   ? (a: PoseConfig) => a
   : function(animation: PoseConfig) {
       Object.keys(animation).forEach(pose => {

src/sentry/utils/pytest/selenium.py

@@ -10,6 +10,7 @@ import pytest
 
 from datetime import datetime
 from django.conf import settings
+from django.utils.text import slugify
 from selenium import webdriver
 from selenium.common.exceptions import NoSuchElementException, WebDriverException
 from selenium.webdriver.support.ui import WebDriverWait
@@ -219,6 +220,11 @@ class Browser(object):
                 click.launch(tf.name)
                 time.sleep(1)
 
+        if os.environ.get("VISUAL_SNAPSHOT_ENABLE") == "1":
+            self.save_screenshot(
+                u".artifacts/visual-snapshots/acceptance/{}.png".format(slugify(name))
+            )
+
         self.percy.snapshot(name=name)
         return self
 

tests/js/spec/utils/getDynamicText.spec.jsx

@@ -10,9 +10,9 @@ describe('getDynamicText', function() {
     ).toEqual('Dynamic Content');
   });
 
-  it('renders fixed content when `process.env.IS_PERCY` is true', function() {
+  it('renders fixed content when `process.env.IS_CI` is true', function() {
     // eslint-disable-next-line no-undef
-    process.env.IS_PERCY = true;
+    process.env.IS_CI = true;
     expect(
       getDynamicText({
         fixed: 'Text',
@@ -20,6 +20,6 @@ describe('getDynamicText', function() {
       })
     ).toEqual('Text');
     // eslint-disable-next-line no-undef
-    process.env.IS_PERCY = null;
+    process.env.IS_CI = null;
   });
 });

webpack.config.js

@@ -309,6 +309,7 @@ let appConfig = {
       'process.env': {
         NODE_ENV: JSON.stringify(env.NODE_ENV),
         IS_PERCY: JSON.stringify(IS_PERCY),
+        IS_CI: JSON.stringify(IS_CI),
         DEPLOY_PREVIEW_CONFIG: JSON.stringify(DEPLOY_PREVIEW_CONFIG),
         EXPERIMENTAL_SPA: JSON.stringify(SENTRY_EXPERIMENTAL_SPA),
         SPA_DSN: JSON.stringify(env.SENTRY_SPA_DSN),
@@ -380,7 +381,7 @@ let appConfig = {
   devtool: IS_PRODUCTION ? 'source-map' : 'cheap-module-eval-source-map',
 };
 
-if (IS_TEST || IS_STORYBOOK) {
+if (IS_TEST || IS_CI || IS_STORYBOOK) {
   appConfig.resolve.alias['integration-docs-platforms'] = path.join(
     __dirname,
     'tests/fixtures/integration-docs/_platforms.json'