feat(ci): Optimizations for dev env workflow (#29658)

- Do not use the Brew cache as it takes a long time
- On CI, only install `libxmlsec1` since we have full coverage via bootstrap-script
- Add some checks as part of the set up step

.github/actions/setup-python/action.yml

@@ -54,18 +54,14 @@ runs:
       with:
         python-version: ${{ steps.python-version.outputs.python-version }}
 
-    - name: Install pip
-      shell: bash
-      env:
-        WORKDIR: ${{ inputs.workdir }}
-      run: |
-        source "$WORKDIR/scripts/lib.sh" && upgrade-pip
-
     - name: Get pip cache dir & version
       id: pip-info
       shell: bash
+      env:
+        WORKDIR: ${{ inputs.workdir }}
       # pip versions before 20.1 do not have `pip cache`, `make upgrade-pip` ensures that version as a minimum
       run: |
+        source "$WORKDIR/scripts/lib.sh" && upgrade-pip
         echo "::set-output name=pip-cache-dir::$(pip cache dir)"
         echo "::set-output name=pip-version::$(pip -V | awk -F ' ' '{print $2}')"
 

.github/workflows/development-environment.yml

@@ -4,28 +4,25 @@ on:
     paths:
       - '.pre-commit-config.yaml'
       - 'Makefile'
-      - '.github/actions/*'
+      - '.github/actions/setup-python'
       - '.github/workflows/development-environment.yml'
       - '.envrc'
       - 'Brewfile'
       - 'scripts/*'
       - 'src/sentry/runner/commands/devserver.py'
       - 'src/sentry/runner/commands/devservices.py'
+      - 'requirements-*.txt'
 
 jobs:
   dev-environment:
     name: dev docs set up
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 90
+    timeout-minutes: 40
     strategy:
       matrix:
-        # Using Ubuntu 18 until I figure out this error:
-        # -> ImportError: libffi.so.6: cannot open shared object file: No such file or directory
-        os: [macos-11.0, ubuntu-18.04]
-        python-version: [3.8.12]
+        os: [macos-11.0, ubuntu-20.04]
       fail-fast: false
     env:
-      PIP_DISABLE_PIP_VERSION_CHECK: on
       # Make the environment more similar to what Mac defaults to
       SHELL: /bin/zsh
 
@@ -36,27 +33,14 @@ jobs:
       - name: Set variables for caches
         id: info
         run: |
-          echo "::set-output name=brew-cache-dir::$(brew --cache)"
           echo "::set-output name=yarn-cache-dir::$(yarn cache dir)"
 
-      - name: Cache (brew)
-        uses: actions/cache@v2
-        with:
-          path: ${{ steps.info.outputs.brew-cache-dir }}
-          key: devenv-${{ runner.os }}-brew-${{ hashFiles('Brewfile') }}
-          restore-keys: devenv-${{ runner.os }}-brew
-
       - name: Install prerequisites
-        # brew can be finicky but it does not always means that the rest of the job will fail
-        # Xcode CLI & brew are already installed, thus, no need to call xcode-select install
-        continue-on-error: true
         run: |
           make prerequisites
 
-      - name: Setup Python ${{ matrix.python-version }}
+      - name: Setup Python
         uses: ./.github/actions/setup-python
-        with:
-          python-version: ${{ matrix.python-version }}
 
       - name: Cache (pyenv)
         uses: actions/cache@v2
@@ -68,7 +52,7 @@ jobs:
         uses: actions/cache@v1 # We are explicitly using v1 due to perf reasons
         with:
           path: ${{ steps.info.outputs.yarn-cache-dir }}
-          key: devenv-${{ matrix.os }}-v2-yarn-${{ hashFiles('yarn.lock') }}
+          key: devenv-${{ matrix.os }}-v1-yarn-${{ hashFiles('yarn.lock') }}
 
       - name: Set up development environment (mostly as per docs)
         run: |
@@ -76,16 +60,19 @@ jobs:
           export VOLTA_HOME="$HOME/.volta"
           export PATH="$HOME/.volta/bin:$PATH"
           make setup-pyenv
-          eval "$(pyenv init --path)"
+          [[ $(which python) != "${HOME}/.pyenv/shims/python" ]]
+          source ~/.zprofile
+          [[ $(which python) == "${HOME}/.pyenv/shims/python" ]]
+          [[ $(python -V) == "Python $(cat .python-version)" ]]
           python -m venv .venv
           source .venv/bin/activate
           make develop init-config
+          pre-commit
 
   bootstrap-script:
     runs-on: macos-11
-    timeout-minutes: 90
+    timeout-minutes: 40
     env:
-      PIP_DISABLE_PIP_VERSION_CHECK: on
       # Make the environment more similar to what Mac defaults to
       SHELL: /bin/zsh
 

.github/workflows/python-deps.yml

@@ -1,48 +0,0 @@
-name: python deps
-on:
-  pull_request:
-    paths:
-      - '.github/actions/setup-python/action.yml'
-      - '.github/workflows/python-deps.yml'
-      - 'requirements-*.txt'
-
-jobs:
-  # This workflow executes faster than the dev env one and focuses on changes
-  # to requirement files
-  # This workflow makes sure that Python dependencies install correctly for
-  # a) our current version
-  python-deps:
-    name: install
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: 90
-    strategy:
-      matrix:
-        os: [macos-11.0, ubuntu-20.04]
-      fail-fast: false
-    env:
-      PIP_DISABLE_PIP_VERSION_CHECK: on
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Install prerequisites
-        env:
-          HOMEBREW_NO_AUTO_UPDATE: on
-        # Necessary packages for some Python deps
-        run: |
-          brew install libxmlsec1
-
-      - name: Setup Python
-        uses: ./.github/actions/setup-python
-        with:
-          # Non-default value since install-py-dev installs all requirements files
-          # We also want to bust the cache if the action and workflow change
-          cache-files-hash: ${{ hashFiles('requirements-*.txt', '.github/actions/setup-python/action.yml', '.github/workflows/python-deps.yml') }}
-
-      - name: Install dependencies
-        run: |
-          python -m venv .venv
-          source .venv/bin/activate
-          make install-py-dev
-          # This exercises the requirements-pre-commit.txt file
-          make setup-git

scripts/lib.sh

@@ -234,7 +234,11 @@ reset-db() {
 }
 
 prerequisites() {
-    brew update -q && brew bundle -q
+    if [ -z "${CI+x}" ]; then
+        brew update -q && brew bundle -q
+    else
+        HOMEBREW_NO_AUTO_UPDATE=on brew install libxmlsec1 pyenv
+    fi
 }
 
 direnv-help() {