build(ci): CI and local dev should upgrade pip the same way (#23795)

* Invalidate CI cache when a new version of pip is released
* CI and local dev should upgrade pip the same way

This will allow catching issues where we change the pip upgrade range in one place but not the
other.

.github/actions/setup-pip/action.yml

@@ -8,7 +8,10 @@ description: 'Installs pip and configures cache dir as output. This is hopefully
 outputs:
   pip-cache-dir:
     description: 'Path to pip cache'
-    value: ${{ steps.pip-cache.outputs.pip-cache }}
+    value: ${{ steps.pip-info.outputs.pip-cache }}
+  pip-version:
+    description: 'Pip version'
+    value: ${{ steps.pip-info.outputs.pip-version }}
 
 runs:
   using: 'composite'
@@ -21,10 +24,12 @@ runs:
     - name: Install pip
       shell: bash
       run: |
-        pip install --no-cache-dir --upgrade "pip>=20.0.2"
+        make upgrade-pip
 
-    - name: Get pip cache dir
-      id: pip-cache
+    - name: Get pip cache dir & version
+      id: pip-info
       shell: bash
+      # pip versions before 20.1 do not have `pip cache`, `make upgrade-pip` ensures that version as a minimum
       run: |
         echo "::set-output name=pip-cache::$(pip cache dir)"
+        echo "::set-output name=pip-version::$(pip -V | awk -F ' ' '{print $2}')"

.github/workflows/acceptance.yml

@@ -106,9 +106,9 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ${{ steps.pip.outputs.pip-cache-dir }}
-          key: ${{ runner.os }}-pip-py${{ steps.python-version.outputs.python-version }}-${{ hashFiles('**/requirements-*.txt') }}
+          key: ${{ runner.os }}-py${{ steps.python-version.outputs.python-version }}-pip${{ steps.pip.outputs.pip-version }}-${{ hashFiles('**/requirements-*.txt') }}
           restore-keys: |
-            ${{ runner.os }}-pip-
+            ${{ runner.os }}-py${{ steps.python-version.outputs.python-version }}-pip${{ steps.pip.outputs.pip-version }}
 
       - name: Setup sentry python env
         uses: ./.github/actions/setup-sentry

.github/workflows/api-docs-test.yml

@@ -48,9 +48,9 @@ jobs:
         if: steps.changes.outputs.api_docs == 'true'
         with:
           path: ${{ steps.pip.outputs.pip-cache-dir }}
-          key: ${{ runner.os }}-pip-py${{ steps.python-version.outputs.python-version }}-${{ hashFiles('**/requirements-*.txt') }}
+          key: ${{ runner.os }}-py${{ steps.python-version.outputs.python-version }}-pip${{ steps.pip.outputs.pip-version }}-${{ hashFiles('**/requirements-*.txt') }}
           restore-keys: |
-            ${{ runner.os }}-pip-py${{ steps.python-version.outputs.python-version }}
+            ${{ runner.os }}-py${{ steps.python-version.outputs.python-version }}-pip${{ steps.pip.outputs.pip-version }}
 
       - name: Setup sentry python env
         uses: ./.github/actions/setup-sentry

.github/workflows/backend-lint.yml

@@ -62,9 +62,9 @@ jobs:
         if: steps.changes.outputs.backend == 'true'
         with:
           path: ${{ steps.pip.outputs.pip-cache-dir }}
-          key: ${{ runner.os }}-pip-py${{ steps.python-version.outputs.python-version }}-${{ hashFiles('**/requirements-*.txt') }}
+          key: ${{ runner.os }}-py${{ steps.python-version.outputs.python-version }}-pip${{ steps.pip.outputs.pip-version }}-${{ hashFiles('**/requirements-*.txt') }}
           restore-keys: |
-            ${{ runner.os }}-pip-py${{ steps.python-version.outputs.python-version }}
+            ${{ runner.os }}-py${{ steps.python-version.outputs.python-version }}-pip${{ steps.pip.outputs.pip-version }}
 
       - name: Setup pre-commit
         if: steps.changes.outputs.backend == 'true'

.github/workflows/backend-test.yml

@@ -58,9 +58,9 @@ jobs:
         if: steps.changes.outputs.backend == 'true'
         with:
           path: ${{ steps.pip.outputs.pip-cache-dir }}
-          key: ${{ runner.os }}-pip-py${{ steps.python-version.outputs.python-version }}-${{ hashFiles('**/requirements-*.txt') }}
+          key: ${{ runner.os }}-py${{ steps.python-version.outputs.python-version }}-pip${{ steps.pip.outputs.pip-version }}-${{ hashFiles('**/requirements-*.txt') }}
           restore-keys: |
-            ${{ runner.os }}-pip-py${{ steps.python-version.outputs.python-version }}
+            ${{ runner.os }}-py${{ steps.python-version.outputs.python-version }}-pip${{ steps.pip.outputs.pip-version }}
 
       - name: Setup sentry env
         uses: ./.github/actions/setup-sentry

.github/workflows/development-environment.yml

@@ -13,7 +13,7 @@ jobs:
   dev-environment:
     name: set up
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 25
+    timeout-minutes: 35
     strategy:
       matrix:
         # macosx-10.15 is Catalina
@@ -64,9 +64,9 @@ jobs:
           path: |
             ${{ steps.pip.outputs.pip-cache-dir }}
             ~/.pyenv
-          key: ${{ runner.os }}-pip-py${{ steps.python-version.outputs.python-version }}-${{ hashFiles('**/requirements-*.txt') }}
+          key: ${{ matrix.os }}-py${{ steps.python-version.outputs.python-version }}-pip${{ steps.pip.outputs.pip-version }}-${{ hashFiles('**/requirements-*.txt') }}
           restore-keys: |
-            ${{ runner.os }}-pip-py${{ steps.python-version.outputs.python-version }}
+            ${{ matrix.os }}-py${{ steps.python-version.outputs.python-version }}-pip${{ steps.pip.outputs.pip-version }}
 
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
@@ -76,9 +76,9 @@ jobs:
         id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
         with:
           path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock', 'api-docs/yarn.lock') }}
+          key: ${{ matrix.os }}-yarn-${{ hashFiles('yarn.lock', 'api-docs/yarn.lock') }}
           restore-keys: |
-            ${{ runner.os }}-yarn-
+            ${{ matrix.os }}-yarn-
 
       - name: Set up development environment (as per docs)
         run: |

.github/workflows/snuba-integration-test.yml

@@ -59,9 +59,9 @@ jobs:
         if: steps.changes.outputs.backend == 'true'
         with:
           path: ${{ steps.pip.outputs.pip-cache-dir }}
-          key: ${{ runner.os }}-pip-py${{ steps.python-version.outputs.python-version }}-${{ hashFiles('**/requirements-*.txt') }}
+          key: ${{ runner.os }}-py${{ steps.python-version.outputs.python-version }}-pip${{ steps.pip.outputs.pip-version }}-${{ hashFiles('**/requirements-*.txt') }}
           restore-keys: |
-            ${{ runner.os }}-pip-py${{ steps.python-version.outputs.python-version }}
+            ${{ runner.os }}-py${{ steps.python-version.outputs.python-version }}-pip${{ steps.pip.outputs.pip-version }}
 
       - name: Setup sentry env
         uses: ./.github/actions/setup-sentry

Makefile

@@ -58,8 +58,12 @@ setup-pyenv:
 ensure-venv:
 	@./scripts/ensure-venv.sh
 
-ensure-pinned-pip: ensure-venv
-	$(PIP) install --no-cache-dir --upgrade "pip>=20.0.2"
+ensure-pinned-pip: ensure-venv upgrade-pip
+
+upgrade-pip:
+	# pip versions before 20.1 do not have `pip cache` as a command which is necessary for the CI
+	# XXX: Temp change to force cache invalidation
+	$(PIP) install --no-cache-dir --upgrade "pip==20.3.3"
 
 setup-git-config:
 	@git config --local branch.autosetuprebase always

scripts/pyenv_setup.sh

@@ -111,7 +111,8 @@ setup_pyenv() {
   echo "Activating pyenv and validating Python version"
   eval "$(pyenv init -)"
   python_version=$(python -V | sed s/Python\ //g)
-  [[ $python_version == $(cat .python-version) ]] || (echo "Wrong Python version: $python_version" && exit 1)
+  [[ $python_version == $(cat .python-version) ]] || \
+    (echo "Wrong Python version: $python_version. Please report in #discuss-dev-tooling" && exit 1)
 }
 
 setup_pyenv