ref: use internal pypi (#38255)

this has a bunch of things:
- speeds up `setup-sentry` by ~30-40 seconds
- removes a bunch of hacks for M1 since everything is prebuilt now
- avoids a whole class of failure modes with local development not
having build deps set up properly
- allows pip-compile to succeed on M1 without needing a bunch of
dependencies
- fully functional python setup on (macos (x86_64, arm64), linux (amd64,
aarch64))
- enables further installation optimizations with everything being a
wheel
- removes a bunch of unneeded apt dependencies

packages are sourced from https://github.com/getsentry/pypi

.github/actions/setup-sentry/action.yml

@@ -107,14 +107,6 @@ runs:
         cache: pip
         cache-dependency-path: ${{ inputs.workdir }}/requirements-dev-frozen.txt
 
-    - name: Install system dependencies
-      shell: bash
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y --no-install-recommends \
-          libxmlsec1-dev \
-          libmaxminddb-dev
-
     - name: Set up outputs
       id: config
       env:

.github/workflows/backend.yml

@@ -448,14 +448,10 @@ jobs:
 
       # We don't call setup-sentry, because we don't need devservices.
       - name: Setup backend typing
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends libxmlsec1-dev
-          pip install -r requirements-dev-frozen.txt
+        run: pip install -r requirements-dev-frozen.txt
 
       - name: Run backend typing (${{ steps.setup.outputs.matrix-instance-number }} of ${{ strategy.job-total }})
-        run: |
-          make backend-typing
+        run: make backend-typing
 
   # This check runs once all dependant jobs have passed
   # It symbolizes that all required Backend checks have succesfully passed (Or skipped)

.github/workflows/development-environment.yml

@@ -50,10 +50,6 @@ jobs:
           sudo /Applications/Docker.app/Contents/MacOS/Docker --unattended --install-privileged-components
           open -a /Applications/Docker.app --args --unattended --accept-license
 
-      - name: Install missing brew packages
-        run: |
-          HOMEBREW_NO_AUTO_UPDATE=1 brew install -v libxmlsec1
-
       - uses: ./.github/actions/setup-volta
 
       # This handles Python's cache
@@ -67,6 +63,9 @@ jobs:
       # This tests starting up the dev services, loading mocks and pre-commit installation
       # This can take over 15 minutes
       - name: make bootstrap
+        # GHA pythons are miscompiled and report macos 10.16
+        env:
+          SYSTEM_VERSION_COMPAT: 0
         run: |
           python -m venv .venv
           source .venv/bin/activate

Brewfile

@@ -10,15 +10,6 @@ brew 'readline'
 # required for yarn test -u
 brew 'watchman'
 
-# required to build some of sentry's dependencies
-brew 'pkgconfig'
-brew 'libxslt'
-brew 'libxmlsec1'
-brew 'geoip'
-
-# Currently needed because on Big Sur there's no wheel for it
-brew 'librdkafka'
-
 # direnv isn't defined here, because we have it configured to check for a bootstrapped environment.
 # If it's installed in the early steps of the setup process, it just leads to confusion.
 # brew 'direnv'

docker/Dockerfile

@@ -58,15 +58,6 @@ RUN set -x \
   wget \
   zlib1g-dev \
   " \
-  # maxminddb
-  && buildDeps="$buildDeps \
-  libmaxminddb-dev \
-  "\
-  # xmlsec
-  && buildDeps="$buildDeps \
-  libxmlsec1-dev \
-  pkg-config \
-  " \
   && apt-get update \
   && apt-get install -y --no-install-recommends $buildDeps \
   && pip install -r /tmp/requirements-frozen.txt \
@@ -78,26 +69,6 @@ RUN set -x \
   && mv dogstatsd_plugin.so /var/lib/uwsgi/ \
   && rm -rf /tmp/requirements-frozen.txt /tmp/uwsgi-dogstatsd .uwsgi_plugins_builder \
   && apt-get purge -y --auto-remove $buildDeps \
-  # We install run-time dependencies strictly after
-  # build dependencies to prevent accidental collusion.
-  # These are also installed last as they are needed
-  # during container run and can have the same deps w/
-  # build deps such as maxminddb.
-  && apt-get install -y --no-install-recommends \
-  # pillow
-  libjpeg-dev \
-  # rust bindings
-  libffi-dev \
-  # maxminddb bindings
-  libmaxminddb-dev \
-  # SAML needs these run-time
-  libxmlsec1-dev \
-  libxslt-dev \
-  # pyyaml needs this run-time
-  libyaml-dev \
-  # other
-  pkg-config \
-  \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/* \
   # Fully verify that the C extension is correctly installed, it unfortunately

requirements-base.txt

@@ -1,3 +1,5 @@
+--index-url https://pypi.devinfra.sentry.io/simple
+
 beautifulsoup4>=4.7.1
 boto3>=1.22.12
 botocore>=1.25.12

requirements-dev-frozen.txt

@@ -1,5 +1,7 @@
 # DO NOT MODIFY. This file was generated with `make freeze-requirements`.
 
+--index-url https://pypi.devinfra.sentry.io/simple
+
 amqp==2.6.1
 async-generator==1.10
 attrs==19.2.0

requirements-dev-only-frozen.txt

@@ -1,5 +1,7 @@
 # DO NOT MODIFY. This file was generated with `make freeze-requirements`.
 
+--index-url https://pypi.devinfra.sentry.io/simple
+
 attrs==21.4.0
 black==22.3.0
 build==0.8.0

requirements-dev.txt

@@ -1,3 +1,5 @@
+--index-url https://pypi.devinfra.sentry.io/simple
+
 docker>=3.7.0,<3.8.0
 exam>=0.10.6
 freezegun>=1.1.0

requirements-frozen.txt

@@ -1,5 +1,7 @@
 # DO NOT MODIFY. This file was generated with `make freeze-requirements`.
 
+--index-url https://pypi.devinfra.sentry.io/simple
+
 amqp==2.6.1
 async-generator==1.10
 attrs==19.2.0