build(deps): Move optional dependencies to base (#15328)

There are only a handful optional dependencies and we need to install them anyway for development and Docker images so there's no need to keep them separate. This PR moves them into the base file and adds an `# optional` comment before each one to tag them.

.travis.yml

@@ -41,7 +41,7 @@ env:
     - NODE_OPTIONS=--max-old-space-size=4096
 
 before_install:
-  - pip install "pip==${PYTHON_PIP_VERSION}"
+  - &pip_install pip install "pip==${PYTHON_PIP_VERSION}"
 
 script:
   # certain commands require sentry init to be run, but this is only true for
@@ -77,12 +77,13 @@ base_postgres: &postgres_default
     - redis-server
     - postgresql
   before_install:
+    - *pip_install
     - docker run -d --network host --name clickhouse-server --ulimit nofile=262144:262144 yandex/clickhouse-server:19.4
     - docker run -d --network host --name snuba --env SNUBA_SETTINGS=test --env CLICKHOUSE_SERVER=localhost:9000 getsentry/snuba
     - docker ps -a
   install:
     - python setup.py install_egg_info
-    - pip install -U -e ".[dev,optional]"
+    - pip install -U -e ".[dev]"
   before_script:
     - psql -c 'create database sentry;' -U postgres
 
@@ -94,6 +95,7 @@ base_acceptance: &acceptance_default
     - redis-server
     - postgresql
   before_install:
+    - *pip_install
     - find "$NODE_DIR" -type d -empty -delete
     - nvm install
     - docker run -d --network host --name clickhouse-server --ulimit nofile=262144:262144 yandex/clickhouse-server:19.4
@@ -102,7 +104,7 @@ base_acceptance: &acceptance_default
   install:
     - ./bin/yarn install --pure-lockfile
     - python setup.py install_egg_info
-    - pip install -U -e ".[dev,optional]"
+    - pip install -U -e ".[dev]"
     - |
       CHROME_MAJOR_VERSION="$(dpkg -s google-chrome-stable | sed -nr 's/Version: ([0-9]+).*/\1/p')"
       wget -N "https://chromedriver.storage.googleapis.com/$(curl https://chromedriver.storage.googleapis.com/LATEST_RELEASE_${CHROME_MAJOR_VERSION})/chromedriver_linux64.zip" -P ~/
@@ -125,7 +127,7 @@ matrix:
       env: TEST_SUITE=lint
       install:
         - python setup.py install_egg_info
-        - SENTRY_LIGHT_BUILD=1 pip install -U -e ".[dev,optional]"
+        - SENTRY_LIGHT_BUILD=1 pip install -U -e ".[dev]"
         - find "$NODE_DIR" -type d -empty -delete
         - nvm install
         - ./bin/yarn install --pure-lockfile
@@ -175,6 +177,7 @@ matrix:
       name: 'Frontend'
       env: TEST_SUITE=js
       before_install:
+        - *pip_install
         - find "$NODE_DIR" -type d -empty -delete
         - nvm install
       install:
@@ -196,6 +199,7 @@ matrix:
       name: 'Distribution build'
       env: TEST_SUITE=dist
       before_install:
+        - *pip_install
         - find "$NODE_DIR" -type d -empty -delete
         - nvm install
 
@@ -217,6 +221,7 @@ matrix:
         - redis-server
         - postgresql
       before_install:
+        - *pip_install
         - docker run -d --network host --name zookeeper -e ZOOKEEPER_CLIENT_PORT=2181 confluentinc/cp-zookeeper:4.1.0
         - docker run -d --network host --name kafka -e KAFKA_ZOOKEEPER_CONNECT=localhost:2181 -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 -e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 confluentinc/cp-kafka:4.1.0
         - docker run -d --network host --name clickhouse-server --ulimit nofile=262144:262144 yandex/clickhouse-server:19.4
@@ -224,7 +229,7 @@ matrix:
         - docker ps -a
       install:
         - python setup.py install_egg_info
-        - pip install -U -e ".[dev,optional]"
+        - pip install -U -e ".[dev]"
         - pip install confluent-kafka
       before_script:
         - psql -c 'create database sentry;' -U postgres
@@ -234,7 +239,7 @@ matrix:
       name: 'Storybook Deploy'
       env: STORYBOOK_BUILD=1
       before_install:
-        # Decrypt the credentials we added to the repo using the key we added with the Travis command line tool
+         # Decrypt the credentials we added to the repo using the key we added with the Travis command line tool
         - openssl aes-256-cbc -K $encrypted_020be61ef175_key -iv $encrypted_020be61ef175_iv -in .travis/storybook-credentials.tar.gz.enc -out credentials.tar.gz -d
         # If the SDK is not already cached, download it and unpack it
         - if [ ! -d ${HOME}/google-cloud-sdk ]; then curl https://sdk.cloud.google.com | bash; fi

Makefile

@@ -94,7 +94,7 @@ install-yarn-pkgs:
 
 install-sentry-dev:
 	@echo "--> Installing Sentry (for development)"
-	$(PIP) install -e ".[dev,optional]" $(PIP_OPTS)
+	$(PIP) install -e ".[dev]" $(PIP_OPTS)
 
 build-js-po: node-version-check
 	mkdir -p build

docker/Dockerfile

@@ -131,6 +131,11 @@ RUN set -x \
     # librabbitmq
     && buildDeps="$buildDeps \
       make \
+    " \
+       # xmlsec
+    && buildDeps="$buildDeps \
+      libxmlsec1-dev \
+      pkg-config \
     " \
     && apt-get update \
     && apt-get install -y --no-install-recommends $buildDeps \

requirements-base.txt

@@ -18,6 +18,14 @@ email-reply-parser>=0.2.0,<0.3.0
 enum34>=1.1.6,<1.2.0
 functools32>=3.2.3,<3.3
 futures>=3.2.0,<4.0.0
+# optional but needed for google stuff below
+google-cloud-core==0.29.1
+# optional
+google-cloud-bigtable>=0.32.1,<0.33.0
+# optional
+google-cloud-pubsub>=0.35.4,<0.36.0
+# optional
+google-cloud-storage>=1.13.2,<1.14
 # broken on python3
 hiredis>=0.1.0,<0.2.0
 ipaddress>=1.0.16,<1.1.0
@@ -25,6 +33,8 @@ jsonschema==2.6.0
 kombu==3.0.35
 loremipsum>=1.0.5,<1.1.0
 lxml>=3.4.1
+# optional
+maxminddb==1.4.1
 # for vsts repo
 mistune>0.7,<0.9
 mmh3>=2.3.1,<2.4
@@ -42,6 +52,8 @@ python-dateutil>=2.0.0,<3.0.0
 python-memcached>=1.53,<2.0.0
 python-openid>=2.2
 python-u2flib-server>=5.0.0,<6.0.0
+# optional
+python3-saml>=1.4.0,<1.5
 PyYAML>=3.11,<3.12
 qrcode>=5.2.2,<6.0.0
 querystring_parser>=1.2.3,<2.0.0

requirements-optional.txt

@@ -1,5 +1 @@
-google-cloud-bigtable>=0.32.1,<0.33.0
-google-cloud-pubsub>=0.35.4,<0.36.0
-google-cloud-storage>=1.13.2,<1.14
-maxminddb==1.4.1
-python3-saml>=1.4.0,<1.5
+# DO NOT EDIT THIS FILE -- TO BE DELETED

setup.py

@@ -73,7 +73,6 @@ def get_requirements(env):
 
 install_requires = get_requirements("base")
 dev_requires = get_requirements("dev")
-optional_requires = get_requirements("optional")
 
 # override django version in requirements file if DJANGO_VERSION is set
 DJANGO_VERSION = os.environ.get("DJANGO_VERSION")
@@ -135,7 +134,7 @@ setup(
     packages=find_packages("src"),
     zip_safe=False,
     install_requires=install_requires,
-    extras_require={"dev": dev_requires, "postgres": [], "optional": optional_requires},
+    extras_require={"dev": dev_requires, "postgres": []},
     cmdclass=cmdclass,
     license="BSD",
     include_package_data=True,