build(travis): Spring Cleaning (#8419)

cool, and nice

.gitignore

@@ -13,7 +13,6 @@ test.conf
 pip-log.txt
 celerybeat-schedule
 sentry-package.json
-/htmlcov
 /coverage/
 /cover
 /build

.travis.yml

@@ -1,19 +1,22 @@
 filter_secrets: false
-group: deprecated-2017Q4
+
 dist: trusty
 sudo: required
+group: deprecated-2017Q4
 language: python
-rvm:
-  - 2.2
+python: 2.7
+
+branches:
+  only:
+  - master
+
 cache:
+  pip: true
   yarn: true
   directories:
     - node_modules
-    - $HOME/.cache/pip/wheels
     - $HOME/google-cloud-sdk
-branches:
-  only:
-  - master
+
 addons:
   apt:
     sources:
@@ -22,29 +25,26 @@ addons:
       - g++-4.8
       - libxmlsec1-dev
   chrome: stable
+
 env:
   global:
-    - NODE_ENV=production
+    - NODE_ENV=development
     - PIP_DISABLE_PIP_VERSION_CHECK=on
     - SENTRY_LIGHT_BUILD=1
     - SENTRY_SKIP_BACKEND_VALIDATION=1
-    - TRAVIS_NODE_VERSION=8.9.1
     - CXX=g++-4.8
     - SOUTH_TESTS_MIGRATE=1
-    - JEST_JUNIT_OUTPUT=jest.junit.xml
-    - DJANGO_VERSION=">=1.6,<1.7"
-install:
-  - 'export PATH=$PATH:~/.bin'
-  - nvm install $TRAVIS_NODE_VERSION
-  - npm install -g yarn@1.3.2
-  - make travis-install-$TEST_SUITE
-before_script:
-  - pip freeze
+    - DJANGO_VERSION=">=1.6.11,<1.7"
+    - NPM_VERSION="8.9.1"
+    - YARN_VERSION="1.3.2"
+
 script:
   - make travis-lint-$TEST_SUITE
   - make travis-test-$TEST_SUITE
   - make travis-scan-$TEST_SUITE
+
 after_success:
+  - pip install codecov
   - codecov -e TEST_SUITE
   - npm install -g @zeus-ci/cli
   - zeus upload -t "text/xml+xunit" junit.xml
@@ -55,6 +55,7 @@ after_success:
   - zeus upload -t "text/plain+pycodestyle" flake8.pycodestyle.log
   - zeus upload -t "text/xml+checkstyle" eslint.checkstyle.xml
   - zeus upload -t "application/webpack-stats+json" webpack-stats.json
+
 after_failure:
   - dmesg | tail -n 100
   - npm install -g @zeus-ci/cli
@@ -66,7 +67,10 @@ after_failure:
   - zeus upload -t "text/plain+pycodestyle" flake8.pycodestyle.log
   - zeus upload -t "text/xml+checkstyle" eslint.checkstyle.xml
   - zeus upload -t "application/webpack-stats+json" webpack-stats.json
-# each attribute in the matrix will override the global attribute
+
+# each job in the matrix inherits `env/global` and uses everything above,
+# but custom `services`, `before_install`, `install`, and `before_script` directives
+# may be defined to define and setup individual job environments with more precision.
 matrix:
   fast_finish: true
   include:
@@ -77,51 +81,104 @@ matrix:
         - memcached
         - riak
         - redis-server
+      install:
+        - pip install -e ".[dev,tests,optional]"
+
     - python: 2.7
       env: TEST_SUITE=postgres DB=postgres
       services:
         - memcached
         - redis-server
         - postgresql
+      install:
+        - pip install -e ".[dev,tests,optional]"
+      before_script:
+        - psql -c 'create database sentry;' -U postgres
+
     - python: 2.7
       env: TEST_SUITE=network DB=postgres
       services:
         - memcached
         - redis-server
         - postgresql
+      install:
+        - pip install -e ".[dev,tests,optional]"
+      before_script:
+        - psql -c 'create database sentry;' -U postgres
+
     - python: 2.7
       env: TEST_SUITE=mysql DB=mysql
       services:
         - memcached
         - mysql
         - redis-server
+      install:
+        - pip install -e ".[dev,tests,optional]"
+        - pip install mysqlclient
+      before_script:
+        - mysql -u root -e 'create database sentry;'
+
     - python: 2.7
       env: TEST_SUITE=acceptance
       services:
         - memcached
         - redis-server
         - postgresql
+      install:
+        - pip install -e ".[dev,tests,optional]"
+        - wget -N "http://chromedriver.storage.googleapis.com/$(curl https://chromedriver.storage.googleapis.com/LATEST_RELEASE)/chromedriver_linux64.zip" -P ~/
+        - unzip ~/chromedriver_linux64.zip -d ~/
+        - rm ~/chromedriver_linux64.zip
+        - sudo install -m755 ~/chromedriver /usr/local/bin/
+      before_script:
+        - psql -c 'create database sentry;' -U postgres
+
     - python: 2.7
       env: TEST_SUITE=js
+      before_install:
+        - nvm install "$NPM_VERSION"
+        - npm install -g "yarn@${YARN_VERSION}"
+      install:
+        - yarn install --pure-lockfile
+        - pip install $(cat requirements*.txt | grep -E 'click|pycodestyle')
+
     - python: 2.7
       env: TEST_SUITE=cli
       services:
         - postgresql
         - redis-server
+      install:
+        - pip install -e .
+      before_script:
+        - psql -c 'create database sentry;' -U postgres
+
     - python: 2.7
       env: TEST_SUITE=dist
+
     - python: 2.7
-      env: TEST_SUITE=postgres DJANGO_VERSION=">=1.8,<1.9"
+      env: SENTRY_TAGSTORE=sentry.tagstore.v2.V2TagStorage TEST_SUITE=postgres DB=postgres
       services:
         - memcached
         - redis-server
         - postgresql
+      install:
+        - pip install -e ".[dev,tests,optional]"
+      before_script:
+        - psql -c 'create database sentry;' -U postgres
+
+    # django 1.8 in testing - allowed to fail
     - python: 2.7
-      env: SENTRY_TAGSTORE=sentry.tagstore.v2.V2TagStorage TEST_SUITE=postgres DB=postgres
+      env: TEST_SUITE=postgres DJANGO_VERSION=">=1.8,<1.9"
       services:
         - memcached
         - redis-server
         - postgresql
+      install:
+        - pip install -e ".[dev,tests,optional]"
+      before_script:
+        - psql -c 'create database sentry;' -U postgres
+
+    # snuba in testing - allowed to fail
     - python: 2.7
       env: TEST_SUITE=snuba SENTRY_TAGSTORE=sentry.tagstore.snuba.SnubaTagStorage
       services:
@@ -133,8 +190,12 @@ matrix:
         - docker run -d --name clickhouse-server -p 9000:9000 -p 9009:9009 -p 8123:8123 --ulimit nofile=262144:262144 yandex/clickhouse-server
         - docker run -d --env SNUBA_SETTINGS=test --env CLICKHOUSE_SERVER=clickhouse-server:9000 --name snuba -p 1218:1218 --link clickhouse-server:clickhouse-server getsentry/snuba
         - docker ps -a
+      install:
+        - pip install -e ".[dev,tests,optional]"
+      before_script:
+        - psql -c 'create database sentry;' -U postgres
 
-    # Deploy 'storybook' (component & style guide)
+    # Deploy 'storybook' (component & style guide) - allowed to fail
     - language: node_js
       env: STORYBOOK_BUILD=1
       before_install:
@@ -145,15 +206,21 @@ matrix:
         - tar -xzf credentials.tar.gz
         # Use the decrypted service account credentials to authenticate the command line tool
         - gcloud auth activate-service-account --key-file client-secret.json
+        # Travis doesn't "inherit" steps like before_install in the build matrix, so we have to explicitly install our package managers
+        - nvm install "$NPM_VERSION"
+        - npm install -g "yarn@${YARN_VERSION}"
       install:
-        - nvm install $TRAVIS_NODE_VERSION
-        - make install-yarn
+        - yarn install --pure-lockfile
         - gcloud version
+      script: bash .travis/deploy-storybook.sh
       after_success: skip
       after_failure: skip
-      script: bash .travis/deploy-storybook.sh
 
+# jobs are defined in matrix/include
+# to specify which jobs are allowed to fail, match the env exactly in matrix/allow_failures
   allow_failures:
+    - python: 2.7
+      env: TEST_SUITE=postgres DJANGO_VERSION=">=1.8,<1.9"
     - python: 2.7
       env: TEST_SUITE=snuba SENTRY_TAGSTORE=sentry.tagstore.snuba.SnubaTagStorage
     - language: node_js

Makefile

@@ -1,6 +1,4 @@
-NPM_ROOT = ./node_modules
 STATIC_DIR = src/sentry/static/sentry
-DJANGO_VERSION := ">=1.6,<1.7"
 
 ifneq "$(wildcard /usr/local/opt/libxmlsec1/lib)" ""
 	LDFLAGS += -L/usr/local/opt/libxmlsec1/lib
@@ -10,40 +8,14 @@ ifneq "$(wildcard /usr/local/opt/openssl/lib)" ""
 endif
 
 PIP = LDFLAGS="$(LDFLAGS)" pip -q
+WEBPACK = NODE_ENV=production ./node_modules/.bin/webpack
 
-develop-only: update-submodules install-brew install-python install-yarn
-
+test: develop lint test-js test-python test-cli
 develop: setup-git develop-only
-	@echo ""
-
-install-yarn:
-	@echo "--> Installing Node dependencies"
-	@hash yarn 2> /dev/null || (echo 'Cannot continue with JavaScript dependencies. Please install yarn before proceeding. For more information refer to https://yarnpkg.com/lang/en/docs/install/'; echo 'If you are on a mac run:'; echo '  brew install yarn'; exit 1)
-	# Use NODE_ENV=development so that yarn installs both dependencies + devDependencies
-	NODE_ENV=development yarn install --pure-lockfile
-
-install-brew:
-	@hash brew 2> /dev/null && brew bundle || (echo '! Homebrew not found, skipping system dependencies.')
+# Used by https://github.com/getsentry/sentry-docs/blob/master/bin/extract-docs
+develop-only: update-submodules install-system-pkgs install-yarn-pkgs install-sentry-dev
 
-install-python:
-	# must be executed serially
-	$(MAKE) install-python-base
-	$(MAKE) install-python-develop
-
-install-python-base:
-	@echo "--> Installing Python dependencies"
-	$(PIP) install "setuptools>=0.9.8" "pip>=8.0.0"
-	# order matters here, base package must install first
-	$(PIP) install -e .
-	$(PIP) install "file://`pwd`#egg=sentry[dev]"
-
-install-python-develop:
-	$(PIP) install "file://`pwd`#egg=sentry[dev,tests]"
-
-install-python-tests:
-	$(PIP) install "file://`pwd`#egg=sentry[dev,tests,optional]"
-
-dev-postgres: install-python
+build: locale
 
 dev-docs:
 	$(PIP) install -r doc-requirements.txt
@@ -56,14 +28,6 @@ reset-db:
 	@echo "--> Applying migrations"
 	sentry upgrade
 
-setup-git:
-	@echo "--> Installing git hooks"
-	git config branch.autosetuprebase always
-	cd .git/hooks && ln -sf ../../config/hooks/* ./
-	@echo ""
-
-build: locale
-
 clean:
 	@echo "--> Cleaning static cache"
 	rm -rf dist/* static/dist/*
@@ -75,9 +39,35 @@ clean:
 	rm -rf build/ dist/ src/sentry/assets.json
 	@echo ""
 
+setup-git:
+	@echo "--> Installing git hooks"
+	git config branch.autosetuprebase always
+	cd .git/hooks && ln -sf ../../config/hooks/* ./
+	@echo ""
+
+update-submodules:
+	@echo "--> Updating git submodules"
+	git submodule init
+	git submodule update
+	@echo ""
+
+install-system-pkgs:
+	@echo "--> Installing system packages (from Brewfile)"
+	@command -v brew 2>&1 > /dev/null && brew bundle || (echo 'WARNING: homebrew not found or brew bundle failed - skipping system dependencies.')
+
+install-yarn-pkgs:
+	@echo "--> Installing Yarn packages (for development)"
+	@command -v yarn 2>&1 > /dev/null || (echo 'yarn not found. Please install it before proceeding.'; exit 1)
+	# Use NODE_ENV=development so that yarn installs both dependencies + devDependencies
+	NODE_ENV=development yarn install --pure-lockfile
+
+install-sentry-dev:
+	@echo "--> Installing Sentry (for development)"
+	$(PIP) install -e ".[dev,tests,optional]"
+
 build-js-po:
 	mkdir -p build
-	SENTRY_EXTRACT_TRANSLATIONS=1 ./node_modules/.bin/webpack
+	SENTRY_EXTRACT_TRANSLATIONS=1 $(WEBPACK)
 
 locale: build-js-po
 	cd src/sentry && sentry django makemessages -i static -l en
@@ -94,23 +84,11 @@ update-transifex: build-js-po
 	./bin/find-good-catalogs src/sentry/locale/catalogs.json
 	cd src/sentry && sentry django compilemessages
 
-update-submodules:
-	@echo "--> Updating git submodules"
-	git submodule init
-	git submodule update
-	@echo ""
-
 build-platform-assets:
 	@echo "--> Building platform assets"
 	sentry init
 	@echo "from sentry.utils.integrationdocs import sync_docs; sync_docs(quiet=True)" | sentry exec
 
-test: develop lint test-js test-python test-cli
-
-testloop: develop
-	pip install -q pytest-xdist
-	py.test tests -f
-
 test-cli:
 	@echo "--> Testing CLI"
 	rm -rf test_cli
@@ -123,7 +101,7 @@ test-cli:
 
 test-js:
 	@echo "--> Building static assets"
-	@${NPM_ROOT}/.bin/webpack --profile --json > webpack-stats.json
+	@$(WEBPACK) --profile --json > webpack-stats.json
 	@echo "--> Running JavaScript tests"
 	@npm run test-ci
 	@echo ""
@@ -154,7 +132,7 @@ test-snuba:
 
 test-acceptance: build-platform-assets
 	@echo "--> Building static assets"
-	@${NPM_ROOT}/.bin/webpack --display errors-only
+	@$(WEBPACK) --display errors-only
 	@echo "--> Running acceptance tests"
 	py.test tests/acceptance --cov . --cov-report="xml:coverage.xml" --junit-xml="junit.xml" --html="pytest.html"
 	@echo ""
@@ -177,68 +155,17 @@ scan-python:
 	bin/scan
 	@echo ""
 
-coverage: develop
-	$(MAKE) test-python
-	coverage html
-
 publish:
 	python setup.py sdist bdist_wheel upload
 
-extract-api-docs:
-	rm -rf api-docs/cache/*
-	cd api-docs; python generator.py
-
-
-.PHONY: develop dev-postgres dev-docs setup-git build clean locale update-transifex update-submodules test testloop test-cli test-js test-styleguide test-python test-acceptance lint lint-python lint-js coverage publish scan-python
-
 
 ############################
 # Halt, Travis stuff below #
 ############################
 
-# Bases for all builds
-travis-upgrade-pip:
-	python -m pip install -q "pip>=9,<10"
-travis-setup-cassandra:
-	echo "create keyspace sentry with replication = {'class' : 'SimpleStrategy', 'replication_factor': 1};" | cqlsh --cqlversion=3.1.7
-	echo 'create table nodestore (key text primary key, value blob, flags int);' | cqlsh -k sentry --cqlversion=3.1.7
-travis-install-python:
-	pip install -q Django${DJANGO_VERSION}
-	$(MAKE) travis-upgrade-pip
-	$(MAKE) install-python-base
-	$(MAKE) install-python-tests
-	python -m pip install -q codecov
 travis-noop:
 	@echo "nothing to do here."
 
-.PHONY: travis-upgrade-pip travis-setup-cassandra travis-install-python travis-noop
-
-travis-install-sqlite: travis-install-python
-travis-install-postgres: travis-install-python dev-postgres
-	psql -c 'create database sentry;' -U postgres
-travis-install-mysql: travis-install-python
-	pip install -q mysqlclient
-	echo 'create database sentry;' | mysql -uroot
-travis-install-acceptance: install-yarn travis-install-postgres
-	wget -N http://chromedriver.storage.googleapis.com/$(shell curl https://chromedriver.storage.googleapis.com/LATEST_RELEASE)/chromedriver_linux64.zip -P ~/
-	unzip ~/chromedriver_linux64.zip -d ~/
-	rm ~/chromedriver_linux64.zip
-	chmod +x ~/chromedriver
-	mkdir -p ~/.bin
-	mv ~/chromedriver ~/.bin/
-travis-install-network: travis-install-postgres
-travis-install-snuba: travis-install-postgres
-travis-install-js:
-	$(MAKE) travis-upgrade-pip
-	$(MAKE) travis-install-python install-yarn
-travis-install-cli: travis-install-postgres
-travis-install-dist:
-	$(MAKE) travis-upgrade-pip
-	$(MAKE) travis-install-python install-yarn
-travis-install-django-18: travis-install-postgres
-
-.PHONY: travis-install-sqlite travis-install-postgres travis-install-js travis-install-cli travis-install-dist
-
 # Lint steps
 travis-lint-sqlite: lint-python
 travis-lint-postgres: lint-python
@@ -249,9 +176,6 @@ travis-lint-snuba: lint-python
 travis-lint-js: lint-js
 travis-lint-cli: travis-noop
 travis-lint-dist: travis-noop
-travis-lint-django-18: travis-lint-postgres
-
-.PHONY: travis-lint-sqlite travis-lint-postgres travis-lint-mysql travis-lint-js travis-lint-cli travis-lint-dist
 
 # Test steps
 travis-test-sqlite: test-python
@@ -260,16 +184,11 @@ travis-test-mysql: test-python
 travis-test-acceptance: test-acceptance
 travis-test-network: test-network
 travis-test-snuba: test-snuba
-travis-test-js:
-	$(MAKE) test-js
+travis-test-js: test-js
 travis-test-cli: test-cli
 travis-test-dist:
 	SENTRY_BUILD=$(TRAVIS_COMMIT) SENTRY_LIGHT_BUILD=0 python setup.py sdist bdist_wheel
 	@ls -lh dist/
-travis-test-django-18: travis-test-postgres
-
-.PHONY: travis-test-sqlite travis-test-postgres travis-test-mysql travis-test-js travis-test-cli travis-test-dist
-
 
 # Scan steps
 travis-scan-sqlite: scan-python
@@ -281,6 +200,5 @@ travis-scan-snuba: scan-python
 travis-scan-js: travis-noop
 travis-scan-cli: travis-noop
 travis-scan-dist: travis-noop
-travis-scan-django-18: travis-noop
 
-.PHONY: travis-scan-sqlite travis-scan-postgres travis-scan-mysql travis-scan-acceptance travis-scan-network travis-scan-snuba travis-scan-js travis-scan-cli travis-scan-dist travis-scan-django-18
+.PHONY: all develop develop-only build dev-docs test testloop reset-db clean setup-git update-submodules install-system-pkgs install-yarn-pkgs install-sentry install-sentry-dev build-js-po locale update-transifex build-platform-assets test-cli test-js test-styleguide test-python test-network test-snuba test-acceptance lint lint-python lint-js scan-python coverage publish extract-api-docs travis-noop travis-setup-cassandra travis-lint-sqlite travis-lint-postgres travis-lint-mysql travis-lint-acceptance travis-lint-network travis-lint-snuba travis-lint-js travis-lint-cli travis-lint-dist travis-test-sqlite travis-test-postgres travis-test-mysql travis-test-acceptance travis-test-network travis-test-snuba travis-test-js travis-test-cli travis-test-dist travis-scan-sqlite travis-scan-postgres travis-scan-mysql travis-scan-acceptance travis-scan-network travis-scan-snuba travis-scan-js travis-scan-cli travis-scan-dist

setup.py

@@ -63,24 +63,22 @@ for m in ('multiprocessing', 'billiard'):
 IS_LIGHT_BUILD = os.environ.get('SENTRY_LIGHT_BUILD') == '1'
 
 # we use pip requirements files to improve Docker layer caching
-
-
 def get_requirements(env):
     with open('requirements-{}.txt'.format(env)) as fp:
         return [x.strip() for x in fp.read().split('\n') if not x.startswith('#')]
 
-
 install_requires = get_requirements('base')
 dev_requires = get_requirements('dev')
 tests_require = get_requirements('test')
 optional_requires = get_requirements('optional')
 
+# override django version in requirements file if DJANGO_VERSION is set
 DJANGO_VERSION = os.environ.get('DJANGO_VERSION')
 if DJANGO_VERSION:
     install_requires = [
-        'Django{}'.format(DJANGO_VERSION[1:-1])
-        if k.startswith('Django>=') else k
-        for k in install_requires
+        'Django{}'.format(DJANGO_VERSION)
+        if r.startswith('Django>=') else r
+        for r in install_requires
     ]