test: upgrade pytest and friends, cleanup configuration (#24315)

Makefile

@@ -128,7 +128,7 @@ fetch-release-registry:
 
 run-acceptance:
 	@echo "--> Running acceptance tests"
-	py.test tests/acceptance --cov . --cov-report="xml:.artifacts/acceptance.coverage.xml" --junit-xml=".artifacts/acceptance.junit.xml"
+	pytest tests/acceptance --cov . --cov-report="xml:.artifacts/acceptance.coverage.xml" --junit-xml=".artifacts/acceptance.junit.xml"
 	@echo ""
 
 test-cli:
@@ -161,22 +161,22 @@ test-js-ci: node-version-check
 test-python:
 	@echo "--> Running Python tests"
 	# This gets called by getsentry
-	py.test tests/integration tests/sentry
+	pytest tests/integration tests/sentry
 
 test-python-ci:
 	make build-platform-assets
 	@echo "--> Running CI Python tests"
-	py.test tests/integration tests/sentry --cov . --cov-report="xml:.artifacts/python.coverage.xml" --junit-xml=".artifacts/python.junit.xml" || exit 1
+	pytest tests/integration tests/sentry --cov . --cov-report="xml:.artifacts/python.coverage.xml" --junit-xml=".artifacts/python.junit.xml" || exit 1
 	@echo ""
 
 test-snuba:
 	@echo "--> Running snuba tests"
-	py.test tests/snuba tests/sentry/eventstream/kafka tests/sentry/snuba/test_discover.py -vv --cov . --cov-report="xml:.artifacts/snuba.coverage.xml" --junit-xml=".artifacts/snuba.junit.xml"
+	pytest tests/snuba tests/sentry/eventstream/kafka tests/sentry/snuba/test_discover.py -vv --cov . --cov-report="xml:.artifacts/snuba.coverage.xml" --junit-xml=".artifacts/snuba.junit.xml"
 	@echo ""
 
 test-symbolicator:
 	@echo "--> Running symbolicator tests"
-	py.test tests/symbolicator -vv --cov . --cov-report="xml:.artifacts/symbolicator.coverage.xml" --junit-xml=".artifacts/symbolicator.junit.xml"
+	pytest tests/symbolicator -vv --cov . --cov-report="xml:.artifacts/symbolicator.coverage.xml" --junit-xml=".artifacts/symbolicator.junit.xml"
 	@echo ""
 
 test-acceptance: node-version-check
@@ -186,7 +186,7 @@ test-acceptance: node-version-check
 
 test-plugins:
 	@echo "--> Running plugin tests"
-	py.test tests/sentry_plugins -vv --cov . --cov-report="xml:.artifacts/plugins.coverage.xml" --junit-xml=".artifacts/plugins.junit.xml" || exit 1
+	pytest tests/sentry_plugins -vv --cov . --cov-report="xml:.artifacts/plugins.coverage.xml" --junit-xml=".artifacts/plugins.junit.xml" || exit 1
 	@echo ""
 
 test-relay-integration:

conftest.py

@@ -12,5 +12,3 @@ def pytest_configure(config):
     # XXX(dcramer): Kombu throws a warning due to transaction.commit_manually
     # being used
     warnings.filterwarnings("error", "", Warning, r"^(?!(|kombu|raven|sentry))")
-
-    config.addinivalue_line("markers", "obsolete: mark test as obsolete and soon to be removed")

pyproject.toml

@@ -6,3 +6,19 @@ build-backend = "setuptools.build_meta"
 # File filtering is taken care of in pre-commit.
 line-length=100
 target-version=['py36']
+
+[tool.pytest.ini_options]
+# note: When updating the traceback format, make sure to update .github/pytest.json
+# We don't use the celery pytest plugin.
+# TODO: Warnings are disabled for now (they always have been)
+#       until the good ones are fixed and the rest filtered for.
+addopts = "-ra --tb=short --strict-markers -p no:celery -p no:warnings"
+# TODO: --import-mode=importlib will become the default soon,
+# currently we have a few relative imports that don't work with that.
+markers = [
+    "snuba: mark a test as requiring snuba",
+]
+selenium_driver = "chrome"
+# This is for people who install pytest-xdist locally,
+# and use the -f/--looponfail feature.
+looponfailroots = ["src", "tests"]

requirements-dev.txt

@@ -3,10 +3,10 @@ exam>=0.5.1
 freezegun==1.1.0
 honcho>=1.0.0,<1.1.0
 openapi-core @ https://github.com/getsentry/openapi-core/archive/master.zip#egg=openapi-core
-pytest==4.6.5
-pytest-cov==2.5.1
-pytest-django==3.5.1
+pytest==6.1.0
+pytest-cov==2.11.1
+pytest-django==3.10.0
 pytest-sentry==0.1.6
-pytest-rerunfailures==8.0
+pytest-rerunfailures==9.1.1
 responses>=0.8.1,<0.9.0
 sqlparse==0.2.4

setup.cfg

@@ -1,15 +1,3 @@
-[tool:pytest]
-python_files = test*.py
-# Note: When updating pytest tweaks, make sure to update getsentry
-# When updating the traceback format, make sure to update .github/pytest.json
-addopts = --tb=short -p no:doctest -p no:warnings
-norecursedirs = bin dist docs htmlcov script hooks node_modules .* {args}
-looponfailroots = src tests
-selenium_driver = chrome
-self-contained-html = true
-markers =
-    snuba: mark a test as requiring snuba
-
 [flake8]
 # File filtering is taken care of in pre-commit.
 # E203 false positive, see https://github.com/PyCQA/pycodestyle/issues/373

src/sentry/utils/pytest/sentry.py

@@ -78,9 +78,8 @@ def pytest_configure(config):
     settings.SENTRY_TSDB = "sentry.tsdb.inmemory.InMemoryTSDB"
     settings.SENTRY_TSDB_OPTIONS = {}
 
-    if settings.SENTRY_NEWSLETTER == "sentry.newsletter.base.Newsletter":
-        settings.SENTRY_NEWSLETTER = "sentry.newsletter.dummy.DummyNewsletter"
-        settings.SENTRY_NEWSLETTER_OPTIONS = {}
+    settings.SENTRY_NEWSLETTER = "sentry.newsletter.dummy.DummyNewsletter"
+    settings.SENTRY_NEWSLETTER_OPTIONS = {}
 
     settings.BROKER_BACKEND = "memory"
     settings.BROKER_URL = "memory://"

tests/sentry/api/endpoints/test_auth_config.py

@@ -49,8 +49,9 @@ class AuthConfigEndpointTest(APITestCase):
         assert not response.data["hasNewsletter"]
         assert response.data["serverHostname"] == "testserver"
 
-    @pytest.mark.skipIf(
-        lambda x: settings.SENTRY_NEWSLETTER != "sentry.newsletter.dummy.DummyNewsletter"
+    @pytest.mark.skipif(
+        settings.SENTRY_NEWSLETTER != "sentry.newsletter.dummy.DummyNewsletter",
+        reason="Requires DummyNewsletter.",
     )
     def test_has_newsletter(self):
         newsletter.backend.enable()

tests/sentry/api/endpoints/test_user_subscriptions.py

@@ -8,8 +8,9 @@ from sentry.models import UserEmail
 from sentry.testutils import APITestCase
 
 
-@pytest.mark.skipIf(
-    lambda x: settings.SENTRY_NEWSLETTER != "sentry.newsletter.dummy.DummyNewsletter"
+@pytest.mark.skipif(
+    settings.SENTRY_NEWSLETTER != "sentry.newsletter.dummy.DummyNewsletter",
+    reason="Requires DummyNewsletter.",
 )
 class UserSubscriptionsNewsletterTest(APITestCase):
     def setUp(self):

tests/sentry/api/test_event_search.py

@@ -2010,13 +2010,15 @@ class GetSnubaQueryArgsTest(TestCase):
         assert _filter.filter_keys == {"project_id": [p1.id, p2.id]}
         assert _filter.project_ids == [p1.id, p2.id]
 
-        with pytest.raises(InvalidSearchQuery) as err:
+        with pytest.raises(InvalidSearchQuery) as exc_info:
             params = {"project_id": []}
             get_filter(f"project.name:{p1.slug}", params)
+
+        exc = exc_info.value
+        exc_str = f"{exc}"
         assert (
-            "Invalid query. Project %s does not exist or is not an actively selected project"
-            % p1.slug
-            in str(err)
+            f"Invalid query. Project {p1.slug} does not exist or is not an actively selected project"
+            in exc_str
         )
 
     def test_not_has_project(self):
@@ -2034,16 +2036,20 @@ class GetSnubaQueryArgsTest(TestCase):
             assert result.conditions == [["transaction.status", "=", key]]
 
     def test_transaction_status_no_wildcard(self):
-        with pytest.raises(InvalidSearchQuery) as err:
+        with pytest.raises(InvalidSearchQuery) as exc_info:
             get_filter("transaction.status:o*")
-        assert "Invalid value" in str(err)
-        assert "cancelled," in str(err)
+        exc = exc_info.value
+        exc_str = f"{exc}"
+        assert "Invalid value" in exc_str
+        assert "cancelled," in exc_str
 
     def test_transaction_status_invalid(self):
-        with pytest.raises(InvalidSearchQuery) as err:
+        with pytest.raises(InvalidSearchQuery) as exc_info:
             get_filter("transaction.status:lol")
-        assert "Invalid value" in str(err)
-        assert "cancelled," in str(err)
+        exc = exc_info.value
+        exc_str = f"{exc}"
+        assert "Invalid value" in exc_str
+        assert "cancelled," in exc_str
 
     def test_error_handled(self):
         result = get_filter("error.handled:true")
@@ -2395,24 +2401,24 @@ class ResolveFieldListTest(unittest.TestCase):
         assert "MAX(user) is not a valid function" in str(err)
 
     def test_aggregate_function_invalid_column(self):
-        with pytest.raises(InvalidSearchQuery) as err:
+        with pytest.raises(InvalidSearchQuery) as exc_info:
             fields = ["min(message)"]
             resolve_field_list(fields, eventstore.Filter())
-        assert (
-            "InvalidSearchQuery: min(message): column argument invalid: message is not a numeric column"
-            in str(err)
-        )
+
+        exc = exc_info.value
+        exc_str = f"{exc}"
+        assert "min(message): column argument invalid: message is not a numeric column" == exc_str
 
     def test_aggregate_function_missing_parameter(self):
-        with pytest.raises(InvalidSearchQuery) as err:
+        with pytest.raises(InvalidSearchQuery) as exc_info:
             fields = ["count_unique()"]
             resolve_field_list(fields, eventstore.Filter())
-        assert (
-            "InvalidSearchQuery: count_unique(): column argument invalid: a column is required"
-            in str(err)
-        )
 
-        with pytest.raises(InvalidSearchQuery) as err:
+        exc = exc_info.value
+        exc_str = f"{exc}"
+        assert "count_unique(): column argument invalid: a column is required" == exc_str
+
+        with pytest.raises(InvalidSearchQuery):
             fields = ["count_unique(  )"]
             resolve_field_list(fields, eventstore.Filter())
 

tests/sentry/tasks/test_enqueue_scheduled_jobs.py

@@ -58,9 +58,7 @@ class EnqueueScheduledJobsTest(TestCase):
         ) == {"sentry.tasks.enqueue_scheduled_jobs", "sentry.tasks.enqueue_scheduled_jobs_followup"}
 
     def test_schedule_job_order(self):
-        with pytest.raises(
-            ValidationError, message="ValidationError raised. Check order of inputs"
-        ):
+        with pytest.raises(ValidationError):
             job = [
                 (
                     "sentry.tasks.enqueue_scheduled_jobs",
@@ -69,3 +67,4 @@ class EnqueueScheduledJobsTest(TestCase):
                 )
             ]
             schedule_jobs(job)
+            pytest.fail("ValidationError raised. Check order of inputs.")