test: enable pytest fail on warnings (#24359)

pyproject.toml

@@ -10,15 +10,38 @@ target-version=['py36']
 [tool.pytest.ini_options]
 # note: When updating the traceback format, make sure to update .github/pytest.json
 # We don't use the celery pytest plugin.
-# TODO: Warnings are disabled for now (they always have been)
-#       until the good ones are fixed and the rest filtered for.
-addopts = "-ra --tb=short --strict-markers -p no:celery -p no:warnings"
+addopts = "-ra --tb=short --strict-markers -p no:celery"
 # TODO: --import-mode=importlib will become the default soon,
 # currently we have a few relative imports that don't work with that.
 markers = [
     "snuba: mark a test as requiring snuba",
 ]
 selenium_driver = "chrome"
+filterwarnings = [
+    # Consider all warnings to be errors other than the ignored ones.
+    "error",
+    # This is just to prevent pytest from exiting if pytest-xdist isn't installed.
+    "ignore:Unknown config option.*looponfailroots:pytest.PytestConfigWarning",
+
+    # The following warning filters should be kept in sync with
+    # sentry.utils.pytest.sentry, and sentry.runner.settings.
+    # TODO(joshuarli): Address these as a prerequisite to testing on Django 2.1.
+    "ignore::django.utils.deprecation.RemovedInDjango20Warning",
+    "ignore::django.utils.deprecation.RemovedInDjango21Warning",
+    # DeprecationWarnings from Python 3.6's sre_parse are just so painful,
+    # and I haven't found a way to ignore it specifically from a module.
+    # This one in particular is from the "cookies" packages as depended
+    # on by an outdated version of responses, and shows up all over tests.
+    # TODO(joshuarli): Upgrade responses, then revisit this.
+    #                  It'll probably show up in other dependencies.
+    "ignore::DeprecationWarning",
+
+    # The following warning filters are for pytest only.
+    # This is so we don't have to wrap most datetime objects in testing code
+    # with django.utils.timezone.
+    "ignore:DateTimeField.*naive datetime:RuntimeWarning",
+    "ignore:.*sentry.digests.backends.dummy.DummyBackend.*:sentry.utils.warnings.UnsupportedBackend",
+]
 # This is for people who install pytest-xdist locally,
 # and use the -f/--looponfail feature.
 looponfailroots = ["src", "tests"]

src/sentry/receivers/onboarding.py

@@ -64,7 +64,7 @@ def record_new_project(project, user, **kwargs):
                 Organization.objects.get(id=project.organization_id).get_default_owner().id
             )
         except IndexError:
-            logging.getLogger("sentry").warn(
+            logging.getLogger("sentry").warning(
                 "Cannot initiate onboarding for organization (%s) due to missing owners",
                 project.organization_id,
             )

src/sentry/runner/settings.py

@@ -87,15 +87,19 @@ def configure(ctx, py, yaml, skip_service_validation=False):
     if __installed:
         return
 
-    # Make sure that our warnings are always displayed
     import warnings
 
+    # Make sure that our warnings are always displayed.
     warnings.filterwarnings("default", "", Warning, r"^sentry")
 
-    # for now, squelch Django 2 warnings so prod logs aren't clogged
-    from django.utils.deprecation import RemovedInDjango20Warning
+    # This should be kept in-sync with sentry.utils.pytest.sentry,
+    # and pytest warningfilters in pyproject.toml.
+    # See pyproject.toml for explanations.
+    from django.utils.deprecation import RemovedInDjango20Warning, RemovedInDjango21Warning
 
     warnings.filterwarnings(action="ignore", category=RemovedInDjango20Warning)
+    warnings.filterwarnings(action="ignore", category=RemovedInDjango21Warning)
+    warnings.filterwarnings(action="ignore", category=DeprecationWarning)
 
     # Add in additional mimetypes that are useful for our static files
     # which aren't common in default system registries

src/sentry/utils/pytest/fixtures.py

@@ -266,7 +266,7 @@ elif _snapshot_writeback != "new":
 _test_base = os.path.realpath(
     os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sentry.__file__))))
 )
-_yaml_snap_re = re.compile(r"^---\r?\n(.*?)\r?\n---\r?\n(.*)$(?s)")
+_yaml_snap_re = re.compile(r"^---\r?\n(.*?)\r?\n---\r?\n(.*)$", re.DOTALL)
 
 
 @pytest.fixture

src/sentry/utils/pytest/sentry.py

@@ -1,10 +1,12 @@
-from sentry.utils.compat import mock
 import os
 from hashlib import md5
 
 from django.conf import settings
 from sentry_sdk import Hub
 
+from sentry.utils.compat import mock
+from sentry.utils.warnings import UnsupportedBackend
+
 
 TEST_ROOT = os.path.normpath(
     os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir, "tests")
@@ -12,6 +14,23 @@ TEST_ROOT = os.path.normpath(
 
 
 def pytest_configure(config):
+    import warnings
+    from django.utils.deprecation import RemovedInDjango20Warning, RemovedInDjango21Warning
+
+    # These warnings should be kept in sync with sentry.runner.settings,
+    # and pytest warningfilters in pyproject.toml.
+    # See pyproject.toml for explanations.
+    warnings.filterwarnings(action="ignore", category=RemovedInDjango20Warning)
+    warnings.filterwarnings(action="ignore", category=RemovedInDjango21Warning)
+    warnings.filterwarnings(action="ignore", category=DeprecationWarning)
+
+    # These warnings are for pytest only.
+    warnings.filterwarnings(
+        action="ignore",
+        message=r".*sentry.digests.backends.dummy.DummyBackend.*",
+        category=UnsupportedBackend,
+    )
+
     # HACK: Only needed for testing!
     os.environ.setdefault("_SENTRY_SKIP_CONFIGURATION", "1")