Revert "Revert "py3(django): pytest startup and test collection on Django 1.10 (#15627)" (#15660)" (#15677)

This reverts commit 98b5bedc880dcbfe34282a2609334c5fe451b6fc.

.travis.yml

@@ -158,6 +158,21 @@ matrix:
       name: 'Acceptance'
       env: TEST_SUITE=acceptance USE_SNUBA=1
 
+    # allowed to fail
+    - <<: *postgres_default
+      name: '[Django 1.10] Backend [Postgres] (1/2)'
+      env: DJANGO_VERSION=">=1.10,<1.11" TEST_SUITE=postgres DB=postgres TOTAL_TEST_GROUPS=2 TEST_GROUP=0
+
+    # allowed to fail
+    - <<: *postgres_default
+      name: '[Django 1.10] Backend [Postgres] (2/2)'
+      env: DJANGO_VERSION=">=1.10,<1.11" TEST_SUITE=postgres DB=postgres TOTAL_TEST_GROUPS=2 TEST_GROUP=1
+
+    # allowed to fail
+    - <<: *acceptance_default
+      name: '[Django 1.10] Acceptance'
+      env: DJANGO_VERSION=">=1.10,<1.11" TEST_SUITE=acceptance USE_SNUBA=1
+
     # allowed to fail
     - <<: *acceptance_default
       name: 'Plugins'
@@ -224,6 +239,29 @@ matrix:
       before_script:
         - psql -c 'create database sentry;' -U postgres
 
+    # allowed to fail
+    - python: 2.7
+      name: '[Django 1.10] Snuba Integration'
+      env: DJANGO_VERSION=">=1.10,<1.11" TEST_SUITE=snuba USE_SNUBA=1 SENTRY_ZOOKEEPER_HOSTS=localhost:2181 SENTRY_KAFKA_HOSTS=localhost:9092
+      services:
+        - docker
+        - memcached
+        - redis-server
+        - postgresql
+      before_install:
+        - *pip_install
+        - docker run -d --network host --name zookeeper -e ZOOKEEPER_CLIENT_PORT=2181 confluentinc/cp-zookeeper:4.1.0
+        - docker run -d --network host --name kafka -e KAFKA_ZOOKEEPER_CONNECT=localhost:2181 -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 -e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 confluentinc/cp-kafka:4.1.0
+        - docker run -d --network host --name clickhouse-server --ulimit nofile=262144:262144 yandex/clickhouse-server:19.11
+        - docker run -d --network host --name snuba --env SNUBA_SETTINGS=test --env CLICKHOUSE_SERVER=localhost:9000 getsentry/snuba
+        - docker ps -a
+      install:
+        - python setup.py install_egg_info
+        - pip install -U -e ".[dev]"
+        - pip install confluent-kafka
+      before_script:
+        - psql -c 'create database sentry;' -U postgres
+
     # Deploy 'storybook' (component & style guide) - allowed to fail
     - name: 'Storybook Deploy'
       language: node_js
@@ -246,6 +284,10 @@ matrix:
   allow_failures:
     - name: 'Storybook Deploy'
     - name: 'Plugins'
+    - name: '[Django 1.10] Backend [Postgres] (1/2)'
+    - name: '[Django 1.10] Backend [Postgres] (2/2)'
+    - name: '[Django 1.10] Acceptance'
+    - name: '[Django 1.10] Snuba Integration'
 
 notifications:
   webhooks:

src/sentry/conf/server.py

@@ -68,7 +68,6 @@ ENVIRONMENT = os.environ.get("SENTRY_ENVIRONMENT", "production")
 IS_DEV = ENVIRONMENT == "development"
 
 DEBUG = IS_DEV
-TEMPLATE_DEBUG = True
 MAINTENANCE = False
 
 ADMINS = ()
@@ -250,12 +249,6 @@ USE_L10N = True
 
 USE_TZ = True
 
-# List of callables that know how to import templates from various sources.
-TEMPLATE_LOADERS = (
-    "django.template.loaders.filesystem.Loader",
-    "django.template.loaders.app_directories.Loader",
-)
-
 MIDDLEWARE_CLASSES = (
     "sentry.middleware.proxy.ChunkedMiddleware",
     "sentry.middleware.proxy.DecompressBodyMiddleware",
@@ -283,23 +276,25 @@ MIDDLEWARE_CLASSES = (
 
 ROOT_URLCONF = "sentry.conf.urls"
 
-TEMPLATE_DIRS = (
-    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
-    # Always use forward slashes, even on Windows.
-    # Don't forget to use absolute paths, not relative paths.
-    os.path.join(PROJECT_ROOT, "templates"),
-)
-
-TEMPLATE_CONTEXT_PROCESSORS = (
-    "django.contrib.auth.context_processors.auth",
-    "django.contrib.messages.context_processors.messages",
-    "django.core.context_processors.csrf",
-    "django.core.context_processors.request",
-    "social_auth.context_processors.social_auth_by_name_backends",
-    "social_auth.context_processors.social_auth_backends",
-    "social_auth.context_processors.social_auth_by_type_backends",
-    "social_auth.context_processors.social_auth_login_redirect",
-)
+TEMPLATES = [
+    {
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
+        "DIRS": [os.path.join(PROJECT_ROOT, "templates")],
+        "APP_DIRS": True,
+        "OPTIONS": {
+            "context_processors": [
+                "django.contrib.auth.context_processors.auth",
+                "django.contrib.messages.context_processors.messages",
+                "django.template.context_processors.csrf",
+                "django.template.context_processors.request",
+                "social_auth.context_processors.social_auth_by_name_backends",
+                "social_auth.context_processors.social_auth_backends",
+                "social_auth.context_processors.social_auth_by_type_backends",
+                "social_auth.context_processors.social_auth_login_redirect",
+            ]
+        },
+    }
+]
 
 INSTALLED_APPS = (
     "django.contrib.admin",

src/sentry/new_migrations/monkey/__init__.py

@@ -7,7 +7,7 @@ from sentry.new_migrations.monkey.writer import SENTRY_MIGRATION_TEMPLATE
 
 from django.db.migrations import migration, executor, writer
 
-LAST_VERIFIED_DJANGO_VERSION = (1, 9)
+LAST_VERIFIED_DJANGO_VERSION = (1, 10)
 CHECK_MESSAGE = """Looks like you're trying to upgrade Django! Since we monkeypatch
 the Django migration library in several places, please verify that we have the latest
 code, and that the monkeypatching still works as expected. Currently the main things

src/sentry/runner/importer.py

@@ -132,9 +132,7 @@ def add_settings(mod, settings):
             continue
 
         setting_value = getattr(mod, setting)
-        if setting in ("INSTALLED_APPS", "TEMPLATE_DIRS") and isinstance(
-            setting_value, six.string_types
-        ):
+        if setting in ("INSTALLED_APPS",) and isinstance(setting_value, six.string_types):
             setting_value = (setting_value,)  # In case the user forgot the comma.
 
         # Any setting that starts with EXTRA_ and matches a setting that is a list or tuple

src/sentry/tasks/merge.py

@@ -223,12 +223,16 @@ def _get_event_environment(event, project, cache):
 def merge_objects(models, group, new_group, limit=1000, logger=None, transaction_id=None):
     has_more = False
     for model in models:
-        all_fields = model._meta.get_all_field_names()
+        all_fields = [f.name for f in model._meta.get_fields()]
 
-        # not all models have a 'project' or 'project_id' field, but we make a best effort
-        # to filter on one if it is available
-        # HACK(mattrobenolt): Our Event table can't handle the extra project_id bit on the query
+        # Not all models have a 'project' or 'project_id' field, but we make a best effort
+        # to filter on one if it is available.
+        # Also note that all_fields doesn't contain f.attname
+        # (django ForeignKeys have only attribute "attname" where "_id" is implicitly appended)
+        # but we still want to check for "project_id" because some models define a project_id bigint.
         has_project = "project_id" in all_fields or "project" in all_fields
+
+        # HACK(mattrobenolt): Our Event table can't handle the extra project_id bit on the query
         if has_project and model.__name__ != "Event":
             project_qs = model.objects.filter(project_id=group.project_id)
         else:

src/sentry/utils/pytest/sentry.py

@@ -45,8 +45,6 @@ def pytest_configure(config):
         else:
             raise RuntimeError("oops, wrong database: %r" % test_db)
 
-    settings.TEMPLATE_DEBUG = True
-
     # Disable static compiling in tests
     settings.STATIC_BUNDLES = {}
 

src/sentry/web/frontend/accounts.py

@@ -8,7 +8,7 @@ import six
 from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth import login as login_user, authenticate
-from django.core.context_processors import csrf
+from django.template.context_processors import csrf
 from django.core.urlresolvers import reverse
 from django.db import transaction
 from django.http import HttpResponseRedirect, Http404, HttpResponse

src/sentry/web/frontend/base.py

@@ -3,7 +3,7 @@ from __future__ import absolute_import
 import logging
 import six
 
-from django.core.context_processors import csrf
+from django.template.context_processors import csrf
 from django.core.urlresolvers import reverse
 from django.http import (
     HttpResponse,

src/social_auth/admin.py

@@ -3,6 +3,7 @@ from __future__ import absolute_import
 from django.contrib import admin
 
 from social_auth.models import UserSocialAuth
+from social_auth.django_compat import get_all_field_names
 
 _User = UserSocialAuth.user_model()
 
@@ -14,7 +15,7 @@ else:
     username_field = None
 
 fieldnames = ("first_name", "last_name", "email") + (username_field,)
-all_names = _User._meta.get_all_field_names()
+all_names = get_all_field_names(_User)
 user_search_fields = ["user__" + name for name in fieldnames if name in all_names]
 
 

src/social_auth/backends/pipeline/user.py

@@ -6,6 +6,7 @@ from uuid import uuid4
 
 from social_auth.utils import setting, module_member
 from social_auth.models import UserSocialAuth
+from social_auth.django_compat import get_all_field_names
 
 
 slugify = module_member(
@@ -87,7 +88,7 @@ def django_orm_maxlength_truncate(backend, details, user=None, is_new=False, *ar
     if user is None:
         return
     out = {}
-    names = user._meta.get_all_field_names()
+    names = get_all_field_names(user)
     for name, value in six.iteritems(details):
         if name in names and not _ignore_field(name, is_new):
             max_length = user._meta.get_field(name).max_length