fix: Don't allow robots to index/follow org login redirects (#12540)

We can't exclude `/` in our robots.txt file as it covers too much surface
area. By using the `X-Robots-Tag` header we can tell crawlers to not
index/follow the redirect which should help keep them out of
organization login pages.

See https://developers.google.com/search/reference/robots_meta_tag#using-the-x-robots-tag-http-header
for reference on the x-robots-tag header.

Refs SEN-343

src/sentry/web/frontend/base.py

@@ -248,7 +248,7 @@ class BaseView(View, OrganizationMixin):
             redirect_to = reverse('sentry-auth-organization', args=[kwargs['organization_slug']])
         else:
             redirect_to = auth.get_login_url()
-        return self.redirect(redirect_to)
+        return self.redirect(redirect_to, headers={'X-Robots-Tag': 'noindex, nofollow'})
 
     def is_sudo_required(self, request, *args, **kwargs):
         return self.sudo_required and not request.is_sudo()
@@ -284,8 +284,12 @@ class BaseView(View, OrganizationMixin):
 
         return render_to_response(template, default_context, self.request, status=status)
 
-    def redirect(self, url):
-        return HttpResponseRedirect(url)
+    def redirect(self, url, headers=None):
+        res = HttpResponseRedirect(url)
+        if headers:
+            for k, v in headers.items():
+                res[k] = v
+        return res
 
     def get_team_list(self, user, organization):
         return Team.objects.get_for_user(

tests/sentry/web/frontend/test_react_page.py

@@ -6,6 +6,19 @@ from sentry.testutils import TestCase
 
 
 class ReactPageViewTest(TestCase):
+    def test_redirects_unauthenticated_request(self):
+        owner = self.create_user('bar@example.com')
+        org = self.create_organization(owner=owner)
+
+        path = reverse('sentry-organization-home', args=[org.slug])
+        resp = self.client.get(path)
+
+        assert resp.status_code == 302
+        assert resp['Location'] == u'http://testserver{}'.format(
+            reverse('sentry-auth-organization', args=[org.slug]),
+        )
+        assert resp['X-Robots-Tag'] == 'noindex, nofollow'
+
     def test_superuser_can_load(self):
         org = self.create_organization(owner=self.user)
         path = reverse('sentry-organization-home', args=[org.slug])