Add safe_urlopen helper

- Disables redirects (by default)
- Disables requests to internal IP ranges
- Default Sentry user-agent

setup.py

@@ -84,7 +84,7 @@ install_requires = [
     'email-reply-parser>=0.2.0,<0.3.0',
     'enum34>=0.9.18,<0.10.0',
     'gunicorn>=0.17.2,<0.18.0',
-    'ua-parser>=0.3.5',
+    'ipaddr>=2.1.11,<2.2.0',
     'logan>=0.5.8.2,<0.6.0',
     'nydus>=0.10.7,<0.11.0',
     'Pygments>=1.6.0,<1.7.0',
@@ -97,6 +97,7 @@ install_requires = [
     'setproctitle>=1.1.7,<1.2.0',
     'South==0.8.2',
     'toronado>=0.0.4,<0.1.0',
+    'ua-parser>=0.3.5',
     'urllib3>=1.7.1,<1.8.0',
 ]
 

src/sentry/conf/server.py

@@ -674,6 +674,12 @@ SENTRY_MAX_STACKTRACE_FRAMES = 25
 # Gravatar service base url
 SENTRY_GRAVATAR_BASE_URL = 'https://secure.gravatar.com'
 
+SENTRY_DISALLOWED_IPS = (
+    '10.0.0.0/8',
+    '172.16.0.0/12',
+    '192.168.0.0/16',
+)
+
 # Configure celery
 import djcelery
 djcelery.setup_loader()

src/sentry/http.py

@@ -0,0 +1,61 @@
+"""
+sentry.utils.http
+~~~~~~~~~~~~~~~~~
+
+:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
+:license: BSD, see LICENSE for more details.
+"""
+
+import sentry
+import socket
+import urllib2
+
+from django.conf import settings
+from django.core.exceptions import SuspiciousOperation
+from ipaddr import IPNetwork
+from urlparse import urlparse
+
+
+DISALLOWED_IPS = map(IPNetwork, settings.SENTRY_DISALLOWED_IPS)
+
+
+class NoRedirectionHandler(urllib2.HTTPErrorProcessor):
+    def http_response(self, request, response):
+        return response
+
+    https_response = http_response
+
+
+def is_valid_url(url):
+    """
+    Tests a URL to ensure it doesn't appear to be a blacklisted IP range.
+    """
+    parsed = urlparse(url)
+    ip_network = IPNetwork(socket.gethostbyname(parsed.hostname))
+    for addr in DISALLOWED_IPS:
+        if ip_network in addr:
+            return False
+    return True
+
+
+def safe_urlopen(url, data, headers=(), user_agent='sentry/%s' % sentry.VERSION,
+                 allow_redirects=False, timeout=30):
+    """
+    A slightly safer version of ``urlib2.urlopen`` which prevents redirection
+    and ensures the URL isn't attempting to hit a blacklisted IP range.
+    """
+    if not is_valid_url(url):
+        raise SuspiciousOperation('%s matches the URL blacklist' % (url,))
+
+    req = urllib2.Request(url, data)
+    req.add_header('User-Agent', user_agent)
+    for key, value in headers:
+        req.add_header(key, value)
+
+    handlers = []
+    if not allow_redirects:
+        handlers.append(NoRedirectionHandler)
+
+    opener = urllib2.build_opener(*handlers)
+
+    return opener.open(req, timeout=timeout)

src/sentry/tasks/fetch_source.py

@@ -10,7 +10,6 @@ import itertools
 import logging
 import hashlib
 import re
-import urllib2
 import zlib
 import base64
 from os.path import splitext
@@ -20,6 +19,7 @@ from urlparse import urljoin, urlsplit
 
 from sentry.app import cache
 from sentry.constants import SOURCE_FETCH_TIMEOUT, MAX_CULPRIT_LENGTH
+from sentry.http import safe_urlopen
 from sentry.utils.sourcemaps import sourcemap_to_index, find_source
 from sentry.utils.strings import truncatechars
 
@@ -122,12 +122,10 @@ def fetch_url_content(url):
     import sentry
 
     try:
-        opener = urllib2.build_opener()
-        opener.addheaders = [
+        req = safe_urlopen(url, headers=[
             ('Accept-Encoding', 'gzip'),
             ('User-Agent', 'Sentry/%s' % sentry.VERSION),
-        ]
-        req = opener.open(url, timeout=SOURCE_FETCH_TIMEOUT)
+        ], allow_redirects=True, timeout=SOURCE_FETCH_TIMEOUT)
         headers = dict(req.headers)
         body = req.read()
         if headers.get('content-encoding') == 'gzip':

src/sentry/utils/http.py

@@ -5,12 +5,12 @@ sentry.utils.http
 :copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
 :license: BSD, see LICENSE for more details.
 """
+
+import six
 import urllib
-from urlparse import urlparse, urljoin
 
 from django.conf import settings
-
-import six
+from urlparse import urlparse, urljoin
 
 
 def absolute_uri(url=None):