geo: Switch to geoip2 mmdb format

requirements-optional.txt

@@ -1,5 +1,5 @@
-GeoIP==1.3.2
 google-cloud-bigtable>=0.32.1,<0.33.0
 google-cloud-pubsub>=0.35.4,<0.36.0
 google-cloud-storage>=1.13.2,<1.14
+maxminddb==1.4.1
 python3-saml>=1.4.0,<1.5

src/sentry/conf/server.py

@@ -1612,9 +1612,6 @@ SENTRY_RELAY_OPEN_REGISTRATION = False
 
 # GeoIP
 # Used for looking up IP addresses.
-# For example /usr/local/share/GeoIP/GeoIPCity.dat
-GEOIP_PATH = None
-# Same file but in the newer format. Both are required.
 # For example /usr/local/share/GeoIP/GeoIPCity.mmdb
 GEOIP_PATH_MMDB = None
 

src/sentry/utils/geo.py

@@ -1,11 +1,13 @@
 from __future__ import absolute_import
 
 import logging
+import six
 
 from django.conf import settings
 
 
 logger = logging.getLogger(__name__)
+geoip_path_mmdb = getattr(settings, "GEOIP_PATH_MMDB", None)
 
 
 # default is no-op
@@ -19,33 +21,40 @@ rust_geoip = None
 def _init_geoip():
     global geo_by_addr
     try:
-        import GeoIP
+        import maxminddb
     except ImportError:
-        logger.warning("GeoIP module not available.")
-        return
-
-    geoip_path = getattr(settings, "GEOIP_PATH", None)
-    if not geoip_path:
-        logger.warning("settings.GEOIP_PATH not configured.")
+        logger.warning("maxminddb module not available.")
         return
 
     try:
-        geo_db = GeoIP.open(geoip_path, GeoIP.GEOIP_MEMORY_CACHE)
+        geo_db = maxminddb.open_database(geoip_path_mmdb, maxminddb.MODE_AUTO)
     except Exception:
-        logger.warning("Error opening GeoIP database: %s" % geoip_path)
+        logger.warning("Error opening GeoIP database: %s" % geoip_path_mmdb)
         return
 
-    geo_by_addr = geo_db.record_by_addr
+    def encode_bytes(data):
+        if isinstance(data, six.text_type):
+            return data.encode("ISO-8859-1")
+        return data
 
+    def _geo_by_addr(ip):
+        geo = geo_db.get(ip)
+        if not geo:
+            return
 
-def _init_geoip_rust():
-    global rust_geoip
+        return {
+            "country_code": encode_bytes(geo["country"]["iso_code"]),
+            "region": encode_bytes(geo.get("subdivisions", [{}])[-1].get("iso_code")),
+            "city": encode_bytes(geo.get("city", {}).get("names", {}).get("en")),
+            "latitude": geo["location"]["latitude"],
+            "longitude": geo["location"]["longitude"],
+        }
 
-    geoip_path_mmdb = getattr(settings, "GEOIP_PATH_MMDB", None)
+    geo_by_addr = _geo_by_addr
 
-    if not geoip_path_mmdb:
-        logger.warning("No GeoIP MMDB database configured")
-        return
+
+def _init_geoip_rust():
+    global rust_geoip
 
     from semaphore.processing import GeoIpLookup
 
@@ -55,5 +64,8 @@ def _init_geoip_rust():
         logger.warning("Error opening GeoIP database in Rust: %s" % geoip_path_mmdb)
 
 
-_init_geoip()
-_init_geoip_rust()
+if geoip_path_mmdb:
+    _init_geoip()
+    _init_geoip_rust()
+else:
+    logger.warning("settings.GEOIP_PATH_MMDB not configured.")