build: update redis and remove redis-py-cluster (#64711)

Updates redis to v4.1.0 since it is the minimum Redis version that
supports/includes `RedisCluster`

fixtures/stubs-for-mypy/rediscluster/__init__.pyi

@@ -1,3 +0,0 @@
-from .client import RedisCluster
-
-__all__ = ('RedisCluster',)

fixtures/stubs-for-mypy/rediscluster/client.pyi

@@ -1,8 +0,0 @@
-from typing import TypeVar
-
-from redis import Redis
-
-T = TypeVar("T", str, bytes)
-
-class RedisCluster(Redis[T]):
-    ...

fixtures/stubs-for-mypy/rediscluster/exceptions.pyi

@@ -1,3 +0,0 @@
-from redis.exceptions import RedisError
-
-class ClusterError(RedisError): ...

requirements-base.txt

@@ -52,8 +52,7 @@ fido2>=0.9.2
 python3-saml>=1.15.0
 PyYAML>=6.0.1
 rb>=1.9.0
-redis-py-cluster>=2.1.0
-redis>=3.4.1
+redis>=4.1.0
 requests-oauthlib>=1.2.0
 requests>=2.25.1
 # [start] jsonschema format validators
@@ -63,7 +62,7 @@ rfc3986-validator>=0.1.1
 sentry-arroyo>=2.16.0
 sentry-kafka-schemas>=0.1.50
 sentry-ophio==0.1.5
-sentry-redis-tools>=0.1.7
+sentry-redis-tools>=0.2.0
 sentry-relay>=0.8.45
 sentry-sdk>=1.39.2
 snuba-sdk>=2.0.25

requirements-dev-frozen.txt

@@ -10,6 +10,7 @@ amqp==5.2.0
 anyio==3.7.1
 asgiref==3.7.2
 async-generator==1.10
+async-timeout==4.0.3
 attrs==23.1.0
 avalara==20.9.0
 beautifulsoup4==4.7.1
@@ -158,8 +159,7 @@ pyuwsgi==2.0.23
 pyvat==1.3.15
 pyyaml==6.0.1
 rb==1.10.0
-redis==3.4.1
-redis-py-cluster==2.1.0
+redis==4.5.4
 referencing==0.30.2
 regex==2022.9.13
 reportlab==4.0.7
@@ -179,7 +179,7 @@ sentry-forked-django-stubs==4.2.7.post3
 sentry-forked-djangorestframework-stubs==3.14.5.post1
 sentry-kafka-schemas==0.1.50
 sentry-ophio==0.1.5
-sentry-redis-tools==0.1.7
+sentry-redis-tools==0.2.0
 sentry-relay==0.8.45
 sentry-sdk==1.39.2
 sentry-usage-accountant==0.0.10
@@ -211,10 +211,11 @@ types-pillow==9.5.0.4
 types-protobuf==4.23.0.1
 types-psutil==5.9.5.16
 types-psycopg2==2.9.21
+types-pyopenssl==23.2.0.2
 types-python-dateutil==2.8.19
 types-pytz==2022.1.2
 types-pyyaml==6.0.11
-types-redis==3.5.18
+types-redis==4.6.0.3
 types-requests==2.31.0.20231231
 types-setuptools==65.3.0
 types-simplejson==3.17.7.2

requirements-dev.txt

@@ -53,7 +53,7 @@ types-python-dateutil
 types-pytz
 types-pyyaml
 # make sure to match close-enough to redis==
-types-redis<4
+types-redis>=4.1
 types-requests>=2.31.0.20231231
 types-setuptools
 types-simplejson>=3.17.7.2

requirements-frozen.txt

@@ -9,6 +9,7 @@
 amqp==5.2.0
 anyio==3.7.1
 asgiref==3.7.2
+async-timeout==4.0.3
 attrs==23.1.0
 avalara==20.9.0
 beautifulsoup4==4.7.1
@@ -105,8 +106,7 @@ pyuwsgi==2.0.23
 pyvat==1.3.15
 pyyaml==6.0.1
 rb==1.10.0
-redis==3.4.1
-redis-py-cluster==2.1.0
+redis==4.5.4
 referencing==0.30.2
 regex==2022.9.13
 reportlab==4.0.7
@@ -120,7 +120,7 @@ s3transfer==0.10.0
 sentry-arroyo==2.16.0
 sentry-kafka-schemas==0.1.50
 sentry-ophio==0.1.5
-sentry-redis-tools==0.1.7
+sentry-redis-tools==0.2.0
 sentry-relay==0.8.45
 sentry-sdk==1.39.2
 sentry-usage-accountant==0.0.10

src/sentry/ingest/transaction_clusterer/rules.py

@@ -60,10 +60,11 @@ class RedisRuleStore:
         key = self._get_rules_key(project)
 
         with client.pipeline() as p:
-            # to be consistent with other stores, clear previous hash entries:
-            p.delete(key)
             if len(rules) > 0:
-                p.hmset(key, rules)
+                p.hset(key, mapping=rules)
+            else:
+                # to be consistent with other stores, clear previous hash entries:
+                p.delete(key)
             p.execute()
 
     def update_rule(self, project: Project, rule: str, last_used: int) -> None:

src/sentry/processing/backpressure/memory.py

@@ -4,7 +4,7 @@ from typing import Any, Union
 
 import rb
 import requests
-from rediscluster import RedisCluster
+from redis.cluster import RedisCluster
 
 
 @dataclass

src/sentry/ratelimits/concurrent.py

@@ -8,6 +8,7 @@ from django.conf import settings
 
 from sentry.exceptions import InvalidConfiguration
 from sentry.utils import redis
+from sentry.utils.redis import disconnect_redis_connection_pools
 
 logger = logging.getLogger(__name__)
 
@@ -34,7 +35,7 @@ class ConcurrentRateLimiter:
     def validate(self) -> None:
         try:
             self.client.ping()
-            self.client.connection_pool.disconnect()
+            disconnect_redis_connection_pools(self.client)
         except Exception as e:
             raise InvalidConfiguration(str(e))