feat(migrations): Clickhouse Config for Distributed Migrations Mode (#25872)

* cadd clickhouse config file for dist mode migrations support

* modified clickhouse config - zookeeper hostname

* try to modify the  kafka setting in sentry setup action

* basic local/dist switch working

* added clickhouse_dist container

* modified volume name for distributed clickhouse container

* revert kafka setting in sentry setup action

* removed repeated comment

* added defined cluster to dist_config

* remove duplication in clickhouse container configs

* added pre-defined dict to better remove duplicate code in server.py

* removed old comments

config/clickhouse/dist_config.xml

@@ -0,0 +1,27 @@
+<yandex>
+    <max_server_memory_usage_to_ram_ratio from_env="MAX_MEMORY_USAGE_RATIO" />
+
+    <remote_servers>
+        <cluster_one_sh>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </cluster_one_sh>
+    </remote_servers>
+
+    <zookeeper>
+        <node>
+            <host>sentry_zookeeper</host>
+            <port>2181</port>
+        </node>
+    </zookeeper>
+
+    <macros>
+        <shard>1</shard>
+        <replica>1</replica>
+    </macros>
+
+</yandex>

src/sentry/conf/server.py

@@ -124,7 +124,14 @@ DEVSERVICES_CONFIG_DIR = os.path.normpath(
     os.path.join(PROJECT_ROOT, os.pardir, os.pardir, "config")
 )
 
-CLICKHOUSE_CONFIG_PATH = os.path.join(DEVSERVICES_CONFIG_DIR, "clickhouse", "config.xml")
+SENTRY_DISTRIBUTED_CLICKHOUSE_TABLES = False
+_common_clickhouse_settings = {
+    "image": "yandex/clickhouse-server:20.3.9.70",
+    "pull": True,
+    "ports": {"9000/tcp": 9000, "9009/tcp": 9009, "8123/tcp": 8123},
+    "ulimits": [{"name": "nofile", "soft": 262144, "hard": 262144}],
+    "environment": {"MAX_MEMORY_USAGE_RATIO": "0.3"},
+}
 
 RELAY_CONFIG_DIR = os.path.join(DEVSERVICES_CONFIG_DIR, "relay")
 
@@ -1662,23 +1669,24 @@ SENTRY_DEVSERVICES = {
         ),
     },
     "clickhouse": {
-        "image": "yandex/clickhouse-server:20.3.9.70",
-        "pull": True,
-        "ports": {"9000/tcp": 9000, "9009/tcp": 9009, "8123/tcp": 8123},
-        "ulimits": [{"name": "nofile", "soft": 262144, "hard": 262144}],
+        **_common_clickhouse_settings,
         "volumes": {
             "clickhouse": {"bind": "/var/lib/clickhouse"},
-            CLICKHOUSE_CONFIG_PATH: {"bind": "/etc/clickhouse-server/config.d/sentry.xml"},
+            os.path.join(DEVSERVICES_CONFIG_DIR, "clickhouse", "loc_config.xml"): {
+                "bind": "/etc/clickhouse-server/config.d/sentry.xml"
+            },
         },
-        "environment": {
-            # This limits Clickhouse's memory to 30% of the host memory
-            # If you have high volume and your search return incomplete results
-            # You might want to change this to a higher value (and ensure your host has enough memory)
-            "MAX_MEMORY_USAGE_RATIO": "0.3"
+        "only_if": lambda settings, options: (not settings.SENTRY_DISTRIBUTED_CLICKHOUSE_TABLES),
+    },
+    "clickhouse_dist": {
+        **_common_clickhouse_settings,
+        "volumes": {
+            "clickhouse_dist": {"bind": "/var/lib/clickhouse"},
+            os.path.join(DEVSERVICES_CONFIG_DIR, "clickhouse", "dist_config.xml"): {
+                "bind": "/etc/clickhouse-server/config.d/sentry.xml"
+            },
         },
-        "only_if": lambda settings, options: (
-            "snuba" in settings.SENTRY_EVENTSTREAM or "kafka" in settings.SENTRY_EVENTSTREAM
-        ),
+        "only_if": lambda settings, options: (settings.SENTRY_DISTRIBUTED_CLICKHOUSE_TABLES),
     },
     "snuba": {
         "image": "getsentry/snuba:nightly",