feat: comprehensive ClickHouse low-memory tuning and switch all storage to ClickHouse

Replace partial memory config with full Altinity low-memory guide
settings. Revert container limit from 6Gi back to 4Gi — proper
tuning (mlock=false, reduced caches/pools/threads, disk spill for
aggregations) makes the original budget sufficient.

Switch all storage feature flags to ClickHouse:
- CAMELEER_STORAGE_SEARCH: opensearch → clickhouse
- CAMELEER_STORAGE_METRICS: postgres → clickhouse
- CAMELEER_STORAGE_STATS: already clickhouse

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

deploy/base/server.yaml

@@ -90,9 +90,9 @@ spec:
                   name: clickhouse-credentials
                   key: CLICKHOUSE_PASSWORD
             - name: CAMELEER_STORAGE_METRICS
-              value: "postgres"
+              value: "clickhouse"
             - name: CAMELEER_STORAGE_SEARCH
-              value: "opensearch"
+              value: "clickhouse"
             - name: CAMELEER_STORAGE_STATS
               value: "clickhouse"
 

deploy/clickhouse.yaml

@@ -41,15 +41,19 @@ spec:
             - name: initdb
               mountPath: /docker-entrypoint-initdb.d
             - name: config
-              mountPath: /etc/clickhouse-server/config.d/memory-tuning.xml
+              mountPath: /etc/clickhouse-server/config.d/low-memory.xml
-              subPath: memory-tuning.xml
+              subPath: low-memory.xml
+              readOnly: true
+            - name: config
+              mountPath: /etc/clickhouse-server/users.d/low-memory-users.xml
+              subPath: low-memory-users.xml
               readOnly: true
           resources:
             requests:
               memory: "2Gi"
               cpu: "500m"
             limits:
-              memory: "6Gi"
+              memory: "4Gi"
               cpu: "2000m"
           livenessProbe:
             httpGet:
@@ -106,28 +110,81 @@ metadata:
   name: clickhouse-config
   namespace: cameleer
 data:
-  memory-tuning.xml: |
+  # Server-level tuning for 4Gi container.
+  # Based on https://kb.altinity.com/altinity-kb-setup-and-maintenance/configure_clickhouse_for_low_mem_envs/
+  low-memory.xml: |
     <clickhouse>
-        <!-- Fit comfortably in a 6Gi container (cgroup limit).                  -->
+        <!-- Let OS page out binary segments instead of pinning them in RAM -->
-        <!-- Without this file ClickHouse auto-sizes caches and buffers from the -->
+        <mlock_executable>false</mlock_executable>
-        <!-- cgroup limit, leaving almost no room for query execution.           -->
 
-        <!-- Cap any single query/insert (including MV processing) at 2 GiB -->
+        <!-- Disable unused protocol ports -->
+        <mysql_port remove="1" />
+        <postgresql_port remove="1" />
+
+        <!-- Disable noisy system log tables -->
+        <query_thread_log remove="1" />
+        <opentelemetry_span_log remove="1" />
+        <processors_profile_log remove="1" />
+
+        <!-- Reserve 25% of cgroup memory for OS/kernel (default 0.9 is too aggressive) -->
+        <max_server_memory_usage_to_ram_ratio>0.75</max_server_memory_usage_to_ram_ratio>
+        <max_server_memory_usage>0</max_server_memory_usage>
+
+        <!-- Shrink caches — defaults are sized for 64+ GiB servers -->
+        <mark_cache_size>268435456</mark_cache_size>           <!-- 256 MiB -->
+        <index_mark_cache_size>67108864</index_mark_cache_size> <!-- 64 MiB -->
+        <uncompressed_cache_size>16777216</uncompressed_cache_size> <!-- 16 MiB -->
+        <compiled_expression_cache_size>67108864</compiled_expression_cache_size> <!-- 64 MiB -->
+
+        <!-- Connection & concurrency limits -->
+        <max_thread_pool_size>2000</max_thread_pool_size>
+        <max_connections>64</max_connections>
+        <max_concurrent_queries>8</max_concurrent_queries>
+
+        <!-- Background merge pools — keep low to avoid memory spikes -->
+        <background_pool_size>2</background_pool_size>
+        <background_merges_mutations_concurrency_ratio>2</background_merges_mutations_concurrency_ratio>
+        <background_buffer_flush_schedule_pool_size>1</background_buffer_flush_schedule_pool_size>
+        <background_move_pool_size>1</background_move_pool_size>
+        <background_fetches_pool_size>1</background_fetches_pool_size>
+        <background_common_pool_size>2</background_common_pool_size>
+        <background_schedule_pool_size>8</background_schedule_pool_size>
+        <background_message_broker_schedule_pool_size>1</background_message_broker_schedule_pool_size>
+        <background_distributed_schedule_pool_size>1</background_distributed_schedule_pool_size>
+        <tables_loader_foreground_pool_size>0</tables_loader_foreground_pool_size>
+        <tables_loader_background_pool_size>0</tables_loader_background_pool_size>
+
+        <!-- MergeTree: limit merge memory and allow more concurrent smaller merges -->
+        <merge_tree>
+            <merge_max_block_size>1024</merge_max_block_size>
+            <max_bytes_to_merge_at_max_space_in_pool>1073741824</max_bytes_to_merge_at_max_space_in_pool> <!-- 1 GiB -->
+            <min_bytes_for_wide_part>134217728</min_bytes_for_wide_part>
+            <vertical_merge_algorithm_min_bytes_to_activate>134217728</vertical_merge_algorithm_min_bytes_to_activate>
+            <number_of_free_entries_in_pool_to_lower_max_size_of_merge>2</number_of_free_entries_in_pool_to_lower_max_size_of_merge>
+            <number_of_free_entries_in_pool_to_execute_mutation>2</number_of_free_entries_in_pool_to_execute_mutation>
+            <number_of_free_entries_in_pool_to_execute_optimize_entire_partition>2</number_of_free_entries_in_pool_to_execute_optimize_entire_partition>
+        </merge_tree>
+    </clickhouse>
+  # User profile settings — per-query limits and parallelism.
+  low-memory-users.xml: |
+    <clickhouse>
         <profiles>
             <default>
-                <max_memory_usage>2147483648</max_memory_usage>
+                <!-- Limit per-query thread count (default = nproc) -->
+                <max_threads>2</max_threads>
+                <!-- Cap per-query memory at 1.5 GiB -->
+                <max_memory_usage>1610612736</max_memory_usage>
+                <max_block_size>8192</max_block_size>
+                <queue_max_wait_ms>1000</queue_max_wait_ms>
+                <max_execution_time>600</max_execution_time>
+                <!-- Disable parallel parse/format to reduce per-query memory -->
+                <input_format_parallel_parsing>0</input_format_parallel_parsing>
+                <output_format_parallel_formatting>0</output_format_parallel_formatting>
+                <!-- Spill to disk instead of OOM for large aggregations/sorts -->
+                <max_bytes_before_external_group_by>1073741824</max_bytes_before_external_group_by>
+                <max_bytes_before_external_sort>1073741824</max_bytes_before_external_sort>
             </default>
         </profiles>
-
-        <!-- Shrink internal caches — defaults are sized for beefy servers -->
-        <mark_cache_size>134217728</mark_cache_size>
-        <index_mark_cache_size>67108864</index_mark_cache_size>
-        <uncompressed_cache_size>0</uncompressed_cache_size>
-        <compiled_expression_cache_size>67108864</compiled_expression_cache_size>
-
-        <merge_tree>
-            <max_bytes_to_merge_at_max_space_in_pool>536870912</max_bytes_to_merge_at_max_space_in_pool>
-        </merge_tree>
     </clickhouse>
 ---
 apiVersion: v1