From e0aac4bf0a43f13578a5394c6aefea50e84c2091 Mon Sep 17 00:00:00 2001
From: hsiegeln <37154749+hsiegeln@users.noreply.github.com>
Date: Fri, 3 Apr 2026 22:33:48 +0200
Subject: [PATCH] perf: enable ClickHouse async_insert to batch small inserts
 server-side

Diagnostics showed 3,200 tiny inserts per 5 minutes (processor_executions:
2,376 at 14 rows avg, logs: 803 at 5 rows avg), each creating a new part
and triggering MV aggregations + background merges. This was the root cause
of ~400m CPU usage at 3 tx/s.

async_insert=1 with 5s busy timeout lets ClickHouse buffer incoming inserts
and consolidate them into fewer, larger parts before writing to disk.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 deploy/clickhouse.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/deploy/clickhouse.yaml b/deploy/clickhouse.yaml
index cb19e974..d11dea02 100644
--- a/deploy/clickhouse.yaml
+++ b/deploy/clickhouse.yaml
@@ -182,6 +182,10 @@ data:
                 <max_block_size>8192</max_block_size>
                 <queue_max_wait_ms>1000</queue_max_wait_ms>
                 <max_execution_time>600</max_execution_time>
+                <!-- Buffer small inserts server-side before creating parts -->
+                <async_insert>1</async_insert>
+                <wait_for_async_insert>0</wait_for_async_insert>
+                <async_insert_busy_timeout_ms>5000</async_insert_busy_timeout_ms>
                 <!-- Disable parallel parse/format to reduce per-query memory -->
                 <input_format_parallel_parsing>0</input_format_parallel_parsing>
                 <output_format_parallel_formatting>0</output_format_parallel_formatting>