Add 5-minute AggregatingMergeTree stats rollup for dashboard queries
All checks were successful
CI / build (push) Successful in 1m14s
CI / docker (push) Successful in 42s
CI / deploy (push) Successful in 30s

Pre-aggregate route execution stats into 5-minute buckets using a
materialized view with -State/-Merge combinators. Rewrite stats() and
timeseries() to query the rollup table instead of scanning the wide
base table. Active count remains a real-time query since RUNNING is
transient. Includes idempotent backfill migration.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
hsiegeln
2026-03-15 10:46:26 +01:00
parent 6f7c92f793
commit adf13f0430
3 changed files with 97 additions and 44 deletions

View File

@@ -0,0 +1,31 @@
-- Pre-aggregated 5-minute stats rollup for route executions.
-- Uses AggregatingMergeTree with -State/-Merge combinators so intermediate
-- aggregates can be merged across arbitrary time windows and dimensions.
CREATE TABLE IF NOT EXISTS route_execution_stats_5m (
bucket DateTime('UTC'),
route_id LowCardinality(String),
agent_id LowCardinality(String),
total_count AggregateFunction(count, UInt64),
failed_count AggregateFunction(countIf, UInt8),
duration_sum AggregateFunction(sum, UInt64),
p99_duration AggregateFunction(quantileTDigest(0.99), UInt64)
)
ENGINE = AggregatingMergeTree()
PARTITION BY toYYYYMMDD(bucket)
ORDER BY (agent_id, route_id, bucket)
TTL bucket + toIntervalDay(30)
SETTINGS ttl_only_drop_parts = 1;
CREATE MATERIALIZED VIEW IF NOT EXISTS route_execution_stats_5m_mv
TO route_execution_stats_5m
AS SELECT
toStartOfFiveMinutes(start_time) AS bucket,
route_id,
agent_id,
countState() AS total_count,
countIfState(status = 'FAILED') AS failed_count,
sumState(duration_ms) AS duration_sum,
quantileTDigestState(0.99)(duration_ms) AS p99_duration
FROM route_executions
GROUP BY bucket, route_id, agent_id;

View File

@@ -0,0 +1,16 @@
-- One-time idempotent backfill of existing route_executions into the
-- 5-minute stats rollup table. Safe for repeated execution — the WHERE
-- clause skips the INSERT if the target table already contains data.
INSERT INTO route_execution_stats_5m
SELECT
toStartOfFiveMinutes(start_time) AS bucket,
route_id,
agent_id,
countState() AS total_count,
countIfState(status = 'FAILED') AS failed_count,
sumState(duration_ms) AS duration_sum,
quantileTDigestState(0.99)(duration_ms) AS p99_duration
FROM route_executions
WHERE (SELECT count() FROM route_execution_stats_5m) = 0
GROUP BY bucket, route_id, agent_id;