Add 5-minute AggregatingMergeTree stats rollup for dashboard queries
Pre-aggregate route execution stats into 5-minute buckets using a materialized view with -State/-Merge combinators. Rewrite stats() and timeseries() to query the rollup table instead of scanning the wide base table. Active count remains a real-time query since RUNNING is transient. Includes idempotent backfill migration. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
-- Pre-aggregated 5-minute stats rollup for route executions.
|
||||
-- Uses AggregatingMergeTree with -State/-Merge combinators so intermediate
|
||||
-- aggregates can be merged across arbitrary time windows and dimensions.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS route_execution_stats_5m (
|
||||
bucket DateTime('UTC'),
|
||||
route_id LowCardinality(String),
|
||||
agent_id LowCardinality(String),
|
||||
total_count AggregateFunction(count, UInt64),
|
||||
failed_count AggregateFunction(countIf, UInt8),
|
||||
duration_sum AggregateFunction(sum, UInt64),
|
||||
p99_duration AggregateFunction(quantileTDigest(0.99), UInt64)
|
||||
)
|
||||
ENGINE = AggregatingMergeTree()
|
||||
PARTITION BY toYYYYMMDD(bucket)
|
||||
ORDER BY (agent_id, route_id, bucket)
|
||||
TTL bucket + toIntervalDay(30)
|
||||
SETTINGS ttl_only_drop_parts = 1;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS route_execution_stats_5m_mv
|
||||
TO route_execution_stats_5m
|
||||
AS SELECT
|
||||
toStartOfFiveMinutes(start_time) AS bucket,
|
||||
route_id,
|
||||
agent_id,
|
||||
countState() AS total_count,
|
||||
countIfState(status = 'FAILED') AS failed_count,
|
||||
sumState(duration_ms) AS duration_sum,
|
||||
quantileTDigestState(0.99)(duration_ms) AS p99_duration
|
||||
FROM route_executions
|
||||
GROUP BY bucket, route_id, agent_id;
|
||||
@@ -0,0 +1,16 @@
|
||||
-- One-time idempotent backfill of existing route_executions into the
|
||||
-- 5-minute stats rollup table. Safe for repeated execution — the WHERE
|
||||
-- clause skips the INSERT if the target table already contains data.
|
||||
|
||||
INSERT INTO route_execution_stats_5m
|
||||
SELECT
|
||||
toStartOfFiveMinutes(start_time) AS bucket,
|
||||
route_id,
|
||||
agent_id,
|
||||
countState() AS total_count,
|
||||
countIfState(status = 'FAILED') AS failed_count,
|
||||
sumState(duration_ms) AS duration_sum,
|
||||
quantileTDigestState(0.99)(duration_ms) AS p99_duration
|
||||
FROM route_executions
|
||||
WHERE (SELECT count() FROM route_execution_stats_5m) = 0
|
||||
GROUP BY bucket, route_id, agent_id;
|
||||
Reference in New Issue
Block a user