fix(pagination): add insert_id UUID tiebreak to cursor keyset
All checks were successful
CI / cleanup-branch (push) Has been skipped
CI / build (push) Successful in 1m26s
CI / docker (push) Successful in 1m12s
CI / deploy-feature (push) Has been skipped
CI / deploy (push) Successful in 43s

Same-millisecond rows were silently skipped between pages because the
log cursor had no tiebreak and the events cursor tied by instance_id
(which also collides when one instance emits multiple events within a
millisecond). Add an insert_id UUID (DEFAULT generateUUIDv4()) column
to both logs and agent_events, order by (timestamp, insert_id)
consistently, and encode the cursor as 'timestamp|insert_id'. Existing
data is materialized via ALTER TABLE MATERIALIZE COLUMN (one-time
background mutation).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
hsiegeln
2026-04-17 14:25:36 +02:00
parent 07dbfb1391
commit 89c9b53edd
5 changed files with 103 additions and 41 deletions

View File

@@ -327,7 +327,8 @@ CREATE TABLE IF NOT EXISTS agent_events (
instance_id LowCardinality(String),
application_id LowCardinality(String),
event_type LowCardinality(String),
detail String DEFAULT ''
detail String DEFAULT '',
insert_id UUID DEFAULT generateUUIDv4()
)
ENGINE = MergeTree()
PARTITION BY (tenant_id, toYYYYMM(timestamp))
@@ -349,6 +350,7 @@ CREATE TABLE IF NOT EXISTS logs (
stack_trace String DEFAULT '',
exchange_id String DEFAULT '',
mdc Map(String, String) DEFAULT map(),
insert_id UUID DEFAULT generateUUIDv4(),
INDEX idx_msg message TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY 4,
INDEX idx_stack stack_trace TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY 4,
@@ -398,3 +400,12 @@ CREATE TABLE IF NOT EXISTS route_catalog (
)
ENGINE = ReplacingMergeTree(last_seen)
ORDER BY (tenant_id, environment, application_id, route_id);
-- insert_id tiebreak for keyset pagination (fixes same-millisecond cursor collision).
-- IF NOT EXISTS on ADD COLUMN is idempotent. MATERIALIZE COLUMN is a background mutation,
-- effectively a no-op once all parts are already materialized.
ALTER TABLE logs ADD COLUMN IF NOT EXISTS insert_id UUID DEFAULT generateUUIDv4();
ALTER TABLE logs MATERIALIZE COLUMN insert_id;
ALTER TABLE agent_events ADD COLUMN IF NOT EXISTS insert_id UUID DEFAULT generateUUIDv4();
ALTER TABLE agent_events MATERIALIZE COLUMN insert_id;