fix(pagination): add insert_id UUID tiebreak to cursor keyset
Same-millisecond rows were silently skipped between pages because the log cursor had no tiebreak and the events cursor tied by instance_id (which also collides when one instance emits multiple events within a millisecond). Add an insert_id UUID (DEFAULT generateUUIDv4()) column to both logs and agent_events, order by (timestamp, insert_id) consistently, and encode the cursor as 'timestamp|insert_id'. Existing data is materialized via ALTER TABLE MATERIALIZE COLUMN (one-time background mutation). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -327,7 +327,8 @@ CREATE TABLE IF NOT EXISTS agent_events (
|
||||
instance_id LowCardinality(String),
|
||||
application_id LowCardinality(String),
|
||||
event_type LowCardinality(String),
|
||||
detail String DEFAULT ''
|
||||
detail String DEFAULT '',
|
||||
insert_id UUID DEFAULT generateUUIDv4()
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY (tenant_id, toYYYYMM(timestamp))
|
||||
@@ -349,6 +350,7 @@ CREATE TABLE IF NOT EXISTS logs (
|
||||
stack_trace String DEFAULT '',
|
||||
exchange_id String DEFAULT '',
|
||||
mdc Map(String, String) DEFAULT map(),
|
||||
insert_id UUID DEFAULT generateUUIDv4(),
|
||||
|
||||
INDEX idx_msg message TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY 4,
|
||||
INDEX idx_stack stack_trace TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY 4,
|
||||
@@ -398,3 +400,12 @@ CREATE TABLE IF NOT EXISTS route_catalog (
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(last_seen)
|
||||
ORDER BY (tenant_id, environment, application_id, route_id);
|
||||
|
||||
-- insert_id tiebreak for keyset pagination (fixes same-millisecond cursor collision).
|
||||
-- IF NOT EXISTS on ADD COLUMN is idempotent. MATERIALIZE COLUMN is a background mutation,
|
||||
-- effectively a no-op once all parts are already materialized.
|
||||
ALTER TABLE logs ADD COLUMN IF NOT EXISTS insert_id UUID DEFAULT generateUUIDv4();
|
||||
ALTER TABLE logs MATERIALIZE COLUMN insert_id;
|
||||
|
||||
ALTER TABLE agent_events ADD COLUMN IF NOT EXISTS insert_id UUID DEFAULT generateUUIDv4();
|
||||
ALTER TABLE agent_events MATERIALIZE COLUMN insert_id;
|
||||
|
||||
Reference in New Issue
Block a user