fix(pagination): add insert_id UUID tiebreak to cursor keyset
All checks were successful
CI / cleanup-branch (push) Has been skipped
CI / build (push) Successful in 1m26s
CI / docker (push) Successful in 1m12s
CI / deploy-feature (push) Has been skipped
CI / deploy (push) Successful in 43s

Same-millisecond rows were silently skipped between pages because the
log cursor had no tiebreak and the events cursor tied by instance_id
(which also collides when one instance emits multiple events within a
millisecond). Add an insert_id UUID (DEFAULT generateUUIDv4()) column
to both logs and agent_events, order by (timestamp, insert_id)
consistently, and encode the cursor as 'timestamp|insert_id'. Existing
data is materialized via ALTER TABLE MATERIALIZE COLUMN (one-time
background mutation).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
hsiegeln
2026-04-17 14:25:36 +02:00
parent 07dbfb1391
commit 89c9b53edd
5 changed files with 103 additions and 41 deletions

View File

@@ -371,4 +371,30 @@ class ClickHouseLogStoreIT {
assertThat(result.data()).extracting(LogEntryResult::message)
.containsExactlyInAnyOrder("app msg", "container msg");
}
@Test
void search_cursorPagination_sameMillisecond_doesNotSkip() {
Instant ts = Instant.parse("2026-04-17T10:00:00Z");
// Insert 5 rows at the exact same timestamp
java.util.List<LogEntry> batch = new java.util.ArrayList<>();
for (int i = 0; i < 5; i++) {
batch.add(entry(ts, "INFO", "logger", "msg-" + i, "t1", null, null));
}
store.indexBatch("agent-1", "my-app", batch);
// Page through with limit 2; across 3 pages we must see all 5 distinct messages, no duplicates
java.util.Set<String> seen = new java.util.HashSet<>();
String cursor = null;
for (int page = 0; page < 10; page++) {
LogSearchResponse resp = store.search(new LogSearchRequest(
null, null, "my-app", null, null, null, null, null,
null, null, cursor, 2, "desc"));
for (LogEntryResult r : resp.data()) {
assertThat(seen.add(r.message())).as("duplicate row returned: " + r.message()).isTrue();
}
cursor = resp.nextCursor();
if (!resp.hasMore()) break;
}
assertThat(seen).containsExactlyInAnyOrder("msg-0", "msg-1", "msg-2", "msg-3", "msg-4");
}
}

View File

@@ -115,25 +115,24 @@ class ClickHouseAgentEventRepositoryIT {
}
@Test
void queryPage_tiebreakByInstanceIdAsc_whenTimestampsEqual() {
void queryPage_tiebreak_sameMillisecond_returnsAllRowsNoDuplicates() {
Instant ts = Instant.parse("2026-04-01T10:00:00Z");
insertAt("agent-z", "app-a", "TICK", "z", ts);
insertAt("agent-a", "app-a", "TICK", "a", ts);
insertAt("agent-m", "app-a", "TICK", "m", ts);
insertAt("agent-b", "app-a", "TICK", "b", ts);
insertAt("agent-c", "app-a", "TICK", "c", ts);
com.cameleer.server.core.agent.AgentEventPage p1 =
repo.queryPage(null, null, null, null, null, null, 2);
assertThat(p1.data()).hasSize(2);
// (timestamp DESC, instance_id ASC): ties resolve to a, m, z
assertThat(p1.data().get(0).instanceId()).isEqualTo("agent-a");
assertThat(p1.data().get(1).instanceId()).isEqualTo("agent-m");
assertThat(p1.hasMore()).isTrue();
com.cameleer.server.core.agent.AgentEventPage p2 =
repo.queryPage(null, null, null, null, null, p1.nextCursor(), 2);
assertThat(p2.data()).hasSize(1);
assertThat(p2.data().get(0).instanceId()).isEqualTo("agent-z");
assertThat(p2.hasMore()).isFalse();
java.util.Set<String> seen = new java.util.HashSet<>();
String cursor = null;
for (int page = 0; page < 10; page++) {
com.cameleer.server.core.agent.AgentEventPage p =
repo.queryPage(null, null, null, null, null, cursor, 1);
for (com.cameleer.server.core.agent.AgentEventRecord r : p.data()) {
assertThat(seen.add(r.instanceId())).as("duplicate row returned: " + r.instanceId()).isTrue();
}
cursor = p.nextCursor();
if (!p.hasMore()) break;
}
assertThat(seen).containsExactlyInAnyOrder("agent-a", "agent-b", "agent-c");
}
@Test
@@ -148,7 +147,7 @@ class ClickHouseAgentEventRepositoryIT {
}
@Test
void queryPage_malformedCursor_emptyInstanceId_throws() {
void queryPage_malformedCursor_emptyInsertId_throws() {
String raw = "2026-04-01T10:00:00Z|";
String cursor = java.util.Base64.getUrlEncoder().withoutPadding()
.encodeToString(raw.getBytes(java.nio.charset.StandardCharsets.UTF_8));