fix(pagination): add insert_id UUID tiebreak to cursor keyset
Same-millisecond rows were silently skipped between pages because the log cursor had no tiebreak and the events cursor tied by instance_id (which also collides when one instance emits multiple events within a millisecond). Add an insert_id UUID (DEFAULT generateUUIDv4()) column to both logs and agent_events, order by (timestamp, insert_id) consistently, and encode the cursor as 'timestamp|insert_id'. Existing data is materialized via ALTER TABLE MATERIALIZE COLUMN (one-time background mutation). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -371,4 +371,30 @@ class ClickHouseLogStoreIT {
|
||||
assertThat(result.data()).extracting(LogEntryResult::message)
|
||||
.containsExactlyInAnyOrder("app msg", "container msg");
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_cursorPagination_sameMillisecond_doesNotSkip() {
|
||||
Instant ts = Instant.parse("2026-04-17T10:00:00Z");
|
||||
// Insert 5 rows at the exact same timestamp
|
||||
java.util.List<LogEntry> batch = new java.util.ArrayList<>();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
batch.add(entry(ts, "INFO", "logger", "msg-" + i, "t1", null, null));
|
||||
}
|
||||
store.indexBatch("agent-1", "my-app", batch);
|
||||
|
||||
// Page through with limit 2; across 3 pages we must see all 5 distinct messages, no duplicates
|
||||
java.util.Set<String> seen = new java.util.HashSet<>();
|
||||
String cursor = null;
|
||||
for (int page = 0; page < 10; page++) {
|
||||
LogSearchResponse resp = store.search(new LogSearchRequest(
|
||||
null, null, "my-app", null, null, null, null, null,
|
||||
null, null, cursor, 2, "desc"));
|
||||
for (LogEntryResult r : resp.data()) {
|
||||
assertThat(seen.add(r.message())).as("duplicate row returned: " + r.message()).isTrue();
|
||||
}
|
||||
cursor = resp.nextCursor();
|
||||
if (!resp.hasMore()) break;
|
||||
}
|
||||
assertThat(seen).containsExactlyInAnyOrder("msg-0", "msg-1", "msg-2", "msg-3", "msg-4");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,25 +115,24 @@ class ClickHouseAgentEventRepositoryIT {
|
||||
}
|
||||
|
||||
@Test
|
||||
void queryPage_tiebreakByInstanceIdAsc_whenTimestampsEqual() {
|
||||
void queryPage_tiebreak_sameMillisecond_returnsAllRowsNoDuplicates() {
|
||||
Instant ts = Instant.parse("2026-04-01T10:00:00Z");
|
||||
insertAt("agent-z", "app-a", "TICK", "z", ts);
|
||||
insertAt("agent-a", "app-a", "TICK", "a", ts);
|
||||
insertAt("agent-m", "app-a", "TICK", "m", ts);
|
||||
insertAt("agent-b", "app-a", "TICK", "b", ts);
|
||||
insertAt("agent-c", "app-a", "TICK", "c", ts);
|
||||
|
||||
com.cameleer.server.core.agent.AgentEventPage p1 =
|
||||
repo.queryPage(null, null, null, null, null, null, 2);
|
||||
assertThat(p1.data()).hasSize(2);
|
||||
// (timestamp DESC, instance_id ASC): ties resolve to a, m, z
|
||||
assertThat(p1.data().get(0).instanceId()).isEqualTo("agent-a");
|
||||
assertThat(p1.data().get(1).instanceId()).isEqualTo("agent-m");
|
||||
assertThat(p1.hasMore()).isTrue();
|
||||
|
||||
com.cameleer.server.core.agent.AgentEventPage p2 =
|
||||
repo.queryPage(null, null, null, null, null, p1.nextCursor(), 2);
|
||||
assertThat(p2.data()).hasSize(1);
|
||||
assertThat(p2.data().get(0).instanceId()).isEqualTo("agent-z");
|
||||
assertThat(p2.hasMore()).isFalse();
|
||||
java.util.Set<String> seen = new java.util.HashSet<>();
|
||||
String cursor = null;
|
||||
for (int page = 0; page < 10; page++) {
|
||||
com.cameleer.server.core.agent.AgentEventPage p =
|
||||
repo.queryPage(null, null, null, null, null, cursor, 1);
|
||||
for (com.cameleer.server.core.agent.AgentEventRecord r : p.data()) {
|
||||
assertThat(seen.add(r.instanceId())).as("duplicate row returned: " + r.instanceId()).isTrue();
|
||||
}
|
||||
cursor = p.nextCursor();
|
||||
if (!p.hasMore()) break;
|
||||
}
|
||||
assertThat(seen).containsExactlyInAnyOrder("agent-a", "agent-b", "agent-c");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -148,7 +147,7 @@ class ClickHouseAgentEventRepositoryIT {
|
||||
}
|
||||
|
||||
@Test
|
||||
void queryPage_malformedCursor_emptyInstanceId_throws() {
|
||||
void queryPage_malformedCursor_emptyInsertId_throws() {
|
||||
String raw = "2026-04-01T10:00:00Z|";
|
||||
String cursor = java.util.Base64.getUrlEncoder().withoutPadding()
|
||||
.encodeToString(raw.getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
||||
|
||||
Reference in New Issue
Block a user