feat(clickhouse): add ClickHouseSearchIndex with ngram-accelerated SQL search
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
hsiegeln
@@ -0,0 +1,319 @@
|
||||
package com.cameleer3.server.app.search;
|
||||
|
||||
import com.cameleer3.server.app.storage.ClickHouseExecutionStore;
|
||||
import com.cameleer3.server.core.ingestion.MergedExecution;
|
||||
import com.cameleer3.server.core.search.ExecutionSummary;
|
||||
import com.cameleer3.server.core.search.SearchRequest;
|
||||
import com.cameleer3.server.core.search.SearchResult;
|
||||
import com.cameleer3.server.core.storage.model.FlatProcessorRecord;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.testcontainers.clickhouse.ClickHouseContainer;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@Testcontainers
|
||||
class ClickHouseSearchIndexIT {
|
||||
|
||||
@Container
|
||||
static final ClickHouseContainer clickhouse =
|
||||
new ClickHouseContainer("clickhouse/clickhouse-server:24.12");
|
||||
|
||||
private JdbcTemplate jdbc;
|
||||
private ClickHouseSearchIndex searchIndex;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
HikariDataSource ds = new HikariDataSource();
|
||||
ds.setJdbcUrl(clickhouse.getJdbcUrl());
|
||||
ds.setUsername(clickhouse.getUsername());
|
||||
ds.setPassword(clickhouse.getPassword());
|
||||
|
||||
jdbc = new JdbcTemplate(ds);
|
||||
|
||||
// Load DDL from classpath resources
|
||||
String executionsDdl = new ClassPathResource("clickhouse/V2__executions.sql")
|
||||
.getContentAsString(StandardCharsets.UTF_8);
|
||||
String processorsDdl = new ClassPathResource("clickhouse/V3__processor_executions.sql")
|
||||
.getContentAsString(StandardCharsets.UTF_8);
|
||||
|
||||
jdbc.execute(executionsDdl);
|
||||
jdbc.execute(processorsDdl);
|
||||
|
||||
jdbc.execute("TRUNCATE TABLE executions");
|
||||
jdbc.execute("TRUNCATE TABLE processor_executions");
|
||||
|
||||
ClickHouseExecutionStore store = new ClickHouseExecutionStore(jdbc);
|
||||
searchIndex = new ClickHouseSearchIndex(jdbc);
|
||||
|
||||
// Seed test data
|
||||
Instant baseTime = Instant.parse("2026-03-31T10:00:00Z");
|
||||
|
||||
// exec-1: COMPLETED, route-timer, agent-a, my-app, corr-1, 500ms, input_body with order number, attributes
|
||||
MergedExecution exec1 = new MergedExecution(
|
||||
"default", 1L, "exec-1", "route-timer", "agent-a", "my-app",
|
||||
"COMPLETED", "corr-1", "exchange-1",
|
||||
baseTime,
|
||||
baseTime.plusMillis(500),
|
||||
500L,
|
||||
"", "", "", "", "", "",
|
||||
"hash-abc", "FULL",
|
||||
"{\"order\":\"12345\"}", "", "", "", "{\"env\":\"prod\"}",
|
||||
"", "",
|
||||
false, false
|
||||
);
|
||||
|
||||
// exec-2: FAILED, route-timer, agent-a, my-app, corr-2, 200ms, with error
|
||||
MergedExecution exec2 = new MergedExecution(
|
||||
"default", 1L, "exec-2", "route-timer", "agent-a", "my-app",
|
||||
"FAILED", "corr-2", "exchange-2",
|
||||
baseTime.plusSeconds(1),
|
||||
baseTime.plusSeconds(1).plusMillis(200),
|
||||
200L,
|
||||
"NullPointerException at line 42",
|
||||
"java.lang.NPE\n at Foo.bar(Foo.java:42)",
|
||||
"NullPointerException", "RUNTIME", "", "",
|
||||
"", "FULL",
|
||||
"", "", "", "", "",
|
||||
"", "",
|
||||
false, false
|
||||
);
|
||||
|
||||
// exec-3: COMPLETED, route-rest, agent-b, other-app, 100ms, no error
|
||||
MergedExecution exec3 = new MergedExecution(
|
||||
"default", 1L, "exec-3", "route-rest", "agent-b", "other-app",
|
||||
"COMPLETED", "", "exchange-3",
|
||||
baseTime.plusSeconds(2),
|
||||
baseTime.plusSeconds(2).plusMillis(100),
|
||||
100L,
|
||||
"", "", "", "", "", "",
|
||||
"", "FULL",
|
||||
"", "", "", "", "",
|
||||
"", "",
|
||||
false, false
|
||||
);
|
||||
|
||||
store.insertExecutionBatch(List.of(exec1, exec2, exec3));
|
||||
|
||||
// Processor for exec-1: seq=1, to, inputBody with "Hello World", inputHeaders with secret-token
|
||||
FlatProcessorRecord proc1 = new FlatProcessorRecord(
|
||||
1, null, null,
|
||||
"proc-1", "to", null, null,
|
||||
"COMPLETED",
|
||||
baseTime, 50L,
|
||||
null,
|
||||
"Hello World request body", "",
|
||||
Map.of("Authorization", "Bearer secret-token"), null,
|
||||
null, null, null, null, null, null,
|
||||
null, null, null, null, null
|
||||
);
|
||||
|
||||
store.insertProcessorBatch("default", "exec-1", "route-timer", "my-app", baseTime, List.of(proc1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_withNoFilters_returnsAllExecutions() {
|
||||
SearchRequest request = new SearchRequest(
|
||||
null, null, null, null, null, null, null, null, null, null,
|
||||
null, null, null, null, null, 0, 50, null, null);
|
||||
|
||||
SearchResult<ExecutionSummary> result = searchIndex.search(request);
|
||||
|
||||
assertThat(result.total()).isEqualTo(3);
|
||||
assertThat(result.data()).hasSize(3);
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_byStatus_filtersCorrectly() {
|
||||
SearchRequest request = new SearchRequest(
|
||||
"FAILED", null, null, null, null, null, null, null, null, null,
|
||||
null, null, null, null, null, 0, 50, null, null);
|
||||
|
||||
SearchResult<ExecutionSummary> result = searchIndex.search(request);
|
||||
|
||||
assertThat(result.total()).isEqualTo(1);
|
||||
assertThat(result.data()).hasSize(1);
|
||||
assertThat(result.data().get(0).executionId()).isEqualTo("exec-2");
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_byTimeRange_filtersCorrectly() {
|
||||
Instant baseTime = Instant.parse("2026-03-31T10:00:00Z");
|
||||
// Time window covering exec-1 and exec-2 but not exec-3
|
||||
SearchRequest request = new SearchRequest(
|
||||
null, baseTime, baseTime.plusMillis(1500), null, null, null, null, null, null, null,
|
||||
null, null, null, null, null, 0, 50, null, null);
|
||||
|
||||
SearchResult<ExecutionSummary> result = searchIndex.search(request);
|
||||
|
||||
assertThat(result.total()).isEqualTo(2);
|
||||
assertThat(result.data()).extracting(ExecutionSummary::executionId)
|
||||
.containsExactlyInAnyOrder("exec-1", "exec-2");
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_fullTextSearch_findsInErrorMessage() {
|
||||
SearchRequest request = new SearchRequest(
|
||||
null, null, null, null, null, null, "NullPointerException", null, null, null,
|
||||
null, null, null, null, null, 0, 50, null, null);
|
||||
|
||||
SearchResult<ExecutionSummary> result = searchIndex.search(request);
|
||||
|
||||
assertThat(result.total()).isEqualTo(1);
|
||||
assertThat(result.data().get(0).executionId()).isEqualTo("exec-2");
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_fullTextSearch_findsInInputBody() {
|
||||
SearchRequest request = new SearchRequest(
|
||||
null, null, null, null, null, null, "12345", null, null, null,
|
||||
null, null, null, null, null, 0, 50, null, null);
|
||||
|
||||
SearchResult<ExecutionSummary> result = searchIndex.search(request);
|
||||
|
||||
assertThat(result.total()).isEqualTo(1);
|
||||
assertThat(result.data().get(0).executionId()).isEqualTo("exec-1");
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_textInBody_searchesProcessorBodies() {
|
||||
SearchRequest request = new SearchRequest(
|
||||
null, null, null, null, null, null, null, "Hello World", null, null,
|
||||
null, null, null, null, null, 0, 50, null, null);
|
||||
|
||||
SearchResult<ExecutionSummary> result = searchIndex.search(request);
|
||||
|
||||
assertThat(result.total()).isEqualTo(1);
|
||||
assertThat(result.data().get(0).executionId()).isEqualTo("exec-1");
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_textInHeaders_searchesProcessorHeaders() {
|
||||
SearchRequest request = new SearchRequest(
|
||||
null, null, null, null, null, null, null, null, "secret-token", null,
|
||||
null, null, null, null, null, 0, 50, null, null);
|
||||
|
||||
SearchResult<ExecutionSummary> result = searchIndex.search(request);
|
||||
|
||||
assertThat(result.total()).isEqualTo(1);
|
||||
assertThat(result.data().get(0).executionId()).isEqualTo("exec-1");
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_textInErrors_searchesErrorFields() {
|
||||
SearchRequest request = new SearchRequest(
|
||||
null, null, null, null, null, null, null, null, null, "Foo.bar",
|
||||
null, null, null, null, null, 0, 50, null, null);
|
||||
|
||||
SearchResult<ExecutionSummary> result = searchIndex.search(request);
|
||||
|
||||
assertThat(result.total()).isEqualTo(1);
|
||||
assertThat(result.data().get(0).executionId()).isEqualTo("exec-2");
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_withHighlight_returnsSnippet() {
|
||||
SearchRequest request = new SearchRequest(
|
||||
null, null, null, null, null, null, "NullPointerException", null, null, null,
|
||||
null, null, null, null, null, 0, 50, null, null);
|
||||
|
||||
SearchResult<ExecutionSummary> result = searchIndex.search(request);
|
||||
|
||||
assertThat(result.total()).isEqualTo(1);
|
||||
assertThat(result.data().get(0).highlight()).contains("NullPointerException");
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_pagination_works() {
|
||||
SearchRequest request = new SearchRequest(
|
||||
null, null, null, null, null, null, null, null, null, null,
|
||||
null, null, null, null, null, 0, 2, null, null);
|
||||
|
||||
SearchResult<ExecutionSummary> result = searchIndex.search(request);
|
||||
|
||||
assertThat(result.total()).isEqualTo(3);
|
||||
assertThat(result.data()).hasSize(2);
|
||||
assertThat(result.offset()).isEqualTo(0);
|
||||
assertThat(result.limit()).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_byApplication_filtersCorrectly() {
|
||||
SearchRequest request = new SearchRequest(
|
||||
null, null, null, null, null, null, null, null, null, null,
|
||||
null, null, null, "other-app", null, 0, 50, null, null);
|
||||
|
||||
SearchResult<ExecutionSummary> result = searchIndex.search(request);
|
||||
|
||||
assertThat(result.total()).isEqualTo(1);
|
||||
assertThat(result.data().get(0).executionId()).isEqualTo("exec-3");
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_byAgentIds_filtersCorrectly() {
|
||||
SearchRequest request = new SearchRequest(
|
||||
null, null, null, null, null, null, null, null, null, null,
|
||||
null, null, null, null, List.of("agent-b"), 0, 50, null, null);
|
||||
|
||||
SearchResult<ExecutionSummary> result = searchIndex.search(request);
|
||||
|
||||
assertThat(result.total()).isEqualTo(1);
|
||||
assertThat(result.data().get(0).executionId()).isEqualTo("exec-3");
|
||||
}
|
||||
|
||||
@Test
|
||||
void count_returnsMatchingCount() {
|
||||
SearchRequest request = new SearchRequest(
|
||||
"COMPLETED", null, null, null, null, null, null, null, null, null,
|
||||
null, null, null, null, null, 0, 50, null, null);
|
||||
|
||||
long count = searchIndex.count(request);
|
||||
|
||||
assertThat(count).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_multipleStatusFilter_works() {
|
||||
SearchRequest request = new SearchRequest(
|
||||
"COMPLETED,FAILED", null, null, null, null, null, null, null, null, null,
|
||||
null, null, null, null, null, 0, 50, null, null);
|
||||
|
||||
SearchResult<ExecutionSummary> result = searchIndex.search(request);
|
||||
|
||||
assertThat(result.total()).isEqualTo(3);
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_byCorrelationId_filtersCorrectly() {
|
||||
SearchRequest request = new SearchRequest(
|
||||
null, null, null, null, null, "corr-1", null, null, null, null,
|
||||
null, null, null, null, null, 0, 50, null, null);
|
||||
|
||||
SearchResult<ExecutionSummary> result = searchIndex.search(request);
|
||||
|
||||
assertThat(result.total()).isEqualTo(1);
|
||||
assertThat(result.data().get(0).executionId()).isEqualTo("exec-1");
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_byDurationRange_filtersCorrectly() {
|
||||
SearchRequest request = new SearchRequest(
|
||||
null, null, null, 300L, 600L, null, null, null, null, null,
|
||||
null, null, null, null, null, 0, 50, null, null);
|
||||
|
||||
SearchResult<ExecutionSummary> result = searchIndex.search(request);
|
||||
|
||||
assertThat(result.total()).isEqualTo(1);
|
||||
assertThat(result.data().get(0).executionId()).isEqualTo("exec-1");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user