feat(clickhouse): add ClickHouseExecutionStore with batch insert for chunked format
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
hsiegeln
@@ -0,0 +1,231 @@
|
||||
package com.cameleer3.server.app.storage;
|
||||
|
||||
import com.cameleer3.server.core.ingestion.MergedExecution;
|
||||
import com.cameleer3.server.core.storage.model.FlatProcessorRecord;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.testcontainers.clickhouse.ClickHouseContainer;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@Testcontainers
|
||||
class ClickHouseExecutionStoreIT {
|
||||
|
||||
@Container
|
||||
static final ClickHouseContainer clickhouse =
|
||||
new ClickHouseContainer("clickhouse/clickhouse-server:24.12");
|
||||
|
||||
private JdbcTemplate jdbc;
|
||||
private ClickHouseExecutionStore store;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
HikariDataSource ds = new HikariDataSource();
|
||||
ds.setJdbcUrl(clickhouse.getJdbcUrl());
|
||||
ds.setUsername(clickhouse.getUsername());
|
||||
ds.setPassword(clickhouse.getPassword());
|
||||
|
||||
jdbc = new JdbcTemplate(ds);
|
||||
|
||||
// Load DDL from classpath resources
|
||||
String executionsDdl = new ClassPathResource("clickhouse/V2__executions.sql")
|
||||
.getContentAsString(StandardCharsets.UTF_8);
|
||||
String processorsDdl = new ClassPathResource("clickhouse/V3__processor_executions.sql")
|
||||
.getContentAsString(StandardCharsets.UTF_8);
|
||||
|
||||
jdbc.execute(executionsDdl);
|
||||
jdbc.execute(processorsDdl);
|
||||
|
||||
jdbc.execute("TRUNCATE TABLE executions");
|
||||
jdbc.execute("TRUNCATE TABLE processor_executions");
|
||||
|
||||
store = new ClickHouseExecutionStore(jdbc);
|
||||
}
|
||||
|
||||
@Test
|
||||
void insertExecutionBatch_writesToClickHouse() {
|
||||
MergedExecution exec = new MergedExecution(
|
||||
"default", 1L, "exec-1", "route-a", "agent-1", "my-app",
|
||||
"COMPLETED", "corr-1", "exchange-1",
|
||||
Instant.parse("2026-03-31T10:00:00Z"),
|
||||
Instant.parse("2026-03-31T10:00:01Z"),
|
||||
1000L,
|
||||
"some error", "stack trace", "IOException", "IO",
|
||||
"FileNotFoundException", "file not found",
|
||||
"hash-abc", "FULL",
|
||||
"{\"key\":\"val\"}", "{\"out\":\"val\"}",
|
||||
"{\"h1\":\"v1\"}", "{\"h2\":\"v2\"}",
|
||||
"{\"attr\":\"val\"}",
|
||||
"trace-123", "span-456",
|
||||
true, false
|
||||
);
|
||||
|
||||
store.insertExecutionBatch(List.of(exec));
|
||||
|
||||
Integer count = jdbc.queryForObject(
|
||||
"SELECT count() FROM executions WHERE execution_id = 'exec-1'",
|
||||
Integer.class);
|
||||
assertThat(count).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void insertProcessorBatch_writesToClickHouse() {
|
||||
FlatProcessorRecord proc = new FlatProcessorRecord(
|
||||
1, null, null,
|
||||
"proc-1", "to", null, null,
|
||||
"COMPLETED",
|
||||
Instant.parse("2026-03-31T10:00:00Z"), 50L,
|
||||
"http://example.com",
|
||||
"input body", "output body",
|
||||
Map.of("h1", "v1"), Map.of("h2", "v2"),
|
||||
null, null, null, null, null, null,
|
||||
Map.of("a1", "v1"),
|
||||
null, null, null, null
|
||||
);
|
||||
|
||||
store.insertProcessorBatch(
|
||||
"default", "exec-1", "route-a", "my-app",
|
||||
Instant.parse("2026-03-31T10:00:00Z"),
|
||||
List.of(proc));
|
||||
|
||||
Integer count = jdbc.queryForObject(
|
||||
"SELECT count() FROM processor_executions WHERE execution_id = 'exec-1'",
|
||||
Integer.class);
|
||||
assertThat(count).isEqualTo(1);
|
||||
|
||||
// Verify seq is stored
|
||||
Integer seq = jdbc.queryForObject(
|
||||
"SELECT seq FROM processor_executions WHERE execution_id = 'exec-1'",
|
||||
Integer.class);
|
||||
assertThat(seq).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void insertProcessorBatch_withIterations() {
|
||||
FlatProcessorRecord splitContainer = new FlatProcessorRecord(
|
||||
1, null, null,
|
||||
"split-1", "split", null, 3,
|
||||
"COMPLETED",
|
||||
Instant.parse("2026-03-31T10:00:00Z"), 300L,
|
||||
null, null, null, null, null,
|
||||
null, null, null, null, null, null,
|
||||
null, null, null, null, null
|
||||
);
|
||||
|
||||
FlatProcessorRecord child0 = new FlatProcessorRecord(
|
||||
2, 1, "split-1",
|
||||
"child-proc", "to", 0, null,
|
||||
"COMPLETED",
|
||||
Instant.parse("2026-03-31T10:00:00.100Z"), 80L,
|
||||
"http://svc-a", "body0", "out0",
|
||||
null, null,
|
||||
null, null, null, null, null, null,
|
||||
null, null, null, null, null
|
||||
);
|
||||
|
||||
FlatProcessorRecord child1 = new FlatProcessorRecord(
|
||||
3, 1, "split-1",
|
||||
"child-proc", "to", 1, null,
|
||||
"COMPLETED",
|
||||
Instant.parse("2026-03-31T10:00:00.200Z"), 90L,
|
||||
"http://svc-a", "body1", "out1",
|
||||
null, null,
|
||||
null, null, null, null, null, null,
|
||||
null, null, null, null, null
|
||||
);
|
||||
|
||||
FlatProcessorRecord child2 = new FlatProcessorRecord(
|
||||
4, 1, "split-1",
|
||||
"child-proc", "to", 2, null,
|
||||
"COMPLETED",
|
||||
Instant.parse("2026-03-31T10:00:00.300Z"), 100L,
|
||||
"http://svc-a", "body2", "out2",
|
||||
null, null,
|
||||
null, null, null, null, null, null,
|
||||
null, null, null, null, null
|
||||
);
|
||||
|
||||
store.insertProcessorBatch(
|
||||
"default", "exec-2", "route-b", "my-app",
|
||||
Instant.parse("2026-03-31T10:00:00Z"),
|
||||
List.of(splitContainer, child0, child1, child2));
|
||||
|
||||
Integer count = jdbc.queryForObject(
|
||||
"SELECT count() FROM processor_executions WHERE execution_id = 'exec-2'",
|
||||
Integer.class);
|
||||
assertThat(count).isEqualTo(4);
|
||||
|
||||
// Verify iteration data on the split container
|
||||
Integer iterationSize = jdbc.queryForObject(
|
||||
"SELECT iteration_size FROM processor_executions " +
|
||||
"WHERE execution_id = 'exec-2' AND seq = 1",
|
||||
Integer.class);
|
||||
assertThat(iterationSize).isEqualTo(3);
|
||||
|
||||
// Verify iteration index on a child
|
||||
Integer iteration = jdbc.queryForObject(
|
||||
"SELECT iteration FROM processor_executions " +
|
||||
"WHERE execution_id = 'exec-2' AND seq = 3",
|
||||
Integer.class);
|
||||
assertThat(iteration).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void insertExecutionBatch_emptyList_doesNothing() {
|
||||
store.insertExecutionBatch(List.of());
|
||||
|
||||
Integer count = jdbc.queryForObject(
|
||||
"SELECT count() FROM executions", Integer.class);
|
||||
assertThat(count).isEqualTo(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void insertExecutionBatch_replacingMergeTree_keepsLatestVersion() {
|
||||
MergedExecution v1 = new MergedExecution(
|
||||
"default", 1L, "exec-r", "route-a", "agent-1", "my-app",
|
||||
"RUNNING", "corr-1", "exchange-1",
|
||||
Instant.parse("2026-03-31T10:00:00Z"),
|
||||
null, null,
|
||||
"", "", "", "", "", "",
|
||||
"", "FULL",
|
||||
"", "", "", "", "",
|
||||
"", "",
|
||||
false, false
|
||||
);
|
||||
|
||||
MergedExecution v2 = new MergedExecution(
|
||||
"default", 2L, "exec-r", "route-a", "agent-1", "my-app",
|
||||
"COMPLETED", "corr-1", "exchange-1",
|
||||
Instant.parse("2026-03-31T10:00:00Z"),
|
||||
Instant.parse("2026-03-31T10:00:05Z"),
|
||||
5000L,
|
||||
"", "", "", "", "", "",
|
||||
"", "FULL",
|
||||
"", "", "", "", "",
|
||||
"", "",
|
||||
false, false
|
||||
);
|
||||
|
||||
store.insertExecutionBatch(List.of(v1));
|
||||
store.insertExecutionBatch(List.of(v2));
|
||||
|
||||
// Force merge to apply ReplacingMergeTree deduplication
|
||||
jdbc.execute("OPTIMIZE TABLE executions FINAL");
|
||||
|
||||
String status = jdbc.queryForObject(
|
||||
"SELECT status FROM executions " +
|
||||
"WHERE execution_id = 'exec-r'",
|
||||
String.class);
|
||||
assertThat(status).isEqualTo("COMPLETED");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user