feat: remove TimescaleDB, dead PG stores, and storage feature flags
Complete the ClickHouse migration by removing all PostgreSQL analytics code. PostgreSQL now serves only RBAC, config, and audit — all observability data is exclusively in ClickHouse. - Delete 6 dead PostgreSQL store classes (executions, stats, diagrams, events, metrics, metrics-query) and 2 integration tests - Delete RetentionScheduler (ClickHouse TTL handles retention) - Remove all 7 cameleer.storage.* feature flags from application.yml - Remove all @ConditionalOnProperty from ClickHouse beans in StorageBeanConfig - Consolidate 14 Flyway migrations (V1-V14) into single clean V1 with only RBAC/config/audit tables (no TimescaleDB, no analytics tables) - Switch from timescale/timescaledb-ha:pg16 to postgres:16 everywhere (docker-compose, deploy/postgres.yaml, test containers) - Remove TimescaleDB check and /metrics-pipeline from DatabaseAdminController - Set clickhouse.enabled default to true Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -8,21 +8,17 @@ import org.springframework.test.context.DynamicPropertyRegistry;
|
||||
import org.springframework.test.context.DynamicPropertySource;
|
||||
import org.testcontainers.clickhouse.ClickHouseContainer;
|
||||
import org.testcontainers.containers.PostgreSQLContainer;
|
||||
import org.testcontainers.utility.DockerImageName;
|
||||
|
||||
|
||||
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
@ActiveProfiles("test")
|
||||
public abstract class AbstractPostgresIT {
|
||||
|
||||
private static final DockerImageName TIMESCALEDB_IMAGE =
|
||||
DockerImageName.parse("timescale/timescaledb-ha:pg16")
|
||||
.asCompatibleSubstituteFor("postgres");
|
||||
|
||||
static final PostgreSQLContainer<?> postgres;
|
||||
static final ClickHouseContainer clickhouse;
|
||||
|
||||
static {
|
||||
postgres = new PostgreSQLContainer<>(TIMESCALEDB_IMAGE)
|
||||
postgres = new PostgreSQLContainer<>("postgres:16")
|
||||
.withDatabaseName("cameleer3")
|
||||
.withUsername("cameleer")
|
||||
.withPassword("test");
|
||||
|
||||
@@ -14,23 +14,34 @@ class FlywayMigrationIT extends AbstractPostgresIT {
|
||||
|
||||
@Test
|
||||
void allMigrationsApplySuccessfully() {
|
||||
// Verify core tables exist
|
||||
Integer execCount = jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM executions", Integer.class);
|
||||
assertEquals(0, execCount);
|
||||
|
||||
Integer procCount = jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM processor_executions", Integer.class);
|
||||
assertEquals(0, procCount);
|
||||
|
||||
// Verify RBAC tables exist
|
||||
Integer userCount = jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM users", Integer.class);
|
||||
assertEquals(0, userCount);
|
||||
|
||||
// Verify continuous aggregates exist
|
||||
Integer caggCount = jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM timescaledb_information.continuous_aggregates",
|
||||
Integer.class);
|
||||
assertEquals(4, caggCount);
|
||||
Integer roleCount = jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM roles", Integer.class);
|
||||
assertEquals(4, roleCount); // AGENT, VIEWER, OPERATOR, ADMIN
|
||||
|
||||
Integer groupCount = jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM groups", Integer.class);
|
||||
assertEquals(1, groupCount); // Admins
|
||||
|
||||
// Verify config/audit tables exist
|
||||
Integer configCount = jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM server_config", Integer.class);
|
||||
assertEquals(0, configCount);
|
||||
|
||||
Integer auditCount = jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM audit_log", Integer.class);
|
||||
assertEquals(0, auditCount);
|
||||
|
||||
Integer appConfigCount = jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM application_config", Integer.class);
|
||||
assertEquals(0, appConfigCount);
|
||||
|
||||
Integer appSettingsCount = jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM app_settings", Integer.class);
|
||||
assertEquals(0, appSettingsCount);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
package com.cameleer3.server.app.storage;
|
||||
|
||||
import com.cameleer3.server.app.AbstractPostgresIT;
|
||||
import com.cameleer3.server.core.storage.ExecutionStore;
|
||||
import com.cameleer3.server.core.storage.ExecutionStore.ExecutionRecord;
|
||||
import com.cameleer3.server.core.storage.ExecutionStore.ProcessorRecord;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class PostgresExecutionStoreIT extends AbstractPostgresIT {
|
||||
|
||||
@Autowired
|
||||
ExecutionStore executionStore;
|
||||
|
||||
@Test
|
||||
void upsertAndFindById() {
|
||||
Instant now = Instant.now();
|
||||
ExecutionRecord record = new ExecutionRecord(
|
||||
"exec-1", "route-a", "agent-1", "app-1",
|
||||
"COMPLETED", "corr-1", "exchange-1",
|
||||
now, now.plusMillis(100), 100L,
|
||||
null, null, null,
|
||||
"REGULAR", null, null, null, null, null,
|
||||
null, null, null, null, null, null, null, false, false);
|
||||
|
||||
executionStore.upsert(record);
|
||||
Optional<ExecutionRecord> found = executionStore.findById("exec-1");
|
||||
|
||||
assertTrue(found.isPresent());
|
||||
assertEquals("exec-1", found.get().executionId());
|
||||
assertEquals("COMPLETED", found.get().status());
|
||||
assertEquals("REGULAR", found.get().engineLevel());
|
||||
}
|
||||
|
||||
@Test
|
||||
void upsertDeduplicatesByExecutionId() {
|
||||
Instant now = Instant.now();
|
||||
ExecutionRecord first = new ExecutionRecord(
|
||||
"exec-dup", "route-a", "agent-1", "app-1",
|
||||
"RUNNING", null, null, now, null, null, null, null, null,
|
||||
null, null, null, null, null, null,
|
||||
null, null, null, null, null, null, null, false, false);
|
||||
ExecutionRecord second = new ExecutionRecord(
|
||||
"exec-dup", "route-a", "agent-1", "app-1",
|
||||
"COMPLETED", null, null, now, now.plusMillis(200), 200L, null, null, null,
|
||||
"COMPLETE", null, null, null, null, null,
|
||||
null, null, null, null, null, null, null, false, false);
|
||||
|
||||
executionStore.upsert(first);
|
||||
executionStore.upsert(second);
|
||||
|
||||
Optional<ExecutionRecord> found = executionStore.findById("exec-dup");
|
||||
assertTrue(found.isPresent());
|
||||
assertEquals("COMPLETED", found.get().status());
|
||||
assertEquals(200L, found.get().durationMs());
|
||||
}
|
||||
|
||||
@Test
|
||||
void upsertProcessorsAndFind() {
|
||||
Instant now = Instant.now();
|
||||
ExecutionRecord exec = new ExecutionRecord(
|
||||
"exec-proc", "route-a", "agent-1", "app-1",
|
||||
"COMPLETED", null, null, now, now.plusMillis(50), 50L, null, null, null,
|
||||
"COMPLETE", null, null, null, null, null,
|
||||
null, null, null, null, null, null, null, false, false);
|
||||
executionStore.upsert(exec);
|
||||
|
||||
List<ProcessorRecord> processors = List.of(
|
||||
new ProcessorRecord("exec-proc", "proc-1", "log",
|
||||
"app-1", "route-a", 0, null, "COMPLETED",
|
||||
now, now.plusMillis(10), 10L, null, null,
|
||||
"input body", "output body", null, null, null,
|
||||
null, null, null, null, null,
|
||||
null, null, null, null, null, null, null, null,
|
||||
null, null, null, null, null, null),
|
||||
new ProcessorRecord("exec-proc", "proc-2", "to",
|
||||
"app-1", "route-a", 1, "proc-1", "COMPLETED",
|
||||
now.plusMillis(10), now.plusMillis(30), 20L, null, null,
|
||||
null, null, null, null, null,
|
||||
null, null, null, null, null,
|
||||
null, null, null, null, null, null, null, null,
|
||||
null, null, null, null, null, null)
|
||||
);
|
||||
executionStore.upsertProcessors("exec-proc", now, "app-1", "route-a", processors);
|
||||
|
||||
List<ProcessorRecord> found = executionStore.findProcessors("exec-proc");
|
||||
assertEquals(2, found.size());
|
||||
assertEquals("proc-1", found.get(0).processorId());
|
||||
assertEquals("proc-2", found.get(1).processorId());
|
||||
assertEquals("proc-1", found.get(1).parentProcessorId());
|
||||
}
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
package com.cameleer3.server.app.storage;
|
||||
|
||||
import com.cameleer3.server.app.AbstractPostgresIT;
|
||||
import com.cameleer3.server.core.search.ExecutionStats;
|
||||
import com.cameleer3.server.core.search.StatsTimeseries;
|
||||
import com.cameleer3.server.core.storage.ExecutionStore;
|
||||
import com.cameleer3.server.core.storage.ExecutionStore.ExecutionRecord;
|
||||
import com.cameleer3.server.core.storage.StatsStore;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class PostgresStatsStoreIT extends AbstractPostgresIT {
|
||||
|
||||
@Autowired StatsStore statsStore;
|
||||
@Autowired ExecutionStore executionStore;
|
||||
@Autowired JdbcTemplate jdbc;
|
||||
|
||||
@Test
|
||||
void statsReturnsCountsForTimeWindow() {
|
||||
// Use a unique route + statsForRoute to avoid data contamination from other tests
|
||||
String uniqueRoute = "stats-route-" + System.nanoTime();
|
||||
Instant base = Instant.now().minus(5, ChronoUnit.MINUTES).truncatedTo(ChronoUnit.SECONDS);
|
||||
insertExecution("stats-1-" + uniqueRoute, uniqueRoute, "app-stats", "COMPLETED", base, 100L);
|
||||
insertExecution("stats-2-" + uniqueRoute, uniqueRoute, "app-stats", "FAILED", base.plusSeconds(10), 200L);
|
||||
insertExecution("stats-3-" + uniqueRoute, uniqueRoute, "app-stats", "COMPLETED", base.plusSeconds(20), 50L);
|
||||
|
||||
// Force continuous aggregate refresh
|
||||
jdbc.execute("CALL refresh_continuous_aggregate('stats_1m_route', NOW() - INTERVAL '1 hour', NOW() + INTERVAL '1 hour')");
|
||||
|
||||
ExecutionStats stats = statsStore.statsForRoute(base.minusSeconds(60), base.plusSeconds(60), uniqueRoute, null);
|
||||
assertEquals(3, stats.totalCount());
|
||||
assertEquals(1, stats.failedCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void timeseriesReturnsBuckets() {
|
||||
String uniqueRoute = "ts-route-" + System.nanoTime();
|
||||
Instant base = Instant.now().minus(10, ChronoUnit.MINUTES).truncatedTo(ChronoUnit.MINUTES);
|
||||
for (int i = 0; i < 10; i++) {
|
||||
insertExecution("ts-" + i + "-" + uniqueRoute, uniqueRoute, "app-ts", "COMPLETED",
|
||||
base.plusSeconds(i * 30), 100L + i);
|
||||
}
|
||||
|
||||
jdbc.execute("CALL refresh_continuous_aggregate('stats_1m_route', NOW() - INTERVAL '1 hour', NOW() + INTERVAL '1 hour')");
|
||||
|
||||
StatsTimeseries ts = statsStore.timeseriesForRoute(base.minus(1, ChronoUnit.MINUTES), base.plus(10, ChronoUnit.MINUTES), 5, uniqueRoute, null);
|
||||
assertNotNull(ts);
|
||||
assertFalse(ts.buckets().isEmpty());
|
||||
}
|
||||
|
||||
private void insertExecution(String id, String routeId, String applicationId,
|
||||
String status, Instant startTime, long durationMs) {
|
||||
executionStore.upsert(new ExecutionRecord(
|
||||
id, routeId, "agent-1", applicationId, status, null, null,
|
||||
startTime, startTime.plusMillis(durationMs), durationMs,
|
||||
status.equals("FAILED") ? "error" : null, null, null,
|
||||
null, null, null, null, null, null,
|
||||
null, null, null, null, null, null, null, false, false));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user