diff --git a/CLAUDE.md b/CLAUDE.md
index 68086069..d890a7f4 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -38,7 +38,8 @@ java -jar cameleer3-server-app/target/cameleer3-server-app-1.0-SNAPSHOT.jar
- Jackson `JavaTimeModule` for `Instant` deserialization
- Communication: receives HTTP POST data from agents (executions, diagrams, metrics, logs), serves SSE event streams for config push/commands (config-update, deep-trace, replay, route-control)
- Maintains agent instance registry (in-memory) with states: LIVE → STALE → DEAD. Auto-heals from JWT claims on heartbeat/SSE after server restart. Route catalog falls back to ClickHouse stats for route discovery when registry has incomplete data.
-- Storage: PostgreSQL for RBAC, config, and audit; ClickHouse for all observability data (executions, search, logs, metrics, stats, diagrams)
+- Storage: PostgreSQL for RBAC, config, and audit; ClickHouse for all observability data (executions, search, logs, metrics, stats, diagrams). ClickHouse schema migrations in `clickhouse/*.sql`, run idempotently on startup by `ClickHouseSchemaInitializer`. Use `IF NOT EXISTS` for CREATE and ADD PROJECTION.
+- Logging: ClickHouse JDBC set to INFO (`com.clickhouse`), HTTP client to WARN (`org.apache.hc.client5`) in application.yml
- Security: JWT auth with RBAC (AGENT/VIEWER/OPERATOR/ADMIN roles), Ed25519 config signing, bootstrap token for registration
- OIDC: Optional external identity provider support (token exchange pattern). Configured via admin API, stored in database (`server_config` table)
- User persistence: PostgreSQL `users` table, admin CRUD at `/api/v1/admin/users`
diff --git a/cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/ClickHouseSchemaInitializer.java b/cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/ClickHouseSchemaInitializer.java
index f8783ee2..20174316 100644
--- a/cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/ClickHouseSchemaInitializer.java
+++ b/cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/ClickHouseSchemaInitializer.java
@@ -11,10 +11,7 @@ import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
-import java.io.IOException;
import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.Comparator;
@Component
@ConditionalOnProperty(name = "clickhouse.enabled", havingValue = "true")
@@ -33,38 +30,24 @@ public class ClickHouseSchemaInitializer {
public void initializeSchema() {
try {
PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
- Resource[] scripts = resolver.getResources("classpath:clickhouse/*.sql");
+ Resource script = resolver.getResource("classpath:clickhouse/init.sql");
- // Sort by numeric version prefix (V1, V2, ..., V10, V11) — not alphabetically
- Arrays.sort(scripts, Comparator.comparingInt(r -> {
- String name = r.getFilename();
- if (name != null && name.startsWith("V")) {
- int end = name.indexOf('_');
- if (end > 1) {
- try { return Integer.parseInt(name.substring(1, end)); } catch (NumberFormatException ignored) {}
- }
- }
- return Integer.MAX_VALUE;
- }));
-
- for (Resource script : scripts) {
- String sql = script.getContentAsString(StandardCharsets.UTF_8);
- log.info("Executing ClickHouse schema script: {}", script.getFilename());
- for (String statement : sql.split(";")) {
- String trimmed = statement.trim();
- // Skip empty segments and comment-only segments
- String withoutComments = trimmed.lines()
- .filter(line -> !line.stripLeading().startsWith("--"))
- .map(String::trim)
- .filter(line -> !line.isEmpty())
- .reduce("", (a, b) -> a + b);
- if (!withoutComments.isEmpty()) {
- clickHouseJdbc.execute(trimmed);
- }
+ String sql = script.getContentAsString(StandardCharsets.UTF_8);
+ log.info("Executing ClickHouse schema: {}", script.getFilename());
+ for (String statement : sql.split(";")) {
+ String trimmed = statement.trim();
+ // Skip empty segments and comment-only segments
+ String withoutComments = trimmed.lines()
+ .filter(line -> !line.stripLeading().startsWith("--"))
+ .map(String::trim)
+ .filter(line -> !line.isEmpty())
+ .reduce("", (a, b) -> a + b);
+ if (!withoutComments.isEmpty()) {
+ clickHouseJdbc.execute(trimmed);
}
}
- log.info("ClickHouse schema initialization complete ({} scripts)", scripts.length);
+ log.info("ClickHouse schema initialization complete");
} catch (Exception e) {
log.error("ClickHouse schema initialization failed — server will continue but ClickHouse features may not work", e);
}
diff --git a/cameleer3-server-app/src/main/java/com/cameleer3/server/app/storage/ClickHouseDiagramStore.java b/cameleer3-server-app/src/main/java/com/cameleer3/server/app/storage/ClickHouseDiagramStore.java
index 2153cc8d..97c47dad 100644
--- a/cameleer3-server-app/src/main/java/com/cameleer3/server/app/storage/ClickHouseDiagramStore.java
+++ b/cameleer3-server-app/src/main/java/com/cameleer3/server/app/storage/ClickHouseDiagramStore.java
@@ -23,6 +23,7 @@ import java.util.HexFormat;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
/**
* ClickHouse implementation of {@link DiagramStore}.
@@ -30,9 +31,9 @@ import java.util.Optional;
* Stores route graphs as JSON with SHA-256 content-hash deduplication.
* Uses ReplacingMergeTree — duplicate inserts are deduplicated on merge.
*
- * {@code findProcessorRouteMapping} fetches all definitions for the application
- * and deserializes them in Java because ClickHouse has no equivalent of
- * PostgreSQL's {@code jsonb_array_elements()}.
+ * In-memory caches avoid expensive ClickHouse scans for the hot lookup paths
+ * (route+instance → hash, hash → graph). Diagrams change rarely (only on
+ * route topology changes), so the caches are effectively append-only.
*/
public class ClickHouseDiagramStore implements DiagramStore {
@@ -66,10 +67,35 @@ public class ClickHouseDiagramStore implements DiagramStore {
private final JdbcTemplate jdbc;
private final ObjectMapper objectMapper;
+ // (routeId + "\0" + instanceId) → contentHash
+ private final ConcurrentHashMap hashCache = new ConcurrentHashMap<>();
+ // contentHash → deserialized RouteGraph
+ private final ConcurrentHashMap graphCache = new ConcurrentHashMap<>();
+
public ClickHouseDiagramStore(JdbcTemplate jdbc) {
this.jdbc = jdbc;
this.objectMapper = new ObjectMapper();
this.objectMapper.registerModule(new JavaTimeModule());
+ warmLoadHashCache();
+ }
+
+ private void warmLoadHashCache() {
+ try {
+ jdbc.query(
+ "SELECT route_id, instance_id, content_hash FROM route_diagrams WHERE tenant_id = ?",
+ rs -> {
+ String key = rs.getString("route_id") + "\0" + rs.getString("instance_id");
+ hashCache.put(key, rs.getString("content_hash"));
+ },
+ TENANT);
+ log.info("Diagram hash cache warmed: {} entries", hashCache.size());
+ } catch (Exception e) {
+ log.warn("Failed to warm diagram hash cache — lookups will fall back to ClickHouse: {}", e.getMessage());
+ }
+ }
+
+ private static String cacheKey(String routeId, String instanceId) {
+ return routeId + "\0" + instanceId;
}
@Override
@@ -90,6 +116,11 @@ public class ClickHouseDiagramStore implements DiagramStore {
applicationId,
json,
Timestamp.from(Instant.now()));
+
+ // Update caches
+ hashCache.put(cacheKey(routeId, agentId), contentHash);
+ graphCache.put(contentHash, graph);
+
log.debug("Stored diagram for route={} agent={} with hash={}", routeId, agentId, contentHash);
} catch (JsonProcessingException e) {
throw new RuntimeException("Failed to serialize RouteGraph to JSON", e);
@@ -98,13 +129,20 @@ public class ClickHouseDiagramStore implements DiagramStore {
@Override
public Optional findByContentHash(String contentHash) {
+ RouteGraph cached = graphCache.get(contentHash);
+ if (cached != null) {
+ return Optional.of(cached);
+ }
+
List