refactor: consolidate ClickHouse schema into single init.sql, cache diagrams

- Merge all V1-V11 migration scripts into one idempotent init.sql
- Simplify ClickHouseSchemaInitializer to load single file
- Replace route_diagrams projection with in-memory caches:
  hashCache (routeId+instanceId → contentHash) warm-loaded on startup,
  graphCache (contentHash → RouteGraph) lazy-populated on access
- Eliminates 9M+ row scans on diagram lookups

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/ClickHouseSchemaInitializer.java

@@ -11,10 +11,7 @@ import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Component;
 
-import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.Comparator;
 
 @Component
 @ConditionalOnProperty(name = "clickhouse.enabled", havingValue = "true")
@@ -33,38 +30,24 @@ public class ClickHouseSchemaInitializer {
     public void initializeSchema() {
         try {
             PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
-            Resource[] scripts = resolver.getResources("classpath:clickhouse/*.sql");
+            Resource script = resolver.getResource("classpath:clickhouse/init.sql");
 
-            // Sort by numeric version prefix (V1, V2, ..., V10, V11) — not alphabetically
-            Arrays.sort(scripts, Comparator.comparingInt(r -> {
-                String name = r.getFilename();
-                if (name != null && name.startsWith("V")) {
-                    int end = name.indexOf('_');
-                    if (end > 1) {
-                        try { return Integer.parseInt(name.substring(1, end)); } catch (NumberFormatException ignored) {}
-                    }
-                }
-                return Integer.MAX_VALUE;
-            }));
-
-            for (Resource script : scripts) {
-                String sql = script.getContentAsString(StandardCharsets.UTF_8);
-                log.info("Executing ClickHouse schema script: {}", script.getFilename());
-                for (String statement : sql.split(";")) {
-                    String trimmed = statement.trim();
-                    // Skip empty segments and comment-only segments
-                    String withoutComments = trimmed.lines()
-                            .filter(line -> !line.stripLeading().startsWith("--"))
-                            .map(String::trim)
-                            .filter(line -> !line.isEmpty())
-                            .reduce("", (a, b) -> a + b);
-                    if (!withoutComments.isEmpty()) {
-                        clickHouseJdbc.execute(trimmed);
-                    }
+            String sql = script.getContentAsString(StandardCharsets.UTF_8);
+            log.info("Executing ClickHouse schema: {}", script.getFilename());
+            for (String statement : sql.split(";")) {
+                String trimmed = statement.trim();
+                // Skip empty segments and comment-only segments
+                String withoutComments = trimmed.lines()
+                        .filter(line -> !line.stripLeading().startsWith("--"))
+                        .map(String::trim)
+                        .filter(line -> !line.isEmpty())
+                        .reduce("", (a, b) -> a + b);
+                if (!withoutComments.isEmpty()) {
+                    clickHouseJdbc.execute(trimmed);
                 }
             }
 
-            log.info("ClickHouse schema initialization complete ({} scripts)", scripts.length);
+            log.info("ClickHouse schema initialization complete");
         } catch (Exception e) {
             log.error("ClickHouse schema initialization failed — server will continue but ClickHouse features may not work", e);
         }

cameleer3-server-app/src/main/java/com/cameleer3/server/app/storage/ClickHouseDiagramStore.java

@@ -23,6 +23,7 @@ import java.util.HexFormat;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
 
 /**
  * ClickHouse implementation of {@link DiagramStore}.
@@ -30,9 +31,9 @@ import java.util.Optional;
  * Stores route graphs as JSON with SHA-256 content-hash deduplication.
  * Uses ReplacingMergeTree — duplicate inserts are deduplicated on merge.
  * <p>
- * {@code findProcessorRouteMapping} fetches all definitions for the application
- * and deserializes them in Java because ClickHouse has no equivalent of
- * PostgreSQL's {@code jsonb_array_elements()}.
+ * In-memory caches avoid expensive ClickHouse scans for the hot lookup paths
+ * (route+instance → hash, hash → graph). Diagrams change rarely (only on
+ * route topology changes), so the caches are effectively append-only.
  */
 public class ClickHouseDiagramStore implements DiagramStore {
 
@@ -66,10 +67,35 @@ public class ClickHouseDiagramStore implements DiagramStore {
     private final JdbcTemplate jdbc;
     private final ObjectMapper objectMapper;
 
+    // (routeId + "\0" + instanceId) → contentHash
+    private final ConcurrentHashMap<String, String> hashCache = new ConcurrentHashMap<>();
+    // contentHash → deserialized RouteGraph
+    private final ConcurrentHashMap<String, RouteGraph> graphCache = new ConcurrentHashMap<>();
+
     public ClickHouseDiagramStore(JdbcTemplate jdbc) {
         this.jdbc = jdbc;
         this.objectMapper = new ObjectMapper();
         this.objectMapper.registerModule(new JavaTimeModule());
+        warmLoadHashCache();
+    }
+
+    private void warmLoadHashCache() {
+        try {
+            jdbc.query(
+                    "SELECT route_id, instance_id, content_hash FROM route_diagrams WHERE tenant_id = ?",
+                    rs -> {
+                        String key = rs.getString("route_id") + "\0" + rs.getString("instance_id");
+                        hashCache.put(key, rs.getString("content_hash"));
+                    },
+                    TENANT);
+            log.info("Diagram hash cache warmed: {} entries", hashCache.size());
+        } catch (Exception e) {
+            log.warn("Failed to warm diagram hash cache — lookups will fall back to ClickHouse: {}", e.getMessage());
+        }
+    }
+
+    private static String cacheKey(String routeId, String instanceId) {
+        return routeId + "\0" + instanceId;
     }
 
     @Override
@@ -90,6 +116,11 @@ public class ClickHouseDiagramStore implements DiagramStore {
                     applicationId,
                     json,
                     Timestamp.from(Instant.now()));
+
+            // Update caches
+            hashCache.put(cacheKey(routeId, agentId), contentHash);
+            graphCache.put(contentHash, graph);
+
             log.debug("Stored diagram for route={} agent={} with hash={}", routeId, agentId, contentHash);
         } catch (JsonProcessingException e) {
             throw new RuntimeException("Failed to serialize RouteGraph to JSON", e);
@@ -98,13 +129,20 @@ public class ClickHouseDiagramStore implements DiagramStore {
 
     @Override
     public Optional<RouteGraph> findByContentHash(String contentHash) {
+        RouteGraph cached = graphCache.get(contentHash);
+        if (cached != null) {
+            return Optional.of(cached);
+        }
+
         List<Map<String, Object>> rows = jdbc.queryForList(SELECT_BY_HASH, TENANT, contentHash);
         if (rows.isEmpty()) {
             return Optional.empty();
         }
         String json = (String) rows.get(0).get("definition");
         try {
-            return Optional.of(objectMapper.readValue(json, RouteGraph.class));
+            RouteGraph graph = objectMapper.readValue(json, RouteGraph.class);
+            graphCache.put(contentHash, graph);
+            return Optional.of(graph);
         } catch (JsonProcessingException e) {
             log.error("Failed to deserialize RouteGraph from ClickHouse", e);
             return Optional.empty();
@@ -113,12 +151,19 @@ public class ClickHouseDiagramStore implements DiagramStore {
 
     @Override
     public Optional<String> findContentHashForRoute(String routeId, String agentId) {
+        String cached = hashCache.get(cacheKey(routeId, agentId));
+        if (cached != null) {
+            return Optional.of(cached);
+        }
+
         List<Map<String, Object>> rows = jdbc.queryForList(
                 SELECT_HASH_FOR_ROUTE, TENANT, routeId, agentId);
         if (rows.isEmpty()) {
             return Optional.empty();
         }
-        return Optional.of((String) rows.get(0).get("content_hash"));
+        String hash = (String) rows.get(0).get("content_hash");
+        hashCache.put(cacheKey(routeId, agentId), hash);
+        return Optional.of(hash);
     }
 
     @Override
@@ -126,6 +171,16 @@ public class ClickHouseDiagramStore implements DiagramStore {
         if (agentIds == null || agentIds.isEmpty()) {
             return Optional.empty();
         }
+
+        // Try cache first — return first hit
+        for (String agentId : agentIds) {
+            String cached = hashCache.get(cacheKey(routeId, agentId));
+            if (cached != null) {
+                return Optional.of(cached);
+            }
+        }
+
+        // Fall back to ClickHouse
         String placeholders = String.join(", ", Collections.nCopies(agentIds.size(), "?"));
         String sql = "SELECT content_hash FROM route_diagrams " +
                 "WHERE tenant_id = ? AND route_id = ? AND instance_id IN (" + placeholders + ") " +
@@ -162,9 +217,6 @@ public class ClickHouseDiagramStore implements DiagramStore {
         return mapping;
     }
 
-    /**
-     * Recursively walks the RouteNode tree and maps each node ID to the given routeId.
-     */
     private void collectNodeIds(RouteNode node, String routeId, Map<String, String> mapping) {
         if (node == null) {
             return;