From e808b567cdc00983b328368de49221d3f1aba719 Mon Sep 17 00:00:00 2001 From: hsiegeln <37154749+hsiegeln@users.noreply.github.com> Date: Wed, 11 Mar 2026 11:33:32 +0100 Subject: [PATCH] docs(01): create phase plan (3 plans, 2 waves) Co-Authored-By: Claude Opus 4.6 --- .planning/ROADMAP.md | 9 +- .../01-01-PLAN.md | 236 +++++++++++++++ .../01-02-PLAN.md | 270 ++++++++++++++++++ .../01-03-PLAN.md | 188 ++++++++++++ 4 files changed, 699 insertions(+), 4 deletions(-) create mode 100644 .planning/phases/01-ingestion-pipeline-api-foundation/01-01-PLAN.md create mode 100644 .planning/phases/01-ingestion-pipeline-api-foundation/01-02-PLAN.md create mode 100644 .planning/phases/01-ingestion-pipeline-api-foundation/01-03-PLAN.md diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index 6c9b5af8..c4e2fd9a 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -29,11 +29,12 @@ Decimal phases appear between their surrounding integers in numeric order. 3. When the write buffer is full, the server returns 503 and does not lose already-buffered data 4. Data older than the configured TTL (default 30 days) is automatically removed by ClickHouse 5. The health endpoint responds at `/api/v1/health`, OpenAPI docs are available, protocol version header is validated, and unknown JSON fields are accepted -**Plans**: TBD +**Plans:** 3 plans Plans: -- [ ] 01-01: ClickHouse schema design + Docker setup + batch write buffer -- [ ] 01-02: Ingestion REST endpoints + API foundation (health, OpenAPI, protocol header, forward compat) +- [ ] 01-01-PLAN.md -- ClickHouse infrastructure, schema, WriteBuffer, repository interfaces, test infrastructure +- [ ] 01-02-PLAN.md -- Ingestion REST endpoints, ClickHouse repositories, flush scheduler, integration tests +- [ ] 01-03-PLAN.md -- API foundation (health, OpenAPI, protocol header, forward compat, TTL verification) ### Phase 2: Transaction Search + Diagrams **Goal**: Users can find any transaction by status, time, duration, correlation ID, or content, view execution detail trees, and see versioned route diagrams linked to transactions @@ -86,7 +87,7 @@ Note: Phases 2 and 3 both depend only on Phase 1 and could execute in parallel. | Phase | Plans Complete | Status | Completed | |-------|----------------|--------|-----------| -| 1. Ingestion Pipeline + API Foundation | 0/2 | Not started | - | +| 1. Ingestion Pipeline + API Foundation | 0/3 | Not started | - | | 2. Transaction Search + Diagrams | 0/2 | Not started | - | | 3. Agent Registry + SSE Push | 0/2 | Not started | - | | 4. Security | 0/1 | Not started | - | diff --git a/.planning/phases/01-ingestion-pipeline-api-foundation/01-01-PLAN.md b/.planning/phases/01-ingestion-pipeline-api-foundation/01-01-PLAN.md new file mode 100644 index 00000000..43dc0e2c --- /dev/null +++ b/.planning/phases/01-ingestion-pipeline-api-foundation/01-01-PLAN.md @@ -0,0 +1,236 @@ +--- +phase: 01-ingestion-pipeline-api-foundation +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: + - pom.xml + - cameleer3-server-core/pom.xml + - cameleer3-server-app/pom.xml + - docker-compose.yml + - clickhouse/init/01-schema.sql + - cameleer3-server-app/src/main/resources/application.yml + - cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/ClickHouseConfig.java + - cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/IngestionConfig.java + - cameleer3-server-core/src/main/java/com/cameleer3/server/core/ingestion/WriteBuffer.java + - cameleer3-server-core/src/main/java/com/cameleer3/server/core/storage/ExecutionRepository.java + - cameleer3-server-core/src/main/java/com/cameleer3/server/core/storage/DiagramRepository.java + - cameleer3-server-core/src/main/java/com/cameleer3/server/core/storage/MetricsRepository.java + - cameleer3-server-core/src/test/java/com/cameleer3/server/core/ingestion/WriteBufferTest.java + - cameleer3-server-app/src/test/resources/application-test.yml + - cameleer3-server-app/src/test/java/com/cameleer3/server/app/AbstractClickHouseIT.java +autonomous: true +requirements: + - INGST-04 + - INGST-05 + - INGST-06 + +must_haves: + truths: + - "WriteBuffer accepts items and returns false when full (backpressure signal)" + - "WriteBuffer drains items in batches for scheduled flush" + - "ClickHouse schema creates route_executions, route_diagrams, and agent_metrics tables with correct column types" + - "TTL clause on tables removes data older than configured days" + - "Docker Compose starts ClickHouse and initializes the schema" + artifacts: + - path: "cameleer3-server-core/src/main/java/com/cameleer3/server/core/ingestion/WriteBuffer.java" + provides: "Generic bounded write buffer with offer/drain/isFull" + min_lines: 30 + - path: "clickhouse/init/01-schema.sql" + provides: "ClickHouse DDL for all three tables" + contains: "CREATE TABLE route_executions" + - path: "docker-compose.yml" + provides: "Local ClickHouse service" + contains: "clickhouse-server" + - path: "cameleer3-server-core/src/main/java/com/cameleer3/server/core/storage/ExecutionRepository.java" + provides: "Repository interface for execution batch inserts" + exports: ["insertBatch"] + - path: "cameleer3-server-app/src/test/java/com/cameleer3/server/app/AbstractClickHouseIT.java" + provides: "Shared Testcontainers base class for integration tests" + min_lines: 20 + key_links: + - from: "cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/ClickHouseConfig.java" + to: "application.yml" + via: "spring.datasource properties" + pattern: "spring\\.datasource" + - from: "cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/IngestionConfig.java" + to: "application.yml" + via: "ingestion.* properties" + pattern: "ingestion\\." +--- + + +Set up ClickHouse infrastructure, schema, WriteBuffer with backpressure, repository interfaces, and test infrastructure. + +Purpose: Establishes the storage foundation that all ingestion endpoints and future search queries depend on. The WriteBuffer is the central throughput mechanism -- all data flows through it before reaching ClickHouse. +Output: Working ClickHouse via Docker Compose, DDL with TTL, WriteBuffer with unit tests, repository interfaces, Testcontainers base class for integration tests. + + + +@C:/Users/Hendrik/.claude/get-shit-done/workflows/execute-plan.md +@C:/Users/Hendrik/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/01-ingestion-pipeline-api-foundation/01-RESEARCH.md + +@pom.xml +@cameleer3-server-core/pom.xml +@cameleer3-server-app/pom.xml + + + + + + Task 1: Dependencies, Docker Compose, ClickHouse schema, and application config + + pom.xml, + cameleer3-server-core/pom.xml, + cameleer3-server-app/pom.xml, + docker-compose.yml, + clickhouse/init/01-schema.sql, + cameleer3-server-app/src/main/resources/application.yml + + + - docker compose up -d starts ClickHouse on ports 8123/9000 + - Connecting to ClickHouse and running SELECT 1 succeeds + - Tables route_executions, route_diagrams, agent_metrics exist after init + - route_executions has TTL clause with configurable interval + - route_executions has PARTITION BY toYYYYMMDD(start_time) and ORDER BY (agent_id, status, start_time, execution_id) + - route_diagrams uses ReplacingMergeTree with ORDER BY (content_hash) + - agent_metrics has TTL and daily partitioning + - Maven compile succeeds with new dependencies + + + 1. Add dependencies to cameleer3-server-app/pom.xml per research: + - clickhouse-jdbc 0.9.7 (classifier: all) + - spring-boot-starter-actuator + - springdoc-openapi-starter-webmvc-ui 2.8.6 + - testcontainers-clickhouse 2.0.2 (test scope) + - junit-jupiter from testcontainers 2.0.2 (test scope) + - awaitility (test scope) + + 2. Add slf4j-api dependency to cameleer3-server-core/pom.xml. + + 3. Create docker-compose.yml at project root with ClickHouse service: + - Image: clickhouse/clickhouse-server:25.3 + - Ports: 8123:8123, 9000:9000 + - Volume mount ./clickhouse/init to /docker-entrypoint-initdb.d + - Environment: CLICKHOUSE_USER=cameleer, CLICKHOUSE_PASSWORD=cameleer_dev, CLICKHOUSE_DB=cameleer3 + - ulimits nofile 262144 + + 4. Create clickhouse/init/01-schema.sql with the three tables from research: + - route_executions: MergeTree, daily partitioning on start_time, ORDER BY (agent_id, status, start_time, execution_id), TTL start_time + INTERVAL 30 DAY, SETTINGS ttl_only_drop_parts=1. Include Array columns for processor executions (processor_ids, processor_types, processor_starts, processor_ends, processor_durations, processor_statuses). Include skip indexes for correlation_id (bloom_filter) and error_message (tokenbf_v1). + - route_diagrams: ReplacingMergeTree(created_at), ORDER BY (content_hash). No TTL. + - agent_metrics: MergeTree, daily partitioning on collected_at, ORDER BY (agent_id, metric_name, collected_at), TTL collected_at + INTERVAL 30 DAY, SETTINGS ttl_only_drop_parts=1. + - All DateTime fields use DateTime64(3, 'UTC'). + + 5. Create cameleer3-server-app/src/main/resources/application.yml with config from research: + - server.port: 8081 + - spring.datasource: url=jdbc:ch://localhost:8123/cameleer3, username/password, driver-class-name + - spring.jackson: write-dates-as-timestamps=false, fail-on-unknown-properties=false + - ingestion: buffer-capacity=50000, batch-size=5000, flush-interval-ms=1000 + - clickhouse.ttl-days: 30 + - springdoc paths under /api/v1/ + - management endpoints (health under /api/v1/, show-details=always) + + 6. Ensure .gitattributes exists with `* text=auto eol=lf`. + + + mvn clean compile -q 2>&1 | tail -5 + + Maven compiles successfully with all new dependencies. Docker Compose file and ClickHouse DDL exist. application.yml configures datasource, ingestion buffer, and springdoc. + + + + Task 2: WriteBuffer, repository interfaces, IngestionConfig, ClickHouseConfig, and test infrastructure + + cameleer3-server-core/src/main/java/com/cameleer3/server/core/ingestion/WriteBuffer.java, + cameleer3-server-core/src/main/java/com/cameleer3/server/core/storage/ExecutionRepository.java, + cameleer3-server-core/src/main/java/com/cameleer3/server/core/storage/DiagramRepository.java, + cameleer3-server-core/src/main/java/com/cameleer3/server/core/storage/MetricsRepository.java, + cameleer3-server-core/src/test/java/com/cameleer3/server/core/ingestion/WriteBufferTest.java, + cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/ClickHouseConfig.java, + cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/IngestionConfig.java, + cameleer3-server-app/src/test/resources/application-test.yml, + cameleer3-server-app/src/test/java/com/cameleer3/server/app/AbstractClickHouseIT.java + + + - WriteBuffer(capacity=10): offer() returns true for first 10 items, false on 11th + - WriteBuffer.drain(5) returns up to 5 items and removes them from the queue + - WriteBuffer.isFull() returns true when at capacity + - WriteBuffer.offerBatch(list) returns false without partial insert if buffer would overflow + - WriteBuffer.size() tracks current queue depth + - ExecutionRepository interface declares insertBatch(List of RouteExecution) + - DiagramRepository interface declares store(RouteGraph) and findByContentHash(String) + - MetricsRepository interface declares insertBatch(List of metric data) + - AbstractClickHouseIT starts a Testcontainers ClickHouse, applies schema, provides JdbcTemplate + + + 1. Create WriteBuffer in core module (no Spring dependency): + - Constructor takes int capacity, creates ArrayBlockingQueue(capacity) + - offer(T item): returns queue.offer(item) -- false when full + - offerBatch(List items): check remainingCapacity() >= items.size() first, then offer each. If insufficient capacity, return false immediately without adding any items. + - drain(int maxBatch): drainTo into ArrayList, return list + - size(), capacity(), isFull(), remainingCapacity() accessors + + 2. Create WriteBufferTest (JUnit 5, no Spring): + - Test offer succeeds until capacity + - Test offer returns false when full + - Test offerBatch all-or-nothing semantics + - Test drain returns items and removes from queue + - Test drain with empty queue returns empty list + - Test isFull/size/remainingCapacity + + 3. Create repository interfaces in core module: + - ExecutionRepository: void insertBatch(List executions) + - DiagramRepository: void store(RouteGraph graph), Optional findByContentHash(String hash), Optional findContentHashForRoute(String routeId, String agentId) + - MetricsRepository: void insertBatch(List metrics) -- use a generic type or the cameleer3-common metrics model if available; if not, create a simple MetricsData record in core module + + 4. Create IngestionConfig as @ConfigurationProperties("ingestion"): + - bufferCapacity (int, default 50000) + - batchSize (int, default 5000) + - flushIntervalMs (long, default 1000) + + 5. Create ClickHouseConfig as @Configuration: + - Exposes JdbcTemplate bean (Spring Boot auto-configures DataSource from spring.datasource) + - No custom bean needed if relying on auto-config; only create if explicit JdbcTemplate customization required + + 6. Create application-test.yml for test profile: + - Placeholder datasource config (overridden by Testcontainers in AbstractClickHouseIT) + - ingestion: small buffer for tests (capacity=100, batch-size=10, flush-interval-ms=100) + + 7. Create AbstractClickHouseIT base class: + - @Testcontainers + @Container with ClickHouseContainer("clickhouse/clickhouse-server:25.3") + - @DynamicPropertySource to override spring.datasource.url/username/password + - @SpringBootTest + - @ActiveProfiles("test") + - @BeforeAll: read clickhouse/init/01-schema.sql and execute it against the container via JDBC + - Expose protected JdbcTemplate for subclasses + + + mvn test -pl cameleer3-server-core -Dtest=WriteBufferTest -q 2>&1 | tail -10 + + WriteBuffer passes all unit tests. Repository interfaces exist with correct method signatures. IngestionConfig reads from application.yml. AbstractClickHouseIT base class exists and can start a Testcontainers ClickHouse instance with schema applied. + + + + + +- `mvn test -pl cameleer3-server-core -q` -- all WriteBuffer unit tests pass +- `mvn clean compile -q` -- full project compiles with new dependencies +- `docker compose config` -- validates Docker Compose file +- clickhouse/init/01-schema.sql contains CREATE TABLE for all three tables with correct ENGINE, ORDER BY, PARTITION BY, and TTL + + + +WriteBuffer unit tests green. Project compiles. ClickHouse DDL defines all three tables with TTL and correct partitioning. Repository interfaces define batch insert contracts. Testcontainers base class ready for integration tests in Plan 02. + + + +After completion, create `.planning/phases/01-ingestion-pipeline-api-foundation/01-01-SUMMARY.md` + diff --git a/.planning/phases/01-ingestion-pipeline-api-foundation/01-02-PLAN.md b/.planning/phases/01-ingestion-pipeline-api-foundation/01-02-PLAN.md new file mode 100644 index 00000000..59c7a00a --- /dev/null +++ b/.planning/phases/01-ingestion-pipeline-api-foundation/01-02-PLAN.md @@ -0,0 +1,270 @@ +--- +phase: 01-ingestion-pipeline-api-foundation +plan: 02 +type: execute +wave: 2 +depends_on: ["01-01"] +files_modified: + - cameleer3-server-app/src/main/java/com/cameleer3/server/app/controller/ExecutionController.java + - cameleer3-server-app/src/main/java/com/cameleer3/server/app/controller/DiagramController.java + - cameleer3-server-app/src/main/java/com/cameleer3/server/app/controller/MetricsController.java + - cameleer3-server-app/src/main/java/com/cameleer3/server/app/storage/ClickHouseExecutionRepository.java + - cameleer3-server-app/src/main/java/com/cameleer3/server/app/storage/ClickHouseDiagramRepository.java + - cameleer3-server-app/src/main/java/com/cameleer3/server/app/storage/ClickHouseMetricsRepository.java + - cameleer3-server-app/src/main/java/com/cameleer3/server/app/ingestion/ClickHouseFlushScheduler.java + - cameleer3-server-core/src/main/java/com/cameleer3/server/core/ingestion/IngestionService.java + - cameleer3-server-app/src/test/java/com/cameleer3/server/app/controller/ExecutionControllerIT.java + - cameleer3-server-app/src/test/java/com/cameleer3/server/app/controller/DiagramControllerIT.java + - cameleer3-server-app/src/test/java/com/cameleer3/server/app/controller/MetricsControllerIT.java + - cameleer3-server-app/src/test/java/com/cameleer3/server/app/controller/BackpressureIT.java +autonomous: true +requirements: + - INGST-01 + - INGST-02 + - INGST-03 + - INGST-05 + +must_haves: + truths: + - "POST /api/v1/data/executions with valid RouteExecution payload returns 202 Accepted" + - "POST /api/v1/data/diagrams with valid RouteGraph payload returns 202 Accepted" + - "POST /api/v1/data/metrics with valid metrics payload returns 202 Accepted" + - "Data posted to endpoints appears in ClickHouse after flush interval" + - "When buffer is full, endpoints return 503 with Retry-After header" + artifacts: + - path: "cameleer3-server-app/src/main/java/com/cameleer3/server/app/controller/ExecutionController.java" + provides: "POST /api/v1/data/executions endpoint" + min_lines: 20 + - path: "cameleer3-server-app/src/main/java/com/cameleer3/server/app/storage/ClickHouseExecutionRepository.java" + provides: "Batch insert to route_executions table via JdbcTemplate" + min_lines: 30 + - path: "cameleer3-server-app/src/main/java/com/cameleer3/server/app/ingestion/ClickHouseFlushScheduler.java" + provides: "Scheduled drain of WriteBuffer into ClickHouse" + min_lines: 20 + - path: "cameleer3-server-core/src/main/java/com/cameleer3/server/core/ingestion/IngestionService.java" + provides: "Routes data to appropriate WriteBuffer instances" + min_lines: 20 + key_links: + - from: "ExecutionController.java" + to: "IngestionService.java" + via: "constructor injection" + pattern: "IngestionService" + - from: "IngestionService.java" + to: "WriteBuffer.java" + via: "offer/offerBatch calls" + pattern: "writeBuffer\\.offer" + - from: "ClickHouseFlushScheduler.java" + to: "WriteBuffer.java" + via: "drain call on scheduled interval" + pattern: "writeBuffer\\.drain" + - from: "ClickHouseFlushScheduler.java" + to: "ClickHouseExecutionRepository.java" + via: "insertBatch call" + pattern: "repository\\.insertBatch" +--- + + +Implement the three ingestion REST endpoints, ClickHouse repository implementations, flush scheduler, and IngestionService that wires controllers to the WriteBuffer. + +Purpose: This is the core data pipeline -- agents POST data to endpoints, IngestionService buffers it, ClickHouseFlushScheduler drains and batch-inserts to ClickHouse. Backpressure returns 503 when buffer full. +Output: Working ingestion flow verified by integration tests against Testcontainers ClickHouse. + + + +@C:/Users/Hendrik/.claude/get-shit-done/workflows/execute-plan.md +@C:/Users/Hendrik/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/01-ingestion-pipeline-api-foundation/01-RESEARCH.md +@.planning/phases/01-ingestion-pipeline-api-foundation/01-01-SUMMARY.md + + + +From cameleer3-server-core WriteBuffer.java: +```java +public class WriteBuffer { + public WriteBuffer(int capacity); + public boolean offer(T item); + public boolean offerBatch(List items); + public List drain(int maxBatch); + public int size(); + public int capacity(); + public boolean isFull(); + public int remainingCapacity(); +} +``` + +From cameleer3-server-core repository interfaces: +```java +public interface ExecutionRepository { + void insertBatch(List executions); +} +public interface DiagramRepository { + void store(RouteGraph graph); + Optional findByContentHash(String hash); + Optional findContentHashForRoute(String routeId, String agentId); +} +public interface MetricsRepository { + void insertBatch(List metrics); +} +``` + +From IngestionConfig: +```java +@ConfigurationProperties("ingestion") +public class IngestionConfig { + int bufferCapacity; // default 50000 + int batchSize; // default 5000 + long flushIntervalMs; // default 1000 +} +``` + + + + + + + Task 1: IngestionService, ClickHouse repositories, and flush scheduler + + cameleer3-server-core/src/main/java/com/cameleer3/server/core/ingestion/IngestionService.java, + cameleer3-server-app/src/main/java/com/cameleer3/server/app/storage/ClickHouseExecutionRepository.java, + cameleer3-server-app/src/main/java/com/cameleer3/server/app/storage/ClickHouseDiagramRepository.java, + cameleer3-server-app/src/main/java/com/cameleer3/server/app/storage/ClickHouseMetricsRepository.java, + cameleer3-server-app/src/main/java/com/cameleer3/server/app/ingestion/ClickHouseFlushScheduler.java + + + - IngestionService.acceptExecutions(list) delegates to WriteBuffer.offerBatch, returns boolean + - IngestionService.acceptDiagram(graph) computes content hash, delegates to WriteBuffer.offer + - IngestionService.acceptMetrics(list) delegates to WriteBuffer.offerBatch + - ClickHouseExecutionRepository.insertBatch uses JdbcTemplate.batchUpdate with all columns including Array columns for processor executions + - ClickHouseDiagramRepository.store computes SHA-256 content hash and does INSERT with content_hash dedup + - ClickHouseFlushScheduler runs on @Scheduled(fixedDelayString="${ingestion.flush-interval-ms:1000}"), drains each buffer and calls respective repository.insertBatch + - ClickHouseFlushScheduler implements SmartLifecycle for graceful shutdown (flush remaining on stop) + + + 1. Create IngestionService in core module (no Spring annotations -- it's a plain class): + - Constructor takes three WriteBuffer instances (executions, diagrams, metrics) + - acceptExecutions(List): calls executionBuffer.offerBatch(), returns boolean + - acceptExecution(RouteExecution): calls executionBuffer.offer(), returns boolean + - acceptDiagram(RouteGraph): calls diagramBuffer.offer(), returns boolean + - acceptDiagrams(List): calls diagramBuffer.offerBatch(), returns boolean + - acceptMetrics(List): calls metricsBuffer.offerBatch(), returns boolean + - getExecutionBufferDepth(), getDiagramBufferDepth(), getMetricsBufferDepth() for monitoring + + 2. Create ClickHouseExecutionRepository implements ExecutionRepository: + - @Repository, inject JdbcTemplate + - insertBatch: INSERT INTO route_executions with all columns. Map RouteExecution fields to ClickHouse columns. + For processor execution arrays: extract from RouteExecution.getProcessorExecutions() into parallel arrays (processor_ids, processor_types, etc.) + Use JdbcTemplate.batchUpdate with BatchPreparedStatementSetter. + For Array columns, use java.sql.Array via connection.createArrayOf() or pass as comma-separated and cast. + Note: ClickHouse JDBC V2 handles Array types -- pass Java arrays directly via ps.setObject(). + + 3. Create ClickHouseDiagramRepository implements DiagramRepository: + - @Repository, inject JdbcTemplate + - store(RouteGraph): serialize graph to JSON (Jackson ObjectMapper), compute SHA-256 hex hash of JSON bytes, INSERT INTO route_diagrams (content_hash, route_id, agent_id, definition) + - findByContentHash: SELECT by content_hash, deserialize definition JSON back to RouteGraph + - findContentHashForRoute: SELECT content_hash WHERE route_id=? AND agent_id=? ORDER BY created_at DESC LIMIT 1 + + 4. Create ClickHouseMetricsRepository implements MetricsRepository: + - @Repository, inject JdbcTemplate + - insertBatch: INSERT INTO agent_metrics with batchUpdate + + 5. Create ClickHouseFlushScheduler: + - @Component, @EnableScheduling on the app config or main class + - Inject three WriteBuffer instances and three repository implementations + - Inject IngestionConfig for batchSize + - @Scheduled(fixedDelayString="${ingestion.flush-interval-ms:1000}") flushAll(): drains each buffer up to batchSize, calls insertBatch if non-empty. Wrap each in try-catch to log errors without stopping the scheduler. + - Implement SmartLifecycle: on stop(), flush all remaining data (loop drain until empty) before returning. + + + mvn clean compile -q 2>&1 | tail -5 + + IngestionService routes data to WriteBuffers. ClickHouse repositories implement batch inserts via JdbcTemplate. FlushScheduler drains buffers on interval and flushes remaining on shutdown. + + + + Task 2: Ingestion REST controllers and integration tests + + cameleer3-server-app/src/main/java/com/cameleer3/server/app/controller/ExecutionController.java, + cameleer3-server-app/src/main/java/com/cameleer3/server/app/controller/DiagramController.java, + cameleer3-server-app/src/main/java/com/cameleer3/server/app/controller/MetricsController.java, + cameleer3-server-app/src/test/java/com/cameleer3/server/app/controller/ExecutionControllerIT.java, + cameleer3-server-app/src/test/java/com/cameleer3/server/app/controller/DiagramControllerIT.java, + cameleer3-server-app/src/test/java/com/cameleer3/server/app/controller/MetricsControllerIT.java, + cameleer3-server-app/src/test/java/com/cameleer3/server/app/controller/BackpressureIT.java + + + - POST /api/v1/data/executions with single RouteExecution JSON returns 202 + - POST /api/v1/data/executions with array of RouteExecutions returns 202 + - POST /api/v1/data/diagrams with single RouteGraph returns 202 + - POST /api/v1/data/diagrams with array of RouteGraphs returns 202 + - POST /api/v1/data/metrics with metrics payload returns 202 + - After flush interval, posted data is queryable in ClickHouse + - When buffer is full, POST returns 503 with Retry-After header + - Unknown JSON fields in request body are accepted (not rejected) + + + 1. Create ExecutionController: + - @RestController @RequestMapping("/api/v1/data") + - POST /executions: accepts @RequestBody that handles both single RouteExecution and List. Use a custom deserializer or accept Object and check type, OR simply always accept as List (require agents to send arrays). Per protocol, accept both single and array. + - Calls ingestionService.acceptExecutions(). If returns false -> 503 with Retry-After: 5 header. If true -> 202 Accepted. + - Add @Operation annotations for OpenAPI documentation. + + 2. Create DiagramController: + - @RestController @RequestMapping("/api/v1/data") + - POST /diagrams: same pattern, accepts single or array of RouteGraph, delegates to ingestionService. + + 3. Create MetricsController: + - @RestController @RequestMapping("/api/v1/data") + - POST /metrics: same pattern. + + 4. Create ExecutionControllerIT (extends AbstractClickHouseIT): + - Use TestRestTemplate or MockMvc with @AutoConfigureMockMvc + - Test: POST valid RouteExecution JSON with X-Cameleer-Protocol-Version:1 header -> 202 + - Test: POST array of executions -> 202 + - Test: After post, wait for flush (use Awaitility), query ClickHouse directly via JdbcTemplate to verify data arrived + - Test: POST with unknown JSON fields -> 202 (forward compat, API-05) + + 5. Create DiagramControllerIT (extends AbstractClickHouseIT): + - Test: POST RouteGraph -> 202 + - Test: After flush, diagram stored in ClickHouse with content_hash + + 6. Create MetricsControllerIT (extends AbstractClickHouseIT): + - Test: POST metrics -> 202 + - Test: After flush, metrics in ClickHouse + + 7. Create BackpressureIT (extends AbstractClickHouseIT): + - Configure test with tiny buffer (ingestion.buffer-capacity=5) + - Fill buffer by posting enough data + - Next POST returns 503 with Retry-After header + - Verify previously buffered data is NOT lost (still flushes to ClickHouse) + + Note: All integration tests must include X-Cameleer-Protocol-Version:1 header (API-04 will be enforced by Plan 03's interceptor, but include the header now for forward compatibility). + + + mvn test -pl cameleer3-server-app -Dtest="ExecutionControllerIT,DiagramControllerIT,MetricsControllerIT,BackpressureIT" -q 2>&1 | tail -15 + + All three ingestion endpoints return 202 on valid data. Data arrives in ClickHouse after flush. Buffer-full returns 503 with Retry-After. Unknown JSON fields accepted. Integration tests green. + + + + + +- `mvn test -pl cameleer3-server-app -Dtest="ExecutionControllerIT,DiagramControllerIT,MetricsControllerIT,BackpressureIT" -q` -- all integration tests pass +- POST to /api/v1/data/executions returns 202 +- POST to /api/v1/data/diagrams returns 202 +- POST to /api/v1/data/metrics returns 202 +- Buffer full returns 503 + + + +All four integration test classes green. Data flows from HTTP POST through WriteBuffer through FlushScheduler to ClickHouse. Backpressure returns 503 when buffer full without losing existing data. + + + +After completion, create `.planning/phases/01-ingestion-pipeline-api-foundation/01-02-SUMMARY.md` + diff --git a/.planning/phases/01-ingestion-pipeline-api-foundation/01-03-PLAN.md b/.planning/phases/01-ingestion-pipeline-api-foundation/01-03-PLAN.md new file mode 100644 index 00000000..eff0bf69 --- /dev/null +++ b/.planning/phases/01-ingestion-pipeline-api-foundation/01-03-PLAN.md @@ -0,0 +1,188 @@ +--- +phase: 01-ingestion-pipeline-api-foundation +plan: 03 +type: execute +wave: 2 +depends_on: ["01-01"] +files_modified: + - cameleer3-server-app/src/main/java/com/cameleer3/server/app/interceptor/ProtocolVersionInterceptor.java + - cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/WebConfig.java + - cameleer3-server-app/src/main/java/com/cameleer3/server/app/Cameleer3ServerApplication.java + - cameleer3-server-app/src/test/java/com/cameleer3/server/app/interceptor/ProtocolVersionIT.java + - cameleer3-server-app/src/test/java/com/cameleer3/server/app/controller/HealthControllerIT.java + - cameleer3-server-app/src/test/java/com/cameleer3/server/app/controller/OpenApiIT.java + - cameleer3-server-app/src/test/java/com/cameleer3/server/app/controller/ForwardCompatIT.java +autonomous: true +requirements: + - API-01 + - API-02 + - API-03 + - API-04 + - API-05 + - INGST-06 + +must_haves: + truths: + - "GET /api/v1/health returns 200 with health status" + - "GET /api/v1/swagger-ui returns Swagger UI page" + - "GET /api/v1/api-docs returns OpenAPI JSON spec listing all endpoints" + - "Requests without X-Cameleer-Protocol-Version header to /api/v1/data/* return 400" + - "Requests with wrong protocol version return 400" + - "Health and swagger endpoints work without protocol version header" + - "Unknown JSON fields in request body do not cause deserialization errors" + - "ClickHouse tables have TTL clause for 30-day retention" + artifacts: + - path: "cameleer3-server-app/src/main/java/com/cameleer3/server/app/interceptor/ProtocolVersionInterceptor.java" + provides: "Validates X-Cameleer-Protocol-Version:1 header on data endpoints" + min_lines: 20 + - path: "cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/WebConfig.java" + provides: "Registers interceptor with path patterns" + min_lines: 10 + key_links: + - from: "WebConfig.java" + to: "ProtocolVersionInterceptor.java" + via: "addInterceptors registration" + pattern: "addInterceptors.*ProtocolVersionInterceptor" + - from: "application.yml" + to: "Actuator health endpoint" + via: "management.endpoints config" + pattern: "management\\.endpoints" + - from: "application.yml" + to: "springdoc" + via: "springdoc.api-docs.path and swagger-ui.path" + pattern: "springdoc" +--- + + +Implement the API foundation: health endpoint, OpenAPI documentation, protocol version header validation, forward compatibility, and TTL verification. + +Purpose: Completes the API scaffolding so all endpoints follow the protocol v1 contract. Health endpoint enables monitoring. OpenAPI enables discoverability. Protocol version interceptor enforces compatibility. TTL verification confirms data retention. +Output: Working health, Swagger UI, protocol header enforcement, and verified TTL retention. + + + +@C:/Users/Hendrik/.claude/get-shit-done/workflows/execute-plan.md +@C:/Users/Hendrik/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/01-ingestion-pipeline-api-foundation/01-RESEARCH.md +@.planning/phases/01-ingestion-pipeline-api-foundation/01-01-SUMMARY.md + + + + + + Task 1: Protocol version interceptor, WebConfig, and Spring Boot application class + + cameleer3-server-app/src/main/java/com/cameleer3/server/app/interceptor/ProtocolVersionInterceptor.java, + cameleer3-server-app/src/main/java/com/cameleer3/server/app/config/WebConfig.java, + cameleer3-server-app/src/main/java/com/cameleer3/server/app/Cameleer3ServerApplication.java + + + - Request to /api/v1/data/* without X-Cameleer-Protocol-Version header returns 400 + - Request to /api/v1/data/* with X-Cameleer-Protocol-Version:2 returns 400 + - Request to /api/v1/data/* with X-Cameleer-Protocol-Version:1 passes through + - Request to /api/v1/health without the header succeeds (excluded from interceptor) + - Request to /api/v1/swagger-ui without the header succeeds (excluded) + - Request to /api/v1/api-docs without the header succeeds (excluded) + + + 1. Create ProtocolVersionInterceptor implementing HandlerInterceptor: + - preHandle: read X-Cameleer-Protocol-Version header + - If null or not "1": set response status 400, write JSON error body {"error": "Missing or unsupported X-Cameleer-Protocol-Version header"}, return false + - If "1": return true + + 2. Create WebConfig implementing WebMvcConfigurer: + - @Configuration + - Inject ProtocolVersionInterceptor (declare it as @Component or @Bean) + - Override addInterceptors: register interceptor with pathPatterns "/api/v1/data/**" and "/api/v1/agents/**" + - Explicitly EXCLUDE: "/api/v1/health", "/api/v1/api-docs/**", "/api/v1/swagger-ui/**", "/api/v1/swagger-ui.html" + + 3. Create or update Cameleer3ServerApplication: + - @SpringBootApplication in package com.cameleer3.server.app + - @EnableScheduling (needed for ClickHouseFlushScheduler from Plan 02) + - @EnableConfigurationProperties(IngestionConfig.class) + - Main method with SpringApplication.run() + - Ensure package scanning covers com.cameleer3.server.app and com.cameleer3.server.core + + + mvn clean compile -pl cameleer3-server-app -q 2>&1 | tail -5 + + ProtocolVersionInterceptor validates header on data/agent paths. Health, swagger, and api-docs paths excluded. Application class enables scheduling and config properties. + + + + Task 2: Health, OpenAPI, protocol version, forward compat, and TTL integration tests + + cameleer3-server-app/src/test/java/com/cameleer3/server/app/controller/HealthControllerIT.java, + cameleer3-server-app/src/test/java/com/cameleer3/server/app/controller/OpenApiIT.java, + cameleer3-server-app/src/test/java/com/cameleer3/server/app/interceptor/ProtocolVersionIT.java, + cameleer3-server-app/src/test/java/com/cameleer3/server/app/controller/ForwardCompatIT.java + + + - GET /api/v1/health returns 200 with JSON containing status field + - GET /api/v1/api-docs returns 200 with OpenAPI JSON containing paths + - GET /api/v1/swagger-ui/index.html returns 200 (Swagger UI page) + - POST /api/v1/data/executions without protocol header returns 400 + - POST /api/v1/data/executions with X-Cameleer-Protocol-Version:1 does NOT return 400 (may return 202 or other based on body) + - POST /api/v1/data/executions with X-Cameleer-Protocol-Version:2 returns 400 + - GET /api/v1/health without protocol header returns 200 (not intercepted) + - POST /api/v1/data/executions with extra unknown JSON fields returns 202 (not 400/422) + - ClickHouse route_executions table SHOW CREATE TABLE includes TTL + + + 1. Create HealthControllerIT (extends AbstractClickHouseIT): + - Test: GET /api/v1/health -> 200 with body containing "status" + - Test: No X-Cameleer-Protocol-Version header required for health + + 2. Create OpenApiIT (extends AbstractClickHouseIT): + - Test: GET /api/v1/api-docs -> 200, body contains "openapi" and "paths" + - Test: GET /api/v1/swagger-ui/index.html -> 200 or 302 (redirect to UI) + - Test: api-docs lists /api/v1/data/executions path + + 3. Create ProtocolVersionIT (extends AbstractClickHouseIT): + - Test: POST /api/v1/data/executions without header -> 400 with error message + - Test: POST /api/v1/data/executions with header value "2" -> 400 + - Test: POST /api/v1/data/executions with header value "1" and valid body -> 202 (not 400) + - Test: GET /api/v1/health without header -> 200 (excluded from interceptor) + - Test: GET /api/v1/api-docs without header -> 200 (excluded from interceptor) + + 4. Create ForwardCompatIT (extends AbstractClickHouseIT): + - Test: POST /api/v1/data/executions with valid RouteExecution JSON plus extra unknown fields (e.g., "futureField": "value") -> 202 (Jackson does not fail on unknown properties) + - This validates API-05 requirement explicitly. + + 5. TTL verification (add to HealthControllerIT or separate test): + - Query ClickHouse: SHOW CREATE TABLE route_executions + - Assert result contains "TTL start_time + toIntervalDay(30)" + - Query: SHOW CREATE TABLE agent_metrics + - Assert result contains TTL clause + + Note: All tests that POST to data endpoints must include X-Cameleer-Protocol-Version:1 header. + + + mvn test -pl cameleer3-server-app -Dtest="HealthControllerIT,OpenApiIT,ProtocolVersionIT,ForwardCompatIT" -q 2>&1 | tail -15 + + Health returns 200. OpenAPI docs are available and list endpoints. Protocol version header enforced on data paths, not on health/docs. Unknown JSON fields accepted. TTL confirmed in ClickHouse DDL. + + + + + +- `mvn test -pl cameleer3-server-app -Dtest="HealthControllerIT,OpenApiIT,ProtocolVersionIT,ForwardCompatIT" -q` -- all tests pass +- GET /api/v1/health returns 200 +- GET /api/v1/api-docs returns OpenAPI spec +- Missing protocol header returns 400 on data endpoints +- Health works without protocol header + + + +All four integration test classes green. Health endpoint accessible. OpenAPI docs list all ingestion endpoints. Protocol version header enforced on data/agent paths but not health/docs. Forward compatibility confirmed. TTL verified in ClickHouse schema. + + + +After completion, create `.planning/phases/01-ingestion-pipeline-api-foundation/01-03-SUMMARY.md` +