cb3ebfea7c
Rename Java packages from com.cameleer3 to com.cameleer, module directories from cameleer3-* to cameleer-*, and all references throughout workflows, Dockerfiles, docs, migrations, and pom.xml. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
355 lines
21 KiB
Markdown
355 lines
21 KiB
Markdown
---
|
|
phase: 02-transaction-search-diagrams
|
|
plan: 03
|
|
type: execute
|
|
wave: 2
|
|
depends_on:
|
|
- "02-01"
|
|
files_modified:
|
|
- cameleer-server-app/src/main/java/com/cameleer/server/app/search/ClickHouseSearchEngine.java
|
|
- cameleer-server-app/src/main/java/com/cameleer/server/app/controller/SearchController.java
|
|
- cameleer-server-app/src/main/java/com/cameleer/server/app/controller/DetailController.java
|
|
- cameleer-server-app/src/main/java/com/cameleer/server/app/config/SearchBeanConfig.java
|
|
- cameleer-server-app/src/main/java/com/cameleer/server/app/storage/ClickHouseExecutionRepository.java
|
|
- cameleer-server-app/src/test/java/com/cameleer/server/app/controller/SearchControllerIT.java
|
|
- cameleer-server-app/src/test/java/com/cameleer/server/app/controller/DetailControllerIT.java
|
|
- cameleer-server-core/src/test/java/com/cameleer/server/core/detail/TreeReconstructionTest.java
|
|
autonomous: true
|
|
requirements:
|
|
- SRCH-01
|
|
- SRCH-02
|
|
- SRCH-03
|
|
- SRCH-04
|
|
- SRCH-05
|
|
- SRCH-06
|
|
|
|
must_haves:
|
|
truths:
|
|
- "User can search by status and get only matching executions"
|
|
- "User can search by time range and get only executions within that window"
|
|
- "User can search by duration range (min/max ms) and get matching executions"
|
|
- "User can search by correlationId to find all related executions"
|
|
- "User can full-text search and find matches in bodies, headers, error messages, stack traces"
|
|
- "User can combine multiple filters in a single search (e.g., status + time + text)"
|
|
- "User can retrieve a transaction detail with nested processor execution tree"
|
|
- "Detail response includes diagramContentHash for linking to diagram endpoint"
|
|
- "Search results are paginated with total count, offset, and limit"
|
|
artifacts:
|
|
- path: "cameleer-server-app/src/main/java/com/cameleer/server/app/search/ClickHouseSearchEngine.java"
|
|
provides: "ClickHouse implementation of SearchEngine with dynamic WHERE building"
|
|
min_lines: 80
|
|
- path: "cameleer-server-app/src/main/java/com/cameleer/server/app/controller/SearchController.java"
|
|
provides: "GET + POST /api/v1/search/executions endpoints"
|
|
exports: ["SearchController"]
|
|
- path: "cameleer-server-app/src/main/java/com/cameleer/server/app/controller/DetailController.java"
|
|
provides: "GET /api/v1/executions/{id} endpoint returning nested tree"
|
|
exports: ["DetailController"]
|
|
- path: "cameleer-server-app/src/test/java/com/cameleer/server/app/controller/SearchControllerIT.java"
|
|
provides: "Integration tests for all search filter combinations"
|
|
min_lines: 100
|
|
key_links:
|
|
- from: "SearchController"
|
|
to: "SearchService"
|
|
via: "constructor injection, delegates search()"
|
|
pattern: "searchService\\.search"
|
|
- from: "SearchService"
|
|
to: "ClickHouseSearchEngine"
|
|
via: "SearchEngine interface"
|
|
pattern: "engine\\.search"
|
|
- from: "ClickHouseSearchEngine"
|
|
to: "route_executions table"
|
|
via: "dynamic SQL with parameterized WHERE"
|
|
pattern: "SELECT.*FROM route_executions.*WHERE"
|
|
- from: "DetailController"
|
|
to: "DetailService"
|
|
via: "constructor injection"
|
|
pattern: "detailService\\.getDetail"
|
|
- from: "DetailService"
|
|
to: "ClickHouseExecutionRepository"
|
|
via: "findRawById for flat data, then reconstructTree"
|
|
pattern: "findRawById|reconstructTree"
|
|
---
|
|
|
|
<objective>
|
|
Implement the search endpoints (GET and POST), the ClickHouse search engine with dynamic SQL, the transaction detail endpoint with nested tree reconstruction, and comprehensive integration tests.
|
|
|
|
Purpose: This is the core query capability of Phase 2 — users need to find transactions by any combination of filters and drill into execution details. The search engine abstraction allows future swap to OpenSearch.
|
|
|
|
Output: SearchController (GET + POST), DetailController, ClickHouseSearchEngine, TreeReconstructionTest, SearchControllerIT, DetailControllerIT.
|
|
</objective>
|
|
|
|
<execution_context>
|
|
@C:/Users/Hendrik/.claude/get-shit-done/workflows/execute-plan.md
|
|
@C:/Users/Hendrik/.claude/get-shit-done/templates/summary.md
|
|
</execution_context>
|
|
|
|
<context>
|
|
@.planning/PROJECT.md
|
|
@.planning/ROADMAP.md
|
|
@.planning/phases/02-transaction-search-diagrams/02-CONTEXT.md
|
|
@.planning/phases/02-transaction-search-diagrams/02-RESEARCH.md
|
|
@.planning/phases/02-transaction-search-diagrams/02-01-SUMMARY.md
|
|
|
|
@clickhouse/init/01-schema.sql
|
|
@clickhouse/init/02-search-columns.sql
|
|
@cameleer-server-app/src/main/java/com/cameleer/server/app/storage/ClickHouseExecutionRepository.java
|
|
@cameleer-server-app/src/main/java/com/cameleer/server/app/controller/ExecutionController.java
|
|
|
|
<interfaces>
|
|
<!-- Core types created by Plan 01 — executor reads these from plan 01 SUMMARY -->
|
|
|
|
From cameleer-server-core/.../search/SearchEngine.java:
|
|
```java
|
|
public interface SearchEngine {
|
|
SearchResult<ExecutionSummary> search(SearchRequest request);
|
|
long count(SearchRequest request);
|
|
}
|
|
```
|
|
|
|
From cameleer-server-core/.../search/SearchRequest.java:
|
|
```java
|
|
public record SearchRequest(
|
|
String status, // nullable, filter by ExecutionStatus name
|
|
Instant timeFrom, // nullable, start_time >= this
|
|
Instant timeTo, // nullable, start_time <= this
|
|
Long durationMin, // nullable, duration_ms >= this
|
|
Long durationMax, // nullable, duration_ms <= this
|
|
String correlationId, // nullable, exact match
|
|
String text, // nullable, global full-text LIKE across all text fields
|
|
String textInBody, // nullable, LIKE on exchange_bodies only
|
|
String textInHeaders, // nullable, LIKE on exchange_headers only
|
|
String textInErrors, // nullable, LIKE on error_message + error_stacktrace
|
|
int offset,
|
|
int limit
|
|
) { /* compact constructor with validation */ }
|
|
```
|
|
|
|
From cameleer-server-core/.../search/SearchResult.java:
|
|
```java
|
|
public record SearchResult<T>(List<T> data, long total, int offset, int limit) {
|
|
public static <T> SearchResult<T> empty(int offset, int limit);
|
|
}
|
|
```
|
|
|
|
From cameleer-server-core/.../search/ExecutionSummary.java:
|
|
```java
|
|
public record ExecutionSummary(
|
|
String executionId, String routeId, String agentId, String status,
|
|
Instant startTime, Instant endTime, long durationMs,
|
|
String correlationId, String errorMessage, String diagramContentHash
|
|
) {}
|
|
```
|
|
|
|
From cameleer-server-core/.../detail/DetailService.java:
|
|
```java
|
|
public class DetailService {
|
|
// Constructor takes ExecutionRepository (or a query interface)
|
|
public Optional<ExecutionDetail> getDetail(String executionId);
|
|
// Internal: reconstructTree(parallel arrays) -> List<ProcessorNode>
|
|
}
|
|
```
|
|
|
|
From cameleer-server-core/.../detail/ExecutionDetail.java:
|
|
```java
|
|
public record ExecutionDetail(
|
|
String executionId, String routeId, String agentId, String status,
|
|
Instant startTime, Instant endTime, long durationMs,
|
|
String correlationId, String exchangeId, String errorMessage,
|
|
String errorStackTrace, String diagramContentHash,
|
|
List<ProcessorNode> processors
|
|
) {}
|
|
```
|
|
|
|
From cameleer-server-core/.../detail/ProcessorNode.java:
|
|
```java
|
|
public record ProcessorNode(
|
|
String processorId, String processorType, String status,
|
|
Instant startTime, Instant endTime, long durationMs,
|
|
String diagramNodeId, String errorMessage, String errorStackTrace,
|
|
List<ProcessorNode> children
|
|
) {}
|
|
```
|
|
|
|
Existing ClickHouse schema (after Plan 01 schema extension):
|
|
```sql
|
|
-- route_executions columns:
|
|
-- execution_id, route_id, agent_id, status, start_time, end_time,
|
|
-- duration_ms, correlation_id, exchange_id, error_message, error_stacktrace,
|
|
-- processor_ids, processor_types, processor_starts, processor_ends,
|
|
-- processor_durations, processor_statuses,
|
|
-- exchange_bodies, exchange_headers,
|
|
-- processor_depths, processor_parent_indexes,
|
|
-- processor_error_messages, processor_error_stacktraces,
|
|
-- processor_input_bodies, processor_output_bodies,
|
|
-- processor_input_headers, processor_output_headers,
|
|
-- processor_diagram_node_ids, diagram_content_hash,
|
|
-- server_received_at
|
|
-- ORDER BY (agent_id, status, start_time, execution_id)
|
|
```
|
|
|
|
Established controller pattern (from Phase 1):
|
|
```java
|
|
// Controllers accept raw String body for single/array flexibility
|
|
// Return 202 for ingestion, standard REST responses for queries
|
|
// ProtocolVersionInterceptor validates X-Cameleer-Protocol-Version: 1 header
|
|
```
|
|
</interfaces>
|
|
</context>
|
|
|
|
<tasks>
|
|
|
|
<task type="auto" tdd="true">
|
|
<name>Task 1: ClickHouseSearchEngine, SearchController, and search integration tests</name>
|
|
<files>
|
|
cameleer-server-app/src/main/java/com/cameleer/server/app/search/ClickHouseSearchEngine.java,
|
|
cameleer-server-app/src/main/java/com/cameleer/server/app/controller/SearchController.java,
|
|
cameleer-server-app/src/main/java/com/cameleer/server/app/config/SearchBeanConfig.java,
|
|
cameleer-server-app/src/test/java/com/cameleer/server/app/controller/SearchControllerIT.java
|
|
</files>
|
|
<behavior>
|
|
- Test searchByStatus: Insert 3 executions (COMPLETED, FAILED, RUNNING). GET /api/v1/search/executions?status=FAILED returns only the FAILED execution. Response has envelope: {"data":[...],"total":1,"offset":0,"limit":50}
|
|
- Test searchByTimeRange: Insert executions at different times. Filter by timeFrom/timeTo returns only those in range
|
|
- Test searchByDuration: Insert executions with different durations. Filter by durationMin=100&durationMax=500 returns only matching
|
|
- Test searchByCorrelationId: Insert executions with different correlationIds. Filter returns only matching
|
|
- Test fullTextSearchGlobal: Insert execution with error_message="NullPointerException in OrderService". Search text=NullPointerException returns it. Search text=nonexistent returns empty
|
|
- Test fullTextSearchInBody: Insert execution with exchange body containing "customer-123". textInBody=customer-123 returns it
|
|
- Test fullTextSearchInHeaders: Insert execution with exchange headers containing "Content-Type". textInHeaders=Content-Type returns it
|
|
- Test fullTextSearchInErrors: Insert execution with error_stacktrace containing "com.example.MyException". textInErrors=MyException returns it
|
|
- Test combinedFilters: status=FAILED + text=NullPointer returns only failed executions with that error
|
|
- Test postAdvancedSearch: POST /api/v1/search/executions with JSON body containing all filters returns correct results
|
|
- Test pagination: Insert 10 executions. Request with offset=2&limit=3 returns 3 items, total=10, offset=2
|
|
- Test emptyResults: Search with no matches returns {"data":[],"total":0,"offset":0,"limit":50}
|
|
</behavior>
|
|
<action>
|
|
1. Create `ClickHouseSearchEngine` in `com.cameleer.server.app.search`:
|
|
- Implements SearchEngine interface from core module.
|
|
- Constructor takes JdbcTemplate.
|
|
- `search(SearchRequest)` method:
|
|
- Build dynamic WHERE clause from non-null SearchRequest fields using ArrayList<String> conditions and ArrayList<Object> params.
|
|
- status: `"status = ?"` with `req.status()`
|
|
- timeFrom: `"start_time >= ?"` with `Timestamp.from(req.timeFrom())`
|
|
- timeTo: `"start_time <= ?"` with `Timestamp.from(req.timeTo())`
|
|
- durationMin: `"duration_ms >= ?"` with `req.durationMin()`
|
|
- durationMax: `"duration_ms <= ?"` with `req.durationMax()`
|
|
- correlationId: `"correlation_id = ?"` with `req.correlationId()`
|
|
- text (global): `"(error_message LIKE ? OR error_stacktrace LIKE ? OR exchange_bodies LIKE ? OR exchange_headers LIKE ?)"` with `"%" + escapeLike(req.text()) + "%"` repeated 4 times
|
|
- textInBody: `"exchange_bodies LIKE ?"` with escaped pattern
|
|
- textInHeaders: `"exchange_headers LIKE ?"` with escaped pattern
|
|
- textInErrors: `"(error_message LIKE ? OR error_stacktrace LIKE ?)"` with escaped pattern repeated 2 times
|
|
- Combine conditions with AND. If empty, no WHERE clause.
|
|
- Count query: `SELECT count() FROM route_executions` + where
|
|
- Data query: `SELECT execution_id, route_id, agent_id, status, start_time, end_time, duration_ms, correlation_id, error_message, diagram_content_hash FROM route_executions` + where + ` ORDER BY start_time DESC LIMIT ? OFFSET ?`
|
|
- Map rows to ExecutionSummary records. Use `rs.getTimestamp("start_time").toInstant()` for Instant fields.
|
|
- Return SearchResult with data, total from count query, offset, limit.
|
|
- `escapeLike(String)` utility: escape `%`, `_`, `\` characters in user input to prevent LIKE injection. Replace `\` with `\\`, `%` with `\%`, `_` with `\_`.
|
|
- `count(SearchRequest)` method: same WHERE building, just count query.
|
|
|
|
2. Create `SearchBeanConfig` in `com.cameleer.server.app.config`:
|
|
- @Configuration class that creates:
|
|
- `ClickHouseSearchEngine` bean (takes JdbcTemplate)
|
|
- `SearchService` bean (takes SearchEngine)
|
|
- `DetailService` bean (takes the execution query interface from Plan 01)
|
|
|
|
3. Create `SearchController` in `com.cameleer.server.app.controller`:
|
|
- Inject SearchService.
|
|
- `GET /api/v1/search/executions` with @RequestParam for basic filters:
|
|
- status (optional String)
|
|
- timeFrom (optional Instant, use @DateTimeFormat or String parsing)
|
|
- timeTo (optional Instant)
|
|
- correlationId (optional String)
|
|
- offset (optional int, default 0)
|
|
- limit (optional int, default 50)
|
|
Build SearchRequest from params, call searchService.search(), return ResponseEntity with SearchResult.
|
|
- `POST /api/v1/search/executions` accepting JSON body:
|
|
- Accept SearchRequest directly (or a DTO that maps to SearchRequest). Jackson will deserialize the JSON body.
|
|
- All filters available including durationMin, durationMax, text, textInBody, textInHeaders, textInErrors.
|
|
- Call searchService.search(), return ResponseEntity with SearchResult.
|
|
- Response format per user decision: `{ "data": [...], "total": N, "offset": 0, "limit": 50 }`
|
|
|
|
4. Create `SearchControllerIT` (extends AbstractClickHouseIT):
|
|
- Use TestRestTemplate (auto-configured by @SpringBootTest with RANDOM_PORT).
|
|
- Seed test data: Insert multiple RouteExecution objects with varying statuses, times, durations, correlationIds, error messages, and exchange snapshot data. Use the POST /api/v1/data/executions endpoint to insert, then wait for flush (Awaitility).
|
|
- Write tests for each behavior listed above. Use GET for basic filter tests, POST for advanced/combined filter tests.
|
|
- All requests include X-Cameleer-Protocol-Version: 1 header per ProtocolVersionInterceptor requirement.
|
|
- Assert response structure matches the envelope format.
|
|
</action>
|
|
<verify>
|
|
<automated>cd C:/Users/Hendrik/Documents/projects/cameleer-server && mvn test -pl cameleer-server-app -Dtest=SearchControllerIT</automated>
|
|
</verify>
|
|
<done>All search filter types work independently and in combination, response envelope has correct format, pagination works correctly, full-text search finds matches in all text fields, LIKE patterns are properly escaped</done>
|
|
</task>
|
|
|
|
<task type="auto" tdd="true">
|
|
<name>Task 2: DetailController, tree reconstruction, exchange snapshot endpoint, and integration tests</name>
|
|
<files>
|
|
cameleer-server-app/src/main/java/com/cameleer/server/app/storage/ClickHouseExecutionRepository.java,
|
|
cameleer-server-app/src/main/java/com/cameleer/server/app/controller/DetailController.java,
|
|
cameleer-server-core/src/test/java/com/cameleer/server/core/detail/TreeReconstructionTest.java,
|
|
cameleer-server-app/src/test/java/com/cameleer/server/app/controller/DetailControllerIT.java
|
|
</files>
|
|
<behavior>
|
|
- Unit test: reconstructTree with [root, child, grandchild], depths=[0,1,2], parents=[-1,0,1] produces single root with one child that has one grandchild
|
|
- Unit test: reconstructTree with [A, B, C], depths=[0,0,0], parents=[-1,-1,-1] produces 3 roots (no nesting)
|
|
- Unit test: reconstructTree with [parent, child1, child2, grandchild], depths=[0,1,1,2], parents=[-1,0,0,2] produces parent with 2 children, second child has one grandchild
|
|
- Unit test: reconstructTree with empty arrays produces empty list
|
|
- Integration test: GET /api/v1/executions/{id} returns ExecutionDetail with nested processors tree matching the ingested structure
|
|
- Integration test: detail response includes diagramContentHash field (can be empty string if not set)
|
|
- Integration test: GET /api/v1/executions/{nonexistent-id} returns 404
|
|
- Integration test: GET /api/v1/executions/{id}/processors/{index}/snapshot returns exchange snapshot data for that processor
|
|
</behavior>
|
|
<action>
|
|
1. Create `TreeReconstructionTest` in core module test directory:
|
|
- Pure unit test (no Spring context needed).
|
|
- Test DetailService.reconstructTree (make it a static method or package-accessible for testing).
|
|
- Cover cases: single root, linear chain, wide tree (multiple roots), branching tree, empty arrays.
|
|
- Verify correct parent-child wiring and that ProcessorNode.children() lists are correctly populated.
|
|
|
|
2. Extend `ClickHouseExecutionRepository` with query methods:
|
|
- Add `findRawById(String executionId)` method that queries all columns from route_executions WHERE execution_id = ?. Return Optional<RawExecutionRow> (use the record created in Plan 01 or create it here if needed). The RawExecutionRow should contain ALL columns including the parallel arrays for processors.
|
|
- Add `findProcessorSnapshot(String executionId, int processorIndex)` method: queries processor_input_bodies[index+1], processor_output_bodies[index+1], processor_input_headers[index+1], processor_output_headers[index+1] for the given execution. Returns a DTO with inputBody, outputBody, inputHeaders, outputHeaders. ClickHouse arrays are 1-indexed in SQL, so add 1 to the Java 0-based index.
|
|
|
|
3. Create `DetailController` in `com.cameleer.server.app.controller`:
|
|
- Inject DetailService.
|
|
- `GET /api/v1/executions/{executionId}`: call detailService.getDetail(executionId). If empty, return 404. Otherwise return 200 with ExecutionDetail JSON. The processors field is a nested tree of ProcessorNode objects.
|
|
- `GET /api/v1/executions/{executionId}/processors/{index}/snapshot`: call repository's findProcessorSnapshot. If execution not found or index out of bounds, return 404. Return JSON with inputBody, outputBody, inputHeaders, outputHeaders. Per user decision: exchange snapshot data fetched separately per processor, not inlined in detail response.
|
|
|
|
4. Create `DetailControllerIT` (extends AbstractClickHouseIT):
|
|
- Seed a RouteExecution with a 3-level processor tree (root with 2 children, one child has a grandchild). Give processors exchange snapshot data (bodies, headers).
|
|
- Also seed a RouteGraph diagram for the route to test diagram hash linking.
|
|
- POST to ingestion endpoints, wait for flush.
|
|
- Test GET /api/v1/executions/{id}: verify response has nested processors tree with correct depths. Root should have 2 children, one child should have 1 grandchild. Verify diagramContentHash is present.
|
|
- Test GET /api/v1/executions/{id}/processors/0/snapshot: returns snapshot data for root processor.
|
|
- Test GET /api/v1/executions/{nonexistent}/: returns 404.
|
|
- Test GET /api/v1/executions/{id}/processors/999/snapshot: returns 404 for out-of-bounds index.
|
|
</action>
|
|
<verify>
|
|
<automated>cd C:/Users/Hendrik/Documents/projects/cameleer-server && mvn test -pl cameleer-server-core -Dtest=TreeReconstructionTest && mvn test -pl cameleer-server-app -Dtest=DetailControllerIT</automated>
|
|
</verify>
|
|
<done>Tree reconstruction correctly rebuilds nested processor trees from flat arrays, detail endpoint returns nested tree with all fields, snapshot endpoint returns per-processor exchange data, diagram hash included in detail response, all tests pass</done>
|
|
</task>
|
|
|
|
</tasks>
|
|
|
|
<verification>
|
|
- `mvn test -pl cameleer-server-core -Dtest=TreeReconstructionTest` passes (unit test for tree rebuild)
|
|
- `mvn test -pl cameleer-server-app -Dtest=SearchControllerIT` passes (all search filters)
|
|
- `mvn test -pl cameleer-server-app -Dtest=DetailControllerIT` passes (detail + snapshot)
|
|
- `mvn clean verify` passes (full suite green)
|
|
</verification>
|
|
|
|
<success_criteria>
|
|
- GET /api/v1/search/executions with status/time/duration/correlationId filters returns correct results
|
|
- POST /api/v1/search/executions with JSON body supports all filters including full-text and per-field targeting
|
|
- Full-text LIKE search finds matches in error_message, error_stacktrace, exchange_bodies, exchange_headers
|
|
- Combined filters work correctly (AND logic)
|
|
- Response envelope: { "data": [...], "total": N, "offset": 0, "limit": 50 }
|
|
- GET /api/v1/executions/{id} returns nested processor tree reconstructed from flat arrays
|
|
- GET /api/v1/executions/{id}/processors/{index}/snapshot returns per-processor exchange data
|
|
- Detail response includes diagramContentHash for linking to diagram render endpoint
|
|
- All tests pass including existing Phase 1 tests
|
|
</success_criteria>
|
|
|
|
<output>
|
|
After completion, create `.planning/phases/02-transaction-search-diagrams/02-03-SUMMARY.md`
|
|
</output>
|