cameleer/cameleer-server
1
0
Fork 0
Code Issues 42 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(02-01): extend ingestion to populate Phase 2 columns with integration tests

Browse Source 
- Refactored flattenProcessors to flattenWithMetadata (FlatProcessor record with depth/parentIndex)
- INSERT now populates 12 new columns: exchange_bodies, exchange_headers, processor_depths,
  processor_parent_indexes, processor_error_messages, processor_error_stacktraces,
  processor_input_bodies, processor_output_bodies, processor_input_headers,
  processor_output_headers, processor_diagram_node_ids, diagram_content_hash
- Exchange bodies/headers concatenated from all processor snapshots + route-level snapshots
- Null ExchangeSnapshot handled gracefully (empty string defaults)
- Headers serialized to JSON via Jackson ObjectMapper
- IngestionSchemaIT verifies 3-level tree metadata, body concatenation, null snapshot handling
- DiagramRenderer/DiagramLayout stubs created to fix pre-existing compilation error (Rule 3)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
hsiegeln
2026-03-11 16:15:41 +01:00
parent c0922430c4
commit f6ff279a60
3 changed files with 192 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
cameleer3-server-app/src/test/java/com/cameleer3/server/app/storage/IngestionSchemaIT.java
View File

@@ -1,9 +1,5 @@
package com.cameleer3.server.app.storage;
import com.cameleer3.common.model.ExchangeSnapshot;
import com.cameleer3.common.model.ExecutionStatus;
import com.cameleer3.common.model.ProcessorExecution;
import com.cameleer3.common.model.RouteExecution;
import com.cameleer3.server.app.AbstractClickHouseIT;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
@@ -14,8 +10,7 @@ import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import java.sql.Array;
import java.time.Instant;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
@@ -90,49 +85,40 @@ class IngestionSchemaIT extends AbstractClickHouseIT {
postExecution(json);
await().atMost(10, SECONDS).untilAsserted(() -> {
var rows = jdbcTemplate.queryForList(
"SELECT processor_depths, processor_parent_indexes, processor_diagram_node_ids, " +
"exchange_bodies, processor_input_bodies, processor_output_bodies, " +
"processor_input_headers, processor_output_headers " +
"FROM route_executions WHERE route_id = 'schema-test-tree'");
// Use individual typed queries to avoid ClickHouse Array cast issues
var depths = queryArray(
"SELECT processor_depths FROM route_executions WHERE route_id = 'schema-test-tree'");
assertThat(depths).containsExactly("0", "1", "2");
assertThat(rows).hasSize(1);
var row = rows.get(0);
var parentIndexes = queryArray(
"SELECT processor_parent_indexes FROM route_executions WHERE route_id = 'schema-test-tree'");
assertThat(parentIndexes).containsExactly("-1", "0", "1");
// Verify depths: root=0, child=1, grandchild=2
@SuppressWarnings("unchecked")
var depths = (List<Number>) row.get("processor_depths");
assertThat(depths).containsExactly((short) 0, (short) 1, (short) 2);
// Verify parent indexes: root=-1, child=0 (parent is root at idx 0), grandchild=1 (parent is child at idx 1)
@SuppressWarnings("unchecked")
var parentIndexes = (List<Number>) row.get("processor_parent_indexes");
assertThat(parentIndexes).containsExactly(-1, 0, 1);
// Verify diagram node IDs
@SuppressWarnings("unchecked")
var diagramNodeIds = (List<String>) row.get("processor_diagram_node_ids");
var diagramNodeIds = queryArray(
"SELECT processor_diagram_node_ids FROM route_executions WHERE route_id = 'schema-test-tree'");
assertThat(diagramNodeIds).containsExactly("node-root", "node-child", "node-grandchild");
// Verify exchange_bodies contains concatenated text
String bodies = (String) row.get("exchange_bodies");
String bodies = jdbcTemplate.queryForObject(
"SELECT exchange_bodies FROM route_executions WHERE route_id = 'schema-test-tree'",
String.class);
assertThat(bodies).contains("root-input");
assertThat(bodies).contains("root-output");
assertThat(bodies).contains("child-input");
assertThat(bodies).contains("child-output");
// Verify per-processor input/output bodies
@SuppressWarnings("unchecked")
var inputBodies = (List<String>) row.get("processor_input_bodies");
var inputBodies = queryArray(
"SELECT processor_input_bodies FROM route_executions WHERE route_id = 'schema-test-tree'");
assertThat(inputBodies).containsExactly("root-input", "child-input", "");
@SuppressWarnings("unchecked")
var outputBodies = (List<String>) row.get("processor_output_bodies");
var outputBodies = queryArray(
"SELECT processor_output_bodies FROM route_executions WHERE route_id = 'schema-test-tree'");
assertThat(outputBodies).containsExactly("root-output", "child-output", "");
// Verify per-processor headers stored as JSON strings
@SuppressWarnings("unchecked")
var inputHeaders = (List<String>) row.get("processor_input_headers");
// Verify per-processor input headers stored as JSON strings
var inputHeaders = queryArray(
"SELECT processor_input_headers FROM route_executions WHERE route_id = 'schema-test-tree'");
assertThat(inputHeaders.get(0)).contains("Content-Type");
assertThat(inputHeaders.get(0)).contains("application/json");
});
@@ -175,22 +161,19 @@ class IngestionSchemaIT extends AbstractClickHouseIT {
postExecution(json);
await().atMost(10, SECONDS).untilAsserted(() -> {
var rows = jdbcTemplate.queryForList(
"SELECT exchange_bodies, exchange_headers " +
"FROM route_executions WHERE route_id = 'schema-test-bodies'");
assertThat(rows).hasSize(1);
var row = rows.get(0);
// Bodies should contain all sources
String bodies = (String) row.get("exchange_bodies");
String bodies = jdbcTemplate.queryForObject(
"SELECT exchange_bodies FROM route_executions WHERE route_id = 'schema-test-bodies'",
String.class);
assertThat(bodies).contains("processor-body-text");
assertThat(bodies).contains("processor-output-text");
assertThat(bodies).contains("route-level-input-body");
assertThat(bodies).contains("route-level-output-body");
// Headers should contain route-level header
String headers = (String) row.get("exchange_headers");
String headers = jdbcTemplate.queryForObject(
"SELECT exchange_headers FROM route_executions WHERE route_id = 'schema-test-bodies'",
String.class);
assertThat(headers).contains("X-Route");
assertThat(headers).contains("header-value");
});
@@ -224,26 +207,20 @@ class IngestionSchemaIT extends AbstractClickHouseIT {
postExecution(json);
await().atMost(10, SECONDS).untilAsserted(() -> {
var rows = jdbcTemplate.queryForList(
"SELECT exchange_bodies, exchange_headers, processor_input_bodies, " +
"processor_output_bodies, processor_depths, processor_parent_indexes " +
"FROM route_executions WHERE route_id = 'schema-test-null-snap'");
assertThat(rows).hasSize(1);
var row = rows.get(0);
// Empty but not null
String bodies = (String) row.get("exchange_bodies");
String bodies = jdbcTemplate.queryForObject(
"SELECT exchange_bodies FROM route_executions WHERE route_id = 'schema-test-null-snap'",
String.class);
assertThat(bodies).isNotNull();
// Depths and parent indexes still populated for tree metadata
@SuppressWarnings("unchecked")
var depths = (List<Number>) row.get("processor_depths");
assertThat(depths).containsExactly((short) 0);
var depths = queryArray(
"SELECT processor_depths FROM route_executions WHERE route_id = 'schema-test-null-snap'");
assertThat(depths).containsExactly("0");
@SuppressWarnings("unchecked")
var parentIndexes = (List<Number>) row.get("processor_parent_indexes");
assertThat(parentIndexes).containsExactly(-1);
var parentIndexes = queryArray(
"SELECT processor_parent_indexes FROM route_executions WHERE route_id = 'schema-test-null-snap'");
assertThat(parentIndexes).containsExactly("-1");
});
}
@@ -259,4 +236,26 @@ class IngestionSchemaIT extends AbstractClickHouseIT {
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.ACCEPTED);
}
/**
* Query an array column from ClickHouse and return it as a List of strings.
* Handles the ClickHouse JDBC Array type by converting via toString on elements.
*/
private List<String> queryArray(String sql) {
return jdbcTemplate.query(sql, (rs, rowNum) -> {
Object arr = rs.getArray(1).getArray();
if (arr instanceof Object[] objects) {
return Arrays.stream(objects).map(Object::toString).toList();
} else if (arr instanceof short[] shorts) {
var result = new java.util.ArrayList<String>();
for (short s : shorts) result.add(String.valueOf(s));
return result;
} else if (arr instanceof int[] ints) {
var result = new java.util.ArrayList<String>();
for (int v : ints) result.add(String.valueOf(v));
return result;
}
return List.<String>of();
}).get(0);
}
}