From 96c52b88d1ea6f2706e490fd1eb1519efe73697a Mon Sep 17 00:00:00 2001 From: hsiegeln <37154749+hsiegeln@users.noreply.github.com> Date: Wed, 11 Mar 2026 11:47:20 +0100 Subject: [PATCH] feat(01-01): add ClickHouse dependencies, Docker Compose, schema, and app config - Add clickhouse-jdbc, springdoc-openapi, actuator, testcontainers deps - Add slf4j-api to core module - Create Docker Compose with ClickHouse service on ports 8123/9000 - Create ClickHouse DDL: route_executions, route_diagrams, agent_metrics - Configure application.yml with datasource, ingestion buffer, springdoc Co-Authored-By: Claude Opus 4.6 --- cameleer3-server-app/pom.xml | 36 ++++++++++++ .../src/main/resources/application.yml | 36 ++++++++++++ cameleer3-server-core/pom.xml | 4 ++ clickhouse/init/01-schema.sql | 57 +++++++++++++++++++ docker-compose.yml | 20 +++++++ 5 files changed, 153 insertions(+) create mode 100644 clickhouse/init/01-schema.sql create mode 100644 docker-compose.yml diff --git a/cameleer3-server-app/pom.xml b/cameleer3-server-app/pom.xml index 039c4c88..3ee49526 100644 --- a/cameleer3-server-app/pom.xml +++ b/cameleer3-server-app/pom.xml @@ -27,11 +27,47 @@ org.springframework.boot spring-boot-starter-websocket + + org.springframework.boot + spring-boot-starter-actuator + + + org.springframework.boot + spring-boot-starter-jdbc + + + com.clickhouse + clickhouse-jdbc + 0.9.7 + all + + + org.springdoc + springdoc-openapi-starter-webmvc-ui + 2.8.6 + org.springframework.boot spring-boot-starter-test test + + org.testcontainers + testcontainers-clickhouse + 2.0.2 + test + + + org.testcontainers + junit-jupiter + 2.0.2 + test + + + org.awaitility + awaitility + test + diff --git a/cameleer3-server-app/src/main/resources/application.yml b/cameleer3-server-app/src/main/resources/application.yml index 11e54c54..024dc99d 100644 --- a/cameleer3-server-app/src/main/resources/application.yml +++ b/cameleer3-server-app/src/main/resources/application.yml @@ -1,2 +1,38 @@ server: port: 8081 + +spring: + datasource: + url: jdbc:ch://localhost:8123/cameleer3 + username: cameleer + password: cameleer_dev + driver-class-name: com.clickhouse.jdbc.ClickHouseDriver + jackson: + serialization: + write-dates-as-timestamps: false + deserialization: + fail-on-unknown-properties: false + +ingestion: + buffer-capacity: 50000 + batch-size: 5000 + flush-interval-ms: 1000 + +clickhouse: + ttl-days: 30 + +springdoc: + api-docs: + path: /api/v1/api-docs + swagger-ui: + path: /api/v1/swagger-ui + +management: + endpoints: + web: + base-path: /api/v1 + exposure: + include: health + endpoint: + health: + show-details: always diff --git a/cameleer3-server-core/pom.xml b/cameleer3-server-core/pom.xml index 3441cfe1..a7f22e23 100644 --- a/cameleer3-server-core/pom.xml +++ b/cameleer3-server-core/pom.xml @@ -23,6 +23,10 @@ com.fasterxml.jackson.core jackson-databind + + org.slf4j + slf4j-api + org.junit.jupiter junit-jupiter diff --git a/clickhouse/init/01-schema.sql b/clickhouse/init/01-schema.sql new file mode 100644 index 00000000..afa94849 --- /dev/null +++ b/clickhouse/init/01-schema.sql @@ -0,0 +1,57 @@ +-- Cameleer3 ClickHouse Schema +-- Tables for route executions, route diagrams, and agent metrics. + +CREATE TABLE IF NOT EXISTS route_executions ( + execution_id String, + route_id LowCardinality(String), + agent_id LowCardinality(String), + status LowCardinality(String), + start_time DateTime64(3, 'UTC'), + end_time Nullable(DateTime64(3, 'UTC')), + duration_ms UInt64, + correlation_id String, + exchange_id String, + error_message Nullable(String), + error_stacktrace Nullable(String), + -- Nested processor executions stored as parallel arrays + processor_ids Array(String), + processor_types Array(LowCardinality(String)), + processor_starts Array(DateTime64(3, 'UTC')), + processor_ends Array(DateTime64(3, 'UTC')), + processor_durations Array(UInt64), + processor_statuses Array(LowCardinality(String)), + -- Metadata + server_received_at DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'), + -- Skip indexes + INDEX idx_correlation correlation_id TYPE bloom_filter GRANULARITY 4, + INDEX idx_error error_message TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 4 +) +ENGINE = MergeTree() +PARTITION BY toYYYYMMDD(start_time) +ORDER BY (agent_id, status, start_time, execution_id) +TTL start_time + INTERVAL 30 DAY +SETTINGS ttl_only_drop_parts = 1; + +CREATE TABLE IF NOT EXISTS route_diagrams ( + content_hash String, + route_id LowCardinality(String), + agent_id LowCardinality(String), + definition String, + created_at DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC') +) +ENGINE = ReplacingMergeTree(created_at) +ORDER BY (content_hash); + +CREATE TABLE IF NOT EXISTS agent_metrics ( + agent_id LowCardinality(String), + collected_at DateTime64(3, 'UTC'), + metric_name LowCardinality(String), + metric_value Float64, + tags Map(String, String), + server_received_at DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC') +) +ENGINE = MergeTree() +PARTITION BY toYYYYMMDD(collected_at) +ORDER BY (agent_id, metric_name, collected_at) +TTL collected_at + INTERVAL 30 DAY +SETTINGS ttl_only_drop_parts = 1; diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..4fa23d89 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,20 @@ +services: + clickhouse: + image: clickhouse/clickhouse-server:25.3 + ports: + - "8123:8123" + - "9000:9000" + volumes: + - clickhouse-data:/var/lib/clickhouse + - ./clickhouse/init:/docker-entrypoint-initdb.d + environment: + CLICKHOUSE_USER: cameleer + CLICKHOUSE_PASSWORD: cameleer_dev + CLICKHOUSE_DB: cameleer3 + ulimits: + nofile: + soft: 262144 + hard: 262144 + +volumes: + clickhouse-data: