cameleer/cameleer-server
1
0
Fork 0
Code Issues 44 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: remove TimescaleDB, dead PG stores, and storage feature flags
Some checks failed
CI / cleanup-branch (push) Has been skipped
Details
CI / build (push) Failing after 32s
Details
CI / docker (push) Has been skipped
Details
CI / deploy (push) Has been skipped
Details
CI / deploy-feature (push) Has been skipped
Details

Browse Source 
Complete the ClickHouse migration by removing all PostgreSQL analytics
code. PostgreSQL now serves only RBAC, config, and audit — all
observability data is exclusively in ClickHouse.

- Delete 6 dead PostgreSQL store classes (executions, stats, diagrams,
  events, metrics, metrics-query) and 2 integration tests
- Delete RetentionScheduler (ClickHouse TTL handles retention)
- Remove all 7 cameleer.storage.* feature flags from application.yml
- Remove all @ConditionalOnProperty from ClickHouse beans in StorageBeanConfig
- Consolidate 14 Flyway migrations (V1-V14) into single clean V1 with
  only RBAC/config/audit tables (no TimescaleDB, no analytics tables)
- Switch from timescale/timescaledb-ha:pg16 to postgres:16 everywhere
  (docker-compose, deploy/postgres.yaml, test containers)
- Remove TimescaleDB check and /metrics-pipeline from DatabaseAdminController
- Set clickhouse.enabled default to true

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
hsiegeln
2026-04-01 20:10:58 +02:00
parent 283e38a20d
commit 188810e54b
37 changed files with 65 additions and 1607 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
cameleer3-server-app/src/main/resources/application.yml
View File

@@ -42,15 +42,6 @@ cameleer:
indexer:
debounce-ms: ${CAMELEER_INDEXER_DEBOUNCE_MS:2000}
queue-size: ${CAMELEER_INDEXER_QUEUE_SIZE:10000}
retention-days: ${CAMELEER_RETENTION_DAYS:30}
storage:
metrics: ${CAMELEER_STORAGE_METRICS:postgres}
search: ${CAMELEER_STORAGE_SEARCH:clickhouse}
stats: ${CAMELEER_STORAGE_STATS:clickhouse}
diagrams: ${CAMELEER_STORAGE_DIAGRAMS:clickhouse}
events: ${CAMELEER_STORAGE_EVENTS:clickhouse}
logs: ${CAMELEER_STORAGE_LOGS:clickhouse}
executions: ${CAMELEER_STORAGE_EXECUTIONS:clickhouse}
security:
access-token-expiry-ms: 3600000
@@ -70,7 +61,7 @@ springdoc:
path: /api/v1/swagger-ui
clickhouse:
enabled: ${CLICKHOUSE_ENABLED:false}
enabled: ${CLICKHOUSE_ENABLED:true}
url: ${CLICKHOUSE_URL:jdbc:clickhouse://localhost:8123/cameleer}
username: ${CLICKHOUSE_USERNAME:default}
password: ${CLICKHOUSE_PASSWORD:}

2
cameleer3-server-app/src/main/resources/clickhouse/V4__stats_tables_and_mvs.sql
View File

@@ -1,5 +1,5 @@
-- V4__stats_tables_and_mvs.sql
-- Materialized views replacing TimescaleDB continuous aggregates.
-- Pre-aggregated statistics tables and materialized views.
-- Tables use AggregatingMergeTree, MVs use -State combinators.
-- stats_1m_all (global)

23
cameleer3-server-app/src/main/resources/db/migration/V10__error_categorization_and_tracing.sql
View File

@@ -1,23 +0,0 @@
-- executions: store raw processor tree for faithful detail response
ALTER TABLE executions ADD COLUMN processors_json JSONB;
-- executions: error categorization + OTel tracing
ALTER TABLE executions ADD COLUMN error_type TEXT;
ALTER TABLE executions ADD COLUMN error_category TEXT;
ALTER TABLE executions ADD COLUMN root_cause_type TEXT;
ALTER TABLE executions ADD COLUMN root_cause_message TEXT;
ALTER TABLE executions ADD COLUMN trace_id TEXT;
ALTER TABLE executions ADD COLUMN span_id TEXT;
-- processor_executions: error categorization + circuit breaker
ALTER TABLE processor_executions ADD COLUMN error_type TEXT;
ALTER TABLE processor_executions ADD COLUMN error_category TEXT;
ALTER TABLE processor_executions ADD COLUMN root_cause_type TEXT;
ALTER TABLE processor_executions ADD COLUMN root_cause_message TEXT;
ALTER TABLE processor_executions ADD COLUMN error_handler_type TEXT;
ALTER TABLE processor_executions ADD COLUMN circuit_breaker_state TEXT;
ALTER TABLE processor_executions ADD COLUMN fallback_triggered BOOLEAN;
-- Remove erroneous depth columns from V9
ALTER TABLE processor_executions DROP COLUMN IF EXISTS split_depth;
ALTER TABLE processor_executions DROP COLUMN IF EXISTS loop_depth;

10
cameleer3-server-app/src/main/resources/db/migration/V11__has_trace_data.sql
View File

@@ -1,10 +0,0 @@
-- Flag indicating whether any processor in this execution captured trace data
ALTER TABLE executions ADD COLUMN IF NOT EXISTS has_trace_data BOOLEAN NOT NULL DEFAULT FALSE;
-- Backfill: set flag for existing executions that have processor trace data
UPDATE executions e SET has_trace_data = TRUE
WHERE EXISTS (
SELECT 1 FROM processor_executions pe
WHERE pe.execution_id = e.execution_id
AND (pe.input_body IS NOT NULL OR pe.output_body IS NOT NULL)
);

11
cameleer3-server-app/src/main/resources/db/migration/V12__app_settings.sql
View File

@@ -1,11 +0,0 @@
-- Per-application dashboard settings (SLA thresholds, health dot thresholds)
CREATE TABLE app_settings (
app_id TEXT PRIMARY KEY,
sla_threshold_ms INTEGER NOT NULL DEFAULT 300,
health_error_warn DOUBLE PRECISION NOT NULL DEFAULT 1.0,
health_error_crit DOUBLE PRECISION NOT NULL DEFAULT 5.0,
health_sla_warn DOUBLE PRECISION NOT NULL DEFAULT 99.0,
health_sla_crit DOUBLE PRECISION NOT NULL DEFAULT 95.0,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);

7
cameleer3-server-app/src/main/resources/db/migration/V13__is_replay.sql
View File

@@ -1,7 +0,0 @@
-- Flag indicating whether this execution is a replayed exchange
ALTER TABLE executions ADD COLUMN IF NOT EXISTS is_replay BOOLEAN NOT NULL DEFAULT FALSE;
-- Backfill: check inputHeaders JSON for X-Cameleer-Replay header
UPDATE executions SET is_replay = TRUE
WHERE input_headers IS NOT NULL
AND input_headers::jsonb ? 'X-Cameleer-Replay';

16
cameleer3-server-app/src/main/resources/db/migration/V14__rename_identity_columns.sql
View File

@@ -1,16 +0,0 @@
-- Rename agent identity columns for protocol v2 alignment.
ALTER TABLE executions RENAME COLUMN agent_id TO instance_id;
ALTER TABLE executions RENAME COLUMN application_name TO application_id;
ALTER TABLE processor_executions RENAME COLUMN application_name TO application_id;
ALTER TABLE agent_metrics RENAME COLUMN agent_id TO instance_id;
ALTER TABLE route_diagrams RENAME COLUMN agent_id TO instance_id;
ALTER TABLE route_diagrams RENAME COLUMN application_name TO application_id;
ALTER TABLE agent_events RENAME COLUMN agent_id TO instance_id;
ALTER TABLE agent_events RENAME COLUMN app_id TO application_id;
ALTER TABLE app_settings RENAME COLUMN app_id TO application_id;

232
cameleer3-server-app/src/main/resources/db/migration/V1__init.sql
View File

@@ -1,8 +1,6 @@
-- V1__init.sql - Consolidated schema for Cameleer3
-- Extensions
CREATE EXTENSION IF NOT EXISTS timescaledb;
CREATE EXTENSION IF NOT EXISTS timescaledb_toolkit;
-- V1__init.sql — PostgreSQL schema for Cameleer3 Server
-- PostgreSQL stores RBAC, configuration, and audit data only.
-- All observability data (executions, metrics, diagrams, logs, stats) is in ClickHouse.
-- =============================================================
-- RBAC
@@ -40,7 +38,6 @@ CREATE TABLE groups (
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
-- Built-in Admins group
INSERT INTO groups (id, name) VALUES
('00000000-0000-0000-0000-000000000010', 'Admins');
@@ -50,7 +47,6 @@ CREATE TABLE group_roles (
PRIMARY KEY (group_id, role_id)
);
-- Assign ADMIN role to Admins group
INSERT INTO group_roles (group_id, role_id) VALUES
('00000000-0000-0000-0000-000000000010', '00000000-0000-0000-0000-000000000004');
@@ -71,113 +67,6 @@ CREATE INDEX idx_user_groups_user_id ON user_groups(user_id);
CREATE INDEX idx_group_roles_group_id ON group_roles(group_id);
CREATE INDEX idx_groups_parent ON groups(parent_group_id);
-- =============================================================
-- Execution data (TimescaleDB hypertables)
-- =============================================================
CREATE TABLE executions (
execution_id TEXT NOT NULL,
route_id TEXT NOT NULL,
agent_id TEXT NOT NULL,
application_name TEXT NOT NULL,
status TEXT NOT NULL,
correlation_id TEXT,
exchange_id TEXT,
start_time TIMESTAMPTZ NOT NULL,
end_time TIMESTAMPTZ,
duration_ms BIGINT,
error_message TEXT,
error_stacktrace TEXT,
diagram_content_hash TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (execution_id, start_time)
);
SELECT create_hypertable('executions', 'start_time', chunk_time_interval => INTERVAL '1 day');
CREATE INDEX idx_executions_agent_time ON executions (agent_id, start_time DESC);
CREATE INDEX idx_executions_route_time ON executions (route_id, start_time DESC);
CREATE INDEX idx_executions_app_time ON executions (application_name, start_time DESC);
CREATE INDEX idx_executions_correlation ON executions (correlation_id);
CREATE TABLE processor_executions (
id BIGSERIAL,
execution_id TEXT NOT NULL,
processor_id TEXT NOT NULL,
processor_type TEXT NOT NULL,
diagram_node_id TEXT,
application_name TEXT NOT NULL,
route_id TEXT NOT NULL,
depth INT NOT NULL,
parent_processor_id TEXT,
status TEXT NOT NULL,
start_time TIMESTAMPTZ NOT NULL,
end_time TIMESTAMPTZ,
duration_ms BIGINT,
error_message TEXT,
error_stacktrace TEXT,
input_body TEXT,
output_body TEXT,
input_headers JSONB,
output_headers JSONB,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
UNIQUE (execution_id, processor_id, start_time)
);
SELECT create_hypertable('processor_executions', 'start_time', chunk_time_interval => INTERVAL '1 day');
CREATE INDEX idx_proc_exec_execution ON processor_executions (execution_id);
CREATE INDEX idx_proc_exec_type_time ON processor_executions (processor_type, start_time DESC);
-- =============================================================
-- Agent metrics
-- =============================================================
CREATE TABLE agent_metrics (
agent_id TEXT NOT NULL,
metric_name TEXT NOT NULL,
metric_value DOUBLE PRECISION NOT NULL,
tags JSONB,
collected_at TIMESTAMPTZ NOT NULL,
server_received_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
SELECT create_hypertable('agent_metrics', 'collected_at', chunk_time_interval => INTERVAL '1 day');
CREATE INDEX idx_metrics_agent_name ON agent_metrics (agent_id, metric_name, collected_at DESC);
-- =============================================================
-- Route diagrams
-- =============================================================
CREATE TABLE route_diagrams (
content_hash TEXT PRIMARY KEY,
route_id TEXT NOT NULL,
agent_id TEXT NOT NULL,
definition TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX idx_diagrams_route_agent ON route_diagrams (route_id, agent_id);
-- =============================================================
-- Agent events
-- =============================================================
CREATE TABLE agent_events (
id BIGSERIAL PRIMARY KEY,
agent_id TEXT NOT NULL,
app_id TEXT NOT NULL,
event_type TEXT NOT NULL,
detail TEXT,
timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX idx_agent_events_agent ON agent_events(agent_id, timestamp DESC);
CREATE INDEX idx_agent_events_app ON agent_events(app_id, timestamp DESC);
CREATE INDEX idx_agent_events_time ON agent_events(timestamp DESC);
-- =============================================================
-- Server configuration
-- =============================================================
@@ -190,7 +79,30 @@ CREATE TABLE server_config (
);
-- =============================================================
-- Admin
-- Application configuration
-- =============================================================
CREATE TABLE application_config (
application TEXT PRIMARY KEY,
config_val JSONB NOT NULL,
version INTEGER NOT NULL DEFAULT 1,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_by TEXT
);
CREATE TABLE app_settings (
application_id TEXT PRIMARY KEY,
sla_threshold_ms INTEGER NOT NULL DEFAULT 300,
health_error_warn DOUBLE PRECISION NOT NULL DEFAULT 1.0,
health_error_crit DOUBLE PRECISION NOT NULL DEFAULT 5.0,
health_sla_warn DOUBLE PRECISION NOT NULL DEFAULT 99.0,
health_sla_crit DOUBLE PRECISION NOT NULL DEFAULT 95.0,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
-- =============================================================
-- Audit log
-- =============================================================
CREATE TABLE audit_log (
@@ -211,93 +123,3 @@ CREATE INDEX idx_audit_log_username ON audit_log (username);
CREATE INDEX idx_audit_log_category ON audit_log (category);
CREATE INDEX idx_audit_log_action ON audit_log (action);
CREATE INDEX idx_audit_log_target ON audit_log (target);
-- =============================================================
-- Continuous aggregates
-- =============================================================
CREATE MATERIALIZED VIEW stats_1m_all
WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS
SELECT
time_bucket('1 minute', start_time) AS bucket,
COUNT(*) AS total_count,
COUNT(*) FILTER (WHERE status = 'FAILED') AS failed_count,
COUNT(*) FILTER (WHERE status = 'RUNNING') AS running_count,
SUM(duration_ms) AS duration_sum,
MAX(duration_ms) AS duration_max,
approx_percentile(0.99, percentile_agg(duration_ms::DOUBLE PRECISION)) AS p99_duration
FROM executions
WHERE status IS NOT NULL
GROUP BY bucket
WITH NO DATA;
CREATE MATERIALIZED VIEW stats_1m_app
WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS
SELECT
time_bucket('1 minute', start_time) AS bucket,
application_name,
COUNT(*) AS total_count,
COUNT(*) FILTER (WHERE status = 'FAILED') AS failed_count,
COUNT(*) FILTER (WHERE status = 'RUNNING') AS running_count,
SUM(duration_ms) AS duration_sum,
MAX(duration_ms) AS duration_max,
approx_percentile(0.99, percentile_agg(duration_ms::DOUBLE PRECISION)) AS p99_duration
FROM executions
WHERE status IS NOT NULL
GROUP BY bucket, application_name
WITH NO DATA;
CREATE MATERIALIZED VIEW stats_1m_route
WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS
SELECT
time_bucket('1 minute', start_time) AS bucket,
application_name,
route_id,
COUNT(*) AS total_count,
COUNT(*) FILTER (WHERE status = 'FAILED') AS failed_count,
COUNT(*) FILTER (WHERE status = 'RUNNING') AS running_count,
SUM(duration_ms) AS duration_sum,
MAX(duration_ms) AS duration_max,
approx_percentile(0.99, percentile_agg(duration_ms::DOUBLE PRECISION)) AS p99_duration
FROM executions
WHERE status IS NOT NULL
GROUP BY bucket, application_name, route_id
WITH NO DATA;
CREATE MATERIALIZED VIEW stats_1m_processor
WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS
SELECT
time_bucket('1 minute', start_time) AS bucket,
application_name,
route_id,
processor_type,
COUNT(*) AS total_count,
COUNT(*) FILTER (WHERE status = 'FAILED') AS failed_count,
SUM(duration_ms) AS duration_sum,
MAX(duration_ms) AS duration_max,
approx_percentile(0.99, percentile_agg(duration_ms::DOUBLE PRECISION)) AS p99_duration
FROM processor_executions
GROUP BY bucket, application_name, route_id, processor_type
WITH NO DATA;
CREATE MATERIALIZED VIEW stats_1m_processor_detail
WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS
SELECT
time_bucket('1 minute', start_time) AS bucket,
application_name,
route_id,
processor_id,
processor_type,
COUNT(*) AS total_count,
COUNT(*) FILTER (WHERE status = 'FAILED') AS failed_count,
SUM(duration_ms) AS duration_sum,
MAX(duration_ms) AS duration_max,
approx_percentile(0.99, percentile_agg(duration_ms)) AS p99_duration
FROM processor_executions
GROUP BY bucket, application_name, route_id, processor_id, processor_type
WITH NO DATA;

38
cameleer3-server-app/src/main/resources/db/migration/V2__policies.sql
View File

@@ -1,38 +0,0 @@
-- V2__policies.sql - TimescaleDB policies (must run outside transaction)
-- flyway:executeInTransaction=false
-- Agent metrics retention & compression
ALTER TABLE agent_metrics SET (timescaledb.compress);
SELECT add_retention_policy('agent_metrics', INTERVAL '90 days', if_not_exists => true);
SELECT add_compression_policy('agent_metrics', INTERVAL '7 days', if_not_exists => true);
-- Continuous aggregate refresh policies
SELECT add_continuous_aggregate_policy('stats_1m_all',
start_offset => INTERVAL '1 hour',
end_offset => INTERVAL '1 minute',
schedule_interval => INTERVAL '1 minute',
if_not_exists => true);
SELECT add_continuous_aggregate_policy('stats_1m_app',
start_offset => INTERVAL '1 hour',
end_offset => INTERVAL '1 minute',
schedule_interval => INTERVAL '1 minute',
if_not_exists => true);
SELECT add_continuous_aggregate_policy('stats_1m_route',
start_offset => INTERVAL '1 hour',
end_offset => INTERVAL '1 minute',
schedule_interval => INTERVAL '1 minute',
if_not_exists => true);
SELECT add_continuous_aggregate_policy('stats_1m_processor',
start_offset => INTERVAL '1 hour',
end_offset => INTERVAL '1 minute',
schedule_interval => INTERVAL '1 minute',
if_not_exists => true);
SELECT add_continuous_aggregate_policy('stats_1m_processor_detail',
start_offset => INTERVAL '1 hour',
end_offset => INTERVAL '1 minute',
schedule_interval => INTERVAL '1 minute',
if_not_exists => true);

9
cameleer3-server-app/src/main/resources/db/migration/V3__engine_level_and_snapshots.sql
View File

@@ -1,9 +0,0 @@
-- Add engine level and route-level snapshot columns to executions table.
-- Required for REGULAR engine level where route-level payloads exist but
-- no processor execution records are created.
ALTER TABLE executions ADD COLUMN IF NOT EXISTS engine_level VARCHAR(16);
ALTER TABLE executions ADD COLUMN IF NOT EXISTS input_body TEXT;
ALTER TABLE executions ADD COLUMN IF NOT EXISTS output_body TEXT;
ALTER TABLE executions ADD COLUMN IF NOT EXISTS input_headers JSONB;
ALTER TABLE executions ADD COLUMN IF NOT EXISTS output_headers JSONB;

9
cameleer3-server-app/src/main/resources/db/migration/V4__application_config.sql
View File

@@ -1,9 +0,0 @@
-- Per-application configuration for agent observability settings.
-- Agents download this at startup and receive updates via SSE CONFIG_UPDATE.
CREATE TABLE application_config (
application TEXT PRIMARY KEY,
config_val JSONB NOT NULL,
version INTEGER NOT NULL DEFAULT 1,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_by TEXT
);

2
cameleer3-server-app/src/main/resources/db/migration/V5__attributes.sql
View File

@@ -1,2 +0,0 @@
ALTER TABLE executions ADD COLUMN IF NOT EXISTS attributes JSONB;
ALTER TABLE processor_executions ADD COLUMN IF NOT EXISTS attributes JSONB;

1
cameleer3-server-app/src/main/resources/db/migration/V6__drop_diagram_node_id.sql
View File

@@ -1 +0,0 @@
ALTER TABLE processor_executions DROP COLUMN IF EXISTS diagram_node_id;

2
cameleer3-server-app/src/main/resources/db/migration/V7__diagram_application_name.sql
View File

@@ -1,2 +0,0 @@
ALTER TABLE route_diagrams ADD COLUMN IF NOT EXISTS application_name TEXT NOT NULL DEFAULT '';
CREATE INDEX IF NOT EXISTS idx_diagrams_application ON route_diagrams (application_name);

5
cameleer3-server-app/src/main/resources/db/migration/V8__processor_iteration_fields.sql
View File

@@ -1,5 +0,0 @@
ALTER TABLE processor_executions ADD COLUMN IF NOT EXISTS loop_index INTEGER;
ALTER TABLE processor_executions ADD COLUMN IF NOT EXISTS loop_size INTEGER;
ALTER TABLE processor_executions ADD COLUMN IF NOT EXISTS split_index INTEGER;
ALTER TABLE processor_executions ADD COLUMN IF NOT EXISTS split_size INTEGER;
ALTER TABLE processor_executions ADD COLUMN IF NOT EXISTS multicast_index INTEGER;

3
cameleer3-server-app/src/main/resources/db/migration/V9__resolved_endpoint_uri.sql
View File

@@ -1,3 +0,0 @@
ALTER TABLE processor_executions ADD COLUMN resolved_endpoint_uri TEXT;
ALTER TABLE processor_executions ADD COLUMN split_depth INTEGER DEFAULT 0;
ALTER TABLE processor_executions ADD COLUMN loop_depth INTEGER DEFAULT 0;