From 8a637df65cfd81661268c6300d920eca05f0677d Mon Sep 17 00:00:00 2001 From: hsiegeln <37154749+hsiegeln@users.noreply.github.com> Date: Mon, 16 Mar 2026 18:13:53 +0100 Subject: [PATCH] feat: add Flyway migrations for PostgreSQL/TimescaleDB schema Co-Authored-By: Claude Sonnet 4.6 --- .../resources/db/migration/V1__extensions.sql | 2 + .../resources/db/migration/V2__executions.sql | 25 ++++++ .../db/migration/V3__processor_executions.sql | 28 +++++++ .../db/migration/V4__agent_metrics.sql | 12 +++ .../db/migration/V5__route_diagrams.sql | 9 ++ .../main/resources/db/migration/V6__users.sql | 9 ++ .../db/migration/V7__oidc_config.sql | 12 +++ .../migration/V8__continuous_aggregates.sql | 83 +++++++++++++++++++ 8 files changed, 180 insertions(+) create mode 100644 cameleer3-server-app/src/main/resources/db/migration/V1__extensions.sql create mode 100644 cameleer3-server-app/src/main/resources/db/migration/V2__executions.sql create mode 100644 cameleer3-server-app/src/main/resources/db/migration/V3__processor_executions.sql create mode 100644 cameleer3-server-app/src/main/resources/db/migration/V4__agent_metrics.sql create mode 100644 cameleer3-server-app/src/main/resources/db/migration/V5__route_diagrams.sql create mode 100644 cameleer3-server-app/src/main/resources/db/migration/V6__users.sql create mode 100644 cameleer3-server-app/src/main/resources/db/migration/V7__oidc_config.sql create mode 100644 cameleer3-server-app/src/main/resources/db/migration/V8__continuous_aggregates.sql diff --git a/cameleer3-server-app/src/main/resources/db/migration/V1__extensions.sql b/cameleer3-server-app/src/main/resources/db/migration/V1__extensions.sql new file mode 100644 index 00000000..26970d8f --- /dev/null +++ b/cameleer3-server-app/src/main/resources/db/migration/V1__extensions.sql @@ -0,0 +1,2 @@ +CREATE EXTENSION IF NOT EXISTS timescaledb; +CREATE EXTENSION IF NOT EXISTS timescaledb_toolkit; diff --git a/cameleer3-server-app/src/main/resources/db/migration/V2__executions.sql b/cameleer3-server-app/src/main/resources/db/migration/V2__executions.sql new file mode 100644 index 00000000..e1eeb2fe --- /dev/null +++ b/cameleer3-server-app/src/main/resources/db/migration/V2__executions.sql @@ -0,0 +1,25 @@ +CREATE TABLE executions ( + execution_id TEXT NOT NULL, + route_id TEXT NOT NULL, + agent_id TEXT NOT NULL, + group_name TEXT NOT NULL, + status TEXT NOT NULL, + correlation_id TEXT, + exchange_id TEXT, + start_time TIMESTAMPTZ NOT NULL, + end_time TIMESTAMPTZ, + duration_ms BIGINT, + error_message TEXT, + error_stacktrace TEXT, + diagram_content_hash TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (execution_id, start_time) +); + +SELECT create_hypertable('executions', 'start_time', chunk_time_interval => INTERVAL '1 day'); + +CREATE INDEX idx_executions_agent_time ON executions (agent_id, start_time DESC); +CREATE INDEX idx_executions_route_time ON executions (route_id, start_time DESC); +CREATE INDEX idx_executions_group_time ON executions (group_name, start_time DESC); +CREATE INDEX idx_executions_correlation ON executions (correlation_id); diff --git a/cameleer3-server-app/src/main/resources/db/migration/V3__processor_executions.sql b/cameleer3-server-app/src/main/resources/db/migration/V3__processor_executions.sql new file mode 100644 index 00000000..433514b0 --- /dev/null +++ b/cameleer3-server-app/src/main/resources/db/migration/V3__processor_executions.sql @@ -0,0 +1,28 @@ +CREATE TABLE processor_executions ( + id BIGSERIAL, + execution_id TEXT NOT NULL, + processor_id TEXT NOT NULL, + processor_type TEXT NOT NULL, + diagram_node_id TEXT, + group_name TEXT NOT NULL, + route_id TEXT NOT NULL, + depth INT NOT NULL, + parent_processor_id TEXT, + status TEXT NOT NULL, + start_time TIMESTAMPTZ NOT NULL, + end_time TIMESTAMPTZ, + duration_ms BIGINT, + error_message TEXT, + error_stacktrace TEXT, + input_body TEXT, + output_body TEXT, + input_headers JSONB, + output_headers JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (execution_id, processor_id, start_time) +); + +SELECT create_hypertable('processor_executions', 'start_time', chunk_time_interval => INTERVAL '1 day'); + +CREATE INDEX idx_proc_exec_execution ON processor_executions (execution_id); +CREATE INDEX idx_proc_exec_type_time ON processor_executions (processor_type, start_time DESC); diff --git a/cameleer3-server-app/src/main/resources/db/migration/V4__agent_metrics.sql b/cameleer3-server-app/src/main/resources/db/migration/V4__agent_metrics.sql new file mode 100644 index 00000000..4ecd6cac --- /dev/null +++ b/cameleer3-server-app/src/main/resources/db/migration/V4__agent_metrics.sql @@ -0,0 +1,12 @@ +CREATE TABLE agent_metrics ( + agent_id TEXT NOT NULL, + metric_name TEXT NOT NULL, + metric_value DOUBLE PRECISION NOT NULL, + tags JSONB, + collected_at TIMESTAMPTZ NOT NULL, + server_received_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +SELECT create_hypertable('agent_metrics', 'collected_at', chunk_time_interval => INTERVAL '1 day'); + +CREATE INDEX idx_metrics_agent_name ON agent_metrics (agent_id, metric_name, collected_at DESC); diff --git a/cameleer3-server-app/src/main/resources/db/migration/V5__route_diagrams.sql b/cameleer3-server-app/src/main/resources/db/migration/V5__route_diagrams.sql new file mode 100644 index 00000000..85eb2355 --- /dev/null +++ b/cameleer3-server-app/src/main/resources/db/migration/V5__route_diagrams.sql @@ -0,0 +1,9 @@ +CREATE TABLE route_diagrams ( + content_hash TEXT PRIMARY KEY, + route_id TEXT NOT NULL, + agent_id TEXT NOT NULL, + definition TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX idx_diagrams_route_agent ON route_diagrams (route_id, agent_id); diff --git a/cameleer3-server-app/src/main/resources/db/migration/V6__users.sql b/cameleer3-server-app/src/main/resources/db/migration/V6__users.sql new file mode 100644 index 00000000..079db7dd --- /dev/null +++ b/cameleer3-server-app/src/main/resources/db/migration/V6__users.sql @@ -0,0 +1,9 @@ +CREATE TABLE users ( + user_id TEXT PRIMARY KEY, + provider TEXT NOT NULL, + email TEXT, + display_name TEXT, + roles TEXT[] NOT NULL DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/cameleer3-server-app/src/main/resources/db/migration/V7__oidc_config.sql b/cameleer3-server-app/src/main/resources/db/migration/V7__oidc_config.sql new file mode 100644 index 00000000..e46a2196 --- /dev/null +++ b/cameleer3-server-app/src/main/resources/db/migration/V7__oidc_config.sql @@ -0,0 +1,12 @@ +CREATE TABLE oidc_config ( + config_id TEXT PRIMARY KEY DEFAULT 'default', + enabled BOOLEAN NOT NULL DEFAULT false, + issuer_uri TEXT, + client_id TEXT, + client_secret TEXT, + roles_claim TEXT, + default_roles TEXT[] NOT NULL DEFAULT '{}', + auto_signup BOOLEAN DEFAULT false, + display_name_claim TEXT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/cameleer3-server-app/src/main/resources/db/migration/V8__continuous_aggregates.sql b/cameleer3-server-app/src/main/resources/db/migration/V8__continuous_aggregates.sql new file mode 100644 index 00000000..6eb5754e --- /dev/null +++ b/cameleer3-server-app/src/main/resources/db/migration/V8__continuous_aggregates.sql @@ -0,0 +1,83 @@ +-- Global stats +CREATE MATERIALIZED VIEW stats_1m_all +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', start_time) AS bucket, + COUNT(*) AS total_count, + COUNT(*) FILTER (WHERE status = 'FAILED') AS failed_count, + COUNT(*) FILTER (WHERE status = 'RUNNING') AS running_count, + SUM(duration_ms) AS duration_sum, + MAX(duration_ms) AS duration_max, + approx_percentile(0.99, percentile_agg(duration_ms::DOUBLE PRECISION)) AS p99_duration +FROM executions +WHERE status IS NOT NULL +GROUP BY bucket; + +SELECT add_continuous_aggregate_policy('stats_1m_all', + start_offset => INTERVAL '1 hour', + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 minute'); + +-- Per-application stats +CREATE MATERIALIZED VIEW stats_1m_app +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', start_time) AS bucket, + group_name, + COUNT(*) AS total_count, + COUNT(*) FILTER (WHERE status = 'FAILED') AS failed_count, + COUNT(*) FILTER (WHERE status = 'RUNNING') AS running_count, + SUM(duration_ms) AS duration_sum, + MAX(duration_ms) AS duration_max, + approx_percentile(0.99, percentile_agg(duration_ms::DOUBLE PRECISION)) AS p99_duration +FROM executions +WHERE status IS NOT NULL +GROUP BY bucket, group_name; + +SELECT add_continuous_aggregate_policy('stats_1m_app', + start_offset => INTERVAL '1 hour', + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 minute'); + +-- Per-route stats +CREATE MATERIALIZED VIEW stats_1m_route +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', start_time) AS bucket, + group_name, + route_id, + COUNT(*) AS total_count, + COUNT(*) FILTER (WHERE status = 'FAILED') AS failed_count, + COUNT(*) FILTER (WHERE status = 'RUNNING') AS running_count, + SUM(duration_ms) AS duration_sum, + MAX(duration_ms) AS duration_max, + approx_percentile(0.99, percentile_agg(duration_ms::DOUBLE PRECISION)) AS p99_duration +FROM executions +WHERE status IS NOT NULL +GROUP BY bucket, group_name, route_id; + +SELECT add_continuous_aggregate_policy('stats_1m_route', + start_offset => INTERVAL '1 hour', + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 minute'); + +-- Per-processor stats (uses denormalized group_name/route_id on processor_executions) +CREATE MATERIALIZED VIEW stats_1m_processor +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 minute', start_time) AS bucket, + group_name, + route_id, + processor_type, + COUNT(*) AS total_count, + COUNT(*) FILTER (WHERE status = 'FAILED') AS failed_count, + SUM(duration_ms) AS duration_sum, + MAX(duration_ms) AS duration_max, + approx_percentile(0.99, percentile_agg(duration_ms::DOUBLE PRECISION)) AS p99_duration +FROM processor_executions +GROUP BY bucket, group_name, route_id, processor_type; + +SELECT add_continuous_aggregate_policy('stats_1m_processor', + start_offset => INTERVAL '1 hour', + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 minute');