diff --git a/cameleer3-server-app/src/main/resources/db/migration/V10__audit_log.sql b/cameleer3-server-app/src/main/resources/db/migration/V10__audit_log.sql deleted file mode 100644 index d10230a6..00000000 --- a/cameleer3-server-app/src/main/resources/db/migration/V10__audit_log.sql +++ /dev/null @@ -1,18 +0,0 @@ -CREATE TABLE audit_log ( - id BIGSERIAL PRIMARY KEY, - timestamp TIMESTAMPTZ NOT NULL DEFAULT now(), - username TEXT NOT NULL, - action TEXT NOT NULL, - category TEXT NOT NULL, - target TEXT, - detail JSONB, - result TEXT NOT NULL, - ip_address TEXT, - user_agent TEXT -); - -CREATE INDEX idx_audit_log_timestamp ON audit_log (timestamp DESC); -CREATE INDEX idx_audit_log_username ON audit_log (username); -CREATE INDEX idx_audit_log_category ON audit_log (category); -CREATE INDEX idx_audit_log_action ON audit_log (action); -CREATE INDEX idx_audit_log_target ON audit_log (target); diff --git a/cameleer3-server-app/src/main/resources/db/migration/V1__extensions.sql b/cameleer3-server-app/src/main/resources/db/migration/V1__extensions.sql deleted file mode 100644 index 26970d8f..00000000 --- a/cameleer3-server-app/src/main/resources/db/migration/V1__extensions.sql +++ /dev/null @@ -1,2 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS timescaledb; -CREATE EXTENSION IF NOT EXISTS timescaledb_toolkit; diff --git a/cameleer3-server-app/src/main/resources/db/migration/V1__init.sql b/cameleer3-server-app/src/main/resources/db/migration/V1__init.sql new file mode 100644 index 00000000..94db463b --- /dev/null +++ b/cameleer3-server-app/src/main/resources/db/migration/V1__init.sql @@ -0,0 +1,289 @@ +-- V1__init.sql - Consolidated schema for Cameleer3 + +-- Extensions +CREATE EXTENSION IF NOT EXISTS timescaledb; +CREATE EXTENSION IF NOT EXISTS timescaledb_toolkit; + +-- ============================================================= +-- RBAC +-- ============================================================= + +CREATE TABLE users ( + user_id TEXT PRIMARY KEY, + provider TEXT NOT NULL, + email TEXT, + display_name TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE roles ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL UNIQUE, + description TEXT NOT NULL DEFAULT '', + scope TEXT NOT NULL DEFAULT 'custom', + system BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +INSERT INTO roles (id, name, description, scope, system) VALUES + ('00000000-0000-0000-0000-000000000001', 'AGENT', 'Agent registration and data ingestion', 'system-wide', true), + ('00000000-0000-0000-0000-000000000002', 'VIEWER', 'Read-only access to dashboards and data', 'system-wide', true), + ('00000000-0000-0000-0000-000000000003', 'OPERATOR', 'Operational commands (start/stop/configure agents)', 'system-wide', true), + ('00000000-0000-0000-0000-000000000004', 'ADMIN', 'Full administrative access', 'system-wide', true); + +CREATE TABLE groups ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL UNIQUE, + parent_group_id UUID REFERENCES groups(id) ON DELETE SET NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE group_roles ( + group_id UUID NOT NULL REFERENCES groups(id) ON DELETE CASCADE, + role_id UUID NOT NULL REFERENCES roles(id) ON DELETE CASCADE, + PRIMARY KEY (group_id, role_id) +); + +CREATE TABLE user_groups ( + user_id TEXT NOT NULL REFERENCES users(user_id) ON DELETE CASCADE, + group_id UUID NOT NULL REFERENCES groups(id) ON DELETE CASCADE, + PRIMARY KEY (user_id, group_id) +); + +CREATE TABLE user_roles ( + user_id TEXT NOT NULL REFERENCES users(user_id) ON DELETE CASCADE, + role_id UUID NOT NULL REFERENCES roles(id) ON DELETE CASCADE, + PRIMARY KEY (user_id, role_id) +); + +CREATE INDEX idx_user_roles_user_id ON user_roles(user_id); +CREATE INDEX idx_user_groups_user_id ON user_groups(user_id); +CREATE INDEX idx_group_roles_group_id ON group_roles(group_id); +CREATE INDEX idx_groups_parent ON groups(parent_group_id); + +-- ============================================================= +-- Execution data (TimescaleDB hypertables) +-- ============================================================= + +CREATE TABLE executions ( + execution_id TEXT NOT NULL, + route_id TEXT NOT NULL, + agent_id TEXT NOT NULL, + group_name TEXT NOT NULL, + status TEXT NOT NULL, + correlation_id TEXT, + exchange_id TEXT, + start_time TIMESTAMPTZ NOT NULL, + end_time TIMESTAMPTZ, + duration_ms BIGINT, + error_message TEXT, + error_stacktrace TEXT, + diagram_content_hash TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (execution_id, start_time) +); + +SELECT create_hypertable('executions', 'start_time', chunk_time_interval => INTERVAL '1 day'); + +CREATE INDEX idx_executions_agent_time ON executions (agent_id, start_time DESC); +CREATE INDEX idx_executions_route_time ON executions (route_id, start_time DESC); +CREATE INDEX idx_executions_group_time ON executions (group_name, start_time DESC); +CREATE INDEX idx_executions_correlation ON executions (correlation_id); + +CREATE TABLE processor_executions ( + id BIGSERIAL, + execution_id TEXT NOT NULL, + processor_id TEXT NOT NULL, + processor_type TEXT NOT NULL, + diagram_node_id TEXT, + group_name TEXT NOT NULL, + route_id TEXT NOT NULL, + depth INT NOT NULL, + parent_processor_id TEXT, + status TEXT NOT NULL, + start_time TIMESTAMPTZ NOT NULL, + end_time TIMESTAMPTZ, + duration_ms BIGINT, + error_message TEXT, + error_stacktrace TEXT, + input_body TEXT, + output_body TEXT, + input_headers JSONB, + output_headers JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (execution_id, processor_id, start_time) +); + +SELECT create_hypertable('processor_executions', 'start_time', chunk_time_interval => INTERVAL '1 day'); + +CREATE INDEX idx_proc_exec_execution ON processor_executions (execution_id); +CREATE INDEX idx_proc_exec_type_time ON processor_executions (processor_type, start_time DESC); + +-- ============================================================= +-- Agent metrics +-- ============================================================= + +CREATE TABLE agent_metrics ( + agent_id TEXT NOT NULL, + metric_name TEXT NOT NULL, + metric_value DOUBLE PRECISION NOT NULL, + tags JSONB, + collected_at TIMESTAMPTZ NOT NULL, + server_received_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +SELECT create_hypertable('agent_metrics', 'collected_at', chunk_time_interval => INTERVAL '1 day'); + +CREATE INDEX idx_metrics_agent_name ON agent_metrics (agent_id, metric_name, collected_at DESC); + +-- ============================================================= +-- Route diagrams +-- ============================================================= + +CREATE TABLE route_diagrams ( + content_hash TEXT PRIMARY KEY, + route_id TEXT NOT NULL, + agent_id TEXT NOT NULL, + definition TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX idx_diagrams_route_agent ON route_diagrams (route_id, agent_id); + +-- ============================================================= +-- OIDC configuration +-- ============================================================= + +CREATE TABLE oidc_config ( + config_id TEXT PRIMARY KEY DEFAULT 'default', + enabled BOOLEAN NOT NULL DEFAULT false, + issuer_uri TEXT, + client_id TEXT, + client_secret TEXT, + roles_claim TEXT, + default_roles TEXT[] NOT NULL DEFAULT '{}', + auto_signup BOOLEAN DEFAULT false, + display_name_claim TEXT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- ============================================================= +-- Continuous aggregates +-- ============================================================= + +CREATE MATERIALIZED VIEW stats_1m_all +WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS +SELECT + time_bucket('1 minute', start_time) AS bucket, + COUNT(*) AS total_count, + COUNT(*) FILTER (WHERE status = 'FAILED') AS failed_count, + COUNT(*) FILTER (WHERE status = 'RUNNING') AS running_count, + SUM(duration_ms) AS duration_sum, + MAX(duration_ms) AS duration_max, + approx_percentile(0.99, percentile_agg(duration_ms::DOUBLE PRECISION)) AS p99_duration +FROM executions +WHERE status IS NOT NULL +GROUP BY bucket +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('stats_1m_all', + start_offset => INTERVAL '1 hour', + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 minute'); + +CREATE MATERIALIZED VIEW stats_1m_app +WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS +SELECT + time_bucket('1 minute', start_time) AS bucket, + group_name, + COUNT(*) AS total_count, + COUNT(*) FILTER (WHERE status = 'FAILED') AS failed_count, + COUNT(*) FILTER (WHERE status = 'RUNNING') AS running_count, + SUM(duration_ms) AS duration_sum, + MAX(duration_ms) AS duration_max, + approx_percentile(0.99, percentile_agg(duration_ms::DOUBLE PRECISION)) AS p99_duration +FROM executions +WHERE status IS NOT NULL +GROUP BY bucket, group_name +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('stats_1m_app', + start_offset => INTERVAL '1 hour', + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 minute'); + +CREATE MATERIALIZED VIEW stats_1m_route +WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS +SELECT + time_bucket('1 minute', start_time) AS bucket, + group_name, + route_id, + COUNT(*) AS total_count, + COUNT(*) FILTER (WHERE status = 'FAILED') AS failed_count, + COUNT(*) FILTER (WHERE status = 'RUNNING') AS running_count, + SUM(duration_ms) AS duration_sum, + MAX(duration_ms) AS duration_max, + approx_percentile(0.99, percentile_agg(duration_ms::DOUBLE PRECISION)) AS p99_duration +FROM executions +WHERE status IS NOT NULL +GROUP BY bucket, group_name, route_id +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('stats_1m_route', + start_offset => INTERVAL '1 hour', + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 minute'); + +CREATE MATERIALIZED VIEW stats_1m_processor +WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS +SELECT + time_bucket('1 minute', start_time) AS bucket, + group_name, + route_id, + processor_type, + COUNT(*) AS total_count, + COUNT(*) FILTER (WHERE status = 'FAILED') AS failed_count, + SUM(duration_ms) AS duration_sum, + MAX(duration_ms) AS duration_max, + approx_percentile(0.99, percentile_agg(duration_ms::DOUBLE PRECISION)) AS p99_duration +FROM processor_executions +GROUP BY bucket, group_name, route_id, processor_type +WITH NO DATA; + +SELECT add_continuous_aggregate_policy('stats_1m_processor', + start_offset => INTERVAL '1 hour', + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 minute'); + +-- ============================================================= +-- Admin +-- ============================================================= + +CREATE TABLE admin_thresholds ( + id INTEGER PRIMARY KEY DEFAULT 1, + config JSONB NOT NULL DEFAULT '{}', + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_by TEXT NOT NULL, + CONSTRAINT single_row CHECK (id = 1) +); + +CREATE TABLE audit_log ( + id BIGSERIAL PRIMARY KEY, + timestamp TIMESTAMPTZ NOT NULL DEFAULT now(), + username TEXT NOT NULL, + action TEXT NOT NULL, + category TEXT NOT NULL, + target TEXT, + detail JSONB, + result TEXT NOT NULL, + ip_address TEXT, + user_agent TEXT +); + +CREATE INDEX idx_audit_log_timestamp ON audit_log (timestamp DESC); +CREATE INDEX idx_audit_log_username ON audit_log (username); +CREATE INDEX idx_audit_log_category ON audit_log (category); +CREATE INDEX idx_audit_log_action ON audit_log (action); +CREATE INDEX idx_audit_log_target ON audit_log (target); diff --git a/cameleer3-server-app/src/main/resources/db/migration/V2__executions.sql b/cameleer3-server-app/src/main/resources/db/migration/V2__executions.sql deleted file mode 100644 index e1eeb2fe..00000000 --- a/cameleer3-server-app/src/main/resources/db/migration/V2__executions.sql +++ /dev/null @@ -1,25 +0,0 @@ -CREATE TABLE executions ( - execution_id TEXT NOT NULL, - route_id TEXT NOT NULL, - agent_id TEXT NOT NULL, - group_name TEXT NOT NULL, - status TEXT NOT NULL, - correlation_id TEXT, - exchange_id TEXT, - start_time TIMESTAMPTZ NOT NULL, - end_time TIMESTAMPTZ, - duration_ms BIGINT, - error_message TEXT, - error_stacktrace TEXT, - diagram_content_hash TEXT, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), - PRIMARY KEY (execution_id, start_time) -); - -SELECT create_hypertable('executions', 'start_time', chunk_time_interval => INTERVAL '1 day'); - -CREATE INDEX idx_executions_agent_time ON executions (agent_id, start_time DESC); -CREATE INDEX idx_executions_route_time ON executions (route_id, start_time DESC); -CREATE INDEX idx_executions_group_time ON executions (group_name, start_time DESC); -CREATE INDEX idx_executions_correlation ON executions (correlation_id); diff --git a/cameleer3-server-app/src/main/resources/db/migration/V3__processor_executions.sql b/cameleer3-server-app/src/main/resources/db/migration/V3__processor_executions.sql deleted file mode 100644 index 433514b0..00000000 --- a/cameleer3-server-app/src/main/resources/db/migration/V3__processor_executions.sql +++ /dev/null @@ -1,28 +0,0 @@ -CREATE TABLE processor_executions ( - id BIGSERIAL, - execution_id TEXT NOT NULL, - processor_id TEXT NOT NULL, - processor_type TEXT NOT NULL, - diagram_node_id TEXT, - group_name TEXT NOT NULL, - route_id TEXT NOT NULL, - depth INT NOT NULL, - parent_processor_id TEXT, - status TEXT NOT NULL, - start_time TIMESTAMPTZ NOT NULL, - end_time TIMESTAMPTZ, - duration_ms BIGINT, - error_message TEXT, - error_stacktrace TEXT, - input_body TEXT, - output_body TEXT, - input_headers JSONB, - output_headers JSONB, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - UNIQUE (execution_id, processor_id, start_time) -); - -SELECT create_hypertable('processor_executions', 'start_time', chunk_time_interval => INTERVAL '1 day'); - -CREATE INDEX idx_proc_exec_execution ON processor_executions (execution_id); -CREATE INDEX idx_proc_exec_type_time ON processor_executions (processor_type, start_time DESC); diff --git a/cameleer3-server-app/src/main/resources/db/migration/V4__agent_metrics.sql b/cameleer3-server-app/src/main/resources/db/migration/V4__agent_metrics.sql deleted file mode 100644 index 4ecd6cac..00000000 --- a/cameleer3-server-app/src/main/resources/db/migration/V4__agent_metrics.sql +++ /dev/null @@ -1,12 +0,0 @@ -CREATE TABLE agent_metrics ( - agent_id TEXT NOT NULL, - metric_name TEXT NOT NULL, - metric_value DOUBLE PRECISION NOT NULL, - tags JSONB, - collected_at TIMESTAMPTZ NOT NULL, - server_received_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - -SELECT create_hypertable('agent_metrics', 'collected_at', chunk_time_interval => INTERVAL '1 day'); - -CREATE INDEX idx_metrics_agent_name ON agent_metrics (agent_id, metric_name, collected_at DESC); diff --git a/cameleer3-server-app/src/main/resources/db/migration/V5__route_diagrams.sql b/cameleer3-server-app/src/main/resources/db/migration/V5__route_diagrams.sql deleted file mode 100644 index 85eb2355..00000000 --- a/cameleer3-server-app/src/main/resources/db/migration/V5__route_diagrams.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE TABLE route_diagrams ( - content_hash TEXT PRIMARY KEY, - route_id TEXT NOT NULL, - agent_id TEXT NOT NULL, - definition TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - -CREATE INDEX idx_diagrams_route_agent ON route_diagrams (route_id, agent_id); diff --git a/cameleer3-server-app/src/main/resources/db/migration/V6__users.sql b/cameleer3-server-app/src/main/resources/db/migration/V6__users.sql deleted file mode 100644 index 079db7dd..00000000 --- a/cameleer3-server-app/src/main/resources/db/migration/V6__users.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE TABLE users ( - user_id TEXT PRIMARY KEY, - provider TEXT NOT NULL, - email TEXT, - display_name TEXT, - roles TEXT[] NOT NULL DEFAULT '{}', - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() -); diff --git a/cameleer3-server-app/src/main/resources/db/migration/V7__oidc_config.sql b/cameleer3-server-app/src/main/resources/db/migration/V7__oidc_config.sql deleted file mode 100644 index e46a2196..00000000 --- a/cameleer3-server-app/src/main/resources/db/migration/V7__oidc_config.sql +++ /dev/null @@ -1,12 +0,0 @@ -CREATE TABLE oidc_config ( - config_id TEXT PRIMARY KEY DEFAULT 'default', - enabled BOOLEAN NOT NULL DEFAULT false, - issuer_uri TEXT, - client_id TEXT, - client_secret TEXT, - roles_claim TEXT, - default_roles TEXT[] NOT NULL DEFAULT '{}', - auto_signup BOOLEAN DEFAULT false, - display_name_claim TEXT, - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() -); diff --git a/cameleer3-server-app/src/main/resources/db/migration/V8__continuous_aggregates.sql b/cameleer3-server-app/src/main/resources/db/migration/V8__continuous_aggregates.sql deleted file mode 100644 index 056ba07c..00000000 --- a/cameleer3-server-app/src/main/resources/db/migration/V8__continuous_aggregates.sql +++ /dev/null @@ -1,87 +0,0 @@ --- Global stats -CREATE MATERIALIZED VIEW stats_1m_all -WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS -SELECT - time_bucket('1 minute', start_time) AS bucket, - COUNT(*) AS total_count, - COUNT(*) FILTER (WHERE status = 'FAILED') AS failed_count, - COUNT(*) FILTER (WHERE status = 'RUNNING') AS running_count, - SUM(duration_ms) AS duration_sum, - MAX(duration_ms) AS duration_max, - approx_percentile(0.99, percentile_agg(duration_ms::DOUBLE PRECISION)) AS p99_duration -FROM executions -WHERE status IS NOT NULL -GROUP BY bucket -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('stats_1m_all', - start_offset => INTERVAL '1 hour', - end_offset => INTERVAL '1 minute', - schedule_interval => INTERVAL '1 minute'); - --- Per-application stats -CREATE MATERIALIZED VIEW stats_1m_app -WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS -SELECT - time_bucket('1 minute', start_time) AS bucket, - group_name, - COUNT(*) AS total_count, - COUNT(*) FILTER (WHERE status = 'FAILED') AS failed_count, - COUNT(*) FILTER (WHERE status = 'RUNNING') AS running_count, - SUM(duration_ms) AS duration_sum, - MAX(duration_ms) AS duration_max, - approx_percentile(0.99, percentile_agg(duration_ms::DOUBLE PRECISION)) AS p99_duration -FROM executions -WHERE status IS NOT NULL -GROUP BY bucket, group_name -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('stats_1m_app', - start_offset => INTERVAL '1 hour', - end_offset => INTERVAL '1 minute', - schedule_interval => INTERVAL '1 minute'); - --- Per-route stats -CREATE MATERIALIZED VIEW stats_1m_route -WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS -SELECT - time_bucket('1 minute', start_time) AS bucket, - group_name, - route_id, - COUNT(*) AS total_count, - COUNT(*) FILTER (WHERE status = 'FAILED') AS failed_count, - COUNT(*) FILTER (WHERE status = 'RUNNING') AS running_count, - SUM(duration_ms) AS duration_sum, - MAX(duration_ms) AS duration_max, - approx_percentile(0.99, percentile_agg(duration_ms::DOUBLE PRECISION)) AS p99_duration -FROM executions -WHERE status IS NOT NULL -GROUP BY bucket, group_name, route_id -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('stats_1m_route', - start_offset => INTERVAL '1 hour', - end_offset => INTERVAL '1 minute', - schedule_interval => INTERVAL '1 minute'); - --- Per-processor stats (uses denormalized group_name/route_id on processor_executions) -CREATE MATERIALIZED VIEW stats_1m_processor -WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS -SELECT - time_bucket('1 minute', start_time) AS bucket, - group_name, - route_id, - processor_type, - COUNT(*) AS total_count, - COUNT(*) FILTER (WHERE status = 'FAILED') AS failed_count, - SUM(duration_ms) AS duration_sum, - MAX(duration_ms) AS duration_max, - approx_percentile(0.99, percentile_agg(duration_ms::DOUBLE PRECISION)) AS p99_duration -FROM processor_executions -GROUP BY bucket, group_name, route_id, processor_type -WITH NO DATA; - -SELECT add_continuous_aggregate_policy('stats_1m_processor', - start_offset => INTERVAL '1 hour', - end_offset => INTERVAL '1 minute', - schedule_interval => INTERVAL '1 minute'); diff --git a/cameleer3-server-app/src/main/resources/db/migration/V9__admin_thresholds.sql b/cameleer3-server-app/src/main/resources/db/migration/V9__admin_thresholds.sql deleted file mode 100644 index 9b618c97..00000000 --- a/cameleer3-server-app/src/main/resources/db/migration/V9__admin_thresholds.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE TABLE admin_thresholds ( - id INTEGER PRIMARY KEY DEFAULT 1, - config JSONB NOT NULL DEFAULT '{}', - updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), - updated_by TEXT NOT NULL, - CONSTRAINT single_row CHECK (id = 1) -);