From d739094a56b9e0a92eac5005f23b04f78c77c6c1 Mon Sep 17 00:00:00 2001 From: hsiegeln <37154749+hsiegeln@users.noreply.github.com> Date: Wed, 1 Apr 2026 12:40:54 +0200 Subject: [PATCH] fix: update ClickHouse DDL files with new column names instead of ALTER RENAME ClickHouse can't rename columns that are part of ORDER BY keys. Updated V1-V8 DDL files directly with new column names (instance_id, application_id) and removed V9 migration. Wipe ClickHouse and restart. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../clickhouse/V1__agent_metrics.sql | 4 +- .../resources/clickhouse/V2__executions.sql | 6 +- .../clickhouse/V3__processor_executions.sql | 4 +- .../clickhouse/V4__stats_tables_and_mvs.sql | 32 +++---- .../clickhouse/V6__route_diagrams.sql | 4 +- .../resources/clickhouse/V7__agent_events.sql | 6 +- .../main/resources/clickhouse/V8__logs.sql | 2 +- .../V9__rename_identity_columns.sql | 87 ------------------- 8 files changed, 29 insertions(+), 116 deletions(-) delete mode 100644 cameleer3-server-app/src/main/resources/clickhouse/V9__rename_identity_columns.sql diff --git a/cameleer3-server-app/src/main/resources/clickhouse/V1__agent_metrics.sql b/cameleer3-server-app/src/main/resources/clickhouse/V1__agent_metrics.sql index b1e620eb..a809a3ba 100644 --- a/cameleer3-server-app/src/main/resources/clickhouse/V1__agent_metrics.sql +++ b/cameleer3-server-app/src/main/resources/clickhouse/V1__agent_metrics.sql @@ -1,7 +1,7 @@ CREATE TABLE IF NOT EXISTS agent_metrics ( tenant_id LowCardinality(String) DEFAULT 'default', collected_at DateTime64(3), - agent_id LowCardinality(String), + instance_id LowCardinality(String), metric_name LowCardinality(String), metric_value Float64, tags Map(String, String) DEFAULT map(), @@ -9,6 +9,6 @@ CREATE TABLE IF NOT EXISTS agent_metrics ( ) ENGINE = MergeTree() PARTITION BY (tenant_id, toYYYYMM(collected_at)) -ORDER BY (tenant_id, agent_id, metric_name, collected_at) +ORDER BY (tenant_id, instance_id, metric_name, collected_at) TTL toDateTime(collected_at) + INTERVAL 365 DAY DELETE SETTINGS index_granularity = 8192; diff --git a/cameleer3-server-app/src/main/resources/clickhouse/V2__executions.sql b/cameleer3-server-app/src/main/resources/clickhouse/V2__executions.sql index e9b31f9e..99e5b3f2 100644 --- a/cameleer3-server-app/src/main/resources/clickhouse/V2__executions.sql +++ b/cameleer3-server-app/src/main/resources/clickhouse/V2__executions.sql @@ -4,8 +4,8 @@ CREATE TABLE IF NOT EXISTS executions ( start_time DateTime64(3), _version UInt64 DEFAULT 1, route_id LowCardinality(String), - agent_id LowCardinality(String), - application_name LowCardinality(String), + instance_id LowCardinality(String), + application_id LowCardinality(String), status LowCardinality(String), correlation_id String DEFAULT '', exchange_id String DEFAULT '', @@ -43,6 +43,6 @@ CREATE TABLE IF NOT EXISTS executions ( ) ENGINE = ReplacingMergeTree(_version) PARTITION BY (tenant_id, toYYYYMM(start_time)) -ORDER BY (tenant_id, start_time, application_name, route_id, execution_id) +ORDER BY (tenant_id, start_time, application_id, route_id, execution_id) TTL toDateTime(start_time) + INTERVAL 365 DAY DELETE SETTINGS index_granularity = 8192; diff --git a/cameleer3-server-app/src/main/resources/clickhouse/V3__processor_executions.sql b/cameleer3-server-app/src/main/resources/clickhouse/V3__processor_executions.sql index 2f4ea1ec..3da0d586 100644 --- a/cameleer3-server-app/src/main/resources/clickhouse/V3__processor_executions.sql +++ b/cameleer3-server-app/src/main/resources/clickhouse/V3__processor_executions.sql @@ -8,7 +8,7 @@ CREATE TABLE IF NOT EXISTS processor_executions ( processor_type LowCardinality(String), start_time DateTime64(3), route_id LowCardinality(String), - application_name LowCardinality(String), + application_id LowCardinality(String), iteration Nullable(Int32), iteration_size Nullable(Int32), status LowCardinality(String), @@ -40,6 +40,6 @@ CREATE TABLE IF NOT EXISTS processor_executions ( ) ENGINE = MergeTree() PARTITION BY (tenant_id, toYYYYMM(start_time)) -ORDER BY (tenant_id, start_time, application_name, route_id, execution_id, seq) +ORDER BY (tenant_id, start_time, application_id, route_id, execution_id, seq) TTL toDateTime(start_time) + INTERVAL 365 DAY DELETE SETTINGS index_granularity = 8192; diff --git a/cameleer3-server-app/src/main/resources/clickhouse/V4__stats_tables_and_mvs.sql b/cameleer3-server-app/src/main/resources/clickhouse/V4__stats_tables_and_mvs.sql index 09450132..2eca3504 100644 --- a/cameleer3-server-app/src/main/resources/clickhouse/V4__stats_tables_and_mvs.sql +++ b/cameleer3-server-app/src/main/resources/clickhouse/V4__stats_tables_and_mvs.sql @@ -36,7 +36,7 @@ GROUP BY tenant_id, bucket; CREATE TABLE IF NOT EXISTS stats_1m_app ( tenant_id LowCardinality(String), - application_name LowCardinality(String), + application_id LowCardinality(String), bucket DateTime, total_count AggregateFunction(count), failed_count AggregateFunction(countIf, UInt8), @@ -47,13 +47,13 @@ CREATE TABLE IF NOT EXISTS stats_1m_app ( ) ENGINE = AggregatingMergeTree() PARTITION BY (tenant_id, toYYYYMM(bucket)) -ORDER BY (tenant_id, application_name, bucket) +ORDER BY (tenant_id, application_id, bucket) TTL bucket + INTERVAL 365 DAY DELETE; CREATE MATERIALIZED VIEW IF NOT EXISTS stats_1m_app_mv TO stats_1m_app AS SELECT tenant_id, - application_name, + application_id, toStartOfMinute(start_time) AS bucket, countState() AS total_count, countIfState(status = 'FAILED') AS failed_count, @@ -62,13 +62,13 @@ SELECT maxState(duration_ms) AS duration_max, quantileState(0.99)(duration_ms) AS p99_duration FROM executions -GROUP BY tenant_id, application_name, bucket; +GROUP BY tenant_id, application_id, bucket; -- stats_1m_route (per-route) CREATE TABLE IF NOT EXISTS stats_1m_route ( tenant_id LowCardinality(String), - application_name LowCardinality(String), + application_id LowCardinality(String), route_id LowCardinality(String), bucket DateTime, total_count AggregateFunction(count), @@ -80,13 +80,13 @@ CREATE TABLE IF NOT EXISTS stats_1m_route ( ) ENGINE = AggregatingMergeTree() PARTITION BY (tenant_id, toYYYYMM(bucket)) -ORDER BY (tenant_id, application_name, route_id, bucket) +ORDER BY (tenant_id, application_id, route_id, bucket) TTL bucket + INTERVAL 365 DAY DELETE; CREATE MATERIALIZED VIEW IF NOT EXISTS stats_1m_route_mv TO stats_1m_route AS SELECT tenant_id, - application_name, + application_id, route_id, toStartOfMinute(start_time) AS bucket, countState() AS total_count, @@ -96,13 +96,13 @@ SELECT maxState(duration_ms) AS duration_max, quantileState(0.99)(duration_ms) AS p99_duration FROM executions -GROUP BY tenant_id, application_name, route_id, bucket; +GROUP BY tenant_id, application_id, route_id, bucket; -- stats_1m_processor (per-processor-type) CREATE TABLE IF NOT EXISTS stats_1m_processor ( tenant_id LowCardinality(String), - application_name LowCardinality(String), + application_id LowCardinality(String), processor_type LowCardinality(String), bucket DateTime, total_count AggregateFunction(count), @@ -113,13 +113,13 @@ CREATE TABLE IF NOT EXISTS stats_1m_processor ( ) ENGINE = AggregatingMergeTree() PARTITION BY (tenant_id, toYYYYMM(bucket)) -ORDER BY (tenant_id, application_name, processor_type, bucket) +ORDER BY (tenant_id, application_id, processor_type, bucket) TTL bucket + INTERVAL 365 DAY DELETE; CREATE MATERIALIZED VIEW IF NOT EXISTS stats_1m_processor_mv TO stats_1m_processor AS SELECT tenant_id, - application_name, + application_id, processor_type, toStartOfMinute(start_time) AS bucket, countState() AS total_count, @@ -128,13 +128,13 @@ SELECT maxState(duration_ms) AS duration_max, quantileState(0.99)(duration_ms) AS p99_duration FROM processor_executions -GROUP BY tenant_id, application_name, processor_type, bucket; +GROUP BY tenant_id, application_id, processor_type, bucket; -- stats_1m_processor_detail (per-processor-id) CREATE TABLE IF NOT EXISTS stats_1m_processor_detail ( tenant_id LowCardinality(String), - application_name LowCardinality(String), + application_id LowCardinality(String), route_id LowCardinality(String), processor_id String, bucket DateTime, @@ -146,13 +146,13 @@ CREATE TABLE IF NOT EXISTS stats_1m_processor_detail ( ) ENGINE = AggregatingMergeTree() PARTITION BY (tenant_id, toYYYYMM(bucket)) -ORDER BY (tenant_id, application_name, route_id, processor_id, bucket) +ORDER BY (tenant_id, application_id, route_id, processor_id, bucket) TTL bucket + INTERVAL 365 DAY DELETE; CREATE MATERIALIZED VIEW IF NOT EXISTS stats_1m_processor_detail_mv TO stats_1m_processor_detail AS SELECT tenant_id, - application_name, + application_id, route_id, processor_id, toStartOfMinute(start_time) AS bucket, @@ -162,4 +162,4 @@ SELECT maxState(duration_ms) AS duration_max, quantileState(0.99)(duration_ms) AS p99_duration FROM processor_executions -GROUP BY tenant_id, application_name, route_id, processor_id, bucket; +GROUP BY tenant_id, application_id, route_id, processor_id, bucket; diff --git a/cameleer3-server-app/src/main/resources/clickhouse/V6__route_diagrams.sql b/cameleer3-server-app/src/main/resources/clickhouse/V6__route_diagrams.sql index 0dd53568..b57df0f2 100644 --- a/cameleer3-server-app/src/main/resources/clickhouse/V6__route_diagrams.sql +++ b/cameleer3-server-app/src/main/resources/clickhouse/V6__route_diagrams.sql @@ -2,8 +2,8 @@ CREATE TABLE IF NOT EXISTS route_diagrams ( tenant_id LowCardinality(String) DEFAULT 'default', content_hash String, route_id LowCardinality(String), - agent_id LowCardinality(String), - application_name LowCardinality(String), + instance_id LowCardinality(String), + application_id LowCardinality(String), definition String, created_at DateTime64(3) DEFAULT now64(3) ) diff --git a/cameleer3-server-app/src/main/resources/clickhouse/V7__agent_events.sql b/cameleer3-server-app/src/main/resources/clickhouse/V7__agent_events.sql index 4fc1d9b3..97278131 100644 --- a/cameleer3-server-app/src/main/resources/clickhouse/V7__agent_events.sql +++ b/cameleer3-server-app/src/main/resources/clickhouse/V7__agent_events.sql @@ -1,12 +1,12 @@ CREATE TABLE IF NOT EXISTS agent_events ( tenant_id LowCardinality(String) DEFAULT 'default', timestamp DateTime64(3) DEFAULT now64(3), - agent_id LowCardinality(String), - app_id LowCardinality(String), + instance_id LowCardinality(String), + application_id LowCardinality(String), event_type LowCardinality(String), detail String DEFAULT '' ) ENGINE = MergeTree() PARTITION BY (tenant_id, toYYYYMM(timestamp)) -ORDER BY (tenant_id, app_id, agent_id, timestamp) +ORDER BY (tenant_id, application_id, instance_id, timestamp) TTL toDateTime(timestamp) + INTERVAL 365 DAY DELETE; diff --git a/cameleer3-server-app/src/main/resources/clickhouse/V8__logs.sql b/cameleer3-server-app/src/main/resources/clickhouse/V8__logs.sql index 41ce45aa..02c1bc32 100644 --- a/cameleer3-server-app/src/main/resources/clickhouse/V8__logs.sql +++ b/cameleer3-server-app/src/main/resources/clickhouse/V8__logs.sql @@ -2,7 +2,7 @@ CREATE TABLE IF NOT EXISTS logs ( tenant_id LowCardinality(String) DEFAULT 'default', timestamp DateTime64(3), application LowCardinality(String), - agent_id LowCardinality(String), + instance_id LowCardinality(String), level LowCardinality(String), logger_name LowCardinality(String) DEFAULT '', message String, diff --git a/cameleer3-server-app/src/main/resources/clickhouse/V9__rename_identity_columns.sql b/cameleer3-server-app/src/main/resources/clickhouse/V9__rename_identity_columns.sql deleted file mode 100644 index 01cf71dd..00000000 --- a/cameleer3-server-app/src/main/resources/clickhouse/V9__rename_identity_columns.sql +++ /dev/null @@ -1,87 +0,0 @@ --- Rename agent identity columns for protocol v2 alignment. - --- Base tables -ALTER TABLE executions RENAME COLUMN agent_id TO instance_id; -ALTER TABLE executions RENAME COLUMN application_name TO application_id; - -ALTER TABLE processor_executions RENAME COLUMN application_name TO application_id; - -ALTER TABLE agent_metrics RENAME COLUMN agent_id TO instance_id; - -ALTER TABLE route_diagrams RENAME COLUMN agent_id TO instance_id; -ALTER TABLE route_diagrams RENAME COLUMN application_name TO application_id; - -ALTER TABLE agent_events RENAME COLUMN agent_id TO instance_id; -ALTER TABLE agent_events RENAME COLUMN app_id TO application_id; - -ALTER TABLE logs RENAME COLUMN agent_id TO instance_id; - --- Stats tables — drop MVs first, rename columns, recreate MVs -DROP VIEW IF EXISTS stats_1m_app_mv; -DROP VIEW IF EXISTS stats_1m_route_mv; -DROP VIEW IF EXISTS stats_1m_processor_mv; -DROP VIEW IF EXISTS stats_1m_processor_detail_mv; - -ALTER TABLE stats_1m_app RENAME COLUMN application_name TO application_id; -ALTER TABLE stats_1m_route RENAME COLUMN application_name TO application_id; -ALTER TABLE stats_1m_processor RENAME COLUMN application_name TO application_id; -ALTER TABLE stats_1m_processor_detail RENAME COLUMN application_name TO application_id; - --- Recreate MVs with renamed columns -CREATE MATERIALIZED VIEW IF NOT EXISTS stats_1m_app_mv TO stats_1m_app AS -SELECT - tenant_id, - application_id, - toStartOfMinute(start_time) AS bucket, - countState() AS total_count, - countIfState(status = 'FAILED') AS failed_count, - countIfState(status = 'RUNNING') AS running_count, - sumState(duration_ms) AS duration_sum, - maxState(duration_ms) AS duration_max, - quantileState(0.99)(duration_ms) AS p99_duration -FROM executions -GROUP BY tenant_id, application_id, bucket; - -CREATE MATERIALIZED VIEW IF NOT EXISTS stats_1m_route_mv TO stats_1m_route AS -SELECT - tenant_id, - application_id, - route_id, - toStartOfMinute(start_time) AS bucket, - countState() AS total_count, - countIfState(status = 'FAILED') AS failed_count, - countIfState(status = 'RUNNING') AS running_count, - sumState(duration_ms) AS duration_sum, - maxState(duration_ms) AS duration_max, - quantileState(0.99)(duration_ms) AS p99_duration -FROM executions -GROUP BY tenant_id, application_id, route_id, bucket; - -CREATE MATERIALIZED VIEW IF NOT EXISTS stats_1m_processor_mv TO stats_1m_processor AS -SELECT - tenant_id, - application_id, - processor_type, - toStartOfMinute(start_time) AS bucket, - countState() AS total_count, - countIfState(status = 'FAILED') AS failed_count, - sumState(duration_ms) AS duration_sum, - maxState(duration_ms) AS duration_max, - quantileState(0.99)(duration_ms) AS p99_duration -FROM processor_executions -GROUP BY tenant_id, application_id, processor_type, bucket; - -CREATE MATERIALIZED VIEW IF NOT EXISTS stats_1m_processor_detail_mv TO stats_1m_processor_detail AS -SELECT - tenant_id, - application_id, - route_id, - processor_id, - toStartOfMinute(start_time) AS bucket, - countState() AS total_count, - countIfState(status = 'FAILED') AS failed_count, - sumState(duration_ms) AS duration_sum, - maxState(duration_ms) AS duration_max, - quantileState(0.99)(duration_ms) AS p99_duration -FROM processor_executions -GROUP BY tenant_id, application_id, route_id, processor_id, bucket;