diff --git a/deploy/clickhouse.yaml b/deploy/clickhouse.yaml index 4992f8ad..339b2026 100644 --- a/deploy/clickhouse.yaml +++ b/deploy/clickhouse.yaml @@ -1,3 +1,84 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: clickhouse-init-schema + namespace: cameleer +data: + 01-schema.sql: | + CREATE TABLE IF NOT EXISTS route_executions ( + execution_id String, + route_id LowCardinality(String), + agent_id LowCardinality(String), + status LowCardinality(String), + start_time DateTime64(3, 'UTC'), + end_time Nullable(DateTime64(3, 'UTC')), + duration_ms UInt64, + correlation_id String, + exchange_id String, + error_message String DEFAULT '', + error_stacktrace String DEFAULT '', + processor_ids Array(String), + processor_types Array(LowCardinality(String)), + processor_starts Array(DateTime64(3, 'UTC')), + processor_ends Array(DateTime64(3, 'UTC')), + processor_durations Array(UInt64), + processor_statuses Array(LowCardinality(String)), + server_received_at DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'), + INDEX idx_correlation correlation_id TYPE bloom_filter GRANULARITY 4, + INDEX idx_error error_message TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 4 + ) + ENGINE = MergeTree() + PARTITION BY toYYYYMMDD(start_time) + ORDER BY (agent_id, status, start_time, execution_id) + TTL toDateTime(start_time) + toIntervalDay(30) + SETTINGS ttl_only_drop_parts = 1; + + CREATE TABLE IF NOT EXISTS route_diagrams ( + content_hash String, + route_id LowCardinality(String), + agent_id LowCardinality(String), + definition String, + created_at DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC') + ) + ENGINE = ReplacingMergeTree(created_at) + ORDER BY (content_hash); + + CREATE TABLE IF NOT EXISTS agent_metrics ( + agent_id LowCardinality(String), + collected_at DateTime64(3, 'UTC'), + metric_name LowCardinality(String), + metric_value Float64, + tags Map(String, String), + server_received_at DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC') + ) + ENGINE = MergeTree() + PARTITION BY toYYYYMMDD(collected_at) + ORDER BY (agent_id, metric_name, collected_at) + TTL toDateTime(collected_at) + toIntervalDay(30) + SETTINGS ttl_only_drop_parts = 1; + + 02-search-columns.sql: | + ALTER TABLE route_executions + ADD COLUMN IF NOT EXISTS exchange_bodies String DEFAULT '', + ADD COLUMN IF NOT EXISTS exchange_headers String DEFAULT '', + ADD COLUMN IF NOT EXISTS processor_depths Array(UInt16) DEFAULT [], + ADD COLUMN IF NOT EXISTS processor_parent_indexes Array(Int32) DEFAULT [], + ADD COLUMN IF NOT EXISTS processor_error_messages Array(String) DEFAULT [], + ADD COLUMN IF NOT EXISTS processor_error_stacktraces Array(String) DEFAULT [], + ADD COLUMN IF NOT EXISTS processor_input_bodies Array(String) DEFAULT [], + ADD COLUMN IF NOT EXISTS processor_output_bodies Array(String) DEFAULT [], + ADD COLUMN IF NOT EXISTS processor_input_headers Array(String) DEFAULT [], + ADD COLUMN IF NOT EXISTS processor_output_headers Array(String) DEFAULT [], + ADD COLUMN IF NOT EXISTS processor_diagram_node_ids Array(String) DEFAULT [], + ADD COLUMN IF NOT EXISTS diagram_content_hash String DEFAULT ''; + + ALTER TABLE route_executions + ADD INDEX IF NOT EXISTS idx_exchange_bodies exchange_bodies TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 4, + ADD INDEX IF NOT EXISTS idx_exchange_headers exchange_headers TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 4; + + ALTER TABLE route_executions + ADD INDEX IF NOT EXISTS idx_error_stacktrace error_stacktrace TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 4; +--- apiVersion: apps/v1 kind: StatefulSet metadata: @@ -32,6 +113,9 @@ spec: volumeMounts: - name: data mountPath: /var/lib/clickhouse + - name: init-schema + mountPath: /docker-entrypoint-initdb.d + readOnly: true resources: requests: memory: "512Mi" @@ -39,6 +123,10 @@ spec: limits: memory: "1Gi" cpu: "1000m" + volumes: + - name: init-schema + configMap: + name: clickhouse-init-schema volumeClaimTemplates: - metadata: name: data