cameleer-server/deploy/clickhouse.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: clickhouse-init
  namespace: cameleer
data:
  01-schema.sql: |
    CREATE TABLE IF NOT EXISTS cameleer3.route_executions (
        execution_id     String,
        route_id         LowCardinality(String),
        agent_id         LowCardinality(String),
        status           LowCardinality(String),
        start_time       DateTime64(3, 'UTC'),
        end_time         Nullable(DateTime64(3, 'UTC')),
        duration_ms      UInt64,
        correlation_id   String,
        exchange_id      String,
        error_message    String DEFAULT '',
        error_stacktrace String DEFAULT '',
        processor_ids       Array(String),
        processor_types     Array(LowCardinality(String)),
        processor_starts    Array(DateTime64(3, 'UTC')),
        processor_ends      Array(DateTime64(3, 'UTC')),
        processor_durations Array(UInt64),
        processor_statuses  Array(LowCardinality(String)),
        server_received_at DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'),
        INDEX idx_correlation correlation_id TYPE bloom_filter GRANULARITY 4,
        INDEX idx_error error_message TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 4
    )
    ENGINE = MergeTree()
    PARTITION BY toYYYYMMDD(start_time)
    ORDER BY (agent_id, status, start_time, execution_id)
    TTL toDateTime(start_time) + toIntervalDay(30)
    SETTINGS ttl_only_drop_parts = 1;

    CREATE TABLE IF NOT EXISTS cameleer3.route_diagrams (
        content_hash     String,
        route_id         LowCardinality(String),
        agent_id         LowCardinality(String),
        definition       String,
        created_at       DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC')
    )
    ENGINE = ReplacingMergeTree(created_at)
    ORDER BY (content_hash);

    CREATE TABLE IF NOT EXISTS cameleer3.agent_metrics (
        agent_id         LowCardinality(String),
        collected_at     DateTime64(3, 'UTC'),
        metric_name      LowCardinality(String),
        metric_value     Float64,
        tags             Map(String, String),
        server_received_at DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC')
    )
    ENGINE = MergeTree()
    PARTITION BY toYYYYMMDD(collected_at)
    ORDER BY (agent_id, metric_name, collected_at)
    TTL toDateTime(collected_at) + toIntervalDay(30)
    SETTINGS ttl_only_drop_parts = 1;

  02-search-columns.sql: |
    ALTER TABLE cameleer3.route_executions
        ADD COLUMN IF NOT EXISTS exchange_bodies String DEFAULT '',
        ADD COLUMN IF NOT EXISTS exchange_headers String DEFAULT '',
        ADD COLUMN IF NOT EXISTS processor_depths Array(UInt16) DEFAULT [],
        ADD COLUMN IF NOT EXISTS processor_parent_indexes Array(Int32) DEFAULT [],
        ADD COLUMN IF NOT EXISTS processor_error_messages Array(String) DEFAULT [],
        ADD COLUMN IF NOT EXISTS processor_error_stacktraces Array(String) DEFAULT [],
        ADD COLUMN IF NOT EXISTS processor_input_bodies Array(String) DEFAULT [],
        ADD COLUMN IF NOT EXISTS processor_output_bodies Array(String) DEFAULT [],
        ADD COLUMN IF NOT EXISTS processor_input_headers Array(String) DEFAULT [],
        ADD COLUMN IF NOT EXISTS processor_output_headers Array(String) DEFAULT [],
        ADD COLUMN IF NOT EXISTS processor_diagram_node_ids Array(String) DEFAULT [],
        ADD COLUMN IF NOT EXISTS diagram_content_hash String DEFAULT '';

    ALTER TABLE cameleer3.route_executions
        ADD INDEX IF NOT EXISTS idx_exchange_bodies exchange_bodies TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 4,
        ADD INDEX IF NOT EXISTS idx_exchange_headers exchange_headers TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 4;

    ALTER TABLE cameleer3.route_executions
        ADD INDEX IF NOT EXISTS idx_error_stacktrace error_stacktrace TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 4;
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: clickhouse
  namespace: cameleer
spec:
  serviceName: clickhouse
  replicas: 1
  selector:
    matchLabels:
      app: clickhouse
  template:
    metadata:
      labels:
        app: clickhouse
    spec:
      containers:
        - name: clickhouse
          image: clickhouse/clickhouse-server:25.3
          ports:
            - containerPort: 8123
              name: http
            - containerPort: 9000
              name: native
          env:
            - name: CLICKHOUSE_USER
              value: cameleer
            - name: CLICKHOUSE_PASSWORD
              value: cameleer_dev
            - name: CLICKHOUSE_DB
              value: cameleer3
          volumeMounts:
            - name: data
              mountPath: /var/lib/clickhouse
            - name: init-scripts
              mountPath: /docker-entrypoint-initdb.d
          resources:
            requests:
              memory: "512Mi"
              cpu: "200m"
            limits:
              memory: "1Gi"
              cpu: "1000m"
      volumes:
        - name: init-scripts
          configMap:
            name: clickhouse-init
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        accessModes: ["ReadWriteOnce"]
        resources:
          requests:
            storage: 2Gi
---
apiVersion: v1
kind: Service
metadata:
  name: clickhouse
  namespace: cameleer
spec:
  clusterIP: None
  selector:
    app: clickhouse
  ports:
    - port: 8123
      targetPort: 8123
      name: http
    - port: 9000
      targetPort: 9000
      name: native