diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..df5d56d0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +**/target/ +.git/ +.gitea/ +.idea/ +*.iml +docs/ +*.md +!pom.xml +.planning/ +.claude/ diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 06ca355e..b422f220 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -3,6 +3,8 @@ name: CI on: push: branches: [main] + tags-ignore: + - 'v*' pull_request: branches: [main] @@ -12,6 +14,10 @@ jobs: container: image: maven:3.9-eclipse-temurin-17 steps: + - name: Install Node.js + run: | + apt-get update && apt-get install -y nodejs + - uses: actions/checkout@v4 - name: Configure Gitea Maven Registry @@ -40,3 +46,101 @@ jobs: - name: Build and Test run: mvn clean verify --batch-mode + + docker: + needs: build + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' + container: + image: docker:27 + steps: + - name: Checkout + run: | + apk add --no-cache git + git clone --depth=1 --branch=${GITHUB_REF_NAME} https://cameleer:${REGISTRY_TOKEN}@gitea.siegeln.net/${GITHUB_REPOSITORY}.git . + env: + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + - name: Login to registry + run: echo "$REGISTRY_TOKEN" | docker login gitea.siegeln.net -u cameleer --password-stdin + env: + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + - name: Set up QEMU for cross-platform builds + run: docker run --rm --privileged tonistiigi/binfmt --install all + - name: Build and push + run: | + docker buildx create --use --name cibuilder + docker buildx build --platform linux/amd64 \ + --build-arg REGISTRY_TOKEN="$REGISTRY_TOKEN" \ + -t gitea.siegeln.net/cameleer/cameleer3-server:${{ github.sha }} \ + -t gitea.siegeln.net/cameleer/cameleer3-server:latest \ + --cache-from type=registry,ref=gitea.siegeln.net/cameleer/cameleer3-server:buildcache \ + --cache-to type=registry,ref=gitea.siegeln.net/cameleer/cameleer3-server:buildcache,mode=max \ + --provenance=false \ + --push . + env: + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + - name: Cleanup local Docker + run: docker system prune -af --filter "until=24h" + if: always() + - name: Cleanup old container images + run: | + apk add --no-cache curl jq + API="https://gitea.siegeln.net/api/v1" + AUTH="Authorization: token ${REGISTRY_TOKEN}" + CURRENT_SHA="${{ github.sha }}" + curl -sf -H "$AUTH" "$API/packages/cameleer/container/cameleer3-server" | \ + jq -r '.[] | "\(.id) \(.version)"' | \ + while read id version; do + if [ "$version" != "latest" ] && [ "$version" != "$CURRENT_SHA" ]; then + echo "Deleting old image tag: $version" + curl -sf -X DELETE -H "$AUTH" "$API/packages/cameleer/container/cameleer3-server/$version" + fi + done + env: + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + if: always() + + deploy: + needs: docker + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' + container: + image: bitnami/kubectl:latest + steps: + - name: Checkout + run: | + git clone --depth=1 --branch=${GITHUB_REF_NAME} https://cameleer:${REGISTRY_TOKEN}@gitea.siegeln.net/${GITHUB_REPOSITORY}.git . + env: + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + - name: Configure kubectl + run: | + mkdir -p ~/.kube + echo "$KUBECONFIG_B64" | base64 -d > ~/.kube/config + env: + KUBECONFIG_B64: ${{ secrets.KUBECONFIG_BASE64 }} + - name: Deploy + run: | + kubectl create namespace cameleer --dry-run=client -o yaml | kubectl apply -f - + + kubectl create secret docker-registry gitea-registry \ + --namespace=cameleer \ + --docker-server=gitea.siegeln.net \ + --docker-username=cameleer \ + --docker-password="$REGISTRY_TOKEN" \ + --dry-run=client -o yaml | kubectl apply -f - + + kubectl create secret generic cameleer-auth \ + --namespace=cameleer \ + --from-literal=CAMELEER_AUTH_TOKEN="$CAMELEER_AUTH_TOKEN" \ + --dry-run=client -o yaml | kubectl apply -f - + + kubectl apply -f deploy/clickhouse.yaml + kubectl -n cameleer rollout status statefulset/clickhouse --timeout=120s + + kubectl apply -f deploy/server.yaml + kubectl -n cameleer set image deployment/cameleer3-server \ + server=gitea.siegeln.net/cameleer/cameleer3-server:${{ github.sha }} + kubectl -n cameleer rollout status deployment/cameleer3-server --timeout=120s + env: + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + CAMELEER_AUTH_TOKEN: ${{ secrets.CAMELEER_AUTH_TOKEN }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..bd3a32b9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM --platform=$BUILDPLATFORM maven:3.9-eclipse-temurin-17 AS build +WORKDIR /build + +# Configure Gitea Maven Registry for cameleer3-common dependency +ARG REGISTRY_TOKEN +RUN mkdir -p ~/.m2 && \ + echo 'giteacameleer'${REGISTRY_TOKEN}'' > ~/.m2/settings.xml + +COPY pom.xml . +COPY cameleer3-server-core/pom.xml cameleer3-server-core/ +COPY cameleer3-server-app/pom.xml cameleer3-server-app/ +# Cache deps — only re-downloaded when POMs change +RUN mvn dependency:go-offline -B || true +COPY . . +RUN mvn clean package -DskipTests -B + +FROM eclipse-temurin:17-jre +WORKDIR /app +COPY --from=build /build/cameleer3-server-app/target/cameleer3-server-app-*.jar /app/server.jar + +ENV SPRING_DATASOURCE_URL=jdbc:ch://clickhouse:8123/cameleer3 +ENV SPRING_DATASOURCE_USERNAME=cameleer +ENV SPRING_DATASOURCE_PASSWORD=cameleer_dev + +EXPOSE 8081 +ENTRYPOINT exec java -jar /app/server.jar diff --git a/deploy/clickhouse.yaml b/deploy/clickhouse.yaml new file mode 100644 index 00000000..1d9784a5 --- /dev/null +++ b/deploy/clickhouse.yaml @@ -0,0 +1,153 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: clickhouse-init + namespace: cameleer +data: + 01-schema.sql: | + CREATE TABLE IF NOT EXISTS route_executions ( + execution_id String, + route_id LowCardinality(String), + agent_id LowCardinality(String), + status LowCardinality(String), + start_time DateTime64(3, 'UTC'), + end_time Nullable(DateTime64(3, 'UTC')), + duration_ms UInt64, + correlation_id String, + exchange_id String, + error_message String DEFAULT '', + error_stacktrace String DEFAULT '', + processor_ids Array(String), + processor_types Array(LowCardinality(String)), + processor_starts Array(DateTime64(3, 'UTC')), + processor_ends Array(DateTime64(3, 'UTC')), + processor_durations Array(UInt64), + processor_statuses Array(LowCardinality(String)), + server_received_at DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC'), + INDEX idx_correlation correlation_id TYPE bloom_filter GRANULARITY 4, + INDEX idx_error error_message TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 4 + ) + ENGINE = MergeTree() + PARTITION BY toYYYYMMDD(start_time) + ORDER BY (agent_id, status, start_time, execution_id) + TTL toDateTime(start_time) + toIntervalDay(30) + SETTINGS ttl_only_drop_parts = 1; + + CREATE TABLE IF NOT EXISTS route_diagrams ( + content_hash String, + route_id LowCardinality(String), + agent_id LowCardinality(String), + definition String, + created_at DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC') + ) + ENGINE = ReplacingMergeTree(created_at) + ORDER BY (content_hash); + + CREATE TABLE IF NOT EXISTS agent_metrics ( + agent_id LowCardinality(String), + collected_at DateTime64(3, 'UTC'), + metric_name LowCardinality(String), + metric_value Float64, + tags Map(String, String), + server_received_at DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC') + ) + ENGINE = MergeTree() + PARTITION BY toYYYYMMDD(collected_at) + ORDER BY (agent_id, metric_name, collected_at) + TTL toDateTime(collected_at) + toIntervalDay(30) + SETTINGS ttl_only_drop_parts = 1; + + 02-search-columns.sql: | + ALTER TABLE route_executions + ADD COLUMN IF NOT EXISTS exchange_bodies String DEFAULT '', + ADD COLUMN IF NOT EXISTS exchange_headers String DEFAULT '', + ADD COLUMN IF NOT EXISTS processor_depths Array(UInt16) DEFAULT [], + ADD COLUMN IF NOT EXISTS processor_parent_indexes Array(Int32) DEFAULT [], + ADD COLUMN IF NOT EXISTS processor_error_messages Array(String) DEFAULT [], + ADD COLUMN IF NOT EXISTS processor_error_stacktraces Array(String) DEFAULT [], + ADD COLUMN IF NOT EXISTS processor_input_bodies Array(String) DEFAULT [], + ADD COLUMN IF NOT EXISTS processor_output_bodies Array(String) DEFAULT [], + ADD COLUMN IF NOT EXISTS processor_input_headers Array(String) DEFAULT [], + ADD COLUMN IF NOT EXISTS processor_output_headers Array(String) DEFAULT [], + ADD COLUMN IF NOT EXISTS processor_diagram_node_ids Array(String) DEFAULT [], + ADD COLUMN IF NOT EXISTS diagram_content_hash String DEFAULT ''; + + ALTER TABLE route_executions + ADD INDEX IF NOT EXISTS idx_exchange_bodies exchange_bodies TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 4, + ADD INDEX IF NOT EXISTS idx_exchange_headers exchange_headers TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 4; + + ALTER TABLE route_executions + ADD INDEX IF NOT EXISTS idx_error_stacktrace error_stacktrace TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 4; +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: clickhouse + namespace: cameleer +spec: + serviceName: clickhouse + replicas: 1 + selector: + matchLabels: + app: clickhouse + template: + metadata: + labels: + app: clickhouse + spec: + containers: + - name: clickhouse + image: clickhouse/clickhouse-server:25.3 + ports: + - containerPort: 8123 + name: http + - containerPort: 9000 + name: native + env: + - name: CLICKHOUSE_USER + value: cameleer + - name: CLICKHOUSE_PASSWORD + value: cameleer_dev + - name: CLICKHOUSE_DB + value: cameleer3 + volumeMounts: + - name: data + mountPath: /var/lib/clickhouse + - name: init-scripts + mountPath: /docker-entrypoint-initdb.d + resources: + requests: + memory: "512Mi" + cpu: "200m" + limits: + memory: "1Gi" + cpu: "1000m" + volumes: + - name: init-scripts + configMap: + name: clickhouse-init + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 2Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: clickhouse + namespace: cameleer +spec: + clusterIP: None + selector: + app: clickhouse + ports: + - port: 8123 + targetPort: 8123 + name: http + - port: 9000 + targetPort: 9000 + name: native diff --git a/deploy/server.yaml b/deploy/server.yaml new file mode 100644 index 00000000..287351ab --- /dev/null +++ b/deploy/server.yaml @@ -0,0 +1,55 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cameleer3-server + namespace: cameleer +spec: + replicas: 1 + selector: + matchLabels: + app: cameleer3-server + template: + metadata: + labels: + app: cameleer3-server + spec: + imagePullSecrets: + - name: gitea-registry + containers: + - name: server + image: gitea.siegeln.net/cameleer/cameleer3-server:latest + ports: + - containerPort: 8081 + env: + - name: SPRING_DATASOURCE_URL + value: "jdbc:ch://clickhouse:8123/cameleer3" + - name: SPRING_DATASOURCE_USERNAME + value: "cameleer" + - name: SPRING_DATASOURCE_PASSWORD + value: "cameleer_dev" + - name: CAMELEER_AUTH_TOKEN + valueFrom: + secretKeyRef: + name: cameleer-auth + key: CAMELEER_AUTH_TOKEN + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" +--- +apiVersion: v1 +kind: Service +metadata: + name: cameleer3-server + namespace: cameleer +spec: + type: NodePort + selector: + app: cameleer3-server + ports: + - port: 8081 + targetPort: 8081 + nodePort: 30081