feat: add feature branch deployments with per-branch isolation
Enable deploying feature branches into isolated environments on the same k3s cluster. Each branch gets its own namespace (cam-<slug>), PostgreSQL schema, and OpenSearch index prefix for data isolation while sharing the underlying infrastructure. - Make OpenSearch index prefix and DB schema configurable via env vars (defaults preserve existing behavior) - Restructure deploy/ into Kustomize base + overlays (main/feature) - Extend CI to build Docker images for all branches, not just main - Add deploy-feature job with namespace creation, secret copying, Traefik Ingress routing (<slug>-api/ui.cameleer.siegeln.net) - Add cleanup-branch job to remove namespace, PG schema, OS indices on branch deletion - Install required tools (git, jq, curl) in CI deploy containers Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -2,15 +2,17 @@ name: CI
|
|||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: [main]
|
branches: [main, 'feature/**', 'fix/**', 'feat/**']
|
||||||
tags-ignore:
|
tags-ignore:
|
||||||
- 'v*'
|
- 'v*'
|
||||||
pull_request:
|
pull_request:
|
||||||
branches: [main]
|
branches: [main]
|
||||||
|
delete:
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build:
|
build:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
if: github.event_name != 'delete'
|
||||||
container:
|
container:
|
||||||
image: maven:3.9-eclipse-temurin-17
|
image: maven:3.9-eclipse-temurin-17
|
||||||
steps:
|
steps:
|
||||||
@@ -60,7 +62,7 @@ jobs:
|
|||||||
docker:
|
docker:
|
||||||
needs: build
|
needs: build
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
if: github.ref == 'refs/heads/main'
|
if: github.event_name == 'push'
|
||||||
container:
|
container:
|
||||||
image: docker:27
|
image: docker:27
|
||||||
steps:
|
steps:
|
||||||
@@ -74,15 +76,36 @@ jobs:
|
|||||||
run: echo "$REGISTRY_TOKEN" | docker login gitea.siegeln.net -u cameleer --password-stdin
|
run: echo "$REGISTRY_TOKEN" | docker login gitea.siegeln.net -u cameleer --password-stdin
|
||||||
env:
|
env:
|
||||||
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||||
|
- name: Compute branch slug
|
||||||
|
run: |
|
||||||
|
sanitize_branch() {
|
||||||
|
echo "$1" | sed -E 's#^(feature|fix|feat|hotfix)/##' \
|
||||||
|
| tr '[:upper:]' '[:lower:]' \
|
||||||
|
| sed 's/[^a-z0-9-]/-/g' \
|
||||||
|
| sed 's/--*/-/g; s/^-//; s/-$//' \
|
||||||
|
| cut -c1-20 \
|
||||||
|
| sed 's/-$//'
|
||||||
|
}
|
||||||
|
if [ "$GITHUB_REF_NAME" = "main" ]; then
|
||||||
|
echo "BRANCH_SLUG=main" >> "$GITHUB_ENV"
|
||||||
|
echo "IMAGE_TAGS=latest" >> "$GITHUB_ENV"
|
||||||
|
else
|
||||||
|
SLUG=$(sanitize_branch "$GITHUB_REF_NAME")
|
||||||
|
echo "BRANCH_SLUG=$SLUG" >> "$GITHUB_ENV"
|
||||||
|
echo "IMAGE_TAGS=branch-$SLUG" >> "$GITHUB_ENV"
|
||||||
|
fi
|
||||||
- name: Set up QEMU for cross-platform builds
|
- name: Set up QEMU for cross-platform builds
|
||||||
run: docker run --rm --privileged tonistiigi/binfmt --install all
|
run: docker run --rm --privileged tonistiigi/binfmt --install all
|
||||||
- name: Build and push server
|
- name: Build and push server
|
||||||
run: |
|
run: |
|
||||||
docker buildx create --use --name cibuilder
|
docker buildx create --use --name cibuilder
|
||||||
|
TAGS="-t gitea.siegeln.net/cameleer/cameleer3-server:${{ github.sha }}"
|
||||||
|
for TAG in $IMAGE_TAGS; do
|
||||||
|
TAGS="$TAGS -t gitea.siegeln.net/cameleer/cameleer3-server:$TAG"
|
||||||
|
done
|
||||||
docker buildx build --platform linux/amd64 \
|
docker buildx build --platform linux/amd64 \
|
||||||
--build-arg REGISTRY_TOKEN="$REGISTRY_TOKEN" \
|
--build-arg REGISTRY_TOKEN="$REGISTRY_TOKEN" \
|
||||||
-t gitea.siegeln.net/cameleer/cameleer3-server:${{ github.sha }} \
|
$TAGS \
|
||||||
-t gitea.siegeln.net/cameleer/cameleer3-server:latest \
|
|
||||||
--cache-from type=registry,ref=gitea.siegeln.net/cameleer/cameleer3-server:buildcache \
|
--cache-from type=registry,ref=gitea.siegeln.net/cameleer/cameleer3-server:buildcache \
|
||||||
--cache-to type=registry,ref=gitea.siegeln.net/cameleer/cameleer3-server:buildcache,mode=max \
|
--cache-to type=registry,ref=gitea.siegeln.net/cameleer/cameleer3-server:buildcache,mode=max \
|
||||||
--provenance=false \
|
--provenance=false \
|
||||||
@@ -91,10 +114,13 @@ jobs:
|
|||||||
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||||
- name: Build and push UI
|
- name: Build and push UI
|
||||||
run: |
|
run: |
|
||||||
|
TAGS="-t gitea.siegeln.net/cameleer/cameleer3-server-ui:${{ github.sha }}"
|
||||||
|
for TAG in $IMAGE_TAGS; do
|
||||||
|
TAGS="$TAGS -t gitea.siegeln.net/cameleer/cameleer3-server-ui:$TAG"
|
||||||
|
done
|
||||||
docker buildx build --platform linux/amd64 \
|
docker buildx build --platform linux/amd64 \
|
||||||
-f ui/Dockerfile \
|
-f ui/Dockerfile \
|
||||||
-t gitea.siegeln.net/cameleer/cameleer3-server-ui:${{ github.sha }} \
|
$TAGS \
|
||||||
-t gitea.siegeln.net/cameleer/cameleer3-server-ui:latest \
|
|
||||||
--cache-from type=registry,ref=gitea.siegeln.net/cameleer/cameleer3-server-ui:buildcache \
|
--cache-from type=registry,ref=gitea.siegeln.net/cameleer/cameleer3-server-ui:buildcache \
|
||||||
--cache-to type=registry,ref=gitea.siegeln.net/cameleer/cameleer3-server-ui:buildcache,mode=max \
|
--cache-to type=registry,ref=gitea.siegeln.net/cameleer/cameleer3-server-ui:buildcache,mode=max \
|
||||||
--provenance=false \
|
--provenance=false \
|
||||||
@@ -110,13 +136,28 @@ jobs:
|
|||||||
API="https://gitea.siegeln.net/api/v1"
|
API="https://gitea.siegeln.net/api/v1"
|
||||||
AUTH="Authorization: token ${REGISTRY_TOKEN}"
|
AUTH="Authorization: token ${REGISTRY_TOKEN}"
|
||||||
CURRENT_SHA="${{ github.sha }}"
|
CURRENT_SHA="${{ github.sha }}"
|
||||||
|
# Build list of tags to keep
|
||||||
|
KEEP_TAGS="latest buildcache $CURRENT_SHA"
|
||||||
|
if [ "$BRANCH_SLUG" != "main" ]; then
|
||||||
|
KEEP_TAGS="$KEEP_TAGS branch-$BRANCH_SLUG"
|
||||||
|
fi
|
||||||
for PKG in cameleer3-server cameleer3-server-ui; do
|
for PKG in cameleer3-server cameleer3-server-ui; do
|
||||||
curl -sf -H "$AUTH" "$API/packages/cameleer/container/$PKG" | \
|
curl -sf -H "$AUTH" "$API/packages/cameleer/container/$PKG" | \
|
||||||
jq -r '.[] | "\(.id) \(.version)"' | \
|
jq -r '.[] | "\(.id) \(.version)"' | \
|
||||||
while read id version; do
|
while read id version; do
|
||||||
if [ "$version" != "latest" ] && [ "$version" != "$CURRENT_SHA" ]; then
|
SHOULD_KEEP=false
|
||||||
echo "Deleting old image tag: $PKG:$version"
|
for KEEP in $KEEP_TAGS; do
|
||||||
curl -sf -X DELETE -H "$AUTH" "$API/packages/cameleer/container/$PKG/$version"
|
if [ "$version" = "$KEEP" ]; then
|
||||||
|
SHOULD_KEEP=true
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [ "$SHOULD_KEEP" = "false" ]; then
|
||||||
|
# Only clean up images for this branch
|
||||||
|
if [ "$BRANCH_SLUG" = "main" ] || echo "$version" | grep -q "branch-$BRANCH_SLUG"; then
|
||||||
|
echo "Deleting old image tag: $PKG:$version"
|
||||||
|
curl -sf -X DELETE -H "$AUTH" "$API/packages/cameleer/container/$PKG/$version"
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
done
|
done
|
||||||
@@ -131,6 +172,9 @@ jobs:
|
|||||||
container:
|
container:
|
||||||
image: bitnami/kubectl:latest
|
image: bitnami/kubectl:latest
|
||||||
steps:
|
steps:
|
||||||
|
- name: Install required tools
|
||||||
|
run: |
|
||||||
|
apt-get update && apt-get install -y --no-install-recommends git ca-certificates
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
run: |
|
run: |
|
||||||
git clone --depth=1 --branch=${GITHUB_REF_NAME} https://cameleer:${REGISTRY_TOKEN}@gitea.siegeln.net/${GITHUB_REPOSITORY}.git .
|
git clone --depth=1 --branch=${GITHUB_REF_NAME} https://cameleer:${REGISTRY_TOKEN}@gitea.siegeln.net/${GITHUB_REPOSITORY}.git .
|
||||||
@@ -198,12 +242,11 @@ jobs:
|
|||||||
kubectl apply -f deploy/authentik.yaml
|
kubectl apply -f deploy/authentik.yaml
|
||||||
kubectl -n cameleer rollout status deployment/authentik-server --timeout=180s
|
kubectl -n cameleer rollout status deployment/authentik-server --timeout=180s
|
||||||
|
|
||||||
kubectl apply -f deploy/server.yaml
|
kubectl apply -k deploy/overlays/main
|
||||||
kubectl -n cameleer set image deployment/cameleer3-server \
|
kubectl -n cameleer set image deployment/cameleer3-server \
|
||||||
server=gitea.siegeln.net/cameleer/cameleer3-server:${{ github.sha }}
|
server=gitea.siegeln.net/cameleer/cameleer3-server:${{ github.sha }}
|
||||||
kubectl -n cameleer rollout status deployment/cameleer3-server --timeout=120s
|
kubectl -n cameleer rollout status deployment/cameleer3-server --timeout=120s
|
||||||
|
|
||||||
kubectl apply -f deploy/ui.yaml
|
|
||||||
kubectl -n cameleer set image deployment/cameleer3-ui \
|
kubectl -n cameleer set image deployment/cameleer3-ui \
|
||||||
ui=gitea.siegeln.net/cameleer/cameleer3-server-ui:${{ github.sha }}
|
ui=gitea.siegeln.net/cameleer/cameleer3-server-ui:${{ github.sha }}
|
||||||
kubectl -n cameleer rollout status deployment/cameleer3-ui --timeout=120s
|
kubectl -n cameleer rollout status deployment/cameleer3-ui --timeout=120s
|
||||||
@@ -225,3 +268,147 @@ jobs:
|
|||||||
CAMELEER_OIDC_ISSUER: ${{ secrets.CAMELEER_OIDC_ISSUER }}
|
CAMELEER_OIDC_ISSUER: ${{ secrets.CAMELEER_OIDC_ISSUER }}
|
||||||
CAMELEER_OIDC_CLIENT_ID: ${{ secrets.CAMELEER_OIDC_CLIENT_ID }}
|
CAMELEER_OIDC_CLIENT_ID: ${{ secrets.CAMELEER_OIDC_CLIENT_ID }}
|
||||||
CAMELEER_OIDC_CLIENT_SECRET: ${{ secrets.CAMELEER_OIDC_CLIENT_SECRET }}
|
CAMELEER_OIDC_CLIENT_SECRET: ${{ secrets.CAMELEER_OIDC_CLIENT_SECRET }}
|
||||||
|
|
||||||
|
deploy-feature:
|
||||||
|
needs: docker
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
if: github.ref != 'refs/heads/main' && github.event_name == 'push'
|
||||||
|
container:
|
||||||
|
image: bitnami/kubectl:latest
|
||||||
|
steps:
|
||||||
|
- name: Install required tools
|
||||||
|
run: |
|
||||||
|
apt-get update && apt-get install -y --no-install-recommends git jq sed ca-certificates
|
||||||
|
- name: Checkout
|
||||||
|
run: |
|
||||||
|
git clone --depth=1 --branch=${GITHUB_REF_NAME} https://cameleer:${REGISTRY_TOKEN}@gitea.siegeln.net/${GITHUB_REPOSITORY}.git .
|
||||||
|
env:
|
||||||
|
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||||
|
- name: Configure kubectl
|
||||||
|
run: |
|
||||||
|
mkdir -p ~/.kube
|
||||||
|
echo "$KUBECONFIG_B64" | base64 -d > ~/.kube/config
|
||||||
|
env:
|
||||||
|
KUBECONFIG_B64: ${{ secrets.KUBECONFIG_BASE64 }}
|
||||||
|
- name: Compute branch variables
|
||||||
|
run: |
|
||||||
|
sanitize_branch() {
|
||||||
|
echo "$1" | sed -E 's#^(feature|fix|feat|hotfix)/##' \
|
||||||
|
| tr '[:upper:]' '[:lower:]' \
|
||||||
|
| sed 's/[^a-z0-9-]/-/g' \
|
||||||
|
| sed 's/--*/-/g; s/^-//; s/-$//' \
|
||||||
|
| cut -c1-20 \
|
||||||
|
| sed 's/-$//'
|
||||||
|
}
|
||||||
|
SLUG=$(sanitize_branch "$GITHUB_REF_NAME")
|
||||||
|
NS="cam-${SLUG}"
|
||||||
|
SCHEMA="cam_$(echo $SLUG | tr '-' '_')"
|
||||||
|
echo "BRANCH_SLUG=$SLUG" >> "$GITHUB_ENV"
|
||||||
|
echo "BRANCH_NS=$NS" >> "$GITHUB_ENV"
|
||||||
|
echo "BRANCH_SCHEMA=$SCHEMA" >> "$GITHUB_ENV"
|
||||||
|
- name: Create namespace
|
||||||
|
run: kubectl create namespace "$BRANCH_NS" --dry-run=client -o yaml | kubectl apply -f -
|
||||||
|
- name: Copy secrets from cameleer namespace
|
||||||
|
run: |
|
||||||
|
for SECRET in gitea-registry postgres-credentials opensearch-credentials cameleer-auth cameleer-oidc; do
|
||||||
|
kubectl get secret "$SECRET" -n cameleer -o json \
|
||||||
|
| jq 'del(.metadata.namespace, .metadata.resourceVersion, .metadata.uid, .metadata.creationTimestamp, .metadata.managedFields)' \
|
||||||
|
| kubectl apply -n "$BRANCH_NS" -f -
|
||||||
|
done
|
||||||
|
- name: Substitute placeholders and deploy
|
||||||
|
run: |
|
||||||
|
# Work on a copy so we don't modify the repo
|
||||||
|
cp -r deploy/overlays/feature /tmp/feature-overlay
|
||||||
|
# Substitute all BRANCH_* placeholders
|
||||||
|
for f in /tmp/feature-overlay/*.yaml; do
|
||||||
|
sed -i \
|
||||||
|
-e "s|BRANCH_NAMESPACE|${BRANCH_NS}|g" \
|
||||||
|
-e "s|BRANCH_SCHEMA|${BRANCH_SCHEMA}|g" \
|
||||||
|
-e "s|BRANCH_SLUG|${BRANCH_SLUG}|g" \
|
||||||
|
-e "s|BRANCH_SHA|${{ github.sha }}|g" \
|
||||||
|
"$f"
|
||||||
|
done
|
||||||
|
# Fix kustomization base path (since we moved the overlay)
|
||||||
|
sed -i 's|../../base|'"$(pwd)"'/deploy/base|g' /tmp/feature-overlay/kustomization.yaml
|
||||||
|
kubectl apply -k /tmp/feature-overlay
|
||||||
|
- name: Wait for init-job
|
||||||
|
run: |
|
||||||
|
kubectl -n "$BRANCH_NS" wait --for=condition=complete job/init-schema --timeout=60s || \
|
||||||
|
echo "Warning: init-schema job did not complete in time"
|
||||||
|
- name: Wait for server rollout
|
||||||
|
run: kubectl -n "$BRANCH_NS" rollout status deployment/cameleer3-server --timeout=120s
|
||||||
|
- name: Wait for UI rollout
|
||||||
|
run: kubectl -n "$BRANCH_NS" rollout status deployment/cameleer3-ui --timeout=60s
|
||||||
|
- name: Print deployment URLs
|
||||||
|
run: |
|
||||||
|
echo "===================================="
|
||||||
|
echo "Feature branch deployed!"
|
||||||
|
echo "API: http://${BRANCH_SLUG}-api.cameleer.siegeln.net"
|
||||||
|
echo "UI: http://${BRANCH_SLUG}.cameleer.siegeln.net"
|
||||||
|
echo "===================================="
|
||||||
|
env:
|
||||||
|
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||||
|
|
||||||
|
cleanup-branch:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
if: github.event_name == 'delete' && github.event.ref_type == 'branch'
|
||||||
|
container:
|
||||||
|
image: bitnami/kubectl:latest
|
||||||
|
steps:
|
||||||
|
- name: Install required tools
|
||||||
|
run: |
|
||||||
|
apt-get update && apt-get install -y --no-install-recommends curl jq ca-certificates
|
||||||
|
- name: Configure kubectl
|
||||||
|
run: |
|
||||||
|
mkdir -p ~/.kube
|
||||||
|
echo "$KUBECONFIG_B64" | base64 -d > ~/.kube/config
|
||||||
|
env:
|
||||||
|
KUBECONFIG_B64: ${{ secrets.KUBECONFIG_BASE64 }}
|
||||||
|
- name: Compute branch variables
|
||||||
|
run: |
|
||||||
|
sanitize_branch() {
|
||||||
|
echo "$1" | sed -E 's#^(feature|fix|feat|hotfix)/##' \
|
||||||
|
| tr '[:upper:]' '[:lower:]' \
|
||||||
|
| sed 's/[^a-z0-9-]/-/g' \
|
||||||
|
| sed 's/--*/-/g; s/^-//; s/-$//' \
|
||||||
|
| cut -c1-20 \
|
||||||
|
| sed 's/-$//'
|
||||||
|
}
|
||||||
|
SLUG=$(sanitize_branch "${{ github.event.ref }}")
|
||||||
|
NS="cam-${SLUG}"
|
||||||
|
SCHEMA="cam_$(echo $SLUG | tr '-' '_')"
|
||||||
|
echo "BRANCH_SLUG=$SLUG" >> "$GITHUB_ENV"
|
||||||
|
echo "BRANCH_NS=$NS" >> "$GITHUB_ENV"
|
||||||
|
echo "BRANCH_SCHEMA=$SCHEMA" >> "$GITHUB_ENV"
|
||||||
|
- name: Delete namespace
|
||||||
|
run: kubectl delete namespace "$BRANCH_NS" --ignore-not-found
|
||||||
|
- name: Drop PostgreSQL schema
|
||||||
|
run: |
|
||||||
|
kubectl run cleanup-schema-${BRANCH_SLUG} \
|
||||||
|
--namespace=cameleer \
|
||||||
|
--image=postgres:16 \
|
||||||
|
--restart=Never \
|
||||||
|
--env="PGPASSWORD=$(kubectl get secret postgres-credentials -n cameleer -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d)" \
|
||||||
|
--command -- sh -c "psql -h postgres -U $(kubectl get secret postgres-credentials -n cameleer -o jsonpath='{.data.POSTGRES_USER}' | base64 -d) -d cameleer3 -c 'DROP SCHEMA IF EXISTS ${BRANCH_SCHEMA} CASCADE'"
|
||||||
|
kubectl wait --for=condition=Ready pod/cleanup-schema-${BRANCH_SLUG} -n cameleer --timeout=30s || true
|
||||||
|
kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/cleanup-schema-${BRANCH_SLUG} -n cameleer --timeout=60s || true
|
||||||
|
kubectl delete pod cleanup-schema-${BRANCH_SLUG} -n cameleer --ignore-not-found
|
||||||
|
- name: Delete OpenSearch indices
|
||||||
|
run: |
|
||||||
|
kubectl run cleanup-indices-${BRANCH_SLUG} \
|
||||||
|
--namespace=cameleer \
|
||||||
|
--image=curlimages/curl:latest \
|
||||||
|
--restart=Never \
|
||||||
|
--command -- curl -sf -X DELETE "http://opensearch:9200/cam-${BRANCH_SLUG}-*"
|
||||||
|
kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/cleanup-indices-${BRANCH_SLUG} -n cameleer --timeout=60s || true
|
||||||
|
kubectl delete pod cleanup-indices-${BRANCH_SLUG} -n cameleer --ignore-not-found
|
||||||
|
- name: Cleanup Docker images
|
||||||
|
run: |
|
||||||
|
API="https://gitea.siegeln.net/api/v1"
|
||||||
|
AUTH="Authorization: token ${REGISTRY_TOKEN}"
|
||||||
|
for PKG in cameleer3-server cameleer3-server-ui; do
|
||||||
|
# Delete branch-specific tag
|
||||||
|
curl -sf -X DELETE -H "$AUTH" "$API/packages/cameleer/container/$PKG/branch-${BRANCH_SLUG}" || true
|
||||||
|
done
|
||||||
|
env:
|
||||||
|
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import org.opensearch.client.opensearch.core.search.Hit;
|
|||||||
import org.opensearch.client.opensearch.indices.*;
|
import org.opensearch.client.opensearch.indices.*;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
import org.springframework.stereotype.Repository;
|
import org.springframework.stereotype.Repository;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
@@ -30,25 +31,29 @@ import java.util.stream.Collectors;
|
|||||||
public class OpenSearchIndex implements SearchIndex {
|
public class OpenSearchIndex implements SearchIndex {
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(OpenSearchIndex.class);
|
private static final Logger log = LoggerFactory.getLogger(OpenSearchIndex.class);
|
||||||
private static final String INDEX_PREFIX = "executions-";
|
|
||||||
private static final DateTimeFormatter DAY_FMT = DateTimeFormatter.ofPattern("yyyy-MM-dd")
|
private static final DateTimeFormatter DAY_FMT = DateTimeFormatter.ofPattern("yyyy-MM-dd")
|
||||||
.withZone(ZoneOffset.UTC);
|
.withZone(ZoneOffset.UTC);
|
||||||
|
|
||||||
private final OpenSearchClient client;
|
private final OpenSearchClient client;
|
||||||
|
private final String indexPrefix;
|
||||||
|
|
||||||
public OpenSearchIndex(OpenSearchClient client) {
|
public OpenSearchIndex(OpenSearchClient client,
|
||||||
|
@Value("${opensearch.index-prefix:executions-}") String indexPrefix) {
|
||||||
this.client = client;
|
this.client = client;
|
||||||
|
this.indexPrefix = indexPrefix;
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostConstruct
|
@PostConstruct
|
||||||
void ensureIndexTemplate() {
|
void ensureIndexTemplate() {
|
||||||
|
String templateName = indexPrefix + "template";
|
||||||
|
String indexPattern = indexPrefix + "*";
|
||||||
try {
|
try {
|
||||||
boolean exists = client.indices().existsIndexTemplate(
|
boolean exists = client.indices().existsIndexTemplate(
|
||||||
ExistsIndexTemplateRequest.of(b -> b.name("executions-template"))).value();
|
ExistsIndexTemplateRequest.of(b -> b.name(templateName))).value();
|
||||||
if (!exists) {
|
if (!exists) {
|
||||||
client.indices().putIndexTemplate(PutIndexTemplateRequest.of(b -> b
|
client.indices().putIndexTemplate(PutIndexTemplateRequest.of(b -> b
|
||||||
.name("executions-template")
|
.name(templateName)
|
||||||
.indexPatterns(List.of("executions-*"))
|
.indexPatterns(List.of(indexPattern))
|
||||||
.template(t -> t
|
.template(t -> t
|
||||||
.settings(s -> s
|
.settings(s -> s
|
||||||
.numberOfShards("3")
|
.numberOfShards("3")
|
||||||
@@ -65,7 +70,7 @@ public class OpenSearchIndex implements SearchIndex {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void index(ExecutionDocument doc) {
|
public void index(ExecutionDocument doc) {
|
||||||
String indexName = INDEX_PREFIX + DAY_FMT.format(doc.startTime());
|
String indexName = indexPrefix + DAY_FMT.format(doc.startTime());
|
||||||
try {
|
try {
|
||||||
client.index(IndexRequest.of(b -> b
|
client.index(IndexRequest.of(b -> b
|
||||||
.index(indexName)
|
.index(indexName)
|
||||||
@@ -98,7 +103,7 @@ public class OpenSearchIndex implements SearchIndex {
|
|||||||
public long count(SearchRequest request) {
|
public long count(SearchRequest request) {
|
||||||
try {
|
try {
|
||||||
var countReq = CountRequest.of(b -> b
|
var countReq = CountRequest.of(b -> b
|
||||||
.index(INDEX_PREFIX + "*")
|
.index(indexPrefix + "*")
|
||||||
.query(buildQuery(request)));
|
.query(buildQuery(request)));
|
||||||
return client.count(countReq).count();
|
return client.count(countReq).count();
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
@@ -111,7 +116,7 @@ public class OpenSearchIndex implements SearchIndex {
|
|||||||
public void delete(String executionId) {
|
public void delete(String executionId) {
|
||||||
try {
|
try {
|
||||||
client.deleteByQuery(DeleteByQueryRequest.of(b -> b
|
client.deleteByQuery(DeleteByQueryRequest.of(b -> b
|
||||||
.index(List.of(INDEX_PREFIX + "*"))
|
.index(List.of(indexPrefix + "*"))
|
||||||
.query(Query.of(q -> q.term(t -> t
|
.query(Query.of(q -> q.term(t -> t
|
||||||
.field("execution_id")
|
.field("execution_id")
|
||||||
.value(FieldValue.of(executionId)))))));
|
.value(FieldValue.of(executionId)))))));
|
||||||
@@ -123,7 +128,7 @@ public class OpenSearchIndex implements SearchIndex {
|
|||||||
private org.opensearch.client.opensearch.core.SearchRequest buildSearchRequest(
|
private org.opensearch.client.opensearch.core.SearchRequest buildSearchRequest(
|
||||||
SearchRequest request, int size) {
|
SearchRequest request, int size) {
|
||||||
return org.opensearch.client.opensearch.core.SearchRequest.of(b -> {
|
return org.opensearch.client.opensearch.core.SearchRequest.of(b -> {
|
||||||
b.index(INDEX_PREFIX + "*")
|
b.index(indexPrefix + "*")
|
||||||
.query(buildQuery(request))
|
.query(buildQuery(request))
|
||||||
.trackTotalHits(th -> th.enabled(true))
|
.trackTotalHits(th -> th.enabled(true))
|
||||||
.size(size)
|
.size(size)
|
||||||
|
|||||||
@@ -3,13 +3,15 @@ server:
|
|||||||
|
|
||||||
spring:
|
spring:
|
||||||
datasource:
|
datasource:
|
||||||
url: jdbc:postgresql://localhost:5432/cameleer3
|
url: jdbc:postgresql://localhost:5432/cameleer3?currentSchema=${CAMELEER_DB_SCHEMA:public}
|
||||||
username: cameleer
|
username: cameleer
|
||||||
password: ${CAMELEER_DB_PASSWORD:cameleer_dev}
|
password: ${CAMELEER_DB_PASSWORD:cameleer_dev}
|
||||||
driver-class-name: org.postgresql.Driver
|
driver-class-name: org.postgresql.Driver
|
||||||
flyway:
|
flyway:
|
||||||
enabled: true
|
enabled: true
|
||||||
locations: classpath:db/migration
|
locations: classpath:db/migration
|
||||||
|
schemas: ${CAMELEER_DB_SCHEMA:public}
|
||||||
|
default-schema: ${CAMELEER_DB_SCHEMA:public}
|
||||||
mvc:
|
mvc:
|
||||||
async:
|
async:
|
||||||
request-timeout: -1
|
request-timeout: -1
|
||||||
@@ -34,6 +36,7 @@ ingestion:
|
|||||||
|
|
||||||
opensearch:
|
opensearch:
|
||||||
url: ${OPENSEARCH_URL:http://localhost:9200}
|
url: ${OPENSEARCH_URL:http://localhost:9200}
|
||||||
|
index-prefix: ${CAMELEER_OPENSEARCH_INDEX_PREFIX:executions-}
|
||||||
queue-size: ${CAMELEER_OPENSEARCH_QUEUE_SIZE:10000}
|
queue-size: ${CAMELEER_OPENSEARCH_QUEUE_SIZE:10000}
|
||||||
debounce-ms: ${CAMELEER_OPENSEARCH_DEBOUNCE_MS:2000}
|
debounce-ms: ${CAMELEER_OPENSEARCH_DEBOUNCE_MS:2000}
|
||||||
|
|
||||||
|
|||||||
5
deploy/base/kustomization.yaml
Normal file
5
deploy/base/kustomization.yaml
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
resources:
|
||||||
|
- server.yaml
|
||||||
|
- ui.yaml
|
||||||
124
deploy/base/server.yaml
Normal file
124
deploy/base/server.yaml
Normal file
@@ -0,0 +1,124 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: cameleer3-server
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: cameleer3-server
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: cameleer3-server
|
||||||
|
spec:
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: gitea-registry
|
||||||
|
containers:
|
||||||
|
- name: server
|
||||||
|
image: gitea.siegeln.net/cameleer/cameleer3-server:latest
|
||||||
|
ports:
|
||||||
|
- containerPort: 8081
|
||||||
|
env:
|
||||||
|
- name: SPRING_DATASOURCE_URL
|
||||||
|
value: "jdbc:postgresql://postgres.cameleer.svc.cluster.local:5432/cameleer3?currentSchema=$(CAMELEER_DB_SCHEMA)"
|
||||||
|
- name: CAMELEER_DB_SCHEMA
|
||||||
|
value: "public"
|
||||||
|
- name: SPRING_DATASOURCE_USERNAME
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: postgres-credentials
|
||||||
|
key: POSTGRES_USER
|
||||||
|
- name: SPRING_DATASOURCE_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: postgres-credentials
|
||||||
|
key: POSTGRES_PASSWORD
|
||||||
|
- name: OPENSEARCH_URL
|
||||||
|
value: "http://opensearch.cameleer.svc.cluster.local:9200"
|
||||||
|
- name: CAMELEER_OPENSEARCH_INDEX_PREFIX
|
||||||
|
value: "executions-"
|
||||||
|
- name: CAMELEER_AUTH_TOKEN
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: cameleer-auth
|
||||||
|
key: CAMELEER_AUTH_TOKEN
|
||||||
|
- name: CAMELEER_UI_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: cameleer-auth
|
||||||
|
key: CAMELEER_UI_USER
|
||||||
|
optional: true
|
||||||
|
- name: CAMELEER_UI_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: cameleer-auth
|
||||||
|
key: CAMELEER_UI_PASSWORD
|
||||||
|
optional: true
|
||||||
|
- name: CAMELEER_UI_ORIGIN
|
||||||
|
value: "http://localhost:5173"
|
||||||
|
- name: CAMELEER_JWT_SECRET
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: cameleer-auth
|
||||||
|
key: CAMELEER_JWT_SECRET
|
||||||
|
optional: true
|
||||||
|
- name: CAMELEER_OIDC_ENABLED
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: cameleer-oidc
|
||||||
|
key: CAMELEER_OIDC_ENABLED
|
||||||
|
optional: true
|
||||||
|
- name: CAMELEER_OIDC_ISSUER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: cameleer-oidc
|
||||||
|
key: CAMELEER_OIDC_ISSUER
|
||||||
|
optional: true
|
||||||
|
- name: CAMELEER_OIDC_CLIENT_ID
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: cameleer-oidc
|
||||||
|
key: CAMELEER_OIDC_CLIENT_ID
|
||||||
|
optional: true
|
||||||
|
- name: CAMELEER_OIDC_CLIENT_SECRET
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: cameleer-oidc
|
||||||
|
key: CAMELEER_OIDC_CLIENT_SECRET
|
||||||
|
optional: true
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: "256Mi"
|
||||||
|
cpu: "100m"
|
||||||
|
limits:
|
||||||
|
memory: "512Mi"
|
||||||
|
cpu: "500m"
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /api/v1/health
|
||||||
|
port: 8081
|
||||||
|
initialDelaySeconds: 30
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 3
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /api/v1/health
|
||||||
|
port: 8081
|
||||||
|
initialDelaySeconds: 10
|
||||||
|
periodSeconds: 5
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 3
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: cameleer3-server
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app: cameleer3-server
|
||||||
|
ports:
|
||||||
|
- port: 8081
|
||||||
|
targetPort: 8081
|
||||||
71
deploy/base/ui.yaml
Normal file
71
deploy/base/ui.yaml
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: cameleer3-ui-config
|
||||||
|
data:
|
||||||
|
config.js: |
|
||||||
|
window.__CAMELEER_CONFIG__ = {
|
||||||
|
apiBaseUrl: 'http://localhost:8081/api/v1',
|
||||||
|
};
|
||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: cameleer3-ui
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: cameleer3-ui
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: cameleer3-ui
|
||||||
|
spec:
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: gitea-registry
|
||||||
|
containers:
|
||||||
|
- name: ui
|
||||||
|
image: gitea.siegeln.net/cameleer/cameleer3-server-ui:latest
|
||||||
|
ports:
|
||||||
|
- containerPort: 80
|
||||||
|
env:
|
||||||
|
- name: CAMELEER_API_URL
|
||||||
|
value: "http://cameleer3-server:8081"
|
||||||
|
volumeMounts:
|
||||||
|
- name: config
|
||||||
|
mountPath: /usr/share/nginx/html/config.js
|
||||||
|
subPath: config.js
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: "32Mi"
|
||||||
|
cpu: "10m"
|
||||||
|
limits:
|
||||||
|
memory: "64Mi"
|
||||||
|
cpu: "100m"
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /healthz
|
||||||
|
port: 80
|
||||||
|
periodSeconds: 10
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /healthz
|
||||||
|
port: 80
|
||||||
|
periodSeconds: 5
|
||||||
|
volumes:
|
||||||
|
- name: config
|
||||||
|
configMap:
|
||||||
|
name: cameleer3-ui-config
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: cameleer3-ui
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app: cameleer3-ui
|
||||||
|
ports:
|
||||||
|
- port: 80
|
||||||
|
targetPort: 80
|
||||||
26
deploy/overlays/feature/ingress.yaml
Normal file
26
deploy/overlays/feature/ingress.yaml
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
name: cameleer-branch-ingress
|
||||||
|
spec:
|
||||||
|
rules:
|
||||||
|
- host: BRANCH_SLUG-api.cameleer.siegeln.net
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: cameleer3-server
|
||||||
|
port:
|
||||||
|
number: 8081
|
||||||
|
- host: BRANCH_SLUG.cameleer.siegeln.net
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: cameleer3-ui
|
||||||
|
port:
|
||||||
|
number: 80
|
||||||
30
deploy/overlays/feature/init-job.yaml
Normal file
30
deploy/overlays/feature/init-job.yaml
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: init-schema
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: Never
|
||||||
|
containers:
|
||||||
|
- name: init
|
||||||
|
image: postgres:16
|
||||||
|
command: ["sh", "-c"]
|
||||||
|
args:
|
||||||
|
- |
|
||||||
|
PGPASSWORD=$POSTGRES_PASSWORD psql \
|
||||||
|
-h postgres.cameleer.svc.cluster.local \
|
||||||
|
-U $POSTGRES_USER -d cameleer3 \
|
||||||
|
-c "CREATE SCHEMA IF NOT EXISTS BRANCH_SCHEMA"
|
||||||
|
env:
|
||||||
|
- name: POSTGRES_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: postgres-credentials
|
||||||
|
key: POSTGRES_USER
|
||||||
|
- name: POSTGRES_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: postgres-credentials
|
||||||
|
key: POSTGRES_PASSWORD
|
||||||
|
backoffLimit: 3
|
||||||
44
deploy/overlays/feature/kustomization.yaml
Normal file
44
deploy/overlays/feature/kustomization.yaml
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
namespace: BRANCH_NAMESPACE
|
||||||
|
resources:
|
||||||
|
- ../../base
|
||||||
|
- ingress.yaml
|
||||||
|
- init-job.yaml
|
||||||
|
images:
|
||||||
|
- name: gitea.siegeln.net/cameleer/cameleer3-server
|
||||||
|
newTag: BRANCH_SHA
|
||||||
|
- name: gitea.siegeln.net/cameleer/cameleer3-server-ui
|
||||||
|
newTag: BRANCH_SHA
|
||||||
|
patches:
|
||||||
|
# Server Deployment: branch-specific schema, index prefix, UI origin, OIDC disabled
|
||||||
|
- patch: |
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: cameleer3-server
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: server
|
||||||
|
env:
|
||||||
|
- name: CAMELEER_DB_SCHEMA
|
||||||
|
value: "BRANCH_SCHEMA"
|
||||||
|
- name: CAMELEER_OPENSEARCH_INDEX_PREFIX
|
||||||
|
value: "cam-BRANCH_SLUG-executions-"
|
||||||
|
- name: CAMELEER_UI_ORIGIN
|
||||||
|
value: "http://BRANCH_SLUG.cameleer.siegeln.net"
|
||||||
|
- name: CAMELEER_OIDC_ENABLED
|
||||||
|
value: "false"
|
||||||
|
# UI ConfigMap: branch-specific API URL
|
||||||
|
- target:
|
||||||
|
kind: ConfigMap
|
||||||
|
name: cameleer3-ui-config
|
||||||
|
patch: |
|
||||||
|
- op: replace
|
||||||
|
path: /data/config.js
|
||||||
|
value: |
|
||||||
|
window.__CAMELEER_CONFIG__ = {
|
||||||
|
apiBaseUrl: 'http://BRANCH_SLUG-api.cameleer.siegeln.net/api/v1',
|
||||||
|
};
|
||||||
57
deploy/overlays/main/kustomization.yaml
Normal file
57
deploy/overlays/main/kustomization.yaml
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
namespace: cameleer
|
||||||
|
resources:
|
||||||
|
- ../../base
|
||||||
|
patches:
|
||||||
|
# Server Service: NodePort 30081
|
||||||
|
- target:
|
||||||
|
kind: Service
|
||||||
|
name: cameleer3-server
|
||||||
|
patch: |
|
||||||
|
- op: replace
|
||||||
|
path: /spec/type
|
||||||
|
value: NodePort
|
||||||
|
- op: add
|
||||||
|
path: /spec/ports/0/nodePort
|
||||||
|
value: 30081
|
||||||
|
# UI Service: NodePort 30090
|
||||||
|
- target:
|
||||||
|
kind: Service
|
||||||
|
name: cameleer3-ui
|
||||||
|
patch: |
|
||||||
|
- op: replace
|
||||||
|
path: /spec/type
|
||||||
|
value: NodePort
|
||||||
|
- op: add
|
||||||
|
path: /spec/ports/0/nodePort
|
||||||
|
value: 30090
|
||||||
|
# Server Deployment: same-namespace DNS + production UI origin
|
||||||
|
- patch: |
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: cameleer3-server
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: server
|
||||||
|
env:
|
||||||
|
- name: SPRING_DATASOURCE_URL
|
||||||
|
value: "jdbc:postgresql://postgres:5432/cameleer3?currentSchema=public"
|
||||||
|
- name: OPENSEARCH_URL
|
||||||
|
value: "http://opensearch:9200"
|
||||||
|
- name: CAMELEER_UI_ORIGIN
|
||||||
|
value: "http://192.168.50.86:30090"
|
||||||
|
# UI ConfigMap: production API URL
|
||||||
|
- target:
|
||||||
|
kind: ConfigMap
|
||||||
|
name: cameleer3-ui-config
|
||||||
|
patch: |
|
||||||
|
- op: replace
|
||||||
|
path: /data/config.js
|
||||||
|
value: |
|
||||||
|
window.__CAMELEER_CONFIG__ = {
|
||||||
|
apiBaseUrl: 'http://192.168.50.86:30081/api/v1',
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user