From 1ddae94930aaa4af7bc2e35bfc5530e4122564cf Mon Sep 17 00:00:00 2001 From: hsiegeln <37154749+hsiegeln@users.noreply.github.com> Date: Mon, 27 Apr 2026 16:06:56 +0200 Subject: [PATCH] feat(runtime): init-container loader pattern + withUsernsMode (#152 hardening close) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tasks 9+10+11 of the init-container-jar-fetch plan, landed atomically because 9 alone leaves the orchestrator+executor referencing removed ContainerRequest fields. ContainerRequest (core) drops jarPath/jarVolumeName/jarVolumeMountPath; adds appVersionId, artifactDownloadUrl, artifactExpectedSize, loaderImage. DockerRuntimeOrchestrator (app): - per-replica named volume "cameleer-jars-{containerName}" - phase 1: loader container with the volume mounted RW at /app/jars, ARTIFACT_URL + ARTIFACT_EXPECTED_SIZE env, full hardening contract - block on waitContainerCmd().awaitStatusCode(120s); on non-zero exit remove the loader, remove the volume, propagate RuntimeException so DeploymentExecutor marks the deployment FAILED. main is never created. - phase 2: main container with the same volume mounted RO at /app/jars - withUsernsMode("host:1000:65536") on BOTH containers — closes the last open hardening gap from issue #152 - main entrypoint paths point at /app/jars/app.jar - extracted baseHardenedHostConfig() so loader and main share the cap_drop / security_opt / readonly / pids / tmpfs contract - removeContainer() also removes the per-replica volume so blue/green doesn't leak volumes DeploymentExecutor (app): - injects ArtifactDownloadTokenSigner; new @Value props loaderimage, artifacttokenttlseconds, artifactbaseurl - replaces the temporary getVersion(...).jarPath() bridge with a signed URL ${artifactBaseUrl}/api/v1/artifacts/{id}?exp&sig - drops the Files.exists pre-flight check; AppVersion.jarSizeBytes is the size-of-record check now - drops jarDockerVolume / jarStoragePath @Value fields and the volume plumbing in startReplica - DeployCtx carries appVersionId / artifactUrl / artifactExpectedSize in place of jarPath Tests: - DockerRuntimeOrchestratorHardeningTest updated for the new shape; captures HostConfig on the MAIN container and asserts cap_drop ALL + no-new-privileges + apparmor + readonly + pids + tmpfs + the new withUsernsMode("host:1000:65536") - DockerRuntimeOrchestratorLoaderTest (new): verifies volume create → loader create with RW bind → loader started → awaited → loader removed → main create with RO bind → main started; verifies abort + cleanup on loader exit != 0 (loader removed, volume removed, main NEVER created); verifies userns_mode applied to both containers. Config: - application.yml replaces jardockervolume with loaderimage, artifacttokenttlseconds, artifactbaseurl Rules updated: .claude/rules/docker-orchestration.md (loader pattern, userns, no more bind-mount); .claude/rules/core-classes.md (ContainerRequest field map). Test counts after change: - cameleer-server-core: 116/116 unit tests pass - cameleer-server-app: 273/273 unit tests pass Co-Authored-By: Claude Opus 4.7 (1M context) --- .claude/rules/core-classes.md | 2 +- .claude/rules/docker-orchestration.md | 18 +- .../app/runtime/DeploymentExecutor.java | 61 ++++-- .../runtime/DockerRuntimeOrchestrator.java | 189 +++++++++++----- .../src/main/resources/application.yml | 8 +- ...ockerRuntimeOrchestratorHardeningTest.java | 90 +++++--- .../DockerRuntimeOrchestratorLoaderTest.java | 204 ++++++++++++++++++ .../server/core/runtime/ContainerRequest.java | 8 +- 8 files changed, 473 insertions(+), 107 deletions(-) create mode 100644 cameleer-server-app/src/test/java/com/cameleer/server/app/runtime/DockerRuntimeOrchestratorLoaderTest.java diff --git a/.claude/rules/core-classes.md b/.claude/rules/core-classes.md index 5c2e9351..d765f4ee 100644 --- a/.claude/rules/core-classes.md +++ b/.claude/rules/core-classes.md @@ -35,7 +35,7 @@ paths: - `DeploymentService` — createDeployment (calls `deleteFailedByAppAndEnvironment` first so FAILED rows don't pile up; STOPPED rows are preserved as restorable checkpoints), markRunning, markFailed, markStopped - `RuntimeType` — enum: AUTO, SPRING_BOOT, QUARKUS, PLAIN_JAVA, NATIVE - `RuntimeDetector` — probes JAR files at upload time: detects runtime from manifest Main-Class (Spring Boot loader, Quarkus entry point, plain Java) or native binary (non-ZIP magic bytes) -- `ContainerRequest` — record: 20 fields for Docker container creation (includes runtimeType, customArgs, mainClass) +- `ContainerRequest` — record: 21 fields for Docker container creation. Replaces the legacy `jarPath`/`jarVolumeName`/`jarVolumeMountPath` triple with `appVersionId` (UUID), `artifactDownloadUrl` (signed), `artifactExpectedSize` (bytes), and `loaderImage`. The orchestrator's loader init-container fetches the JAR from the URL into a per-replica named volume; the main container reads it from `/app/jars/app.jar`. - `ContainerStatus` — record: state, running, exitCode, error - `ResolvedContainerConfig` — record: typed config with memoryLimitMb, memoryReserveMb, cpuRequest, cpuLimit, appPort, exposedPorts, customEnvVars, stripPathPrefix, sslOffloading, routingMode, routingDomain, serverUrl, replicas, deploymentStrategy, routeControlEnabled, replayEnabled, runtimeType, customArgs, extraNetworks, externalRouting (default `true`; when `false`, `TraefikLabelBuilder` strips all `traefik.*` labels so the container is not publicly routed), certResolver (server-wide, sourced from `CAMELEER_SERVER_RUNTIME_CERTRESOLVER`; when blank the `tls.certresolver` label is omitted — use for dev installs with a static TLS store) - `RoutingMode` — enum for routing strategies diff --git a/.claude/rules/docker-orchestration.md b/.claude/rules/docker-orchestration.md index 89ecbc72..65d95c79 100644 --- a/.claude/rules/docker-orchestration.md +++ b/.claude/rules/docker-orchestration.md @@ -25,16 +25,29 @@ When deployed via the cameleer-saas platform, this server orchestrates customer ## Container Hardening (issue #152) -`DockerRuntimeOrchestrator.startContainer` applies an unconditional hardening contract to every tenant container — Java 17 has no SecurityManager so the JVM is not a security boundary, and isolation must live below it. Defaults are fail-closed and have no opt-out: +`DockerRuntimeOrchestrator.startContainer` applies an unconditional hardening contract to BOTH the loader init-container AND the main tenant container (`baseHardenedHostConfig()` is the shared helper). Java 17 has no SecurityManager so the JVM is not a security boundary, and isolation must live below it. Defaults are fail-closed and have no opt-out: - `cap_drop` = every `Capability.values()` (effectively ALL — docker-java's enum has no `ALL` constant). Outbound TCP still works (no caps needed); raw sockets, ptrace, mounts, and bind <1024 are denied. - `security_opt`: `no-new-privileges:true`, `apparmor=docker-default`. Default seccomp profile is applied implicitly when `seccomp=` is absent. - `read_only` rootfs = true. - `pids_limit` = 512 (`PIDS_LIMIT` constant). - `tmpfs` mount: `/tmp` with `rw,nosuid,size=256m`. **No `noexec`** — Netty/tcnative, Snappy, LZ4, Zstd dlopen native libs from `/tmp` via `mmap(PROT_EXEC)` which `noexec` blocks. Issue #153 will add per-app `writeableVolumes` for stateful tenants (Kafka Streams etc.). +- `userns_mode` = `host:1000:65536` on both loader and main. Container root is never UID 0 on the host — closes the last open hardening item from issue #152. **Sandboxed runtime auto-detect**: at construction the orchestrator calls `dockerClient.infoCmd().exec().getRuntimes()` and uses `runsc` (gVisor) when present. Override with `cameleer.server.runtime.dockerruntime` (e.g. `kata` to force Kata Containers, or any other registered runtime). Empty/blank = auto. The override always wins over auto-detect. The `DockerRuntimeOrchestrator(DockerClient, String)` constructor is the canonical entry point; the single-arg constructor exists only as a convenience for tests that don't need an override. +## Init-Container Loader Pattern (JAR fetch) + +`startContainer` is now a two-phase op per replica: + +1. **Volume create** — `cameleer-jars-{containerName}` named volume (per-replica, deterministic so cleanup in `removeContainer` can derive it). +2. **Loader container** — `loaderImage` (default `gitea.siegeln.net/cameleer/cameleer-runtime-loader:latest`), name `{containerName}-loader`, mount the volume **RW at `/app/jars`**, env vars `ARTIFACT_URL` + `ARTIFACT_EXPECTED_SIZE`. Loader downloads the JAR from the signed URL into the volume and exits 0. Orchestrator blocks on `waitContainerCmd().exec(WaitContainerResultCallback).awaitStatusCode(120, SECONDS)`. Loader container is removed in a `finally` block; on non-zero exit the volume is also removed and `RuntimeException` propagates so `DeploymentExecutor` marks the deployment FAILED. +3. **Main container** — same hardening contract, mount the same volume **RO at `/app/jars`**, entrypoint reads `/app/jars/app.jar` (Spring Boot/Quarkus: `-jar /app/jars/app.jar`; plain Java: `-cp /app/jars/app.jar `; native: `exec /app/jars/app.jar`). + +`removeContainer(id)` derives the volume name from the inspected container name (Docker prefixes it with `/`) and removes the volume after the container removes — blue/green doesn't leak volumes. + +`DeploymentExecutor` generates the signed URL via `ArtifactDownloadTokenSigner.sign(appVersion.id(), Duration.ofSeconds(artifactTokenTtlSeconds))` and passes `appVersion.id()`, the URL, `appVersion.jarSizeBytes()`, and the loader image into `ContainerRequest`. The host filesystem is no longer involved at deploy time. + ## DeploymentExecutor Details Primary network for app containers is set via `CAMELEER_SERVER_RUNTIME_DOCKERNETWORK` env var (in SaaS mode: `cameleer-tenant-{slug}`); apps also connect to `cameleer-traefik` (routing) and `cameleer-env-{tenantId}-{envSlug}` (per-environment discovery) as additional networks. Resolves `runtimeType: auto` to concrete type from `AppVersion.detectedRuntimeType` at PRE_FLIGHT (fails deployment if unresolvable). Builds Docker entrypoint per runtime type (all JVM types use `-javaagent:/app/agent.jar -jar`, plain Java uses `-cp` with main class, native runs binary directly). Sets per-replica `CAMELEER_AGENT_INSTANCEID` env var to `{envSlug}-{appSlug}-{replicaIndex}-{generation}` so container logs and agent logs share the same instance identity. Sets `CAMELEER_AGENT_*` env vars from `ResolvedContainerConfig` (routeControlEnabled, replayEnabled, health port). These are startup-only agent properties — changing them requires redeployment. @@ -67,7 +80,8 @@ Traffic routing is implicit: Traefik labels (`cameleer.app`, `cameleer.environme - **Retention policy** per environment: configurable maximum number of JAR versions to keep. Older JARs are deleted automatically. - **Nightly cleanup job** (`JarRetentionJob`, Spring `@Scheduled` 03:00): purges JARs exceeding the retention limit and removes orphaned files not referenced by any app version. Skips versions currently deployed. -- **Volume-based JAR mounting** for Docker-in-Docker setups: set `CAMELEER_SERVER_RUNTIME_JARDOCKERVOLUME` to the Docker volume name that contains the JAR storage directory. When set, the orchestrator mounts this volume into the container instead of bind-mounting the host path (required when the SaaS container itself runs inside Docker and the host path is not accessible from sibling containers). +- **Storage abstraction**: `ArtifactStore` (in `cameleer-server-core/storage`) is the only path that touches JAR bytes. `FilesystemArtifactStore` writes under `cameleer.server.runtime.jarstoragepath` (default `/data/jars`); the orchestrator never reads the host filesystem at deploy time. +- **Loader-fetch at deploy time**: tenant containers no longer bind-mount JARs from the host. The loader init-container streams the JAR via a signed URL (HMAC-SHA256, TTL `cameleer.server.runtime.artifacttokenttlseconds`, default 600s) into a per-replica named volume; main mounts that volume RO. This works without host-path access and is the single path supported in Docker-in-Docker SaaS deployments. ## Runtime Type Detection diff --git a/cameleer-server-app/src/main/java/com/cameleer/server/app/runtime/DeploymentExecutor.java b/cameleer-server-app/src/main/java/com/cameleer/server/app/runtime/DeploymentExecutor.java index 01b5d134..78841946 100644 --- a/cameleer-server-app/src/main/java/com/cameleer/server/app/runtime/DeploymentExecutor.java +++ b/cameleer-server-app/src/main/java/com/cameleer/server/app/runtime/DeploymentExecutor.java @@ -6,6 +6,7 @@ import com.cameleer.server.app.license.LicenseUsageReader; import com.cameleer.server.app.metrics.ServerMetrics; import com.cameleer.server.app.storage.PostgresApplicationConfigRepository; import com.cameleer.server.app.storage.PostgresDeploymentRepository; +import com.cameleer.server.app.web.ArtifactDownloadTokenSigner; import com.cameleer.server.core.runtime.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,8 +15,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; -import java.nio.file.Files; -import java.nio.file.Path; +import java.time.Duration; import java.util.*; @Service @@ -69,11 +69,14 @@ public class DeploymentExecutor { @Value("${cameleer.server.runtime.certresolver:}") private String globalCertResolver; - @Value("${cameleer.server.runtime.jardockervolume:}") - private String jarDockerVolume; + @Value("${cameleer.server.runtime.loaderimage:gitea.siegeln.net/cameleer/cameleer-runtime-loader:latest}") + private String loaderImage; - @Value("${cameleer.server.runtime.jarstoragepath:/data/jars}") - private String jarStoragePath; + @Value("${cameleer.server.runtime.artifacttokenttlseconds:600}") + private long artifactTokenTtlSeconds; + + @Value("${cameleer.server.runtime.artifactbaseurl:}") + private String artifactBaseUrl; @Value("${cameleer.server.tenant.id:default}") private String tenantId; @@ -81,6 +84,9 @@ public class DeploymentExecutor { @Autowired private ServerMetrics serverMetrics; + @Autowired + private ArtifactDownloadTokenSigner artifactTokenSigner; + public DeploymentExecutor(RuntimeOrchestrator orchestrator, DeploymentService deploymentService, AppService appService, @@ -117,7 +123,9 @@ public class DeploymentExecutor { App app, Environment env, ResolvedContainerConfig config, - String jarPath, + UUID appVersionId, + String artifactUrl, + long artifactExpectedSize, String resolvedRuntimeType, String mainClass, String generation, @@ -134,10 +142,19 @@ public class DeploymentExecutor { try { App app = appService.getById(deployment.appId()); Environment env = envService.getById(deployment.environmentId()); - // TODO Task 11: replace with signed download URL via ArtifactDownloadController. - // This leaks the filesystem locator out of ArtifactStore — tactical bridge until - // the loader-init-container pattern lands. - String jarPath = appService.getVersion(deployment.appVersionId()).jarPath(); + // Resolve the artifact via a signed download URL — the loader + // container fetches the JAR from this URL and writes it into the + // shared per-replica volume. The orchestrator no longer needs a + // host filesystem path or a Docker volume for JAR mounting. + AppVersion appVersion = appService.getVersion(deployment.appVersionId()); + String artifactBase = artifactBaseUrl.isBlank() + ? (globalServerUrl.isBlank() ? "http://cameleer-server:8081" : globalServerUrl) + : artifactBaseUrl; + ArtifactDownloadTokenSigner.SignedToken token = artifactTokenSigner.sign( + appVersion.id(), Duration.ofSeconds(artifactTokenTtlSeconds)); + String artifactUrl = artifactBase + "/api/v1/artifacts/" + appVersion.id() + + "?exp=" + token.exp() + "&sig=" + token.sig(); + long artifactExpectedSize = appVersion.jarSizeBytes() == null ? 0L : appVersion.jarSizeBytes(); String generation = generationOf(deployment); var globalDefaults = new ConfigMerger.GlobalRuntimeDefaults( @@ -156,7 +173,7 @@ public class DeploymentExecutor { // === PRE-FLIGHT === updateStage(deployment.id(), DeployStage.PRE_FLIGHT); - preFlightChecks(jarPath, config); + preFlightChecks(appVersion, config); // === LICENSE COMPUTE CAPS === // Spec §4.1: sum cpu/memory/replicas across non-stopped deployments + new request @@ -171,11 +188,10 @@ public class DeploymentExecutor { licenseEnforcer.assertWithinCap("max_total_memory_mb", usage.memoryMb(), (long) reqMem * reqReps); licenseEnforcer.assertWithinCap("max_total_replicas", usage.replicas(), reqReps); - // Resolve runtime type + // Resolve runtime type — use the appVersion we already fetched above. String resolvedRuntimeType = config.runtimeType(); String mainClass = null; if ("auto".equalsIgnoreCase(resolvedRuntimeType)) { - AppVersion appVersion = appService.getVersion(deployment.appVersionId()); if (appVersion.detectedRuntimeType() == null) { throw new IllegalStateException( "Could not detect runtime type for JAR '" + appVersion.jarFilename() + @@ -184,7 +200,6 @@ public class DeploymentExecutor { resolvedRuntimeType = appVersion.detectedRuntimeType(); mainClass = appVersion.detectedMainClass(); } else if ("plain-java".equals(resolvedRuntimeType)) { - AppVersion appVersion = appService.getVersion(deployment.appVersionId()); mainClass = appVersion.detectedMainClass(); if (mainClass == null) { throw new IllegalStateException( @@ -224,7 +239,8 @@ public class DeploymentExecutor { } DeployCtx ctx = new DeployCtx( - deployment, app, env, config, jarPath, + deployment, app, env, config, + appVersion.id(), artifactUrl, artifactExpectedSize, resolvedRuntimeType, mainClass, generation, primaryNetwork, additionalNets, buildEnvVars(app, env, config), @@ -450,10 +466,10 @@ public class DeploymentExecutor { Map replicaEnvVars = new LinkedHashMap<>(ctx.baseEnvVars()); replicaEnvVars.put("CAMELEER_AGENT_INSTANCEID", instanceId); - String volumeName = jarDockerVolume != null && !jarDockerVolume.isBlank() ? jarDockerVolume : null; ContainerRequest request = new ContainerRequest( - containerName, baseImage, ctx.jarPath(), - volumeName, jarStoragePath, + containerName, baseImage, + ctx.appVersionId(), ctx.artifactUrl(), ctx.artifactExpectedSize(), + loaderImage, ctx.primaryNetwork(), ctx.additionalNets(), replicaEnvVars, labels, @@ -536,9 +552,10 @@ public class DeploymentExecutor { } } - private void preFlightChecks(String jarPath, ResolvedContainerConfig config) { - if (!Files.exists(Path.of(jarPath))) { - throw new IllegalStateException("JAR file not found: " + jarPath); + private void preFlightChecks(AppVersion appVersion, ResolvedContainerConfig config) { + if (appVersion.jarSizeBytes() == null || appVersion.jarSizeBytes() <= 0) { + throw new IllegalStateException( + "AppVersion " + appVersion.id() + " has no recorded jarSizeBytes"); } if (config.memoryLimitMb() <= 0) { throw new IllegalStateException("Memory limit must be positive, got: " + config.memoryLimitMb()); diff --git a/cameleer-server-app/src/main/java/com/cameleer/server/app/runtime/DockerRuntimeOrchestrator.java b/cameleer-server-app/src/main/java/com/cameleer/server/app/runtime/DockerRuntimeOrchestrator.java index 702a6602..bd24d90b 100644 --- a/cameleer-server-app/src/main/java/com/cameleer/server/app/runtime/DockerRuntimeOrchestrator.java +++ b/cameleer-server-app/src/main/java/com/cameleer/server/app/runtime/DockerRuntimeOrchestrator.java @@ -13,6 +13,7 @@ import com.github.dockerjava.api.model.HealthCheck; import com.github.dockerjava.api.model.HostConfig; import com.github.dockerjava.api.model.RestartPolicy; import com.github.dockerjava.api.model.Volume; +import com.github.dockerjava.core.command.WaitContainerResultCallback; import jakarta.annotation.PreDestroy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; public class DockerRuntimeOrchestrator implements RuntimeOrchestrator { @@ -41,6 +43,20 @@ public class DockerRuntimeOrchestrator implements RuntimeOrchestrator { * `noexec` would block dlopen via mmap(PROT_EXEC) — keep it off. */ private static final String TMPFS_TMP_OPTS = "rw,nosuid,size=256m"; + /** Mount path for the per-replica JAR volume. Loader writes here RW; main + * container mounts the same volume RO and reads /app/jars/app.jar. */ + private static final String LOADER_VOLUME_MOUNT = "/app/jars"; + + /** User-namespace remap applied to BOTH loader and main containers + * (issue #152). `host:1000:65536` maps the in-container UID range starting + * at 1000 onto the host so root inside the container is never UID 0 on the + * host. */ + private static final String USERNS_MODE = "host:1000:65536"; + + /** How long the orchestrator will wait for the loader container to fetch + * the JAR before giving up. */ + private static final long LOADER_WAIT_TIMEOUT_SECONDS = 120; + private final DockerClient dockerClient; private final String dockerRuntime; @@ -112,6 +128,62 @@ public class DockerRuntimeOrchestrator implements RuntimeOrchestrator { @Override public String startContainer(ContainerRequest request) { + // Per-replica named volume — shared between loader (RW) and main (RO). + // Naming convention pins it to the replica's container name so volume + // cleanup in removeContainer() can derive it deterministically. + String volumeName = "cameleer-jars-" + request.containerName(); + dockerClient.createVolumeCmd().withName(volumeName).exec(); + + // Phase 1: Loader container fetches the JAR from the signed URL into + // the shared volume. Hardened identically to the main container, plus + // RW bind on /app/jars and the artifact env vars the loader entrypoint + // expects. We block on its exit code before bringing the main up. + String loaderId = createAndStartLoader(request, volumeName); + int exitCode; + try { + exitCode = dockerClient.waitContainerCmd(loaderId) + .exec(new WaitContainerResultCallback()) + .awaitStatusCode(LOADER_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } catch (Exception e) { + cleanup(loaderId, volumeName); + throw new RuntimeException("Loader wait failed for " + request.containerName(), e); + } finally { + try { + dockerClient.removeContainerCmd(loaderId).withForce(true).exec(); + } catch (Exception e) { + log.warn("Failed to remove loader {}: {}", loaderId, e.getMessage()); + } + } + if (exitCode != 0) { + cleanupVolume(volumeName); + throw new RuntimeException("Loader exited " + exitCode + " for " + request.containerName()); + } + + // Phase 2: Main container — RO on the shared volume. + return createAndStartMain(request, volumeName); + } + + private String createAndStartLoader(ContainerRequest request, String volumeName) { + HostConfig hc = baseHardenedHostConfig() + .withBinds(new Bind(volumeName, new Volume(LOADER_VOLUME_MOUNT), AccessMode.rw)) + .withUsernsMode(USERNS_MODE) + .withNetworkMode(request.network()); + if (!dockerRuntime.isBlank()) { + hc.withRuntime(dockerRuntime); + } + + var create = dockerClient.createContainerCmd(request.loaderImage()) + .withName(request.containerName() + "-loader") + .withEnv(List.of( + "ARTIFACT_URL=" + request.artifactDownloadUrl(), + "ARTIFACT_EXPECTED_SIZE=" + request.artifactExpectedSize())) + .withHostConfig(hc); + String id = create.exec().getId(); + dockerClient.startContainerCmd(id).exec(); + return id; + } + + private String createAndStartMain(ContainerRequest request, String volumeName) { List envList = request.envVars().entrySet().stream() .map(e -> e.getKey() + "=" + e.getValue()).toList(); @@ -128,63 +200,26 @@ public class DockerRuntimeOrchestrator implements RuntimeOrchestrator { // - readonly rootfs + /tmp tmpfs: persistence-via-write defeated; apps // needing durable state declare writeableVolumes (issue #153). // - pids-limit: fork bombs cannot exhaust the host PID namespace. - HostConfig hostConfig = HostConfig.newHostConfig() + // - userns_mode host:1000:65536: container root is never UID 0 on the host. + HostConfig hc = baseHardenedHostConfig() .withMemory(request.memoryLimitBytes()) .withMemorySwap(request.memoryLimitBytes()) .withCpuShares(request.cpuShares()) .withNetworkMode(request.network()) .withRestartPolicy(RestartPolicy.onFailureRestart(request.restartPolicyMaxRetries())) - .withCapDrop(Capability.values()) - .withSecurityOpts(List.of( - "no-new-privileges:true", - "apparmor=docker-default")) - .withReadonlyRootfs(true) - .withPidsLimit(PIDS_LIMIT) - .withTmpFs(Map.of("/tmp", TMPFS_TMP_OPTS)); - + .withUsernsMode(USERNS_MODE) + .withBinds(new Bind(volumeName, new Volume(LOADER_VOLUME_MOUNT), AccessMode.ro)); if (!dockerRuntime.isBlank()) { - hostConfig.withRuntime(dockerRuntime); + hc.withRuntime(dockerRuntime); } - - // JAR mounting: volume mount (Docker-in-Docker) or bind mount (host path) - if (request.jarVolumeName() != null && !request.jarVolumeName().isBlank()) { - // Mount the named volume at the jar storage base path - Bind volumeBind = new Bind(request.jarVolumeName(), new Volume(request.jarVolumeMountPath()), AccessMode.ro); - hostConfig.withBinds(volumeBind); - } else { - Bind jarBind = new Bind(request.jarPath(), new Volume("/app/app.jar"), AccessMode.ro); - hostConfig.withBinds(jarBind); - } - if (request.memoryReserveBytes() != null) { - hostConfig.withMemoryReservation(request.memoryReserveBytes()); + hc.withMemoryReservation(request.memoryReserveBytes()); } if (request.cpuQuota() != null) { - hostConfig.withCpuQuota(request.cpuQuota()); + hc.withCpuQuota(request.cpuQuota()); } - // Resolve the JAR path for the entrypoint - String appJarPath; - if (request.jarVolumeName() != null && !request.jarVolumeName().isBlank()) { - appJarPath = request.jarPath(); - } else { - appJarPath = "/app/app.jar"; - } - - var createCmd = dockerClient.createContainerCmd(request.baseImage()) - .withName(request.containerName()) - .withEnv(envList) - .withLabels(request.labels() != null ? request.labels() : Map.of()) - .withHostConfig(hostConfig) - .withHealthcheck(new HealthCheck() - .withTest(List.of("CMD-SHELL", - "wget -qO- http://localhost:" + request.healthCheckPort() + "/cameleer/health || exit 1")) - .withInterval(10_000_000_000L) - .withTimeout(5_000_000_000L) - .withRetries(3) - .withStartPeriod(30_000_000_000L)); - - // Build entrypoint based on runtime type + String appJarPath = LOADER_VOLUME_MOUNT + "/app.jar"; String customArgs = request.customArgs() != null && !request.customArgs().isBlank() ? " " + request.customArgs() : ""; String entrypoint = switch (request.runtimeType()) { @@ -194,20 +229,55 @@ public class DockerRuntimeOrchestrator implements RuntimeOrchestrator { default -> // spring-boot, quarkus, and others all use -jar "exec java -javaagent:/app/agent.jar" + customArgs + " -jar " + appJarPath; }; - createCmd.withEntrypoint("sh", "-c", entrypoint); + + var createCmd = dockerClient.createContainerCmd(request.baseImage()) + .withName(request.containerName()) + .withEnv(envList) + .withLabels(request.labels() != null ? request.labels() : Map.of()) + .withHostConfig(hc) + .withHealthcheck(new HealthCheck() + .withTest(List.of("CMD-SHELL", + "wget -qO- http://localhost:" + request.healthCheckPort() + "/cameleer/health || exit 1")) + .withInterval(10_000_000_000L) + .withTimeout(5_000_000_000L) + .withRetries(3) + .withStartPeriod(30_000_000_000L)) + .withEntrypoint("sh", "-c", entrypoint); if (request.exposedPorts() != null && !request.exposedPorts().isEmpty()) { var ports = request.exposedPorts().stream() - .map(p -> com.github.dockerjava.api.model.ExposedPort.tcp(p)) + .map(com.github.dockerjava.api.model.ExposedPort::tcp) .toArray(com.github.dockerjava.api.model.ExposedPort[]::new); createCmd.withExposedPorts(ports); } - var container = createCmd.exec(); - dockerClient.startContainerCmd(container.getId()).exec(); + String id = createCmd.exec().getId(); + dockerClient.startContainerCmd(id).exec(); + log.info("Started container {} ({})", request.containerName(), id); + return id; + } - log.info("Started container {} ({})", request.containerName(), container.getId()); - return container.getId(); + /** Hardening contract from issue #152 — applied uniformly to loader + main. */ + private HostConfig baseHardenedHostConfig() { + return HostConfig.newHostConfig() + .withCapDrop(Capability.values()) + .withSecurityOpts(List.of("no-new-privileges:true", "apparmor=docker-default")) + .withReadonlyRootfs(true) + .withPidsLimit(PIDS_LIMIT) + .withTmpFs(Map.of("/tmp", TMPFS_TMP_OPTS)); + } + + private void cleanup(String loaderId, String volumeName) { + try { + dockerClient.removeContainerCmd(loaderId).withForce(true).exec(); + } catch (Exception ignored) { /* best effort */ } + cleanupVolume(volumeName); + } + + private void cleanupVolume(String volumeName) { + try { + dockerClient.removeVolumeCmd(volumeName).exec(); + } catch (Exception ignored) { /* best effort */ } } public DockerClient getDockerClient() { @@ -226,12 +296,29 @@ public class DockerRuntimeOrchestrator implements RuntimeOrchestrator { @Override public void removeContainer(String containerId) { + // Look up the container name first so we can derive the per-replica + // volume name. Do this before removing the container — afterward the + // inspect would 404. Volume removal happens after container removal so + // no in-flight bind keeps the volume busy. + String volumeName = null; + try { + String name = dockerClient.inspectContainerCmd(containerId).exec().getName(); + if (name != null) { + // Docker prefixes inspected names with '/'. + volumeName = "cameleer-jars-" + name.replaceFirst("^/", ""); + } + } catch (Exception e) { + log.warn("Could not inspect {} for volume name: {}", containerId, e.getMessage()); + } try { dockerClient.removeContainerCmd(containerId).withForce(true).exec(); log.info("Removed container {}", containerId); } catch (Exception e) { log.warn("Failed to remove container {}: {}", containerId, e.getMessage()); } + if (volumeName != null) { + cleanupVolume(volumeName); + } } @Override diff --git a/cameleer-server-app/src/main/resources/application.yml b/cameleer-server-app/src/main/resources/application.yml index a7e29444..91d72085 100644 --- a/cameleer-server-app/src/main/resources/application.yml +++ b/cameleer-server-app/src/main/resources/application.yml @@ -61,7 +61,13 @@ cameleer: routingdomain: ${CAMELEER_SERVER_RUNTIME_ROUTINGDOMAIN:localhost} serverurl: ${CAMELEER_SERVER_RUNTIME_SERVERURL:} certresolver: ${CAMELEER_SERVER_RUNTIME_CERTRESOLVER:} - jardockervolume: ${CAMELEER_SERVER_RUNTIME_JARDOCKERVOLUME:} + # Init-container loader for tenant JAR fetch. The loader runs as a + # short-lived sidecar that downloads the JAR from a signed URL into a + # per-replica named volume, which the main container then mounts RO at + # /app/jars. See issue #152 close-out + .claude/rules/docker-orchestration.md. + loaderimage: ${CAMELEER_SERVER_RUNTIME_LOADERIMAGE:gitea.siegeln.net/cameleer/cameleer-runtime-loader:latest} + artifacttokenttlseconds: ${CAMELEER_SERVER_RUNTIME_ARTIFACTTOKENTTLSECONDS:600} + artifactbaseurl: ${CAMELEER_SERVER_RUNTIME_ARTIFACTBASEURL:} indexer: debouncems: ${CAMELEER_SERVER_INDEXER_DEBOUNCEMS:2000} queuesize: ${CAMELEER_SERVER_INDEXER_QUEUESIZE:10000} diff --git a/cameleer-server-app/src/test/java/com/cameleer/server/app/runtime/DockerRuntimeOrchestratorHardeningTest.java b/cameleer-server-app/src/test/java/com/cameleer/server/app/runtime/DockerRuntimeOrchestratorHardeningTest.java index b865a6dd..72a1c165 100644 --- a/cameleer-server-app/src/test/java/com/cameleer/server/app/runtime/DockerRuntimeOrchestratorHardeningTest.java +++ b/cameleer-server-app/src/test/java/com/cameleer/server/app/runtime/DockerRuntimeOrchestratorHardeningTest.java @@ -4,11 +4,15 @@ import com.cameleer.server.core.runtime.ContainerRequest; import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.CreateContainerCmd; import com.github.dockerjava.api.command.CreateContainerResponse; +import com.github.dockerjava.api.command.CreateVolumeCmd; import com.github.dockerjava.api.command.InfoCmd; +import com.github.dockerjava.api.command.RemoveContainerCmd; import com.github.dockerjava.api.command.StartContainerCmd; +import com.github.dockerjava.api.command.WaitContainerCmd; import com.github.dockerjava.api.model.Capability; import com.github.dockerjava.api.model.HostConfig; import com.github.dockerjava.api.model.Info; +import com.github.dockerjava.core.command.WaitContainerResultCallback; import org.junit.jupiter.api.Test; import org.mockito.Answers; import org.mockito.ArgumentCaptor; @@ -16,9 +20,13 @@ import org.mockito.ArgumentCaptor; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -27,15 +35,25 @@ import static org.mockito.Mockito.when; * container is launched with cap_drop ALL, no-new-privileges, AppArmor profile, * read-only rootfs, a pids limit, and a writeable /tmp tmpfs. Also verifies the * runsc auto-detect via `docker info` and the explicit override. + * + *

Since the init-container loader pattern landed, {@link DockerRuntimeOrchestrator#startContainer} + * creates a per-replica volume + a loader container before the main container + * launches. The hardening assertions in this test target the **main** container's + * HostConfig — that's the one that runs untrusted tenant code. */ class DockerRuntimeOrchestratorHardeningTest { + private static final String LOADER_IMAGE = "gitea.siegeln.net/cameleer/cameleer-runtime-loader:latest"; + private static final String BASE_IMAGE = "registry.example/runtime:latest"; + private static ContainerRequest sampleRequest() { return new ContainerRequest( "tenant-app-0-abcd1234", - "registry.example/runtime:latest", - "/data/jars/app.jar", - null, null, + BASE_IMAGE, + UUID.randomUUID(), + "https://srv/api/v1/artifacts/uuid?exp=1&sig=x", + 12345L, + LOADER_IMAGE, "tenant-net", List.of(), Map.of("CAMELEER_AGENT_APPLICATION", "myapp"), @@ -63,6 +81,36 @@ class DockerRuntimeOrchestratorHardeningTest { return dockerClient; } + /** + * Wires the loader path to a successful exit (0) and returns the {@link CreateContainerCmd} + * for the *main* container so callers can capture its HostConfig. + */ + private static CreateContainerCmd wireLoaderAndMain(DockerClient dockerClient) { + CreateContainerCmd loaderCreate = mock(CreateContainerCmd.class, Answers.RETURNS_SELF); + CreateContainerCmd mainCreate = mock(CreateContainerCmd.class, Answers.RETURNS_SELF); + when(dockerClient.createContainerCmd(eq(LOADER_IMAGE))).thenReturn(loaderCreate); + when(dockerClient.createContainerCmd(eq(BASE_IMAGE))).thenReturn(mainCreate); + + CreateContainerResponse loaderResp = mock(CreateContainerResponse.class); + when(loaderResp.getId()).thenReturn("loader-id"); + when(loaderCreate.exec()).thenReturn(loaderResp); + CreateContainerResponse mainResp = mock(CreateContainerResponse.class); + when(mainResp.getId()).thenReturn("main-id"); + when(mainCreate.exec()).thenReturn(mainResp); + + when(dockerClient.startContainerCmd(anyString())).thenReturn(mock(StartContainerCmd.class)); + when(dockerClient.removeContainerCmd(anyString())) + .thenReturn(mock(RemoveContainerCmd.class, Answers.RETURNS_SELF)); + when(dockerClient.createVolumeCmd()).thenReturn(mock(CreateVolumeCmd.class, Answers.RETURNS_SELF)); + + WaitContainerCmd waitCmd = mock(WaitContainerCmd.class); + when(dockerClient.waitContainerCmd("loader-id")).thenReturn(waitCmd); + WaitContainerResultCallback waitCallback = mock(WaitContainerResultCallback.class); + when(waitCmd.exec(any())).thenReturn(waitCallback); + when(waitCallback.awaitStatusCode(anyLong(), any())).thenReturn(0); + return mainCreate; + } + @Test void resolveRuntime_picksRunscWhenDaemonHasIt() { DockerClient dockerClient = mockDockerClientWithRuntimes(Map.of( @@ -116,22 +164,15 @@ class DockerRuntimeOrchestratorHardeningTest { } @Test - void startContainer_appliesHardeningContractToHostConfig() { + void startContainer_appliesHardeningContractToMainContainer() { DockerClient dockerClient = mockDockerClientWithRuntimes(new HashMap<>()); - - CreateContainerCmd createCmd = mock(CreateContainerCmd.class, Answers.RETURNS_SELF); - when(dockerClient.createContainerCmd(anyString())).thenReturn(createCmd); - CreateContainerResponse createResponse = mock(CreateContainerResponse.class); - when(createResponse.getId()).thenReturn("container-id-1"); - when(createCmd.exec()).thenReturn(createResponse); - StartContainerCmd startCmd = mock(StartContainerCmd.class); - when(dockerClient.startContainerCmd(anyString())).thenReturn(startCmd); + CreateContainerCmd mainCreate = wireLoaderAndMain(dockerClient); DockerRuntimeOrchestrator orchestrator = new DockerRuntimeOrchestrator(dockerClient, ""); orchestrator.startContainer(sampleRequest()); ArgumentCaptor hostCaptor = ArgumentCaptor.forClass(HostConfig.class); - org.mockito.Mockito.verify(createCmd).withHostConfig(hostCaptor.capture()); + org.mockito.Mockito.verify(mainCreate).withHostConfig(hostCaptor.capture()); HostConfig hc = hostCaptor.getValue(); // cap_drop ALL — every capability the SDK knows about @@ -161,23 +202,23 @@ class DockerRuntimeOrchestratorHardeningTest { .containsKey("/tmp"); String tmpOpts = hc.getTmpFs().get("/tmp"); assertThat(tmpOpts).contains("rw").contains("nosuid").doesNotContain("noexec"); + + // userns remap applied uniformly to the main container — issue #152 + assertThat(hc.getUsernsMode()) + .as("withUsernsMode pins container UIDs onto a non-root host range") + .isEqualTo("host:1000:65536"); } @Test void startContainer_doesNotForceRuntimeWhenAutoDetectFindsNothing() { DockerClient dockerClient = mockDockerClientWithRuntimes(Map.of("runc", new Object())); - CreateContainerCmd createCmd = mock(CreateContainerCmd.class, Answers.RETURNS_SELF); - when(dockerClient.createContainerCmd(anyString())).thenReturn(createCmd); - CreateContainerResponse createResponse = mock(CreateContainerResponse.class); - when(createResponse.getId()).thenReturn("c"); - when(createCmd.exec()).thenReturn(createResponse); - when(dockerClient.startContainerCmd(anyString())).thenReturn(mock(StartContainerCmd.class)); + CreateContainerCmd mainCreate = wireLoaderAndMain(dockerClient); DockerRuntimeOrchestrator orchestrator = new DockerRuntimeOrchestrator(dockerClient, ""); orchestrator.startContainer(sampleRequest()); ArgumentCaptor hostCaptor = ArgumentCaptor.forClass(HostConfig.class); - org.mockito.Mockito.verify(createCmd).withHostConfig(hostCaptor.capture()); + org.mockito.Mockito.verify(mainCreate).withHostConfig(hostCaptor.capture()); // When daemon has no sandboxed runtime, we leave runtime null/empty so Docker picks its default. String runtime = hostCaptor.getValue().getRuntime(); @@ -189,18 +230,13 @@ class DockerRuntimeOrchestratorHardeningTest { @Test void startContainer_appliesRunscWhenAvailable() { DockerClient dockerClient = mockDockerClientWithRuntimes(Map.of("runsc", new Object())); - CreateContainerCmd createCmd = mock(CreateContainerCmd.class, Answers.RETURNS_SELF); - when(dockerClient.createContainerCmd(anyString())).thenReturn(createCmd); - CreateContainerResponse createResponse = mock(CreateContainerResponse.class); - when(createResponse.getId()).thenReturn("c"); - when(createCmd.exec()).thenReturn(createResponse); - when(dockerClient.startContainerCmd(anyString())).thenReturn(mock(StartContainerCmd.class)); + CreateContainerCmd mainCreate = wireLoaderAndMain(dockerClient); DockerRuntimeOrchestrator orchestrator = new DockerRuntimeOrchestrator(dockerClient, ""); orchestrator.startContainer(sampleRequest()); ArgumentCaptor hostCaptor = ArgumentCaptor.forClass(HostConfig.class); - org.mockito.Mockito.verify(createCmd).withHostConfig(hostCaptor.capture()); + org.mockito.Mockito.verify(mainCreate).withHostConfig(hostCaptor.capture()); assertThat(hostCaptor.getValue().getRuntime()).isEqualTo("runsc"); } diff --git a/cameleer-server-app/src/test/java/com/cameleer/server/app/runtime/DockerRuntimeOrchestratorLoaderTest.java b/cameleer-server-app/src/test/java/com/cameleer/server/app/runtime/DockerRuntimeOrchestratorLoaderTest.java new file mode 100644 index 00000000..480fe8f8 --- /dev/null +++ b/cameleer-server-app/src/test/java/com/cameleer/server/app/runtime/DockerRuntimeOrchestratorLoaderTest.java @@ -0,0 +1,204 @@ +package com.cameleer.server.app.runtime; + +import com.cameleer.server.core.runtime.ContainerRequest; +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.CreateContainerCmd; +import com.github.dockerjava.api.command.CreateContainerResponse; +import com.github.dockerjava.api.command.CreateVolumeCmd; +import com.github.dockerjava.api.command.InfoCmd; +import com.github.dockerjava.api.command.RemoveContainerCmd; +import com.github.dockerjava.api.command.RemoveVolumeCmd; +import com.github.dockerjava.api.command.StartContainerCmd; +import com.github.dockerjava.api.command.WaitContainerCmd; +import com.github.dockerjava.api.model.AccessMode; +import com.github.dockerjava.api.model.HostConfig; +import com.github.dockerjava.api.model.Info; +import com.github.dockerjava.core.command.WaitContainerResultCallback; +import org.junit.jupiter.api.Test; +import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Behaviour tests for the loader-init-container pattern (issue #152 close-out). + * The orchestrator must: + *

    + *
  1. create a per-replica named volume,
    2. + *
  2. start the loader container with the volume mounted RW at /app/jars,
    3. + *
  3. wait for the loader to exit 0 before bringing up the main container,
    4. + *
  4. remove the loader (always) and remove the volume + abort if the loader fails,
    5. + *
  5. apply {@code withUsernsMode("host:1000:65536")} to BOTH containers.
    6. + *
+ */ +class DockerRuntimeOrchestratorLoaderTest { + + private static final String LOADER_IMAGE = "gitea.siegeln.net/cameleer/cameleer-runtime-loader:latest"; + private static final String BASE_IMAGE = "base-image:latest"; + private static final String CONTAINER_NAME = "tenant-app-0-abcd1234"; + private static final String EXPECTED_VOLUME = "cameleer-jars-" + CONTAINER_NAME; + + private static ContainerRequest sampleRequest() { + return new ContainerRequest( + CONTAINER_NAME, + BASE_IMAGE, + UUID.randomUUID(), + "https://srv/api/v1/artifacts/uuid?exp=1&sig=x", + 12345L, + LOADER_IMAGE, + "tenant-net", + List.of(), + Map.of("CAMELEER_AGENT_APPLICATION", "myapp"), + Map.of(), + 512L * 1024 * 1024, + null, + 512, + null, + List.of(8080), + 9464, + "on-failure", + 3, + "spring-boot", + "", + null); + } + + private static DockerClient mockDockerClientWithRuntimes(Map runtimes) { + DockerClient dockerClient = mock(DockerClient.class); + InfoCmd infoCmd = mock(InfoCmd.class); + Info info = mock(Info.class); + when(dockerClient.infoCmd()).thenReturn(infoCmd); + when(infoCmd.exec()).thenReturn(info); + when(info.getRuntimes()).thenReturn((Map) runtimes); + return dockerClient; + } + + /** Wires loader path + main path. Returns the loader create cmd, the main + * create cmd, and the wait-callback so each test can poke them directly. */ + private static record Wired(CreateContainerCmd loaderCreate, CreateContainerCmd mainCreate, + WaitContainerResultCallback waitCallback, + RemoveVolumeCmd removeVolumeCmd) {} + + private static Wired wire(DockerClient dockerClient, int loaderExitCode) { + CreateContainerCmd loaderCreate = mock(CreateContainerCmd.class, Answers.RETURNS_SELF); + CreateContainerCmd mainCreate = mock(CreateContainerCmd.class, Answers.RETURNS_SELF); + when(dockerClient.createContainerCmd(eq(LOADER_IMAGE))).thenReturn(loaderCreate); + when(dockerClient.createContainerCmd(eq(BASE_IMAGE))).thenReturn(mainCreate); + + CreateContainerResponse loaderResp = mock(CreateContainerResponse.class); + when(loaderResp.getId()).thenReturn("loader-id"); + when(loaderCreate.exec()).thenReturn(loaderResp); + CreateContainerResponse mainResp = mock(CreateContainerResponse.class); + when(mainResp.getId()).thenReturn("main-id"); + when(mainCreate.exec()).thenReturn(mainResp); + + when(dockerClient.startContainerCmd(anyString())).thenReturn(mock(StartContainerCmd.class)); + when(dockerClient.removeContainerCmd(anyString())) + .thenReturn(mock(RemoveContainerCmd.class, Answers.RETURNS_SELF)); + + when(dockerClient.createVolumeCmd()).thenReturn(mock(CreateVolumeCmd.class, Answers.RETURNS_SELF)); + RemoveVolumeCmd removeVolumeCmd = mock(RemoveVolumeCmd.class); + when(dockerClient.removeVolumeCmd(anyString())).thenReturn(removeVolumeCmd); + + WaitContainerCmd waitCmd = mock(WaitContainerCmd.class); + when(dockerClient.waitContainerCmd("loader-id")).thenReturn(waitCmd); + WaitContainerResultCallback waitCallback = mock(WaitContainerResultCallback.class); + when(waitCmd.exec(any())).thenReturn(waitCallback); + when(waitCallback.awaitStatusCode(anyLong(), any())).thenReturn(loaderExitCode); + + return new Wired(loaderCreate, mainCreate, waitCallback, removeVolumeCmd); + } + + @Test + void startContainerCreatesVolumeRunsLoaderThenStartsMain() { + DockerClient docker = mockDockerClientWithRuntimes(Map.of()); + Wired w = wire(docker, 0); + + var orchestrator = new DockerRuntimeOrchestrator(docker, ""); + String returnedId = orchestrator.startContainer(sampleRequest()); + + assertThat(returnedId).isEqualTo("main-id"); + + // Loader's HostConfig binds the named volume RW at /app/jars + ArgumentCaptor loaderHc = ArgumentCaptor.forClass(HostConfig.class); + verify(w.loaderCreate()).withHostConfig(loaderHc.capture()); + assertThat(loaderHc.getValue().getBinds()) + .as("loader must mount the per-replica volume RW at /app/jars") + .anyMatch(b -> b.getVolume().getPath().equals("/app/jars") + && b.getAccessMode() == AccessMode.rw + && b.getPath().equals(EXPECTED_VOLUME)); + + // Main's HostConfig binds the same volume RO at /app/jars + ArgumentCaptor mainHc = ArgumentCaptor.forClass(HostConfig.class); + verify(w.mainCreate()).withHostConfig(mainHc.capture()); + assertThat(mainHc.getValue().getBinds()) + .as("main must mount the per-replica volume RO at /app/jars") + .anyMatch(b -> b.getVolume().getPath().equals("/app/jars") + && b.getAccessMode() == AccessMode.ro + && b.getPath().equals(EXPECTED_VOLUME)); + + // Order: loader started, awaited, removed; then main started. + InOrder order = inOrder(docker); + order.verify(docker).createVolumeCmd(); + order.verify(docker).createContainerCmd(LOADER_IMAGE); + order.verify(docker).startContainerCmd("loader-id"); + order.verify(docker).waitContainerCmd("loader-id"); + order.verify(docker).removeContainerCmd("loader-id"); + order.verify(docker).createContainerCmd(BASE_IMAGE); + order.verify(docker).startContainerCmd("main-id"); + } + + @Test + void startContainerAbortsAndCleansUpWhenLoaderFails() { + DockerClient docker = mockDockerClientWithRuntimes(Map.of()); + Wired w = wire(docker, 2); + + var orchestrator = new DockerRuntimeOrchestrator(docker, ""); + + assertThatThrownBy(() -> orchestrator.startContainer(sampleRequest())) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("Loader exited 2"); + + // Loader was removed, volume was removed, main container never created. + verify(docker).removeContainerCmd("loader-id"); + verify(docker).removeVolumeCmd(EXPECTED_VOLUME); + verify(w.removeVolumeCmd()).exec(); + verify(docker, never()).createContainerCmd(BASE_IMAGE); + } + + @Test + void startContainerAppliesUsernsModeToBothContainers() { + DockerClient docker = mockDockerClientWithRuntimes(Map.of()); + Wired w = wire(docker, 0); + + var orchestrator = new DockerRuntimeOrchestrator(docker, ""); + orchestrator.startContainer(sampleRequest()); + + ArgumentCaptor loaderHc = ArgumentCaptor.forClass(HostConfig.class); + verify(w.loaderCreate()).withHostConfig(loaderHc.capture()); + assertThat(loaderHc.getValue().getUsernsMode()) + .as("loader must run with the userns remap") + .isEqualTo("host:1000:65536"); + + ArgumentCaptor mainHc = ArgumentCaptor.forClass(HostConfig.class); + verify(w.mainCreate()).withHostConfig(mainHc.capture()); + assertThat(mainHc.getValue().getUsernsMode()) + .as("main must run with the userns remap") + .isEqualTo("host:1000:65536"); + } +} diff --git a/cameleer-server-core/src/main/java/com/cameleer/server/core/runtime/ContainerRequest.java b/cameleer-server-core/src/main/java/com/cameleer/server/core/runtime/ContainerRequest.java index 8aa8b715..56f32c3b 100644 --- a/cameleer-server-core/src/main/java/com/cameleer/server/core/runtime/ContainerRequest.java +++ b/cameleer-server-core/src/main/java/com/cameleer/server/core/runtime/ContainerRequest.java @@ -2,13 +2,15 @@ package com.cameleer.server.core.runtime; import java.util.List; import java.util.Map; +import java.util.UUID; public record ContainerRequest( String containerName, String baseImage, - String jarPath, - String jarVolumeName, - String jarVolumeMountPath, + UUID appVersionId, + String artifactDownloadUrl, + long artifactExpectedSize, + String loaderImage, String network, List additionalNetworks, Map envVars,