From e9f523f2b820c0ca6c9c1342bd11f9ea3650621a Mon Sep 17 00:00:00 2001 From: hsiegeln <37154749+hsiegeln@users.noreply.github.com> Date: Thu, 23 Apr 2026 10:00:00 +0200 Subject: [PATCH] test(deploy): blue-green + rolling strategy ITs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four ITs covering strategy behavior: - BlueGreenStrategyIT#blueGreen_allHealthy_stopsOldAfterNew: old is stopped only after all new replicas are healthy. - BlueGreenStrategyIT#blueGreen_partialHealthy_preservesOldAndMarksFailed: strict all-healthy — one starting replica aborts the deploy and leaves the previous deployment RUNNING untouched. - RollingStrategyIT#rolling_allHealthy_replacesOneByOne: InOrder on stopContainer confirms old-0 stops before old-1 (the interleaving that distinguishes rolling from blue-green). - RollingStrategyIT#rolling_failsMidRollout_preservesRemainingOld: mid-rollout health failure stops only the in-flight new containers and the already-replaced old-0; old-1 stays untouched. Shortens healthchecktimeout to 2s via @TestPropertySource so failure paths complete in ~25s instead of ~60s. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../app/runtime/BlueGreenStrategyIT.java | 190 +++++++++++++++++ .../server/app/runtime/RollingStrategyIT.java | 194 ++++++++++++++++++ 2 files changed, 384 insertions(+) create mode 100644 cameleer-server-app/src/test/java/com/cameleer/server/app/runtime/BlueGreenStrategyIT.java create mode 100644 cameleer-server-app/src/test/java/com/cameleer/server/app/runtime/RollingStrategyIT.java diff --git a/cameleer-server-app/src/test/java/com/cameleer/server/app/runtime/BlueGreenStrategyIT.java b/cameleer-server-app/src/test/java/com/cameleer/server/app/runtime/BlueGreenStrategyIT.java new file mode 100644 index 00000000..26605a42 --- /dev/null +++ b/cameleer-server-app/src/test/java/com/cameleer/server/app/runtime/BlueGreenStrategyIT.java @@ -0,0 +1,190 @@ +package com.cameleer.server.app.runtime; + +import com.cameleer.server.app.AbstractPostgresIT; +import com.cameleer.server.app.TestSecurityHelper; +import com.cameleer.server.app.storage.PostgresDeploymentRepository; +import com.cameleer.server.core.runtime.ContainerStatus; +import com.cameleer.server.core.runtime.Deployment; +import com.cameleer.server.core.runtime.DeploymentStatus; +import com.cameleer.server.core.runtime.RuntimeOrchestrator; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.web.client.TestRestTemplate; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.test.context.TestPropertySource; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Verifies the blue-green deployment strategy: start all new → health-check + * all → stop old. Strict all-healthy — partial failure preserves the previous + * deployment untouched. + */ +@TestPropertySource(properties = "cameleer.server.runtime.healthchecktimeout=2") +class BlueGreenStrategyIT extends AbstractPostgresIT { + + @MockBean + RuntimeOrchestrator runtimeOrchestrator; + + @Autowired private TestRestTemplate restTemplate; + @Autowired private ObjectMapper objectMapper; + @Autowired private TestSecurityHelper securityHelper; + @Autowired private PostgresDeploymentRepository deploymentRepository; + + private String operatorJwt; + private String appSlug; + private String versionId; + + @BeforeEach + void setUp() throws Exception { + operatorJwt = securityHelper.operatorToken(); + + jdbcTemplate.update("DELETE FROM deployments"); + jdbcTemplate.update("DELETE FROM app_versions"); + jdbcTemplate.update("DELETE FROM apps"); + jdbcTemplate.update("DELETE FROM application_config WHERE environment = 'default'"); + + when(runtimeOrchestrator.isEnabled()).thenReturn(true); + + appSlug = "bg-" + UUID.randomUUID().toString().substring(0, 8); + post("/api/v1/environments/default/apps", String.format(""" + {"slug": "%s", "displayName": "BG App"} + """, appSlug), operatorJwt); + put("/api/v1/environments/default/apps/" + appSlug + "/container-config", """ + {"runtimeType": "spring-boot", "appPort": 8081, "replicas": 2, "deploymentStrategy": "blue-green"} + """, operatorJwt); + versionId = uploadJar(appSlug, ("bg-jar-" + appSlug).getBytes()); + } + + @Test + void blueGreen_allHealthy_stopsOldAfterNew() throws Exception { + when(runtimeOrchestrator.startContainer(any())) + .thenReturn("old-0", "old-1", "new-0", "new-1"); + ContainerStatus healthy = new ContainerStatus("healthy", true, 0, null); + when(runtimeOrchestrator.getContainerStatus("old-0")).thenReturn(healthy); + when(runtimeOrchestrator.getContainerStatus("old-1")).thenReturn(healthy); + when(runtimeOrchestrator.getContainerStatus("new-0")).thenReturn(healthy); + when(runtimeOrchestrator.getContainerStatus("new-1")).thenReturn(healthy); + + String firstDeployId = triggerDeploy(); + awaitStatus(firstDeployId, DeploymentStatus.RUNNING); + + String secondDeployId = triggerDeploy(); + awaitStatus(secondDeployId, DeploymentStatus.RUNNING); + + // Previous deployment was stopped once new was healthy + Deployment first = deploymentRepository.findById(UUID.fromString(firstDeployId)).orElseThrow(); + assertThat(first.status()).isEqualTo(DeploymentStatus.STOPPED); + + verify(runtimeOrchestrator).stopContainer("old-0"); + verify(runtimeOrchestrator).stopContainer("old-1"); + verify(runtimeOrchestrator, never()).stopContainer("new-0"); + verify(runtimeOrchestrator, never()).stopContainer("new-1"); + + // New deployment has both new replicas recorded + Deployment second = deploymentRepository.findById(UUID.fromString(secondDeployId)).orElseThrow(); + assertThat(second.replicaStates()).hasSize(2); + } + + @Test + void blueGreen_partialHealthy_preservesOldAndMarksFailed() throws Exception { + when(runtimeOrchestrator.startContainer(any())) + .thenReturn("old-0", "old-1", "new-0", "new-1"); + ContainerStatus healthy = new ContainerStatus("healthy", true, 0, null); + ContainerStatus starting = new ContainerStatus("starting", true, 0, null); + when(runtimeOrchestrator.getContainerStatus("old-0")).thenReturn(healthy); + when(runtimeOrchestrator.getContainerStatus("old-1")).thenReturn(healthy); + when(runtimeOrchestrator.getContainerStatus("new-0")).thenReturn(healthy); + when(runtimeOrchestrator.getContainerStatus("new-1")).thenReturn(starting); + + String firstDeployId = triggerDeploy(); + awaitStatus(firstDeployId, DeploymentStatus.RUNNING); + + String secondDeployId = triggerDeploy(); + awaitStatus(secondDeployId, DeploymentStatus.FAILED); + + Deployment second = deploymentRepository.findById(UUID.fromString(secondDeployId)).orElseThrow(); + assertThat(second.errorMessage()) + .contains("blue-green") + .contains("1/2"); + + // Previous deployment stays RUNNING — blue-green's safety promise. + Deployment first = deploymentRepository.findById(UUID.fromString(firstDeployId)).orElseThrow(); + assertThat(first.status()).isEqualTo(DeploymentStatus.RUNNING); + + verify(runtimeOrchestrator, never()).stopContainer("old-0"); + verify(runtimeOrchestrator, never()).stopContainer("old-1"); + // Cleanup ran on both new replicas. + verify(runtimeOrchestrator).stopContainer("new-0"); + verify(runtimeOrchestrator).stopContainer("new-1"); + } + + // ---- helpers ---- + + private String triggerDeploy() throws Exception { + JsonNode deployResponse = post( + "/api/v1/environments/default/apps/" + appSlug + "/deployments", + String.format("{\"appVersionId\": \"%s\"}", versionId), operatorJwt); + return deployResponse.path("id").asText(); + } + + private void awaitStatus(String deployId, DeploymentStatus expected) { + await().atMost(30, TimeUnit.SECONDS) + .pollInterval(500, TimeUnit.MILLISECONDS) + .untilAsserted(() -> { + Deployment d = deploymentRepository.findById(UUID.fromString(deployId)) + .orElseThrow(() -> new AssertionError("Deployment not found: " + deployId)); + assertThat(d.status()).isEqualTo(expected); + }); + } + + private JsonNode post(String path, String json, String jwt) throws Exception { + HttpHeaders headers = securityHelper.authHeaders(jwt); + var response = restTemplate.exchange(path, HttpMethod.POST, + new HttpEntity<>(json, headers), String.class); + return objectMapper.readTree(response.getBody()); + } + + private void put(String path, String json, String jwt) { + HttpHeaders headers = securityHelper.authHeaders(jwt); + restTemplate.exchange(path, HttpMethod.PUT, + new HttpEntity<>(json, headers), String.class); + } + + private String uploadJar(String appSlug, byte[] content) throws Exception { + ByteArrayResource resource = new ByteArrayResource(content) { + @Override public String getFilename() { return "app.jar"; } + }; + MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("file", resource); + + HttpHeaders headers = new HttpHeaders(); + headers.set("Authorization", "Bearer " + operatorJwt); + headers.set("X-Cameleer-Protocol-Version", "1"); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + + var response = restTemplate.exchange( + "/api/v1/environments/default/apps/" + appSlug + "/versions", + HttpMethod.POST, new HttpEntity<>(body, headers), String.class); + JsonNode versionNode = objectMapper.readTree(response.getBody()); + return versionNode.path("id").asText(); + } +} diff --git a/cameleer-server-app/src/test/java/com/cameleer/server/app/runtime/RollingStrategyIT.java b/cameleer-server-app/src/test/java/com/cameleer/server/app/runtime/RollingStrategyIT.java new file mode 100644 index 00000000..e8cba532 --- /dev/null +++ b/cameleer-server-app/src/test/java/com/cameleer/server/app/runtime/RollingStrategyIT.java @@ -0,0 +1,194 @@ +package com.cameleer.server.app.runtime; + +import com.cameleer.server.app.AbstractPostgresIT; +import com.cameleer.server.app.TestSecurityHelper; +import com.cameleer.server.app.storage.PostgresDeploymentRepository; +import com.cameleer.server.core.runtime.ContainerStatus; +import com.cameleer.server.core.runtime.Deployment; +import com.cameleer.server.core.runtime.DeploymentStatus; +import com.cameleer.server.core.runtime.RuntimeOrchestrator; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.web.client.TestRestTemplate; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.test.context.TestPropertySource; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Verifies the rolling deployment strategy: per-replica start → health → stop + * old. Mid-rollout health failure preserves remaining un-replaced old replicas; + * already-stopped old replicas are not restored. + */ +@TestPropertySource(properties = "cameleer.server.runtime.healthchecktimeout=2") +class RollingStrategyIT extends AbstractPostgresIT { + + @MockBean + RuntimeOrchestrator runtimeOrchestrator; + + @Autowired private TestRestTemplate restTemplate; + @Autowired private ObjectMapper objectMapper; + @Autowired private TestSecurityHelper securityHelper; + @Autowired private PostgresDeploymentRepository deploymentRepository; + + private String operatorJwt; + private String appSlug; + private String versionId; + + @BeforeEach + void setUp() throws Exception { + operatorJwt = securityHelper.operatorToken(); + + jdbcTemplate.update("DELETE FROM deployments"); + jdbcTemplate.update("DELETE FROM app_versions"); + jdbcTemplate.update("DELETE FROM apps"); + jdbcTemplate.update("DELETE FROM application_config WHERE environment = 'default'"); + + when(runtimeOrchestrator.isEnabled()).thenReturn(true); + + appSlug = "roll-" + UUID.randomUUID().toString().substring(0, 8); + post("/api/v1/environments/default/apps", String.format(""" + {"slug": "%s", "displayName": "Rolling App"} + """, appSlug), operatorJwt); + put("/api/v1/environments/default/apps/" + appSlug + "/container-config", """ + {"runtimeType": "spring-boot", "appPort": 8081, "replicas": 2, "deploymentStrategy": "rolling"} + """, operatorJwt); + versionId = uploadJar(appSlug, ("roll-jar-" + appSlug).getBytes()); + } + + @Test + void rolling_allHealthy_replacesOneByOne() throws Exception { + when(runtimeOrchestrator.startContainer(any())) + .thenReturn("old-0", "old-1", "new-0", "new-1"); + ContainerStatus healthy = new ContainerStatus("healthy", true, 0, null); + when(runtimeOrchestrator.getContainerStatus("old-0")).thenReturn(healthy); + when(runtimeOrchestrator.getContainerStatus("old-1")).thenReturn(healthy); + when(runtimeOrchestrator.getContainerStatus("new-0")).thenReturn(healthy); + when(runtimeOrchestrator.getContainerStatus("new-1")).thenReturn(healthy); + + String firstDeployId = triggerDeploy(); + awaitStatus(firstDeployId, DeploymentStatus.RUNNING); + + String secondDeployId = triggerDeploy(); + awaitStatus(secondDeployId, DeploymentStatus.RUNNING); + + // Rolling invariant: old-0 is stopped BEFORE old-1 (replicas replaced + // one at a time, not all at once). Checking stop order is sufficient — + // a blue-green path would have both stops adjacent at the end with no + // interleaved starts; rolling interleaves starts between stops. + InOrder inOrder = inOrder(runtimeOrchestrator); + inOrder.verify(runtimeOrchestrator).stopContainer("old-0"); + inOrder.verify(runtimeOrchestrator).stopContainer("old-1"); + + // Total of 4 startContainer calls: 2 for first deploy, 2 for rolling. + verify(runtimeOrchestrator, times(4)).startContainer(any()); + // New replicas were not stopped — they're the running ones now. + verify(runtimeOrchestrator, never()).stopContainer("new-0"); + verify(runtimeOrchestrator, never()).stopContainer("new-1"); + + Deployment first = deploymentRepository.findById(UUID.fromString(firstDeployId)).orElseThrow(); + assertThat(first.status()).isEqualTo(DeploymentStatus.STOPPED); + } + + @Test + void rolling_failsMidRollout_preservesRemainingOld() throws Exception { + when(runtimeOrchestrator.startContainer(any())) + .thenReturn("old-0", "old-1", "new-0", "new-1"); + ContainerStatus healthy = new ContainerStatus("healthy", true, 0, null); + ContainerStatus starting = new ContainerStatus("starting", true, 0, null); + when(runtimeOrchestrator.getContainerStatus("old-0")).thenReturn(healthy); + when(runtimeOrchestrator.getContainerStatus("old-1")).thenReturn(healthy); + when(runtimeOrchestrator.getContainerStatus("new-0")).thenReturn(healthy); + when(runtimeOrchestrator.getContainerStatus("new-1")).thenReturn(starting); + + String firstDeployId = triggerDeploy(); + awaitStatus(firstDeployId, DeploymentStatus.RUNNING); + + String secondDeployId = triggerDeploy(); + awaitStatus(secondDeployId, DeploymentStatus.FAILED); + + Deployment second = deploymentRepository.findById(UUID.fromString(secondDeployId)).orElseThrow(); + assertThat(second.errorMessage()) + .contains("rolling") + .contains("replica 1"); + + // old-0 was replaced before the failure; old-1 was never touched. + verify(runtimeOrchestrator).stopContainer("old-0"); + verify(runtimeOrchestrator, never()).stopContainer("old-1"); + // Cleanup stops both new replicas started so far. + verify(runtimeOrchestrator).stopContainer("new-0"); + verify(runtimeOrchestrator).stopContainer("new-1"); + } + + // ---- helpers (same pattern as BlueGreenStrategyIT) ---- + + private String triggerDeploy() throws Exception { + JsonNode deployResponse = post( + "/api/v1/environments/default/apps/" + appSlug + "/deployments", + String.format("{\"appVersionId\": \"%s\"}", versionId), operatorJwt); + return deployResponse.path("id").asText(); + } + + private void awaitStatus(String deployId, DeploymentStatus expected) { + await().atMost(30, TimeUnit.SECONDS) + .pollInterval(500, TimeUnit.MILLISECONDS) + .untilAsserted(() -> { + Deployment d = deploymentRepository.findById(UUID.fromString(deployId)) + .orElseThrow(() -> new AssertionError("Deployment not found: " + deployId)); + assertThat(d.status()).isEqualTo(expected); + }); + } + + private JsonNode post(String path, String json, String jwt) throws Exception { + HttpHeaders headers = securityHelper.authHeaders(jwt); + var response = restTemplate.exchange(path, HttpMethod.POST, + new HttpEntity<>(json, headers), String.class); + return objectMapper.readTree(response.getBody()); + } + + private void put(String path, String json, String jwt) { + HttpHeaders headers = securityHelper.authHeaders(jwt); + restTemplate.exchange(path, HttpMethod.PUT, + new HttpEntity<>(json, headers), String.class); + } + + private String uploadJar(String appSlug, byte[] content) throws Exception { + ByteArrayResource resource = new ByteArrayResource(content) { + @Override public String getFilename() { return "app.jar"; } + }; + MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("file", resource); + + HttpHeaders headers = new HttpHeaders(); + headers.set("Authorization", "Bearer " + operatorJwt); + headers.set("X-Cameleer-Protocol-Version", "1"); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + + var response = restTemplate.exchange( + "/api/v1/environments/default/apps/" + appSlug + "/versions", + HttpMethod.POST, new HttpEntity<>(body, headers), String.class); + JsonNode versionNode = objectMapper.readTree(response.getBody()); + return versionNode.path("id").asText(); + } +}