test(deploy): blue-green + rolling strategy ITs
Four ITs covering strategy behavior: - BlueGreenStrategyIT#blueGreen_allHealthy_stopsOldAfterNew: old is stopped only after all new replicas are healthy. - BlueGreenStrategyIT#blueGreen_partialHealthy_preservesOldAndMarksFailed: strict all-healthy — one starting replica aborts the deploy and leaves the previous deployment RUNNING untouched. - RollingStrategyIT#rolling_allHealthy_replacesOneByOne: InOrder on stopContainer confirms old-0 stops before old-1 (the interleaving that distinguishes rolling from blue-green). - RollingStrategyIT#rolling_failsMidRollout_preservesRemainingOld: mid-rollout health failure stops only the in-flight new containers and the already-replaced old-0; old-1 stays untouched. Shortens healthchecktimeout to 2s via @TestPropertySource so failure paths complete in ~25s instead of ~60s. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
hsiegeln
@@ -0,0 +1,190 @@
|
||||
package com.cameleer.server.app.runtime;
|
||||
|
||||
import com.cameleer.server.app.AbstractPostgresIT;
|
||||
import com.cameleer.server.app.TestSecurityHelper;
|
||||
import com.cameleer.server.app.storage.PostgresDeploymentRepository;
|
||||
import com.cameleer.server.core.runtime.ContainerStatus;
|
||||
import com.cameleer.server.core.runtime.Deployment;
|
||||
import com.cameleer.server.core.runtime.DeploymentStatus;
|
||||
import com.cameleer.server.core.runtime.RuntimeOrchestrator;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.mock.mockito.MockBean;
|
||||
import org.springframework.boot.test.web.client.TestRestTemplate;
|
||||
import org.springframework.core.io.ByteArrayResource;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.test.context.TestPropertySource;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.awaitility.Awaitility.await;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Verifies the blue-green deployment strategy: start all new → health-check
|
||||
* all → stop old. Strict all-healthy — partial failure preserves the previous
|
||||
* deployment untouched.
|
||||
*/
|
||||
@TestPropertySource(properties = "cameleer.server.runtime.healthchecktimeout=2")
|
||||
class BlueGreenStrategyIT extends AbstractPostgresIT {
|
||||
|
||||
@MockBean
|
||||
RuntimeOrchestrator runtimeOrchestrator;
|
||||
|
||||
@Autowired private TestRestTemplate restTemplate;
|
||||
@Autowired private ObjectMapper objectMapper;
|
||||
@Autowired private TestSecurityHelper securityHelper;
|
||||
@Autowired private PostgresDeploymentRepository deploymentRepository;
|
||||
|
||||
private String operatorJwt;
|
||||
private String appSlug;
|
||||
private String versionId;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
operatorJwt = securityHelper.operatorToken();
|
||||
|
||||
jdbcTemplate.update("DELETE FROM deployments");
|
||||
jdbcTemplate.update("DELETE FROM app_versions");
|
||||
jdbcTemplate.update("DELETE FROM apps");
|
||||
jdbcTemplate.update("DELETE FROM application_config WHERE environment = 'default'");
|
||||
|
||||
when(runtimeOrchestrator.isEnabled()).thenReturn(true);
|
||||
|
||||
appSlug = "bg-" + UUID.randomUUID().toString().substring(0, 8);
|
||||
post("/api/v1/environments/default/apps", String.format("""
|
||||
{"slug": "%s", "displayName": "BG App"}
|
||||
""", appSlug), operatorJwt);
|
||||
put("/api/v1/environments/default/apps/" + appSlug + "/container-config", """
|
||||
{"runtimeType": "spring-boot", "appPort": 8081, "replicas": 2, "deploymentStrategy": "blue-green"}
|
||||
""", operatorJwt);
|
||||
versionId = uploadJar(appSlug, ("bg-jar-" + appSlug).getBytes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void blueGreen_allHealthy_stopsOldAfterNew() throws Exception {
|
||||
when(runtimeOrchestrator.startContainer(any()))
|
||||
.thenReturn("old-0", "old-1", "new-0", "new-1");
|
||||
ContainerStatus healthy = new ContainerStatus("healthy", true, 0, null);
|
||||
when(runtimeOrchestrator.getContainerStatus("old-0")).thenReturn(healthy);
|
||||
when(runtimeOrchestrator.getContainerStatus("old-1")).thenReturn(healthy);
|
||||
when(runtimeOrchestrator.getContainerStatus("new-0")).thenReturn(healthy);
|
||||
when(runtimeOrchestrator.getContainerStatus("new-1")).thenReturn(healthy);
|
||||
|
||||
String firstDeployId = triggerDeploy();
|
||||
awaitStatus(firstDeployId, DeploymentStatus.RUNNING);
|
||||
|
||||
String secondDeployId = triggerDeploy();
|
||||
awaitStatus(secondDeployId, DeploymentStatus.RUNNING);
|
||||
|
||||
// Previous deployment was stopped once new was healthy
|
||||
Deployment first = deploymentRepository.findById(UUID.fromString(firstDeployId)).orElseThrow();
|
||||
assertThat(first.status()).isEqualTo(DeploymentStatus.STOPPED);
|
||||
|
||||
verify(runtimeOrchestrator).stopContainer("old-0");
|
||||
verify(runtimeOrchestrator).stopContainer("old-1");
|
||||
verify(runtimeOrchestrator, never()).stopContainer("new-0");
|
||||
verify(runtimeOrchestrator, never()).stopContainer("new-1");
|
||||
|
||||
// New deployment has both new replicas recorded
|
||||
Deployment second = deploymentRepository.findById(UUID.fromString(secondDeployId)).orElseThrow();
|
||||
assertThat(second.replicaStates()).hasSize(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void blueGreen_partialHealthy_preservesOldAndMarksFailed() throws Exception {
|
||||
when(runtimeOrchestrator.startContainer(any()))
|
||||
.thenReturn("old-0", "old-1", "new-0", "new-1");
|
||||
ContainerStatus healthy = new ContainerStatus("healthy", true, 0, null);
|
||||
ContainerStatus starting = new ContainerStatus("starting", true, 0, null);
|
||||
when(runtimeOrchestrator.getContainerStatus("old-0")).thenReturn(healthy);
|
||||
when(runtimeOrchestrator.getContainerStatus("old-1")).thenReturn(healthy);
|
||||
when(runtimeOrchestrator.getContainerStatus("new-0")).thenReturn(healthy);
|
||||
when(runtimeOrchestrator.getContainerStatus("new-1")).thenReturn(starting);
|
||||
|
||||
String firstDeployId = triggerDeploy();
|
||||
awaitStatus(firstDeployId, DeploymentStatus.RUNNING);
|
||||
|
||||
String secondDeployId = triggerDeploy();
|
||||
awaitStatus(secondDeployId, DeploymentStatus.FAILED);
|
||||
|
||||
Deployment second = deploymentRepository.findById(UUID.fromString(secondDeployId)).orElseThrow();
|
||||
assertThat(second.errorMessage())
|
||||
.contains("blue-green")
|
||||
.contains("1/2");
|
||||
|
||||
// Previous deployment stays RUNNING — blue-green's safety promise.
|
||||
Deployment first = deploymentRepository.findById(UUID.fromString(firstDeployId)).orElseThrow();
|
||||
assertThat(first.status()).isEqualTo(DeploymentStatus.RUNNING);
|
||||
|
||||
verify(runtimeOrchestrator, never()).stopContainer("old-0");
|
||||
verify(runtimeOrchestrator, never()).stopContainer("old-1");
|
||||
// Cleanup ran on both new replicas.
|
||||
verify(runtimeOrchestrator).stopContainer("new-0");
|
||||
verify(runtimeOrchestrator).stopContainer("new-1");
|
||||
}
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
private String triggerDeploy() throws Exception {
|
||||
JsonNode deployResponse = post(
|
||||
"/api/v1/environments/default/apps/" + appSlug + "/deployments",
|
||||
String.format("{\"appVersionId\": \"%s\"}", versionId), operatorJwt);
|
||||
return deployResponse.path("id").asText();
|
||||
}
|
||||
|
||||
private void awaitStatus(String deployId, DeploymentStatus expected) {
|
||||
await().atMost(30, TimeUnit.SECONDS)
|
||||
.pollInterval(500, TimeUnit.MILLISECONDS)
|
||||
.untilAsserted(() -> {
|
||||
Deployment d = deploymentRepository.findById(UUID.fromString(deployId))
|
||||
.orElseThrow(() -> new AssertionError("Deployment not found: " + deployId));
|
||||
assertThat(d.status()).isEqualTo(expected);
|
||||
});
|
||||
}
|
||||
|
||||
private JsonNode post(String path, String json, String jwt) throws Exception {
|
||||
HttpHeaders headers = securityHelper.authHeaders(jwt);
|
||||
var response = restTemplate.exchange(path, HttpMethod.POST,
|
||||
new HttpEntity<>(json, headers), String.class);
|
||||
return objectMapper.readTree(response.getBody());
|
||||
}
|
||||
|
||||
private void put(String path, String json, String jwt) {
|
||||
HttpHeaders headers = securityHelper.authHeaders(jwt);
|
||||
restTemplate.exchange(path, HttpMethod.PUT,
|
||||
new HttpEntity<>(json, headers), String.class);
|
||||
}
|
||||
|
||||
private String uploadJar(String appSlug, byte[] content) throws Exception {
|
||||
ByteArrayResource resource = new ByteArrayResource(content) {
|
||||
@Override public String getFilename() { return "app.jar"; }
|
||||
};
|
||||
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
|
||||
body.add("file", resource);
|
||||
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.set("Authorization", "Bearer " + operatorJwt);
|
||||
headers.set("X-Cameleer-Protocol-Version", "1");
|
||||
headers.setContentType(MediaType.MULTIPART_FORM_DATA);
|
||||
|
||||
var response = restTemplate.exchange(
|
||||
"/api/v1/environments/default/apps/" + appSlug + "/versions",
|
||||
HttpMethod.POST, new HttpEntity<>(body, headers), String.class);
|
||||
JsonNode versionNode = objectMapper.readTree(response.getBody());
|
||||
return versionNode.path("id").asText();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
package com.cameleer.server.app.runtime;
|
||||
|
||||
import com.cameleer.server.app.AbstractPostgresIT;
|
||||
import com.cameleer.server.app.TestSecurityHelper;
|
||||
import com.cameleer.server.app.storage.PostgresDeploymentRepository;
|
||||
import com.cameleer.server.core.runtime.ContainerStatus;
|
||||
import com.cameleer.server.core.runtime.Deployment;
|
||||
import com.cameleer.server.core.runtime.DeploymentStatus;
|
||||
import com.cameleer.server.core.runtime.RuntimeOrchestrator;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.InOrder;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.mock.mockito.MockBean;
|
||||
import org.springframework.boot.test.web.client.TestRestTemplate;
|
||||
import org.springframework.core.io.ByteArrayResource;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.test.context.TestPropertySource;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.awaitility.Awaitility.await;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.inOrder;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Verifies the rolling deployment strategy: per-replica start → health → stop
|
||||
* old. Mid-rollout health failure preserves remaining un-replaced old replicas;
|
||||
* already-stopped old replicas are not restored.
|
||||
*/
|
||||
@TestPropertySource(properties = "cameleer.server.runtime.healthchecktimeout=2")
|
||||
class RollingStrategyIT extends AbstractPostgresIT {
|
||||
|
||||
@MockBean
|
||||
RuntimeOrchestrator runtimeOrchestrator;
|
||||
|
||||
@Autowired private TestRestTemplate restTemplate;
|
||||
@Autowired private ObjectMapper objectMapper;
|
||||
@Autowired private TestSecurityHelper securityHelper;
|
||||
@Autowired private PostgresDeploymentRepository deploymentRepository;
|
||||
|
||||
private String operatorJwt;
|
||||
private String appSlug;
|
||||
private String versionId;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
operatorJwt = securityHelper.operatorToken();
|
||||
|
||||
jdbcTemplate.update("DELETE FROM deployments");
|
||||
jdbcTemplate.update("DELETE FROM app_versions");
|
||||
jdbcTemplate.update("DELETE FROM apps");
|
||||
jdbcTemplate.update("DELETE FROM application_config WHERE environment = 'default'");
|
||||
|
||||
when(runtimeOrchestrator.isEnabled()).thenReturn(true);
|
||||
|
||||
appSlug = "roll-" + UUID.randomUUID().toString().substring(0, 8);
|
||||
post("/api/v1/environments/default/apps", String.format("""
|
||||
{"slug": "%s", "displayName": "Rolling App"}
|
||||
""", appSlug), operatorJwt);
|
||||
put("/api/v1/environments/default/apps/" + appSlug + "/container-config", """
|
||||
{"runtimeType": "spring-boot", "appPort": 8081, "replicas": 2, "deploymentStrategy": "rolling"}
|
||||
""", operatorJwt);
|
||||
versionId = uploadJar(appSlug, ("roll-jar-" + appSlug).getBytes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void rolling_allHealthy_replacesOneByOne() throws Exception {
|
||||
when(runtimeOrchestrator.startContainer(any()))
|
||||
.thenReturn("old-0", "old-1", "new-0", "new-1");
|
||||
ContainerStatus healthy = new ContainerStatus("healthy", true, 0, null);
|
||||
when(runtimeOrchestrator.getContainerStatus("old-0")).thenReturn(healthy);
|
||||
when(runtimeOrchestrator.getContainerStatus("old-1")).thenReturn(healthy);
|
||||
when(runtimeOrchestrator.getContainerStatus("new-0")).thenReturn(healthy);
|
||||
when(runtimeOrchestrator.getContainerStatus("new-1")).thenReturn(healthy);
|
||||
|
||||
String firstDeployId = triggerDeploy();
|
||||
awaitStatus(firstDeployId, DeploymentStatus.RUNNING);
|
||||
|
||||
String secondDeployId = triggerDeploy();
|
||||
awaitStatus(secondDeployId, DeploymentStatus.RUNNING);
|
||||
|
||||
// Rolling invariant: old-0 is stopped BEFORE old-1 (replicas replaced
|
||||
// one at a time, not all at once). Checking stop order is sufficient —
|
||||
// a blue-green path would have both stops adjacent at the end with no
|
||||
// interleaved starts; rolling interleaves starts between stops.
|
||||
InOrder inOrder = inOrder(runtimeOrchestrator);
|
||||
inOrder.verify(runtimeOrchestrator).stopContainer("old-0");
|
||||
inOrder.verify(runtimeOrchestrator).stopContainer("old-1");
|
||||
|
||||
// Total of 4 startContainer calls: 2 for first deploy, 2 for rolling.
|
||||
verify(runtimeOrchestrator, times(4)).startContainer(any());
|
||||
// New replicas were not stopped — they're the running ones now.
|
||||
verify(runtimeOrchestrator, never()).stopContainer("new-0");
|
||||
verify(runtimeOrchestrator, never()).stopContainer("new-1");
|
||||
|
||||
Deployment first = deploymentRepository.findById(UUID.fromString(firstDeployId)).orElseThrow();
|
||||
assertThat(first.status()).isEqualTo(DeploymentStatus.STOPPED);
|
||||
}
|
||||
|
||||
@Test
|
||||
void rolling_failsMidRollout_preservesRemainingOld() throws Exception {
|
||||
when(runtimeOrchestrator.startContainer(any()))
|
||||
.thenReturn("old-0", "old-1", "new-0", "new-1");
|
||||
ContainerStatus healthy = new ContainerStatus("healthy", true, 0, null);
|
||||
ContainerStatus starting = new ContainerStatus("starting", true, 0, null);
|
||||
when(runtimeOrchestrator.getContainerStatus("old-0")).thenReturn(healthy);
|
||||
when(runtimeOrchestrator.getContainerStatus("old-1")).thenReturn(healthy);
|
||||
when(runtimeOrchestrator.getContainerStatus("new-0")).thenReturn(healthy);
|
||||
when(runtimeOrchestrator.getContainerStatus("new-1")).thenReturn(starting);
|
||||
|
||||
String firstDeployId = triggerDeploy();
|
||||
awaitStatus(firstDeployId, DeploymentStatus.RUNNING);
|
||||
|
||||
String secondDeployId = triggerDeploy();
|
||||
awaitStatus(secondDeployId, DeploymentStatus.FAILED);
|
||||
|
||||
Deployment second = deploymentRepository.findById(UUID.fromString(secondDeployId)).orElseThrow();
|
||||
assertThat(second.errorMessage())
|
||||
.contains("rolling")
|
||||
.contains("replica 1");
|
||||
|
||||
// old-0 was replaced before the failure; old-1 was never touched.
|
||||
verify(runtimeOrchestrator).stopContainer("old-0");
|
||||
verify(runtimeOrchestrator, never()).stopContainer("old-1");
|
||||
// Cleanup stops both new replicas started so far.
|
||||
verify(runtimeOrchestrator).stopContainer("new-0");
|
||||
verify(runtimeOrchestrator).stopContainer("new-1");
|
||||
}
|
||||
|
||||
// ---- helpers (same pattern as BlueGreenStrategyIT) ----
|
||||
|
||||
private String triggerDeploy() throws Exception {
|
||||
JsonNode deployResponse = post(
|
||||
"/api/v1/environments/default/apps/" + appSlug + "/deployments",
|
||||
String.format("{\"appVersionId\": \"%s\"}", versionId), operatorJwt);
|
||||
return deployResponse.path("id").asText();
|
||||
}
|
||||
|
||||
private void awaitStatus(String deployId, DeploymentStatus expected) {
|
||||
await().atMost(30, TimeUnit.SECONDS)
|
||||
.pollInterval(500, TimeUnit.MILLISECONDS)
|
||||
.untilAsserted(() -> {
|
||||
Deployment d = deploymentRepository.findById(UUID.fromString(deployId))
|
||||
.orElseThrow(() -> new AssertionError("Deployment not found: " + deployId));
|
||||
assertThat(d.status()).isEqualTo(expected);
|
||||
});
|
||||
}
|
||||
|
||||
private JsonNode post(String path, String json, String jwt) throws Exception {
|
||||
HttpHeaders headers = securityHelper.authHeaders(jwt);
|
||||
var response = restTemplate.exchange(path, HttpMethod.POST,
|
||||
new HttpEntity<>(json, headers), String.class);
|
||||
return objectMapper.readTree(response.getBody());
|
||||
}
|
||||
|
||||
private void put(String path, String json, String jwt) {
|
||||
HttpHeaders headers = securityHelper.authHeaders(jwt);
|
||||
restTemplate.exchange(path, HttpMethod.PUT,
|
||||
new HttpEntity<>(json, headers), String.class);
|
||||
}
|
||||
|
||||
private String uploadJar(String appSlug, byte[] content) throws Exception {
|
||||
ByteArrayResource resource = new ByteArrayResource(content) {
|
||||
@Override public String getFilename() { return "app.jar"; }
|
||||
};
|
||||
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
|
||||
body.add("file", resource);
|
||||
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.set("Authorization", "Bearer " + operatorJwt);
|
||||
headers.set("X-Cameleer-Protocol-Version", "1");
|
||||
headers.setContentType(MediaType.MULTIPART_FORM_DATA);
|
||||
|
||||
var response = restTemplate.exchange(
|
||||
"/api/v1/environments/default/apps/" + appSlug + "/versions",
|
||||
HttpMethod.POST, new HttpEntity<>(body, headers), String.class);
|
||||
JsonNode versionNode = objectMapper.readTree(response.getBody());
|
||||
return versionNode.path("id").asText();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user