feat(01-02): implement ingestion REST controllers with backpressure

- ExecutionController: POST /api/v1/data/executions (single or array)
- DiagramController: POST /api/v1/data/diagrams (single or array)
- MetricsController: POST /api/v1/data/metrics (array)
- All return 202 Accepted or 503 with Retry-After when buffer full
- Fix duplicate IngestionConfig bean (remove @Configuration, use @EnableConfigurationProperties)
- Fix BackpressureIT timing by using batch POST and 60s flush interval
- All 11 integration tests green

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
hsiegeln
2026-03-11 12:13:27 +01:00
parent d55ebc1f57
commit 8fe65f083c
5 changed files with 266 additions and 45 deletions

View File

@@ -1,13 +1,13 @@
package com.cameleer3.server.app.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
/**
* Configuration properties for the ingestion write buffer.
* Bound from the {@code ingestion.*} namespace in application.yml.
* <p>
* Registered via {@code @EnableConfigurationProperties} on the application class.
*/
@Configuration
@ConfigurationProperties(prefix = "ingestion")
public class IngestionConfig {

View File

@@ -0,0 +1,77 @@
package com.cameleer3.server.app.controller;
import com.cameleer3.common.graph.RouteGraph;
import com.cameleer3.server.core.ingestion.IngestionService;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
/**
* Ingestion endpoint for route diagrams.
* <p>
* Accepts both single {@link RouteGraph} and arrays. Data is buffered
* and flushed to ClickHouse by the flush scheduler.
*/
@RestController
@RequestMapping("/api/v1/data")
@Tag(name = "Ingestion", description = "Data ingestion endpoints")
public class DiagramController {
private static final Logger log = LoggerFactory.getLogger(DiagramController.class);
private final IngestionService ingestionService;
private final ObjectMapper objectMapper;
public DiagramController(IngestionService ingestionService, ObjectMapper objectMapper) {
this.ingestionService = ingestionService;
this.objectMapper = objectMapper;
}
@PostMapping("/diagrams")
@Operation(summary = "Ingest route diagram data",
description = "Accepts a single RouteGraph or an array of RouteGraphs")
@ApiResponse(responseCode = "202", description = "Data accepted for processing")
@ApiResponse(responseCode = "503", description = "Buffer full, retry later")
public ResponseEntity<Void> ingestDiagrams(@RequestBody String body) throws JsonProcessingException {
List<RouteGraph> graphs = parsePayload(body);
boolean accepted;
if (graphs.size() == 1) {
accepted = ingestionService.acceptDiagram(graphs.get(0));
} else {
accepted = ingestionService.acceptDiagrams(graphs);
}
if (!accepted) {
log.warn("Diagram buffer full, returning 503");
return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE)
.header("Retry-After", "5")
.build();
}
return ResponseEntity.accepted().build();
}
private List<RouteGraph> parsePayload(String body) throws JsonProcessingException {
String trimmed = body.strip();
if (trimmed.startsWith("[")) {
return objectMapper.readValue(trimmed, new TypeReference<>() {});
} else {
RouteGraph single = objectMapper.readValue(trimmed, RouteGraph.class);
return List.of(single);
}
}
}

View File

@@ -0,0 +1,78 @@
package com.cameleer3.server.app.controller;
import com.cameleer3.common.model.RouteExecution;
import com.cameleer3.server.core.ingestion.IngestionService;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
/**
* Ingestion endpoint for route execution data.
* <p>
* Accepts both single {@link RouteExecution} and arrays. Data is buffered
* in a {@link com.cameleer3.server.core.ingestion.WriteBuffer} and flushed
* to ClickHouse by the flush scheduler.
*/
@RestController
@RequestMapping("/api/v1/data")
@Tag(name = "Ingestion", description = "Data ingestion endpoints")
public class ExecutionController {
private static final Logger log = LoggerFactory.getLogger(ExecutionController.class);
private final IngestionService ingestionService;
private final ObjectMapper objectMapper;
public ExecutionController(IngestionService ingestionService, ObjectMapper objectMapper) {
this.ingestionService = ingestionService;
this.objectMapper = objectMapper;
}
@PostMapping("/executions")
@Operation(summary = "Ingest route execution data",
description = "Accepts a single RouteExecution or an array of RouteExecutions")
@ApiResponse(responseCode = "202", description = "Data accepted for processing")
@ApiResponse(responseCode = "503", description = "Buffer full, retry later")
public ResponseEntity<Void> ingestExecutions(@RequestBody String body) throws JsonProcessingException {
List<RouteExecution> executions = parsePayload(body);
boolean accepted;
if (executions.size() == 1) {
accepted = ingestionService.acceptExecution(executions.get(0));
} else {
accepted = ingestionService.acceptExecutions(executions);
}
if (!accepted) {
log.warn("Execution buffer full, returning 503");
return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE)
.header("Retry-After", "5")
.build();
}
return ResponseEntity.accepted().build();
}
private List<RouteExecution> parsePayload(String body) throws JsonProcessingException {
String trimmed = body.strip();
if (trimmed.startsWith("[")) {
return objectMapper.readValue(trimmed, new TypeReference<>() {});
} else {
RouteExecution single = objectMapper.readValue(trimmed, RouteExecution.class);
return List.of(single);
}
}
}

View File

@@ -0,0 +1,71 @@
package com.cameleer3.server.app.controller;
import com.cameleer3.server.core.ingestion.IngestionService;
import com.cameleer3.server.core.storage.model.MetricsSnapshot;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
/**
* Ingestion endpoint for agent metrics.
* <p>
* Accepts an array of {@link MetricsSnapshot}. Data is buffered
* and flushed to ClickHouse by the flush scheduler.
*/
@RestController
@RequestMapping("/api/v1/data")
@Tag(name = "Ingestion", description = "Data ingestion endpoints")
public class MetricsController {
private static final Logger log = LoggerFactory.getLogger(MetricsController.class);
private final IngestionService ingestionService;
private final ObjectMapper objectMapper;
public MetricsController(IngestionService ingestionService, ObjectMapper objectMapper) {
this.ingestionService = ingestionService;
this.objectMapper = objectMapper;
}
@PostMapping("/metrics")
@Operation(summary = "Ingest agent metrics",
description = "Accepts an array of MetricsSnapshot objects")
@ApiResponse(responseCode = "202", description = "Data accepted for processing")
@ApiResponse(responseCode = "503", description = "Buffer full, retry later")
public ResponseEntity<Void> ingestMetrics(@RequestBody String body) throws JsonProcessingException {
List<MetricsSnapshot> metrics = parsePayload(body);
boolean accepted = ingestionService.acceptMetrics(metrics);
if (!accepted) {
log.warn("Metrics buffer full, returning 503");
return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE)
.header("Retry-After", "5")
.build();
}
return ResponseEntity.accepted().build();
}
private List<MetricsSnapshot> parsePayload(String body) throws JsonProcessingException {
String trimmed = body.strip();
if (trimmed.startsWith("[")) {
return objectMapper.readValue(trimmed, new TypeReference<>() {});
} else {
MetricsSnapshot single = objectMapper.readValue(trimmed, MetricsSnapshot.class);
return List.of(single);
}
}
}