feat(server): REST API over server_metrics for SaaS dashboards
Adds /api/v1/admin/server-metrics/{catalog,instances,query} so SaaS control
planes can build the server-health dashboard without direct ClickHouse
access. One generic /query endpoint covers every panel in the
server-self-metrics doc: aggregation (avg/sum/max/min/latest), group-by-tag,
filter-by-tag, counter-delta mode with per-server_instance_id rotation
handling, and a derived 'mean' statistic for timers. Regex-validated
identifiers, parameterised literals, 31-day range cap, 500-series response
cap. ADMIN-only via the existing /api/v1/admin/** RBAC gate. Docs updated:
all 17 suggested panels now expressed as single-endpoint queries.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
hsiegeln
@@ -0,0 +1,314 @@
|
||||
package com.cameleer.server.app.controller;
|
||||
|
||||
import com.cameleer.server.app.AbstractPostgresIT;
|
||||
import com.cameleer.server.app.TestSecurityHelper;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.web.client.TestRestTemplate;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class ServerMetricsAdminControllerIT extends AbstractPostgresIT {
|
||||
|
||||
@Autowired
|
||||
private TestRestTemplate restTemplate;
|
||||
|
||||
@Autowired
|
||||
private TestSecurityHelper securityHelper;
|
||||
|
||||
private final ObjectMapper mapper = new ObjectMapper();
|
||||
|
||||
private HttpHeaders adminJson;
|
||||
private HttpHeaders adminGet;
|
||||
private HttpHeaders viewerGet;
|
||||
|
||||
@BeforeEach
|
||||
void seedAndAuth() {
|
||||
adminJson = securityHelper.adminHeaders();
|
||||
adminGet = securityHelper.authHeadersNoBody(securityHelper.adminToken());
|
||||
viewerGet = securityHelper.authHeadersNoBody(securityHelper.viewerToken());
|
||||
|
||||
// Fresh rows for each test. The Spring-context ClickHouse JdbcTemplate
|
||||
// lives in a different bean; reach for it here by executing through
|
||||
// the same JdbcTemplate used by the store via the ClickHouseConfig bean.
|
||||
org.springframework.jdbc.core.JdbcTemplate ch = clickhouseJdbc();
|
||||
ch.execute("TRUNCATE TABLE server_metrics");
|
||||
|
||||
Instant t0 = Instant.parse("2026-04-23T10:00:00Z");
|
||||
// Gauge: cameleer.agents.connected, two states, two buckets.
|
||||
insert(ch, "default", t0, "srv-A", "cameleer.agents.connected", "gauge", "value", 3.0,
|
||||
Map.of("state", "live"));
|
||||
insert(ch, "default", t0.plusSeconds(60), "srv-A", "cameleer.agents.connected", "gauge", "value", 4.0,
|
||||
Map.of("state", "live"));
|
||||
insert(ch, "default", t0, "srv-A", "cameleer.agents.connected", "gauge", "value", 1.0,
|
||||
Map.of("state", "stale"));
|
||||
insert(ch, "default", t0.plusSeconds(60), "srv-A", "cameleer.agents.connected", "gauge", "value", 0.0,
|
||||
Map.of("state", "stale"));
|
||||
|
||||
// Counter: cumulative drops, +5 per minute on srv-A.
|
||||
insert(ch, "default", t0, "srv-A", "cameleer.ingestion.drops", "counter", "count", 0.0, Map.of("reason", "buffer_full"));
|
||||
insert(ch, "default", t0.plusSeconds(60), "srv-A", "cameleer.ingestion.drops", "counter", "count", 5.0, Map.of("reason", "buffer_full"));
|
||||
insert(ch, "default", t0.plusSeconds(120), "srv-A", "cameleer.ingestion.drops", "counter", "count", 10.0, Map.of("reason", "buffer_full"));
|
||||
// Simulated restart to srv-B: counter resets to 0, then climbs to 2.
|
||||
insert(ch, "default", t0.plusSeconds(180), "srv-B", "cameleer.ingestion.drops", "counter", "count", 0.0, Map.of("reason", "buffer_full"));
|
||||
insert(ch, "default", t0.plusSeconds(240), "srv-B", "cameleer.ingestion.drops", "counter", "count", 2.0, Map.of("reason", "buffer_full"));
|
||||
|
||||
// Timer mean inputs: two buckets, 2 samples each (count=2, total_time=30).
|
||||
insert(ch, "default", t0, "srv-A", "cameleer.ingestion.flush.duration", "timer", "count", 2.0, Map.of("type", "execution"));
|
||||
insert(ch, "default", t0, "srv-A", "cameleer.ingestion.flush.duration", "timer", "total_time", 30.0, Map.of("type", "execution"));
|
||||
insert(ch, "default", t0.plusSeconds(60), "srv-A", "cameleer.ingestion.flush.duration", "timer", "count", 4.0, Map.of("type", "execution"));
|
||||
insert(ch, "default", t0.plusSeconds(60), "srv-A", "cameleer.ingestion.flush.duration", "timer", "total_time", 100.0, Map.of("type", "execution"));
|
||||
}
|
||||
|
||||
// ── catalog ─────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void catalog_listsSeededMetricsWithStatisticsAndTagKeys() throws Exception {
|
||||
ResponseEntity<String> r = restTemplate.exchange(
|
||||
"/api/v1/admin/server-metrics/catalog?from=2026-04-23T09:00:00Z&to=2026-04-23T11:00:00Z",
|
||||
HttpMethod.GET, new HttpEntity<>(adminGet), String.class);
|
||||
assertThat(r.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
|
||||
JsonNode body = mapper.readTree(r.getBody());
|
||||
assertThat(body.isArray()).isTrue();
|
||||
|
||||
JsonNode drops = findByField(body, "metricName", "cameleer.ingestion.drops");
|
||||
assertThat(drops.get("metricType").asText()).isEqualTo("counter");
|
||||
assertThat(asStringList(drops.get("statistics"))).contains("count");
|
||||
assertThat(asStringList(drops.get("tagKeys"))).contains("reason");
|
||||
|
||||
JsonNode timer = findByField(body, "metricName", "cameleer.ingestion.flush.duration");
|
||||
assertThat(asStringList(timer.get("statistics"))).contains("count", "total_time");
|
||||
}
|
||||
|
||||
// ── instances ───────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void instances_listsDistinctServerInstanceIdsWithFirstAndLastSeen() throws Exception {
|
||||
ResponseEntity<String> r = restTemplate.exchange(
|
||||
"/api/v1/admin/server-metrics/instances?from=2026-04-23T09:00:00Z&to=2026-04-23T11:00:00Z",
|
||||
HttpMethod.GET, new HttpEntity<>(adminGet), String.class);
|
||||
assertThat(r.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
|
||||
JsonNode body = mapper.readTree(r.getBody());
|
||||
assertThat(body.isArray()).isTrue();
|
||||
assertThat(body.size()).isEqualTo(2);
|
||||
// Ordered by last_seen DESC — srv-B saw a later row.
|
||||
assertThat(body.get(0).get("serverInstanceId").asText()).isEqualTo("srv-B");
|
||||
assertThat(body.get(1).get("serverInstanceId").asText()).isEqualTo("srv-A");
|
||||
}
|
||||
|
||||
// ── query — gauge with group-by-tag ─────────────────────────────────
|
||||
|
||||
@Test
|
||||
void query_gaugeWithGroupByTag_returnsSeriesPerTagValue() throws Exception {
|
||||
String requestBody = """
|
||||
{
|
||||
"metric": "cameleer.agents.connected",
|
||||
"statistic": "value",
|
||||
"from": "2026-04-23T09:59:00Z",
|
||||
"to": "2026-04-23T10:02:00Z",
|
||||
"stepSeconds": 60,
|
||||
"groupByTags": ["state"],
|
||||
"aggregation": "avg",
|
||||
"mode": "raw"
|
||||
}
|
||||
""";
|
||||
|
||||
ResponseEntity<String> r = restTemplate.postForEntity(
|
||||
"/api/v1/admin/server-metrics/query",
|
||||
new HttpEntity<>(requestBody, adminJson), String.class);
|
||||
assertThat(r.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
|
||||
JsonNode body = mapper.readTree(r.getBody());
|
||||
assertThat(body.get("metric").asText()).isEqualTo("cameleer.agents.connected");
|
||||
assertThat(body.get("statistic").asText()).isEqualTo("value");
|
||||
assertThat(body.get("mode").asText()).isEqualTo("raw");
|
||||
assertThat(body.get("stepSeconds").asInt()).isEqualTo(60);
|
||||
|
||||
JsonNode series = body.get("series");
|
||||
assertThat(series.isArray()).isTrue();
|
||||
assertThat(series.size()).isEqualTo(2);
|
||||
|
||||
JsonNode live = findByTag(series, "state", "live");
|
||||
assertThat(live.get("points").size()).isEqualTo(2);
|
||||
assertThat(live.get("points").get(0).get("v").asDouble()).isEqualTo(3.0);
|
||||
assertThat(live.get("points").get(1).get("v").asDouble()).isEqualTo(4.0);
|
||||
}
|
||||
|
||||
// ── query — counter delta across instance rotation ──────────────────
|
||||
|
||||
@Test
|
||||
void query_counterDelta_clipsNegativesAcrossInstanceRotation() throws Exception {
|
||||
String requestBody = """
|
||||
{
|
||||
"metric": "cameleer.ingestion.drops",
|
||||
"statistic": "count",
|
||||
"from": "2026-04-23T09:59:00Z",
|
||||
"to": "2026-04-23T10:05:00Z",
|
||||
"stepSeconds": 60,
|
||||
"groupByTags": ["reason"],
|
||||
"aggregation": "sum",
|
||||
"mode": "delta"
|
||||
}
|
||||
""";
|
||||
|
||||
ResponseEntity<String> r = restTemplate.postForEntity(
|
||||
"/api/v1/admin/server-metrics/query",
|
||||
new HttpEntity<>(requestBody, adminJson), String.class);
|
||||
assertThat(r.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
|
||||
JsonNode body = mapper.readTree(r.getBody());
|
||||
JsonNode reason = findByTag(body.get("series"), "reason", "buffer_full");
|
||||
// Deltas: 0 (first bucket on srv-A), 5, 5, 0 (first on srv-B, clipped), 2.
|
||||
// Sum across the window should be 12 if we tally all positive deltas.
|
||||
double sum = 0;
|
||||
for (JsonNode p : reason.get("points")) sum += p.get("v").asDouble();
|
||||
assertThat(sum).isEqualTo(12.0);
|
||||
// No individual point may be negative.
|
||||
for (JsonNode p : reason.get("points")) {
|
||||
assertThat(p.get("v").asDouble()).isGreaterThanOrEqualTo(0.0);
|
||||
}
|
||||
}
|
||||
|
||||
// ── query — derived 'mean' statistic for timers ─────────────────────
|
||||
|
||||
@Test
|
||||
void query_timerMeanStatistic_computesTotalOverCountPerBucket() throws Exception {
|
||||
String requestBody = """
|
||||
{
|
||||
"metric": "cameleer.ingestion.flush.duration",
|
||||
"statistic": "mean",
|
||||
"from": "2026-04-23T09:59:00Z",
|
||||
"to": "2026-04-23T10:02:00Z",
|
||||
"stepSeconds": 60,
|
||||
"groupByTags": ["type"],
|
||||
"aggregation": "avg",
|
||||
"mode": "raw"
|
||||
}
|
||||
""";
|
||||
|
||||
ResponseEntity<String> r = restTemplate.postForEntity(
|
||||
"/api/v1/admin/server-metrics/query",
|
||||
new HttpEntity<>(requestBody, adminJson), String.class);
|
||||
assertThat(r.getStatusCode()).isEqualTo(HttpStatus.OK);
|
||||
|
||||
JsonNode body = mapper.readTree(r.getBody());
|
||||
JsonNode points = findByTag(body.get("series"), "type", "execution").get("points");
|
||||
// Bucket 0: 30 / 2 = 15.0
|
||||
// Bucket 1: 100 / 4 = 25.0
|
||||
assertThat(points.get(0).get("v").asDouble()).isEqualTo(15.0);
|
||||
assertThat(points.get(1).get("v").asDouble()).isEqualTo(25.0);
|
||||
}
|
||||
|
||||
// ── query — input validation ────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void query_rejectsUnsafeMetricName() {
|
||||
String requestBody = """
|
||||
{
|
||||
"metric": "cameleer.agents; DROP TABLE server_metrics",
|
||||
"from": "2026-04-23T09:59:00Z",
|
||||
"to": "2026-04-23T10:02:00Z"
|
||||
}
|
||||
""";
|
||||
|
||||
ResponseEntity<String> r = restTemplate.postForEntity(
|
||||
"/api/v1/admin/server-metrics/query",
|
||||
new HttpEntity<>(requestBody, adminJson), String.class);
|
||||
assertThat(r.getStatusCode()).isEqualTo(HttpStatus.BAD_REQUEST);
|
||||
}
|
||||
|
||||
@Test
|
||||
void query_rejectsRangeBeyondMax() {
|
||||
String requestBody = """
|
||||
{
|
||||
"metric": "cameleer.agents.connected",
|
||||
"from": "2026-01-01T00:00:00Z",
|
||||
"to": "2026-04-23T00:00:00Z"
|
||||
}
|
||||
""";
|
||||
|
||||
ResponseEntity<String> r = restTemplate.postForEntity(
|
||||
"/api/v1/admin/server-metrics/query",
|
||||
new HttpEntity<>(requestBody, adminJson), String.class);
|
||||
assertThat(r.getStatusCode()).isEqualTo(HttpStatus.BAD_REQUEST);
|
||||
}
|
||||
|
||||
// ── authorization ───────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void allEndpoints_requireAdminRole() {
|
||||
ResponseEntity<String> catalog = restTemplate.exchange(
|
||||
"/api/v1/admin/server-metrics/catalog",
|
||||
HttpMethod.GET, new HttpEntity<>(viewerGet), String.class);
|
||||
assertThat(catalog.getStatusCode()).isEqualTo(HttpStatus.FORBIDDEN);
|
||||
|
||||
ResponseEntity<String> instances = restTemplate.exchange(
|
||||
"/api/v1/admin/server-metrics/instances",
|
||||
HttpMethod.GET, new HttpEntity<>(viewerGet), String.class);
|
||||
assertThat(instances.getStatusCode()).isEqualTo(HttpStatus.FORBIDDEN);
|
||||
|
||||
HttpHeaders viewerPost = securityHelper.authHeaders(securityHelper.viewerToken());
|
||||
ResponseEntity<String> query = restTemplate.exchange(
|
||||
"/api/v1/admin/server-metrics/query",
|
||||
HttpMethod.POST, new HttpEntity<>("{}", viewerPost), String.class);
|
||||
assertThat(query.getStatusCode()).isEqualTo(HttpStatus.FORBIDDEN);
|
||||
}
|
||||
|
||||
// ── helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
private org.springframework.jdbc.core.JdbcTemplate clickhouseJdbc() {
|
||||
return org.springframework.test.util.AopTestUtils.getTargetObject(
|
||||
applicationContext.getBean("clickHouseJdbcTemplate"));
|
||||
}
|
||||
|
||||
@Autowired
|
||||
private org.springframework.context.ApplicationContext applicationContext;
|
||||
|
||||
private static void insert(org.springframework.jdbc.core.JdbcTemplate jdbc,
|
||||
String tenantId, Instant collectedAt, String serverInstanceId,
|
||||
String metricName, String metricType, String statistic,
|
||||
double value, Map<String, String> tags) {
|
||||
jdbc.update("""
|
||||
INSERT INTO server_metrics
|
||||
(tenant_id, collected_at, server_instance_id,
|
||||
metric_name, metric_type, statistic, metric_value, tags)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
tenantId, Timestamp.from(collectedAt), serverInstanceId,
|
||||
metricName, metricType, statistic, value, tags);
|
||||
}
|
||||
|
||||
private static JsonNode findByField(JsonNode array, String field, String value) {
|
||||
for (JsonNode n : array) {
|
||||
if (value.equals(n.path(field).asText())) return n;
|
||||
}
|
||||
throw new AssertionError("no element with " + field + "=" + value);
|
||||
}
|
||||
|
||||
private static JsonNode findByTag(JsonNode seriesArray, String tagKey, String tagValue) {
|
||||
for (JsonNode s : seriesArray) {
|
||||
if (tagValue.equals(s.path("tags").path(tagKey).asText())) return s;
|
||||
}
|
||||
throw new AssertionError("no series with tag " + tagKey + "=" + tagValue);
|
||||
}
|
||||
|
||||
private static java.util.List<String> asStringList(JsonNode arr) {
|
||||
java.util.List<String> out = new java.util.ArrayList<>();
|
||||
if (arr != null) for (JsonNode n : arr) out.add(n.asText());
|
||||
return out;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user