feat(alerting): WebhookDispatcher with HMAC + TLS + retry classification
Renders URL/headers/body with Mustache, optionally HMAC-signs the body (X-Cameleer-Signature), supports POST/PUT/PATCH, classifies 2xx/4xx/5xx into DELIVERED/FAILED/retry. 8 WireMock-backed IT tests including HTTPS TRUST_ALL against WireMock self-signed cert. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
hsiegeln
@@ -0,0 +1,213 @@
|
||||
package com.cameleer.server.app.alerting.notify;
|
||||
|
||||
import com.cameleer.server.app.alerting.config.AlertingProperties;
|
||||
import com.cameleer.server.app.outbound.crypto.SecretCipher;
|
||||
import com.cameleer.server.core.alerting.AlertInstance;
|
||||
import com.cameleer.server.core.alerting.AlertNotification;
|
||||
import com.cameleer.server.core.alerting.AlertRule;
|
||||
import com.cameleer.server.core.alerting.NotificationStatus;
|
||||
import com.cameleer.server.core.alerting.WebhookBinding;
|
||||
import com.cameleer.server.core.http.OutboundHttpClientFactory;
|
||||
import com.cameleer.server.core.http.OutboundHttpRequestContext;
|
||||
import com.cameleer.server.core.outbound.OutboundConnection;
|
||||
import com.cameleer.server.core.outbound.OutboundMethod;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.apache.hc.client5.http.classic.methods.HttpPatch;
|
||||
import org.apache.hc.client5.http.classic.methods.HttpPost;
|
||||
import org.apache.hc.client5.http.classic.methods.HttpPut;
|
||||
import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase;
|
||||
import org.apache.hc.core5.http.io.entity.EntityUtils;
|
||||
import org.apache.hc.core5.http.io.entity.StringEntity;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Renders, signs, and dispatches webhook notifications over HTTP.
|
||||
* <p>
|
||||
* Classification:
|
||||
* <ul>
|
||||
* <li>2xx → {@link NotificationStatus#DELIVERED}</li>
|
||||
* <li>4xx → {@link NotificationStatus#FAILED} (retry won't help)</li>
|
||||
* <li>5xx / network / timeout → {@code null} status (caller retries up to max attempts)</li>
|
||||
* </ul>
|
||||
*/
|
||||
@Component
|
||||
public class WebhookDispatcher {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(WebhookDispatcher.class);
|
||||
|
||||
/** baseDelay that callers multiply by attempt count: 30s, 60s, 90s, … */
|
||||
static final Duration BASE_RETRY_DELAY = Duration.ofSeconds(30);
|
||||
|
||||
private static final int SNIPPET_LIMIT = 512;
|
||||
private static final String DEFAULT_CONTENT_TYPE = "application/json";
|
||||
|
||||
private final OutboundHttpClientFactory clientFactory;
|
||||
private final SecretCipher secretCipher;
|
||||
private final MustacheRenderer renderer;
|
||||
private final AlertingProperties props;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public WebhookDispatcher(OutboundHttpClientFactory clientFactory,
|
||||
SecretCipher secretCipher,
|
||||
MustacheRenderer renderer,
|
||||
AlertingProperties props,
|
||||
ObjectMapper objectMapper) {
|
||||
this.clientFactory = clientFactory;
|
||||
this.secretCipher = secretCipher;
|
||||
this.renderer = renderer;
|
||||
this.props = props;
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
public record Outcome(
|
||||
NotificationStatus status,
|
||||
int httpStatus,
|
||||
String snippet,
|
||||
Duration retryAfter) {}
|
||||
|
||||
/**
|
||||
* Dispatch a single webhook notification.
|
||||
*
|
||||
* @param notif the outbox record (contains webhookId used to find per-rule overrides)
|
||||
* @param rule the alert rule (may be null when rule was deleted)
|
||||
* @param instance the alert instance
|
||||
* @param conn the resolved outbound connection
|
||||
* @param context the Mustache rendering context
|
||||
*/
|
||||
public Outcome dispatch(AlertNotification notif,
|
||||
AlertRule rule,
|
||||
AlertInstance instance,
|
||||
OutboundConnection conn,
|
||||
Map<String, Object> context) {
|
||||
try {
|
||||
// 1. Determine per-binding overrides
|
||||
WebhookBinding binding = findBinding(rule, notif);
|
||||
|
||||
// 2. Render URL
|
||||
String url = renderer.render(conn.url(), context);
|
||||
|
||||
// 3. Build body
|
||||
String body = buildBody(conn, binding, context);
|
||||
|
||||
// 4. Build headers
|
||||
Map<String, String> headers = buildHeaders(conn, binding, context);
|
||||
|
||||
// 5. HMAC sign if configured
|
||||
if (conn.hmacSecretCiphertext() != null) {
|
||||
String secret = secretCipher.decrypt(conn.hmacSecretCiphertext());
|
||||
String sig = new HmacSigner().sign(secret, body.getBytes(StandardCharsets.UTF_8));
|
||||
headers.put("X-Cameleer-Signature", sig);
|
||||
}
|
||||
|
||||
// 6. Build HTTP request
|
||||
Duration timeout = Duration.ofMillis(props.effectiveWebhookTimeoutMs());
|
||||
OutboundHttpRequestContext ctx = new OutboundHttpRequestContext(
|
||||
conn.tlsTrustMode(), conn.tlsCaPemPaths(), timeout, timeout);
|
||||
|
||||
var client = clientFactory.clientFor(ctx);
|
||||
HttpUriRequestBase request = buildRequest(conn.method(), url);
|
||||
for (var e : headers.entrySet()) {
|
||||
request.setHeader(e.getKey(), e.getValue());
|
||||
}
|
||||
request.setEntity(new StringEntity(body, StandardCharsets.UTF_8));
|
||||
|
||||
// 7. Execute and classify
|
||||
try (var response = client.execute(request)) {
|
||||
int code = response.getCode();
|
||||
String snippet = snippet(response.getEntity() != null
|
||||
? EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)
|
||||
: "");
|
||||
|
||||
if (code >= 200 && code < 300) {
|
||||
return new Outcome(NotificationStatus.DELIVERED, code, snippet, null);
|
||||
} else if (code >= 400 && code < 500) {
|
||||
return new Outcome(NotificationStatus.FAILED, code, snippet, null);
|
||||
} else {
|
||||
return new Outcome(null, code, snippet, BASE_RETRY_DELAY);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
log.warn("WebhookDispatcher: network/timeout error dispatching notification {}: {}",
|
||||
notif.id(), e.getMessage());
|
||||
return new Outcome(null, 0, snippet(e.getMessage()), BASE_RETRY_DELAY);
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
private WebhookBinding findBinding(AlertRule rule, AlertNotification notif) {
|
||||
if (rule == null || notif.webhookId() == null) return null;
|
||||
return rule.webhooks().stream()
|
||||
.filter(w -> w.id().equals(notif.webhookId()))
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
}
|
||||
|
||||
private String buildBody(OutboundConnection conn, WebhookBinding binding, Map<String, Object> context) {
|
||||
// Priority: per-binding override > connection default > built-in JSON envelope
|
||||
String tmpl = null;
|
||||
if (binding != null && binding.bodyOverride() != null) {
|
||||
tmpl = binding.bodyOverride();
|
||||
} else if (conn.defaultBodyTmpl() != null) {
|
||||
tmpl = conn.defaultBodyTmpl();
|
||||
}
|
||||
|
||||
if (tmpl != null) {
|
||||
return renderer.render(tmpl, context);
|
||||
}
|
||||
|
||||
// Built-in default: serialize the entire context map as JSON
|
||||
try {
|
||||
return objectMapper.writeValueAsString(context);
|
||||
} catch (Exception e) {
|
||||
log.warn("WebhookDispatcher: failed to serialize context as JSON, using empty object", e);
|
||||
return "{}";
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, String> buildHeaders(OutboundConnection conn, WebhookBinding binding,
|
||||
Map<String, Object> context) {
|
||||
Map<String, String> headers = new LinkedHashMap<>();
|
||||
|
||||
// Default content-type
|
||||
headers.put("Content-Type", DEFAULT_CONTENT_TYPE);
|
||||
|
||||
// Connection-level default headers (keys are literal, values are Mustache-rendered)
|
||||
for (var e : conn.defaultHeaders().entrySet()) {
|
||||
headers.put(e.getKey(), renderer.render(e.getValue(), context));
|
||||
}
|
||||
|
||||
// Per-binding overrides (also Mustache-rendered values)
|
||||
if (binding != null) {
|
||||
for (var e : binding.headerOverrides().entrySet()) {
|
||||
headers.put(e.getKey(), renderer.render(e.getValue(), context));
|
||||
}
|
||||
}
|
||||
|
||||
return headers;
|
||||
}
|
||||
|
||||
private HttpUriRequestBase buildRequest(OutboundMethod method, String url) {
|
||||
if (method == null) method = OutboundMethod.POST;
|
||||
return switch (method) {
|
||||
case PUT -> new HttpPut(url);
|
||||
case PATCH -> new HttpPatch(url);
|
||||
default -> new HttpPost(url);
|
||||
};
|
||||
}
|
||||
|
||||
private String snippet(String text) {
|
||||
if (text == null) return "";
|
||||
return text.length() <= SNIPPET_LIMIT ? text : text.substring(0, SNIPPET_LIMIT);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
package com.cameleer.server.app.alerting.notify;
|
||||
|
||||
import com.cameleer.server.app.alerting.config.AlertingProperties;
|
||||
import com.cameleer.server.app.http.ApacheOutboundHttpClientFactory;
|
||||
import com.cameleer.server.app.http.SslContextBuilder;
|
||||
import com.cameleer.server.app.outbound.crypto.SecretCipher;
|
||||
import com.cameleer.server.core.alerting.*;
|
||||
import com.cameleer.server.core.http.OutboundHttpProperties;
|
||||
import com.cameleer.server.core.http.TrustMode;
|
||||
import com.cameleer.server.core.outbound.OutboundAuth;
|
||||
import com.cameleer.server.core.outbound.OutboundConnection;
|
||||
import com.cameleer.server.core.outbound.OutboundMethod;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.github.tomakehurst.wiremock.WireMockServer;
|
||||
import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
import static com.github.tomakehurst.wiremock.client.WireMock.*;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* WireMock-backed integration tests for {@link WebhookDispatcher}.
|
||||
* Each test spins its own WireMock server (HTTP on random port, or HTTPS for TLS test).
|
||||
*/
|
||||
class WebhookDispatcherIT {
|
||||
|
||||
private static final String JWT_SECRET = "very-secret-jwt-key-for-test-only-32chars";
|
||||
|
||||
private WireMockServer wm;
|
||||
private WebhookDispatcher dispatcher;
|
||||
private SecretCipher cipher;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
wm = new WireMockServer(WireMockConfiguration.options().dynamicPort());
|
||||
wm.start();
|
||||
|
||||
OutboundHttpProperties props = new OutboundHttpProperties(
|
||||
false, List.of(), Duration.ofSeconds(2), Duration.ofSeconds(5), null, null, null);
|
||||
cipher = new SecretCipher(JWT_SECRET);
|
||||
dispatcher = new WebhookDispatcher(
|
||||
new ApacheOutboundHttpClientFactory(props, new SslContextBuilder()),
|
||||
cipher,
|
||||
new MustacheRenderer(),
|
||||
new AlertingProperties(null, null, null, null, null, null, null, null, null, null, null, null, null),
|
||||
new ObjectMapper()
|
||||
);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
if (wm != null) wm.stop();
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Tests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
@Test
|
||||
void twoHundredRespond_isDelivered() {
|
||||
wm.stubFor(post("/webhook").willReturn(aResponse().withStatus(200).withBody("accepted")));
|
||||
|
||||
var outcome = dispatcher.dispatch(
|
||||
notif(null), null, instance(), conn(wm.port(), OutboundMethod.POST, null, Map.of(), null), ctx());
|
||||
|
||||
assertThat(outcome.status()).isEqualTo(NotificationStatus.DELIVERED);
|
||||
assertThat(outcome.httpStatus()).isEqualTo(200);
|
||||
assertThat(outcome.snippet()).isEqualTo("accepted");
|
||||
assertThat(outcome.retryAfter()).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void fourOhFour_isFailedImmediately() {
|
||||
wm.stubFor(post("/webhook").willReturn(aResponse().withStatus(404).withBody("not found")));
|
||||
|
||||
var outcome = dispatcher.dispatch(
|
||||
notif(null), null, instance(), conn(wm.port(), OutboundMethod.POST, null, Map.of(), null), ctx());
|
||||
|
||||
assertThat(outcome.status()).isEqualTo(NotificationStatus.FAILED);
|
||||
assertThat(outcome.httpStatus()).isEqualTo(404);
|
||||
assertThat(outcome.retryAfter()).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void fiveOhThree_hasNullStatusAndRetryDelay() {
|
||||
wm.stubFor(post("/webhook").willReturn(aResponse().withStatus(503).withBody("unavailable")));
|
||||
|
||||
var outcome = dispatcher.dispatch(
|
||||
notif(null), null, instance(), conn(wm.port(), OutboundMethod.POST, null, Map.of(), null), ctx());
|
||||
|
||||
assertThat(outcome.status()).isNull();
|
||||
assertThat(outcome.httpStatus()).isEqualTo(503);
|
||||
assertThat(outcome.retryAfter()).isEqualTo(Duration.ofSeconds(30));
|
||||
}
|
||||
|
||||
@Test
|
||||
void hmacHeader_presentWhenSecretSet() {
|
||||
wm.stubFor(post("/webhook").willReturn(ok("ok")));
|
||||
|
||||
// Encrypt a test secret
|
||||
String ciphertext = cipher.encrypt("my-signing-secret");
|
||||
var outcome = dispatcher.dispatch(
|
||||
notif(null), null, instance(), conn(wm.port(), OutboundMethod.POST, ciphertext, Map.of(), null), ctx());
|
||||
|
||||
assertThat(outcome.status()).isEqualTo(NotificationStatus.DELIVERED);
|
||||
wm.verify(postRequestedFor(urlEqualTo("/webhook"))
|
||||
.withHeader("X-Cameleer-Signature", matching("sha256=[0-9a-f]{64}")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void hmacHeader_absentWhenNoSecret() {
|
||||
wm.stubFor(post("/webhook").willReturn(ok("ok")));
|
||||
|
||||
dispatcher.dispatch(
|
||||
notif(null), null, instance(), conn(wm.port(), OutboundMethod.POST, null, Map.of(), null), ctx());
|
||||
|
||||
wm.verify(postRequestedFor(urlEqualTo("/webhook"))
|
||||
.withoutHeader("X-Cameleer-Signature"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void putMethod_isRespected() {
|
||||
wm.stubFor(put("/webhook").willReturn(ok("ok")));
|
||||
|
||||
var outcome = dispatcher.dispatch(
|
||||
notif(null), null, instance(), conn(wm.port(), OutboundMethod.PUT, null, Map.of(), null), ctx());
|
||||
|
||||
assertThat(outcome.status()).isEqualTo(NotificationStatus.DELIVERED);
|
||||
wm.verify(putRequestedFor(urlEqualTo("/webhook")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void customHeaderRenderedWithMustache() {
|
||||
wm.stubFor(post("/webhook").willReturn(ok("ok")));
|
||||
|
||||
// "{{env.slug}}" in the defaultHeaders value should resolve to "dev" from context
|
||||
var headers = Map.of("X-Env", "{{env.slug}}");
|
||||
var outcome = dispatcher.dispatch(
|
||||
notif(null), null, instance(),
|
||||
conn(wm.port(), OutboundMethod.POST, null, headers, null),
|
||||
ctxWithEnv("dev"));
|
||||
|
||||
assertThat(outcome.status()).isEqualTo(NotificationStatus.DELIVERED);
|
||||
wm.verify(postRequestedFor(urlEqualTo("/webhook"))
|
||||
.withHeader("X-Env", equalTo("dev")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void tlsTrustAll_worksAgainstSelfSignedCert() throws Exception {
|
||||
// Separate WireMock instance with HTTPS only
|
||||
WireMockServer wmHttps = new WireMockServer(
|
||||
WireMockConfiguration.options().httpDisabled(true).dynamicHttpsPort());
|
||||
wmHttps.start();
|
||||
wmHttps.stubFor(post("/webhook").willReturn(ok("secure-ok")));
|
||||
|
||||
try {
|
||||
// Connection with TRUST_ALL so the self-signed cert is accepted
|
||||
var conn = connHttps(wmHttps.httpsPort(), OutboundMethod.POST, null, Map.of());
|
||||
var outcome = dispatcher.dispatch(notif(null), null, instance(), conn, ctx());
|
||||
assertThat(outcome.status()).isEqualTo(NotificationStatus.DELIVERED);
|
||||
assertThat(outcome.snippet()).isEqualTo("secure-ok");
|
||||
} finally {
|
||||
wmHttps.stop();
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Builders
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
private AlertNotification notif(UUID webhookId) {
|
||||
return new AlertNotification(
|
||||
UUID.randomUUID(), UUID.randomUUID(),
|
||||
webhookId, UUID.randomUUID(),
|
||||
NotificationStatus.PENDING, 0, Instant.now(),
|
||||
null, null, null, null, Map.of(), null, Instant.now());
|
||||
}
|
||||
|
||||
private AlertInstance instance() {
|
||||
return new AlertInstance(
|
||||
UUID.randomUUID(), UUID.randomUUID(), Map.of(),
|
||||
UUID.randomUUID(), AlertState.FIRING, AlertSeverity.WARNING,
|
||||
Instant.now(), null, null, null, null, false,
|
||||
null, null, Map.of(), "Alert", "Message",
|
||||
List.of(), List.of(), List.of());
|
||||
}
|
||||
|
||||
private OutboundConnection conn(int port, OutboundMethod method, String hmacCiphertext,
|
||||
Map<String, String> defaultHeaders, String bodyTmpl) {
|
||||
return new OutboundConnection(
|
||||
UUID.randomUUID(), "default", "test-conn", null,
|
||||
"http://localhost:" + port + "/webhook",
|
||||
method, defaultHeaders, bodyTmpl,
|
||||
TrustMode.SYSTEM_DEFAULT, List.of(),
|
||||
hmacCiphertext, new OutboundAuth.None(),
|
||||
List.of(), Instant.now(), "system", Instant.now(), "system");
|
||||
}
|
||||
|
||||
private OutboundConnection connHttps(int port, OutboundMethod method, String hmacCiphertext,
|
||||
Map<String, String> defaultHeaders) {
|
||||
return new OutboundConnection(
|
||||
UUID.randomUUID(), "default", "test-conn-https", null,
|
||||
"https://localhost:" + port + "/webhook",
|
||||
method, defaultHeaders, null,
|
||||
TrustMode.TRUST_ALL, List.of(),
|
||||
hmacCiphertext, new OutboundAuth.None(),
|
||||
List.of(), Instant.now(), "system", Instant.now(), "system");
|
||||
}
|
||||
|
||||
private Map<String, Object> ctx() {
|
||||
return Map.of(
|
||||
"env", Map.of("slug", "prod", "id", UUID.randomUUID().toString()),
|
||||
"rule", Map.of("name", "test-rule", "severity", "WARNING", "id", UUID.randomUUID().toString(), "description", ""),
|
||||
"alert", Map.of("id", UUID.randomUUID().toString(), "state", "FIRING", "firedAt", Instant.now().toString(),
|
||||
"resolvedAt", "", "ackedBy", "", "link", "/alerts/inbox/x", "currentValue", "", "threshold", "")
|
||||
);
|
||||
}
|
||||
|
||||
private Map<String, Object> ctxWithEnv(String envSlug) {
|
||||
return Map.of(
|
||||
"env", Map.of("slug", envSlug, "id", UUID.randomUUID().toString()),
|
||||
"rule", Map.of("name", "test-rule", "severity", "WARNING", "id", UUID.randomUUID().toString(), "description", ""),
|
||||
"alert", Map.of("id", UUID.randomUUID().toString(), "state", "FIRING", "firedAt", Instant.now().toString(),
|
||||
"resolvedAt", "", "ackedBy", "", "link", "/alerts/inbox/x", "currentValue", "", "threshold", "")
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user