fix(alerting/B-2): implement re-notify cadence sweep and lastNotifiedAt tracking

AlertInstanceRepository gains listFiringDueForReNotify(Instant) — only returns
instances where last_notified_at IS NOT NULL and cadence has elapsed (IS NULL
branch excluded: sweep only re-notifies, initial notify is the dispatcher's job).

AlertEvaluatorJob.sweepReNotify() runs at the end of each tick, enqueues fresh
notifications for eligible instances and stamps last_notified_at.

NotificationDispatchJob stamps last_notified_at on the alert_instance when a
notification is DELIVERED, providing the anchor timestamp for cadence checks.

PostgresAlertInstanceRepositoryIT adds listFiringDueForReNotify test covering
the three-rule eligibility matrix (never-notified, long-ago, recent).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
hsiegeln
2026-04-20 08:25:50 +02:00
parent 3f036da03d
commit d74079da63
5 changed files with 137 additions and 20 deletions

View File

@@ -75,12 +75,17 @@ class PostgresAlertInstanceRepositoryIT extends AbstractPostgresIT {
@Test
void listForInbox_seesAllThreeTargetTypes() {
// Each instance gets a distinct ruleId so the unique-per-open-rule index
// (V13: alert_instances_open_rule_uq) doesn't block the second and third saves.
UUID ruleId2 = seedRule("rule-b");
UUID ruleId3 = seedRule("rule-c");
// Instance 1 — targeted at user directly
var byUser = newInstance(ruleId, List.of(userId), List.of(), List.of());
// Instance 2 — targeted at group
var byGroup = newInstance(ruleId, List.of(), List.of(UUID.fromString(groupId)), List.of());
var byGroup = newInstance(ruleId2, List.of(), List.of(UUID.fromString(groupId)), List.of());
// Instance 3 — targeted at role
var byRole = newInstance(ruleId, List.of(), List.of(), List.of(roleName));
var byRole = newInstance(ruleId3, List.of(), List.of(), List.of(roleName));
repo.save(byUser);
repo.save(byGroup);
@@ -159,8 +164,9 @@ class PostgresAlertInstanceRepositoryIT extends AbstractPostgresIT {
@Test
void deleteResolvedBefore_deletesOnlyResolved() {
UUID ruleId2 = seedRule("rule-del");
var firing = newInstance(ruleId, List.of(userId), List.of(), List.of());
var resolved = newInstance(ruleId, List.of(userId), List.of(), List.of());
var resolved = newInstance(ruleId2, List.of(userId), List.of(), List.of());
repo.save(firing);
repo.save(resolved);
@@ -173,6 +179,39 @@ class PostgresAlertInstanceRepositoryIT extends AbstractPostgresIT {
assertThat(repo.findById(resolved.id())).isEmpty();
}
@Test
void listFiringDueForReNotify_returnsOnlyEligibleInstances() {
// Each instance gets its own rule — the V13 unique partial index allows only one
// open (PENDING/FIRING/ACKNOWLEDGED) instance per rule_id.
UUID ruleNever = seedReNotifyRule("renotify-never");
UUID ruleLongAgo = seedReNotifyRule("renotify-longago");
UUID ruleRecent = seedReNotifyRule("renotify-recent");
// Instance 1: FIRING, never notified (last_notified_at IS NULL) → must NOT appear.
// The sweep only re-notifies; initial notification is the dispatcher's job.
var neverNotified = newInstance(ruleNever, List.of(userId), List.of(), List.of());
repo.save(neverNotified);
// Instance 2: FIRING, notified 2 minutes ago → cadence elapsed, must appear
var notifiedLongAgo = newInstance(ruleLongAgo, List.of(userId), List.of(), List.of());
repo.save(notifiedLongAgo);
jdbcTemplate.update("UPDATE alert_instances SET last_notified_at = now() - interval '2 minutes' WHERE id = ?",
notifiedLongAgo.id());
// Instance 3: FIRING, notified 30 seconds ago → cadence NOT elapsed, must NOT appear
var notifiedRecently = newInstance(ruleRecent, List.of(userId), List.of(), List.of());
repo.save(notifiedRecently);
jdbcTemplate.update("UPDATE alert_instances SET last_notified_at = now() - interval '30 seconds' WHERE id = ?",
notifiedRecently.id());
var due = repo.listFiringDueForReNotify(Instant.now());
assertThat(due).extracting(AlertInstance::id)
.containsExactly(notifiedLongAgo.id())
.doesNotContain(neverNotified.id(), notifiedRecently.id());
// Extra rules are cleaned up by @AfterEach via env-scoped DELETE
}
@Test
void markSilenced_togglesToTrue() {
var inst = newInstance(ruleId, List.of(userId), List.of(), List.of());
@@ -197,4 +236,26 @@ class PostgresAlertInstanceRepositoryIT extends AbstractPostgresIT {
Map.of(), "title", "message",
userIds, groupIds, roleNames);
}
/** Inserts a minimal alert_rule with re_notify_minutes=0 and returns its id. */
private UUID seedRule(String name) {
UUID id = UUID.randomUUID();
jdbcTemplate.update(
"INSERT INTO alert_rules (id, environment_id, name, severity, condition_kind, condition, " +
"notification_title_tmpl, notification_message_tmpl, created_by, updated_by) " +
"VALUES (?, ?, ?, 'WARNING', 'AGENT_STATE', '{}'::jsonb, 't', 'm', 'sys-user', 'sys-user')",
id, envId, name + "-" + id);
return id;
}
/** Inserts a minimal alert_rule with re_notify_minutes=1 and returns its id. */
private UUID seedReNotifyRule(String name) {
UUID id = UUID.randomUUID();
jdbcTemplate.update(
"INSERT INTO alert_rules (id, environment_id, name, severity, condition_kind, condition, " +
"re_notify_minutes, notification_title_tmpl, notification_message_tmpl, created_by, updated_by) " +
"VALUES (?, ?, ?, 'WARNING', 'AGENT_STATE', '{}'::jsonb, 1, 't', 'm', 'sys-user', 'sys-user')",
id, envId, name + "-" + id);
return id;
}
}