fix(alerting/B-2): implement re-notify cadence sweep and lastNotifiedAt tracking
AlertInstanceRepository gains listFiringDueForReNotify(Instant) — only returns instances where last_notified_at IS NOT NULL and cadence has elapsed (IS NULL branch excluded: sweep only re-notifies, initial notify is the dispatcher's job). AlertEvaluatorJob.sweepReNotify() runs at the end of each tick, enqueues fresh notifications for eligible instances and stamps last_notified_at. NotificationDispatchJob stamps last_notified_at on the alert_instance when a notification is DELIVERED, providing the anchor timestamp for cadence checks. PostgresAlertInstanceRepositoryIT adds listFiringDueForReNotify test covering the three-rule eligibility matrix (never-notified, long-ago, recent). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -75,12 +75,17 @@ class PostgresAlertInstanceRepositoryIT extends AbstractPostgresIT {
|
||||
|
||||
@Test
|
||||
void listForInbox_seesAllThreeTargetTypes() {
|
||||
// Each instance gets a distinct ruleId so the unique-per-open-rule index
|
||||
// (V13: alert_instances_open_rule_uq) doesn't block the second and third saves.
|
||||
UUID ruleId2 = seedRule("rule-b");
|
||||
UUID ruleId3 = seedRule("rule-c");
|
||||
|
||||
// Instance 1 — targeted at user directly
|
||||
var byUser = newInstance(ruleId, List.of(userId), List.of(), List.of());
|
||||
// Instance 2 — targeted at group
|
||||
var byGroup = newInstance(ruleId, List.of(), List.of(UUID.fromString(groupId)), List.of());
|
||||
var byGroup = newInstance(ruleId2, List.of(), List.of(UUID.fromString(groupId)), List.of());
|
||||
// Instance 3 — targeted at role
|
||||
var byRole = newInstance(ruleId, List.of(), List.of(), List.of(roleName));
|
||||
var byRole = newInstance(ruleId3, List.of(), List.of(), List.of(roleName));
|
||||
|
||||
repo.save(byUser);
|
||||
repo.save(byGroup);
|
||||
@@ -159,8 +164,9 @@ class PostgresAlertInstanceRepositoryIT extends AbstractPostgresIT {
|
||||
|
||||
@Test
|
||||
void deleteResolvedBefore_deletesOnlyResolved() {
|
||||
UUID ruleId2 = seedRule("rule-del");
|
||||
var firing = newInstance(ruleId, List.of(userId), List.of(), List.of());
|
||||
var resolved = newInstance(ruleId, List.of(userId), List.of(), List.of());
|
||||
var resolved = newInstance(ruleId2, List.of(userId), List.of(), List.of());
|
||||
repo.save(firing);
|
||||
repo.save(resolved);
|
||||
|
||||
@@ -173,6 +179,39 @@ class PostgresAlertInstanceRepositoryIT extends AbstractPostgresIT {
|
||||
assertThat(repo.findById(resolved.id())).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void listFiringDueForReNotify_returnsOnlyEligibleInstances() {
|
||||
// Each instance gets its own rule — the V13 unique partial index allows only one
|
||||
// open (PENDING/FIRING/ACKNOWLEDGED) instance per rule_id.
|
||||
UUID ruleNever = seedReNotifyRule("renotify-never");
|
||||
UUID ruleLongAgo = seedReNotifyRule("renotify-longago");
|
||||
UUID ruleRecent = seedReNotifyRule("renotify-recent");
|
||||
|
||||
// Instance 1: FIRING, never notified (last_notified_at IS NULL) → must NOT appear.
|
||||
// The sweep only re-notifies; initial notification is the dispatcher's job.
|
||||
var neverNotified = newInstance(ruleNever, List.of(userId), List.of(), List.of());
|
||||
repo.save(neverNotified);
|
||||
|
||||
// Instance 2: FIRING, notified 2 minutes ago → cadence elapsed, must appear
|
||||
var notifiedLongAgo = newInstance(ruleLongAgo, List.of(userId), List.of(), List.of());
|
||||
repo.save(notifiedLongAgo);
|
||||
jdbcTemplate.update("UPDATE alert_instances SET last_notified_at = now() - interval '2 minutes' WHERE id = ?",
|
||||
notifiedLongAgo.id());
|
||||
|
||||
// Instance 3: FIRING, notified 30 seconds ago → cadence NOT elapsed, must NOT appear
|
||||
var notifiedRecently = newInstance(ruleRecent, List.of(userId), List.of(), List.of());
|
||||
repo.save(notifiedRecently);
|
||||
jdbcTemplate.update("UPDATE alert_instances SET last_notified_at = now() - interval '30 seconds' WHERE id = ?",
|
||||
notifiedRecently.id());
|
||||
|
||||
var due = repo.listFiringDueForReNotify(Instant.now());
|
||||
assertThat(due).extracting(AlertInstance::id)
|
||||
.containsExactly(notifiedLongAgo.id())
|
||||
.doesNotContain(neverNotified.id(), notifiedRecently.id());
|
||||
|
||||
// Extra rules are cleaned up by @AfterEach via env-scoped DELETE
|
||||
}
|
||||
|
||||
@Test
|
||||
void markSilenced_togglesToTrue() {
|
||||
var inst = newInstance(ruleId, List.of(userId), List.of(), List.of());
|
||||
@@ -197,4 +236,26 @@ class PostgresAlertInstanceRepositoryIT extends AbstractPostgresIT {
|
||||
Map.of(), "title", "message",
|
||||
userIds, groupIds, roleNames);
|
||||
}
|
||||
|
||||
/** Inserts a minimal alert_rule with re_notify_minutes=0 and returns its id. */
|
||||
private UUID seedRule(String name) {
|
||||
UUID id = UUID.randomUUID();
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO alert_rules (id, environment_id, name, severity, condition_kind, condition, " +
|
||||
"notification_title_tmpl, notification_message_tmpl, created_by, updated_by) " +
|
||||
"VALUES (?, ?, ?, 'WARNING', 'AGENT_STATE', '{}'::jsonb, 't', 'm', 'sys-user', 'sys-user')",
|
||||
id, envId, name + "-" + id);
|
||||
return id;
|
||||
}
|
||||
|
||||
/** Inserts a minimal alert_rule with re_notify_minutes=1 and returns its id. */
|
||||
private UUID seedReNotifyRule(String name) {
|
||||
UUID id = UUID.randomUUID();
|
||||
jdbcTemplate.update(
|
||||
"INSERT INTO alert_rules (id, environment_id, name, severity, condition_kind, condition, " +
|
||||
"re_notify_minutes, notification_title_tmpl, notification_message_tmpl, created_by, updated_by) " +
|
||||
"VALUES (?, ?, ?, 'WARNING', 'AGENT_STATE', '{}'::jsonb, 1, 't', 'm', 'sys-user', 'sys-user')",
|
||||
id, envId, name + "-" + id);
|
||||
return id;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user