Alerting: Persist AlertInstance ResolvedAt & LastSentAt (#89135)

* Alerting: Persist AlertInstance ResolvedAt & LastSentAt

* Fix test

* Modify existing tests

* Fix merge conflicts from nullable LastSentAt & ResolvedAt

pkg/services/ngalert/models/instance.go

@@ -14,6 +14,8 @@ type AlertInstance struct {
 	CurrentStateSince time.Time
 	CurrentStateEnd   time.Time
 	LastEvalTime      time.Time
+	LastSentAt        *time.Time
+	ResolvedAt        *time.Time
 	ResultFingerprint string
 }
 

pkg/services/ngalert/models/testing.go

@@ -17,6 +17,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	alertingModels "github.com/grafana/alerting/models"
+
 	"github.com/grafana/grafana/pkg/expr"
 	"github.com/grafana/grafana/pkg/services/datasources"
 	"github.com/grafana/grafana/pkg/services/folder"
@@ -840,6 +841,11 @@ func AlertInstanceGen(mutators ...AlertInstanceMutator) *AlertInstance {
 		CurrentStateSince: currentStateSince,
 		CurrentStateEnd:   currentStateSince.Add(time.Duration(rand.Intn(100) + 200)),
 		LastEvalTime:      time.Now().Add(-time.Duration(rand.Intn(100) + 50)),
+		LastSentAt:        util.Pointer(time.Now().Add(-time.Duration(rand.Intn(100) + 50))),
+	}
+
+	if instance.CurrentState == InstanceStateNormal && rand.Intn(2) == 1 {
+		instance.ResolvedAt = util.Pointer(time.Now().Add(-time.Duration(rand.Intn(100) + 50)))
 	}
 
 	for _, mutator := range mutators {

pkg/services/ngalert/state/cache.go

@@ -365,6 +365,8 @@ func (c *cache) asInstances(skipNormalState bool) []ngModels.AlertInstance {
 					LastEvalTime:      v2.LastEvaluationTime,
 					CurrentStateSince: v2.StartsAt,
 					CurrentStateEnd:   v2.EndsAt,
+					ResolvedAt:        v2.ResolvedAt,
+					LastSentAt:        v2.LastSentAt,
 					ResultFingerprint: v2.ResultFingerprint.String(),
 				})
 			}

pkg/services/ngalert/state/manager.go

@@ -215,6 +215,8 @@ func (st *Manager) Warm(ctx context.Context, rulesReader RuleReader) {
 				LastEvaluationTime:   entry.LastEvalTime,
 				Annotations:          ruleForEntry.Annotations,
 				ResultFingerprint:    resultFp,
+				ResolvedAt:           entry.ResolvedAt,
+				LastSentAt:           entry.LastSentAt,
 			}
 			statesCount++
 		}

pkg/services/ngalert/state/manager_test.go

@@ -62,6 +62,8 @@ func TestWarmStateCache(t *testing.T) {
 			StartsAt:           evaluationTime.Add(-1 * time.Minute),
 			EndsAt:             evaluationTime.Add(1 * time.Minute),
 			LastEvaluationTime: evaluationTime,
+			LastSentAt:         util.Pointer(evaluationTime),
+			ResolvedAt:         util.Pointer(evaluationTime),
 			Annotations:        map[string]string{"testAnnoKey": "testAnnoValue"},
 			ResultFingerprint:  data.Fingerprint(math.MaxUint64),
 		}, {
@@ -73,6 +75,8 @@ func TestWarmStateCache(t *testing.T) {
 			StartsAt:           evaluationTime.Add(-1 * time.Minute),
 			EndsAt:             evaluationTime.Add(1 * time.Minute),
 			LastEvaluationTime: evaluationTime,
+			LastSentAt:         util.Pointer(evaluationTime.Add(-1 * time.Minute)),
+			ResolvedAt:         nil,
 			Annotations:        map[string]string{"testAnnoKey": "testAnnoValue"},
 			ResultFingerprint:  data.Fingerprint(math.MaxUint64 - 1),
 		},
@@ -85,6 +89,8 @@ func TestWarmStateCache(t *testing.T) {
 			StartsAt:           evaluationTime.Add(-1 * time.Minute),
 			EndsAt:             evaluationTime.Add(1 * time.Minute),
 			LastEvaluationTime: evaluationTime,
+			LastSentAt:         util.Pointer(evaluationTime.Add(-1 * time.Minute)),
+			ResolvedAt:         nil,
 			Annotations:        map[string]string{"testAnnoKey": "testAnnoValue"},
 			ResultFingerprint:  data.Fingerprint(0),
 		},
@@ -97,6 +103,8 @@ func TestWarmStateCache(t *testing.T) {
 			StartsAt:           evaluationTime.Add(-1 * time.Minute),
 			EndsAt:             evaluationTime.Add(1 * time.Minute),
 			LastEvaluationTime: evaluationTime,
+			LastSentAt:         util.Pointer(evaluationTime.Add(-1 * time.Minute)),
+			ResolvedAt:         nil,
 			Annotations:        map[string]string{"testAnnoKey": "testAnnoValue"},
 			ResultFingerprint:  data.Fingerprint(1),
 		},
@@ -109,6 +117,8 @@ func TestWarmStateCache(t *testing.T) {
 			StartsAt:           evaluationTime.Add(-1 * time.Minute),
 			EndsAt:             evaluationTime.Add(1 * time.Minute),
 			LastEvaluationTime: evaluationTime,
+			LastSentAt:         nil,
+			ResolvedAt:         nil,
 			Annotations:        map[string]string{"testAnnoKey": "testAnnoValue"},
 			ResultFingerprint:  data.Fingerprint(2),
 		},
@@ -128,6 +138,8 @@ func TestWarmStateCache(t *testing.T) {
 		LastEvalTime:      evaluationTime,
 		CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
 		CurrentStateEnd:   evaluationTime.Add(1 * time.Minute),
+		LastSentAt:        &evaluationTime,
+		ResolvedAt:        &evaluationTime,
 		Labels:            labels,
 		ResultFingerprint: data.Fingerprint(math.MaxUint64).String(),
 	})
@@ -144,6 +156,8 @@ func TestWarmStateCache(t *testing.T) {
 		LastEvalTime:      evaluationTime,
 		CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
 		CurrentStateEnd:   evaluationTime.Add(1 * time.Minute),
+		LastSentAt:        util.Pointer(evaluationTime.Add(-1 * time.Minute)),
+		ResolvedAt:        nil,
 		Labels:            labels,
 		ResultFingerprint: data.Fingerprint(math.MaxUint64 - 1).String(),
 	})
@@ -160,6 +174,8 @@ func TestWarmStateCache(t *testing.T) {
 		LastEvalTime:      evaluationTime,
 		CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
 		CurrentStateEnd:   evaluationTime.Add(1 * time.Minute),
+		LastSentAt:        util.Pointer(evaluationTime.Add(-1 * time.Minute)),
+		ResolvedAt:        nil,
 		Labels:            labels,
 		ResultFingerprint: data.Fingerprint(0).String(),
 	})
@@ -176,6 +192,8 @@ func TestWarmStateCache(t *testing.T) {
 		LastEvalTime:      evaluationTime,
 		CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
 		CurrentStateEnd:   evaluationTime.Add(1 * time.Minute),
+		LastSentAt:        util.Pointer(evaluationTime.Add(-1 * time.Minute)),
+		ResolvedAt:        nil,
 		Labels:            labels,
 		ResultFingerprint: data.Fingerprint(1).String(),
 	})
@@ -192,6 +210,8 @@ func TestWarmStateCache(t *testing.T) {
 		LastEvalTime:      evaluationTime,
 		CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
 		CurrentStateEnd:   evaluationTime.Add(1 * time.Minute),
+		LastSentAt:        nil,
+		ResolvedAt:        nil,
 		Labels:            labels,
 		ResultFingerprint: data.Fingerprint(2).String(),
 	})
@@ -1656,7 +1676,7 @@ func printAllAnnotations(annos map[int64]annotations.Item) string {
 }
 
 func TestStaleResultsHandler(t *testing.T) {
-	evaluationTime := time.Now()
+	evaluationTime := time.Now().Truncate(time.Second).UTC() // Truncate to the second since we don't store sub-second precision.
 	interval := time.Minute
 
 	ctx := context.Background()
@@ -1666,9 +1686,19 @@ func TestStaleResultsHandler(t *testing.T) {
 	rule := tests.CreateTestAlertRule(t, ctx, dbstore, int64(interval.Seconds()), mainOrgID)
 	lastEval := evaluationTime.Add(-2 * interval)
 
-	labels1 := models.InstanceLabels{"test1": "testValue1"}
+	labels1 := models.InstanceLabels{
+		"__alert_rule_namespace_uid__": "namespace",
+		"__alert_rule_uid__":           rule.UID,
+		"alertname":                    rule.Title,
+		"test1":                        "testValue1",
+	}
 	_, hash1, _ := labels1.StringAndHash()
-	labels2 := models.InstanceLabels{"test2": "testValue2"}
+	labels2 := models.InstanceLabels{
+		"__alert_rule_namespace_uid__": "namespace",
+		"__alert_rule_uid__":           rule.UID,
+		"alertname":                    rule.Title,
+		"test2":                        "testValue2",
+	}
 	_, hash2, _ := labels2.StringAndHash()
 	instances := []models.AlertInstance{
 		{
@@ -1682,6 +1712,9 @@ func TestStaleResultsHandler(t *testing.T) {
 			LastEvalTime:      lastEval,
 			CurrentStateSince: lastEval,
 			CurrentStateEnd:   lastEval.Add(3 * interval),
+			LastSentAt:        &lastEval,
+			ResolvedAt:        &lastEval,
+			ResultFingerprint: data.Labels{"test1": "testValue1"}.Fingerprint().String(),
 		},
 		{
 			AlertInstanceKey: models.AlertInstanceKey{
@@ -1694,6 +1727,9 @@ func TestStaleResultsHandler(t *testing.T) {
 			LastEvalTime:      lastEval,
 			CurrentStateSince: lastEval,
 			CurrentStateEnd:   lastEval.Add(3 * interval),
+			LastSentAt:        &lastEval,
+			ResolvedAt:        nil,
+			ResultFingerprint: data.Labels{"test2": "testValue2"}.Fingerprint().String(),
 		},
 	}
 
@@ -1723,23 +1759,20 @@ func TestStaleResultsHandler(t *testing.T) {
 				{
 					AlertRuleUID: rule.UID,
 					OrgID:        1,
-					Labels: data.Labels{
+					Labels:       data.Labels(labels1),
-						"__alert_rule_namespace_uid__": "namespace",
+					Values:       make(map[string]float64),
-						"__alert_rule_uid__":           rule.UID,
+					State:        eval.Normal,
-						"alertname":                    rule.Title,
-						"test1":                        "testValue1",
-					},
-					Values: make(map[string]float64),
-					State:  eval.Normal,
 					LatestResult: &state.Evaluation{
 						EvaluationTime:  evaluationTime,
 						EvaluationState: eval.Normal,
 						Values:          make(map[string]float64),
 						Condition:       "A",
 					},
-					StartsAt:           evaluationTime,
+					StartsAt:           lastEval,
-					EndsAt:             evaluationTime,
+					EndsAt:             lastEval.Add(3 * interval),
 					LastEvaluationTime: evaluationTime,
+					LastSentAt:         &lastEval,
+					ResolvedAt:         &lastEval,
 					EvaluationDuration: 0,
 					Annotations:        map[string]string{"testAnnoKey": "testAnnoValue"},
 					ResultFingerprint:  data.Labels{"test1": "testValue1"}.Fingerprint(),

pkg/services/ngalert/state/persister_sync.go

@@ -102,6 +102,8 @@ func (a *SyncStatePersister) saveAlertStates(ctx context.Context, states ...Stat
 			LastEvalTime:      s.LastEvaluationTime,
 			CurrentStateSince: s.StartsAt,
 			CurrentStateEnd:   s.EndsAt,
+			ResolvedAt:        s.ResolvedAt,
+			LastSentAt:        s.LastSentAt,
 		}
 
 		err = a.store.SaveAlertInstance(ctx, instance)

pkg/services/ngalert/store/instance_database.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
@@ -55,12 +56,25 @@ func (st DBstore) SaveAlertInstance(ctx context.Context, alertInstance models.Al
 		if err != nil {
 			return err
 		}
-		params := append(make([]any, 0), alertInstance.RuleOrgID, alertInstance.RuleUID, labelTupleJSON, alertInstance.LabelsHash, alertInstance.CurrentState, alertInstance.CurrentReason, alertInstance.CurrentStateSince.Unix(), alertInstance.CurrentStateEnd.Unix(), alertInstance.LastEvalTime.Unix(), alertInstance.ResultFingerprint)
+		params := append(make([]any, 0),
+			alertInstance.RuleOrgID,
+			alertInstance.RuleUID,
+			labelTupleJSON,
+			alertInstance.LabelsHash,
+			alertInstance.CurrentState,
+			alertInstance.CurrentReason,
+			alertInstance.CurrentStateSince.Unix(),
+			alertInstance.CurrentStateEnd.Unix(),
+			alertInstance.LastEvalTime.Unix(),
+			nullableTimeToUnix(alertInstance.ResolvedAt),
+			nullableTimeToUnix(alertInstance.LastSentAt),
+			alertInstance.ResultFingerprint,
+		)
 
 		upsertSQL := st.SQLStore.GetDialect().UpsertSQL(
 			"alert_instance",
 			[]string{"rule_org_id", "rule_uid", "labels_hash"},
-			[]string{"rule_org_id", "rule_uid", "labels", "labels_hash", "current_state", "current_reason", "current_state_since", "current_state_end", "last_eval_time", "result_fingerprint"})
+			[]string{"rule_org_id", "rule_uid", "labels", "labels_hash", "current_state", "current_reason", "current_state_since", "current_state_end", "last_eval_time", "resolved_at", "last_sent_at", "result_fingerprint"})
 		_, err = sess.SQL(upsertSQL, params...).Query()
 		if err != nil {
 			return err
@@ -219,8 +233,20 @@ func (st DBstore) FullSync(ctx context.Context, instances []models.AlertInstance
 				continue
 			}
 
-			_, err = sess.Exec("INSERT INTO alert_instance (rule_org_id, rule_uid, labels, labels_hash, current_state, current_reason, current_state_since, current_state_end, last_eval_time) VALUES (?,?,?,?,?,?,?,?,?)",
+			_, err = sess.Exec(
-				alertInstance.RuleOrgID, alertInstance.RuleUID, labelTupleJSON, alertInstance.LabelsHash, alertInstance.CurrentState, alertInstance.CurrentReason, alertInstance.CurrentStateSince.Unix(), alertInstance.CurrentStateEnd.Unix(), alertInstance.LastEvalTime.Unix())
+				"INSERT INTO alert_instance (rule_org_id, rule_uid, labels, labels_hash, current_state, current_reason, current_state_since, current_state_end, last_eval_time, resolved_at, last_sent_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)",
+				alertInstance.RuleOrgID,
+				alertInstance.RuleUID,
+				labelTupleJSON,
+				alertInstance.LabelsHash,
+				alertInstance.CurrentState,
+				alertInstance.CurrentReason,
+				alertInstance.CurrentStateSince.Unix(),
+				alertInstance.CurrentStateEnd.Unix(),
+				alertInstance.LastEvalTime.Unix(),
+				nullableTimeToUnix(alertInstance.ResolvedAt),
+				nullableTimeToUnix(alertInstance.LastSentAt),
+			)
 			if err != nil {
 				return fmt.Errorf("failed to insert into alert_instance table: %w", err)
 			}
@@ -231,3 +257,12 @@ func (st DBstore) FullSync(ctx context.Context, instances []models.AlertInstance
 		return nil
 	})
 }
+
+// nullableTimeToUnix converts a nullable time.Time to nil, if it is nil, otherwise it converts the time.Time to a unix timestamp.
+func nullableTimeToUnix(t *time.Time) *int64 {
+	if t == nil {
+		return nil
+	}
+	unix := t.Unix()
+	return &unix
+}

pkg/services/ngalert/store/instance_database_test.go

@@ -386,6 +386,8 @@ func generateTestAlertInstance(orgID int64, ruleID string) models.AlertInstance
 		CurrentStateEnd:   time.Now(),
 		CurrentStateSince: time.Now(),
 		LastEvalTime:      time.Now(),
+		LastSentAt:        util.Pointer(time.Now()),
+		ResolvedAt:        util.Pointer(time.Now()),
 		CurrentReason:     "abc",
 	}
 }

pkg/services/sqlstore/migrations/migrations.go

@@ -123,6 +123,8 @@ func (oss *OSSMigrations) AddMigration(mg *Migrator) {
 	accesscontrol.AddManagedFolderAlertingSilencesActionsMigrator(mg)
 
 	ualert.AddRecordingRuleColumns(mg)
+
+	ualert.AddStateResolvedAtColumns(mg)
 }
 
 func addStarMigrations(mg *Migrator) {

pkg/services/sqlstore/migrations/ualert/state_resolvedat_mig.go

@@ -0,0 +1,18 @@
+package ualert
+
+import "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
+
+// AddStateResolvedAtColumns adds columns to alert_instance to represent ResolvedAt and LastSentAt.
+func AddStateResolvedAtColumns(mg *migrator.Migrator) {
+	mg.AddMigration("add resolved_at column to alert_instance table", migrator.NewAddColumnMigration(migrator.Table{Name: "alert_instance"}, &migrator.Column{
+		Name:     "resolved_at",
+		Type:     migrator.DB_BigInt, // BigInt, to match existing time fields.
+		Nullable: true,
+	}))
+
+	mg.AddMigration("add last_sent_at column to alert_instance table", migrator.NewAddColumnMigration(migrator.Table{Name: "alert_instance"}, &migrator.Column{
+		Name:     "last_sent_at",
+		Type:     migrator.DB_BigInt,
+		Nullable: true,
+	}))
+}