diff --git a/pkg/services/ngalert/models/instance.go b/pkg/services/ngalert/models/instance.go index ea731dd950b..d2537ba7f03 100644 --- a/pkg/services/ngalert/models/instance.go +++ b/pkg/services/ngalert/models/instance.go @@ -14,6 +14,8 @@ type AlertInstance struct { CurrentStateSince time.Time CurrentStateEnd time.Time LastEvalTime time.Time + LastSentAt *time.Time + ResolvedAt *time.Time ResultFingerprint string } diff --git a/pkg/services/ngalert/models/testing.go b/pkg/services/ngalert/models/testing.go index 8dfaa17807b..657d0546644 100644 --- a/pkg/services/ngalert/models/testing.go +++ b/pkg/services/ngalert/models/testing.go @@ -17,6 +17,7 @@ import ( "github.com/stretchr/testify/require" alertingModels "github.com/grafana/alerting/models" + "github.com/grafana/grafana/pkg/expr" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/folder" @@ -840,6 +841,11 @@ func AlertInstanceGen(mutators ...AlertInstanceMutator) *AlertInstance { CurrentStateSince: currentStateSince, CurrentStateEnd: currentStateSince.Add(time.Duration(rand.Intn(100) + 200)), LastEvalTime: time.Now().Add(-time.Duration(rand.Intn(100) + 50)), + LastSentAt: util.Pointer(time.Now().Add(-time.Duration(rand.Intn(100) + 50))), + } + + if instance.CurrentState == InstanceStateNormal && rand.Intn(2) == 1 { + instance.ResolvedAt = util.Pointer(time.Now().Add(-time.Duration(rand.Intn(100) + 50))) } for _, mutator := range mutators { diff --git a/pkg/services/ngalert/state/cache.go b/pkg/services/ngalert/state/cache.go index 4799e6ab127..99f0dcb885f 100644 --- a/pkg/services/ngalert/state/cache.go +++ b/pkg/services/ngalert/state/cache.go @@ -365,6 +365,8 @@ func (c *cache) asInstances(skipNormalState bool) []ngModels.AlertInstance { LastEvalTime: v2.LastEvaluationTime, CurrentStateSince: v2.StartsAt, CurrentStateEnd: v2.EndsAt, + ResolvedAt: v2.ResolvedAt, + LastSentAt: v2.LastSentAt, ResultFingerprint: v2.ResultFingerprint.String(), }) } diff --git a/pkg/services/ngalert/state/manager.go b/pkg/services/ngalert/state/manager.go index 19c61e64e6b..35aeff3c4d7 100644 --- a/pkg/services/ngalert/state/manager.go +++ b/pkg/services/ngalert/state/manager.go @@ -215,6 +215,8 @@ func (st *Manager) Warm(ctx context.Context, rulesReader RuleReader) { LastEvaluationTime: entry.LastEvalTime, Annotations: ruleForEntry.Annotations, ResultFingerprint: resultFp, + ResolvedAt: entry.ResolvedAt, + LastSentAt: entry.LastSentAt, } statesCount++ } diff --git a/pkg/services/ngalert/state/manager_test.go b/pkg/services/ngalert/state/manager_test.go index 96f58326986..7905204cc71 100644 --- a/pkg/services/ngalert/state/manager_test.go +++ b/pkg/services/ngalert/state/manager_test.go @@ -62,6 +62,8 @@ func TestWarmStateCache(t *testing.T) { StartsAt: evaluationTime.Add(-1 * time.Minute), EndsAt: evaluationTime.Add(1 * time.Minute), LastEvaluationTime: evaluationTime, + LastSentAt: util.Pointer(evaluationTime), + ResolvedAt: util.Pointer(evaluationTime), Annotations: map[string]string{"testAnnoKey": "testAnnoValue"}, ResultFingerprint: data.Fingerprint(math.MaxUint64), }, { @@ -73,6 +75,8 @@ func TestWarmStateCache(t *testing.T) { StartsAt: evaluationTime.Add(-1 * time.Minute), EndsAt: evaluationTime.Add(1 * time.Minute), LastEvaluationTime: evaluationTime, + LastSentAt: util.Pointer(evaluationTime.Add(-1 * time.Minute)), + ResolvedAt: nil, Annotations: map[string]string{"testAnnoKey": "testAnnoValue"}, ResultFingerprint: data.Fingerprint(math.MaxUint64 - 1), }, @@ -85,6 +89,8 @@ func TestWarmStateCache(t *testing.T) { StartsAt: evaluationTime.Add(-1 * time.Minute), EndsAt: evaluationTime.Add(1 * time.Minute), LastEvaluationTime: evaluationTime, + LastSentAt: util.Pointer(evaluationTime.Add(-1 * time.Minute)), + ResolvedAt: nil, Annotations: map[string]string{"testAnnoKey": "testAnnoValue"}, ResultFingerprint: data.Fingerprint(0), }, @@ -97,6 +103,8 @@ func TestWarmStateCache(t *testing.T) { StartsAt: evaluationTime.Add(-1 * time.Minute), EndsAt: evaluationTime.Add(1 * time.Minute), LastEvaluationTime: evaluationTime, + LastSentAt: util.Pointer(evaluationTime.Add(-1 * time.Minute)), + ResolvedAt: nil, Annotations: map[string]string{"testAnnoKey": "testAnnoValue"}, ResultFingerprint: data.Fingerprint(1), }, @@ -109,6 +117,8 @@ func TestWarmStateCache(t *testing.T) { StartsAt: evaluationTime.Add(-1 * time.Minute), EndsAt: evaluationTime.Add(1 * time.Minute), LastEvaluationTime: evaluationTime, + LastSentAt: nil, + ResolvedAt: nil, Annotations: map[string]string{"testAnnoKey": "testAnnoValue"}, ResultFingerprint: data.Fingerprint(2), }, @@ -128,6 +138,8 @@ func TestWarmStateCache(t *testing.T) { LastEvalTime: evaluationTime, CurrentStateSince: evaluationTime.Add(-1 * time.Minute), CurrentStateEnd: evaluationTime.Add(1 * time.Minute), + LastSentAt: &evaluationTime, + ResolvedAt: &evaluationTime, Labels: labels, ResultFingerprint: data.Fingerprint(math.MaxUint64).String(), }) @@ -144,6 +156,8 @@ func TestWarmStateCache(t *testing.T) { LastEvalTime: evaluationTime, CurrentStateSince: evaluationTime.Add(-1 * time.Minute), CurrentStateEnd: evaluationTime.Add(1 * time.Minute), + LastSentAt: util.Pointer(evaluationTime.Add(-1 * time.Minute)), + ResolvedAt: nil, Labels: labels, ResultFingerprint: data.Fingerprint(math.MaxUint64 - 1).String(), }) @@ -160,6 +174,8 @@ func TestWarmStateCache(t *testing.T) { LastEvalTime: evaluationTime, CurrentStateSince: evaluationTime.Add(-1 * time.Minute), CurrentStateEnd: evaluationTime.Add(1 * time.Minute), + LastSentAt: util.Pointer(evaluationTime.Add(-1 * time.Minute)), + ResolvedAt: nil, Labels: labels, ResultFingerprint: data.Fingerprint(0).String(), }) @@ -176,6 +192,8 @@ func TestWarmStateCache(t *testing.T) { LastEvalTime: evaluationTime, CurrentStateSince: evaluationTime.Add(-1 * time.Minute), CurrentStateEnd: evaluationTime.Add(1 * time.Minute), + LastSentAt: util.Pointer(evaluationTime.Add(-1 * time.Minute)), + ResolvedAt: nil, Labels: labels, ResultFingerprint: data.Fingerprint(1).String(), }) @@ -192,6 +210,8 @@ func TestWarmStateCache(t *testing.T) { LastEvalTime: evaluationTime, CurrentStateSince: evaluationTime.Add(-1 * time.Minute), CurrentStateEnd: evaluationTime.Add(1 * time.Minute), + LastSentAt: nil, + ResolvedAt: nil, Labels: labels, ResultFingerprint: data.Fingerprint(2).String(), }) @@ -1656,7 +1676,7 @@ func printAllAnnotations(annos map[int64]annotations.Item) string { } func TestStaleResultsHandler(t *testing.T) { - evaluationTime := time.Now() + evaluationTime := time.Now().Truncate(time.Second).UTC() // Truncate to the second since we don't store sub-second precision. interval := time.Minute ctx := context.Background() @@ -1666,9 +1686,19 @@ func TestStaleResultsHandler(t *testing.T) { rule := tests.CreateTestAlertRule(t, ctx, dbstore, int64(interval.Seconds()), mainOrgID) lastEval := evaluationTime.Add(-2 * interval) - labels1 := models.InstanceLabels{"test1": "testValue1"} + labels1 := models.InstanceLabels{ + "__alert_rule_namespace_uid__": "namespace", + "__alert_rule_uid__": rule.UID, + "alertname": rule.Title, + "test1": "testValue1", + } _, hash1, _ := labels1.StringAndHash() - labels2 := models.InstanceLabels{"test2": "testValue2"} + labels2 := models.InstanceLabels{ + "__alert_rule_namespace_uid__": "namespace", + "__alert_rule_uid__": rule.UID, + "alertname": rule.Title, + "test2": "testValue2", + } _, hash2, _ := labels2.StringAndHash() instances := []models.AlertInstance{ { @@ -1682,6 +1712,9 @@ func TestStaleResultsHandler(t *testing.T) { LastEvalTime: lastEval, CurrentStateSince: lastEval, CurrentStateEnd: lastEval.Add(3 * interval), + LastSentAt: &lastEval, + ResolvedAt: &lastEval, + ResultFingerprint: data.Labels{"test1": "testValue1"}.Fingerprint().String(), }, { AlertInstanceKey: models.AlertInstanceKey{ @@ -1694,6 +1727,9 @@ func TestStaleResultsHandler(t *testing.T) { LastEvalTime: lastEval, CurrentStateSince: lastEval, CurrentStateEnd: lastEval.Add(3 * interval), + LastSentAt: &lastEval, + ResolvedAt: nil, + ResultFingerprint: data.Labels{"test2": "testValue2"}.Fingerprint().String(), }, } @@ -1723,23 +1759,20 @@ func TestStaleResultsHandler(t *testing.T) { { AlertRuleUID: rule.UID, OrgID: 1, - Labels: data.Labels{ - "__alert_rule_namespace_uid__": "namespace", - "__alert_rule_uid__": rule.UID, - "alertname": rule.Title, - "test1": "testValue1", - }, - Values: make(map[string]float64), - State: eval.Normal, + Labels: data.Labels(labels1), + Values: make(map[string]float64), + State: eval.Normal, LatestResult: &state.Evaluation{ EvaluationTime: evaluationTime, EvaluationState: eval.Normal, Values: make(map[string]float64), Condition: "A", }, - StartsAt: evaluationTime, - EndsAt: evaluationTime, + StartsAt: lastEval, + EndsAt: lastEval.Add(3 * interval), LastEvaluationTime: evaluationTime, + LastSentAt: &lastEval, + ResolvedAt: &lastEval, EvaluationDuration: 0, Annotations: map[string]string{"testAnnoKey": "testAnnoValue"}, ResultFingerprint: data.Labels{"test1": "testValue1"}.Fingerprint(), diff --git a/pkg/services/ngalert/state/persister_sync.go b/pkg/services/ngalert/state/persister_sync.go index 5a53d6ddd7d..754a0d26eff 100644 --- a/pkg/services/ngalert/state/persister_sync.go +++ b/pkg/services/ngalert/state/persister_sync.go @@ -102,6 +102,8 @@ func (a *SyncStatePersister) saveAlertStates(ctx context.Context, states ...Stat LastEvalTime: s.LastEvaluationTime, CurrentStateSince: s.StartsAt, CurrentStateEnd: s.EndsAt, + ResolvedAt: s.ResolvedAt, + LastSentAt: s.LastSentAt, } err = a.store.SaveAlertInstance(ctx, instance) diff --git a/pkg/services/ngalert/store/instance_database.go b/pkg/services/ngalert/store/instance_database.go index 57766b9f6f2..c5f1b654bc4 100644 --- a/pkg/services/ngalert/store/instance_database.go +++ b/pkg/services/ngalert/store/instance_database.go @@ -5,6 +5,7 @@ import ( "fmt" "sort" "strings" + "time" "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/services/featuremgmt" @@ -55,12 +56,25 @@ func (st DBstore) SaveAlertInstance(ctx context.Context, alertInstance models.Al if err != nil { return err } - params := append(make([]any, 0), alertInstance.RuleOrgID, alertInstance.RuleUID, labelTupleJSON, alertInstance.LabelsHash, alertInstance.CurrentState, alertInstance.CurrentReason, alertInstance.CurrentStateSince.Unix(), alertInstance.CurrentStateEnd.Unix(), alertInstance.LastEvalTime.Unix(), alertInstance.ResultFingerprint) + params := append(make([]any, 0), + alertInstance.RuleOrgID, + alertInstance.RuleUID, + labelTupleJSON, + alertInstance.LabelsHash, + alertInstance.CurrentState, + alertInstance.CurrentReason, + alertInstance.CurrentStateSince.Unix(), + alertInstance.CurrentStateEnd.Unix(), + alertInstance.LastEvalTime.Unix(), + nullableTimeToUnix(alertInstance.ResolvedAt), + nullableTimeToUnix(alertInstance.LastSentAt), + alertInstance.ResultFingerprint, + ) upsertSQL := st.SQLStore.GetDialect().UpsertSQL( "alert_instance", []string{"rule_org_id", "rule_uid", "labels_hash"}, - []string{"rule_org_id", "rule_uid", "labels", "labels_hash", "current_state", "current_reason", "current_state_since", "current_state_end", "last_eval_time", "result_fingerprint"}) + []string{"rule_org_id", "rule_uid", "labels", "labels_hash", "current_state", "current_reason", "current_state_since", "current_state_end", "last_eval_time", "resolved_at", "last_sent_at", "result_fingerprint"}) _, err = sess.SQL(upsertSQL, params...).Query() if err != nil { return err @@ -219,8 +233,20 @@ func (st DBstore) FullSync(ctx context.Context, instances []models.AlertInstance continue } - _, err = sess.Exec("INSERT INTO alert_instance (rule_org_id, rule_uid, labels, labels_hash, current_state, current_reason, current_state_since, current_state_end, last_eval_time) VALUES (?,?,?,?,?,?,?,?,?)", - alertInstance.RuleOrgID, alertInstance.RuleUID, labelTupleJSON, alertInstance.LabelsHash, alertInstance.CurrentState, alertInstance.CurrentReason, alertInstance.CurrentStateSince.Unix(), alertInstance.CurrentStateEnd.Unix(), alertInstance.LastEvalTime.Unix()) + _, err = sess.Exec( + "INSERT INTO alert_instance (rule_org_id, rule_uid, labels, labels_hash, current_state, current_reason, current_state_since, current_state_end, last_eval_time, resolved_at, last_sent_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)", + alertInstance.RuleOrgID, + alertInstance.RuleUID, + labelTupleJSON, + alertInstance.LabelsHash, + alertInstance.CurrentState, + alertInstance.CurrentReason, + alertInstance.CurrentStateSince.Unix(), + alertInstance.CurrentStateEnd.Unix(), + alertInstance.LastEvalTime.Unix(), + nullableTimeToUnix(alertInstance.ResolvedAt), + nullableTimeToUnix(alertInstance.LastSentAt), + ) if err != nil { return fmt.Errorf("failed to insert into alert_instance table: %w", err) } @@ -231,3 +257,12 @@ func (st DBstore) FullSync(ctx context.Context, instances []models.AlertInstance return nil }) } + +// nullableTimeToUnix converts a nullable time.Time to nil, if it is nil, otherwise it converts the time.Time to a unix timestamp. +func nullableTimeToUnix(t *time.Time) *int64 { + if t == nil { + return nil + } + unix := t.Unix() + return &unix +} diff --git a/pkg/services/ngalert/store/instance_database_test.go b/pkg/services/ngalert/store/instance_database_test.go index bfe44560cc7..eefc5f080c0 100644 --- a/pkg/services/ngalert/store/instance_database_test.go +++ b/pkg/services/ngalert/store/instance_database_test.go @@ -386,6 +386,8 @@ func generateTestAlertInstance(orgID int64, ruleID string) models.AlertInstance CurrentStateEnd: time.Now(), CurrentStateSince: time.Now(), LastEvalTime: time.Now(), + LastSentAt: util.Pointer(time.Now()), + ResolvedAt: util.Pointer(time.Now()), CurrentReason: "abc", } } diff --git a/pkg/services/sqlstore/migrations/migrations.go b/pkg/services/sqlstore/migrations/migrations.go index ae88297da96..571d47f145a 100644 --- a/pkg/services/sqlstore/migrations/migrations.go +++ b/pkg/services/sqlstore/migrations/migrations.go @@ -123,6 +123,8 @@ func (oss *OSSMigrations) AddMigration(mg *Migrator) { accesscontrol.AddManagedFolderAlertingSilencesActionsMigrator(mg) ualert.AddRecordingRuleColumns(mg) + + ualert.AddStateResolvedAtColumns(mg) } func addStarMigrations(mg *Migrator) { diff --git a/pkg/services/sqlstore/migrations/ualert/state_resolvedat_mig.go b/pkg/services/sqlstore/migrations/ualert/state_resolvedat_mig.go new file mode 100644 index 00000000000..6b7070022e5 --- /dev/null +++ b/pkg/services/sqlstore/migrations/ualert/state_resolvedat_mig.go @@ -0,0 +1,18 @@ +package ualert + +import "github.com/grafana/grafana/pkg/services/sqlstore/migrator" + +// AddStateResolvedAtColumns adds columns to alert_instance to represent ResolvedAt and LastSentAt. +func AddStateResolvedAtColumns(mg *migrator.Migrator) { + mg.AddMigration("add resolved_at column to alert_instance table", migrator.NewAddColumnMigration(migrator.Table{Name: "alert_instance"}, &migrator.Column{ + Name: "resolved_at", + Type: migrator.DB_BigInt, // BigInt, to match existing time fields. + Nullable: true, + })) + + mg.AddMigration("add last_sent_at column to alert_instance table", migrator.NewAddColumnMigration(migrator.Table{Name: "alert_instance"}, &migrator.Column{ + Name: "last_sent_at", + Type: migrator.DB_BigInt, + Nullable: true, + })) +}