IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Persist AlertInstance ResolvedAt & LastSentAt (#89135)

Browse Source 
* Alerting: Persist AlertInstance ResolvedAt & LastSentAt

* Fix test

* Modify existing tests

* Fix merge conflicts from nullable LastSentAt & ResolvedAt
This commit is contained in:
Matthew Jacobson 2024-07-12 12:26:58 -04:00 committed by GitHub
parent e1f030592f
commit ba800692c6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 121 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/services/ngalert/models/instance.go
View File

@ -14,6 +14,8 @@ type AlertInstance struct {
CurrentStateSince time.Time
CurrentStateEnd time.Time
LastEvalTime time.Time
LastSentAt *time.Time
ResolvedAt *time.Time
ResultFingerprint string
}

6
pkg/services/ngalert/models/testing.go
View File

@ -17,6 +17,7 @@ import (
"github.com/stretchr/testify/require"
alertingModels "github.com/grafana/alerting/models"
"github.com/grafana/grafana/pkg/expr"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/folder"
@ -840,6 +841,11 @@ func AlertInstanceGen(mutators ...AlertInstanceMutator) *AlertInstance {
CurrentStateSince: currentStateSince,
CurrentStateEnd: currentStateSince.Add(time.Duration(rand.Intn(100) + 200)),
LastEvalTime: time.Now().Add(-time.Duration(rand.Intn(100) + 50)),
LastSentAt: util.Pointer(time.Now().Add(-time.Duration(rand.Intn(100) + 50))),
}
if instance.CurrentState == InstanceStateNormal && rand.Intn(2) == 1 {
instance.ResolvedAt = util.Pointer(time.Now().Add(-time.Duration(rand.Intn(100) + 50)))
}
for _, mutator := range mutators {

2
pkg/services/ngalert/state/cache.go
View File

@ -365,6 +365,8 @@ func (c *cache) asInstances(skipNormalState bool) []ngModels.AlertInstance {
LastEvalTime: v2.LastEvaluationTime,
CurrentStateSince: v2.StartsAt,
CurrentStateEnd: v2.EndsAt,
ResolvedAt: v2.ResolvedAt,
LastSentAt: v2.LastSentAt,
ResultFingerprint: v2.ResultFingerprint.String(),
})
}

2
pkg/services/ngalert/state/manager.go
View File

@ -215,6 +215,8 @@ func (st *Manager) Warm(ctx context.Context, rulesReader RuleReader) {
LastEvaluationTime: entry.LastEvalTime,
Annotations: ruleForEntry.Annotations,
ResultFingerprint: resultFp,
ResolvedAt: entry.ResolvedAt,
LastSentAt: entry.LastSentAt,
}
statesCount++
}

55
pkg/services/ngalert/state/manager_test.go
View File

@ -62,6 +62,8 @@ func TestWarmStateCache(t *testing.T) {
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
LastSentAt: util.Pointer(evaluationTime),
ResolvedAt: util.Pointer(evaluationTime),
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Fingerprint(math.MaxUint64),
}, {
@ -73,6 +75,8 @@ func TestWarmStateCache(t *testing.T) {
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
LastSentAt: util.Pointer(evaluationTime.Add(-1 * time.Minute)),
ResolvedAt: nil,
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Fingerprint(math.MaxUint64 - 1),
},
@ -85,6 +89,8 @@ func TestWarmStateCache(t *testing.T) {
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
LastSentAt: util.Pointer(evaluationTime.Add(-1 * time.Minute)),
ResolvedAt: nil,
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Fingerprint(0),
},
@ -97,6 +103,8 @@ func TestWarmStateCache(t *testing.T) {
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
LastSentAt: util.Pointer(evaluationTime.Add(-1 * time.Minute)),
ResolvedAt: nil,
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Fingerprint(1),
},
@ -109,6 +117,8 @@ func TestWarmStateCache(t *testing.T) {
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
LastSentAt: nil,
ResolvedAt: nil,
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Fingerprint(2),
},
@ -128,6 +138,8 @@ func TestWarmStateCache(t *testing.T) {
LastEvalTime: evaluationTime,
CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
CurrentStateEnd: evaluationTime.Add(1 * time.Minute),
LastSentAt: &evaluationTime,
ResolvedAt: &evaluationTime,
Labels: labels,
ResultFingerprint: data.Fingerprint(math.MaxUint64).String(),
})
@ -144,6 +156,8 @@ func TestWarmStateCache(t *testing.T) {
LastEvalTime: evaluationTime,
CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
CurrentStateEnd: evaluationTime.Add(1 * time.Minute),
LastSentAt: util.Pointer(evaluationTime.Add(-1 * time.Minute)),
ResolvedAt: nil,
Labels: labels,
ResultFingerprint: data.Fingerprint(math.MaxUint64 - 1).String(),
})
@ -160,6 +174,8 @@ func TestWarmStateCache(t *testing.T) {
LastEvalTime: evaluationTime,
CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
CurrentStateEnd: evaluationTime.Add(1 * time.Minute),
LastSentAt: util.Pointer(evaluationTime.Add(-1 * time.Minute)),
ResolvedAt: nil,
Labels: labels,
ResultFingerprint: data.Fingerprint(0).String(),
})
@ -176,6 +192,8 @@ func TestWarmStateCache(t *testing.T) {
LastEvalTime: evaluationTime,
CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
CurrentStateEnd: evaluationTime.Add(1 * time.Minute),
LastSentAt: util.Pointer(evaluationTime.Add(-1 * time.Minute)),
ResolvedAt: nil,
Labels: labels,
ResultFingerprint: data.Fingerprint(1).String(),
})
@ -192,6 +210,8 @@ func TestWarmStateCache(t *testing.T) {
LastEvalTime: evaluationTime,
CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
CurrentStateEnd: evaluationTime.Add(1 * time.Minute),
LastSentAt: nil,
ResolvedAt: nil,
Labels: labels,
ResultFingerprint: data.Fingerprint(2).String(),
})
@ -1656,7 +1676,7 @@ func printAllAnnotations(annos map[int64]annotations.Item) string {
}
func TestStaleResultsHandler(t *testing.T) {
evaluationTime := time.Now()
evaluationTime := time.Now().Truncate(time.Second).UTC() // Truncate to the second since we don't store sub-second precision.
interval := time.Minute
ctx := context.Background()
@ -1666,9 +1686,19 @@ func TestStaleResultsHandler(t *testing.T) {
rule := tests.CreateTestAlertRule(t, ctx, dbstore, int64(interval.Seconds()), mainOrgID)
lastEval := evaluationTime.Add(-2 * interval)
labels1 := models.InstanceLabels{"test1": "testValue1"}
labels1 := models.InstanceLabels{
"__alert_rule_namespace_uid__": "namespace",
"__alert_rule_uid__": rule.UID,
"alertname": rule.Title,
"test1": "testValue1",
}
_, hash1, _ := labels1.StringAndHash()
labels2 := models.InstanceLabels{"test2": "testValue2"}
labels2 := models.InstanceLabels{
"__alert_rule_namespace_uid__": "namespace",
"__alert_rule_uid__": rule.UID,
"alertname": rule.Title,
"test2": "testValue2",
}
_, hash2, _ := labels2.StringAndHash()
instances := []models.AlertInstance{
{
@ -1682,6 +1712,9 @@ func TestStaleResultsHandler(t *testing.T) {
LastEvalTime: lastEval,
CurrentStateSince: lastEval,
CurrentStateEnd: lastEval.Add(3 * interval),
LastSentAt: &lastEval,
ResolvedAt: &lastEval,
ResultFingerprint: data.Labels{"test1": "testValue1"}.Fingerprint().String(),
},
{
AlertInstanceKey: models.AlertInstanceKey{
@ -1694,6 +1727,9 @@ func TestStaleResultsHandler(t *testing.T) {
LastEvalTime: lastEval,
CurrentStateSince: lastEval,
CurrentStateEnd: lastEval.Add(3 * interval),
LastSentAt: &lastEval,
ResolvedAt: nil,
ResultFingerprint: data.Labels{"test2": "testValue2"}.Fingerprint().String(),
},
}
@ -1723,12 +1759,7 @@ func TestStaleResultsHandler(t *testing.T) {
{
AlertRuleUID: rule.UID,
OrgID: 1,
Labels: data.Labels{
"__alert_rule_namespace_uid__": "namespace",
"__alert_rule_uid__": rule.UID,
"alertname": rule.Title,
"test1": "testValue1",
},
Labels: data.Labels(labels1),
Values: make(map[string]float64),
State: eval.Normal,
LatestResult: &state.Evaluation{
@ -1737,9 +1768,11 @@ func TestStaleResultsHandler(t *testing.T) {
Values: make(map[string]float64),
Condition: "A",
},
StartsAt: evaluationTime,
EndsAt: evaluationTime,
StartsAt: lastEval,
EndsAt: lastEval.Add(3 * interval),
LastEvaluationTime: evaluationTime,
LastSentAt: &lastEval,
ResolvedAt: &lastEval,
EvaluationDuration: 0,
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Labels{"test1": "testValue1"}.Fingerprint(),

2
pkg/services/ngalert/state/persister_sync.go
View File

@ -102,6 +102,8 @@ func (a *SyncStatePersister) saveAlertStates(ctx context.Context, states ...Stat
LastEvalTime: s.LastEvaluationTime,
CurrentStateSince: s.StartsAt,
CurrentStateEnd: s.EndsAt,
ResolvedAt: s.ResolvedAt,
LastSentAt: s.LastSentAt,
}
err = a.store.SaveAlertInstance(ctx, instance)

43
pkg/services/ngalert/store/instance_database.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"sort"
"strings"
"time"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/services/featuremgmt"
@ -55,12 +56,25 @@ func (st DBstore) SaveAlertInstance(ctx context.Context, alertInstance models.Al
if err != nil {
return err
}
params := append(make([]any, 0), alertInstance.RuleOrgID, alertInstance.RuleUID, labelTupleJSON, alertInstance.LabelsHash, alertInstance.CurrentState, alertInstance.CurrentReason, alertInstance.CurrentStateSince.Unix(), alertInstance.CurrentStateEnd.Unix(), alertInstance.LastEvalTime.Unix(), alertInstance.ResultFingerprint)
params := append(make([]any, 0),
alertInstance.RuleOrgID,
alertInstance.RuleUID,
labelTupleJSON,
alertInstance.LabelsHash,
alertInstance.CurrentState,
alertInstance.CurrentReason,
alertInstance.CurrentStateSince.Unix(),
alertInstance.CurrentStateEnd.Unix(),
alertInstance.LastEvalTime.Unix(),
nullableTimeToUnix(alertInstance.ResolvedAt),
nullableTimeToUnix(alertInstance.LastSentAt),
alertInstance.ResultFingerprint,
)
upsertSQL := st.SQLStore.GetDialect().UpsertSQL(
"alert_instance",
[]string{"rule_org_id", "rule_uid", "labels_hash"},
[]string{"rule_org_id", "rule_uid", "labels", "labels_hash", "current_state", "current_reason", "current_state_since", "current_state_end", "last_eval_time", "result_fingerprint"})
[]string{"rule_org_id", "rule_uid", "labels", "labels_hash", "current_state", "current_reason", "current_state_since", "current_state_end", "last_eval_time", "resolved_at", "last_sent_at", "result_fingerprint"})
_, err = sess.SQL(upsertSQL, params...).Query()
if err != nil {
return err
@ -219,8 +233,20 @@ func (st DBstore) FullSync(ctx context.Context, instances []models.AlertInstance
continue
}
_, err = sess.Exec("INSERT INTO alert_instance (rule_org_id, rule_uid, labels, labels_hash, current_state, current_reason, current_state_since, current_state_end, last_eval_time) VALUES (?,?,?,?,?,?,?,?,?)",
alertInstance.RuleOrgID, alertInstance.RuleUID, labelTupleJSON, alertInstance.LabelsHash, alertInstance.CurrentState, alertInstance.CurrentReason, alertInstance.CurrentStateSince.Unix(), alertInstance.CurrentStateEnd.Unix(), alertInstance.LastEvalTime.Unix())
_, err = sess.Exec(
"INSERT INTO alert_instance (rule_org_id, rule_uid, labels, labels_hash, current_state, current_reason, current_state_since, current_state_end, last_eval_time, resolved_at, last_sent_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)",
alertInstance.RuleOrgID,
alertInstance.RuleUID,
labelTupleJSON,
alertInstance.LabelsHash,
alertInstance.CurrentState,
alertInstance.CurrentReason,
alertInstance.CurrentStateSince.Unix(),
alertInstance.CurrentStateEnd.Unix(),
alertInstance.LastEvalTime.Unix(),
nullableTimeToUnix(alertInstance.ResolvedAt),
nullableTimeToUnix(alertInstance.LastSentAt),
)
if err != nil {
return fmt.Errorf("failed to insert into alert_instance table: %w", err)
}
@ -231,3 +257,12 @@ func (st DBstore) FullSync(ctx context.Context, instances []models.AlertInstance
return nil
})
}
// nullableTimeToUnix converts a nullable time.Time to nil, if it is nil, otherwise it converts the time.Time to a unix timestamp.
func nullableTimeToUnix(t *time.Time) *int64 {
if t == nil {
return nil
}
unix := t.Unix()
return &unix
}

2
pkg/services/ngalert/store/instance_database_test.go
View File

@ -386,6 +386,8 @@ func generateTestAlertInstance(orgID int64, ruleID string) models.AlertInstance
CurrentStateEnd: time.Now(),
CurrentStateSince: time.Now(),
LastEvalTime: time.Now(),
LastSentAt: util.Pointer(time.Now()),
ResolvedAt: util.Pointer(time.Now()),
CurrentReason: "abc",
}
}

2
pkg/services/sqlstore/migrations/migrations.go
View File

@ -123,6 +123,8 @@ func (oss *OSSMigrations) AddMigration(mg *Migrator) {
accesscontrol.AddManagedFolderAlertingSilencesActionsMigrator(mg)
ualert.AddRecordingRuleColumns(mg)
ualert.AddStateResolvedAtColumns(mg)
}
func addStarMigrations(mg *Migrator) {

18
pkg/services/sqlstore/migrations/ualert/state_resolvedat_mig.go Normal file
View File

@ -0,0 +1,18 @@
package ualert
import "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
// AddStateResolvedAtColumns adds columns to alert_instance to represent ResolvedAt and LastSentAt.
func AddStateResolvedAtColumns(mg *migrator.Migrator) {
mg.AddMigration("add resolved_at column to alert_instance table", migrator.NewAddColumnMigration(migrator.Table{Name: "alert_instance"}, &migrator.Column{
Name: "resolved_at",
Type: migrator.DB_BigInt, // BigInt, to match existing time fields.
Nullable: true,
}))
mg.AddMigration("add last_sent_at column to alert_instance table", migrator.NewAddColumnMigration(migrator.Table{Name: "alert_instance"}, &migrator.Column{
Name: "last_sent_at",
Type: migrator.DB_BigInt,
Nullable: true,
}))
}