From c10713ea76f537f99cf82bf4244aae2eb1bb654e Mon Sep 17 00:00:00 2001 From: Alexander Weaver Date: Thu, 19 Jan 2023 03:45:31 -0600 Subject: [PATCH] Alerting: Create query interface for state history along with annotation-based implementation (#61646) --- pkg/services/ngalert/models/history.go | 12 +++ pkg/services/ngalert/ngalert.go | 6 +- .../ngalert/state/historian/annotation.go | 93 ++++++++++++++++++- .../state/historian/annotation_test.go | 70 ++++++++++++++ pkg/services/ngalert/state/historian/loki.go | 5 + pkg/services/ngalert/state/historian/query.go | 15 +++ pkg/services/ngalert/state/historian/sql.go | 5 + pkg/services/ngalert/state/manager_test.go | 4 +- 8 files changed, 204 insertions(+), 6 deletions(-) create mode 100644 pkg/services/ngalert/models/history.go create mode 100644 pkg/services/ngalert/state/historian/annotation_test.go create mode 100644 pkg/services/ngalert/state/historian/query.go diff --git a/pkg/services/ngalert/models/history.go b/pkg/services/ngalert/models/history.go new file mode 100644 index 00000000000..c49ebd8a9ce --- /dev/null +++ b/pkg/services/ngalert/models/history.go @@ -0,0 +1,12 @@ +package models + +import "time" + +// HistoryQuery represents a query for alert state history. +type HistoryQuery struct { + RuleUID string + OrgID int64 + Labels map[string]string + From time.Time + To time.Time +} diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index 652f74d3d1f..a5a9fa894db 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -205,7 +205,7 @@ func (ng *AlertNG) init() error { Tracer: ng.tracer, } - history, err := configureHistorianBackend(ng.Cfg.UnifiedAlerting.StateHistory, ng.annotationsRepo, ng.dashboardService) + history, err := configureHistorianBackend(ng.Cfg.UnifiedAlerting.StateHistory, ng.annotationsRepo, ng.dashboardService, ng.store) if err != nil { return err } @@ -378,13 +378,13 @@ func readQuotaConfig(cfg *setting.Cfg) (*quota.Map, error) { return limits, nil } -func configureHistorianBackend(cfg setting.UnifiedAlertingStateHistorySettings, ar annotations.Repository, ds dashboards.DashboardService) (state.Historian, error) { +func configureHistorianBackend(cfg setting.UnifiedAlertingStateHistorySettings, ar annotations.Repository, ds dashboards.DashboardService, rs historian.RuleStore) (state.Historian, error) { if !cfg.Enabled { return historian.NewNopHistorian(), nil } if cfg.Backend == "annotations" { - return historian.NewAnnotationBackend(ar, ds), nil + return historian.NewAnnotationBackend(ar, ds, rs), nil } if cfg.Backend == "loki" { baseURL, err := url.Parse(cfg.LokiRemoteURL) diff --git a/pkg/services/ngalert/state/historian/annotation.go b/pkg/services/ngalert/state/historian/annotation.go index b49146ad424..544f6a45a17 100644 --- a/pkg/services/ngalert/state/historian/annotation.go +++ b/pkg/services/ngalert/state/historian/annotation.go @@ -2,11 +2,13 @@ package historian import ( "context" + "encoding/json" "fmt" "sort" "strings" "time" + "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/services/annotations" @@ -20,13 +22,19 @@ import ( type AnnotationBackend struct { annotations annotations.Repository dashboards *dashboardResolver + rules RuleStore log log.Logger } -func NewAnnotationBackend(annotations annotations.Repository, dashboards dashboards.DashboardService) *AnnotationBackend { +type RuleStore interface { + GetAlertRuleByUID(ctx context.Context, query *ngmodels.GetAlertRuleByUIDQuery) error +} + +func NewAnnotationBackend(annotations annotations.Repository, dashboards dashboards.DashboardService, rules RuleStore) *AnnotationBackend { return &AnnotationBackend{ annotations: annotations, dashboards: newDashboardResolver(dashboards, defaultDashboardCacheExpiry), + rules: rules, log: log.New("ngalert.state.historian"), } } @@ -40,6 +48,89 @@ func (h *AnnotationBackend) RecordStatesAsync(ctx context.Context, rule *ngmodel go h.recordAnnotationsSync(ctx, panel, annotations, logger) } +func (h *AnnotationBackend) QueryStates(ctx context.Context, query ngmodels.HistoryQuery) (*data.Frame, error) { + logger := h.log.FromContext(ctx) + if query.RuleUID == "" { + return nil, fmt.Errorf("ruleUID is required to query annotations") + } + + if query.Labels != nil { + logger.Warn("Annotation state history backend does not support label queries, ignoring that filter") + } + + rq := ngmodels.GetAlertRuleByUIDQuery{ + UID: query.RuleUID, + OrgID: query.OrgID, + } + err := h.rules.GetAlertRuleByUID(ctx, &rq) + if err != nil { + return nil, fmt.Errorf("failed to look up the requested rule") + } + if rq.Result == nil { + return nil, fmt.Errorf("no such rule exists") + } + + q := annotations.ItemQuery{ + AlertId: rq.Result.ID, + OrgId: query.OrgID, + From: query.From.Unix(), + To: query.To.Unix(), + } + items, err := h.annotations.Find(ctx, &q) + if err != nil { + return nil, fmt.Errorf("failed to query annotations for state history: %w", err) + } + + frame := data.NewFrame("states") + + // Annotations only support querying for a single rule's history. + // Since we are guaranteed to have a single rule, we can return it as a single series. + // Also, annotations don't store labels in a strongly defined format. They are formatted into the label's text. + // We are not guaranteed that a given annotation has parseable text, so we instead use the entire text as an opaque value. + + lbls := data.Labels(map[string]string{ + "from": "state-history", + "ruleUID": fmt.Sprint(query.RuleUID), + }) + + // TODO: In the future, we probably want to have one series per unique text string, instead. For simplicity, let's just make it a new column. + // + // TODO: This is a really naive mapping that will evolve in the next couple changes. + // TODO: It will converge over time with the other implementations. + // + // We represent state history as five vectors: + // 1. `time` - when the transition happened + // 2. `text` - a text string containing metadata about the rule + // 3. `prev` - the previous state and reason + // 4. `next` - the next state and reason + // 5. `data` - a JSON string, containing the annotation's contents. analogous to item.Data + times := make([]time.Time, 0, len(items)) + texts := make([]string, 0, len(items)) + prevStates := make([]string, 0, len(items)) + nextStates := make([]string, 0, len(items)) + values := make([]string, 0, len(items)) + for _, item := range items { + data, err := json.Marshal(item.Data) + if err != nil { + logger.Error("Annotation service gave an annotation with unparseable data, skipping", "id", item.Id, "err", err) + continue + } + times = append(times, time.Unix(item.Time, 0)) + texts = append(texts, item.Text) + prevStates = append(prevStates, item.PrevState) + nextStates = append(nextStates, item.NewState) + values = append(values, string(data)) + } + + frame.Fields = append(frame.Fields, data.NewField("time", lbls, times)) + frame.Fields = append(frame.Fields, data.NewField("text", lbls, texts)) + frame.Fields = append(frame.Fields, data.NewField("prev", lbls, prevStates)) + frame.Fields = append(frame.Fields, data.NewField("next", lbls, nextStates)) + frame.Fields = append(frame.Fields, data.NewField("data", lbls, values)) + + return frame, nil +} + func (h *AnnotationBackend) buildAnnotations(rule *ngmodels.AlertRule, states []state.StateTransition, logger log.Logger) []annotations.Item { items := make([]annotations.Item, 0, len(states)) for _, state := range states { diff --git a/pkg/services/ngalert/state/historian/annotation_test.go b/pkg/services/ngalert/state/historian/annotation_test.go new file mode 100644 index 00000000000..bee69be9d56 --- /dev/null +++ b/pkg/services/ngalert/state/historian/annotation_test.go @@ -0,0 +1,70 @@ +package historian + +import ( + "context" + "testing" + "time" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/annotations" + "github.com/grafana/grafana/pkg/services/annotations/annotationstest" + "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes" + "github.com/stretchr/testify/require" +) + +func TestAnnotationHistorian_Integration(t *testing.T) { + t.Run("alert annotations are queryable", func(t *testing.T) { + anns := createTestAnnotationBackendSut(t) + items := []annotations.Item{createAnnotation()} + anns.recordAnnotationsSync(context.Background(), nil, items, log.NewNopLogger()) + + q := models.HistoryQuery{ + RuleUID: "my-rule", + OrgID: 1, + } + frame, err := anns.QueryStates(context.Background(), q) + + require.NoError(t, err) + require.NotNil(t, frame) + require.Len(t, frame.Fields, 5) + for i := 0; i < 5; i++ { + require.Equal(t, frame.Fields[i].Len(), 1) + } + }) +} + +func createTestAnnotationBackendSut(t *testing.T) *AnnotationBackend { + t.Helper() + fakeAnnoRepo := annotationstest.NewFakeAnnotationsRepo() + rules := fakes.NewRuleStore(t) + rules.Rules[1] = []*models.AlertRule{ + models.AlertRuleGen(withOrgID(1), withUID("my-rule"))(), + } + return NewAnnotationBackend(fakeAnnoRepo, &dashboards.FakeDashboardService{}, rules) +} + +func createAnnotation() annotations.Item { + return annotations.Item{ + Id: 1, + OrgId: 1, + AlertId: 1, + Text: "MyAlert {a=b} - No data", + Data: simplejson.New(), + Epoch: time.Now().UnixNano() / int64(time.Millisecond), + } +} + +func withOrgID(orgId int64) func(rule *models.AlertRule) { + return func(rule *models.AlertRule) { + rule.OrgID = orgId + } +} + +func withUID(uid string) func(rule *models.AlertRule) { + return func(rule *models.AlertRule) { + rule.UID = uid + } +} diff --git a/pkg/services/ngalert/state/historian/loki.go b/pkg/services/ngalert/state/historian/loki.go index 33cf92cd89c..e18b58d3305 100644 --- a/pkg/services/ngalert/state/historian/loki.go +++ b/pkg/services/ngalert/state/historian/loki.go @@ -3,6 +3,7 @@ package historian import ( "context" + "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/ngalert/state" @@ -33,3 +34,7 @@ func (h *RemoteLokiBackend) RecordStatesAsync(ctx context.Context, _ *models.Ale logger := h.log.FromContext(ctx) logger.Debug("Remote Loki state history backend was called with states") } + +func (h *RemoteLokiBackend) QueryStates(ctx context.Context, query models.HistoryQuery) (*data.Frame, error) { + return data.NewFrame("states"), nil +} diff --git a/pkg/services/ngalert/state/historian/query.go b/pkg/services/ngalert/state/historian/query.go new file mode 100644 index 00000000000..26849c88a5f --- /dev/null +++ b/pkg/services/ngalert/state/historian/query.go @@ -0,0 +1,15 @@ +package historian + +import ( + "context" + + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/services/ngalert/models" +) + +// Querier represents the ability to query state history. +// TODO: This package also contains implementations of this interface. +// TODO: This type should be moved to the side of the consumer, when the consumer is created in the future. We add it here temporarily to more clearly define this package's interface. +type Querier interface { + QueryStates(ctx context.Context, query models.HistoryQuery) (*data.Frame, error) +} diff --git a/pkg/services/ngalert/state/historian/sql.go b/pkg/services/ngalert/state/historian/sql.go index 6e03f277044..09cfe436ac7 100644 --- a/pkg/services/ngalert/state/historian/sql.go +++ b/pkg/services/ngalert/state/historian/sql.go @@ -3,6 +3,7 @@ package historian import ( "context" + "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/ngalert/state" @@ -20,3 +21,7 @@ func NewSqlBackend() *SqlBackend { func (h *SqlBackend) RecordStatesAsync(ctx context.Context, _ *models.AlertRule, _ []state.StateTransition) { } + +func (h *SqlBackend) QueryStates(ctx context.Context, query models.HistoryQuery) (*data.Frame, error) { + return data.NewFrame("states"), nil +} diff --git a/pkg/services/ngalert/state/manager_test.go b/pkg/services/ngalert/state/manager_test.go index 79a4dafef3e..ca96f4f6de9 100644 --- a/pkg/services/ngalert/state/manager_test.go +++ b/pkg/services/ngalert/state/manager_test.go @@ -219,7 +219,7 @@ func TestDashboardAnnotations(t *testing.T) { _, dbstore := tests.SetupTestEnv(t, 1) fakeAnnoRepo := annotationstest.NewFakeAnnotationsRepo() - hist := historian.NewAnnotationBackend(fakeAnnoRepo, &dashboards.FakeDashboardService{}) + hist := historian.NewAnnotationBackend(fakeAnnoRepo, &dashboards.FakeDashboardService{}, nil) cfg := state.ManagerCfg{ Metrics: testMetrics.GetStateMetrics(), ExternalURL: nil, @@ -2208,7 +2208,7 @@ func TestProcessEvalResults(t *testing.T) { for _, tc := range testCases { fakeAnnoRepo := annotationstest.NewFakeAnnotationsRepo() - hist := historian.NewAnnotationBackend(fakeAnnoRepo, &dashboards.FakeDashboardService{}) + hist := historian.NewAnnotationBackend(fakeAnnoRepo, &dashboards.FakeDashboardService{}, nil) cfg := state.ManagerCfg{ Metrics: testMetrics.GetStateMetrics(), ExternalURL: nil,