mirror of
https://github.com/grafana/grafana.git
synced 2025-02-15 01:53:33 -06:00
265 lines
8.7 KiB
Go
265 lines
8.7 KiB
Go
package historian
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"math"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/benbjohnson/clock"
|
|
"github.com/grafana/grafana-plugin-sdk-go/data"
|
|
|
|
"github.com/grafana/grafana/pkg/components/simplejson"
|
|
"github.com/grafana/grafana/pkg/infra/log"
|
|
"github.com/grafana/grafana/pkg/services/annotations"
|
|
"github.com/grafana/grafana/pkg/services/dashboards"
|
|
"github.com/grafana/grafana/pkg/services/ngalert/eval"
|
|
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
|
|
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
|
|
"github.com/grafana/grafana/pkg/services/ngalert/state"
|
|
history_model "github.com/grafana/grafana/pkg/services/ngalert/state/historian/model"
|
|
)
|
|
|
|
// AnnotationBackend is an implementation of state.Historian that uses Grafana Annotations as the backing datastore.
|
|
type AnnotationBackend struct {
|
|
annotations AnnotationStore
|
|
dashboards *dashboardResolver
|
|
rules RuleStore
|
|
clock clock.Clock
|
|
metrics *metrics.Historian
|
|
log log.Logger
|
|
}
|
|
|
|
type RuleStore interface {
|
|
GetAlertRuleByUID(ctx context.Context, query *ngmodels.GetAlertRuleByUIDQuery) error
|
|
}
|
|
|
|
type AnnotationStore interface {
|
|
Find(ctx context.Context, query *annotations.ItemQuery) ([]*annotations.ItemDTO, error)
|
|
SaveMany(ctx context.Context, items []annotations.Item) error
|
|
}
|
|
|
|
func NewAnnotationBackend(annotations AnnotationStore, dashboards dashboards.DashboardService, rules RuleStore, metrics *metrics.Historian) *AnnotationBackend {
|
|
logger := log.New("ngalert.state.historian", "backend", "annotations")
|
|
return &AnnotationBackend{
|
|
annotations: annotations,
|
|
dashboards: newDashboardResolver(dashboards, defaultDashboardCacheExpiry),
|
|
rules: rules,
|
|
clock: clock.New(),
|
|
metrics: metrics,
|
|
log: logger,
|
|
}
|
|
}
|
|
|
|
// Record writes a number of state transitions for a given rule to state history.
|
|
func (h *AnnotationBackend) Record(ctx context.Context, rule history_model.RuleMeta, states []state.StateTransition) <-chan error {
|
|
logger := h.log.FromContext(ctx)
|
|
// Build annotations before starting goroutine, to make sure all data is copied and won't mutate underneath us.
|
|
annotations := buildAnnotations(rule, states, logger)
|
|
panel := parsePanelKey(rule, logger)
|
|
|
|
errCh := make(chan error, 1)
|
|
if len(annotations) == 0 {
|
|
close(errCh)
|
|
return errCh
|
|
}
|
|
|
|
go func() {
|
|
defer close(errCh)
|
|
errCh <- h.recordAnnotations(ctx, panel, annotations, rule.OrgID, logger)
|
|
}()
|
|
return errCh
|
|
}
|
|
|
|
// Query filters state history annotations and formats them into a dataframe.
|
|
func (h *AnnotationBackend) Query(ctx context.Context, query ngmodels.HistoryQuery) (*data.Frame, error) {
|
|
logger := h.log.FromContext(ctx)
|
|
if query.RuleUID == "" {
|
|
return nil, fmt.Errorf("ruleUID is required to query annotations")
|
|
}
|
|
|
|
if query.Labels != nil {
|
|
logger.Warn("Annotation state history backend does not support label queries, ignoring that filter")
|
|
}
|
|
|
|
rq := ngmodels.GetAlertRuleByUIDQuery{
|
|
UID: query.RuleUID,
|
|
OrgID: query.OrgID,
|
|
}
|
|
err := h.rules.GetAlertRuleByUID(ctx, &rq)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to look up the requested rule")
|
|
}
|
|
if rq.Result == nil {
|
|
return nil, fmt.Errorf("no such rule exists")
|
|
}
|
|
|
|
q := annotations.ItemQuery{
|
|
AlertID: rq.Result.ID,
|
|
OrgID: query.OrgID,
|
|
From: query.From.Unix(),
|
|
To: query.To.Unix(),
|
|
SignedInUser: query.SignedInUser,
|
|
}
|
|
items, err := h.annotations.Find(ctx, &q)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to query annotations for state history: %w", err)
|
|
}
|
|
|
|
frame := data.NewFrame("states")
|
|
|
|
// Annotations only support querying for a single rule's history.
|
|
// Since we are guaranteed to have a single rule, we can return it as a single series.
|
|
// Also, annotations don't store labels in a strongly defined format. They are formatted into the label's text.
|
|
// We are not guaranteed that a given annotation has parseable text, so we instead use the entire text as an opaque value.
|
|
|
|
lbls := data.Labels(map[string]string{
|
|
"from": "state-history",
|
|
"ruleUID": fmt.Sprint(query.RuleUID),
|
|
})
|
|
|
|
// TODO: In the future, we probably want to have one series per unique text string, instead. For simplicity, let's just make it a new column.
|
|
//
|
|
// TODO: This is a really naive mapping that will evolve in the next couple changes.
|
|
// TODO: It will converge over time with the other implementations.
|
|
//
|
|
// We represent state history as five vectors:
|
|
// 1. `time` - when the transition happened
|
|
// 2. `text` - a text string containing metadata about the rule
|
|
// 3. `prev` - the previous state and reason
|
|
// 4. `next` - the next state and reason
|
|
// 5. `data` - a JSON string, containing the annotation's contents. analogous to item.Data
|
|
times := make([]time.Time, 0, len(items))
|
|
texts := make([]string, 0, len(items))
|
|
prevStates := make([]string, 0, len(items))
|
|
nextStates := make([]string, 0, len(items))
|
|
values := make([]string, 0, len(items))
|
|
for _, item := range items {
|
|
data, err := json.Marshal(item.Data)
|
|
if err != nil {
|
|
logger.Error("Annotation service gave an annotation with unparseable data, skipping", "id", item.ID, "err", err)
|
|
continue
|
|
}
|
|
times = append(times, time.Unix(item.Time, 0))
|
|
texts = append(texts, item.Text)
|
|
prevStates = append(prevStates, item.PrevState)
|
|
nextStates = append(nextStates, item.NewState)
|
|
values = append(values, string(data))
|
|
}
|
|
|
|
frame.Fields = append(frame.Fields, data.NewField("time", lbls, times))
|
|
frame.Fields = append(frame.Fields, data.NewField("text", lbls, texts))
|
|
frame.Fields = append(frame.Fields, data.NewField("prev", lbls, prevStates))
|
|
frame.Fields = append(frame.Fields, data.NewField("next", lbls, nextStates))
|
|
frame.Fields = append(frame.Fields, data.NewField("data", lbls, values))
|
|
|
|
return frame, nil
|
|
}
|
|
|
|
func buildAnnotations(rule history_model.RuleMeta, states []state.StateTransition, logger log.Logger) []annotations.Item {
|
|
items := make([]annotations.Item, 0, len(states))
|
|
for _, state := range states {
|
|
if !shouldRecord(state) {
|
|
continue
|
|
}
|
|
logger.Debug("Alert state changed creating annotation", "newState", state.Formatted(), "oldState", state.PreviousFormatted())
|
|
|
|
annotationText, annotationData := buildAnnotationTextAndData(rule, state.State)
|
|
|
|
item := annotations.Item{
|
|
AlertID: rule.ID,
|
|
OrgID: state.OrgID,
|
|
PrevState: state.PreviousFormatted(),
|
|
NewState: state.Formatted(),
|
|
Text: annotationText,
|
|
Data: annotationData,
|
|
Epoch: state.LastEvaluationTime.UnixNano() / int64(time.Millisecond),
|
|
}
|
|
|
|
items = append(items, item)
|
|
}
|
|
return items
|
|
}
|
|
|
|
func (h *AnnotationBackend) recordAnnotations(ctx context.Context, panel *panelKey, annotations []annotations.Item, orgID int64, logger log.Logger) error {
|
|
if panel != nil {
|
|
dashID, err := h.dashboards.getID(ctx, panel.orgID, panel.dashUID)
|
|
if err != nil {
|
|
logger.Error("Error getting dashboard for alert annotation", "dashboardUID", panel.dashUID, "error", err)
|
|
dashID = 0
|
|
}
|
|
|
|
for i := range annotations {
|
|
annotations[i].DashboardID = dashID
|
|
annotations[i].PanelID = panel.panelID
|
|
}
|
|
}
|
|
|
|
org := fmt.Sprint(orgID)
|
|
h.metrics.WritesTotal.WithLabelValues(org).Inc()
|
|
h.metrics.TransitionsTotal.WithLabelValues(org).Add(float64(len(annotations)))
|
|
if err := h.annotations.SaveMany(ctx, annotations); err != nil {
|
|
logger.Error("Error saving alert annotation batch", "error", err)
|
|
h.metrics.WritesFailed.WithLabelValues(org).Inc()
|
|
h.metrics.TransitionsFailed.WithLabelValues(org).Add(float64(len(annotations)))
|
|
return fmt.Errorf("error saving alert annotation batch: %w", err)
|
|
}
|
|
|
|
logger.Debug("Done saving alert annotation batch")
|
|
return nil
|
|
}
|
|
|
|
func buildAnnotationTextAndData(rule history_model.RuleMeta, currentState *state.State) (string, *simplejson.Json) {
|
|
jsonData := simplejson.New()
|
|
var value string
|
|
|
|
switch currentState.State {
|
|
case eval.Error:
|
|
if currentState.Error == nil {
|
|
jsonData.Set("error", nil)
|
|
} else {
|
|
jsonData.Set("error", currentState.Error.Error())
|
|
}
|
|
value = "Error"
|
|
case eval.NoData:
|
|
jsonData.Set("noData", true)
|
|
value = "No data"
|
|
default:
|
|
keys := make([]string, 0, len(currentState.Values))
|
|
for k := range currentState.Values {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
|
|
var values []string
|
|
for _, k := range keys {
|
|
values = append(values, fmt.Sprintf("%s=%f", k, currentState.Values[k]))
|
|
}
|
|
jsonData.Set("values", jsonifyValues(currentState.Values))
|
|
value = strings.Join(values, ", ")
|
|
}
|
|
|
|
labels := removePrivateLabels(currentState.Labels)
|
|
return fmt.Sprintf("%s {%s} - %s", rule.Title, labels.String(), value), jsonData
|
|
}
|
|
|
|
func jsonifyValues(vs map[string]float64) *simplejson.Json {
|
|
if vs == nil {
|
|
return nil
|
|
}
|
|
|
|
j := simplejson.New()
|
|
for k, v := range vs {
|
|
switch {
|
|
case math.IsInf(v, 0), math.IsNaN(v):
|
|
j.Set(k, fmt.Sprintf("%f", v))
|
|
default:
|
|
j.Set(k, v)
|
|
}
|
|
}
|
|
return j
|
|
}
|