IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Add values to annotations (#57738)

Browse Source 
* Add values to annotations

* Fix imports

* Use State attrs instead of Result attrs

* Remove unnecessary variable
This commit is contained in:
Alex Moreno 2022-11-03 10:35:34 +01:00 committed by GitHub
parent b8303fd431
commit ba15d675e7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 55 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
pkg/services/ngalert/state/historian/annotation.go
View File

@ -3,14 +3,17 @@ package historian
import (
"context"
"fmt"
"sort"
"strconv"
"strings"
"time"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/annotations"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/state"
)
@ -30,19 +33,18 @@ func NewAnnotationHistorian(annotations annotations.Repository, dashboards dashb
}
}
func (h *AnnotationStateHistorian) RecordState(ctx context.Context, rule *ngmodels.AlertRule, labels data.Labels, evaluatedAt time.Time, currentData, previousData state.InstanceStateAndReason) {
func (h *AnnotationStateHistorian) RecordState(ctx context.Context, rule *ngmodels.AlertRule, currentState *state.State, evaluatedAt time.Time, currentData, previousData state.InstanceStateAndReason) {
logger := h.log.New(rule.GetKey().LogContext()...)
logger.Debug("Alert state changed creating annotation", "newState", currentData.String(), "oldState", previousData.String())
labels = removePrivateLabels(labels)
annotationText := fmt.Sprintf("%s {%s} - %s", rule.Title, labels.String(), currentData.String())
annotationText, annotationData := buildAnnotationTextAndData(rule, currentState)
item := &annotations.Item{
AlertId: rule.ID,
OrgId: rule.OrgID,
PrevState: previousData.String(),
NewState: currentData.String(),
Text: annotationText,
Data: annotationData,
Epoch: evaluatedAt.UnixNano() / int64(time.Millisecond),
}
@ -72,6 +74,40 @@ func (h *AnnotationStateHistorian) RecordState(ctx context.Context, rule *ngmode
}
}
func buildAnnotationTextAndData(rule *ngmodels.AlertRule, currentState *state.State) (string, *simplejson.Json) {
jsonData := simplejson.New()
var value string
switch currentState.State {
case eval.Error:
if currentState.Error == nil {
jsonData.Set("error", nil)
} else {
jsonData.Set("error", currentState.Error.Error())
}
value = "Error"
case eval.NoData:
jsonData.Set("noData", true)
value = "No data"
default:
keys := make([]string, 0, len(currentState.Values))
for k := range currentState.Values {
keys = append(keys, k)
}
sort.Strings(keys)
var values []string
for _, k := range keys {
values = append(values, fmt.Sprintf("%s=%f", k, currentState.Values[k]))
}
jsonData.Set("values", simplejson.NewFromAny(currentState.Values))
value = strings.Join(values, ", ")
}
labels := removePrivateLabels(currentState.Labels)
return fmt.Sprintf("%s {%s} - %s", rule.Title, labels.String(), value), jsonData
}
func removePrivateLabels(labels data.Labels) data.Labels {
result := make(data.Labels)
for k, v := range labels {

6
pkg/services/ngalert/state/manager.go
View File

@ -243,7 +243,9 @@ func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRu
shouldUpdateAnnotation := oldState != currentState.State || oldReason != currentState.StateReason
if shouldUpdateAnnotation && st.historian != nil {
go st.historian.RecordState(ctx, alertRule, currentState.Labels, result.EvaluatedAt, InstanceStateAndReason{State: currentState.State, Reason: currentState.StateReason}, InstanceStateAndReason{State: oldState, Reason: oldReason})
go st.historian.RecordState(ctx, alertRule, currentState, result.EvaluatedAt,
InstanceStateAndReason{State: currentState.State, Reason: currentState.StateReason},
InstanceStateAndReason{State: oldState, Reason: oldReason})
}
return currentState
}
@ -387,7 +389,7 @@ func (st *Manager) staleResultsHandler(ctx context.Context, evaluatedAt time.Tim
s.EndsAt = evaluatedAt
s.Resolved = true
if st.historian != nil {
st.historian.RecordState(ctx, alertRule, s.Labels, evaluatedAt,
st.historian.RecordState(ctx, alertRule, s, evaluatedAt,
InstanceStateAndReason{State: eval.Normal, Reason: s.StateReason},
previousState,
)

10
pkg/services/ngalert/state/manager_test.go
View File

@ -49,15 +49,21 @@ func TestDashboardAnnotations(t *testing.T) {
})
st.Warm(ctx)
bValue := float64(42)
cValue := float64(1)
_ = st.ProcessEvalResults(ctx, evaluationTime, rule, eval.Results{{
Instance: data.Labels{"instance_label": "testValue2"},
State: eval.Alerting,
EvaluatedAt: evaluationTime,
Values: map[string]eval.NumberValueCapture{
"B": {Var: "B", Value: &bValue, Labels: data.Labels{"job": "prometheus"}},
"C": {Var: "C", Value: &cValue, Labels: data.Labels{"job": "prometheus"}},
},
}}, data.Labels{
"alertname": rule.Title,
})
expected := []string{rule.Title + " {alertname=" + rule.Title + ", instance_label=testValue2, test1=testValue1, test2=testValue2} - Alerting"}
expected := []string{rule.Title + " {alertname=" + rule.Title + ", instance_label=testValue2, test1=testValue1, test2=testValue2} - B=42.000000, C=1.000000"}
sort.Strings(expected)
require.Eventuallyf(t, func() bool {
var actual []string
@ -1316,6 +1322,7 @@ func TestProcessEvalResults(t *testing.T) {
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Error,
Error: errors.New("test error"),
EvaluatedAt: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
},
@ -1337,6 +1344,7 @@ func TestProcessEvalResults(t *testing.T) {
Values: make(map[string]float64),
State: eval.Pending,
StateReason: eval.Error.String(),
Error: errors.New("test error"),
Results: []state.Evaluation{
{
EvaluationTime: evaluationTime,

3
pkg/services/ngalert/state/persist.go
View File

@ -4,7 +4,6 @@ import (
"context"
"time"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/services/ngalert/models"
)
@ -24,5 +23,5 @@ type RuleReader interface {
// Historian maintains an audit log of alert state history.
type Historian interface {
RecordState(ctx context.Context, rule *models.AlertRule, labels data.Labels, evaluatedAt time.Time, currentData, previousData InstanceStateAndReason)
RecordState(ctx context.Context, rule *models.AlertRule, currentState *State, evaluatedAt time.Time, currentData, previousData InstanceStateAndReason)
}

3
pkg/services/ngalert/state/testing.go
View File

@ -5,7 +5,6 @@ import (
"sync"
"time"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/services/ngalert/models"
)
@ -50,5 +49,5 @@ func (f *FakeRuleReader) ListAlertRules(_ context.Context, q *models.ListAlertRu
type FakeHistorian struct{}
func (f *FakeHistorian) RecordState(ctx context.Context, rule *models.AlertRule, labels data.Labels, evaluatedAt time.Time, currentData, previousData InstanceStateAndReason) {
func (f *FakeHistorian) RecordState(ctx context.Context, rule *models.AlertRule, currentState *State, evaluatedAt time.Time, currentData, previousData InstanceStateAndReason) {
}