IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-12 00:25:46 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Fix state cache id duplication (#33480)

Browse Source
This commit is contained in:
Kyle Brandt 2021-04-28 11:42:19 -04:00 committed by GitHub
parent 7509c6cbc4
commit 914443c816
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 53 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pkg/services/ngalert/models/instance_labels.go
View File

@ -38,6 +38,15 @@ func (il *InstanceLabels) ToDB() ([]byte, error) {
return []byte{}, fmt.Errorf("database serialization of alerting ng Instance labels is not implemented") return []byte{}, fmt.Errorf("database serialization of alerting ng Instance labels is not implemented")
} }
func (il *InstanceLabels) StringKey() (string, error) {
tl := labelsToTupleLabels(*il)
b, err := json.Marshal(tl)
if err != nil {
return "", fmt.Errorf("can not gereate key due to failure to encode labels: %w", err)
}
return string(b), nil
}
// StringAndHash returns a the json representation of the labels as tuples // StringAndHash returns a the json representation of the labels as tuples
// sorted by key. It also returns the a hash of that representation. // sorted by key. It also returns the a hash of that representation.
func (il *InstanceLabels) StringAndHash() (string, string, error) { func (il *InstanceLabels) StringAndHash() (string, string, error) {

6
pkg/services/ngalert/schedule/schedule.go
View File

@ -355,10 +355,14 @@ func (sch *schedule) WarmStateCache(st *state.Manager) {
} }
for _, entry := range cmd.Result { for _, entry := range cmd.Result {
lbs := map[string]string(entry.Labels) lbs := map[string]string(entry.Labels)
cacheId, err := entry.Labels.StringKey()
if err != nil {
sch.log.Error("error getting cacheId for entry", "msg", err.Error())
}
stateForEntry := &state.State{ stateForEntry := &state.State{
AlertRuleUID: entry.DefinitionUID, AlertRuleUID: entry.DefinitionUID,
OrgID: entry.DefinitionOrgID, OrgID: entry.DefinitionOrgID,
CacheId: fmt.Sprintf("%s %s", entry.DefinitionUID, lbs), CacheId: cacheId,
Labels: lbs, Labels: lbs,
State: translateInstanceState(entry.CurrentState), State: translateInstanceState(entry.CurrentState),
Results: []state.Evaluation{}, Results: []state.Evaluation{},

12
pkg/services/ngalert/state/cache.go
View File

@ -6,6 +6,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/ngalert/eval" "github.com/grafana/grafana/pkg/services/ngalert/eval"
ngModels "github.com/grafana/grafana/pkg/services/ngalert/models" ngModels "github.com/grafana/grafana/pkg/services/ngalert/models"
prometheusModel "github.com/prometheus/common/model" prometheusModel "github.com/prometheus/common/model"
@ -14,11 +15,13 @@ import (
type cache struct { type cache struct {
states map[string]*State states map[string]*State
mtxStates sync.RWMutex mtxStates sync.RWMutex
log log.Logger
} }
func newCache() *cache { func newCache(logger log.Logger) *cache {
return &cache{ return &cache{
states: make(map[string]*State), states: make(map[string]*State),
log: logger,
} }
} }
@ -32,7 +35,12 @@ func (c *cache) getOrCreate(alertRule *ngModels.AlertRule, result eval.Result) *
lbs[ngModels.NamespaceUIDLabel] = alertRule.NamespaceUID lbs[ngModels.NamespaceUIDLabel] = alertRule.NamespaceUID
lbs[prometheusModel.AlertNameLabel] = alertRule.Title lbs[prometheusModel.AlertNameLabel] = alertRule.Title
id := fmt.Sprintf("%s", map[string]string(lbs)) il := ngModels.InstanceLabels(lbs)
id, err := il.StringKey()
if err != nil {
c.log.Error("error getting cacheId for entry", "msg", err.Error())
}
if state, ok := c.states[id]; ok { if state, ok := c.states[id]; ok {
return state return state
} }

2
pkg/services/ngalert/state/manager.go
View File

@ -17,7 +17,7 @@ type Manager struct {
func NewManager(logger log.Logger) *Manager { func NewManager(logger log.Logger) *Manager {
manager := &Manager{ manager := &Manager{
cache: newCache(), cache: newCache(logger),
quit: make(chan struct{}), quit: make(chan struct{}),
Log: logger, Log: logger,
} }

52
pkg/services/ngalert/tests/manager_test.go
View File

@ -51,10 +51,10 @@ func TestProcessEvalResults(t *testing.T) {
}, },
}, },
expectedStates: map[string]*state.State{ expectedStates: map[string]*state.State{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label:test label:test]": { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid", AlertRuleUID: "test_alert_rule_uid",
OrgID: 1, OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label:test label:test]", CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
Labels: data.Labels{ Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid", "__alert_rule_uid__": "test_alert_rule_uid",
@ -103,10 +103,10 @@ func TestProcessEvalResults(t *testing.T) {
}, },
}, },
expectedStates: map[string]*state.State{ expectedStates: map[string]*state.State{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label_1:test label:test]": { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label_1","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid", AlertRuleUID: "test_alert_rule_uid",
OrgID: 1, OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label_1:test label:test]", CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label_1","test"],["label","test"]]`,
Labels: data.Labels{ Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid", "__alert_rule_uid__": "test_alert_rule_uid",
@ -125,10 +125,10 @@ func TestProcessEvalResults(t *testing.T) {
EvaluationDuration: evaluationDuration, EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"}, Annotations: map[string]string{"annotation": "test"},
}, },
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label_2:test label:test]": { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label_2","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid", AlertRuleUID: "test_alert_rule_uid",
OrgID: 1, OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label_2:test label:test]", CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid"],["alertname","test_title"],["instance_label_2","test"],["label","test"]]`,
Labels: data.Labels{ Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid", "__alert_rule_uid__": "test_alert_rule_uid",
@ -181,10 +181,10 @@ func TestProcessEvalResults(t *testing.T) {
}, },
}, },
expectedStates: map[string]*state.State{ expectedStates: map[string]*state.State{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_1 alertname:test_title instance_label:test label:test]": { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_1"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_1", AlertRuleUID: "test_alert_rule_uid_1",
OrgID: 1, OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_1 alertname:test_title instance_label:test label:test]", CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_1"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
Labels: data.Labels{ Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_1", "__alert_rule_uid__": "test_alert_rule_uid_1",
@ -239,10 +239,10 @@ func TestProcessEvalResults(t *testing.T) {
}, },
}, },
expectedStates: map[string]*state.State{ expectedStates: map[string]*state.State{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_2", AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1, OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]", CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
Labels: data.Labels{ Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2", "__alert_rule_uid__": "test_alert_rule_uid_2",
@ -308,10 +308,10 @@ func TestProcessEvalResults(t *testing.T) {
}, },
}, },
expectedStates: map[string]*state.State{ expectedStates: map[string]*state.State{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_2", AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1, OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]", CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
Labels: data.Labels{ Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2", "__alert_rule_uid__": "test_alert_rule_uid_2",
@ -373,10 +373,10 @@ func TestProcessEvalResults(t *testing.T) {
}, },
}, },
expectedStates: map[string]*state.State{ expectedStates: map[string]*state.State{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_2", AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1, OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]", CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
Labels: data.Labels{ Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2", "__alert_rule_uid__": "test_alert_rule_uid_2",
@ -434,10 +434,10 @@ func TestProcessEvalResults(t *testing.T) {
}, },
}, },
expectedStates: map[string]*state.State{ expectedStates: map[string]*state.State{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_2", AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1, OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]", CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
Labels: data.Labels{ Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2", "__alert_rule_uid__": "test_alert_rule_uid_2",
@ -495,10 +495,10 @@ func TestProcessEvalResults(t *testing.T) {
}, },
}, },
expectedStates: map[string]*state.State{ expectedStates: map[string]*state.State{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_2", AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1, OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]", CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
Labels: data.Labels{ Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2", "__alert_rule_uid__": "test_alert_rule_uid_2",
@ -556,10 +556,10 @@ func TestProcessEvalResults(t *testing.T) {
}, },
}, },
expectedStates: map[string]*state.State{ expectedStates: map[string]*state.State{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_2", AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1, OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]", CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
Labels: data.Labels{ Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2", "__alert_rule_uid__": "test_alert_rule_uid_2",
@ -618,10 +618,10 @@ func TestProcessEvalResults(t *testing.T) {
}, },
}, },
expectedStates: map[string]*state.State{ expectedStates: map[string]*state.State{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_2", AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1, OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]", CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
Labels: data.Labels{ Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2", "__alert_rule_uid__": "test_alert_rule_uid_2",
@ -680,10 +680,10 @@ func TestProcessEvalResults(t *testing.T) {
}, },
}, },
expectedStates: map[string]*state.State{ expectedStates: map[string]*state.State{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_2", AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1, OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]", CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
Labels: data.Labels{ Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2", "__alert_rule_uid__": "test_alert_rule_uid_2",
@ -742,10 +742,10 @@ func TestProcessEvalResults(t *testing.T) {
}, },
}, },
expectedStates: map[string]*state.State{ expectedStates: map[string]*state.State{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": { `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_2", AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1, OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]", CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
Labels: data.Labels{ Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid", "__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2", "__alert_rule_uid__": "test_alert_rule_uid_2",

4
pkg/services/ngalert/tests/schedule_test.go
View File

@ -37,7 +37,7 @@ func TestWarmStateCache(t *testing.T) {
{ {
AlertRuleUID: "test_uid", AlertRuleUID: "test_uid",
OrgID: 123, OrgID: 123,
CacheId: "test_uid map[test1:testValue1]", CacheId: `[["test1","testValue1"]]`,
Labels: data.Labels{"test1": "testValue1"}, Labels: data.Labels{"test1": "testValue1"},
State: eval.Normal, State: eval.Normal,
Results: []state.Evaluation{ Results: []state.Evaluation{
@ -49,7 +49,7 @@ func TestWarmStateCache(t *testing.T) {
}, { }, {
AlertRuleUID: "test_uid", AlertRuleUID: "test_uid",
OrgID: 123, OrgID: 123,
CacheId: "test_uid map[test2:testValue2]", CacheId: `[["test2","testValue2"]]`,
Labels: data.Labels{"test2": "testValue2"}, Labels: data.Labels{"test2": "testValue2"},
State: eval.Alerting, State: eval.Alerting,
Results: []state.Evaluation{ Results: []state.Evaluation{