IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Handle NoData and Error evaluation results (#33194)

Browse Source 
* set processing time

* merge labels and set on response

* use state cache for adding alerts to rules

* minor cleanup

* add support for NoData and Error results

* rename test

* bring in changes from other PRs tha have been merged

* pr feedback

* add integration test

* close state tracker cleanup on context.Done

* fixup test

* not those annotations
This commit is contained in:
David Parrott 2021-04-23 11:47:52 -07:00 committed by GitHub
parent 948cba199b
commit ca79206498
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 1044 additions and 280 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/services/ngalert/api/api_prometheus.go
View File

@ -51,7 +51,9 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res
DiscoveryBase: apimodels.DiscoveryBase{
Status: "success",
},
Data: apimodels.RuleDiscovery{},
Data: apimodels.RuleDiscovery{
RuleGroups: []*apimodels.RuleGroup{},
},
}
ruleGroupQuery := ngmodels.ListOrgRuleGroupsQuery{
@ -99,7 +101,7 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res
newRule := apimodels.Rule{
Name: rule.Title,
Labels: rule.Labels,
Health: "ok", // TODO: update this in the future when error and noData states are being evaluated and set
Health: "ok",
Type: apiv1.RuleTypeAlerting,
LastEvaluation: time.Time{},
}
@ -131,9 +133,9 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res
case eval.Alerting:
alertingRule.State = "firing"
case eval.Error:
// handle Error case based on configuration in alertRule
newRule.Health = "error"
case eval.NoData:
// handle NoData case based on configuration in alertRule
newRule.Health = "nodata"
}
alertingRule.Alerts = append(alertingRule.Alerts, alert)
}

2
pkg/services/ngalert/eval/eval.go
View File

@ -78,7 +78,7 @@ const (
// Pending is the eval state for an alert instance condition
// that evaluated to true (Alerting) but has not yet met
// the For duration defined in AlertRule
// the For duration defined in AlertRule.
Pending
// NoData is the eval state for an alert rule condition

1
pkg/services/ngalert/schedule/schedule.go
View File

@ -307,6 +307,7 @@ func (sch *schedule) Ticker(grafanaCtx context.Context, stateTracker *state.Stat
case <-grafanaCtx.Done():
err := dispatcherGroup.Wait()
sch.saveAlertStates(stateTracker.GetAll())
stateTracker.Close()
return err
}
}

166
pkg/services/ngalert/state/state_tracker.go
View File

@ -56,6 +56,10 @@ func NewStateTracker(logger log.Logger) *StateTracker {
return tracker
}
func (st *StateTracker) Close() {
st.quit <- struct{}{}
}
func (st *StateTracker) getOrCreate(alertRule *ngModels.AlertRule, result eval.Result, evaluationDuration time.Duration) AlertState {
st.cache.mtxStates.Lock()
defer st.cache.mtxStates.Unlock()
@ -76,13 +80,8 @@ func (st *StateTracker) getOrCreate(alertRule *ngModels.AlertRule, result eval.R
annotations = alertRule.Annotations
}
newResults := []StateEvaluation{
{
EvaluationTime: result.EvaluatedAt,
EvaluationState: result.State,
},
}
// If the first result we get is alerting, set StartsAt to EvaluatedAt because we
// do not have data for determining StartsAt otherwise
st.Log.Debug("adding new alert state cache entry", "cacheId", id, "state", result.State.String(), "evaluatedAt", result.EvaluatedAt.String())
newState := AlertState{
AlertRuleUID: alertRule.UID,
@ -90,7 +89,6 @@ func (st *StateTracker) getOrCreate(alertRule *ngModels.AlertRule, result eval.R
CacheId: id,
Labels: lbs,
State: result.State,
Results: newResults,
Annotations: annotations,
EvaluationDuration: evaluationDuration,
}
@ -136,57 +134,111 @@ func (st *StateTracker) ProcessEvalResults(alertRule *ngModels.AlertRule, result
//Set the current state based on evaluation results
func (st *StateTracker) setNextState(alertRule *ngModels.AlertRule, result eval.Result, evaluationDuration time.Duration) AlertState {
currentState := st.getOrCreate(alertRule, result, evaluationDuration)
currentState.LastEvaluationTime = result.EvaluatedAt
currentState.EvaluationDuration = evaluationDuration
currentState.Results = append(currentState.Results, StateEvaluation{
EvaluationTime: result.EvaluatedAt,
EvaluationState: result.State,
})
st.Log.Debug("setting alert state", "uid", alertRule.UID)
switch {
case currentState.State == result.State:
st.Log.Debug("no state transition", "cacheId", currentState.CacheId, "state", currentState.State.String())
currentState.LastEvaluationTime = result.EvaluatedAt
currentState.EvaluationDuration = evaluationDuration
currentState.Results = append(currentState.Results, StateEvaluation{
EvaluationTime: result.EvaluatedAt,
EvaluationState: result.State,
})
if currentState.State == eval.Alerting {
//TODO: Move me and unify me with the top level constant
// 10 seconds is the base evaluation interval. We use 2 times that interval to make sure we send an alert
// that would expire after at least 2 iterations and avoid flapping.
resendDelay := 10 * 2 * time.Second
if alertRule.For > resendDelay {
resendDelay = alertRule.For * 2
}
currentState.EndsAt = result.EvaluatedAt.Add(resendDelay)
}
st.set(currentState)
return currentState
case currentState.State == eval.Normal && result.State == eval.Alerting:
st.Log.Debug("state transition from normal to alerting", "cacheId", currentState.CacheId)
currentState.State = eval.Alerting
currentState.LastEvaluationTime = result.EvaluatedAt
currentState.StartsAt = result.EvaluatedAt
currentState.EndsAt = result.EvaluatedAt.Add(alertRule.For * time.Second)
currentState.EvaluationDuration = evaluationDuration
currentState.Results = append(currentState.Results, StateEvaluation{
EvaluationTime: result.EvaluatedAt,
EvaluationState: result.State,
})
currentState.Annotations["alerting"] = result.EvaluatedAt.String()
st.set(currentState)
return currentState
case currentState.State == eval.Alerting && result.State == eval.Normal:
st.Log.Debug("state transition from alerting to normal", "cacheId", currentState.CacheId)
currentState.State = eval.Normal
currentState.LastEvaluationTime = result.EvaluatedAt
currentState.EndsAt = result.EvaluatedAt
currentState.EvaluationDuration = evaluationDuration
currentState.Results = append(currentState.Results, StateEvaluation{
EvaluationTime: result.EvaluatedAt,
EvaluationState: result.State,
})
st.set(currentState)
return currentState
default:
return currentState
switch result.State {
case eval.Normal:
currentState = resultNormal(currentState, result)
case eval.Alerting:
currentState = currentState.resultAlerting(alertRule, result)
case eval.Error:
currentState = currentState.resultError(alertRule, result)
case eval.NoData:
currentState = currentState.resultNoData(alertRule, result)
case eval.Pending: // we do not emit results with this state
}
st.set(currentState)
return currentState
}
func resultNormal(alertState AlertState, result eval.Result) AlertState {
newState := alertState
if alertState.State != eval.Normal {
newState.EndsAt = result.EvaluatedAt
}
newState.State = eval.Normal
return newState
}
func (a AlertState) resultAlerting(alertRule *ngModels.AlertRule, result eval.Result) AlertState {
switch a.State {
case eval.Alerting:
if !(alertRule.For > 0) {
// If there is not For set, we will set EndsAt to be twice the evaluation interval
// to avoid flapping with every evaluation
a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second)
return a
}
a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
case eval.Pending:
if result.EvaluatedAt.Sub(a.StartsAt) > alertRule.For {
a.State = eval.Alerting
a.StartsAt = result.EvaluatedAt
a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
}
default:
a.StartsAt = result.EvaluatedAt
if !(alertRule.For > 0) {
a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second)
a.State = eval.Alerting
} else {
a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
if result.EvaluatedAt.Sub(a.StartsAt) > alertRule.For {
a.State = eval.Alerting
} else {
a.State = eval.Pending
}
}
}
return a
}
func (a AlertState) resultError(alertRule *ngModels.AlertRule, result eval.Result) AlertState {
if a.StartsAt.IsZero() {
a.StartsAt = result.EvaluatedAt
}
if !(alertRule.For > 0) {
a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second)
} else {
a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
}
switch alertRule.ExecErrState {
case ngModels.AlertingErrState:
a.State = eval.Alerting
case ngModels.KeepLastStateErrState:
}
return a
}
func (a AlertState) resultNoData(alertRule *ngModels.AlertRule, result eval.Result) AlertState {
if a.StartsAt.IsZero() {
a.StartsAt = result.EvaluatedAt
}
if !(alertRule.For > 0) {
a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second)
} else {
a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
}
switch alertRule.NoDataState {
case ngModels.Alerting:
a.State = eval.Alerting
case ngModels.NoData:
a.State = eval.NoData
case ngModels.KeepLastState:
case ngModels.OK:
a.State = eval.Normal
}
return a
}
func (st *StateTracker) GetAll() []AlertState {

907
pkg/services/ngalert/tests/state_tracker_test.go
View File

@ -1,7 +1,6 @@
package tests
import (
"fmt"
"testing"
"time"
@ -16,276 +15,748 @@ import (
)
func TestProcessEvalResults(t *testing.T) {
t.Skip()
evaluationTime, err := time.Parse("2006-01-02", "2021-03-25")
if err != nil {
t.Fatalf("error parsing date format: %s", err.Error())
}
cacheId := "map[__alert_rule_namespace_uid__:test_namespace __alert_rule_uid__:test_uid alertname:test_title label1:value1 label2:value2 rule_label:rule_value]"
evaluationDuration := 10 * time.Millisecond
ruleLabels := map[string]string{
"rule_label": "rule_value",
}
alertRule := models.AlertRule{
ID: 1,
OrgID: 123,
Title: "test_title",
Condition: "A",
UID: "test_uid",
NamespaceUID: "test_namespace",
For: 10 * time.Second,
Labels: ruleLabels,
}
processingTime := 10 * time.Millisecond
expectedLabels := data.Labels{
"label1": "value1",
"label2": "value2",
"rule_label": "rule_value",
"__alert_rule_uid__": "test_uid",
"__alert_rule_namespace_uid__": "test_namespace",
"alertname": "test_title",
}
testCases := []struct {
desc string
uid string
evalResults eval.Results
expectedState eval.State
expectedReturnedStateCount int
expectedResultCount int
expectedCacheEntries []state.AlertState
desc string
alertRule *models.AlertRule
evalResults []eval.Results
expectedStates map[string]state.AlertState
}{
{
desc: "given a single evaluation result",
uid: "test_uid",
evalResults: eval.Results{
eval.Result{
Instance: data.Labels{"label1": "value1", "label2": "value2"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
desc: "a cache entry is correctly created",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid",
NamespaceUID: "test_namespace_uid",
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
},
},
},
expectedState: eval.Normal,
expectedReturnedStateCount: 0,
expectedResultCount: 1,
expectedCacheEntries: []state.AlertState{
{
AlertRuleUID: "test_uid",
OrgID: 123,
CacheId: cacheId,
Labels: expectedLabels,
State: eval.Normal,
Results: []state.StateEvaluation{
{EvaluationTime: evaluationTime, EvaluationState: eval.Normal},
expectedStates: map[string]state.AlertState{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label:test label:test]": {
AlertRuleUID: "test_alert_rule_uid",
OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label:test label:test]",
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid",
"alertname": "test_title",
"label": "test",
"instance_label": "test",
},
State: eval.Normal,
Results: []state.StateEvaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
},
},
StartsAt: time.Time{},
EndsAt: time.Time{},
LastEvaluationTime: evaluationTime,
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "given a state change from normal to alerting for a single entity",
uid: "test_uid",
evalResults: eval.Results{
eval.Result{
Instance: data.Labels{"label1": "value1", "label2": "value2"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
},
eval.Result{
Instance: data.Labels{"label1": "value1", "label2": "value2"},
State: eval.Alerting,
EvaluatedAt: evaluationTime.Add(1 * time.Minute),
desc: "two results create two correct cache entries",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid",
NamespaceUID: "test_namespace_uid",
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label_1": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
},
eval.Result{
Instance: data.Labels{"instance_label_2": "test"},
State: eval.Alerting,
EvaluatedAt: evaluationTime,
},
},
},
expectedState: eval.Alerting,
expectedReturnedStateCount: 1,
expectedResultCount: 2,
expectedCacheEntries: []state.AlertState{
{
AlertRuleUID: "test_uid",
OrgID: 123,
CacheId: cacheId,
Labels: expectedLabels,
State: eval.Alerting,
expectedStates: map[string]state.AlertState{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label_1:test label:test]": {
AlertRuleUID: "test_alert_rule_uid",
OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label_1:test label:test]",
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid",
"alertname": "test_title",
"label": "test",
"instance_label_1": "test",
},
State: eval.Normal,
Results: []state.StateEvaluation{
{EvaluationTime: evaluationTime, EvaluationState: eval.Normal},
{EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Alerting},
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
},
},
LastEvaluationTime: evaluationTime,
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label_2:test label:test]": {
AlertRuleUID: "test_alert_rule_uid",
OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label_2:test label:test]",
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid",
"alertname": "test_title",
"label": "test",
"instance_label_2": "test",
},
State: eval.Alerting,
Results: []state.StateEvaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Alerting,
},
},
StartsAt: evaluationTime,
EndsAt: evaluationTime.Add(20 * time.Second),
LastEvaluationTime: evaluationTime,
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "state is maintained",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_1",
NamespaceUID: "test_namespace_uid",
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime.Add(1 * time.Minute),
},
},
},
expectedStates: map[string]state.AlertState{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_1 alertname:test_title instance_label:test label:test]": {
AlertRuleUID: "test_alert_rule_uid_1",
OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_1 alertname:test_title instance_label:test label:test]",
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_1",
"alertname": "test_title",
"label": "test",
"instance_label": "test",
},
State: eval.Normal,
Results: []state.StateEvaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
},
{
EvaluationTime: evaluationTime.Add(1 * time.Minute),
EvaluationState: eval.Normal,
},
},
LastEvaluationTime: evaluationTime.Add(1 * time.Minute),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "normal -> alerting transition when For is unset",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_2",
NamespaceUID: "test_namespace_uid",
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Alerting,
EvaluatedAt: evaluationTime.Add(1 * time.Minute),
},
},
},
expectedStates: map[string]state.AlertState{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": {
AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]",
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2",
"alertname": "test_title",
"label": "test",
"instance_label": "test",
},
State: eval.Alerting,
Results: []state.StateEvaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
},
{
EvaluationTime: evaluationTime.Add(1 * time.Minute),
EvaluationState: eval.Alerting,
},
},
StartsAt: evaluationTime.Add(1 * time.Minute),
EndsAt: evaluationTime.Add(alertRule.For * time.Second).Add(1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute).Add(time.Duration(20) * time.Second),
LastEvaluationTime: evaluationTime.Add(1 * time.Minute),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "given a state change from alerting to normal for a single entity",
uid: "test_uid",
evalResults: eval.Results{
eval.Result{
Instance: data.Labels{"label1": "value1", "label2": "value2"},
State: eval.Alerting,
EvaluatedAt: evaluationTime,
desc: "normal -> alerting when For is set",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_2",
NamespaceUID: "test_namespace_uid",
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
For: 1 * time.Minute,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
},
},
eval.Result{
Instance: data.Labels{"label1": "value1", "label2": "value2"},
State: eval.Normal,
EvaluatedAt: evaluationTime.Add(1 * time.Minute),
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Alerting,
EvaluatedAt: evaluationTime.Add(10 * time.Second),
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Alerting,
EvaluatedAt: evaluationTime.Add(80 * time.Second),
},
},
},
expectedState: eval.Normal,
expectedReturnedStateCount: 1,
expectedResultCount: 2,
expectedCacheEntries: []state.AlertState{
{
AlertRuleUID: "test_uid",
OrgID: 123,
CacheId: cacheId,
Labels: expectedLabels,
State: eval.Normal,
Results: []state.StateEvaluation{
{EvaluationTime: evaluationTime, EvaluationState: eval.Alerting},
{EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Normal},
expectedStates: map[string]state.AlertState{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": {
AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]",
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2",
"alertname": "test_title",
"label": "test",
"instance_label": "test",
},
StartsAt: evaluationTime,
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime.Add(1 * time.Minute),
State: eval.Alerting,
Results: []state.StateEvaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
},
{
EvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationState: eval.Alerting,
},
{
EvaluationTime: evaluationTime.Add(80 * time.Second),
EvaluationState: eval.Alerting,
},
},
StartsAt: evaluationTime.Add(80 * time.Second),
EndsAt: evaluationTime.Add(80 * time.Second).Add(1 * time.Minute),
LastEvaluationTime: evaluationTime.Add(80 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "given a constant alerting state for a single entity",
uid: "test_uid",
evalResults: eval.Results{
eval.Result{
Instance: data.Labels{"label1": "value1", "label2": "value2"},
State: eval.Alerting,
EvaluatedAt: evaluationTime,
desc: "normal -> pending when For is set but not exceeded",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_2",
NamespaceUID: "test_namespace_uid",
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
For: 1 * time.Minute,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
},
},
eval.Result{
Instance: data.Labels{"label1": "value1", "label2": "value2"},
State: eval.Alerting,
EvaluatedAt: evaluationTime.Add(1 * time.Minute),
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Alerting,
EvaluatedAt: evaluationTime.Add(10 * time.Second),
},
},
},
expectedState: eval.Alerting,
expectedReturnedStateCount: 0,
expectedResultCount: 2,
expectedCacheEntries: []state.AlertState{
{
AlertRuleUID: "test_uid",
OrgID: 123,
CacheId: cacheId,
Labels: expectedLabels,
State: eval.Alerting,
Results: []state.StateEvaluation{
{EvaluationTime: evaluationTime, EvaluationState: eval.Alerting},
{EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Alerting},
expectedStates: map[string]state.AlertState{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": {
AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]",
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2",
"alertname": "test_title",
"label": "test",
"instance_label": "test",
},
StartsAt: evaluationTime,
EndsAt: evaluationTime.Add(alertRule.For * time.Second).Add(1 * time.Minute),
LastEvaluationTime: evaluationTime.Add(1 * time.Minute),
State: eval.Pending,
Results: []state.StateEvaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
},
{
EvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationState: eval.Alerting,
},
},
StartsAt: evaluationTime.Add(10 * time.Second),
EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute),
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "given a constant normal state for a single entity",
uid: "test_uid",
evalResults: eval.Results{
eval.Result{
Instance: data.Labels{"label1": "value1", "label2": "value2"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
desc: "normal -> alerting when result is NoData and NoDataState is alerting",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_2",
NamespaceUID: "test_namespace_uid",
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
NoDataState: models.Alerting,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
},
},
eval.Result{
Instance: data.Labels{"label1": "value1", "label2": "value2"},
State: eval.Normal,
EvaluatedAt: evaluationTime.Add(1 * time.Minute),
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.NoData,
EvaluatedAt: evaluationTime.Add(10 * time.Second),
},
},
},
expectedState: eval.Normal,
expectedReturnedStateCount: 0,
expectedResultCount: 2,
expectedCacheEntries: []state.AlertState{
{
AlertRuleUID: "test_uid",
OrgID: 123,
CacheId: cacheId,
Labels: expectedLabels,
State: eval.Normal,
Results: []state.StateEvaluation{
{EvaluationTime: evaluationTime, EvaluationState: eval.Normal},
{EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Normal},
expectedStates: map[string]state.AlertState{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": {
AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]",
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2",
"alertname": "test_title",
"label": "test",
"instance_label": "test",
},
StartsAt: time.Time{},
EndsAt: time.Time{},
LastEvaluationTime: evaluationTime.Add(1 * time.Minute),
State: eval.Alerting,
Results: []state.StateEvaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
},
{
EvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationState: eval.NoData,
},
},
StartsAt: evaluationTime.Add(10 * time.Second),
EndsAt: evaluationTime.Add(10 * time.Second).Add(20 * time.Second),
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "normal -> nodata when result is NoData and NoDataState is nodata",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_2",
NamespaceUID: "test_namespace_uid",
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
NoDataState: models.NoData,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.NoData,
EvaluatedAt: evaluationTime.Add(10 * time.Second),
},
},
},
expectedStates: map[string]state.AlertState{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": {
AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]",
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2",
"alertname": "test_title",
"label": "test",
"instance_label": "test",
},
State: eval.NoData,
Results: []state.StateEvaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
},
{
EvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationState: eval.NoData,
},
},
StartsAt: evaluationTime.Add(10 * time.Second),
EndsAt: evaluationTime.Add(10 * time.Second).Add(20 * time.Second),
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "normal -> normal when result is NoData and NoDataState is ok",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_2",
NamespaceUID: "test_namespace_uid",
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
NoDataState: models.OK,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.NoData,
EvaluatedAt: evaluationTime.Add(10 * time.Second),
},
},
},
expectedStates: map[string]state.AlertState{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": {
AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]",
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2",
"alertname": "test_title",
"label": "test",
"instance_label": "test",
},
State: eval.Normal,
Results: []state.StateEvaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
},
{
EvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationState: eval.NoData,
},
},
StartsAt: evaluationTime.Add(10 * time.Second),
EndsAt: evaluationTime.Add(10 * time.Second).Add(20 * time.Second),
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "EndsAt set correctly. normal -> alerting when result is NoData and NoDataState is alerting and For is set and For is breached",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_2",
NamespaceUID: "test_namespace_uid",
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
For: 1 * time.Minute,
NoDataState: models.Alerting,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.NoData,
EvaluatedAt: evaluationTime.Add(10 * time.Second),
},
},
},
expectedStates: map[string]state.AlertState{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": {
AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]",
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2",
"alertname": "test_title",
"label": "test",
"instance_label": "test",
},
State: eval.Alerting,
Results: []state.StateEvaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
},
{
EvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationState: eval.NoData,
},
},
StartsAt: evaluationTime.Add(10 * time.Second),
EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute),
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "normal -> normal when result is NoData and NoDataState is KeepLastState",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_2",
NamespaceUID: "test_namespace_uid",
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
For: 1 * time.Minute,
NoDataState: models.KeepLastState,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.NoData,
EvaluatedAt: evaluationTime.Add(10 * time.Second),
},
},
},
expectedStates: map[string]state.AlertState{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": {
AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]",
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2",
"alertname": "test_title",
"label": "test",
"instance_label": "test",
},
State: eval.Normal,
Results: []state.StateEvaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
},
{
EvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationState: eval.NoData,
},
},
StartsAt: evaluationTime.Add(10 * time.Second),
EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute),
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "normal -> normal when result is NoData and NoDataState is KeepLastState",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_2",
NamespaceUID: "test_namespace_uid",
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
For: 1 * time.Minute,
NoDataState: models.KeepLastState,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.NoData,
EvaluatedAt: evaluationTime.Add(10 * time.Second),
},
},
},
expectedStates: map[string]state.AlertState{
"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": {
AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1,
CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]",
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2",
"alertname": "test_title",
"label": "test",
"instance_label": "test",
},
State: eval.Normal,
Results: []state.StateEvaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
},
{
EvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationState: eval.NoData,
},
},
StartsAt: evaluationTime.Add(10 * time.Second),
EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute),
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
}
for _, tc := range testCases {
t.Run("all fields for a cache entry are set correctly", func(t *testing.T) {
st := state.NewStateTracker(log.New("test_state_tracker"))
_ = st.ProcessEvalResults(&alertRule, tc.evalResults, processingTime)
for _, entry := range tc.expectedCacheEntries {
if !entry.Equals(st.Get(entry.CacheId)) {
t.Log(tc.desc)
printEntryDiff(entry, st.Get(entry.CacheId), t)
}
assert.True(t, entry.Equals(st.Get(entry.CacheId)))
st := state.NewStateTracker(log.New("test_state_tracker"))
t.Run(tc.desc, func(t *testing.T) {
for _, res := range tc.evalResults {
_ = st.ProcessEvalResults(tc.alertRule, res, evaluationDuration)
}
for id, s := range tc.expectedStates {
assert.Equal(t, s, st.Get(id))
}
})
t.Run("the expected number of entries are added to the cache", func(t *testing.T) {
st := state.NewStateTracker(log.New("test_state_tracker"))
st.ProcessEvalResults(&alertRule, tc.evalResults, processingTime)
assert.Equal(t, len(tc.expectedCacheEntries), len(st.GetAll()))
})
//This test, as configured, does not quite represent the behavior of the system.
//It is expected that each batch of evaluation results will have only one result
//for a unique set of labels.
t.Run("the expected number of states are returned to the caller", func(t *testing.T) {
st := state.NewStateTracker(log.New("test_state_tracker"))
results := st.ProcessEvalResults(&alertRule, tc.evalResults, processingTime)
assert.Equal(t, len(tc.evalResults), len(results))
})
}
}
func printEntryDiff(a, b state.AlertState, t *testing.T) {
if a.AlertRuleUID != b.AlertRuleUID {
t.Log(fmt.Sprintf("%v \t %v\n", a.AlertRuleUID, b.AlertRuleUID))
}
if a.OrgID != b.OrgID {
t.Log(fmt.Sprintf("%v \t %v\n", a.OrgID, b.OrgID))
}
if a.CacheId != b.CacheId {
t.Log(fmt.Sprintf("%v \t %v\n", a.CacheId, b.CacheId))
}
if !a.Labels.Equals(b.Labels) {
t.Log(fmt.Sprintf("%v \t %v\n", a.Labels, b.Labels))
}
if a.StartsAt != b.StartsAt {
t.Log(fmt.Sprintf("%v \t %v\n", a.StartsAt, b.StartsAt))
}
if a.EndsAt != b.EndsAt {
t.Log(fmt.Sprintf("%v \t %v\n", a.EndsAt, b.EndsAt))
}
if a.LastEvaluationTime != b.LastEvaluationTime {
t.Log(fmt.Sprintf("%v \t %v\n", a.LastEvaluationTime, b.LastEvaluationTime))
}
if len(a.Results) != len(b.Results) {
t.Log(fmt.Sprintf("a: %d b: %d", len(a.Results), len(b.Results)))
t.Log("a")
for i := 0; i < len(a.Results); i++ {
t.Log(fmt.Sprintf("%v\n", a.Results[i]))
}
t.Log("b")
for i := 0; i < len(b.Results); i++ {
t.Log(fmt.Sprintf("%v\n", b.Results[i]))
}
}
}

238
pkg/tests/api/alerting/api_prometheus_test.go Normal file
View File

@ -0,0 +1,238 @@
package alerting
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"testing"
"time"
"github.com/grafana/grafana/pkg/models"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/tests/testinfra"
"github.com/prometheus/common/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestPrometheusRules(t *testing.T) {
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
EnableFeatureToggles: []string{"ngalert"},
AnonymousUserRole: models.ROLE_EDITOR,
})
store := testinfra.SetUpDatabase(t, dir)
grafanaListedAddr := testinfra.StartGrafana(t, dir, path, store)
// Create the namespace we'll save our alerts to.
require.NoError(t, createFolder(t, store, 0, "default"))
interval, err := model.ParseDuration("10s")
require.NoError(t, err)
// When we have no alerting rules, it returns an empty list.
{
promRulesURL := fmt.Sprintf("http://%s/api/prometheus/grafana/api/v1/rules", grafanaListedAddr)
// nolint:gosec
resp, err := http.Get(promRulesURL)
require.NoError(t, err)
t.Cleanup(func() {
err := resp.Body.Close()
require.NoError(t, err)
})
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
assert.Equal(t, 200, resp.StatusCode)
require.JSONEq(t, `{"status": "success", "data": {"groups": []}}`, string(b))
}
// Now, let's create some rules
{
rules := apimodels.PostableRuleGroupConfig{
Name: "arulegroup",
Rules: []apimodels.PostableExtendedRuleNode{
{
ApiRuleNode: &apimodels.ApiRuleNode{
For: interval,
Labels: map[string]string{"label1": "val1"},
Annotations: map[string]string{"annotation1": "val1"},
},
// this rule does not explicitly set no data and error states
// therefore it should get the default values
GrafanaManagedAlert: &apimodels.PostableGrafanaRule{
Title: "AlwaysFiring",
Condition: "A",
Data: []ngmodels.AlertQuery{
{
RefID: "A",
RelativeTimeRange: ngmodels.RelativeTimeRange{
From: ngmodels.Duration(time.Duration(5) * time.Hour),
To: ngmodels.Duration(time.Duration(3) * time.Hour),
},
Model: json.RawMessage(`{
"datasourceUid": "-100",
"type": "math",
"expression": "2 + 3 > 1"
}`),
},
},
},
},
{
GrafanaManagedAlert: &apimodels.PostableGrafanaRule{
Title: "AlwaysFiringButSilenced",
Condition: "A",
Data: []ngmodels.AlertQuery{
{
RefID: "A",
RelativeTimeRange: ngmodels.RelativeTimeRange{
From: ngmodels.Duration(time.Duration(5) * time.Hour),
To: ngmodels.Duration(time.Duration(3) * time.Hour),
},
Model: json.RawMessage(`{
"datasourceUid": "-100",
"type": "math",
"expression": "2 + 3 > 1"
}`),
},
},
NoDataState: apimodels.NoDataState(ngmodels.Alerting),
ExecErrState: apimodels.ExecutionErrorState(ngmodels.KeepLastStateErrState),
},
},
},
}
buf := bytes.Buffer{}
enc := json.NewEncoder(&buf)
err := enc.Encode(&rules)
require.NoError(t, err)
u := fmt.Sprintf("http://%s/api/ruler/grafana/api/v1/rules/default", grafanaListedAddr)
// nolint:gosec
resp, err := http.Post(u, "application/json", &buf)
require.NoError(t, err)
t.Cleanup(func() {
err := resp.Body.Close()
require.NoError(t, err)
})
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
assert.Equal(t, resp.StatusCode, 202)
require.JSONEq(t, `{"message":"rule group updated successfully"}`, string(b))
}
// Now, let's see how this looks like.
{
promRulesURL := fmt.Sprintf("http://%s/api/prometheus/grafana/api/v1/rules", grafanaListedAddr)
// nolint:gosec
resp, err := http.Get(promRulesURL)
require.NoError(t, err)
t.Cleanup(func() {
err := resp.Body.Close()
require.NoError(t, err)
})
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.Equal(t, 200, resp.StatusCode)
require.JSONEq(t, `
{
"status": "success",
"data": {
"groups": [{
"name": "arulegroup",
"file": "default",
"rules": [{
"state": "inactive",
"name": "AlwaysFiring",
"query": "[{\"datasourceUid\":\"-100\",\"expression\":\"2 + 3 \\u003e 1\",\"intervalMs\":1000,\"maxDataPoints\":100,\"type\":\"math\"}]",
"duration": 10,
"annotations": {
"annotation1": "val1"
},
"labels": {
"label1": "val1"
},
"health": "ok",
"lastError": "",
"type": "alerting",
"lastEvaluation": "0001-01-01T00:00:00Z",
"evaluationTime": 0
}, {
"state": "inactive",
"name": "AlwaysFiringButSilenced",
"query": "[{\"datasourceUid\":\"-100\",\"expression\":\"2 + 3 \\u003e 1\",\"intervalMs\":1000,\"maxDataPoints\":100,\"type\":\"math\"}]",
"labels": null,
"health": "ok",
"lastError": "",
"type": "alerting",
"lastEvaluation": "0001-01-01T00:00:00Z",
"evaluationTime": 0
}],
"interval": 60,
"lastEvaluation": "0001-01-01T00:00:00Z",
"evaluationTime": 0
}]
}
}`, string(b))
}
{
promRulesURL := fmt.Sprintf("http://%s/api/prometheus/grafana/api/v1/rules", grafanaListedAddr)
// nolint:gosec
require.Eventually(t, func() bool {
resp, err := http.Get(promRulesURL)
require.NoError(t, err)
t.Cleanup(func() {
err := resp.Body.Close()
require.NoError(t, err)
})
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.Equal(t, 200, resp.StatusCode)
require.JSONEq(t, `
{
"status": "success",
"data": {
"groups": [{
"name": "arulegroup",
"file": "default",
"rules": [{
"state": "inactive",
"name": "AlwaysFiring",
"query": "[{\"datasourceUid\":\"-100\",\"expression\":\"2 + 3 \\u003e 1\",\"intervalMs\":1000,\"maxDataPoints\":100,\"type\":\"math\"}]",
"duration": 10,
"annotations": {
"annotation1": "val1"
},
"labels": {
"label1": "val1"
},
"health": "ok",
"lastError": "",
"type": "alerting",
"lastEvaluation": "0001-01-01T00:00:00Z",
"evaluationTime": 0
}, {
"state": "inactive",
"name": "AlwaysFiringButSilenced",
"query": "[{\"datasourceUid\":\"-100\",\"expression\":\"2 + 3 \\u003e 1\",\"intervalMs\":1000,\"maxDataPoints\":100,\"type\":\"math\"}]",
"labels": null,
"health": "ok",
"lastError": "",
"type": "alerting",
"lastEvaluation": "0001-01-01T00:00:00Z",
"evaluationTime": 0
}],
"interval": 60,
"lastEvaluation": "0001-01-01T00:00:00Z",
"evaluationTime": 0
}]
}
}`, string(b))
return true
}, 18*time.Second, 2*time.Second)
}
}