mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
3228b64fe6
* Simple replace of State.Resolved with State.ResolvedAt
* Retain ResolvedAt time between Normal->Normal transition
* Introduce ResolvedRetention to keep sending recently resolved alerts
* Make ResolvedRetention configurable with resolved_alert_retention
* Tick-based LastSentAt for testing of ResendDelay and ResolvedRetention
* Do not reset ResolvedAt during Normal->Pending transition
Initially this was done to be inline with Prom ruler. However, Prom ruler
doesn't keep track of Inactive->Pending/Alerting using the same alert instance,
so it's more understandable that they choose not to retain ResolvedAt. In our
case, since we use the same cached instance to represent the transition, it
makes more sense to retain it.
This should help alleviate some odd situations where temporarily entering
Pending will stop future resolved notifications that would have happened
because of ResolvedRetention.
* Pointers for ResolvedAt & LastSentAt
To avoid awkward time.Time{}.Unix() defaults on persist
352 lines
12 KiB
Go
352 lines
12 KiB
Go
package state
|
|
|
|
import (
|
|
"fmt"
|
|
"math/rand"
|
|
"net/url"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/benbjohnson/clock"
|
|
"github.com/go-openapi/strfmt"
|
|
"github.com/grafana/grafana-plugin-sdk-go/data"
|
|
"github.com/prometheus/alertmanager/api/v2/models"
|
|
"github.com/prometheus/common/model"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
alertingModels "github.com/grafana/alerting/models"
|
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/eval"
|
|
ngModels "github.com/grafana/grafana/pkg/services/ngalert/models"
|
|
"github.com/grafana/grafana/pkg/util"
|
|
)
|
|
|
|
func Test_StateToPostableAlert(t *testing.T) {
|
|
appURL := &url.URL{
|
|
Scheme: "http:",
|
|
Host: fmt.Sprintf("host-%d", rand.Int()),
|
|
Path: fmt.Sprintf("path-%d", rand.Int()),
|
|
}
|
|
|
|
testCases := []struct {
|
|
name string
|
|
state eval.State
|
|
}{
|
|
{
|
|
name: "when state is Normal",
|
|
state: eval.Normal,
|
|
},
|
|
{
|
|
name: "when state is Alerting",
|
|
state: eval.Alerting,
|
|
},
|
|
{
|
|
name: "when state is Pending",
|
|
state: eval.Pending,
|
|
},
|
|
{
|
|
name: "when state is NoData",
|
|
state: eval.NoData,
|
|
},
|
|
{
|
|
name: "when state is Error",
|
|
state: eval.Error,
|
|
},
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Run("it generates proper URL", func(t *testing.T) {
|
|
t.Run("to alert rule", func(t *testing.T) {
|
|
alertState := randomTransition(eval.Normal, tc.state)
|
|
alertState.Labels[alertingModels.RuleUIDLabel] = alertState.AlertRuleUID
|
|
result := StateToPostableAlert(alertState, appURL)
|
|
u := *appURL
|
|
u.Path = u.Path + "/alerting/grafana/" + alertState.AlertRuleUID + "/view"
|
|
require.Equal(t, u.String(), result.Alert.GeneratorURL.String())
|
|
})
|
|
|
|
t.Run("app URL as is if rule UID is not specified", func(t *testing.T) {
|
|
alertState := randomTransition(eval.Normal, tc.state)
|
|
alertState.Labels[alertingModels.RuleUIDLabel] = ""
|
|
result := StateToPostableAlert(alertState, appURL)
|
|
require.Equal(t, appURL.String(), result.Alert.GeneratorURL.String())
|
|
|
|
delete(alertState.Labels, alertingModels.RuleUIDLabel)
|
|
result = StateToPostableAlert(alertState, appURL)
|
|
require.Equal(t, appURL.String(), result.Alert.GeneratorURL.String())
|
|
})
|
|
|
|
t.Run("empty string if app URL is not provided", func(t *testing.T) {
|
|
alertState := randomTransition(eval.Normal, tc.state)
|
|
alertState.Labels[alertingModels.RuleUIDLabel] = alertState.AlertRuleUID
|
|
result := StateToPostableAlert(alertState, nil)
|
|
require.Equal(t, "", result.Alert.GeneratorURL.String())
|
|
})
|
|
})
|
|
|
|
t.Run("Start and End timestamps should be the same", func(t *testing.T) {
|
|
alertState := randomTransition(eval.Normal, tc.state)
|
|
result := StateToPostableAlert(alertState, appURL)
|
|
require.Equal(t, strfmt.DateTime(alertState.StartsAt), result.StartsAt)
|
|
require.Equal(t, strfmt.DateTime(alertState.EndsAt), result.EndsAt)
|
|
})
|
|
|
|
t.Run("should copy annotations", func(t *testing.T) {
|
|
alertState := randomTransition(eval.Normal, tc.state)
|
|
alertState.Annotations = randomMapOfStrings()
|
|
result := StateToPostableAlert(alertState, appURL)
|
|
require.Equal(t, models.LabelSet(alertState.Annotations), result.Annotations)
|
|
|
|
t.Run("add __value_string__ if it has results", func(t *testing.T) {
|
|
alertState := randomTransition(eval.Normal, tc.state)
|
|
alertState.Annotations = randomMapOfStrings()
|
|
expectedValueString := util.GenerateShortUID()
|
|
alertState.LastEvaluationString = expectedValueString
|
|
|
|
result := StateToPostableAlert(alertState, appURL)
|
|
|
|
expected := make(models.LabelSet, len(alertState.Annotations)+1)
|
|
for k, v := range alertState.Annotations {
|
|
expected[k] = v
|
|
}
|
|
expected["__value_string__"] = expectedValueString
|
|
|
|
require.Equal(t, expected, result.Annotations)
|
|
|
|
// even overwrites
|
|
alertState.Annotations["__value_string__"] = util.GenerateShortUID()
|
|
result = StateToPostableAlert(alertState, appURL)
|
|
require.Equal(t, expected, result.Annotations)
|
|
})
|
|
|
|
t.Run("add __alertImageToken__ if there is an image token", func(t *testing.T) {
|
|
alertState := randomTransition(eval.Normal, tc.state)
|
|
alertState.Annotations = randomMapOfStrings()
|
|
alertState.Image = &ngModels.Image{Token: "test_token"}
|
|
|
|
result := StateToPostableAlert(alertState, appURL)
|
|
|
|
expected := make(models.LabelSet, len(alertState.Annotations)+1)
|
|
for k, v := range alertState.Annotations {
|
|
expected[k] = v
|
|
}
|
|
expected["__alertImageToken__"] = alertState.Image.Token
|
|
|
|
require.Equal(t, expected, result.Annotations)
|
|
})
|
|
|
|
t.Run("don't add __alertImageToken__ if there's no image token", func(t *testing.T) {
|
|
alertState := randomTransition(eval.Normal, tc.state)
|
|
alertState.Annotations = randomMapOfStrings()
|
|
alertState.Image = &ngModels.Image{}
|
|
|
|
result := StateToPostableAlert(alertState, appURL)
|
|
|
|
expected := make(models.LabelSet, len(alertState.Annotations)+1)
|
|
for k, v := range alertState.Annotations {
|
|
expected[k] = v
|
|
}
|
|
|
|
require.Equal(t, expected, result.Annotations)
|
|
})
|
|
})
|
|
|
|
t.Run("should add state reason annotation if not empty", func(t *testing.T) {
|
|
alertState := randomTransition(eval.Normal, tc.state)
|
|
alertState.StateReason = "TEST_STATE_REASON"
|
|
result := StateToPostableAlert(alertState, appURL)
|
|
require.Equal(t, alertState.StateReason, result.Annotations[ngModels.StateReasonAnnotation])
|
|
})
|
|
|
|
switch tc.state {
|
|
case eval.NoData:
|
|
t.Run("should keep existing labels and change name", func(t *testing.T) {
|
|
alertState := randomTransition(eval.Normal, tc.state)
|
|
alertState.Labels = randomMapOfStrings()
|
|
alertName := util.GenerateShortUID()
|
|
alertState.Labels[model.AlertNameLabel] = alertName
|
|
|
|
result := StateToPostableAlert(alertState, appURL)
|
|
|
|
expected := make(models.LabelSet, len(alertState.Labels)+1)
|
|
for k, v := range alertState.Labels {
|
|
expected[k] = v
|
|
}
|
|
expected[model.AlertNameLabel] = NoDataAlertName
|
|
expected[Rulename] = alertName
|
|
|
|
require.Equal(t, expected, result.Labels)
|
|
|
|
t.Run("should not backup original alert name if it does not exist", func(t *testing.T) {
|
|
alertState := randomTransition(eval.Normal, tc.state)
|
|
alertState.Labels = randomMapOfStrings()
|
|
delete(alertState.Labels, model.AlertNameLabel)
|
|
|
|
result := StateToPostableAlert(alertState, appURL)
|
|
|
|
require.Equal(t, NoDataAlertName, result.Labels[model.AlertNameLabel])
|
|
require.NotContains(t, result.Labels[model.AlertNameLabel], Rulename)
|
|
})
|
|
})
|
|
case eval.Error:
|
|
t.Run("should keep existing labels and change name", func(t *testing.T) {
|
|
alertState := randomTransition(eval.Normal, tc.state)
|
|
alertState.Labels = randomMapOfStrings()
|
|
alertName := util.GenerateShortUID()
|
|
alertState.Labels[model.AlertNameLabel] = alertName
|
|
|
|
result := StateToPostableAlert(alertState, appURL)
|
|
|
|
expected := make(models.LabelSet, len(alertState.Labels)+1)
|
|
for k, v := range alertState.Labels {
|
|
expected[k] = v
|
|
}
|
|
expected[model.AlertNameLabel] = ErrorAlertName
|
|
expected[Rulename] = alertName
|
|
|
|
require.Equal(t, expected, result.Labels)
|
|
|
|
t.Run("should not backup original alert name if it does not exist", func(t *testing.T) {
|
|
alertState := randomTransition(eval.Normal, tc.state)
|
|
alertState.Labels = randomMapOfStrings()
|
|
delete(alertState.Labels, model.AlertNameLabel)
|
|
|
|
result := StateToPostableAlert(alertState, appURL)
|
|
|
|
require.Equal(t, ErrorAlertName, result.Labels[model.AlertNameLabel])
|
|
require.NotContains(t, result.Labels[model.AlertNameLabel], Rulename)
|
|
})
|
|
})
|
|
default:
|
|
t.Run("should copy labels as is", func(t *testing.T) {
|
|
alertState := randomTransition(eval.Normal, tc.state)
|
|
alertState.Labels = randomMapOfStrings()
|
|
result := StateToPostableAlert(alertState, appURL)
|
|
require.Equal(t, models.LabelSet(alertState.Labels), result.Labels)
|
|
})
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestStateToPostableAlertFromNodataError(t *testing.T) {
|
|
appURL := &url.URL{
|
|
Scheme: "http:",
|
|
Host: fmt.Sprintf("host-%d", rand.Int()),
|
|
Path: fmt.Sprintf("path-%d", rand.Int()),
|
|
}
|
|
|
|
standardLabels := models.LabelSet{model.AlertNameLabel: "name"}
|
|
noDataLabels := models.LabelSet{Rulename: "name", model.AlertNameLabel: NoDataAlertName}
|
|
errorLabels := models.LabelSet{Rulename: "name", model.AlertNameLabel: ErrorAlertName}
|
|
|
|
testCases := []struct {
|
|
name string
|
|
resolved bool
|
|
from eval.State
|
|
to eval.State
|
|
expectedLabels models.LabelSet
|
|
}{
|
|
// These are the important cases.
|
|
{name: "from NoData to Normal resolved", resolved: true, from: eval.NoData, to: eval.Normal, expectedLabels: noDataLabels},
|
|
{name: "from Error to Normal resolved", resolved: true, from: eval.Error, to: eval.Normal, expectedLabels: errorLabels},
|
|
|
|
// Regressions.
|
|
{name: "from NoData to Normal unresolved", resolved: false, from: eval.NoData, to: eval.Normal, expectedLabels: standardLabels},
|
|
{name: "from Error to Normal unresolved", resolved: false, from: eval.Error, to: eval.Normal, expectedLabels: standardLabels},
|
|
{name: "from NoData to Alerting unresolved", resolved: false, from: eval.NoData, to: eval.Alerting, expectedLabels: standardLabels},
|
|
{name: "from Error to Alerting unresolved", resolved: false, from: eval.Error, to: eval.Alerting, expectedLabels: standardLabels},
|
|
{name: "from NoData to Pending unresolved", resolved: false, from: eval.NoData, to: eval.Pending, expectedLabels: standardLabels},
|
|
{name: "from Error to Pending unresolved", resolved: false, from: eval.Error, to: eval.Pending, expectedLabels: standardLabels},
|
|
{name: "from NoData to NoData unresolved", resolved: false, from: eval.NoData, to: eval.NoData, expectedLabels: noDataLabels},
|
|
{name: "from Error to NoData unresolved", resolved: false, from: eval.Error, to: eval.NoData, expectedLabels: noDataLabels},
|
|
{name: "from NoData to Error unresolved", resolved: false, from: eval.NoData, to: eval.Error, expectedLabels: errorLabels},
|
|
{name: "from Error to Error unresolved", resolved: false, from: eval.Error, to: eval.Error, expectedLabels: errorLabels},
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
alertState := randomTransition(tc.from, tc.to)
|
|
if tc.resolved {
|
|
alertState.ResolvedAt = &alertState.LastEvaluationTime
|
|
}
|
|
alertState.Labels = data.Labels(standardLabels)
|
|
result := StateToPostableAlert(alertState, appURL)
|
|
require.Equal(t, tc.expectedLabels, result.Labels)
|
|
})
|
|
}
|
|
}
|
|
|
|
func Test_FromAlertsStateToStoppedAlert(t *testing.T) {
|
|
appURL := &url.URL{
|
|
Scheme: "http:",
|
|
Host: fmt.Sprintf("host-%d", rand.Int()),
|
|
Path: fmt.Sprintf("path-%d", rand.Int()),
|
|
}
|
|
|
|
evalStates := [...]eval.State{eval.Normal, eval.Alerting, eval.Pending, eval.Error, eval.NoData}
|
|
states := make([]StateTransition, 0, len(evalStates)*len(evalStates))
|
|
for _, to := range evalStates {
|
|
for _, from := range evalStates {
|
|
states = append(states, randomTransition(from, to))
|
|
}
|
|
}
|
|
|
|
clk := clock.NewMock()
|
|
clk.Set(time.Now())
|
|
|
|
expected := make([]models.PostableAlert, 0, len(states))
|
|
for _, s := range states {
|
|
if !(s.PreviousState == eval.Alerting || s.PreviousState == eval.Error || s.PreviousState == eval.NoData) {
|
|
continue
|
|
}
|
|
alert := StateToPostableAlert(s, appURL)
|
|
alert.EndsAt = strfmt.DateTime(clk.Now())
|
|
expected = append(expected, *alert)
|
|
}
|
|
|
|
result := FromAlertsStateToStoppedAlert(states, appURL, clk)
|
|
|
|
require.Equal(t, expected, result.PostableAlerts)
|
|
}
|
|
|
|
func randomMapOfStrings() map[string]string {
|
|
max := 5
|
|
result := make(map[string]string, max)
|
|
for i := 0; i < max; i++ {
|
|
result[util.GenerateShortUID()] = util.GenerateShortUID()
|
|
}
|
|
return result
|
|
}
|
|
|
|
func randomDuration() time.Duration {
|
|
return time.Duration(rand.Int63n(599)+1) * time.Second
|
|
}
|
|
|
|
func randomTimeInFuture() time.Time {
|
|
return time.Now().Add(randomDuration())
|
|
}
|
|
|
|
func randomTimeInPast() time.Time {
|
|
return time.Now().Add(-randomDuration())
|
|
}
|
|
|
|
func randomTransition(from, to eval.State) StateTransition {
|
|
return StateTransition{
|
|
PreviousState: from,
|
|
State: &State{
|
|
State: to,
|
|
AlertRuleUID: util.GenerateShortUID(),
|
|
StartsAt: time.Now(),
|
|
EndsAt: randomTimeInFuture(),
|
|
LastEvaluationTime: randomTimeInPast(),
|
|
EvaluationDuration: randomDuration(),
|
|
LastSentAt: util.Pointer(randomTimeInPast()),
|
|
Annotations: make(map[string]string),
|
|
Labels: make(map[string]string),
|
|
Values: make(map[string]float64),
|
|
},
|
|
}
|
|
}
|