IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2024-12-28 18:01:40 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Support hysteresis command expression (#75189)

Browse Source 
Backend: 

* Update the Grafana Alerting engine to provide feedback to HysteresisCommand. The feedback information is stored in state.Manager as a fingerprint of each state. The fingerprint is persisted to the database. Only fingerprints that belong to Pending and Alerting states are considered as "loaded" and provided back to the command.
   - add ResultFingerprint to state.State. It's different from other fingerprints we store in the state because it is calculated from the result labels.
  -  add rule_fingerprint column to alert_instance
   - update alerting evaluator to accept AlertingResultsReader via context, and update scheduler to provide it.
   - add AlertingResultsFromRuleState that implements the new interface in eval package
   - update getExprRequest to patch the hysteresis command.

* Only one "Recovery Threshold" query is allowed to be used in the alert rule and it must be the Condition.


Frontend:

* Add hysteresis option to Threshold in UI. It's called "Recovery Threshold"
* Add test for getUnloadEvaluatorTypeFromCondition
* Hide hysteresis in panel expressions

* Refactor isInvalid and add test for it
* Remove unnecesary React.memo
* Add tests for updateEvaluatorConditions

---------

Co-authored-by: Sonia Aguilar <soniaaguilarpeiron@gmail.com>
This commit is contained in:
Yuri Tseretyan 2024-01-04 11:47:13 -05:00 committed by GitHub
parent 29c251851d
commit f6a46744a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 1804 additions and 201 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.betterer.results
View File

@ -3715,10 +3715,6 @@ exports[`better eslint`] = {
[0, 0, 0, "Styles should be written using objects.", "5"],
[0, 0, 0, "Styles should be written using objects.", "6"]
],
"public/app/features/expressions/components/Threshold.tsx:5381": [
[0, 0, 0, "Styles should be written using objects.", "0"],
[0, 0, 0, "Styles should be written using objects.", "1"]
],
"public/app/features/expressions/guards.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],

17
pkg/services/ngalert/backtesting/engine.go
View File

@ -17,6 +17,7 @@ import (
"github.com/grafana/grafana/pkg/services/auth/identity"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/schedule"
"github.com/grafana/grafana/pkg/services/ngalert/state"
)
@ -35,6 +36,7 @@ type backtestingEvaluator interface {
type stateManager interface {
ProcessEvalResults(ctx context.Context, evaluatedAt time.Time, alertRule *models.AlertRule, results eval.Results, extraLabels data.Labels) []state.StateTransition
schedule.RuleStateProvider
}
type Engine struct {
@ -74,13 +76,16 @@ func (e *Engine) Test(ctx context.Context, user identity.Requester, rule *models
}
length := int(to.Sub(from).Seconds()) / int(rule.IntervalSeconds)
evaluator, err := backtestingEvaluatorFactory(ruleCtx, e.evalFactory, user, rule.GetEvalCondition())
stateManager := e.createStateManager()
evaluator, err := backtestingEvaluatorFactory(ruleCtx, e.evalFactory, user, rule.GetEvalCondition(), &schedule.AlertingResultsFromRuleState{
Manager: stateManager,
Rule: rule,
})
if err != nil {
return nil, errors.Join(ErrInvalidInputData, err)
}
stateManager := e.createStateManager()
logger.Info("Start testing alert rule", "from", from, "to", to, "interval", rule.IntervalSeconds, "evaluations", length)
start := time.Now()
@ -126,7 +131,7 @@ func (e *Engine) Test(ctx context.Context, user identity.Requester, rule *models
return result, nil
}
func newBacktestingEvaluator(ctx context.Context, evalFactory eval.EvaluatorFactory, user identity.Requester, condition models.Condition) (backtestingEvaluator, error) {
func newBacktestingEvaluator(ctx context.Context, evalFactory eval.EvaluatorFactory, user identity.Requester, condition models.Condition, reader eval.AlertingResultsReader) (backtestingEvaluator, error) {
for _, q := range condition.Data {
if q.DatasourceUID == "__data__" || q.QueryType == "__data__" {
if len(condition.Data) != 1 {
@ -152,9 +157,7 @@ func newBacktestingEvaluator(ctx context.Context, evalFactory eval.EvaluatorFact
}
}
evaluator, err := evalFactory.Create(eval.EvaluationContext{Ctx: ctx,
User: user,
}, condition)
evaluator, err := evalFactory.Create(eval.NewContextWithPreviousResults(ctx, user, reader), condition)
if err != nil {
return nil, err

8
pkg/services/ngalert/backtesting/engine_test.go
View File

@ -145,7 +145,7 @@ func TestNewBacktestingEvaluator(t *testing.T) {
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
e, err := newBacktestingEvaluator(context.Background(), evalFactory, nil, testCase.condition)
e, err := newBacktestingEvaluator(context.Background(), evalFactory, nil, testCase.condition, nil)
if testCase.error {
require.Error(t, err)
return
@ -175,7 +175,7 @@ func TestEvaluatorTest(t *testing.T) {
}
manager := &fakeStateManager{}
backtestingEvaluatorFactory = func(ctx context.Context, evalFactory eval.EvaluatorFactory, user identity.Requester, condition models.Condition) (backtestingEvaluator, error) {
backtestingEvaluatorFactory = func(ctx context.Context, evalFactory eval.EvaluatorFactory, user identity.Requester, condition models.Condition, r eval.AlertingResultsReader) (backtestingEvaluator, error) {
return evaluator, nil
}
@ -386,6 +386,10 @@ func (f *fakeStateManager) ProcessEvalResults(_ context.Context, evaluatedAt tim
return f.stateCallback(evaluatedAt)
}
func (f *fakeStateManager) GetStatesForRuleUID(orgID int64, alertRuleUID string) []*state.State {
return nil
}
type fakeBacktestingEvaluator struct {
evalCallback func(now time.Time) (eval.Results, error)
}

21
pkg/services/ngalert/eval/context.go
View File

@ -3,13 +3,22 @@ package eval
import (
"context"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/services/auth/identity"
)
// AlertingResultsReader provides fingerprints of results that are in alerting state.
// It is used during the evaluation of queries.
type AlertingResultsReader interface {
Read() map[data.Fingerprint]struct{}
}
// EvaluationContext represents the context in which a condition is evaluated.
type EvaluationContext struct {
Ctx context.Context
User identity.Requester
Ctx context.Context
User identity.Requester
AlertingResultsReader AlertingResultsReader
}
func NewContext(ctx context.Context, user identity.Requester) EvaluationContext {
@ -18,3 +27,11 @@ func NewContext(ctx context.Context, user identity.Requester) EvaluationContext
User: user,
}
}
func NewContextWithPreviousResults(ctx context.Context, user identity.Requester, reader AlertingResultsReader) EvaluationContext {
return EvaluationContext{
Ctx: ctx,
User: user,
AlertingResultsReader: reader,
}
}

65
pkg/services/ngalert/eval/eval.go
View File

@ -298,30 +298,16 @@ func buildDatasourceHeaders(ctx context.Context) map[string]string {
}
// getExprRequest validates the condition, gets the datasource information and creates an expr.Request from it.
func getExprRequest(ctx EvaluationContext, data []models.AlertQuery, dsCacheService datasources.CacheService) (*expr.Request, error) {
func getExprRequest(ctx EvaluationContext, condition models.Condition, dsCacheService datasources.CacheService, reader AlertingResultsReader) (*expr.Request, error) {
req := &expr.Request{
OrgId: ctx.User.GetOrgID(),
Headers: buildDatasourceHeaders(ctx.Ctx),
User: ctx.User,
}
datasources := make(map[string]*datasources.DataSource, len(condition.Data))
datasources := make(map[string]*datasources.DataSource, len(data))
for _, q := range data {
model, err := q.GetModel()
if err != nil {
return nil, fmt.Errorf("failed to get query model from '%s': %w", q.RefID, err)
}
interval, err := q.GetIntervalDuration()
if err != nil {
return nil, fmt.Errorf("failed to retrieve intervalMs from '%s': %w", q.RefID, err)
}
maxDatapoints, err := q.GetMaxDatapoints()
if err != nil {
return nil, fmt.Errorf("failed to retrieve maxDatapoints from '%s': %w", q.RefID, err)
}
for _, q := range condition.Data {
var err error
ds, ok := datasources[q.DatasourceUID]
if !ok {
switch nodeType := expr.NodeTypeFromDatasourceUID(q.DatasourceUID); nodeType {
@ -336,6 +322,45 @@ func getExprRequest(ctx EvaluationContext, data []models.AlertQuery, dsCacheServ
datasources[q.DatasourceUID] = ds
}
// TODO rewrite the code below and remove the mutable component from AlertQuery
// if the query is command expression and it's a hysteresis, patch it with the current state
// it's important to do this before GetModel
if ds.Type == expr.DatasourceType {
isHysteresis, err := q.IsHysteresisExpression()
if err != nil {
return nil, fmt.Errorf("failed to build query '%s': %w", q.RefID, err)
}
if isHysteresis {
// make sure we allow hysteresis expressions to be specified only as the alert condition.
// This guarantees us that the AlertResultsReader can be correctly applied to the expression tree.
if q.RefID != condition.Condition {
return nil, fmt.Errorf("recovery threshold '%s' is only allowed to be the alert condition", q.RefID)
}
if reader != nil {
logger.FromContext(ctx.Ctx).Debug("Detected hysteresis threshold command. Populating with the results")
err = q.PatchHysteresisExpression(reader.Read())
if err != nil {
return nil, fmt.Errorf("failed to amend hysteresis command '%s': %w", q.RefID, err)
}
}
}
}
model, err := q.GetModel()
if err != nil {
return nil, fmt.Errorf("failed to get query model from '%s': %w", q.RefID, err)
}
interval, err := q.GetIntervalDuration()
if err != nil {
return nil, fmt.Errorf("failed to retrieve intervalMs from '%s': %w", q.RefID, err)
}
maxDatapoints, err := q.GetMaxDatapoints()
if err != nil {
return nil, fmt.Errorf("failed to retrieve maxDatapoints from '%s': %w", q.RefID, err)
}
req.Queries = append(req.Queries, expr.Query{
TimeRange: q.RelativeTimeRange.ToTimeRange(),
DataSource: ds,
@ -724,7 +749,7 @@ func (evalResults Results) AsDataFrame() data.Frame {
}
func (e *evaluatorImpl) Validate(ctx EvaluationContext, condition models.Condition) error {
req, err := getExprRequest(ctx, condition.Data, e.dataSourceCache)
req, err := getExprRequest(ctx, condition, e.dataSourceCache, ctx.AlertingResultsReader)
if err != nil {
return err
}
@ -760,7 +785,7 @@ func (e *evaluatorImpl) Create(ctx EvaluationContext, condition models.Condition
if len(condition.Condition) == 0 {
return nil, errors.New("condition must not be empty")
}
req, err := getExprRequest(ctx, condition.Data, e.dataSourceCache)
req, err := getExprRequest(ctx, condition, e.dataSourceCache, ctx.AlertingResultsReader)
if err != nil {
return nil, err
}

180
pkg/services/ngalert/eval/eval_test.go
View File

@ -524,6 +524,59 @@ func TestValidate(t *testing.T) {
}
},
},
{
name: "fail if hysteresis command is not the condition",
error: true,
condition: func(services services) models.Condition {
dsQuery := models.GenerateAlertQuery()
ds := &datasources.DataSource{
UID: dsQuery.DatasourceUID,
Type: util.GenerateShortUID(),
}
services.cache.DataSources = append(services.cache.DataSources, ds)
services.pluginsStore.PluginList = append(services.pluginsStore.PluginList, pluginstore.Plugin{
JSONData: plugins.JSONData{
ID: ds.Type,
Backend: true,
},
})
return models.Condition{
Condition: "C",
Data: []models.AlertQuery{
dsQuery,
models.CreateHysteresisExpression(t, "B", dsQuery.RefID, 4, 1),
models.CreateClassicConditionExpression("C", "B", "last", "gt", rand.Int()),
},
}
},
},
{
name: "pass if hysteresis command and it is the condition",
error: false,
condition: func(services services) models.Condition {
dsQuery := models.GenerateAlertQuery()
ds := &datasources.DataSource{
UID: dsQuery.DatasourceUID,
Type: util.GenerateShortUID(),
}
services.cache.DataSources = append(services.cache.DataSources, ds)
services.pluginsStore.PluginList = append(services.pluginsStore.PluginList, pluginstore.Plugin{
JSONData: plugins.JSONData{
ID: ds.Type,
Backend: true,
},
})
return models.Condition{
Condition: "B",
Data: []models.AlertQuery{
dsQuery,
models.CreateHysteresisExpression(t, "B", dsQuery.RefID, 4, 1),
},
}
},
},
}
for _, testCase := range testCases {
@ -550,6 +603,133 @@ func TestValidate(t *testing.T) {
}
}
func TestCreate_HysteresisCommand(t *testing.T) {
type services struct {
cache *fakes.FakeCacheService
pluginsStore *pluginstore.FakePluginStore
}
testCases := []struct {
name string
reader AlertingResultsReader
condition func(services services) models.Condition
error bool
}{
{
name: "fail if hysteresis command is not the condition",
error: true,
condition: func(services services) models.Condition {
dsQuery := models.GenerateAlertQuery()
ds := &datasources.DataSource{
UID: dsQuery.DatasourceUID,
Type: util.GenerateShortUID(),
}
services.cache.DataSources = append(services.cache.DataSources, ds)
services.pluginsStore.PluginList = append(services.pluginsStore.PluginList, pluginstore.Plugin{
JSONData: plugins.JSONData{
ID: ds.Type,
Backend: true,
},
})
return models.Condition{
Condition: "C",
Data: []models.AlertQuery{
dsQuery,
models.CreateHysteresisExpression(t, "B", dsQuery.RefID, 4, 1),
models.CreateClassicConditionExpression("C", "B", "last", "gt", rand.Int()),
},
}
},
},
{
name: "populate with loaded metrics",
error: false,
reader: FakeLoadedMetricsReader{fingerprints: map[data.Fingerprint]struct{}{1: {}, 2: {}, 3: {}}},
condition: func(services services) models.Condition {
dsQuery := models.GenerateAlertQuery()
ds := &datasources.DataSource{
UID: dsQuery.DatasourceUID,
Type: util.GenerateShortUID(),
}
services.cache.DataSources = append(services.cache.DataSources, ds)
services.pluginsStore.PluginList = append(services.pluginsStore.PluginList, pluginstore.Plugin{
JSONData: plugins.JSONData{
ID: ds.Type,
Backend: true,
},
})
return models.Condition{
Condition: "B",
Data: []models.AlertQuery{
dsQuery,
models.CreateHysteresisExpression(t, "B", dsQuery.RefID, 4, 1),
},
}
},
},
{
name: "do nothing if reader is not specified",
error: false,
reader: nil,
condition: func(services services) models.Condition {
dsQuery := models.GenerateAlertQuery()
ds := &datasources.DataSource{
UID: dsQuery.DatasourceUID,
Type: util.GenerateShortUID(),
}
services.cache.DataSources = append(services.cache.DataSources, ds)
services.pluginsStore.PluginList = append(services.pluginsStore.PluginList, pluginstore.Plugin{
JSONData: plugins.JSONData{
ID: ds.Type,
Backend: true,
},
})
return models.Condition{
Condition: "B",
Data: []models.AlertQuery{
dsQuery,
models.CreateHysteresisExpression(t, "B", dsQuery.RefID, 4, 1),
},
}
},
},
}
for _, testCase := range testCases {
u := &user.SignedInUser{}
t.Run(testCase.name, func(t *testing.T) {
cacheService := &fakes.FakeCacheService{}
store := &pluginstore.FakePluginStore{}
condition := testCase.condition(services{
cache: cacheService,
pluginsStore: store,
})
evaluator := NewEvaluatorFactory(setting.UnifiedAlertingSettings{}, cacheService, expr.ProvideService(&setting.Cfg{ExpressionsEnabled: true}, nil, nil, featuremgmt.WithFeatures(featuremgmt.FlagRecoveryThreshold), nil, tracing.InitializeTracerForTest()), store)
evalCtx := NewContextWithPreviousResults(context.Background(), u, testCase.reader)
eval, err := evaluator.Create(evalCtx, condition)
if testCase.error {
require.Error(t, err)
return
}
require.IsType(t, &conditionEvaluator{}, eval)
ce := eval.(*conditionEvaluator)
cmds := expr.GetCommandsFromPipeline[*expr.HysteresisCommand](ce.pipeline)
require.Len(t, cmds, 1)
if testCase.reader == nil {
require.Empty(t, cmds[0].LoadedDimensions)
} else {
require.EqualValues(t, testCase.reader.Read(), cmds[0].LoadedDimensions)
}
})
}
}
func TestEvaluate(t *testing.T) {
cases := []struct {
name string

8
pkg/services/ngalert/eval/testing.go
View File

@ -79,3 +79,11 @@ func WithLabels(labels data.Labels) ResultMutator {
r.Instance = labels
}
}
type FakeLoadedMetricsReader struct {
fingerprints map[data.Fingerprint]struct{}
}
func (f FakeLoadedMetricsReader) Read() map[data.Fingerprint]struct{} {
return f.fingerprints
}

24
pkg/services/ngalert/models/alert_query.go
View File

@ -6,6 +6,8 @@ import (
"fmt"
"time"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/expr"
)
@ -116,6 +118,28 @@ func (aq *AlertQuery) IsExpression() (bool, error) {
return expr.NodeTypeFromDatasourceUID(aq.DatasourceUID) == expr.TypeCMDNode, nil
}
// IsHysteresisExpression returns true if the model describes a hysteresis command expression. Returns error if the Model is not a valid JSON
func (aq *AlertQuery) IsHysteresisExpression() (bool, error) {
if aq.modelProps == nil {
err := aq.setModelProps()
if err != nil {
return false, err
}
}
return expr.IsHysteresisExpression(aq.modelProps), nil
}
// PatchHysteresisExpression updates the AlertQuery to include loaded metrics into hysteresis
func (aq *AlertQuery) PatchHysteresisExpression(loadedMetrics map[data.Fingerprint]struct{}) error {
if aq.modelProps == nil {
err := aq.setModelProps()
if err != nil {
return err
}
}
return expr.SetLoadedDimensionsToHysteresisCommand(aq.modelProps, loadedMetrics)
}
// setMaxDatapoints sets the model maxDataPoints if it's missing or invalid
func (aq *AlertQuery) setMaxDatapoints() error {
if aq.modelProps == nil {

80
pkg/services/ngalert/models/alert_query_test.go
View File

@ -7,9 +7,11 @@ import (
"testing"
"time"
"github.com/grafana/grafana/pkg/expr"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/expr"
)
func TestAlertQuery(t *testing.T) {
@ -17,6 +19,7 @@ func TestAlertQuery(t *testing.T) {
desc string
alertQuery AlertQuery
expectedIsExpression bool
expectedIsHysteresis bool
expectedDatasource string
expectedMaxPoints int64
expectedIntervalMS int64
@ -133,6 +136,64 @@ func TestAlertQuery(t *testing.T) {
expectedMaxPoints: int64(defaultMaxDataPoints),
expectedIntervalMS: int64(defaultIntervalMS),
},
{
desc: "given a query with threshold expression",
alertQuery: AlertQuery{
RefID: "A",
DatasourceUID: expr.DatasourceType,
Model: json.RawMessage(`{
"type": "threshold",
"queryType": "metricQuery",
"extraParam": "some text",
"conditions": [
{
"evaluator": {
"params": [
4
],
"type": "gt"
}
}
]
}`),
},
expectedIsExpression: true,
expectedIsHysteresis: false,
expectedMaxPoints: int64(defaultMaxDataPoints),
expectedIntervalMS: int64(defaultIntervalMS),
},
{
desc: "given a query with hysteresis expression",
alertQuery: AlertQuery{
RefID: "A",
DatasourceUID: expr.DatasourceType,
Model: json.RawMessage(`{
"type": "threshold",
"queryType": "metricQuery",
"extraParam": "some text",
"conditions": [
{
"evaluator": {
"params": [
4
],
"type": "gt"
},
"unloadEvaluator": {
"params": [
2
],
"type": "lt"
}
}
]
}`),
},
expectedMaxPoints: int64(defaultMaxDataPoints),
expectedIntervalMS: int64(defaultIntervalMS),
expectedIsExpression: true,
expectedIsHysteresis: true,
},
}
for _, tc := range testCases {
@ -143,6 +204,12 @@ func TestAlertQuery(t *testing.T) {
assert.Equal(t, tc.expectedIsExpression, isExpression)
})
t.Run("can recognize if it's a hysteresis expression", func(t *testing.T) {
isExpression, err := tc.alertQuery.IsHysteresisExpression()
require.NoError(t, err)
assert.Equal(t, tc.expectedIsHysteresis, isExpression)
})
t.Run("can set queryType for expression", func(t *testing.T) {
err := tc.alertQuery.setQueryType()
require.NoError(t, err)
@ -186,6 +253,17 @@ func TestAlertQuery(t *testing.T) {
require.True(t, ok)
require.Equal(t, "some text", extraParam)
})
if tc.expectedIsHysteresis {
t.Run("can patch the command with loaded metrics", func(t *testing.T) {
require.NoError(t, tc.alertQuery.PatchHysteresisExpression(map[data.Fingerprint]struct{}{1: {}, 2: {}, 3: {}}))
data, ok := tc.alertQuery.modelProps["conditions"].([]any)[0].(map[string]any)["loadedDimensions"]
require.True(t, ok)
require.NotNil(t, data)
_, err := tc.alertQuery.GetModel()
require.NoError(t, err)
})
}
})
}
}

1
pkg/services/ngalert/models/instance.go
View File

@ -14,6 +14,7 @@ type AlertInstance struct {
CurrentStateSince time.Time
CurrentStateEnd time.Time
LastEvalTime time.Time
ResultFingerprint string
}
type AlertInstanceKey struct {

42
pkg/services/ngalert/models/testing.go
View File

@ -6,10 +6,12 @@ import (
"math/rand"
"slices"
"sync"
"testing"
"time"
"github.com/google/uuid"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/expr"
"github.com/grafana/grafana/pkg/services/datasources"
@ -511,6 +513,46 @@ func CreateLokiQuery(refID string, expr string, intervalMs int64, maxDataPoints
}
}
func CreateHysteresisExpression(t *testing.T, refID string, inputRefID string, threshold int, recoveryThreshold int) AlertQuery {
t.Helper()
q := AlertQuery{
RefID: refID,
QueryType: expr.DatasourceType,
DatasourceUID: expr.DatasourceUID,
Model: json.RawMessage(fmt.Sprintf(`
{
"refId": "%[1]s",
"type": "threshold",
"datasource": {
"uid": "%[5]s",
"type": "%[6]s"
},
"expression": "%[2]s",
"conditions": [
{
"type": "query",
"evaluator": {
"params": [
%[3]d
],
"type": "gt"
},
"unloadEvaluator": {
"params": [
%[4]d
],
"type": "lt"
}
}
]
}`, refID, inputRefID, threshold, recoveryThreshold, expr.DatasourceUID, expr.DatasourceType)),
}
h, err := q.IsHysteresisExpression()
require.NoError(t, err)
require.Truef(t, h, "test model is expected to be a hysteresis expression")
return q
}
type AlertInstanceMutator func(*AlertInstance)
// AlertInstanceGen provides a factory function that generates a random AlertInstance.

44
pkg/services/ngalert/schedule/loaded_metrics_reader.go Normal file
View File

@ -0,0 +1,44 @@
package schedule
import (
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/state"
)
var _ eval.AlertingResultsReader = AlertingResultsFromRuleState{}
func (sch *schedule) newLoadedMetricsReader(rule *ngmodels.AlertRule) eval.AlertingResultsReader {
return &AlertingResultsFromRuleState{
Manager: sch.stateManager,
Rule: rule,
}
}
type RuleStateProvider interface {
GetStatesForRuleUID(orgID int64, alertRuleUID string) []*state.State
}
// AlertingResultsFromRuleState implements eval.AlertingResultsReader that gets the data from state manager.
// It returns results fingerprints only for Alerting and Pending states that have empty StateReason.
type AlertingResultsFromRuleState struct {
Manager RuleStateProvider
Rule *ngmodels.AlertRule
}
func (n AlertingResultsFromRuleState) Read() map[data.Fingerprint]struct{} {
states := n.Manager.GetStatesForRuleUID(n.Rule.OrgID, n.Rule.UID)
active := map[data.Fingerprint]struct{}{}
for _, st := range states {
if st.StateReason != "" {
continue
}
if st.State == eval.Alerting || st.State == eval.Pending {
active[st.ResultFingerprint] = struct{}{}
}
}
return active
}

65
pkg/services/ngalert/schedule/loaded_metrics_reader_test.go Normal file
View File

@ -0,0 +1,65 @@
package schedule
import (
"testing"
"github.com/google/uuid"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/state"
)
func TestLoadedResultsFromRuleState(t *testing.T) {
rule := ngmodels.AlertRuleGen()()
p := &FakeRuleStateProvider{
map[ngmodels.AlertRuleKey][]*state.State{
rule.GetKey(): {
{State: eval.Alerting, ResultFingerprint: data.Fingerprint(1)},
{State: eval.Pending, ResultFingerprint: data.Fingerprint(2)},
{State: eval.Normal, ResultFingerprint: data.Fingerprint(3)},
{State: eval.NoData, ResultFingerprint: data.Fingerprint(4)},
{State: eval.Error, ResultFingerprint: data.Fingerprint(5)},
},
},
}
reader := AlertingResultsFromRuleState{
Manager: p,
Rule: rule,
}
t.Run("should return pending and alerting states", func(t *testing.T) {
loaded := reader.Read()
require.Len(t, loaded, 2)
require.Contains(t, loaded, data.Fingerprint(1))
require.Contains(t, loaded, data.Fingerprint(2))
})
t.Run("should not return any states with reason", func(t *testing.T) {
for _, s := range p.states[rule.GetKey()] {
s.StateReason = uuid.NewString()
}
loaded := reader.Read()
require.Empty(t, loaded)
})
t.Run("empty if no states", func(t *testing.T) {
p.states[rule.GetKey()] = nil
loaded := reader.Read()
require.Empty(t, loaded)
})
}
type FakeRuleStateProvider struct {
states map[ngmodels.AlertRuleKey][]*state.State
}
func (f FakeRuleStateProvider) GetStatesForRuleUID(orgID int64, UID string) []*state.State {
return f.states[ngmodels.AlertRuleKey{
OrgID: orgID,
UID: UID,
}]
}

2
pkg/services/ngalert/schedule/schedule.go
View File

@ -382,7 +382,7 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
logger := logger.New("version", e.rule.Version, "fingerprint", f, "attempt", attempt, "now", e.scheduledAt).FromContext(ctx)
start := sch.clock.Now()
evalCtx := eval.NewContext(ctx, SchedulerUserFor(e.rule.OrgID))
evalCtx := eval.NewContextWithPreviousResults(ctx, SchedulerUserFor(e.rule.OrgID), sch.newLoadedMetricsReader(e.rule))
ruleEval, err := sch.evaluatorFactory.Create(evalCtx, e.rule.GetEvalCondition())
var results eval.Results
var dur time.Duration

1
pkg/services/ngalert/state/cache.go
View File

@ -190,6 +190,7 @@ func calculateState(ctx context.Context, log log.Logger, alertRule *ngModels.Ale
Values: values,
StartsAt: result.EvaluatedAt,
EndsAt: result.EvaluatedAt,
ResultFingerprint: result.Instance.Fingerprint(),
}
return newState
}

11
pkg/services/ngalert/state/manager.go
View File

@ -3,6 +3,7 @@ package state
import (
"context"
"net/url"
"strconv"
"time"
"github.com/benbjohnson/clock"
@ -158,6 +159,14 @@ func (st *Manager) Warm(ctx context.Context, rulesReader RuleReader) {
if err != nil {
st.log.Error("Error getting cacheId for entry", "error", err)
}
var resultFp data.Fingerprint
if entry.ResultFingerprint != "" {
fp, err := strconv.ParseUint(entry.ResultFingerprint, 16, 64)
if err != nil {
st.log.Error("Failed to parse result fingerprint of alert instance", "error", err, "ruleUID", entry.RuleUID)
}
resultFp = data.Fingerprint(fp)
}
rulesStates.states[cacheID] = &State{
AlertRuleUID: entry.RuleUID,
OrgID: entry.RuleOrgID,
@ -170,6 +179,7 @@ func (st *Manager) Warm(ctx context.Context, rulesReader RuleReader) {
EndsAt: entry.CurrentStateEnd,
LastEvaluationTime: entry.LastEvalTime,
Annotations: ruleForEntry.Annotations,
ResultFingerprint: resultFp,
}
statesCount++
}
@ -458,6 +468,7 @@ func (st *Manager) saveAlertStates(ctx context.Context, logger log.Logger, state
LastEvalTime: s.LastEvaluationTime,
CurrentStateSince: s.StartsAt,
CurrentStateEnd: s.EndsAt,
ResultFingerprint: s.ResultFingerprint.String(),
}
err = st.instanceStore.SaveAlertInstance(ctx, instance)

17
pkg/services/ngalert/state/manager_private_test.go
View File

@ -286,6 +286,15 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
"system + rule + datasource-error": mergeLabels(mergeLabels(expectedDatasourceErrorLabels, baseRule.Labels), systemLabels),
}
resultFingerprints := map[string]data.Fingerprint{
"system + rule": data.Labels{}.Fingerprint(),
"system + rule + labels1": labels1.Fingerprint(),
"system + rule + labels2": labels2.Fingerprint(),
"system + rule + labels3": labels3.Fingerprint(),
"system + rule + no-data": noDataLabels.Fingerprint(),
"system + rule + datasource-error": data.Labels{}.Fingerprint(),
}
patchState := func(r *ngmodels.AlertRule, s *State) {
// patch all optional fields of the expected state
setCacheID(s)
@ -304,6 +313,14 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
if s.Values == nil {
s.Values = make(map[string]float64)
}
if s.ResultFingerprint == data.Fingerprint(0) {
for key, set := range labels {
if set.Fingerprint() == s.Labels.Fingerprint() {
s.ResultFingerprint = resultFingerprints[key]
break
}
}
}
}
executeTest := func(t *testing.T, alertRule *ngmodels.AlertRule, resultsAtTime map[time.Time]eval.Results, expectedTransitionsAtTime map[time.Time][]StateTransition, applyNoDataErrorToAllStates bool) {

185
pkg/services/ngalert/state/manager_test.go
View File

@ -5,6 +5,7 @@ import (
"context"
"errors"
"fmt"
"math"
"math/rand"
"sort"
"strings"
@ -58,6 +59,7 @@ func TestWarmStateCache(t *testing.T) {
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Fingerprint(math.MaxUint64),
}, {
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
@ -70,6 +72,7 @@ func TestWarmStateCache(t *testing.T) {
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Fingerprint(math.MaxUint64 - 1),
},
{
AlertRuleUID: rule.UID,
@ -83,6 +86,7 @@ func TestWarmStateCache(t *testing.T) {
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Fingerprint(0),
},
{
AlertRuleUID: rule.UID,
@ -96,6 +100,7 @@ func TestWarmStateCache(t *testing.T) {
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Fingerprint(1),
},
{
AlertRuleUID: rule.UID,
@ -109,6 +114,7 @@ func TestWarmStateCache(t *testing.T) {
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Fingerprint(2),
},
}
@ -127,6 +133,7 @@ func TestWarmStateCache(t *testing.T) {
CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
CurrentStateEnd: evaluationTime.Add(1 * time.Minute),
Labels: labels,
ResultFingerprint: data.Fingerprint(math.MaxUint64).String(),
})
labels = models.InstanceLabels{"test2": "testValue2"}
@ -142,6 +149,7 @@ func TestWarmStateCache(t *testing.T) {
CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
CurrentStateEnd: evaluationTime.Add(1 * time.Minute),
Labels: labels,
ResultFingerprint: data.Fingerprint(math.MaxUint64 - 1).String(),
})
labels = models.InstanceLabels{"test3": "testValue3"}
@ -157,6 +165,7 @@ func TestWarmStateCache(t *testing.T) {
CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
CurrentStateEnd: evaluationTime.Add(1 * time.Minute),
Labels: labels,
ResultFingerprint: data.Fingerprint(0).String(),
})
labels = models.InstanceLabels{"test4": "testValue4"}
@ -172,6 +181,7 @@ func TestWarmStateCache(t *testing.T) {
CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
CurrentStateEnd: evaluationTime.Add(1 * time.Minute),
Labels: labels,
ResultFingerprint: data.Fingerprint(1).String(),
})
labels = models.InstanceLabels{"test5": "testValue5"}
@ -187,6 +197,7 @@ func TestWarmStateCache(t *testing.T) {
CurrentStateSince: evaluationTime.Add(-1 * time.Minute),
CurrentStateEnd: evaluationTime.Add(1 * time.Minute),
Labels: labels,
ResultFingerprint: data.Fingerprint(2).String(),
})
for _, instance := range instances {
_ = dbstore.SaveAlertInstance(ctx, instance)
@ -384,8 +395,9 @@ func TestProcessEvalResults(t *testing.T) {
},
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
},
@ -407,8 +419,9 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
},
@ -417,8 +430,9 @@ func TestProcessEvalResults(t *testing.T) {
LastEvaluationTime: t1,
},
{
Labels: labels["system + rule + labels2"],
State: eval.Alerting,
Labels: labels["system + rule + labels2"],
ResultFingerprint: labels2.Fingerprint(),
State: eval.Alerting,
Results: []state.Evaluation{
newEvaluation(t1, eval.Alerting),
},
@ -441,8 +455,9 @@ func TestProcessEvalResults(t *testing.T) {
},
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(tn(6), eval.Normal),
@ -467,8 +482,9 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Alerting,
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Alerting),
@ -499,8 +515,9 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Alerting,
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
Results: []state.Evaluation{
newEvaluation(t3, eval.Alerting),
newEvaluation(tn(4), eval.Alerting),
@ -534,8 +551,9 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 3, // Normal -> Pending, Pending -> NoData, NoData -> Pending
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Pending,
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
Results: []state.Evaluation{
newEvaluation(tn(4), eval.Alerting),
newEvaluation(tn(5), eval.Alerting),
@ -566,8 +584,9 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 3,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.NoData,
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData,
Results: []state.Evaluation{
newEvaluation(t3, eval.Alerting),
newEvaluation(tn(4), eval.NoData),
@ -592,7 +611,8 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
Results: []state.Evaluation{
@ -619,8 +639,9 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Pending,
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
Results: []state.Evaluation{
newEvaluation(t1, eval.Alerting),
newEvaluation(t2, eval.Alerting),
@ -645,9 +666,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Pending,
StateReason: eval.NoData.String(),
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
StateReason: eval.NoData.String(),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
@ -681,9 +703,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Alerting,
StateReason: eval.NoData.String(),
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: eval.NoData.String(),
Results: []state.Evaluation{
newEvaluation(t3, eval.NoData),
newEvaluation(tn(4), eval.NoData),
@ -709,8 +732,9 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.NoData,
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
@ -735,8 +759,9 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
},
@ -745,8 +770,9 @@ func TestProcessEvalResults(t *testing.T) {
LastEvaluationTime: t1,
},
{
Labels: labels["system + rule"],
State: eval.NoData,
Labels: labels["system + rule"],
ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.NoData,
Results: []state.Evaluation{
newEvaluation(t2, eval.NoData),
},
@ -771,8 +797,9 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
},
@ -781,8 +808,9 @@ func TestProcessEvalResults(t *testing.T) {
LastEvaluationTime: t1,
},
{
Labels: labels["system + rule + labels2"],
State: eval.Normal,
Labels: labels["system + rule + labels2"],
ResultFingerprint: labels2.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
},
@ -791,8 +819,9 @@ func TestProcessEvalResults(t *testing.T) {
LastEvaluationTime: t1,
},
{
Labels: labels["system + rule"],
State: eval.NoData,
Labels: labels["system + rule"],
ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.NoData,
Results: []state.Evaluation{
newEvaluation(t2, eval.NoData),
},
@ -819,8 +848,9 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t3, eval.Normal),
@ -830,8 +860,9 @@ func TestProcessEvalResults(t *testing.T) {
LastEvaluationTime: t3,
},
{
Labels: labels["system + rule + no-data"],
State: eval.NoData,
Labels: labels["system + rule + no-data"],
ResultFingerprint: noDataLabels.Fingerprint(),
State: eval.NoData,
Results: []state.Evaluation{
newEvaluation(t2, eval.NoData),
},
@ -855,9 +886,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 0,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
StateReason: eval.NoData.String(),
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
StateReason: eval.NoData.String(),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
@ -882,10 +914,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Pending,
StateReason: eval.Error.String(),
Error: errors.New("with_state_error"),
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
StateReason: eval.Error.String(),
Error: errors.New("with_state_error"),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Error),
@ -919,10 +952,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Alerting,
StateReason: eval.Error.String(),
Error: errors.New("with_state_error"),
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: eval.Error.String(),
Error: errors.New("with_state_error"),
Results: []state.Evaluation{
newEvaluation(t3, eval.Error),
newEvaluation(tn(4), eval.Error),
@ -960,8 +994,9 @@ func TestProcessEvalResults(t *testing.T) {
"datasource_uid": "datasource_uid_1",
"ref_id": "A",
}),
State: eval.Error,
Error: expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error")),
ResultFingerprint: labels1.Fingerprint(),
State: eval.Error,
Error: expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error")),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Error),
@ -988,9 +1023,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
StateReason: eval.Error.String(),
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
StateReason: eval.Error.String(),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Error),
@ -1015,9 +1051,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Normal,
StateReason: eval.Error.String(),
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
StateReason: eval.Error.String(),
Results: []state.Evaluation{
newEvaluation(t1, eval.Alerting),
newEvaluation(t2, eval.Error),
@ -1054,9 +1091,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 3,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Error,
Error: fmt.Errorf("with_state_error"),
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Error,
Error: fmt.Errorf("with_state_error"),
Results: []state.Evaluation{
newEvaluation(tn(5), eval.Error),
newEvaluation(tn(6), eval.Error),
@ -1087,8 +1125,9 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 3,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.Pending,
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
Results: []state.Evaluation{
newEvaluation(tn(4), eval.Alerting),
newEvaluation(tn(5), eval.Error),
@ -1120,8 +1159,9 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 3,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
State: eval.NoData,
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData,
Results: []state.Evaluation{
newEvaluation(tn(4), eval.Alerting),
newEvaluation(tn(5), eval.Error),
@ -1166,6 +1206,11 @@ func TestProcessEvalResults(t *testing.T) {
LastEvaluationTime: t1,
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"summary": "grafana is down in us-central-1 cluster -> prod namespace"},
ResultFingerprint: data.Labels{
"cluster": "us-central-1",
"namespace": "prod",
"pod": "grafana",
}.Fingerprint(),
},
},
},
@ -1186,8 +1231,9 @@ func TestProcessEvalResults(t *testing.T) {
},
expectedStates: []*state.State{
{
Labels: labels["system + rule"],
State: eval.Alerting,
Labels: labels["system + rule"],
ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.Alerting,
Results: []state.Evaluation{
newEvaluation(t1, eval.Alerting),
newEvaluation(t2, eval.Error),
@ -1454,6 +1500,7 @@ func TestStaleResultsHandler(t *testing.T) {
LastEvaluationTime: evaluationTime,
EvaluationDuration: 0,
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Labels{"test1": "testValue1"}.Fingerprint(),
},
},
startingStateCount: 2,

3
pkg/services/ngalert/state/state.go
View File

@ -34,6 +34,9 @@ type State struct {
// StateReason is a textual description to explain why the state has its current state.
StateReason string
// ResultFingerprint is a hash of labels of the result before it is processed by
ResultFingerprint data.Fingerprint
// Results contains the result of the current and previous evaluations.
Results []Evaluation

4
pkg/services/ngalert/store/instance_database.go
View File

@ -54,12 +54,12 @@ func (st DBstore) SaveAlertInstance(ctx context.Context, alertInstance models.Al
if err != nil {
return err
}
params := append(make([]any, 0), alertInstance.RuleOrgID, alertInstance.RuleUID, labelTupleJSON, alertInstance.LabelsHash, alertInstance.CurrentState, alertInstance.CurrentReason, alertInstance.CurrentStateSince.Unix(), alertInstance.CurrentStateEnd.Unix(), alertInstance.LastEvalTime.Unix())
params := append(make([]any, 0), alertInstance.RuleOrgID, alertInstance.RuleUID, labelTupleJSON, alertInstance.LabelsHash, alertInstance.CurrentState, alertInstance.CurrentReason, alertInstance.CurrentStateSince.Unix(), alertInstance.CurrentStateEnd.Unix(), alertInstance.LastEvalTime.Unix(), alertInstance.ResultFingerprint)
upsertSQL := st.SQLStore.GetDialect().UpsertSQL(
"alert_instance",
[]string{"rule_org_id", "rule_uid", "labels_hash"},
[]string{"rule_org_id", "rule_uid", "labels", "labels_hash", "current_state", "current_reason", "current_state_since", "current_state_end", "last_eval_time"})
[]string{"rule_org_id", "rule_uid", "labels", "labels_hash", "current_state", "current_reason", "current_state_since", "current_state_end", "last_eval_time", "result_fingerprint"})
_, err = sess.SQL(upsertSQL, params...).Query()
if err != nil {
return err

4
pkg/services/sqlstore/migrations/ualert/tables.go
View File

@ -189,6 +189,10 @@ func alertInstanceMigration(mg *migrator.Migrator) {
migrator.NewAddColumnMigration(alertInstance, &migrator.Column{
Name: "current_reason", Type: migrator.DB_NVarchar, Length: DefaultFieldMaxLength, Nullable: true,
}))
mg.AddMigration("add result_fingerprint column to alert_instance", migrator.NewAddColumnMigration(alertInstance, &migrator.Column{
Name: "result_fingerprint", Type: migrator.DB_NVarchar, Length: 16, Nullable: true,
}))
}
func addAlertRuleMigrations(mg *migrator.Migrator, defaultIntervalSeconds int64) {

75
pkg/tests/api/alerting/api_ruler_test.go
View File

@ -15,6 +15,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/google/uuid"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/prometheus/common/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -1315,3 +1316,77 @@ func TestIntegrationRulePause(t *testing.T) {
})
}
}
func TestIntegrationHysteresisRule(t *testing.T) {
testinfra.SQLiteIntegrationTest(t)
// Setup Grafana and its Database. Scheduler is set to evaluate every 1 second
dir, p := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
DisableLegacyAlerting: true,
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
NGAlertSchedulerBaseInterval: 1 * time.Second,
EnableFeatureToggles: []string{featuremgmt.FlagConfigurableSchedulerTick, featuremgmt.FlagRecoveryThreshold},
})
grafanaListedAddr, store := testinfra.StartGrafana(t, dir, p)
// Create a user to make authenticated requests
createUser(t, store, user.CreateUserCommand{
DefaultOrgRole: string(org.RoleAdmin),
Password: "password",
Login: "grafana",
})
apiClient := newAlertingApiClient(grafanaListedAddr, "grafana", "password")
folder := "hysteresis"
testDs := apiClient.CreateTestDatasource(t)
apiClient.CreateFolder(t, folder, folder)
bodyRaw, err := testData.ReadFile("test-data/hysteresis_rule.json")
require.NoError(t, err)
var postData apimodels.PostableRuleGroupConfig
require.NoError(t, json.Unmarshal(bodyRaw, &postData))
for _, rule := range postData.Rules {
for i := range rule.GrafanaManagedAlert.Data {
rule.GrafanaManagedAlert.Data[i].DatasourceUID = strings.ReplaceAll(rule.GrafanaManagedAlert.Data[i].DatasourceUID, "REPLACE_ME", testDs.Body.Datasource.UID)
}
}
changes, status, body := apiClient.PostRulesGroupWithStatus(t, folder, &postData)
require.Equalf(t, http.StatusAccepted, status, body)
require.Len(t, changes.Created, 1)
ruleUid := changes.Created[0]
var frame data.Frame
require.Eventuallyf(t, func() bool {
frame, status, body = apiClient.GetRuleHistoryWithStatus(t, ruleUid)
require.Equalf(t, http.StatusOK, status, body)
return frame.Rows() > 1
}, 15*time.Second, 1*time.Second, "Alert state history expected to have more than one record but got %d. Body: %s", frame.Rows(), body)
f, _ := frame.FieldByName("next")
alertingIdx := 0
normalIdx := 1
if f.At(alertingIdx).(string) != "Alerting" {
alertingIdx = 1
normalIdx = 0
}
assert.Equalf(t, "Alerting", f.At(alertingIdx).(string), body)
assert.Equalf(t, "Normal", f.At(normalIdx).(string), body)
type HistoryData struct {
Values map[string]int64
}
f, _ = frame.FieldByName("data")
var d HistoryData
require.NoErrorf(t, json.Unmarshal([]byte(f.At(alertingIdx).(string)), &d), body)
assert.EqualValuesf(t, 5, d.Values["B"], body)
require.NoErrorf(t, json.Unmarshal([]byte(f.At(normalIdx).(string)), &d), body)
assert.EqualValuesf(t, 1, d.Values["B"], body)
}

71
pkg/tests/api/alerting/test-data/hysteresis_rule.json Normal file
View File

@ -0,0 +1,71 @@
{
"name": "Default",
"interval": "1s",
"rules": [
{
"grafana_alert": {
"title": "Hysteresis Test",
"condition": "C",
"no_data_state": "NoData",
"exec_err_state": "Error",
"data": [
{
"refId": "A",
"datasourceUid": "REPLACE_ME",
"queryType": "",
"relativeTimeRange": {
"from": 600,
"to": 0
},
"model": {
"refId": "A",
"scenarioId": "predictable_csv_wave",
"csvWave": [
{
"timeStep": 1,
"valuesCSV": "5,3,2,1"
}
],
"seriesCount": 1
}
},
{
"refId": "B",
"datasourceUid": "__expr__",
"model": {
"refId": "B",
"type": "reduce",
"reducer": "last",
"expression": "A"
}
},
{
"refId": "C",
"datasourceUid": "__expr__",
"model": {
"refId": "C",
"type": "threshold",
"conditions": [
{
"evaluator": {
"params": [
4
],
"type": "gt"
},
"unloadEvaluator": {
"params": [
2
],
"type": "lt"
}
}
],
"expression": "B"
}
}
]
}
}
]
}

15
pkg/tests/api/alerting/testing.go
View File

@ -12,6 +12,7 @@ import (
"time"
"github.com/google/uuid"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/prometheus/common/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -695,6 +696,20 @@ func (a apiClient) UpdateRouteWithStatus(t *testing.T, route apimodels.Route) (i
return resp.StatusCode, string(body)
}
func (a apiClient) GetRuleHistoryWithStatus(t *testing.T, ruleUID string) (data.Frame, int, string) {
t.Helper()
u, err := url.Parse(fmt.Sprintf("%s/api/v1/rules/history", a.url))
require.NoError(t, err)
q := url.Values{}
q.Set("ruleUID", ruleUID)
u.RawQuery = q.Encode()
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
require.NoError(t, err)
return sendRequest[data.Frame](t, req, http.StatusOK)
}
func sendRequest[T any](t *testing.T, req *http.Request, successStatusCode int) (T, int, string) {
client := &http.Client{}
resp, err := client.Do(req)

10
pkg/tests/testinfra/testinfra.go
View File

@ -394,6 +394,15 @@ func CreateGrafDir(t *testing.T, opts ...GrafanaOpts) (string, string) {
require.NoError(t, err)
_, err = logSection.NewKey("query_retries", fmt.Sprintf("%d", queryRetries))
require.NoError(t, err)
if o.NGAlertSchedulerBaseInterval > 0 {
unifiedAlertingSection, err := getOrCreateSection("unified_alerting")
require.NoError(t, err)
_, err = unifiedAlertingSection.NewKey("scheduler_tick_interval", o.NGAlertSchedulerBaseInterval.String())
require.NoError(t, err)
_, err = unifiedAlertingSection.NewKey("min_interval", o.NGAlertSchedulerBaseInterval.String())
require.NoError(t, err)
}
}
cfgPath := filepath.Join(cfgDir, "test.ini")
@ -419,6 +428,7 @@ type GrafanaOpts struct {
EnableFeatureToggles []string
NGAlertAdminConfigPollInterval time.Duration
NGAlertAlertmanagerConfigPollInterval time.Duration
NGAlertSchedulerBaseInterval time.Duration
AnonymousUserRole org.RoleType
EnableQuota bool
DashboardOrgQuota *int64

50
public/app/features/alerting/unified/GrafanaRuleQueryViewer.tsx
View File

@ -377,20 +377,44 @@ function ThresholdExpressionViewer({ model }: { model: ExpressionQuery }) {
const isRange = evaluator ? isRangeEvaluator(evaluator) : false;
return (
<div className={styles.container}>
<div className={styles.label}>Input</div>
<div className={styles.value}>{expression}</div>
const unloadEvaluator = conditions && conditions[0]?.unloadEvaluator;
const unloadThresholdFunction = thresholdFunctions.find((tf) => tf.value === unloadEvaluator?.type);
{evaluator && (
<>
<div className={styles.blue}>{thresholdFunction?.label}</div>
<div className={styles.bold}>
{isRange ? `(${evaluator.params[0]}; ${evaluator.params[1]})` : evaluator.params[0]}
</div>
</>
)}
</div>
const unloadIsRange = unloadEvaluator ? isRangeEvaluator(unloadEvaluator) : false;
return (
<>
<div className={styles.container}>
<div className={styles.label}>Input</div>
<div className={styles.value}>{expression}</div>
{evaluator && (
<>
<div className={styles.blue}>{thresholdFunction?.label}</div>
<div className={styles.bold}>
{isRange ? `(${evaluator.params[0]}; ${evaluator.params[1]})` : evaluator.params[0]}
</div>
</>
)}
</div>
<div className={styles.container}>
{unloadEvaluator && (
<>
<div className={styles.label}>Stop alerting when </div>
<div className={styles.value}>{expression}</div>
<>
<div className={styles.blue}>{unloadThresholdFunction?.label}</div>
<div className={styles.bold}>
{unloadIsRange
? `(${unloadEvaluator.params[0]}; ${unloadEvaluator.params[1]})`
: unloadEvaluator.params[0]}
</div>
</>
</>
)}
</div>
</>
);
}

29
public/app/features/alerting/unified/components/expressions/Expression.tsx
View File

@ -1,9 +1,10 @@
import { css, cx } from '@emotion/css';
import { uniqueId } from 'lodash';
import React, { FC, useCallback, useState } from 'react';
import { useFormContext } from 'react-hook-form';
import { DataFrame, dateTimeFormat, GrafanaTheme2, isTimeSeriesFrames, LoadingState, PanelData } from '@grafana/data';
import { AutoSizeInput, Button, clearButtonStyles, IconButton, useStyles2, Stack } from '@grafana/ui';
import { AutoSizeInput, Button, clearButtonStyles, IconButton, Stack, useStyles2 } from '@grafana/ui';
import { ClassicConditions } from 'app/features/expressions/components/ClassicConditions';
import { Math } from 'app/features/expressions/components/Math';
import { Reduce } from 'app/features/expressions/components/Reduce';
@ -56,6 +57,19 @@ export const Expression: FC<ExpressionProps> = ({
const queryType = query?.type;
const { setError, clearErrors } = useFormContext();
const onQueriesValidationError = useCallback(
(errorMsg: string | undefined) => {
if (errorMsg) {
setError('queries', { type: 'custom', message: errorMsg });
} else {
clearErrors('queries');
}
},
[setError, clearErrors]
);
const isLoading = data && Object.values(data).some((d) => Boolean(d) && d.state === LoadingState.Loading);
const hasResults = Array.isArray(data?.series) && !isLoading;
const series = data?.series ?? [];
@ -85,13 +99,22 @@ export const Expression: FC<ExpressionProps> = ({
return <ClassicConditions onChange={onChangeQuery} query={query} refIds={availableRefIds} />;
case ExpressionQueryType.threshold:
return <Threshold onChange={onChangeQuery} query={query} labelWidth={'auto'} refIds={availableRefIds} />;
return (
<Threshold
onChange={onChangeQuery}
query={query}
labelWidth={'auto'}
refIds={availableRefIds}
onError={onQueriesValidationError}
useHysteresis={true}
/>
);
default:
return <>Expression not supported: {query.type}</>;
}
},
[onChangeQuery, queries]
[onChangeQuery, queries, onQueriesValidationError]
);
const selectedExpressionType = expressionTypes.find((o) => o.value === queryType);
const selectedExpressionDescription = selectedExpressionType?.description ?? '';

6
public/app/features/alerting/unified/components/rule-editor/alert-rule-form/ModifyExportRuleForm.tsx
View File

@ -46,6 +46,10 @@ export function ModifyExportRuleForm({ ruleForm, alertUid }: ModifyExportRuleFor
const [conditionErrorMsg, setConditionErrorMsg] = useState('');
const [evaluateEvery, setEvaluateEvery] = useState(ruleForm?.evaluateEvery ?? MINUTE);
const onInvalid = (): void => {
notifyApp.error('There are errors in the form. Please correct them and try again!');
};
const checkAlertCondition = (msg = '') => {
setConditionErrorMsg(msg);
};
@ -66,7 +70,7 @@ export function ModifyExportRuleForm({ ruleForm, alertUid }: ModifyExportRuleFor
<LinkButton href={returnTo} key="cancel" size="sm" variant="secondary" onClick={() => submit(undefined)}>
Cancel
</LinkButton>,
<Button key="export-rule" size="sm" onClick={formAPI.handleSubmit((formValues) => submit(formValues))}>
<Button key="export-rule" size="sm" onClick={formAPI.handleSubmit((formValues) => submit(formValues), onInvalid)}>
Export
</Button>,
];

355
public/app/features/expressions/components/Threshold.tsx
View File

@ -1,70 +1,86 @@
import { css } from '@emotion/css';
import React, { FormEvent } from 'react';
import { AnyAction } from '@reduxjs/toolkit';
import React, { FormEvent, useEffect, useReducer } from 'react';
import { GrafanaTheme2, SelectableValue } from '@grafana/data';
import { ButtonSelect, InlineField, InlineFieldRow, Input, Select, useStyles2 } from '@grafana/ui';
import { Stack } from '@grafana/experimental';
import { ButtonSelect, InlineField, InlineFieldRow, InlineSwitch, Input, Select, useStyles2 } from '@grafana/ui';
import { config } from 'app/core/config';
import { EvalFunction } from 'app/features/alerting/state/alertDef';
import { ClassicCondition, ExpressionQuery, thresholdFunctions } from '../types';
import {
isInvalid,
thresholdReducer,
updateHysteresisChecked,
updateRefId,
updateThresholdParams,
updateThresholdType,
updateUnloadParams,
} from './thresholdReducer';
interface Props {
labelWidth: number | 'auto';
refIds: Array<SelectableValue<string>>;
query: ExpressionQuery;
onChange: (query: ExpressionQuery) => void;
onError?: (error: string | undefined) => void;
useHysteresis?: boolean;
}
const defaultThresholdFunction = EvalFunction.IsAbove;
export const Threshold = ({ labelWidth, onChange, refIds, query }: Props) => {
const defaultEvaluator: ClassicCondition = {
type: 'query',
evaluator: {
type: defaultThresholdFunction,
params: [0, 0],
},
query: {
params: [],
},
reducer: {
params: [],
type: 'last',
},
};
export const Threshold = ({ labelWidth, onChange, refIds, query, onError, useHysteresis = false }: Props) => {
const styles = useStyles2(getStyles);
const defaultEvaluator: ClassicCondition = {
type: 'query',
evaluator: {
type: defaultThresholdFunction,
params: [0, 0],
},
query: {
params: [],
},
reducer: {
params: [],
type: 'last',
},
};
const initialExpression = { ...query, conditions: query.conditions?.length ? query.conditions : [defaultEvaluator] };
const conditions = query.conditions?.length ? query.conditions : [defaultEvaluator];
const condition = conditions[0];
// this queryState is the source of truth for the threshold component.
// All the changes are made to this object through the dispatch function with the thresholdReducer.
const [queryState, dispatch] = useReducer(thresholdReducer, initialExpression);
const conditionInState = queryState.conditions[0];
const thresholdFunction = thresholdFunctions.find((fn) => fn.value === conditions[0].evaluator?.type);
const thresholdFunction = thresholdFunctions.find((fn) => fn.value === queryState.conditions[0].evaluator?.type);
const onRefIdChange = (value: SelectableValue<string>) => {
onChange({ ...query, expression: value.value });
dispatch(updateRefId(value.value));
};
const onEvalFunctionChange = (value: SelectableValue<EvalFunction>) => {
const type = value.value ?? defaultThresholdFunction;
// any change in the queryState will trigger the onChange function.
useEffect(() => {
queryState && onChange(queryState);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [queryState]);
onChange({
...query,
conditions: updateConditions(conditions, { type }),
});
const onEvalFunctionChange = (value: SelectableValue<EvalFunction>) => {
dispatch(updateThresholdType({ evalFunction: value.value ?? defaultThresholdFunction, onError }));
};
const onEvaluateValueChange = (event: FormEvent<HTMLInputElement>, index: number) => {
const newValue = parseFloat(event.currentTarget.value);
const newParams = [...condition.evaluator.params];
newParams[index] = newValue;
onChange({
...query,
conditions: updateConditions(conditions, { params: newParams }),
});
dispatch(updateThresholdParams({ param: parseFloat(event.currentTarget.value), index }));
};
const isRange =
condition.evaluator.type === EvalFunction.IsWithinRange || condition.evaluator.type === EvalFunction.IsOutsideRange;
conditionInState.evaluator.type === EvalFunction.IsWithinRange ||
conditionInState.evaluator.type === EvalFunction.IsOutsideRange;
const hysteresisEnabled = Boolean(config.featureToggles?.recoveryThreshold) && useHysteresis;
return (
<>
@ -86,14 +102,14 @@ export const Threshold = ({ labelWidth, onChange, refIds, query }: Props) => {
type="number"
width={10}
onChange={(event) => onEvaluateValueChange(event, 0)}
defaultValue={condition.evaluator.params[0]}
defaultValue={conditionInState.evaluator.params[0]}
/>
<div className={styles.button}>TO</div>
<Input
type="number"
width={10}
onChange={(event) => onEvaluateValueChange(event, 1)}
defaultValue={condition.evaluator.params[1]}
defaultValue={conditionInState.evaluator.params[1]}
/>
</>
) : (
@ -101,52 +117,239 @@ export const Threshold = ({ labelWidth, onChange, refIds, query }: Props) => {
type="number"
width={10}
onChange={(event) => onEvaluateValueChange(event, 0)}
defaultValue={conditions[0].evaluator.params[0] || 0}
defaultValue={conditionInState.evaluator.params[0] || 0}
/>
)}
</InlineFieldRow>
{hysteresisEnabled && <HysteresisSection isRange={isRange} onError={onError} />}
</>
);
interface HysteresisSectionProps {
isRange: boolean;
onError?: (error: string | undefined) => void;
}
function HysteresisSection({ isRange, onError }: HysteresisSectionProps) {
const hasHysteresis = Boolean(conditionInState.unloadEvaluator);
const onHysteresisCheckChange = (event: FormEvent<HTMLInputElement>) => {
dispatch(updateHysteresisChecked({ hysteresisChecked: event.currentTarget.checked, onError }));
allowOnblurFromUnload.current = true;
};
const allowOnblurFromUnload = React.useRef(true);
const onHysteresisCheckDown: React.MouseEventHandler<HTMLDivElement> | undefined = () => {
allowOnblurFromUnload.current = false;
};
return (
<div className={styles.hysteresis}>
{/* This is to enhance the user experience for mouse users.
The onBlur event in RecoveryThresholdRow inputs triggers validations,
but we want to skip them when the switch is clicked as this click should inmount this component.
To achieve this, we use the onMouseDown event to set a flag, which is later utilized in the onBlur event to bypass validations.
The onMouseDown event precedes the onBlur event, unlike onchange. */}
{/*Disabling the a11y rules here as the InlineSwitch handles keyboard interactions */}
{/* eslint-disable-next-line jsx-a11y/click-events-have-key-events, jsx-a11y/no-static-element-interactions */}
<div onMouseDown={onHysteresisCheckDown}>
<InlineSwitch
showLabel={true}
label="Custom recovery threshold"
value={hasHysteresis}
onChange={onHysteresisCheckChange}
className={styles.switch}
/>
</div>
{hasHysteresis && (
<RecoveryThresholdRow
isRange={isRange}
condition={conditionInState}
onError={onError}
dispatch={dispatch}
allowOnblur={allowOnblurFromUnload}
/>
)}
</div>
);
}
};
function updateConditions(
conditions: ClassicCondition[],
update: Partial<{
params: number[];
type: EvalFunction;
}>
): ClassicCondition[] {
return [
{
...conditions[0],
evaluator: {
...conditions[0].evaluator,
...update,
},
},
];
interface RecoveryThresholdRowProps {
isRange: boolean;
condition: ClassicCondition;
onError?: (error: string | undefined) => void;
dispatch: React.Dispatch<AnyAction>;
allowOnblur: React.MutableRefObject<boolean>;
}
function RecoveryThresholdRow({ isRange, condition, onError, dispatch, allowOnblur }: RecoveryThresholdRowProps) {
const styles = useStyles2(getStyles);
const onUnloadValueChange = (event: FormEvent<HTMLInputElement>, paramIndex: number) => {
const newValue = parseFloat(event.currentTarget.value);
dispatch(updateUnloadParams({ param: newValue, index: paramIndex, onError }));
};
// check if is valid for the current unload evaluator params
const error = isInvalid(condition);
// get the error message depending on the unload evaluator type
const { errorMsg: invalidErrorMsg, errorMsgFrom, errorMsgTo } = error ?? {};
if (isRange) {
return <RecoveryForRange allowOnblur={allowOnblur} />;
} else {
return <RecoveryForSingleValue allowOnblur={allowOnblur} />;
}
/* We prioritize the onMouseDown event over the onBlur event. This is because the onBlur event is executed before the onChange event that we have
in the hysteresis checkbox, and because of that, we were validating when unchecking the switch.
We need to uncheck the switch before the onBlur event is executed.*/
interface RecoveryProps {
allowOnblur: React.MutableRefObject<boolean>;
}
function RecoveryForRange({ allowOnblur }: RecoveryProps) {
if (condition.evaluator.type === EvalFunction.IsWithinRange) {
return (
<InlineFieldRow className={styles.hysteresis}>
<InlineField label="Stop alerting when outside range" labelWidth={'auto'}>
<Stack direction="row" gap={0}>
<div className={styles.range}>
<InlineField invalid={Boolean(errorMsgFrom)} error={errorMsgFrom} className={styles.noMargin}>
<Input
type="number"
width={10}
onBlur={(event) => allowOnblur.current && onUnloadValueChange(event, 0)}
defaultValue={condition.unloadEvaluator?.params[0]}
/>
</InlineField>
</div>
<div className={styles.button}>TO</div>
<div className={styles.range}>
<InlineField invalid={Boolean(errorMsgTo)} error={errorMsgTo}>
<Input
type="number"
width={10}
onBlur={(event) => allowOnblur.current && onUnloadValueChange(event, 1)}
defaultValue={condition.unloadEvaluator?.params[1]}
/>
</InlineField>
</div>
</Stack>
</InlineField>
</InlineFieldRow>
);
} else {
return (
<InlineFieldRow className={styles.hysteresis}>
<InlineField label="Stop alerting when inside range" labelWidth={'auto'}>
<Stack direction="row" gap={0}>
<div className={styles.range}>
<InlineField invalid={Boolean(errorMsgFrom)} error={errorMsgFrom}>
<Input
type="number"
width={10}
onBlur={(event) => allowOnblur.current && onUnloadValueChange(event, 0)}
defaultValue={condition.unloadEvaluator?.params[0]}
/>
</InlineField>
</div>
<div className={styles.button}>TO</div>
<div className={styles.range}>
<InlineField invalid={Boolean(errorMsgTo)} error={errorMsgTo}>
<Input
type="number"
width={10}
onBlur={(event) => allowOnblur.current && onUnloadValueChange(event, 1)}
defaultValue={condition.unloadEvaluator?.params[1]}
/>
</InlineField>
</div>
</Stack>
</InlineField>
</InlineFieldRow>
);
}
}
function RecoveryForSingleValue({ allowOnblur }: RecoveryProps) {
if (condition.evaluator.type === EvalFunction.IsAbove) {
return (
<InlineFieldRow className={styles.hysteresis}>
<InlineField
label="Stop alerting when below"
labelWidth={'auto'}
invalid={Boolean(invalidErrorMsg)}
error={invalidErrorMsg}
>
<Input
type="number"
width={10}
onBlur={(event) => {
allowOnblur.current && onUnloadValueChange(event, 0);
}}
defaultValue={condition.unloadEvaluator?.params[0]}
/>
</InlineField>
</InlineFieldRow>
);
} else {
return (
<InlineFieldRow className={styles.hysteresis}>
<InlineField
label="Stop alerting when above"
labelWidth={'auto'}
invalid={Boolean(invalidErrorMsg)}
error={invalidErrorMsg}
>
<Input
type="number"
width={10}
onBlur={(event) => {
allowOnblur.current && onUnloadValueChange(event, 0);
}}
defaultValue={condition.unloadEvaluator?.params[0]}
/>
</InlineField>
</InlineFieldRow>
);
}
}
}
const getStyles = (theme: GrafanaTheme2) => ({
buttonSelectText: css`
color: ${theme.colors.primary.text};
font-size: ${theme.typography.bodySmall.fontSize};
text-transform: uppercase;
`,
button: css`
height: 32px;
color: ${theme.colors.primary.text};
font-size: ${theme.typography.bodySmall.fontSize};
text-transform: uppercase;
display: flex;
align-items: center;
border-radius: ${theme.shape.radius.default};
font-weight: ${theme.typography.fontWeightBold};
border: 1px solid ${theme.colors.border.medium};
white-space: nowrap;
padding: 0 ${theme.spacing(1)};
background-color: ${theme.colors.background.primary};
`,
buttonSelectText: css({
color: theme.colors.primary.text,
fontSize: theme.typography.bodySmall.fontSize,
textTransform: 'uppercase',
padding: `0 ${theme.spacing(1)}`,
}),
button: css({
height: '32px',
color: theme.colors.primary.text,
fontSize: theme.typography.bodySmall.fontSize,
textTransform: 'uppercase',
display: 'flex',
alignItems: 'center',
borderRadius: theme.shape.radius.default,
fontWeight: theme.typography.fontWeightBold,
border: `1px solid ${theme.colors.border.medium}`,
whiteSpace: 'nowrap',
padding: `0 ${theme.spacing(1)}`,
backgroundColor: theme.colors.background.primary,
}),
range: css({
width: 'min-content',
}),
hysteresis: css({
marginTop: theme.spacing(2),
}),
switch: css({
paddingLeft: theme.spacing(1),
}),
noMargin: css({
margin: 0,
}),
});

212
public/app/features/expressions/components/__snapshots__/hysteresis.test.ts.snap Normal file
View File

@ -0,0 +1,212 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`thresholdReducer Should update unlooadEvaluator when checking hysteresis 1`] = `
{
"conditions": [
{
"evaluator": {
"params": [
10,
0,
],
"type": "gt",
},
"query": {
"params": [
"A",
"B",
],
},
"reducer": {
"params": [],
"type": "avg",
},
"type": "query",
"unloadEvaluator": {
"params": [
10,
0,
],
"type": "lt",
},
},
],
"refId": "A",
"type": "threshold",
}
`;
exports[`thresholdReducer Should update unlooadEvaluator when unchecking hysteresis 1`] = `
{
"conditions": [
{
"evaluator": {
"params": [
10,
0,
],
"type": "gt",
},
"query": {
"params": [
"A",
"B",
],
},
"reducer": {
"params": [],
"type": "avg",
},
"type": "query",
"unloadEvaluator": undefined,
},
],
"refId": "A",
"type": "threshold",
}
`;
exports[`thresholdReducer should update Threshold Type, and unloadEvaluator params and type 1`] = `
{
"conditions": [
{
"evaluator": {
"params": [
10,
0,
],
"type": "lt",
},
"query": {
"params": [
"A",
"B",
],
},
"reducer": {
"params": [],
"type": "avg",
},
"type": "query",
"unloadEvaluator": {
"params": [
10,
0,
],
"type": "gt",
},
},
],
"refId": "A",
"type": "threshold",
}
`;
exports[`thresholdReducer should update expression with RefId 1`] = `
{
"conditions": [
{
"evaluator": {
"params": [
10,
0,
],
"type": "gt",
},
"query": {
"params": [
"A",
"B",
],
},
"reducer": {
"params": [],
"type": "avg",
},
"type": "query",
"unloadEvaluator": {
"params": [
10,
0,
],
"type": "lt",
},
},
],
"expression": "B",
"refId": "A",
"type": "threshold",
}
`;
exports[`thresholdReducer should update unloadParams no error when are invalid 1`] = `
{
"conditions": [
{
"evaluator": {
"params": [
10,
0,
],
"type": "gt",
},
"query": {
"params": [
"A",
"B",
],
},
"reducer": {
"params": [],
"type": "avg",
},
"type": "query",
"unloadEvaluator": {
"params": [
20,
0,
],
"type": "lt",
},
},
],
"refId": "A",
"type": "threshold",
}
`;
exports[`thresholdReducer should update unloadParams with no error when are valid 1`] = `
{
"conditions": [
{
"evaluator": {
"params": [
10,
0,
],
"type": "gt",
},
"query": {
"params": [
"A",
"B",
],
},
"reducer": {
"params": [],
"type": "avg",
},
"type": "query",
"unloadEvaluator": {
"params": [
9,
0,
],
"type": "lt",
},
},
],
"refId": "A",
"type": "threshold",
}
`;

217
public/app/features/expressions/components/hysteresis.test.ts Normal file
View File

@ -0,0 +1,217 @@
import { EvalFunction } from 'app/features/alerting/state/alertDef';
import { ClassicCondition, ExpressionQueryType, ThresholdExpressionQuery } from '../types';
import {
isInvalid,
thresholdReducer,
updateHysteresisChecked,
updateRefId,
updateThresholdType,
updateUnloadParams,
} from './thresholdReducer';
describe('isInvalid', () => {
it('returns an error message if unloadEvaluator.params[0] is undefined', () => {
const condition: ClassicCondition = {
unloadEvaluator: {
type: EvalFunction.IsAbove,
params: [],
},
evaluator: { type: EvalFunction.IsAbove, params: [10] },
query: { params: ['A', 'B'] },
reducer: { type: 'avg', params: [] },
type: 'query',
};
expect(isInvalid(condition)).toEqual({ errorMsg: 'This value cannot be empty' });
});
it('When using is above, returns an error message if the value in unloadevaluator is above the threshold', () => {
const condition: ClassicCondition = {
unloadEvaluator: {
type: EvalFunction.IsAbove,
params: [15],
},
evaluator: { type: EvalFunction.IsAbove, params: [10] },
query: { params: ['A', 'B'] },
reducer: { type: 'avg', params: [] },
type: 'query',
};
expect(isInvalid(condition)).toEqual({ errorMsg: 'Enter a number less than or equal to 10' });
});
it('When using is below, returns an error message if the value in unloadevaluator is below the threshold', () => {
const condition: ClassicCondition = {
unloadEvaluator: {
type: EvalFunction.IsAbove,
params: [9],
},
evaluator: { type: EvalFunction.IsBelow, params: [10] },
query: { params: ['A', 'B'] },
reducer: { type: 'avg', params: [] },
type: 'query',
};
expect(isInvalid(condition)).toEqual({ errorMsg: 'Enter a number more than or equal to 10' });
});
it('When using is within range, returns an error message if the value in unloadevaluator is within the range', () => {
// first parameter is wrong
const condition: ClassicCondition = {
unloadEvaluator: {
type: EvalFunction.IsOutsideRange,
params: [11, 21],
},
evaluator: { type: EvalFunction.IsWithinRange, params: [10, 20] },
query: { params: ['A', 'B'] },
reducer: { type: 'avg', params: [] },
type: 'query',
};
expect(isInvalid(condition)).toEqual({ errorMsgFrom: 'Enter a number less than or equal to 10' });
// second parameter is wrong
const condition2: ClassicCondition = {
unloadEvaluator: {
type: EvalFunction.IsOutsideRange,
params: [9, 19],
},
evaluator: { type: EvalFunction.IsWithinRange, params: [10, 20] },
query: { params: ['A', 'B'] },
reducer: { type: 'avg', params: [] },
type: 'query',
};
expect(isInvalid(condition2)).toEqual({ errorMsgTo: 'Enter a number be more than or equal to 20' });
});
it('When using is outside range, returns an error message if the value in unloadevaluator is outside the range', () => {
// first parameter is wrong
const condition: ClassicCondition = {
unloadEvaluator: {
type: EvalFunction.IsWithinRange,
params: [8, 19],
},
evaluator: { type: EvalFunction.IsOutsideRange, params: [10, 20] },
query: { params: ['A', 'B'] },
reducer: { type: 'avg', params: [] },
type: 'query',
};
expect(isInvalid(condition)).toEqual({ errorMsgFrom: 'Enter a number more than or equal to 10' });
// second parameter is wrong
const condition2: ClassicCondition = {
unloadEvaluator: {
type: EvalFunction.IsWithinRange,
params: [11, 21],
},
evaluator: { type: EvalFunction.IsOutsideRange, params: [10, 20] },
query: { params: ['A', 'B'] },
reducer: { type: 'avg', params: [] },
type: 'query',
};
expect(isInvalid(condition2)).toEqual({ errorMsgTo: 'Enter a number less than or equal to 20' });
});
});
describe('thresholdReducer', () => {
beforeEach(() => {
jest.clearAllMocks();
});
const onError = jest.fn();
const thresholdCondition: ClassicCondition = {
evaluator: { type: EvalFunction.IsAbove, params: [10, 0] },
unloadEvaluator: {
type: EvalFunction.IsBelow,
params: [10, 0],
},
query: { params: ['A', 'B'] },
reducer: { type: 'avg', params: [] },
type: 'query',
};
it('should return initial state', () => {
expect(thresholdReducer(undefined, { type: undefined })).toEqual({
type: ExpressionQueryType.threshold,
conditions: [],
refId: '',
});
});
it('should update expression with RefId', () => {
const initialState: ThresholdExpressionQuery = {
type: ExpressionQueryType.threshold,
refId: 'A',
conditions: [thresholdCondition],
};
const newState = thresholdReducer(initialState, updateRefId('B'));
expect(newState).toMatchSnapshot();
expect(newState.expression).toEqual('B');
});
it('should update Threshold Type, and unloadEvaluator params and type ', () => {
const initialState: ThresholdExpressionQuery = {
type: ExpressionQueryType.threshold,
refId: 'A',
conditions: [thresholdCondition],
};
const newState = thresholdReducer(
initialState,
updateThresholdType({ evalFunction: EvalFunction.IsBelow, onError })
);
expect(newState).toMatchSnapshot();
expect(newState.conditions[0].evaluator.type).toEqual(EvalFunction.IsBelow);
expect(newState.conditions[0].unloadEvaluator?.type).toEqual(EvalFunction.IsAbove);
expect(onError).toHaveBeenCalledWith(undefined);
expect(newState.conditions[0].unloadEvaluator?.params[0]).toEqual(10);
});
it('Should update unlooadEvaluator when checking hysteresis', () => {
const initialState: ThresholdExpressionQuery = {
type: ExpressionQueryType.threshold,
refId: 'A',
conditions: [thresholdCondition],
};
const newState = thresholdReducer(initialState, updateHysteresisChecked({ hysteresisChecked: true, onError }));
expect(newState).toMatchSnapshot();
expect(newState.conditions[0].unloadEvaluator?.type).toEqual(EvalFunction.IsBelow);
expect(newState.conditions[0].unloadEvaluator?.params[0]).toEqual(10);
});
it('Should update unlooadEvaluator when unchecking hysteresis', () => {
const initialState: ThresholdExpressionQuery = {
type: ExpressionQueryType.threshold,
refId: 'A',
conditions: [thresholdCondition],
};
const newState = thresholdReducer(initialState, updateHysteresisChecked({ hysteresisChecked: false, onError }));
expect(newState).toMatchSnapshot();
expect(newState.conditions[0].unloadEvaluator).toEqual(undefined);
expect(onError).toHaveBeenCalledWith(undefined);
});
it('should update unloadParams with no error when are valid', () => {
const initialState: ThresholdExpressionQuery = {
type: ExpressionQueryType.threshold,
refId: 'A',
conditions: [thresholdCondition],
};
const newState = thresholdReducer(initialState, updateUnloadParams({ param: 9, index: 0, onError }));
expect(newState).toMatchSnapshot();
expect(newState.conditions[0].unloadEvaluator?.params[0]).toEqual(9);
expect(onError).toHaveBeenCalledWith(undefined);
});
it('should update unloadParams no error when are invalid', () => {
const initialState: ThresholdExpressionQuery = {
type: ExpressionQueryType.threshold,
refId: 'A',
conditions: [thresholdCondition],
};
const newState = thresholdReducer(initialState, updateUnloadParams({ param: 20, index: 0, onError }));
expect(newState).toMatchSnapshot();
expect(newState.conditions[0].unloadEvaluator?.params[0]).toEqual(20);
expect(onError).toHaveBeenCalledWith('Enter a number less than or equal to 10');
});
});

172
public/app/features/expressions/components/thresholdReducer.ts Normal file
View File

@ -0,0 +1,172 @@
import { createAction, createReducer } from '@reduxjs/toolkit';
import { EvalFunction } from 'app/features/alerting/state/alertDef';
import { ClassicCondition, ExpressionQueryType, ThresholdExpressionQuery } from '../types';
export const updateRefId = createAction<string | undefined>('thresold/updateRefId');
export const updateThresholdType = createAction<{
evalFunction: EvalFunction;
onError: ((error: string | undefined) => void) | undefined;
}>('thresold/updateThresholdType');
export const updateThresholdParams = createAction<{ param: number; index: number }>('thresold/updateThresholdParams');
export const updateHysteresisChecked = createAction<{
hysteresisChecked: boolean;
onError: ((error: string | undefined) => void) | undefined;
}>('thresold/updateHysteresis');
export const updateUnloadParams = createAction<{
param: number;
index: number;
onError: ((error: string | undefined) => void) | undefined;
}>('thresold/updateUnloadParams');
export const thresholdReducer = createReducer<ThresholdExpressionQuery>(
{ type: ExpressionQueryType.threshold, refId: '', conditions: [] },
(builder) => {
builder.addCase(updateRefId, (state, action) => {
state.expression = action.payload;
});
builder.addCase(updateThresholdType, (state, action) => {
const typeInPayload = action.payload.evalFunction;
const onError = action.payload.onError;
//set new type in evaluator
state.conditions[0].evaluator.type = typeInPayload;
// check if hysteresis is checked
const hsyteresisIsChecked = Boolean(state.conditions[0].unloadEvaluator);
if (hsyteresisIsChecked) {
// when type whas changed and hsyteresIsChecked, we need to update the type for the unload evaluator with the opposite type
const updatedUnloadType = getUnloadEvaluatorTypeFromEvaluatorType(state.conditions[0].evaluator.type);
// set error to undefined when type is changed as we default to the new type that is valid
if (onError) {
onError(undefined); //clear error
}
// set newtype in evaluator
state.conditions[0].evaluator.type = typeInPayload;
// set new type and params in unload evaluator
const defaultUnloadEvaluator = {
type: updatedUnloadType,
params: state.conditions[0].evaluator?.params ?? [0, 0],
};
state.conditions[0].unloadEvaluator = defaultUnloadEvaluator;
}
});
builder.addCase(updateThresholdParams, (state, action) => {
const { param, index } = action.payload;
state.conditions[0].evaluator.params[index] = param;
});
builder.addCase(updateHysteresisChecked, (state, action) => {
const { hysteresisChecked, onError } = action.payload;
if (!hysteresisChecked) {
state.conditions[0].unloadEvaluator = undefined;
if (onError) {
onError(undefined); // clear error
}
} else {
state.conditions[0].unloadEvaluator = {
type: getUnloadEvaluatorTypeFromEvaluatorType(state.conditions[0].evaluator.type),
params: state.conditions[0].evaluator?.params ?? [0, 0],
};
}
});
builder.addCase(updateUnloadParams, (state, action) => {
const { param, index, onError } = action.payload;
// if there is no unload evaluator, we use the default evaluator params
if (!state.conditions[0].unloadEvaluator) {
state.conditions[0].unloadEvaluator = {
type: getUnloadEvaluatorTypeFromEvaluatorType(state.conditions[0].evaluator.type),
params: state.conditions[0].evaluator?.params ?? [0, 0],
};
} else {
// only update the param
state.conditions[0].unloadEvaluator!.params[index] = param;
}
// check if is valid for the new unload evaluator params
const error = isInvalid(state.conditions[0]);
const { errorMsg: invalidErrorMsg, errorMsgFrom, errorMsgTo } = error ?? {};
const errorMsg = invalidErrorMsg || errorMsgFrom || errorMsgTo;
// set error in form manually as we don't have a field for the unload evaluator
if (onError) {
onError(errorMsg);
}
});
}
);
function getUnloadEvaluatorTypeFromEvaluatorType(type: EvalFunction) {
// we don't let the user change the unload evaluator type. We just change it to the opposite of the evaluator type
if (type === EvalFunction.IsAbove) {
return EvalFunction.IsBelow;
}
if (type === EvalFunction.IsBelow) {
return EvalFunction.IsAbove;
}
if (type === EvalFunction.IsWithinRange) {
return EvalFunction.IsOutsideRange;
}
if (type === EvalFunction.IsOutsideRange) {
return EvalFunction.IsWithinRange;
}
return EvalFunction.IsBelow;
}
export function isInvalid(condition: ClassicCondition) {
// first check if the unload evaluator values are not empty
const { unloadEvaluator, evaluator } = condition;
if (!evaluator) {
return;
}
if (unloadEvaluator?.params[0] === undefined || Number.isNaN(unloadEvaluator?.params[0])) {
return { errorMsg: 'This value cannot be empty' };
}
const { type, params: loadParams } = evaluator;
const { params: unloadParams } = unloadEvaluator;
if (type === EvalFunction.IsWithinRange || type === EvalFunction.IsOutsideRange) {
if (unloadParams[0] === undefined || Number.isNaN(unloadParams[0])) {
return { errorMsgFrom: 'This value cannot be empty' };
}
if (unloadParams[1] === undefined || Number.isNaN(unloadParams[1])) {
return { errorMsgTo: 'This value cannot be empty' };
}
}
// check if the unload evaluator values are valid for the current load evaluator values
const [firstParamInUnloadEvaluator, secondParamInUnloadEvaluator] = unloadEvaluator.params;
const [firstParamInEvaluator, secondParamInEvaluator] = loadParams;
switch (type) {
case EvalFunction.IsAbove:
if (firstParamInUnloadEvaluator > firstParamInEvaluator) {
return { errorMsg: `Enter a number less than or equal to ${firstParamInEvaluator}` };
}
break;
case EvalFunction.IsBelow:
if (firstParamInUnloadEvaluator < firstParamInEvaluator) {
return { errorMsg: `Enter a number more than or equal to ${firstParamInEvaluator}` };
}
break;
case EvalFunction.IsOutsideRange:
if (firstParamInUnloadEvaluator < firstParamInEvaluator) {
return { errorMsgFrom: `Enter a number more than or equal to ${firstParamInEvaluator}` };
}
if (secondParamInUnloadEvaluator > secondParamInEvaluator) {
return { errorMsgTo: `Enter a number less than or equal to ${secondParamInEvaluator}` };
}
break;
case EvalFunction.IsWithinRange:
if (firstParamInUnloadEvaluator > firstParamInEvaluator) {
return { errorMsgFrom: `Enter a number less than or equal to ${firstParamInEvaluator}` };
}
if (secondParamInUnloadEvaluator < secondParamInEvaluator) {
return { errorMsgTo: `Enter a number be more than or equal to ${secondParamInEvaluator}` };
}
break;
default:
throw new Error(`evaluator function type ${type} not supported.`);
}
return;
}

7
public/app/features/expressions/types.ts
View File

@ -130,6 +130,9 @@ export interface ExpressionQuery extends DataQuery {
settings?: ExpressionQuerySettings;
}
export interface ThresholdExpressionQuery extends ExpressionQuery {
conditions: ClassicCondition[];
}
export interface ExpressionQuerySettings {
mode?: ReducerMode;
replaceWithValue?: number;
@ -140,6 +143,10 @@ export interface ClassicCondition {
params: number[];
type: EvalFunction;
};
unloadEvaluator?: {
params: number[];
type: EvalFunction;
};
operator?: {
type: string;
};