Alerting: Support hysteresis command expression (#75189)

Backend: 

* Update the Grafana Alerting engine to provide feedback to HysteresisCommand. The feedback information is stored in state.Manager as a fingerprint of each state. The fingerprint is persisted to the database. Only fingerprints that belong to Pending and Alerting states are considered as "loaded" and provided back to the command.
   - add ResultFingerprint to state.State. It's different from other fingerprints we store in the state because it is calculated from the result labels.
  -  add rule_fingerprint column to alert_instance
   - update alerting evaluator to accept AlertingResultsReader via context, and update scheduler to provide it.
   - add AlertingResultsFromRuleState that implements the new interface in eval package
   - update getExprRequest to patch the hysteresis command.

* Only one "Recovery Threshold" query is allowed to be used in the alert rule and it must be the Condition.


Frontend:

* Add hysteresis option to Threshold in UI. It's called "Recovery Threshold"
* Add test for getUnloadEvaluatorTypeFromCondition
* Hide hysteresis in panel expressions

* Refactor isInvalid and add test for it
* Remove unnecesary React.memo
* Add tests for updateEvaluatorConditions

---------

Co-authored-by: Sonia Aguilar <soniaaguilarpeiron@gmail.com>
This commit is contained in:
Yuri Tseretyan
2024-01-04 11:47:13 -05:00
committed by GitHub
parent 29c251851d
commit f6a46744a6
33 changed files with 1804 additions and 201 deletions

View File

@@ -15,6 +15,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/google/uuid"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/prometheus/common/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -1315,3 +1316,77 @@ func TestIntegrationRulePause(t *testing.T) {
})
}
}
func TestIntegrationHysteresisRule(t *testing.T) {
testinfra.SQLiteIntegrationTest(t)
// Setup Grafana and its Database. Scheduler is set to evaluate every 1 second
dir, p := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
DisableLegacyAlerting: true,
EnableUnifiedAlerting: true,
DisableAnonymous: true,
AppModeProduction: true,
NGAlertSchedulerBaseInterval: 1 * time.Second,
EnableFeatureToggles: []string{featuremgmt.FlagConfigurableSchedulerTick, featuremgmt.FlagRecoveryThreshold},
})
grafanaListedAddr, store := testinfra.StartGrafana(t, dir, p)
// Create a user to make authenticated requests
createUser(t, store, user.CreateUserCommand{
DefaultOrgRole: string(org.RoleAdmin),
Password: "password",
Login: "grafana",
})
apiClient := newAlertingApiClient(grafanaListedAddr, "grafana", "password")
folder := "hysteresis"
testDs := apiClient.CreateTestDatasource(t)
apiClient.CreateFolder(t, folder, folder)
bodyRaw, err := testData.ReadFile("test-data/hysteresis_rule.json")
require.NoError(t, err)
var postData apimodels.PostableRuleGroupConfig
require.NoError(t, json.Unmarshal(bodyRaw, &postData))
for _, rule := range postData.Rules {
for i := range rule.GrafanaManagedAlert.Data {
rule.GrafanaManagedAlert.Data[i].DatasourceUID = strings.ReplaceAll(rule.GrafanaManagedAlert.Data[i].DatasourceUID, "REPLACE_ME", testDs.Body.Datasource.UID)
}
}
changes, status, body := apiClient.PostRulesGroupWithStatus(t, folder, &postData)
require.Equalf(t, http.StatusAccepted, status, body)
require.Len(t, changes.Created, 1)
ruleUid := changes.Created[0]
var frame data.Frame
require.Eventuallyf(t, func() bool {
frame, status, body = apiClient.GetRuleHistoryWithStatus(t, ruleUid)
require.Equalf(t, http.StatusOK, status, body)
return frame.Rows() > 1
}, 15*time.Second, 1*time.Second, "Alert state history expected to have more than one record but got %d. Body: %s", frame.Rows(), body)
f, _ := frame.FieldByName("next")
alertingIdx := 0
normalIdx := 1
if f.At(alertingIdx).(string) != "Alerting" {
alertingIdx = 1
normalIdx = 0
}
assert.Equalf(t, "Alerting", f.At(alertingIdx).(string), body)
assert.Equalf(t, "Normal", f.At(normalIdx).(string), body)
type HistoryData struct {
Values map[string]int64
}
f, _ = frame.FieldByName("data")
var d HistoryData
require.NoErrorf(t, json.Unmarshal([]byte(f.At(alertingIdx).(string)), &d), body)
assert.EqualValuesf(t, 5, d.Values["B"], body)
require.NoErrorf(t, json.Unmarshal([]byte(f.At(normalIdx).(string)), &d), body)
assert.EqualValuesf(t, 1, d.Values["B"], body)
}