Alerting: Configure recording rule writer from config.ini (#89056)

This commit is contained in:
William Wernert
2024-06-12 16:04:46 -04:00
committed by GitHub
parent 7664b89209
commit c62cc25513
10 changed files with 136 additions and 49 deletions

View File

@@ -59,9 +59,10 @@ const (
// with intervals that are not exactly divided by this number not to be evaluated
SchedulerBaseInterval = 10 * time.Second
// DefaultRuleEvaluationInterval indicates a default interval of for how long a rule should be evaluated to change state from Pending to Alerting
DefaultRuleEvaluationInterval = SchedulerBaseInterval * 6 // == 60 seconds
stateHistoryDefaultEnabled = true
lokiDefaultMaxQueryLength = 721 * time.Hour // 30d1h, matches the default value in Loki
DefaultRuleEvaluationInterval = SchedulerBaseInterval * 6 // == 60 seconds
stateHistoryDefaultEnabled = true
lokiDefaultMaxQueryLength = 721 * time.Hour // 30d1h, matches the default value in Loki
defaultRecordingRequestTimeout = 10 * time.Second
)
type UnifiedAlertingSettings struct {
@@ -103,6 +104,8 @@ type UnifiedAlertingSettings struct {
SkipClustering bool
StateHistory UnifiedAlertingStateHistorySettings
RemoteAlertmanager RemoteAlertmanagerSettings
RecordingRules RecordingRuleSettings
// MaxStateSaveConcurrency controls the number of goroutines (per rule) that can save alert state in parallel.
MaxStateSaveConcurrency int
StatePeriodicSaveInterval time.Duration
@@ -112,6 +115,14 @@ type UnifiedAlertingSettings struct {
NotificationLogRetention time.Duration
}
type RecordingRuleSettings struct {
URL string
BasicAuthUsername string
BasicAuthPassword string
CustomHeaders map[string]string
Timeout time.Duration
}
// RemoteAlertmanagerSettings contains the configuration needed
// to disable the internal Alertmanager and use an external one instead.
type RemoteAlertmanagerSettings struct {
@@ -395,6 +406,23 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
}
uaCfg.StateHistory = uaCfgStateHistory
rr := iniFile.Section("recording_rules")
uaCfgRecordingRules := RecordingRuleSettings{
URL: rr.Key("url").MustString(""),
BasicAuthUsername: rr.Key("basic_auth_username").MustString(""),
BasicAuthPassword: rr.Key("basic_auth_password").MustString(""),
Timeout: rr.Key("timeout").MustDuration(defaultRecordingRequestTimeout),
}
rrHeaders := iniFile.Section("recording_rules.custom_headers")
rrHeadersKeys := rrHeaders.Keys()
uaCfgRecordingRules.CustomHeaders = make(map[string]string, len(rrHeadersKeys))
for _, key := range rrHeadersKeys {
uaCfgRecordingRules.CustomHeaders[key.Name()] = key.Value()
}
uaCfg.RecordingRules = uaCfgRecordingRules
uaCfg.MaxStateSaveConcurrency = ua.Key("max_state_save_concurrency").MustInt(1)
uaCfg.StatePeriodicSaveInterval, err = gtime.ParseDuration(valueAsString(ua, "state_periodic_save_interval", (time.Minute * 5).String()))