IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Update state manager to support nil stores and metrics (#57791)

Browse Source
This commit is contained in:
Yuriy Tseretyan 2022-10-28 13:10:28 -04:00 committed by GitHub
parent d848cc629b
commit 3294918e9f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 26 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
pkg/services/ngalert/state/manager.go
View File

@ -58,7 +58,9 @@ func NewManager(metrics *metrics.State, externalURL *url.URL,
clock: clock, clock: clock,
externalURL: externalURL, externalURL: externalURL,
} }
go manager.recordMetrics() if manager.metrics != nil {
go manager.recordMetrics()
}
return manager return manager
} }
@ -67,6 +69,12 @@ func (st *Manager) Close() {
} }
func (st *Manager) Warm(ctx context.Context) { func (st *Manager) Warm(ctx context.Context) {
if st.instanceStore == nil {
st.log.Info("Skip warming the state because instance store is not configured")
}
if st.ruleStore == nil {
st.log.Info("Skip warming the state because rule store is not configured")
}
startTime := time.Now() startTime := time.Now()
st.log.Info("Warming state cache for startup") st.log.Info("Warming state cache for startup")
@ -149,7 +157,7 @@ func (st *Manager) ResetStateByRuleUID(ctx context.Context, ruleKey ngModels.Ale
logger := st.log.New(ruleKey.LogContext()...) logger := st.log.New(ruleKey.LogContext()...)
logger.Debug("Resetting state of the rule") logger.Debug("Resetting state of the rule")
states := st.cache.removeByRuleUID(ruleKey.OrgID, ruleKey.UID) states := st.cache.removeByRuleUID(ruleKey.OrgID, ruleKey.UID)
if len(states) > 0 { if len(states) > 0 && st.instanceStore != nil {
err := st.instanceStore.DeleteAlertInstancesByRule(ctx, ruleKey) err := st.instanceStore.DeleteAlertInstancesByRule(ctx, ruleKey)
if err != nil { if err != nil {
logger.Error("Failed to delete states that belong to a rule from database", "error", err) logger.Error("Failed to delete states that belong to a rule from database", "error", err)
@ -172,7 +180,7 @@ func (st *Manager) ProcessEvalResults(ctx context.Context, evaluatedAt time.Time
processedResults[s.CacheID] = s processedResults[s.CacheID] = s
} }
resolvedStates := st.staleResultsHandler(ctx, evaluatedAt, alertRule, processedResults, logger) resolvedStates := st.staleResultsHandler(ctx, evaluatedAt, alertRule, processedResults, logger)
if len(states) > 0 { if len(states) > 0 && st.instanceStore != nil {
logger.Debug("Saving new states to the database", "count", len(states)) logger.Debug("Saving new states to the database", "count", len(states))
_, _ = st.saveAlertStates(ctx, states...) _, _ = st.saveAlertStates(ctx, states...)
} }
@ -264,7 +272,7 @@ func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRu
st.cache.set(currentState) st.cache.set(currentState)
shouldUpdateAnnotation := oldState != currentState.State || oldReason != currentState.StateReason shouldUpdateAnnotation := oldState != currentState.State || oldReason != currentState.StateReason
if shouldUpdateAnnotation { if shouldUpdateAnnotation && st.historian != nil {
go st.historian.RecordState(ctx, alertRule, currentState.Labels, result.EvaluatedAt, InstanceStateAndReason{State: currentState.State, Reason: currentState.StateReason}, InstanceStateAndReason{State: oldState, Reason: oldReason}) go st.historian.RecordState(ctx, alertRule, currentState.Labels, result.EvaluatedAt, InstanceStateAndReason{State: currentState.State, Reason: currentState.StateReason}, InstanceStateAndReason{State: oldState, Reason: oldReason})
} }
return currentState return currentState
@ -304,6 +312,10 @@ func (st *Manager) Put(states []*State) {
// TODO: Is the `State` type necessary? Should it embed the instance? // TODO: Is the `State` type necessary? Should it embed the instance?
func (st *Manager) saveAlertStates(ctx context.Context, states ...*State) (saved, failed int) { func (st *Manager) saveAlertStates(ctx context.Context, states ...*State) (saved, failed int) {
if st.instanceStore == nil {
return 0, 0
}
st.log.Debug("Saving alert states", "count", len(states)) st.log.Debug("Saving alert states", "count", len(states))
instances := make([]ngModels.AlertInstance, 0, len(states)) instances := make([]ngModels.AlertInstance, 0, len(states))
@ -400,17 +412,21 @@ func (st *Manager) staleResultsHandler(ctx context.Context, evaluatedAt time.Tim
s.StateReason = ngModels.StateReasonMissingSeries s.StateReason = ngModels.StateReasonMissingSeries
s.EndsAt = evaluatedAt s.EndsAt = evaluatedAt
s.Resolved = true s.Resolved = true
st.historian.RecordState(ctx, alertRule, s.Labels, evaluatedAt, if st.historian != nil {
InstanceStateAndReason{State: eval.Normal, Reason: s.StateReason}, st.historian.RecordState(ctx, alertRule, s.Labels, evaluatedAt,
previousState, InstanceStateAndReason{State: eval.Normal, Reason: s.StateReason},
) previousState,
)
}
resolvedStates = append(resolvedStates, s) resolvedStates = append(resolvedStates, s)
} }
} }
} }
if err := st.instanceStore.DeleteAlertInstances(ctx, toDelete...); err != nil { if st.instanceStore != nil {
logger.Error("Unable to delete stale instances from database", "error", err, "count", len(toDelete)) if err := st.instanceStore.DeleteAlertInstances(ctx, toDelete...); err != nil {
logger.Error("Unable to delete stale instances from database", "error", err, "count", len(toDelete))
}
} }
return resolvedStates return resolvedStates
} }