mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
Alerting: Update state manager to support nil stores and metrics (#57791)
This commit is contained in:
parent
d848cc629b
commit
3294918e9f
@ -58,7 +58,9 @@ func NewManager(metrics *metrics.State, externalURL *url.URL,
|
||||
clock: clock,
|
||||
externalURL: externalURL,
|
||||
}
|
||||
go manager.recordMetrics()
|
||||
if manager.metrics != nil {
|
||||
go manager.recordMetrics()
|
||||
}
|
||||
return manager
|
||||
}
|
||||
|
||||
@ -67,6 +69,12 @@ func (st *Manager) Close() {
|
||||
}
|
||||
|
||||
func (st *Manager) Warm(ctx context.Context) {
|
||||
if st.instanceStore == nil {
|
||||
st.log.Info("Skip warming the state because instance store is not configured")
|
||||
}
|
||||
if st.ruleStore == nil {
|
||||
st.log.Info("Skip warming the state because rule store is not configured")
|
||||
}
|
||||
startTime := time.Now()
|
||||
st.log.Info("Warming state cache for startup")
|
||||
|
||||
@ -149,7 +157,7 @@ func (st *Manager) ResetStateByRuleUID(ctx context.Context, ruleKey ngModels.Ale
|
||||
logger := st.log.New(ruleKey.LogContext()...)
|
||||
logger.Debug("Resetting state of the rule")
|
||||
states := st.cache.removeByRuleUID(ruleKey.OrgID, ruleKey.UID)
|
||||
if len(states) > 0 {
|
||||
if len(states) > 0 && st.instanceStore != nil {
|
||||
err := st.instanceStore.DeleteAlertInstancesByRule(ctx, ruleKey)
|
||||
if err != nil {
|
||||
logger.Error("Failed to delete states that belong to a rule from database", "error", err)
|
||||
@ -172,7 +180,7 @@ func (st *Manager) ProcessEvalResults(ctx context.Context, evaluatedAt time.Time
|
||||
processedResults[s.CacheID] = s
|
||||
}
|
||||
resolvedStates := st.staleResultsHandler(ctx, evaluatedAt, alertRule, processedResults, logger)
|
||||
if len(states) > 0 {
|
||||
if len(states) > 0 && st.instanceStore != nil {
|
||||
logger.Debug("Saving new states to the database", "count", len(states))
|
||||
_, _ = st.saveAlertStates(ctx, states...)
|
||||
}
|
||||
@ -264,7 +272,7 @@ func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRu
|
||||
st.cache.set(currentState)
|
||||
|
||||
shouldUpdateAnnotation := oldState != currentState.State || oldReason != currentState.StateReason
|
||||
if shouldUpdateAnnotation {
|
||||
if shouldUpdateAnnotation && st.historian != nil {
|
||||
go st.historian.RecordState(ctx, alertRule, currentState.Labels, result.EvaluatedAt, InstanceStateAndReason{State: currentState.State, Reason: currentState.StateReason}, InstanceStateAndReason{State: oldState, Reason: oldReason})
|
||||
}
|
||||
return currentState
|
||||
@ -304,6 +312,10 @@ func (st *Manager) Put(states []*State) {
|
||||
|
||||
// TODO: Is the `State` type necessary? Should it embed the instance?
|
||||
func (st *Manager) saveAlertStates(ctx context.Context, states ...*State) (saved, failed int) {
|
||||
if st.instanceStore == nil {
|
||||
return 0, 0
|
||||
}
|
||||
|
||||
st.log.Debug("Saving alert states", "count", len(states))
|
||||
instances := make([]ngModels.AlertInstance, 0, len(states))
|
||||
|
||||
@ -400,17 +412,21 @@ func (st *Manager) staleResultsHandler(ctx context.Context, evaluatedAt time.Tim
|
||||
s.StateReason = ngModels.StateReasonMissingSeries
|
||||
s.EndsAt = evaluatedAt
|
||||
s.Resolved = true
|
||||
st.historian.RecordState(ctx, alertRule, s.Labels, evaluatedAt,
|
||||
InstanceStateAndReason{State: eval.Normal, Reason: s.StateReason},
|
||||
previousState,
|
||||
)
|
||||
if st.historian != nil {
|
||||
st.historian.RecordState(ctx, alertRule, s.Labels, evaluatedAt,
|
||||
InstanceStateAndReason{State: eval.Normal, Reason: s.StateReason},
|
||||
previousState,
|
||||
)
|
||||
}
|
||||
resolvedStates = append(resolvedStates, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := st.instanceStore.DeleteAlertInstances(ctx, toDelete...); err != nil {
|
||||
logger.Error("Unable to delete stale instances from database", "error", err, "count", len(toDelete))
|
||||
if st.instanceStore != nil {
|
||||
if err := st.instanceStore.DeleteAlertInstances(ctx, toDelete...); err != nil {
|
||||
logger.Error("Unable to delete stale instances from database", "error", err, "count", len(toDelete))
|
||||
}
|
||||
}
|
||||
return resolvedStates
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user