mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
Alerting: Update state manager to support nil stores and metrics (#57791)
This commit is contained in:
parent
d848cc629b
commit
3294918e9f
@ -58,7 +58,9 @@ func NewManager(metrics *metrics.State, externalURL *url.URL,
|
|||||||
clock: clock,
|
clock: clock,
|
||||||
externalURL: externalURL,
|
externalURL: externalURL,
|
||||||
}
|
}
|
||||||
|
if manager.metrics != nil {
|
||||||
go manager.recordMetrics()
|
go manager.recordMetrics()
|
||||||
|
}
|
||||||
return manager
|
return manager
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -67,6 +69,12 @@ func (st *Manager) Close() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (st *Manager) Warm(ctx context.Context) {
|
func (st *Manager) Warm(ctx context.Context) {
|
||||||
|
if st.instanceStore == nil {
|
||||||
|
st.log.Info("Skip warming the state because instance store is not configured")
|
||||||
|
}
|
||||||
|
if st.ruleStore == nil {
|
||||||
|
st.log.Info("Skip warming the state because rule store is not configured")
|
||||||
|
}
|
||||||
startTime := time.Now()
|
startTime := time.Now()
|
||||||
st.log.Info("Warming state cache for startup")
|
st.log.Info("Warming state cache for startup")
|
||||||
|
|
||||||
@ -149,7 +157,7 @@ func (st *Manager) ResetStateByRuleUID(ctx context.Context, ruleKey ngModels.Ale
|
|||||||
logger := st.log.New(ruleKey.LogContext()...)
|
logger := st.log.New(ruleKey.LogContext()...)
|
||||||
logger.Debug("Resetting state of the rule")
|
logger.Debug("Resetting state of the rule")
|
||||||
states := st.cache.removeByRuleUID(ruleKey.OrgID, ruleKey.UID)
|
states := st.cache.removeByRuleUID(ruleKey.OrgID, ruleKey.UID)
|
||||||
if len(states) > 0 {
|
if len(states) > 0 && st.instanceStore != nil {
|
||||||
err := st.instanceStore.DeleteAlertInstancesByRule(ctx, ruleKey)
|
err := st.instanceStore.DeleteAlertInstancesByRule(ctx, ruleKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Error("Failed to delete states that belong to a rule from database", "error", err)
|
logger.Error("Failed to delete states that belong to a rule from database", "error", err)
|
||||||
@ -172,7 +180,7 @@ func (st *Manager) ProcessEvalResults(ctx context.Context, evaluatedAt time.Time
|
|||||||
processedResults[s.CacheID] = s
|
processedResults[s.CacheID] = s
|
||||||
}
|
}
|
||||||
resolvedStates := st.staleResultsHandler(ctx, evaluatedAt, alertRule, processedResults, logger)
|
resolvedStates := st.staleResultsHandler(ctx, evaluatedAt, alertRule, processedResults, logger)
|
||||||
if len(states) > 0 {
|
if len(states) > 0 && st.instanceStore != nil {
|
||||||
logger.Debug("Saving new states to the database", "count", len(states))
|
logger.Debug("Saving new states to the database", "count", len(states))
|
||||||
_, _ = st.saveAlertStates(ctx, states...)
|
_, _ = st.saveAlertStates(ctx, states...)
|
||||||
}
|
}
|
||||||
@ -264,7 +272,7 @@ func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRu
|
|||||||
st.cache.set(currentState)
|
st.cache.set(currentState)
|
||||||
|
|
||||||
shouldUpdateAnnotation := oldState != currentState.State || oldReason != currentState.StateReason
|
shouldUpdateAnnotation := oldState != currentState.State || oldReason != currentState.StateReason
|
||||||
if shouldUpdateAnnotation {
|
if shouldUpdateAnnotation && st.historian != nil {
|
||||||
go st.historian.RecordState(ctx, alertRule, currentState.Labels, result.EvaluatedAt, InstanceStateAndReason{State: currentState.State, Reason: currentState.StateReason}, InstanceStateAndReason{State: oldState, Reason: oldReason})
|
go st.historian.RecordState(ctx, alertRule, currentState.Labels, result.EvaluatedAt, InstanceStateAndReason{State: currentState.State, Reason: currentState.StateReason}, InstanceStateAndReason{State: oldState, Reason: oldReason})
|
||||||
}
|
}
|
||||||
return currentState
|
return currentState
|
||||||
@ -304,6 +312,10 @@ func (st *Manager) Put(states []*State) {
|
|||||||
|
|
||||||
// TODO: Is the `State` type necessary? Should it embed the instance?
|
// TODO: Is the `State` type necessary? Should it embed the instance?
|
||||||
func (st *Manager) saveAlertStates(ctx context.Context, states ...*State) (saved, failed int) {
|
func (st *Manager) saveAlertStates(ctx context.Context, states ...*State) (saved, failed int) {
|
||||||
|
if st.instanceStore == nil {
|
||||||
|
return 0, 0
|
||||||
|
}
|
||||||
|
|
||||||
st.log.Debug("Saving alert states", "count", len(states))
|
st.log.Debug("Saving alert states", "count", len(states))
|
||||||
instances := make([]ngModels.AlertInstance, 0, len(states))
|
instances := make([]ngModels.AlertInstance, 0, len(states))
|
||||||
|
|
||||||
@ -400,18 +412,22 @@ func (st *Manager) staleResultsHandler(ctx context.Context, evaluatedAt time.Tim
|
|||||||
s.StateReason = ngModels.StateReasonMissingSeries
|
s.StateReason = ngModels.StateReasonMissingSeries
|
||||||
s.EndsAt = evaluatedAt
|
s.EndsAt = evaluatedAt
|
||||||
s.Resolved = true
|
s.Resolved = true
|
||||||
|
if st.historian != nil {
|
||||||
st.historian.RecordState(ctx, alertRule, s.Labels, evaluatedAt,
|
st.historian.RecordState(ctx, alertRule, s.Labels, evaluatedAt,
|
||||||
InstanceStateAndReason{State: eval.Normal, Reason: s.StateReason},
|
InstanceStateAndReason{State: eval.Normal, Reason: s.StateReason},
|
||||||
previousState,
|
previousState,
|
||||||
)
|
)
|
||||||
|
}
|
||||||
resolvedStates = append(resolvedStates, s)
|
resolvedStates = append(resolvedStates, s)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if st.instanceStore != nil {
|
||||||
if err := st.instanceStore.DeleteAlertInstances(ctx, toDelete...); err != nil {
|
if err := st.instanceStore.DeleteAlertInstances(ctx, toDelete...); err != nil {
|
||||||
logger.Error("Unable to delete stale instances from database", "error", err, "count", len(toDelete))
|
logger.Error("Unable to delete stale instances from database", "error", err, "count", len(toDelete))
|
||||||
}
|
}
|
||||||
|
}
|
||||||
return resolvedStates
|
return resolvedStates
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user