grafana/pkg/services/ngalert/state/state.go
David Parrott 788bc2a793
Alerting: refactor state tracker (#33292)
* set processing time

* merge labels and set on response

* use state cache for adding alerts to rules

* minor cleanup

* add support for NoData and Error results

* rename test

* bring in changes from other PRs tha have been merged

* pr feedback

* add integration test

* close state tracker cleanup on context.Done

* fixup test

* rename state tracker

* set EvaluationDuration on Result

* default labels set as constants

* separate cache and state from manager

* use RWMutex in cache
2021-04-23 21:32:25 +02:00

131 lines
3.6 KiB
Go

package state
import (
"time"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
ngModels "github.com/grafana/grafana/pkg/services/ngalert/models"
)
type State struct {
AlertRuleUID string
OrgID int64
CacheId string
Labels data.Labels
State eval.State
Results []Evaluation
StartsAt time.Time
EndsAt time.Time
LastEvaluationTime time.Time
EvaluationDuration time.Duration
Annotations map[string]string
}
type Evaluation struct {
EvaluationTime time.Time
EvaluationState eval.State
}
func resultNormal(alertState *State, result eval.Result) *State {
newState := alertState
if alertState.State != eval.Normal {
newState.EndsAt = result.EvaluatedAt
}
newState.State = eval.Normal
return newState
}
func (a *State) resultAlerting(alertRule *ngModels.AlertRule, result eval.Result) *State {
switch a.State {
case eval.Alerting:
if !(alertRule.For > 0) {
// If there is not For set, we will set EndsAt to be twice the evaluation interval
// to avoid flapping with every evaluation
a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second)
return a
}
a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
case eval.Pending:
if result.EvaluatedAt.Sub(a.StartsAt) > alertRule.For {
a.State = eval.Alerting
a.StartsAt = result.EvaluatedAt
a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
a.Annotations["alerting_at"] = result.EvaluatedAt.String()
}
default:
a.StartsAt = result.EvaluatedAt
if !(alertRule.For > 0) {
a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second)
a.State = eval.Alerting
a.Annotations["alerting_at"] = result.EvaluatedAt.String()
} else {
a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
if result.EvaluatedAt.Sub(a.StartsAt) > alertRule.For {
a.State = eval.Alerting
a.Annotations["alerting_at"] = result.EvaluatedAt.String()
} else {
a.State = eval.Pending
}
}
}
return a
}
func (a *State) resultError(alertRule *ngModels.AlertRule, result eval.Result) *State {
if a.StartsAt.IsZero() {
a.StartsAt = result.EvaluatedAt
}
if !(alertRule.For > 0) {
a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second)
} else {
a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
}
if a.State != eval.Error {
a.Annotations["last_error"] = result.EvaluatedAt.String()
}
switch alertRule.ExecErrState {
case ngModels.AlertingErrState:
a.State = eval.Alerting
case ngModels.KeepLastStateErrState:
}
return a
}
func (a *State) resultNoData(alertRule *ngModels.AlertRule, result eval.Result) *State {
if a.StartsAt.IsZero() {
a.StartsAt = result.EvaluatedAt
}
if !(alertRule.For > 0) {
a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second)
} else {
a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
}
if a.State != eval.NoData {
a.Annotations["no_data"] = result.EvaluatedAt.String()
}
switch alertRule.NoDataState {
case ngModels.Alerting:
a.State = eval.Alerting
case ngModels.NoData:
a.State = eval.NoData
case ngModels.KeepLastState:
case ngModels.OK:
a.State = eval.Normal
}
return a
}
func (a *State) Equals(b *State) bool {
return a.AlertRuleUID == b.AlertRuleUID &&
a.OrgID == b.OrgID &&
a.CacheId == b.CacheId &&
a.Labels.String() == b.Labels.String() &&
a.State.String() == b.State.String() &&
a.StartsAt == b.StartsAt &&
a.EndsAt == b.EndsAt &&
a.LastEvaluationTime == b.LastEvaluationTime
}