mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
Alerting: Simplify scheduler configuration and remove dependency on Grafana-wide settings (#59735)
* Make scheduler not depend directly on grafana-wide settings * Re-add missing interval
This commit is contained in:
parent
e7bdcb4ba8
commit
9977c7ea43
@ -100,7 +100,7 @@ func (ng *NGAlert) GetMultiOrgAlertmanagerMetrics() *MultiOrgAlertmanager {
|
|||||||
func NewNGAlert(r prometheus.Registerer) *NGAlert {
|
func NewNGAlert(r prometheus.Registerer) *NGAlert {
|
||||||
return &NGAlert{
|
return &NGAlert{
|
||||||
Registerer: r,
|
Registerer: r,
|
||||||
schedulerMetrics: newSchedulerMetrics(r),
|
schedulerMetrics: NewSchedulerMetrics(r),
|
||||||
stateMetrics: newStateMetrics(r),
|
stateMetrics: newStateMetrics(r),
|
||||||
multiOrgAlertmanagerMetrics: newMultiOrgAlertmanagerMetrics(r),
|
multiOrgAlertmanagerMetrics: newMultiOrgAlertmanagerMetrics(r),
|
||||||
apiMetrics: newAPIMetrics(r),
|
apiMetrics: newAPIMetrics(r),
|
||||||
@ -125,7 +125,7 @@ func (moa *MultiOrgAlertmanager) GetOrCreateOrgRegistry(id int64) prometheus.Reg
|
|||||||
return moa.registries.GetOrCreateOrgRegistry(id)
|
return moa.registries.GetOrCreateOrgRegistry(id)
|
||||||
}
|
}
|
||||||
|
|
||||||
func newSchedulerMetrics(r prometheus.Registerer) *Scheduler {
|
func NewSchedulerMetrics(r prometheus.Registerer) *Scheduler {
|
||||||
return &Scheduler{
|
return &Scheduler{
|
||||||
Registerer: r,
|
Registerer: r,
|
||||||
BehindSeconds: promauto.With(r).NewGauge(prometheus.GaugeOpts{
|
BehindSeconds: promauto.With(r).NewGauge(prometheus.GaugeOpts{
|
||||||
|
@ -184,17 +184,21 @@ func (ng *AlertNG) init() error {
|
|||||||
|
|
||||||
evalFactory := eval.NewEvaluatorFactory(ng.Cfg.UnifiedAlerting, ng.DataSourceCache, ng.ExpressionService)
|
evalFactory := eval.NewEvaluatorFactory(ng.Cfg.UnifiedAlerting, ng.DataSourceCache, ng.ExpressionService)
|
||||||
schedCfg := schedule.SchedulerCfg{
|
schedCfg := schedule.SchedulerCfg{
|
||||||
Cfg: ng.Cfg.UnifiedAlerting,
|
MaxAttempts: ng.Cfg.UnifiedAlerting.MaxAttempts,
|
||||||
C: clk,
|
C: clk,
|
||||||
EvaluatorFactory: evalFactory,
|
BaseInterval: ng.Cfg.UnifiedAlerting.BaseInterval,
|
||||||
RuleStore: store,
|
MinRuleInterval: ng.Cfg.UnifiedAlerting.MinInterval,
|
||||||
Metrics: ng.Metrics.GetSchedulerMetrics(),
|
DisableGrafanaFolder: ng.Cfg.UnifiedAlerting.ReservedLabels.IsReservedLabelDisabled(models.FolderTitleLabel),
|
||||||
AlertSender: alertsRouter,
|
AppURL: appUrl,
|
||||||
|
EvaluatorFactory: evalFactory,
|
||||||
|
RuleStore: store,
|
||||||
|
Metrics: ng.Metrics.GetSchedulerMetrics(),
|
||||||
|
AlertSender: alertsRouter,
|
||||||
}
|
}
|
||||||
|
|
||||||
historian := historian.NewAnnotationHistorian(ng.annotationsRepo, ng.dashboardService)
|
historian := historian.NewAnnotationHistorian(ng.annotationsRepo, ng.dashboardService)
|
||||||
stateManager := state.NewManager(ng.Metrics.GetStateMetrics(), appUrl, store, ng.imageService, clk, historian)
|
stateManager := state.NewManager(ng.Metrics.GetStateMetrics(), appUrl, store, ng.imageService, clk, historian)
|
||||||
scheduler := schedule.NewScheduler(schedCfg, appUrl, stateManager)
|
scheduler := schedule.NewScheduler(schedCfg, stateManager)
|
||||||
|
|
||||||
// if it is required to include folder title to the alerts, we need to subscribe to changes of alert title
|
// if it is required to include folder title to the alerts, we need to subscribe to changes of alert title
|
||||||
if !ng.Cfg.UnifiedAlerting.ReservedLabels.IsReservedLabelDisabled(models.FolderTitleLabel) {
|
if !ng.Cfg.UnifiedAlerting.ReservedLabels.IsReservedLabelDisabled(models.FolderTitleLabel) {
|
||||||
|
@ -18,7 +18,6 @@ import (
|
|||||||
"github.com/grafana/grafana/pkg/services/ngalert/state"
|
"github.com/grafana/grafana/pkg/services/ngalert/state"
|
||||||
"github.com/grafana/grafana/pkg/services/org"
|
"github.com/grafana/grafana/pkg/services/org"
|
||||||
"github.com/grafana/grafana/pkg/services/user"
|
"github.com/grafana/grafana/pkg/services/user"
|
||||||
"github.com/grafana/grafana/pkg/setting"
|
|
||||||
"github.com/grafana/grafana/pkg/util/ticker"
|
"github.com/grafana/grafana/pkg/util/ticker"
|
||||||
|
|
||||||
"github.com/benbjohnson/clock"
|
"github.com/benbjohnson/clock"
|
||||||
@ -98,29 +97,33 @@ type schedule struct {
|
|||||||
|
|
||||||
// SchedulerCfg is the scheduler configuration.
|
// SchedulerCfg is the scheduler configuration.
|
||||||
type SchedulerCfg struct {
|
type SchedulerCfg struct {
|
||||||
Cfg setting.UnifiedAlertingSettings
|
MaxAttempts int64
|
||||||
C clock.Clock
|
BaseInterval time.Duration
|
||||||
EvaluatorFactory eval.EvaluatorFactory
|
C clock.Clock
|
||||||
RuleStore RulesStore
|
MinRuleInterval time.Duration
|
||||||
Metrics *metrics.Scheduler
|
DisableGrafanaFolder bool
|
||||||
AlertSender AlertsSender
|
AppURL *url.URL
|
||||||
|
EvaluatorFactory eval.EvaluatorFactory
|
||||||
|
RuleStore RulesStore
|
||||||
|
Metrics *metrics.Scheduler
|
||||||
|
AlertSender AlertsSender
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewScheduler returns a new schedule.
|
// NewScheduler returns a new schedule.
|
||||||
func NewScheduler(cfg SchedulerCfg, appURL *url.URL, stateManager *state.Manager) *schedule {
|
func NewScheduler(cfg SchedulerCfg, stateManager *state.Manager) *schedule {
|
||||||
sch := schedule{
|
sch := schedule{
|
||||||
registry: alertRuleInfoRegistry{alertRuleInfo: make(map[ngmodels.AlertRuleKey]*alertRuleInfo)},
|
registry: alertRuleInfoRegistry{alertRuleInfo: make(map[ngmodels.AlertRuleKey]*alertRuleInfo)},
|
||||||
maxAttempts: cfg.Cfg.MaxAttempts,
|
maxAttempts: cfg.MaxAttempts,
|
||||||
clock: cfg.C,
|
clock: cfg.C,
|
||||||
baseInterval: cfg.Cfg.BaseInterval,
|
baseInterval: cfg.BaseInterval,
|
||||||
log: log.New("ngalert.scheduler"),
|
log: log.New("ngalert.scheduler"),
|
||||||
evaluatorFactory: cfg.EvaluatorFactory,
|
evaluatorFactory: cfg.EvaluatorFactory,
|
||||||
ruleStore: cfg.RuleStore,
|
ruleStore: cfg.RuleStore,
|
||||||
metrics: cfg.Metrics,
|
metrics: cfg.Metrics,
|
||||||
appURL: appURL,
|
appURL: cfg.AppURL,
|
||||||
disableGrafanaFolder: cfg.Cfg.ReservedLabels.IsReservedLabelDisabled(ngmodels.FolderTitleLabel),
|
disableGrafanaFolder: cfg.DisableGrafanaFolder,
|
||||||
stateManager: stateManager,
|
stateManager: stateManager,
|
||||||
minRuleInterval: cfg.Cfg.MinInterval,
|
minRuleInterval: cfg.MinRuleInterval,
|
||||||
schedulableAlertRules: alertRulesRegistry{rules: make(map[ngmodels.AlertRuleKey]*ngmodels.AlertRule)},
|
schedulableAlertRules: alertRulesRegistry{rules: make(map[ngmodels.AlertRuleKey]*ngmodels.AlertRule)},
|
||||||
alertsSender: cfg.AlertSender,
|
alertsSender: cfg.AlertSender,
|
||||||
}
|
}
|
||||||
|
@ -54,20 +54,22 @@ func TestProcessTicks(t *testing.T) {
|
|||||||
notifier := &AlertsSenderMock{}
|
notifier := &AlertsSenderMock{}
|
||||||
notifier.EXPECT().Send(mock.Anything, mock.Anything).Return()
|
notifier.EXPECT().Send(mock.Anything, mock.Anything).Return()
|
||||||
|
|
||||||
schedCfg := SchedulerCfg{
|
|
||||||
Cfg: cfg,
|
|
||||||
C: mockedClock,
|
|
||||||
RuleStore: ruleStore,
|
|
||||||
Metrics: testMetrics.GetSchedulerMetrics(),
|
|
||||||
AlertSender: notifier,
|
|
||||||
}
|
|
||||||
st := state.NewManager(testMetrics.GetStateMetrics(), nil, nil, &state.NoopImageService{}, mockedClock, &state.FakeHistorian{})
|
|
||||||
|
|
||||||
appUrl := &url.URL{
|
appUrl := &url.URL{
|
||||||
Scheme: "http",
|
Scheme: "http",
|
||||||
Host: "localhost",
|
Host: "localhost",
|
||||||
}
|
}
|
||||||
sched := NewScheduler(schedCfg, appUrl, st)
|
|
||||||
|
schedCfg := SchedulerCfg{
|
||||||
|
BaseInterval: cfg.BaseInterval,
|
||||||
|
C: mockedClock,
|
||||||
|
AppURL: appUrl,
|
||||||
|
RuleStore: ruleStore,
|
||||||
|
Metrics: testMetrics.GetSchedulerMetrics(),
|
||||||
|
AlertSender: notifier,
|
||||||
|
}
|
||||||
|
st := state.NewManager(testMetrics.GetStateMetrics(), nil, nil, &state.NoopImageService{}, mockedClock, &state.FakeHistorian{})
|
||||||
|
|
||||||
|
sched := NewScheduler(schedCfg, st)
|
||||||
|
|
||||||
evalAppliedCh := make(chan evalAppliedInfo, 1)
|
evalAppliedCh := make(chan evalAppliedInfo, 1)
|
||||||
stopAppliedCh := make(chan models.AlertRuleKey, 1)
|
stopAppliedCh := make(chan models.AlertRuleKey, 1)
|
||||||
@ -673,8 +675,10 @@ func setupScheduler(t *testing.T, rs *fakeRulesStore, is *state.FakeInstanceStor
|
|||||||
}
|
}
|
||||||
|
|
||||||
schedCfg := SchedulerCfg{
|
schedCfg := SchedulerCfg{
|
||||||
Cfg: cfg,
|
BaseInterval: cfg.BaseInterval,
|
||||||
|
MaxAttempts: cfg.MaxAttempts,
|
||||||
C: mockedClock,
|
C: mockedClock,
|
||||||
|
AppURL: appUrl,
|
||||||
EvaluatorFactory: evaluator,
|
EvaluatorFactory: evaluator,
|
||||||
RuleStore: rs,
|
RuleStore: rs,
|
||||||
Metrics: m.GetSchedulerMetrics(),
|
Metrics: m.GetSchedulerMetrics(),
|
||||||
@ -682,7 +686,7 @@ func setupScheduler(t *testing.T, rs *fakeRulesStore, is *state.FakeInstanceStor
|
|||||||
}
|
}
|
||||||
|
|
||||||
st := state.NewManager(m.GetStateMetrics(), nil, is, &state.NoopImageService{}, mockedClock, &state.FakeHistorian{})
|
st := state.NewManager(m.GetStateMetrics(), nil, is, &state.NoopImageService{}, mockedClock, &state.FakeHistorian{})
|
||||||
return NewScheduler(schedCfg, appUrl, st)
|
return NewScheduler(schedCfg, st)
|
||||||
}
|
}
|
||||||
|
|
||||||
func withQueryForState(t *testing.T, evalResult eval.State) models.AlertRuleMutator {
|
func withQueryForState(t *testing.T, evalResult eval.State) models.AlertRuleMutator {
|
||||||
|
Loading…
Reference in New Issue
Block a user