2020-11-12 07:11:30 -06:00
|
|
|
package ngalert
|
|
|
|
|
|
|
|
import (
|
2020-12-17 08:00:09 -06:00
|
|
|
"context"
|
|
|
|
"time"
|
|
|
|
|
2021-04-30 11:28:06 -05:00
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
|
2021-03-24 17:34:18 -05:00
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/state"
|
|
|
|
|
2020-12-17 08:00:09 -06:00
|
|
|
"github.com/benbjohnson/clock"
|
2020-11-12 07:11:30 -06:00
|
|
|
|
|
|
|
"github.com/grafana/grafana/pkg/api/routing"
|
|
|
|
"github.com/grafana/grafana/pkg/infra/log"
|
|
|
|
"github.com/grafana/grafana/pkg/registry"
|
2021-03-24 09:20:44 -05:00
|
|
|
"github.com/grafana/grafana/pkg/services/datasourceproxy"
|
2020-11-12 07:11:30 -06:00
|
|
|
"github.com/grafana/grafana/pkg/services/datasources"
|
2021-03-24 09:20:44 -05:00
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/api"
|
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/eval"
|
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/notifier"
|
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/schedule"
|
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/store"
|
|
|
|
"github.com/grafana/grafana/pkg/services/sqlstore"
|
2020-11-12 07:11:30 -06:00
|
|
|
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
|
|
|
"github.com/grafana/grafana/pkg/setting"
|
2021-03-24 09:20:44 -05:00
|
|
|
"github.com/grafana/grafana/pkg/tsdb"
|
2020-11-12 07:11:30 -06:00
|
|
|
)
|
|
|
|
|
2020-12-17 08:00:09 -06:00
|
|
|
const (
|
|
|
|
maxAttempts int64 = 3
|
|
|
|
// scheduler interval
|
|
|
|
// changing this value is discouraged
|
|
|
|
// because this could cause existing alert definition
|
|
|
|
// with intervals that are not exactly divided by this number
|
|
|
|
// not to be evaluated
|
|
|
|
baseIntervalSeconds = 10
|
|
|
|
// default alert definiiton interval
|
|
|
|
defaultIntervalSeconds int64 = 6 * baseIntervalSeconds
|
|
|
|
)
|
|
|
|
|
2020-11-12 07:11:30 -06:00
|
|
|
// AlertNG is the service for evaluating the condition of an alert definition.
|
|
|
|
type AlertNG struct {
|
2021-03-19 09:32:13 -05:00
|
|
|
Cfg *setting.Cfg `inject:""`
|
|
|
|
DatasourceCache datasources.CacheService `inject:""`
|
|
|
|
RouteRegister routing.RouteRegister `inject:""`
|
|
|
|
SQLStore *sqlstore.SQLStore `inject:""`
|
|
|
|
DataService *tsdb.Service `inject:""`
|
2021-03-24 09:20:44 -05:00
|
|
|
Alertmanager *notifier.Alertmanager `inject:""`
|
2021-03-19 09:32:13 -05:00
|
|
|
DataProxy *datasourceproxy.DatasourceProxyService `inject:""`
|
2021-04-30 11:28:06 -05:00
|
|
|
Metrics *metrics.Metrics `inject:""`
|
2021-03-08 14:19:21 -06:00
|
|
|
Log log.Logger
|
|
|
|
schedule schedule.ScheduleService
|
2021-04-23 14:32:25 -05:00
|
|
|
stateManager *state.Manager
|
2020-11-12 07:11:30 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
registry.RegisterService(&AlertNG{})
|
|
|
|
}
|
|
|
|
|
|
|
|
// Init initializes the AlertingService.
|
|
|
|
func (ng *AlertNG) Init() error {
|
2021-03-08 14:19:21 -06:00
|
|
|
ng.Log = log.New("ngalert")
|
2021-04-30 11:28:06 -05:00
|
|
|
ng.stateManager = state.NewManager(ng.Log, ng.Metrics)
|
2021-03-03 09:52:19 -06:00
|
|
|
baseInterval := baseIntervalSeconds * time.Second
|
|
|
|
|
2021-03-08 14:19:21 -06:00
|
|
|
store := store.DBstore{BaseInterval: baseInterval, DefaultIntervalSeconds: defaultIntervalSeconds, SQLStore: ng.SQLStore}
|
2021-03-03 09:52:19 -06:00
|
|
|
|
2021-03-08 14:19:21 -06:00
|
|
|
schedCfg := schedule.SchedulerCfg{
|
2021-05-03 06:19:15 -05:00
|
|
|
C: clock.New(),
|
|
|
|
BaseInterval: baseInterval,
|
|
|
|
Logger: ng.Log,
|
|
|
|
MaxAttempts: maxAttempts,
|
|
|
|
Evaluator: eval.Evaluator{Cfg: ng.Cfg},
|
|
|
|
InstanceStore: store,
|
|
|
|
RuleStore: store,
|
|
|
|
Notifier: ng.Alertmanager,
|
2021-01-22 11:27:33 -06:00
|
|
|
}
|
2021-03-08 14:19:21 -06:00
|
|
|
ng.schedule = schedule.NewScheduler(schedCfg, ng.DataService)
|
2021-03-03 09:52:19 -06:00
|
|
|
|
2021-03-08 14:19:21 -06:00
|
|
|
api := api.API{
|
2021-03-03 09:52:19 -06:00
|
|
|
Cfg: ng.Cfg,
|
|
|
|
DatasourceCache: ng.DatasourceCache,
|
|
|
|
RouteRegister: ng.RouteRegister,
|
2021-03-08 00:02:49 -06:00
|
|
|
DataService: ng.DataService,
|
2021-03-08 14:19:21 -06:00
|
|
|
Schedule: ng.schedule,
|
2021-03-19 09:32:13 -05:00
|
|
|
DataProxy: ng.DataProxy,
|
2021-05-03 06:19:15 -05:00
|
|
|
InstanceStore: store,
|
2021-04-01 03:11:45 -05:00
|
|
|
RuleStore: store,
|
2021-03-31 15:00:56 -05:00
|
|
|
AlertingStore: store,
|
2021-03-24 09:20:44 -05:00
|
|
|
Alertmanager: ng.Alertmanager,
|
2021-04-23 14:32:25 -05:00
|
|
|
StateManager: ng.stateManager,
|
2021-03-24 09:20:44 -05:00
|
|
|
}
|
2021-04-30 11:28:06 -05:00
|
|
|
api.RegisterAPIEndpoints(ng.Metrics)
|
2021-03-03 09:52:19 -06:00
|
|
|
|
2020-11-12 07:11:30 -06:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-12-17 08:00:09 -06:00
|
|
|
// Run starts the scheduler
|
|
|
|
func (ng *AlertNG) Run(ctx context.Context) error {
|
2021-03-08 14:19:21 -06:00
|
|
|
ng.Log.Debug("ngalert starting")
|
2021-04-23 14:32:25 -05:00
|
|
|
ng.schedule.WarmStateCache(ng.stateManager)
|
|
|
|
return ng.schedule.Ticker(ctx, ng.stateManager)
|
2020-12-17 08:00:09 -06:00
|
|
|
}
|
|
|
|
|
2020-11-12 07:11:30 -06:00
|
|
|
// IsDisabled returns true if the alerting service is disable for this instance.
|
|
|
|
func (ng *AlertNG) IsDisabled() bool {
|
|
|
|
if ng.Cfg == nil {
|
2020-12-17 08:00:09 -06:00
|
|
|
return true
|
2020-11-12 07:11:30 -06:00
|
|
|
}
|
|
|
|
return !ng.Cfg.IsNgAlertEnabled()
|
|
|
|
}
|
|
|
|
|
|
|
|
// AddMigration defines database migrations.
|
|
|
|
// If Alerting NG is not enabled does nothing.
|
|
|
|
func (ng *AlertNG) AddMigration(mg *migrator.Migrator) {
|
|
|
|
if ng.IsDisabled() {
|
|
|
|
return
|
|
|
|
}
|
2021-03-31 15:00:56 -05:00
|
|
|
store.AddAlertDefinitionMigrations(mg, defaultIntervalSeconds)
|
|
|
|
store.AddAlertDefinitionVersionMigrations(mg)
|
2021-01-18 12:57:17 -06:00
|
|
|
// Create alert_instance table
|
2021-03-31 15:00:56 -05:00
|
|
|
store.AlertInstanceMigration(mg)
|
|
|
|
|
2021-04-01 03:11:45 -05:00
|
|
|
// Create alert_rule
|
|
|
|
store.AddAlertRuleMigrations(mg, defaultIntervalSeconds)
|
|
|
|
store.AddAlertRuleVersionMigrations(mg)
|
2020-11-12 07:11:30 -06:00
|
|
|
}
|