2020-11-12 07:11:30 -06:00
|
|
|
package ngalert
|
|
|
|
|
|
|
|
import (
|
2020-12-17 08:00:09 -06:00
|
|
|
"context"
|
2021-09-30 11:51:20 -05:00
|
|
|
"net/url"
|
2020-12-17 08:00:09 -06:00
|
|
|
|
2021-10-07 09:33:50 -05:00
|
|
|
"github.com/benbjohnson/clock"
|
2022-03-08 08:22:16 -06:00
|
|
|
"golang.org/x/sync/errgroup"
|
|
|
|
|
2020-11-12 07:11:30 -06:00
|
|
|
"github.com/grafana/grafana/pkg/api/routing"
|
2021-11-10 04:52:16 -06:00
|
|
|
"github.com/grafana/grafana/pkg/expr"
|
2021-09-09 11:25:22 -05:00
|
|
|
"github.com/grafana/grafana/pkg/infra/kvstore"
|
2020-11-12 07:11:30 -06:00
|
|
|
"github.com/grafana/grafana/pkg/infra/log"
|
2022-03-08 08:22:16 -06:00
|
|
|
"github.com/grafana/grafana/pkg/services/accesscontrol"
|
2022-02-16 07:15:44 -06:00
|
|
|
"github.com/grafana/grafana/pkg/services/dashboards"
|
2021-03-24 09:20:44 -05:00
|
|
|
"github.com/grafana/grafana/pkg/services/datasourceproxy"
|
2020-11-12 07:11:30 -06:00
|
|
|
"github.com/grafana/grafana/pkg/services/datasources"
|
2021-09-09 11:25:22 -05:00
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/api"
|
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/eval"
|
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
|
2021-03-24 09:20:44 -05:00
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/notifier"
|
2022-04-05 16:48:51 -05:00
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/provisioning"
|
2021-03-24 09:20:44 -05:00
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/schedule"
|
2021-09-09 11:25:22 -05:00
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/state"
|
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/store"
|
2022-01-26 09:42:40 -06:00
|
|
|
"github.com/grafana/grafana/pkg/services/notifications"
|
2021-09-09 11:25:22 -05:00
|
|
|
"github.com/grafana/grafana/pkg/services/quota"
|
2021-11-04 11:47:21 -05:00
|
|
|
"github.com/grafana/grafana/pkg/services/secrets"
|
2021-03-24 09:20:44 -05:00
|
|
|
"github.com/grafana/grafana/pkg/services/sqlstore"
|
2020-11-12 07:11:30 -06:00
|
|
|
"github.com/grafana/grafana/pkg/setting"
|
|
|
|
)
|
|
|
|
|
2021-08-25 08:11:22 -05:00
|
|
|
func ProvideService(cfg *setting.Cfg, dataSourceCache datasources.CacheService, routeRegister routing.RouteRegister,
|
2021-11-10 04:52:16 -06:00
|
|
|
sqlStore *sqlstore.SQLStore, kvStore kvstore.KVStore, expressionService *expr.Service, dataProxy *datasourceproxy.DataSourceProxyService,
|
2022-03-08 08:22:16 -06:00
|
|
|
quotaService *quota.QuotaService, secretsService secrets.Service, notificationService notifications.Service, m *metrics.NGAlert,
|
2022-05-17 13:52:22 -05:00
|
|
|
folderService dashboards.FolderService, ac accesscontrol.AccessControl, dashboardService dashboards.DashboardService) (*AlertNG, error) {
|
2021-08-25 08:11:22 -05:00
|
|
|
ng := &AlertNG{
|
2022-01-26 09:42:40 -06:00
|
|
|
Cfg: cfg,
|
|
|
|
DataSourceCache: dataSourceCache,
|
|
|
|
RouteRegister: routeRegister,
|
|
|
|
SQLStore: sqlStore,
|
|
|
|
KVStore: kvStore,
|
|
|
|
ExpressionService: expressionService,
|
|
|
|
DataProxy: dataProxy,
|
|
|
|
QuotaService: quotaService,
|
|
|
|
SecretsService: secretsService,
|
|
|
|
Metrics: m,
|
|
|
|
Log: log.New("ngalert"),
|
2022-02-16 07:15:44 -06:00
|
|
|
NotificationService: notificationService,
|
|
|
|
folderService: folderService,
|
2022-03-08 08:22:16 -06:00
|
|
|
accesscontrol: ac,
|
2022-05-18 04:04:17 -05:00
|
|
|
dashboardService: dashboardService,
|
2021-08-25 08:11:22 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
if ng.IsDisabled() {
|
|
|
|
return ng, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := ng.init(); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return ng, nil
|
|
|
|
}
|
|
|
|
|
2020-11-12 07:11:30 -06:00
|
|
|
// AlertNG is the service for evaluating the condition of an alert definition.
|
|
|
|
type AlertNG struct {
|
2022-01-26 09:42:40 -06:00
|
|
|
Cfg *setting.Cfg
|
|
|
|
DataSourceCache datasources.CacheService
|
|
|
|
RouteRegister routing.RouteRegister
|
|
|
|
SQLStore *sqlstore.SQLStore
|
|
|
|
KVStore kvstore.KVStore
|
|
|
|
ExpressionService *expr.Service
|
|
|
|
DataProxy *datasourceproxy.DataSourceProxyService
|
|
|
|
QuotaService *quota.QuotaService
|
|
|
|
SecretsService secrets.Service
|
|
|
|
Metrics *metrics.NGAlert
|
|
|
|
NotificationService notifications.Service
|
|
|
|
Log log.Logger
|
|
|
|
schedule schedule.ScheduleService
|
|
|
|
stateManager *state.Manager
|
2022-02-16 07:15:44 -06:00
|
|
|
folderService dashboards.FolderService
|
2022-05-17 13:52:22 -05:00
|
|
|
dashboardService dashboards.DashboardService
|
2021-08-06 07:06:56 -05:00
|
|
|
|
|
|
|
// Alerting notification services
|
2021-08-24 05:28:09 -05:00
|
|
|
MultiOrgAlertmanager *notifier.MultiOrgAlertmanager
|
2022-03-08 08:22:16 -06:00
|
|
|
accesscontrol accesscontrol.AccessControl
|
2020-11-12 07:11:30 -06:00
|
|
|
}
|
|
|
|
|
2021-08-25 08:11:22 -05:00
|
|
|
func (ng *AlertNG) init() error {
|
2021-09-16 09:33:51 -05:00
|
|
|
var err error
|
|
|
|
|
2021-05-14 15:13:44 -05:00
|
|
|
store := &store.DBstore{
|
2022-02-11 15:13:49 -06:00
|
|
|
BaseInterval: ng.Cfg.UnifiedAlerting.BaseInterval,
|
2022-02-18 09:05:06 -06:00
|
|
|
DefaultInterval: ng.Cfg.UnifiedAlerting.DefaultRuleEvaluationInterval,
|
2021-09-28 05:00:16 -05:00
|
|
|
SQLStore: ng.SQLStore,
|
|
|
|
Logger: ng.Log,
|
2022-02-16 07:15:44 -06:00
|
|
|
FolderService: ng.folderService,
|
2022-04-01 18:33:26 -05:00
|
|
|
AccessControl: ng.accesscontrol,
|
2021-05-14 15:13:44 -05:00
|
|
|
}
|
2021-05-13 13:01:38 -05:00
|
|
|
|
2021-11-04 11:47:21 -05:00
|
|
|
decryptFn := ng.SecretsService.GetDecryptedValue
|
2021-09-16 09:33:51 -05:00
|
|
|
multiOrgMetrics := ng.Metrics.GetMultiOrgAlertmanagerMetrics()
|
2022-04-22 11:57:56 -05:00
|
|
|
ng.MultiOrgAlertmanager, err = notifier.NewMultiOrgAlertmanager(ng.Cfg, store, store, ng.KVStore, store, decryptFn, multiOrgMetrics, ng.NotificationService, log.New("ngalert.multiorg.alertmanager"), ng.SecretsService)
|
2021-09-16 09:33:51 -05:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2021-08-24 05:28:09 -05:00
|
|
|
|
|
|
|
// Let's make sure we're able to complete an initial sync of Alertmanagers before we start the alerting components.
|
|
|
|
if err := ng.MultiOrgAlertmanager.LoadAndSyncAlertmanagersForOrgs(context.Background()); err != nil {
|
2021-05-13 13:01:38 -05:00
|
|
|
return err
|
|
|
|
}
|
2021-03-03 09:52:19 -06:00
|
|
|
|
2021-03-08 14:19:21 -06:00
|
|
|
schedCfg := schedule.SchedulerCfg{
|
2021-08-13 07:14:36 -05:00
|
|
|
C: clock.New(),
|
2022-02-11 15:13:49 -06:00
|
|
|
BaseInterval: ng.Cfg.UnifiedAlerting.BaseInterval,
|
2021-09-28 05:00:16 -05:00
|
|
|
Logger: ng.Log,
|
|
|
|
MaxAttempts: ng.Cfg.UnifiedAlerting.MaxAttempts,
|
2022-02-04 07:56:37 -06:00
|
|
|
Evaluator: eval.NewEvaluator(ng.Cfg, ng.Log, ng.DataSourceCache, ng.SecretsService),
|
2021-08-13 07:14:36 -05:00
|
|
|
InstanceStore: store,
|
|
|
|
RuleStore: store,
|
|
|
|
AdminConfigStore: store,
|
2021-08-24 05:28:09 -05:00
|
|
|
OrgStore: store,
|
|
|
|
MultiOrgNotifier: ng.MultiOrgAlertmanager,
|
2021-09-14 06:55:01 -05:00
|
|
|
Metrics: ng.Metrics.GetSchedulerMetrics(),
|
2021-09-20 02:12:21 -05:00
|
|
|
AdminConfigPollInterval: ng.Cfg.UnifiedAlerting.AdminConfigPollInterval,
|
2021-09-29 09:16:40 -05:00
|
|
|
DisabledOrgs: ng.Cfg.UnifiedAlerting.DisabledOrgs,
|
2022-02-11 15:13:49 -06:00
|
|
|
MinRuleInterval: ng.Cfg.UnifiedAlerting.MinInterval,
|
2021-01-22 11:27:33 -06:00
|
|
|
}
|
2021-09-30 11:51:20 -05:00
|
|
|
|
|
|
|
appUrl, err := url.Parse(ng.Cfg.AppURL)
|
|
|
|
if err != nil {
|
|
|
|
ng.Log.Error("Failed to parse application URL. Continue without it.", "error", err)
|
|
|
|
appUrl = nil
|
|
|
|
}
|
2022-05-17 13:52:22 -05:00
|
|
|
stateManager := state.NewManager(ng.Log, ng.Metrics.GetStateMetrics(), appUrl, store, store, ng.SQLStore, ng.dashboardService)
|
2021-11-10 04:52:16 -06:00
|
|
|
scheduler := schedule.NewScheduler(schedCfg, ng.ExpressionService, appUrl, stateManager)
|
2021-08-13 07:14:36 -05:00
|
|
|
|
2021-08-25 08:11:22 -05:00
|
|
|
ng.stateManager = stateManager
|
2021-09-30 11:51:20 -05:00
|
|
|
ng.schedule = scheduler
|
2021-03-03 09:52:19 -06:00
|
|
|
|
2022-04-05 16:48:51 -05:00
|
|
|
// Provisioning
|
|
|
|
policyService := provisioning.NewNotificationPolicyService(store, store, store, ng.Log)
|
2022-04-13 15:15:55 -05:00
|
|
|
contactPointService := provisioning.NewContactPointService(store, ng.SecretsService, store, store, ng.Log)
|
2022-04-28 13:51:57 -05:00
|
|
|
templateService := provisioning.NewTemplateService(store, store, store, ng.Log)
|
2022-05-17 13:42:48 -05:00
|
|
|
muteTimingService := provisioning.NewMuteTimingService(store, store, store, ng.Log)
|
2022-04-05 16:48:51 -05:00
|
|
|
|
2021-03-08 14:19:21 -06:00
|
|
|
api := api.API{
|
2021-08-24 05:28:09 -05:00
|
|
|
Cfg: ng.Cfg,
|
2021-08-25 08:11:22 -05:00
|
|
|
DatasourceCache: ng.DataSourceCache,
|
2021-08-24 05:28:09 -05:00
|
|
|
RouteRegister: ng.RouteRegister,
|
2021-11-10 04:52:16 -06:00
|
|
|
ExpressionService: ng.ExpressionService,
|
2021-08-24 05:28:09 -05:00
|
|
|
Schedule: ng.schedule,
|
|
|
|
DataProxy: ng.DataProxy,
|
|
|
|
QuotaService: ng.QuotaService,
|
2021-11-04 11:47:21 -05:00
|
|
|
SecretsService: ng.SecretsService,
|
2022-03-15 11:48:42 -05:00
|
|
|
TransactionManager: store,
|
2021-08-24 05:28:09 -05:00
|
|
|
InstanceStore: store,
|
|
|
|
RuleStore: store,
|
|
|
|
AlertingStore: store,
|
|
|
|
AdminConfigStore: store,
|
2022-04-28 14:27:34 -05:00
|
|
|
ProvenanceStore: store,
|
2021-08-24 05:28:09 -05:00
|
|
|
MultiOrgAlertmanager: ng.MultiOrgAlertmanager,
|
|
|
|
StateManager: ng.stateManager,
|
2022-03-08 08:22:16 -06:00
|
|
|
AccessControl: ng.accesscontrol,
|
2022-04-05 16:48:51 -05:00
|
|
|
Policies: policyService,
|
2022-04-13 15:15:55 -05:00
|
|
|
ContactPointService: contactPointService,
|
2022-04-28 13:51:57 -05:00
|
|
|
Templates: templateService,
|
2022-05-17 13:42:48 -05:00
|
|
|
MuteTimings: muteTimingService,
|
2021-03-24 09:20:44 -05:00
|
|
|
}
|
2021-09-14 06:55:01 -05:00
|
|
|
api.RegisterAPIEndpoints(ng.Metrics.GetAPIMetrics())
|
2021-03-03 09:52:19 -06:00
|
|
|
|
2022-03-15 13:30:32 -05:00
|
|
|
return DeclareFixedRoles(ng.accesscontrol)
|
2020-11-12 07:11:30 -06:00
|
|
|
}
|
|
|
|
|
2021-07-27 05:52:59 -05:00
|
|
|
// Run starts the scheduler and Alertmanager.
|
2020-12-17 08:00:09 -06:00
|
|
|
func (ng *AlertNG) Run(ctx context.Context) error {
|
2021-03-08 14:19:21 -06:00
|
|
|
ng.Log.Debug("ngalert starting")
|
2022-02-08 02:52:03 -06:00
|
|
|
ng.stateManager.Warm(ctx)
|
2021-05-13 13:01:38 -05:00
|
|
|
|
|
|
|
children, subCtx := errgroup.WithContext(ctx)
|
2021-09-28 05:00:16 -05:00
|
|
|
|
|
|
|
if ng.Cfg.UnifiedAlerting.ExecuteAlerts {
|
|
|
|
children.Go(func() error {
|
|
|
|
return ng.schedule.Run(subCtx)
|
|
|
|
})
|
|
|
|
}
|
2021-05-13 13:01:38 -05:00
|
|
|
children.Go(func() error {
|
2021-08-24 05:28:09 -05:00
|
|
|
return ng.MultiOrgAlertmanager.Run(subCtx)
|
2021-05-13 13:01:38 -05:00
|
|
|
})
|
|
|
|
return children.Wait()
|
2020-12-17 08:00:09 -06:00
|
|
|
}
|
|
|
|
|
2020-11-12 07:11:30 -06:00
|
|
|
// IsDisabled returns true if the alerting service is disable for this instance.
|
|
|
|
func (ng *AlertNG) IsDisabled() bool {
|
|
|
|
if ng.Cfg == nil {
|
2020-12-17 08:00:09 -06:00
|
|
|
return true
|
2020-11-12 07:11:30 -06:00
|
|
|
}
|
2021-11-24 13:56:07 -06:00
|
|
|
return !ng.Cfg.UnifiedAlerting.IsEnabled()
|
2020-11-12 07:11:30 -06:00
|
|
|
}
|