package alerting import ( "context" "errors" "time" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/metrics" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/annotations" "github.com/grafana/grafana/pkg/services/rendering" ) type resultHandler interface { handle(evalContext *EvalContext) error } type defaultResultHandler struct { notifier *notificationService log log.Logger } func newResultHandler(renderService rendering.Service, decryptFn GetDecryptedValueFn) *defaultResultHandler { return &defaultResultHandler{ log: log.New("alerting.resultHandler"), notifier: newNotificationService(renderService, decryptFn), } } func (handler *defaultResultHandler) handle(evalContext *EvalContext) error { executionError := "" annotationData := simplejson.New() if len(evalContext.EvalMatches) > 0 { annotationData.Set("evalMatches", simplejson.NewFromAny(evalContext.EvalMatches)) } if evalContext.Error != nil { executionError = evalContext.Error.Error() annotationData.Set("error", executionError) } else if evalContext.NoDataFound { annotationData.Set("noData", true) } metrics.MAlertingResultState.WithLabelValues(string(evalContext.Rule.State)).Inc() if evalContext.shouldUpdateAlertState() { handler.log.Info("New state change", "ruleId", evalContext.Rule.ID, "newState", evalContext.Rule.State, "prev state", evalContext.PrevAlertState) cmd := &models.SetAlertStateCommand{ AlertId: evalContext.Rule.ID, OrgId: evalContext.Rule.OrgID, State: evalContext.Rule.State, Error: executionError, EvalData: annotationData, } if err := bus.Dispatch(evalContext.Ctx, cmd); err != nil { if errors.Is(err, models.ErrCannotChangeStateOnPausedAlert) { handler.log.Error("Cannot change state on alert that's paused", "error", err) return err } if errors.Is(err, models.ErrRequiresNewState) { handler.log.Info("Alert already updated") return nil } handler.log.Error("Failed to save state", "error", err) } else { // StateChanges is used for de duping alert notifications // when two servers are raising. This makes sure that the server // with the last state change always sends a notification. evalContext.Rule.StateChanges = cmd.Result.StateChanges // Update the last state change of the alert rule in memory evalContext.Rule.LastStateChange = time.Now() } // save annotation item := annotations.Item{ OrgId: evalContext.Rule.OrgID, DashboardId: evalContext.Rule.DashboardID, PanelId: evalContext.Rule.PanelID, AlertId: evalContext.Rule.ID, Text: "", NewState: string(evalContext.Rule.State), PrevState: string(evalContext.PrevAlertState), Epoch: time.Now().UnixNano() / int64(time.Millisecond), Data: annotationData, } annotationRepo := annotations.GetRepository() if err := annotationRepo.Save(&item); err != nil { handler.log.Error("Failed to save annotation for new alert state", "error", err) } } if err := handler.notifier.SendIfNeeded(evalContext); err != nil { switch { case errors.Is(err, context.Canceled): handler.log.Debug("handler.notifier.SendIfNeeded returned context.Canceled") case errors.Is(err, context.DeadlineExceeded): handler.log.Debug("handler.notifier.SendIfNeeded returned context.DeadlineExceeded") default: handler.log.Error("handler.notifier.SendIfNeeded failed", "err", err) } } return nil }