From c0d83fc01e9e8cd072d64f5e5a3c789385804bda Mon Sep 17 00:00:00 2001 From: David Parrott Date: Mon, 5 Apr 2021 15:05:39 -0700 Subject: [PATCH] Alerting: Return cached alerts for prometheus/api/v1/alerts (#32654) * Return cached alerts for prometheus/api/v1/alerts * Return not implemented for /prometheus/grafana/api/v1/rules * Set StartsAt for already alerting states * Fix tests --- pkg/services/ngalert/api/api.go | 5 +- pkg/services/ngalert/api/api_prometheus.go | 44 ++++++ .../ngalert/api/api_prometheus_mock.go | 133 ------------------ pkg/services/ngalert/ngalert.go | 1 + pkg/services/ngalert/state/state_tracker.go | 3 + .../ngalert/state/state_tracker_test.go | 4 +- 6 files changed, 54 insertions(+), 136 deletions(-) create mode 100644 pkg/services/ngalert/api/api_prometheus.go delete mode 100644 pkg/services/ngalert/api/api_prometheus_mock.go diff --git a/pkg/services/ngalert/api/api.go b/pkg/services/ngalert/api/api.go index 3423f8bfb88..b3458e83386 100644 --- a/pkg/services/ngalert/api/api.go +++ b/pkg/services/ngalert/api/api.go @@ -4,6 +4,8 @@ import ( "fmt" "time" + "github.com/grafana/grafana/pkg/services/ngalert/state" + "github.com/go-macaron/binding" apimodels "github.com/grafana/alerting-api/pkg/api" @@ -47,6 +49,7 @@ type API struct { AlertingStore store.AlertingStore DataProxy *datasourceproxy.DatasourceProxyService Alertmanager Alertmanager + StateTracker *state.StateTracker } // RegisterAPIEndpoints registers API handlers @@ -63,7 +66,7 @@ func (api *API) RegisterAPIEndpoints() { api.RegisterPrometheusApiEndpoints(NewForkedProm( api.DatasourceCache, NewLotexProm(proxy, logger), - PrometheusApiMock{log: logger}, + PrometheusSrv{log: logger, stateTracker: api.StateTracker}, )) api.RegisterRulerApiEndpoints(NewForkedRuler( api.DatasourceCache, diff --git a/pkg/services/ngalert/api/api_prometheus.go b/pkg/services/ngalert/api/api_prometheus.go new file mode 100644 index 00000000000..3f6a545698f --- /dev/null +++ b/pkg/services/ngalert/api/api_prometheus.go @@ -0,0 +1,44 @@ +package api + +import ( + "net/http" + + apimodels "github.com/grafana/alerting-api/pkg/api" + "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/ngalert/state" +) + +type PrometheusSrv struct { + log log.Logger + stateTracker *state.StateTracker +} + +func (srv PrometheusSrv) RouteGetAlertStatuses(c *models.ReqContext) response.Response { + alertResponse := apimodels.AlertResponse{ + DiscoveryBase: apimodels.DiscoveryBase{ + Status: "success", + }, + Data: apimodels.AlertDiscovery{ + Alerts: []*apimodels.Alert{}, + }, + } + for _, alertState := range srv.stateTracker.GetAll() { + startsAt := alertState.StartsAt + alertResponse.Data.Alerts = append(alertResponse.Data.Alerts, &apimodels.Alert{ + Labels: map[string]string(alertState.Labels), + Annotations: map[string]string{}, //TODO: Once annotations are added to the evaluation result, set them here + State: alertState.State.String(), + ActiveAt: &startsAt, + Value: "", //TODO: once the result of the evaluation is added to the evaluation result, set it here + }) + } + return response.JSON(http.StatusOK, alertResponse) +} + +func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Response { + recipient := c.Params(":Recipient") + srv.log.Info("RouteGetRuleStatuses: ", "Recipient", recipient) + return response.Error(http.StatusNotImplemented, "", nil) +} diff --git a/pkg/services/ngalert/api/api_prometheus_mock.go b/pkg/services/ngalert/api/api_prometheus_mock.go deleted file mode 100644 index 52c3717665e..00000000000 --- a/pkg/services/ngalert/api/api_prometheus_mock.go +++ /dev/null @@ -1,133 +0,0 @@ -package api - -import ( - "net/http" - "time" - - apimodels "github.com/grafana/alerting-api/pkg/api" - "github.com/grafana/grafana/pkg/api/response" - "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/models" -) - -type PrometheusApiMock struct { - log log.Logger -} - -func (mock PrometheusApiMock) RouteGetAlertStatuses(c *models.ReqContext) response.Response { - recipient := c.Params(":Recipient") - mock.log.Info("RouteGetAlertStatuses: ", "Recipient", recipient) - now := time.Now() - result := apimodels.AlertResponse{ - Data: apimodels.AlertDiscovery{ - Alerts: []*apimodels.Alert{ - { - Labels: map[string]string{ - "label 1": "value 1", - "label 2": "value 2", - }, - Annotations: map[string]string{ - "alert annotation 1": "value 1", - "alert annotation 2": "value 2", - }, - State: "firing", - ActiveAt: &now, - Value: "", - }, - { - Labels: map[string]string{ - "label 1-2": "value 1", - "label 2-2": "value 2", - }, - Annotations: map[string]string{ - "alert annotation 1-2": "value 1", - "alert annotation 2-2": "value 2", - }, - State: "inactive", - ActiveAt: &now, - Value: "", - }, - }, - }, - } - return response.JSON(http.StatusOK, result) -} - -func (mock PrometheusApiMock) RouteGetRuleStatuses(c *models.ReqContext) response.Response { - recipient := c.Params(":Recipient") - mock.log.Info("RouteGetRuleStatuses: ", "Recipient", recipient) - now := time.Now() - result := apimodels.RuleResponse{ - Data: apimodels.RuleDiscovery{ - RuleGroups: []*apimodels.RuleGroup{ - { - Name: "group1", - Interval: 60, - LastEvaluation: now.Add(-time.Minute), - EvaluationTime: 1, - Rules: []apimodels.AlertingRule{ - { - State: "firing", - Name: "rule 1-1", - Query: `{ - "title": "rule 1-1", - "condition": "A", - "data": [ - { - "refId": "A", - "queryType": "", - "relativeTimeRange": { - "from": 18000, - "to": 10800 - }, - "model": { - "datasource": "__expr__", - "type": "math", - "expression": "2 + 2 > 1" - } - } - ], - "no_data_state": "NoData", - "exec_err_state": "Alerting" - }`, - Duration: 600, - Annotations: map[string]string{ - "annotation 1": "value 1", - "annotation 2": "value 2", - }, - Alerts: []*apimodels.Alert{ - { - Labels: map[string]string{ - "label 1": "value 1", - "label 2": "value 2", - }, - Annotations: map[string]string{ - "alert annotation 1": "value 1", - "alert annotation 2": "value 2", - }, - State: "firing", - ActiveAt: &now, - Value: "", - }, - { - Labels: map[string]string{ - "label 1-2": "value 1", - "label 2-2": "value 2", - }, - Annotations: map[string]string{ - "alert annotation 1-2": "value 1", - "alert annotation 2-2": "value 2", - }, - State: "inactive", - ActiveAt: &now, - Value: "", - }, - }, - }, - }, - }, - }, - }, - } - return response.JSON(http.StatusOK, result) -} diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index 73890d2d8b0..365de0e468e 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -85,6 +85,7 @@ func (ng *AlertNG) Init() error { RuleStore: store, AlertingStore: store, Alertmanager: ng.Alertmanager, + StateTracker: ng.stateTracker, } api.RegisterAPIEndpoints() diff --git a/pkg/services/ngalert/state/state_tracker.go b/pkg/services/ngalert/state/state_tracker.go index c96185e1e76..f136778a910 100644 --- a/pkg/services/ngalert/state/state_tracker.go +++ b/pkg/services/ngalert/state/state_tracker.go @@ -70,6 +70,9 @@ func (st *StateTracker) getOrCreate(uid string, orgId int64, result eval.Result) State: result.State, Results: []StateEvaluation{}, } + if result.State == eval.Alerting { + newState.StartsAt = result.EvaluatedAt + } st.stateCache.cacheMap[idString] = newState return newState } diff --git a/pkg/services/ngalert/state/state_tracker_test.go b/pkg/services/ngalert/state/state_tracker_test.go index f4450572dd2..d89bce52dc7 100644 --- a/pkg/services/ngalert/state/state_tracker_test.go +++ b/pkg/services/ngalert/state/state_tracker_test.go @@ -133,7 +133,7 @@ func TestProcessEvalResults(t *testing.T) { {EvaluationTime: evaluationTime, EvaluationState: eval.Alerting}, {EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Normal}, }, - StartsAt: time.Time{}, + StartsAt: evaluationTime, EndsAt: evaluationTime.Add(1 * time.Minute), LastEvaluationTime: evaluationTime.Add(1 * time.Minute), }, @@ -172,7 +172,7 @@ func TestProcessEvalResults(t *testing.T) { {EvaluationTime: evaluationTime, EvaluationState: eval.Alerting}, {EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Alerting}, }, - StartsAt: time.Time{}, + StartsAt: evaluationTime, EndsAt: evaluationTime.Add(100 * time.Second), LastEvaluationTime: evaluationTime.Add(1 * time.Minute), },