IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Fetch alerts from a remote Alertmanager (#75844)

Browse Source 
* Alerting: post alerts to the remote Alertmanager and fetch them

* fix broken tests

* Alerting: Add Mimir Backend image to devenv (blocks)

* add alerting as code owner for mimir_backend block

* Alerting: Use Mimir image to run integration tests for the remote Alertmanager

* skip integration test when running all tests

* skipping integration test when no Alertmanager URL is provided

* fix bad host for mimir_backend

* remove basic auth testing until we have an nginx image in our CI

* add integration tests for alerts

* fix tests

* change SendCtx -> Send, add context.Context to Send, fix CI

* add reover() for functions from the Prometheus Alertmanager HTTP client that could panic

* add TODO to implement PutAlerts in a way that mimicks what Prometheus does

* fix log format
This commit is contained in:
Santiago
2023-10-19 11:27:37 +02:00
committed by GitHub
parent 241996b9ba
commit 61cb26711e
11 changed files with 238 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
pkg/services/ngalert/schedule/alerts_sender_mock.go
View File

@@ -1,11 +1,13 @@
// Code generated by mockery v2.10.0. DO NOT EDIT.
// Code generated by mockery v2.34.2. DO NOT EDIT.
package schedule
import (
mock "github.com/stretchr/testify/mock"
context "context"
definitions "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
mock "github.com/stretchr/testify/mock"
models "github.com/grafana/grafana/pkg/services/ngalert/models"
)
@@ -22,9 +24,9 @@ func (_m *AlertsSenderMock) EXPECT() *AlertsSenderMock_Expecter {
return &AlertsSenderMock_Expecter{mock: &_m.Mock}
}
// Send provides a mock function with given fields: key, alerts
func (_m *AlertsSenderMock) Send(key models.AlertRuleKey, alerts definitions.PostableAlerts) {
_m.Called(key, alerts)
// Send provides a mock function with given fields: ctx, key, alerts
func (_m *AlertsSenderMock) Send(ctx context.Context, key models.AlertRuleKey, alerts definitions.PostableAlerts) {
_m.Called(ctx, key, alerts)
}
// AlertsSenderMock_Send_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Send'
@@ -33,15 +35,16 @@ type AlertsSenderMock_Send_Call struct {
}
// Send is a helper method to define mock.On call
// - ctx context.Context
// - key models.AlertRuleKey
// - alerts definitions.PostableAlerts
func (_e *AlertsSenderMock_Expecter) Send(key any, alerts any) *AlertsSenderMock_Send_Call {
return &AlertsSenderMock_Send_Call{Call: _e.mock.On("Send", key, alerts)}
func (_e *AlertsSenderMock_Expecter) Send(ctx interface{}, key interface{}, alerts interface{}) *AlertsSenderMock_Send_Call {
return &AlertsSenderMock_Send_Call{Call: _e.mock.On("Send", ctx, key, alerts)}
}
func (_c *AlertsSenderMock_Send_Call) Run(run func(key models.AlertRuleKey, alerts definitions.PostableAlerts)) *AlertsSenderMock_Send_Call {
func (_c *AlertsSenderMock_Send_Call) Run(run func(ctx context.Context, key models.AlertRuleKey, alerts definitions.PostableAlerts)) *AlertsSenderMock_Send_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(models.AlertRuleKey), args[1].(definitions.PostableAlerts))
run(args[0].(context.Context), args[1].(models.AlertRuleKey), args[2].(definitions.PostableAlerts))
})
return _c
}
@@ -50,3 +53,22 @@ func (_c *AlertsSenderMock_Send_Call) Return() *AlertsSenderMock_Send_Call {
_c.Call.Return()
return _c
}
func (_c *AlertsSenderMock_Send_Call) RunAndReturn(run func(context.Context, models.AlertRuleKey, definitions.PostableAlerts)) *AlertsSenderMock_Send_Call {
_c.Call.Return(run)
return _c
}
// NewAlertsSenderMock creates a new instance of AlertsSenderMock. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewAlertsSenderMock(t interface {
mock.TestingT
Cleanup(func())
}) *AlertsSenderMock {
mock := &AlertsSenderMock{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}

6
pkg/services/ngalert/schedule/schedule.go
View File

@@ -38,7 +38,7 @@ type ScheduleService interface {
//
//go:generate mockery --name AlertsSender --structname AlertsSenderMock --inpackage --filename alerts_sender_mock.go --with-expecter
type AlertsSender interface {
Send(key ngmodels.AlertRuleKey, alerts definitions.PostableAlerts)
Send(ctx context.Context, key ngmodels.AlertRuleKey, alerts definitions.PostableAlerts)
}
// RulesStore is a store that provides alert rules for scheduling
@@ -360,7 +360,7 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
notify := func(states []state.StateTransition) {
expiredAlerts := state.FromAlertsStateToStoppedAlert(states, sch.appURL, sch.clock)
if len(expiredAlerts.PostableAlerts) > 0 {
sch.alertsSender.Send(key, expiredAlerts)
sch.alertsSender.Send(grafanaCtx, key, expiredAlerts)
}
}
@@ -437,7 +437,7 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
attribute.Int64("alerts_to_send", int64(len(alerts.PostableAlerts))),
))
if len(alerts.PostableAlerts) > 0 {
sch.alertsSender.Send(key, alerts)
sch.alertsSender.Send(ctx, key, alerts)
}
sendDuration.Observe(sch.clock.Now().Sub(start).Seconds())
}

24
pkg/services/ngalert/schedule/schedule_unit_test.go
View File

@@ -59,7 +59,7 @@ func TestProcessTicks(t *testing.T) {
mockedClock := clock.NewMock()
notifier := &AlertsSenderMock{}
notifier.EXPECT().Send(mock.Anything, mock.Anything).Return()
notifier.EXPECT().Send(mock.Anything, mock.Anything, mock.Anything).Return()
appUrl := &url.URL{
Scheme: "http",
@@ -578,7 +578,7 @@ func TestSchedule_ruleRoutine(t *testing.T) {
updateChan := make(chan ruleVersionAndPauseStatus)
sender := AlertsSenderMock{}
sender.EXPECT().Send(rule.GetKey(), mock.Anything).Return()
sender.EXPECT().Send(mock.Anything, rule.GetKey(), mock.Anything).Return()
sch, ruleStore, _, _ := createSchedule(evalAppliedChan, &sender)
ruleStore.PutRule(context.Background(), rule)
@@ -645,8 +645,8 @@ func TestSchedule_ruleRoutine(t *testing.T) {
require.Empty(t, sch.stateManager.GetStatesForRuleUID(rule.OrgID, rule.UID))
sender.AssertNumberOfCalls(t, "Send", 1)
args, ok := sender.Calls[0].Arguments[1].(definitions.PostableAlerts)
require.Truef(t, ok, fmt.Sprintf("expected argument of function was supposed to be 'definitions.PostableAlerts' but got %T", sender.Calls[0].Arguments[1]))
args, ok := sender.Calls[0].Arguments[2].(definitions.PostableAlerts)
require.Truef(t, ok, fmt.Sprintf("expected argument of function was supposed to be 'definitions.PostableAlerts' but got %T", sender.Calls[0].Arguments[2]))
require.Len(t, args.PostableAlerts, expectedToBeSent)
})
})
@@ -659,7 +659,7 @@ func TestSchedule_ruleRoutine(t *testing.T) {
evalAppliedChan := make(chan time.Time)
sender := AlertsSenderMock{}
sender.EXPECT().Send(rule.GetKey(), mock.Anything).Return()
sender.EXPECT().Send(mock.Anything, rule.GetKey(), mock.Anything).Return()
sch, ruleStore, _, reg := createSchedule(evalAppliedChan, &sender)
ruleStore.PutRule(context.Background(), rule)
@@ -748,8 +748,8 @@ func TestSchedule_ruleRoutine(t *testing.T) {
t.Run("it should send special alert DatasourceError", func(t *testing.T) {
sender.AssertNumberOfCalls(t, "Send", 1)
args, ok := sender.Calls[0].Arguments[1].(definitions.PostableAlerts)
require.Truef(t, ok, fmt.Sprintf("expected argument of function was supposed to be 'definitions.PostableAlerts' but got %T", sender.Calls[0].Arguments[1]))
args, ok := sender.Calls[0].Arguments[2].(definitions.PostableAlerts)
require.Truef(t, ok, fmt.Sprintf("expected argument of function was supposed to be 'definitions.PostableAlerts' but got %T", sender.Calls[0].Arguments[2]))
assert.Len(t, args.PostableAlerts, 1)
assert.Equal(t, state.ErrorAlertName, args.PostableAlerts[0].Labels[prometheusModel.AlertNameLabel])
})
@@ -764,7 +764,7 @@ func TestSchedule_ruleRoutine(t *testing.T) {
evalAppliedChan := make(chan time.Time)
sender := AlertsSenderMock{}
sender.EXPECT().Send(rule.GetKey(), mock.Anything).Return()
sender.EXPECT().Send(mock.Anything, rule.GetKey(), mock.Anything).Return()
sch, ruleStore, _, _ := createSchedule(evalAppliedChan, &sender)
ruleStore.PutRule(context.Background(), rule)
@@ -783,8 +783,8 @@ func TestSchedule_ruleRoutine(t *testing.T) {
waitForTimeChannel(t, evalAppliedChan)
sender.AssertNumberOfCalls(t, "Send", 1)
args, ok := sender.Calls[0].Arguments[1].(definitions.PostableAlerts)
require.Truef(t, ok, fmt.Sprintf("expected argument of function was supposed to be 'definitions.PostableAlerts' but got %T", sender.Calls[0].Arguments[1]))
args, ok := sender.Calls[0].Arguments[2].(definitions.PostableAlerts)
require.Truef(t, ok, fmt.Sprintf("expected argument of function was supposed to be 'definitions.PostableAlerts' but got %T", sender.Calls[0].Arguments[2]))
require.Len(t, args.PostableAlerts, 1)
})
@@ -797,7 +797,7 @@ func TestSchedule_ruleRoutine(t *testing.T) {
evalAppliedChan := make(chan time.Time)
sender := AlertsSenderMock{}
sender.EXPECT().Send(rule.GetKey(), mock.Anything).Return()
sender.EXPECT().Send(mock.Anything, rule.GetKey(), mock.Anything).Return()
sch, ruleStore, _, _ := createSchedule(evalAppliedChan, &sender)
ruleStore.PutRule(context.Background(), rule)
@@ -873,7 +873,7 @@ func setupScheduler(t *testing.T, rs *fakeRulesStore, is *state.FakeInstanceStor
if senderMock == nil {
senderMock = &AlertsSenderMock{}
senderMock.EXPECT().Send(mock.Anything, mock.Anything).Return()
senderMock.EXPECT().Send(mock.Anything, mock.Anything, mock.Anything).Return()
}
cfg := setting.UnifiedAlertingSettings{