From 364154d81fc40b14a1aff87e0ce024a69a8d69e6 Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 28 Jan 2019 07:56:31 +0100 Subject: [PATCH] moves timeout tests to an integration test --- pkg/services/alerting/engine.go | 6 + .../alerting/engine_integration_test.go | 148 ++++++++++++++++++ pkg/services/alerting/engine_test.go | 127 --------------- 3 files changed, 154 insertions(+), 127 deletions(-) create mode 100644 pkg/services/alerting/engine_integration_test.go diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index f784a322513..22cbe2456b7 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -205,8 +205,14 @@ func (e *AlertingService) processJob(attemptID int, attemptChan chan int, cancel } } + // create new context with timeout for notifications resultHandleCtx, resultHandleCancelFn := context.WithTimeout(context.Background(), resultHandleTimeout) cancelChan <- resultHandleCancelFn + + // override the context used for evaluation with a new context for notifications. + // This makes it possible for notifiers to execute when datasources + // dont respond within the timeout limit. We should rewrite this so notifications + // dont reuse the evalContext and get its own context. evalContext.Ctx = resultHandleCtx evalContext.Rule.State = evalContext.GetNewState() e.resultHandler.Handle(evalContext) diff --git a/pkg/services/alerting/engine_integration_test.go b/pkg/services/alerting/engine_integration_test.go new file mode 100644 index 00000000000..aa518baae24 --- /dev/null +++ b/pkg/services/alerting/engine_integration_test.go @@ -0,0 +1,148 @@ +// +build integration + +package alerting + +import ( + "context" + "errors" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestEngineTimeouts(t *testing.T) { + Convey("Alerting engine timeout tests", t, func() { + engine := NewEngine() + engine.resultHandler = &FakeResultHandler{} + job := &Job{Running: true, Rule: &Rule{}} + + Convey("Should trigger as many retries as needed", func() { + Convey("pended alert for datasource -> result handler should be worked", func() { + // reduce alert timeout to test quickly + originAlertTimeout := alertTimeout + alertTimeout = 2 * time.Second + transportTimeoutInterval := 2 * time.Second + serverBusySleepDuration := 1 * time.Second + + evalHandler := NewFakeCommonTimeoutHandler(transportTimeoutInterval, serverBusySleepDuration) + resultHandler := NewFakeCommonTimeoutHandler(transportTimeoutInterval, serverBusySleepDuration) + engine.evalHandler = evalHandler + engine.resultHandler = resultHandler + + engine.processJobWithRetry(context.TODO(), job) + + So(evalHandler.EvalSucceed, ShouldEqual, true) + So(resultHandler.ResultHandleSucceed, ShouldEqual, true) + + // initialize for other tests. + alertTimeout = originAlertTimeout + engine.resultHandler = &FakeResultHandler{} + }) + }) + }) +} + +type FakeCommonTimeoutHandler struct { + TransportTimeoutDuration time.Duration + ServerBusySleepDuration time.Duration + EvalSucceed bool + ResultHandleSucceed bool +} + +func NewFakeCommonTimeoutHandler(transportTimeoutDuration time.Duration, serverBusySleepDuration time.Duration) *FakeCommonTimeoutHandler { + return &FakeCommonTimeoutHandler{ + TransportTimeoutDuration: transportTimeoutDuration, + ServerBusySleepDuration: serverBusySleepDuration, + EvalSucceed: false, + ResultHandleSucceed: false, + } +} + +func (handler *FakeCommonTimeoutHandler) Eval(evalContext *EvalContext) { + // 1. prepare mock server + path := "/evaltimeout" + srv := runBusyServer(path, handler.ServerBusySleepDuration) + defer srv.Close() + + // 2. send requests + url := srv.URL + path + res, err := sendRequest(evalContext.Ctx, url, handler.TransportTimeoutDuration) + if res != nil { + defer res.Body.Close() + } + + if err != nil { + evalContext.Error = errors.New("Fake evaluation timeout test failure") + return + } + + if res.StatusCode == 200 { + handler.EvalSucceed = true + } + + evalContext.Error = errors.New("Fake evaluation timeout test failure; wrong response") +} + +func (handler *FakeCommonTimeoutHandler) Handle(evalContext *EvalContext) error { + // 1. prepare mock server + path := "/resulthandle" + srv := runBusyServer(path, handler.ServerBusySleepDuration) + defer srv.Close() + + // 2. send requests + url := srv.URL + path + res, err := sendRequest(evalContext.Ctx, url, handler.TransportTimeoutDuration) + if res != nil { + defer res.Body.Close() + } + + if err != nil { + evalContext.Error = errors.New("Fake result handle timeout test failure") + return evalContext.Error + } + + if res.StatusCode == 200 { + handler.ResultHandleSucceed = true + return nil + } + + evalContext.Error = errors.New("Fake result handle timeout test failure; wrong response") + + return evalContext.Error +} + +func runBusyServer(path string, serverBusySleepDuration time.Duration) *httptest.Server { + mux := http.NewServeMux() + server := httptest.NewServer(mux) + + mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + time.Sleep(serverBusySleepDuration) + }) + + return server +} + +func sendRequest(context context.Context, url string, transportTimeoutInterval time.Duration) (resp *http.Response, err error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + req = req.WithContext(context) + + transport := http.Transport{ + Dial: (&net.Dialer{ + Timeout: transportTimeoutInterval, + KeepAlive: transportTimeoutInterval, + }).Dial, + } + client := http.Client{ + Transport: &transport, + } + + return client.Do(req) +} diff --git a/pkg/services/alerting/engine_test.go b/pkg/services/alerting/engine_test.go index 7c2a0888c04..63108bbb9aa 100644 --- a/pkg/services/alerting/engine_test.go +++ b/pkg/services/alerting/engine_test.go @@ -4,11 +4,7 @@ import ( "context" "errors" "math" - "net" - "net/http" - "net/http/httptest" "testing" - "time" . "github.com/smartystreets/goconvey/convey" ) @@ -38,106 +34,6 @@ func (handler *FakeResultHandler) Handle(evalContext *EvalContext) error { return nil } -type FakeCommonTimeoutHandler struct { - TransportTimeoutDuration time.Duration - ServerBusySleepDuration time.Duration - EvalSucceed bool - ResultHandleSucceed bool -} - -func NewFakeCommonTimeoutHandler(transportTimeoutDuration time.Duration, serverBusySleepDuration time.Duration) *FakeCommonTimeoutHandler { - return &FakeCommonTimeoutHandler{ - TransportTimeoutDuration: transportTimeoutDuration, - ServerBusySleepDuration: serverBusySleepDuration, - EvalSucceed: false, - ResultHandleSucceed: false, - } -} - -func (handler *FakeCommonTimeoutHandler) Eval(evalContext *EvalContext) { - // 1. prepare mock server - path := "/evaltimeout" - srv := runBusyServer(path, handler.ServerBusySleepDuration) - defer srv.Close() - - // 2. send requests - url := srv.URL + path - res, err := sendRequest(evalContext.Ctx, url, handler.TransportTimeoutDuration) - if res != nil { - defer res.Body.Close() - } - - if err != nil { - evalContext.Error = errors.New("Fake evaluation timeout test failure") - return - } - - if res.StatusCode == 200 { - handler.EvalSucceed = true - } - - evalContext.Error = errors.New("Fake evaluation timeout test failure; wrong response") -} - -func (handler *FakeCommonTimeoutHandler) Handle(evalContext *EvalContext) error { - // 1. prepare mock server - path := "/resulthandle" - srv := runBusyServer(path, handler.ServerBusySleepDuration) - defer srv.Close() - - // 2. send requests - url := srv.URL + path - res, err := sendRequest(evalContext.Ctx, url, handler.TransportTimeoutDuration) - if res != nil { - defer res.Body.Close() - } - - if err != nil { - evalContext.Error = errors.New("Fake result handle timeout test failure") - return evalContext.Error - } - - if res.StatusCode == 200 { - handler.ResultHandleSucceed = true - return nil - } - - evalContext.Error = errors.New("Fake result handle timeout test failure; wrong response") - - return evalContext.Error -} - -func runBusyServer(path string, serverBusySleepDuration time.Duration) *httptest.Server { - mux := http.NewServeMux() - server := httptest.NewServer(mux) - - mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { - time.Sleep(serverBusySleepDuration) - }) - - return server -} - -func sendRequest(context context.Context, url string, transportTimeoutInterval time.Duration) (resp *http.Response, err error) { - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, err - } - - req = req.WithContext(context) - - transport := http.Transport{ - Dial: (&net.Dialer{ - Timeout: transportTimeoutInterval, - KeepAlive: transportTimeoutInterval, - }).Dial, - } - client := http.Client{ - Transport: &transport, - } - - return client.Do(req) -} func TestEngineProcessJob(t *testing.T) { Convey("Alerting engine job processing", t, func() { engine := NewEngine() @@ -217,29 +113,6 @@ func TestEngineProcessJob(t *testing.T) { engine.processJobWithRetry(context.TODO(), job) So(evalHandler.CallNb, ShouldEqual, expectedAttempts) }) - - Convey("pended alert for datasource -> result handler should be worked", func() { - // reduce alert timeout to test quickly - originAlertTimeout := alertTimeout - alertTimeout = 5 * time.Second - transportTimeoutInterval := 5 * time.Second - serverBusySleepDuration := 4 * time.Second - - evalHandler := NewFakeCommonTimeoutHandler(transportTimeoutInterval, serverBusySleepDuration) - resultHandler := NewFakeCommonTimeoutHandler(transportTimeoutInterval, serverBusySleepDuration) - engine.evalHandler = evalHandler - engine.resultHandler = resultHandler - - engine.processJobWithRetry(context.TODO(), job) - - So(evalHandler.EvalSucceed, ShouldEqual, true) - So(resultHandler.ResultHandleSucceed, ShouldEqual, true) - - // initialize for other tests. - alertTimeout = originAlertTimeout - engine.resultHandler = &FakeResultHandler{} - }) - }) }) }