From 5713048f489f424c12458b7cc720c00ba6910a82 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Tue, 25 Jun 2019 08:52:17 +0200 Subject: [PATCH] Alerting: Improve alert rule testing (#16286) * tsdb: add support for setting debug flag of tsdb query * alerting: adds debug flag in eval context Debug flag is set when testing an alert rule and this debug flag is used to return more debug information in test aler rule response. This debug flag is also provided to tsdb queries so datasources can optionally add support for returning additional debug data * alerting: improve test alert rule ui Adds buttons for expand/collapse json and copy json to clipboard, very similar to how the query inspector works. * elasticsearch: implement support for tsdb query debug flag * elasticsearch: embedding client response in struct * alerting: return proper query model when testing rule --- pkg/api/dtos/models.go | 1 + pkg/api/metrics.go | 2 +- pkg/services/alerting/conditions/query.go | 54 +++++++++++++- pkg/services/alerting/eval_context.go | 1 + pkg/services/alerting/test_rule.go | 1 + pkg/tsdb/elasticsearch/client/client.go | 73 ++++++++++++++++--- pkg/tsdb/elasticsearch/client/models.go | 25 +++++++ pkg/tsdb/elasticsearch/elasticsearch.go | 4 + pkg/tsdb/elasticsearch/response_parser.go | 11 ++- .../elasticsearch/response_parser_test.go | 2 +- pkg/tsdb/elasticsearch/time_series_query.go | 2 +- .../elasticsearch/time_series_query_test.go | 2 + pkg/tsdb/models.go | 1 + .../app/features/alerting/TestRuleResult.tsx | 72 +++++++++++++++++- 14 files changed, 232 insertions(+), 19 deletions(-) diff --git a/pkg/api/dtos/models.go b/pkg/api/dtos/models.go index 6a130e62158..44daed0c3e6 100644 --- a/pkg/api/dtos/models.go +++ b/pkg/api/dtos/models.go @@ -44,6 +44,7 @@ type MetricRequest struct { From string `json:"from"` To string `json:"to"` Queries []*simplejson.Json `json:"queries"` + Debug bool `json:"debug"` } type UserStars struct { diff --git a/pkg/api/metrics.go b/pkg/api/metrics.go index 0bdf7496489..a307ef877f4 100644 --- a/pkg/api/metrics.go +++ b/pkg/api/metrics.go @@ -34,7 +34,7 @@ func (hs *HTTPServer) QueryMetrics(c *m.ReqContext, reqDto dtos.MetricRequest) R return Error(500, "Unable to load datasource meta data", err) } - request := &tsdb.TsdbQuery{TimeRange: timeRange} + request := &tsdb.TsdbQuery{TimeRange: timeRange, Debug: reqDto.Debug} for _, query := range reqDto.Queries { request.Queries = append(request.Queries, &tsdb.Query{ diff --git a/pkg/services/alerting/conditions/query.go b/pkg/services/alerting/conditions/query.go index b29f39b4916..8e1916f3c3f 100644 --- a/pkg/services/alerting/conditions/query.go +++ b/pkg/services/alerting/conditions/query.go @@ -114,9 +114,46 @@ func (c *QueryCondition) executeQuery(context *alerting.EvalContext, timeRange * return nil, fmt.Errorf("Could not find datasource %v", err) } - req := c.getRequestForAlertRule(getDsInfo.Result, timeRange) + req := c.getRequestForAlertRule(getDsInfo.Result, timeRange, context.IsDebug) result := make(tsdb.TimeSeriesSlice, 0) + if context.IsDebug { + data := simplejson.New() + if req.TimeRange != nil { + data.Set("from", req.TimeRange.GetFromAsMsEpoch()) + data.Set("to", req.TimeRange.GetToAsMsEpoch()) + } + + type queryDto struct { + RefId string `json:"refId"` + Model *simplejson.Json `json:"model"` + Datasource *simplejson.Json `json:"datasource"` + MaxDataPoints int64 `json:"maxDataPoints"` + IntervalMs int64 `json:"intervalMs"` + } + + queries := []*queryDto{} + for _, q := range req.Queries { + queries = append(queries, &queryDto{ + RefId: q.RefId, + Model: q.Model, + Datasource: simplejson.NewFromAny(map[string]interface{}{ + "id": q.DataSource.Id, + "name": q.DataSource.Name, + }), + MaxDataPoints: q.MaxDataPoints, + IntervalMs: q.IntervalMs, + }) + } + + data.Set("queries", queries) + + context.Logs = append(context.Logs, &alerting.ResultLogEntry{ + Message: fmt.Sprintf("Condition[%d]: Query", c.Index), + Data: data, + }) + } + resp, err := c.HandleRequest(context.Ctx, getDsInfo.Result, req) if err != nil { if err == gocontext.DeadlineExceeded { @@ -133,10 +170,20 @@ func (c *QueryCondition) executeQuery(context *alerting.EvalContext, timeRange * result = append(result, v.Series...) + queryResultData := map[string]interface{}{} + if context.IsTestRun { + queryResultData["series"] = v.Series + } + + if context.IsDebug && v.Meta != nil { + queryResultData["meta"] = v.Meta + } + + if context.IsTestRun || context.IsDebug { context.Logs = append(context.Logs, &alerting.ResultLogEntry{ Message: fmt.Sprintf("Condition[%d]: Query Result", c.Index), - Data: v.Series, + Data: simplejson.NewFromAny(queryResultData), }) } } @@ -144,7 +191,7 @@ func (c *QueryCondition) executeQuery(context *alerting.EvalContext, timeRange * return result, nil } -func (c *QueryCondition) getRequestForAlertRule(datasource *models.DataSource, timeRange *tsdb.TimeRange) *tsdb.TsdbQuery { +func (c *QueryCondition) getRequestForAlertRule(datasource *models.DataSource, timeRange *tsdb.TimeRange, debug bool) *tsdb.TsdbQuery { req := &tsdb.TsdbQuery{ TimeRange: timeRange, Queries: []*tsdb.Query{ @@ -154,6 +201,7 @@ func (c *QueryCondition) getRequestForAlertRule(datasource *models.DataSource, t DataSource: datasource, }, }, + Debug: debug, } return req diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go index 8436e9c9a78..2ec2f1e60ee 100644 --- a/pkg/services/alerting/eval_context.go +++ b/pkg/services/alerting/eval_context.go @@ -15,6 +15,7 @@ import ( type EvalContext struct { Firing bool IsTestRun bool + IsDebug bool EvalMatches []*EvalMatch Logs []*ResultLogEntry Error error diff --git a/pkg/services/alerting/test_rule.go b/pkg/services/alerting/test_rule.go index 1575490ea32..79659d6a8ca 100644 --- a/pkg/services/alerting/test_rule.go +++ b/pkg/services/alerting/test_rule.go @@ -54,6 +54,7 @@ func testAlertRule(rule *Rule) *EvalContext { context := NewEvalContext(context.Background(), rule) context.IsTestRun = true + context.IsDebug = true handler.Eval(context) context.Rule.State = context.GetNewState() diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index 48f9cce0a5a..103ef452a4b 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "io/ioutil" "net/http" "net/url" "path" @@ -37,6 +38,7 @@ type Client interface { GetMinInterval(queryInterval string) (time.Duration, error) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error) MultiSearch() *MultiSearchRequestBuilder + EnableDebug() } // NewClient creates a new elasticsearch client @@ -80,12 +82,13 @@ var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange *tsdb } type baseClientImpl struct { - ctx context.Context - ds *models.DataSource - version int - timeField string - indices []string - timeRange *tsdb.TimeRange + ctx context.Context + ds *models.DataSource + version int + timeField string + indices []string + timeRange *tsdb.TimeRange + debugEnabled bool } func (c *baseClientImpl) GetVersion() int { @@ -112,7 +115,7 @@ type multiRequest struct { interval tsdb.Interval } -func (c *baseClientImpl) executeBatchRequest(uriPath, uriQuery string, requests []*multiRequest) (*http.Response, error) { +func (c *baseClientImpl) executeBatchRequest(uriPath, uriQuery string, requests []*multiRequest) (*response, error) { bytes, err := c.encodeBatchRequests(requests) if err != nil { return nil, err @@ -150,7 +153,7 @@ func (c *baseClientImpl) encodeBatchRequests(requests []*multiRequest) ([]byte, return payload.Bytes(), nil } -func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body []byte) (*http.Response, error) { +func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body []byte) (*response, error) { u, _ := url.Parse(c.ds.Url) u.Path = path.Join(u.Path, uriPath) u.RawQuery = uriQuery @@ -168,6 +171,15 @@ func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body [ clientLog.Debug("Executing request", "url", req.URL.String(), "method", method) + var reqInfo *SearchRequestInfo + if c.debugEnabled { + reqInfo = &SearchRequestInfo{ + Method: req.Method, + Url: req.URL.String(), + Data: string(body), + } + } + req.Header.Set("User-Agent", "Grafana") req.Header.Set("Content-Type", "application/json") @@ -191,7 +203,11 @@ func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body [ elapsed := time.Since(start) clientLog.Debug("Executed request", "took", elapsed) }() - return ctxhttp.Do(c.ctx, httpClient, req) + res, err := ctxhttp.Do(c.ctx, httpClient, req) + return &response{ + httpResponse: res, + reqInfo: reqInfo, + }, err } func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error) { @@ -199,18 +215,31 @@ func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearch multiRequests := c.createMultiSearchRequests(r.Requests) queryParams := c.getMultiSearchQueryParameters() - res, err := c.executeBatchRequest("_msearch", queryParams, multiRequests) + clientRes, err := c.executeBatchRequest("_msearch", queryParams, multiRequests) if err != nil { return nil, err } + res := clientRes.httpResponse + defer res.Body.Close() clientLog.Debug("Received multisearch response", "code", res.StatusCode, "status", res.Status, "content-length", res.ContentLength) start := time.Now() clientLog.Debug("Decoding multisearch json response") + var bodyBytes []byte + if c.debugEnabled { + tmpBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + clientLog.Error("failed to read http response bytes", "error", err) + } else { + bodyBytes = make([]byte, len(tmpBytes)) + copy(bodyBytes, tmpBytes) + res.Body = ioutil.NopCloser(bytes.NewBuffer(tmpBytes)) + } + } + var msr MultiSearchResponse - defer res.Body.Close() dec := json.NewDecoder(res.Body) err = dec.Decode(&msr) if err != nil { @@ -222,6 +251,24 @@ func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearch msr.Status = res.StatusCode + if c.debugEnabled { + bodyJSON, err := simplejson.NewFromReader(bytes.NewBuffer(bodyBytes)) + var data *simplejson.Json + if err != nil { + clientLog.Error("failed to decode http response into json", "error", err) + } else { + data = bodyJSON + } + + msr.DebugInfo = &SearchDebugInfo{ + Request: clientRes.reqInfo, + Response: &SearchResponseInfo{ + Status: res.StatusCode, + Data: data, + }, + } + } + return &msr, nil } @@ -266,3 +313,7 @@ func (c *baseClientImpl) getMultiSearchQueryParameters() string { func (c *baseClientImpl) MultiSearch() *MultiSearchRequestBuilder { return NewMultiSearchRequestBuilder(c.GetVersion()) } + +func (c *baseClientImpl) EnableDebug() { + c.debugEnabled = true +} diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go index a9b0dddd880..809b30a23b9 100644 --- a/pkg/tsdb/elasticsearch/client/models.go +++ b/pkg/tsdb/elasticsearch/client/models.go @@ -2,10 +2,34 @@ package es import ( "encoding/json" + "net/http" + + "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/tsdb" ) +type response struct { + httpResponse *http.Response + reqInfo *SearchRequestInfo +} + +type SearchRequestInfo struct { + Method string `json:"method"` + Url string `json:"url"` + Data string `json:"data"` +} + +type SearchResponseInfo struct { + Status int `json:"status"` + Data *simplejson.Json `json:"data"` +} + +type SearchDebugInfo struct { + Request *SearchRequestInfo `json:"request"` + Response *SearchResponseInfo `json:"response"` +} + // SearchRequest represents a search request type SearchRequest struct { Index string @@ -60,6 +84,7 @@ type MultiSearchRequest struct { type MultiSearchResponse struct { Status int `json:"status,omitempty"` Responses []*SearchResponse `json:"responses"` + DebugInfo *SearchDebugInfo `json:"-"` } // Query represents a query diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go index 01c9194877c..127f93620a2 100644 --- a/pkg/tsdb/elasticsearch/elasticsearch.go +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -40,6 +40,10 @@ func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSo return nil, err } + if tsdbQuery.Debug { + client.EnableDebug() + } + query := newTimeSeriesQuery(client, tsdbQuery, intervalCalculator) return query.execute() } diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index 6bbaa3df34b..974c6dfbec6 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -29,12 +29,14 @@ const ( type responseParser struct { Responses []*es.SearchResponse Targets []*Query + DebugInfo *es.SearchDebugInfo } -var newResponseParser = func(responses []*es.SearchResponse, targets []*Query) *responseParser { +var newResponseParser = func(responses []*es.SearchResponse, targets []*Query, debugInfo *es.SearchDebugInfo) *responseParser { return &responseParser{ Responses: responses, Targets: targets, + DebugInfo: debugInfo, } } @@ -49,12 +51,19 @@ func (rp *responseParser) getTimeSeries() (*tsdb.Response, error) { for i, res := range rp.Responses { target := rp.Targets[i] + var debugInfo *simplejson.Json + if rp.DebugInfo != nil && i == 0 { + debugInfo = simplejson.NewFromAny(rp.DebugInfo) + } + if res.Error != nil { result.Results[target.RefID] = getErrorFromElasticResponse(res) + result.Results[target.RefID].Meta = debugInfo continue } queryRes := tsdb.NewQueryResult() + queryRes.Meta = debugInfo props := make(map[string]string) table := tsdb.Table{ Columns: make([]tsdb.TableColumn, 0), diff --git a/pkg/tsdb/elasticsearch/response_parser_test.go b/pkg/tsdb/elasticsearch/response_parser_test.go index e9cd8ad0980..43024d7b135 100644 --- a/pkg/tsdb/elasticsearch/response_parser_test.go +++ b/pkg/tsdb/elasticsearch/response_parser_test.go @@ -954,5 +954,5 @@ func newResponseParserForTest(tsdbQueries map[string]string, responseBody string return nil, err } - return newResponseParser(response.Responses, queries), nil + return newResponseParser(response.Responses, queries, nil), nil } diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go index e1cd466275e..ebe89bd6f8b 100644 --- a/pkg/tsdb/elasticsearch/time_series_query.go +++ b/pkg/tsdb/elasticsearch/time_series_query.go @@ -163,7 +163,7 @@ func (e *timeSeriesQuery) execute() (*tsdb.Response, error) { return nil, err } - rp := newResponseParser(res.Responses, queries) + rp := newResponseParser(res.Responses, queries, res.DebugInfo) return rp.getTimeSeries() } diff --git a/pkg/tsdb/elasticsearch/time_series_query_test.go b/pkg/tsdb/elasticsearch/time_series_query_test.go index 3a558c32782..5de0fb22be3 100644 --- a/pkg/tsdb/elasticsearch/time_series_query_test.go +++ b/pkg/tsdb/elasticsearch/time_series_query_test.go @@ -635,6 +635,8 @@ func newFakeClient(version int) *fakeClient { } } +func (c *fakeClient) EnableDebug() {} + func (c *fakeClient) GetVersion() int { return c.version } diff --git a/pkg/tsdb/models.go b/pkg/tsdb/models.go index dee7289af7f..8a81d6fb237 100644 --- a/pkg/tsdb/models.go +++ b/pkg/tsdb/models.go @@ -9,6 +9,7 @@ import ( type TsdbQuery struct { TimeRange *TimeRange Queries []*Query + Debug bool } type Query struct { diff --git a/public/app/features/alerting/TestRuleResult.tsx b/public/app/features/alerting/TestRuleResult.tsx index 509ea1721cb..2f914cdd365 100644 --- a/public/app/features/alerting/TestRuleResult.tsx +++ b/public/app/features/alerting/TestRuleResult.tsx @@ -1,5 +1,7 @@ import React, { PureComponent } from 'react'; import { JSONFormatter } from 'app/core/components/JSONFormatter/JSONFormatter'; +import appEvents from 'app/core/app_events'; +import { CopyToClipboard } from 'app/core/components/CopyToClipboard/CopyToClipboard'; import { getBackendSrv } from '@grafana/runtime'; import { DashboardModel } from '../dashboard/state/DashboardModel'; import { LoadingPlaceholder } from '@grafana/ui/src'; @@ -11,15 +13,20 @@ export interface Props { interface State { isLoading: boolean; + allNodesExpanded: boolean; testRuleResponse: {}; } export class TestRuleResult extends PureComponent { readonly state: State = { isLoading: false, + allNodesExpanded: null, testRuleResponse: {}, }; + formattedJson: any; + clipboard: any; + componentDidMount() { this.testRule(); } @@ -33,6 +40,50 @@ export class TestRuleResult extends PureComponent { this.setState({ isLoading: false, testRuleResponse }); } + setFormattedJson = formattedJson => { + this.formattedJson = formattedJson; + }; + + getTextForClipboard = () => { + return JSON.stringify(this.formattedJson, null, 2); + }; + + onClipboardSuccess = () => { + appEvents.emit('alert-success', ['Content copied to clipboard']); + }; + + onToggleExpand = () => { + this.setState(prevState => ({ + ...prevState, + allNodesExpanded: !this.state.allNodesExpanded, + })); + }; + + getNrOfOpenNodes = () => { + if (this.state.allNodesExpanded === null) { + return 3; // 3 is default, ie when state is null + } else if (this.state.allNodesExpanded) { + return 20; + } + return 1; + }; + + renderExpandCollapse = () => { + const { allNodesExpanded } = this.state; + + const collapse = ( + <> + Collapse All + + ); + const expand = ( + <> + Expand All + + ); + return allNodesExpanded ? collapse : expand; + }; + render() { const { testRuleResponse, isLoading } = this.state; @@ -40,6 +91,25 @@ export class TestRuleResult extends PureComponent { return ; } - return ; + const openNodes = this.getNrOfOpenNodes(); + + return ( + <> +
+ + + Copy to Clipboard + +
+ + + + ); } }