Alerting: Improve alert rule testing (#16286)

* tsdb: add support for setting debug flag of tsdb query

* alerting: adds debug flag in eval context

Debug flag is set when testing an alert rule and this debug
flag is used to return more debug information in test aler rule
response. This debug flag is also provided to tsdb queries so
datasources can optionally add support for returning additional
debug data

* alerting: improve test alert rule ui

Adds buttons for expand/collapse json and copy json to clipboard,
very similar to how the query inspector works.

* elasticsearch: implement support for tsdb query debug flag

* elasticsearch: embedding client response in struct

* alerting: return proper query model when testing rule

pkg/api/dtos/models.go

@@ -44,6 +44,7 @@ type MetricRequest struct {
 	From    string             `json:"from"`
 	To      string             `json:"to"`
 	Queries []*simplejson.Json `json:"queries"`
+	Debug   bool               `json:"debug"`
 }
 
 type UserStars struct {

pkg/api/metrics.go

@@ -34,7 +34,7 @@ func (hs *HTTPServer) QueryMetrics(c *m.ReqContext, reqDto dtos.MetricRequest) R
 		return Error(500, "Unable to load datasource meta data", err)
 	}
 
-	request := &tsdb.TsdbQuery{TimeRange: timeRange}
+	request := &tsdb.TsdbQuery{TimeRange: timeRange, Debug: reqDto.Debug}
 
 	for _, query := range reqDto.Queries {
 		request.Queries = append(request.Queries, &tsdb.Query{

pkg/services/alerting/conditions/query.go

@@ -114,9 +114,46 @@ func (c *QueryCondition) executeQuery(context *alerting.EvalContext, timeRange *
 		return nil, fmt.Errorf("Could not find datasource %v", err)
 	}
 
-	req := c.getRequestForAlertRule(getDsInfo.Result, timeRange)
+	req := c.getRequestForAlertRule(getDsInfo.Result, timeRange, context.IsDebug)
 	result := make(tsdb.TimeSeriesSlice, 0)
 
+	if context.IsDebug {
+		data := simplejson.New()
+		if req.TimeRange != nil {
+			data.Set("from", req.TimeRange.GetFromAsMsEpoch())
+			data.Set("to", req.TimeRange.GetToAsMsEpoch())
+		}
+
+		type queryDto struct {
+			RefId         string           `json:"refId"`
+			Model         *simplejson.Json `json:"model"`
+			Datasource    *simplejson.Json `json:"datasource"`
+			MaxDataPoints int64            `json:"maxDataPoints"`
+			IntervalMs    int64            `json:"intervalMs"`
+		}
+
+		queries := []*queryDto{}
+		for _, q := range req.Queries {
+			queries = append(queries, &queryDto{
+				RefId: q.RefId,
+				Model: q.Model,
+				Datasource: simplejson.NewFromAny(map[string]interface{}{
+					"id":   q.DataSource.Id,
+					"name": q.DataSource.Name,
+				}),
+				MaxDataPoints: q.MaxDataPoints,
+				IntervalMs:    q.IntervalMs,
+			})
+		}
+
+		data.Set("queries", queries)
+
+		context.Logs = append(context.Logs, &alerting.ResultLogEntry{
+			Message: fmt.Sprintf("Condition[%d]: Query", c.Index),
+			Data:    data,
+		})
+	}
+
 	resp, err := c.HandleRequest(context.Ctx, getDsInfo.Result, req)
 	if err != nil {
 		if err == gocontext.DeadlineExceeded {
@@ -133,10 +170,20 @@ func (c *QueryCondition) executeQuery(context *alerting.EvalContext, timeRange *
 
 		result = append(result, v.Series...)
 
+		queryResultData := map[string]interface{}{}
+
 		if context.IsTestRun {
+			queryResultData["series"] = v.Series
+		}
+
+		if context.IsDebug && v.Meta != nil {
+			queryResultData["meta"] = v.Meta
+		}
+
+		if context.IsTestRun || context.IsDebug {
 			context.Logs = append(context.Logs, &alerting.ResultLogEntry{
 				Message: fmt.Sprintf("Condition[%d]: Query Result", c.Index),
-				Data:    v.Series,
+				Data:    simplejson.NewFromAny(queryResultData),
 			})
 		}
 	}
@@ -144,7 +191,7 @@ func (c *QueryCondition) executeQuery(context *alerting.EvalContext, timeRange *
 	return result, nil
 }
 
-func (c *QueryCondition) getRequestForAlertRule(datasource *models.DataSource, timeRange *tsdb.TimeRange) *tsdb.TsdbQuery {
+func (c *QueryCondition) getRequestForAlertRule(datasource *models.DataSource, timeRange *tsdb.TimeRange, debug bool) *tsdb.TsdbQuery {
 	req := &tsdb.TsdbQuery{
 		TimeRange: timeRange,
 		Queries: []*tsdb.Query{
@@ -154,6 +201,7 @@ func (c *QueryCondition) getRequestForAlertRule(datasource *models.DataSource, t
 				DataSource: datasource,
 			},
 		},
+		Debug: debug,
 	}
 
 	return req

pkg/services/alerting/eval_context.go

@@ -15,6 +15,7 @@ import (
 type EvalContext struct {
 	Firing         bool
 	IsTestRun      bool
+	IsDebug        bool
 	EvalMatches    []*EvalMatch
 	Logs           []*ResultLogEntry
 	Error          error

pkg/services/alerting/test_rule.go

@@ -54,6 +54,7 @@ func testAlertRule(rule *Rule) *EvalContext {
 
 	context := NewEvalContext(context.Background(), rule)
 	context.IsTestRun = true
+	context.IsDebug = true
 
 	handler.Eval(context)
 	context.Rule.State = context.GetNewState()

pkg/tsdb/elasticsearch/client/client.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -37,6 +38,7 @@ type Client interface {
 	GetMinInterval(queryInterval string) (time.Duration, error)
 	ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error)
 	MultiSearch() *MultiSearchRequestBuilder
+	EnableDebug()
 }
 
 // NewClient creates a new elasticsearch client
@@ -80,12 +82,13 @@ var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange *tsdb
 }
 
 type baseClientImpl struct {
-	ctx       context.Context
+	ctx          context.Context
-	ds        *models.DataSource
+	ds           *models.DataSource
-	version   int
+	version      int
-	timeField string
+	timeField    string
-	indices   []string
+	indices      []string
-	timeRange *tsdb.TimeRange
+	timeRange    *tsdb.TimeRange
+	debugEnabled bool
 }
 
 func (c *baseClientImpl) GetVersion() int {
@@ -112,7 +115,7 @@ type multiRequest struct {
 	interval tsdb.Interval
 }
 
-func (c *baseClientImpl) executeBatchRequest(uriPath, uriQuery string, requests []*multiRequest) (*http.Response, error) {
+func (c *baseClientImpl) executeBatchRequest(uriPath, uriQuery string, requests []*multiRequest) (*response, error) {
 	bytes, err := c.encodeBatchRequests(requests)
 	if err != nil {
 		return nil, err
@@ -150,7 +153,7 @@ func (c *baseClientImpl) encodeBatchRequests(requests []*multiRequest) ([]byte,
 	return payload.Bytes(), nil
 }
 
-func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body []byte) (*http.Response, error) {
+func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body []byte) (*response, error) {
 	u, _ := url.Parse(c.ds.Url)
 	u.Path = path.Join(u.Path, uriPath)
 	u.RawQuery = uriQuery
@@ -168,6 +171,15 @@ func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body [
 
 	clientLog.Debug("Executing request", "url", req.URL.String(), "method", method)
 
+	var reqInfo *SearchRequestInfo
+	if c.debugEnabled {
+		reqInfo = &SearchRequestInfo{
+			Method: req.Method,
+			Url:    req.URL.String(),
+			Data:   string(body),
+		}
+	}
+
 	req.Header.Set("User-Agent", "Grafana")
 	req.Header.Set("Content-Type", "application/json")
 
@@ -191,7 +203,11 @@ func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body [
 		elapsed := time.Since(start)
 		clientLog.Debug("Executed request", "took", elapsed)
 	}()
-	return ctxhttp.Do(c.ctx, httpClient, req)
+	res, err := ctxhttp.Do(c.ctx, httpClient, req)
+	return &response{
+		httpResponse: res,
+		reqInfo:      reqInfo,
+	}, err
 }
 
 func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error) {
@@ -199,18 +215,31 @@ func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearch
 
 	multiRequests := c.createMultiSearchRequests(r.Requests)
 	queryParams := c.getMultiSearchQueryParameters()
-	res, err := c.executeBatchRequest("_msearch", queryParams, multiRequests)
+	clientRes, err := c.executeBatchRequest("_msearch", queryParams, multiRequests)
 	if err != nil {
 		return nil, err
 	}
+	res := clientRes.httpResponse
+	defer res.Body.Close()
 
 	clientLog.Debug("Received multisearch response", "code", res.StatusCode, "status", res.Status, "content-length", res.ContentLength)
 
 	start := time.Now()
 	clientLog.Debug("Decoding multisearch json response")
 
+	var bodyBytes []byte
+	if c.debugEnabled {
+		tmpBytes, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			clientLog.Error("failed to read http response bytes", "error", err)
+		} else {
+			bodyBytes = make([]byte, len(tmpBytes))
+			copy(bodyBytes, tmpBytes)
+			res.Body = ioutil.NopCloser(bytes.NewBuffer(tmpBytes))
+		}
+	}
+
 	var msr MultiSearchResponse
-	defer res.Body.Close()
 	dec := json.NewDecoder(res.Body)
 	err = dec.Decode(&msr)
 	if err != nil {
@@ -222,6 +251,24 @@ func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearch
 
 	msr.Status = res.StatusCode
 
+	if c.debugEnabled {
+		bodyJSON, err := simplejson.NewFromReader(bytes.NewBuffer(bodyBytes))
+		var data *simplejson.Json
+		if err != nil {
+			clientLog.Error("failed to decode http response into json", "error", err)
+		} else {
+			data = bodyJSON
+		}
+
+		msr.DebugInfo = &SearchDebugInfo{
+			Request: clientRes.reqInfo,
+			Response: &SearchResponseInfo{
+				Status: res.StatusCode,
+				Data:   data,
+			},
+		}
+	}
+
 	return &msr, nil
 }
 
@@ -266,3 +313,7 @@ func (c *baseClientImpl) getMultiSearchQueryParameters() string {
 func (c *baseClientImpl) MultiSearch() *MultiSearchRequestBuilder {
 	return NewMultiSearchRequestBuilder(c.GetVersion())
 }
+
+func (c *baseClientImpl) EnableDebug() {
+	c.debugEnabled = true
+}

pkg/tsdb/elasticsearch/client/models.go

@@ -2,10 +2,34 @@ package es
 
 import (
 	"encoding/json"
+	"net/http"
+
+	"github.com/grafana/grafana/pkg/components/simplejson"
 
 	"github.com/grafana/grafana/pkg/tsdb"
 )
 
+type response struct {
+	httpResponse *http.Response
+	reqInfo      *SearchRequestInfo
+}
+
+type SearchRequestInfo struct {
+	Method string `json:"method"`
+	Url    string `json:"url"`
+	Data   string `json:"data"`
+}
+
+type SearchResponseInfo struct {
+	Status int              `json:"status"`
+	Data   *simplejson.Json `json:"data"`
+}
+
+type SearchDebugInfo struct {
+	Request  *SearchRequestInfo  `json:"request"`
+	Response *SearchResponseInfo `json:"response"`
+}
+
 // SearchRequest represents a search request
 type SearchRequest struct {
 	Index       string
@@ -60,6 +84,7 @@ type MultiSearchRequest struct {
 type MultiSearchResponse struct {
 	Status    int               `json:"status,omitempty"`
 	Responses []*SearchResponse `json:"responses"`
+	DebugInfo *SearchDebugInfo  `json:"-"`
 }
 
 // Query represents a query

pkg/tsdb/elasticsearch/elasticsearch.go

@@ -40,6 +40,10 @@ func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSo
 		return nil, err
 	}
 
+	if tsdbQuery.Debug {
+		client.EnableDebug()
+	}
+
 	query := newTimeSeriesQuery(client, tsdbQuery, intervalCalculator)
 	return query.execute()
 }

pkg/tsdb/elasticsearch/response_parser.go

@@ -29,12 +29,14 @@ const (
 type responseParser struct {
 	Responses []*es.SearchResponse
 	Targets   []*Query
+	DebugInfo *es.SearchDebugInfo
 }
 
-var newResponseParser = func(responses []*es.SearchResponse, targets []*Query) *responseParser {
+var newResponseParser = func(responses []*es.SearchResponse, targets []*Query, debugInfo *es.SearchDebugInfo) *responseParser {
 	return &responseParser{
 		Responses: responses,
 		Targets:   targets,
+		DebugInfo: debugInfo,
 	}
 }
 
@@ -49,12 +51,19 @@ func (rp *responseParser) getTimeSeries() (*tsdb.Response, error) {
 	for i, res := range rp.Responses {
 		target := rp.Targets[i]
 
+		var debugInfo *simplejson.Json
+		if rp.DebugInfo != nil && i == 0 {
+			debugInfo = simplejson.NewFromAny(rp.DebugInfo)
+		}
+
 		if res.Error != nil {
 			result.Results[target.RefID] = getErrorFromElasticResponse(res)
+			result.Results[target.RefID].Meta = debugInfo
 			continue
 		}
 
 		queryRes := tsdb.NewQueryResult()
+		queryRes.Meta = debugInfo
 		props := make(map[string]string)
 		table := tsdb.Table{
 			Columns: make([]tsdb.TableColumn, 0),

pkg/tsdb/elasticsearch/response_parser_test.go

@@ -954,5 +954,5 @@ func newResponseParserForTest(tsdbQueries map[string]string, responseBody string
 		return nil, err
 	}
 
-	return newResponseParser(response.Responses, queries), nil
+	return newResponseParser(response.Responses, queries, nil), nil
 }

pkg/tsdb/elasticsearch/time_series_query.go

@@ -163,7 +163,7 @@ func (e *timeSeriesQuery) execute() (*tsdb.Response, error) {
 		return nil, err
 	}
 
-	rp := newResponseParser(res.Responses, queries)
+	rp := newResponseParser(res.Responses, queries, res.DebugInfo)
 	return rp.getTimeSeries()
 }
 

pkg/tsdb/elasticsearch/time_series_query_test.go

@@ -635,6 +635,8 @@ func newFakeClient(version int) *fakeClient {
 	}
 }
 
+func (c *fakeClient) EnableDebug() {}
+
 func (c *fakeClient) GetVersion() int {
 	return c.version
 }

pkg/tsdb/models.go

@@ -9,6 +9,7 @@ import (
 type TsdbQuery struct {
 	TimeRange *TimeRange
 	Queries   []*Query
+	Debug     bool
 }
 
 type Query struct {

public/app/features/alerting/TestRuleResult.tsx

@@ -1,5 +1,7 @@
 import React, { PureComponent } from 'react';
 import { JSONFormatter } from 'app/core/components/JSONFormatter/JSONFormatter';
+import appEvents from 'app/core/app_events';
+import { CopyToClipboard } from 'app/core/components/CopyToClipboard/CopyToClipboard';
 import { getBackendSrv } from '@grafana/runtime';
 import { DashboardModel } from '../dashboard/state/DashboardModel';
 import { LoadingPlaceholder } from '@grafana/ui/src';
@@ -11,15 +13,20 @@ export interface Props {
 
 interface State {
   isLoading: boolean;
+  allNodesExpanded: boolean;
   testRuleResponse: {};
 }
 
 export class TestRuleResult extends PureComponent<Props, State> {
   readonly state: State = {
     isLoading: false,
+    allNodesExpanded: null,
     testRuleResponse: {},
   };
 
+  formattedJson: any;
+  clipboard: any;
+
   componentDidMount() {
     this.testRule();
   }
@@ -33,6 +40,50 @@ export class TestRuleResult extends PureComponent<Props, State> {
     this.setState({ isLoading: false, testRuleResponse });
   }
 
+  setFormattedJson = formattedJson => {
+    this.formattedJson = formattedJson;
+  };
+
+  getTextForClipboard = () => {
+    return JSON.stringify(this.formattedJson, null, 2);
+  };
+
+  onClipboardSuccess = () => {
+    appEvents.emit('alert-success', ['Content copied to clipboard']);
+  };
+
+  onToggleExpand = () => {
+    this.setState(prevState => ({
+      ...prevState,
+      allNodesExpanded: !this.state.allNodesExpanded,
+    }));
+  };
+
+  getNrOfOpenNodes = () => {
+    if (this.state.allNodesExpanded === null) {
+      return 3; // 3 is default, ie when state is null
+    } else if (this.state.allNodesExpanded) {
+      return 20;
+    }
+    return 1;
+  };
+
+  renderExpandCollapse = () => {
+    const { allNodesExpanded } = this.state;
+
+    const collapse = (
+      <>
+        <i className="fa fa-minus-square-o" /> Collapse All
+      </>
+    );
+    const expand = (
+      <>
+        <i className="fa fa-plus-square-o" /> Expand All
+      </>
+    );
+    return allNodesExpanded ? collapse : expand;
+  };
+
   render() {
     const { testRuleResponse, isLoading } = this.state;
 
@@ -40,6 +91,25 @@ export class TestRuleResult extends PureComponent<Props, State> {
       return <LoadingPlaceholder text="Evaluating rule" />;
     }
 
-    return <JSONFormatter json={testRuleResponse} />;
+    const openNodes = this.getNrOfOpenNodes();
+
+    return (
+      <>
+        <div className="pull-right">
+          <button className="btn btn-transparent btn-p-x-0 m-r-1" onClick={this.onToggleExpand}>
+            {this.renderExpandCollapse()}
+          </button>
+          <CopyToClipboard
+            className="btn btn-transparent btn-p-x-0"
+            text={this.getTextForClipboard}
+            onSuccess={this.onClipboardSuccess}
+          >
+            <i className="fa fa-clipboard" /> Copy to Clipboard
+          </CopyToClipboard>
+        </div>
+
+        <JSONFormatter json={testRuleResponse} open={openNodes} onDidRender={this.setFormattedJson} />
+      </>
+    );
   }
 }