Alerting: Improve alert rule testing (#16286)

* tsdb: add support for setting debug flag of tsdb query

* alerting: adds debug flag in eval context

Debug flag is set when testing an alert rule and this debug
flag is used to return more debug information in test aler rule
response. This debug flag is also provided to tsdb queries so
datasources can optionally add support for returning additional
debug data

* alerting: improve test alert rule ui

Adds buttons for expand/collapse json and copy json to clipboard,
very similar to how the query inspector works.

* elasticsearch: implement support for tsdb query debug flag

* elasticsearch: embedding client response in struct

* alerting: return proper query model when testing rule

pkg/tsdb/elasticsearch/client/client.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -37,6 +38,7 @@ type Client interface {
 	GetMinInterval(queryInterval string) (time.Duration, error)
 	ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error)
 	MultiSearch() *MultiSearchRequestBuilder
+	EnableDebug()
 }
 
 // NewClient creates a new elasticsearch client
@@ -80,12 +82,13 @@ var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange *tsdb
 }
 
 type baseClientImpl struct {
-	ctx       context.Context
-	ds        *models.DataSource
-	version   int
-	timeField string
-	indices   []string
-	timeRange *tsdb.TimeRange
+	ctx          context.Context
+	ds           *models.DataSource
+	version      int
+	timeField    string
+	indices      []string
+	timeRange    *tsdb.TimeRange
+	debugEnabled bool
 }
 
 func (c *baseClientImpl) GetVersion() int {
@@ -112,7 +115,7 @@ type multiRequest struct {
 	interval tsdb.Interval
 }
 
-func (c *baseClientImpl) executeBatchRequest(uriPath, uriQuery string, requests []*multiRequest) (*http.Response, error) {
+func (c *baseClientImpl) executeBatchRequest(uriPath, uriQuery string, requests []*multiRequest) (*response, error) {
 	bytes, err := c.encodeBatchRequests(requests)
 	if err != nil {
 		return nil, err
@@ -150,7 +153,7 @@ func (c *baseClientImpl) encodeBatchRequests(requests []*multiRequest) ([]byte,
 	return payload.Bytes(), nil
 }
 
-func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body []byte) (*http.Response, error) {
+func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body []byte) (*response, error) {
 	u, _ := url.Parse(c.ds.Url)
 	u.Path = path.Join(u.Path, uriPath)
 	u.RawQuery = uriQuery
@@ -168,6 +171,15 @@ func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body [
 
 	clientLog.Debug("Executing request", "url", req.URL.String(), "method", method)
 
+	var reqInfo *SearchRequestInfo
+	if c.debugEnabled {
+		reqInfo = &SearchRequestInfo{
+			Method: req.Method,
+			Url:    req.URL.String(),
+			Data:   string(body),
+		}
+	}
+
 	req.Header.Set("User-Agent", "Grafana")
 	req.Header.Set("Content-Type", "application/json")
 
@@ -191,7 +203,11 @@ func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body [
 		elapsed := time.Since(start)
 		clientLog.Debug("Executed request", "took", elapsed)
 	}()
-	return ctxhttp.Do(c.ctx, httpClient, req)
+	res, err := ctxhttp.Do(c.ctx, httpClient, req)
+	return &response{
+		httpResponse: res,
+		reqInfo:      reqInfo,
+	}, err
 }
 
 func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error) {
@@ -199,18 +215,31 @@ func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearch
 
 	multiRequests := c.createMultiSearchRequests(r.Requests)
 	queryParams := c.getMultiSearchQueryParameters()
-	res, err := c.executeBatchRequest("_msearch", queryParams, multiRequests)
+	clientRes, err := c.executeBatchRequest("_msearch", queryParams, multiRequests)
 	if err != nil {
 		return nil, err
 	}
+	res := clientRes.httpResponse
+	defer res.Body.Close()
 
 	clientLog.Debug("Received multisearch response", "code", res.StatusCode, "status", res.Status, "content-length", res.ContentLength)
 
 	start := time.Now()
 	clientLog.Debug("Decoding multisearch json response")
 
+	var bodyBytes []byte
+	if c.debugEnabled {
+		tmpBytes, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			clientLog.Error("failed to read http response bytes", "error", err)
+		} else {
+			bodyBytes = make([]byte, len(tmpBytes))
+			copy(bodyBytes, tmpBytes)
+			res.Body = ioutil.NopCloser(bytes.NewBuffer(tmpBytes))
+		}
+	}
+
 	var msr MultiSearchResponse
-	defer res.Body.Close()
 	dec := json.NewDecoder(res.Body)
 	err = dec.Decode(&msr)
 	if err != nil {
@@ -222,6 +251,24 @@ func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearch
 
 	msr.Status = res.StatusCode
 
+	if c.debugEnabled {
+		bodyJSON, err := simplejson.NewFromReader(bytes.NewBuffer(bodyBytes))
+		var data *simplejson.Json
+		if err != nil {
+			clientLog.Error("failed to decode http response into json", "error", err)
+		} else {
+			data = bodyJSON
+		}
+
+		msr.DebugInfo = &SearchDebugInfo{
+			Request: clientRes.reqInfo,
+			Response: &SearchResponseInfo{
+				Status: res.StatusCode,
+				Data:   data,
+			},
+		}
+	}
+
 	return &msr, nil
 }
 
@@ -266,3 +313,7 @@ func (c *baseClientImpl) getMultiSearchQueryParameters() string {
 func (c *baseClientImpl) MultiSearch() *MultiSearchRequestBuilder {
 	return NewMultiSearchRequestBuilder(c.GetVersion())
 }
+
+func (c *baseClientImpl) EnableDebug() {
+	c.debugEnabled = true
+}

pkg/tsdb/elasticsearch/client/models.go

@@ -2,10 +2,34 @@ package es
 
 import (
 	"encoding/json"
+	"net/http"
+
+	"github.com/grafana/grafana/pkg/components/simplejson"
 
 	"github.com/grafana/grafana/pkg/tsdb"
 )
 
+type response struct {
+	httpResponse *http.Response
+	reqInfo      *SearchRequestInfo
+}
+
+type SearchRequestInfo struct {
+	Method string `json:"method"`
+	Url    string `json:"url"`
+	Data   string `json:"data"`
+}
+
+type SearchResponseInfo struct {
+	Status int              `json:"status"`
+	Data   *simplejson.Json `json:"data"`
+}
+
+type SearchDebugInfo struct {
+	Request  *SearchRequestInfo  `json:"request"`
+	Response *SearchResponseInfo `json:"response"`
+}
+
 // SearchRequest represents a search request
 type SearchRequest struct {
 	Index       string
@@ -60,6 +84,7 @@ type MultiSearchRequest struct {
 type MultiSearchResponse struct {
 	Status    int               `json:"status,omitempty"`
 	Responses []*SearchResponse `json:"responses"`
+	DebugInfo *SearchDebugInfo  `json:"-"`
 }
 
 // Query represents a query

pkg/tsdb/elasticsearch/elasticsearch.go

@@ -40,6 +40,10 @@ func (e *ElasticsearchExecutor) Query(ctx context.Context, dsInfo *models.DataSo
 		return nil, err
 	}
 
+	if tsdbQuery.Debug {
+		client.EnableDebug()
+	}
+
 	query := newTimeSeriesQuery(client, tsdbQuery, intervalCalculator)
 	return query.execute()
 }

pkg/tsdb/elasticsearch/response_parser.go

@@ -29,12 +29,14 @@ const (
 type responseParser struct {
 	Responses []*es.SearchResponse
 	Targets   []*Query
+	DebugInfo *es.SearchDebugInfo
 }
 
-var newResponseParser = func(responses []*es.SearchResponse, targets []*Query) *responseParser {
+var newResponseParser = func(responses []*es.SearchResponse, targets []*Query, debugInfo *es.SearchDebugInfo) *responseParser {
 	return &responseParser{
 		Responses: responses,
 		Targets:   targets,
+		DebugInfo: debugInfo,
 	}
 }
 
@@ -49,12 +51,19 @@ func (rp *responseParser) getTimeSeries() (*tsdb.Response, error) {
 	for i, res := range rp.Responses {
 		target := rp.Targets[i]
 
+		var debugInfo *simplejson.Json
+		if rp.DebugInfo != nil && i == 0 {
+			debugInfo = simplejson.NewFromAny(rp.DebugInfo)
+		}
+
 		if res.Error != nil {
 			result.Results[target.RefID] = getErrorFromElasticResponse(res)
+			result.Results[target.RefID].Meta = debugInfo
 			continue
 		}
 
 		queryRes := tsdb.NewQueryResult()
+		queryRes.Meta = debugInfo
 		props := make(map[string]string)
 		table := tsdb.Table{
 			Columns: make([]tsdb.TableColumn, 0),

pkg/tsdb/elasticsearch/response_parser_test.go

@@ -954,5 +954,5 @@ func newResponseParserForTest(tsdbQueries map[string]string, responseBody string
 		return nil, err
 	}
 
-	return newResponseParser(response.Responses, queries), nil
+	return newResponseParser(response.Responses, queries, nil), nil
 }

pkg/tsdb/elasticsearch/time_series_query.go

@@ -163,7 +163,7 @@ func (e *timeSeriesQuery) execute() (*tsdb.Response, error) {
 		return nil, err
 	}
 
-	rp := newResponseParser(res.Responses, queries)
+	rp := newResponseParser(res.Responses, queries, res.DebugInfo)
 	return rp.getTimeSeries()
 }
 

pkg/tsdb/elasticsearch/time_series_query_test.go

@@ -635,6 +635,8 @@ func newFakeClient(version int) *fakeClient {
 	}
 }
 
+func (c *fakeClient) EnableDebug() {}
+
 func (c *fakeClient) GetVersion() int {
 	return c.version
 }

pkg/tsdb/models.go

@@ -9,6 +9,7 @@ import (
 type TsdbQuery struct {
 	TimeRange *TimeRange
 	Queries   []*Query
+	Debug     bool
 }
 
 type Query struct {