elasticsearch: add 7.x version support (#16646)

Adds a new version option 7.0+ (70 internally).
Version 7.0+ doesn't include document types in index mappings 
so support for handling this have been added.
Version 7.0+ returns number of hits in a different way so 
support for handling this have been added.
Version 7.0+ doesn't support sending 
max_concurrent_shard_requests in multisearch header so 
support for sending this in query string have been added.
Update elastic6 docker block and dashboards (devenv) to use 
6.7.1 images, filebeat index name is now filebeat-YYYY.MM.DD 
and dashboard include correct tags and links.
Add elastic7 docker block and provisioning (devenv).
Updates documentation regarding new version.

Closes #15622

pkg/tsdb/elasticsearch/client/client.go

@@ -65,7 +65,7 @@ var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange *tsdb
 	clientLog.Debug("Creating new client", "version", version, "timeField", timeField, "indices", strings.Join(indices, ", "))
 
 	switch version {
-	case 2, 5, 56, 60:
+	case 2, 5, 56, 60, 70:
 		return &baseClientImpl{
 			ctx:       ctx,
 			ds:        ds,
@@ -112,12 +112,12 @@ type multiRequest struct {
 	interval tsdb.Interval
 }
 
-func (c *baseClientImpl) executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) {
+func (c *baseClientImpl) executeBatchRequest(uriPath, uriQuery string, requests []*multiRequest) (*http.Response, error) {
 	bytes, err := c.encodeBatchRequests(requests)
 	if err != nil {
 		return nil, err
 	}
-	return c.executeRequest(http.MethodPost, uriPath, bytes)
+	return c.executeRequest(http.MethodPost, uriPath, uriQuery, bytes)
 }
 
 func (c *baseClientImpl) encodeBatchRequests(requests []*multiRequest) ([]byte, error) {
@@ -150,9 +150,10 @@ func (c *baseClientImpl) encodeBatchRequests(requests []*multiRequest) ([]byte,
 	return payload.Bytes(), nil
 }
 
-func (c *baseClientImpl) executeRequest(method, uriPath string, body []byte) (*http.Response, error) {
+func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body []byte) (*http.Response, error) {
 	u, _ := url.Parse(c.ds.Url)
 	u.Path = path.Join(u.Path, uriPath)
+	u.RawQuery = uriQuery
 
 	var req *http.Request
 	var err error
@@ -197,7 +198,8 @@ func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearch
 	clientLog.Debug("Executing multisearch", "search requests", len(r.Requests))
 
 	multiRequests := c.createMultiSearchRequests(r.Requests)
-	res, err := c.executeBatchRequest("_msearch", multiRequests)
+	queryParams := c.getMultiSearchQueryParameters()
+	res, err := c.executeBatchRequest("_msearch", queryParams, multiRequests)
 	if err != nil {
 		return nil, err
 	}
@@ -241,7 +243,7 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque
 			mr.header["search_type"] = "count"
 		}
 
-		if c.version >= 56 {
+		if c.version >= 56 && c.version < 70 {
 			maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(256)
 			mr.header["max_concurrent_shard_requests"] = maxConcurrentShardRequests
 		}
@@ -252,6 +254,15 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque
 	return multiRequests
 }
 
+func (c *baseClientImpl) getMultiSearchQueryParameters() string {
+	if c.version >= 70 {
+		maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(5)
+		return fmt.Sprintf("max_concurrent_shard_requests=%d", maxConcurrentShardRequests)
+	}
+
+	return ""
+}
+
 func (c *baseClientImpl) MultiSearch() *MultiSearchRequestBuilder {
 	return NewMultiSearchRequestBuilder(c.GetVersion())
 }

pkg/tsdb/elasticsearch/client/client_test.go

@@ -103,6 +103,19 @@ func TestClient(t *testing.T) {
 				So(err, ShouldBeNil)
 				So(c.GetVersion(), ShouldEqual, 60)
 			})
+
+			Convey("When version 70 should return v7.0 client", func() {
+				ds := &models.DataSource{
+					JsonData: simplejson.NewFromAny(map[string]interface{}{
+						"esVersion": 70,
+						"timeField": "@timestamp",
+					}),
+				}
+
+				c, err := NewClient(context.Background(), ds, nil)
+				So(err, ShouldBeNil)
+				So(c.GetVersion(), ShouldEqual, 70)
+			})
 		})
 
 		Convey("Given a fake http client", func() {
@@ -290,6 +303,60 @@ func TestClient(t *testing.T) {
 				})
 			})
 
+			Convey("and a v7.0 client", func() {
+				ds := models.DataSource{
+					Database: "[metrics-]YYYY.MM.DD",
+					Url:      ts.URL,
+					JsonData: simplejson.NewFromAny(map[string]interface{}{
+						"esVersion":                  70,
+						"maxConcurrentShardRequests": 6,
+						"timeField":                  "@timestamp",
+						"interval":                   "Daily",
+					}),
+				}
+
+				c, err := NewClient(context.Background(), &ds, timeRange)
+				So(err, ShouldBeNil)
+				So(c, ShouldNotBeNil)
+
+				Convey("When executing multi search", func() {
+					ms, err := createMultisearchForTest(c)
+					So(err, ShouldBeNil)
+					c.ExecuteMultisearch(ms)
+
+					Convey("Should send correct request and payload", func() {
+						So(req, ShouldNotBeNil)
+						So(req.Method, ShouldEqual, http.MethodPost)
+						So(req.URL.Path, ShouldEqual, "/_msearch")
+						So(req.URL.RawQuery, ShouldEqual, "max_concurrent_shard_requests=6")
+
+						So(responseBuffer, ShouldNotBeNil)
+
+						headerBytes, err := responseBuffer.ReadBytes('\n')
+						So(err, ShouldBeNil)
+						bodyBytes := responseBuffer.Bytes()
+
+						jHeader, err := simplejson.NewJson(headerBytes)
+						So(err, ShouldBeNil)
+
+						jBody, err := simplejson.NewJson(bodyBytes)
+						So(err, ShouldBeNil)
+
+						So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15")
+						So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true)
+						So(jHeader.Get("search_type").MustString(), ShouldEqual, "query_then_fetch")
+
+						Convey("and replace $__interval variable", func() {
+							So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname")
+						})
+
+						Convey("and replace $__interval_ms variable", func() {
+							So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s")
+						})
+					})
+				})
+			})
+
 			Reset(func() {
 				newDatasourceHttpClient = currentNewDatasourceHttpClient
 			})

pkg/tsdb/elasticsearch/client/models.go

@@ -42,7 +42,7 @@ func (r *SearchRequest) MarshalJSON() ([]byte, error) {
 // SearchResponseHits represents search response hits
 type SearchResponseHits struct {
 	Hits  []map[string]interface{}
-	Total int64
+	Total map[string]interface{}
 }
 
 // SearchResponse represents a search response