From 61a3160c34f659be848d67a10bd9e0e5be6e1b58 Mon Sep 17 00:00:00 2001 From: Giordano Ricci Date: Fri, 18 Sep 2020 18:27:25 +0100 Subject: [PATCH] Elasticsearch: add frozen indices search support (#27472) * Elasticsearch: Add frozen indices support for alerting --- pkg/tsdb/elasticsearch/client/client.go | 12 +++- pkg/tsdb/elasticsearch/client/client_test.go | 7 ++- .../configuration/ConfigEditor.tsx | 1 + .../configuration/ElasticDetails.tsx | 23 ++++++- .../elasticsearch/datasource.test.ts | 63 +++++++++++++++++++ .../datasource/elasticsearch/datasource.ts | 12 +++- .../plugins/datasource/elasticsearch/types.ts | 1 + 7 files changed, 112 insertions(+), 7 deletions(-) diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index 1b4457cd087..2f840796531 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -308,12 +308,20 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque } func (c *baseClientImpl) getMultiSearchQueryParameters() string { + var qs []string + if c.version >= 70 { maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(5) - return fmt.Sprintf("max_concurrent_shard_requests=%d", maxConcurrentShardRequests) + qs = append(qs, fmt.Sprintf("max_concurrent_shard_requests=%d", maxConcurrentShardRequests)) + + includeFrozen := c.getSettings().Get("includeFrozen").MustBool(false) + + if includeFrozen { + qs = append(qs, "ignore_throttled=false") + } } - return "" + return strings.Join(qs, "&") } func (c *baseClientImpl) MultiSearch() *MultiSearchRequestBuilder { diff --git a/pkg/tsdb/elasticsearch/client/client_test.go b/pkg/tsdb/elasticsearch/client/client_test.go index 993310d6beb..222272a33f1 100644 --- a/pkg/tsdb/elasticsearch/client/client_test.go +++ b/pkg/tsdb/elasticsearch/client/client_test.go @@ -250,6 +250,7 @@ func TestClient(t *testing.T) { "maxConcurrentShardRequests": 100, "timeField": "@timestamp", "interval": "Daily", + "includeFrozen": true, }), }, func(sc *scenarioContext) { sc.responseBody = `{ @@ -271,6 +272,7 @@ func TestClient(t *testing.T) { So(sc.request, ShouldNotBeNil) So(sc.request.Method, ShouldEqual, http.MethodPost) So(sc.request.URL.Path, ShouldEqual, "/_msearch") + So(sc.request.URL.RawQuery, ShouldNotContainSubstring, "ignore_throttled=") So(sc.requestBody, ShouldNotBeNil) @@ -312,6 +314,7 @@ func TestClient(t *testing.T) { "maxConcurrentShardRequests": 6, "timeField": "@timestamp", "interval": "Daily", + "includeFrozen": true, }), }, func(sc *scenarioContext) { sc.responseBody = `{ @@ -333,7 +336,8 @@ func TestClient(t *testing.T) { So(sc.request, ShouldNotBeNil) So(sc.request.Method, ShouldEqual, http.MethodPost) So(sc.request.URL.Path, ShouldEqual, "/_msearch") - So(sc.request.URL.RawQuery, ShouldEqual, "max_concurrent_shard_requests=6") + So(sc.request.URL.RawQuery, ShouldContainSubstring, "max_concurrent_shard_requests=6") + So(sc.request.URL.RawQuery, ShouldContainSubstring, "ignore_throttled=false") So(sc.requestBody, ShouldNotBeNil) @@ -350,6 +354,7 @@ func TestClient(t *testing.T) { So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15") So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true) So(jHeader.Get("search_type").MustString(), ShouldEqual, "query_then_fetch") + So(jHeader.Get("ignore_throttled").MustBool(), ShouldBeFalse) Convey("and replace $__interval variable", func() { So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname") diff --git a/public/app/plugins/datasource/elasticsearch/configuration/ConfigEditor.tsx b/public/app/plugins/datasource/elasticsearch/configuration/ConfigEditor.tsx index 9d6e87e8f72..328d0696b81 100644 --- a/public/app/plugins/datasource/elasticsearch/configuration/ConfigEditor.tsx +++ b/public/app/plugins/datasource/elasticsearch/configuration/ConfigEditor.tsx @@ -18,6 +18,7 @@ export const ConfigEditor = (props: Props) => { jsonData: { ...options.jsonData, timeField: options.jsonData.timeField || '@timestamp', + includeFrozen: options.jsonData.includeFrozen ?? false, esVersion, maxConcurrentShardRequests: options.jsonData.maxConcurrentShardRequests || defaultMaxConcurrentShardRequests(esVersion), diff --git a/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx b/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx index 2c26f3007db..7cf35144874 100644 --- a/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx +++ b/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx @@ -1,8 +1,8 @@ import React from 'react'; import { EventsWithValidation, regexValidation, LegacyForms } from '@grafana/ui'; -const { Select, Input, FormField } = LegacyForms; +const { Select, Input, FormField, Switch } = LegacyForms; import { ElasticsearchOptions } from '../types'; -import { DataSourceSettings, SelectableValue } from '@grafana/data'; +import { DataSourceSettings, onUpdateDatasourceJsonDataOptionChecked, SelectableValue } from '@grafana/data'; const indexPatternTypes = [ { label: 'No pattern', value: 'none' }, @@ -144,11 +144,29 @@ export const ElasticDetails = (props: Props) => { /> + {value.jsonData.esVersion >= 70 && ( +
+
+ +
+
+ )} ); }; +// TODO: Use change handlers from @grafana/data const changeHandler = ( key: keyof DataSourceSettings, value: Props['value'], @@ -160,6 +178,7 @@ const changeHandler = ( }); }; +// TODO: Use change handlers from @grafana/data const jsonDataChangeHandler = (key: keyof ElasticsearchOptions, value: Props['value'], onChange: Props['onChange']) => ( event: React.SyntheticEvent ) => { diff --git a/public/app/plugins/datasource/elasticsearch/datasource.test.ts b/public/app/plugins/datasource/elasticsearch/datasource.test.ts index 4ac588e0fc7..3c5f5c185fa 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.test.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.test.ts @@ -864,6 +864,69 @@ describe('ElasticDatasource', function(this: any) { expect(typeof JSON.parse(query.split('\n')[1]).query.bool.filter[0].range['@time'].gte).toBe('number'); }); }); + + describe('getMultiSearchUrl', () => { + describe('When esVersion >= 70', () => { + it('Should add correct params to URL if "includeFrozen" is enabled', () => { + const datasSurce = new ElasticDatasource( + { + jsonData: { + esVersion: 70, + includeFrozen: true, + }, + } as DataSourceInstanceSettings, + templateSrv, + timeSrv + ); + + expect(datasSurce.getMultiSearchUrl()).toMatch(/ignore_throttled=false/); + }); + + it('Should NOT add ignore_throttled if "includeFrozen" is disabled', () => { + const datasSurce = new ElasticDatasource( + { + jsonData: { + esVersion: 70, + includeFrozen: false, + }, + } as DataSourceInstanceSettings, + templateSrv, + timeSrv + ); + + expect(datasSurce.getMultiSearchUrl()).not.toMatch(/ignore_throttled=false/); + }); + }); + + describe('When esVersion <= 70', () => { + it('Should NOT add ignore_throttled params regardless of includeFrozen', () => { + const datasSurceWithIncludeFrozen = new ElasticDatasource( + { + jsonData: { + esVersion: 60, + includeFrozen: true, + }, + } as DataSourceInstanceSettings, + templateSrv, + timeSrv + ); + + const datasSurceWithoutIncludeFrozen = new ElasticDatasource( + { + jsonData: { + esVersion: 60, + includeFrozen: false, + }, + } as DataSourceInstanceSettings, + templateSrv, + timeSrv + ); + + expect(datasSurceWithIncludeFrozen.getMultiSearchUrl()).not.toMatch(/ignore_throttled=false/); + expect(datasSurceWithoutIncludeFrozen.getMultiSearchUrl()).not.toMatch(/ignore_throttled=false/); + }); + }); + }); }); describe('enhanceDataFrame', () => { diff --git a/public/app/plugins/datasource/elasticsearch/datasource.ts b/public/app/plugins/datasource/elasticsearch/datasource.ts index 7960ba72c5d..603c8d07193 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.ts @@ -52,6 +52,7 @@ export class ElasticDatasource extends DataSourceApi= 70 && this.maxConcurrentShardRequests) { - return `_msearch?max_concurrent_shard_requests=${this.maxConcurrentShardRequests}`; + searchParams.append('max_concurrent_shard_requests', '' + this.maxConcurrentShardRequests); } - return '_msearch'; + if (this.esVersion >= 70 && this.includeFrozen) { + searchParams.append('ignore_throttled', 'false'); + } + + return (`_msearch?` + searchParams.toString()).replace(/\?+$/, ''); } metricFindQuery(query: any) { diff --git a/public/app/plugins/datasource/elasticsearch/types.ts b/public/app/plugins/datasource/elasticsearch/types.ts index ec07da2c737..af896bc5b6c 100644 --- a/public/app/plugins/datasource/elasticsearch/types.ts +++ b/public/app/plugins/datasource/elasticsearch/types.ts @@ -9,6 +9,7 @@ export interface ElasticsearchOptions extends DataSourceJsonData { logMessageField?: string; logLevelField?: string; dataLinks?: DataLinkConfig[]; + includeFrozen?: boolean; } export interface ElasticsearchAggregation {