Elasticsearch: Add frozen indices search support (#36018)

* Revert "Revert "Elasticsearch: add frozen indices search support (#27472)" (#27726)"

This reverts commit 4c7131425b.

* Make label width a bit more consistent

* Add documentation for X-Pack & Frozen Indices support in Elasticsearch

* Change UI & docs casing

* create default empty dataframe

* Remove backticks and simplify regex

* small doc improvement

docs/sources/datasources/elasticsearch.md

@@ -80,6 +80,13 @@ number followed by a valid time identifier, e.g. `1m` (1 minute) or `30s` (30 se
 | `s`        | second      |
 | `ms`       | millisecond |
 
+### X-Pack enabled
+
+Enables `X-Pack` specific features and options, providing the query editor with additional aggregations such as `Rate` and `Top Metrics`.
+
+#### Include frozen indices
+
+When `X-Pack enabled` is active and the configured Elasticsearch version is higher than `6.6.0`, you can configure Grafana to not ignore [frozen indices](https://www.elastic.co/guide/en/elasticsearch/reference/7.13/frozen-indices.html) when performing search requests.
 ### Logs
 
 There are two parameters, `Message field name` and `Level field name`, that can optionally be configured from the data source settings page that determine

pkg/tsdb/elasticsearch/client/client.go

@@ -334,12 +334,23 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque
 }
 
 func (c *baseClientImpl) getMultiSearchQueryParameters() string {
+	var qs []string
+
 	if c.version.Major() >= 7 {
 		maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(5)
-		return fmt.Sprintf("max_concurrent_shard_requests=%d", maxConcurrentShardRequests)
+		qs = append(qs, fmt.Sprintf("max_concurrent_shard_requests=%d", maxConcurrentShardRequests))
 	}
 
-	return ""
+	// Querying frozen indices was added in 6.6 with xpack
+	includeFrozen := c.getSettings().Get("includeFrozen").MustBool(false)
+	xpack := c.getSettings().Get("xpack").MustBool(false)
+	allowedFrozenIndicesVersionRange, _ := semver.NewConstraint(">=6.6.0")
+
+	if (allowedFrozenIndicesVersionRange.Check(c.version)) && includeFrozen && xpack {
+		qs = append(qs, "ignore_throttled=false")
+	}
+
+	return strings.Join(qs, "&")
 }
 
 func (c *baseClientImpl) MultiSearch() *MultiSearchRequestBuilder {

pkg/tsdb/elasticsearch/client/client_test.go

@@ -253,10 +253,12 @@ func TestClient_ExecuteMultisearch(t *testing.T) {
 	httpClientScenario(t, "Given a fake http client and a v5.6 client with response", &models.DataSource{
 		Database: "[metrics-]YYYY.MM.DD",
 		JsonData: simplejson.NewFromAny(map[string]interface{}{
-			"esVersion":                  56,
+			"esVersion":                  "5.6.0",
 			"maxConcurrentShardRequests": 100,
 			"timeField":                  "@timestamp",
 			"interval":                   "Daily",
+			"includeFrozen":              true,
+			"xpack":                      true,
 		}),
 	}, func(sc *scenarioContext) {
 		sc.responseBody = `{
@@ -276,6 +278,7 @@ func TestClient_ExecuteMultisearch(t *testing.T) {
 		require.NotNil(t, sc.request)
 		assert.Equal(t, http.MethodPost, sc.request.Method)
 		assert.Equal(t, "/_msearch", sc.request.URL.Path)
+		assert.NotContains(t, sc.request.URL.RawQuery, "ignore_throttled=")
 
 		require.NotNil(t, sc.requestBody)
 
@@ -305,10 +308,12 @@ func TestClient_ExecuteMultisearch(t *testing.T) {
 	httpClientScenario(t, "Given a fake http client and a v7.0 client with response", &models.DataSource{
 		Database: "[metrics-]YYYY.MM.DD",
 		JsonData: simplejson.NewFromAny(map[string]interface{}{
-			"esVersion":                  70,
+			"esVersion":                  "7.0.0",
 			"maxConcurrentShardRequests": 6,
 			"timeField":                  "@timestamp",
 			"interval":                   "Daily",
+			"includeFrozen":              true,
+			"xpack":                      true,
 		}),
 	}, func(sc *scenarioContext) {
 		sc.responseBody = `{
@@ -328,7 +333,7 @@ func TestClient_ExecuteMultisearch(t *testing.T) {
 		require.NotNil(t, sc.request)
 		assert.Equal(t, http.MethodPost, sc.request.Method)
 		assert.Equal(t, "/_msearch", sc.request.URL.Path)
-		assert.Equal(t, "max_concurrent_shard_requests=6", sc.request.URL.RawQuery)
+		assert.Equal(t, "max_concurrent_shard_requests=6&ignore_throttled=false", sc.request.URL.RawQuery)
 
 		require.NotNil(t, sc.requestBody)
 
@@ -346,6 +351,7 @@ func TestClient_ExecuteMultisearch(t *testing.T) {
 		assert.True(t, jHeader.Get("ignore_unavailable").MustBool(false))
 		assert.Equal(t, "query_then_fetch", jHeader.Get("search_type").MustString())
 		assert.Empty(t, jHeader.Get("max_concurrent_shard_requests"))
+		assert.False(t, jHeader.Get("ignore_throttled").MustBool())
 
 		assert.Equal(t, "15000*@hostname", jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString())
 

pkg/tsdb/elasticsearch/response_parser.go

@@ -67,7 +67,8 @@ func (rp *responseParser) getTimeSeries() (plugins.DataResponse, error) {
 		}
 
 		queryRes := plugins.DataQueryResult{
-			Meta: debugInfo,
+			Meta:       debugInfo,
+			Dataframes: plugins.NewDecodedDataFrames(data.Frames{}),
 		}
 		props := make(map[string]string)
 		err := rp.processBuckets(res.Aggregations, target, &queryRes, props, 0)

public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx

@@ -145,17 +145,29 @@ export const ElasticDetails = ({ value, onChange }: Props) => {
         </div>
         <div className="gf-form-inline">
           <Switch
-            label="X-Pack Enabled"
-            labelClass="width-13"
+            label="X-Pack enabled"
+            labelClass="width-10"
             checked={value.jsonData.xpack || false}
             onChange={jsonDataSwitchChangeHandler('xpack', value, onChange)}
           />
         </div>
+
+        {gte(value.jsonData.esVersion, '6.6.0') && value.jsonData.xpack && (
+          <div className="gf-form-inline">
+            <Switch
+              label="Include frozen indices"
+              labelClass="width-10"
+              checked={value.jsonData.includeFrozen ?? false}
+              onChange={jsonDataSwitchChangeHandler('includeFrozen', value, onChange)}
+            />
+          </div>
+        )}
       </div>
     </>
   );
 };
 
+// TODO: Use change handlers from @grafana/data
 const changeHandler = (
   key: keyof DataSourceSettings<ElasticsearchOptions>,
   value: Props['value'],
@@ -167,6 +179,7 @@ const changeHandler = (
   });
 };
 
+// TODO: Use change handlers from @grafana/data
 const jsonDataChangeHandler = (key: keyof ElasticsearchOptions, value: Props['value'], onChange: Props['onChange']) => (
   event: React.SyntheticEvent<HTMLInputElement | HTMLSelectElement>
 ) => {

public/app/plugins/datasource/elasticsearch/configuration/utils.ts

@@ -19,6 +19,7 @@ export const coerceOptions = (
         options.jsonData.maxConcurrentShardRequests || defaultMaxConcurrentShardRequests(esVersion),
       logMessageField: options.jsonData.logMessageField || '',
       logLevelField: options.jsonData.logLevelField || '',
+      includeFrozen: options.jsonData.includeFrozen ?? false,
     },
   };
 };

public/app/plugins/datasource/elasticsearch/datasource.test.ts

@@ -896,6 +896,42 @@ describe('ElasticDatasource', function (this: any) {
   });
 });
 
+describe('getMultiSearchUrl', () => {
+  describe('When esVersion >= 6.6.0', () => {
+    it('Should add correct params to URL if "includeFrozen" is enabled', () => {
+      const { ds } = getTestContext({ jsonData: { esVersion: '6.6.0', includeFrozen: true, xpack: true } });
+
+      expect(ds.getMultiSearchUrl()).toMatch(/ignore_throttled=false/);
+    });
+
+    it('Should NOT add ignore_throttled if "includeFrozen" is disabled', () => {
+      const { ds } = getTestContext({ jsonData: { esVersion: '6.6.0', includeFrozen: false, xpack: true } });
+
+      expect(ds.getMultiSearchUrl()).not.toMatch(/ignore_throttled=false/);
+    });
+
+    it('Should NOT add ignore_throttled if "xpack" is disabled', () => {
+      const { ds } = getTestContext({ jsonData: { esVersion: '6.6.0', includeFrozen: true, xpack: false } });
+
+      expect(ds.getMultiSearchUrl()).not.toMatch(/ignore_throttled=false/);
+    });
+  });
+
+  describe('When esVersion < 6.6.0', () => {
+    it('Should NOT add ignore_throttled params regardless of includeFrozen', () => {
+      const { ds: dsWithIncludeFrozen } = getTestContext({
+        jsonData: { esVersion: '5.6.0', includeFrozen: false, xpack: true },
+      });
+      const { ds: dsWithoutIncludeFrozen } = getTestContext({
+        jsonData: { esVersion: '5.6.0', includeFrozen: true, xpack: true },
+      });
+
+      expect(dsWithIncludeFrozen.getMultiSearchUrl()).not.toMatch(/ignore_throttled=false/);
+      expect(dsWithoutIncludeFrozen.getMultiSearchUrl()).not.toMatch(/ignore_throttled=false/);
+    });
+  });
+});
+
 describe('enhanceDataFrame', () => {
   it('adds links to dataframe', () => {
     const df = new MutableDataFrame({

public/app/plugins/datasource/elasticsearch/datasource.ts

@@ -73,6 +73,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
   logLevelField?: string;
   dataLinks: DataLinkConfig[];
   languageProvider: LanguageProvider;
+  includeFrozen: boolean;
 
   constructor(
     instanceSettings: DataSourceInstanceSettings<ElasticsearchOptions>,
@@ -99,6 +100,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
     this.logMessageField = settingsData.logMessageField || '';
     this.logLevelField = settingsData.logLevelField || '';
     this.dataLinks = settingsData.dataLinks || [];
+    this.includeFrozen = settingsData.includeFrozen ?? false;
 
     if (this.logMessageField === '') {
       this.logMessageField = undefined;
@@ -761,11 +763,17 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
   }
 
   getMultiSearchUrl() {
+    const searchParams = new URLSearchParams();
+
     if (gte(this.esVersion, '7.0.0') && this.maxConcurrentShardRequests) {
-      return `_msearch?max_concurrent_shard_requests=${this.maxConcurrentShardRequests}`;
+      searchParams.append('max_concurrent_shard_requests', `${this.maxConcurrentShardRequests}`);
     }
 
-    return '_msearch';
+    if (gte(this.esVersion, '6.6.0') && this.xpack && this.includeFrozen) {
+      searchParams.append('ignore_throttled', 'false');
+    }
+
+    return ('_msearch?' + searchParams.toString()).replace(/\?$/, '');
   }
 
   metricFindQuery(query: string, options?: any): Promise<MetricFindValue[]> {

public/app/plugins/datasource/elasticsearch/types.ts

@@ -20,6 +20,7 @@ export interface ElasticsearchOptions extends DataSourceJsonData {
   logMessageField?: string;
   logLevelField?: string;
   dataLinks?: DataLinkConfig[];
+  includeFrozen?: boolean;
 }
 
 interface MetricConfiguration<T extends MetricAggregationType> {