Elasticsearch: add rate aggregation (#33311)

* Elasticsearch: add first version of rate aggregation

* Add test for rate aggreation settings editor

* Fix unit

* Add default value for mode for rate agg

* Add todo

* Revert "Add default value for mode for rate agg"

This reverts commit a44c985d8b.

* Fix width when select is empty

* Add rate aggregation to elasticsearch models

* Remove rate aggregation from extended stats

* Move elastic2 docker block & add elastic (latest)

* Add versionRange to rate aggregation

* add 7.10 elasticsearch version option

* Set supportsInlineScript to true for rate aggregation

* Remove fixed width from select

* Change seconds label for rate aggregation unit options dropdown

Co-authored-by: Piotr Jamróz <pm.jamroz@gmail.com>

Co-authored-by: Elfo404 <me@giordanoricci.com>
Co-authored-by: Piotr Jamróz <pm.jamroz@gmail.com>

devenv/docker/blocks/elastic/docker-compose.yaml

@@ -1,15 +1,19 @@
-  elasticsearch:
-    image: elasticsearch:2.4.1
-    command: elasticsearch -Des.network.host=0.0.0.0
-    ports:
-      - "9200:9200"
-      - "9300:9300"
-    volumes:
-      - ./docker/blocks/elastic/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
+# You need to run 'sysctl -w vm.max_map_count=262144' on the host machine
 
-  fake-elastic-data:
-    image: grafana/fake-data-gen
-    network_mode: bridge
+  elasticsearch-latest:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.2
+    command: elasticsearch
     environment:
-      FD_DATASOURCE: elasticsearch
-      FD_PORT: 9200
+      - "discovery.type=single-node"
+    ports:
+      - "14200:9200"
+      - "14300:9300"
+
+  fake-elastic-latest-data:
+    image: grafana/fake-data-gen
+    links:
+      - elasticsearch-latest
+    environment:
+      FD_SERVER: elasticsearch-latest
+      FD_DATASOURCE: elasticsearch7
+      FD_PORT: 9200

devenv/docker/blocks/elastic/elasticsearch.yml

@@ -1,2 +1,3 @@
 script.inline: on
 script.indexed: on
+xpack.license.self_generated.type: basic

devenv/docker/blocks/elastic2/docker-compose.yaml

@@ -0,0 +1,15 @@
+  elasticsearch2:
+    image: elasticsearch:2.4.1
+    command: elasticsearch -Des.network.host=0.0.0.0
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+    volumes:
+      - ./docker/blocks/elastic2/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
+
+  fake-elastic-data:
+    image: grafana/fake-data-gen
+    network_mode: bridge
+    environment:
+      FD_DATASOURCE: elasticsearch2
+      FD_PORT: 9200

devenv/docker/blocks/elastic2/elasticsearch.yml

@@ -0,0 +1,2 @@
+script.inline: on
+script.indexed: on

docs/sources/datasources/elasticsearch.md

@@ -57,7 +57,7 @@ a time pattern for the index name or a wildcard.
 ### Elasticsearch version
 
 Select the version of your Elasticsearch data source from the version selection dropdown. Different query compositions and functionalities are available in the query editor for different versions.
-Available Elasticsearch versions are `2.x`, `5.x`, `5.6+`, `6.0+`, `7.0+` and `7.7+`. Select the option that best matches your data source version.
+Available Elasticsearch versions are `2.x`, `5.x`, `5.6+`, `6.0+`, `7.0+`, `7.7+` and `7.10+`. Select the option that best matches your data source version.
 
 Grafana assumes that you are running the lowest possible version for a specified range. This ensures that new features or breaking changes in a future Elasticsearch release will not affect your configuration.
 

pkg/tsdb/elasticsearch/models.go

@@ -52,6 +52,7 @@ var metricAggType = map[string]string{
 	"serial_diff":    "Serial Difference",
 	"bucket_script":  "Bucket Script",
 	"raw_document":   "Raw Document",
+	"rate":           "Rate",
 }
 
 var extendedStats = map[string]string{

public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/SettingsEditor/index.test.tsx

@@ -82,4 +82,45 @@ describe('Settings Editor', () => {
       expect(settingsButtonEl.textContent).toBe(`Size: ${newSizeValue}`);
     });
   });
+
+  describe('Rate aggregation', () => {
+    it('should render correct settings', () => {
+      const metricId = '1';
+      const query: ElasticsearchQuery = {
+        refId: 'A',
+        query: '',
+        metrics: [
+          {
+            id: metricId,
+            type: 'rate',
+            settings: {},
+          },
+        ],
+        bucketAggs: [],
+      };
+
+      const onChange = jest.fn();
+
+      render(
+        <ElasticsearchProvider
+          query={query}
+          datasource={{} as ElasticDatasource}
+          onChange={onChange}
+          onRunQuery={() => {}}
+          range={getDefaultTimeRange()}
+        >
+          <SettingsEditor metric={query.metrics![0]} previousMetrics={[]} />
+        </ElasticsearchProvider>
+      );
+
+      let settingsButtonEl = screen.getByRole('button');
+      fireEvent.click(settingsButtonEl);
+
+      const unitSelectElement = screen.getByTestId('unit-select');
+      const modeSelectElement = screen.getByTestId('mode-select');
+
+      expect(unitSelectElement).toBeInTheDocument();
+      expect(modeSelectElement).toBeInTheDocument();
+    });
+  });
 });

public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/SettingsEditor/index.tsx

@@ -1,4 +1,4 @@
-import { InlineField, Input, InlineSwitch } from '@grafana/ui';
+import { InlineField, Input, InlineSwitch, Select } from '@grafana/ui';
 import React, { ComponentProps, useState } from 'react';
 import { extendedStats } from '../../../../query_def';
 import { useDispatch } from '../../../../hooks/useStatelessReducer';
@@ -34,6 +34,22 @@ export const SettingsEditor = ({ metric, previousMetrics }: Props) => {
   const description = useDescription(metric);
   const query = useQuery();
 
+  const rateAggUnitOptions = [
+    { value: 'second', label: 'Second' },
+    { value: 'minute', label: 'Minute' },
+    { value: 'hour', label: 'Hour' },
+    { value: 'day', label: 'Day' },
+    { value: 'week', label: 'Week' },
+    { value: 'month', label: 'Month' },
+    { value: 'quarter', label: 'Quarter' },
+    { value: 'Year', label: 'Year' },
+  ];
+
+  const rateAggModeOptions = [
+    { value: 'sum', label: 'Sum' },
+    { value: 'value_count', label: 'Value count' },
+  ];
+
   return (
     <SettingsEditorContainer label={description} hidden={metric.hide}>
       {metric.type === 'derivative' && <SettingField label="Unit" metric={metric} settingName="unit" />}
@@ -105,6 +121,28 @@ export const SettingsEditor = ({ metric, previousMetrics }: Props) => {
         </InlineField>
       )}
 
+      {metric.type === 'rate' && (
+        <>
+          <InlineField label="Unit" {...inlineFieldProps} data-testid="unit-select">
+            <Select
+              id={`ES-query-${query.refId}_metric-${metric.id}-unit`}
+              onChange={(e) => dispatch(changeMetricSetting(metric, 'unit', e.value))}
+              options={rateAggUnitOptions}
+              value={metric.settings?.unit}
+            />
+          </InlineField>
+
+          <InlineField label="Mode" {...inlineFieldProps} data-testid="mode-select">
+            <Select
+              id={`ES-query-${query.refId}_metric-${metric.id}-mode`}
+              onChange={(e) => dispatch(changeMetricSetting(metric, 'mode', e.value))}
+              options={rateAggModeOptions}
+              value={metric.settings?.unit}
+            />
+          </InlineField>
+        </>
+      )}
+
       {isMetricAggregationWithInlineScript(metric) && (
         <SettingField label="Script" metric={metric} settingName="script" placeholder="_value * 1" />
       )}

public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/aggregations.ts

@@ -20,6 +20,7 @@ export type MetricAggregationType =
   | 'raw_document'
   | 'raw_data'
   | 'logs'
+  | 'rate'
   | 'top_metrics'
   | PipelineMetricAggregationType;
 
@@ -154,6 +155,14 @@ export interface Logs extends BaseMetricAggregation {
   };
 }
 
+export interface Rate extends MetricAggregationWithField {
+  type: 'rate';
+  settings?: {
+    unit?: string;
+    mode?: string;
+  };
+}
+
 export interface BasePipelineMetricAggregation extends MetricAggregationWithField {
   type: PipelineMetricAggregationType;
   pipelineAgg?: string;
@@ -311,6 +320,7 @@ export type MetricAggregationWithSettings =
   | MovingAverage
   | MovingFunction
   | Logs
+  | Rate
   | TopMetrics;
 
 export type MetricAggregationWithMeta = ExtendedStats;
@@ -373,6 +383,7 @@ export const METRIC_AGGREGATION_TYPES: MetricAggregationType[] = [
   'serial_diff',
   'cumulative_sum',
   'bucket_script',
+  'rate',
   'top_metrics',
 ];
 

public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/utils.ts

@@ -257,6 +257,19 @@ export const metricAggregationConfig: MetricsConfiguration = {
       },
     },
   },
+  rate: {
+    label: 'Rate',
+    xpack: true,
+    versionRange: '>=7.10.0',
+    requiresField: true,
+    isPipelineAgg: false,
+    supportsMissing: false,
+    supportsMultipleBucketPaths: false,
+    hasSettings: true,
+    supportsInlineScript: true,
+    hasMeta: false,
+    defaults: {},
+  },
 };
 
 interface PipelineOption {

public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx

@@ -21,6 +21,7 @@ const esVersions = [
   { label: '6.0+', value: '6.0.0' },
   { label: '7.0+', value: '7.0.0' },
   { label: '7.7+', value: '7.7.0' },
+  { label: '7.10+', value: '7.10.0' },
 ];
 
 type Props = {