elasticsearch: support bucket script pipeline aggregations

2025-02-25 18:55:37 -06:00 · 2018-11-16 16:54:25 +01:00
parent 2325c5dbed
commit b45f72a140
19 changed files with 1151 additions and 62 deletions
--- a/public/app/plugins/datasource/elasticsearch/elastic_response.ts
+++ b/public/app/plugins/datasource/elasticsearch/elastic_response.ts
@@ -88,6 +88,7 @@ export class ElasticResponse {
            datapoints: [],
            metric: metric.type,
            field: metric.field,
+            metricId: metric.id,
            props: props,
          };
          for (i = 0; i < esAgg.buckets.length; i++) {
@@ -240,7 +241,7 @@ export class ElasticResponse {
          return metricName;
        }
        if (group === 'field') {
-          return series.field;
+          return series.field || '';
        }

        return match;
@@ -248,11 +249,27 @@ export class ElasticResponse {
    }

    if (series.field && queryDef.isPipelineAgg(series.metric)) {
-      const appliedAgg = _.find(target.metrics, { id: series.field });
-      if (appliedAgg) {
-        metricName += ' ' + queryDef.describeMetric(appliedAgg);
+      if (series.metric && queryDef.isPipelineAggWithMultipleBucketPaths(series.metric)) {
+        const agg = _.find(target.metrics, { id: series.metricId });
+        if (agg && agg.settings.script) {
+          metricName = agg.settings.script;
+
+          for (const pv of agg.pipelineVariables) {
+            const appliedAgg = _.find(target.metrics, { id: pv.pipelineAgg });
+            if (appliedAgg) {
+              metricName = metricName.replace('params.' + pv.name, queryDef.describeMetric(appliedAgg));
+            }
+          }
+        } else {
+          metricName = 'Unset';
+        }
      } else {
-        metricName = 'Unset';
+        const appliedAgg = _.find(target.metrics, { id: series.field });
+        if (appliedAgg) {
+          metricName += ' ' + queryDef.describeMetric(appliedAgg);
+        } else {
+          metricName = 'Unset';
+        }
      }
    } else if (series.field) {
      metricName += ' ' + series.field;
--- a/public/app/plugins/datasource/elasticsearch/metric_agg.ts
+++ b/public/app/plugins/datasource/elasticsearch/metric_agg.ts
@@ -35,11 +35,20 @@ export class ElasticMetricAggCtrl {
      $scope.isFirst = $scope.index === 0;
      $scope.isSingle = metricAggs.length === 1;
      $scope.settingsLinkText = '';
+      $scope.variablesLinkText = '';
      $scope.aggDef = _.find($scope.metricAggTypes, { value: $scope.agg.type });

      if (queryDef.isPipelineAgg($scope.agg.type)) {
-        $scope.agg.pipelineAgg = $scope.agg.pipelineAgg || 'select metric';
-        $scope.agg.field = $scope.agg.pipelineAgg;
+        if (queryDef.isPipelineAggWithMultipleBucketPaths($scope.agg.type)) {
+          $scope.variablesLinkText = 'Options';
+
+          if ($scope.agg.settings.script) {
+            $scope.variablesLinkText = 'Script: ' + $scope.agg.settings.script.replace(new RegExp('params.', 'g'), '');
+          }
+        } else {
+          $scope.agg.pipelineAgg = $scope.agg.pipelineAgg || 'select metric';
+          $scope.agg.field = $scope.agg.pipelineAgg;
+        }

        const pipelineOptions = queryDef.getPipelineOptions($scope.agg);
        if (pipelineOptions.length > 0) {
@@ -119,6 +128,10 @@ export class ElasticMetricAggCtrl {
      $scope.updatePipelineAggOptions();
    };

+    $scope.toggleVariables = () => {
+      $scope.showVariables = !$scope.showVariables;
+    };
+
    $scope.onChangeInternal = () => {
      $scope.onChange();
    };
@@ -152,6 +165,7 @@ export class ElasticMetricAggCtrl {
        $scope.target.bucketAggs = [queryDef.defaultBucketAgg()];
      }

+      $scope.showVariables = queryDef.isPipelineAggWithMultipleBucketPaths($scope.agg.type);
      $scope.updatePipelineAggOptions();
      $scope.onChange();
    };
--- a/public/app/plugins/datasource/elasticsearch/partials/metric_agg.html
+++ b/public/app/plugins/datasource/elasticsearch/partials/metric_agg.html
@@ -13,7 +13,17 @@
 	<div class="gf-form">
 		<metric-segment-model property="agg.type" options="metricAggTypes" on-change="onTypeChange()" custom="false" css-class="width-10"></metric-segment-model>
 		<metric-segment-model ng-if="aggDef.requiresField" property="agg.field" get-options="getFieldsInternal()" on-change="onChange()" css-class="width-12"></metric-segment-model>
-		<metric-segment-model ng-if="aggDef.isPipelineAgg" property="agg.pipelineAgg" options="pipelineAggOptions" on-change="onChangeInternal()" custom="false" css-class="width-12"></metric-segment-model>
+		<metric-segment-model ng-if="aggDef.isPipelineAgg && !aggDef.supportsMultipleBucketPaths" property="agg.pipelineAgg" options="pipelineAggOptions" on-change="onChangeInternal()" custom="false" css-class="width-12"></metric-segment-model>
+	</div>
+
+  <div class="gf-form gf-form--grow" ng-if="aggDef.isPipelineAgg && aggDef.supportsMultipleBucketPaths">
+		<label class="gf-form-label gf-form-label--grow">
+			<a ng-click="toggleVariables()">
+				<i class="fa fa-caret-down" ng-show="showVariables"></i>
+				<i class="fa fa-caret-right" ng-hide="showVariables"></i>
+        {{variablesLinkText}}
+			</a>
+		</label>
 	</div>

 	<div class="gf-form gf-form--grow">
@@ -36,6 +46,20 @@
 	</div>
 </div>

+<div class="gf-form-group" ng-if="showVariables">
+	<elastic-pipeline-variables variables="agg.pipelineVariables" options="pipelineAggOptions" on-change="onChangeInternal()" />
+  <div class="gf-form offset-width-7">
+    <label class="gf-form-label width-10">
+      Script
+      <info-popover mode="right-normal">
+        Elasticsearch v5.0 and above: Scripting language is Painless. Use <i>params.&lt;var&gt;</i> to reference a variable.<br/><br/>
+        Elasticsearch pre-v5.0: Scripting language is per default Groovy if not changed. For Groovy use <i>&lt;var&gt;</i> to reference a variable.
+      </info-popover>
+    </label>
+    <input type="text" class="gf-form-input max-width-24" empty-to-null ng-model="agg.settings.script" ng-blur="onChangeInternal()" spellcheck='false' placeholder="params.var1 / params.var2">
+  </div>
+</div>
+
 <div class="gf-form-group" ng-if="showOptions">
 	<div class="gf-form offset-width-7" ng-if="agg.type === 'derivative'">
 		<label class="gf-form-label width-10">Unit</label>
@@ -103,5 +127,5 @@
 			<tip>The missing parameter defines how documents that are missing a value should be treated. By default they will be ignored but it is also possible to treat them as if they had a value</tip>
 		</label>
 		<input type="number" class="gf-form-input max-width-12" empty-to-null ng-model="agg.settings.missing" ng-blur="onChangeInternal()" spellcheck='false'>
-	</div>
+  </div>
 </div>
--- a/public/app/plugins/datasource/elasticsearch/partials/pipeline_variables.html
+++ b/public/app/plugins/datasource/elasticsearch/partials/pipeline_variables.html
@@ -0,0 +1,20 @@
+<div ng-repeat="var in variables">
+  <div class="gf-form offset-width-7" ng-if="$index === 0">
+    <label class="gf-form-label width-10">Variables</label>
+    <input type="text" class="gf-form-input max-width-12" ng-model="var.name" placeholder="Variable name" ng-blur="onChangeInternal()" spellcheck='false'>
+    <metric-segment-model property="var.pipelineAgg" options="options" on-change="onChangeInternal()" custom="false" css-class="width-12"></metric-segment-model>
+    <label class="gf-form-label">
+      <a class="pointer" ng-click="remove($index)"><i class="fa fa-minus"></i></a>
+    </label>
+    <label class="gf-form-label">
+      <a class="pointer" ng-click="add()"><i class="fa fa-plus"></i></a>
+    </label>
+  </div>
+  <div class="gf-form offset-width-17" ng-if="$index !== 0">
+    <input type="text" class="gf-form-input max-width-12" ng-model="var.name" placeholder="Variable name" ng-blur="onChangeInternal()" spellcheck='false'>
+    <metric-segment-model property="var.pipelineAgg" options="options" on-change="onChangeInternal()" custom="false" css-class="width-12"></metric-segment-model>
+    <label class="gf-form-label">
+      <a class="pointer" ng-click="remove($index)"><i class="fa fa-minus"></i></a>
+    </label>
+  </div>
+</div>
--- a/public/app/plugins/datasource/elasticsearch/pipeline_variables.ts
+++ b/public/app/plugins/datasource/elasticsearch/pipeline_variables.ts
@@ -0,0 +1,45 @@
+import coreModule from 'app/core/core_module';
+import _ from 'lodash';
+
+export function elasticPipelineVariables() {
+  return {
+    templateUrl: 'public/app/plugins/datasource/elasticsearch/partials/pipeline_variables.html',
+    controller: 'ElasticPipelineVariablesCtrl',
+    restrict: 'E',
+    scope: {
+      onChange: '&',
+      variables: '=',
+      options: '=',
+    },
+  };
+}
+
+const newVariable = index => {
+  return {
+    name: 'var' + index,
+    pipelineAgg: 'select metric',
+  };
+};
+
+export class ElasticPipelineVariablesCtrl {
+  constructor($scope) {
+    $scope.variables = $scope.variables || [newVariable(1)];
+
+    $scope.onChangeInternal = () => {
+      $scope.onChange();
+    };
+
+    $scope.add = () => {
+      $scope.variables.push(newVariable($scope.variables.length + 1));
+      $scope.onChange();
+    };
+
+    $scope.remove = index => {
+      $scope.variables.splice(index, 1);
+      $scope.onChange();
+    };
+  }
+}
+
+coreModule.directive('elasticPipelineVariables', elasticPipelineVariables);
+coreModule.controller('ElasticPipelineVariablesCtrl', ElasticPipelineVariablesCtrl);
--- a/public/app/plugins/datasource/elasticsearch/query_builder.ts
+++ b/public/app/plugins/datasource/elasticsearch/query_builder.ts
@@ -189,7 +189,7 @@ export class ElasticQueryBuilder {
    target.bucketAggs = target.bucketAggs || [queryDef.defaultBucketAgg()];
    target.timeField = this.timeField;

-    let i, nestedAggs, metric;
+    let i, j, pv, nestedAggs, metric;
    const query = {
      size: 0,
      query: {
@@ -269,17 +269,42 @@ export class ElasticQueryBuilder {
      let metricAgg = null;

      if (queryDef.isPipelineAgg(metric.type)) {
-        if (metric.pipelineAgg && /^\d*$/.test(metric.pipelineAgg)) {
-          const appliedAgg = queryDef.findMetricById(target.metrics, metric.pipelineAgg);
-          if (appliedAgg) {
-            if (appliedAgg.type === 'count') {
-              metricAgg = { buckets_path: '_count' };
-            } else {
-              metricAgg = { buckets_path: metric.pipelineAgg };
+        if (queryDef.isPipelineAggWithMultipleBucketPaths(metric.type)) {
+          if (metric.pipelineVariables) {
+            metricAgg = {
+              buckets_path: {},
+            };
+
+            for (j = 0; j < metric.pipelineVariables.length; j++) {
+              pv = metric.pipelineVariables[j];
+
+              if (pv.name && pv.pipelineAgg && /^\d*$/.test(pv.pipelineAgg)) {
+                const appliedAgg = queryDef.findMetricById(target.metrics, pv.pipelineAgg);
+                if (appliedAgg) {
+                  if (appliedAgg.type === 'count') {
+                    metricAgg.buckets_path[pv.name] = '_count';
+                  } else {
+                    metricAgg.buckets_path[pv.name] = pv.pipelineAgg;
+                  }
+                }
+              }
            }
+          } else {
+            continue;
          }
        } else {
-          continue;
+          if (metric.pipelineAgg && /^\d*$/.test(metric.pipelineAgg)) {
+            const appliedAgg = queryDef.findMetricById(target.metrics, metric.pipelineAgg);
+            if (appliedAgg) {
+              if (appliedAgg.type === 'count') {
+                metricAgg = { buckets_path: '_count' };
+              } else {
+                metricAgg = { buckets_path: metric.pipelineAgg };
+              }
+            }
+          } else {
+            continue;
+          }
        }
      } else {
        metricAgg = { field: metric.field };
--- a/public/app/plugins/datasource/elasticsearch/query_ctrl.ts
+++ b/public/app/plugins/datasource/elasticsearch/query_ctrl.ts
@@ -1,5 +1,6 @@
 import './bucket_agg';
 import './metric_agg';
+import './pipeline_variables';

 import angular from 'angular';
 import _ from 'lodash';
@@ -70,6 +71,9 @@ export class ElasticQueryCtrl extends QueryCtrl {
      if (aggDef.requiresField) {
        text += metric.field;
      }
+      if (aggDef.supportsMultipleBucketPaths) {
+        text += metric.settings.script.replace(new RegExp('params.', 'g'), '');
+      }
      text += '), ';
    });

--- a/public/app/plugins/datasource/elasticsearch/query_def.ts
+++ b/public/app/plugins/datasource/elasticsearch/query_def.ts
@@ -64,6 +64,14 @@ export const metricAggTypes = [
    isPipelineAgg: true,
    minVersion: 2,
  },
+  {
+    text: 'Bucket Script',
+    value: 'bucket_script',
+    requiresField: false,
+    isPipelineAgg: true,
+    supportsMultipleBucketPaths: true,
+    minVersion: 2,
+  },
  { text: 'Raw Document', value: 'raw_document', requiresField: false },
 ];

@@ -128,6 +136,7 @@ export const pipelineOptions = {
    { text: 'minimize', default: false },
  ],
  derivative: [{ text: 'unit', default: undefined }],
+  bucket_script: [],
 };

 export const movingAvgModelSettings = {
@@ -171,6 +180,14 @@ export function isPipelineAgg(metricType) {
  return false;
 }

+export function isPipelineAggWithMultipleBucketPaths(metricType) {
+  if (metricType) {
+    return metricAggTypes.find(t => t.value === metricType && t.supportsMultipleBucketPaths) !== undefined;
+  }
+
+  return false;
+}
+
 export function getPipelineAggOptions(targets) {
  const result = [];
  _.each(targets.metrics, metric => {
--- a/public/app/plugins/datasource/elasticsearch/specs/elastic_response.test.ts
+++ b/public/app/plugins/datasource/elasticsearch/specs/elastic_response.test.ts
@@ -665,4 +665,70 @@ describe('ElasticResponse', () => {
      expect(result.data[0].datapoints[0].fieldProp).toBe('field');
    });
  });
+
+  describe('with bucket_script ', () => {
+    let result;
+
+    beforeEach(() => {
+      targets = [
+        {
+          refId: 'A',
+          metrics: [
+            { id: '1', type: 'sum', field: '@value' },
+            { id: '3', type: 'max', field: '@value' },
+            {
+              id: '4',
+              field: 'select field',
+              pipelineVariables: [{ name: 'var1', pipelineAgg: '1' }, { name: 'var2', pipelineAgg: '3' }],
+              settings: { script: 'params.var1 * params.var2' },
+              type: 'bucket_script',
+            },
+          ],
+          bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }],
+        },
+      ];
+      response = {
+        responses: [
+          {
+            aggregations: {
+              '2': {
+                buckets: [
+                  {
+                    1: { value: 2 },
+                    3: { value: 3 },
+                    4: { value: 6 },
+                    doc_count: 60,
+                    key: 1000,
+                  },
+                  {
+                    1: { value: 3 },
+                    3: { value: 4 },
+                    4: { value: 12 },
+                    doc_count: 60,
+                    key: 2000,
+                  },
+                ],
+              },
+            },
+          },
+        ],
+      };
+
+      result = new ElasticResponse(targets, response).getTimeSeries();
+    });
+
+    it('should return 3 series', () => {
+      expect(result.data.length).toBe(3);
+      expect(result.data[0].datapoints.length).toBe(2);
+      expect(result.data[0].target).toBe('Sum @value');
+      expect(result.data[1].target).toBe('Max @value');
+      expect(result.data[2].target).toBe('Sum @value * Max @value');
+      expect(result.data[0].datapoints[0][0]).toBe(2);
+      expect(result.data[1].datapoints[0][0]).toBe(3);
+      expect(result.data[2].datapoints[0][0]).toBe(6);
+      expect(result.data[0].datapoints[1][0]).toBe(3);
+      expect(result.data[1].datapoints[1][0]).toBe(4);
+      expect(result.data[2].datapoints[1][0]).toBe(12);
+    });
+  });
 });
--- a/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts
+++ b/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts
@@ -353,6 +353,83 @@ describe('ElasticQueryBuilder', () => {
    expect(firstLevel.aggs['2'].derivative.buckets_path).toBe('_count');
  });

+  it('with bucket_script', () => {
+    const query = builder.build({
+      metrics: [
+        {
+          id: '1',
+          type: 'sum',
+          field: '@value',
+        },
+        {
+          id: '3',
+          type: 'max',
+          field: '@value',
+        },
+        {
+          field: 'select field',
+          id: '4',
+          meta: {},
+          pipelineVariables: [
+            {
+              name: 'var1',
+              pipelineAgg: '1',
+            },
+            {
+              name: 'var2',
+              pipelineAgg: '3',
+            },
+          ],
+          settings: {
+            script: 'params.var1 * params.var2',
+          },
+          type: 'bucket_script',
+        },
+      ],
+      bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }],
+    });
+
+    const firstLevel = query.aggs['2'];
+
+    expect(firstLevel.aggs['4']).not.toBe(undefined);
+    expect(firstLevel.aggs['4'].bucket_script).not.toBe(undefined);
+    expect(firstLevel.aggs['4'].bucket_script.buckets_path).toMatchObject({ var1: '1', var2: '3' });
+  });
+
+  it('with bucket_script doc count', () => {
+    const query = builder.build({
+      metrics: [
+        {
+          id: '3',
+          type: 'count',
+          field: 'select field',
+        },
+        {
+          field: 'select field',
+          id: '4',
+          meta: {},
+          pipelineVariables: [
+            {
+              name: 'var1',
+              pipelineAgg: '3',
+            },
+          ],
+          settings: {
+            script: 'params.var1 * 1000',
+          },
+          type: 'bucket_script',
+        },
+      ],
+      bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }],
+    });
+
+    const firstLevel = query.aggs['2'];
+
+    expect(firstLevel.aggs['4']).not.toBe(undefined);
+    expect(firstLevel.aggs['4'].bucket_script).not.toBe(undefined);
+    expect(firstLevel.aggs['4'].bucket_script.buckets_path).toMatchObject({ var1: '_count' });
+  });
+
  it('with histogram', () => {
    const query = builder.build({
      metrics: [{ id: '1', type: 'count' }],
--- a/public/app/plugins/datasource/elasticsearch/specs/query_def.test.ts
+++ b/public/app/plugins/datasource/elasticsearch/specs/query_def.test.ts
@@ -65,6 +65,24 @@ describe('ElasticQueryDef', () => {
    });
  });

+  describe('isPipelineAggWithMultipleBucketPaths', () => {
+    describe('bucket_script', () => {
+      const result = queryDef.isPipelineAggWithMultipleBucketPaths('bucket_script');
+
+      test('should have multiple bucket paths support', () => {
+        expect(result).toBe(true);
+      });
+    });
+
+    describe('moving_avg', () => {
+      const result = queryDef.isPipelineAggWithMultipleBucketPaths('moving_avg');
+
+      test('should not have multiple bucket paths support', () => {
+        expect(result).toBe(false);
+      });
+    });
+  });
+
  describe('pipeline aggs depending on esverison', () => {
    describe('using esversion undefined', () => {
      test('should not get pipeline aggs', () => {
@@ -80,13 +98,13 @@ describe('ElasticQueryDef', () => {

    describe('using esversion 2', () => {
      test('should get pipeline aggs', () => {
-        expect(queryDef.getMetricAggTypes(2).length).toBe(11);
+        expect(queryDef.getMetricAggTypes(2).length).toBe(12);
      });
    });

    describe('using esversion 5', () => {
      test('should get pipeline aggs', () => {
-        expect(queryDef.getMetricAggTypes(5).length).toBe(11);
+        expect(queryDef.getMetricAggTypes(5).length).toBe(12);
      });
    });
  });