feat(elasticsearch): groundwork for a much more sophisticated elasticsearch query editor

public/app/plugins/datasource/elasticsearch/datasource.js

@@ -174,39 +174,32 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
 
     // This is quite complex
     // neeed to recurise down the nested buckets to build series
-    ElasticDatasource.prototype._processBuckets = function(buckets, target, series, level, parentName, parentTime) {
+    ElasticDatasource.prototype._processBuckets = function(buckets, target, series, level, parentName) {
-      var groupBy = target.groupByFields[level];
+      var seriesName, value, metric, i, y, bucket, childBucket;
-      var seriesName, time, value, select, i, y, bucket;
 
       for (i = 0; i < buckets.length; i++) {
         bucket = buckets[i];
+        childBucket = bucket['b' + level];
 
-        if (groupBy) {
+        if (childBucket && childBucket.buckets) {
-          seriesName = level > 0 ? parentName + ' ' + bucket.key : parentName;
+          seriesName = parentName + ' ' + bucket.key;
-          time = parentTime || bucket.key;
+          this._processBuckets(childBucket.buckets, target, series, level+1, seriesName);
-          this._processBuckets(bucket[groupBy.field].buckets, target, series, level+1, seriesName, time);
         } else {
 
-          for (y = 0; y < target.select.length; y++) {
+          for (y = 0; y < target.metrics.length; y++) {
-            select = target.select[y];
+            metric = target.metrics[y];
             seriesName = parentName;
 
-            if (level > 0) {
+            if (metric.field) {
-              seriesName +=  ' ' + bucket.key;
+              seriesName += ' ' + metric.field + ' ' + metric.agg;
-            } else {
+              value = bucket['m' + y.toString()].value;
-              parentTime = bucket.key;
-            }
-
-            if (select.field) {
-              seriesName += ' ' + select.field + ' ' + select.agg;
-              value = bucket[y.toString()].value;
             } else {
               seriesName += ' count';
               value = bucket.doc_count;
             }
 
             var serie = series[seriesName] = series[seriesName] || {target: seriesName, datapoints: []};
-            serie.datapoints.push([value, parentTime]);
+            serie.datapoints.push([value, bucket.key]);
           }
         }
       }
@@ -221,11 +214,11 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
           throw { message: response.error };
         }
 
-        var buckets = response.aggregations.histogram.buckets;
+        var buckets = response.aggregations["b0"].buckets;
         var target = targets[i];
         var querySeries = {};
 
-        this._processBuckets(buckets, target, querySeries, 0, target.refId);
+        this._processBuckets(buckets, target, querySeries, 1, target.refId);
 
         for (var prop in querySeries) {
           if (querySeries.hasOwnProperty(prop)) {

public/app/plugins/datasource/elasticsearch/directives.js

@@ -1,5 +1,6 @@
 define([
   'angular',
+  './queryComponent',
 ],
 function (angular) {
   'use strict';

public/app/plugins/datasource/elasticsearch/partials/query.editor.html

@@ -51,7 +51,7 @@
 			Time field
 		</li>
 		<li>
-			<metric-segment segment="timeSegment" get-alt-segments="getTimeFields()" on-value-changed="timeFieldChanged()"></metric-segment>
+			<metric-segment segment="timeSegment" get-alt-segments="getFields()" on-value-changed="timeFieldChanged()"></metric-segment>
 		</li>
 	</ul>
 
@@ -62,28 +62,39 @@
 	</div>
 </div>
 
-<div class="tight-form" ng-hide="target.rawQuery">
+<div ng-hide="target.rawQuery">
-	<ul class="tight-form-list">
+	<div class="tight-form">
-		<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
+		<ul class="tight-form-list">
-			Select
+			<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
-		</li>
+				Metrics
-		<li ng-repeat="segment in selectSegments">
+			</li>
-			<metric-segment segment="segment" get-alt-segments="getSelectSegments(segment, $index)"  on-value-changed="selectChanged(segment, $index)"></metric-segment>
+			<li ng-repeat="segment in selectSegments">
-		</li>
+				<metric-segment segment="segment" get-alt-segments="getSelectSegments(segment, $index)"  on-value-changed="selectChanged(segment, $index)"></metric-segment>
-	</ul>
+			</li>
+		</ul>
 
-	<div class="clearfix"></div>
+		<div class="clearfix"></div>
+	</div>
+
+	<div class="tight-form" ng-repeat="agg in target.bucketAggs">
+		<ul class="tight-form-list">
+			<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
+				<span ng-show="$first">Group by</span>
+				<span ng-show="!$first">Then by</span>
+			</li>
+			<li>
+				<elastic-query-component model="agg" get-fields="getFields()" on-change="queryUpdated()"></elastic-query-component>
+			</li>
+		</ul>
+
+		<ul class="tight-form-list pull-right">
+			<li class="tight-form-item" ng-if="$index === 0">
+				<a class="pointer" ng-click="addBucketAgg()"><i class="fa fa-plus"></i></a>
+			</li>
+			<li class="tight-form-item" ng-if="!$last">
+				<a class="pointer" ng-click="removeBucketAgg(agg, $index)"><i class="fa fa-minus"></i></a>
+			</li>
+		</ul>
+		<div class="clearfix"></div>
+	</div>
 </div>
-
-<div class="tight-form" ng-hide="target.rawQuery">
-	<ul class="tight-form-list">
-		<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
-			Group by
-		</li>
-		<li ng-repeat="segment in groupBySegments">
-			<metric-segment segment="segment" get-alt-segments="getGroupByFields(segment, $index)" on-value-changed="groupByChanged(segment, $index)"></metric-segment>
-		</li>
-	</ul>
-	<div class="clearfix"></div>
-</div>
-

public/app/plugins/datasource/elasticsearch/queryBuilder.js

@@ -17,6 +17,7 @@ function (angular) {
       return angular.fromJson(target.rawQuery);
     }
 
+    var i, nestedAggs;
     var query = {
       "size": 0,
       "query": {
@@ -36,43 +37,42 @@ function (angular) {
       }
     };
 
-    query.aggs = {
+    nestedAggs = query;
-      "histogram": {
+
-        "date_histogram": {
+    for (i = 0; i < target.bucketAggs.length; i++) {
-          "interval": target.interval || "$interval",
+      var aggDef = target.bucketAggs[i];
-          "field": target.timeField,
+      var esAgg = {};
-          "min_doc_count": 0,
+
-          "extended_bounds": {
+      switch(aggDef.type) {
-            "min": "$timeFrom",
+        case 'date_histogram': {
-            "max": "$timeTo"
+          esAgg["date_histogram"] = {
-          }
+            "interval": target.interval || "$interval",
+            "field": aggDef.field,
+            "min_doc_count": 0,
+            "extended_bounds": { "min": "$timeFrom", "max": "$timeTo" }
+          };
+          break;
         }
-      },
+        case 'terms': {
-    };
+          esAgg["terms"] = { "field": aggDef.field };
-
+          break;
-    var nestedAggs = query.aggs.histogram;
+        }
-    var i;
+      }
-
-    target.groupByFields = target.groupByFields || [];
-
-    for (i = 0; i < target.groupByFields.length; i++) {
-      var field = target.groupByFields[i].field;
-      var aggs = {terms: {field: field}};
 
       nestedAggs.aggs = {};
-      nestedAggs.aggs[field] = aggs;
+      nestedAggs.aggs['b' + i] = esAgg;
-      nestedAggs = aggs;
+      nestedAggs = esAgg;
     }
 
     nestedAggs.aggs = {};
 
-    for (i = 0; i < target.select.length; i++) {
+    for (i = 0; i < target.metrics.length; i++) {
-      var select = target.select[i];
+      var metric = target.metrics[i];
-      if (select.field) {
+      if (metric.field) {
         var aggField = {};
-        aggField[select.agg] = {field: select.field};
+        aggField[metric.agg] = {field: metric.field};
 
-        nestedAggs.aggs[i.toString()] = aggField;
+        nestedAggs.aggs['m' + i] = aggField;
       }
     }
 

public/app/plugins/datasource/elasticsearch/queryComponent.js

@@ -0,0 +1,60 @@
+define([
+  'angular',
+  'lodash',
+  'jquery',
+],
+function (angular, _, $) {
+  'use strict';
+
+  angular
+    .module('grafana.directives')
+    .directive('elasticQueryComponent', function($compile, uiSegmentSrv, $q) {
+
+      //var linkTemplate = '<a class="tight-form-item tabindex="1" ng-bind-html="textRep"></a>';
+      /* jshint maxlen:false */
+      var template1 = '<metric-segment segment="typeSegment" get-alt-segments="getBucketAggTypes()" on-value-changed="bucketAggTypeChanged()"></metric-segment>';
+      /* jshint maxlen:false */
+      var template2 = '<metric-segment segment="fieldSegment" get-alt-segments="getFields()" on-value-changed="fieldChanged()"></metric-segment>';
+
+      return {
+        restrict: 'E',
+        scope: {
+          model: "=",
+          onChange: "&",
+          getFields: "&",
+        },
+        link: function postLink($scope, elem) {
+
+          $scope.getBucketAggTypes = function() {
+            return $q.when([
+              uiSegmentSrv.newSegment({value: 'terms'}),
+              uiSegmentSrv.newSegment({value: 'date_histogram'}),
+            ]);
+          };
+
+          $scope.fieldChanged = function() {
+            $scope.model.field = $scope.fieldSegment.value;
+            $scope.onChange();
+          };
+
+          $scope.bucketAggTypeChanged = function() {
+            $scope.model.type = $scope.typeSegment.value;
+            $scope.onChange();
+          };
+
+          function addElementsAndCompile() {
+            var $html = $(template1 + template2);
+
+            $scope.fieldSegment = uiSegmentSrv.newSegment($scope.model.field);
+            $scope.typeSegment = uiSegmentSrv.newSegment($scope.model.type);
+
+            $html.appendTo(elem);
+
+            $compile(elem.contents())($scope);
+          }
+
+          addElementsAndCompile();
+        }
+      };
+    });
+});

public/app/plugins/datasource/elasticsearch/queryCtrl.js

@@ -15,17 +15,22 @@ function (angular, _, ElasticQueryBuilder) {
       if (!target) { return; }
 
       target.timeField = target.timeField || '@timestamp';
-      target.select = target.select || [{ agg: 'count' }];
+      target.metrics = target.metrics || [{ agg: 'count' }];
-      target.groupByFields = target.groupByFields || [];
+      target.bucketAggs = target.bucketAggs || [];
+      target.bucketAggs = [
+        {
+          type: 'terms',
+          field: '@hostname'
+        },
+        {
+          type: 'date_histogram',
+          field: '@timestamp'
+        },
+      ];
 
       $scope.timeSegment = uiSegmentSrv.newSegment(target.timeField);
 
-      $scope.groupBySegments = _.map(target.groupByFields, function(group) {
-        return uiSegmentSrv.newSegment(group.field);
-      });
-
       $scope.initSelectSegments();
-      $scope.groupBySegments.push(uiSegmentSrv.newPlusButton());
       $scope.removeSelectSegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove select --'});
       $scope.resetSelectSegment = uiSegmentSrv.newSegment({fake: true, value: '-- reset --'});
       $scope.removeGroupBySegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove group by --'});
@@ -36,7 +41,7 @@ function (angular, _, ElasticQueryBuilder) {
 
     $scope.initSelectSegments = function() {
       $scope.selectSegments = [];
-      _.each($scope.target.select, function(select) {
+      _.each($scope.target.metrics, function(select) {
         if ($scope.selectSegments.length > 0) {
           $scope.selectSegments.push(uiSegmentSrv.newCondition(" and "));
         }
@@ -55,9 +60,10 @@ function (angular, _, ElasticQueryBuilder) {
       if (segment.type === 'agg' || segment.type === 'plus-button') {
         var options = [
           uiSegmentSrv.newSegment({value: 'count', type: 'agg'}),
+          uiSegmentSrv.newSegment({value: 'avg',   type: 'agg', reqField: true}),
+          uiSegmentSrv.newSegment({value: 'sum',   type: 'agg', reqField: true}),
           uiSegmentSrv.newSegment({value: 'min',   type: 'agg', reqField: true}),
           uiSegmentSrv.newSegment({value: 'max',   type: 'agg', reqField: true}),
-          uiSegmentSrv.newSegment({value: 'avg',   type: 'agg', reqField: true}),
         ];
         // if we have other selects and this is not a plus button add remove option
         if (segment.type !== 'plus-button' && $scope.selectSegments.length > 3) {
@@ -78,7 +84,7 @@ function (angular, _, ElasticQueryBuilder) {
     $scope.selectChanged = function(segment, index) {
       // reset
       if (segment.value === $scope.resetSelectSegment.value) {
-        $scope.target.select = [{ agg: 'count' }];
+        $scope.target.metrics = [{ agg: 'count' }];
         $scope.initSelectSegments();
         $scope.queryUpdated();
         return;
@@ -125,7 +131,7 @@ function (angular, _, ElasticQueryBuilder) {
     };
 
     $scope.rebuildTargetSelects = function() {
-      $scope.target.select = [];
+      $scope.target.metrics = [];
       for (var i = 0; i < $scope.selectSegments.length; i++) {
         var segment = $scope.selectSegments[i];
         var select = {agg: segment.value };
@@ -138,7 +144,7 @@ function (angular, _, ElasticQueryBuilder) {
         }
 
         if (select.field === 'select field') { continue; }
-        $scope.target.select.push(select);
+        $scope.target.metrics.push(select);
       }
     };
 
@@ -154,7 +160,7 @@ function (angular, _, ElasticQueryBuilder) {
       .then(null, $scope.handleQueryError);
     };
 
-    $scope.getTimeFields = function() {
+    $scope.getFields = function() {
       return $scope.datasource.metricFindQuery('fields()')
       .then($scope.transformToSegments(false))
       .then(null, $scope.handleQueryError);
@@ -165,22 +171,20 @@ function (angular, _, ElasticQueryBuilder) {
       $scope.queryUpdated();
     };
 
-    $scope.groupByChanged = function(segment, index) {
+    $scope.addBucketAgg = function() {
-      if (segment.value === $scope.removeGroupBySegment.value) {
+      // if last is date histogram add it before
-        $scope.target.groupByFields.splice(index, 1);
+      var lastBucket = $scope.target.bucketAggs[$scope.target.bucketAggs.length - 1];
-        $scope.groupBySegments.splice(index, 1);
+      var addIndex = $scope.target.bucketAggs.length - 1;
-        $scope.queryUpdated();
+
-        return;
+      if (lastBucket && lastBucket.type === 'date_histogram') {
+        addIndex - 1;
       }
 
-      if (index === $scope.groupBySegments.length-1) {
+      $scope.target.bucketAggs.splice(addIndex, 0, {type: "terms", field: "select field" });
-        $scope.groupBySegments.push(uiSegmentSrv.newPlusButton());
+    };
-      }
 
-      segment.type = 'group-by-key';
+    $scope.removeBucketAgg = function(index) {
-      segment.fake = false;
+      $scope.target.bucketAggs.splice(index, 1);
-
-      $scope.target.groupByFields[index] = {field: segment.value};
       $scope.queryUpdated();
     };
 

public/test/specs/elasticsearch-querybuilder-specs.js

@@ -9,25 +9,42 @@ define([
       var builder = new ElasticQueryBuilder();
 
       var query = builder.build({
+        metrics: [{agg: 'Count'}],
         timeField: '@timestamp',
-        select: [{agg: 'Count'}],
+        bucketAggs: [{type: 'date_histogram', field: '@timestamp'}],
-        groupByFields: [],
       });
 
       expect(query.query.filtered.filter.bool.must[0].range["@timestamp"].gte).to.be("$timeFrom");
-      expect(query.aggs.histogram.date_histogram.extended_bounds.min).to.be("$timeFrom");
+      expect(query.aggs["b0"].date_histogram.extended_bounds.min).to.be("$timeFrom");
     });
 
+    it('with multiple bucket aggs', function() {
+      var builder = new ElasticQueryBuilder();
+
+      var query = builder.build({
+        metrics: [{agg: 'Count'}],
+        timeField: '@timestamp',
+        bucketAggs: [
+          {type: 'terms', field: '@host'},
+          {type: 'date_histogram', field: '@timestamp'}
+        ],
+      });
+
+      expect(query.aggs["b0"].terms.field).to.be("@host");
+      expect(query.aggs["b0"].aggs["b1"].date_histogram.field).to.be("@timestamp");
+    });
+
+
     it('with select field', function() {
       var builder = new ElasticQueryBuilder();
 
       var query = builder.build({
-        select: [{agg: 'avg', field: '@value'}],
+        metrics: [{agg: 'avg', field: '@value'}],
-        groupByFields: [],
+        bucketAggs: [{type: 'date_histogram', field: '@timestamp'}],
       }, 100, 1000);
 
-      var aggs = query.aggs.histogram.aggs;
+      var aggs = query.aggs["b0"].aggs;
-      expect(aggs["0"].avg.field).to.be("@value");
+      expect(aggs["m0"].avg.field).to.be("@value");
     });
 
 

public/test/specs/elasticsearch-queryctrl-specs.js

@@ -33,7 +33,7 @@ define([
       describe('initSelectSegments with 2 selects', function() {
 
         it('init selectSegments', function() {
-          ctx.scope.target.select = [
+          ctx.scope.target.metrics = [
             {agg: 'count'},
             {agg: 'avg', field: 'value'},
           ];

public/test/specs/elasticsearch-specs.js

@@ -23,12 +23,12 @@ define([
         beforeEach(function() {
           result = ctx.ds._processTimeSeries([{
             refId: 'A',
-            select: [{agg: 'count'}],
+            metrics: [{agg: 'count'}],
-            groupByFields: [],
+            bucketAggs: [{type: 'date_histogram', field: '@timestamp'}],
           }], {
             responses: [{
               aggregations: {
-                histogram: {
+                "b0": {
                   buckets: [
                     {
                       doc_count: 10,
@@ -60,20 +60,20 @@ define([
         beforeEach(function() {
           result = ctx.ds._processTimeSeries([{
             refId: 'A',
-            select: [{agg: 'count'}, {agg: 'avg', field: 'value'}],
+            metrics: [{agg: 'count'}, {agg: 'avg', field: 'value'}],
-            groupByFields: [],
+            bucketAggs: [{type: 'date_histogram', field: '@timestamp'}],
           }], {
             responses: [{
               aggregations: {
-                histogram: {
+                "b0": {
                   buckets: [
                     {
-                      "1": {value: 88},
+                      "m1": {value: 88},
                       doc_count: 10,
                       key: 1000
                     },
                     {
-                      "1": {value: 99},
+                      "m1": {value: 99},
                       doc_count: 15,
                       key: 2000
                     }
@@ -103,33 +103,33 @@ define([
         beforeEach(function() {
           result = ctx.ds._processTimeSeries([{
             refId: 'A',
-            select: [{agg: 'count'}],
+            metrics: [{agg: 'count'}],
-            groupByFields: [{field: 'host' }]
+            bucketAggs: [{type: 'terms', field: 'host'}, {type: 'date_histogram', field: '@timestamp'}],
           }], {
             responses: [{
               aggregations: {
-                histogram: {
+                "b0": {
                   buckets: [
                     {
-                      host: {
+                      "b1": {
                         buckets: [
-                           {doc_count: 4, key: 'server1'},
+                          {doc_count: 1, key: 1000},
-                           {doc_count: 6, key: 'server2'},
+                          {doc_count: 3, key: 2000}
+                        ]
+                      },
+                      doc_count: 4,
+                      key: 'server1',
+                    },
+                    {
+                      "b1": {
+                        buckets: [
+                          {doc_count: 2, key: 1000},
+                          {doc_count: 8, key: 2000}
                         ]
                       },
                       doc_count: 10,
-                      key: 1000
+                      key: 'server2',
                     },
-                    {
-                      host: {
-                        buckets: [
-                           {doc_count: 4, key: 'server1'},
-                           {doc_count: 6, key: 'server2'},
-                        ]
-                      },
-                      doc_count: 15,
-                      key: 2000
-                    }
                   ]
                 }
               }
@@ -145,90 +145,6 @@ define([
         });
       });
 
-      describe('group by query 2 fields', function() {
-        var result;
-
-        beforeEach(function() {
-          result = ctx.ds._processTimeSeries([{
-              refId: 'A',
-              select: [{agg: 'count'}],
-              groupByFields: [{field: 'host'}, {field: 'site'}]
-            }], {
-            responses: [{
-              aggregations: {
-                histogram: {
-                  buckets: [
-                    {
-                      host: {
-                        buckets: [
-                           {
-                             site: {
-                               buckets: [
-                                 {doc_count: 3, key: 'backend'},
-                                 {doc_count: 1, key: 'frontend'},
-                               ],
-                             },
-                             doc_count: 4, key: 'server1'
-                           },
-                           {
-                             site: {
-                               buckets: [
-                                 {doc_count: 3, key: 'backend'},
-                                 {doc_count: 1, key: 'frontend'},
-                               ],
-                             },
-                             doc_count: 6, key: 'server2'
-                           },
-                        ]
-                      },
-                      doc_count: 10,
-                      key: 1000
-                    },
-                    {
-                      host: {
-                        buckets: [
-                          {
-                            site: {
-                               buckets: [
-                                 {doc_count: 3, key: 'backend'},
-                                 {doc_count: 1, key: 'frontend'},
-                               ],
-                            },
-                            doc_count: 4,
-                            key: 'server1'
-                          },
-                          {
-                            site: {
-                               buckets: [
-                                 {doc_count: 3, key: 'backend'},
-                                 {doc_count: 1, key: 'frontend'},
-                               ],
-                            },
-                            doc_count: 6,
-                            key: 'server2'
-                          },
-                        ]
-                      },
-                      doc_count: 15,
-                      key: 2000
-                    }
-                  ]
-                }
-              }
-            }]
-          });
-        });
-
-        it('should return 2 series', function() {
-          expect(result.data.length).to.be(4);
-          expect(result.data[0].datapoints.length).to.be(2);
-          expect(result.data[0].target).to.be('A server1 backend count');
-          expect(result.data[1].target).to.be('A server1 frontend count');
-          expect(result.data[2].target).to.be('A server2 backend count');
-          expect(result.data[3].target).to.be('A server2 frontend count');
-        });
-      });
-
     });
   });
 });