feat(elasticsearch): groundwork for a much more sophisticated elasticsearch query editor

public/app/plugins/datasource/elasticsearch/datasource.js

@@ -174,39 +174,32 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
 
     // This is quite complex
     // neeed to recurise down the nested buckets to build series
-    ElasticDatasource.prototype._processBuckets = function(buckets, target, series, level, parentName, parentTime) {
-      var groupBy = target.groupByFields[level];
-      var seriesName, time, value, select, i, y, bucket;
+    ElasticDatasource.prototype._processBuckets = function(buckets, target, series, level, parentName) {
+      var seriesName, value, metric, i, y, bucket, childBucket;
 
       for (i = 0; i < buckets.length; i++) {
         bucket = buckets[i];
+        childBucket = bucket['b' + level];
 
-        if (groupBy) {
-          seriesName = level > 0 ? parentName + ' ' + bucket.key : parentName;
-          time = parentTime || bucket.key;
-          this._processBuckets(bucket[groupBy.field].buckets, target, series, level+1, seriesName, time);
+        if (childBucket && childBucket.buckets) {
+          seriesName = parentName + ' ' + bucket.key;
+          this._processBuckets(childBucket.buckets, target, series, level+1, seriesName);
         } else {
 
-          for (y = 0; y < target.select.length; y++) {
-            select = target.select[y];
+          for (y = 0; y < target.metrics.length; y++) {
+            metric = target.metrics[y];
             seriesName = parentName;
 
-            if (level > 0) {
-              seriesName +=  ' ' + bucket.key;
-            } else {
-              parentTime = bucket.key;
-            }
-
-            if (select.field) {
-              seriesName += ' ' + select.field + ' ' + select.agg;
-              value = bucket[y.toString()].value;
+            if (metric.field) {
+              seriesName += ' ' + metric.field + ' ' + metric.agg;
+              value = bucket['m' + y.toString()].value;
             } else {
               seriesName += ' count';
               value = bucket.doc_count;
             }
 
             var serie = series[seriesName] = series[seriesName] || {target: seriesName, datapoints: []};
-            serie.datapoints.push([value, parentTime]);
+            serie.datapoints.push([value, bucket.key]);
           }
         }
       }
@@ -221,11 +214,11 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
           throw { message: response.error };
         }
 
-        var buckets = response.aggregations.histogram.buckets;
+        var buckets = response.aggregations["b0"].buckets;
         var target = targets[i];
         var querySeries = {};
 
-        this._processBuckets(buckets, target, querySeries, 0, target.refId);
+        this._processBuckets(buckets, target, querySeries, 1, target.refId);
 
         for (var prop in querySeries) {
           if (querySeries.hasOwnProperty(prop)) {

public/app/plugins/datasource/elasticsearch/directives.js

@@ -1,5 +1,6 @@
 define([
   'angular',
+  './queryComponent',
 ],
 function (angular) {
   'use strict';

public/app/plugins/datasource/elasticsearch/partials/query.editor.html

@@ -51,7 +51,7 @@
 			Time field
 		</li>
 		<li>
-			<metric-segment segment="timeSegment" get-alt-segments="getTimeFields()" on-value-changed="timeFieldChanged()"></metric-segment>
+			<metric-segment segment="timeSegment" get-alt-segments="getFields()" on-value-changed="timeFieldChanged()"></metric-segment>
 		</li>
 	</ul>
 
@@ -62,28 +62,39 @@
 	</div>
 </div>
 
-<div class="tight-form" ng-hide="target.rawQuery">
-	<ul class="tight-form-list">
-		<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
-			Select
-		</li>
-		<li ng-repeat="segment in selectSegments">
-			<metric-segment segment="segment" get-alt-segments="getSelectSegments(segment, $index)"  on-value-changed="selectChanged(segment, $index)"></metric-segment>
-		</li>
-	</ul>
+<div ng-hide="target.rawQuery">
+	<div class="tight-form">
+		<ul class="tight-form-list">
+			<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
+				Metrics
+			</li>
+			<li ng-repeat="segment in selectSegments">
+				<metric-segment segment="segment" get-alt-segments="getSelectSegments(segment, $index)"  on-value-changed="selectChanged(segment, $index)"></metric-segment>
+			</li>
+		</ul>
 
-	<div class="clearfix"></div>
+		<div class="clearfix"></div>
+	</div>
+
+	<div class="tight-form" ng-repeat="agg in target.bucketAggs">
+		<ul class="tight-form-list">
+			<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
+				<span ng-show="$first">Group by</span>
+				<span ng-show="!$first">Then by</span>
+			</li>
+			<li>
+				<elastic-query-component model="agg" get-fields="getFields()" on-change="queryUpdated()"></elastic-query-component>
+			</li>
+		</ul>
+
+		<ul class="tight-form-list pull-right">
+			<li class="tight-form-item" ng-if="$index === 0">
+				<a class="pointer" ng-click="addBucketAgg()"><i class="fa fa-plus"></i></a>
+			</li>
+			<li class="tight-form-item" ng-if="!$last">
+				<a class="pointer" ng-click="removeBucketAgg(agg, $index)"><i class="fa fa-minus"></i></a>
+			</li>
+		</ul>
+		<div class="clearfix"></div>
+	</div>
 </div>
-
-<div class="tight-form" ng-hide="target.rawQuery">
-	<ul class="tight-form-list">
-		<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
-			Group by
-		</li>
-		<li ng-repeat="segment in groupBySegments">
-			<metric-segment segment="segment" get-alt-segments="getGroupByFields(segment, $index)" on-value-changed="groupByChanged(segment, $index)"></metric-segment>
-		</li>
-	</ul>
-	<div class="clearfix"></div>
-</div>
-

public/app/plugins/datasource/elasticsearch/queryBuilder.js

@@ -17,6 +17,7 @@ function (angular) {
       return angular.fromJson(target.rawQuery);
     }
 
+    var i, nestedAggs;
     var query = {
       "size": 0,
       "query": {
@@ -36,43 +37,42 @@ function (angular) {
       }
     };
 
-    query.aggs = {
-      "histogram": {
-        "date_histogram": {
-          "interval": target.interval || "$interval",
-          "field": target.timeField,
-          "min_doc_count": 0,
-          "extended_bounds": {
-            "min": "$timeFrom",
-            "max": "$timeTo"
-          }
+    nestedAggs = query;
+
+    for (i = 0; i < target.bucketAggs.length; i++) {
+      var aggDef = target.bucketAggs[i];
+      var esAgg = {};
+
+      switch(aggDef.type) {
+        case 'date_histogram': {
+          esAgg["date_histogram"] = {
+            "interval": target.interval || "$interval",
+            "field": aggDef.field,
+            "min_doc_count": 0,
+            "extended_bounds": { "min": "$timeFrom", "max": "$timeTo" }
+          };
+          break;
         }
-      },
-    };
-
-    var nestedAggs = query.aggs.histogram;
-    var i;
-
-    target.groupByFields = target.groupByFields || [];
-
-    for (i = 0; i < target.groupByFields.length; i++) {
-      var field = target.groupByFields[i].field;
-      var aggs = {terms: {field: field}};
+        case 'terms': {
+          esAgg["terms"] = { "field": aggDef.field };
+          break;
+        }
+      }
 
       nestedAggs.aggs = {};
-      nestedAggs.aggs[field] = aggs;
-      nestedAggs = aggs;
+      nestedAggs.aggs['b' + i] = esAgg;
+      nestedAggs = esAgg;
     }
 
     nestedAggs.aggs = {};
 
-    for (i = 0; i < target.select.length; i++) {
-      var select = target.select[i];
-      if (select.field) {
+    for (i = 0; i < target.metrics.length; i++) {
+      var metric = target.metrics[i];
+      if (metric.field) {
         var aggField = {};
-        aggField[select.agg] = {field: select.field};
+        aggField[metric.agg] = {field: metric.field};
 
-        nestedAggs.aggs[i.toString()] = aggField;
+        nestedAggs.aggs['m' + i] = aggField;
       }
     }
 

public/app/plugins/datasource/elasticsearch/queryComponent.js

@@ -0,0 +1,60 @@
+define([
+  'angular',
+  'lodash',
+  'jquery',
+],
+function (angular, _, $) {
+  'use strict';
+
+  angular
+    .module('grafana.directives')
+    .directive('elasticQueryComponent', function($compile, uiSegmentSrv, $q) {
+
+      //var linkTemplate = '<a class="tight-form-item tabindex="1" ng-bind-html="textRep"></a>';
+      /* jshint maxlen:false */
+      var template1 = '<metric-segment segment="typeSegment" get-alt-segments="getBucketAggTypes()" on-value-changed="bucketAggTypeChanged()"></metric-segment>';
+      /* jshint maxlen:false */
+      var template2 = '<metric-segment segment="fieldSegment" get-alt-segments="getFields()" on-value-changed="fieldChanged()"></metric-segment>';
+
+      return {
+        restrict: 'E',
+        scope: {
+          model: "=",
+          onChange: "&",
+          getFields: "&",
+        },
+        link: function postLink($scope, elem) {
+
+          $scope.getBucketAggTypes = function() {
+            return $q.when([
+              uiSegmentSrv.newSegment({value: 'terms'}),
+              uiSegmentSrv.newSegment({value: 'date_histogram'}),
+            ]);
+          };
+
+          $scope.fieldChanged = function() {
+            $scope.model.field = $scope.fieldSegment.value;
+            $scope.onChange();
+          };
+
+          $scope.bucketAggTypeChanged = function() {
+            $scope.model.type = $scope.typeSegment.value;
+            $scope.onChange();
+          };
+
+          function addElementsAndCompile() {
+            var $html = $(template1 + template2);
+
+            $scope.fieldSegment = uiSegmentSrv.newSegment($scope.model.field);
+            $scope.typeSegment = uiSegmentSrv.newSegment($scope.model.type);
+
+            $html.appendTo(elem);
+
+            $compile(elem.contents())($scope);
+          }
+
+          addElementsAndCompile();
+        }
+      };
+    });
+});

public/app/plugins/datasource/elasticsearch/queryCtrl.js

@@ -15,17 +15,22 @@ function (angular, _, ElasticQueryBuilder) {
       if (!target) { return; }
 
       target.timeField = target.timeField || '@timestamp';
-      target.select = target.select || [{ agg: 'count' }];
-      target.groupByFields = target.groupByFields || [];
+      target.metrics = target.metrics || [{ agg: 'count' }];
+      target.bucketAggs = target.bucketAggs || [];
+      target.bucketAggs = [
+        {
+          type: 'terms',
+          field: '@hostname'
+        },
+        {
+          type: 'date_histogram',
+          field: '@timestamp'
+        },
+      ];
 
       $scope.timeSegment = uiSegmentSrv.newSegment(target.timeField);
 
-      $scope.groupBySegments = _.map(target.groupByFields, function(group) {
-        return uiSegmentSrv.newSegment(group.field);
-      });
-
       $scope.initSelectSegments();
-      $scope.groupBySegments.push(uiSegmentSrv.newPlusButton());
       $scope.removeSelectSegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove select --'});
       $scope.resetSelectSegment = uiSegmentSrv.newSegment({fake: true, value: '-- reset --'});
       $scope.removeGroupBySegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove group by --'});
@@ -36,7 +41,7 @@ function (angular, _, ElasticQueryBuilder) {
 
     $scope.initSelectSegments = function() {
       $scope.selectSegments = [];
-      _.each($scope.target.select, function(select) {
+      _.each($scope.target.metrics, function(select) {
         if ($scope.selectSegments.length > 0) {
           $scope.selectSegments.push(uiSegmentSrv.newCondition(" and "));
         }
@@ -55,9 +60,10 @@ function (angular, _, ElasticQueryBuilder) {
       if (segment.type === 'agg' || segment.type === 'plus-button') {
         var options = [
           uiSegmentSrv.newSegment({value: 'count', type: 'agg'}),
+          uiSegmentSrv.newSegment({value: 'avg',   type: 'agg', reqField: true}),
+          uiSegmentSrv.newSegment({value: 'sum',   type: 'agg', reqField: true}),
           uiSegmentSrv.newSegment({value: 'min',   type: 'agg', reqField: true}),
           uiSegmentSrv.newSegment({value: 'max',   type: 'agg', reqField: true}),
-          uiSegmentSrv.newSegment({value: 'avg',   type: 'agg', reqField: true}),
         ];
         // if we have other selects and this is not a plus button add remove option
         if (segment.type !== 'plus-button' && $scope.selectSegments.length > 3) {
@@ -78,7 +84,7 @@ function (angular, _, ElasticQueryBuilder) {
     $scope.selectChanged = function(segment, index) {
       // reset
       if (segment.value === $scope.resetSelectSegment.value) {
-        $scope.target.select = [{ agg: 'count' }];
+        $scope.target.metrics = [{ agg: 'count' }];
         $scope.initSelectSegments();
         $scope.queryUpdated();
         return;
@@ -125,7 +131,7 @@ function (angular, _, ElasticQueryBuilder) {
     };
 
     $scope.rebuildTargetSelects = function() {
-      $scope.target.select = [];
+      $scope.target.metrics = [];
       for (var i = 0; i < $scope.selectSegments.length; i++) {
         var segment = $scope.selectSegments[i];
         var select = {agg: segment.value };
@@ -138,7 +144,7 @@ function (angular, _, ElasticQueryBuilder) {
         }
 
         if (select.field === 'select field') { continue; }
-        $scope.target.select.push(select);
+        $scope.target.metrics.push(select);
       }
     };
 
@@ -154,7 +160,7 @@ function (angular, _, ElasticQueryBuilder) {
       .then(null, $scope.handleQueryError);
     };
 
-    $scope.getTimeFields = function() {
+    $scope.getFields = function() {
       return $scope.datasource.metricFindQuery('fields()')
       .then($scope.transformToSegments(false))
       .then(null, $scope.handleQueryError);
@@ -165,22 +171,20 @@ function (angular, _, ElasticQueryBuilder) {
       $scope.queryUpdated();
     };
 
-    $scope.groupByChanged = function(segment, index) {
-      if (segment.value === $scope.removeGroupBySegment.value) {
-        $scope.target.groupByFields.splice(index, 1);
-        $scope.groupBySegments.splice(index, 1);
-        $scope.queryUpdated();
-        return;
+    $scope.addBucketAgg = function() {
+      // if last is date histogram add it before
+      var lastBucket = $scope.target.bucketAggs[$scope.target.bucketAggs.length - 1];
+      var addIndex = $scope.target.bucketAggs.length - 1;
+
+      if (lastBucket && lastBucket.type === 'date_histogram') {
+        addIndex - 1;
       }
 
-      if (index === $scope.groupBySegments.length-1) {
-        $scope.groupBySegments.push(uiSegmentSrv.newPlusButton());
-      }
+      $scope.target.bucketAggs.splice(addIndex, 0, {type: "terms", field: "select field" });
+    };
 
-      segment.type = 'group-by-key';
-      segment.fake = false;
-
-      $scope.target.groupByFields[index] = {field: segment.value};
+    $scope.removeBucketAgg = function(index) {
+      $scope.target.bucketAggs.splice(index, 1);
       $scope.queryUpdated();
     };