feat(elasticsearch): lots of work on elasticsearch metrics queries, #1034

public/app/plugins/datasource/elasticsearch/datasource.js

@@ -135,14 +135,33 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
     };
 
     ElasticDatasource.prototype.query = function(options) {
-      var queryBuilder = new ElasticQueryBuilder;
+      var queryBuilder = new ElasticQueryBuilder();
-      var query = queryBuilder.build(options.targets);
+      var header = '{"index":"' + this.index + '","search_type":"count","ignore_unavailable":true}'
-      query = query.replace(/\$interval/g, options.interval);
+      var payload = ""
-      query = query.replace(/\$rangeFrom/g, this.translateTime(options.range.from));
+      var sentTargets = [];
-      query = query.replace(/\$rangeTo/g, this.translateTime(options.range.to));
+      var timeFrom = this.translateTime(options.range.from);
-      query = query.replace(/\$maxDataPoints/g, options.maxDataPoints);
+      var timeTo = this.translateTime(options.range.to);
-      query = templateSrv.replace(query, options.scopedVars);
+
-      return this._post('/_search?search_type=count', query).then(this._getTimeSeries);
+      _.each(options.targets, function(target) {
+        if (target.hide) {
+          return;
+        }
+
+        var esQuery = queryBuilder.build(target, timeFrom, timeTo);
+        payload += header + '\n';
+        payload += esQuery + '\n';
+
+        sentTargets.push(target);
+      });
+
+      payload = payload.replace(/\$interval/g, options.interval);
+      payload = payload.replace(/\$rangeFrom/g, this.translateTime(options.range.from));
+      payload = payload.replace(/\$rangeTo/g, this.translateTime(options.range.to));
+      payload = payload.replace(/\$maxDataPoints/g, options.maxDataPoints);
+      payload = templateSrv.replace(payload, options.scopedVars);
+
+      var processTimeSeries = _.partial(this._processTimeSeries, sentTargets);
+      return this._post('/_msearch?search_type=count', payload).then(processTimeSeries);
     };
 
     ElasticDatasource.prototype.translateTime = function(date) {
@@ -150,31 +169,37 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
         return date;
       }
 
-      return date.toJSON();
+      return date.getTime();
     };
 
-    ElasticDatasource.prototype._getTimeSeries = function(results) {
+    ElasticDatasource._aggToSeries = function(agg) {
-      var _aggregation2timeSeries = function(aggregation) {
+      var datapoints = agg.date_histogram.buckets.map(function(entry) {
-        var datapoints = aggregation.date_histogram.buckets.map(function(entry) {
+        return [entry.stats.avg, entry.key];
-          return [entry.stats.avg, entry.key];
+      });
-        });
+      return { target: agg.key, datapoints: datapoints };
-        return { target: aggregation.key, datapoints: datapoints };
+    };
-      };
-      var data = [];
 
-      if (results && results.aggregations) {
+
-        for (var target in results.aggregations) {
+    ElasticDatasource.prototype._processTimeSeries = function(targets, results) {
-          if (!results.aggregations.hasOwnProperty(target)) {
+      var series = [];
-            continue;
+
-          }
+      _.each(results.responses, function(response, index) {
-          if (results.aggregations[target].date_histogram && results.aggregations[target].date_histogram.buckets) {
+        var buckets = response.aggregations.date_histogram.buckets;
-            data.push(_aggregation2timeSeries(results.aggregations[target]));
+        var target = targets[index];
-          } else if (results.aggregations[target].terms && results.aggregations[target].terms.buckets) {
+        var points = [];
-            [].push.apply(data, results.aggregations[target].terms.buckets.map(_aggregation2timeSeries));
+
-          }
+        for (var i = 0; i < buckets.length; i++) {
+          var bucket = buckets[i];
+          points[i] = [bucket.doc_count, bucket.key];
         }
-      }
+
-      return { data: data };
+        series.push({target: 'name', datapoints: points})
+        console.log('Nr DataPoints: ' + points.length);
+      });
+
+      console.log(series);
+
+      return { data: series };
     };
 
     ElasticDatasource.prototype.metricFindQuery = function(query) {

public/app/plugins/datasource/elasticsearch/partials/query.editor.html

@@ -37,12 +37,35 @@
 		</li>
 	</ul>
 
-	<ul class="tight-form-list">
+	<ul class="tight-form-list" ng-hide="target.rawQuery">
+		<li class="tight-form-item query-keyword" style="width: 75px">
+			Query
+		</li>
+		<li>
+			<input type="text" class="input-xlarge tight-form-input" ng-model="target.query" spellcheck='false' placeholder="Lucence query" ng-blur="get_data()">
+		</li>
 		<li class="tight-form-item query-keyword">
-			Timestamp field
+			Time field
 		</li>
 		<li>
-			<metric-segment segment="timestampSegment" get-alt-segments="getFields()" on-value-changed="timestampChanged()"></metric-segment>
+			<metric-segment segment="timeSegment" get-alt-segments="getFields()" on-value-changed="timestampChanged()"></metric-segment>
+		</li>
+	</ul>
+
+	<div class="clearfix"></div>
+
+	<div style="padding: 10px" ng-if="target.rawQuery">
+		<textarea ng-model="target.rawJson" rows="8" spellcheck="false" style="width: 100%; box-sizing: border-box;"></textarea>
+	</div>
+</div>
+
+<div class="tight-form" ng-hide="target.rawQuery">
+	<ul class="tight-form-list">
+		<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
+			Select
+		</li>
+		<li ng-repeat="segment in selectSegments">
+			<metric-segment segment="segment" get-alt-segments="getSelectSegments(segment, $index)"  on-value-changed="selectSegmentChanged()"></metric-segment>
 		</li>
 	</ul>
 
@@ -52,43 +75,6 @@
 <div class="tight-form" ng-hide="target.rawQuery">
 	<ul class="tight-form-list">
 		<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
-			Value field
-		</li>
-		<li>
-			<metric-segment segment="valueFieldSegment" get-alt-segments="getFields()"  on-value-changed="valueFieldChanged()"></metric-segment>
-		</li>
-	</ul>
-
-	<div class="clearfix"></div>
-</div>
-
-<div class="tight-form" ng-hide="target.rawQuery">
-	<ul class="tight-form-list">
-		<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
-			Term
-		</li>
-
-		<li>
-			<metric-segment segment="termKeySegment" on-value-changed="termKeySegmentChanged()"></metric-segment>
-		</li>
-
-		<li>
-			<span class="tight-form-item">:</span>
-		</li>
-
-		<li>
-			<metric-segment segment="termValueSegment" on-value-changed="termValueSegmentChanged()"></metric-segment>
-		</li>
-	</ul>
-	<div class="clearfix"></div>
-</div>
-
-<div class="tight-form" ng-hide="target.rawQuery">
-	<ul class="tight-form-list">
-		<li class="tight-form-item">
-			<i class="fa fa-eye invisible"></i>
-		</li>
-		<li class="tight-form-item query-keyword" style="width: 75px;">
 			Group By
 		</li>
 		<li>
@@ -98,5 +84,4 @@
 
 	<div class="clearfix"></div>
 </div>
-	</div>
+
-</div>

public/app/plugins/datasource/elasticsearch/queryBuilder.js

@@ -5,54 +5,56 @@ function () {
 
   function ElasticQueryBuilder() { }
 
-  ElasticQueryBuilder.prototype.build = function(targets) {
+  ElasticQueryBuilder.prototype.build = function(target, timeFrom, timeTo) {
+    if (target.rawQuery) {
+      return angular.fromJson(target.rawJson);
+    }
+
     var query = {
-      "aggs": {},
+      "size": 0,
-      "size": "$maxDataPoints"
+      "query": {
-    };
+        "filtered": {
-
+          "query": {
-    var self = this;
+            "query_string": {
-    targets.forEach(function(target, index) {
+              "analyze_wildcard": true,
-      if (target.hide) {
+              "query": target.query || '*' ,
-        return;
+            }
-      }
+          },
-
+          "filter": {
-      var esQuery = {
+            "bool": {
-        "filter": {
+              "must": [
-          "and": [
+                {
-            self._buildRangeFilter(target)
+                  "range": {
-          ]
+                    "@timestamp": {
-        },
+                      "gte": timeFrom,
-        "aggs": {
+                      "lte": timeTo
-          "date_histogram": {
+                    }
-            "date_histogram": {
+                  }
-              "interval": target.interval || "$interval",
-              "field": target.timestampField,
-              "min_doc_count": 0,
-            },
-            "aggs": {
-              "stats": {
-                "stats": {
-                  "field": target.valueField
                 }
-              }
+              ],
+              "must_not": [
+
+              ]
             }
           }
         }
-      };
-      if (target.groupByField) {
-        query["aggs"][target.termKey + "_" + target.termValue]["aggs"] = {
-          "terms": {
-            "terms": {
-              "field": target.groupByField
-            },
-            "aggs": query["aggs"][target.termKey + "_" + target.termValue]["aggs"]
-          }
-        };
       }
+    };
+
+    query.aggs = {
+      "date_histogram": {
+        "date_histogram": {
+          "interval": target.interval || "$interval",
+          "field": target.timeField,
+          "min_doc_count": 0,
+          "extended_bounds": {
+            "min": timeFrom,
+            "max": timeTo
+          }
+        }
+      }
+    };
 
-      query["aggs"]['query:' + index] = esQuery;
-    });
     query = JSON.stringify(query);
     return query;
   };

public/app/plugins/datasource/elasticsearch/queryCtrl.js

@@ -21,15 +21,15 @@ function (angular, _, ElasticQueryBuilder) {
 
       var target = $scope.target;
       target.function = target.function || 'mean';
-      target.timestampField = target.timestampField || '@timestamp';
+      target.timeField = target.timeField || '@timestamp';
-      target.valueField = target.valueField || 'value' ;
+      target.select = target.select || [{ agg: 'count' }];
 
-      $scope.timestampSegment = uiSegmentSrv.newSegment(target.timestampField);
+      $scope.timeSegment = uiSegmentSrv.newSegment(target.timeField);
-      $scope.valueFieldSegment = uiSegmentSrv.newSegment(target.valueField);
-
-      $scope.termKeySegment = uiSegmentSrv.getSegmentForValue(target.termKey, 'select term field');
-      $scope.termValueSegment = uiSegmentSrv.getSegmentForValue(target.termValue, 'select term value');
       $scope.groupByFieldSegment = uiSegmentSrv.getSegmentForValue(target.groupByField, 'add group by');
+
+      $scope.selectSegments = _.map(target.select, function(select) {
+        return uiSegmentSrv.newSegment(select.agg);
+      });
     };
 
     $scope.getFields = function() {
@@ -101,6 +101,10 @@ function (angular, _, ElasticQueryBuilder) {
       return segments;
     };
 
+    $scope.toggleQueryMode = function () {
+      $scope.target.rawQuery = !$scope.target.rawQuery;
+    };
+
     $scope.init();
 
   });