From cc1e3d0101c225f41e81691ba1842e22c89f0ca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Fri, 4 Sep 2015 16:05:47 +0200 Subject: [PATCH] feat(elasticsearch): groundwork for a much more sophisticated elasticsearch query editor --- .../datasource/elasticsearch/datasource.js | 35 ++--- .../datasource/elasticsearch/directives.js | 1 + .../elasticsearch/partials/query.editor.html | 59 ++++---- .../datasource/elasticsearch/queryBuilder.js | 56 ++++---- .../elasticsearch/queryComponent.js | 60 ++++++++ .../datasource/elasticsearch/queryCtrl.js | 56 ++++---- .../specs/elasticsearch-querybuilder-specs.js | 31 +++- .../specs/elasticsearch-queryctrl-specs.js | 2 +- public/test/specs/elasticsearch-specs.js | 134 ++++-------------- 9 files changed, 218 insertions(+), 216 deletions(-) create mode 100644 public/app/plugins/datasource/elasticsearch/queryComponent.js diff --git a/public/app/plugins/datasource/elasticsearch/datasource.js b/public/app/plugins/datasource/elasticsearch/datasource.js index 1f194ad0284..be503c02cc5 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.js +++ b/public/app/plugins/datasource/elasticsearch/datasource.js @@ -174,39 +174,32 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) { // This is quite complex // neeed to recurise down the nested buckets to build series - ElasticDatasource.prototype._processBuckets = function(buckets, target, series, level, parentName, parentTime) { - var groupBy = target.groupByFields[level]; - var seriesName, time, value, select, i, y, bucket; + ElasticDatasource.prototype._processBuckets = function(buckets, target, series, level, parentName) { + var seriesName, value, metric, i, y, bucket, childBucket; for (i = 0; i < buckets.length; i++) { bucket = buckets[i]; + childBucket = bucket['b' + level]; - if (groupBy) { - seriesName = level > 0 ? parentName + ' ' + bucket.key : parentName; - time = parentTime || bucket.key; - this._processBuckets(bucket[groupBy.field].buckets, target, series, level+1, seriesName, time); + if (childBucket && childBucket.buckets) { + seriesName = parentName + ' ' + bucket.key; + this._processBuckets(childBucket.buckets, target, series, level+1, seriesName); } else { - for (y = 0; y < target.select.length; y++) { - select = target.select[y]; + for (y = 0; y < target.metrics.length; y++) { + metric = target.metrics[y]; seriesName = parentName; - if (level > 0) { - seriesName += ' ' + bucket.key; - } else { - parentTime = bucket.key; - } - - if (select.field) { - seriesName += ' ' + select.field + ' ' + select.agg; - value = bucket[y.toString()].value; + if (metric.field) { + seriesName += ' ' + metric.field + ' ' + metric.agg; + value = bucket['m' + y.toString()].value; } else { seriesName += ' count'; value = bucket.doc_count; } var serie = series[seriesName] = series[seriesName] || {target: seriesName, datapoints: []}; - serie.datapoints.push([value, parentTime]); + serie.datapoints.push([value, bucket.key]); } } } @@ -221,11 +214,11 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) { throw { message: response.error }; } - var buckets = response.aggregations.histogram.buckets; + var buckets = response.aggregations["b0"].buckets; var target = targets[i]; var querySeries = {}; - this._processBuckets(buckets, target, querySeries, 0, target.refId); + this._processBuckets(buckets, target, querySeries, 1, target.refId); for (var prop in querySeries) { if (querySeries.hasOwnProperty(prop)) { diff --git a/public/app/plugins/datasource/elasticsearch/directives.js b/public/app/plugins/datasource/elasticsearch/directives.js index 2d6825a70c7..ee6e4a77625 100644 --- a/public/app/plugins/datasource/elasticsearch/directives.js +++ b/public/app/plugins/datasource/elasticsearch/directives.js @@ -1,5 +1,6 @@ define([ 'angular', + './queryComponent', ], function (angular) { 'use strict'; diff --git a/public/app/plugins/datasource/elasticsearch/partials/query.editor.html b/public/app/plugins/datasource/elasticsearch/partials/query.editor.html index 0ecf582a50c..8b30fb7e6f4 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/query.editor.html +++ b/public/app/plugins/datasource/elasticsearch/partials/query.editor.html @@ -51,7 +51,7 @@ Time field
  • - +
    • @@ -62,28 +62,39 @@ -
    - +
    +
    +
      +
    • + Metrics +
      • +
    • + +
      • +
    -
    +
    +
    + +
    +
      +
    • + Group by + Then by +
      • +
    • + +
      • +
    + +
      +
    • + +
      • +
    • + +
      • +
    +
    +
    - -
    -
      -
    • - Group by -
      • -
    • - -
      • -
    -
    -
    - diff --git a/public/app/plugins/datasource/elasticsearch/queryBuilder.js b/public/app/plugins/datasource/elasticsearch/queryBuilder.js index 2c19f5db65d..39a67c4a415 100644 --- a/public/app/plugins/datasource/elasticsearch/queryBuilder.js +++ b/public/app/plugins/datasource/elasticsearch/queryBuilder.js @@ -17,6 +17,7 @@ function (angular) { return angular.fromJson(target.rawQuery); } + var i, nestedAggs; var query = { "size": 0, "query": { @@ -36,43 +37,42 @@ function (angular) { } }; - query.aggs = { - "histogram": { - "date_histogram": { - "interval": target.interval || "$interval", - "field": target.timeField, - "min_doc_count": 0, - "extended_bounds": { - "min": "$timeFrom", - "max": "$timeTo" - } + nestedAggs = query; + + for (i = 0; i < target.bucketAggs.length; i++) { + var aggDef = target.bucketAggs[i]; + var esAgg = {}; + + switch(aggDef.type) { + case 'date_histogram': { + esAgg["date_histogram"] = { + "interval": target.interval || "$interval", + "field": aggDef.field, + "min_doc_count": 0, + "extended_bounds": { "min": "$timeFrom", "max": "$timeTo" } + }; + break; } - }, - }; - - var nestedAggs = query.aggs.histogram; - var i; - - target.groupByFields = target.groupByFields || []; - - for (i = 0; i < target.groupByFields.length; i++) { - var field = target.groupByFields[i].field; - var aggs = {terms: {field: field}}; + case 'terms': { + esAgg["terms"] = { "field": aggDef.field }; + break; + } + } nestedAggs.aggs = {}; - nestedAggs.aggs[field] = aggs; - nestedAggs = aggs; + nestedAggs.aggs['b' + i] = esAgg; + nestedAggs = esAgg; } nestedAggs.aggs = {}; - for (i = 0; i < target.select.length; i++) { - var select = target.select[i]; - if (select.field) { + for (i = 0; i < target.metrics.length; i++) { + var metric = target.metrics[i]; + if (metric.field) { var aggField = {}; - aggField[select.agg] = {field: select.field}; + aggField[metric.agg] = {field: metric.field}; - nestedAggs.aggs[i.toString()] = aggField; + nestedAggs.aggs['m' + i] = aggField; } } diff --git a/public/app/plugins/datasource/elasticsearch/queryComponent.js b/public/app/plugins/datasource/elasticsearch/queryComponent.js new file mode 100644 index 00000000000..7d3b09a6e65 --- /dev/null +++ b/public/app/plugins/datasource/elasticsearch/queryComponent.js @@ -0,0 +1,60 @@ +define([ + 'angular', + 'lodash', + 'jquery', +], +function (angular, _, $) { + 'use strict'; + + angular + .module('grafana.directives') + .directive('elasticQueryComponent', function($compile, uiSegmentSrv, $q) { + + //var linkTemplate = ''; + /* jshint maxlen:false */ + var template1 = ''; + /* jshint maxlen:false */ + var template2 = ''; + + return { + restrict: 'E', + scope: { + model: "=", + onChange: "&", + getFields: "&", + }, + link: function postLink($scope, elem) { + + $scope.getBucketAggTypes = function() { + return $q.when([ + uiSegmentSrv.newSegment({value: 'terms'}), + uiSegmentSrv.newSegment({value: 'date_histogram'}), + ]); + }; + + $scope.fieldChanged = function() { + $scope.model.field = $scope.fieldSegment.value; + $scope.onChange(); + }; + + $scope.bucketAggTypeChanged = function() { + $scope.model.type = $scope.typeSegment.value; + $scope.onChange(); + }; + + function addElementsAndCompile() { + var $html = $(template1 + template2); + + $scope.fieldSegment = uiSegmentSrv.newSegment($scope.model.field); + $scope.typeSegment = uiSegmentSrv.newSegment($scope.model.type); + + $html.appendTo(elem); + + $compile(elem.contents())($scope); + } + + addElementsAndCompile(); + } + }; + }); +}); diff --git a/public/app/plugins/datasource/elasticsearch/queryCtrl.js b/public/app/plugins/datasource/elasticsearch/queryCtrl.js index 03c6ad4d66d..60191fc2620 100644 --- a/public/app/plugins/datasource/elasticsearch/queryCtrl.js +++ b/public/app/plugins/datasource/elasticsearch/queryCtrl.js @@ -15,17 +15,22 @@ function (angular, _, ElasticQueryBuilder) { if (!target) { return; } target.timeField = target.timeField || '@timestamp'; - target.select = target.select || [{ agg: 'count' }]; - target.groupByFields = target.groupByFields || []; + target.metrics = target.metrics || [{ agg: 'count' }]; + target.bucketAggs = target.bucketAggs || []; + target.bucketAggs = [ + { + type: 'terms', + field: '@hostname' + }, + { + type: 'date_histogram', + field: '@timestamp' + }, + ]; $scope.timeSegment = uiSegmentSrv.newSegment(target.timeField); - $scope.groupBySegments = _.map(target.groupByFields, function(group) { - return uiSegmentSrv.newSegment(group.field); - }); - $scope.initSelectSegments(); - $scope.groupBySegments.push(uiSegmentSrv.newPlusButton()); $scope.removeSelectSegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove select --'}); $scope.resetSelectSegment = uiSegmentSrv.newSegment({fake: true, value: '-- reset --'}); $scope.removeGroupBySegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove group by --'}); @@ -36,7 +41,7 @@ function (angular, _, ElasticQueryBuilder) { $scope.initSelectSegments = function() { $scope.selectSegments = []; - _.each($scope.target.select, function(select) { + _.each($scope.target.metrics, function(select) { if ($scope.selectSegments.length > 0) { $scope.selectSegments.push(uiSegmentSrv.newCondition(" and ")); } @@ -55,9 +60,10 @@ function (angular, _, ElasticQueryBuilder) { if (segment.type === 'agg' || segment.type === 'plus-button') { var options = [ uiSegmentSrv.newSegment({value: 'count', type: 'agg'}), + uiSegmentSrv.newSegment({value: 'avg', type: 'agg', reqField: true}), + uiSegmentSrv.newSegment({value: 'sum', type: 'agg', reqField: true}), uiSegmentSrv.newSegment({value: 'min', type: 'agg', reqField: true}), uiSegmentSrv.newSegment({value: 'max', type: 'agg', reqField: true}), - uiSegmentSrv.newSegment({value: 'avg', type: 'agg', reqField: true}), ]; // if we have other selects and this is not a plus button add remove option if (segment.type !== 'plus-button' && $scope.selectSegments.length > 3) { @@ -78,7 +84,7 @@ function (angular, _, ElasticQueryBuilder) { $scope.selectChanged = function(segment, index) { // reset if (segment.value === $scope.resetSelectSegment.value) { - $scope.target.select = [{ agg: 'count' }]; + $scope.target.metrics = [{ agg: 'count' }]; $scope.initSelectSegments(); $scope.queryUpdated(); return; @@ -125,7 +131,7 @@ function (angular, _, ElasticQueryBuilder) { }; $scope.rebuildTargetSelects = function() { - $scope.target.select = []; + $scope.target.metrics = []; for (var i = 0; i < $scope.selectSegments.length; i++) { var segment = $scope.selectSegments[i]; var select = {agg: segment.value }; @@ -138,7 +144,7 @@ function (angular, _, ElasticQueryBuilder) { } if (select.field === 'select field') { continue; } - $scope.target.select.push(select); + $scope.target.metrics.push(select); } }; @@ -154,7 +160,7 @@ function (angular, _, ElasticQueryBuilder) { .then(null, $scope.handleQueryError); }; - $scope.getTimeFields = function() { + $scope.getFields = function() { return $scope.datasource.metricFindQuery('fields()') .then($scope.transformToSegments(false)) .then(null, $scope.handleQueryError); @@ -165,22 +171,20 @@ function (angular, _, ElasticQueryBuilder) { $scope.queryUpdated(); }; - $scope.groupByChanged = function(segment, index) { - if (segment.value === $scope.removeGroupBySegment.value) { - $scope.target.groupByFields.splice(index, 1); - $scope.groupBySegments.splice(index, 1); - $scope.queryUpdated(); - return; + $scope.addBucketAgg = function() { + // if last is date histogram add it before + var lastBucket = $scope.target.bucketAggs[$scope.target.bucketAggs.length - 1]; + var addIndex = $scope.target.bucketAggs.length - 1; + + if (lastBucket && lastBucket.type === 'date_histogram') { + addIndex - 1; } - if (index === $scope.groupBySegments.length-1) { - $scope.groupBySegments.push(uiSegmentSrv.newPlusButton()); - } + $scope.target.bucketAggs.splice(addIndex, 0, {type: "terms", field: "select field" }); + }; - segment.type = 'group-by-key'; - segment.fake = false; - - $scope.target.groupByFields[index] = {field: segment.value}; + $scope.removeBucketAgg = function(index) { + $scope.target.bucketAggs.splice(index, 1); $scope.queryUpdated(); }; diff --git a/public/test/specs/elasticsearch-querybuilder-specs.js b/public/test/specs/elasticsearch-querybuilder-specs.js index edce5806f6c..c18b598c45b 100644 --- a/public/test/specs/elasticsearch-querybuilder-specs.js +++ b/public/test/specs/elasticsearch-querybuilder-specs.js @@ -9,25 +9,42 @@ define([ var builder = new ElasticQueryBuilder(); var query = builder.build({ + metrics: [{agg: 'Count'}], timeField: '@timestamp', - select: [{agg: 'Count'}], - groupByFields: [], + bucketAggs: [{type: 'date_histogram', field: '@timestamp'}], }); expect(query.query.filtered.filter.bool.must[0].range["@timestamp"].gte).to.be("$timeFrom"); - expect(query.aggs.histogram.date_histogram.extended_bounds.min).to.be("$timeFrom"); + expect(query.aggs["b0"].date_histogram.extended_bounds.min).to.be("$timeFrom"); }); + it('with multiple bucket aggs', function() { + var builder = new ElasticQueryBuilder(); + + var query = builder.build({ + metrics: [{agg: 'Count'}], + timeField: '@timestamp', + bucketAggs: [ + {type: 'terms', field: '@host'}, + {type: 'date_histogram', field: '@timestamp'} + ], + }); + + expect(query.aggs["b0"].terms.field).to.be("@host"); + expect(query.aggs["b0"].aggs["b1"].date_histogram.field).to.be("@timestamp"); + }); + + it('with select field', function() { var builder = new ElasticQueryBuilder(); var query = builder.build({ - select: [{agg: 'avg', field: '@value'}], - groupByFields: [], + metrics: [{agg: 'avg', field: '@value'}], + bucketAggs: [{type: 'date_histogram', field: '@timestamp'}], }, 100, 1000); - var aggs = query.aggs.histogram.aggs; - expect(aggs["0"].avg.field).to.be("@value"); + var aggs = query.aggs["b0"].aggs; + expect(aggs["m0"].avg.field).to.be("@value"); }); diff --git a/public/test/specs/elasticsearch-queryctrl-specs.js b/public/test/specs/elasticsearch-queryctrl-specs.js index ce4079af2fd..f8775529335 100644 --- a/public/test/specs/elasticsearch-queryctrl-specs.js +++ b/public/test/specs/elasticsearch-queryctrl-specs.js @@ -33,7 +33,7 @@ define([ describe('initSelectSegments with 2 selects', function() { it('init selectSegments', function() { - ctx.scope.target.select = [ + ctx.scope.target.metrics = [ {agg: 'count'}, {agg: 'avg', field: 'value'}, ]; diff --git a/public/test/specs/elasticsearch-specs.js b/public/test/specs/elasticsearch-specs.js index 58db123846d..84e25a67814 100644 --- a/public/test/specs/elasticsearch-specs.js +++ b/public/test/specs/elasticsearch-specs.js @@ -23,12 +23,12 @@ define([ beforeEach(function() { result = ctx.ds._processTimeSeries([{ refId: 'A', - select: [{agg: 'count'}], - groupByFields: [], + metrics: [{agg: 'count'}], + bucketAggs: [{type: 'date_histogram', field: '@timestamp'}], }], { responses: [{ aggregations: { - histogram: { + "b0": { buckets: [ { doc_count: 10, @@ -60,20 +60,20 @@ define([ beforeEach(function() { result = ctx.ds._processTimeSeries([{ refId: 'A', - select: [{agg: 'count'}, {agg: 'avg', field: 'value'}], - groupByFields: [], + metrics: [{agg: 'count'}, {agg: 'avg', field: 'value'}], + bucketAggs: [{type: 'date_histogram', field: '@timestamp'}], }], { responses: [{ aggregations: { - histogram: { + "b0": { buckets: [ { - "1": {value: 88}, + "m1": {value: 88}, doc_count: 10, key: 1000 }, { - "1": {value: 99}, + "m1": {value: 99}, doc_count: 15, key: 2000 } @@ -103,33 +103,33 @@ define([ beforeEach(function() { result = ctx.ds._processTimeSeries([{ refId: 'A', - select: [{agg: 'count'}], - groupByFields: [{field: 'host' }] + metrics: [{agg: 'count'}], + bucketAggs: [{type: 'terms', field: 'host'}, {type: 'date_histogram', field: '@timestamp'}], }], { responses: [{ aggregations: { - histogram: { + "b0": { buckets: [ { - host: { + "b1": { buckets: [ - {doc_count: 4, key: 'server1'}, - {doc_count: 6, key: 'server2'}, + {doc_count: 1, key: 1000}, + {doc_count: 3, key: 2000} + ] + }, + doc_count: 4, + key: 'server1', + }, + { + "b1": { + buckets: [ + {doc_count: 2, key: 1000}, + {doc_count: 8, key: 2000} ] }, doc_count: 10, - key: 1000 + key: 'server2', }, - { - host: { - buckets: [ - {doc_count: 4, key: 'server1'}, - {doc_count: 6, key: 'server2'}, - ] - }, - doc_count: 15, - key: 2000 - } ] } } @@ -145,90 +145,6 @@ define([ }); }); - describe('group by query 2 fields', function() { - var result; - - beforeEach(function() { - result = ctx.ds._processTimeSeries([{ - refId: 'A', - select: [{agg: 'count'}], - groupByFields: [{field: 'host'}, {field: 'site'}] - }], { - responses: [{ - aggregations: { - histogram: { - buckets: [ - { - host: { - buckets: [ - { - site: { - buckets: [ - {doc_count: 3, key: 'backend'}, - {doc_count: 1, key: 'frontend'}, - ], - }, - doc_count: 4, key: 'server1' - }, - { - site: { - buckets: [ - {doc_count: 3, key: 'backend'}, - {doc_count: 1, key: 'frontend'}, - ], - }, - doc_count: 6, key: 'server2' - }, - ] - }, - doc_count: 10, - key: 1000 - }, - { - host: { - buckets: [ - { - site: { - buckets: [ - {doc_count: 3, key: 'backend'}, - {doc_count: 1, key: 'frontend'}, - ], - }, - doc_count: 4, - key: 'server1' - }, - { - site: { - buckets: [ - {doc_count: 3, key: 'backend'}, - {doc_count: 1, key: 'frontend'}, - ], - }, - doc_count: 6, - key: 'server2' - }, - ] - }, - doc_count: 15, - key: 2000 - } - ] - } - } - }] - }); - }); - - it('should return 2 series', function() { - expect(result.data.length).to.be(4); - expect(result.data[0].datapoints.length).to.be(2); - expect(result.data[0].target).to.be('A server1 backend count'); - expect(result.data[1].target).to.be('A server1 frontend count'); - expect(result.data[2].target).to.be('A server2 backend count'); - expect(result.data[3].target).to.be('A server2 frontend count'); - }); - }); - }); }); });