IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

feat(elasticsearch): lots of work on elasticsearch metrics processing, handling grouped responses, etc, #1034

Browse Source
This commit is contained in:
Torkel Ödegaard
2015-09-03 11:14:25 +02:00
parent d099d8950f
commit df1d56e7b1
7 changed files with 255 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
public/app/plugins/datasource/elasticsearch/datasource.js
View File

@@ -13,7 +13,7 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
var module = angular.module('grafana.services');
module.factory('ElasticDatasource', function($q, backendSrv, templateSrv) {
module.factory('ElasticDatasource', function($q, backendSrv, templateSrv, timeSrv) {
function ElasticDatasource(datasource) {
this.type = 'elasticsearch';
@@ -160,7 +160,7 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
payload = payload.replace(/\$maxDataPoints/g, options.maxDataPoints);
payload = templateSrv.replace(payload, options.scopedVars);
var processTimeSeries = _.partial(this._processTimeSeries, sentTargets);
var processTimeSeries = _.bind(this._processTimeSeries, this, sentTargets);
return this._post('/_msearch?search_type=count', payload).then(processTimeSeries);
};
@@ -172,54 +172,109 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
return date.getTime();
};
ElasticDatasource._aggToSeries = function(agg) {
var datapoints = agg.date_histogram.buckets.map(function(entry) {
return [entry.stats.avg, entry.key];
});
return { target: agg.key, datapoints: datapoints };
};
ElasticDatasource.prototype._processBuckets = function(buckets, groupByFields, series, level, parentName, parentTime) {
var points = [];
var groupBy = groupByFields[level];
for (var i = 0; i < buckets.length; i++) {
var bucket = buckets[i];
if (groupBy) {
var seriesName = "";
var time = parentTime || bucket.key;
this._processBuckets(bucket[groupBy.field].buckets, groupByFields, series, level+1, seriesName, time)
} else {
var seriesName = parentName;
if (level > 0) {
seriesName += bucket.key;
} else {
parentTime = bucket.key;
}
var serie = series[seriesName] = series[seriesName] || {target: seriesName, datapoints: []};
serie.datapoints.push([bucket.doc_count, parentTime]);
}
}
};
ElasticDatasource.prototype._processTimeSeries = function(targets, results) {
var series = [];
_.each(results.responses, function(response, index) {
var buckets = response.aggregations.date_histogram.buckets;
var target = targets[index];
for (var i = 0; i < results.responses.length; i++) {
var buckets = results.responses[i].aggregations.histogram.buckets;
var target = targets[i];
var points = [];
var querySeries = {}
for (var i = 0; i < buckets.length; i++) {
var bucket = buckets[i];
points[i] = [bucket.doc_count, bucket.key];
}
this._processBuckets(buckets, target.groupByFields, querySeries, 0, target.refId);
series.push({target: 'name', datapoints: points})
console.log('Nr DataPoints: ' + points.length);
});
console.log(series);
_.each(querySeries, function(value) {
series.push(value);
});
};
return { data: series };
};
ElasticDatasource.prototype.metricFindQuery = function(query) {
var region;
var namespace;
var metricName;
var timeFrom = this.translateTime(timeSrv.time.from);
var timeTo = this.translateTime(timeSrv.time.to);
var transformSuggestData = function(suggestData) {
return _.map(suggestData, function(v) {
return { text: v };
});
var query = {
size: 10,
"query": {
"filtered": {
"filter": {
"bool": {
"must": [
{
"range": {
"@timestamp": {
"gte": timeFrom,
"lte": timeTo
}
}
}
],
}
}
}
}
};
var d = $q.defer();
return this._post('/_search?', query).then(function(res) {
var fields = {};
var regionQuery = query.match(/^region\(\)/);
if (regionQuery) {
d.resolve(transformSuggestData(this.performSuggestRegion()));
return d.promise;
}
for (var i = 0; i < res.hits.hits.length; i++) {
var hit = res.hits.hits[i];
for (var field in hit) {
if (hit.hasOwnProperty(field) && field[0] !== '_') {
fields[field] = 1;
}
}
if (hit._source) {
for (var field in hit._source) {
if (hit._source.hasOwnProperty(field)) {
fields[field] = 1;
}
}
}
}
fields = _.map(_.keys(fields), function(field) {
return {text: field};
})
console.log('metricFindQuery:', fields);
return fields;
});
// var d = $q.defer();
//
// var fieldsQuery = query.match(/^fields\(\)/);
// if (fieldsQuery) {
// return d.promise;
// }
};
return ElasticDatasource;

5
public/app/plugins/datasource/elasticsearch/partials/query.editor.html
View File

@@ -77,11 +77,10 @@
<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
Group By
</li>
<li>
<metric-segment segment="groupByFieldSegment" on-value-changed="groupByFieldChanged()"></metric-segment>
<li ng-repeat="segment in groupBySegments">
<metric-segment segment="segment" get-alt-segments="getGroupByFields(segment, $index)" on-value-changed="groupByChanged(segment, $index)"></metric-segment>
</li>
</ul>
<div class="clearfix"></div>
</div>

15
public/app/plugins/datasource/elasticsearch/queryBuilder.js
View File

@@ -42,7 +42,7 @@ function () {
};
query.aggs = {
"date_histogram": {
"histogram": {
"date_histogram": {
"interval": target.interval || "$interval",
"field": target.timeField,
@@ -55,6 +55,19 @@ function () {
}
};
var nestedAggs = {};
if (target.groupByFields && target.groupByFields.length > 0) {
var field = target.groupByFields[0].field;
nestedAggs[field] = {
terms: {
field: field,
}
}
}
query.aggs.histogram.aggs = nestedAggs;
query = JSON.stringify(query);
return query;
};

56
public/app/plugins/datasource/elasticsearch/queryCtrl.js
View File

@@ -23,18 +23,25 @@ function (angular, _, ElasticQueryBuilder) {
target.function = target.function || 'mean';
target.timeField = target.timeField || '@timestamp';
target.select = target.select || [{ agg: 'count' }];
target.groupByFields = target.groupByFields || [];
$scope.timeSegment = uiSegmentSrv.newSegment(target.timeField);
$scope.groupByFieldSegment = uiSegmentSrv.getSegmentForValue(target.groupByField, 'add group by');
$scope.groupBySegments = _.map(target.groupByFields, function(group) {
return uiSegmentSrv.newSegment(group.field);
});
$scope.selectSegments = _.map(target.select, function(select) {
return uiSegmentSrv.newSegment(select.agg);
});
$scope.groupBySegments.push(uiSegmentSrv.newPlusButton());
$scope.removeSelectSegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove select --'});
$scope.removeGroupBySegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove group by --'});
};
$scope.getFields = function() {
return $scope.datasource.metricFindQuery('fields()')
.then($scope.transformToSegments(true));
return $scope.datasource.metricFindQuery('fields()').then($scope.transformToSegments(true));
};
$scope.transformToSegments = function(addTemplateVars) {
@@ -53,6 +60,36 @@ function (angular, _, ElasticQueryBuilder) {
};
};
$scope.getGroupByFields = function(segment) {
return $scope.datasource.metricFindQuery('fields()').then($scope.transformToSegments(false))
.then(function(results) {
if (segment.type !== 'plus-button') {
results.splice(0, 0, angular.copy($scope.removeGroupBySegment));
}
return results;
})
.then(null, $scope.handleQueryError);
};
$scope.groupByChanged = function(segment, index) {
if (segment.value === $scope.removeGroupBySegment.value) {
$scope.target.groupByFields.splice(index, 1);
$scope.groupBySegments.splice(index, 1);
$scope.$parent.get_data();
return;
}
if (index === $scope.groupBySegments.length-1) {
$scope.groupBySegments.push(uiSegmentSrv.newPlusButton());
}
segment.type = 'group-by-key';
segment.fake = false;
$scope.target.groupByFields[index] = {field: segment.value};
$scope.$parent.get_data();
};
$scope.valueFieldChanged = function() {
$scope.target.valueField = $scope.valueFieldSegment.value;
$scope.$parent.get_data();
@@ -88,19 +125,6 @@ function (angular, _, ElasticQueryBuilder) {
return [];
};
$scope.transformToSegments = function(results) {
return _.map(results, function(segment) {
return new MetricSegment({ value: segment.text, expandable: segment.expandable });
});
};
$scope.addTemplateVariableSegments = function(segments) {
_.each(templateSrv.variables, function(variable) {
segments.unshift(new MetricSegment({ type: 'template', value: '$' + variable.name, expandable: true }));
});
return segments;
};
$scope.toggleQueryMode = function () {
$scope.target.rawQuery = !$scope.target.rawQuery;
};