IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2026-07-30 00:08:10 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'elastic5_support'

Browse Source
This commit is contained in:
bergquist
2016-12-09 11:42:15 +01:00
parent e58b6989c2 619c5c4f1b
commit cfb6033765
9 changed files with 204 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/blocks/elastic1/elasticsearch.yml
+2
View File
@@ -0,0 +1,2 @@
script.inline: on
script.indexed: on
docker/blocks/elastic1/fig
+8
View File
@@ -0,0 +1,8 @@
elasticsearch1:
image: elasticsearch:1.7.6
command: elasticsearch -Des.network.host=0.0.0.0
ports:
- "11200:9200"
- "11300:9300"
volumes:
- ./blocks/elastic/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
docker/blocks/elastic5/fig
+2 -2
View File
@@ -4,5 +4,5 @@ elasticsearch5:
image: elasticsearch:5
command: elasticsearch
ports:
- "9200:9200"
- "9300:9300"
- "10200:9200"
- "10300:9300"
public/app/plugins/datasource/elasticsearch/config_ctrl.ts
+1 -1
View File
@@ -22,8 +22,8 @@ export class ElasticConfigCtrl {
];
esVersions = [
{name: '1.x', value: 1},
{name: '2.x', value: 2},
{name: '5.x', value: 5},
];
indexPatternTypeChanged() {
public/app/plugins/datasource/elasticsearch/datasource.js
+27 -15
View File
@@ -81,21 +81,31 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
range[timeField]= {
from: options.range.from.valueOf(),
to: options.range.to.valueOf(),
format: "epoch_millis",
};
if (this.esVersion >= 2) {
range[timeField]["format"] = "epoch_millis";
}
var queryInterpolated = templateSrv.replace(queryString, {}, 'lucene');
var filter = { "bool": { "must": [{ "range": range }] } };
var query = { "bool": { "should": [{ "query_string": { "query": queryInterpolated } }] } };
var query = {
"bool": {
"must": [
{ "range": range },
{ "query_string": {
"query": queryInterpolated }
}
]
}
};
var data = {
"fields": [timeField, "_source"],
"query" : { "filtered": { "query" : query, "filter": filter } },
"query" : query,
"size": 10000
};
// fields field not supported on ES 5.x
if (this.esVersion < 5) {
data["fields"] = [timeField, "_source"];
}
var header = {search_type: "query_then_fetch", "ignore_unavailable": true};
// old elastic annotations had index specified on them
@@ -133,11 +143,12 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
for (var i = 0; i < hits.length; i++) {
var source = hits[i]._source;
var fields = hits[i].fields;
var time = source[timeField];
if (_.isString(fields[timeField]) || _.isNumber(fields[timeField])) {
time = fields[timeField];
if (typeof hits[i].fields !== 'undefined') {
var fields = hits[i].fields;
if (_.isString(fields[timeField]) || _.isNumber(fields[timeField])) {
time = fields[timeField];
}
}
var event = {
@@ -194,7 +205,7 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
luceneQuery = luceneQuery.substr(1, luceneQuery.length - 2);
esQuery = esQuery.replace("$lucene_query", luceneQuery);
var searchType = queryObj.size === 0 ? 'count' : 'query_then_fetch';
var searchType = (queryObj.size === 0 && this.esVersion < 5) ? 'count' : 'query_then_fetch';
var header = this.getQueryHeader(searchType, options.range.from, options.range.to);
payload += header + '\n';
@@ -277,14 +288,15 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
this.getTerms = function(queryDef) {
var range = timeSrv.timeRange();
var header = this.getQueryHeader('count', range.from, range.to);
var searchType = this.esVersion >= 5 ? 'query_then_fetch' : 'count' ;
var header = this.getQueryHeader(searchType, range.from, range.to);
var esQuery = angular.toJson(this.queryBuilder.getTermsQuery(queryDef));
esQuery = esQuery.replace(/\$timeFrom/g, range.from.valueOf());
esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf());
esQuery = header + '\n' + esQuery + '\n';
return this._post('_msearch?search_type=count', esQuery).then(function(res) {
return this._post('_msearch?search_type=' + searchType, esQuery).then(function(res) {
if (!res.responses[0].aggregations) {
return [];
}
public/app/plugins/datasource/elasticsearch/query_builder.js
+31 -39
View File
@@ -11,11 +11,11 @@ function (queryDef) {
ElasticQueryBuilder.prototype.getRangeFilter = function() {
var filter = {};
filter[this.timeField] = {"gte": "$timeFrom", "lte": "$timeTo"};
if (this.esVersion >= 2) {
filter[this.timeField]["format"] = "epoch_millis";
}
filter[this.timeField] = {
gte: "$timeFrom",
lte: "$timeTo",
format: "epoch_millis",
};
return filter;
};
@@ -28,7 +28,8 @@ function (queryDef) {
return queryNode;
}
queryNode.terms.size = parseInt(aggDef.settings.size, 10);
queryNode.terms.size = parseInt(aggDef.settings.size, 10) === 0 ? 500 : parseInt(aggDef.settings.size, 10);
if (aggDef.settings.orderBy !== void 0) {
queryNode.terms.order = {};
queryNode.terms.order[aggDef.settings.orderBy] = aggDef.settings.order;
@@ -62,15 +63,12 @@ function (queryDef) {
esAgg.field = this.timeField;
esAgg.min_doc_count = settings.min_doc_count || 0;
esAgg.extended_bounds = {min: "$timeFrom", max: "$timeTo"};
esAgg.format = "epoch_millis";
if (esAgg.interval === 'auto') {
esAgg.interval = "$interval";
}
if (this.esVersion >= 2) {
esAgg.format = "epoch_millis";
}
if (settings.missing) {
esAgg.missing = settings.missing;
}
@@ -80,15 +78,13 @@ function (queryDef) {
ElasticQueryBuilder.prototype.getFiltersAgg = function(aggDef) {
var filterObj = {};
for (var i = 0; i < aggDef.settings.filters.length; i++) {
var query = aggDef.settings.filters[i].query;
filterObj[query] = {
query: {
query_string: {
query: query,
analyze_wildcard: true
}
query_string: {
query: query,
analyze_wildcard: true
}
};
}
@@ -100,7 +96,12 @@ function (queryDef) {
query.size = 500;
query.sort = {};
query.sort[this.timeField] = {order: 'desc', unmapped_type: 'boolean'};
query.fields = ["*", "_source"];
// fields field not supported on ES 5.x
if (this.esVersion < 5) {
query.fields = ["*", "_source"];
}
query.script_fields = {},
query.fielddata_fields = [this.timeField];
return query;
@@ -112,13 +113,11 @@ function (queryDef) {
}
var i, filter, condition;
var must = query.query.filtered.filter.bool.must;
for (i = 0; i < adhocFilters.length; i++) {
filter = adhocFilters[i];
condition = {};
condition[filter.key] = filter.value;
must.push({"term": condition});
query.query.bool.must.push({"term": condition});
}
};
@@ -133,18 +132,15 @@ function (queryDef) {
var query = {
"size": 0,
"query": {
"filtered": {
"query": {
"query_string": {
"bool": {
"must": [
{"range": this.getRangeFilter()},
{"query_string": {
"analyze_wildcard": true,
"query": '$lucene_query',
"query": '$lucene_query'
}
}
},
"filter": {
"bool": {
"must": [{"range": this.getRangeFilter()}]
}
}
]
}
}
};
@@ -228,30 +224,26 @@ function (queryDef) {
var query = {
"size": 0,
"query": {
"filtered": {
"filter": {
"bool": {
"must": [{"range": this.getRangeFilter()}]
}
}
"bool": {
"must": [{"range": this.getRangeFilter()}]
}
}
};
if (queryDef.query) {
query.query.filtered.query = {
query.query.bool.must.push({
"query_string": {
"analyze_wildcard": true,
"query": queryDef.query,
}
};
});
}
query.aggs = {
"1": {
"terms": {
"field": queryDef.field,
"size": 0,
"size": 500,
"order": {
"_term": "asc"
}
public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts
+79 -4
View File
@@ -28,7 +28,7 @@ describe('ElasticDatasource', function() {
describe('When testing datasource with index pattern', function() {
beforeEach(function() {
createDatasource({url: 'http://es.com', index: '[asd-]YYYY.MM.DD', jsonData: {interval: 'Daily'}});
createDatasource({url: 'http://es.com', index: '[asd-]YYYY.MM.DD', jsonData: {interval: 'Daily', esVersion: '2'}});
});
it('should translate index pattern to current day', function() {
@@ -50,7 +50,7 @@ describe('ElasticDatasource', function() {
var requestOptions, parts, header;
beforeEach(function() {
createDatasource({url: 'http://es.com', index: '[asd-]YYYY.MM.DD', jsonData: {interval: 'Daily'}});
createDatasource({url: 'http://es.com', index: '[asd-]YYYY.MM.DD', jsonData: {interval: 'Daily', esVersion: '2'}});
ctx.backendSrv.datasourceRequest = function(options) {
requestOptions = options;
@@ -77,7 +77,7 @@ describe('ElasticDatasource', function() {
it('should json escape lucene query', function() {
var body = angular.fromJson(parts[1]);
expect(body.query.filtered.query.query_string.query).to.be('escape\\:test');
expect(body.query.bool.must[1].query_string.query).to.be('escape\\:test');
});
});
@@ -85,7 +85,7 @@ describe('ElasticDatasource', function() {
var requestOptions, parts, header;
beforeEach(function() {
createDatasource({url: 'http://es.com', index: 'test'});
createDatasource({url: 'http://es.com', index: 'test', jsonData: {esVersion: '2'}});
ctx.backendSrv.datasourceRequest = function(options) {
requestOptions = options;
@@ -209,4 +209,79 @@ describe('ElasticDatasource', function() {
});
});
describe('When issuing aggregation query on es5.x', function() {
var requestOptions, parts, header;
beforeEach(function() {
createDatasource({url: 'http://es.com', index: 'test', jsonData: {esVersion: '5'}});
ctx.backendSrv.datasourceRequest = function(options) {
requestOptions = options;
return ctx.$q.when({data: {responses: []}});
};
ctx.ds.query({
range: { from: moment([2015, 4, 30, 10]), to: moment([2015, 5, 1, 10]) },
targets: [{
bucketAggs: [
{type: 'date_histogram', field: '@timestamp', id: '2'}
],
metrics: [
{type: 'count'}], query: 'test' }
]
});
ctx.$rootScope.$apply();
parts = requestOptions.data.split('\n');
header = angular.fromJson(parts[0]);
});
it('should not set search type to count', function() {
expect(header.search_type).to.not.eql('count');
});
it('should set size to 0', function() {
var body = angular.fromJson(parts[1]);
expect(body.size).to.be(0);
});
});
describe('When issuing metricFind query on es5.x', function() {
var requestOptions, parts, header, body;
beforeEach(function() {
createDatasource({url: 'http://es.com', index: 'test', jsonData: {esVersion: '5'}});
ctx.backendSrv.datasourceRequest = function(options) {
requestOptions = options;
return ctx.$q.when({
data: {
responses: [{aggregations: {"1": [{buckets: {text: 'test', value: '1'}}]}}]
}
});
};
ctx.ds.metricFindQuery('{"find": "terms", "field": "test"}');
ctx.$rootScope.$apply();
parts = requestOptions.data.split('\n');
header = angular.fromJson(parts[0]);
body = angular.fromJson(parts[1]);
});
it('should not set search type to count', function() {
expect(header.search_type).to.not.eql('count');
});
it('should set size to 0', function() {
expect(body.size).to.be(0);
});
it('should not set terms aggregation size to 0', function() {
expect(body['aggs']['1']['terms'].size).to.not.be(0);
});
});
});
public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts
+48 -38
View File
@@ -16,7 +16,23 @@ describe('ElasticQueryBuilder', function() {
bucketAggs: [{type: 'date_histogram', field: '@timestamp', id: '1'}],
});
expect(query.query.filtered.filter.bool.must[0].range["@timestamp"].gte).to.be("$timeFrom");
expect(query.query.bool.must[0].range["@timestamp"].gte).to.be("$timeFrom");
expect(query.aggs["1"].date_histogram.extended_bounds.min).to.be("$timeFrom");
});
it('with defaults on es5.x', function() {
var builder_5x = new ElasticQueryBuilder({
timeField: '@timestamp',
esVersion: 5
});
var query = builder_5x.build({
metrics: [{type: 'Count', id: '0'}],
timeField: '@timestamp',
bucketAggs: [{type: 'date_histogram', field: '@timestamp', id: '1'}],
});
expect(query.query.bool.must[0].range["@timestamp"].gte).to.be("$timeFrom");
expect(query.aggs["1"].date_histogram.extended_bounds.min).to.be("$timeFrom");
});
@@ -34,39 +50,6 @@ describe('ElasticQueryBuilder', function() {
expect(query.aggs["2"].aggs["3"].date_histogram.field).to.be("@timestamp");
});
it('with es1.x and es2.x date histogram queries check time format', function() {
var builder_2x = new ElasticQueryBuilder({
timeField: '@timestamp',
esVersion: 2
});
var query_params = {
metrics: [],
bucketAggs: [
{type: 'date_histogram', field: '@timestamp', id: '1'}
],
};
// format should not be specified in 1.x queries
expect("format" in builder.build(query_params)["aggs"]["1"]["date_histogram"]).to.be(false);
// 2.x query should specify format to be "epoch_millis"
expect(builder_2x.build(query_params)["aggs"]["1"]["date_histogram"]["format"]).to.be("epoch_millis");
});
it('with es1.x and es2.x range filter check time format', function() {
var builder_2x = new ElasticQueryBuilder({
timeField: '@timestamp',
esVersion: 2
});
// format should not be specified in 1.x queries
expect("format" in builder.getRangeFilter()["@timestamp"]).to.be(false);
// 2.x query should specify format to be "epoch_millis"
expect(builder_2x.getRangeFilter()["@timestamp"]["format"]).to.be("epoch_millis");
});
it('with select field', function() {
var query = builder.build({
metrics: [{type: 'avg', field: '@value', id: '1'}],
@@ -138,8 +121,36 @@ describe('ElasticQueryBuilder', function() {
],
});
expect(query.aggs["2"].filters.filters["@metric:cpu"].query.query_string.query).to.be("@metric:cpu");
expect(query.aggs["2"].filters.filters["@metric:logins.count"].query.query_string.query).to.be("@metric:logins.count");
expect(query.aggs["2"].filters.filters["@metric:cpu"].query_string.query).to.be("@metric:cpu");
expect(query.aggs["2"].filters.filters["@metric:logins.count"].query_string.query).to.be("@metric:logins.count");
expect(query.aggs["2"].aggs["4"].date_histogram.field).to.be("@timestamp");
});
it('with filters aggs on es5.x', function() {
var builder_5x = new ElasticQueryBuilder({
timeField: '@timestamp',
esVersion: 5
});
var query = builder_5x.build({
metrics: [{type: 'count', id: '1'}],
timeField: '@timestamp',
bucketAggs: [
{
id: '2',
type: 'filters',
settings: {
filters: [
{query: '@metric:cpu' },
{query: '@metric:logins.count' },
]
}
},
{type: 'date_histogram', field: '@timestamp', id: '4'}
],
});
expect(query.aggs["2"].filters.filters["@metric:cpu"].query_string.query).to.be("@metric:cpu");
expect(query.aggs["2"].filters.filters["@metric:logins.count"].query_string.query).to.be("@metric:logins.count");
expect(query.aggs["2"].aggs["4"].date_histogram.field).to.be("@timestamp");
});
@@ -247,7 +258,6 @@ describe('ElasticQueryBuilder', function() {
{key: 'key1', operator: '=', value: 'value1'}
]);
expect(query.query.filtered.filter.bool.must[1].term["key1"]).to.be("value1");
expect(query.query.bool.must[2].term["key1"]).to.be("value1");
});
});
public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts
+6
View File
@@ -95,5 +95,11 @@ describe('ElasticQueryDef', function() {
expect(queryDef.getMetricAggTypes(2).length).to.be(11);
});
});
describe('using esversion 5', function() {
it('should get pipeline aggs', function() {
expect(queryDef.getMetricAggTypes(5).length).to.be(11);
});
});
});
});