IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Elasticsearch: Add support for Elasticsearch 8.0 (Beta) (#41729)

Browse Source 
* use fixed_interval in date_histogram

* Add 8.0 to available versions in datasource settings

* Remove moving_avg from available metric aggregations

* Add ES8 devenv

* Update public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/utils.ts

Co-authored-by: Piotr Jamróz <pm.jamroz@gmail.com>

* Add FE tests

* Add BE test

* fix FE test

* fix BE test

Co-authored-by: Piotr Jamróz <pm.jamroz@gmail.com>
This commit is contained in:
Giordano Ricci 2021-11-17 19:30:53 +00:00 committed by GitHub
parent ef1fb64bf5
commit 1b99d88337
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 138 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
devenv/docker/blocks/elastic/docker-compose.yaml
View File

@ -1,19 +1,22 @@
# You need to run 'sysctl -w vm.max_map_count=262144' on the host machine
elasticsearch-latest:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.2
image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0-beta1
command: elasticsearch
environment:
- "discovery.type=single-node"
- "xpack.license.self_generated.type=basic"
- "xpack.security.enabled=false"
ports:
- "14200:9200"
- "14300:9300"
fake-elastic-latest-data:
image: grafana/fake-data-gen
links:
- elasticsearch-latest
environment:
FD_SERVER: elasticsearch-latest
FD_DATASOURCE: elasticsearch7
FD_PORT: 9200
# TODO: uncomment when https://github.com/grafana/fake-data-gen/pull/20 is merged
# fake-elastic-latest-data:
# image: grafana/fake-data-gen
# links:
# - elasticsearch-latest
# environment:
# FD_SERVER: elasticsearch-latest
# FD_DATASOURCE: elasticsearch8
# FD_PORT: 9200

3
devenv/docker/blocks/elastic/elasticsearch.yml
View File

@ -1,3 +0,0 @@
script.inline: on
script.indexed: on
xpack.license.self_generated.type: basic

1
pkg/tsdb/elasticsearch/client/models.go
View File

@ -239,6 +239,7 @@ type HistogramAgg struct {
type DateHistogramAgg struct {
Field string `json:"field"`
Interval string `json:"interval,omitempty"`
FixedInterval string `json:"fixed_interval,omitempty"`
MinDocCount int `json:"min_doc_count"`
Missing *string `json:"missing,omitempty"`
ExtendedBounds *ExtendedBounds `json:"extended_bounds"`

5
pkg/tsdb/elasticsearch/client/search_request.go
View File

@ -342,6 +342,11 @@ func (b *aggBuilderImpl) DateHistogram(key, field string, fn func(a *DateHistogr
fn(innerAgg, builder)
}
if b.version.Major() >= 8 {
innerAgg.FixedInterval = innerAgg.Interval
innerAgg.Interval = ""
}
b.aggDefs = append(b.aggDefs, aggDef)
return b

56
pkg/tsdb/elasticsearch/time_series_query_test.go
View File

@ -1049,6 +1049,62 @@ func TestSettingsCasting(t *testing.T) {
assert.Equal(t, 10, dateHistogramAgg.MinDocCount)
})
t.Run("interval parameter", func(t *testing.T) {
t.Run("Uses interval with ES < 8.0.0", func(t *testing.T) {
c := newFakeClient("7.7.0")
_, err := executeTsdbQuery(c, `{
"timeField": "@timestamp",
"bucketAggs": [
{
"type": "date_histogram",
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "1d"
}
}
],
"metrics": [
{ "id": "1", "type": "average", "field": "@value" }
]
}`, from, to, 15*time.Second)
assert.Nil(t, err)
sr := c.multisearchRequests[0].Requests[0]
dateHistogramAgg := sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg)
assert.Zero(t, dateHistogramAgg.FixedInterval)
assert.NotZero(t, dateHistogramAgg.Interval)
})
t.Run("Uses fixed_interval with ES >= 8.0.0", func(t *testing.T) {
c := newFakeClient("8.0.0")
_, err := executeTsdbQuery(c, `{
"timeField": "@timestamp",
"bucketAggs": [
{
"type": "date_histogram",
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "1d"
}
}
],
"metrics": [
{ "id": "1", "type": "average", "field": "@value" }
]
}`, from, to, 15*time.Second)
assert.Nil(t, err)
sr := c.multisearchRequests[0].Requests[0]
dateHistogramAgg := sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg)
assert.NotZero(t, dateHistogramAgg.FixedInterval)
assert.Zero(t, dateHistogramAgg.Interval)
})
})
})
t.Run("Inline Script", func(t *testing.T) {

2
public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/utils.ts
View File

@ -112,7 +112,7 @@ export const metricAggregationConfig: MetricsConfiguration = {
label: 'Moving Average',
requiresField: true,
isPipelineAgg: true,
versionRange: '>=2.0.0',
versionRange: '>=2.0.0 <8.0.0',
supportsMissing: false,
supportsMultipleBucketPaths: false,
hasSettings: true,

7
public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx
View File

@ -14,7 +14,7 @@ const indexPatternTypes: Array<SelectableValue<'none' | Interval>> = [
{ label: 'Yearly', value: 'Yearly', example: '[logstash-]YYYY' },
];
const esVersions = [
const esVersions: SelectableValue[] = [
{ label: '2.x', value: '2.0.0' },
{ label: '5.x', value: '5.0.0' },
{ label: '5.6+', value: '5.6.0' },
@ -22,6 +22,11 @@ const esVersions = [
{ label: '7.0+', value: '7.0.0' },
{ label: '7.7+', value: '7.7.0' },
{ label: '7.10+', value: '7.10.0' },
{
label: '8.0+',
value: '8.0.0',
description: 'support for Elasticsearch 8 is currently experimental',
},
];
type Props = {

12
public/app/plugins/datasource/elasticsearch/query_builder.ts
View File

@ -95,7 +95,7 @@ export class ElasticQueryBuilder {
getDateHistogramAgg(aggDef: DateHistogram) {
const esAgg: any = {};
const settings = aggDef.settings || {};
esAgg.interval = settings.interval;
esAgg.field = aggDef.field || this.timeField;
esAgg.min_doc_count = settings.min_doc_count || 0;
esAgg.extended_bounds = { min: '$timeFrom', max: '$timeTo' };
@ -108,8 +108,14 @@ export class ElasticQueryBuilder {
esAgg.offset = settings.offset;
}
if (esAgg.interval === 'auto') {
esAgg.interval = '$__interval';
const interval = settings.interval === 'auto' ? '$__interval' : settings.interval;
if (gte(this.esVersion, '8.0.0')) {
// The deprecation was actually introduced in 7.0.0, we might want to use that instead of the removal date,
// but it woudl be a breaking change on our side.
esAgg.fixed_interval = interval;
} else {
esAgg.interval = interval;
}
return esAgg;

50
public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts
View File

@ -9,8 +9,9 @@ describe('ElasticQueryBuilder', () => {
const builder6x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '6.0.0' });
const builder7x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '7.0.0' });
const builder77 = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '7.7.0' });
const builder8 = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '8.0.0' });
const allBuilders = [builder, builder5x, builder56, builder6x, builder7x, builder77];
const allBuilders = [builder, builder5x, builder56, builder6x, builder7x, builder77, builder8];
allBuilders.forEach((builder) => {
describe(`version ${builder.esVersion}`, () => {
@ -728,7 +729,7 @@ describe('ElasticQueryBuilder', () => {
{ _doc: { order: 'desc' } },
]);
const expectedAggs = {
const expectedAggs: any = {
// FIXME: It's pretty weak to include this '1' in the test as it's not part of what we are testing here and
// might change as a cause of unrelated changes
1: {
@ -742,6 +743,11 @@ describe('ElasticQueryBuilder', () => {
},
},
};
if (gte(builder.esVersion, '8.0.0')) {
expectedAggs['1'].date_histogram.fixed_interval = expectedAggs['1'].date_histogram.interval;
delete expectedAggs['1'].date_histogram.interval;
}
expect(query.aggs).toMatchObject(expectedAggs);
});
@ -921,6 +927,46 @@ describe('ElasticQueryBuilder', () => {
expect(query.aggs['2'].date_histogram.field).toBe('@time');
});
describe('interval parameter', () => {
it('should use interval if Elasticsearch version <8.0.0', () => {
const query = builder77.build({
refId: 'A',
metrics: [{ type: 'count', id: '1' }],
timeField: '@timestamp',
bucketAggs: [
{
type: 'date_histogram',
id: '2',
field: '@time',
settings: { min_doc_count: '1', interval: '1d' },
},
],
});
expect(query.aggs['2'].date_histogram.interval).toBe('1d');
expect(query.aggs['2'].date_histogram.fixed_interval).toBeUndefined();
});
});
it('should use fixed_interval if Elasticsearch version >=8.0.0', () => {
const query = builder8.build({
refId: 'A',
metrics: [{ type: 'count', id: '1' }],
timeField: '@timestamp',
bucketAggs: [
{
type: 'date_histogram',
id: '2',
field: '@time',
settings: { min_doc_count: '1', interval: '1d' },
},
],
});
expect(query.aggs['2'].date_histogram.interval).toBeUndefined();
expect(query.aggs['2'].date_histogram.fixed_interval).toBe('1d');
});
});
});
});