mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
Elasticsearch: Add support for Elasticsearch 8.0 (Beta) (#41729)
* use fixed_interval in date_histogram * Add 8.0 to available versions in datasource settings * Remove moving_avg from available metric aggregations * Add ES8 devenv * Update public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/utils.ts Co-authored-by: Piotr Jamróz <pm.jamroz@gmail.com> * Add FE tests * Add BE test * fix FE test * fix BE test Co-authored-by: Piotr Jamróz <pm.jamroz@gmail.com>
This commit is contained in:
parent
ef1fb64bf5
commit
1b99d88337
@ -1,19 +1,22 @@
|
||||
# You need to run 'sysctl -w vm.max_map_count=262144' on the host machine
|
||||
|
||||
elasticsearch-latest:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.2
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0-beta1
|
||||
command: elasticsearch
|
||||
environment:
|
||||
- "discovery.type=single-node"
|
||||
- "xpack.license.self_generated.type=basic"
|
||||
- "xpack.security.enabled=false"
|
||||
ports:
|
||||
- "14200:9200"
|
||||
- "14300:9300"
|
||||
|
||||
fake-elastic-latest-data:
|
||||
image: grafana/fake-data-gen
|
||||
links:
|
||||
- elasticsearch-latest
|
||||
environment:
|
||||
FD_SERVER: elasticsearch-latest
|
||||
FD_DATASOURCE: elasticsearch7
|
||||
FD_PORT: 9200
|
||||
# TODO: uncomment when https://github.com/grafana/fake-data-gen/pull/20 is merged
|
||||
# fake-elastic-latest-data:
|
||||
# image: grafana/fake-data-gen
|
||||
# links:
|
||||
# - elasticsearch-latest
|
||||
# environment:
|
||||
# FD_SERVER: elasticsearch-latest
|
||||
# FD_DATASOURCE: elasticsearch8
|
||||
# FD_PORT: 9200
|
||||
|
@ -1,3 +0,0 @@
|
||||
script.inline: on
|
||||
script.indexed: on
|
||||
xpack.license.self_generated.type: basic
|
@ -239,6 +239,7 @@ type HistogramAgg struct {
|
||||
type DateHistogramAgg struct {
|
||||
Field string `json:"field"`
|
||||
Interval string `json:"interval,omitempty"`
|
||||
FixedInterval string `json:"fixed_interval,omitempty"`
|
||||
MinDocCount int `json:"min_doc_count"`
|
||||
Missing *string `json:"missing,omitempty"`
|
||||
ExtendedBounds *ExtendedBounds `json:"extended_bounds"`
|
||||
|
@ -342,6 +342,11 @@ func (b *aggBuilderImpl) DateHistogram(key, field string, fn func(a *DateHistogr
|
||||
fn(innerAgg, builder)
|
||||
}
|
||||
|
||||
if b.version.Major() >= 8 {
|
||||
innerAgg.FixedInterval = innerAgg.Interval
|
||||
innerAgg.Interval = ""
|
||||
}
|
||||
|
||||
b.aggDefs = append(b.aggDefs, aggDef)
|
||||
|
||||
return b
|
||||
|
@ -1049,6 +1049,62 @@ func TestSettingsCasting(t *testing.T) {
|
||||
|
||||
assert.Equal(t, 10, dateHistogramAgg.MinDocCount)
|
||||
})
|
||||
|
||||
t.Run("interval parameter", func(t *testing.T) {
|
||||
t.Run("Uses interval with ES < 8.0.0", func(t *testing.T) {
|
||||
c := newFakeClient("7.7.0")
|
||||
_, err := executeTsdbQuery(c, `{
|
||||
"timeField": "@timestamp",
|
||||
"bucketAggs": [
|
||||
{
|
||||
"type": "date_histogram",
|
||||
"field": "@timestamp",
|
||||
"id": "2",
|
||||
"settings": {
|
||||
"interval": "1d"
|
||||
}
|
||||
}
|
||||
],
|
||||
"metrics": [
|
||||
{ "id": "1", "type": "average", "field": "@value" }
|
||||
]
|
||||
}`, from, to, 15*time.Second)
|
||||
assert.Nil(t, err)
|
||||
sr := c.multisearchRequests[0].Requests[0]
|
||||
|
||||
dateHistogramAgg := sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg)
|
||||
|
||||
assert.Zero(t, dateHistogramAgg.FixedInterval)
|
||||
assert.NotZero(t, dateHistogramAgg.Interval)
|
||||
})
|
||||
|
||||
t.Run("Uses fixed_interval with ES >= 8.0.0", func(t *testing.T) {
|
||||
c := newFakeClient("8.0.0")
|
||||
_, err := executeTsdbQuery(c, `{
|
||||
"timeField": "@timestamp",
|
||||
"bucketAggs": [
|
||||
{
|
||||
"type": "date_histogram",
|
||||
"field": "@timestamp",
|
||||
"id": "2",
|
||||
"settings": {
|
||||
"interval": "1d"
|
||||
}
|
||||
}
|
||||
],
|
||||
"metrics": [
|
||||
{ "id": "1", "type": "average", "field": "@value" }
|
||||
]
|
||||
}`, from, to, 15*time.Second)
|
||||
assert.Nil(t, err)
|
||||
sr := c.multisearchRequests[0].Requests[0]
|
||||
|
||||
dateHistogramAgg := sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg)
|
||||
|
||||
assert.NotZero(t, dateHistogramAgg.FixedInterval)
|
||||
assert.Zero(t, dateHistogramAgg.Interval)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("Inline Script", func(t *testing.T) {
|
||||
|
@ -112,7 +112,7 @@ export const metricAggregationConfig: MetricsConfiguration = {
|
||||
label: 'Moving Average',
|
||||
requiresField: true,
|
||||
isPipelineAgg: true,
|
||||
versionRange: '>=2.0.0',
|
||||
versionRange: '>=2.0.0 <8.0.0',
|
||||
supportsMissing: false,
|
||||
supportsMultipleBucketPaths: false,
|
||||
hasSettings: true,
|
||||
|
@ -14,7 +14,7 @@ const indexPatternTypes: Array<SelectableValue<'none' | Interval>> = [
|
||||
{ label: 'Yearly', value: 'Yearly', example: '[logstash-]YYYY' },
|
||||
];
|
||||
|
||||
const esVersions = [
|
||||
const esVersions: SelectableValue[] = [
|
||||
{ label: '2.x', value: '2.0.0' },
|
||||
{ label: '5.x', value: '5.0.0' },
|
||||
{ label: '5.6+', value: '5.6.0' },
|
||||
@ -22,6 +22,11 @@ const esVersions = [
|
||||
{ label: '7.0+', value: '7.0.0' },
|
||||
{ label: '7.7+', value: '7.7.0' },
|
||||
{ label: '7.10+', value: '7.10.0' },
|
||||
{
|
||||
label: '8.0+',
|
||||
value: '8.0.0',
|
||||
description: 'support for Elasticsearch 8 is currently experimental',
|
||||
},
|
||||
];
|
||||
|
||||
type Props = {
|
||||
|
@ -95,7 +95,7 @@ export class ElasticQueryBuilder {
|
||||
getDateHistogramAgg(aggDef: DateHistogram) {
|
||||
const esAgg: any = {};
|
||||
const settings = aggDef.settings || {};
|
||||
esAgg.interval = settings.interval;
|
||||
|
||||
esAgg.field = aggDef.field || this.timeField;
|
||||
esAgg.min_doc_count = settings.min_doc_count || 0;
|
||||
esAgg.extended_bounds = { min: '$timeFrom', max: '$timeTo' };
|
||||
@ -108,8 +108,14 @@ export class ElasticQueryBuilder {
|
||||
esAgg.offset = settings.offset;
|
||||
}
|
||||
|
||||
if (esAgg.interval === 'auto') {
|
||||
esAgg.interval = '$__interval';
|
||||
const interval = settings.interval === 'auto' ? '$__interval' : settings.interval;
|
||||
|
||||
if (gte(this.esVersion, '8.0.0')) {
|
||||
// The deprecation was actually introduced in 7.0.0, we might want to use that instead of the removal date,
|
||||
// but it woudl be a breaking change on our side.
|
||||
esAgg.fixed_interval = interval;
|
||||
} else {
|
||||
esAgg.interval = interval;
|
||||
}
|
||||
|
||||
return esAgg;
|
||||
|
@ -9,8 +9,9 @@ describe('ElasticQueryBuilder', () => {
|
||||
const builder6x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '6.0.0' });
|
||||
const builder7x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '7.0.0' });
|
||||
const builder77 = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '7.7.0' });
|
||||
const builder8 = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '8.0.0' });
|
||||
|
||||
const allBuilders = [builder, builder5x, builder56, builder6x, builder7x, builder77];
|
||||
const allBuilders = [builder, builder5x, builder56, builder6x, builder7x, builder77, builder8];
|
||||
|
||||
allBuilders.forEach((builder) => {
|
||||
describe(`version ${builder.esVersion}`, () => {
|
||||
@ -728,7 +729,7 @@ describe('ElasticQueryBuilder', () => {
|
||||
{ _doc: { order: 'desc' } },
|
||||
]);
|
||||
|
||||
const expectedAggs = {
|
||||
const expectedAggs: any = {
|
||||
// FIXME: It's pretty weak to include this '1' in the test as it's not part of what we are testing here and
|
||||
// might change as a cause of unrelated changes
|
||||
1: {
|
||||
@ -742,6 +743,11 @@ describe('ElasticQueryBuilder', () => {
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
if (gte(builder.esVersion, '8.0.0')) {
|
||||
expectedAggs['1'].date_histogram.fixed_interval = expectedAggs['1'].date_histogram.interval;
|
||||
delete expectedAggs['1'].date_histogram.interval;
|
||||
}
|
||||
expect(query.aggs).toMatchObject(expectedAggs);
|
||||
});
|
||||
|
||||
@ -921,6 +927,46 @@ describe('ElasticQueryBuilder', () => {
|
||||
|
||||
expect(query.aggs['2'].date_histogram.field).toBe('@time');
|
||||
});
|
||||
|
||||
describe('interval parameter', () => {
|
||||
it('should use interval if Elasticsearch version <8.0.0', () => {
|
||||
const query = builder77.build({
|
||||
refId: 'A',
|
||||
metrics: [{ type: 'count', id: '1' }],
|
||||
timeField: '@timestamp',
|
||||
bucketAggs: [
|
||||
{
|
||||
type: 'date_histogram',
|
||||
id: '2',
|
||||
field: '@time',
|
||||
settings: { min_doc_count: '1', interval: '1d' },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(query.aggs['2'].date_histogram.interval).toBe('1d');
|
||||
expect(query.aggs['2'].date_histogram.fixed_interval).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
it('should use fixed_interval if Elasticsearch version >=8.0.0', () => {
|
||||
const query = builder8.build({
|
||||
refId: 'A',
|
||||
metrics: [{ type: 'count', id: '1' }],
|
||||
timeField: '@timestamp',
|
||||
bucketAggs: [
|
||||
{
|
||||
type: 'date_histogram',
|
||||
id: '2',
|
||||
field: '@time',
|
||||
settings: { min_doc_count: '1', interval: '1d' },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(query.aggs['2'].date_histogram.interval).toBeUndefined();
|
||||
expect(query.aggs['2'].date_histogram.fixed_interval).toBe('1d');
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
Loading…
Reference in New Issue
Block a user