Elasticsearch: Run getTerms trough resource call if enableElasticsearchBackendQuerying enabled (#67848)

* Elasticsearch: Run getTerms torugh backend if toggle enabled

* Add template variables to devenv dashboard for easier testing

* Add TODO

* Run feature toggle gen to fix build
This commit is contained in:
Ivana Huckova 2023-05-06 10:00:43 +02:00 committed by GitHub
parent b2e1b3ad91
commit 3145660f5a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 591 additions and 8 deletions

View File

@ -5274,18 +5274,520 @@
],
"title": "Logs",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 13
},
"id": 97,
"panels": [
{
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 1
},
"id": 98,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"alias": "",
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto"
},
"type": "date_histogram"
}
],
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"metrics": [
{
"id": "1",
"type": "count"
}
],
"query": "hostname: $hostname",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "$hostname count",
"type": "timeseries"
},
{
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 1
},
"id": 99,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"alias": "",
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto"
},
"type": "date_histogram"
}
],
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"metrics": [
{
"field": "float",
"id": "1",
"type": "avg"
}
],
"query": "hostname: $hostname",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "$hostname average float",
"type": "timeseries"
},
{
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 9
},
"id": 100,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"alias": "",
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto"
},
"type": "date_histogram"
}
],
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"metrics": [
{
"id": "1",
"type": "count"
}
],
"query": "hostname: $hostname AND level: $level_by_hostname",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "$hostname and $level_by_hostname count",
"type": "timeseries"
},
{
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 9
},
"id": 101,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"alias": "",
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto"
},
"type": "date_histogram"
}
],
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"metrics": [
{
"field": "float",
"id": "1",
"type": "avg"
}
],
"query": "hostname: $hostname AND level: $level_by_hostname",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "$hostname and $level_by_hostname average float",
"type": "timeseries"
}
],
"title": "Using template variables",
"type": "row"
}
],
"refresh": "",
"schemaVersion": 38,
"style": "dark",
"tags": [
"tags": [
"gdev",
"elasticsearch",
"datasource-test"
],
"templating": {
"list": []
"list": [
{
"current": {
"selected": true,
"text": "hostname2",
"value": "hostname2"
},
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"definition": "{\"find\": \"terms\", \"field\": \"hostname\"}",
"hide": 0,
"includeAll": false,
"multi": false,
"name": "hostname",
"options": [],
"query": "{\"find\": \"terms\", \"field\": \"hostname\"}",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
},
{
"current": {
"selected": false,
"text": "error",
"value": "error"
},
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"definition": "{\"find\": \"terms\", \"field\": \"level\", \"query\": \"hostname:$hostname\"}",
"hide": 0,
"includeAll": false,
"multi": false,
"name": "level_by_hostname",
"options": [],
"query": "{\"find\": \"terms\", \"field\": \"level\", \"query\": \"hostname:$hostname\"}",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
},
{
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"filters": [],
"hide": 0,
"name": "adhoc",
"skipUrlSync": false,
"type": "adhoc"
}
]
},
"time": {
"from": "now-6h",
@ -5293,7 +5795,7 @@
},
"timepicker": {},
"timezone": "",
"title": "Datasource tests - Elasticsearch complex",
"title": "Datasource tests - Elasticsearch complex with template variables",
"uid": "es_complex",
"version": 1,
"weekStart": ""

View File

@ -183,8 +183,9 @@ func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceReq
logger := eslog.FromContext(ctx)
// allowed paths for resource calls:
// - empty string for fetching db version
// - /?/_mapping for fetching index mapping
if req.Path != "" && !strings.HasSuffix(req.Path, "/_mapping") {
// - ?/_mapping for fetching index mapping
// - _msearch for executing getTerms queries
if req.Path != "" && !strings.HasSuffix(req.Path, "/_mapping") && req.Path != "_msearch" {
return fmt.Errorf("invalid resource URL: %s", req.Path)
}

View File

@ -1327,6 +1327,69 @@ describe('ElasticDatasource using backend', () => {
});
});
describe('metricFindQuery', () => {
async function runScenario() {
const data = {
responses: [
{
aggregations: {
'1': {
buckets: [
{ doc_count: 1, key: 'test' },
{
doc_count: 2,
key: 'test2',
key_as_string: 'test2_as_string',
},
],
},
},
},
],
};
const { ds } = getTestContext();
const postResourceMock = jest.spyOn(ds, 'postResource');
postResourceMock.mockResolvedValue(data);
const results = await ds.metricFindQuery('{"find": "terms", "field": "test"}');
expect(ds.postResource).toHaveBeenCalledTimes(1);
const requestOptions = postResourceMock.mock.calls[0][1];
const parts = requestOptions.split('\n');
const header = JSON.parse(parts[0]);
const body = JSON.parse(parts[1]);
return { results, body, header };
}
it('should get results', async () => {
const { results } = await runScenario();
expect(results.length).toEqual(2);
});
it('should use key or key_as_string', async () => {
const { results } = await runScenario();
expect(results[0].text).toEqual('test');
expect(results[1].text).toEqual('test2_as_string');
});
it('should not set search type to count', async () => {
const { header } = await runScenario();
expect(header.search_type).not.toEqual('count');
});
it('should set size to 0', async () => {
const { body } = await runScenario();
expect(body.size).toBe(0);
});
it('should not set terms aggregation size to 0', async () => {
const { body } = await runScenario();
expect(body['aggs']['1']['terms'].size).not.toBe(0);
});
});
describe('getFields', () => {
const getFieldsMockData = {
'[test-]YYYY.MM.DD': {

View File

@ -30,7 +30,7 @@ import {
LogRowContextOptions,
SupplementaryQueryOptions,
} from '@grafana/data';
import { DataSourceWithBackend, getDataSourceSrv, config } from '@grafana/runtime';
import { DataSourceWithBackend, getDataSourceSrv, config, BackendSrvRequest } from '@grafana/runtime';
import { queryLogsVolume } from 'app/core/logsModel';
import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv';
import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv';
@ -148,6 +148,18 @@ export class ElasticDatasource
this.legacyQueryRunner = new LegacyQueryRunner(this, this.templateSrv);
}
getResourceRequest(path: string, params?: BackendSrvRequest['params'], options?: Partial<BackendSrvRequest>) {
return this.getResource(path, params, options);
}
postResourceRequest(path: string, data?: BackendSrvRequest['data'], options?: Partial<BackendSrvRequest>) {
const resourceOptions = options ?? {};
resourceOptions.headers = resourceOptions.headers ?? {};
resourceOptions.headers['content-type'] = 'application/x-ndjson';
return this.postResource(path, data, resourceOptions);
}
async importFromAbstractQueries(abstractQueries: AbstractQuery[]): Promise<ElasticsearchQuery[]> {
return abstractQueries.map((abstractQuery) => this.languageProvider.importFromAbstractQuery(abstractQuery));
}
@ -532,7 +544,12 @@ export class ElasticDatasource
const url = this.getMultiSearchUrl();
return this.legacyQueryRunner.request('POST', url, esQuery).pipe(
const termsObservable = config.featureToggles.enableElasticsearchBackendQuerying
? // TODO: This is run trough resource call, but maybe should run trough query
from(this.postResourceRequest(url, esQuery))
: this.legacyQueryRunner.request('POST', url, esQuery);
return termsObservable.pipe(
map((res) => {
if (!res.responses[0].aggregations) {
return [];
@ -743,7 +760,7 @@ export class ElasticDatasource
private getDatabaseVersionUncached(): Promise<SemVer | null> {
// we want this function to never fail
const getDbVersionObservable = config.featureToggles.enableElasticsearchBackendQuerying
? from(this.getResource(''))
? from(this.getResourceRequest(''))
: this.legacyQueryRunner.request('GET', '/');
return lastValueFrom(getDbVersionObservable).then(