mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
Elasticsearch: Run getTerms trough resource call if enableElasticsearchBackendQuerying enabled (#67848)
* Elasticsearch: Run getTerms torugh backend if toggle enabled * Add template variables to devenv dashboard for easier testing * Add TODO * Run feature toggle gen to fix build
This commit is contained in:
Ivana Huckova
GitHub
parent
b2e1b3ad91
commit
3145660f5a
@ -5274,18 +5274,520 @@
|
||||
],
|
||||
"title": "Logs",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"collapsed": true,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 13
|
||||
},
|
||||
"id": 97,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "elasticsearch",
|
||||
"uid": "gdev-elasticsearch"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"id": 98,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"alias": "",
|
||||
"bucketAggs": [
|
||||
{
|
||||
"field": "@timestamp",
|
||||
"id": "2",
|
||||
"settings": {
|
||||
"interval": "auto"
|
||||
},
|
||||
"type": "date_histogram"
|
||||
}
|
||||
],
|
||||
"datasource": {
|
||||
"type": "elasticsearch",
|
||||
"uid": "gdev-elasticsearch"
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"id": "1",
|
||||
"type": "count"
|
||||
}
|
||||
],
|
||||
"query": "hostname: $hostname",
|
||||
"refId": "A",
|
||||
"timeField": "@timestamp"
|
||||
}
|
||||
],
|
||||
"title": "$hostname count",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "elasticsearch",
|
||||
"uid": "gdev-elasticsearch"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 1
|
||||
},
|
||||
"id": 99,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"alias": "",
|
||||
"bucketAggs": [
|
||||
{
|
||||
"field": "@timestamp",
|
||||
"id": "2",
|
||||
"settings": {
|
||||
"interval": "auto"
|
||||
},
|
||||
"type": "date_histogram"
|
||||
}
|
||||
],
|
||||
"datasource": {
|
||||
"type": "elasticsearch",
|
||||
"uid": "gdev-elasticsearch"
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "float",
|
||||
"id": "1",
|
||||
"type": "avg"
|
||||
}
|
||||
],
|
||||
"query": "hostname: $hostname",
|
||||
"refId": "A",
|
||||
"timeField": "@timestamp"
|
||||
}
|
||||
],
|
||||
"title": "$hostname average float",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "elasticsearch",
|
||||
"uid": "gdev-elasticsearch"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
},
|
||||
"id": 100,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"alias": "",
|
||||
"bucketAggs": [
|
||||
{
|
||||
"field": "@timestamp",
|
||||
"id": "2",
|
||||
"settings": {
|
||||
"interval": "auto"
|
||||
},
|
||||
"type": "date_histogram"
|
||||
}
|
||||
],
|
||||
"datasource": {
|
||||
"type": "elasticsearch",
|
||||
"uid": "gdev-elasticsearch"
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"id": "1",
|
||||
"type": "count"
|
||||
}
|
||||
],
|
||||
"query": "hostname: $hostname AND level: $level_by_hostname",
|
||||
"refId": "A",
|
||||
"timeField": "@timestamp"
|
||||
}
|
||||
],
|
||||
"title": "$hostname and $level_by_hostname count",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "elasticsearch",
|
||||
"uid": "gdev-elasticsearch"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 9
|
||||
},
|
||||
"id": 101,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"alias": "",
|
||||
"bucketAggs": [
|
||||
{
|
||||
"field": "@timestamp",
|
||||
"id": "2",
|
||||
"settings": {
|
||||
"interval": "auto"
|
||||
},
|
||||
"type": "date_histogram"
|
||||
}
|
||||
],
|
||||
"datasource": {
|
||||
"type": "elasticsearch",
|
||||
"uid": "gdev-elasticsearch"
|
||||
},
|
||||
"metrics": [
|
||||
{
|
||||
"field": "float",
|
||||
"id": "1",
|
||||
"type": "avg"
|
||||
}
|
||||
],
|
||||
"query": "hostname: $hostname AND level: $level_by_hostname",
|
||||
"refId": "A",
|
||||
"timeField": "@timestamp"
|
||||
}
|
||||
],
|
||||
"title": "$hostname and $level_by_hostname average float",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"title": "Using template variables",
|
||||
"type": "row"
|
||||
}
|
||||
],
|
||||
"refresh": "",
|
||||
"schemaVersion": 38,
|
||||
"style": "dark",
|
||||
"tags": [
|
||||
"tags": [
|
||||
"gdev",
|
||||
"elasticsearch",
|
||||
"datasource-test"
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"selected": true,
|
||||
"text": "hostname2",
|
||||
"value": "hostname2"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "elasticsearch",
|
||||
"uid": "gdev-elasticsearch"
|
||||
},
|
||||
"definition": "{\"find\": \"terms\", \"field\": \"hostname\"}",
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"multi": false,
|
||||
"name": "hostname",
|
||||
"options": [],
|
||||
"query": "{\"find\": \"terms\", \"field\": \"hostname\"}",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 0,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"selected": false,
|
||||
"text": "error",
|
||||
"value": "error"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "elasticsearch",
|
||||
"uid": "gdev-elasticsearch"
|
||||
},
|
||||
"definition": "{\"find\": \"terms\", \"field\": \"level\", \"query\": \"hostname:$hostname\"}",
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"multi": false,
|
||||
"name": "level_by_hostname",
|
||||
"options": [],
|
||||
"query": "{\"find\": \"terms\", \"field\": \"level\", \"query\": \"hostname:$hostname\"}",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 0,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "elasticsearch",
|
||||
"uid": "gdev-elasticsearch"
|
||||
},
|
||||
"filters": [],
|
||||
"hide": 0,
|
||||
"name": "adhoc",
|
||||
"skipUrlSync": false,
|
||||
"type": "adhoc"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
@ -5293,7 +5795,7 @@
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "Datasource tests - Elasticsearch complex",
|
||||
"title": "Datasource tests - Elasticsearch complex with template variables",
|
||||
"uid": "es_complex",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
|
||||
@ -183,8 +183,9 @@ func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceReq
|
||||
logger := eslog.FromContext(ctx)
|
||||
// allowed paths for resource calls:
|
||||
// - empty string for fetching db version
|
||||
// - /?/_mapping for fetching index mapping
|
||||
if req.Path != "" && !strings.HasSuffix(req.Path, "/_mapping") {
|
||||
// - ?/_mapping for fetching index mapping
|
||||
// - _msearch for executing getTerms queries
|
||||
if req.Path != "" && !strings.HasSuffix(req.Path, "/_mapping") && req.Path != "_msearch" {
|
||||
return fmt.Errorf("invalid resource URL: %s", req.Path)
|
||||
}
|
||||
|
||||
|
||||
@ -1327,6 +1327,69 @@ describe('ElasticDatasource using backend', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('metricFindQuery', () => {
|
||||
async function runScenario() {
|
||||
const data = {
|
||||
responses: [
|
||||
{
|
||||
aggregations: {
|
||||
'1': {
|
||||
buckets: [
|
||||
{ doc_count: 1, key: 'test' },
|
||||
{
|
||||
doc_count: 2,
|
||||
key: 'test2',
|
||||
key_as_string: 'test2_as_string',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const { ds } = getTestContext();
|
||||
const postResourceMock = jest.spyOn(ds, 'postResource');
|
||||
postResourceMock.mockResolvedValue(data);
|
||||
|
||||
const results = await ds.metricFindQuery('{"find": "terms", "field": "test"}');
|
||||
|
||||
expect(ds.postResource).toHaveBeenCalledTimes(1);
|
||||
const requestOptions = postResourceMock.mock.calls[0][1];
|
||||
const parts = requestOptions.split('\n');
|
||||
const header = JSON.parse(parts[0]);
|
||||
const body = JSON.parse(parts[1]);
|
||||
|
||||
return { results, body, header };
|
||||
}
|
||||
|
||||
it('should get results', async () => {
|
||||
const { results } = await runScenario();
|
||||
expect(results.length).toEqual(2);
|
||||
});
|
||||
|
||||
it('should use key or key_as_string', async () => {
|
||||
const { results } = await runScenario();
|
||||
expect(results[0].text).toEqual('test');
|
||||
expect(results[1].text).toEqual('test2_as_string');
|
||||
});
|
||||
|
||||
it('should not set search type to count', async () => {
|
||||
const { header } = await runScenario();
|
||||
expect(header.search_type).not.toEqual('count');
|
||||
});
|
||||
|
||||
it('should set size to 0', async () => {
|
||||
const { body } = await runScenario();
|
||||
expect(body.size).toBe(0);
|
||||
});
|
||||
|
||||
it('should not set terms aggregation size to 0', async () => {
|
||||
const { body } = await runScenario();
|
||||
expect(body['aggs']['1']['terms'].size).not.toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getFields', () => {
|
||||
const getFieldsMockData = {
|
||||
'[test-]YYYY.MM.DD': {
|
||||
|
||||
@ -30,7 +30,7 @@ import {
|
||||
LogRowContextOptions,
|
||||
SupplementaryQueryOptions,
|
||||
} from '@grafana/data';
|
||||
import { DataSourceWithBackend, getDataSourceSrv, config } from '@grafana/runtime';
|
||||
import { DataSourceWithBackend, getDataSourceSrv, config, BackendSrvRequest } from '@grafana/runtime';
|
||||
import { queryLogsVolume } from 'app/core/logsModel';
|
||||
import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv';
|
||||
import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv';
|
||||
@ -148,6 +148,18 @@ export class ElasticDatasource
|
||||
this.legacyQueryRunner = new LegacyQueryRunner(this, this.templateSrv);
|
||||
}
|
||||
|
||||
getResourceRequest(path: string, params?: BackendSrvRequest['params'], options?: Partial<BackendSrvRequest>) {
|
||||
return this.getResource(path, params, options);
|
||||
}
|
||||
|
||||
postResourceRequest(path: string, data?: BackendSrvRequest['data'], options?: Partial<BackendSrvRequest>) {
|
||||
const resourceOptions = options ?? {};
|
||||
resourceOptions.headers = resourceOptions.headers ?? {};
|
||||
resourceOptions.headers['content-type'] = 'application/x-ndjson';
|
||||
|
||||
return this.postResource(path, data, resourceOptions);
|
||||
}
|
||||
|
||||
async importFromAbstractQueries(abstractQueries: AbstractQuery[]): Promise<ElasticsearchQuery[]> {
|
||||
return abstractQueries.map((abstractQuery) => this.languageProvider.importFromAbstractQuery(abstractQuery));
|
||||
}
|
||||
@ -532,7 +544,12 @@ export class ElasticDatasource
|
||||
|
||||
const url = this.getMultiSearchUrl();
|
||||
|
||||
return this.legacyQueryRunner.request('POST', url, esQuery).pipe(
|
||||
const termsObservable = config.featureToggles.enableElasticsearchBackendQuerying
|
||||
? // TODO: This is run trough resource call, but maybe should run trough query
|
||||
from(this.postResourceRequest(url, esQuery))
|
||||
: this.legacyQueryRunner.request('POST', url, esQuery);
|
||||
|
||||
return termsObservable.pipe(
|
||||
map((res) => {
|
||||
if (!res.responses[0].aggregations) {
|
||||
return [];
|
||||
@ -743,7 +760,7 @@ export class ElasticDatasource
|
||||
private getDatabaseVersionUncached(): Promise<SemVer | null> {
|
||||
// we want this function to never fail
|
||||
const getDbVersionObservable = config.featureToggles.enableElasticsearchBackendQuerying
|
||||
? from(this.getResource(''))
|
||||
? from(this.getResourceRequest(''))
|
||||
: this.legacyQueryRunner.request('GET', '/');
|
||||
|
||||
return lastValueFrom(getDbVersionObservable).then(
|
||||
|
||||
Loading…
Reference in New Issue
Block a user