IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-15 18:13:32 -06:00
Code Issues Packages Projects Releases Wiki Activity

Elasticsearch: Run requestAllIndices trough resource call if enableElasticsearchBackendQuerying enabled (#67825)

Browse Source 
* Elasticsearch: Run requestAllIndices trough resource call if enabled

* Unlock resource call path

* Fix lint
This commit is contained in:
Ivana Huckova 2023-05-05 11:35:30 +02:00 committed by GitHub
parent 20217db100
commit f5ac099907
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 184 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/tsdb/elasticsearch/elasticsearch.go
View File

@ -11,6 +11,7 @@ import (
"net/url"
"path"
"strconv"
"strings"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
@ -182,7 +183,8 @@ func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceReq
logger := eslog.FromContext(ctx)
// allowed paths for resource calls:
// - empty string for fetching db version
if req.Path != "" {
// - /?/_mapping for fetching index mapping
if req.Path != "" && !strings.HasSuffix(req.Path, "/_mapping") {
return fmt.Errorf("invalid resource URL: %s", req.Path)
}

174
public/app/plugins/datasource/elasticsearch/datasource.test.ts
View File

@ -1326,4 +1326,178 @@ describe('ElasticDatasource using backend', () => {
expect(version).toBe(null);
});
});
describe('getFields', () => {
const getFieldsMockData = {
'[test-]YYYY.MM.DD': {
mappings: {
properties: {
'@timestamp_millis': {
type: 'date',
format: 'epoch_millis',
},
classification_terms: {
type: 'keyword',
},
ip_address: {
type: 'ip',
},
justification_blob: {
properties: {
criterion: {
type: 'text',
fields: {
keyword: {
type: 'keyword',
ignore_above: 256,
},
},
},
shallow: {
properties: {
jsi: {
properties: {
sdb: {
properties: {
dsel2: {
properties: {
'bootlegged-gille': {
properties: {
botness: {
type: 'float',
},
general_algorithm_score: {
type: 'float',
},
},
},
'uncombed-boris': {
properties: {
botness: {
type: 'float',
},
general_algorithm_score: {
type: 'float',
},
},
},
},
},
},
},
},
},
},
},
},
},
overall_vote_score: {
type: 'float',
},
},
},
},
};
it('should not retry when ES is down', async () => {
const twoDaysBefore = toUtc().subtract(2, 'day').format('YYYY.MM.DD');
const { ds, timeSrv } = getTestContext({
from: 'now-2w',
jsonData: { interval: 'Daily' },
});
ds.getResource = jest.fn().mockImplementation((options) => {
if (options.url === `test-${twoDaysBefore}/_mapping`) {
return of({
data: {},
});
}
return throwError({ status: 500 });
});
const range = timeSrv.timeRange();
await expect(ds.getFields(undefined, range)).toEmitValuesWith((received) => {
expect(received.length).toBe(1);
expect(received[0]).toStrictEqual({ status: 500 });
expect(ds.getResource).toBeCalledTimes(1);
});
});
it('should not retry more than 7 indices', async () => {
const { ds, timeSrv } = getTestContext({
from: 'now-2w',
jsonData: { interval: 'Daily' },
});
const range = timeSrv.timeRange();
ds.getResource = jest.fn().mockImplementation(() => {
return throwError({ status: 404 });
});
await expect(ds.getFields(undefined, range)).toEmitValuesWith((received) => {
expect(received.length).toBe(1);
expect(received[0]).toStrictEqual('Could not find an available index for this time range.');
expect(ds.getResource).toBeCalledTimes(7);
});
});
it('should return nested fields', async () => {
const { ds } = getTestContext({
from: 'now-2w',
jsonData: { interval: 'Daily' },
});
ds.getResource = jest.fn().mockResolvedValue(getFieldsMockData);
await expect(ds.getFields()).toEmitValuesWith((received) => {
expect(received.length).toBe(1);
const fieldObjects = received[0];
const fields = map(fieldObjects, 'text');
expect(fields).toEqual([
'@timestamp_millis',
'classification_terms',
'ip_address',
'justification_blob.criterion.keyword',
'justification_blob.criterion',
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness',
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score',
'overall_vote_score',
]);
});
});
it('should return number fields', async () => {
const { ds } = getTestContext({});
ds.getResource = jest.fn().mockResolvedValue(getFieldsMockData);
await expect(ds.getFields(['number'])).toEmitValuesWith((received) => {
expect(received.length).toBe(1);
const fieldObjects = received[0];
const fields = map(fieldObjects, 'text');
expect(fields).toEqual([
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness',
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score',
'overall_vote_score',
]);
});
});
it('should return date fields', async () => {
const { ds } = getTestContext({});
ds.getResource = jest.fn().mockResolvedValue(getFieldsMockData);
await expect(ds.getFields(['date'])).toEmitValuesWith((received) => {
expect(received.length).toBe(1);
const fieldObjects = received[0];
const fields = map(fieldObjects, 'text');
expect(fields).toEqual(['@timestamp_millis']);
});
});
});
});

11
public/app/plugins/datasource/elasticsearch/datasource.ts
View File

@ -179,9 +179,12 @@ export class ElasticDatasource
}).pipe(
mergeMap((index) => {
// catch all errors and emit an object with an err property to simplify checks later in the pipeline
return this.legacyQueryRunner
.request('GET', indexUrlList[listLen - index - 1])
.pipe(catchError((err) => of({ err })));
const path = indexUrlList[listLen - index - 1];
const requestObservable = config.featureToggles.enableElasticsearchBackendQuerying
? from(this.getResource(path))
: this.legacyQueryRunner.request('GET', path);
return requestObservable.pipe(catchError((err) => of({ err })));
}),
skipWhile((resp) => resp?.err?.status === 404), // skip all requests that fail because missing Elastic index
throwIfEmpty(() => 'Could not find an available index for this time range.'), // when i === Math.min(listLen, maxTraversals) generate will complete but without emitting any values which means we didn't find a valid index
@ -462,7 +465,7 @@ export class ElasticDatasource
return true;
}
// equal query type filter, or via typemap translation
// equal query type filter, or via type map translation
return type.includes(obj.type) || type.includes(typeMap[obj.type]);
};