From d6e4f2a504239fc16e33a647b04d2951b329df84 Mon Sep 17 00:00:00 2001 From: Gareth Dawson Date: Thu, 22 Jun 2023 12:53:05 +0100 Subject: [PATCH] Elasticsearch: Enable logs samples for metric queries (#70258) * enable logs samples on elastic ds * add tests for getSupplementaryQuery * only display log samples for date_hostogram queries * changes * test * Update public/app/plugins/datasource/elasticsearch/datasource.test.ts Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> * Update public/app/plugins/datasource/elasticsearch/datasource.test.ts Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> * Update public/app/plugins/datasource/elasticsearch/datasource.test.ts Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> * Update public/app/plugins/datasource/elasticsearch/datasource.test.ts Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> * Update public/app/plugins/datasource/elasticsearch/datasource.test.ts Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> * address feedback / tests --------- Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> --- .../components/QueryEditor/index.tsx | 7 +- .../elasticsearch/datasource.test.ts | 97 +++++++++++++++++++ .../datasource/elasticsearch/datasource.ts | 45 ++++++++- .../datasource/elasticsearch/utils.test.ts | 47 ++++++++- .../plugins/datasource/elasticsearch/utils.ts | 7 +- 5 files changed, 194 insertions(+), 9 deletions(-) diff --git a/public/app/plugins/datasource/elasticsearch/components/QueryEditor/index.tsx b/public/app/plugins/datasource/elasticsearch/components/QueryEditor/index.tsx index cdb72b42017..108f5695160 100644 --- a/public/app/plugins/datasource/elasticsearch/components/QueryEditor/index.tsx +++ b/public/app/plugins/datasource/elasticsearch/components/QueryEditor/index.tsx @@ -9,7 +9,7 @@ import { ElasticDatasource } from '../../datasource'; import { useNextId } from '../../hooks/useNextId'; import { useDispatch } from '../../hooks/useStatelessReducer'; import { ElasticsearchOptions, ElasticsearchQuery } from '../../types'; -import { isSupportedVersion, unsupportedVersionMessage } from '../../utils'; +import { isSupportedVersion, isTimeSeriesQuery, unsupportedVersionMessage } from '../../utils'; import { BucketAggregationsEditor } from './BucketAggregationsEditor'; import { ElasticsearchProvider } from './ElasticsearchQueryContext'; @@ -92,8 +92,7 @@ const QueryEditorForm = ({ value }: Props) => { const nextId = useNextId(); const styles = useStyles2(getStyles); - // To be considered a time series query, the last bucked aggregation must be a Date Histogram - const isTimeSeriesQuery = value?.bucketAggs?.slice(-1)[0]?.type === 'date_histogram'; + const isTimeSeries = isTimeSeriesQuery(value); const showBucketAggregationsEditor = value.metrics?.every( (metric) => metricAggregationConfig[metric.type].impliedQueryType === 'metrics' @@ -111,7 +110,7 @@ const QueryEditorForm = ({ value }: Props) => { Lucene Query dispatch(changeQuery(query))} value={value?.query} /> - {isTimeSeriesQuery && ( + {isTimeSeries && ( { timeField: '', }); }); + + it('does not return logs samples for non time series queries', () => { + expect( + ds.getSupplementaryQuery( + { type: SupplementaryQueryType.LogsSample, limit: 100 }, + { + refId: 'A', + bucketAggs: [{ type: 'filters', id: '1' }], + query: '', + } + ) + ).toEqual(undefined); + }); + + it('returns logs samples for time series queries', () => { + expect( + ds.getSupplementaryQuery( + { type: SupplementaryQueryType.LogsSample, limit: 100 }, + { + refId: 'A', + query: '', + bucketAggs: [{ type: 'date_histogram', id: '1' }], + } + ) + ).toEqual({ + refId: `log-sample-A`, + query: '', + metrics: [{ type: 'logs', id: '1', settings: { limit: '100' } }], + }); + }); + }); + + describe('getDataProvider', () => { + let ds: ElasticDatasource; + beforeEach(() => { + ds = getTestContext().ds; + }); + + it('does not create a logs sample provider for non time series query', () => { + const options = getQueryOptions({ + targets: [ + { + refId: 'A', + metrics: [{ type: 'logs', id: '1', settings: { limit: '100' } }], + }, + ], + }); + + expect(ds.getDataProvider(SupplementaryQueryType.LogsSample, options)).not.toBeDefined(); + }); + + it('does create a logs sample provider for time series query', () => { + const options = getQueryOptions({ + targets: [ + { + refId: 'A', + bucketAggs: [{ type: 'date_histogram', id: '1' }], + }, + ], + }); + + expect(ds.getDataProvider(SupplementaryQueryType.LogsSample, options)).toBeDefined(); + }); + }); + + describe('getLogsSampleDataProvider', () => { + let ds: ElasticDatasource; + beforeEach(() => { + ds = getTestContext().ds; + }); + + it("doesn't return a logs sample provider given a non time series query", () => { + const request = getQueryOptions({ + targets: [ + { + refId: 'A', + metrics: [{ type: 'logs', id: '1', settings: { limit: '100' } }], + }, + ], + }); + + expect(ds.getLogsSampleDataProvider(request)).not.toBeDefined(); + }); + + it('returns a logs sample provider given a time series query', () => { + const request = getQueryOptions({ + targets: [ + { + refId: 'A', + bucketAggs: [{ type: 'date_histogram', id: '1' }], + }, + ], + }); + + expect(ds.getLogsSampleDataProvider(request)).toBeDefined(); + }); }); }); diff --git a/public/app/plugins/datasource/elasticsearch/datasource.ts b/public/app/plugins/datasource/elasticsearch/datasource.ts index 2343b342467..2999a2f9a4c 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.ts @@ -33,7 +33,7 @@ import { AnnotationEvent, } from '@grafana/data'; import { DataSourceWithBackend, getDataSourceSrv, config, BackendSrvRequest } from '@grafana/runtime'; -import { queryLogsVolume } from 'app/core/logsModel'; +import { queryLogsSample, queryLogsVolume } from 'app/core/logsModel'; import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv'; import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv'; @@ -64,9 +64,11 @@ import { ElasticsearchAnnotationQuery, RangeMap, } from './types'; -import { getScriptValue, isSupportedVersion, unsupportedVersionMessage } from './utils'; +import { getScriptValue, isSupportedVersion, isTimeSeriesQuery, unsupportedVersionMessage } from './utils'; export const REF_ID_STARTER_LOG_VOLUME = 'log-volume-'; +export const REF_ID_STARTER_LOG_SAMPLE = 'log-sample-'; + // Those are metadata fields as defined in https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-fields.html#_identity_metadata_fields. // custom fields can start with underscores, therefore is not safe to exclude anything that starts with one. const ELASTIC_META_FIELDS = [ @@ -520,13 +522,15 @@ export class ElasticDatasource switch (type) { case SupplementaryQueryType.LogsVolume: return this.getLogsVolumeDataProvider(request); + case SupplementaryQueryType.LogsSample: + return this.getLogsSampleDataProvider(request); default: return undefined; } } getSupportedSupplementaryQueryTypes(): SupplementaryQueryType[] { - return [SupplementaryQueryType.LogsVolume]; + return [SupplementaryQueryType.LogsVolume, SupplementaryQueryType.LogsSample]; } getSupplementaryQuery(options: SupplementaryQueryOptions, query: ElasticsearchQuery): ElasticsearchQuery | undefined { @@ -579,6 +583,27 @@ export class ElasticDatasource bucketAggs, }; + case SupplementaryQueryType.LogsSample: + isQuerySuitable = isTimeSeriesQuery(query); + + if (!isQuerySuitable) { + return undefined; + } + + if (options.limit) { + return { + refId: `${REF_ID_STARTER_LOG_SAMPLE}${query.refId}`, + query: query.query, + metrics: [{ type: 'logs', id: '1', settings: { limit: options.limit.toString() } }], + }; + } + + return { + refId: `${REF_ID_STARTER_LOG_SAMPLE}${query.refId}`, + query: query.query, + metrics: [{ type: 'logs', id: '1' }], + }; + default: return undefined; } @@ -605,6 +630,20 @@ export class ElasticDatasource ); } + getLogsSampleDataProvider(request: DataQueryRequest): Observable | undefined { + const logsSampleRequest = cloneDeep(request); + const targets = logsSampleRequest.targets; + const queries = targets.map((query) => { + return this.getSupplementaryQuery({ type: SupplementaryQueryType.LogsSample, limit: 100 }, query); + }); + const elasticQueries = queries.filter((query): query is ElasticsearchQuery => !!query); + + if (!elasticQueries.length) { + return undefined; + } + return queryLogsSample(this, { ...logsSampleRequest, targets: elasticQueries }); + } + query(request: DataQueryRequest): Observable { const { enableElasticsearchBackendQuerying } = config.featureToggles; if (enableElasticsearchBackendQuerying) { diff --git a/public/app/plugins/datasource/elasticsearch/utils.test.ts b/public/app/plugins/datasource/elasticsearch/utils.test.ts index 22f396b3c18..dcf10900c64 100644 --- a/public/app/plugins/datasource/elasticsearch/utils.test.ts +++ b/public/app/plugins/datasource/elasticsearch/utils.test.ts @@ -1,4 +1,5 @@ -import { removeEmpty } from './utils'; +import { ElasticsearchQuery } from './types'; +import { isTimeSeriesQuery, removeEmpty } from './utils'; describe('removeEmpty', () => { it('Should remove all empty', () => { @@ -34,3 +35,47 @@ describe('removeEmpty', () => { expect(removeEmpty(original)).toStrictEqual(expectedResult); }); }); + +describe('isTimeSeriesQuery', () => { + it('should return false when given a log query', () => { + const logsQuery: ElasticsearchQuery = { + refId: `A`, + metrics: [{ type: 'logs', id: '1' }], + }; + + expect(isTimeSeriesQuery(logsQuery)).toBe(false); + }); + + it('should return false when bucket aggs are empty', () => { + const query: ElasticsearchQuery = { + refId: `A`, + bucketAggs: [], + }; + + expect(isTimeSeriesQuery(query)).toBe(false); + }); + + it('returns false when empty date_histogram is not last', () => { + const query: ElasticsearchQuery = { + refId: `A`, + bucketAggs: [ + { id: '1', type: 'date_histogram' }, + { id: '2', type: 'terms' }, + ], + }; + + expect(isTimeSeriesQuery(query)).toBe(false); + }); + + it('returns true when empty date_histogram is last', () => { + const query: ElasticsearchQuery = { + refId: `A`, + bucketAggs: [ + { id: '1', type: 'terms' }, + { id: '2', type: 'date_histogram' }, + ], + }; + + expect(isTimeSeriesQuery(query)).toBe(true); + }); +}); diff --git a/public/app/plugins/datasource/elasticsearch/utils.ts b/public/app/plugins/datasource/elasticsearch/utils.ts index 029fafc737a..63e3ac25feb 100644 --- a/public/app/plugins/datasource/elasticsearch/utils.ts +++ b/public/app/plugins/datasource/elasticsearch/utils.ts @@ -2,7 +2,7 @@ import { gte, SemVer } from 'semver'; import { isMetricAggregationWithField } from './components/QueryEditor/MetricAggregationsEditor/aggregations'; import { metricAggregationConfig } from './components/QueryEditor/MetricAggregationsEditor/utils'; -import { MetricAggregation, MetricAggregationWithInlineScript } from './types'; +import { ElasticsearchQuery, MetricAggregation, MetricAggregationWithInlineScript } from './types'; export const describeMetric = (metric: MetricAggregation) => { if (!isMetricAggregationWithField(metric)) { @@ -101,3 +101,8 @@ export const isSupportedVersion = (version: SemVer): boolean => { export const unsupportedVersionMessage = 'Support for Elasticsearch versions after their end-of-life (currently versions < 7.16) was removed. Using unsupported version of Elasticsearch may lead to unexpected and incorrect results.'; + +// To be considered a time series query, the last bucked aggregation must be a Date Histogram +export const isTimeSeriesQuery = (query: ElasticsearchQuery): boolean => { + return query?.bucketAggs?.slice(-1)[0]?.type === 'date_histogram'; +};