From 2a734a9e3f37b0a6ab2aedd425c4e3203c1f9599 Mon Sep 17 00:00:00 2001 From: Matias Chomicki Date: Tue, 30 Jan 2024 11:36:20 +0100 Subject: [PATCH] Supplementary queries: allow plugin decoupling by allowing providers to return a request instance (#80281) * Supplementary queries: add support for providers returning a request instance * Formatting * DataSourceWithSupplementaryQueriesSupport: update getDataProvider signature * getLogLevelFromLabels: fix buggy implementation * getLogLevelFromKey: fix key type Why number?? * Revert "getLogLevelFromKey: fix key type" This reverts commit 14a95298a6f803cc3270e0421b2e04dd0d65f131. * getSupplementaryQueryProvider: remove observable support * Datasources: remove unnecessary check The switch is doing the same job * Supplementary queries: update unit test * datasource_srv: sync mock with real api * Formatting * Supplementary queries: pass targets from getSupplementaryQueryProvider * LogsVolumeQueryOptions: remove range and make extract level optional * logsModel: add missing range to test data * query: sync tests with changes * Formatting * DataSourceWithSupplementaryQueriesSupport: update interface with deprecated and new methods * DataSourceWithSupplementaryQueriesSupport: sync Loki and Elasticsearch * queryLogsVolume: extractLevel no longer customizable * Loki: update test * Supplementary queries: add support for the new method * hasSupplementaryQuerySupport: update signature * Formatting * Betterer * Query: update test * Supplementary queries: add test for the legacy API * Update public/app/features/explore/utils/supplementaryQueries.ts Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> --------- Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> --- .betterer.results | 3 - packages/grafana-data/src/types/logs.ts | 28 +- .../features/explore/Logs/LogsSamplePanel.tsx | 3 + .../app/features/explore/state/query.test.ts | 68 +++- .../utils/supplementaryQueries.test.ts | 81 ++-- .../explore/utils/supplementaryQueries.ts | 16 +- .../utils/supplementaryQueries_legacy.test.ts | 365 ++++++++++++++++++ public/app/features/logs/logsModel.test.ts | 18 +- public/app/features/logs/logsModel.ts | 27 +- .../elasticsearch/datasource.test.ts | 8 +- .../datasource/elasticsearch/datasource.ts | 46 +-- .../datasource/loki/datasource.test.ts | 14 +- .../app/plugins/datasource/loki/datasource.ts | 57 +-- public/test/mocks/datasource_srv.ts | 2 +- 14 files changed, 567 insertions(+), 169 deletions(-) create mode 100644 public/app/features/explore/utils/supplementaryQueries_legacy.test.ts diff --git a/.betterer.results b/.betterer.results index bab1eb40406..f77ae021038 100644 --- a/.betterer.results +++ b/.betterer.results @@ -328,9 +328,6 @@ exports[`better eslint`] = { [0, 0, 0, "Do not use any type assertions.", "7"], [0, 0, 0, "Do not use any type assertions.", "8"] ], - "packages/grafana-data/src/types/logs.ts:5381": [ - [0, 0, 0, "Do not use any type assertions.", "0"] - ], "packages/grafana-data/src/types/options.ts:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"], [0, 0, 0, "Unexpected any. Specify a different type.", "1"] diff --git a/packages/grafana-data/src/types/logs.ts b/packages/grafana-data/src/types/logs.ts index e578e482091..e63e5512976 100644 --- a/packages/grafana-data/src/types/logs.ts +++ b/packages/grafana-data/src/types/logs.ts @@ -4,7 +4,7 @@ import { DataQuery } from '@grafana/schema'; import { KeyValue, Labels } from './data'; import { DataFrame } from './dataFrame'; -import { DataQueryRequest, DataQueryResponse, QueryFixAction, QueryFixType } from './datasource'; +import { DataQueryRequest, DataQueryResponse, DataSourceApi, QueryFixAction, QueryFixType } from './datasource'; import { AbsoluteTimeRange } from './time'; export { LogsDedupStrategy, LogsSortOrder } from '@grafana/schema'; @@ -225,36 +225,44 @@ export interface DataSourceWithSupplementaryQueriesSupport ): Observable | undefined; + /** + * Receives a SupplementaryQueryType and a DataQueryRequest and returns a new DataQueryRequest to fetch supplementary data. + * If provided type or request is not suitable for a supplementary data request, returns undefined. + */ + getSupplementaryRequest?( + type: SupplementaryQueryType, + request: DataQueryRequest + ): DataQueryRequest | undefined; /** * Returns supplementary query types that data source supports. */ getSupportedSupplementaryQueryTypes(): SupplementaryQueryType[]; /** * Returns a supplementary query to be used to fetch supplementary data based on the provided type and original query. - * If provided query is not suitable for provided supplementary query type, undefined should be returned. + * If the provided query is not suitable for the provided supplementary query type, undefined should be returned. */ getSupplementaryQuery(options: SupplementaryQueryOptions, originalQuery: TQuery): TQuery | undefined; } export const hasSupplementaryQuerySupport = ( - datasource: unknown, + datasource: DataSourceApi | (DataSourceApi & DataSourceWithSupplementaryQueriesSupport), type: SupplementaryQueryType -): datasource is DataSourceWithSupplementaryQueriesSupport => { +): datasource is DataSourceApi & DataSourceWithSupplementaryQueriesSupport => { if (!datasource) { return false; } - const withSupplementaryQueriesSupport = datasource as DataSourceWithSupplementaryQueriesSupport; - return ( - withSupplementaryQueriesSupport.getDataProvider !== undefined && - withSupplementaryQueriesSupport.getSupplementaryQuery !== undefined && - withSupplementaryQueriesSupport.getSupportedSupplementaryQueryTypes().includes(type) + ('getDataProvider' in datasource || 'getSupplementaryRequest' in datasource) && + 'getSupplementaryQuery' in datasource && + 'getSupportedSupplementaryQueryTypes' in datasource && + datasource.getSupportedSupplementaryQueryTypes().includes(type) ); }; diff --git a/public/app/features/explore/Logs/LogsSamplePanel.tsx b/public/app/features/explore/Logs/LogsSamplePanel.tsx index 857e5668be7..5e628b38c4b 100644 --- a/public/app/features/explore/Logs/LogsSamplePanel.tsx +++ b/public/app/features/explore/Logs/LogsSamplePanel.tsx @@ -45,6 +45,9 @@ export function LogsSamplePanel(props: Props) { }; const OpenInSplitViewButton = () => { + if (!datasourceInstance) { + return null; + } if (!hasSupplementaryQuerySupport(datasourceInstance, SupplementaryQueryType.LogsSample)) { return null; } diff --git a/public/app/features/explore/state/query.test.ts b/public/app/features/explore/state/query.test.ts index ffcd3c85a2e..18bd129cba0 100644 --- a/public/app/features/explore/state/query.test.ts +++ b/public/app/features/explore/state/query.test.ts @@ -16,6 +16,7 @@ import { SupplementaryQueryType, } from '@grafana/data'; import { DataQuery, DataSourceRef } from '@grafana/schema'; +import { queryLogsSample, queryLogsVolume } from 'app/features/logs/logsModel'; import { createAsyncThunk, ExploreItemState, StoreState, ThunkDispatch } from 'app/types'; import { reducerTester } from '../../../../test/core/redux/reducerTester'; @@ -49,6 +50,8 @@ import { import * as actions from './query'; import { makeExplorePaneState } from './utils'; +jest.mock('app/features/logs/logsModel'); + const { testRange, defaultInitialState } = createDefaultInitialState(); const exploreId = 'left'; @@ -846,7 +849,7 @@ describe('reducer', () => { }); }); - describe('supplementary queries', () => { + describe('legacy supplementary queries', () => { let dispatch: ThunkDispatch, getState: () => StoreState, unsubscribes: Function[], @@ -878,7 +881,7 @@ describe('reducer', () => { meta: { id: 'something', }, - getDataProvider: () => { + getDataProvider: (_: SupplementaryQueryType, request: DataQueryRequest) => { return mockDataProvider(); }, getSupportedSupplementaryQueryTypes: () => [ @@ -898,6 +901,67 @@ describe('reducer', () => { setupQueryResponse(getState()); }); + it('should load supplementary queries after running the query', async () => { + await dispatch(runQueries({ exploreId: 'left' })); + expect(unsubscribes).toHaveLength(2); + }); + }); + + describe('supplementary queries', () => { + let dispatch: ThunkDispatch, + getState: () => StoreState, + unsubscribes: Function[], + mockDataProvider: () => Observable; + + beforeEach(() => { + unsubscribes = []; + mockDataProvider = () => { + return { + subscribe: () => { + const unsubscribe = jest.fn(); + unsubscribes.push(unsubscribe); + return { + unsubscribe, + }; + }, + } as unknown as Observable; + }; + + jest.mocked(queryLogsVolume).mockImplementation(() => mockDataProvider()); + jest.mocked(queryLogsSample).mockImplementation(() => mockDataProvider()); + + const store: { dispatch: ThunkDispatch; getState: () => StoreState } = configureStore({ + ...defaultInitialState, + explore: { + panes: { + left: { + ...defaultInitialState.explore.panes.left, + datasourceInstance: { + query: jest.fn(), + getRef: jest.fn(), + meta: { + id: 'something', + }, + getSupplementaryRequest: (_: SupplementaryQueryType, request: DataQueryRequest) => { + return request; + }, + getSupportedSupplementaryQueryTypes: () => [ + SupplementaryQueryType.LogsVolume, + SupplementaryQueryType.LogsSample, + ], + getSupplementaryQuery: jest.fn(), + }, + }, + }, + }, + } as unknown as Partial); + + dispatch = store.dispatch; + getState = store.getState; + + setupQueryResponse(getState()); + }); + it('should cancel any unfinished supplementary queries when a new query is run', async () => { await dispatch(runQueries({ exploreId: 'left' })); // first query is run automatically diff --git a/public/app/features/explore/utils/supplementaryQueries.test.ts b/public/app/features/explore/utils/supplementaryQueries.test.ts index 3f13e0d2f51..6c231f4d206 100644 --- a/public/app/features/explore/utils/supplementaryQueries.test.ts +++ b/public/app/features/explore/utils/supplementaryQueries.test.ts @@ -1,10 +1,9 @@ import { flatten } from 'lodash'; -import { from, Observable } from 'rxjs'; +import { Observable, from } from 'rxjs'; import { DataFrame, DataQueryRequest, - DataQueryResponse, DataSourceApi, DataSourceWithSupplementaryQueriesSupport, FieldType, @@ -15,8 +14,8 @@ import { SupplementaryQueryType, SupplementaryQueryOptions, toDataFrame, + DataQueryResponse, } from '@grafana/data'; -import { getDataSourceSrv } from '@grafana/runtime'; import { DataQuery } from '@grafana/schema'; import { MockDataSourceApi } from '../../../../test/mocks/datasource_srv'; @@ -40,21 +39,26 @@ class MockDataSourceWithSupplementaryQuerySupport return this; } - getDataProvider( + query(_: DataQueryRequest): Observable { + const data = + this.supplementaryQueriesResults[SupplementaryQueryType.LogsVolume] || + this.supplementaryQueriesResults[SupplementaryQueryType.LogsSample] || + []; + return from([{ state: LoadingState.Done, data }]); + } + + getSupplementaryRequest( type: SupplementaryQueryType, request: DataQueryRequest - ): Observable | undefined { + ): DataQueryRequest | undefined { const data = this.supplementaryQueriesResults[type]; if (data) { - return from([ - { state: LoadingState.Loading, data: [] }, - { state: LoadingState.Done, data }, - ]); + return request; } return undefined; } - getSupplementaryQuery(options: SupplementaryQueryOptions, query: DataQuery): DataQuery | undefined { + getSupplementaryQuery(_: SupplementaryQueryOptions, query: DataQuery): DataQuery | undefined { return query; } @@ -136,38 +140,27 @@ const datasources: DataSourceApi[] = [ new MockDataSourceApi('no-data-providers-2'), ]; -jest.mock('@grafana/runtime', () => ({ - ...jest.requireActual('@grafana/runtime'), - getDataSourceSrv: () => { - return { - get: async ({ uid }: { uid: string }) => datasources.find((ds) => ds.name === uid) || undefined, - }; - }, -})); - -const setup = async (targetSources: string[], type: SupplementaryQueryType) => { +const setup = (targetSources: string[], type: SupplementaryQueryType) => { const requestMock = new MockDataQueryRequest({ targets: targetSources.map((source, i) => new MockQuery(`${i}`, 'a', { uid: source })), }); const explorePanelDataMock: Observable = mockExploreDataWithLogs(); - const datasources = await Promise.all( - targetSources.map(async (source, i) => { - const datasource = await getDataSourceSrv().get({ uid: source }); - return { - datasource, - targets: [new MockQuery(`${i}`, 'a', { uid: source })], - }; - }) - ); + const groupedQueries = targetSources.map((source, i) => { + const datasource = datasources.find((datasource) => datasource.name === source) || datasources[0]; + return { + datasource, + targets: [new MockQuery(`${i}`, 'a', { uid: datasource.name })], + }; + }); - return getSupplementaryQueryProvider(datasources, type, requestMock, explorePanelDataMock); + return getSupplementaryQueryProvider(groupedQueries, type, requestMock, explorePanelDataMock); }; const assertDataFrom = (type: SupplementaryQueryType, ...datasources: string[]) => { return flatten( datasources.map((name: string) => { - return [{ refId: `1-${type}-${name}` }, { refId: `2-${type}-${name}` }]; + return createSupplementaryQueryResponse(type, name); }) ); }; @@ -179,7 +172,7 @@ const assertDataFromLogsResults = () => { describe('SupplementaryQueries utils', function () { describe('Non-mixed data source', function () { it('Returns result from the provider', async () => { - const testProvider = await setup(['logs-volume-a'], SupplementaryQueryType.LogsVolume); + const testProvider = setup(['logs-volume-a'], SupplementaryQueryType.LogsVolume); await expect(testProvider).toEmitValuesWith((received) => { expect(received).toMatchObject([ @@ -192,7 +185,7 @@ describe('SupplementaryQueries utils', function () { }); }); it('Uses fallback for logs volume', async () => { - const testProvider = await setup(['no-data-providers'], SupplementaryQueryType.LogsVolume); + const testProvider = setup(['no-data-providers'], SupplementaryQueryType.LogsVolume); await expect(testProvider).toEmitValuesWith((received) => { expect(received).toMatchObject([ @@ -204,11 +197,11 @@ describe('SupplementaryQueries utils', function () { }); }); it('Returns undefined for logs sample', async () => { - const testProvider = await setup(['no-data-providers'], SupplementaryQueryType.LogsSample); + const testProvider = setup(['no-data-providers'], SupplementaryQueryType.LogsSample); await expect(testProvider).toBe(undefined); }); it('Creates single fallback result', async () => { - const testProvider = await setup(['no-data-providers', 'no-data-providers-2'], SupplementaryQueryType.LogsVolume); + const testProvider = setup(['no-data-providers', 'no-data-providers-2'], SupplementaryQueryType.LogsVolume); await expect(testProvider).toEmitValuesWith((received) => { expect(received).toMatchObject([ @@ -229,7 +222,7 @@ describe('SupplementaryQueries utils', function () { describe('Logs volume', function () { describe('All data sources support full range logs volume', function () { it('Merges all data frames into a single response', async () => { - const testProvider = await setup(['logs-volume-a', 'logs-volume-b'], SupplementaryQueryType.LogsVolume); + const testProvider = setup(['logs-volume-a', 'logs-volume-b'], SupplementaryQueryType.LogsVolume); await expect(testProvider).toEmitValuesWith((received) => { expect(received).toMatchObject([ { data: [], state: LoadingState.Loading }, @@ -248,10 +241,7 @@ describe('SupplementaryQueries utils', function () { describe('All data sources do not support full range logs volume', function () { it('Creates single fallback result', async () => { - const testProvider = await setup( - ['no-data-providers', 'no-data-providers-2'], - SupplementaryQueryType.LogsVolume - ); + const testProvider = setup(['no-data-providers', 'no-data-providers-2'], SupplementaryQueryType.LogsVolume); await expect(testProvider).toEmitValuesWith((received) => { expect(received).toMatchObject([ @@ -270,7 +260,7 @@ describe('SupplementaryQueries utils', function () { describe('Some data sources support full range logs volume, while others do not', function () { it('Creates merged result containing full range and limited logs volume', async () => { - const testProvider = await setup( + const testProvider = setup( ['logs-volume-a', 'no-data-providers', 'logs-volume-b', 'no-data-providers-2'], SupplementaryQueryType.LogsVolume ); @@ -308,7 +298,7 @@ describe('SupplementaryQueries utils', function () { describe('Logs sample', function () { describe('All data sources support logs sample', function () { it('Merges all responses into single result', async () => { - const testProvider = await setup(['logs-sample-a', 'logs-sample-b'], SupplementaryQueryType.LogsSample); + const testProvider = setup(['logs-sample-a', 'logs-sample-b'], SupplementaryQueryType.LogsSample); await expect(testProvider).toEmitValuesWith((received) => { expect(received).toMatchObject([ { data: [], state: LoadingState.Loading }, @@ -327,17 +317,14 @@ describe('SupplementaryQueries utils', function () { describe('All data sources do not support full range logs volume', function () { it('Does not provide fallback result', async () => { - const testProvider = await setup( - ['no-data-providers', 'no-data-providers-2'], - SupplementaryQueryType.LogsSample - ); + const testProvider = setup(['no-data-providers', 'no-data-providers-2'], SupplementaryQueryType.LogsSample); await expect(testProvider).toBeUndefined(); }); }); describe('Some data sources support full range logs volume, while others do not', function () { it('Returns results only for data sources supporting logs sample', async () => { - const testProvider = await setup( + const testProvider = setup( ['logs-sample-a', 'no-data-providers', 'logs-sample-b', 'no-data-providers-2'], SupplementaryQueryType.LogsSample ); diff --git a/public/app/features/explore/utils/supplementaryQueries.ts b/public/app/features/explore/utils/supplementaryQueries.ts index 9deaca32e20..ae25b8902b4 100644 --- a/public/app/features/explore/utils/supplementaryQueries.ts +++ b/public/app/features/explore/utils/supplementaryQueries.ts @@ -18,7 +18,7 @@ import { import store from 'app/core/store'; import { ExplorePanelData, SupplementaryQueries } from 'app/types'; -import { makeDataFramesForLogs } from '../../logs/logsModel'; +import { makeDataFramesForLogs, queryLogsSample, queryLogsVolume } from '../../logs/logsModel'; export const supplementaryQueryTypes: SupplementaryQueryType[] = [ SupplementaryQueryType.LogsVolume, @@ -130,7 +130,19 @@ export const getSupplementaryQueryProvider = ( dsRequest.targets = targets; if (hasSupplementaryQuerySupport(datasource, type)) { - return datasource.getDataProvider(type, dsRequest); + if (datasource.getDataProvider) { + return datasource.getDataProvider(type, dsRequest); + } else if (datasource.getSupplementaryRequest) { + const request = datasource.getSupplementaryRequest(type, dsRequest); + if (!request) { + return undefined; + } + return type === SupplementaryQueryType.LogsVolume + ? queryLogsVolume(datasource, request, { targets: dsRequest.targets }) + : queryLogsSample(datasource, request); + } else { + return undefined; + } } else { return getSupplementaryQueryFallback(type, explorePanelData, targets, datasource.name); } diff --git a/public/app/features/explore/utils/supplementaryQueries_legacy.test.ts b/public/app/features/explore/utils/supplementaryQueries_legacy.test.ts new file mode 100644 index 00000000000..fbb79119bf8 --- /dev/null +++ b/public/app/features/explore/utils/supplementaryQueries_legacy.test.ts @@ -0,0 +1,365 @@ +/** + * Test file to be removed when `getDataProvider` is removed from DataSourceWithSupplementaryQueriesSupport + * in packages/grafana-data/src/types/logs.ts + */ +import { flatten } from 'lodash'; +import { from, Observable } from 'rxjs'; + +import { + DataFrame, + DataQueryRequest, + DataQueryResponse, + DataSourceApi, + DataSourceWithSupplementaryQueriesSupport, + FieldType, + LoadingState, + LogLevel, + LogsVolumeType, + MutableDataFrame, + SupplementaryQueryType, + SupplementaryQueryOptions, + toDataFrame, +} from '@grafana/data'; +import { getDataSourceSrv } from '@grafana/runtime'; +import { DataQuery } from '@grafana/schema'; + +import { MockDataSourceApi } from '../../../../test/mocks/datasource_srv'; +import { MockDataQueryRequest, MockQuery } from '../../../../test/mocks/query'; +import { ExplorePanelData } from '../../../types'; +import { mockExplorePanelData } from '../__mocks__/data'; + +import { getSupplementaryQueryProvider } from './supplementaryQueries'; + +class MockDataSourceWithSupplementaryQuerySupport + extends MockDataSourceApi + implements DataSourceWithSupplementaryQueriesSupport +{ + private supplementaryQueriesResults: Record = { + [SupplementaryQueryType.LogsVolume]: undefined, + [SupplementaryQueryType.LogsSample]: undefined, + }; + + withSupplementaryQuerySupport(type: SupplementaryQueryType, data: DataFrame[]) { + this.supplementaryQueriesResults[type] = data; + return this; + } + + getDataProvider( + type: SupplementaryQueryType, + request: DataQueryRequest + ): Observable | undefined { + const data = this.supplementaryQueriesResults[type]; + if (data) { + return from([ + { state: LoadingState.Loading, data: [] }, + { state: LoadingState.Done, data }, + ]); + } + return undefined; + } + + getSupplementaryQuery(options: SupplementaryQueryOptions, query: DataQuery): DataQuery | undefined { + return query; + } + + getSupportedSupplementaryQueryTypes(): SupplementaryQueryType[] { + return Object.values(SupplementaryQueryType).filter((type) => this.supplementaryQueriesResults[type]); + } +} + +const createSupplementaryQueryResponse = (type: SupplementaryQueryType, id: string) => { + return [ + toDataFrame({ + refId: `1-${type}-${id}`, + fields: [{ name: 'value', type: FieldType.string, values: [1] }], + meta: { + custom: { + logsVolumeType: LogsVolumeType.FullRange, + }, + }, + }), + toDataFrame({ + refId: `2-${type}-${id}`, + fields: [{ name: 'value', type: FieldType.string, values: [2] }], + meta: { + custom: { + logsVolumeType: LogsVolumeType.FullRange, + }, + }, + }), + ]; +}; + +const mockRow = (refId: string) => { + return { + rowIndex: 0, + entryFieldIndex: 0, + dataFrame: new MutableDataFrame({ refId, fields: [{ name: 'A', values: [] }] }), + entry: '', + hasAnsi: false, + hasUnescapedContent: false, + labels: {}, + logLevel: LogLevel.info, + raw: '', + timeEpochMs: 0, + timeEpochNs: '0', + timeFromNow: '', + timeLocal: '', + timeUtc: '', + uid: '1', + }; +}; + +const mockExploreDataWithLogs = () => + mockExplorePanelData({ + logsResult: { + rows: [mockRow('0'), mockRow('1')], + visibleRange: { from: 0, to: 1 }, + bucketSize: 1000, + }, + }); + +const datasources: DataSourceApi[] = [ + new MockDataSourceWithSupplementaryQuerySupport('logs-volume-a').withSupplementaryQuerySupport( + SupplementaryQueryType.LogsVolume, + createSupplementaryQueryResponse(SupplementaryQueryType.LogsVolume, 'logs-volume-a') + ), + new MockDataSourceWithSupplementaryQuerySupport('logs-volume-b').withSupplementaryQuerySupport( + SupplementaryQueryType.LogsVolume, + createSupplementaryQueryResponse(SupplementaryQueryType.LogsVolume, 'logs-volume-b') + ), + new MockDataSourceWithSupplementaryQuerySupport('logs-sample-a').withSupplementaryQuerySupport( + SupplementaryQueryType.LogsSample, + createSupplementaryQueryResponse(SupplementaryQueryType.LogsSample, 'logs-sample-a') + ), + new MockDataSourceWithSupplementaryQuerySupport('logs-sample-b').withSupplementaryQuerySupport( + SupplementaryQueryType.LogsSample, + createSupplementaryQueryResponse(SupplementaryQueryType.LogsSample, 'logs-sample-b') + ), + new MockDataSourceApi('no-data-providers'), + new MockDataSourceApi('no-data-providers-2'), +]; + +jest.mock('@grafana/runtime', () => ({ + ...jest.requireActual('@grafana/runtime'), + getDataSourceSrv: () => { + return { + get: async ({ uid }: { uid: string }) => datasources.find((ds) => ds.name === uid) || undefined, + }; + }, +})); + +const setup = async (targetSources: string[], type: SupplementaryQueryType) => { + const requestMock = new MockDataQueryRequest({ + targets: targetSources.map((source, i) => new MockQuery(`${i}`, 'a', { uid: source })), + }); + const explorePanelDataMock: Observable = mockExploreDataWithLogs(); + + const datasources = await Promise.all( + targetSources.map(async (source, i) => { + const datasource = await getDataSourceSrv().get({ uid: source }); + return { + datasource, + targets: [new MockQuery(`${i}`, 'a', { uid: source })], + }; + }) + ); + + return getSupplementaryQueryProvider(datasources, type, requestMock, explorePanelDataMock); +}; + +const assertDataFrom = (type: SupplementaryQueryType, ...datasources: string[]) => { + return flatten( + datasources.map((name: string) => { + return [{ refId: `1-${type}-${name}` }, { refId: `2-${type}-${name}` }]; + }) + ); +}; + +const assertDataFromLogsResults = () => { + return [{ meta: { custom: { logsVolumeType: LogsVolumeType.Limited } } }]; +}; + +describe('SupplementaryQueries utils', function () { + describe('Non-mixed data source', function () { + it('Returns result from the provider', async () => { + const testProvider = await setup(['logs-volume-a'], SupplementaryQueryType.LogsVolume); + + await expect(testProvider).toEmitValuesWith((received) => { + expect(received).toMatchObject([ + { data: [], state: LoadingState.Loading }, + { + data: assertDataFrom(SupplementaryQueryType.LogsVolume, 'logs-volume-a'), + state: LoadingState.Done, + }, + ]); + }); + }); + it('Uses fallback for logs volume', async () => { + const testProvider = await setup(['no-data-providers'], SupplementaryQueryType.LogsVolume); + + await expect(testProvider).toEmitValuesWith((received) => { + expect(received).toMatchObject([ + { + data: assertDataFromLogsResults(), + state: LoadingState.Done, + }, + ]); + }); + }); + it('Returns undefined for logs sample', async () => { + const testProvider = await setup(['no-data-providers'], SupplementaryQueryType.LogsSample); + await expect(testProvider).toBe(undefined); + }); + it('Creates single fallback result', async () => { + const testProvider = await setup(['no-data-providers', 'no-data-providers-2'], SupplementaryQueryType.LogsVolume); + + await expect(testProvider).toEmitValuesWith((received) => { + expect(received).toMatchObject([ + { + data: assertDataFromLogsResults(), + state: LoadingState.Done, + }, + { + data: [...assertDataFromLogsResults(), ...assertDataFromLogsResults()], + state: LoadingState.Done, + }, + ]); + }); + }); + }); + + describe('Mixed data source', function () { + describe('Logs volume', function () { + describe('All data sources support full range logs volume', function () { + it('Merges all data frames into a single response', async () => { + const testProvider = await setup(['logs-volume-a', 'logs-volume-b'], SupplementaryQueryType.LogsVolume); + await expect(testProvider).toEmitValuesWith((received) => { + expect(received).toMatchObject([ + { data: [], state: LoadingState.Loading }, + { + data: assertDataFrom(SupplementaryQueryType.LogsVolume, 'logs-volume-a'), + state: LoadingState.Done, + }, + { + data: assertDataFrom(SupplementaryQueryType.LogsVolume, 'logs-volume-a', 'logs-volume-b'), + state: LoadingState.Done, + }, + ]); + }); + }); + }); + + describe('All data sources do not support full range logs volume', function () { + it('Creates single fallback result', async () => { + const testProvider = await setup( + ['no-data-providers', 'no-data-providers-2'], + SupplementaryQueryType.LogsVolume + ); + + await expect(testProvider).toEmitValuesWith((received) => { + expect(received).toMatchObject([ + { + data: assertDataFromLogsResults(), + state: LoadingState.Done, + }, + { + data: [...assertDataFromLogsResults(), ...assertDataFromLogsResults()], + state: LoadingState.Done, + }, + ]); + }); + }); + }); + + describe('Some data sources support full range logs volume, while others do not', function () { + it('Creates merged result containing full range and limited logs volume', async () => { + const testProvider = await setup( + ['logs-volume-a', 'no-data-providers', 'logs-volume-b', 'no-data-providers-2'], + SupplementaryQueryType.LogsVolume + ); + await expect(testProvider).toEmitValuesWith((received) => { + expect(received).toMatchObject([ + { + data: [], + state: LoadingState.Loading, + }, + { + data: assertDataFrom(SupplementaryQueryType.LogsVolume, 'logs-volume-a'), + state: LoadingState.Done, + }, + { + data: [ + ...assertDataFrom(SupplementaryQueryType.LogsVolume, 'logs-volume-a'), + ...assertDataFromLogsResults(), + ], + state: LoadingState.Done, + }, + { + data: [ + ...assertDataFrom(SupplementaryQueryType.LogsVolume, 'logs-volume-a'), + ...assertDataFromLogsResults(), + ...assertDataFrom(SupplementaryQueryType.LogsVolume, 'logs-volume-b'), + ], + state: LoadingState.Done, + }, + ]); + }); + }); + }); + }); + + describe('Logs sample', function () { + describe('All data sources support logs sample', function () { + it('Merges all responses into single result', async () => { + const testProvider = await setup(['logs-sample-a', 'logs-sample-b'], SupplementaryQueryType.LogsSample); + await expect(testProvider).toEmitValuesWith((received) => { + expect(received).toMatchObject([ + { data: [], state: LoadingState.Loading }, + { + data: assertDataFrom(SupplementaryQueryType.LogsSample, 'logs-sample-a'), + state: LoadingState.Done, + }, + { + data: assertDataFrom(SupplementaryQueryType.LogsSample, 'logs-sample-a', 'logs-sample-b'), + state: LoadingState.Done, + }, + ]); + }); + }); + }); + + describe('All data sources do not support full range logs volume', function () { + it('Does not provide fallback result', async () => { + const testProvider = await setup( + ['no-data-providers', 'no-data-providers-2'], + SupplementaryQueryType.LogsSample + ); + await expect(testProvider).toBeUndefined(); + }); + }); + + describe('Some data sources support full range logs volume, while others do not', function () { + it('Returns results only for data sources supporting logs sample', async () => { + const testProvider = await setup( + ['logs-sample-a', 'no-data-providers', 'logs-sample-b', 'no-data-providers-2'], + SupplementaryQueryType.LogsSample + ); + await expect(testProvider).toEmitValuesWith((received) => { + expect(received).toMatchObject([ + { data: [], state: LoadingState.Loading }, + { + data: assertDataFrom(SupplementaryQueryType.LogsSample, 'logs-sample-a'), + state: LoadingState.Done, + }, + { + data: assertDataFrom(SupplementaryQueryType.LogsSample, 'logs-sample-a', 'logs-sample-b'), + state: LoadingState.Done, + }, + ]); + }); + }); + }); + }); + }); +}); diff --git a/public/app/features/logs/logsModel.test.ts b/public/app/features/logs/logsModel.test.ts index d5894aeae39..05883b14849 100644 --- a/public/app/features/logs/logsModel.test.ts +++ b/public/app/features/logs/logsModel.test.ts @@ -1317,16 +1317,22 @@ describe('logs volume', () => { { refId: 'B', target: 'volume query 2' }, ], scopedVars: {}, - } as unknown as DataQueryRequest; - volumeProvider = queryLogsVolume(datasource, request, { - extractLevel: (dataFrame: DataFrame) => { - return dataFrame.fields[1]!.labels!.level === 'error' ? LogLevel.error : LogLevel.unknown; - }, + requestId: '', + interval: '', + intervalMs: 0, range: { from: FROM, to: TO, - raw: { from: '0', to: '1' }, + raw: { + from: FROM, + to: TO, + }, }, + timezone: '', + app: '', + startTime: 0, + }; + volumeProvider = queryLogsVolume(datasource, request, { targets: request.targets, }); } diff --git a/public/app/features/logs/logsModel.ts b/public/app/features/logs/logsModel.ts index d0afaf5c95c..396553207dc 100644 --- a/public/app/features/logs/logsModel.ts +++ b/public/app/features/logs/logsModel.ts @@ -35,7 +35,6 @@ import { ScopedVars, sortDataFrame, textUtil, - TimeRange, toDataFrame, toUtc, } from '@grafana/data'; @@ -641,11 +640,22 @@ const updateLogsVolumeConfig = ( }; type LogsVolumeQueryOptions = { - extractLevel: (dataFrame: DataFrame) => LogLevel; targets: T[]; - range: TimeRange; }; +function defaultExtractLevel(dataFrame: DataFrame): LogLevel { + let valueField; + try { + valueField = new FieldCache(dataFrame).getFirstFieldOfType(FieldType.number); + } catch {} + return valueField?.labels ? getLogLevelFromLabels(valueField.labels) : LogLevel.unknown; +} + +function getLogLevelFromLabels(labels: Labels): LogLevel { + const level = labels['level'] ?? labels['lvl'] ?? labels['loglevel'] ?? ''; + return level ? getLogLevelFromKey(level) : LogLevel.unknown; +} + /** * Creates an observable, which makes requests to get logs volume and aggregates results. */ @@ -654,7 +664,10 @@ export function queryLogsVolume, options: LogsVolumeQueryOptions ): Observable { - const timespan = options.range.to.valueOf() - options.range.from.valueOf(); + const range = logsVolumeRequest.range; + const targets = options.targets; + const extractLevel = defaultExtractLevel; + const timespan = range.to.valueOf() - range.from.valueOf(); const intervalInfo = getIntervalInfo(logsVolumeRequest.scopedVars, timespan); logsVolumeRequest.interval = intervalInfo.interval; @@ -705,9 +718,9 @@ export function queryLogsVolume dataQuery.refId === sourceRefId)!, + sourceQuery: targets.find((dataQuery) => dataQuery.refId === sourceRefId)!, }; dataFrame.meta = { @@ -717,7 +730,7 @@ export function queryLogsVolume { ], }); - expect(ds.getDataProvider(SupplementaryQueryType.LogsSample, options)).not.toBeDefined(); + expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsSample, options)).not.toBeDefined(); }); it('does create a logs sample provider for time series query', () => { @@ -999,7 +999,7 @@ describe('ElasticDatasource', () => { ], }); - expect(ds.getDataProvider(SupplementaryQueryType.LogsSample, options)).toBeDefined(); + expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsSample, options)).toBeDefined(); }); }); @@ -1019,7 +1019,7 @@ describe('ElasticDatasource', () => { ], }); - expect(ds.getLogsSampleDataProvider(request)).not.toBeDefined(); + expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsSample, request)).not.toBeDefined(); }); it('returns a logs sample provider given a time series query', () => { @@ -1032,7 +1032,7 @@ describe('ElasticDatasource', () => { ], }); - expect(ds.getLogsSampleDataProvider(request)).toBeDefined(); + expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsSample, request)).toBeDefined(); }); }); }); diff --git a/public/app/plugins/datasource/elasticsearch/datasource.ts b/public/app/plugins/datasource/elasticsearch/datasource.ts index 7aa1ece56a4..04a98ab3f12 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.ts @@ -31,7 +31,6 @@ import { SupplementaryQueryOptions, toUtc, AnnotationEvent, - FieldType, DataSourceWithToggleableQueryFiltersSupport, QueryFilterOptions, ToggleFilterAction, @@ -48,9 +47,6 @@ import { getTemplateSrv, } from '@grafana/runtime'; -import { queryLogsSample, queryLogsVolume } from '../../../features/logs/logsModel'; -import { getLogLevelFromKey } from '../../../features/logs/utils'; - import { IndexPattern, intervalMap } from './IndexPattern'; import LanguageProvider from './LanguageProvider'; import { LegacyQueryRunner } from './LegacyQueryRunner'; @@ -538,13 +534,15 @@ export class ElasticDatasource } }; - getDataProvider( + /** + * Implemented for DataSourceWithSupplementaryQueriesSupport. + * It generates a DataQueryRequest for a specific supplementary query type. + * @returns A DataQueryRequest for the supplementary queries or undefined if not supported. + */ + getSupplementaryRequest( type: SupplementaryQueryType, request: DataQueryRequest - ): Observable | undefined { - if (!this.getSupportedSupplementaryQueryTypes().includes(type)) { - return undefined; - } + ): DataQueryRequest | undefined { switch (type) { case SupplementaryQueryType.LogsVolume: return this.getLogsVolumeDataProvider(request); @@ -560,10 +558,6 @@ export class ElasticDatasource } getSupplementaryQuery(options: SupplementaryQueryOptions, query: ElasticsearchQuery): ElasticsearchQuery | undefined { - if (!this.getSupportedSupplementaryQueryTypes().includes(options.type)) { - return undefined; - } - let isQuerySuitable = false; switch (options.type) { @@ -635,7 +629,9 @@ export class ElasticDatasource } } - getLogsVolumeDataProvider(request: DataQueryRequest): Observable | undefined { + private getLogsVolumeDataProvider( + request: DataQueryRequest + ): DataQueryRequest | undefined { const logsVolumeRequest = cloneDeep(request); const targets = logsVolumeRequest.targets .map((target) => this.getSupplementaryQuery({ type: SupplementaryQueryType.LogsVolume }, target)) @@ -645,18 +641,12 @@ export class ElasticDatasource return undefined; } - return queryLogsVolume( - this, - { ...logsVolumeRequest, targets }, - { - range: request.range, - targets: request.targets, - extractLevel, - } - ); + return { ...logsVolumeRequest, targets }; } - getLogsSampleDataProvider(request: DataQueryRequest): Observable | undefined { + private getLogsSampleDataProvider( + request: DataQueryRequest + ): DataQueryRequest | undefined { const logsSampleRequest = cloneDeep(request); const targets = logsSampleRequest.targets; const queries = targets.map((query) => { @@ -667,7 +657,7 @@ export class ElasticDatasource if (!elasticQueries.length) { return undefined; } - return queryLogsSample(this, { ...logsSampleRequest, targets: elasticQueries }); + return { ...logsSampleRequest, targets: elasticQueries }; } query(request: DataQueryRequest): Observable { @@ -1178,9 +1168,3 @@ function createContextTimeRange(rowTimeEpochMs: number, direction: string, inter } } } - -function extractLevel(dataFrame: DataFrame): LogLevel { - const valueField = dataFrame.fields.find((f) => f.type === FieldType.number); - const name = valueField?.labels?.['level'] ?? ''; - return getLogLevelFromKey(name); -} diff --git a/public/app/plugins/datasource/loki/datasource.test.ts b/public/app/plugins/datasource/loki/datasource.test.ts index f1f175f18cc..ee3b164db4b 100644 --- a/public/app/plugins/datasource/loki/datasource.test.ts +++ b/public/app/plugins/datasource/loki/datasource.test.ts @@ -1255,7 +1255,7 @@ describe('LokiDatasource', () => { targets: [{ expr: '{label="value"}', refId: 'A', queryType: LokiQueryType.Range }], }; - expect(ds.getDataProvider(SupplementaryQueryType.LogsVolume, options)).toBeDefined(); + expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsVolume, options)).toBeDefined(); }); it('does not create provider for metrics query', () => { @@ -1264,7 +1264,7 @@ describe('LokiDatasource', () => { targets: [{ expr: 'rate({label="value"}[1m])', refId: 'A' }], }; - expect(ds.getDataProvider(SupplementaryQueryType.LogsVolume, options)).not.toBeDefined(); + expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsVolume, options)).not.toBeDefined(); }); it('creates provider if at least one query is a logs query', () => { @@ -1276,7 +1276,7 @@ describe('LokiDatasource', () => { ], }; - expect(ds.getDataProvider(SupplementaryQueryType.LogsVolume, options)).toBeDefined(); + expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsVolume, options)).toBeDefined(); }); it('does not create provider if there is only an instant logs query', () => { @@ -1285,7 +1285,7 @@ describe('LokiDatasource', () => { targets: [{ expr: '{label="value"', refId: 'A', queryType: LokiQueryType.Instant }], }; - expect(ds.getDataProvider(SupplementaryQueryType.LogsVolume, options)).not.toBeDefined(); + expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsVolume, options)).not.toBeDefined(); }); }); @@ -1301,7 +1301,7 @@ describe('LokiDatasource', () => { targets: [{ expr: 'rate({label="value"}[5m])', refId: 'A' }], }; - expect(ds.getDataProvider(SupplementaryQueryType.LogsSample, options)).toBeDefined(); + expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsSample, options)).toBeDefined(); }); it('does not create provider for log query', () => { @@ -1310,7 +1310,7 @@ describe('LokiDatasource', () => { targets: [{ expr: '{label="value"}', refId: 'A' }], }; - expect(ds.getDataProvider(SupplementaryQueryType.LogsSample, options)).not.toBeDefined(); + expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsSample, options)).not.toBeDefined(); }); it('creates provider if at least one query is a metric query', () => { @@ -1322,7 +1322,7 @@ describe('LokiDatasource', () => { ], }; - expect(ds.getDataProvider(SupplementaryQueryType.LogsSample, options)).toBeDefined(); + expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsSample, options)).toBeDefined(); }); }); diff --git a/public/app/plugins/datasource/loki/datasource.ts b/public/app/plugins/datasource/loki/datasource.ts index d07fd7cb295..8277285cd5c 100644 --- a/public/app/plugins/datasource/loki/datasource.ts +++ b/public/app/plugins/datasource/loki/datasource.ts @@ -17,11 +17,8 @@ import { SupplementaryQueryType, DataSourceWithQueryExportSupport, DataSourceWithQueryImportSupport, - FieldCache, - FieldType, Labels, LoadingState, - LogLevel, LogRowModel, QueryFixAction, QueryHint, @@ -46,9 +43,6 @@ import { Duration } from '@grafana/lezer-logql'; import { BackendSrvRequest, config, DataSourceWithBackend, getTemplateSrv, TemplateSrv } from '@grafana/runtime'; import { DataQuery } from '@grafana/schema'; -import { queryLogsSample, queryLogsVolume } from '../../../features/logs/logsModel'; -import { getLogLevelFromKey } from '../../../features/logs/utils'; - import LanguageProvider from './LanguageProvider'; import { LiveStreams, LokiLiveTarget } from './LiveStreams'; import { LogContextProvider } from './LogContextProvider'; @@ -168,16 +162,13 @@ export class LokiDatasource /** * Implemented for DataSourceWithSupplementaryQueriesSupport. - * It retrieves a data provider for a specific supplementary query type. - * @returns An Observable of DataQueryResponse or undefined if the specified query type is not supported. + * It generates a DataQueryRequest for a specific supplementary query type. + * @returns A DataQueryRequest for the supplementary queries or undefined if not supported. */ - getDataProvider( + getSupplementaryRequest( type: SupplementaryQueryType, request: DataQueryRequest - ): Observable | undefined { - if (!this.getSupportedSupplementaryQueryTypes().includes(type)) { - return undefined; - } + ): DataQueryRequest | undefined { switch (type) { case SupplementaryQueryType.LogsVolume: return this.getLogsVolumeDataProvider(request); @@ -203,10 +194,6 @@ export class LokiDatasource * @returns A supplemented Loki query or undefined if unsupported. */ getSupplementaryQuery(options: SupplementaryQueryOptions, query: LokiQuery): LokiQuery | undefined { - if (!this.getSupportedSupplementaryQueryTypes().includes(options.type)) { - return undefined; - } - const normalizedQuery = getNormalizedLokiQuery(query); let expr = removeCommentsFromQuery(normalizedQuery.expr); let isQuerySuitable = false; @@ -255,7 +242,7 @@ export class LokiDatasource * Private method used in the `getDataProvider` for DataSourceWithSupplementaryQueriesSupport, specifically for Logs volume queries. * @returns An Observable of DataQueryResponse or undefined if no suitable queries are found. */ - private getLogsVolumeDataProvider(request: DataQueryRequest): Observable | undefined { + private getLogsVolumeDataProvider(request: DataQueryRequest): DataQueryRequest | undefined { const logsVolumeRequest = cloneDeep(request); const targets = logsVolumeRequest.targets .map((query) => this.getSupplementaryQuery({ type: SupplementaryQueryType.LogsVolume }, query)) @@ -265,22 +252,14 @@ export class LokiDatasource return undefined; } - return queryLogsVolume( - this, - { ...logsVolumeRequest, targets }, - { - extractLevel, - range: request.range, - targets: request.targets, - } - ); + return { ...logsVolumeRequest, targets }; } /** * Private method used in the `getDataProvider` for DataSourceWithSupplementaryQueriesSupport, specifically for Logs sample queries. * @returns An Observable of DataQueryResponse or undefined if no suitable queries are found. */ - private getLogsSampleDataProvider(request: DataQueryRequest): Observable | undefined { + private getLogsSampleDataProvider(request: DataQueryRequest): DataQueryRequest | undefined { const logsSampleRequest = cloneDeep(request); const targets = logsSampleRequest.targets .map((query) => this.getSupplementaryQuery({ type: SupplementaryQueryType.LogsSample, limit: 100 }, query)) @@ -289,7 +268,7 @@ export class LokiDatasource if (!targets.length) { return undefined; } - return queryLogsSample(this, { ...logsSampleRequest, targets }); + return { ...logsSampleRequest, targets }; } /** @@ -1171,23 +1150,3 @@ export function lokiSpecialRegexEscape(value: any) { } return value; } - -function extractLevel(dataFrame: DataFrame): LogLevel { - let valueField; - try { - valueField = new FieldCache(dataFrame).getFirstFieldOfType(FieldType.number); - } catch {} - return valueField?.labels ? getLogLevelFromLabels(valueField.labels) : LogLevel.unknown; -} - -function getLogLevelFromLabels(labels: Labels): LogLevel { - const labelNames = ['level', 'lvl', 'loglevel']; - let levelLabel; - for (let labelName of labelNames) { - if (labelName in labels) { - levelLabel = labelName; - break; - } - } - return levelLabel ? getLogLevelFromKey(labels[levelLabel]) : LogLevel.unknown; -} diff --git a/public/test/mocks/datasource_srv.ts b/public/test/mocks/datasource_srv.ts index 9da05b83ac2..91ed06201e6 100644 --- a/public/test/mocks/datasource_srv.ts +++ b/public/test/mocks/datasource_srv.ts @@ -49,7 +49,7 @@ export class MockDataSourceApi extends DataSourceApi { this.meta = meta || ({} as DataSourcePluginMeta); } - query(request: DataQueryRequest): Promise { + query(request: DataQueryRequest): Promise | Observable { if (this.error) { return Promise.reject(this.error); }