IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Supplementary queries: allow plugin decoupling by allowing providers to return a request instance (#80281)

Browse Source 
* Supplementary queries: add support for providers returning a request instance

* Formatting

* DataSourceWithSupplementaryQueriesSupport: update getDataProvider signature

* getLogLevelFromLabels: fix buggy implementation

* getLogLevelFromKey: fix key type

Why number??

* Revert "getLogLevelFromKey: fix key type"

This reverts commit 14a95298a6f803cc3270e0421b2e04dd0d65f131.

* getSupplementaryQueryProvider: remove observable support

* Datasources: remove unnecessary check

The switch is doing the same job

* Supplementary queries: update unit test

* datasource_srv: sync mock with real api

* Formatting

* Supplementary queries: pass targets from getSupplementaryQueryProvider

* LogsVolumeQueryOptions: remove range and make extract level optional

* logsModel: add missing range to test data

* query: sync tests with changes

* Formatting

* DataSourceWithSupplementaryQueriesSupport: update interface with deprecated and new methods

* DataSourceWithSupplementaryQueriesSupport: sync Loki and Elasticsearch

* queryLogsVolume: extractLevel no longer customizable

* Loki: update test

* Supplementary queries: add support for the new method

* hasSupplementaryQuerySupport: update signature

* Formatting

* Betterer

* Query: update test

* Supplementary queries: add test for the legacy API

* Update public/app/features/explore/utils/supplementaryQueries.ts

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

---------

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>
This commit is contained in:
Matias Chomicki 2024-01-30 11:36:20 +01:00 committed by GitHub
parent ecf0c2c1c9
commit 2a734a9e3f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 567 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.betterer.results
View File

@ -328,9 +328,6 @@ exports[`better eslint`] = {
[0, 0, 0, "Do not use any type assertions.", "7"],
[0, 0, 0, "Do not use any type assertions.", "8"]
],
"packages/grafana-data/src/types/logs.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"packages/grafana-data/src/types/options.ts:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"],
[0, 0, 0, "Unexpected any. Specify a different type.", "1"]

28
packages/grafana-data/src/types/logs.ts
View File

@ -4,7 +4,7 @@ import { DataQuery } from '@grafana/schema';
import { KeyValue, Labels } from './data';
import { DataFrame } from './dataFrame';
import { DataQueryRequest, DataQueryResponse, QueryFixAction, QueryFixType } from './datasource';
import { DataQueryRequest, DataQueryResponse, DataSourceApi, QueryFixAction, QueryFixType } from './datasource';
import { AbsoluteTimeRange } from './time';
export { LogsDedupStrategy, LogsSortOrder } from '@grafana/schema';
@ -225,36 +225,44 @@ export interface DataSourceWithSupplementaryQueriesSupport<TQuery extends DataQu
/**
* Returns an observable that will be used to fetch supplementary data based on the provided
* supplementary query type and original request.
* @deprecated Use getSupplementaryQueryRequest() instead
*/
getDataProvider(
getDataProvider?(
type: SupplementaryQueryType,
request: DataQueryRequest<TQuery>
): Observable<DataQueryResponse> | undefined;
/**
* Receives a SupplementaryQueryType and a DataQueryRequest and returns a new DataQueryRequest to fetch supplementary data.
* If provided type or request is not suitable for a supplementary data request, returns undefined.
*/
getSupplementaryRequest?(
type: SupplementaryQueryType,
request: DataQueryRequest<TQuery>
): DataQueryRequest<TQuery> | undefined;
/**
* Returns supplementary query types that data source supports.
*/
getSupportedSupplementaryQueryTypes(): SupplementaryQueryType[];
/**
* Returns a supplementary query to be used to fetch supplementary data based on the provided type and original query.
* If provided query is not suitable for provided supplementary query type, undefined should be returned.
* If the provided query is not suitable for the provided supplementary query type, undefined should be returned.
*/
getSupplementaryQuery(options: SupplementaryQueryOptions, originalQuery: TQuery): TQuery | undefined;
}
export const hasSupplementaryQuerySupport = <TQuery extends DataQuery>(
datasource: unknown,
datasource: DataSourceApi | (DataSourceApi & DataSourceWithSupplementaryQueriesSupport<TQuery>),
type: SupplementaryQueryType
): datasource is DataSourceWithSupplementaryQueriesSupport<TQuery> => {
): datasource is DataSourceApi & DataSourceWithSupplementaryQueriesSupport<TQuery> => {
if (!datasource) {
return false;
}
const withSupplementaryQueriesSupport = datasource as DataSourceWithSupplementaryQueriesSupport<TQuery>;
return (
withSupplementaryQueriesSupport.getDataProvider !== undefined &&
withSupplementaryQueriesSupport.getSupplementaryQuery !== undefined &&
withSupplementaryQueriesSupport.getSupportedSupplementaryQueryTypes().includes(type)
('getDataProvider' in datasource || 'getSupplementaryRequest' in datasource) &&
'getSupplementaryQuery' in datasource &&
'getSupportedSupplementaryQueryTypes' in datasource &&
datasource.getSupportedSupplementaryQueryTypes().includes(type)
);
};

3
public/app/features/explore/Logs/LogsSamplePanel.tsx
View File

@ -45,6 +45,9 @@ export function LogsSamplePanel(props: Props) {
};
const OpenInSplitViewButton = () => {
if (!datasourceInstance) {
return null;
}
if (!hasSupplementaryQuerySupport(datasourceInstance, SupplementaryQueryType.LogsSample)) {
return null;
}

68
public/app/features/explore/state/query.test.ts
View File

@ -16,6 +16,7 @@ import {
SupplementaryQueryType,
} from '@grafana/data';
import { DataQuery, DataSourceRef } from '@grafana/schema';
import { queryLogsSample, queryLogsVolume } from 'app/features/logs/logsModel';
import { createAsyncThunk, ExploreItemState, StoreState, ThunkDispatch } from 'app/types';
import { reducerTester } from '../../../../test/core/redux/reducerTester';
@ -49,6 +50,8 @@ import {
import * as actions from './query';
import { makeExplorePaneState } from './utils';
jest.mock('app/features/logs/logsModel');
const { testRange, defaultInitialState } = createDefaultInitialState();
const exploreId = 'left';
@ -846,7 +849,7 @@ describe('reducer', () => {
});
});
describe('supplementary queries', () => {
describe('legacy supplementary queries', () => {
let dispatch: ThunkDispatch,
getState: () => StoreState,
unsubscribes: Function[],
@ -878,7 +881,7 @@ describe('reducer', () => {
meta: {
id: 'something',
},
getDataProvider: () => {
getDataProvider: (_: SupplementaryQueryType, request: DataQueryRequest<DataQuery>) => {
return mockDataProvider();
},
getSupportedSupplementaryQueryTypes: () => [
@ -898,6 +901,67 @@ describe('reducer', () => {
setupQueryResponse(getState());
});
it('should load supplementary queries after running the query', async () => {
await dispatch(runQueries({ exploreId: 'left' }));
expect(unsubscribes).toHaveLength(2);
});
});
describe('supplementary queries', () => {
let dispatch: ThunkDispatch,
getState: () => StoreState,
unsubscribes: Function[],
mockDataProvider: () => Observable<DataQueryResponse>;
beforeEach(() => {
unsubscribes = [];
mockDataProvider = () => {
return {
subscribe: () => {
const unsubscribe = jest.fn();
unsubscribes.push(unsubscribe);
return {
unsubscribe,
};
},
} as unknown as Observable<DataQueryResponse>;
};
jest.mocked(queryLogsVolume).mockImplementation(() => mockDataProvider());
jest.mocked(queryLogsSample).mockImplementation(() => mockDataProvider());
const store: { dispatch: ThunkDispatch; getState: () => StoreState } = configureStore({
...defaultInitialState,
explore: {
panes: {
left: {
...defaultInitialState.explore.panes.left,
datasourceInstance: {
query: jest.fn(),
getRef: jest.fn(),
meta: {
id: 'something',
},
getSupplementaryRequest: (_: SupplementaryQueryType, request: DataQueryRequest<DataQuery>) => {
return request;
},
getSupportedSupplementaryQueryTypes: () => [
SupplementaryQueryType.LogsVolume,
SupplementaryQueryType.LogsSample,
],
getSupplementaryQuery: jest.fn(),
},
},
},
},
} as unknown as Partial<StoreState>);
dispatch = store.dispatch;
getState = store.getState;
setupQueryResponse(getState());
});
it('should cancel any unfinished supplementary queries when a new query is run', async () => {
await dispatch(runQueries({ exploreId: 'left' }));
// first query is run automatically

81
public/app/features/explore/utils/supplementaryQueries.test.ts
View File

@ -1,10 +1,9 @@
import { flatten } from 'lodash';
import { from, Observable } from 'rxjs';
import { Observable, from } from 'rxjs';
import {
DataFrame,
DataQueryRequest,
DataQueryResponse,
DataSourceApi,
DataSourceWithSupplementaryQueriesSupport,
FieldType,
@ -15,8 +14,8 @@ import {
SupplementaryQueryType,
SupplementaryQueryOptions,
toDataFrame,
DataQueryResponse,
} from '@grafana/data';
import { getDataSourceSrv } from '@grafana/runtime';
import { DataQuery } from '@grafana/schema';
import { MockDataSourceApi } from '../../../../test/mocks/datasource_srv';
@ -40,21 +39,26 @@ class MockDataSourceWithSupplementaryQuerySupport
return this;
}
getDataProvider(
query(_: DataQueryRequest): Observable<DataQueryResponse> {
const data =
this.supplementaryQueriesResults[SupplementaryQueryType.LogsVolume] ||
this.supplementaryQueriesResults[SupplementaryQueryType.LogsSample] ||
[];
return from([{ state: LoadingState.Done, data }]);
}
getSupplementaryRequest(
type: SupplementaryQueryType,
request: DataQueryRequest<DataQuery>
): Observable<DataQueryResponse> | undefined {
): DataQueryRequest<DataQuery> | undefined {
const data = this.supplementaryQueriesResults[type];
if (data) {
return from([
{ state: LoadingState.Loading, data: [] },
{ state: LoadingState.Done, data },
]);
return request;
}
return undefined;
}
getSupplementaryQuery(options: SupplementaryQueryOptions, query: DataQuery): DataQuery | undefined {
getSupplementaryQuery(_: SupplementaryQueryOptions, query: DataQuery): DataQuery | undefined {
return query;
}
@ -136,38 +140,27 @@ const datasources: DataSourceApi[] = [
new MockDataSourceApi('no-data-providers-2'),
];
jest.mock('@grafana/runtime', () => ({
...jest.requireActual('@grafana/runtime'),
getDataSourceSrv: () => {
return {
get: async ({ uid }: { uid: string }) => datasources.find((ds) => ds.name === uid) || undefined,
};
},
}));
const setup = async (targetSources: string[], type: SupplementaryQueryType) => {
const setup = (targetSources: string[], type: SupplementaryQueryType) => {
const requestMock = new MockDataQueryRequest({
targets: targetSources.map((source, i) => new MockQuery(`${i}`, 'a', { uid: source })),
});
const explorePanelDataMock: Observable<ExplorePanelData> = mockExploreDataWithLogs();
const datasources = await Promise.all(
targetSources.map(async (source, i) => {
const datasource = await getDataSourceSrv().get({ uid: source });
return {
datasource,
targets: [new MockQuery(`${i}`, 'a', { uid: source })],
};
})
);
const groupedQueries = targetSources.map((source, i) => {
const datasource = datasources.find((datasource) => datasource.name === source) || datasources[0];
return {
datasource,
targets: [new MockQuery(`${i}`, 'a', { uid: datasource.name })],
};
});
return getSupplementaryQueryProvider(datasources, type, requestMock, explorePanelDataMock);
return getSupplementaryQueryProvider(groupedQueries, type, requestMock, explorePanelDataMock);
};
const assertDataFrom = (type: SupplementaryQueryType, ...datasources: string[]) => {
return flatten(
datasources.map((name: string) => {
return [{ refId: `1-${type}-${name}` }, { refId: `2-${type}-${name}` }];
return createSupplementaryQueryResponse(type, name);
})
);
};
@ -179,7 +172,7 @@ const assertDataFromLogsResults = () => {
describe('SupplementaryQueries utils', function () {
describe('Non-mixed data source', function () {
it('Returns result from the provider', async () => {
const testProvider = await setup(['logs-volume-a'], SupplementaryQueryType.LogsVolume);
const testProvider = setup(['logs-volume-a'], SupplementaryQueryType.LogsVolume);
await expect(testProvider).toEmitValuesWith((received) => {
expect(received).toMatchObject([
@ -192,7 +185,7 @@ describe('SupplementaryQueries utils', function () {
});
});
it('Uses fallback for logs volume', async () => {
const testProvider = await setup(['no-data-providers'], SupplementaryQueryType.LogsVolume);
const testProvider = setup(['no-data-providers'], SupplementaryQueryType.LogsVolume);
await expect(testProvider).toEmitValuesWith((received) => {
expect(received).toMatchObject([
@ -204,11 +197,11 @@ describe('SupplementaryQueries utils', function () {
});
});
it('Returns undefined for logs sample', async () => {
const testProvider = await setup(['no-data-providers'], SupplementaryQueryType.LogsSample);
const testProvider = setup(['no-data-providers'], SupplementaryQueryType.LogsSample);
await expect(testProvider).toBe(undefined);
});
it('Creates single fallback result', async () => {
const testProvider = await setup(['no-data-providers', 'no-data-providers-2'], SupplementaryQueryType.LogsVolume);
const testProvider = setup(['no-data-providers', 'no-data-providers-2'], SupplementaryQueryType.LogsVolume);
await expect(testProvider).toEmitValuesWith((received) => {
expect(received).toMatchObject([
@ -229,7 +222,7 @@ describe('SupplementaryQueries utils', function () {
describe('Logs volume', function () {
describe('All data sources support full range logs volume', function () {
it('Merges all data frames into a single response', async () => {
const testProvider = await setup(['logs-volume-a', 'logs-volume-b'], SupplementaryQueryType.LogsVolume);
const testProvider = setup(['logs-volume-a', 'logs-volume-b'], SupplementaryQueryType.LogsVolume);
await expect(testProvider).toEmitValuesWith((received) => {
expect(received).toMatchObject([
{ data: [], state: LoadingState.Loading },
@ -248,10 +241,7 @@ describe('SupplementaryQueries utils', function () {
describe('All data sources do not support full range logs volume', function () {
it('Creates single fallback result', async () => {
const testProvider = await setup(
['no-data-providers', 'no-data-providers-2'],
SupplementaryQueryType.LogsVolume
);
const testProvider = setup(['no-data-providers', 'no-data-providers-2'], SupplementaryQueryType.LogsVolume);
await expect(testProvider).toEmitValuesWith((received) => {
expect(received).toMatchObject([
@ -270,7 +260,7 @@ describe('SupplementaryQueries utils', function () {
describe('Some data sources support full range logs volume, while others do not', function () {
it('Creates merged result containing full range and limited logs volume', async () => {
const testProvider = await setup(
const testProvider = setup(
['logs-volume-a', 'no-data-providers', 'logs-volume-b', 'no-data-providers-2'],
SupplementaryQueryType.LogsVolume
);
@ -308,7 +298,7 @@ describe('SupplementaryQueries utils', function () {
describe('Logs sample', function () {
describe('All data sources support logs sample', function () {
it('Merges all responses into single result', async () => {
const testProvider = await setup(['logs-sample-a', 'logs-sample-b'], SupplementaryQueryType.LogsSample);
const testProvider = setup(['logs-sample-a', 'logs-sample-b'], SupplementaryQueryType.LogsSample);
await expect(testProvider).toEmitValuesWith((received) => {
expect(received).toMatchObject([
{ data: [], state: LoadingState.Loading },
@ -327,17 +317,14 @@ describe('SupplementaryQueries utils', function () {
describe('All data sources do not support full range logs volume', function () {
it('Does not provide fallback result', async () => {
const testProvider = await setup(
['no-data-providers', 'no-data-providers-2'],
SupplementaryQueryType.LogsSample
);
const testProvider = setup(['no-data-providers', 'no-data-providers-2'], SupplementaryQueryType.LogsSample);
await expect(testProvider).toBeUndefined();
});
});
describe('Some data sources support full range logs volume, while others do not', function () {
it('Returns results only for data sources supporting logs sample', async () => {
const testProvider = await setup(
const testProvider = setup(
['logs-sample-a', 'no-data-providers', 'logs-sample-b', 'no-data-providers-2'],
SupplementaryQueryType.LogsSample
);

16
public/app/features/explore/utils/supplementaryQueries.ts
View File

@ -18,7 +18,7 @@ import {
import store from 'app/core/store';
import { ExplorePanelData, SupplementaryQueries } from 'app/types';
import { makeDataFramesForLogs } from '../../logs/logsModel';
import { makeDataFramesForLogs, queryLogsSample, queryLogsVolume } from '../../logs/logsModel';
export const supplementaryQueryTypes: SupplementaryQueryType[] = [
SupplementaryQueryType.LogsVolume,
@ -130,7 +130,19 @@ export const getSupplementaryQueryProvider = (
dsRequest.targets = targets;
if (hasSupplementaryQuerySupport(datasource, type)) {
return datasource.getDataProvider(type, dsRequest);
if (datasource.getDataProvider) {
return datasource.getDataProvider(type, dsRequest);
} else if (datasource.getSupplementaryRequest) {
const request = datasource.getSupplementaryRequest(type, dsRequest);
if (!request) {
return undefined;
}
return type === SupplementaryQueryType.LogsVolume
? queryLogsVolume(datasource, request, { targets: dsRequest.targets })
: queryLogsSample(datasource, request);
} else {
return undefined;
}
} else {
return getSupplementaryQueryFallback(type, explorePanelData, targets, datasource.name);
}

365
public/app/features/explore/utils/supplementaryQueries_legacy.test.ts Normal file
View File

@ -0,0 +1,365 @@
/**
* Test file to be removed when `getDataProvider` is removed from DataSourceWithSupplementaryQueriesSupport
* in packages/grafana-data/src/types/logs.ts
*/
import { flatten } from 'lodash';
import { from, Observable } from 'rxjs';
import {
DataFrame,
DataQueryRequest,
DataQueryResponse,
DataSourceApi,
DataSourceWithSupplementaryQueriesSupport,
FieldType,
LoadingState,
LogLevel,
LogsVolumeType,
MutableDataFrame,
SupplementaryQueryType,
SupplementaryQueryOptions,
toDataFrame,
} from '@grafana/data';
import { getDataSourceSrv } from '@grafana/runtime';
import { DataQuery } from '@grafana/schema';
import { MockDataSourceApi } from '../../../../test/mocks/datasource_srv';
import { MockDataQueryRequest, MockQuery } from '../../../../test/mocks/query';
import { ExplorePanelData } from '../../../types';
import { mockExplorePanelData } from '../__mocks__/data';
import { getSupplementaryQueryProvider } from './supplementaryQueries';
class MockDataSourceWithSupplementaryQuerySupport
extends MockDataSourceApi
implements DataSourceWithSupplementaryQueriesSupport<DataQuery>
{
private supplementaryQueriesResults: Record<SupplementaryQueryType, DataFrame[] | undefined> = {
[SupplementaryQueryType.LogsVolume]: undefined,
[SupplementaryQueryType.LogsSample]: undefined,
};
withSupplementaryQuerySupport(type: SupplementaryQueryType, data: DataFrame[]) {
this.supplementaryQueriesResults[type] = data;
return this;
}
getDataProvider(
type: SupplementaryQueryType,
request: DataQueryRequest<DataQuery>
): Observable<DataQueryResponse> | undefined {
const data = this.supplementaryQueriesResults[type];
if (data) {
return from([
{ state: LoadingState.Loading, data: [] },
{ state: LoadingState.Done, data },
]);
}
return undefined;
}
getSupplementaryQuery(options: SupplementaryQueryOptions, query: DataQuery): DataQuery | undefined {
return query;
}
getSupportedSupplementaryQueryTypes(): SupplementaryQueryType[] {
return Object.values(SupplementaryQueryType).filter((type) => this.supplementaryQueriesResults[type]);
}
}
const createSupplementaryQueryResponse = (type: SupplementaryQueryType, id: string) => {
return [
toDataFrame({
refId: `1-${type}-${id}`,
fields: [{ name: 'value', type: FieldType.string, values: [1] }],
meta: {
custom: {
logsVolumeType: LogsVolumeType.FullRange,
},
},
}),
toDataFrame({
refId: `2-${type}-${id}`,
fields: [{ name: 'value', type: FieldType.string, values: [2] }],
meta: {
custom: {
logsVolumeType: LogsVolumeType.FullRange,
},
},
}),
];
};
const mockRow = (refId: string) => {
return {
rowIndex: 0,
entryFieldIndex: 0,
dataFrame: new MutableDataFrame({ refId, fields: [{ name: 'A', values: [] }] }),
entry: '',
hasAnsi: false,
hasUnescapedContent: false,
labels: {},
logLevel: LogLevel.info,
raw: '',
timeEpochMs: 0,
timeEpochNs: '0',
timeFromNow: '',
timeLocal: '',
timeUtc: '',
uid: '1',
};
};
const mockExploreDataWithLogs = () =>
mockExplorePanelData({
logsResult: {
rows: [mockRow('0'), mockRow('1')],
visibleRange: { from: 0, to: 1 },
bucketSize: 1000,
},
});
const datasources: DataSourceApi[] = [
new MockDataSourceWithSupplementaryQuerySupport('logs-volume-a').withSupplementaryQuerySupport(
SupplementaryQueryType.LogsVolume,
createSupplementaryQueryResponse(SupplementaryQueryType.LogsVolume, 'logs-volume-a')
),
new MockDataSourceWithSupplementaryQuerySupport('logs-volume-b').withSupplementaryQuerySupport(
SupplementaryQueryType.LogsVolume,
createSupplementaryQueryResponse(SupplementaryQueryType.LogsVolume, 'logs-volume-b')
),
new MockDataSourceWithSupplementaryQuerySupport('logs-sample-a').withSupplementaryQuerySupport(
SupplementaryQueryType.LogsSample,
createSupplementaryQueryResponse(SupplementaryQueryType.LogsSample, 'logs-sample-a')
),
new MockDataSourceWithSupplementaryQuerySupport('logs-sample-b').withSupplementaryQuerySupport(
SupplementaryQueryType.LogsSample,
createSupplementaryQueryResponse(SupplementaryQueryType.LogsSample, 'logs-sample-b')
),
new MockDataSourceApi('no-data-providers'),
new MockDataSourceApi('no-data-providers-2'),
];
jest.mock('@grafana/runtime', () => ({
...jest.requireActual('@grafana/runtime'),
getDataSourceSrv: () => {
return {
get: async ({ uid }: { uid: string }) => datasources.find((ds) => ds.name === uid) || undefined,
};
},
}));
const setup = async (targetSources: string[], type: SupplementaryQueryType) => {
const requestMock = new MockDataQueryRequest({
targets: targetSources.map((source, i) => new MockQuery(`${i}`, 'a', { uid: source })),
});
const explorePanelDataMock: Observable<ExplorePanelData> = mockExploreDataWithLogs();
const datasources = await Promise.all(
targetSources.map(async (source, i) => {
const datasource = await getDataSourceSrv().get({ uid: source });
return {
datasource,
targets: [new MockQuery(`${i}`, 'a', { uid: source })],
};
})
);
return getSupplementaryQueryProvider(datasources, type, requestMock, explorePanelDataMock);
};
const assertDataFrom = (type: SupplementaryQueryType, ...datasources: string[]) => {
return flatten(
datasources.map((name: string) => {
return [{ refId: `1-${type}-${name}` }, { refId: `2-${type}-${name}` }];
})
);
};
const assertDataFromLogsResults = () => {
return [{ meta: { custom: { logsVolumeType: LogsVolumeType.Limited } } }];
};
describe('SupplementaryQueries utils', function () {
describe('Non-mixed data source', function () {
it('Returns result from the provider', async () => {
const testProvider = await setup(['logs-volume-a'], SupplementaryQueryType.LogsVolume);
await expect(testProvider).toEmitValuesWith((received) => {
expect(received).toMatchObject([
{ data: [], state: LoadingState.Loading },
{
data: assertDataFrom(SupplementaryQueryType.LogsVolume, 'logs-volume-a'),
state: LoadingState.Done,
},
]);
});
});
it('Uses fallback for logs volume', async () => {
const testProvider = await setup(['no-data-providers'], SupplementaryQueryType.LogsVolume);
await expect(testProvider).toEmitValuesWith((received) => {
expect(received).toMatchObject([
{
data: assertDataFromLogsResults(),
state: LoadingState.Done,
},
]);
});
});
it('Returns undefined for logs sample', async () => {
const testProvider = await setup(['no-data-providers'], SupplementaryQueryType.LogsSample);
await expect(testProvider).toBe(undefined);
});
it('Creates single fallback result', async () => {
const testProvider = await setup(['no-data-providers', 'no-data-providers-2'], SupplementaryQueryType.LogsVolume);
await expect(testProvider).toEmitValuesWith((received) => {
expect(received).toMatchObject([
{
data: assertDataFromLogsResults(),
state: LoadingState.Done,
},
{
data: [...assertDataFromLogsResults(), ...assertDataFromLogsResults()],
state: LoadingState.Done,
},
]);
});
});
});
describe('Mixed data source', function () {
describe('Logs volume', function () {
describe('All data sources support full range logs volume', function () {
it('Merges all data frames into a single response', async () => {
const testProvider = await setup(['logs-volume-a', 'logs-volume-b'], SupplementaryQueryType.LogsVolume);
await expect(testProvider).toEmitValuesWith((received) => {
expect(received).toMatchObject([
{ data: [], state: LoadingState.Loading },
{
data: assertDataFrom(SupplementaryQueryType.LogsVolume, 'logs-volume-a'),
state: LoadingState.Done,
},
{
data: assertDataFrom(SupplementaryQueryType.LogsVolume, 'logs-volume-a', 'logs-volume-b'),
state: LoadingState.Done,
},
]);
});
});
});
describe('All data sources do not support full range logs volume', function () {
it('Creates single fallback result', async () => {
const testProvider = await setup(
['no-data-providers', 'no-data-providers-2'],
SupplementaryQueryType.LogsVolume
);
await expect(testProvider).toEmitValuesWith((received) => {
expect(received).toMatchObject([
{
data: assertDataFromLogsResults(),
state: LoadingState.Done,
},
{
data: [...assertDataFromLogsResults(), ...assertDataFromLogsResults()],
state: LoadingState.Done,
},
]);
});
});
});
describe('Some data sources support full range logs volume, while others do not', function () {
it('Creates merged result containing full range and limited logs volume', async () => {
const testProvider = await setup(
['logs-volume-a', 'no-data-providers', 'logs-volume-b', 'no-data-providers-2'],
SupplementaryQueryType.LogsVolume
);
await expect(testProvider).toEmitValuesWith((received) => {
expect(received).toMatchObject([
{
data: [],
state: LoadingState.Loading,
},
{
data: assertDataFrom(SupplementaryQueryType.LogsVolume, 'logs-volume-a'),
state: LoadingState.Done,
},
{
data: [
...assertDataFrom(SupplementaryQueryType.LogsVolume, 'logs-volume-a'),
...assertDataFromLogsResults(),
],
state: LoadingState.Done,
},
{
data: [
...assertDataFrom(SupplementaryQueryType.LogsVolume, 'logs-volume-a'),
...assertDataFromLogsResults(),
...assertDataFrom(SupplementaryQueryType.LogsVolume, 'logs-volume-b'),
],
state: LoadingState.Done,
},
]);
});
});
});
});
describe('Logs sample', function () {
describe('All data sources support logs sample', function () {
it('Merges all responses into single result', async () => {
const testProvider = await setup(['logs-sample-a', 'logs-sample-b'], SupplementaryQueryType.LogsSample);
await expect(testProvider).toEmitValuesWith((received) => {
expect(received).toMatchObject([
{ data: [], state: LoadingState.Loading },
{
data: assertDataFrom(SupplementaryQueryType.LogsSample, 'logs-sample-a'),
state: LoadingState.Done,
},
{
data: assertDataFrom(SupplementaryQueryType.LogsSample, 'logs-sample-a', 'logs-sample-b'),
state: LoadingState.Done,
},
]);
});
});
});
describe('All data sources do not support full range logs volume', function () {
it('Does not provide fallback result', async () => {
const testProvider = await setup(
['no-data-providers', 'no-data-providers-2'],
SupplementaryQueryType.LogsSample
);
await expect(testProvider).toBeUndefined();
});
});
describe('Some data sources support full range logs volume, while others do not', function () {
it('Returns results only for data sources supporting logs sample', async () => {
const testProvider = await setup(
['logs-sample-a', 'no-data-providers', 'logs-sample-b', 'no-data-providers-2'],
SupplementaryQueryType.LogsSample
);
await expect(testProvider).toEmitValuesWith((received) => {
expect(received).toMatchObject([
{ data: [], state: LoadingState.Loading },
{
data: assertDataFrom(SupplementaryQueryType.LogsSample, 'logs-sample-a'),
state: LoadingState.Done,
},
{
data: assertDataFrom(SupplementaryQueryType.LogsSample, 'logs-sample-a', 'logs-sample-b'),
state: LoadingState.Done,
},
]);
});
});
});
});
});
});

18
public/app/features/logs/logsModel.test.ts
View File

@ -1317,16 +1317,22 @@ describe('logs volume', () => {
{ refId: 'B', target: 'volume query 2' },
],
scopedVars: {},
} as unknown as DataQueryRequest<TestDataQuery>;
volumeProvider = queryLogsVolume(datasource, request, {
extractLevel: (dataFrame: DataFrame) => {
return dataFrame.fields[1]!.labels!.level === 'error' ? LogLevel.error : LogLevel.unknown;
},
requestId: '',
interval: '',
intervalMs: 0,
range: {
from: FROM,
to: TO,
raw: { from: '0', to: '1' },
raw: {
from: FROM,
to: TO,
},
},
timezone: '',
app: '',
startTime: 0,
};
volumeProvider = queryLogsVolume(datasource, request, {
targets: request.targets,
});
}

27
public/app/features/logs/logsModel.ts
View File

@ -35,7 +35,6 @@ import {
ScopedVars,
sortDataFrame,
textUtil,
TimeRange,
toDataFrame,
toUtc,
} from '@grafana/data';
@ -641,11 +640,22 @@ const updateLogsVolumeConfig = (
};
type LogsVolumeQueryOptions<T extends DataQuery> = {
extractLevel: (dataFrame: DataFrame) => LogLevel;
targets: T[];
range: TimeRange;
};
function defaultExtractLevel(dataFrame: DataFrame): LogLevel {
let valueField;
try {
valueField = new FieldCache(dataFrame).getFirstFieldOfType(FieldType.number);
} catch {}
return valueField?.labels ? getLogLevelFromLabels(valueField.labels) : LogLevel.unknown;
}
function getLogLevelFromLabels(labels: Labels): LogLevel {
const level = labels['level'] ?? labels['lvl'] ?? labels['loglevel'] ?? '';
return level ? getLogLevelFromKey(level) : LogLevel.unknown;
}
/**
* Creates an observable, which makes requests to get logs volume and aggregates results.
*/
@ -654,7 +664,10 @@ export function queryLogsVolume<TQuery extends DataQuery, TOptions extends DataS
logsVolumeRequest: DataQueryRequest<TQuery>,
options: LogsVolumeQueryOptions<TQuery>
): Observable<DataQueryResponse> {
const timespan = options.range.to.valueOf() - options.range.from.valueOf();
const range = logsVolumeRequest.range;
const targets = options.targets;
const extractLevel = defaultExtractLevel;
const timespan = range.to.valueOf() - range.from.valueOf();
const intervalInfo = getIntervalInfo(logsVolumeRequest.scopedVars, timespan);
logsVolumeRequest.interval = intervalInfo.interval;
@ -705,9 +718,9 @@ export function queryLogsVolume<TQuery extends DataQuery, TOptions extends DataS
const logsVolumeCustomMetaData: LogsVolumeCustomMetaData = {
logsVolumeType: LogsVolumeType.FullRange,
absoluteRange: { from: options.range.from.valueOf(), to: options.range.to.valueOf() },
absoluteRange: { from: range.from.valueOf(), to: range.to.valueOf() },
datasourceName: datasource.name,
sourceQuery: options.targets.find((dataQuery) => dataQuery.refId === sourceRefId)!,
sourceQuery: targets.find((dataQuery) => dataQuery.refId === sourceRefId)!,
};
dataFrame.meta = {
@ -717,7 +730,7 @@ export function queryLogsVolume<TQuery extends DataQuery, TOptions extends DataS
...logsVolumeCustomMetaData,
},
};
return updateLogsVolumeConfig(dataFrame, options.extractLevel, framesByRefId[dataFrame.refId].length === 1);
return updateLogsVolumeConfig(dataFrame, extractLevel, framesByRefId[dataFrame.refId].length === 1);
});
observer.next({

8
public/app/plugins/datasource/elasticsearch/datasource.test.ts
View File

@ -986,7 +986,7 @@ describe('ElasticDatasource', () => {
],
});
expect(ds.getDataProvider(SupplementaryQueryType.LogsSample, options)).not.toBeDefined();
expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsSample, options)).not.toBeDefined();
});
it('does create a logs sample provider for time series query', () => {
@ -999,7 +999,7 @@ describe('ElasticDatasource', () => {
],
});
expect(ds.getDataProvider(SupplementaryQueryType.LogsSample, options)).toBeDefined();
expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsSample, options)).toBeDefined();
});
});
@ -1019,7 +1019,7 @@ describe('ElasticDatasource', () => {
],
});
expect(ds.getLogsSampleDataProvider(request)).not.toBeDefined();
expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsSample, request)).not.toBeDefined();
});
it('returns a logs sample provider given a time series query', () => {
@ -1032,7 +1032,7 @@ describe('ElasticDatasource', () => {
],
});
expect(ds.getLogsSampleDataProvider(request)).toBeDefined();
expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsSample, request)).toBeDefined();
});
});
});

46
public/app/plugins/datasource/elasticsearch/datasource.ts
View File

@ -31,7 +31,6 @@ import {
SupplementaryQueryOptions,
toUtc,
AnnotationEvent,
FieldType,
DataSourceWithToggleableQueryFiltersSupport,
QueryFilterOptions,
ToggleFilterAction,
@ -48,9 +47,6 @@ import {
getTemplateSrv,
} from '@grafana/runtime';
import { queryLogsSample, queryLogsVolume } from '../../../features/logs/logsModel';
import { getLogLevelFromKey } from '../../../features/logs/utils';
import { IndexPattern, intervalMap } from './IndexPattern';
import LanguageProvider from './LanguageProvider';
import { LegacyQueryRunner } from './LegacyQueryRunner';
@ -538,13 +534,15 @@ export class ElasticDatasource
}
};
getDataProvider(
/**
* Implemented for DataSourceWithSupplementaryQueriesSupport.
* It generates a DataQueryRequest for a specific supplementary query type.
* @returns A DataQueryRequest for the supplementary queries or undefined if not supported.
*/
getSupplementaryRequest(
type: SupplementaryQueryType,
request: DataQueryRequest<ElasticsearchQuery>
): Observable<DataQueryResponse> | undefined {
if (!this.getSupportedSupplementaryQueryTypes().includes(type)) {
return undefined;
}
): DataQueryRequest<ElasticsearchQuery> | undefined {
switch (type) {
case SupplementaryQueryType.LogsVolume:
return this.getLogsVolumeDataProvider(request);
@ -560,10 +558,6 @@ export class ElasticDatasource
}
getSupplementaryQuery(options: SupplementaryQueryOptions, query: ElasticsearchQuery): ElasticsearchQuery | undefined {
if (!this.getSupportedSupplementaryQueryTypes().includes(options.type)) {
return undefined;
}
let isQuerySuitable = false;
switch (options.type) {
@ -635,7 +629,9 @@ export class ElasticDatasource
}
}
getLogsVolumeDataProvider(request: DataQueryRequest<ElasticsearchQuery>): Observable<DataQueryResponse> | undefined {
private getLogsVolumeDataProvider(
request: DataQueryRequest<ElasticsearchQuery>
): DataQueryRequest<ElasticsearchQuery> | undefined {
const logsVolumeRequest = cloneDeep(request);
const targets = logsVolumeRequest.targets
.map((target) => this.getSupplementaryQuery({ type: SupplementaryQueryType.LogsVolume }, target))
@ -645,18 +641,12 @@ export class ElasticDatasource
return undefined;
}
return queryLogsVolume(
this,
{ ...logsVolumeRequest, targets },
{
range: request.range,
targets: request.targets,
extractLevel,
}
);
return { ...logsVolumeRequest, targets };
}
getLogsSampleDataProvider(request: DataQueryRequest<ElasticsearchQuery>): Observable<DataQueryResponse> | undefined {
private getLogsSampleDataProvider(
request: DataQueryRequest<ElasticsearchQuery>
): DataQueryRequest<ElasticsearchQuery> | undefined {
const logsSampleRequest = cloneDeep(request);
const targets = logsSampleRequest.targets;
const queries = targets.map((query) => {
@ -667,7 +657,7 @@ export class ElasticDatasource
if (!elasticQueries.length) {
return undefined;
}
return queryLogsSample(this, { ...logsSampleRequest, targets: elasticQueries });
return { ...logsSampleRequest, targets: elasticQueries };
}
query(request: DataQueryRequest<ElasticsearchQuery>): Observable<DataQueryResponse> {
@ -1178,9 +1168,3 @@ function createContextTimeRange(rowTimeEpochMs: number, direction: string, inter
}
}
}
function extractLevel(dataFrame: DataFrame): LogLevel {
const valueField = dataFrame.fields.find((f) => f.type === FieldType.number);
const name = valueField?.labels?.['level'] ?? '';
return getLogLevelFromKey(name);
}

14
public/app/plugins/datasource/loki/datasource.test.ts
View File

@ -1255,7 +1255,7 @@ describe('LokiDatasource', () => {
targets: [{ expr: '{label="value"}', refId: 'A', queryType: LokiQueryType.Range }],
};
expect(ds.getDataProvider(SupplementaryQueryType.LogsVolume, options)).toBeDefined();
expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsVolume, options)).toBeDefined();
});
it('does not create provider for metrics query', () => {
@ -1264,7 +1264,7 @@ describe('LokiDatasource', () => {
targets: [{ expr: 'rate({label="value"}[1m])', refId: 'A' }],
};
expect(ds.getDataProvider(SupplementaryQueryType.LogsVolume, options)).not.toBeDefined();
expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsVolume, options)).not.toBeDefined();
});
it('creates provider if at least one query is a logs query', () => {
@ -1276,7 +1276,7 @@ describe('LokiDatasource', () => {
],
};
expect(ds.getDataProvider(SupplementaryQueryType.LogsVolume, options)).toBeDefined();
expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsVolume, options)).toBeDefined();
});
it('does not create provider if there is only an instant logs query', () => {
@ -1285,7 +1285,7 @@ describe('LokiDatasource', () => {
targets: [{ expr: '{label="value"', refId: 'A', queryType: LokiQueryType.Instant }],
};
expect(ds.getDataProvider(SupplementaryQueryType.LogsVolume, options)).not.toBeDefined();
expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsVolume, options)).not.toBeDefined();
});
});
@ -1301,7 +1301,7 @@ describe('LokiDatasource', () => {
targets: [{ expr: 'rate({label="value"}[5m])', refId: 'A' }],
};
expect(ds.getDataProvider(SupplementaryQueryType.LogsSample, options)).toBeDefined();
expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsSample, options)).toBeDefined();
});
it('does not create provider for log query', () => {
@ -1310,7 +1310,7 @@ describe('LokiDatasource', () => {
targets: [{ expr: '{label="value"}', refId: 'A' }],
};
expect(ds.getDataProvider(SupplementaryQueryType.LogsSample, options)).not.toBeDefined();
expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsSample, options)).not.toBeDefined();
});
it('creates provider if at least one query is a metric query', () => {
@ -1322,7 +1322,7 @@ describe('LokiDatasource', () => {
],
};
expect(ds.getDataProvider(SupplementaryQueryType.LogsSample, options)).toBeDefined();
expect(ds.getSupplementaryRequest(SupplementaryQueryType.LogsSample, options)).toBeDefined();
});
});

57
public/app/plugins/datasource/loki/datasource.ts
View File

@ -17,11 +17,8 @@ import {
SupplementaryQueryType,
DataSourceWithQueryExportSupport,
DataSourceWithQueryImportSupport,
FieldCache,
FieldType,
Labels,
LoadingState,
LogLevel,
LogRowModel,
QueryFixAction,
QueryHint,
@ -46,9 +43,6 @@ import { Duration } from '@grafana/lezer-logql';
import { BackendSrvRequest, config, DataSourceWithBackend, getTemplateSrv, TemplateSrv } from '@grafana/runtime';
import { DataQuery } from '@grafana/schema';
import { queryLogsSample, queryLogsVolume } from '../../../features/logs/logsModel';
import { getLogLevelFromKey } from '../../../features/logs/utils';
import LanguageProvider from './LanguageProvider';
import { LiveStreams, LokiLiveTarget } from './LiveStreams';
import { LogContextProvider } from './LogContextProvider';
@ -168,16 +162,13 @@ export class LokiDatasource
/**
* Implemented for DataSourceWithSupplementaryQueriesSupport.
* It retrieves a data provider for a specific supplementary query type.
* @returns An Observable of DataQueryResponse or undefined if the specified query type is not supported.
* It generates a DataQueryRequest for a specific supplementary query type.
* @returns A DataQueryRequest for the supplementary queries or undefined if not supported.
*/
getDataProvider(
getSupplementaryRequest(
type: SupplementaryQueryType,
request: DataQueryRequest<LokiQuery>
): Observable<DataQueryResponse> | undefined {
if (!this.getSupportedSupplementaryQueryTypes().includes(type)) {
return undefined;
}
): DataQueryRequest<LokiQuery> | undefined {
switch (type) {
case SupplementaryQueryType.LogsVolume:
return this.getLogsVolumeDataProvider(request);
@ -203,10 +194,6 @@ export class LokiDatasource
* @returns A supplemented Loki query or undefined if unsupported.
*/
getSupplementaryQuery(options: SupplementaryQueryOptions, query: LokiQuery): LokiQuery | undefined {
if (!this.getSupportedSupplementaryQueryTypes().includes(options.type)) {
return undefined;
}
const normalizedQuery = getNormalizedLokiQuery(query);
let expr = removeCommentsFromQuery(normalizedQuery.expr);
let isQuerySuitable = false;
@ -255,7 +242,7 @@ export class LokiDatasource
* Private method used in the `getDataProvider` for DataSourceWithSupplementaryQueriesSupport, specifically for Logs volume queries.
* @returns An Observable of DataQueryResponse or undefined if no suitable queries are found.
*/
private getLogsVolumeDataProvider(request: DataQueryRequest<LokiQuery>): Observable<DataQueryResponse> | undefined {
private getLogsVolumeDataProvider(request: DataQueryRequest<LokiQuery>): DataQueryRequest<LokiQuery> | undefined {
const logsVolumeRequest = cloneDeep(request);
const targets = logsVolumeRequest.targets
.map((query) => this.getSupplementaryQuery({ type: SupplementaryQueryType.LogsVolume }, query))
@ -265,22 +252,14 @@ export class LokiDatasource
return undefined;
}
return queryLogsVolume(
this,
{ ...logsVolumeRequest, targets },
{
extractLevel,
range: request.range,
targets: request.targets,
}
);
return { ...logsVolumeRequest, targets };
}
/**
* Private method used in the `getDataProvider` for DataSourceWithSupplementaryQueriesSupport, specifically for Logs sample queries.
* @returns An Observable of DataQueryResponse or undefined if no suitable queries are found.
*/
private getLogsSampleDataProvider(request: DataQueryRequest<LokiQuery>): Observable<DataQueryResponse> | undefined {
private getLogsSampleDataProvider(request: DataQueryRequest<LokiQuery>): DataQueryRequest<LokiQuery> | undefined {
const logsSampleRequest = cloneDeep(request);
const targets = logsSampleRequest.targets
.map((query) => this.getSupplementaryQuery({ type: SupplementaryQueryType.LogsSample, limit: 100 }, query))
@ -289,7 +268,7 @@ export class LokiDatasource
if (!targets.length) {
return undefined;
}
return queryLogsSample(this, { ...logsSampleRequest, targets });
return { ...logsSampleRequest, targets };
}
/**
@ -1171,23 +1150,3 @@ export function lokiSpecialRegexEscape(value: any) {
}
return value;
}
function extractLevel(dataFrame: DataFrame): LogLevel {
let valueField;
try {
valueField = new FieldCache(dataFrame).getFirstFieldOfType(FieldType.number);
} catch {}
return valueField?.labels ? getLogLevelFromLabels(valueField.labels) : LogLevel.unknown;
}
function getLogLevelFromLabels(labels: Labels): LogLevel {
const labelNames = ['level', 'lvl', 'loglevel'];
let levelLabel;
for (let labelName of labelNames) {
if (labelName in labels) {
levelLabel = labelName;
break;
}
}
return levelLabel ? getLogLevelFromKey(labels[levelLabel]) : LogLevel.unknown;
}

2
public/test/mocks/datasource_srv.ts
View File

@ -49,7 +49,7 @@ export class MockDataSourceApi extends DataSourceApi {
this.meta = meta || ({} as DataSourcePluginMeta);
}
query(request: DataQueryRequest): Promise<DataQueryResponse> {
query(request: DataQueryRequest): Promise<DataQueryResponse> | Observable<DataQueryResponse> {
if (this.error) {
return Promise.reject(this.error);
}