Glue: Enrich query results data frames in Explore with correlations to generate static links from correlations (#56295)

* Attach static generic data link to data frames in Explore

* WIP

* Always load correlations config when the query is run

This will be moved to Wrapper.tsx and called only once Explore is mounted

* remove comment

* Load the config when Explore is loaded

* Clean up

* Check for feature toggle, simplify cod

* Simplify the code

* Remove unused code

* Fix types

* Add a test for attaching links

* Revert package.json changes

* Display title provided in the correlation label

* Add missing mocks

* Fix tests

* Merge branch 'main' into ifrost/integration/attach-generic-data-link

# Conflicts:
#	public/app/features/explore/Wrapper.tsx
#	public/app/features/explore/state/main.ts

* Remove redundant async calls

* Do not block Wrapper before correlations are loaded (only delay the query)

* Test showing results after correlations are loaded

* Post-merge fix

* Use more consistent naming

* Avoid null assertions

Co-authored-by: Elfo404 <me@giordanoricci.com>
This commit is contained in:
Piotr Jamróz 2022-10-11 14:40:55 +02:00 committed by GitHub
parent 95b9fa3346
commit ae927eab73
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 278 additions and 38 deletions

View File

@ -4314,11 +4314,7 @@ exports[`better eslint`] = {
[0, 0, 0, "Unexpected any. Specify a different type.", "5"],
[0, 0, 0, "Unexpected any. Specify a different type.", "6"],
[0, 0, 0, "Unexpected any. Specify a different type.", "7"],
[0, 0, 0, "Unexpected any. Specify a different type.", "8"],
[0, 0, 0, "Unexpected any. Specify a different type.", "9"],
[0, 0, 0, "Unexpected any. Specify a different type.", "10"],
[0, 0, 0, "Unexpected any. Specify a different type.", "11"],
[0, 0, 0, "Unexpected any. Specify a different type.", "12"]
[0, 0, 0, "Unexpected any. Specify a different type.", "8"]
],
"public/app/features/explore/state/query.ts:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"]

View File

@ -245,7 +245,7 @@ datasources:
type: query
target:
expr: "{ job=\"test\" }"
field: "labels"
field: "traceID"
jsonData:
manageAlerts: false
derivedFields:

View File

@ -0,0 +1,94 @@
import { DataFrame, DataSourceInstanceSettings, FieldType, toDataFrame } from '@grafana/data';
import { CorrelationData } from './useCorrelations';
import { attachCorrelationsToDataFrames } from './utils';
describe('correlations utils', () => {
it('attaches correlations defined in the configuration', () => {
const loki = { uid: 'loki-uid', name: 'loki' } as DataSourceInstanceSettings;
const elastic = { uid: 'elastic-uid', name: 'elastic' } as DataSourceInstanceSettings;
const prometheus = { uid: 'prometheus-uid', name: 'prometheus' } as DataSourceInstanceSettings;
const refIdMap = {
'Loki Query': loki.uid,
'Elastic Query': elastic.uid,
'Prometheus Query': prometheus.uid,
};
const testDataFrames: DataFrame[] = [
toDataFrame({
name: 'Loki Logs',
refId: 'Loki Query',
fields: [
{ name: 'line', values: [] },
{ name: 'traceId', values: [] },
],
}),
toDataFrame({
name: 'Elastic Logs',
refId: 'Elastic Query',
fields: [
{ name: 'line', values: [] },
{ name: 'traceId', values: [] },
],
}),
toDataFrame({
name: 'Prometheus Metrics',
refId: 'Prometheus Query',
fields: [{ name: 'value', type: FieldType.number, values: [1, 2, 3, 4, 5] }],
}),
];
const correlations: CorrelationData[] = [
{
uid: 'loki-to-prometheus',
label: 'logs to metrics',
source: loki,
target: prometheus,
config: { type: 'query', field: 'traceId', target: { expr: 'target Prometheus query' } },
},
{
uid: 'prometheus-to-elastic',
label: 'metrics to logs',
source: prometheus,
target: elastic,
config: { type: 'query', field: 'value', target: { expr: 'target Elastic query' } },
},
];
attachCorrelationsToDataFrames(testDataFrames, correlations, refIdMap);
// Loki line (no links)
expect(testDataFrames[0].fields[0].config.links).toBeUndefined();
// Loki traceId (linked to Prometheus)
expect(testDataFrames[0].fields[1].config.links).toHaveLength(1);
expect(testDataFrames[0].fields[1].config.links![0]).toMatchObject({
title: 'logs to metrics',
internal: {
datasourceUid: prometheus.uid,
datasourceName: prometheus.name,
query: {
expr: 'target Prometheus query',
},
},
});
// Elastic line (no links)
expect(testDataFrames[1].fields[0].config.links).toBeUndefined();
// Elastic traceId (no links)
expect(testDataFrames[1].fields[0].config.links).toBeUndefined();
// Prometheus value (linked to Elastic)
expect(testDataFrames[2].fields[0].config.links).toHaveLength(1);
expect(testDataFrames[2].fields[0].config.links![0]).toMatchObject({
title: 'metrics to logs',
internal: {
datasourceUid: elastic.uid,
datasourceName: elastic.name,
query: {
expr: 'target Elastic query',
},
},
});
});
});

View File

@ -0,0 +1,49 @@
import { DataFrame } from '@grafana/data';
import { CorrelationData } from './useCorrelations';
type DataFrameRefIdToDataSourceUid = Record<string, string>;
/**
* Creates data links from provided CorrelationData object
*
* @param dataFrames list of data frames to be processed
* @param correlations list of of possible correlations that can be applied
* @param dataFrameRefIdToDataSourceUid a map that for provided refId references corresponding data source ui
*/
export const attachCorrelationsToDataFrames = (
dataFrames: DataFrame[],
correlations: CorrelationData[],
dataFrameRefIdToDataSourceUid: DataFrameRefIdToDataSourceUid
): DataFrame[] => {
dataFrames.forEach((dataFrame) => {
const frameRefId = dataFrame.refId;
if (!frameRefId) {
return;
}
const dataSourceUid = dataFrameRefIdToDataSourceUid[frameRefId];
const sourceCorrelations = correlations.filter((correlation) => correlation.source.uid === dataSourceUid);
decorateDataFrameWithInternalDataLinks(dataFrame, sourceCorrelations);
});
return dataFrames;
};
const decorateDataFrameWithInternalDataLinks = (dataFrame: DataFrame, correlations: CorrelationData[]) => {
dataFrame.fields.forEach((field) => {
correlations.map((correlation) => {
if (correlation.config?.field === field.name) {
field.config.links = field.config.links || [];
field.config.links.push({
internal: {
query: correlation.config?.target,
datasourceUid: correlation.target.uid,
datasourceName: correlation.target.name,
},
url: '',
title: correlation.label || correlation.target.name,
});
}
});
});
};

View File

@ -57,6 +57,7 @@ function setup(queries: DataQuery[]) {
queries,
},
syncedTimes: false,
correlations: [],
right: undefined,
richHistoryStorageFull: false,
richHistoryLimitExceededWarningShown: false,

View File

@ -4,17 +4,19 @@ import React, { useEffect } from 'react';
import { locationService } from '@grafana/runtime';
import { ErrorBoundaryAlert } from '@grafana/ui';
import { useGrafana } from 'app/core/context/GrafanaContext';
import { useAppNotification } from 'app/core/copy/appNotification';
import { useNavModel } from 'app/core/hooks/useNavModel';
import { GrafanaRouteComponentProps } from 'app/core/navigation/types';
import { isTruthy } from 'app/core/utils/types';
import { useSelector, useDispatch } from 'app/types';
import { useDispatch, useSelector } from 'app/types';
import { ExploreId, ExploreQueryParams } from 'app/types/explore';
import { Branding } from '../../core/components/Branding/Branding';
import { useCorrelations } from '../correlations/useCorrelations';
import { ExploreActions } from './ExploreActions';
import { ExplorePaneContainer } from './ExplorePaneContainer';
import { lastSavedUrl, resetExploreAction } from './state/main';
import { lastSavedUrl, resetExploreAction, saveCorrelationsAction } from './state/main';
const styles = {
pageScrollbarWrapper: css`
@ -32,8 +34,10 @@ function Wrapper(props: GrafanaRouteComponentProps<{}, ExploreQueryParams>) {
useExplorePageTitle();
const dispatch = useDispatch();
const queryParams = props.queryParams;
const { keybindings, chrome } = useGrafana();
const { keybindings, chrome, config } = useGrafana();
const navModel = useNavModel('explore');
const { get } = useCorrelations();
const { warning } = useAppNotification();
useEffect(() => {
//This is needed for breadcrumbs and topnav.
@ -45,6 +49,27 @@ function Wrapper(props: GrafanaRouteComponentProps<{}, ExploreQueryParams>) {
keybindings.setupTimeRangeBindings(false);
}, [keybindings]);
useEffect(() => {
if (!config.featureToggles.correlations) {
dispatch(saveCorrelationsAction([]));
} else {
get.execute();
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
useEffect(() => {
if (get.value) {
dispatch(saveCorrelationsAction(get.value));
} else if (get.error) {
dispatch(saveCorrelationsAction([]));
warning(
'Could not load correlations.',
'Correlations data could not be loaded, DataLinks may have partial data.'
);
}
}, [get.value, get.error, dispatch, warning]);
useEffect(() => {
lastSavedUrl.left = undefined;
lastSavedUrl.right = undefined;
@ -95,8 +120,7 @@ const useExplorePageTitle = () => {
[state.explore.left.datasourceInstance?.name, state.explore.right?.datasourceInstance?.name].filter(isTruthy)
);
const documentTitle = `${navModel.main.text} - ${datasources.join(' | ')} - ${Branding.AppTitle}`;
document.title = documentTitle;
document.title = `${navModel.main.text} - ${datasources.join(' | ')} - ${Branding.AppTitle}`;
};
export default Wrapper;

View File

@ -58,8 +58,8 @@ export const selectOnlyActiveDataSource = async (exploreId: ExploreId = ExploreI
await userEvent.click(checkbox);
};
export const starQueryHistory = (queryIndex: number, exploreId: ExploreId = ExploreId.left) => {
invokeAction(queryIndex, 'Star query', exploreId);
export const starQueryHistory = async (queryIndex: number, exploreId: ExploreId = ExploreId.left) => {
await invokeAction(queryIndex, 'Star query', exploreId);
};
export const commentQueryHistory = async (
@ -74,8 +74,8 @@ export const commentQueryHistory = async (
await invokeAction(queryIndex, 'Submit button', exploreId);
};
export const deleteQueryHistory = (queryIndex: number, exploreId: ExploreId = ExploreId.left) => {
invokeAction(queryIndex, 'Delete query', exploreId);
export const deleteQueryHistory = async (queryIndex: number, exploreId: ExploreId = ExploreId.left) => {
await invokeAction(queryIndex, 'Delete query', exploreId);
};
export const loadMoreQueryHistory = async (exploreId: ExploreId = ExploreId.left) => {

View File

@ -168,7 +168,7 @@ describe('Explore: Query History', () => {
await assertQueryHistory(['{"expr":"query #2"}', '{"expr":"query #1"}'], ExploreId.right);
// star one one query
starQueryHistory(1, ExploreId.left);
await starQueryHistory(1, ExploreId.left);
await assertQueryHistoryIsStarred([false, true], ExploreId.left);
await assertQueryHistoryIsStarred([false, true], ExploreId.right);
expect(reportInteractionMock).toBeCalledWith('grafana_explore_query_history_starred', {
@ -176,7 +176,7 @@ describe('Explore: Query History', () => {
newValue: true,
});
deleteQueryHistory(0, ExploreId.left);
await deleteQueryHistory(0, ExploreId.left);
await assertQueryHistory(['{"expr":"query #1"}'], ExploreId.left);
await assertQueryHistory(['{"expr":"query #1"}'], ExploreId.right);
expect(reportInteractionMock).toBeCalledWith('grafana_explore_query_history_deleted', {

View File

@ -10,6 +10,7 @@ import { ExploreId, ExploreItemState, ExploreState } from 'app/types/explore';
import { RichHistoryResults } from '../../../core/history/RichHistoryStorage';
import { RichHistorySearchFilters, RichHistorySettings } from '../../../core/utils/richHistoryTypes';
import { ThunkResult } from '../../../types';
import { CorrelationData } from '../../correlations/useCorrelations';
import { TimeSrv } from '../../dashboard/services/TimeSrv';
import { paneReducer } from './explorePane';
@ -37,6 +38,8 @@ export const richHistorySearchFiltersUpdatedAction = createAction<{
filters?: RichHistorySearchFilters;
}>('explore/richHistorySearchFiltersUpdatedAction');
export const saveCorrelationsAction = createAction<CorrelationData[]>('explore/saveCorrelationsAction');
/**
* Resets state for explore.
*/
@ -156,6 +159,7 @@ export const initialExploreState: ExploreState = {
syncedTimes: false,
left: initialExploreItemState,
right: undefined,
correlations: undefined,
richHistoryStorageFull: false,
richHistoryLimitExceededWarningShown: false,
richHistoryMigrationFailed: false,
@ -178,6 +182,13 @@ export const exploreReducer = (state = initialExploreState, action: AnyAction):
};
}
if (saveCorrelationsAction.match(action)) {
return {
...state,
correlations: action.payload,
};
}
if (syncTimesAction.match(action)) {
return { ...state, syncedTimes: action.payload.syncedTimes };
}

View File

@ -22,6 +22,7 @@ import { configureStore } from '../../../store/configureStore';
import { setTimeSrv } from '../../dashboard/services/TimeSrv';
import { createDefaultInitialState } from './helpers';
import { saveCorrelationsAction } from './main';
import {
addQueryRowAction,
addResultsToCache,
@ -99,23 +100,26 @@ function setupQueryResponse(state: StoreState) {
}
describe('runQueries', () => {
it('should pass dataFrames to state even if there is error in response', async () => {
const setupTests = () => {
setTimeSrv({ init() {} } as any);
const { dispatch, getState } = configureStore({
return configureStore({
...(defaultInitialState as any),
});
};
it('should pass dataFrames to state even if there is error in response', async () => {
const { dispatch, getState } = setupTests();
setupQueryResponse(getState());
await dispatch(saveCorrelationsAction([]));
await dispatch(runQueries(ExploreId.left));
expect(getState().explore[ExploreId.left].showMetrics).toBeTruthy();
expect(getState().explore[ExploreId.left].graphResult).toBeDefined();
});
it('should modify the request-id for log-volume queries', async () => {
setTimeSrv({ init() {} } as any);
const { dispatch, getState } = configureStore({
...(defaultInitialState as any),
});
const { dispatch, getState } = setupTests();
setupQueryResponse(getState());
await dispatch(saveCorrelationsAction([]));
await dispatch(runQueries(ExploreId.left));
const state = getState().explore[ExploreId.left];
@ -129,16 +133,23 @@ describe('runQueries', () => {
});
it('should set state to done if query completes without emitting', async () => {
setTimeSrv({ init() {} } as any);
const { dispatch, getState } = configureStore({
...(defaultInitialState as any),
});
const { dispatch, getState } = setupTests();
const leftDatasourceInstance = assertIsDefined(getState().explore[ExploreId.left].datasourceInstance);
jest.mocked(leftDatasourceInstance.query).mockReturnValueOnce(EMPTY);
await dispatch(saveCorrelationsAction([]));
await dispatch(runQueries(ExploreId.left));
await new Promise((resolve) => setTimeout(() => resolve(''), 500));
expect(getState().explore[ExploreId.left].queryResponse.state).toBe(LoadingState.Done);
});
it('shows results only after correlations are loaded', async () => {
const { dispatch, getState } = setupTests();
setupQueryResponse(getState());
await dispatch(runQueries(ExploreId.left));
expect(getState().explore[ExploreId.left].graphResult).not.toBeDefined();
await dispatch(saveCorrelationsAction([]));
expect(getState().explore[ExploreId.left].graphResult).toBeDefined();
});
});
describe('running queries', () => {

View File

@ -1,7 +1,7 @@
import { AnyAction, createAction, PayloadAction } from '@reduxjs/toolkit';
import deepEqual from 'fast-deep-equal';
import { flatten, groupBy } from 'lodash';
import { identity, Observable, of, SubscriptionLike, Unsubscribable } from 'rxjs';
import { identity, Observable, of, SubscriptionLike, Unsubscribable, combineLatest } from 'rxjs';
import { mergeMap, throttleTime } from 'rxjs/operators';
import {
@ -15,7 +15,6 @@ import {
hasQueryImportSupport,
HistoryItem,
LoadingState,
PanelData,
PanelEvents,
QueryFixAction,
toLegacyResponseData,
@ -32,8 +31,10 @@ import {
updateHistory,
} from 'app/core/utils/explore';
import { getShiftedTimeRange } from 'app/core/utils/timePicker';
import { CorrelationData } from 'app/features/correlations/useCorrelations';
import { getTimeZone } from 'app/features/profile/state/selectors';
import { MIXED_DATASOURCE_NAME } from 'app/plugins/datasource/mixed/MixedDataSource';
import { store } from 'app/store/store';
import { ExploreItemState, ExplorePanelData, ThunkDispatch, ThunkResult } from 'app/types';
import { ExploreId, ExploreState, QueryOptions } from 'app/types/explore';
@ -401,6 +402,8 @@ export const runQueries = (
return (dispatch, getState) => {
dispatch(updateTime({ exploreId }));
const correlations$ = getCorrelations();
// We always want to clear cache unless we explicitly pass preserveCache parameter
const preserveCache = options?.preserveCache === true;
if (!preserveCache) {
@ -438,15 +441,16 @@ export const runQueries = (
// If we have results saved in cache, we are going to use those results instead of running queries
if (cachedValue) {
newQuerySub = of(cachedValue)
newQuerySub = combineLatest([of(cachedValue), correlations$])
.pipe(
mergeMap((data: PanelData) =>
mergeMap(([data, correlations]) =>
decorateData(
data,
queryResponse,
absoluteRange,
refreshInterval,
queries,
correlations,
datasourceInstance != null && hasLogsVolumeSupport(datasourceInstance)
)
)
@ -493,19 +497,23 @@ export const runQueries = (
dispatch(changeLoadingStateAction({ exploreId, loadingState: LoadingState.Loading }));
newQuerySub = runRequest(datasourceInstance, transaction.request)
.pipe(
newQuerySub = combineLatest([
runRequest(datasourceInstance, transaction.request)
// Simple throttle for live tailing, in case of > 1000 rows per interval we spend about 200ms on processing and
// rendering. In case this is optimized this can be tweaked, but also it should be only as fast as user
// actually can see what is happening.
live ? throttleTime(500) : identity,
mergeMap((data: PanelData) =>
.pipe(live ? throttleTime(500) : identity),
correlations$,
])
.pipe(
mergeMap(([data, correlations]) =>
decorateData(
data,
queryResponse,
absoluteRange,
refreshInterval,
queries,
correlations,
datasourceInstance != null && hasLogsVolumeSupport(datasourceInstance)
)
)
@ -904,6 +912,28 @@ export const queryReducer = (state: ExploreItemState, action: AnyAction): Explor
return state;
};
/**
* Creates an observable that emits correlations once they are loaded
*/
const getCorrelations = () => {
return new Observable<CorrelationData[]>((subscriber) => {
const existingCorrelations = store.getState().explore.correlations;
if (existingCorrelations) {
subscriber.next(existingCorrelations);
subscriber.complete();
} else {
const unsubscribe = store.subscribe(() => {
const { correlations } = store.getState().explore;
if (correlations) {
unsubscribe();
subscriber.next(correlations);
subscriber.complete();
}
});
}
});
};
export const processQueryResponse = (
state: ExploreItemState,
action: PayloadAction<QueryEndedPayload>

View File

@ -1,4 +1,4 @@
import { groupBy } from 'lodash';
import { groupBy, mapValues } from 'lodash';
import { Observable, of } from 'rxjs';
import { map, mergeMap } from 'rxjs/operators';
@ -15,8 +15,10 @@ import { config } from '@grafana/runtime';
import { dataFrameToLogsModel } from '../../../core/logsModel';
import { refreshIntervalToSortOrder } from '../../../core/utils/explore';
import { sortLogsResult } from '../../../features/logs/utils';
import { ExplorePanelData } from '../../../types';
import { CorrelationData } from '../../correlations/useCorrelations';
import { attachCorrelationsToDataFrames } from '../../correlations/utils';
import { sortLogsResult } from '../../logs/utils';
import { preProcessPanelData } from '../../query/state/runRequest';
/**
@ -77,6 +79,22 @@ export const decorateWithFrameTypeMetadata = (data: PanelData): ExplorePanelData
};
};
export const decorateWithCorrelations = ({
queries,
correlations,
}: {
queries: DataQuery[] | undefined;
correlations: CorrelationData[] | undefined;
}) => {
return (data: PanelData): PanelData => {
if (queries?.length && correlations?.length) {
const queryRefIdToDataSourceUid = mapValues(groupBy(queries, 'refId'), '0.datasource.uid');
attachCorrelationsToDataFrames(data.series, correlations, queryRefIdToDataSourceUid);
}
return data;
};
};
export const decorateWithGraphResult = (data: ExplorePanelData): ExplorePanelData => {
if (!data.graphFrames.length) {
return { ...data, graphResult: null };
@ -169,10 +187,12 @@ export function decorateData(
absoluteRange: AbsoluteTimeRange,
refreshInterval: string | undefined,
queries: DataQuery[] | undefined,
correlations: CorrelationData[] | undefined,
fullRangeLogsVolumeAvailable: boolean
): Observable<ExplorePanelData> {
return of(data).pipe(
map((data: PanelData) => preProcessPanelData(data, queryResponse)),
map(decorateWithCorrelations({ queries, correlations })),
map(decorateWithFrameTypeMetadata),
map(decorateWithGraphResult),
map(decorateWithLogsResult({ absoluteRange, refreshInterval, queries, fullRangeLogsVolumeAvailable })),

View File

@ -18,6 +18,8 @@ import {
} from '@grafana/data';
import { RichHistorySearchFilters, RichHistorySettings } from 'app/core/utils/richHistoryTypes';
import { CorrelationData } from '../features/correlations/useCorrelations';
export enum ExploreId {
left = 'left',
right = 'right',
@ -45,6 +47,8 @@ export interface ExploreState {
*/
right?: ExploreItemState;
correlations?: CorrelationData[];
/**
* Settings for rich history (note: filters are stored per each pane separately)
*/

View File

@ -13,7 +13,7 @@ export function getGrafanaContextMock(overrides: Partial<GrafanaContextType> = {
// eslint-disable-next-line
location: {} as LocationService,
// eslint-disable-next-line
config: {} as GrafanaConfig,
config: { featureToggles: {} } as GrafanaConfig,
// eslint-disable-next-line
keybindings: {
clearAndInitGlobalBindings: jest.fn(),