IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Loki: Preserve pipeline stages in context query (#70472)

Browse Source 
* add pipeline stages to context query

* add ui

* improve `Postion`

* add `processPipelineStagesToExpr` to logcontextprovider

* add ui toggle

* fix lokicontextui tests

* remove import

* contextually hide the toggle

* Update `SHOULD_INCLUDE_PIPELINE_OPERATIONS` name

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

* add setIncludePipelineOperations to false in revert

* add prepareExpression method

* remove unused method

* fix test and add `runContextQuery`

* set correct revert state

* let let be const

* remove argument

---------

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>
This commit is contained in:
Sven Grossmann 2023-06-22 17:34:43 +02:00 committed by GitHub
parent 8079c1456f
commit d7337e4f9c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 589 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

189
public/app/plugins/datasource/loki/LogContextProvider.test.ts
View File

@ -3,14 +3,26 @@ import { of } from 'rxjs';
import { DataQueryResponse, FieldType, LogRowContextQueryDirection, LogRowModel, createDataFrame } from '@grafana/data';
import LokiLanguageProvider from './LanguageProvider';
import { LogContextProvider, LOKI_LOG_CONTEXT_PRESERVED_LABELS } from './LogContextProvider';
import {
LogContextProvider,
LOKI_LOG_CONTEXT_PRESERVED_LABELS,
SHOULD_INCLUDE_PIPELINE_OPERATIONS,
} from './LogContextProvider';
import { createLokiDatasource } from './mocks';
import { LokiQuery } from './types';
jest.mock('app/core/store', () => {
return {
get() {
return window.localStorage.getItem(LOKI_LOG_CONTEXT_PRESERVED_LABELS);
get(item: string) {
return window.localStorage.getItem(item);
},
getBool(key: string, defaultValue?: boolean) {
const item = window.localStorage.getItem(key);
if (item === null) {
return defaultValue;
} else {
return item === 'true';
}
},
};
});
@ -205,6 +217,143 @@ describe('LogContextProvider', () => {
expect(contextQuery.query.expr).toEqual(`{bar="baz"}`);
});
it('should not apply line_format if flag is not set by default', async () => {
logContextProvider.appliedContextFilters = [{ value: 'baz', enabled: true, fromParser: false, label: 'bar' }];
const contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
defaultLogRow,
10,
LogRowContextQueryDirection.Backward,
{
expr: '{bar="baz"} | logfmt | line_format = "foo"',
} as unknown as LokiQuery
);
expect(contextQuery.query.expr).toEqual(`{bar="baz"} | logfmt`);
});
it('should not apply line_format if flag is not set', async () => {
window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'false');
logContextProvider.appliedContextFilters = [{ value: 'baz', enabled: true, fromParser: false, label: 'bar' }];
const contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
defaultLogRow,
10,
LogRowContextQueryDirection.Backward,
{
expr: '{bar="baz"} | logfmt | line_format = "foo"',
} as unknown as LokiQuery
);
expect(contextQuery.query.expr).toEqual(`{bar="baz"} | logfmt`);
});
it('should apply line_format if flag is set', async () => {
window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'true');
logContextProvider.appliedContextFilters = [{ value: 'baz', enabled: true, fromParser: false, label: 'bar' }];
const contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
defaultLogRow,
10,
LogRowContextQueryDirection.Backward,
{
expr: '{bar="baz"} | logfmt | line_format = "foo"',
} as unknown as LokiQuery
);
expect(contextQuery.query.expr).toEqual(`{bar="baz"} | logfmt | line_format = "foo"`);
});
it('should not apply line filters if flag is set', async () => {
window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'true');
logContextProvider.appliedContextFilters = [{ value: 'baz', enabled: true, fromParser: false, label: 'bar' }];
let contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
defaultLogRow,
10,
LogRowContextQueryDirection.Backward,
{
expr: '{bar="baz"} | logfmt | line_format = "foo" |= "bar"',
} as unknown as LokiQuery
);
expect(contextQuery.query.expr).toEqual(`{bar="baz"} | logfmt | line_format = "foo"`);
contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
defaultLogRow,
10,
LogRowContextQueryDirection.Backward,
{
expr: '{bar="baz"} | logfmt | line_format = "foo" |~ "bar"',
} as unknown as LokiQuery
);
expect(contextQuery.query.expr).toEqual(`{bar="baz"} | logfmt | line_format = "foo"`);
contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
defaultLogRow,
10,
LogRowContextQueryDirection.Backward,
{
expr: '{bar="baz"} | logfmt | line_format = "foo" !~ "bar"',
} as unknown as LokiQuery
);
expect(contextQuery.query.expr).toEqual(`{bar="baz"} | logfmt | line_format = "foo"`);
contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
defaultLogRow,
10,
LogRowContextQueryDirection.Backward,
{
expr: '{bar="baz"} | logfmt | line_format = "foo" != "bar"',
} as unknown as LokiQuery
);
expect(contextQuery.query.expr).toEqual(`{bar="baz"} | logfmt | line_format = "foo"`);
});
it('should not apply line filters if nested between two operations', async () => {
window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'true');
logContextProvider.appliedContextFilters = [{ value: 'baz', enabled: true, fromParser: false, label: 'bar' }];
const contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
defaultLogRow,
10,
LogRowContextQueryDirection.Backward,
{
expr: '{bar="baz"} | logfmt | line_format "foo" |= "bar" | label_format a="baz"',
} as unknown as LokiQuery
);
expect(contextQuery.query.expr).toEqual(`{bar="baz"} | logfmt | line_format "foo" | label_format a="baz"`);
});
it('should not apply label filters', async () => {
window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'true');
logContextProvider.appliedContextFilters = [{ value: 'baz', enabled: true, fromParser: false, label: 'bar' }];
const contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
defaultLogRow,
10,
LogRowContextQueryDirection.Backward,
{
expr: '{bar="baz"} | logfmt | line_format "foo" | bar > 1 | label_format a="baz"',
} as unknown as LokiQuery
);
expect(contextQuery.query.expr).toEqual(`{bar="baz"} | logfmt | line_format "foo" | label_format a="baz"`);
});
it('should not apply additional parsers', async () => {
window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'true');
logContextProvider.appliedContextFilters = [{ value: 'baz', enabled: true, fromParser: false, label: 'bar' }];
const contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
defaultLogRow,
10,
LogRowContextQueryDirection.Backward,
{
expr: '{bar="baz"} | logfmt | line_format "foo" | json | label_format a="baz"',
} as unknown as LokiQuery
);
expect(contextQuery.query.expr).toEqual(`{bar="baz"}`);
});
});
describe('getInitContextFiltersFromLabels', () => {
@ -312,4 +461,38 @@ describe('LogContextProvider', () => {
});
});
});
describe('queryContainsValidPipelineStages', () => {
it('should return true if query contains a line_format stage', () => {
expect(
logContextProvider.queryContainsValidPipelineStages({ expr: '{foo="bar"} | line_format "foo"' } as LokiQuery)
).toBe(true);
});
it('should return true if query contains a label_format stage', () => {
expect(
logContextProvider.queryContainsValidPipelineStages({ expr: '{foo="bar"} | label_format a="foo"' } as LokiQuery)
).toBe(true);
});
it('should return false if query contains a parser', () => {
expect(logContextProvider.queryContainsValidPipelineStages({ expr: '{foo="bar"} | json' } as LokiQuery)).toBe(
false
);
});
it('should return false if query contains a line filter', () => {
expect(logContextProvider.queryContainsValidPipelineStages({ expr: '{foo="bar"} |= "test"' } as LokiQuery)).toBe(
false
);
});
it('should return true if query contains a line filter and a label_format', () => {
expect(
logContextProvider.queryContainsValidPipelineStages({
expr: '{foo="bar"} |= "test" | label_format a="foo"',
} as LokiQuery)
).toBe(true);
});
});
});

69
public/app/plugins/datasource/loki/LogContextProvider.ts
View File

@ -14,6 +14,7 @@ import {
LogRowContextQueryDirection,
LogRowContextOptions,
} from '@grafana/data';
import { LabelParser, LabelFilter, LineFilters, PipelineStage } from '@grafana/lezer-logql';
import { Labels } from '@grafana/schema';
import { notifyApp } from 'app/core/actions';
import { createSuccessNotification } from 'app/core/copy/appNotification';
@ -24,11 +25,18 @@ import { LokiContextUi } from './components/LokiContextUi';
import { LokiDatasource, makeRequest, REF_ID_STARTER_LOG_ROW_CONTEXT } from './datasource';
import { escapeLabelValueInExactSelector } from './languageUtils';
import { addLabelToQuery, addParserToQuery } from './modifyQuery';
import { getParserFromQuery, getStreamSelectorsFromQuery, isQueryWithParser } from './queryUtils';
import {
getNodePositionsFromQuery,
getParserFromQuery,
getStreamSelectorsFromQuery,
isQueryWithParser,
} from './queryUtils';
import { sortDataFrameByTime, SortDirection } from './sortDataFrame';
import { ContextFilter, LokiQuery, LokiQueryDirection, LokiQueryType } from './types';
export const LOKI_LOG_CONTEXT_PRESERVED_LABELS = 'lokiLogContextPreservedLabels';
export const SHOULD_INCLUDE_PIPELINE_OPERATIONS = 'lokiLogContextShouldIncludePipelineOperations';
export type PreservedLabels = {
removedLabels: string[];
selectedExtractedLabels: string[];
@ -109,7 +117,8 @@ export class LogContextProvider {
direction: LogRowContextQueryDirection,
origQuery?: LokiQuery
): Promise<{ query: LokiQuery; range: TimeRange }> {
const expr = this.processContextFiltersToExpr(row, this.appliedContextFilters, origQuery);
const expr = this.prepareExpression(this.appliedContextFilters, origQuery);
const contextTimeBuffer = 2 * 60 * 60 * 1000; // 2h buffer
const queryDirection =
@ -180,10 +189,19 @@ export class LogContextProvider {
updateFilter,
onClose: this.onContextClose,
logContextProvider: this,
runContextQuery,
});
}
processContextFiltersToExpr = (row: LogRowModel, contextFilters: ContextFilter[], query: LokiQuery | undefined) => {
prepareExpression(contextFilters: ContextFilter[], query: LokiQuery | undefined): string {
let preparedExpression = this.processContextFiltersToExpr(contextFilters, query);
if (store.getBool(SHOULD_INCLUDE_PIPELINE_OPERATIONS, false)) {
preparedExpression = this.processPipelineStagesToExpr(preparedExpression, query);
}
return preparedExpression;
}
processContextFiltersToExpr = (contextFilters: ContextFilter[], query: LokiQuery | undefined): string => {
const labelFilters = contextFilters
.map((filter) => {
if (!filter.fromParser && filter.enabled) {
@ -216,6 +234,51 @@ export class LogContextProvider {
return expr;
};
processPipelineStagesToExpr = (currentExpr: string, query: LokiQuery | undefined): string => {
let newExpr = currentExpr;
const origExpr = query?.expr ?? '';
if (isQueryWithParser(origExpr).parserCount > 1) {
return newExpr;
}
const allNodePositions = getNodePositionsFromQuery(origExpr, [
PipelineStage,
LabelParser,
LineFilters,
LabelFilter,
]);
const pipelineStagePositions = allNodePositions.filter((position) => position.type?.id === PipelineStage);
const otherNodePositions = allNodePositions.filter((position) => position.type?.id !== PipelineStage);
for (const pipelineStagePosition of pipelineStagePositions) {
// we don't process pipeline stages that contain label parsers, line filters or label filters
if (otherNodePositions.some((position) => pipelineStagePosition.contains(position))) {
continue;
}
newExpr += ` ${pipelineStagePosition.getExpression(origExpr)}`;
}
return newExpr;
};
queryContainsValidPipelineStages = (query: LokiQuery | undefined): boolean => {
const origExpr = query?.expr ?? '';
const allNodePositions = getNodePositionsFromQuery(origExpr, [
PipelineStage,
LabelParser,
LineFilters,
LabelFilter,
]);
const pipelineStagePositions = allNodePositions.filter((position) => position.type?.id === PipelineStage);
const otherNodePositions = allNodePositions.filter((position) => position.type?.id !== PipelineStage);
return pipelineStagePositions.some((pipelineStagePosition) =>
otherNodePositions.every((position) => pipelineStagePosition.contains(position) === false)
);
};
getInitContextFilters = async (labels: Labels, query?: LokiQuery) => {
if (!query || isEmpty(labels)) {
return [];

104
public/app/plugins/datasource/loki/components/LokiContextUi.test.tsx
View File

@ -5,10 +5,10 @@ import { selectOptionInTest } from 'test/helpers/selectOptionInTest';
import { LogRowModel } from '@grafana/data';
import { LogContextProvider } from '../LogContextProvider';
import { LogContextProvider, SHOULD_INCLUDE_PIPELINE_OPERATIONS } from '../LogContextProvider';
import { ContextFilter, LokiQuery } from '../types';
import { LokiContextUi, LokiContextUiProps } from './LokiContextUi';
import { IS_LOKI_LOG_CONTEXT_UI_OPEN, LokiContextUi, LokiContextUiProps } from './LokiContextUi';
// we have to mock out reportInteraction, otherwise it crashes the test.
jest.mock('@grafana/runtime', () => ({
@ -19,8 +19,13 @@ jest.mock('@grafana/runtime', () => ({
jest.mock('app/core/store', () => {
return {
set() {},
getBool() {
return true;
getBool(key: string, defaultValue?: boolean) {
const item = window.localStorage.getItem(key);
if (item === null) {
return defaultValue;
} else {
return item === 'true';
}
},
delete() {},
};
@ -28,7 +33,7 @@ jest.mock('app/core/store', () => {
const setupProps = (): LokiContextUiProps => {
const defaults: LokiContextUiProps = {
logContextProvider: mockLogContextProvider as unknown as LogContextProvider,
logContextProvider: Object.assign({}, mockLogContextProvider) as unknown as LogContextProvider,
updateFilter: jest.fn(),
row: {
entry: 'WARN test 1.23 on [xxx]',
@ -42,6 +47,7 @@ const setupProps = (): LokiContextUiProps => {
expr: '{label1="value1"} | logfmt',
refId: 'A',
},
runContextQuery: jest.fn(),
};
return defaults;
@ -55,13 +61,24 @@ const mockLogContextProvider = {
])
),
processContextFiltersToExpr: jest.fn().mockImplementation(
(row: LogRowModel, contextFilters: ContextFilter[], query: LokiQuery | undefined) =>
(contextFilters: ContextFilter[], query: LokiQuery | undefined) =>
`{${contextFilters
.filter((filter) => filter.enabled)
.map((filter) => `${filter.label}="${filter.value}"`)
.join('` ')}}`
),
processPipelineStagesToExpr: jest
.fn()
.mockImplementation((currentExpr: string, query: LokiQuery | undefined) => `${currentExpr} | newOperation`),
getLogRowContext: jest.fn(),
queryContainsValidPipelineStages: jest.fn().mockReturnValue(true),
prepareExpression: jest.fn().mockImplementation(
(contextFilters: ContextFilter[], query: LokiQuery | undefined) =>
`{${contextFilters
.filter((filter) => filter.enabled)
.map((filter) => `${filter.label}="${filter.value}"`)
.join('` ')}}`
),
};
describe('LokiContextUi', () => {
@ -80,6 +97,15 @@ describe('LokiContextUi', () => {
global = savedGlobal;
});
beforeEach(() => {
window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'true');
window.localStorage.setItem(IS_LOKI_LOG_CONTEXT_UI_OPEN, 'true');
});
afterEach(() => {
window.localStorage.clear();
});
it('renders and shows executed query text', async () => {
const props = setupProps();
render(<LokiContextUi {...props} />);
@ -193,6 +219,72 @@ describe('LokiContextUi', () => {
});
});
it('renders pipeline operations switch as enabled when saved in localstorage', async () => {
const props = setupProps();
const newProps = {
...props,
origQuery: {
expr: '{label1="value1"} | logfmt',
refId: 'A',
},
};
window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'true');
render(<LokiContextUi {...newProps} />);
await waitFor(() => {
expect((screen.getByRole('checkbox') as HTMLInputElement).checked).toBe(true);
});
});
it('renders pipeline operations switch as disabled when saved in localstorage', async () => {
const props = setupProps();
const newProps = {
...props,
origQuery: {
expr: '{label1="value1"} | logfmt',
refId: 'A',
},
};
window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'false');
render(<LokiContextUi {...newProps} />);
await waitFor(() => {
expect((screen.getByRole('checkbox') as HTMLInputElement).checked).toBe(false);
});
});
it('renders pipeline operations switch if query contains valid pipeline stages', async () => {
const props = setupProps();
(props.logContextProvider.queryContainsValidPipelineStages as jest.Mock).mockReturnValue(true);
const newProps = {
...props,
origQuery: {
expr: '{label1="value1"} | logfmt',
refId: 'A',
},
};
window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'true');
render(<LokiContextUi {...newProps} />);
await waitFor(() => {
expect(screen.getByRole('checkbox')).toBeInTheDocument();
});
});
it('does not render pipeline operations switch if query does not contain valid pipeline stages', async () => {
const props = setupProps();
(props.logContextProvider.queryContainsValidPipelineStages as jest.Mock).mockReturnValue(false);
const newProps = {
...props,
origQuery: {
expr: '{label1="value1"} | logfmt',
refId: 'A',
},
};
window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'true');
render(<LokiContextUi {...newProps} />);
await waitFor(() => {
expect(screen.queryByRole('checkbox')).toBeNull();
});
});
it('does not show parsed labels section if origQuery has 2 parsers', async () => {
const props = setupProps();
const newProps = {

99
public/app/plugins/datasource/loki/components/LokiContextUi.tsx
View File

@ -2,13 +2,31 @@ import { css } from '@emotion/css';
import React, { useCallback, useEffect, useMemo, useState } from 'react';
import { useAsync } from 'react-use';
import { GrafanaTheme2, LogRowModel, SelectableValue } from '@grafana/data';
import { GrafanaTheme2, LogRowModel, renderMarkdown, SelectableValue } from '@grafana/data';
import { reportInteraction } from '@grafana/runtime';
import { Button, Collapse, Icon, Label, MultiSelect, Spinner, Tooltip, useStyles2 } from '@grafana/ui';
import {
Button,
Collapse,
Icon,
InlineField,
InlineFieldRow,
InlineSwitch,
Label,
MultiSelect,
RenderUserContentAsHTML,
Spinner,
Tooltip,
useStyles2,
} from '@grafana/ui';
import store from 'app/core/store';
import { RawQuery } from '../../prometheus/querybuilder/shared/RawQuery';
import { LogContextProvider, LOKI_LOG_CONTEXT_PRESERVED_LABELS, PreservedLabels } from '../LogContextProvider';
import {
LogContextProvider,
LOKI_LOG_CONTEXT_PRESERVED_LABELS,
PreservedLabels,
SHOULD_INCLUDE_PIPELINE_OPERATIONS,
} from '../LogContextProvider';
import { escapeLabelValueInSelector } from '../languageUtils';
import { isQueryWithParser } from '../queryUtils';
import { lokiGrammar } from '../syntax';
@ -20,6 +38,7 @@ export interface LokiContextUiProps {
updateFilter: (value: ContextFilter[]) => void;
onClose: () => void;
origQuery?: LokiQuery;
runContextQuery?: () => void;
}
function getStyles(theme: GrafanaTheme2) {
@ -74,13 +93,22 @@ function getStyles(theme: GrafanaTheme2) {
right: ${theme.spacing(1)};
z-index: ${theme.zIndex.navbarFixed};
`,
operationsToggle: css`
margin: ${theme.spacing(1)} 0 ${theme.spacing(-1)} 0;
& > div {
margin: 0;
& > label {
padding: 0;
}
}
`,
};
}
const IS_LOKI_LOG_CONTEXT_UI_OPEN = 'isLogContextQueryUiOpen';
export const IS_LOKI_LOG_CONTEXT_UI_OPEN = 'isLogContextQueryUiOpen';
export function LokiContextUi(props: LokiContextUiProps) {
const { row, logContextProvider, updateFilter, onClose, origQuery } = props;
const { row, logContextProvider, updateFilter, onClose, origQuery, runContextQuery } = props;
const styles = useStyles2(getStyles);
const [contextFilters, setContextFilters] = useState<ContextFilter[]>([]);
@ -88,19 +116,27 @@ export function LokiContextUi(props: LokiContextUiProps) {
const [initialized, setInitialized] = useState(false);
const [loading, setLoading] = useState(false);
const [isOpen, setIsOpen] = useState(store.getBool(IS_LOKI_LOG_CONTEXT_UI_OPEN, false));
const [includePipelineOperations, setIncludePipelineOperations] = useState(
store.getBool(SHOULD_INCLUDE_PIPELINE_OPERATIONS, false)
);
const timerHandle = React.useRef<number>();
const previousInitialized = React.useRef<boolean>(false);
const previousContextFilters = React.useRef<ContextFilter[]>([]);
const isInitialQuery = useMemo(() => {
const isInitialState = useMemo(() => {
// Initial query has all regular labels enabled and all parsed labels disabled
if (initialized && contextFilters.some((filter) => filter.fromParser === filter.enabled)) {
return false;
}
// if we include pipeline operations, we also want to enable the revert button
if (includePipelineOperations && logContextProvider.queryContainsValidPipelineStages(origQuery)) {
return false;
}
return true;
}, [contextFilters, initialized]);
}, [contextFilters, includePipelineOperations, initialized, logContextProvider, origQuery]);
useEffect(() => {
if (!initialized) {
@ -200,6 +236,10 @@ export function LokiContextUi(props: LokiContextUiProps) {
// Currently we support adding of parser and showing parsed labels only if there is 1 parser
const showParsedLabels = origQuery && isQueryWithParser(origQuery.expr).parserCount === 1 && parsedLabels.length > 0;
let queryExpr = logContextProvider.prepareExpression(
contextFilters.filter(({ enabled }) => enabled),
origQuery
);
return (
<div className={styles.wrapper}>
<Tooltip content={'Revert to initial log context query.'}>
@ -208,7 +248,7 @@ export function LokiContextUi(props: LokiContextUiProps) {
data-testid="revert-button"
icon="history-alt"
variant="secondary"
disabled={isInitialQuery}
disabled={isInitialState}
onClick={(e) => {
reportInteraction('grafana_explore_logs_loki_log_context_reverted', {
logRowUid: row.uid,
@ -222,6 +262,8 @@ export function LokiContextUi(props: LokiContextUiProps) {
});
// We are removing the preserved labels from local storage so we can preselect the labels in the UI
store.delete(LOKI_LOG_CONTEXT_PRESERVED_LABELS);
store.delete(SHOULD_INCLUDE_PIPELINE_OPERATIONS);
setIncludePipelineOperations(false);
}}
/>
</div>
@ -242,15 +284,7 @@ export function LokiContextUi(props: LokiContextUiProps) {
<div className={styles.rawQueryContainer}>
{initialized ? (
<>
<RawQuery
lang={{ grammar: lokiGrammar, name: 'loki' }}
query={logContextProvider.processContextFiltersToExpr(
row,
contextFilters.filter(({ enabled }) => enabled),
origQuery
)}
className={styles.rawQuery}
/>
<RawQuery lang={{ grammar: lokiGrammar, name: 'loki' }} query={queryExpr} className={styles.rawQuery} />
<Tooltip content="The initial log context query is created from all labels defining the stream for the selected log line. Use the editor below to customize the log context query.">
<Icon name="info-circle" size="sm" className={styles.queryDescription} />
</Tooltip>
@ -345,6 +379,37 @@ export function LokiContextUi(props: LokiContextUiProps) {
/>
</>
)}
{logContextProvider.queryContainsValidPipelineStages(origQuery) && (
<InlineFieldRow className={styles.operationsToggle}>
<InlineField
label="Include LogQL pipeline operations"
tooltip={
<RenderUserContentAsHTML
content={renderMarkdown(
"This will include LogQL operations such as `line_format` or `label_format`. It won't include line or label filter operations."
)}
/>
}
>
<InlineSwitch
value={includePipelineOperations}
showLabel={true}
transparent={true}
onChange={(e) => {
reportInteraction('grafana_explore_logs_loki_log_context_pipeline_toggled', {
logRowUid: row.uid,
action: e.currentTarget.checked ? 'enable' : 'disable',
});
store.set(SHOULD_INCLUDE_PIPELINE_OPERATIONS, e.currentTarget.checked);
setIncludePipelineOperations(e.currentTarget.checked);
if (runContextQuery) {
runContextQuery();
}
}}
/>
</InlineField>
</InlineFieldRow>
)}
</div>
</Collapse>
</div>

59
public/app/plugins/datasource/loki/modifyQuery.test.ts
View File

@ -1,8 +1,11 @@
import { SyntaxNode } from '@lezer/common';
import {
addLabelFormatToQuery,
addLabelToQuery,
addNoPipelineErrorToQuery,
addParserToQuery,
NodePosition,
removeCommentsFromQuery,
} from './modifyQuery';
@ -187,3 +190,59 @@ describe('removeCommentsFromQuery', () => {
expect(removeCommentsFromQuery(query)).toBe(expectedResult);
});
});
describe('NodePosition', () => {
describe('contains', () => {
it('should return true if the position is contained within the current position', () => {
const position = new NodePosition(5, 10);
const containedPosition = new NodePosition(6, 9);
const result = position.contains(containedPosition);
expect(result).toBe(true);
});
it('should return false if the position is not contained within the current position', () => {
const position = new NodePosition(5, 10);
const outsidePosition = new NodePosition(11, 15);
const result = position.contains(outsidePosition);
expect(result).toBe(false);
});
it('should return true if the position is the same as the current position', () => {
const position = new NodePosition(5, 10);
const samePosition = new NodePosition(5, 10);
const result = position.contains(samePosition);
expect(result).toBe(true);
});
});
describe('getExpression', () => {
it('should return the substring of the query within the given position', () => {
const position = new NodePosition(7, 12);
const query = 'Hello, world!';
const result = position.getExpression(query);
expect(result).toBe('world');
});
it('should return an empty string if the position is out of range', () => {
const position = new NodePosition(15, 20);
const query = 'Hello, world!';
const result = position.getExpression(query);
expect(result).toBe('');
});
});
describe('fromNode', () => {
it('should create a new NodePosition instance from a SyntaxNode', () => {
const syntaxNode = {
from: 5,
to: 10,
type: 'identifier',
} as unknown as SyntaxNode;
const result = NodePosition.fromNode(syntaxNode);
expect(result).toBeInstanceOf(NodePosition);
expect(result.from).toBe(5);
expect(result.to).toBe(10);
expect(result.type).toBe('identifier');
});
});
});

94
public/app/plugins/datasource/loki/modifyQuery.ts
View File

@ -1,4 +1,4 @@
import { SyntaxNode } from '@lezer/common';
import { NodeType, SyntaxNode } from '@lezer/common';
import { sortBy } from 'lodash';
import {
@ -22,7 +22,29 @@ import { unescapeLabelValue } from './languageUtils';
import { LokiQueryModeller } from './querybuilder/LokiQueryModeller';
import { buildVisualQueryFromString } from './querybuilder/parsing';
export type Position = { from: number; to: number };
export class NodePosition {
from: number;
to: number;
type?: NodeType;
constructor(from: number, to: number, type?: NodeType) {
this.from = from;
this.to = to;
this.type = type;
}
static fromNode(node: SyntaxNode): NodePosition {
return new NodePosition(node.from, node.to, node.type);
}
contains(position: NodePosition): boolean {
return this.from <= position.from && this.to >= position.to;
}
getExpression(query: string): string {
return query.substring(this.from, this.to);
}
}
/**
* Adds label filter to existing query. Useful for query modification for example for ad hoc filters.
*
@ -139,13 +161,13 @@ export function removeCommentsFromQuery(query: string): string {
* selector.
* @param query
*/
export function getStreamSelectorPositions(query: string): Position[] {
export function getStreamSelectorPositions(query: string): NodePosition[] {
const tree = parser.parse(query);
const positions: Position[] = [];
const positions: NodePosition[] = [];
tree.iterate({
enter: ({ type, from, to }): false | void => {
if (type.id === Selector) {
positions.push({ from, to });
positions.push(new NodePosition(from, to, type));
return false;
}
},
@ -153,9 +175,9 @@ export function getStreamSelectorPositions(query: string): Position[] {
return positions;
}
function getMatcherInStreamPositions(query: string): Position[] {
function getMatcherInStreamPositions(query: string): NodePosition[] {
const tree = parser.parse(query);
const positions: Position[] = [];
const positions: NodePosition[] = [];
tree.iterate({
enter: ({ node }): false | void => {
if (node.type.id === Selector) {
@ -170,13 +192,13 @@ function getMatcherInStreamPositions(query: string): Position[] {
* Parse the string and get all LabelParser positions in the query.
* @param query
*/
export function getParserPositions(query: string): Position[] {
export function getParserPositions(query: string): NodePosition[] {
const tree = parser.parse(query);
const positions: Position[] = [];
const positions: NodePosition[] = [];
tree.iterate({
enter: ({ type, from, to }): false | void => {
if (type.id === LabelParser || type.id === JsonExpressionParser) {
positions.push({ from, to });
positions.push(new NodePosition(from, to, type));
return false;
}
},
@ -188,13 +210,13 @@ export function getParserPositions(query: string): Position[] {
* Parse the string and get all LabelFilter positions in the query.
* @param query
*/
export function getLabelFilterPositions(query: string): Position[] {
export function getLabelFilterPositions(query: string): NodePosition[] {
const tree = parser.parse(query);
const positions: Position[] = [];
const positions: NodePosition[] = [];
tree.iterate({
enter: ({ type, from, to }): false | void => {
if (type.id === LabelFilter) {
positions.push({ from, to });
positions.push(new NodePosition(from, to, type));
return false;
}
},
@ -206,13 +228,13 @@ export function getLabelFilterPositions(query: string): Position[] {
* Parse the string and get all Line filter positions in the query.
* @param query
*/
function getLineFiltersPositions(query: string): Position[] {
function getLineFiltersPositions(query: string): NodePosition[] {
const tree = parser.parse(query);
const positions: Position[] = [];
const positions: NodePosition[] = [];
tree.iterate({
enter: ({ type, node }): false | void => {
enter: ({ type, from, to }): false | void => {
if (type.id === LineFilters) {
positions.push({ from: node.from, to: node.to });
positions.push(new NodePosition(from, to, type));
return false;
}
},
@ -224,13 +246,13 @@ function getLineFiltersPositions(query: string): Position[] {
* Parse the string and get all Log query positions in the query.
* @param query
*/
function getLogQueryPositions(query: string): Position[] {
function getLogQueryPositions(query: string): NodePosition[] {
const tree = parser.parse(query);
const positions: Position[] = [];
const positions: NodePosition[] = [];
tree.iterate({
enter: ({ type, from, to, node }): false | void => {
if (type.id === LogExpr) {
positions.push({ from, to });
positions.push(new NodePosition(from, to, type));
return false;
}
@ -238,25 +260,25 @@ function getLogQueryPositions(query: string): Position[] {
if (type.id === LogRangeExpr) {
// Unfortunately, LogRangeExpr includes both log and non-log (e.g. Duration/Range/...) parts of query.
// We get position of all log-parts within LogRangeExpr: Selector, PipelineExpr and UnwrapExpr.
const logPartsPositions: Position[] = [];
const logPartsPositions: NodePosition[] = [];
const selector = node.getChild(Selector);
if (selector) {
logPartsPositions.push({ from: selector.from, to: selector.to });
logPartsPositions.push(NodePosition.fromNode(selector));
}
const pipeline = node.getChild(PipelineExpr);
if (pipeline) {
logPartsPositions.push({ from: pipeline.from, to: pipeline.to });
logPartsPositions.push(NodePosition.fromNode(pipeline));
}
const unwrap = node.getChild(UnwrapExpr);
if (unwrap) {
logPartsPositions.push({ from: unwrap.from, to: unwrap.to });
logPartsPositions.push(NodePosition.fromNode(unwrap));
}
// We sort them and then pick "from" from first position and "to" from last position.
const sorted = sortBy(logPartsPositions, (position) => position.to);
positions.push({ from: sorted[0].from, to: sorted[sorted.length - 1].to });
positions.push(new NodePosition(sorted[0].from, sorted[sorted.length - 1].to));
return false;
}
},
@ -277,7 +299,7 @@ export function toLabelFilter(key: string, value: string, operator: string): Que
*/
function addFilterToStreamSelector(
query: string,
vectorSelectorPositions: Position[],
vectorSelectorPositions: NodePosition[],
filter: QueryBuilderLabelFilter
): string {
const modeller = new LokiQueryModeller();
@ -313,7 +335,7 @@ function addFilterToStreamSelector(
*/
export function addFilterAsLabelFilter(
query: string,
positionsToAddAfter: Position[],
positionsToAddAfter: NodePosition[],
filter: QueryBuilderLabelFilter
): string {
let newQuery = '';
@ -349,7 +371,7 @@ export function addFilterAsLabelFilter(
* @param queryPartPositions
* @param parser
*/
function addParser(query: string, queryPartPositions: Position[], parser: string): string {
function addParser(query: string, queryPartPositions: NodePosition[], parser: string): string {
let newQuery = '';
let prev = 0;
@ -376,7 +398,7 @@ function addParser(query: string, queryPartPositions: Position[], parser: string
*/
function addLabelFormat(
query: string,
logQueryPositions: Position[],
logQueryPositions: NodePosition[],
labelFormat: { originalLabel: string; renameTo: string }
): string {
let newQuery = '';
@ -405,13 +427,13 @@ export function addLineFilter(query: string): string {
return newQueryExpr;
}
function getLineCommentPositions(query: string): Position[] {
function getLineCommentPositions(query: string): NodePosition[] {
const tree = parser.parse(query);
const positions: Position[] = [];
const positions: NodePosition[] = [];
tree.iterate({
enter: ({ type, from, to }): false | void => {
if (type.id === LineComment) {
positions.push({ from, to });
positions.push(new NodePosition(from, to, type));
return false;
}
},
@ -432,16 +454,16 @@ function labelExists(labels: QueryBuilderLabelFilter[], filter: QueryBuilderLabe
* Return the last position based on "to" property
* @param positions
*/
export function findLastPosition(positions: Position[]): Position {
export function findLastPosition(positions: NodePosition[]): NodePosition {
return positions.reduce((prev, current) => (prev.to > current.to ? prev : current));
}
function getAllPositionsInNodeByType(query: string, node: SyntaxNode, type: number): Position[] {
function getAllPositionsInNodeByType(query: string, node: SyntaxNode, type: number): NodePosition[] {
if (node.type.id === type) {
return [{ from: node.from, to: node.to }];
return [NodePosition.fromNode(node)];
}
const positions: Position[] = [];
const positions: NodePosition[] = [];
let pos = 0;
let child = node.childAfter(pos);
while (child) {

24
public/app/plugins/datasource/loki/queryUtils.test.ts
View File

@ -1,3 +1,5 @@
import { String } from '@grafana/lezer-logql';
import {
getHighlighterExpressionsFromQuery,
getLokiQueryType,
@ -14,6 +16,7 @@ import {
isQueryPipelineErrorFiltering,
getLogQueryFromMetricsQuery,
getNormalizedLokiQuery,
getNodePositionsFromQuery,
} from './queryUtils';
import { LokiQuery, LokiQueryType } from './types';
@ -416,3 +419,24 @@ describe('getLogQueryFromMetricsQuery', () => {
).toBe('{label="$var"} | logfmt | __error__=``');
});
});
describe('getNodePositionsFromQuery', () => {
it('returns the right amount of positions without type', () => {
// LogQL, Expr, LogExpr, Selector, Matchers, Matcher, Identifier, Eq, String
expect(getNodePositionsFromQuery('{job="grafana"}').length).toBe(9);
});
it('returns the right position of a string in a stream selector', () => {
// LogQL, Expr, LogExpr, Selector, Matchers, Matcher, Identifier, Eq, String
const nodePositions = getNodePositionsFromQuery('{job="grafana"}', [String]);
expect(nodePositions.length).toBe(1);
expect(nodePositions[0].from).toBe(5);
expect(nodePositions[0].to).toBe(14);
});
it('returns an empty array with a wrong expr', () => {
// LogQL, Expr, LogExpr, Selector, Matchers, Matcher, Identifier, Eq, String
const nodePositions = getNodePositionsFromQuery('not loql', [String]);
expect(nodePositions.length).toBe(0);
});
});

19
public/app/plugins/datasource/loki/queryUtils.ts
View File

@ -24,7 +24,7 @@ import { DataQuery } from '@grafana/schema';
import { ErrorId } from '../prometheus/querybuilder/shared/parsingUtils';
import { getStreamSelectorPositions } from './modifyQuery';
import { getStreamSelectorPositions, NodePosition } from './modifyQuery';
import { LokiQuery, LokiQueryType } from './types';
export function formatQuery(selector: string | undefined): string {
@ -145,12 +145,12 @@ export function isQueryWithNode(query: string, nodeType: number): boolean {
return isQueryWithNode;
}
export function getNodesFromQuery(query: string, nodeTypes: number[]): SyntaxNode[] {
export function getNodesFromQuery(query: string, nodeTypes?: number[]): SyntaxNode[] {
const nodes: SyntaxNode[] = [];
const tree = parser.parse(query);
tree.iterate({
enter: (node): false | void => {
if (nodeTypes.includes(node.type.id)) {
if (nodeTypes === undefined || nodeTypes.includes(node.type.id)) {
nodes.push(node.node);
}
},
@ -158,6 +158,19 @@ export function getNodesFromQuery(query: string, nodeTypes: number[]): SyntaxNod
return nodes;
}
export function getNodePositionsFromQuery(query: string, nodeTypes?: number[]): NodePosition[] {
const positions: NodePosition[] = [];
const tree = parser.parse(query);
tree.iterate({
enter: (node): false | void => {
if (nodeTypes === undefined || nodeTypes.includes(node.type.id)) {
positions.push(NodePosition.fromNode(node.node));
}
},
});
return positions;
}
export function getNodeFromQuery(query: string, nodeType: number): SyntaxNode | undefined {
const nodes = getNodesFromQuery(query, [nodeType]);
return nodes.length > 0 ? nodes[0] : undefined;