IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Loki: Move log context to separate log context provider (#66357)

Browse Source 
* Loki: Move log context to separate provider

* Update public/app/plugins/datasource/loki/LogContextProvider.ts

Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com>

* Fix lint

---------

Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com>
This commit is contained in:
Ivana Huckova 2023-04-12 15:09:37 +02:00 committed by GitHub
parent 2c21090931
commit e12598f55c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 228 additions and 248 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.betterer.results
View File

@ -4606,6 +4606,9 @@ exports[`better eslint`] = {
"public/app/plugins/datasource/loki/LiveStreams.ts:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"]
],
"public/app/plugins/datasource/loki/LogContextProvider.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"]
],
"public/app/plugins/datasource/loki/components/LokiLabelBrowser.tsx:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"],
[0, 0, 0, "Do not use any type assertions.", "1"]
@ -4632,10 +4635,9 @@ exports[`better eslint`] = {
[0, 0, 0, "Unexpected any. Specify a different type.", "1"],
[0, 0, 0, "Unexpected any. Specify a different type.", "2"],
[0, 0, 0, "Unexpected any. Specify a different type.", "3"],
[0, 0, 0, "Do not use any type assertions.", "4"],
[0, 0, 0, "Unexpected any. Specify a different type.", "4"],
[0, 0, 0, "Unexpected any. Specify a different type.", "5"],
[0, 0, 0, "Unexpected any. Specify a different type.", "6"],
[0, 0, 0, "Unexpected any. Specify a different type.", "7"]
[0, 0, 0, "Unexpected any. Specify a different type.", "6"]
],
"public/app/plugins/datasource/loki/getDerivedFields.ts:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"]

152
public/app/plugins/datasource/loki/LogContextProvider.ts Normal file
View File

@ -0,0 +1,152 @@
import { FieldCache, FieldType, LogRowModel, TimeRange, toUtc } from '@grafana/data';
import { DataQuery } from '@grafana/schema';
import LokiLanguageProvider from './LanguageProvider';
import { LokiContextUi } from './components/LokiContextUi';
import { REF_ID_STARTER_LOG_ROW_CONTEXT } from './datasource';
import { escapeLabelValueInExactSelector } from './languageUtils';
import { addLabelToQuery, addParserToQuery } from './modifyQuery';
import { getParserFromQuery } from './queryUtils';
import { ContextFilter, LokiQuery, LokiQueryDirection, LokiQueryType } from './types';
export class LogContextProvider {
languageProvider: LokiLanguageProvider;
onContextClose: (() => void) | undefined;
constructor(languageProvider: LokiLanguageProvider) {
this.languageProvider = languageProvider;
}
async prepareLogRowContextQueryTarget(
row: LogRowModel,
limit: number,
direction: 'BACKWARD' | 'FORWARD',
origQuery?: DataQuery
): Promise<{ query: LokiQuery; range: TimeRange }> {
let expr = await this.prepareContextExpr(row, origQuery);
const contextTimeBuffer = 2 * 60 * 60 * 1000; // 2h buffer
const queryDirection = direction === 'FORWARD' ? LokiQueryDirection.Forward : LokiQueryDirection.Backward;
const query: LokiQuery = {
expr,
queryType: LokiQueryType.Range,
refId: `${REF_ID_STARTER_LOG_ROW_CONTEXT}${row.dataFrame.refId || ''}`,
maxLines: limit,
direction: queryDirection,
};
const fieldCache = new FieldCache(row.dataFrame);
const tsField = fieldCache.getFirstFieldOfType(FieldType.time);
if (tsField === undefined) {
throw new Error('loki: data frame missing time-field, should never happen');
}
const tsValue = tsField.values.get(row.rowIndex);
const timestamp = toUtc(tsValue);
const range =
queryDirection === LokiQueryDirection.Forward
? {
// start param in Loki API is inclusive so we'll have to filter out the row that this request is based from
// and any other that were logged in the same ns but before the row. Right now these rows will be lost
// because the are before but came it he response that should return only rows after.
from: timestamp,
// convert to ns, we lose some precision here but it is not that important at the far points of the context
to: toUtc(row.timeEpochMs + contextTimeBuffer),
}
: {
// convert to ns, we lose some precision here but it is not that important at the far points of the context
from: toUtc(row.timeEpochMs - contextTimeBuffer),
to: timestamp,
};
return {
query,
range: {
from: range.from,
to: range.to,
raw: range,
},
};
}
getLogRowContextUi(row: LogRowModel, runContextQuery: () => void): React.ReactNode {
const updateFilter = (contextFilters: ContextFilter[]) => {
this.prepareContextExpr = async (row: LogRowModel, origQuery?: DataQuery) => {
await this.languageProvider.start();
const labels = this.languageProvider.getLabelKeys();
let expr = contextFilters
.map((filter) => {
const label = filter.value;
if (filter && !filter.fromParser && filter.enabled && labels.includes(label)) {
// escape backslashes in label as users can't escape them by themselves
return `${label}="${escapeLabelValueInExactSelector(row.labels[label])}"`;
}
return '';
})
// Filter empty strings
.filter((label) => !!label)
.join(',');
expr = `{${expr}}`;
const parserContextFilters = contextFilters.filter((filter) => filter.fromParser && filter.enabled);
if (parserContextFilters.length) {
// we should also filter for labels from parsers, let's find the right parser
if (origQuery) {
const parser = getParserFromQuery((origQuery as LokiQuery).expr);
if (parser) {
expr = addParserToQuery(expr, parser);
}
}
for (const filter of parserContextFilters) {
if (filter.enabled) {
expr = addLabelToQuery(expr, filter.label, '=', row.labels[filter.label]);
}
}
}
return expr;
};
if (runContextQuery) {
runContextQuery();
}
};
// we need to cache this function so that it doesn't get recreated on every render
this.onContextClose =
this.onContextClose ??
(() => {
this.prepareContextExpr = this.prepareContextExprWithoutParsedLabels;
});
return LokiContextUi({
row,
updateFilter,
languageProvider: this.languageProvider,
onClose: this.onContextClose,
});
}
async prepareContextExpr(row: LogRowModel, origQuery?: DataQuery): Promise<string> {
return await this.prepareContextExprWithoutParsedLabels(row, origQuery);
}
private async prepareContextExprWithoutParsedLabels(row: LogRowModel, origQuery?: DataQuery): Promise<string> {
await this.languageProvider.start();
const labels = this.languageProvider.getLabelKeys();
const expr = Object.keys(row.labels)
.map((label: string) => {
if (labels.includes(label)) {
// escape backslashes in label as users can't escape them by themselves
return `${label}="${escapeLabelValueInExactSelector(row.labels[label])}"`;
}
return '';
})
.filter((label) => !!label)
.join(',');
return `{${expr}}`;
}
}

60
public/app/plugins/datasource/loki/LogContextProvidet.test.ts Normal file
View File

@ -0,0 +1,60 @@
import { FieldType, LogRowModel, MutableDataFrame } from '@grafana/data';
import LokiLanguageProvider from './LanguageProvider';
import { LogContextProvider } from './LogContextProvider';
const defaultLanguageProviderMock = {
start: jest.fn(),
getLabelKeys: jest.fn(() => ['foo']),
} as unknown as LokiLanguageProvider;
const defaultLogRow = {
rowIndex: 0,
dataFrame: new MutableDataFrame({
fields: [
{
name: 'ts',
type: FieldType.time,
values: [0],
},
],
}),
labels: { bar: 'baz', foo: 'uniqueParsedLabel' },
uid: '1',
} as unknown as LogRowModel;
describe('new context ui', () => {
it('returns expression with 1 label', async () => {
const lcp = new LogContextProvider(defaultLanguageProviderMock);
const result = await lcp.prepareContextExpr(defaultLogRow);
expect(result).toEqual('{foo="uniqueParsedLabel"}');
});
it('returns empty expression for parsed labels', async () => {
const languageProviderMock = {
...defaultLanguageProviderMock,
getLabelKeys: jest.fn(() => []),
} as unknown as LokiLanguageProvider;
const lcp = new LogContextProvider(languageProviderMock);
const result = await lcp.prepareContextExpr(defaultLogRow);
expect(result).toEqual('{}');
});
});
describe('prepareLogRowContextQueryTarget', () => {
const lcp = new LogContextProvider(defaultLanguageProviderMock);
it('creates query with only labels from /labels API', async () => {
const contextQuery = await lcp.prepareLogRowContextQueryTarget(defaultLogRow, 10, 'BACKWARD');
expect(contextQuery.query.expr).toContain('uniqueParsedLabel');
expect(contextQuery.query.expr).not.toContain('baz');
});
it('should call languageProvider.start to fetch labels', async () => {
await lcp.prepareLogRowContextQueryTarget(defaultLogRow, 10, 'BACKWARD');
expect(lcp.languageProvider.start).toBeCalled();
});
});

107
public/app/plugins/datasource/loki/datasource.test.ts
View File

@ -13,8 +13,6 @@ import {
DataSourceInstanceSettings,
dateTime,
FieldType,
LogRowModel,
MutableDataFrame,
SupplementaryQueryType,
} from '@grafana/data';
import {
@ -839,57 +837,6 @@ describe('LokiDatasource', () => {
});
});
describe('prepareLogRowContextQueryTarget', () => {
const ds = createLokiDatasource(templateSrvStub);
it('creates query with only labels from /labels API', async () => {
const row: LogRowModel = {
rowIndex: 0,
dataFrame: new MutableDataFrame({
fields: [
{
name: 'ts',
type: FieldType.time,
values: [0],
},
],
}),
labels: { bar: 'baz', foo: 'uniqueParsedLabel' },
uid: '1',
} as unknown as LogRowModel;
//Mock stored labels to only include "bar" label
jest.spyOn(ds.languageProvider, 'start').mockImplementation(() => Promise.resolve([]));
jest.spyOn(ds.languageProvider, 'getLabelKeys').mockImplementation(() => ['bar']);
const contextQuery = await ds.prepareLogRowContextQueryTarget(row, 10, 'BACKWARD');
expect(contextQuery.query.expr).toContain('baz');
expect(contextQuery.query.expr).not.toContain('uniqueParsedLabel');
});
it('should call languageProvider.start to fetch labels', async () => {
const row: LogRowModel = {
rowIndex: 0,
dataFrame: new MutableDataFrame({
fields: [
{
name: 'ts',
type: FieldType.time,
values: [0],
},
],
}),
labels: { bar: 'baz', foo: 'uniqueParsedLabel' },
uid: '1',
} as unknown as LogRowModel;
//Mock stored labels to only include "bar" label
jest.spyOn(ds.languageProvider, 'start').mockImplementation(() => Promise.resolve([]));
await ds.prepareLogRowContextQueryTarget(row, 10, 'BACKWARD');
expect(ds.languageProvider.start).toBeCalled();
});
});
describe('logs volume data provider', () => {
let ds: LokiDatasource;
beforeEach(() => {
@ -1228,57 +1175,3 @@ function makeAnnotationQueryRequest(options = {}): AnnotationQueryRequest<LokiQu
rangeRaw: timeRange,
};
}
describe('new context ui', () => {
it('returns expression with 1 label', async () => {
const ds = createLokiDatasource(templateSrvStub);
const row: LogRowModel = {
rowIndex: 0,
dataFrame: new MutableDataFrame({
fields: [
{
name: 'ts',
type: FieldType.time,
values: [0],
},
],
}),
labels: { bar: 'baz', foo: 'uniqueParsedLabel' },
uid: '1',
} as unknown as LogRowModel;
jest.spyOn(ds.languageProvider, 'start').mockImplementation(() => Promise.resolve([]));
jest.spyOn(ds.languageProvider, 'getLabelKeys').mockImplementation(() => ['foo']);
const result = await ds.prepareContextExpr(row);
expect(result).toEqual('{foo="uniqueParsedLabel"}');
});
it('returns empty expression for parsed labels', async () => {
const ds = createLokiDatasource(templateSrvStub);
const row: LogRowModel = {
rowIndex: 0,
dataFrame: new MutableDataFrame({
fields: [
{
name: 'ts',
type: FieldType.time,
values: [0],
},
],
}),
labels: { bar: 'baz', foo: 'uniqueParsedLabel' },
uid: '1',
} as unknown as LogRowModel;
jest.spyOn(ds.languageProvider, 'start').mockImplementation(() => Promise.resolve([]));
jest.spyOn(ds.languageProvider, 'getLabelKeys').mockImplementation(() => []);
const result = await ds.prepareContextExpr(row);
expect(result).toEqual('{}');
});
});

149
public/app/plugins/datasource/loki/datasource.ts
View File

@ -31,7 +31,6 @@ import {
rangeUtil,
ScopedVars,
TimeRange,
toUtc,
} from '@grafana/data';
import { BackendSrvRequest, config, DataSourceWithBackend, FetchError } from '@grafana/runtime';
import { DataQuery } from '@grafana/schema';
@ -48,10 +47,10 @@ import { replaceVariables, returnVariables } from '../prometheus/querybuilder/sh
import LanguageProvider from './LanguageProvider';
import { LiveStreams, LokiLiveTarget } from './LiveStreams';
import { LogContextProvider } from './LogContextProvider';
import { transformBackendResult } from './backendResultTransformer';
import { LokiAnnotationsQueryEditor } from './components/AnnotationsQueryEditor';
import { LokiContextUi } from './components/LokiContextUi';
import { escapeLabelValueInExactSelector, escapeLabelValueInSelector, isRegexSelector } from './languageUtils';
import { escapeLabelValueInSelector, isRegexSelector } from './languageUtils';
import { labelNamesRegex, labelValuesRegex } from './migrations/variableQueryMigrations';
import {
addLabelFormatToQuery,
@ -72,7 +71,6 @@ import {
getLogQueryFromMetricsQuery,
getNormalizedLokiQuery,
getStreamSelectorsFromQuery,
getParserFromQuery,
isLogsQuery,
isValidQuery,
requestSupportsSplitting,
@ -81,10 +79,8 @@ import { sortDataFrameByTime, SortDirection } from './sortDataFrame';
import { doLokiChannelStream } from './streaming';
import { trackQuery } from './tracking';
import {
ContextFilter,
LokiOptions,
LokiQuery,
LokiQueryDirection,
LokiQueryType,
LokiVariableQuery,
LokiVariableQueryType,
@ -135,8 +131,8 @@ export class LokiDatasource
{
private streams = new LiveStreams();
languageProvider: LanguageProvider;
logContextProvider: LogContextProvider;
maxLines: number;
onContextClose: (() => void) | undefined;
constructor(
private instanceSettings: DataSourceInstanceSettings<LokiOptions>,
@ -152,6 +148,7 @@ export class LokiDatasource
QueryEditor: LokiAnnotationsQueryEditor,
};
this.variables = new LokiVariableSupport(this);
this.logContextProvider = new LogContextProvider(this.languageProvider);
}
getDataProvider(
@ -654,7 +651,12 @@ export class LokiDatasource
): Promise<{ data: DataFrame[] }> => {
const direction = (options && options.direction) || 'BACKWARD';
const limit = (options && options.limit) || 10;
const { query, range } = await this.prepareLogRowContextQueryTarget(row, limit, direction, origQuery);
const { query, range } = await this.logContextProvider.prepareLogRowContextQueryTarget(
row,
limit,
direction,
origQuery
);
const processDataFrame = (frame: DataFrame): DataFrame => {
// log-row-context requires specific field-names to work, so we set them here: "ts", "line", "id"
@ -717,137 +719,8 @@ export class LokiDatasource
);
};
prepareLogRowContextQueryTarget = async (
row: LogRowModel,
limit: number,
direction: 'BACKWARD' | 'FORWARD',
origQuery?: DataQuery
): Promise<{ query: LokiQuery; range: TimeRange }> => {
let expr = await this.prepareContextExpr(row, origQuery);
const contextTimeBuffer = 2 * 60 * 60 * 1000; // 2h buffer
const queryDirection = direction === 'FORWARD' ? LokiQueryDirection.Forward : LokiQueryDirection.Backward;
const query: LokiQuery = {
expr,
queryType: LokiQueryType.Range,
refId: `${REF_ID_STARTER_LOG_ROW_CONTEXT}${row.dataFrame.refId || ''}`,
maxLines: limit,
direction: queryDirection,
};
const fieldCache = new FieldCache(row.dataFrame);
const tsField = fieldCache.getFirstFieldOfType(FieldType.time);
if (tsField === undefined) {
throw new Error('loki: dataframe missing time-field, should never happen');
}
const tsValue = tsField.values.get(row.rowIndex);
const timestamp = toUtc(tsValue);
const range =
queryDirection === LokiQueryDirection.Forward
? {
// start param in Loki API is inclusive so we'll have to filter out the row that this request is based from
// and any other that were logged in the same ns but before the row. Right now these rows will be lost
// because the are before but came it he response that should return only rows after.
from: timestamp,
// convert to ns, we lose some precision here but it is not that important at the far points of the context
to: toUtc(row.timeEpochMs + contextTimeBuffer),
}
: {
// convert to ns, we lose some precision here but it is not that important at the far points of the context
from: toUtc(row.timeEpochMs - contextTimeBuffer),
to: timestamp,
};
return {
query,
range: {
from: range.from,
to: range.to,
raw: range,
},
};
};
async prepareContextExprWithoutParsedLabels(row: LogRowModel, origQuery?: DataQuery): Promise<string> {
await this.languageProvider.start();
const labels = this.languageProvider.getLabelKeys();
const expr = Object.keys(row.labels)
.map((label: string) => {
if (labels.includes(label)) {
// escape backslashes in label as users can't escape them by themselves
return `${label}="${escapeLabelValueInExactSelector(row.labels[label])}"`;
}
return '';
})
.filter((label) => !!label)
.join(',');
return `{${expr}}`;
}
async prepareContextExpr(row: LogRowModel, origQuery?: DataQuery): Promise<string> {
return await this.prepareContextExprWithoutParsedLabels(row, origQuery);
}
getLogRowContextUi(row: LogRowModel, runContextQuery: () => void): React.ReactNode {
const updateFilter = (contextFilters: ContextFilter[]) => {
this.prepareContextExpr = async (row: LogRowModel, origQuery?: DataQuery) => {
await this.languageProvider.start();
const labels = this.languageProvider.getLabelKeys();
let expr = contextFilters
.map((filter) => {
const label = filter.value;
if (filter && !filter.fromParser && filter.enabled && labels.includes(label)) {
// escape backslashes in label as users can't escape them by themselves
return `${label}="${escapeLabelValueInExactSelector(row.labels[label])}"`;
}
return '';
})
// Filter empty strings
.filter((label) => !!label)
.join(',');
expr = `{${expr}}`;
const parserContextFilters = contextFilters.filter((filter) => filter.fromParser && filter.enabled);
if (parserContextFilters.length) {
// we should also filter for labels from parsers, let's find the right parser
if (origQuery) {
const parser = getParserFromQuery((origQuery as LokiQuery).expr);
if (parser) {
expr = addParserToQuery(expr, parser);
}
}
for (const filter of parserContextFilters) {
if (filter.enabled) {
expr = addLabelToQuery(expr, filter.label, '=', row.labels[filter.label]);
}
}
}
return expr;
};
if (runContextQuery) {
runContextQuery();
}
};
// we need to cache this function so that it doesn't get recreated on every render
this.onContextClose =
this.onContextClose ??
(() => {
this.prepareContextExpr = this.prepareContextExprWithoutParsedLabels;
});
return LokiContextUi({
row,
updateFilter,
languageProvider: this.languageProvider,
onClose: this.onContextClose,
});
return this.logContextProvider.getLogRowContextUi(row, runContextQuery);
}
testDatasource(): Promise<{ status: string; message: string }> {