GrafanaData: Remove obsolete logs exports (#66271)

* grafana-data: removed obsolete logs exports

* updated betterer checksums
This commit is contained in:
Gábor Farkas
2023-04-13 13:32:09 +02:00
committed by GitHub
parent 93c252e0fe
commit 83289065f6
5 changed files with 0 additions and 652 deletions

View File

@@ -103,35 +103,6 @@ export interface LogLabelStatsModel {
value: string;
}
/** @deprecated will be removed in the next major version */
export interface LogsParser {
/**
* Value-agnostic matcher for a field label.
* Used to filter rows, and first capture group contains the value.
*/
buildMatcher: (label: string) => RegExp;
/**
* Returns all parsable substrings from a line, used for highlighting
*/
getFields: (line: string) => string[];
/**
* Gets the label name from a parsable substring of a line
*/
getLabelFromField: (field: string) => string;
/**
* Gets the label value from a parsable substring of a line
*/
getValueFromField: (field: string) => string;
/**
* Function to verify if this is a valid parser for the given line.
* The parser accepts the line if it returns true.
*/
test: (line: string) => boolean;
}
export enum LogsDedupDescription {
none = 'No de-duplication',
exact = 'De-duplication of successive lines that are identical, ignoring ISO datetimes.',

View File

@@ -4,7 +4,6 @@ export * from './Registry';
export * from './datasource';
export * from './deprecationWarning';
export * from './csv';
export * from './logs';
export * from './labels';
export * from './numbers';
export * from './object';

View File

@@ -1,372 +0,0 @@
import { MutableDataFrame } from '../dataframe/MutableDataFrame';
import { Labels } from '../types/data';
import { LogLevel, LogsModel, LogRowModel, LogsSortOrder } from '../types/logs';
import {
getLogLevel,
calculateLogsLabelStats,
calculateFieldStats,
getParser,
LogsParsers,
calculateStats,
getLogLevelFromKey,
sortLogsResult,
checkLogsError,
} from './logs';
describe('getLoglevel()', () => {
it('returns no log level on empty line', () => {
expect(getLogLevel('')).toBe(LogLevel.unknown);
});
it('returns no log level on when level is part of a word', () => {
expect(getLogLevel('who warns us')).toBe(LogLevel.unknown);
});
it('returns same log level for long and short version', () => {
expect(getLogLevel('[Warn]')).toBe(LogLevel.warning);
expect(getLogLevel('[Warning]')).toBe(LogLevel.warning);
expect(getLogLevel('[Warn]')).toBe('warning');
});
it('returns correct log level when level is capitalized', () => {
expect(getLogLevel('WARN')).toBe(LogLevel.warn);
});
it('returns log level on line contains a log level', () => {
expect(getLogLevel('warn: it is looking bad')).toBe(LogLevel.warn);
expect(getLogLevel('2007-12-12 12:12:12 [WARN]: it is looking bad')).toBe(LogLevel.warn);
});
it('returns first log level found', () => {
expect(getLogLevel('WARN this could be a debug message')).toBe(LogLevel.warn);
expect(getLogLevel('WARN this is a non-critical message')).toBe(LogLevel.warn);
});
});
describe('getLogLevelFromKey()', () => {
it('returns correct log level', () => {
expect(getLogLevelFromKey('info')).toBe(LogLevel.info);
});
it('returns correct log level when level is capitalized', () => {
expect(getLogLevelFromKey('INFO')).toBe(LogLevel.info);
});
it('returns unknown log level when level is integer', () => {
expect(getLogLevelFromKey(1)).toBe(LogLevel.unknown);
});
});
describe('calculateLogsLabelStats()', () => {
test('should return no stats for empty rows', () => {
expect(calculateLogsLabelStats([], '')).toEqual([]);
});
test('should return no stats of label is not found', () => {
const rows = [
{
entry: 'foo 1',
labels: {
foo: 'bar',
} as Labels,
},
] as LogRowModel[];
expect(calculateLogsLabelStats(rows, 'baz')).toEqual([]);
});
test('should return stats for found labels', () => {
const rows = [
{
entry: 'foo 1',
labels: {
foo: 'bar',
} as Labels,
},
{
entry: 'foo 0',
labels: {
foo: 'xxx',
} as Labels,
},
{
entry: 'foo 2',
labels: {
foo: 'bar',
} as Labels,
},
] as LogRowModel[];
expect(calculateLogsLabelStats(rows, 'foo')).toMatchObject([
{
value: 'bar',
count: 2,
},
{
value: 'xxx',
count: 1,
},
]);
});
});
describe('LogsParsers', () => {
describe('logfmt', () => {
const parser = LogsParsers.logfmt;
test('should detect format', () => {
expect(parser.test('foo')).toBeFalsy();
expect(parser.test('foo=bar')).toBeTruthy();
});
test('should return detected fields', () => {
expect(
parser.getFields(
'foo=bar baz="42 + 1" msg="[resolver] received A record \\"127.0.0.1\\" for \\"localhost.\\" from udp:192.168.65.1" time(ms)=50 label{foo}=bar'
)
).toEqual([
'foo=bar',
'baz="42 + 1"',
'msg="[resolver] received A record \\"127.0.0.1\\" for \\"localhost.\\" from udp:192.168.65.1"',
'time(ms)=50',
'label{foo}=bar',
]);
});
test('should return label for field', () => {
expect(parser.getLabelFromField('foo=bar')).toBe('foo');
expect(parser.getLabelFromField('time(ms)=50')).toBe('time(ms)');
});
test('should return value for field', () => {
expect(parser.getValueFromField('foo=bar')).toBe('bar');
expect(parser.getValueFromField('time(ms)=50')).toBe('50');
expect(
parser.getValueFromField(
'msg="[resolver] received A record \\"127.0.0.1\\" for \\"localhost.\\" from udp:192.168.65.1"'
)
).toBe('"[resolver] received A record \\"127.0.0.1\\" for \\"localhost.\\" from udp:192.168.65.1"');
});
test('should build a valid value matcher', () => {
const matcher = parser.buildMatcher('foo');
const match = 'foo=bar'.match(matcher);
expect(match).toBeDefined();
expect(match![1]).toBe('bar');
});
test('should build a valid complex value matcher', () => {
const matcher = parser.buildMatcher('time(ms)');
const match = 'time(ms)=50'.match(matcher);
expect(match).toBeDefined();
expect(match![1]).toBe('50');
});
});
describe('JSON', () => {
const parser = LogsParsers.JSON;
test('should detect format', () => {
expect(parser.test('foo')).toBeFalsy();
expect(parser.test('"foo"')).toBeFalsy();
expect(parser.test('{"foo":"bar"}')).toBeTruthy();
});
test('should return detected fields', () => {
expect(parser.getFields('{ "foo" : "bar", "baz" : 42 }')).toEqual(['"foo":"bar"', '"baz":42']);
});
test('should return detected fields for nested quotes', () => {
expect(parser.getFields(`{"foo":"bar: '[value=\\"42\\"]'"}`)).toEqual([`"foo":"bar: '[value=\\"42\\"]'"`]);
});
test('should return label for field', () => {
expect(parser.getLabelFromField('"foo" : "bar"')).toBe('foo');
expect(parser.getLabelFromField('"docker.memory.fail.count":0')).toBe('docker.memory.fail.count');
});
test('should return value for field', () => {
expect(parser.getValueFromField('"foo" : "bar"')).toBe('"bar"');
expect(parser.getValueFromField('"foo" : 42')).toBe('42');
expect(parser.getValueFromField('"foo" : 42.1')).toBe('42.1');
});
test('should build a valid value matcher for strings', () => {
const matcher = parser.buildMatcher('foo');
const match = '{"foo":"bar"}'.match(matcher);
expect(match).toBeDefined();
expect(match![1]).toBe('bar');
});
test('should build a valid value matcher for integers', () => {
const matcher = parser.buildMatcher('foo');
const match = '{"foo":42.1}'.match(matcher);
expect(match).toBeDefined();
expect(match![1]).toBe('42.1');
});
});
});
describe('calculateFieldStats()', () => {
test('should return no stats for empty rows', () => {
expect(calculateFieldStats([], /foo=(.*)/)).toEqual([]);
});
test('should return no stats if extractor does not match', () => {
const rows = [
{
entry: 'foo=bar',
},
] as LogRowModel[];
expect(calculateFieldStats(rows, /baz=(.*)/)).toEqual([]);
});
test('should return stats for found field', () => {
const rows = [
{
entry: 'foo="42 + 1"',
},
{
entry: 'foo=503 baz=foo',
},
{
entry: 'foo="42 + 1"',
},
{
entry: 't=2018-12-05T07:44:59+0000 foo=503',
},
] as LogRowModel[];
expect(calculateFieldStats(rows, /foo=("[^"]*"|\S+)/)).toMatchObject([
{
value: '"42 + 1"',
count: 2,
},
{
value: '503',
count: 2,
},
]);
});
});
describe('calculateStats()', () => {
test('should return no stats for empty array', () => {
expect(calculateStats([])).toEqual([]);
});
test('should return correct stats', () => {
const values = ['one', 'one', null, undefined, 'two'];
expect(calculateStats(values)).toMatchObject([
{
value: 'one',
count: 2,
proportion: 2 / 3,
},
{
value: 'two',
count: 1,
proportion: 1 / 3,
},
]);
});
});
describe('getParser()', () => {
test('should return no parser on empty line', () => {
expect(getParser('')).toBeUndefined();
});
test('should return no parser on unknown line pattern', () => {
expect(getParser('To Be or not to be')).toBeUndefined();
});
test('should return logfmt parser on key value patterns', () => {
expect(getParser('foo=bar baz="41 + 1')).toEqual(LogsParsers.logfmt);
});
test('should return JSON parser on JSON log lines', () => {
// TODO implement other JSON value types than string
expect(getParser('{"foo": "bar", "baz": "41 + 1"}')).toEqual(LogsParsers.JSON);
});
});
describe('sortLogsResult', () => {
const firstRow: LogRowModel = {
rowIndex: 0,
entryFieldIndex: 0,
dataFrame: new MutableDataFrame(),
entry: '',
hasAnsi: false,
hasUnescapedContent: false,
labels: {},
logLevel: LogLevel.info,
raw: '',
timeEpochMs: 0,
timeEpochNs: '0',
timeFromNow: '',
timeLocal: '',
timeUtc: '',
uid: '1',
};
const sameAsFirstRow = firstRow;
const secondRow: LogRowModel = {
rowIndex: 1,
entryFieldIndex: 0,
dataFrame: new MutableDataFrame(),
entry: '',
hasAnsi: false,
hasUnescapedContent: false,
labels: {},
logLevel: LogLevel.info,
raw: '',
timeEpochMs: 10,
timeEpochNs: '10000000',
timeFromNow: '',
timeLocal: '',
timeUtc: '',
uid: '2',
};
describe('when called with LogsSortOrder.Descending', () => {
it('then it should sort descending', () => {
const logsResult: LogsModel = {
rows: [firstRow, sameAsFirstRow, secondRow],
hasUniqueLabels: false,
};
const result = sortLogsResult(logsResult, LogsSortOrder.Descending);
expect(result).toEqual({
rows: [secondRow, firstRow, sameAsFirstRow],
hasUniqueLabels: false,
});
});
});
describe('when called with LogsSortOrder.Ascending', () => {
it('then it should sort ascending', () => {
const logsResult: LogsModel = {
rows: [secondRow, firstRow, sameAsFirstRow],
hasUniqueLabels: false,
};
const result = sortLogsResult(logsResult, LogsSortOrder.Ascending);
expect(result).toEqual({
rows: [firstRow, sameAsFirstRow, secondRow],
hasUniqueLabels: false,
});
});
});
});
describe('checkLogsError()', () => {
const log = {
labels: {
__error__: 'Error Message',
foo: 'boo',
} as Labels,
} as LogRowModel;
test('should return correct error if error is present', () => {
expect(checkLogsError(log)).toStrictEqual({ hasError: true, errorMessage: 'Error Message' });
});
});

View File

@@ -1,246 +0,0 @@
import { countBy, chain, escapeRegExp } from 'lodash';
import { DataFrame, FieldType } from '../types/index';
import { LogLevel, LogRowModel, LogLabelStatsModel, LogsParser, LogsModel, LogsSortOrder } from '../types/logs';
import { ArrayVector } from '../vector/ArrayVector';
// This matches:
// first a label from start of the string or first white space, then any word chars until "="
// second either an empty quotes, or anything that starts with quote and ends with unescaped quote,
// or any non whitespace chars that do not start with quote
const LOGFMT_REGEXP = /(?:^|\s)([\w\(\)\[\]\{\}]+)=(""|(?:".*?[^\\]"|[^"\s]\S*))/;
/**
* Returns the log level of a log line.
* Parse the line for level words. If no level is found, it returns `LogLevel.unknown`.
*
* Example: `getLogLevel('WARN 1999-12-31 this is great') // LogLevel.warn`
*/
/** @deprecated will be removed in the next major version */
export function getLogLevel(line: string): LogLevel {
if (!line) {
return LogLevel.unknown;
}
let level = LogLevel.unknown;
let currentIndex: number | undefined = undefined;
for (const key of Object.keys(LogLevel)) {
const regexp = new RegExp(`\\b${key}\\b`, 'i');
const result = regexp.exec(line);
if (result) {
if (currentIndex === undefined || result.index < currentIndex) {
level = LogLevel[key as keyof typeof LogLevel];
currentIndex = result.index;
}
}
}
return level;
}
/** @deprecated will be removed in the next major version */
export function getLogLevelFromKey(key: string | number): LogLevel {
const level = LogLevel[key.toString().toLowerCase() as keyof typeof LogLevel];
if (level) {
return level;
}
return LogLevel.unknown;
}
/** @deprecated will be removed in the next major version */
export function addLogLevelToSeries(series: DataFrame, lineIndex: number): DataFrame {
const levels = new ArrayVector<LogLevel>();
const lines = series.fields[lineIndex];
for (let i = 0; i < lines.values.length; i++) {
const line = lines.values.get(lineIndex);
levels.buffer.push(getLogLevel(line));
}
return {
...series, // Keeps Tags, RefID etc
fields: [
...series.fields,
{
name: 'LogLevel',
type: FieldType.string,
values: levels,
config: {},
},
],
};
}
/** @deprecated will be removed in the next major version */
export const LogsParsers: { [name: string]: LogsParser } = {
JSON: {
buildMatcher: (label) => new RegExp(`(?:{|,)\\s*"${label}"\\s*:\\s*"?([\\d\\.]+|[^"]*)"?`),
getFields: (line) => {
try {
const parsed = JSON.parse(line);
return Object.keys(parsed).map((key) => {
return `"${key}":${JSON.stringify(parsed[key])}`;
});
} catch {}
return [];
},
getLabelFromField: (field) => (field.match(/^"([^"]+)"\s*:/) || [])[1],
getValueFromField: (field) => (field.match(/:\s*(.*)$/) || [])[1],
test: (line) => {
let parsed;
try {
parsed = JSON.parse(line);
} catch (error) {}
// The JSON parser should only be used for log lines that are valid serialized JSON objects.
// If it would be used for a string, detected fields would include each letter as a separate field.
return typeof parsed === 'object';
},
},
logfmt: {
buildMatcher: (label) => new RegExp(`(?:^|\\s)${escapeRegExp(label)}=("[^"]*"|\\S+)`),
getFields: (line) => {
const fields: string[] = [];
line.replace(new RegExp(LOGFMT_REGEXP, 'g'), (substring) => {
fields.push(substring.trim());
return '';
});
return fields;
},
getLabelFromField: (field) => (field.match(LOGFMT_REGEXP) || [])[1],
getValueFromField: (field) => (field.match(LOGFMT_REGEXP) || [])[2],
test: (line) => LOGFMT_REGEXP.test(line),
},
};
/** @deprecated will be removed in the next major version */
export function calculateFieldStats(rows: LogRowModel[], extractor: RegExp): LogLabelStatsModel[] {
// Consider only rows that satisfy the matcher
const rowsWithField = rows.filter((row) => extractor.test(row.entry));
const rowCount = rowsWithField.length;
// Get field value counts for eligible rows
const countsByValue = countBy(rowsWithField, (r) => {
const row: LogRowModel = r;
const match = row.entry.match(extractor);
return match ? match[1] : null;
});
return getSortedCounts(countsByValue, rowCount);
}
/** @deprecated will be removed in the next major version */
export function calculateLogsLabelStats(rows: LogRowModel[], label: string): LogLabelStatsModel[] {
// Consider only rows that have the given label
const rowsWithLabel = rows.filter((row) => row.labels[label] !== undefined);
const rowCount = rowsWithLabel.length;
// Get label value counts for eligible rows
const countsByValue = countBy(rowsWithLabel, (row) => row.labels[label]);
return getSortedCounts(countsByValue, rowCount);
}
/** @deprecated will be removed in the next major version */
export function calculateStats(values: unknown[]): LogLabelStatsModel[] {
const nonEmptyValues = values.filter((value) => value !== undefined && value !== null);
const countsByValue = countBy(nonEmptyValues);
return getSortedCounts(countsByValue, nonEmptyValues.length);
}
const getSortedCounts = (countsByValue: { [value: string]: number }, rowCount: number) => {
return chain(countsByValue)
.map((count, value) => ({ count, value, proportion: count / rowCount }))
.sortBy('count')
.reverse()
.value();
};
/** @deprecated will be removed in the next major version */
export function getParser(line: string): LogsParser | undefined {
let parser;
try {
if (LogsParsers.JSON.test(line)) {
parser = LogsParsers.JSON;
}
} catch (error) {}
if (!parser && LogsParsers.logfmt.test(line)) {
parser = LogsParsers.logfmt;
}
return parser;
}
/** @deprecated will be removed in the next major version */
export const sortInAscendingOrder = (a: LogRowModel, b: LogRowModel) => {
// compare milliseconds
if (a.timeEpochMs < b.timeEpochMs) {
return -1;
}
if (a.timeEpochMs > b.timeEpochMs) {
return 1;
}
// if milliseconds are equal, compare nanoseconds
if (a.timeEpochNs < b.timeEpochNs) {
return -1;
}
if (a.timeEpochNs > b.timeEpochNs) {
return 1;
}
return 0;
};
/** @deprecated will be removed in the next major version */
export const sortInDescendingOrder = (a: LogRowModel, b: LogRowModel) => {
// compare milliseconds
if (a.timeEpochMs > b.timeEpochMs) {
return -1;
}
if (a.timeEpochMs < b.timeEpochMs) {
return 1;
}
// if milliseconds are equal, compare nanoseconds
if (a.timeEpochNs > b.timeEpochNs) {
return -1;
}
if (a.timeEpochNs < b.timeEpochNs) {
return 1;
}
return 0;
};
/** @deprecated will be removed in the next major version */
export const sortLogsResult = (logsResult: LogsModel | null, sortOrder: LogsSortOrder): LogsModel => {
const rows = logsResult ? sortLogRows(logsResult.rows, sortOrder) : [];
return logsResult ? { ...logsResult, rows } : { hasUniqueLabels: false, rows };
};
/** @deprecated will be removed in the next major version */
export const sortLogRows = (logRows: LogRowModel[], sortOrder: LogsSortOrder) =>
sortOrder === LogsSortOrder.Ascending ? logRows.sort(sortInAscendingOrder) : logRows.sort(sortInDescendingOrder);
// Currently supports only error condition in Loki logs
/** @deprecated will be removed in the next major version */
export const checkLogsError = (logRow: LogRowModel): { hasError: boolean; errorMessage?: string } => {
if (logRow.labels.__error__) {
return {
hasError: true,
errorMessage: logRow.labels.__error__,
};
}
return {
hasError: false,
};
};
/** @deprecated will be removed in the next major version */
export const escapeUnescapedString = (string: string) =>
string.replace(/\\r\\n|\\n|\\t|\\r/g, (match: string) => (match.slice(1) === 't' ? '\t' : '\n'));