logs: improve logs-frame parsing (#71450)

* logs: improve logs-frame parsing

* renamed fields
This commit is contained in:
Gábor Farkas 2023-07-17 14:42:33 +02:00 committed by GitHub
parent 409eae6ff9
commit ab58466d09
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 86 additions and 34 deletions

View File

@ -1,28 +1,35 @@
import { DataFrame, FieldCache, FieldType, Field, Labels } from '@grafana/data';
import { DataFrame, FieldCache, FieldType, Field, Labels, FieldWithIndex } from '@grafana/data';
import type { LogsFrame } from './logsFrame';
function getLabels(frame: DataFrame, cache: FieldCache, lineField: Field): Labels[] | null {
const useLabelsField = frame.meta?.custom?.frameType === 'LabeledTimeValues';
if (!useLabelsField) {
const lineLabels = lineField.labels;
if (lineLabels !== undefined) {
const result = new Array(frame.length);
result.fill(lineLabels);
return result;
} else {
return null;
}
}
const labelsField = cache.getFieldByName('labels');
if (labelsField === undefined) {
// take the labels from the line-field, and "stretch" it into an array
// with the length of the frame (so there are the same labels for every row)
function makeLabelsArray(lineField: Field, length: number): Labels[] | null {
const lineLabels = lineField.labels;
if (lineLabels !== undefined) {
const result = new Array(length);
result.fill(lineLabels);
return result;
} else {
return null;
}
}
return labelsField.values;
// we decide if the frame is old-loki-style frame, and adjust the behavior.
// we also have to return the labels-field (if we used it),
// to be able to remove it from the unused-fields, later.
function makeLabelsGetter(
cache: FieldCache,
lineField: Field,
frame: DataFrame
): [FieldWithIndex | null, () => Labels[] | null] {
if (frame.meta?.custom?.frameType === 'LabeledTimeValues') {
const labelsField = cache.getFieldByName('labels');
return labelsField === undefined ? [null, () => null] : [labelsField, () => labelsField.values];
} else {
// we use the labels on the line-field, and make an array with it
return [null, () => makeLabelsArray(lineField, frame.length)];
}
}
export function parseLegacyLogsFrame(frame: DataFrame): LogsFrame | null {
@ -39,7 +46,21 @@ export function parseLegacyLogsFrame(frame: DataFrame): LogsFrame | null {
const severityField = cache.getFieldByName('level') ?? null;
const idField = cache.getFieldByName('id') ?? null;
const labels = getLabels(frame, cache, bodyField);
// extracting the labels is done very differently for old-loki-style and simple-style
// dataframes, so it's a little awkward to handle it,
// we both need to on-demand extract the labels, and also get teh labelsField,
// but only if the labelsField is used.
const [labelsField, getL] = makeLabelsGetter(cache, bodyField, frame);
const extraFields = cache.fields.filter(
(_, i) =>
i !== timeField.index &&
i !== bodyField.index &&
i !== timeNanosecondField?.index &&
i !== severityField?.index &&
i !== idField?.index &&
i !== labelsField?.index
);
return {
timeField,
@ -47,7 +68,8 @@ export function parseLegacyLogsFrame(frame: DataFrame): LogsFrame | null {
timeNanosecondField,
severityField,
idField,
attributes: labels,
getAttributesAsLabels: () => labels,
getAttributes: getL,
getAttributesAsLabels: getL,
extraFields,
};
}

View File

@ -37,8 +37,8 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
const severity = makeString('severity', ['info', 'debug']);
const id = makeString('id', ['id1', 'id2']);
const attributes = makeObject('attributes', [
{ counter: '38141', label: 'val2', level: 'warning' },
{ counter: '38143', label: 'val2', level: 'info' },
{ counter: '38141', label: 'val2', level: 'warning', nested: { a: '1', b: ['2', '3'] } },
{ counter: '38143', label: 'val2', level: 'info', nested: { a: '11', b: ['12', '13'] } },
]);
const result = parseLogsFrame({
@ -56,10 +56,15 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
expect(result!.idField?.values[0]).toBe(id.values[0]);
expect(result!.timeNanosecondField).toBeNull();
expect(result!.severityField?.values[0]).toBe(severity.values[0]);
expect(result!.attributes).toStrictEqual([
{ counter: '38141', label: 'val2', level: 'warning' },
{ counter: '38143', label: 'val2', level: 'info' },
expect(result!.getAttributes()).toStrictEqual([
{ counter: '38141', label: 'val2', level: 'warning', nested: { a: '1', b: ['2', '3'] } },
{ counter: '38143', label: 'val2', level: 'info', nested: { a: '11', b: ['12', '13'] } },
]);
expect(result!.getAttributesAsLabels()).toStrictEqual([
{ counter: '38141', label: 'val2', level: 'warning', nested: `{"a":"1","b":["2","3"]}` },
{ counter: '38143', label: 'val2', level: 'info', nested: `{"a":"11","b":["12","13"]}` },
]);
expect(result?.extraFields).toStrictEqual([]);
});
it('should parse old Loki-style (grafana8.x) frames ( multi-frame, but here we only parse a single frame )', () => {
@ -80,10 +85,15 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
expect(result!.idField?.values[0]).toBe(id.values[0]);
expect(result!.timeNanosecondField?.values[0]).toBe(ns.values[0]);
expect(result!.severityField).toBeNull();
expect(result!.attributes).toStrictEqual([
expect(result!.getAttributes()).toStrictEqual([
{ counter: '34543', lable: 'val3', level: 'info' },
{ counter: '34543', lable: 'val3', level: 'info' },
]);
expect(result!.getAttributesAsLabels()).toStrictEqual([
{ counter: '34543', lable: 'val3', level: 'info' },
{ counter: '34543', lable: 'val3', level: 'info' },
]);
expect(result?.extraFields).toStrictEqual([]);
});
it('should parse a Loki-style frame (single-frame, labels-in-json)', () => {
@ -113,13 +123,18 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
expect(result!.idField?.values[0]).toBe(id.values[0]);
expect(result!.timeNanosecondField?.values[0]).toBe(ns.values[0]);
expect(result!.severityField).toBeNull();
expect(result!.attributes).toStrictEqual([
expect(result!.getAttributes()).toStrictEqual([
{ counter: '38141', label: 'val2', level: 'warning' },
{ counter: '38143', label: 'val2', level: 'info' },
]);
expect(result!.getAttributesAsLabels()).toStrictEqual([
{ counter: '38141', label: 'val2', level: 'warning' },
{ counter: '38143', label: 'val2', level: 'info' },
]);
expect(result?.extraFields).toStrictEqual([]);
});
it('should parse elastic-style frame (has level-field, no labels parsed, extra fields ignored)', () => {
it('should parse elastic-style frame (has level-field, no labels parsed, with extra unused fields)', () => {
const time = makeTime('Time', [1687185711795, 1687185711995]);
const line = makeString('Line', ['line1', 'line2']);
const source = makeObject('_source', [
@ -146,7 +161,9 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
expect(result!.severityField?.values[0]).toBe(level.values[0]);
expect(result!.idField).toBeNull();
expect(result!.timeNanosecondField).toBeNull();
expect(result!.attributes).toBeNull();
expect(result!.getAttributesAsLabels()).toBeNull();
expect(result!.getAttributes()).toBeNull();
expect(result?.extraFields.map((f) => f.name)).toStrictEqual(['_source', 'hostname']);
});
it('should parse a minimal old-style frame (only two fields, time and line)', () => {
@ -165,7 +182,9 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
expect(result!.severityField).toBeNull();
expect(result!.idField).toBeNull();
expect(result!.timeNanosecondField).toBeNull();
expect(result!.attributes).toBeNull();
expect(result!.getAttributesAsLabels()).toBeNull();
expect(result!.getAttributes()).toBeNull();
expect(result?.extraFields).toStrictEqual([]);
});
});

View File

@ -14,8 +14,9 @@ export type LogsFrame = {
timeNanosecondField: FieldWithIndex | null;
severityField: FieldWithIndex | null;
idField: FieldWithIndex | null;
attributes: Attributes[] | null;
getAttributes: () => Attributes[] | null; // may be slow, so we only do it when asked for it explicitly
getAttributesAsLabels: () => Labels[] | null; // temporarily exists to make the labels=>attributes migration simpler
extraFields: FieldWithIndex[];
};
function getField(cache: FieldCache, name: string, fieldType: FieldType): FieldWithIndex | undefined {
@ -60,14 +61,24 @@ function parseDataplaneLogsFrame(frame: DataFrame): LogsFrame | null {
const attributes = attributesField === null ? null : attributesField.values;
const extraFields = cache.fields.filter(
(_, i) =>
i !== timestampField.index &&
i !== bodyField.index &&
i !== severityField?.index &&
i !== idField?.index &&
i !== attributesField?.index
);
return {
timeField: timestampField,
bodyField,
severityField,
idField,
attributes,
getAttributes: () => attributes,
timeNanosecondField: null,
getAttributesAsLabels: () => (attributes !== null ? attributes.map(attributesToLabels) : null),
extraFields,
};
return null;
}