mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
logs: improve logs-frame parsing (#71450)
* logs: improve logs-frame parsing * renamed fields
This commit is contained in:
parent
409eae6ff9
commit
ab58466d09
@ -1,28 +1,35 @@
|
||||
import { DataFrame, FieldCache, FieldType, Field, Labels } from '@grafana/data';
|
||||
import { DataFrame, FieldCache, FieldType, Field, Labels, FieldWithIndex } from '@grafana/data';
|
||||
|
||||
import type { LogsFrame } from './logsFrame';
|
||||
|
||||
function getLabels(frame: DataFrame, cache: FieldCache, lineField: Field): Labels[] | null {
|
||||
const useLabelsField = frame.meta?.custom?.frameType === 'LabeledTimeValues';
|
||||
|
||||
if (!useLabelsField) {
|
||||
const lineLabels = lineField.labels;
|
||||
if (lineLabels !== undefined) {
|
||||
const result = new Array(frame.length);
|
||||
result.fill(lineLabels);
|
||||
return result;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const labelsField = cache.getFieldByName('labels');
|
||||
|
||||
if (labelsField === undefined) {
|
||||
// take the labels from the line-field, and "stretch" it into an array
|
||||
// with the length of the frame (so there are the same labels for every row)
|
||||
function makeLabelsArray(lineField: Field, length: number): Labels[] | null {
|
||||
const lineLabels = lineField.labels;
|
||||
if (lineLabels !== undefined) {
|
||||
const result = new Array(length);
|
||||
result.fill(lineLabels);
|
||||
return result;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return labelsField.values;
|
||||
// we decide if the frame is old-loki-style frame, and adjust the behavior.
|
||||
// we also have to return the labels-field (if we used it),
|
||||
// to be able to remove it from the unused-fields, later.
|
||||
function makeLabelsGetter(
|
||||
cache: FieldCache,
|
||||
lineField: Field,
|
||||
frame: DataFrame
|
||||
): [FieldWithIndex | null, () => Labels[] | null] {
|
||||
if (frame.meta?.custom?.frameType === 'LabeledTimeValues') {
|
||||
const labelsField = cache.getFieldByName('labels');
|
||||
return labelsField === undefined ? [null, () => null] : [labelsField, () => labelsField.values];
|
||||
} else {
|
||||
// we use the labels on the line-field, and make an array with it
|
||||
return [null, () => makeLabelsArray(lineField, frame.length)];
|
||||
}
|
||||
}
|
||||
|
||||
export function parseLegacyLogsFrame(frame: DataFrame): LogsFrame | null {
|
||||
@ -39,7 +46,21 @@ export function parseLegacyLogsFrame(frame: DataFrame): LogsFrame | null {
|
||||
const severityField = cache.getFieldByName('level') ?? null;
|
||||
const idField = cache.getFieldByName('id') ?? null;
|
||||
|
||||
const labels = getLabels(frame, cache, bodyField);
|
||||
// extracting the labels is done very differently for old-loki-style and simple-style
|
||||
// dataframes, so it's a little awkward to handle it,
|
||||
// we both need to on-demand extract the labels, and also get teh labelsField,
|
||||
// but only if the labelsField is used.
|
||||
const [labelsField, getL] = makeLabelsGetter(cache, bodyField, frame);
|
||||
|
||||
const extraFields = cache.fields.filter(
|
||||
(_, i) =>
|
||||
i !== timeField.index &&
|
||||
i !== bodyField.index &&
|
||||
i !== timeNanosecondField?.index &&
|
||||
i !== severityField?.index &&
|
||||
i !== idField?.index &&
|
||||
i !== labelsField?.index
|
||||
);
|
||||
|
||||
return {
|
||||
timeField,
|
||||
@ -47,7 +68,8 @@ export function parseLegacyLogsFrame(frame: DataFrame): LogsFrame | null {
|
||||
timeNanosecondField,
|
||||
severityField,
|
||||
idField,
|
||||
attributes: labels,
|
||||
getAttributesAsLabels: () => labels,
|
||||
getAttributes: getL,
|
||||
getAttributesAsLabels: getL,
|
||||
extraFields,
|
||||
};
|
||||
}
|
||||
|
@ -37,8 +37,8 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
|
||||
const severity = makeString('severity', ['info', 'debug']);
|
||||
const id = makeString('id', ['id1', 'id2']);
|
||||
const attributes = makeObject('attributes', [
|
||||
{ counter: '38141', label: 'val2', level: 'warning' },
|
||||
{ counter: '38143', label: 'val2', level: 'info' },
|
||||
{ counter: '38141', label: 'val2', level: 'warning', nested: { a: '1', b: ['2', '3'] } },
|
||||
{ counter: '38143', label: 'val2', level: 'info', nested: { a: '11', b: ['12', '13'] } },
|
||||
]);
|
||||
|
||||
const result = parseLogsFrame({
|
||||
@ -56,10 +56,15 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
|
||||
expect(result!.idField?.values[0]).toBe(id.values[0]);
|
||||
expect(result!.timeNanosecondField).toBeNull();
|
||||
expect(result!.severityField?.values[0]).toBe(severity.values[0]);
|
||||
expect(result!.attributes).toStrictEqual([
|
||||
{ counter: '38141', label: 'val2', level: 'warning' },
|
||||
{ counter: '38143', label: 'val2', level: 'info' },
|
||||
expect(result!.getAttributes()).toStrictEqual([
|
||||
{ counter: '38141', label: 'val2', level: 'warning', nested: { a: '1', b: ['2', '3'] } },
|
||||
{ counter: '38143', label: 'val2', level: 'info', nested: { a: '11', b: ['12', '13'] } },
|
||||
]);
|
||||
expect(result!.getAttributesAsLabels()).toStrictEqual([
|
||||
{ counter: '38141', label: 'val2', level: 'warning', nested: `{"a":"1","b":["2","3"]}` },
|
||||
{ counter: '38143', label: 'val2', level: 'info', nested: `{"a":"11","b":["12","13"]}` },
|
||||
]);
|
||||
expect(result?.extraFields).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it('should parse old Loki-style (grafana8.x) frames ( multi-frame, but here we only parse a single frame )', () => {
|
||||
@ -80,10 +85,15 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
|
||||
expect(result!.idField?.values[0]).toBe(id.values[0]);
|
||||
expect(result!.timeNanosecondField?.values[0]).toBe(ns.values[0]);
|
||||
expect(result!.severityField).toBeNull();
|
||||
expect(result!.attributes).toStrictEqual([
|
||||
expect(result!.getAttributes()).toStrictEqual([
|
||||
{ counter: '34543', lable: 'val3', level: 'info' },
|
||||
{ counter: '34543', lable: 'val3', level: 'info' },
|
||||
]);
|
||||
expect(result!.getAttributesAsLabels()).toStrictEqual([
|
||||
{ counter: '34543', lable: 'val3', level: 'info' },
|
||||
{ counter: '34543', lable: 'val3', level: 'info' },
|
||||
]);
|
||||
expect(result?.extraFields).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it('should parse a Loki-style frame (single-frame, labels-in-json)', () => {
|
||||
@ -113,13 +123,18 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
|
||||
expect(result!.idField?.values[0]).toBe(id.values[0]);
|
||||
expect(result!.timeNanosecondField?.values[0]).toBe(ns.values[0]);
|
||||
expect(result!.severityField).toBeNull();
|
||||
expect(result!.attributes).toStrictEqual([
|
||||
expect(result!.getAttributes()).toStrictEqual([
|
||||
{ counter: '38141', label: 'val2', level: 'warning' },
|
||||
{ counter: '38143', label: 'val2', level: 'info' },
|
||||
]);
|
||||
expect(result!.getAttributesAsLabels()).toStrictEqual([
|
||||
{ counter: '38141', label: 'val2', level: 'warning' },
|
||||
{ counter: '38143', label: 'val2', level: 'info' },
|
||||
]);
|
||||
expect(result?.extraFields).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it('should parse elastic-style frame (has level-field, no labels parsed, extra fields ignored)', () => {
|
||||
it('should parse elastic-style frame (has level-field, no labels parsed, with extra unused fields)', () => {
|
||||
const time = makeTime('Time', [1687185711795, 1687185711995]);
|
||||
const line = makeString('Line', ['line1', 'line2']);
|
||||
const source = makeObject('_source', [
|
||||
@ -146,7 +161,9 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
|
||||
expect(result!.severityField?.values[0]).toBe(level.values[0]);
|
||||
expect(result!.idField).toBeNull();
|
||||
expect(result!.timeNanosecondField).toBeNull();
|
||||
expect(result!.attributes).toBeNull();
|
||||
expect(result!.getAttributesAsLabels()).toBeNull();
|
||||
expect(result!.getAttributes()).toBeNull();
|
||||
expect(result?.extraFields.map((f) => f.name)).toStrictEqual(['_source', 'hostname']);
|
||||
});
|
||||
|
||||
it('should parse a minimal old-style frame (only two fields, time and line)', () => {
|
||||
@ -165,7 +182,9 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
|
||||
expect(result!.severityField).toBeNull();
|
||||
expect(result!.idField).toBeNull();
|
||||
expect(result!.timeNanosecondField).toBeNull();
|
||||
expect(result!.attributes).toBeNull();
|
||||
expect(result!.getAttributesAsLabels()).toBeNull();
|
||||
expect(result!.getAttributes()).toBeNull();
|
||||
expect(result?.extraFields).toStrictEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
|
@ -14,8 +14,9 @@ export type LogsFrame = {
|
||||
timeNanosecondField: FieldWithIndex | null;
|
||||
severityField: FieldWithIndex | null;
|
||||
idField: FieldWithIndex | null;
|
||||
attributes: Attributes[] | null;
|
||||
getAttributes: () => Attributes[] | null; // may be slow, so we only do it when asked for it explicitly
|
||||
getAttributesAsLabels: () => Labels[] | null; // temporarily exists to make the labels=>attributes migration simpler
|
||||
extraFields: FieldWithIndex[];
|
||||
};
|
||||
|
||||
function getField(cache: FieldCache, name: string, fieldType: FieldType): FieldWithIndex | undefined {
|
||||
@ -60,14 +61,24 @@ function parseDataplaneLogsFrame(frame: DataFrame): LogsFrame | null {
|
||||
|
||||
const attributes = attributesField === null ? null : attributesField.values;
|
||||
|
||||
const extraFields = cache.fields.filter(
|
||||
(_, i) =>
|
||||
i !== timestampField.index &&
|
||||
i !== bodyField.index &&
|
||||
i !== severityField?.index &&
|
||||
i !== idField?.index &&
|
||||
i !== attributesField?.index
|
||||
);
|
||||
|
||||
return {
|
||||
timeField: timestampField,
|
||||
bodyField,
|
||||
severityField,
|
||||
idField,
|
||||
attributes,
|
||||
getAttributes: () => attributes,
|
||||
timeNanosecondField: null,
|
||||
getAttributesAsLabels: () => (attributes !== null ? attributes.map(attributesToLabels) : null),
|
||||
extraFields,
|
||||
};
|
||||
return null;
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user