From 4548b0d9fc1e0065c021812b811dc1c25aa10553 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Farkas?= Date: Thu, 29 Jun 2023 15:33:41 +0200 Subject: [PATCH] logs: better nanosecond handling (#70878) * logs: simplify code * refactor * handle nanoseconds --- public/app/core/logsModel.ts | 35 ++++- public/app/core/logsModel_parse.test.ts | 194 ++++++++++++++++++++++++ 2 files changed, 224 insertions(+), 5 deletions(-) diff --git a/public/app/core/logsModel.ts b/public/app/core/logsModel.ts index a642e4e239d..473ca66bce9 100644 --- a/public/app/core/logsModel.ts +++ b/public/app/core/logsModel.ts @@ -11,6 +11,8 @@ import { DataSourceJsonData, dateTimeFormat, dateTimeFormatTimeAgo, + DateTimeInput, + Field, FieldCache, FieldColorModeId, FieldType, @@ -316,6 +318,32 @@ interface LogInfo { frameLabels?: Labels[]; } +function parseTime( + timeField: Field, + timeNsField: Field | undefined, + index: number +): { ts: DateTimeInput; timeEpochMs: number; timeEpochNs: string } { + const ts = timeField.values[index]; + const time = toUtc(ts); + const timeEpochMs = time.valueOf(); + + if (timeNsField) { + return { ts, timeEpochMs, timeEpochNs: timeNsField.values[index] }; + } + + if (timeField.nanos !== undefined) { + const ns = timeField.nanos[index].toString().padStart(6, '0'); + const timeEpochNs = `${timeEpochMs}${ns}`; + return { ts, timeEpochMs, timeEpochNs }; + } + + return { + ts, + timeEpochMs, + timeEpochNs: timeEpochMs + '000000', + }; +} + /** * Converts dataFrames into LogsModel. This involves merging them into one list, sorting them and computing metadata * like common labels. @@ -364,10 +392,7 @@ export function logSeriesToLogsModel(logSeries: DataFrame[], queries: DataQuery[ const { timeField, timeNanosecondField, bodyField: stringField, severityField: logLevelField, idField } = logsFrame; for (let j = 0; j < series.length; j++) { - const ts = timeField.values[j]; - const time = toUtc(ts); - const tsNs = timeNanosecondField ? timeNanosecondField.values[j] : undefined; - const timeEpochNs = tsNs ? tsNs : time.valueOf() + '000000'; + const { ts, timeEpochMs, timeEpochNs } = parseTime(timeField, timeNanosecondField ?? undefined, j); // In edge cases, this can be undefined. If undefined, we want to replace it with empty string. const messageValue: unknown = stringField.values[j] ?? ''; @@ -405,7 +430,7 @@ export function logSeriesToLogsModel(logSeries: DataFrame[], queries: DataQuery[ dataFrame: series, logLevel, timeFromNow: dateTimeFormatTimeAgo(ts), - timeEpochMs: time.valueOf(), + timeEpochMs, timeEpochNs, timeLocal: dateTimeFormat(ts, { timeZone: 'browser' }), timeUtc: dateTimeFormat(ts, { timeZone: 'utc' }), diff --git a/public/app/core/logsModel_parse.test.ts b/public/app/core/logsModel_parse.test.ts index 29434537312..9461b6bf9b3 100644 --- a/public/app/core/logsModel_parse.test.ts +++ b/public/app/core/logsModel_parse.test.ts @@ -731,4 +731,198 @@ describe('logSeriesToLogsModel should parse different logs-dataframe formats', ( expect(logSeriesToLogsModel(frames)).toStrictEqual(expected); }); + + it('should parse timestamps when nanosecond data in the time field and no nanosecond field', () => { + const frames: DataFrame[] = [ + { + refId: 'A', + fields: [ + { + name: 'timestamp', + type: FieldType.time, + config: {}, + values: [1686142519756, 1686142520411, 1686142519997], + nanos: [641, 0, 123456], + }, + { + name: 'body', + type: FieldType.string, + config: {}, + values: ['line1', 'line2', 'line3'], + }, + ], + length: 3, + }, + ]; + + const expected = { + hasUniqueLabels: false, + meta: [], + rows: [ + { + dataFrame: frames[0], + datasourceType: undefined, + entry: 'line1', + entryFieldIndex: 1, + hasAnsi: false, + hasUnescapedContent: false, + labels: {}, + logLevel: 'unknown', + raw: 'line1', + rowIndex: 0, + searchWords: [], + timeEpochMs: 1686142519756, + timeEpochNs: '1686142519756000641', + timeFromNow: 'mock:dateTimeFormatTimeAgo:2023-06-07T06:55:19-06:00', + timeLocal: '2023-06-07 06:55:19', + timeUtc: '2023-06-07 12:55:19', + uid: 'A_0', + uniqueLabels: {}, + }, + { + dataFrame: frames[0], + datasourceType: undefined, + entry: 'line2', + entryFieldIndex: 1, + hasAnsi: false, + hasUnescapedContent: false, + labels: {}, + logLevel: 'unknown', + raw: 'line2', + rowIndex: 1, + searchWords: [], + timeEpochMs: 1686142520411, + timeEpochNs: '1686142520411000000', + timeFromNow: 'mock:dateTimeFormatTimeAgo:2023-06-07T06:55:20-06:00', + timeLocal: '2023-06-07 06:55:20', + timeUtc: '2023-06-07 12:55:20', + uid: 'A_1', + uniqueLabels: {}, + }, + { + dataFrame: frames[0], + datasourceType: undefined, + entry: 'line3', + entryFieldIndex: 1, + hasAnsi: false, + hasUnescapedContent: false, + labels: {}, + logLevel: 'unknown', + raw: 'line3', + rowIndex: 2, + searchWords: [], + timeEpochMs: 1686142519997, + timeEpochNs: '1686142519997123456', + timeFromNow: 'mock:dateTimeFormatTimeAgo:2023-06-07T06:55:19-06:00', + timeLocal: '2023-06-07 06:55:19', + timeUtc: '2023-06-07 12:55:19', + uid: 'A_2', + uniqueLabels: {}, + }, + ], + }; + + expect(logSeriesToLogsModel(frames)).toStrictEqual(expected); + }); + + it('should parse timestamps when both nanosecond data and nanosecond field, the field wins', () => { + // whether the dataframe-field wins or the nanosecond-data in the time-field wins, + // is arbitrary at the end. we simply have to pick one option, and keep doing that. + const frames: DataFrame[] = [ + { + refId: 'A', + fields: [ + { + name: 'timestamp', + type: FieldType.time, + config: {}, + values: [1686142519756, 1686142520411, 1686142519997], + nanos: [1, 2, 3], + }, + { + name: 'body', + type: FieldType.string, + config: {}, + values: ['line1', 'line2', 'line3'], + }, + { + name: 'tsNs', + type: FieldType.string, + config: {}, + values: ['1686142519756000004', '1686142520411000005', '1686142519997000006'], + }, + ], + length: 3, + }, + ]; + + const expected = { + hasUniqueLabels: false, + meta: [], + rows: [ + { + dataFrame: frames[0], + datasourceType: undefined, + entry: 'line1', + entryFieldIndex: 1, + hasAnsi: false, + hasUnescapedContent: false, + labels: {}, + logLevel: 'unknown', + raw: 'line1', + rowIndex: 0, + searchWords: [], + timeEpochMs: 1686142519756, + timeEpochNs: '1686142519756000004', + timeFromNow: 'mock:dateTimeFormatTimeAgo:2023-06-07T06:55:19-06:00', + timeLocal: '2023-06-07 06:55:19', + timeUtc: '2023-06-07 12:55:19', + uid: 'A_0', + uniqueLabels: {}, + }, + { + dataFrame: frames[0], + datasourceType: undefined, + entry: 'line2', + entryFieldIndex: 1, + hasAnsi: false, + hasUnescapedContent: false, + labels: {}, + logLevel: 'unknown', + raw: 'line2', + rowIndex: 1, + searchWords: [], + timeEpochMs: 1686142520411, + timeEpochNs: '1686142520411000005', + timeFromNow: 'mock:dateTimeFormatTimeAgo:2023-06-07T06:55:20-06:00', + timeLocal: '2023-06-07 06:55:20', + timeUtc: '2023-06-07 12:55:20', + uid: 'A_1', + uniqueLabels: {}, + }, + { + dataFrame: frames[0], + datasourceType: undefined, + entry: 'line3', + entryFieldIndex: 1, + hasAnsi: false, + hasUnescapedContent: false, + labels: {}, + logLevel: 'unknown', + raw: 'line3', + rowIndex: 2, + searchWords: [], + timeEpochMs: 1686142519997, + timeEpochNs: '1686142519997000006', + timeFromNow: 'mock:dateTimeFormatTimeAgo:2023-06-07T06:55:19-06:00', + timeLocal: '2023-06-07 06:55:19', + timeUtc: '2023-06-07 12:55:19', + uid: 'A_2', + uniqueLabels: {}, + }, + ], + }; + + expect(logSeriesToLogsModel(frames)).toStrictEqual(expected); + }); });