logs: better nanosecond handling (#70878)

* logs: simplify code

* refactor

* handle nanoseconds

public/app/core/logsModel.ts

@@ -11,6 +11,8 @@ import {
   DataSourceJsonData,
   dateTimeFormat,
   dateTimeFormatTimeAgo,
+  DateTimeInput,
+  Field,
   FieldCache,
   FieldColorModeId,
   FieldType,
@@ -316,6 +318,32 @@ interface LogInfo {
   frameLabels?: Labels[];
 }
 
+function parseTime(
+  timeField: Field,
+  timeNsField: Field | undefined,
+  index: number
+): { ts: DateTimeInput; timeEpochMs: number; timeEpochNs: string } {
+  const ts = timeField.values[index];
+  const time = toUtc(ts);
+  const timeEpochMs = time.valueOf();
+
+  if (timeNsField) {
+    return { ts, timeEpochMs, timeEpochNs: timeNsField.values[index] };
+  }
+
+  if (timeField.nanos !== undefined) {
+    const ns = timeField.nanos[index].toString().padStart(6, '0');
+    const timeEpochNs = `${timeEpochMs}${ns}`;
+    return { ts, timeEpochMs, timeEpochNs };
+  }
+
+  return {
+    ts,
+    timeEpochMs,
+    timeEpochNs: timeEpochMs + '000000',
+  };
+}
+
 /**
  * Converts dataFrames into LogsModel. This involves merging them into one list, sorting them and computing metadata
  * like common labels.
@@ -364,10 +392,7 @@ export function logSeriesToLogsModel(logSeries: DataFrame[], queries: DataQuery[
     const { timeField, timeNanosecondField, bodyField: stringField, severityField: logLevelField, idField } = logsFrame;
 
     for (let j = 0; j < series.length; j++) {
-      const ts = timeField.values[j];
+      const { ts, timeEpochMs, timeEpochNs } = parseTime(timeField, timeNanosecondField ?? undefined, j);
-      const time = toUtc(ts);
-      const tsNs = timeNanosecondField ? timeNanosecondField.values[j] : undefined;
-      const timeEpochNs = tsNs ? tsNs : time.valueOf() + '000000';
 
       // In edge cases, this can be undefined. If undefined, we want to replace it with empty string.
       const messageValue: unknown = stringField.values[j] ?? '';
@@ -405,7 +430,7 @@ export function logSeriesToLogsModel(logSeries: DataFrame[], queries: DataQuery[
         dataFrame: series,
         logLevel,
         timeFromNow: dateTimeFormatTimeAgo(ts),
-        timeEpochMs: time.valueOf(),
+        timeEpochMs,
         timeEpochNs,
         timeLocal: dateTimeFormat(ts, { timeZone: 'browser' }),
         timeUtc: dateTimeFormat(ts, { timeZone: 'utc' }),

public/app/core/logsModel_parse.test.ts

@@ -731,4 +731,198 @@ describe('logSeriesToLogsModel should parse different logs-dataframe formats', (
 
     expect(logSeriesToLogsModel(frames)).toStrictEqual(expected);
   });
+
+  it('should parse timestamps when nanosecond data in the time field and no nanosecond field', () => {
+    const frames: DataFrame[] = [
+      {
+        refId: 'A',
+        fields: [
+          {
+            name: 'timestamp',
+            type: FieldType.time,
+            config: {},
+            values: [1686142519756, 1686142520411, 1686142519997],
+            nanos: [641, 0, 123456],
+          },
+          {
+            name: 'body',
+            type: FieldType.string,
+            config: {},
+            values: ['line1', 'line2', 'line3'],
+          },
+        ],
+        length: 3,
+      },
+    ];
+
+    const expected = {
+      hasUniqueLabels: false,
+      meta: [],
+      rows: [
+        {
+          dataFrame: frames[0],
+          datasourceType: undefined,
+          entry: 'line1',
+          entryFieldIndex: 1,
+          hasAnsi: false,
+          hasUnescapedContent: false,
+          labels: {},
+          logLevel: 'unknown',
+          raw: 'line1',
+          rowIndex: 0,
+          searchWords: [],
+          timeEpochMs: 1686142519756,
+          timeEpochNs: '1686142519756000641',
+          timeFromNow: 'mock:dateTimeFormatTimeAgo:2023-06-07T06:55:19-06:00',
+          timeLocal: '2023-06-07 06:55:19',
+          timeUtc: '2023-06-07 12:55:19',
+          uid: 'A_0',
+          uniqueLabels: {},
+        },
+        {
+          dataFrame: frames[0],
+          datasourceType: undefined,
+          entry: 'line2',
+          entryFieldIndex: 1,
+          hasAnsi: false,
+          hasUnescapedContent: false,
+          labels: {},
+          logLevel: 'unknown',
+          raw: 'line2',
+          rowIndex: 1,
+          searchWords: [],
+          timeEpochMs: 1686142520411,
+          timeEpochNs: '1686142520411000000',
+          timeFromNow: 'mock:dateTimeFormatTimeAgo:2023-06-07T06:55:20-06:00',
+          timeLocal: '2023-06-07 06:55:20',
+          timeUtc: '2023-06-07 12:55:20',
+          uid: 'A_1',
+          uniqueLabels: {},
+        },
+        {
+          dataFrame: frames[0],
+          datasourceType: undefined,
+          entry: 'line3',
+          entryFieldIndex: 1,
+          hasAnsi: false,
+          hasUnescapedContent: false,
+          labels: {},
+          logLevel: 'unknown',
+          raw: 'line3',
+          rowIndex: 2,
+          searchWords: [],
+          timeEpochMs: 1686142519997,
+          timeEpochNs: '1686142519997123456',
+          timeFromNow: 'mock:dateTimeFormatTimeAgo:2023-06-07T06:55:19-06:00',
+          timeLocal: '2023-06-07 06:55:19',
+          timeUtc: '2023-06-07 12:55:19',
+          uid: 'A_2',
+          uniqueLabels: {},
+        },
+      ],
+    };
+
+    expect(logSeriesToLogsModel(frames)).toStrictEqual(expected);
+  });
+
+  it('should parse timestamps when both nanosecond data and nanosecond field, the field wins', () => {
+    // whether the dataframe-field wins or the nanosecond-data in the time-field wins,
+    // is arbitrary at the end. we simply have to pick one option, and keep doing that.
+    const frames: DataFrame[] = [
+      {
+        refId: 'A',
+        fields: [
+          {
+            name: 'timestamp',
+            type: FieldType.time,
+            config: {},
+            values: [1686142519756, 1686142520411, 1686142519997],
+            nanos: [1, 2, 3],
+          },
+          {
+            name: 'body',
+            type: FieldType.string,
+            config: {},
+            values: ['line1', 'line2', 'line3'],
+          },
+          {
+            name: 'tsNs',
+            type: FieldType.string,
+            config: {},
+            values: ['1686142519756000004', '1686142520411000005', '1686142519997000006'],
+          },
+        ],
+        length: 3,
+      },
+    ];
+
+    const expected = {
+      hasUniqueLabels: false,
+      meta: [],
+      rows: [
+        {
+          dataFrame: frames[0],
+          datasourceType: undefined,
+          entry: 'line1',
+          entryFieldIndex: 1,
+          hasAnsi: false,
+          hasUnescapedContent: false,
+          labels: {},
+          logLevel: 'unknown',
+          raw: 'line1',
+          rowIndex: 0,
+          searchWords: [],
+          timeEpochMs: 1686142519756,
+          timeEpochNs: '1686142519756000004',
+          timeFromNow: 'mock:dateTimeFormatTimeAgo:2023-06-07T06:55:19-06:00',
+          timeLocal: '2023-06-07 06:55:19',
+          timeUtc: '2023-06-07 12:55:19',
+          uid: 'A_0',
+          uniqueLabels: {},
+        },
+        {
+          dataFrame: frames[0],
+          datasourceType: undefined,
+          entry: 'line2',
+          entryFieldIndex: 1,
+          hasAnsi: false,
+          hasUnescapedContent: false,
+          labels: {},
+          logLevel: 'unknown',
+          raw: 'line2',
+          rowIndex: 1,
+          searchWords: [],
+          timeEpochMs: 1686142520411,
+          timeEpochNs: '1686142520411000005',
+          timeFromNow: 'mock:dateTimeFormatTimeAgo:2023-06-07T06:55:20-06:00',
+          timeLocal: '2023-06-07 06:55:20',
+          timeUtc: '2023-06-07 12:55:20',
+          uid: 'A_1',
+          uniqueLabels: {},
+        },
+        {
+          dataFrame: frames[0],
+          datasourceType: undefined,
+          entry: 'line3',
+          entryFieldIndex: 1,
+          hasAnsi: false,
+          hasUnescapedContent: false,
+          labels: {},
+          logLevel: 'unknown',
+          raw: 'line3',
+          rowIndex: 2,
+          searchWords: [],
+          timeEpochMs: 1686142519997,
+          timeEpochNs: '1686142519997000006',
+          timeFromNow: 'mock:dateTimeFormatTimeAgo:2023-06-07T06:55:19-06:00',
+          timeLocal: '2023-06-07 06:55:19',
+          timeUtc: '2023-06-07 12:55:19',
+          uid: 'A_2',
+          uniqueLabels: {},
+        },
+      ],
+    };
+
+    expect(logSeriesToLogsModel(frames)).toStrictEqual(expected);
+  });
 });