loki: use single-dataframe format on the backend (#47069)

public/app/plugins/datasource/loki/backendResultTransformer.test.ts

@@ -1,100 +1,95 @@
-import { ArrayVector, CoreApp, DataFrame, DataQueryRequest, DataQueryResponse, FieldType, toUtc } from '@grafana/data';
+import { ArrayVector, DataFrame, DataQueryResponse, FieldType } from '@grafana/data';
 import { cloneDeep } from 'lodash';
 
 import { transformBackendResult } from './backendResultTransformer';
-import { LokiQuery } from './types';
 
-const frame: DataFrame = {
-  name: 'frame1',
+const LOKI_EXPR = '{level="info"} |= "thing1"';
+const inputFrame: DataFrame = {
   refId: 'A',
   meta: {
-    executedQueryString: 'something1',
+    executedQueryString: LOKI_EXPR,
   },
   fields: [
     {
-      name: 'Time',
+      name: 'time',
       type: FieldType.time,
       config: {},
-      values: new ArrayVector([1645029699311, 1645029699312, 1645029699313]),
+      values: new ArrayVector([1645030244810, 1645030247027, 1645030246277, 1645030245539, 1645030244091]),
     },
     {
-      name: 'Value',
+      name: 'value',
+      type: FieldType.string,
+      config: {},
+      values: new ArrayVector(['line1', 'line2', 'line3', 'line4', 'line5']),
+    },
+    {
+      name: 'labels',
       type: FieldType.string,
-      labels: {
-        level: 'error',
-        location: 'moon',
-        protocol: 'http',
-      },
       config: {
-        displayNameFromDS: '{level="error", location="moon", protocol="http"}',
+        custom: {
+          json: true,
+        },
       },
-      values: new ArrayVector(['line1', 'line2', 'line3']),
+      values: new ArrayVector([
+        `[["level", "info"],["code", "41🌙"]]`,
+        `[["level", "error"],["code", "41🌙"]]`,
+        `[["level", "error"],["code", "43🌙"]]`,
+        `[["level", "error"],["code", "41🌙"]]`,
+        `[["level", "info"],["code", "41🌙"]]`,
+      ]),
     },
     {
       name: 'tsNs',
+      type: FieldType.time,
+      config: {},
+      values: new ArrayVector([
+        '1645030244810757120',
+        '1645030247027735040',
+        '1645030246277587968',
+        '1645030245539423744',
+        '1645030244091700992',
+      ]),
+    },
+    {
+      name: 'id',
       type: FieldType.string,
       config: {},
-      values: new ArrayVector(['1645029699311000500', '1645029699312000500', '1645029699313000500']),
+      values: new ArrayVector(['id1', 'id2', 'id3', 'id4', 'id5']),
     },
   ],
-  length: 3,
+  length: 5,
 };
 
-function makeRequest(expr: string): DataQueryRequest<LokiQuery> {
-  return {
-    requestId: 'test1',
-    interval: '1s',
-    intervalMs: 1000,
-    range: {
-      from: toUtc('2022-02-22T13:14:15'),
-      to: toUtc('2022-02-22T13:15:15'),
-      raw: {
-        from: toUtc('2022-02-22T13:14:15'),
-        to: toUtc('2022-02-22T13:15:15'),
-      },
-    },
-    scopedVars: {},
-    targets: [
-      {
-        refId: 'A',
-        expr,
-      },
-    ],
-    timezone: 'UTC',
-    app: CoreApp.Explore,
-    startTime: 0,
-  };
-}
-
 describe('loki backendResultTransformer', () => {
   it('processes a logs-dataframe correctly', () => {
-    const response: DataQueryResponse = { data: [cloneDeep(frame)] };
-    const request = makeRequest('{level="info"} |= "thing1"');
+    const response: DataQueryResponse = { data: [cloneDeep(inputFrame)] };
 
-    const expectedFrame = cloneDeep(frame);
+    const expectedFrame = cloneDeep(inputFrame);
     expectedFrame.meta = {
-      executedQueryString: 'something1',
+      ...expectedFrame.meta,
       preferredVisualisationType: 'logs',
       searchWords: ['thing1'],
       custom: {
         lokiQueryStatKey: 'Summary: total bytes processed',
       },
     };
-    expectedFrame.fields[2].type = FieldType.time;
-    expectedFrame.fields.push({
-      name: 'id',
-      type: FieldType.string,
-      config: {},
-      values: new ArrayVector([
-        '6b099923-25a6-5336-96fa-c84a14b7c351_A',
-        '0e1b7c47-a956-5cf2-a803-d487679745bd_A',
-        '6f9a840c-6a00-525b-9ed4-cceea29e62af_A',
-      ]),
-    });
+    expectedFrame.fields[2].type = FieldType.other;
+    expectedFrame.fields[2].values = new ArrayVector([
+      { level: 'info', code: '41🌙' },
+      { level: 'error', code: '41🌙' },
+      { level: 'error', code: '43🌙' },
+      { level: 'error', code: '41🌙' },
+      { level: 'info', code: '41🌙' },
+    ]);
 
     const expected: DataQueryResponse = { data: [expectedFrame] };
 
-    const result = transformBackendResult(response, request);
+    const result = transformBackendResult(response, [
+      {
+        refId: 'A',
+        expr: LOKI_EXPR,
+      },
+    ]);
     expect(result).toEqual(expected);
   });
 });

public/app/plugins/datasource/loki/backendResultTransformer.ts

@@ -1,8 +1,15 @@
-import { DataQueryRequest, DataQueryResponse, DataFrame, isDataFrame, FieldType, QueryResultMeta } from '@grafana/data';
+import {
+  DataQueryResponse,
+  DataFrame,
+  isDataFrame,
+  FieldType,
+  QueryResultMeta,
+  ArrayVector,
+  Labels,
+} from '@grafana/data';
 import { LokiQuery, LokiQueryType } from './types';
 import { makeTableFrames } from './makeTableFrames';
 import { formatQuery, getHighlighterExpressionsFromQuery } from './query_utils';
-import { makeIdField } from './makeIdField';
 
 function isMetricFrame(frame: DataFrame): boolean {
   return frame.fields.every((field) => field.type === FieldType.time || field.type === FieldType.number);
@@ -19,6 +26,12 @@ function setFrameMeta(frame: DataFrame, meta: QueryResultMeta): DataFrame {
   };
 }
 
+function decodeLabelsInJson(text: string): Labels {
+  const array: Array<[string, string]> = JSON.parse(text);
+  // NOTE: maybe we should go with maps, those have guaranteed ordering
+  return Object.fromEntries(array);
+}
+
 function processStreamFrame(frame: DataFrame, query: LokiQuery | undefined): DataFrame {
   const meta: QueryResultMeta = {
     preferredVisualisationType: 'logs',
@@ -29,21 +42,36 @@ function processStreamFrame(frame: DataFrame, query: LokiQuery | undefined): Dat
     },
   };
   const newFrame = setFrameMeta(frame, meta);
-  const newFields = frame.fields.map((field) => {
-    // the nanosecond-timestamp field must have a type-time
-    if (field.name === 'tsNs') {
-      return {
-        ...field,
-        type: FieldType.time,
-      };
-    } else {
-      return field;
+
+  const newFields = newFrame.fields.map((field) => {
+    switch (field.name) {
+      case 'labels': {
+        // the labels, when coming from the server, are json-encoded.
+        // here we decode them if needed.
+        return field.config.custom.json
+          ? {
+              name: field.name,
+              type: FieldType.other,
+              config: field.config,
+              // we are parsing the labels the same way as streaming-dataframes do
+              values: new ArrayVector(field.values.toArray().map((text) => decodeLabelsInJson(text))),
+            }
+          : field;
+      }
+      case 'tsNs': {
+        // we need to switch the field-type to be `time`
+        return {
+          ...field,
+          type: FieldType.time,
+        };
+      }
+      default: {
+        // no modification needed
+        return field;
+      }
     }
   });
 
-  // we add a calculated id-field
-  newFields.push(makeIdField(frame));
-
   return {
     ...newFrame,
     fields: newFields,
@@ -96,10 +124,7 @@ function groupFrames(
   return { streamsFrames, metricInstantFrames, metricRangeFrames };
 }
 
-export function transformBackendResult(
-  response: DataQueryResponse,
-  request: DataQueryRequest<LokiQuery>
-): DataQueryResponse {
+export function transformBackendResult(response: DataQueryResponse, queries: LokiQuery[]): DataQueryResponse {
   const { data, ...rest } = response;
 
   // in the typescript type, data is an array of basically anything.
@@ -112,7 +137,7 @@ export function transformBackendResult(
     return d;
   });
 
-  const queryMap = new Map(request.targets.map((query) => [query.refId, query]));
+  const queryMap = new Map(queries.map((query) => [query.refId, query]));
 
   const { streamsFrames, metricInstantFrames, metricRangeFrames } = groupFrames(dataFrames, queryMap);
 

public/app/plugins/datasource/loki/datasource.ts

@@ -161,7 +161,14 @@ export class LokiDatasource
         ...request,
         targets: request.targets.map(getNormalizedLokiQuery),
       };
-      return super.query(fixedRequest).pipe(map((response) => transformBackendResult(response, fixedRequest)));
+
+      if (fixedRequest.liveStreaming) {
+        return this.runLiveQueryThroughBackend(fixedRequest);
+      } else {
+        return super
+          .query(fixedRequest)
+          .pipe(map((response) => transformBackendResult(response, fixedRequest.targets)));
+      }
     }
 
     const filteredTargets = request.targets
@@ -199,6 +206,27 @@ export class LokiDatasource
     return merge(...subQueries);
   }
 
+  runLiveQueryThroughBackend(request: DataQueryRequest<LokiQuery>): Observable<DataQueryResponse> {
+    // this only works in explore-mode, so variables don't need to be handled,
+    //  and only for logs-queries, not metric queries
+    const logsQueries = request.targets.filter((query) => query.expr !== '' && !isMetricsQuery(query.expr));
+
+    if (logsQueries.length === 0) {
+      return of({
+        data: [],
+        state: LoadingState.Done,
+      });
+    }
+
+    const subQueries = logsQueries.map((query) => {
+      const maxDataPoints = query.maxLines || this.maxLines;
+      // FIXME: currently we are running it through the frontend still.
+      return this.runLiveQuery(query, maxDataPoints);
+    });
+
+    return merge(...subQueries);
+  }
+
   runInstantQuery = (
     target: LokiQuery,
     options: DataQueryRequest<LokiQuery>,

public/app/plugins/datasource/loki/makeIdField.test.ts

@@ -1,87 +0,0 @@
-import { ArrayVector, DataFrame, FieldType } from '@grafana/data';
-import { makeIdField } from './makeIdField';
-
-function makeFrame(timestamps: number[], values: string[], timestampNss: string[], refId?: string): DataFrame {
-  return {
-    name: 'frame',
-    refId,
-    meta: {
-      executedQueryString: 'something1',
-    },
-    fields: [
-      {
-        name: 'Time',
-        type: FieldType.time,
-        config: {},
-        values: new ArrayVector(timestamps),
-      },
-      {
-        name: 'Value',
-        type: FieldType.string,
-        config: {},
-        labels: {
-          foo: 'bar',
-        },
-        values: new ArrayVector(values),
-      },
-      {
-        name: 'tsNs',
-        type: FieldType.time,
-        config: {},
-        values: new ArrayVector(timestampNss),
-      },
-    ],
-    length: timestamps.length,
-  };
-}
-
-describe('loki makeIdField', () => {
-  it('should always generate unique ids for logs', () => {
-    const frame = makeFrame(
-      [1579857562021, 1579857562021, 1579857562021, 1579857562021],
-      [
-        't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"',
-        't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"',
-        't=2020-02-12T15:04:51+0000 lvl=info msg="Non-Duplicated"',
-        't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"',
-      ],
-      ['1579857562021616000', '1579857562021616000', '1579857562021616000', '1579857562021616000']
-    );
-    expect(makeIdField(frame)).toEqual({
-      config: {},
-      name: 'id',
-      type: 'string',
-      values: new ArrayVector([
-        '75fceace-9f98-5134-b222-643fdcde2877',
-        '75fceace-9f98-5134-b222-643fdcde2877_1',
-        '4a081a89-040d-5f64-9477-a4d846ce9f6b',
-        '75fceace-9f98-5134-b222-643fdcde2877_2',
-      ]),
-    });
-  });
-
-  it('should append refId to the unique ids if refId is provided', () => {
-    const frame = makeFrame(
-      [1579857562021, 1579857562021, 1579857562021, 1579857562021],
-      [
-        't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"',
-        't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"',
-        't=2020-02-12T15:04:51+0000 lvl=info msg="Non-Duplicated"',
-        't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"',
-      ],
-      ['1579857562021616000', '1579857562021616000', '1579857562021616000', '1579857562021616000'],
-      'X'
-    );
-    expect(makeIdField(frame)).toEqual({
-      config: {},
-      name: 'id',
-      type: 'string',
-      values: new ArrayVector([
-        '75fceace-9f98-5134-b222-643fdcde2877_X',
-        '75fceace-9f98-5134-b222-643fdcde2877_1_X',
-        '4a081a89-040d-5f64-9477-a4d846ce9f6b_X',
-        '75fceace-9f98-5134-b222-643fdcde2877_2_X',
-      ]),
-    });
-  });
-});

public/app/plugins/datasource/loki/makeIdField.ts

@@ -1,54 +0,0 @@
-import { v5 as uuidv5 } from 'uuid';
-
-import { ArrayVector, DataFrame, Field, FieldType, Labels } from '@grafana/data';
-
-const UUID_NAMESPACE = '6ec946da-0f49-47a8-983a-1d76d17e7c92';
-
-function createUid(text: string, usedUids: Map<string, number>, refId?: string): string {
-  const id = uuidv5(text, UUID_NAMESPACE);
-
-  // check how many times have we seen this id before,
-  // set the count to zero, if never.
-  const count = usedUids.get(id) ?? 0;
-
-  // if we have seen this id before, we need to make
-  // it unique by appending the seen-count
-  // (starts with 1, and goes up)
-  const uniqueId = count > 0 ? `${id}_${count}` : id;
-
-  // we increment the counter for this id, to be used when we are called the next time
-  usedUids.set(id, count + 1);
-
-  // we add refId to the end, if it is available
-  return refId !== undefined ? `${uniqueId}_${refId}` : uniqueId;
-}
-
-export function makeIdField(frame: DataFrame): Field {
-  const allLabels: Labels = {};
-
-  // collect labels from every field
-  frame.fields.forEach((field) => {
-    Object.assign(allLabels, field.labels);
-  });
-
-  const labelsString = Object.entries(allLabels)
-    .map(([key, val]) => `${key}="${val}"`)
-    .sort()
-    .join('');
-
-  const usedUids = new Map<string, number>();
-
-  const { length } = frame;
-
-  const uids: string[] = new Array(length);
-
-  // we need to go through the dataframe "row by row"
-  for (let i = 0; i < length; i++) {
-    const row = frame.fields.map((f) => String(f.values.get(i)));
-    const text = `${labelsString}_${row.join('_')}`;
-    const uid = createUid(text, usedUids, frame.refId);
-    uids[i] = uid;
-  }
-
-  return { name: 'id', type: FieldType.string, config: {}, values: new ArrayVector(uids) };
-}