loki: backend-mode: handle derived fields (#48873)

public/app/plugins/datasource/loki/backendResultTransformer.test.ts

@@ -1,9 +1,22 @@
 import { cloneDeep } from 'lodash';
 
-import { ArrayVector, DataFrame, DataQueryResponse, FieldType } from '@grafana/data';
+import { ArrayVector, DataFrame, DataQueryResponse, Field, FieldType } from '@grafana/data';
 
 import { transformBackendResult } from './backendResultTransformer';
 
+// needed because the derived-fields functionality calls it
+jest.mock('@grafana/runtime', () => ({
+  // @ts-ignore
+  ...jest.requireActual('@grafana/runtime'),
+  getDataSourceSrv: () => {
+    return {
+      getInstanceSettings: () => {
+        return { name: 'Loki1' };
+      },
+    };
+  },
+}));
+
 const LOKI_EXPR = '{level="info"} |= "thing1"';
 const inputFrame: DataFrame = {
   refId: 'A',
@@ -65,12 +78,52 @@ describe('loki backendResultTransformer', () => {
 
     const expected: DataQueryResponse = { data: [expectedFrame] };
 
-    const result = transformBackendResult(response, [
-      {
-        refId: 'A',
-        expr: LOKI_EXPR,
-      },
-    ]);
+    const result = transformBackendResult(
+      response,
+      [
+        {
+          refId: 'A',
+          expr: LOKI_EXPR,
+        },
+      ],
+      []
+    );
     expect(result).toEqual(expected);
   });
+
+  it('processed derived fields correctly', () => {
+    const input: DataFrame = {
+      length: 1,
+      fields: [
+        {
+          name: 'time',
+          config: {},
+          values: new ArrayVector([1]),
+          type: FieldType.time,
+        },
+        {
+          name: 'line',
+          config: {},
+          values: new ArrayVector(['line1']),
+          type: FieldType.string,
+        },
+      ],
+    };
+    const response: DataQueryResponse = { data: [input] };
+    const result = transformBackendResult(
+      response,
+      [{ refId: 'A', expr: '' }],
+      [
+        {
+          matcherRegex: 'trace=(w+)',
+          name: 'derived1',
+          url: 'example.com',
+        },
+      ]
+    );
+
+    expect(
+      result.data[0].fields.filter((field: Field) => field.name === 'derived1' && field.type === 'string').length
+    ).toBe(1);
+  });
 });

public/app/plugins/datasource/loki/backendResultTransformer.ts

@@ -1,8 +1,9 @@
 import { DataQueryResponse, DataFrame, isDataFrame, FieldType, QueryResultMeta } from '@grafana/data';
 
+import { getDerivedFields } from './getDerivedFields';
 import { makeTableFrames } from './makeTableFrames';
 import { formatQuery, getHighlighterExpressionsFromQuery } from './query_utils';
-import { LokiQuery, LokiQueryType } from './types';
+import { DerivedFieldConfig, LokiQuery, LokiQueryType } from './types';
 
 function isMetricFrame(frame: DataFrame): boolean {
   return frame.fields.every((field) => field.type === FieldType.time || field.type === FieldType.number);
@@ -19,7 +20,11 @@ function setFrameMeta(frame: DataFrame, meta: QueryResultMeta): DataFrame {
   };
 }
 
-function processStreamFrame(frame: DataFrame, query: LokiQuery | undefined): DataFrame {
+function processStreamFrame(
+  frame: DataFrame,
+  query: LokiQuery | undefined,
+  derivedFieldConfigs: DerivedFieldConfig[]
+): DataFrame {
   const meta: QueryResultMeta = {
     preferredVisualisationType: 'logs',
     searchWords: query !== undefined ? getHighlighterExpressionsFromQuery(formatQuery(query.expr)) : undefined,
@@ -29,13 +34,22 @@ function processStreamFrame(frame: DataFrame, query: LokiQuery | undefined): Dat
     },
   };
 
-  return setFrameMeta(frame, meta);
+  const newFrame = setFrameMeta(frame, meta);
+  const derivedFields = getDerivedFields(newFrame, derivedFieldConfigs);
+  return {
+    ...newFrame,
+    fields: [...newFrame.fields, ...derivedFields],
+  };
 }
 
-function processStreamsFrames(frames: DataFrame[], queryMap: Map<string, LokiQuery>): DataFrame[] {
+function processStreamsFrames(
+  frames: DataFrame[],
+  queryMap: Map<string, LokiQuery>,
+  derivedFieldConfigs: DerivedFieldConfig[]
+): DataFrame[] {
   return frames.map((frame) => {
     const query = frame.refId !== undefined ? queryMap.get(frame.refId) : undefined;
-    return processStreamFrame(frame, query);
+    return processStreamFrame(frame, query, derivedFieldConfigs);
   });
 }
 
@@ -78,7 +92,11 @@ function groupFrames(
   return { streamsFrames, metricInstantFrames, metricRangeFrames };
 }
 
-export function transformBackendResult(response: DataQueryResponse, queries: LokiQuery[]): DataQueryResponse {
+export function transformBackendResult(
+  response: DataQueryResponse,
+  queries: LokiQuery[],
+  derivedFieldConfigs: DerivedFieldConfig[]
+): DataQueryResponse {
   const { data, ...rest } = response;
 
   // in the typescript type, data is an array of basically anything.
@@ -100,7 +118,7 @@ export function transformBackendResult(response: DataQueryResponse, queries: Lok
     data: [
       ...processMetricRangeFrames(metricRangeFrames),
       ...processMetricInstantFrames(metricInstantFrames),
-      ...processStreamsFrames(streamsFrames, queryMap),
+      ...processStreamsFrames(streamsFrames, queryMap, derivedFieldConfigs),
     ],
   };
 }

public/app/plugins/datasource/loki/datasource.ts

@@ -180,7 +180,11 @@ export class LokiDatasource
       } else {
         return super
           .query(fixedRequest)
-          .pipe(map((response) => transformBackendResult(response, fixedRequest.targets)));
+          .pipe(
+            map((response) =>
+              transformBackendResult(response, fixedRequest.targets, this.instanceSettings.jsonData.derivedFields ?? [])
+            )
+          );
       }
     }
 

public/app/plugins/datasource/loki/getDerivedFields.test.ts

@@ -0,0 +1,62 @@
+import { MutableDataFrame } from '@grafana/data';
+
+import { getDerivedFields } from './getDerivedFields';
+
+jest.mock('@grafana/runtime', () => ({
+  // @ts-ignore
+  ...jest.requireActual('@grafana/runtime'),
+  getDataSourceSrv: () => {
+    return {
+      getInstanceSettings: () => {
+        return { name: 'Loki1' };
+      },
+    };
+  },
+}));
+
+describe('getDerivedFields', () => {
+  it('adds links to fields', () => {
+    const df = new MutableDataFrame({ fields: [{ name: 'line', values: ['nothing', 'trace1=1234', 'trace2=foo'] }] });
+    const newFields = getDerivedFields(df, [
+      {
+        matcherRegex: 'trace1=(\\w+)',
+        name: 'trace1',
+        url: 'http://localhost/${__value.raw}',
+      },
+      {
+        matcherRegex: 'trace2=(\\w+)',
+        name: 'trace2',
+        url: 'test',
+        datasourceUid: 'uid',
+      },
+      {
+        matcherRegex: 'trace2=(\\w+)',
+        name: 'trace2',
+        url: 'test',
+        datasourceUid: 'uid2',
+        urlDisplayLabel: 'Custom Label',
+      },
+    ]);
+    expect(newFields.length).toBe(2);
+    const trace1 = newFields.find((f) => f.name === 'trace1');
+    expect(trace1!.values.toArray()).toEqual([null, '1234', null]);
+    expect(trace1!.config.links![0]).toEqual({
+      url: 'http://localhost/${__value.raw}',
+      title: '',
+    });
+
+    const trace2 = newFields.find((f) => f.name === 'trace2');
+    expect(trace2!.values.toArray()).toEqual([null, null, 'foo']);
+    expect(trace2!.config.links!.length).toBe(2);
+    expect(trace2!.config.links![0]).toEqual({
+      title: '',
+      internal: { datasourceName: 'Loki1', datasourceUid: 'uid', query: { query: 'test' } },
+      url: '',
+    });
+    expect(trace2!.config.links![1]).toEqual({
+      title: 'Custom Label',
+      internal: { datasourceName: 'Loki1', datasourceUid: 'uid2', query: { query: 'test' } },
+      url: '',
+    });
+  });
+});

public/app/plugins/datasource/loki/getDerivedFields.ts

@@ -0,0 +1,77 @@
+import { groupBy } from 'lodash';
+
+import { FieldType, DataFrame, ArrayVector, DataLink, Field } from '@grafana/data';
+import { getDataSourceSrv } from '@grafana/runtime';
+
+import { DerivedFieldConfig } from './types';
+
+export function getDerivedFields(dataFrame: DataFrame, derivedFieldConfigs: DerivedFieldConfig[]): Field[] {
+  if (!derivedFieldConfigs.length) {
+    return [];
+  }
+  const derivedFieldsGrouped = groupBy(derivedFieldConfigs, 'name');
+
+  const newFields = Object.values(derivedFieldsGrouped).map(fieldFromDerivedFieldConfig);
+
+  // line-field is the first string-field
+  // NOTE: we should create some common log-frame-extra-string-field code somewhere
+  const lineField = dataFrame.fields.find((f) => f.type === FieldType.string);
+
+  if (lineField === undefined) {
+    // if this is happening, something went wrong, let's raise an error
+    throw new Error('invalid logs-dataframe, string-field missing');
+  }
+
+  lineField.values.toArray().forEach((line) => {
+    for (const field of newFields) {
+      const logMatch = line.match(derivedFieldsGrouped[field.name][0].matcherRegex);
+      field.values.add(logMatch && logMatch[1]);
+    }
+  });
+
+  return newFields;
+}
+
+/**
+ * Transform derivedField config into dataframe field with config that contains link.
+ */
+function fieldFromDerivedFieldConfig(derivedFieldConfigs: DerivedFieldConfig[]): Field<any, ArrayVector> {
+  const dataSourceSrv = getDataSourceSrv();
+
+  const dataLinks = derivedFieldConfigs.reduce((acc, derivedFieldConfig) => {
+    // Having field.datasourceUid means it is an internal link.
+    if (derivedFieldConfig.datasourceUid) {
+      const dsSettings = dataSourceSrv.getInstanceSettings(derivedFieldConfig.datasourceUid);
+
+      acc.push({
+        // Will be filled out later
+        title: derivedFieldConfig.urlDisplayLabel || '',
+        url: '',
+        // This is hardcoded for Jaeger or Zipkin not way right now to specify datasource specific query object
+        internal: {
+          query: { query: derivedFieldConfig.url },
+          datasourceUid: derivedFieldConfig.datasourceUid,
+          datasourceName: dsSettings?.name ?? 'Data source not found',
+        },
+      });
+    } else if (derivedFieldConfig.url) {
+      acc.push({
+        // We do not know what title to give here so we count on presentation layer to create a title from metadata.
+        title: derivedFieldConfig.urlDisplayLabel || '',
+        // This is hardcoded for Jaeger or Zipkin not way right now to specify datasource specific query object
+        url: derivedFieldConfig.url,
+      });
+    }
+    return acc;
+  }, [] as DataLink[]);
+
+  return {
+    name: derivedFieldConfigs[0].name,
+    type: FieldType.string,
+    config: {
+      links: dataLinks,
+    },
+    // We are adding values later on
+    values: new ArrayVector<string>([]),
+  };
+}