Loki: Option to add derived fields based on labels (#76162)

* Plugin: Deriving fields by name from parsed logs

Loki only derives fields by a regex matcher, this limits its usage when functions such as `line_formatter` is used on
top of the logs.
Some users already have logs parsed in json or logfmt structure which are detected as fields in loki.
This pull request allows the mapping between detected fields values and derived values by matching the fields' names.
Currently the feature is behind `lokiEnableNameMatcherOption` feature toggle.

* improve settings page to have a `fieldType`

* improve derived fields getter to use `matcherRegex`

* fix failing test

* rename feature toggle to `lokiDerivedFieldsFromLabels`

* added suggestions from review

* add empty config object

* remove feature flag

* fix width of select

* default to `regex` derived field

* fix failing test

---------

Co-authored-by: Sven Grossmann <svennergr@gmail.com>

public/app/plugins/datasource/loki/configuration/DerivedField.tsx

@@ -3,11 +3,13 @@ import React, { ChangeEvent, useEffect, useState } from 'react';
 import { usePrevious } from 'react-use';
 
 import { GrafanaTheme2, DataSourceInstanceSettings, VariableSuggestion } from '@grafana/data';
-import { Button, DataLinkInput, Field, Icon, Input, Label, Tooltip, useStyles2, Switch } from '@grafana/ui';
+import { Button, DataLinkInput, Field, Icon, Input, Label, Tooltip, useStyles2, Select, Switch } from '@grafana/ui';
 import { DataSourcePicker } from 'app/features/datasources/components/picker/DataSourcePicker';
 
 import { DerivedFieldConfig } from '../types';
 
+type MatcherType = 'label' | 'regex';
+
 const getStyles = (theme: GrafanaTheme2) => ({
   row: css`
     display: flex;
@@ -32,6 +34,10 @@ const getStyles = (theme: GrafanaTheme2) => ({
     margin-right: ${theme.spacing(1)};
   `,
   dataSource: css``,
+  nameMatcherField: css({
+    width: theme.spacing(20),
+    marginRight: theme.spacing(0.5),
+  }),
 });
 
 type Props = {
@@ -47,6 +53,7 @@ export const DerivedField = (props: Props) => {
   const styles = useStyles2(getStyles);
   const [showInternalLink, setShowInternalLink] = useState(!!value.datasourceUid);
   const previousUid = usePrevious(value.datasourceUid);
+  const [fieldType, setFieldType] = useState<MatcherType>(value.matcherType ?? 'regex');
 
   // Force internal link visibility change if uid changed outside of this component.
   useEffect(() => {
@@ -74,13 +81,46 @@ export const DerivedField = (props: Props) => {
           <Input value={value.name} onChange={handleChange('name')} placeholder="Field name" invalid={invalidName} />
         </Field>
         <Field
-          className={styles.regexField}
+          className={styles.nameMatcherField}
           label={
             <TooltipLabel
-              label="Regex"
+              label="Type"
-              content="Use to parse and capture some part of the log message. You can use the captured groups in the template."
+              content="Derived fields can be created from labels or by applying a regular expression to the log message."
             />
           }
+        >
+          <Select
+            options={[
+              { label: 'Regex in log line', value: 'regex' },
+              { label: 'Label', value: 'label' },
+            ]}
+            value={fieldType}
+            onChange={(type) => {
+              // make sure this is a valid MatcherType
+              if (type.value === 'label' || type.value === 'regex') {
+                setFieldType(type.value);
+                onChange({
+                  ...value,
+                  matcherType: type.value,
+                });
+              }
+            }}
+          />
+        </Field>
+        <Field
+          className={styles.regexField}
+          label={
+            <>
+              {fieldType === 'regex' && (
+                <TooltipLabel
+                  label="Regex"
+                  content="Use to parse and capture some part of the log message. You can use the captured groups in the template."
+                />
+              )}
+
+              {fieldType === 'label' && <TooltipLabel label="Label" content="Use to derive the field from a label." />}
+            </>
+          }
         >
           <Input value={value.matcherRegex} onChange={handleChange('matcherRegex')} />
         </Field>

public/app/plugins/datasource/loki/configuration/DerivedFields.tsx

@@ -91,7 +91,14 @@ export const DerivedFields = ({ fields = [], onChange }: Props) => {
             icon="plus"
             onClick={(event) => {
               event.preventDefault();
-              const newDerivedFields = [...fields, { name: '', matcherRegex: '', urlDisplayLabel: '', url: '' }];
+              const emptyConfig: DerivedFieldConfig = {
+                name: '',
+                matcherRegex: '',
+                urlDisplayLabel: '',
+                url: '',
+                matcherType: 'regex',
+              };
+              const newDerivedFields = [...fields, emptyConfig];
               onChange(newDerivedFields);
             }}
           >

public/app/plugins/datasource/loki/getDerivedFields.test.ts

@@ -105,4 +105,47 @@ describe('getDerivedFields', () => {
       url: '',
     });
   });
+  it('adds links to fields with labels', () => {
+    const df = createDataFrame({
+      fields: [
+        { name: 'labels', values: [{ trace3: 'bar', trace4: 'blank' }, { trace3: 'tar' }, {}, null] },
+        { name: 'line', values: ['nothing', 'trace1=1234', 'trace2=aa', ''] },
+      ],
+    });
+    const newFields = getDerivedFields(df, [
+      {
+        matcherRegex: 'trace1=(\\w+)',
+        name: 'trace1',
+        url: 'http://localhost/${__value.raw}',
+      },
+      {
+        matcherRegex: 'trace3',
+        name: 'trace3Name',
+        url: 'http://localhost:8080/${__value.raw}',
+        matcherType: 'label',
+      },
+      {
+        matcherRegex: 'trace4',
+        name: 'trace4Name',
+        matcherType: 'regex',
+      },
+    ]);
+    expect(newFields.length).toBe(3);
+    const trace1 = newFields.find((f) => f.name === 'trace1');
+    expect(trace1!.values).toEqual([null, '1234', null, null]);
+    expect(trace1!.config.links![0]).toEqual({
+      url: 'http://localhost/${__value.raw}',
+      title: '',
+    });
+
+    const trace3 = newFields.find((f) => f.name === 'trace3Name');
+    expect(trace3!.values).toEqual(['bar', 'tar']);
+    expect(trace3!.config.links![0]).toEqual({
+      url: 'http://localhost:8080/${__value.raw}',
+      title: '',
+    });
+
+    const trace4 = newFields.find((f) => f.name === 'trace4Name');
+    expect(trace4!.values).toEqual([]);
+  });
 });

public/app/plugins/datasource/loki/getDerivedFields.ts

@@ -22,12 +22,38 @@ export function getDerivedFields(dataFrame: DataFrame, derivedFieldConfigs: Deri
     throw new Error('invalid logs-dataframe, string-field missing');
   }
 
-  lineField.values.forEach((line) => {
+  const labelFields = dataFrame.fields.find((f) => f.type === FieldType.other && f.name === 'labels');
+
+  for (let i = 0; i < lineField.values.length; i++) {
     for (const field of newFields) {
-      const logMatch = line.match(derivedFieldsGrouped[field.name][0].matcherRegex);
+      // `matcherRegex` can be either a RegExp that is used to extract the value from the log line, or it can be a label key to derive the field from the labels
-      field.values.push(logMatch && logMatch[1]);
+      if (derivedFieldsGrouped[field.name][0].matcherType === 'label' && labelFields) {
+        const label = labelFields.values[i];
+        if (label) {
+          // Find the key that matches both, the `matcherRegex` and the label key
+          const intersectingKey = Object.keys(label).find(
+            (key) => derivedFieldsGrouped[field.name][0].matcherRegex === key
+          );
+
+          if (intersectingKey) {
+            field.values.push(label[intersectingKey]);
+            continue;
+          }
+        }
+      } else if (derivedFieldsGrouped[field.name][0].matcherType !== 'regex') {
+        // `matcherRegex` will actually be used as a RegExp here
+        const line = lineField.values[i];
+        const logMatch = line.match(derivedFieldsGrouped[field.name][0].matcherRegex);
+
+        if (logMatch && logMatch[1]) {
+          field.values.push(logMatch[1]);
+          continue;
+        }
+
+        field.values.push(null);
+      }
     }
-  });
+  }
 
   return newFields;
 }

public/app/plugins/datasource/loki/types.ts

@@ -54,6 +54,7 @@ export type DerivedFieldConfig = {
   url?: string;
   urlDisplayLabel?: string;
   datasourceUid?: string;
+  matcherType?: 'label' | 'regex';
 };
 
 export enum LokiVariableQueryType {