Transformers: extract fields from JSON and text (alpha) (#41791)

Co-authored-by: Leon Sorokin <leeoniya@gmail.com>

packages/grafana-data/src/dataframe/processDataFrame.ts

@@ -32,7 +32,7 @@ function convertTableToDataFrame(table: TableData): DataFrame {
     // TODO: should be Column but type does not exists there so not sure whats up here.
     const { text, type, ...disp } = c as any;
     return {
-      name: text, // rename 'text' to the 'name' field
+      name: text?.length ? text : c, // rename 'text' to the 'name' field
       config: (disp || {}) as FieldConfig,
       values: new ArrayVector(),
       type: type && Object.values(FieldType).includes(type as FieldType) ? (type as FieldType) : FieldType.other,

packages/grafana-data/src/transformations/transformers/ids.ts

@@ -28,4 +28,5 @@ export enum DataTransformerID {
   prepareTimeSeries = 'prepareTimeSeries',
   convertFieldType = 'convertFieldType',
   fieldLookup = 'fieldLookup',
+  extractFields = 'extractFields',
 }

public/app/core/components/TransformersUI/extractFields/ExtractFieldsTransformerEditor.tsx

@@ -0,0 +1,94 @@
+import React from 'react';
+import {
+  DataTransformerID,
+  FieldNamePickerConfigSettings,
+  PluginState,
+  SelectableValue,
+  StandardEditorsRegistryItem,
+  TransformerRegistryItem,
+  TransformerUIProps,
+} from '@grafana/data';
+
+import { InlineField, InlineFieldRow, InlineSwitch, Select } from '@grafana/ui';
+import { FieldNamePicker } from '@grafana/ui/src/components/MatchersUI/FieldNamePicker';
+import { ExtractFieldsOptions, extractFieldsTransformer } from './extractFields';
+import { FieldExtractorID, fieldExtractors } from './fieldExtractors';
+
+const fieldNamePickerSettings: StandardEditorsRegistryItem<string, FieldNamePickerConfigSettings> = {
+  settings: {
+    width: 30,
+    placeholderText: 'Select field',
+  },
+  name: '',
+  id: '',
+  editor: () => null,
+};
+
+export const extractFieldsTransformerEditor: React.FC<TransformerUIProps<ExtractFieldsOptions>> = ({
+  input,
+  options,
+  onChange,
+}) => {
+  const onPickSourceField = (source?: string) => {
+    onChange({
+      ...options,
+      source,
+    });
+  };
+
+  const onFormatChange = (format?: SelectableValue<FieldExtractorID>) => {
+    onChange({
+      ...options,
+      format: format?.value,
+    });
+  };
+
+  const onToggleReplace = () => {
+    onChange({
+      ...options,
+      replace: !options.replace,
+    });
+  };
+
+  const format = fieldExtractors.selectOptions(options.format ? [options.format] : undefined);
+
+  return (
+    <div>
+      <InlineFieldRow>
+        <InlineField label={'Source'} labelWidth={16}>
+          <FieldNamePicker
+            context={{ data: input }}
+            value={options.source ?? ''}
+            onChange={onPickSourceField}
+            item={fieldNamePickerSettings as any}
+          />
+        </InlineField>
+      </InlineFieldRow>
+      <InlineFieldRow>
+        <InlineField label={'Format'} labelWidth={16}>
+          <Select
+            value={format.current[0] as any}
+            options={format.options as any}
+            onChange={onFormatChange}
+            width={24}
+            placeholder={'Auto'}
+          />
+        </InlineField>
+      </InlineFieldRow>
+      <InlineFieldRow>
+        <InlineField label={'Replace all fields'} labelWidth={16}>
+          <InlineSwitch value={options.replace ?? false} onChange={onToggleReplace} />
+        </InlineField>
+      </InlineFieldRow>
+    </div>
+  );
+};
+
+export const extractFieldsTransformRegistryItem: TransformerRegistryItem<ExtractFieldsOptions> = {
+  id: DataTransformerID.extractFields,
+  editor: extractFieldsTransformerEditor,
+  transformation: extractFieldsTransformer,
+  name: 'Extract fields',
+  description: `Parse fields from content (JSON, labels, etc)`,
+  state: PluginState.alpha,
+};

public/app/core/components/TransformersUI/extractFields/extractFields.test.ts

@@ -0,0 +1,75 @@
+import { toDataFrame } from '@grafana/data/src/dataframe/processDataFrame';
+import { ExtractFieldsOptions, extractFieldsTransformer } from './extractFields';
+
+describe('Fields from JSON', () => {
+  it('adds fields from JSON in string', async () => {
+    const cfg: ExtractFieldsOptions = {
+      source: 'line',
+      replace: true,
+    };
+    const data = toDataFrame({
+      columns: ['ts', 'line'],
+      rows: appl,
+    });
+
+    const frames = extractFieldsTransformer.transformer(cfg)([data]);
+    expect(frames.length).toEqual(1);
+    expect(frames[0].fields.map((v) => v.name)).toMatchInlineSnapshot(`
+      Array [
+        "a",
+        "av",
+        "c",
+        "e",
+        "ev",
+        "h",
+        "l",
+        "o",
+        "op",
+        "s",
+        "sym",
+        "v",
+        "vw",
+        "z",
+      ]
+    `);
+  });
+});
+
+const appl = [
+  [
+    '1636678740000000000',
+    '{"a":"148.1673","av":41941752,"c":"148.25","e":1636678800000,"ev":"AM","h":"148.28","l":"148.22","o":"148.25","op":"148.96","s":1636678740000,"sym":"AAPL","v":2903,"vw":"148.2545","z":152}',
+  ],
+  [
+    '1636678680000000000',
+    '{"a":"148.1673","av":41938849,"c":"148.25","e":1636678740000,"ev":"AM","h":"148.27","l":"148.25","o":"148.26","op":"148.96","s":1636678680000,"sym":"AAPL","v":7589,"vw":"148.2515","z":329}',
+  ],
+  [
+    '1636678620000000000',
+    '{"a":"148.1672","av":41931260,"c":"148.27","e":1636678680000,"ev":"AM","h":"148.27","l":"148.25","o":"148.27","op":"148.96","s":1636678620000,"sym":"AAPL","v":6138,"vw":"148.2541","z":245}',
+  ],
+  [
+    '1636678560000000000',
+    '{"a":"148.1672","av":41925122,"c":"148.28","e":1636678620000,"ev":"AM","h":"148.29","l":"148.27","o":"148.27","op":"148.96","s":1636678560000,"sym":"AAPL","v":1367,"vw":"148.2816","z":56}',
+  ],
+  [
+    '1636678500000000000',
+    '{"a":"148.1672","av":41923755,"c":"148.25","e":1636678560000,"ev":"AM","h":"148.27","l":"148.25","o":"148.25","op":"148.96","s":1636678500000,"sym":"AAPL","v":556,"vw":"148.2539","z":55}',
+  ],
+  [
+    '1636678440000000000',
+    '{"a":"148.1672","av":41923199,"c":"148.28","e":1636678500000,"ev":"AM","h":"148.28","l":"148.25","o":"148.25","op":"148.96","s":1636678440000,"sym":"AAPL","v":451,"vw":"148.2614","z":56}',
+  ],
+  [
+    '1636678380000000000',
+    '{"a":"148.1672","av":41922748,"c":"148.24","e":1636678440000,"ev":"AM","h":"148.24","l":"148.24","o":"148.24","op":"148.96","s":1636678380000,"sym":"AAPL","v":344,"vw":"148.2521","z":24}',
+  ],
+  [
+    '1636678320000000000',
+    '{"a":"148.1672","av":41922404,"c":"148.28","e":1636678380000,"ev":"AM","h":"148.28","l":"148.24","o":"148.24","op":"148.96","s":1636678320000,"sym":"AAPL","v":705,"vw":"148.2543","z":64}',
+  ],
+  [
+    '1636678260000000000',
+    '{"a":"148.1672","av":41921699,"c":"148.25","e":1636678320000,"ev":"AM","h":"148.25","l":"148.25","o":"148.25","op":"148.96","s":1636678260000,"sym":"AAPL","v":1054,"vw":"148.2513","z":131}',
+  ],
+];

public/app/core/components/TransformersUI/extractFields/extractFields.ts

@@ -0,0 +1,92 @@
+import {
+  ArrayVector,
+  DataFrame,
+  DataTransformerID,
+  Field,
+  FieldType,
+  guessFieldTypeForField,
+  SynchronousDataTransformerInfo,
+} from '@grafana/data';
+import { findField } from 'app/features/dimensions';
+import { isString } from 'lodash';
+import { map } from 'rxjs/operators';
+import { FieldExtractorID, fieldExtractors } from './fieldExtractors';
+
+export interface ExtractFieldsOptions {
+  source?: string;
+  format?: FieldExtractorID;
+  replace?: boolean;
+}
+
+export const extractFieldsTransformer: SynchronousDataTransformerInfo<ExtractFieldsOptions> = {
+  id: DataTransformerID.extractFields,
+  name: 'Extract fields',
+  description: 'Parse fields from the contends of another',
+  defaultOptions: {},
+
+  operator: (options) => (source) => source.pipe(map((data) => extractFieldsTransformer.transformer(options)(data))),
+
+  transformer: (options: ExtractFieldsOptions) => {
+    return (data: DataFrame[]) => {
+      return data.map((v) => addExtractedFields(v, options));
+    };
+  },
+};
+
+function addExtractedFields(frame: DataFrame, options: ExtractFieldsOptions): DataFrame {
+  if (!options.source) {
+    return frame;
+  }
+  const source = findField(frame, options.source);
+  if (!source) {
+    throw new Error('json field not found');
+  }
+
+  const ext = fieldExtractors.getIfExists(options.format ?? FieldExtractorID.Auto);
+  if (!ext) {
+    throw new Error('unkonwn extractor');
+  }
+
+  const count = frame.length;
+  const names: string[] = []; // keep order
+  const values = new Map<string, any[]>();
+
+  for (let i = 0; i < count; i++) {
+    let obj = source.values.get(i);
+    if (isString(obj)) {
+      try {
+        obj = ext.parse(obj);
+      } catch {
+        obj = {}; // empty
+      }
+    }
+    for (const [key, val] of Object.entries(obj)) {
+      let buffer = values.get(key);
+      if (buffer == null) {
+        buffer = new Array(count);
+        values.set(key, buffer);
+        names.push(key);
+      }
+      buffer[i] = val;
+    }
+  }
+
+  const fields = names.map((name) => {
+    const f: Field = {
+      name,
+      values: new ArrayVector(values.get(name)),
+      type: FieldType.boolean,
+      config: {},
+    };
+    f.type = guessFieldTypeForField(f) ?? FieldType.other;
+    return f;
+  });
+
+  if (!options.replace) {
+    fields.unshift(...frame.fields);
+  }
+  return {
+    ...frame,
+    fields,
+  };
+}

public/app/core/components/TransformersUI/extractFields/fieldExtractor.test.ts

@@ -0,0 +1,53 @@
+import { fieldExtractors, FieldExtractorID } from './fieldExtractors';
+
+describe('Extract fields from text', () => {
+  it('JSON extractor', async () => {
+    const extractor = fieldExtractors.get(FieldExtractorID.JSON);
+    const out = extractor.parse('{"a":"148.1672","av":41923755,"c":148.25}');
+
+    expect(out).toMatchInlineSnapshot(`
+      Object {
+        "a": "148.1672",
+        "av": 41923755,
+        "c": 148.25,
+      }
+    `);
+  });
+
+  it('Split key+values', async () => {
+    const extractor = fieldExtractors.get(FieldExtractorID.KeyValues);
+    const out = extractor.parse('a="1",   "b"=\'2\',c=3  x:y ;\r\nz="7"');
+    expect(out).toMatchInlineSnapshot(`
+      Object {
+        "a": "1",
+        "b": "2",
+        "c": "3",
+        "x": "y",
+        "z": "7",
+      }
+    `);
+  });
+
+  it('Split URL style parameters', async () => {
+    const extractor = fieldExtractors.get(FieldExtractorID.KeyValues);
+    const out = extractor.parse('a=b&c=d&x=123');
+    expect(out).toMatchInlineSnapshot(`
+      Object {
+        "a": "b",
+        "c": "d",
+        "x": "123",
+      }
+    `);
+  });
+
+  it('Prometheus labels style (not really supported)', async () => {
+    const extractor = fieldExtractors.get(FieldExtractorID.KeyValues);
+    const out = extractor.parse('{foo="bar", baz="42"}');
+    expect(out).toMatchInlineSnapshot(`
+      Object {
+        "baz": "42",
+        "foo": "bar",
+      }
+    `);
+  });
+});

public/app/core/components/TransformersUI/extractFields/fieldExtractors.ts

@@ -0,0 +1,70 @@
+import { Registry, RegistryItem } from '@grafana/data';
+
+export enum FieldExtractorID {
+  JSON = 'json',
+  KeyValues = 'kvp',
+  Auto = 'auto',
+}
+
+export interface FieldExtractor extends RegistryItem {
+  parse: (v: string) => Record<string, any> | undefined;
+}
+
+const extJSON: FieldExtractor = {
+  id: FieldExtractorID.JSON,
+  name: 'JSON',
+  description: 'Parse JSON string',
+  parse: (v: string) => {
+    return JSON.parse(v);
+  },
+};
+
+// strips quotes and leading/trailing braces in prom labels
+const stripDecor = /['"]|^\{|\}$/g;
+// splits on whitespace and other label pair delimiters
+const splitLines = /[\s,;&]+/g;
+// splits kv pairs
+const splitPair = /[=:]/g;
+
+const extLabels: FieldExtractor = {
+  id: FieldExtractorID.KeyValues,
+  name: 'Key+value pairs',
+  description: 'Look for a=b, c: d values in the line',
+  parse: (v: string) => {
+    const obj: Record<string, any> = {};
+
+    v.trim()
+      .replace(stripDecor, '')
+      .split(splitLines)
+      .forEach((pair) => {
+        let [k, v] = pair.split(splitPair);
+
+        if (k != null) {
+          obj[k] = v;
+        }
+      });
+
+    return obj;
+  },
+};
+
+const fmts = [extJSON, extLabels];
+
+const extAuto: FieldExtractor = {
+  id: FieldExtractorID.Auto,
+  name: 'Auto',
+  description: 'parse new fields automatically',
+  parse: (v: string) => {
+    for (const f of fmts) {
+      try {
+        const r = f.parse(v);
+        if (r != null) {
+          return r;
+        }
+      } catch {} // ignore errors
+    }
+    return undefined;
+  },
+};
+
+export const fieldExtractors = new Registry<FieldExtractor>(() => [...fmts, extAuto]);

public/app/core/utils/standardTransformers.ts

@@ -19,6 +19,7 @@ import { configFromQueryTransformRegistryItem } from '../components/Transformers
 import { prepareTimeseriesTransformerRegistryItem } from '../components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor';
 import { convertFieldTypeTransformRegistryItem } from '../components/TransformersUI/ConvertFieldTypeTransformerEditor';
 import { fieldLookupTransformRegistryItem } from '../components/TransformersUI/lookupGazetteer/FieldLookupTransformerEditor';
+import { extractFieldsTransformRegistryItem } from '../components/TransformersUI/extractFields/ExtractFieldsTransformerEditor';
 
 export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> => {
   return [
@@ -42,5 +43,6 @@ export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> =
     prepareTimeseriesTransformerRegistryItem,
     convertFieldTypeTransformRegistryItem,
     fieldLookupTransformRegistryItem,
+    extractFieldsTransformRegistryItem,
   ];
 };