Transformations: Extract JSON Paths (#59400)

* Added new extractJSONPath transformer
Co-authored-by: Galen <galen.kistler@grafana.com>

public/app/features/transformers/extractFields/ExtractFieldsTransformerEditor.tsx

@@ -2,17 +2,19 @@ import React from 'react';
 
 import {
   DataTransformerID,
+  TransformerRegistryItem,
+  TransformerUIProps,
   FieldNamePickerConfigSettings,
   SelectableValue,
   StandardEditorsRegistryItem,
-  TransformerRegistryItem,
-  TransformerUIProps,
 } from '@grafana/data';
-import { InlineField, InlineFieldRow, InlineSwitch, Select } from '@grafana/ui';
+import { InlineField, InlineFieldRow, Select, InlineSwitch } from '@grafana/ui';
 import { FieldNamePicker } from '@grafana/ui/src/components/MatchersUI/FieldNamePicker';
 
-import { ExtractFieldsOptions, extractFieldsTransformer } from './extractFields';
-import { FieldExtractorID, fieldExtractors } from './fieldExtractors';
+import { JSONPathEditor } from './components/JSONPathEditor';
+import { extractFieldsTransformer } from './extractFields';
+import { fieldExtractors } from './fieldExtractors';
+import { ExtractFieldsOptions, FieldExtractorID, JSONPath } from './types';
 
 const fieldNamePickerSettings: StandardEditorsRegistryItem<string, FieldNamePickerConfigSettings> = {
   settings: {
@@ -43,13 +45,31 @@ export const extractFieldsTransformerEditor: React.FC<TransformerUIProps<Extract
     });
   };
 
+  const onJSONPathsChange = (jsonPaths: JSONPath[]) => {
+    onChange({
+      ...options,
+      jsonPaths,
+    });
+  };
+
   const onToggleReplace = () => {
+    if (options.replace) {
+      options.keepTime = false;
+    }
+
     onChange({
       ...options,
       replace: !options.replace,
     });
   };
 
+  const onToggleKeepTime = () => {
+    onChange({
+      ...options,
+      keepTime: !options.keepTime,
+    });
+  };
+
   const format = fieldExtractors.selectOptions(options.format ? [options.format] : undefined);
 
   return (
@@ -75,11 +95,19 @@ export const extractFieldsTransformerEditor: React.FC<TransformerUIProps<Extract
           />
         </InlineField>
       </InlineFieldRow>
+      {options.format === 'json' && <JSONPathEditor options={options.jsonPaths ?? []} onChange={onJSONPathsChange} />}
       <InlineFieldRow>
         <InlineField label={'Replace all fields'} labelWidth={16}>
           <InlineSwitch value={options.replace ?? false} onChange={onToggleReplace} />
         </InlineField>
       </InlineFieldRow>
+      {options.replace && (
+        <InlineFieldRow>
+          <InlineField label={'Keep time'} labelWidth={16}>
+            <InlineSwitch value={options.keepTime ?? false} onChange={onToggleKeepTime} />
+          </InlineField>
+        </InlineFieldRow>
+      )}
     </div>
   );
 };

public/app/features/transformers/extractFields/components/JSONPathEditor.tsx

@@ -0,0 +1,130 @@
+import { css, cx } from '@emotion/css';
+import React, { useState } from 'react';
+
+import { Button, InlineField, InlineFieldRow, IconButton, Input } from '@grafana/ui';
+
+import { JSONPath } from '../types';
+
+interface Props {
+  options: JSONPath[];
+  onChange: (options: JSONPath[]) => void;
+}
+
+export function JSONPathEditor({ options, onChange }: Props) {
+  const [paths, setPaths] = useState<JSONPath[]>(options);
+  const tooltips = getTooltips();
+  const style = getStyle();
+
+  const addJSONPath = () => {
+    paths.push({ path: '' });
+    setPaths([...paths]);
+    onBlur();
+  };
+
+  const removeJSONPath = (keyPath: number) => {
+    if (paths) {
+      paths.splice(keyPath, 1);
+      setPaths([...paths]);
+      onBlur();
+    }
+  };
+
+  const onJSONPathChange = (event: React.SyntheticEvent<HTMLInputElement>, keyPath: number, type: 'alias' | 'path') => {
+    if (paths) {
+      if (type === 'alias') {
+        paths[keyPath].alias = event.currentTarget.value ?? '';
+      } else {
+        paths[keyPath].path = event.currentTarget.value ?? '';
+      }
+      setPaths([...paths]);
+    }
+  };
+
+  const onBlur = () => {
+    onChange(paths);
+  };
+
+  return (
+    <ol className={cx(style.list)}>
+      {paths &&
+        paths.map((path: JSONPath, key: number) => (
+          <li key={key}>
+            <InlineFieldRow>
+              <InlineField label="Field" tooltip={tooltips.field} grow>
+                <Input
+                  onBlur={onBlur}
+                  onChange={(event: React.SyntheticEvent<HTMLInputElement>) => onJSONPathChange(event, key, 'path')}
+                  value={path.path}
+                  placeholder='A valid json path, e.g. "object.value1" or "object.value2[0]"'
+                />
+              </InlineField>
+              <InlineField label="Alias" tooltip={tooltips.alias}>
+                <Input
+                  width={12}
+                  value={path.alias}
+                  onBlur={onBlur}
+                  onChange={(event: React.SyntheticEvent<HTMLInputElement>) => onJSONPathChange(event, key, 'alias')}
+                />
+              </InlineField>
+              <InlineField className={cx(style.removeIcon)}>
+                <IconButton onClick={() => removeJSONPath(key)} name={'trash-alt'} />
+              </InlineField>
+            </InlineFieldRow>
+          </li>
+        ))}
+      <InlineField>
+        <Button icon={'plus'} onClick={() => addJSONPath()} variant={'secondary'}>
+          Add path
+        </Button>
+      </InlineField>
+    </ol>
+  );
+}
+
+const getTooltips = () => {
+  const mapValidPaths = [
+    { path: 'object', description: '=> extract fields from object' },
+    { path: 'object.value1', description: '=> extract value1' },
+    { path: 'object.value2', description: '=> extract value2' },
+    { path: 'object.value2[0]', description: '=> extract value2 first element' },
+    { path: 'object.value2[1]', description: '=> extract value2 second element' },
+  ];
+
+  return {
+    field: (
+      <div>
+        A valid path of an json object.
+        <div>
+          <strong>JSON Value:</strong>
+        </div>
+        <pre>
+          <code>
+            {['{', '  "object": {', '    "value1": "hello world"', '    "value2": [1, 2, 3, 4]', '  }', '}'].join('\n')}
+          </code>
+        </pre>
+        <strong>Valid Paths:</strong>
+        {mapValidPaths.map((value, key) => {
+          return (
+            <p key={key}>
+              <code>{value.path}</code>
+              <i>{value.description}</i>
+            </p>
+          );
+        })}
+      </div>
+    ),
+    alias: 'An alias name for the variable in the dashboard. If left blank the given path will be used.',
+  };
+};
+
+function getStyle() {
+  return {
+    list: css`
+      margin-left: 20px;
+    `,
+    removeIcon: css`
+      margin: 0 0 0 4px;
+      align-items: center;
+    `,
+  };
+}

public/app/features/transformers/extractFields/extractFields.test.ts

@@ -1,6 +1,8 @@
+import { ArrayVector, DataFrame, Field, FieldType } from '@grafana/data';
 import { toDataFrame } from '@grafana/data/src/dataframe/processDataFrame';
 
-import { ExtractFieldsOptions, extractFieldsTransformer } from './extractFields';
+import { extractFieldsTransformer } from './extractFields';
+import { ExtractFieldsOptions, FieldExtractorID } from './types';
 
 describe('Fields from JSON', () => {
   it('adds fields from JSON in string', async () => {
@@ -40,8 +42,186 @@ describe('Fields from JSON', () => {
       }
     `);
   });
+
+  it('Get nested path values', () => {
+    const cfg: ExtractFieldsOptions = {
+      replace: true,
+      source: 'JSON',
+      format: FieldExtractorID.JSON,
+      jsonPaths: [
+        { path: 'object.nestedArray[0]' },
+        { path: 'object.nestedArray[1]' },
+        { path: 'object.nestedString' },
+      ],
+    };
+    const ctx = { interpolate: (v: string) => v };
+
+    const frames = extractFieldsTransformer.transformer(cfg, ctx)([testDataFrame]);
+    expect(frames.length).toEqual(1);
+    expect(frames[0]).toMatchInlineSnapshot(`
+      {
+        "fields": [
+          {
+            "config": {},
+            "name": "object.nestedArray[0]",
+            "type": "number",
+            "values": [
+              1,
+            ],
+          },
+          {
+            "config": {},
+            "name": "object.nestedArray[1]",
+            "type": "number",
+            "values": [
+              2,
+            ],
+          },
+          {
+            "config": {},
+            "name": "object.nestedString",
+            "type": "string",
+            "values": [
+              "Hallo World",
+            ],
+          },
+        ],
+        "length": 1,
+      }
+    `);
+  });
+
+  it('Keep time field on replace', () => {
+    const cfg: ExtractFieldsOptions = {
+      replace: true,
+      keepTime: true,
+      source: 'JSON',
+      format: FieldExtractorID.JSON,
+      jsonPaths: [
+        { path: 'object.nestedArray[2]' },
+        { path: 'object.nestedArray[3]' },
+        { path: 'object.nestedString' },
+      ],
+    };
+    const ctx = { interpolate: (v: string) => v };
+
+    const frames = extractFieldsTransformer.transformer(cfg, ctx)([testDataFrame]);
+    expect(frames.length).toEqual(1);
+    expect(frames[0]).toMatchInlineSnapshot(`
+      {
+        "fields": [
+          {
+            "config": {},
+            "name": "Time",
+            "state": {
+              "displayName": "Time",
+              "multipleFrames": false,
+            },
+            "type": "time",
+            "values": [
+              1669638911691,
+            ],
+          },
+          {
+            "config": {},
+            "name": "object.nestedArray[2]",
+            "type": "number",
+            "values": [
+              3,
+            ],
+          },
+          {
+            "config": {},
+            "name": "object.nestedArray[3]",
+            "type": "number",
+            "values": [
+              4,
+            ],
+          },
+          {
+            "config": {},
+            "name": "object.nestedString",
+            "type": "string",
+            "values": [
+              "Hallo World",
+            ],
+          },
+        ],
+        "length": 1,
+      }
+    `);
+  });
+
+  it('Path is invalid', () => {
+    const cfg: ExtractFieldsOptions = {
+      replace: true,
+      source: 'JSON',
+      format: FieldExtractorID.JSON,
+      jsonPaths: [{ path: 'object.nestedString' }, { path: 'invalid.path' }],
+    };
+    const ctx = { interpolate: (v: string) => v };
+
+    const frames = extractFieldsTransformer.transformer(cfg, ctx)([testDataFrame]);
+    expect(frames.length).toEqual(1);
+    expect(frames[0]).toMatchInlineSnapshot(`
+      {
+        "fields": [
+          {
+            "config": {},
+            "name": "object.nestedString",
+            "type": "string",
+            "values": [
+              "Hallo World",
+            ],
+          },
+          {
+            "config": {},
+            "name": "invalid.path",
+            "type": "string",
+            "values": [
+              "Not Found",
+            ],
+          },
+        ],
+        "length": 1,
+      }
+    `);
+  });
 });
 
+const testFieldTime: Field = {
+  config: {},
+  name: 'Time',
+  type: FieldType.time,
+  values: new ArrayVector([1669638911691]),
+};
+
+const testFieldString: Field = {
+  config: {},
+  name: 'String',
+  type: FieldType.string,
+  values: new ArrayVector(['Hallo World']),
+};
+
+const testFieldJSON: Field = {
+  config: {},
+  name: 'JSON',
+  type: FieldType.string,
+  values: new ArrayVector([
+    JSON.stringify({
+      object: {
+        nestedArray: [1, 2, 3, 4],
+        nestedString: 'Hallo World',
+      },
+    }),
+  ]),
+};
+
+const testDataFrame: DataFrame = {
+  fields: [testFieldTime, testFieldString, testFieldJSON],
+  length: 1,
+};
+
 const appl = [
   [
     '1636678740000000000',

public/app/features/transformers/extractFields/extractFields.ts

@@ -1,4 +1,4 @@
-import { isString } from 'lodash';
+import { isString, get } from 'lodash';
 import { map } from 'rxjs/operators';
 
 import {
@@ -12,13 +12,8 @@ import {
 } from '@grafana/data';
 import { findField } from 'app/features/dimensions';
 
-import { FieldExtractorID, fieldExtractors } from './fieldExtractors';
-
-export interface ExtractFieldsOptions {
-  source?: string;
-  format?: FieldExtractorID;
-  replace?: boolean;
-}
+import { fieldExtractors } from './fieldExtractors';
+import { ExtractFieldsOptions, FieldExtractorID, JSONPath } from './types';
 
 export const extractFieldsTransformer: SynchronousDataTransformerInfo<ExtractFieldsOptions> = {
   id: DataTransformerID.extractFields,
@@ -40,7 +35,9 @@ function addExtractedFields(frame: DataFrame, options: ExtractFieldsOptions): Da
   if (!options.source) {
     return frame;
   }
+
   const source = findField(frame, options.source);
+
   if (!source) {
     // this case can happen when there are multiple queries
     return frame;
@@ -57,6 +54,7 @@ function addExtractedFields(frame: DataFrame, options: ExtractFieldsOptions): Da
 
   for (let i = 0; i < count; i++) {
     let obj = source.values.get(i);
+
     if (isString(obj)) {
       try {
         obj = ext.parse(obj);
@@ -64,6 +62,22 @@ function addExtractedFields(frame: DataFrame, options: ExtractFieldsOptions): Da
         obj = {}; // empty
       }
     }
+
+    if (options.format === FieldExtractorID.JSON && options.jsonPaths && options.jsonPaths?.length > 0) {
+      const newObj: { [k: string]: unknown } = {};
+      // filter out empty paths
+      const filteredPaths = options.jsonPaths.filter((path: JSONPath) => path.path);
+
+      if (filteredPaths.length > 0) {
+        filteredPaths.forEach((path: JSONPath) => {
+          const key = path.alias && path.alias.length > 0 ? path.alias : path.path;
+          newObj[key] = get(obj, path.path) ?? 'Not Found';
+        });
+
+        obj = newObj;
+      }
+    }
+
     for (const [key, val] of Object.entries(obj)) {
       let buffer = values.get(key);
       if (buffer == null) {
@@ -85,9 +99,17 @@ function addExtractedFields(frame: DataFrame, options: ExtractFieldsOptions): Da
     } as Field;
   });
 
+  if (options.keepTime) {
+    const sourceTime = findField(frame, 'Time') || findField(frame, 'time');
+    if (sourceTime) {
+      fields.unshift(sourceTime);
+    }
+  }
+
   if (!options.replace) {
     fields.unshift(...frame.fields);
   }
+
   return {
     ...frame,
     fields,

public/app/features/transformers/extractFields/fieldExtractor.test.ts

@@ -1,4 +1,5 @@
-import { fieldExtractors, FieldExtractorID } from './fieldExtractors';
+import { fieldExtractors } from './fieldExtractors';
+import { FieldExtractorID } from './types';
 
 describe('Extract fields from text', () => {
   it('JSON extractor', async () => {

public/app/features/transformers/extractFields/fieldExtractors.ts

@@ -1,10 +1,6 @@
 import { Registry, RegistryItem } from '@grafana/data';
 
-export enum FieldExtractorID {
-  JSON = 'json',
-  KeyValues = 'kvp',
-  Auto = 'auto',
-}
+import { FieldExtractorID } from './types';
 
 export interface FieldExtractor extends RegistryItem {
   parse: (v: string) => Record<string, any> | undefined;

public/app/features/transformers/extractFields/types.ts

@@ -0,0 +1,17 @@
+export enum FieldExtractorID {
+  JSON = 'json',
+  KeyValues = 'kvp',
+  Auto = 'auto',
+}
+
+export interface JSONPath {
+  path: string;
+  alias?: string;
+}
+export interface ExtractFieldsOptions {
+  source?: string;
+  jsonPaths?: JSONPath[];
+  format?: FieldExtractorID;
+  replace?: boolean;
+  keepTime?: boolean;
+}