Transformations: Add Delimiter format option to Extract fields (#97340)

* Extract to fields: Add CSV extractor

* Support escaping

* Split at delimiter

* Fix unused import

* PR feedback

* Rename extractor

* Use number instead

* update

* Add comment about regex match

* Add deimiter selector in editor

* Default delimiter to comma

---------

Co-authored-by: Leon Sorokin <leeoniya@gmail.com>

public/app/features/transformers/extractFields/ExtractFieldsTransformerEditor.tsx

@@ -9,7 +9,7 @@ import {
   StandardEditorsRegistryItem,
   TransformerCategory,
 } from '@grafana/data';
-import { InlineField, InlineFieldRow, Select, InlineSwitch, Input } from '@grafana/ui';
+import { InlineField, InlineFieldRow, Select, InlineSwitch, Input, Combobox, ComboboxOption } from '@grafana/ui';
 import { FieldNamePicker } from '@grafana/ui/src/components/MatchersUI/FieldNamePicker';
 
 import { getTransformationContent } from '../docs/getTransformationContent';
@@ -31,7 +31,7 @@ const fieldNamePickerSettings: StandardEditorsRegistryItem<string, FieldNamePick
 
 export const extractFieldsTransformerEditor = ({
   input,
-  options,
+  options = { delimiter: ',' },
   onChange,
 }: TransformerUIProps<ExtractFieldsOptions>) => {
   const onPickSourceField = (source?: string) => {
@@ -62,6 +62,13 @@ export const extractFieldsTransformerEditor = ({
     });
   };
 
+  const onDelimiterChange = (val: ComboboxOption) => {
+    onChange({
+      ...options,
+      delimiter: val.value,
+    });
+  };
+
   const onToggleReplace = () => {
     if (options.replace) {
       options.keepTime = false;
@@ -115,6 +122,19 @@ export const extractFieldsTransformerEditor = ({
       {options.format === FieldExtractorID.JSON && (
         <JSONPathEditor options={options.jsonPaths ?? []} onChange={onJSONPathsChange} />
       )}
+      {options.format === FieldExtractorID.Delimiter && (
+        <InlineFieldRow>
+          <InlineField label="Delimiter" labelWidth={16}>
+            <Combobox
+              value={options.delimiter}
+              options={[{ value: ',' }, { value: ';' }, { value: '|' }]}
+              onChange={onDelimiterChange}
+              placeholder="Select delimiter..."
+              width={24}
+            />
+          </InlineField>
+        </InlineFieldRow>
+      )}
       <InlineFieldRow>
         <InlineField label={'Replace all fields'} labelWidth={16}>
           <InlineSwitch value={options.replace ?? false} onChange={onToggleReplace} />

public/app/features/transformers/extractFields/extractFields.ts

@@ -20,7 +20,9 @@ export const extractFieldsTransformer: SynchronousDataTransformerInfo<ExtractFie
   id: DataTransformerID.extractFields,
   name: 'Extract fields',
   description: 'Parse fields from the contends of another',
-  defaultOptions: {},
+  defaultOptions: {
+    delimiter: ',',
+  },
 
   operator: (options, ctx) => (source) =>
     source.pipe(map((data) => extractFieldsTransformer.transformer(options, ctx)(data))),

public/app/features/transformers/extractFields/fieldExtractor.test.ts

@@ -151,4 +151,35 @@ describe('Extract fields from text', () => {
       }
     `);
   });
+
+  describe('Delimiter', () => {
+    it('splits by comma', async () => {
+      const extractor = fieldExtractors.get(FieldExtractorID.Delimiter);
+      const parse = extractor.getParser({});
+      const out = parse('a,b,c');
+
+      expect(out).toMatchInlineSnapshot(`
+        {
+          "a": 1,
+          "b": 1,
+          "c": 1,
+        }
+      `);
+    });
+
+    it('trims whitespace', async () => {
+      const extractor = fieldExtractors.get(FieldExtractorID.Delimiter);
+      const parse = extractor.getParser({});
+      const out = parse(` a, b,c, d `);
+
+      expect(out).toMatchInlineSnapshot(`
+        {
+          "a": 1,
+          "b": 1,
+          "c": 1,
+          "d": 1,
+        }
+      `);
+    });
+  });
 });

public/app/features/transformers/extractFields/fieldExtractors.ts

@@ -1,4 +1,4 @@
-import { Registry, RegistryItem, stringStartsAsRegEx, stringToJsRegex } from '@grafana/data';
+import { escapeStringForRegex, Registry, RegistryItem, stringStartsAsRegEx, stringToJsRegex } from '@grafana/data';
 
 import { ExtractFieldsOptions, FieldExtractorID } from './types';
 
@@ -133,7 +133,27 @@ const extLabels: FieldExtractor = {
   getParser: (options) => parseKeyValuePairs,
 };
 
-const fmts = [extJSON, extLabels, extRegExp];
+const extDelimiter: FieldExtractor = {
+  id: FieldExtractorID.Delimiter,
+  name: 'Split by delimiter',
+  description: 'Splits at delimited values, such as commas',
+  getParser: ({ delimiter = ',' }) => {
+    // Match for delimiter with surrounding whitesapce (\s)
+    const splitRegExp = new RegExp(`\\s*${escapeStringForRegex(delimiter)}\\s*`, 'g');
+
+    return (raw: string) => {
+      // Try to split delimited values
+      const parts = raw.trim().split(splitRegExp);
+      const acc: Record<string, number> = {};
+      for (const part of parts) {
+        acc[part] = 1;
+      }
+      return acc;
+    };
+  },
+};
+
+const fmts = [extJSON, extLabels, extDelimiter, extRegExp];
 
 const extAuto: FieldExtractor = {
   id: FieldExtractorID.Auto,

public/app/features/transformers/extractFields/types.ts

@@ -3,6 +3,7 @@ export enum FieldExtractorID {
   KeyValues = 'kvp',
   Auto = 'auto',
   RegExp = 'regexp',
+  Delimiter = 'delimiter',
 }
 
 export interface JSONPath {
@@ -12,6 +13,7 @@ export interface JSONPath {
 export interface ExtractFieldsOptions {
   source?: string;
   jsonPaths?: JSONPath[];
+  delimiter?: string;
   regExp?: string;
   format?: FieldExtractorID;
   replace?: boolean;