Transformers: PartitionByValues (#56767)

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>

packages/grafana-data/src/transformations/transformers/filterByName.ts

@@ -32,7 +32,8 @@ export const filterFieldsByNameTransformer: DataTransformerInfo<FilterFieldsByNa
     ),
 };
 
-const getMatcherConfig = (options?: RegexpOrNamesMatcherOptions): MatcherConfig | undefined => {
+// Exported to share with other implementations, but not exported to `@grafana/data`
+export const getMatcherConfig = (options?: RegexpOrNamesMatcherOptions): MatcherConfig | undefined => {
   if (!options) {
     return undefined;
   }

packages/grafana-data/src/transformations/transformers/ids.ts

@@ -35,4 +35,5 @@ export enum DataTransformerID {
   extractFields = 'extractFields',
   groupingToMatrix = 'groupingToMatrix',
   limit = 'limit',
+  partitionByValues = 'partitionByValues',
 }

packages/grafana-data/src/transformations/transformers/noop.ts

@@ -1,21 +1,19 @@
-import { DataTransformerInfo } from '../../types/transformations';
+import { DataFrame } from '../../types';
+import { SynchronousDataTransformerInfo } from '../../types/transformations';
 
 import { DataTransformerID } from './ids';
 
-export interface NoopTransformerOptions {
-  include?: string;
-  exclude?: string;
-}
+export interface NoopTransformerOptions {}
 
-export const noopTransformer: DataTransformerInfo<NoopTransformerOptions> = {
+export const noopTransformer: SynchronousDataTransformerInfo<NoopTransformerOptions> = {
   id: DataTransformerID.noop,
   name: 'noop',
   description: 'No-operation transformer',
   defaultOptions: {},
 
-  /**
-   * Return a modified copy of the series.  If the transform is not or should not
-   * be applied, just return the input series
-   */
+  /** no operation */
   operator: (options: NoopTransformerOptions) => (source) => source,
+
+  /** no operation */
+  transformer: (options: NoopTransformerOptions) => (data: DataFrame[]) => data,
 };

public/app/features/transformers/partitionByValues/PartitionByValuesEditor.tsx

@@ -0,0 +1,107 @@
+import React, { useCallback, useMemo } from 'react';
+
+import {
+  DataTransformerID,
+  PluginState,
+  TransformerRegistryItem,
+  TransformerUIProps,
+  SelectableValue,
+} from '@grafana/data';
+import { InlineField, InlineFieldRow, ValuePicker, Button, HorizontalGroup, FieldValidationMessage } from '@grafana/ui';
+import { useFieldDisplayNames, useSelectOptions } from '@grafana/ui/src/components/MatchersUI/utils';
+
+import { partitionByValuesTransformer, PartitionByValuesTransformerOptions } from './partitionByValues';
+
+export function PartitionByValuesEditor({
+  input,
+  options,
+  onChange,
+}: TransformerUIProps<PartitionByValuesTransformerOptions>) {
+  const names = useFieldDisplayNames(input);
+  const allSelectOptions = useSelectOptions(names);
+  const selectOptions = useMemo(() => {
+    const fieldNames = new Set(options.fields);
+
+    if (fieldNames.size < 1) {
+      return allSelectOptions;
+    }
+    return allSelectOptions.filter((v) => !fieldNames.has(v.value!));
+  }, [allSelectOptions, options]);
+
+  const addField = useCallback(
+    (v: SelectableValue<string>) => {
+      if (!v.value) {
+        return;
+      }
+
+      const fieldNames = new Set(options.fields);
+
+      fieldNames.add(v.value);
+
+      onChange({
+        ...options,
+        fields: [...fieldNames],
+      });
+    },
+    [onChange, options]
+  );
+
+  const removeField = useCallback(
+    (v: string) => {
+      if (!v) {
+        return;
+      }
+
+      const fieldNames = new Set(options.fields);
+
+      fieldNames.delete(v);
+
+      onChange({
+        ...options,
+        fields: [...fieldNames],
+      });
+    },
+    [onChange, options]
+  );
+
+  if (input.length > 1) {
+    return <FieldValidationMessage>Partition by values only works with a single frame.</FieldValidationMessage>;
+  }
+
+  const fieldNames = [...new Set(options.fields)];
+
+  return (
+    <div>
+      <InlineFieldRow>
+        <InlineField label="Field" labelWidth={10} grow={true}>
+          <HorizontalGroup>
+            {fieldNames.map((name) => (
+              <Button key={name} icon="times" variant="secondary" size="md" onClick={() => removeField(name)}>
+                {name}
+              </Button>
+            ))}
+            {selectOptions.length && (
+              <ValuePicker
+                variant="secondary"
+                size="md"
+                options={selectOptions}
+                onChange={addField}
+                label="Select field"
+                icon="plus"
+              />
+            )}
+          </HorizontalGroup>
+        </InlineField>
+      </InlineFieldRow>
+    </div>
+  );
+}
+
+export const partitionByValuesTransformRegistryItem: TransformerRegistryItem<PartitionByValuesTransformerOptions> = {
+  id: DataTransformerID.partitionByValues,
+  editor: PartitionByValuesEditor,
+  transformation: partitionByValuesTransformer,
+  name: partitionByValuesTransformer.name,
+  description: partitionByValuesTransformer.description,
+  state: PluginState.alpha,
+};

public/app/features/transformers/partitionByValues/partition.ts

@@ -0,0 +1,52 @@
+type Idxs = number[];
+type KeyMap = Map<unknown, KeyMap | Idxs>;
+type Accum = Idxs[];
+
+/** The totally type-aware flavor is much slower, so we prefer to disable the lint rule in this case */
+/* eslint-disable @typescript-eslint/consistent-type-assertions */
+const digArrs = (map: KeyMap | Idxs, depth: number, acc: Accum = []) => {
+  // the leaf nodes are always Idxs
+  if (depth === 0) {
+    acc.push(map as Idxs);
+  }
+  // the branch nodes are always KeyMaps
+  else {
+    (map as KeyMap).forEach((v) => {
+      digArrs(v, depth - 1, acc);
+    });
+  }
+
+  return acc;
+};
+
+// in:  [['a','b','z','b'], ['c','c','x','c']]
+// out: [[0], [1,3], [2]]
+export function partition(keys: unknown[][]) {
+  const len = keys[0].length;
+  const klen = keys.length;
+
+  const rootMap: KeyMap = new Map();
+
+  for (let i = 0; i < len; i++) {
+    let cur: KeyMap | Idxs = rootMap;
+
+    for (let j = 0; j < klen; j++) {
+      let key = keys[j][i];
+
+      let next: KeyMap | Idxs | undefined = (cur as KeyMap).get(key);
+
+      if (next == null) {
+        next = j === klen - 1 ? [] : new Map();
+        (cur as KeyMap).set(key, next);
+      }
+
+      cur = next;
+    }
+
+    (cur as Idxs).push(i);
+  }
+
+  return digArrs(rootMap, klen);
+}
+
+/* eslint-enable @typescript-eslint/consistent-type-assertions */

public/app/features/transformers/partitionByValues/partitionByValues.test.ts

@@ -0,0 +1,183 @@
+import { toDataFrame, FieldType } from '@grafana/data';
+
+import { partitionByValuesTransformer, PartitionByValuesTransformerOptions } from './partitionByValues';
+
+describe('Partition by values transformer', () => {
+  it('should partition by one field', () => {
+    const source = [
+      toDataFrame({
+        name: 'XYZ',
+        refId: 'A',
+        fields: [
+          { name: 'model', type: FieldType.string, values: ['E1', 'E2', 'C1', 'E3', 'C2', 'C3'] },
+          { name: 'region', type: FieldType.string, values: ['Europe', 'Europe', 'China', 'Europe', 'China', 'China'] },
+        ],
+      }),
+    ];
+
+    const config: PartitionByValuesTransformerOptions = {
+      fields: ['region'],
+    };
+
+    let partitioned = partitionByValuesTransformer.transformer(config)(source);
+
+    expect(partitioned.length).toEqual(2);
+
+    expect(partitioned[0].length).toEqual(3);
+    expect(partitioned[0].name).toEqual('Europe');
+    expect(partitioned[0].fields[0].name).toEqual('model');
+    expect(partitioned[0].fields[1].name).toEqual('region');
+    expect(partitioned[0].fields[0].values.toArray()).toEqual(['E1', 'E2', 'E3']);
+    expect(partitioned[0].fields[1].values.toArray()).toEqual(['Europe', 'Europe', 'Europe']);
+
+    expect(partitioned[1].length).toEqual(3);
+    expect(partitioned[1].name).toEqual('China');
+    expect(partitioned[1].fields[0].name).toEqual('model');
+    expect(partitioned[1].fields[1].name).toEqual('region');
+    expect(partitioned[1].fields[0].values.toArray()).toEqual(['C1', 'C2', 'C3']);
+    expect(partitioned[1].fields[1].values.toArray()).toEqual(['China', 'China', 'China']);
+  });
+
+  it('should partition by multiple fields', () => {
+    const source = [
+      toDataFrame({
+        name: 'XYZ',
+        refId: 'A',
+        fields: [
+          { name: 'model', type: FieldType.string, values: ['E1', 'E2', 'C1', 'E3', 'C2', 'C3'] },
+          { name: 'region', type: FieldType.string, values: ['Europe', 'Europe', 'China', 'Europe', 'China', 'China'] },
+          { name: 'status', type: FieldType.string, values: ['OK', 'FAIL', 'OK', 'FAIL', 'OK', 'FAIL'] },
+        ],
+      }),
+    ];
+
+    const config: PartitionByValuesTransformerOptions = {
+      fields: ['region', 'status'],
+    };
+
+    let partitioned = partitionByValuesTransformer.transformer(config)(source);
+
+    expect(partitioned.length).toEqual(4);
+
+    expect(partitioned[0].length).toEqual(1);
+    expect(partitioned[0].name).toEqual('Europe OK');
+    expect(partitioned[0].fields[0].name).toEqual('model');
+    expect(partitioned[0].fields[1].name).toEqual('region');
+    expect(partitioned[0].fields[2].name).toEqual('status');
+    expect(partitioned[0].fields[0].values.toArray()).toEqual(['E1']);
+    expect(partitioned[0].fields[1].values.toArray()).toEqual(['Europe']);
+    expect(partitioned[0].fields[2].values.toArray()).toEqual(['OK']);
+
+    expect(partitioned[1].length).toEqual(2);
+    expect(partitioned[1].name).toEqual('Europe FAIL');
+    expect(partitioned[1].fields[0].name).toEqual('model');
+    expect(partitioned[1].fields[1].name).toEqual('region');
+    expect(partitioned[1].fields[2].name).toEqual('status');
+    expect(partitioned[1].fields[0].values.toArray()).toEqual(['E2', 'E3']);
+    expect(partitioned[1].fields[1].values.toArray()).toEqual(['Europe', 'Europe']);
+    expect(partitioned[1].fields[2].values.toArray()).toEqual(['FAIL', 'FAIL']);
+
+    expect(partitioned[2].length).toEqual(2);
+    expect(partitioned[2].name).toEqual('China OK');
+    expect(partitioned[2].fields[0].name).toEqual('model');
+    expect(partitioned[2].fields[1].name).toEqual('region');
+    expect(partitioned[2].fields[2].name).toEqual('status');
+    expect(partitioned[2].fields[0].values.toArray()).toEqual(['C1', 'C2']);
+    expect(partitioned[2].fields[1].values.toArray()).toEqual(['China', 'China']);
+    expect(partitioned[2].fields[2].values.toArray()).toEqual(['OK', 'OK']);
+
+    expect(partitioned[3].length).toEqual(1);
+    expect(partitioned[3].name).toEqual('China FAIL');
+    expect(partitioned[3].fields[0].name).toEqual('model');
+    expect(partitioned[3].fields[1].name).toEqual('region');
+    expect(partitioned[3].fields[2].name).toEqual('status');
+    expect(partitioned[3].fields[0].values.toArray()).toEqual(['C3']);
+    expect(partitioned[3].fields[1].values.toArray()).toEqual(['China']);
+    expect(partitioned[3].fields[2].values.toArray()).toEqual(['FAIL']);
+  });
+
+  it('should partition by multiple fields with custom frame naming {withFields: true}', () => {
+    const source = [
+      toDataFrame({
+        name: 'XYZ',
+        refId: 'A',
+        fields: [
+          { name: 'model', type: FieldType.string, values: ['E1', 'E2', 'C1', 'E3', 'C2', 'C3'] },
+          { name: 'region', type: FieldType.string, values: ['Europe', 'Europe', 'China', 'Europe', 'China', 'China'] },
+          { name: 'status', type: FieldType.string, values: ['OK', 'FAIL', 'OK', 'FAIL', 'OK', 'FAIL'] },
+        ],
+      }),
+    ];
+
+    const config: PartitionByValuesTransformerOptions = {
+      fields: ['region', 'status'],
+      naming: {
+        withFields: true,
+      },
+    };
+
+    let partitioned = partitionByValuesTransformer.transformer(config)(source);
+
+    expect(partitioned[0].name).toEqual('region=Europe status=OK');
+    expect(partitioned[1].name).toEqual('region=Europe status=FAIL');
+    expect(partitioned[2].name).toEqual('region=China status=OK');
+    expect(partitioned[3].name).toEqual('region=China status=FAIL');
+  });
+
+  it('should partition by multiple fields with custom frame naming {append: true}', () => {
+    const source = [
+      toDataFrame({
+        name: 'XYZ',
+        refId: 'A',
+        fields: [
+          { name: 'model', type: FieldType.string, values: ['E1', 'E2', 'C1', 'E3', 'C2', 'C3'] },
+          { name: 'region', type: FieldType.string, values: ['Europe', 'Europe', 'China', 'Europe', 'China', 'China'] },
+          { name: 'status', type: FieldType.string, values: ['OK', 'FAIL', 'OK', 'FAIL', 'OK', 'FAIL'] },
+        ],
+      }),
+    ];
+
+    const config: PartitionByValuesTransformerOptions = {
+      fields: ['region', 'status'],
+      naming: {
+        append: true,
+      },
+    };
+
+    let partitioned = partitionByValuesTransformer.transformer(config)(source);
+
+    expect(partitioned[0].name).toEqual('XYZ Europe OK');
+    expect(partitioned[1].name).toEqual('XYZ Europe FAIL');
+    expect(partitioned[2].name).toEqual('XYZ China OK');
+    expect(partitioned[3].name).toEqual('XYZ China FAIL');
+  });
+
+  it('should partition by multiple fields with custom frame naming {withFields: true, append: true}', () => {
+    const source = [
+      toDataFrame({
+        name: 'XYZ',
+        refId: 'A',
+        fields: [
+          { name: 'model', type: FieldType.string, values: ['E1', 'E2', 'C1', 'E3', 'C2', 'C3'] },
+          { name: 'region', type: FieldType.string, values: ['Europe', 'Europe', 'China', 'Europe', 'China', 'China'] },
+          { name: 'status', type: FieldType.string, values: ['OK', 'FAIL', 'OK', 'FAIL', 'OK', 'FAIL'] },
+        ],
+      }),
+    ];
+
+    const config: PartitionByValuesTransformerOptions = {
+      fields: ['region', 'status'],
+      naming: {
+        withFields: true,
+        append: true,
+      },
+    };
+
+    let partitioned = partitionByValuesTransformer.transformer(config)(source);
+
+    expect(partitioned[0].name).toEqual('XYZ region=Europe status=OK');
+    expect(partitioned[1].name).toEqual('XYZ region=Europe status=FAIL');
+    expect(partitioned[2].name).toEqual('XYZ region=China status=OK');
+    expect(partitioned[3].name).toEqual('XYZ region=China status=FAIL');
+  });
+});

public/app/features/transformers/partitionByValues/partitionByValues.ts

@@ -0,0 +1,112 @@
+import { map } from 'rxjs';
+
+import {
+  ArrayVector,
+  DataFrame,
+  DataTransformerID,
+  SynchronousDataTransformerInfo,
+  getFieldMatcher,
+} from '@grafana/data';
+import { getMatcherConfig } from '@grafana/data/src/transformations/transformers/filterByName';
+import { noopTransformer } from '@grafana/data/src/transformations/transformers/noop';
+
+import { partition } from './partition';
+
+export interface FrameNamingOptions {
+  /** whether to append to existing frame name, false -> replace */
+  append?: boolean; // false
+  /** whether to include discriminator field names, e.g. true -> Region=Europe Profession=Chef, false -> 'Europe Chef'  */
+  withFields?: boolean; // false
+  /** name/value separator, e.g. '=' in 'Region=Europe' */
+  separator1?: string;
+  /** name/value pair separator, e.g. ' ' in 'Region=Europe Profession=Chef' */
+  separator2?: string;
+}
+
+const defaultFrameNameOptions: FrameNamingOptions = {
+  append: false,
+  withFields: false,
+  separator1: '=',
+  separator2: ' ',
+};
+
+export interface PartitionByValuesTransformerOptions {
+  /** field names whose values should be used as discriminator keys (typically enum fields) */
+  fields: string[];
+  /** how the split frames' names should be suffixed (ends up as field prefixes) */
+  naming?: FrameNamingOptions;
+}
+
+function buildFrameName(opts: FrameNamingOptions, names: string[], values: unknown[]): string {
+  return names
+    .map((name, i) => (opts.withFields ? `${name}${opts.separator1}${values[i]}` : values[i]))
+    .join(opts.separator2);
+}
+
+export const partitionByValuesTransformer: SynchronousDataTransformerInfo<PartitionByValuesTransformerOptions> = {
+  id: DataTransformerID.partitionByValues,
+  name: 'Partition by values',
+  description: `Splits a one-frame dataset into multiple series discriminated by unique/enum values in one or more fields.`,
+  defaultOptions: {},
+
+  operator: (options) => (source) =>
+    source.pipe(map((data) => partitionByValuesTransformer.transformer(options)(data))),
+
+  transformer: (options: PartitionByValuesTransformerOptions) => {
+    const matcherConfig = getMatcherConfig({ names: options.fields });
+
+    if (!matcherConfig) {
+      return noopTransformer.transformer({});
+    }
+
+    const matcher = getFieldMatcher(matcherConfig);
+
+    return (data: DataFrame[]) => {
+      if (!data.length) {
+        return data;
+      }
+
+      const frame = data[0];
+      const keyFields = frame.fields.filter((f) => matcher!(f, frame, data))!;
+      const keyFieldsVals = keyFields.map((f) => f.values.toArray());
+      const names = keyFields.map((f) => f.name);
+
+      const frameNameOpts = {
+        ...defaultFrameNameOptions,
+        ...options.naming,
+      };
+
+      return partition(keyFieldsVals).map((idxs: number[]) => {
+        let name = buildFrameName(
+          frameNameOpts,
+          names,
+          keyFields.map((f, i) => keyFieldsVals[i][idxs[0]])
+        );
+
+        if (options.naming?.append && frame.name) {
+          name = `${frame.name} ${name}`;
+        }
+
+        return {
+          ...frame,
+          name,
+          length: idxs.length,
+          fields: frame.fields.map((f) => {
+            const vals = f.values.toArray();
+            const vals2 = Array(idxs.length);
+
+            for (let i = 0; i < idxs.length; i++) {
+              vals2[i] = vals[idxs[i]];
+            }
+
+            return {
+              ...f,
+              state: undefined,
+              values: new ArrayVector(vals2),
+            };
+          }),
+        };
+      });
+    };
+  },
+};

public/app/features/transformers/standardTransformers.ts

@@ -23,6 +23,7 @@ import { sortByTransformRegistryItem } from './editors/SortByTransformerEditor';
 import { extractFieldsTransformRegistryItem } from './extractFields/ExtractFieldsTransformerEditor';
 import { joinByLabelsTransformRegistryItem } from './joinByLabels/JoinByLabelsTransformerEditor';
 import { fieldLookupTransformRegistryItem } from './lookupGazetteer/FieldLookupTransformerEditor';
+import { partitionByValuesTransformRegistryItem } from './partitionByValues/PartitionByValuesEditor';
 import { prepareTimeseriesTransformerRegistryItem } from './prepareTimeSeries/PrepareTimeSeriesEditor';
 import { rowsToFieldsTransformRegistryItem } from './rowsToFields/RowsToFieldsTransformerEditor';
 import { spatialTransformRegistryItem } from './spatial/SpatialTransformerEditor';
@@ -55,5 +56,6 @@ export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> =
     groupingToMatrixTransformRegistryItem,
     limitTransformRegistryItem,
     joinByLabelsTransformRegistryItem,
+    partitionByValuesTransformRegistryItem,
   ];
 };