Transforms: Add Join by label transformation (#52670)

packages/grafana-data/src/transformations/transformers/ids.ts

@@ -30,6 +30,7 @@ export enum DataTransformerID {
   fieldLookup = 'fieldLookup',
   heatmap = 'heatmap',
   spatial = 'spatial',
+  joinByLabels = 'joinByLabels',
   extractFields = 'extractFields',
   groupingToMatrix = 'groupingToMatrix',
 }

public/app/features/transformers/joinByLabels/JoinByLabelsTransformerEditor.tsx

@@ -0,0 +1,164 @@
+import React, { useMemo } from 'react';
+
+import { PluginState, SelectableValue, TransformerRegistryItem, TransformerUIProps } from '@grafana/data';
+import { Alert, HorizontalGroup, InlineField, InlineFieldRow, Select, ValuePicker } from '@grafana/ui';
+
+import { getDistinctLabels } from '../utils';
+
+import { joinByLabelsTransformer, JoinByLabelsTransformOptions } from './joinByLabels';
+
+export interface Props extends TransformerUIProps<JoinByLabelsTransformOptions> {}
+
+export function JoinByLabelsTransformerEditor({ input, options, onChange }: Props) {
+  const info = useMemo(() => {
+    let warn: React.ReactNode = undefined;
+    const distinct = getDistinctLabels(input);
+    const valueOptions = Array.from(distinct).map((value) => ({ label: value, value }));
+    let valueOption = valueOptions.find((v) => v.value === options.value);
+    if (!valueOption && options.value) {
+      valueOption = { label: `${options.value} (not found)`, value: options.value };
+      valueOptions.push(valueOption);
+    }
+
+    if (!input.length) {
+      warn = <Alert title="No input found">No input (or labels) found</Alert>;
+    } else if (distinct.size === 0) {
+      warn = <Alert title="No labels found">The input does not contain any labels</Alert>;
+    }
+
+    // Show the selected values
+    distinct.delete(options.value);
+    const joinOptions = Array.from(distinct).map((value) => ({ label: value, value }));
+
+    let addOptions = joinOptions;
+    const hasJoin = Boolean(options.join?.length);
+    let addText = 'Join';
+    if (hasJoin) {
+      addOptions = joinOptions.filter((v) => !options.join!.includes(v.value));
+    } else {
+      addText = joinOptions.map((v) => v.value).join(', '); // all the fields
+    }
+
+    return { warn, valueOptions, valueOption, joinOptions, addOptions, addText, hasJoin, key: Date.now() };
+  }, [options, input]);
+
+  const updateJoinValue = (idx: number, value?: string) => {
+    if (!options.join) {
+      return; // nothing to do
+    }
+
+    const join = options.join.slice();
+    if (!value) {
+      join.splice(idx, 1);
+      if (!join.length) {
+        onChange({ ...options, join: undefined });
+        return;
+      }
+    } else {
+      join[idx] = value;
+    }
+
+    // Remove duplicates and the value field
+    const t = new Set(join);
+    if (options.value) {
+      t.delete(options.value);
+    }
+    onChange({ ...options, join: Array.from(t) });
+  };
+
+  const addJoin = (sel: SelectableValue<string>) => {
+    const v = sel?.value;
+    if (!v) {
+      return;
+    }
+    const join = options.join ? options.join.slice() : [];
+    join.push(v);
+    onChange({ ...options, join });
+  };
+
+  const labelWidth = 10;
+  const noOptionsMessage = 'No labels found';
+
+  return (
+    <div>
+      {info.warn}
+
+      <InlineFieldRow>
+        <InlineField
+          error="required"
+          invalid={!Boolean(options.value?.length)}
+          label={'Value'}
+          labelWidth={labelWidth}
+          tooltip="Select the label indicating the values name"
+        >
+          <Select
+            options={info.valueOptions}
+            value={info.valueOption}
+            onChange={(v) => onChange({ ...options, value: v.value! })}
+            noOptionsMessage={noOptionsMessage}
+          />
+        </InlineField>
+      </InlineFieldRow>
+      {info.hasJoin ? (
+        options.join!.map((v, idx) => (
+          <InlineFieldRow key={v + idx}>
+            <InlineField
+              label={'Join'}
+              labelWidth={labelWidth}
+              error="Unable to join by the value label"
+              invalid={v === options.value}
+            >
+              <HorizontalGroup>
+                <Select
+                  options={info.joinOptions}
+                  value={info.joinOptions.find((o) => o.value === v)}
+                  isClearable={true}
+                  onChange={(v) => updateJoinValue(idx, v?.value)}
+                  noOptionsMessage={noOptionsMessage}
+                />
+                {Boolean(info.addOptions.length && idx === options.join!.length - 1) && (
+                  <ValuePicker
+                    icon="plus"
+                    label={''}
+                    options={info.addOptions}
+                    onChange={addJoin}
+                    variant="secondary"
+                  />
+                )}
+              </HorizontalGroup>
+            </InlineField>
+          </InlineFieldRow>
+        ))
+      ) : (
+        <>
+          {Boolean(info.addOptions.length) && (
+            <InlineFieldRow>
+              <InlineField label={'Join'} labelWidth={labelWidth}>
+                <Select
+                  options={info.addOptions}
+                  placeholder={info.addText}
+                  onChange={addJoin}
+                  noOptionsMessage={noOptionsMessage}
+                />
+              </InlineField>
+            </InlineFieldRow>
+          )}
+        </>
+      )}
+    </div>
+  );
+}
+
+export const joinByLabelsTransformRegistryItem: TransformerRegistryItem<JoinByLabelsTransformOptions> = {
+  id: joinByLabelsTransformer.id,
+  editor: JoinByLabelsTransformerEditor,
+  transformation: joinByLabelsTransformer,
+  name: joinByLabelsTransformer.name,
+  description: joinByLabelsTransformer.description,
+  state: PluginState.beta,
+  //   help: `
+  // ### Use cases
+
+  // This transforms labeled results into a table
+  // `,
+};

public/app/features/transformers/joinByLabels/joinByLabels.test.ts

@@ -0,0 +1,142 @@
+import { toDataFrame, FieldType, DataFrame } from '@grafana/data';
+
+import { joinByLabels } from './joinByLabels';
+
+describe('Join by labels', () => {
+  it('Simple join', () => {
+    const input = [
+      toDataFrame({
+        fields: [
+          { name: 'Time', type: FieldType.time, values: [1, 2] },
+          {
+            name: 'Value',
+            type: FieldType.number,
+            values: [10, 200],
+            labels: { what: 'Temp', cluster: 'A', job: 'J1' },
+          },
+        ],
+      }),
+      toDataFrame({
+        fields: [
+          { name: 'Time', type: FieldType.time, values: [1, 2] },
+          {
+            name: 'Value',
+            type: FieldType.number,
+            values: [10, 200],
+            labels: { what: 'Temp', cluster: 'B', job: 'J1' },
+          },
+        ],
+      }),
+      toDataFrame({
+        fields: [
+          { name: 'Time', type: FieldType.time, values: [22, 28] },
+          {
+            name: 'Value',
+            type: FieldType.number,
+            values: [22, 77],
+            labels: { what: 'Speed', cluster: 'B', job: 'J1' },
+          },
+        ],
+      }),
+    ];
+
+    const result = joinByLabels(
+      {
+        value: 'what',
+      },
+      input
+    );
+    expect(toRowsSnapshow(result)).toMatchInlineSnapshot(`
+      Object {
+        "columns": Array [
+          "cluster",
+          "job",
+          "Temp",
+          "Speed",
+        ],
+        "rows": Array [
+          Array [
+            "A",
+            "J1",
+            10,
+            undefined,
+          ],
+          Array [
+            "A",
+            "J1",
+            200,
+            undefined,
+          ],
+          Array [
+            "B",
+            "J1",
+            10,
+            22,
+          ],
+          Array [
+            "B",
+            "J1",
+            200,
+            77,
+          ],
+        ],
+      }
+    `);
+  });
+
+  it('Error handling (no labels)', () => {
+    const input = [
+      toDataFrame({
+        fields: [
+          { name: 'Time', type: FieldType.time, values: [1, 2] },
+          {
+            name: 'Value',
+            type: FieldType.number,
+            values: [10, 200],
+          },
+        ],
+      }),
+    ];
+
+    const result = joinByLabels(
+      {
+        value: 'what',
+      },
+      input
+    );
+    expect(result).toMatchInlineSnapshot(`
+      Object {
+        "fields": Array [
+          Object {
+            "config": Object {},
+            "name": "Error",
+            "type": "string",
+            "values": Array [
+              "No labels in result",
+            ],
+          },
+        ],
+        "length": 0,
+        "meta": Object {
+          "notices": Array [
+            Object {
+              "severity": "error",
+              "text": "No labels in result",
+            },
+          ],
+        },
+      }
+    `);
+  });
+});
+
+function toRowsSnapshow(frame: DataFrame) {
+  const columns = frame.fields.map((f) => f.name);
+  const rows = frame.fields[0].values.toArray().map((v, idx) => {
+    return frame.fields.map((f) => f.values.get(idx));
+  });
+  return {
+    columns,
+    rows,
+  };
+}

public/app/features/transformers/joinByLabels/joinByLabels.ts

@@ -0,0 +1,144 @@
+import { map } from 'rxjs/operators';
+
+import {
+  ArrayVector,
+  DataFrame,
+  DataTransformerID,
+  Field,
+  FieldType,
+  SynchronousDataTransformerInfo,
+} from '@grafana/data';
+
+import { getDistinctLabels } from '../utils';
+
+export interface JoinByLabelsTransformOptions {
+  value: string; // something must be defined
+  join?: string[];
+}
+
+export const joinByLabelsTransformer: SynchronousDataTransformerInfo<JoinByLabelsTransformOptions> = {
+  id: DataTransformerID.joinByLabels,
+  name: 'Join by labels',
+  description: 'Flatten labeled results into a table joined by labels',
+  defaultOptions: {},
+
+  operator: (options) => (source) => source.pipe(map((data) => joinByLabelsTransformer.transformer(options)(data))),
+
+  transformer: (options: JoinByLabelsTransformOptions) => {
+    return (data: DataFrame[]) => {
+      if (!data || !data.length) {
+        return data;
+      }
+      return [joinByLabels(options, data)];
+    };
+  },
+};
+
+interface JoinValues {
+  keys: string[];
+  values: Record<string, number[]>;
+}
+
+export function joinByLabels(options: JoinByLabelsTransformOptions, data: DataFrame[]): DataFrame {
+  if (!options.value?.length) {
+    return getErrorFrame('No value labele configured');
+  }
+  const distinctLabels = getDistinctLabels(data);
+  if (distinctLabels.size < 1) {
+    return getErrorFrame('No labels in result');
+  }
+  if (!distinctLabels.has(options.value)) {
+    return getErrorFrame('Value label not found');
+  }
+
+  let join = options.join?.length ? options.join : Array.from(distinctLabels);
+  join = join.filter((f) => f !== options.value);
+
+  const names = new Set<string>();
+  const found = new Map<string, JoinValues>();
+  const inputFields: Record<string, Field> = {};
+  for (const frame of data) {
+    for (const field of frame.fields) {
+      if (field.labels && field.type !== FieldType.time) {
+        const keys = join.map((v) => field.labels![v]);
+        const key = keys.join(',');
+        let item = found.get(key);
+        if (!item) {
+          item = {
+            keys,
+            values: {},
+          };
+          found.set(key, item);
+        }
+        const name = field.labels[options.value];
+        const vals = field.values.toArray();
+        const old = item.values[name];
+        if (old) {
+          item.values[name] = old.concat(vals);
+        } else {
+          item.values[name] = vals;
+        }
+        if (!inputFields[name]) {
+          inputFields[name] = field; // keep the config
+        }
+        names.add(name);
+      }
+    }
+  }
+
+  const allNames = Array.from(names);
+  const joinValues = join.map((): string[] => []);
+  const nameValues = allNames.map((): number[] => []);
+
+  for (const item of found.values()) {
+    let valueOffset = -1;
+    let done = false;
+    while (!done) {
+      valueOffset++;
+      done = true;
+      for (let i = 0; i < join.length; i++) {
+        joinValues[i].push(item.keys[i]);
+      }
+      for (let i = 0; i < allNames.length; i++) {
+        const name = allNames[i];
+        const values = item.values[name] ?? [];
+        nameValues[i].push(values[valueOffset]);
+        if (values.length > valueOffset + 1) {
+          done = false;
+        }
+      }
+    }
+  }
+
+  const frame: DataFrame = { fields: [], length: nameValues[0].length };
+  for (let i = 0; i < join.length; i++) {
+    frame.fields.push({
+      name: join[i],
+      config: {},
+      type: FieldType.string,
+      values: new ArrayVector(joinValues[i]),
+    });
+  }
+
+  for (let i = 0; i < allNames.length; i++) {
+    const old = inputFields[allNames[i]];
+    frame.fields.push({
+      name: allNames[i],
+      config: old.config ?? {},
+      type: old.type ?? FieldType.number,
+      values: new ArrayVector(nameValues[i]),
+    });
+  }
+
+  return frame;
+}
+
+function getErrorFrame(text: string): DataFrame {
+  return {
+    meta: {
+      notices: [{ severity: 'error', text }],
+    },
+    fields: [{ name: 'Error', type: FieldType.string, config: {}, values: new ArrayVector([text]) }],
+    length: 0,
+  };
+}

public/app/features/transformers/standardTransformers.ts

@@ -20,6 +20,7 @@ import { seriesToFieldsTransformerRegistryItem } from './editors/SeriesToFieldsT
 import { seriesToRowsTransformerRegistryItem } from './editors/SeriesToRowsTransformerEditor';
 import { sortByTransformRegistryItem } from './editors/SortByTransformerEditor';
 import { extractFieldsTransformRegistryItem } from './extractFields/ExtractFieldsTransformerEditor';
+import { joinByLabelsTransformRegistryItem } from './joinByLabels/JoinByLabelsTransformerEditor';
 import { fieldLookupTransformRegistryItem } from './lookupGazetteer/FieldLookupTransformerEditor';
 import { prepareTimeseriesTransformerRegistryItem } from './prepareTimeSeries/PrepareTimeSeriesEditor';
 import { rowsToFieldsTransformRegistryItem } from './rowsToFields/RowsToFieldsTransformerEditor';
@@ -51,5 +52,6 @@ export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> =
     extractFieldsTransformRegistryItem,
     heatmapTransformRegistryItem,
     groupingToMatrixTransformRegistryItem,
+    joinByLabelsTransformRegistryItem,
   ];
 };

public/app/features/transformers/utils.ts

@@ -23,3 +23,17 @@ export function useAllFieldNamesFromDataFrames(input: DataFrame[]): string[] {
     );
   }, [input]);
 }
+
+export function getDistinctLabels(input: DataFrame[]): Set<string> {
+  const distinct = new Set<string>();
+  for (const frame of input) {
+    for (const field of frame.fields) {
+      if (field.labels) {
+        for (const k of Object.keys(field.labels)) {
+          distinct.add(k);
+        }
+      }
+    }
+  }
+  return distinct;
+}