Transformations: improve the reduce transformer (#27875)

2025-02-25 18:55:37 -06:00 · 2020-10-14 08:27:58 -07:00 · 2020-10-14 08:27:58 -07:00 · f8e0adb168
commit f8e0adb168
parent db071e4939
5 changed files with 272 additions and 106 deletions
--- a/packages/grafana-data/src/transformations/matchers/predicates.ts
+++ b/packages/grafana-data/src/transformations/matchers/predicates.ts
@ -1,4 +1,4 @@
-import { Field, DataFrame } from '../../types/dataFrame';
+import { Field, DataFrame, FieldType } from '../../types/dataFrame';
 import { MatcherID } from './ids';
 import { getFieldMatcher, fieldMatchers, getFrameMatchers, frameMatchers } from '../matchers';
 import { FieldMatcherInfo, MatcherConfig, FrameMatcherInfo } from '../../types/transformations';
@ -191,6 +191,10 @@ export const neverFieldMatcher = (field: Field) => {
  return false;
 };

+export const notTimeFieldMatcher = (field: Field) => {
+  return field.type !== FieldType.time;
+};
+
 export const neverFrameMatcher = (frame: DataFrame) => {
  return false;
 };
--- a/packages/grafana-data/src/transformations/transformers/reduce.test.ts
+++ b/packages/grafana-data/src/transformations/transformers/reduce.test.ts
@ -2,11 +2,13 @@ import { ReducerID } from '../fieldReducer';
 import { DataTransformerID } from './ids';
 import { toDataFrame } from '../../dataframe/processDataFrame';
 import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
-import { reduceTransformer } from './reduce';
+import { reduceFields, reduceTransformer } from './reduce';
 import { transformDataFrame } from '../transformDataFrame';
 import { Field, FieldType } from '../../types';
 import { ArrayVector } from '../../vector';
 import { observableTester } from '../../utils/tests/observableTester';
+import { notTimeFieldMatcher } from '../matchers/predicates';
+import { DataFrameView } from '../../dataframe';

 const seriesAWithSingleField = toDataFrame({
  name: 'A',
@ -254,4 +256,31 @@ describe('Reducer Transformer', () => {
      done,
    });
  });
+
+  it('reduces fields with single calculator', () => {
+    const frames = reduceFields(
+      [seriesAWithSingleField, seriesAWithMultipleFields], // data
+      notTimeFieldMatcher, // skip time fields
+      [ReducerID.last] // only one
+    );
+
+    // Convert each frame to a structure with the same fields
+    expect(frames.length).toEqual(2);
+    expect(frames[0].length).toEqual(1);
+    expect(frames[1].length).toEqual(1);
+
+    const view0 = new DataFrameView<any>(frames[0]);
+    const view1 = new DataFrameView<any>(frames[1]);
+    expect({ ...view0.get(0) }).toMatchInlineSnapshot(`
+      Object {
+        "temperature": 6,
+      }
+    `);
+    expect({ ...view1.get(0) }).toMatchInlineSnapshot(`
+      Object {
+        "humidity": 10000.6,
+        "temperature": 6,
+      }
+    `);
+  });
 });
--- a/packages/grafana-data/src/transformations/transformers/reduce.ts
+++ b/packages/grafana-data/src/transformations/transformers/reduce.ts
@ -3,19 +3,24 @@ import { map } from 'rxjs/operators';
 import { DataTransformerID } from './ids';
 import { DataTransformerInfo, MatcherConfig } from '../../types/transformations';
 import { fieldReducers, reduceField, ReducerID } from '../fieldReducer';
-import { alwaysFieldMatcher } from '../matchers/predicates';
+import { alwaysFieldMatcher, notTimeFieldMatcher } from '../matchers/predicates';
 import { DataFrame, Field, FieldType } from '../../types/dataFrame';
 import { ArrayVector } from '../../vector/ArrayVector';
 import { KeyValue } from '../../types/data';
 import { guessFieldTypeForField } from '../../dataframe/processDataFrame';
 import { getFieldMatcher } from '../matchers';
-import { FieldMatcherID } from '../matchers/ids';
-import { filterFieldsTransformer } from './filter';
 import { getFieldDisplayName } from '../../field';
+import { FieldMatcher } from '../../types/transformations';

+export enum ReduceTransformerMode {
+  SeriesToRows = 'seriesToRows', // default
+  ReduceFields = 'reduceFields', // same structure, add additional row for each type
+}
 export interface ReduceTransformerOptions {
  reducers: ReducerID[];
  fields?: MatcherConfig; // Assume all fields
+  mode?: ReduceTransformerMode;
+  includeTimeField?: boolean;
 }

 export const reduceTransformer: DataTransformerInfo<ReduceTransformerOptions> = {
@ -33,89 +38,111 @@ export const reduceTransformer: DataTransformerInfo<ReduceTransformerOptions> =
  operator: options => source =>
    source.pipe(
      map(data => {
-        const matcher = options.fields ? getFieldMatcher(options.fields) : alwaysFieldMatcher;
-        const calculators = options.reducers && options.reducers.length ? fieldReducers.list(options.reducers) : [];
-        const reducers = calculators.map(c => c.id);
-
-        const processed: DataFrame[] = [];
-
-        for (let seriesIndex = 0; seriesIndex < data.length; seriesIndex++) {
-          const series = data[seriesIndex];
-          const values: ArrayVector[] = [];
-          const fields: Field[] = [];
-          const byId: KeyValue<ArrayVector> = {};
-
-          values.push(new ArrayVector()); // The name
-          fields.push({
-            name: 'Field',
-            type: FieldType.string,
-            values: values[0],
-            config: {},
-          });
-
-          for (const info of calculators) {
-            const vals = new ArrayVector();
-            byId[info.id] = vals;
-            values.push(vals);
-
-            fields.push({
-              name: info.name,
-              type: FieldType.other, // UNKNOWN until after we call the functions
-              values: values[values.length - 1],
-              config: {},
-            });
-          }
-
-          for (let i = 0; i < series.fields.length; i++) {
-            const field = series.fields[i];
-
-            if (field.type === FieldType.time) {
-              continue;
-            }
-
-            if (matcher(field, series, data)) {
-              const results = reduceField({
-                field,
-                reducers,
-              });
-
-              // Update the name list
-              const fieldName = getFieldDisplayName(field, series, data);
-
-              values[0].buffer.push(fieldName);
-
-              for (const info of calculators) {
-                const v = results[info.id];
-                byId[info.id].buffer.push(v);
-              }
-            }
-          }
-
-          for (const f of fields) {
-            const t = guessFieldTypeForField(f);
-
-            if (t) {
-              f.type = t;
-            }
-          }
-
-          processed.push({
-            ...series, // Same properties, different fields
-            fields,
-            length: values[0].length,
-          });
+        if (!options?.reducers?.length) {
+          return data; // nothing selected
        }

-        return processed;
-      }),
-      filterFieldsTransformer.operator({ exclude: { id: FieldMatcherID.time } }),
-      map(mergeResults)
+        const matcher = options.fields
+          ? getFieldMatcher(options.fields)
+          : options.includeTimeField && options.mode === ReduceTransformerMode.ReduceFields
+          ? alwaysFieldMatcher
+          : notTimeFieldMatcher;
+
+        // Collapse all matching fields into a single row
+        if (options.mode === ReduceTransformerMode.ReduceFields) {
+          return reduceFields(data, matcher, options.reducers);
+        }
+
+        // Add a row for each series
+        const res = reduceSeriesToRows(data, matcher, options.reducers);
+        return res ? [res] : [];
+      })
    ),
 };

-const mergeResults = (data: DataFrame[]) => {
-  if (data.length <= 1) {
-    return data;
+/**
+ * @internal only exported for testing
+ */
+export function reduceSeriesToRows(
+  data: DataFrame[],
+  matcher: FieldMatcher,
+  reducerId: ReducerID[]
+): DataFrame | undefined {
+  const calculators = fieldReducers.list(reducerId);
+  const reducers = calculators.map(c => c.id);
+  const processed: DataFrame[] = [];
+
+  for (const series of data) {
+    const values: ArrayVector[] = [];
+    const fields: Field[] = [];
+    const byId: KeyValue<ArrayVector> = {};
+
+    values.push(new ArrayVector()); // The name
+    fields.push({
+      name: 'Field',
+      type: FieldType.string,
+      values: values[0],
+      config: {},
+    });
+
+    for (const info of calculators) {
+      const vals = new ArrayVector();
+      byId[info.id] = vals;
+      values.push(vals);
+
+      fields.push({
+        name: info.name,
+        type: FieldType.other, // UNKNOWN until after we call the functions
+        values: values[values.length - 1],
+        config: {},
+      });
+    }
+
+    for (let i = 0; i < series.fields.length; i++) {
+      const field = series.fields[i];
+
+      if (matcher(field, series, data)) {
+        const results = reduceField({
+          field,
+          reducers,
+        });
+
+        // Update the name list
+        const fieldName = getFieldDisplayName(field, series, data);
+
+        values[0].buffer.push(fieldName);
+
+        for (const info of calculators) {
+          const v = results[info.id];
+          byId[info.id].buffer.push(v);
+        }
+      }
+    }
+
+    for (const f of fields) {
+      const t = guessFieldTypeForField(f);
+
+      if (t) {
+        f.type = t;
+      }
+    }
+
+    processed.push({
+      ...series, // Same properties, different fields
+      fields,
+      length: values[0].length,
+    });
+  }
+
+  return mergeResults(processed);
+}
+
+/**
+ * @internal only exported for testing
+ */
+export function mergeResults(data: DataFrame[]): DataFrame | undefined {
+  if (!data?.length) {
+    return undefined;
  }

  const baseFrame = data[0];
@ -138,6 +165,50 @@ const mergeResults = (data: DataFrame[]) => {

  baseFrame.name = undefined;
  baseFrame.length = baseFrame.fields[0].values.length;
+  return baseFrame;
+}

-  return [baseFrame];
-};
+/**
+ * @internal -- only exported for testing
+ */
+export function reduceFields(data: DataFrame[], matcher: FieldMatcher, reducerId: ReducerID[]): DataFrame[] {
+  const calculators = fieldReducers.list(reducerId);
+  const reducers = calculators.map(c => c.id);
+  const processed: DataFrame[] = [];
+
+  for (const series of data) {
+    const fields: Field[] = [];
+    for (const field of series.fields) {
+      if (matcher(field, series, data)) {
+        const results = reduceField({
+          field,
+          reducers,
+        });
+        for (const reducer of reducers) {
+          const value = results[reducer];
+          const copy = {
+            ...field,
+            values: new ArrayVector([value]),
+          };
+          copy.state = undefined;
+          if (reducers.length > 1) {
+            if (!copy.labels) {
+              copy.labels = {};
+            }
+            copy.labels['reducer'] = fieldReducers.get(reducer).name;
+          }
+          fields.push(copy);
+        }
+      }
+    }
+    if (fields.length) {
+      processed.push({
+        ...series,
+        fields,
+        length: 1, // always one row
+      });
+    }
+  }
+
+  return processed;
+}
--- a/packages/grafana-e2e-selectors/src/selectors/components.ts
+++ b/packages/grafana-e2e-selectors/src/selectors/components.ts
@ -110,6 +110,7 @@ export const Components = {
  },
  Transforms: {
    Reduce: {
+      modeLabel: 'Transform mode label',
      calculationsLabel: 'Transform calculations label',
    },
  },
--- a/public/app/core/components/TransformersUI/ReduceTransformerEditor.tsx
+++ b/public/app/core/components/TransformersUI/ReduceTransformerEditor.tsx
@ -1,14 +1,15 @@
-import React from 'react';
-import { StatsPicker } from '@grafana/ui';
+import React, { useCallback } from 'react';
+import { StatsPicker, Select, LegacyForms } from '@grafana/ui';
 import {
  DataTransformerID,
  ReducerID,
  standardTransformers,
  TransformerRegistyItem,
  TransformerUIProps,
+  SelectableValue,
 } from '@grafana/data';

-import { ReduceTransformerOptions } from '@grafana/data/src/transformations/transformers/reduce';
+import { ReduceTransformerOptions, ReduceTransformerMode } from '@grafana/data/src/transformations/transformers/reduce';
 import { selectors } from '@grafana/e2e-selectors';

 // TODO:  Minimal implementation, needs some <3
@ -16,27 +17,87 @@ export const ReduceTransformerEditor: React.FC<TransformerUIProps<ReduceTransfor
  options,
  onChange,
 }) => {
+  const modes: Array<SelectableValue<ReduceTransformerMode>> = [
+    {
+      label: 'Series to rows',
+      value: ReduceTransformerMode.SeriesToRows,
+      description: 'Create a table with one row for each series value',
+    },
+    {
+      label: 'Reduce fields',
+      value: ReduceTransformerMode.ReduceFields,
+      description: 'Collapse each field into a single value',
+    },
+  ];
+
+  const onSelectMode = useCallback(
+    (value: SelectableValue<ReduceTransformerMode>) => {
+      const mode = value.value!;
+      onChange({
+        ...options,
+        mode,
+        includeTimeField: mode === ReduceTransformerMode.ReduceFields ? !!options.includeTimeField : false,
+      });
+    },
+    [onChange, options]
+  );
+
+  const onToggleTime = useCallback(() => {
+    onChange({
+      ...options,
+      includeTimeField: !options.includeTimeField,
+    });
+  }, [onChange, options]);
+
  return (
-    <div className="gf-form-inline">
-      <div className="gf-form gf-form--grow">
-        <div className="gf-form-label width-8" aria-label={selectors.components.Transforms.Reduce.calculationsLabel}>
-          Calculations
+    <>
+      <div>
+        <div className="gf-form gf-form--grow">
+          <div className="gf-form-label width-8" aria-label={selectors.components.Transforms.Reduce.modeLabel}>
+            Mode
+          </div>
+          <Select
+            options={modes}
+            value={modes.find(v => v.value === options.mode) || modes[0]}
+            onChange={onSelectMode}
+            menuPlacement="bottom"
+            className="flex-grow-1"
+          />
        </div>
-        <StatsPicker
-          className="flex-grow-1"
-          placeholder="Choose Stat"
-          allowMultiple
-          stats={options.reducers || []}
-          onChange={stats => {
-            onChange({
-              ...options,
-              reducers: stats as ReducerID[],
-            });
-          }}
-          menuPlacement="bottom"
-        />
      </div>
-    </div>
+      <div className="gf-form-inline">
+        <div className="gf-form gf-form--grow">
+          <div className="gf-form-label width-8" aria-label={selectors.components.Transforms.Reduce.calculationsLabel}>
+            Calculations
+          </div>
+          <StatsPicker
+            className="flex-grow-1"
+            placeholder="Choose Stat"
+            allowMultiple
+            stats={options.reducers || []}
+            onChange={stats => {
+              onChange({
+                ...options,
+                reducers: stats as ReducerID[],
+              });
+            }}
+            menuPlacement="bottom"
+          />
+        </div>
+      </div>
+      {options.mode === ReduceTransformerMode.ReduceFields && (
+        <div className="gf-form-inline">
+          <div className="gf-form">
+            <LegacyForms.Switch
+              label="Include time"
+              labelClass="width-8"
+              checked={!!options.includeTimeField}
+              onChange={onToggleTime}
+            />
+          </div>
+        </div>
+      )}
+    </>
  );
 };