Transformations: Binary operation on all numbers (#92172)

* Transformations: Binary operation on all numbers

* Handle replaceFields option

* Change left clear out to string

* Handle time field

* Fix filtering

* Update new field names to remove double space

* Add tests

* Add BinaryValue interface and update editor

* Fix initial behavior

* Rollback rendering standards

* Add ctx interpolate

* Fix fixed value variable

* Add function to convert old binary value type

* Update tests for new structures

* Add bullet for all number field option

* baldm0mma/run content build script

* Disable alias control for type matching

docs/sources/panels-visualizations/query-transform-data/transform-data/index.md

@@ -188,6 +188,7 @@ Use this transformation to add a new field calculated from two other fields. Eac
 - **Field name** - Select the names of fields you want to use in the calculation for the new field.
 - **Calculation** - If you select **Reduce row** mode, then the **Calculation** field appears. Click in the field to see a list of calculation choices you can use to create the new field. For information about available calculations, refer to [Calculation types][].
 - **Operation** - If you select **Binary operation** or **Unary operation** mode, then the **Operation** fields appear. These fields allow you to apply basic math operations on values in a single row from selected fields. You can also use numerical values for binary operations.
+  - **All number fields** - Set the left side of a **Binary operation** to apply the calculation to all number fields.
 - **As percentile** - If you select **Row index** mode, then the **As percentile** switch appears. This switch allows you to transform the row index as a percentage of the total number of rows.
 - **Alias** - (Optional) Enter the name of your new field. If you leave this blank, then the field will be named to match the calculation.
 - **Replace all fields** - (Optional) Select this option if you want to hide all other fields and display only your calculated field in the visualization.

packages/grafana-data/src/transformations/transformers/calculateField.test.ts

@@ -7,6 +7,7 @@ import { BinaryOperationID } from '../../utils/binaryOperators';
 import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
 import { UnaryOperationID } from '../../utils/unaryOperators';
 import { ReducerID } from '../fieldReducer';
+import { FieldMatcherID } from '../matchers/ids';
 import { transformDataFrame } from '../transformDataFrame';
 
 import {
@@ -197,6 +198,72 @@ describe('calculateField transformer w/ timeseries', () => {
     });
   });
 
+  it('all numbers + static number', async () => {
+    const cfg = {
+      id: DataTransformerID.calculateField,
+      options: {
+        mode: CalculateFieldMode.BinaryOperation,
+        binary: {
+          left: { matcher: { id: FieldMatcherID.byType, options: FieldType.number } },
+          operator: BinaryOperationID.Add,
+          right: '2',
+        },
+        replaceFields: true,
+      },
+    };
+
+    await expect(transformDataFrame([cfg], [seriesBC])).toEmitValuesWith((received) => {
+      const data = received[0];
+      const filtered = data[0];
+      const rows = new DataFrameView(filtered).toArray();
+      expect(rows).toEqual([
+        {
+          'B + 2': 4,
+          'C + 2': 5,
+          TheTime: 1000,
+        },
+        {
+          'B + 2': 202,
+          'C + 2': 302,
+          TheTime: 2000,
+        },
+      ]);
+    });
+  });
+
+  it('all numbers + field number', async () => {
+    const cfg = {
+      id: DataTransformerID.calculateField,
+      options: {
+        mode: CalculateFieldMode.BinaryOperation,
+        binary: {
+          left: { matcher: { id: FieldMatcherID.byType, options: FieldType.number } },
+          operator: BinaryOperationID.Add,
+          right: 'C',
+        },
+        replaceFields: true,
+      },
+    };
+
+    await expect(transformDataFrame([cfg], [seriesBC])).toEmitValuesWith((received) => {
+      const data = received[0];
+      const filtered = data[0];
+      const rows = new DataFrameView(filtered).toArray();
+      expect(rows).toEqual([
+        {
+          'B + C': 5,
+          'C + C': 6,
+          TheTime: 1000,
+        },
+        {
+          'B + C': 500,
+          'C + C': 600,
+          TheTime: 2000,
+        },
+      ]);
+    });
+  });
+
   it('unary math', async () => {
     const unarySeries = toDataFrame({
       fields: [

packages/grafana-data/src/transformations/transformers/calculateField.ts

@@ -5,7 +5,7 @@ import { getTimeField } from '../../dataframe/processDataFrame';
 import { getFieldDisplayName } from '../../field/fieldState';
 import { NullValueMode } from '../../types/data';
 import { DataFrame, FieldType, Field } from '../../types/dataFrame';
-import { DataTransformerInfo } from '../../types/transformations';
+import { DataTransformContext, DataTransformerInfo } from '../../types/transformations';
 import { BinaryOperationID, binaryOperators } from '../../utils/binaryOperators';
 import { UnaryOperationID, unaryOperators } from '../../utils/unaryOperators';
 import { doStandardCalcs, fieldReducers, ReducerID } from '../fieldReducer';
@@ -58,9 +58,14 @@ export interface UnaryOptions {
 }
 
 export interface BinaryOptions {
-  left: string;
+  left: BinaryValue;
   operator: BinaryOperationID;
-  right: string;
+  right: BinaryValue;
+}
+
+export interface BinaryValue {
+  fixed?: string;
+  matcher?: { id?: FieldMatcherID; options?: string };
 }
 
 interface IndexOptions {
@@ -79,9 +84,9 @@ export const defaultWindowOptions: WindowOptions = {
 };
 
 const defaultBinaryOptions: BinaryOptions = {
-  left: '',
+  left: { fixed: '' },
   operator: BinaryOperationID.Add,
-  right: '',
+  right: { fixed: '' },
 };
 
 const defaultUnaryOptions: UnaryOptions = {
@@ -152,13 +157,62 @@ export const calculateFieldTransformer: DataTransformerInfo<CalculateFieldTransf
             creator = getUnaryCreator(defaults(options.unary, defaultUnaryOptions), data);
             break;
           case CalculateFieldMode.BinaryOperation:
+            const fieldNames: string[] = [];
+            data.map((frame) => {
+              frame.fields.map((field) => {
+                fieldNames.push(field.name);
+              });
+            });
             const binaryOptions = {
-              ...options.binary,
-              left: ctx.interpolate(options.binary?.left!),
-              right: ctx.interpolate(options.binary?.right!),
+              left: checkBinaryValueType(options.binary?.left ?? '', fieldNames),
+              operator: options.binary?.operator ?? defaultBinaryOptions.operator,
+              right: checkBinaryValueType(options.binary?.right ?? '', fieldNames),
             };
+            options.binary = binaryOptions;
+            if (binaryOptions.left?.matcher?.id && binaryOptions.left?.matcher.id === FieldMatcherID.byType) {
+              const fieldType = binaryOptions.left.matcher.options;
+              const operator = binaryOperators.getIfExists(binaryOptions.operator);
+              return data.map((frame) => {
+                const { timeField } = getTimeField(frame);
+                const newFields: Field[] = [];
+                if (timeField && options.timeSeries !== false) {
+                  newFields.push(timeField);
+                }
+                // For each field of type match, apply operator
+                frame.fields.map((field, index) => {
+                  if (!options.replaceFields) {
+                    newFields.push(field);
+                  }
+                  if (field.type === fieldType) {
+                    const left = field.values;
+                    // TODO consolidate common creator logic
+                    const right = findFieldValuesWithNameOrConstant(
+                      frame,
+                      binaryOptions.right ?? defaultBinaryOptions.right,
+                      data,
+                      ctx
+                    );
+                    if (!left || !right || !operator) {
+                      return undefined;
+                    }
 
-            creator = getBinaryCreator(defaults(binaryOptions, defaultBinaryOptions), data);
+                    const arr = new Array(left.length);
+                    for (let i = 0; i < arr.length; i++) {
+                      arr[i] = operator.operation(left[i], right[i]);
+                    }
+                    const newField = {
+                      ...field,
+                      name: `${field.name} ${options.binary?.operator ?? ''} ${options.binary?.right.matcher?.options ?? options.binary?.right.fixed}`,
+                      values: arr,
+                    };
+                    newFields.push(newField);
+                  }
+                });
+                return { ...frame, fields: newFields };
+              });
+            } else {
+              creator = getBinaryCreator(defaults(binaryOptions, defaultBinaryOptions), data, ctx);
+            }
             break;
           case CalculateFieldMode.Index:
             return data.map((frame) => {
@@ -488,23 +542,27 @@ function getReduceRowCreator(options: ReduceOptions, allFrames: DataFrame[]): Va
 
 function findFieldValuesWithNameOrConstant(
   frame: DataFrame,
-  name: string,
-  allFrames: DataFrame[]
+  value: BinaryValue,
+  allFrames: DataFrame[],
+  ctx: DataTransformContext
 ): number[] | undefined {
-  if (!name) {
+  if (!value) {
     return undefined;
   }
 
-  for (const f of frame.fields) {
-    if (name === getFieldDisplayName(f, frame, allFrames)) {
-      if (f.type === FieldType.boolean) {
-        return f.values.map((v) => (v ? 1 : 0));
+  if (value.matcher && value.matcher.id === FieldMatcherID.byName) {
+    const name = ctx.interpolate(value.matcher.options ?? '');
+    for (const f of frame.fields) {
+      if (name === getFieldDisplayName(f, frame, allFrames)) {
+        if (f.type === FieldType.boolean) {
+          return f.values.map((v) => (v ? 1 : 0));
+        }
+        return f.values;
       }
-      return f.values;
     }
   }
 
-  const v = parseFloat(name);
+  const v = parseFloat(value.fixed ?? ctx.interpolate(value.matcher?.options ?? ''));
   if (!isNaN(v)) {
     return new Array(frame.length).fill(v);
   }
@@ -512,12 +570,12 @@ function findFieldValuesWithNameOrConstant(
   return undefined;
 }
 
-function getBinaryCreator(options: BinaryOptions, allFrames: DataFrame[]): ValuesCreator {
+function getBinaryCreator(options: BinaryOptions, allFrames: DataFrame[], ctx: DataTransformContext): ValuesCreator {
   const operator = binaryOperators.getIfExists(options.operator);
 
   return (frame: DataFrame) => {
-    const left = findFieldValuesWithNameOrConstant(frame, options.left, allFrames);
-    const right = findFieldValuesWithNameOrConstant(frame, options.right, allFrames);
+    const left = findFieldValuesWithNameOrConstant(frame, options.left, allFrames, ctx);
+    const right = findFieldValuesWithNameOrConstant(frame, options.right, allFrames, ctx);
     if (!left || !right || !operator) {
       return undefined;
     }
@@ -530,6 +588,24 @@ function getBinaryCreator(options: BinaryOptions, allFrames: DataFrame[]): Value
   };
 }
 
+export function checkBinaryValueType(value: BinaryValue | string, names: string[]): BinaryValue {
+  // Support old binary value structure
+  if (typeof value === 'string') {
+    if (isNaN(Number(value))) {
+      return { matcher: { id: FieldMatcherID.byName, options: value } };
+    } else {
+      // If it's a number, check if matches name, otherwise store as fixed number value
+      if (names.includes(value)) {
+        return { matcher: { id: FieldMatcherID.byName, options: value } };
+      } else {
+        return { fixed: value };
+      }
+    }
+  }
+  // Pass through new BinaryValue structure
+  return value;
+}
+
 function getUnaryCreator(options: UnaryOptions, allFrames: DataFrame[]): ValuesCreator {
   const operator = unaryOperators.getIfExists(options.operator);
 
@@ -577,7 +653,7 @@ export function getNameFromOptions(options: CalculateFieldTransformerOptions) {
     }
     case CalculateFieldMode.BinaryOperation: {
       const { binary } = options;
-      const alias = `${binary?.left ?? ''} ${binary?.operator ?? ''} ${binary?.right ?? ''}`;
+      const alias = `${binary?.left?.matcher?.options ?? binary?.left?.fixed ?? ''} ${binary?.operator ?? ''} ${binary?.right?.matcher?.options ?? binary?.right?.fixed ?? ''}`;
 
       //Remove $ signs as they will be interpolated and cause issues. Variables can still be used
       //in alias but shouldn't in the autogenerated name

public/app/features/transformers/docs/content.ts

@@ -64,6 +64,7 @@ Use this transformation to add a new field calculated from two other fields. Eac
 - **Field name** - Select the names of fields you want to use in the calculation for the new field.
 - **Calculation** - If you select **Reduce row** mode, then the **Calculation** field appears. Click in the field to see a list of calculation choices you can use to create the new field. For information about available calculations, refer to [Calculation types][].
 - **Operation** - If you select **Binary operation** or **Unary operation** mode, then the **Operation** fields appear. These fields allow you to apply basic math operations on values in a single row from selected fields. You can also use numerical values for binary operations.
+  - **All number fields** - Set the left side of a **Binary operation** to apply the calculation to all number fields.
 - **As percentile** - If you select **Row index** mode, then the **As percentile** switch appears. This switch allows you to transform the row index as a percentage of the total number of rows.
 - **Alias** - (Optional) Enter the name of your new field. If you leave this blank, then the field will be named to match the calculation.
 - **Replace all fields** - (Optional) Select this option if you want to hide all other fields and display only your calculated field in the visualization.

public/app/features/transformers/editors/CalculateFieldTransformerEditor/BinaryOperationOptionsEditor.tsx

@@ -1,10 +1,12 @@
-import { BinaryOperationID, binaryOperators, SelectableValue } from '@grafana/data';
+import { BinaryOperationID, binaryOperators, FieldMatcherID, FieldType, SelectableValue } from '@grafana/data';
 import {
+  BinaryValue,
   BinaryOptions,
   CalculateFieldMode,
   CalculateFieldTransformerOptions,
+  checkBinaryValueType,
 } from '@grafana/data/src/transformations/transformers/calculateField';
-import { InlineField, InlineFieldRow, Select } from '@grafana/ui';
+import { getFieldTypeIconName, InlineField, InlineFieldRow, Select } from '@grafana/ui';
 
 import { LABEL_WIDTH } from './constants';
 
@@ -14,21 +16,65 @@ export const BinaryOperationOptionsEditor = (props: {
   names: string[];
 }) => {
   const { options, onChange } = props;
+  const newLeft = checkBinaryValueType(props.options.binary?.left ?? '', props.names);
+  const newRight = checkBinaryValueType(props.options.binary?.right ?? '', props.names);
+  // If there is a change due to migration, update save model
+  if (newLeft !== props.options.binary?.left || newRight !== props.options.binary?.right) {
+    onChange({
+      ...options,
+      mode: CalculateFieldMode.BinaryOperation,
+      binary: { operator: options.binary?.operator!, left: newLeft, right: newRight },
+    });
+  }
+
   const { binary } = options;
 
   let foundLeft = !binary?.left;
   let foundRight = !binary?.right;
+
+  const fixedValueLeft = !binary?.left?.matcher;
+  const fixedValueRight = !binary?.right?.matcher;
+  const matcherOptionsLeft = binary?.left?.matcher?.options;
+  const matcherOptionsRight = binary?.right?.matcher?.options;
+
+  const byNameLeft = binary?.left?.matcher?.id === FieldMatcherID.byName;
+  const byNameRight = binary?.right?.matcher?.id === FieldMatcherID.byName;
   const names = props.names.map((v) => {
-    if (v === binary?.left) {
+    if (byNameLeft && v === matcherOptionsLeft) {
       foundLeft = true;
     }
-    if (v === binary?.right) {
+    if (byNameRight && v === matcherOptionsRight) {
       foundRight = true;
     }
-    return { label: v, value: v };
+    return { label: v, value: JSON.stringify({ matcher: { id: FieldMatcherID.byName, options: v } }) };
   });
-  const leftNames = foundLeft ? names : [...names, { label: binary?.left, value: binary?.left }];
-  const rightNames = foundRight ? names : [...names, { label: binary?.right, value: binary?.right }];
+
+  // Populate left and right names with missing name only for byName
+  const leftNames = foundLeft
+    ? [...names]
+    : byNameLeft
+      ? [...names, { label: matcherOptionsLeft, value: JSON.stringify(binary.left), icon: '' }]
+      : [...names];
+  const rightNames = foundRight
+    ? [...names]
+    : byNameRight
+      ? [...names, { label: matcherOptionsRight, value: JSON.stringify(binary.right), icon: '' }]
+      : [...names];
+
+  // Add byTypes to left names ONLY - avoid all number fields operated by all number fields
+  leftNames.push({
+    label: `All ${FieldType.number} fields`,
+    value: JSON.stringify({ matcher: { id: FieldMatcherID.byType, options: FieldType.number } }),
+    icon: getFieldTypeIconName(FieldType.number),
+  });
+
+  // Add fixed values to left and right names
+  if (fixedValueLeft && binary?.left?.fixed) {
+    leftNames.push({ label: binary.left.fixed, value: JSON.stringify(binary.left) ?? '', icon: '' });
+  }
+  if (fixedValueRight && binary?.right?.fixed) {
+    rightNames.push({ label: binary.right.fixed, value: JSON.stringify(binary.right) ?? '', icon: '' });
+  }
 
   const ops = binaryOperators.list().map((v) => {
     return { label: v.binaryOperationID, value: v.binaryOperationID };
@@ -43,17 +89,35 @@ export const BinaryOperationOptionsEditor = (props: {
   };
 
   const onBinaryLeftChanged = (v: SelectableValue<string>) => {
-    updateBinaryOptions({
-      ...binary!,
-      left: v.value!,
-    });
+    const vObject: BinaryValue = JSON.parse(v.value ?? '');
+    // If no matcher, treat as fixed value
+    if (!vObject.matcher) {
+      updateBinaryOptions({
+        ...binary!,
+        left: { fixed: vObject.fixed ?? v.value?.toString() },
+      });
+    } else {
+      updateBinaryOptions({
+        ...binary!,
+        left: vObject,
+      });
+    }
   };
 
   const onBinaryRightChanged = (v: SelectableValue<string>) => {
-    updateBinaryOptions({
-      ...binary!,
-      right: v.value!,
-    });
+    const vObject: BinaryValue = JSON.parse(v.value ?? '');
+    // If no matcher, treat as fixed value
+    if (!vObject.matcher) {
+      updateBinaryOptions({
+        ...binary!,
+        right: { fixed: vObject.fixed ?? v.value?.toString() },
+      });
+    } else {
+      updateBinaryOptions({
+        ...binary!,
+        right: vObject,
+      });
+    }
   };
 
   const onBinaryOperationChanged = (v: SelectableValue<BinaryOperationID>) => {
@@ -69,10 +133,10 @@ export const BinaryOperationOptionsEditor = (props: {
         <InlineField label="Operation" labelWidth={LABEL_WIDTH}>
           <Select
             allowCustomValue={true}
-            placeholder="Field or number"
+            placeholder="Field(s) or number"
             options={leftNames}
             className="min-width-18"
-            value={binary?.left}
+            value={JSON.stringify(binary?.left)}
             onChange={onBinaryLeftChanged}
           />
         </InlineField>
@@ -90,7 +154,7 @@ export const BinaryOperationOptionsEditor = (props: {
             placeholder="Field or number"
             className="min-width-10"
             options={rightNames}
-            value={binary?.right}
+            value={JSON.stringify(binary?.right)}
             onChange={onBinaryRightChanged}
           />
         </InlineField>

public/app/features/transformers/editors/CalculateFieldTransformerEditor/CalculateFieldTransformerEditor.tsx

@@ -14,6 +14,7 @@ import {
   TransformerRegistryItem,
   TransformerUIProps,
   TransformerCategory,
+  FieldMatcherID,
 } from '@grafana/data';
 import {
   CalculateFieldMode,
@@ -171,6 +172,9 @@ export const CalculateFieldTransformerEditor = (props: CalculateFieldTransformer
   };
 
   const mode = options.mode ?? CalculateFieldMode.BinaryOperation;
+  // For binary operation with type matching, disable alias input
+  const disableAlias =
+    mode === CalculateFieldMode.BinaryOperation && options.binary?.left.matcher?.id === FieldMatcherID.byType;
 
   return (
     <>
@@ -213,7 +217,7 @@ export const CalculateFieldTransformerEditor = (props: CalculateFieldTransformer
       {mode === CalculateFieldMode.Index && (
         <IndexOptionsEditor options={options} onChange={props.onChange}></IndexOptionsEditor>
       )}
-      <InlineField labelWidth={LABEL_WIDTH} label="Alias">
+      <InlineField labelWidth={LABEL_WIDTH} label="Alias" disabled={disableAlias}>
         <Input
           className="width-18"
           value={options.alias ?? ''}