Transformers: allow label fields in the reduce processor (#37373)

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>

packages/grafana-data/src/transformations/transformers/reduce.test.ts

@@ -2,9 +2,9 @@ import { ReducerID } from '../fieldReducer';
 import { DataTransformerID } from './ids';
 import { toDataFrame } from '../../dataframe/processDataFrame';
 import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
-import { reduceFields, reduceTransformer } from './reduce';
+import { reduceFields, reduceTransformer, ReduceTransformerOptions } from './reduce';
 import { transformDataFrame } from '../transformDataFrame';
-import { Field, FieldType } from '../../types';
+import { DataTransformerConfig, Field, FieldType } from '../../types';
 import { ArrayVector } from '../../vector';
 import { notTimeFieldMatcher } from '../matchers/predicates';
 import { DataFrameView } from '../../dataframe';
@@ -366,4 +366,84 @@ describe('Reducer Transformer', () => {
       expect(processed[0].fields).toEqual(expected);
     });
   });
+
+  it('reduces keeping label field', async () => {
+    const cfg: DataTransformerConfig<ReduceTransformerOptions> = {
+      id: DataTransformerID.reduce,
+      options: {
+        reducers: [ReducerID.max],
+        labelsToFields: true,
+      },
+    };
+
+    const seriesA = toDataFrame({
+      fields: [
+        { name: 'time', type: FieldType.time, values: [3000, 4000, 5000, 6000] },
+        { name: 'value', labels: { state: 'CA' }, type: FieldType.number, values: [3, 4, 5, 6] },
+        { name: 'value', labels: { state: 'NY' }, type: FieldType.number, values: [3, 4, 5, 6] },
+      ],
+    });
+
+    const seriesB = toDataFrame({
+      fields: [
+        { name: 'time', type: FieldType.time, values: [3000, 4000, 5000, 6000] },
+        { name: 'value', labels: { state: 'CA', country: 'USA' }, type: FieldType.number, values: [3, 4, 5, 6] },
+        { name: 'value', labels: { country: 'USA' }, type: FieldType.number, values: [3, 4, 5, 6] },
+      ],
+    });
+
+    await expect(transformDataFrame([cfg], [seriesA, seriesB])).toEmitValuesWith((received) => {
+      const processed = received[0];
+
+      expect(processed.length).toEqual(1);
+      expect(processed[0].fields).toMatchInlineSnapshot(`
+        Array [
+          Object {
+            "config": Object {},
+            "name": "Field",
+            "type": "string",
+            "values": Array [
+              "value",
+              "value",
+              "value",
+              "value",
+            ],
+          },
+          Object {
+            "config": Object {},
+            "name": "state",
+            "type": "string",
+            "values": Array [
+              "CA",
+              "NY",
+              "CA",
+              undefined,
+            ],
+          },
+          Object {
+            "config": Object {},
+            "name": "country",
+            "type": "string",
+            "values": Array [
+              undefined,
+              undefined,
+              "USA",
+              "USA",
+            ],
+          },
+          Object {
+            "config": Object {},
+            "name": "Max",
+            "type": "number",
+            "values": Array [
+              6,
+              6,
+              6,
+              6,
+            ],
+          },
+        ]
+      `);
+    });
+  });
 });

packages/grafana-data/src/transformations/transformers/reduce.ts

@@ -20,6 +20,7 @@ export interface ReduceTransformerOptions {
   fields?: MatcherConfig; // Assume all fields
   mode?: ReduceTransformerMode;
   includeTimeField?: boolean;
+  labelsToFields?: boolean;
 }
 
 export const reduceTransformer: DataTransformerInfo<ReduceTransformerOptions> = {
@@ -53,7 +54,7 @@ export const reduceTransformer: DataTransformerInfo<ReduceTransformerOptions> =
         }
 
         // Add a row for each series
-        const res = reduceSeriesToRows(data, matcher, options.reducers);
+        const res = reduceSeriesToRows(data, matcher, options.reducers, options.labelsToFields);
         return res ? [res] : [];
       })
     ),
@@ -65,77 +66,113 @@ export const reduceTransformer: DataTransformerInfo<ReduceTransformerOptions> =
 export function reduceSeriesToRows(
   data: DataFrame[],
   matcher: FieldMatcher,
-  reducerId: ReducerID[]
+  reducerId: ReducerID[],
+  labelsToFields?: boolean
 ): DataFrame | undefined {
   const calculators = fieldReducers.list(reducerId);
   const reducers = calculators.map((c) => c.id);
   const processed: DataFrame[] = [];
+  const distinctLabels = labelsToFields ? getDistinctLabelKeys(data) : [];
 
   for (const series of data) {
-    const values: ArrayVector[] = [];
+    const source = series.fields.filter((f) => matcher(f, series, data));
-    const fields: Field[] = [];
-    const byId: KeyValue<ArrayVector> = {};
 
-    values.push(new ArrayVector()); // The name
+    const size = source.length;
+    const fields: Field[] = [];
+    const names = new ArrayVector<string>(new Array(size));
     fields.push({
       name: 'Field',
       type: FieldType.string,
-      values: values[0],
+      values: names,
       config: {},
     });
 
-    for (const info of calculators) {
+    const labels: KeyValue<ArrayVector> = {};
-      const vals = new ArrayVector();
+    if (labelsToFields) {
-      byId[info.id] = vals;
+      for (const key of distinctLabels) {
-      values.push(vals);
+        labels[key] = new ArrayVector<string>(new Array(size));
+        fields.push({
+          name: key,
+          type: FieldType.string,
+          values: labels[key],
+          config: {},
+        });
+      }
+    }
 
+    const calcs: KeyValue<ArrayVector> = {};
+    for (const info of calculators) {
+      calcs[info.id] = new ArrayVector(new Array(size));
       fields.push({
         name: info.name,
         type: FieldType.other, // UNKNOWN until after we call the functions
-        values: values[values.length - 1],
+        values: calcs[info.id],
         config: {},
       });
     }
 
-    for (let i = 0; i < series.fields.length; i++) {
+    for (let i = 0; i < source.length; i++) {
-      const field = series.fields[i];
+      const field = source[i];
-
-      if (matcher(field, series, data)) {
       const results = reduceField({
         field,
         reducers,
       });
 
-        // Update the name list
+      if (labelsToFields) {
-        const fieldName = getFieldDisplayName(field, series, data);
+        names.buffer[i] = field.name;
-
+        if (field.labels) {
-        values[0].buffer.push(fieldName);
+          for (const key of Object.keys(field.labels)) {
+            labels[key].set(i, field.labels[key]);
+          }
+        }
+      } else {
+        names.buffer[i] = getFieldDisplayName(field, series, data);
+      }
 
       for (const info of calculators) {
         const v = results[info.id];
-          byId[info.id].buffer.push(v);
+        calcs[info.id].buffer[i] = v;
-        }
       }
     }
 
+    // For reduced fields, we don't know the type until we see the value
     for (const f of fields) {
+      if (f.type === FieldType.other) {
         const t = guessFieldTypeForField(f);
-
         if (t) {
           f.type = t;
         }
       }
+    }
 
     processed.push({
       ...series, // Same properties, different fields
       fields,
-      length: values[0].length,
+      length: size,
     });
   }
 
   return mergeResults(processed);
 }
 
+export function getDistinctLabelKeys(frames: DataFrame[]): string[] {
+  const ordered: string[] = [];
+  const keys = new Set<string>();
+  for (const frame of frames) {
+    for (const field of frame.fields) {
+      if (field.labels) {
+        for (const k of Object.keys(field.labels)) {
+          if (!keys.has(k)) {
+            ordered.push(k);
+            keys.add(k);
+          }
+        }
+      }
+    }
+  }
+  return ordered;
+}
+
 /**
  * @internal only exported for testing
  */

public/app/core/components/TransformersUI/ReduceTransformerEditor.tsx

@@ -49,6 +49,13 @@ export const ReduceTransformerEditor: React.FC<TransformerUIProps<ReduceTransfor
     });
   }, [onChange, options]);
 
+  const onToggleLabels = useCallback(() => {
+    onChange({
+      ...options,
+      labelsToFields: !options.labelsToFields,
+    });
+  }, [onChange, options]);
+
   return (
     <>
       <div>
@@ -95,6 +102,18 @@ export const ReduceTransformerEditor: React.FC<TransformerUIProps<ReduceTransfor
           </div>
         </div>
       )}
+      {options.mode !== ReduceTransformerMode.ReduceFields && (
+        <div className="gf-form-inline">
+          <div className="gf-form">
+            <LegacyForms.Switch
+              label="Labels to fields"
+              labelClass="width-8"
+              checked={!!options.labelsToFields}
+              onChange={onToggleLabels}
+            />
+          </div>
+        </div>
+      )}
     </>
   );
 };