Transformations: add 'prepare time series' transformer (#36737)

* adding transformer that will stretch a data frame from wide to long.

* added a UI for the stretch frames transformer.

* refactored according to feedback.

* removed unused dep.

* making sure it is being displayed.

* minor adjustments.

* move stretch to prepare

* improved readability of tests.

* refactored to use a function component syntax.

Co-authored-by: Marcus Andersson <marcus.andersson@grafana.com>

packages/grafana-data/src/transformations/transformers/ids.ts

@@ -25,4 +25,5 @@ export enum DataTransformerID {
   histogram = 'histogram',
   configFromData = 'configFromData',
   rowsToFields = 'rowsToFields',
+  prepareTimeSeries = 'prepareTimeSeries',
 }

public/app/core/components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor.tsx

@@ -0,0 +1,94 @@
+import React, { useCallback } from 'react';
+import { GrafanaTheme2, SelectableValue, TransformerRegistryItem, TransformerUIProps } from '@grafana/data';
+import { prepareTimeSeriesTransformer, PrepareTimeSeriesOptions, timeSeriesFormat } from './prepareTimeSeries';
+import { InlineField, InlineFieldRow, Select, useStyles2 } from '@grafana/ui';
+import { css } from '@emotion/css';
+
+const wideInfo = {
+  label: 'Wide time series',
+  value: timeSeriesFormat.TimeSeriesWide,
+  description: 'Creates a single frame joined by time',
+  info: (
+    <ul>
+      <li>Single frame</li>
+      <li>1st field is shared time field</li>
+      <li>Time in ascending order</li>
+      <li>Multiple value fields of any type</li>
+    </ul>
+  ),
+};
+
+const manyInfo = {
+  label: 'Multi-frame time series',
+  value: timeSeriesFormat.TimeSeriesMany,
+  description: 'Creates a new frame for each time/number pair',
+  info: (
+    <ul>
+      <li>Multiple frames</li>
+      <li>Each frame has two fields: time, value</li>
+      <li>Time in ascending order</li>
+      <li>All values are numeric</li>
+    </ul>
+  ),
+};
+
+const formats: Array<SelectableValue<timeSeriesFormat>> = [wideInfo, manyInfo];
+
+export function PrepareTimeSeriesEditor(props: TransformerUIProps<PrepareTimeSeriesOptions>): React.ReactElement {
+  const { options, onChange } = props;
+  const styles = useStyles2(getStyles);
+
+  const onSelectFormat = useCallback(
+    (value: SelectableValue<timeSeriesFormat>) => {
+      onChange({
+        ...options,
+        format: value.value!,
+      });
+    },
+    [onChange, options]
+  );
+
+  return (
+    <>
+      <InlineFieldRow>
+        <InlineField label="Format" labelWidth={12}>
+          <Select
+            width={35}
+            options={formats}
+            value={formats.find((v) => v.value === options.format) || formats[0]}
+            onChange={onSelectFormat}
+            className="flex-grow-1"
+          />
+        </InlineField>
+      </InlineFieldRow>
+      <InlineFieldRow>
+        <InlineField label="Info" labelWidth={12}>
+          <div className={styles.info}>
+            {options.format === timeSeriesFormat.TimeSeriesMany ? manyInfo.info : wideInfo.info}
+          </div>
+        </InlineField>
+      </InlineFieldRow>
+    </>
+  );
+}
+
+const getStyles = (theme: GrafanaTheme2) => ({
+  info: css`
+    margin-left: 20px;
+  `,
+});
+
+export const prepareTimeseriesTransformerRegistryItem: TransformerRegistryItem<PrepareTimeSeriesOptions> = {
+  id: prepareTimeSeriesTransformer.id,
+  editor: PrepareTimeSeriesEditor,
+  transformation: prepareTimeSeriesTransformer,
+  name: prepareTimeSeriesTransformer.name,
+  description: prepareTimeSeriesTransformer.description,
+  help: `
+  ### Use cases 
+  
+  This will take query results and transform them into a predictable timeseries format.  
+  This transformer may be especially useful when using old panels that only expect the
+  many-frame timeseries format.
+  `,
+};

public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.test.ts

@@ -0,0 +1,214 @@
+import { toDataFrame, ArrayVector, DataFrame, FieldType, toDataFrameDTO, DataFrameDTO } from '@grafana/data';
+import { prepareTimeSeries, PrepareTimeSeriesOptions, timeSeriesFormat } from './prepareTimeSeries';
+
+describe('Prepair time series transformer', () => {
+  it('should transform wide to many', () => {
+    const source = [
+      toDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+          { name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+      }),
+    ];
+
+    const config: PrepareTimeSeriesOptions = {
+      format: timeSeriesFormat.TimeSeriesMany,
+    };
+
+    expect(prepareTimeSeries(source, config)).toEqual([
+      toEquableDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+        ],
+        length: 6,
+      }),
+      toEquableDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+        length: 6,
+      }),
+    ]);
+  });
+
+  it('should remove string fields since time series format is expected to be time/number fields', () => {
+    const source = [
+      toDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+          { name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+      }),
+    ];
+
+    const config: PrepareTimeSeriesOptions = {
+      format: timeSeriesFormat.TimeSeriesMany,
+    };
+
+    expect(prepareTimeSeries(source, config)).toEqual([
+      toEquableDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+        ],
+        length: 6,
+      }),
+      toEquableDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+        length: 6,
+      }),
+    ]);
+  });
+
+  it('should transform all wide to many when mixed', () => {
+    const source = [
+      toDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+          { name: 'another', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+      }),
+      toDataFrame({
+        name: 'long',
+        refId: 'B',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [100, 90, 80, 70, 60, 50] },
+          { name: 'value', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+      }),
+    ];
+
+    const config: PrepareTimeSeriesOptions = {
+      format: timeSeriesFormat.TimeSeriesMany,
+    };
+
+    expect(prepareTimeSeries(source, config)).toEqual([
+      toEquableDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+        ],
+        length: 6,
+      }),
+      toEquableDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'another', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+        length: 6,
+      }),
+      toEquableDataFrame({
+        name: 'long',
+        refId: 'B',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [100, 90, 80, 70, 60, 50] },
+          { name: 'value', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+        ],
+        length: 6,
+      }),
+    ]);
+  });
+
+  it('should transform none when source only has long frames', () => {
+    const source = [
+      toDataFrame({
+        name: 'long',
+        refId: 'A',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+        ],
+      }),
+      toDataFrame({
+        name: 'long',
+        refId: 'B',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+        ],
+      }),
+    ];
+
+    const config: PrepareTimeSeriesOptions = {
+      format: timeSeriesFormat.TimeSeriesMany,
+    };
+
+    expect(toEquableDataFrames(prepareTimeSeries(source, config))).toEqual(toEquableDataFrames(source));
+  });
+
+  it('should return empty array when no timeseries exist', () => {
+    const source = [
+      toDataFrame({
+        name: 'wide',
+        refId: 'A',
+        fields: [
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+        ],
+      }),
+      toDataFrame({
+        name: 'wide',
+        refId: 'B',
+        fields: [
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+        ],
+      }),
+    ];
+
+    const config: PrepareTimeSeriesOptions = {
+      format: timeSeriesFormat.TimeSeriesMany,
+    };
+
+    expect(prepareTimeSeries(source, config)).toEqual([]);
+  });
+});
+
+function toEquableDataFrame(source: any): DataFrame {
+  return toDataFrame({
+    meta: undefined,
+    ...source,
+    fields: source.fields.map((field: any) => {
+      return {
+        ...field,
+        values: new ArrayVector(field.values),
+        config: {},
+      };
+    }),
+  });
+}
+
+function toEquableDataFrames(data: DataFrame[]): DataFrameDTO[] {
+  return data.map((frame) => toDataFrameDTO(frame));
+}

public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.ts

@@ -0,0 +1,93 @@
+import {
+  DataTransformerInfo,
+  DataFrame,
+  FieldType,
+  DataTransformerID,
+  outerJoinDataFrames,
+  fieldMatchers,
+  FieldMatcherID,
+} from '@grafana/data';
+import { map } from 'rxjs/operators';
+
+/**
+ * There is currently an effort to figure out consistent names
+ * for the various formats/types we produce and use.
+ *
+ * This transformer will eventually include the required metadata that can assert
+ * a DataFrame[] is of a given type
+ *
+ * @internal -- TBD
+ */
+export enum timeSeriesFormat {
+  TimeSeriesWide = 'wide', // [time,...values]
+  TimeSeriesMany = 'many', // All frames have [time,number]
+  //  TimeSeriesLong = 'long',
+}
+
+export type PrepareTimeSeriesOptions = {
+  format: timeSeriesFormat;
+};
+
+/**
+ * Convert to [][time,number]
+ */
+export function toTimeSeriesMany(data: DataFrame[]): DataFrame[] {
+  if (!Array.isArray(data) || data.length === 0) {
+    return data;
+  }
+
+  const result: DataFrame[] = [];
+  for (const frame of data) {
+    const timeField = frame.fields.find((field) => {
+      return field.type === FieldType.time;
+    });
+
+    if (!timeField) {
+      continue;
+    }
+
+    for (const field of frame.fields) {
+      if (field.type !== FieldType.number) {
+        continue;
+      }
+
+      result.push({
+        name: frame.name,
+        refId: frame.refId,
+        meta: frame.meta,
+        fields: [timeField, field],
+        length: frame.length,
+      });
+    }
+  }
+  return result;
+}
+
+export function prepareTimeSeries(data: DataFrame[], options: PrepareTimeSeriesOptions): DataFrame[] {
+  const format = options?.format ?? timeSeriesFormat.TimeSeriesWide;
+  if (format === timeSeriesFormat.TimeSeriesMany) {
+    return toTimeSeriesMany(data);
+  }
+
+  // Join by the first frame
+  const frame = outerJoinDataFrames({
+    frames: data,
+    joinBy: fieldMatchers.get(FieldMatcherID.firstTimeField).get({}),
+    enforceSort: true,
+    keepOriginIndices: true,
+  });
+  return frame ? [frame] : [];
+}
+
+export const prepareTimeSeriesTransformer: DataTransformerInfo<PrepareTimeSeriesOptions> = {
+  id: DataTransformerID.prepareTimeSeries,
+  name: 'Prepare time series',
+  description: `Will stretch data frames from the wide format into the long format. This is really helpful to be able to keep backwards compatability for panels not supporting the new wide format.`,
+  defaultOptions: {},
+
+  /**
+   * Return a modified copy of the series.  If the transform is not or should not
+   * be applied, just return the input series
+   */
+  operator: (options) => (source) => source.pipe(map((data) => prepareTimeSeries(data, options))),
+};

public/app/core/utils/standardTransformers.ts

@@ -16,6 +16,7 @@ import { renameByRegexTransformRegistryItem } from '../components/TransformersUI
 import { histogramTransformRegistryItem } from '../components/TransformersUI/HistogramTransformerEditor';
 import { rowsToFieldsTransformRegistryItem } from '../components/TransformersUI/rowsToFields/RowsToFieldsTransformerEditor';
 import { configFromQueryTransformRegistryItem } from '../components/TransformersUI/configFromQuery/ConfigFromQueryTransformerEditor';
+import { prepareTimeseriesTransformerRegistryItem } from '../components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor';
 
 export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> => {
   return [
@@ -36,5 +37,6 @@ export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> =
     histogramTransformRegistryItem,
     rowsToFieldsTransformRegistryItem,
     configFromQueryTransformRegistryItem,
+    prepareTimeseriesTransformerRegistryItem,
   ];
 };