Timeseries: add wide-to-long, and fix the multi-frame output (#38670)

Co-authored-by: Douglas Thrift <dthrift@flexera.com>

public/app/core/components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor.tsx

@@ -27,12 +27,28 @@ const manyInfo = {
       <li>Multiple frames</li>
       <li>Each frame has two fields: time, value</li>
       <li>Time in ascending order</li>
+      <li>String values are represented as labels</li>
       <li>All values are numeric</li>
     </ul>
   ),
 };
 
-const formats: Array<SelectableValue<timeSeriesFormat>> = [wideInfo, manyInfo];
+const longInfo = {
+  label: 'Long time series',
+  value: timeSeriesFormat.TimeSeriesLong,
+  description: 'Convert each frame to long format',
+  info: (
+    <ul>
+      <li>Single frame</li>
+      <li>1st field is time field</li>
+      <li>Time in ascending order, but may have duplictes</li>
+      <li>String values are represented as separate fields rather than as labels</li>
+      <li>Multiple value fields may exist</li>
+    </ul>
+  ),
+};
+
+const formats: Array<SelectableValue<timeSeriesFormat>> = [wideInfo, manyInfo, longInfo];
 
 export function PrepareTimeSeriesEditor(props: TransformerUIProps<PrepareTimeSeriesOptions>): React.ReactElement {
   const { options, onChange } = props;
@@ -64,9 +80,7 @@ export function PrepareTimeSeriesEditor(props: TransformerUIProps<PrepareTimeSer
       </InlineFieldRow>
       <InlineFieldRow>
         <InlineField label="Info" labelWidth={12}>
-          <div className={styles.info}>
+          <div className={styles.info}>{(formats.find((v) => v.value === options.format) || formats[0]).info}</div>
-            {options.format === timeSeriesFormat.TimeSeriesMany ? manyInfo.info : wideInfo.info}
-          </div>
         </InlineField>
       </InlineFieldRow>
     </>

public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.test.ts

@@ -6,18 +6,19 @@ import {
   toDataFrameDTO,
   DataFrameDTO,
   DataFrameType,
+  getFrameDisplayName,
 } from '@grafana/data';
 import { prepareTimeSeriesTransformer, PrepareTimeSeriesOptions, timeSeriesFormat } from './prepareTimeSeries';
 
-describe('Prepair time series transformer', () => {
+describe('Prepare time series transformer', () => {
   it('should transform wide to many', () => {
     const source = [
       toDataFrame({
         name: 'wide',
         refId: 'A',
         fields: [
-          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'time', type: FieldType.time, values: [1, 2, 3, 4, 5, 6] },
-          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+          { name: 'count', type: FieldType.number, values: [10, 20, 30, 40, 50, 60] },
           { name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
         ],
       }),
@@ -32,8 +33,8 @@ describe('Prepair time series transformer', () => {
         name: 'wide',
         refId: 'A',
         fields: [
-          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'time', type: FieldType.time, values: [1, 2, 3, 4, 5, 6] },
-          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+          { name: 'count', type: FieldType.number, values: [10, 20, 30, 40, 50, 60] },
         ],
         meta: {
           type: DataFrameType.TimeSeriesMany,
@@ -44,7 +45,7 @@ describe('Prepair time series transformer', () => {
         name: 'wide',
         refId: 'A',
         fields: [
-          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'time', type: FieldType.time, values: [1, 2, 3, 4, 5, 6] },
           { name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
         ],
         meta: {
@@ -55,16 +56,16 @@ describe('Prepair time series transformer', () => {
     ]);
   });
 
-  it('should remove string fields since time series format is expected to be time/number fields', () => {
+  it('should treat string fields as labels', () => {
     const source = [
       toDataFrame({
         name: 'wide',
         refId: 'A',
         fields: [
-          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'time', type: FieldType.time, values: [1, 1, 2, 2] },
-          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'region', type: FieldType.string, values: ['a', 'b', 'a', 'b'] },
-          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+          { name: 'count', type: FieldType.number, values: [10, 20, 30, 40] },
-          { name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+          { name: 'more', type: FieldType.number, values: [2, 3, 4, 5] },
         ],
       }),
     ];
@@ -73,32 +74,75 @@ describe('Prepair time series transformer', () => {
       format: timeSeriesFormat.TimeSeriesMany,
     };
 
-    expect(prepareTimeSeriesTransformer.transformer(config)(source)).toEqual([
+    const frames = prepareTimeSeriesTransformer.transformer(config)(source);
-      toEquableDataFrame({
+    expect(frames.length).toEqual(4);
-        name: 'wide',
+    expect(
-        refId: 'A',
+      frames.map((f) => ({
-        fields: [
+        name: getFrameDisplayName(f),
-          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+        labels: f.fields[1].labels,
-          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+        time: f.fields[0].values.toArray(),
-        ],
+        values: f.fields[1].values.toArray(),
-        length: 6,
+      }))
-        meta: {
+    ).toMatchInlineSnapshot(`
-          type: DataFrameType.TimeSeriesMany,
+      Array [
+        Object {
+          "labels": Object {
+            "region": "a",
+          },
+          "name": "wide",
+          "time": Array [
+            1,
+            2,
+          ],
+          "values": Array [
+            10,
+            30,
+          ],
         },
-      }),
+        Object {
-      toEquableDataFrame({
+          "labels": Object {
-        name: 'wide',
+            "region": "b",
-        refId: 'A',
+          },
-        fields: [
+          "name": "wide",
-          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          "time": Array [
-          { name: 'more', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+            1,
-        ],
+            2,
-        length: 6,
+          ],
-        meta: {
+          "values": Array [
-          type: DataFrameType.TimeSeriesMany,
+            20,
+            40,
+          ],
         },
-      }),
+        Object {
-    ]);
+          "labels": Object {
+            "region": "a",
+          },
+          "name": "wide",
+          "time": Array [
+            1,
+            2,
+          ],
+          "values": Array [
+            2,
+            4,
+          ],
+        },
+        Object {
+          "labels": Object {
+            "region": "b",
+          },
+          "name": "wide",
+          "time": Array [
+            1,
+            2,
+          ],
+          "values": Array [
+            3,
+            5,
+          ],
+        },
+      ]
+    `);
   });
 
   it('should transform all wide to many when mixed', () => {
@@ -107,9 +151,8 @@ describe('Prepair time series transformer', () => {
         name: 'wide',
         refId: 'A',
         fields: [
-          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'time', type: FieldType.time, values: [0, 1, 2, 3, 4, 5] },
-          { name: 'text', type: FieldType.string, values: ['a', 'z', 'b', 'x', 'c', 'b'] },
+          { name: 'count', type: FieldType.number, values: [10, 20, 30, 40, 50, 60] },
-          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
           { name: 'another', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
         ],
       }),
@@ -117,7 +160,7 @@ describe('Prepair time series transformer', () => {
         name: 'long',
         refId: 'B',
         fields: [
-          { name: 'time', type: FieldType.time, values: [100, 90, 80, 70, 60, 50] },
+          { name: 'time', type: FieldType.time, values: [4, 5, 6, 7, 8, 9] },
           { name: 'value', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
         ],
       }),
@@ -132,8 +175,8 @@ describe('Prepair time series transformer', () => {
         name: 'wide',
         refId: 'A',
         fields: [
-          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'time', type: FieldType.time, values: [0, 1, 2, 3, 4, 5] },
-          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+          { name: 'another', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
         ],
         length: 6,
         meta: {
@@ -144,8 +187,8 @@ describe('Prepair time series transformer', () => {
         name: 'wide',
         refId: 'A',
         fields: [
-          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'time', type: FieldType.time, values: [0, 1, 2, 3, 4, 5] },
-          { name: 'another', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
+          { name: 'count', type: FieldType.number, values: [10, 20, 30, 40, 50, 60] },
         ],
         length: 6,
         meta: {
@@ -156,7 +199,7 @@ describe('Prepair time series transformer', () => {
         name: 'long',
         refId: 'B',
         fields: [
-          { name: 'time', type: FieldType.time, values: [100, 90, 80, 70, 60, 50] },
+          { name: 'time', type: FieldType.time, values: [4, 5, 6, 7, 8, 9] },
           { name: 'value', type: FieldType.number, values: [2, 3, 4, 5, 6, 7] },
         ],
         length: 6,
@@ -173,16 +216,16 @@ describe('Prepair time series transformer', () => {
         name: 'long',
         refId: 'A',
         fields: [
-          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'time', type: FieldType.time, values: [1, 2, 3, 4, 5, 6] },
-          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+          { name: 'count', type: FieldType.number, values: [10, 20, 30, 40, 50, 60] },
         ],
       }),
       toDataFrame({
         name: 'long',
         refId: 'B',
         fields: [
-          { name: 'time', type: FieldType.time, values: [10, 9, 8, 7, 6, 5] },
+          { name: 'time', type: FieldType.time, values: [1, 2, 3, 4, 5, 6] },
-          { name: 'count', type: FieldType.number, values: [1, 2, 3, 4, 5, 6] },
+          { name: 'count', type: FieldType.number, values: [10, 20, 30, 40, 50, 60] },
         ],
       }),
     ];
@@ -231,6 +274,52 @@ describe('Prepair time series transformer', () => {
 
     expect(prepareTimeSeriesTransformer.transformer(config)(source)).toEqual([]);
   });
+
+  it('should convert long to many', () => {
+    const source = [
+      toDataFrame({
+        name: 'long',
+        refId: 'X',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [1, 1, 2, 2, 3, 3] },
+          { name: 'value', type: FieldType.number, values: [10, 20, 30, 40, 50, 60] },
+          { name: 'region', type: FieldType.string, values: ['a', 'b', 'a', 'b', 'a', 'b'] },
+        ],
+      }),
+    ];
+
+    const config: PrepareTimeSeriesOptions = {
+      format: timeSeriesFormat.TimeSeriesMany,
+    };
+
+    const frames = prepareTimeSeriesTransformer.transformer(config)(source);
+    expect(frames).toEqual([
+      toEquableDataFrame({
+        name: 'long',
+        refId: 'X',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [1, 2, 3] },
+          { name: 'value', labels: { region: 'a' }, type: FieldType.number, values: [10, 30, 50] },
+        ],
+        length: 3,
+        meta: {
+          type: DataFrameType.TimeSeriesMany,
+        },
+      }),
+      toEquableDataFrame({
+        name: 'long',
+        refId: 'X',
+        fields: [
+          { name: 'time', type: FieldType.time, values: [1, 2, 3] },
+          { name: 'value', labels: { region: 'b' }, type: FieldType.number, values: [20, 40, 60] },
+        ],
+        length: 3,
+        meta: {
+          type: DataFrameType.TimeSeriesMany,
+        },
+      }),
+    ]);
+  });
 });
 
 function toEquableDataFrame(source: any): DataFrame {

public/app/core/components/TransformersUI/prepareTimeSeries/prepareTimeSeries.ts

@@ -7,7 +7,11 @@ import {
   outerJoinDataFrames,
   fieldMatchers,
   FieldMatcherID,
+  Field,
+  MutableDataFrame,
+  ArrayVector,
 } from '@grafana/data';
+import { Labels } from 'app/types/unified-alerting-dto';
 import { map } from 'rxjs/operators';
 
 /**
@@ -22,7 +26,7 @@ import { map } from 'rxjs/operators';
 export enum timeSeriesFormat {
   TimeSeriesWide = 'wide', // [time,...values]
   TimeSeriesMany = 'many', // All frames have [time,number]
-  //  TimeSeriesLong = 'long',
+  TimeSeriesLong = 'long',
 }
 
 export type PrepareTimeSeriesOptions = {
@@ -37,40 +41,248 @@ export function toTimeSeriesMany(data: DataFrame[]): DataFrame[] {
     return data;
   }
 
+  const result: DataFrame[] = [];
+  for (const frame of toTimeSeriesLong(data)) {
+    const timeField = frame.fields[0];
+    if (!timeField || timeField.type !== FieldType.time) {
+      continue;
+    }
+    const valueFields: Field[] = [];
+    const labelFields: Field[] = [];
+    for (const field of frame.fields) {
+      switch (field.type) {
+        case FieldType.number:
+        case FieldType.boolean:
+          valueFields.push(field);
+          break;
+        case FieldType.string:
+          labelFields.push(field);
+          break;
+      }
+    }
+
+    for (const field of valueFields) {
+      if (labelFields.length) {
+        // new frame for each label key
+        type frameBuilder = {
+          time: number[];
+          value: number[];
+          key: string;
+          labels: Labels;
+        };
+        const builders = new Map<string, frameBuilder>();
+        for (let i = 0; i < frame.length; i++) {
+          const time = timeField.values.get(i);
+          const value = field.values.get(i);
+          if (value === undefined || time == null) {
+            continue; // skip values left over from join
+          }
+
+          const key = labelFields.map((f) => f.values.get(i)).join('/');
+          let builder = builders.get(key);
+          if (!builder) {
+            builder = {
+              key,
+              time: [],
+              value: [],
+              labels: {},
+            };
+            for (const label of labelFields) {
+              builder.labels[label.name] = label.values.get(i);
+            }
+            builders.set(key, builder);
+          }
+          builder.time.push(time);
+          builder.value.push(value);
+        }
+
+        // Add a frame for each distinct value
+        for (const b of builders.values()) {
+          result.push({
+            name: frame.name,
+            refId: frame.refId,
+            meta: {
+              ...frame.meta,
+              type: DataFrameType.TimeSeriesMany,
+            },
+            fields: [
+              {
+                ...timeField,
+                values: new ArrayVector(b.time),
+              },
+              {
+                ...field,
+                values: new ArrayVector(b.value),
+                labels: b.labels,
+              },
+            ],
+            length: b.time.length,
+          });
+        }
+      } else {
+        result.push({
+          name: frame.name,
+          refId: frame.refId,
+          meta: {
+            ...frame.meta,
+            type: DataFrameType.TimeSeriesMany,
+          },
+          fields: [timeField, field],
+          length: frame.length,
+        });
+      }
+    }
+  }
+  return result;
+}
+
+export function toTimeSeriesLong(data: DataFrame[]): DataFrame[] {
+  if (!Array.isArray(data) || data.length === 0) {
+    return data;
+  }
+
   const result: DataFrame[] = [];
   for (const frame of data) {
-    const timeField = frame.fields.find((field) => {
+    let timeField: Field | undefined;
-      return field.type === FieldType.time;
+    const uniqueValueNames: string[] = [];
-    });
+    const uniqueValueNamesToType: Record<string, FieldType> = {};
+    const uniqueLabelKeys: Record<string, boolean> = {};
+    const labelKeyToWideIndices: Record<string, number[]> = {};
+    const uniqueFactorNamesToWideIndex: Record<string, number> = {};
+
+    for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
+      const field = frame.fields[fieldIndex];
+
+      switch (field.type) {
+        case FieldType.string:
+        case FieldType.boolean:
+          if (field.name in uniqueFactorNamesToWideIndex) {
+            // TODO error?
+          } else {
+            uniqueFactorNamesToWideIndex[field.name] = fieldIndex;
+            uniqueLabelKeys[field.name] = true;
+          }
+          break;
+        case FieldType.time:
+          if (!timeField) {
+            timeField = field;
+            break;
+          }
+        default:
+          if (field.name in uniqueValueNamesToType) {
+            const type = uniqueValueNamesToType[field.name];
+
+            if (field.type !== type) {
+              // TODO error?
+              continue;
+            }
+          } else {
+            uniqueValueNamesToType[field.name] = field.type;
+            uniqueValueNames.push(field.name);
+          }
+
+          const tKey = JSON.stringify(field.labels);
+          const wideIndices = labelKeyToWideIndices[tKey];
+
+          if (wideIndices !== undefined) {
+            wideIndices.push(fieldIndex);
+          } else {
+            labelKeyToWideIndices[tKey] = [fieldIndex];
+          }
+
+          if (field.labels != null) {
+            for (const labelKey in field.labels) {
+              uniqueLabelKeys[labelKey] = true;
+            }
+          }
+      }
+    }
 
     if (!timeField) {
       continue;
     }
 
-    for (const field of frame.fields) {
+    type TimeWideRowIndex = {
-      if (field.type !== FieldType.number) {
+      time: any;
-        continue;
+      wideRowIndex: number;
-      }
+    };
+    const sortedTimeRowIndices: TimeWideRowIndex[] = [];
+    const sortedUniqueLabelKeys: string[] = [];
+    const uniqueFactorNames: string[] = [];
+    const uniqueFactorNamesWithWideIndices: string[] = [];
 
-      result.push({
+    for (let wideRowIndex = 0; wideRowIndex < frame.length; wideRowIndex++) {
-        name: frame.name,
+      sortedTimeRowIndices.push({ time: timeField.values.get(wideRowIndex), wideRowIndex: wideRowIndex });
-        refId: frame.refId,
-        meta: {
-          ...frame.meta,
-          type: DataFrameType.TimeSeriesMany,
-        },
-        fields: [timeField, field],
-        length: frame.length,
-      });
     }
+
+    for (const labelKeys in labelKeyToWideIndices) {
+      sortedUniqueLabelKeys.push(labelKeys);
+    }
+    for (const labelKey in uniqueLabelKeys) {
+      uniqueFactorNames.push(labelKey);
+    }
+    for (const name in uniqueFactorNamesToWideIndex) {
+      uniqueFactorNamesWithWideIndices.push(name);
+    }
+
+    sortedTimeRowIndices.sort((a, b) => a.time - b.time);
+    sortedUniqueLabelKeys.sort();
+    uniqueFactorNames.sort();
+    uniqueValueNames.sort();
+
+    const longFrame = new MutableDataFrame({
+      ...frame,
+      meta: { ...frame.meta, type: DataFrameType.TimeSeriesLong },
+      fields: [{ name: timeField.name, type: timeField.type }],
+    });
+
+    for (const name of uniqueValueNames) {
+      longFrame.addField({ name: name, type: uniqueValueNamesToType[name] });
+    }
+
+    for (const name of uniqueFactorNames) {
+      longFrame.addField({ name: name, type: FieldType.string });
+    }
+
+    for (const timeWideRowIndex of sortedTimeRowIndices) {
+      const { time, wideRowIndex } = timeWideRowIndex;
+
+      for (const labelKeys of sortedUniqueLabelKeys) {
+        const rowValues: Record<string, any> = {};
+
+        for (const name of uniqueFactorNamesWithWideIndices) {
+          rowValues[name] = frame.fields[uniqueFactorNamesToWideIndex[name]].values.get(wideRowIndex);
+        }
+
+        let index = 0;
+
+        for (const wideFieldIndex of labelKeyToWideIndices[labelKeys]) {
+          const wideField = frame.fields[wideFieldIndex];
+
+          if (index++ === 0 && wideField.labels != null) {
+            for (const labelKey in wideField.labels) {
+              rowValues[labelKey] = wideField.labels[labelKey];
+            }
+          }
+
+          rowValues[wideField.name] = wideField.values.get(wideRowIndex);
+        }
+
+        rowValues[timeField.name] = time;
+        longFrame.add(rowValues);
+      }
+    }
+
+    result.push(longFrame);
   }
+
   return result;
 }
 
 export const prepareTimeSeriesTransformer: SynchronousDataTransformerInfo<PrepareTimeSeriesOptions> = {
   id: DataTransformerID.prepareTimeSeries,
   name: 'Prepare time series',
-  description: `Will stretch data frames from the wide format into the long format. This is really helpful to be able to keep backwards compatability for panels not supporting the new wide format.`,
+  description: `Will stretch data frames from the wide format into the long format. This is really helpful to be able to keep backwards compatibility for panels not supporting the new wide format.`,
   defaultOptions: {},
 
   operator: (options) => (source) =>
@@ -80,6 +292,8 @@ export const prepareTimeSeriesTransformer: SynchronousDataTransformerInfo<Prepar
     const format = options?.format ?? timeSeriesFormat.TimeSeriesWide;
     if (format === timeSeriesFormat.TimeSeriesMany) {
       return toTimeSeriesMany;
+    } else if (format === timeSeriesFormat.TimeSeriesLong) {
+      return toTimeSeriesLong;
     }
 
     return (data: DataFrame[]) => {