DataFrame: round trip metadata to arrow Table (#21277)

packages/grafana-data/rollup.config.ts

@@ -20,7 +20,7 @@ const buildCjsPackage = ({ env }) => {
         globals: {},
       },
     ],
-    external: ['lodash'], // Use Lodash from grafana
+    external: ['lodash', 'apache-arrow'], // Use Lodash & arrow from grafana
     plugins: [
       commonjs({
         include: /node_modules/,

packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts

@@ -1,5 +1,6 @@
-import { resultsToDataFrames } from './ArrowDataFrame';
+import { resultsToDataFrames, grafanaDataFrameToArrowTable, arrowTableToDataFrame } from './ArrowDataFrame';
-import { toDataFrameDTO } from '../processDataFrame';
+import { toDataFrameDTO, toDataFrame } from './processDataFrame';
+import { FieldType } from '../types';
 
 /* tslint:disable */
 const resp = {
@@ -33,3 +34,29 @@ describe('GEL Utils', () => {
     expect(norm).toMatchSnapshot();
   });
 });
+
+describe('Read/Write arrow Table to DataFrame', () => {
+  test('should parse output with dataframe', () => {
+    const frame = toDataFrame({
+      name: 'Hello',
+      refId: 'XYZ',
+      meta: {
+        aaa: 'xyz',
+        anything: 'xxx',
+      },
+      fields: [
+        { name: 'time', config: {}, type: FieldType.time, values: [1, 2, 3] },
+        { name: 'value', config: { min: 0, max: 50, unit: 'somthing' }, type: FieldType.number, values: [1, 2, 3] },
+        { name: 'str', config: {}, type: FieldType.string, values: ['a', 'b', 'c'] },
+      ],
+    });
+
+    const table = grafanaDataFrameToArrowTable(frame);
+    expect(table.length).toEqual(frame.length);
+
+    // Now back to DataFrame
+    const before = JSON.stringify(toDataFrameDTO(frame), null, 2);
+    const after = JSON.stringify(toDataFrameDTO(arrowTableToDataFrame(table)), null, 2);
+    expect(after).toEqual(before);
+  });
+});

packages/grafana-data/src/dataframe/ArrowDataFrame.ts

@@ -0,0 +1,150 @@
+import { DataFrame, FieldType, Field, Vector, FieldConfig, Labels } from '../types';
+import {
+  Table,
+  ArrowType,
+  Builder,
+  Vector as ArrowVector,
+  Float64,
+  DataType,
+  Utf8,
+  TimestampMillisecond,
+  Bool,
+  Column,
+} from 'apache-arrow';
+
+export interface ArrowDataFrame extends DataFrame {
+  table: Table;
+}
+
+export function base64StringToArrowTable(text: string): Table {
+  const b64 = atob(text);
+  const arr = Uint8Array.from(b64, c => {
+    return c.charCodeAt(0);
+  });
+  return Table.from(arr);
+}
+
+function valueOrUndefined(val?: string) {
+  return val ? val : undefined;
+}
+
+export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
+  const fields: Field[] = [];
+
+  for (let i = 0; i < table.numCols; i++) {
+    const col = table.getColumnAt(i);
+    if (col) {
+      const schema = table.schema.fields[i];
+      let type = FieldType.other;
+      const values: Vector<any> = col;
+      switch ((schema.typeId as unknown) as ArrowType) {
+        case ArrowType.Decimal:
+        case ArrowType.Int:
+        case ArrowType.FloatingPoint: {
+          type = FieldType.number;
+          break;
+        }
+        case ArrowType.Bool: {
+          type = FieldType.boolean;
+          break;
+        }
+        case ArrowType.Timestamp: {
+          type = FieldType.time;
+          break;
+        }
+        case ArrowType.Utf8: {
+          type = FieldType.string;
+          break;
+        }
+        default:
+          console.log('UNKNOWN Type:', schema);
+      }
+      const labelsJson = col.metadata.get('labels');
+      const configJson = col.metadata.get('config');
+
+      let config: FieldConfig = {};
+      let labels: Labels | undefined = undefined;
+      if (labelsJson) {
+        labels = JSON.parse(labelsJson);
+      }
+      if (configJson) {
+        config = JSON.parse(configJson);
+      }
+
+      fields.push({
+        name: col.name,
+        type,
+        config,
+        values,
+        labels,
+      });
+    }
+  }
+  const meta = table.schema.metadata;
+  const metaJson = valueOrUndefined(meta.get('meta'));
+  return {
+    fields,
+    length: table.length,
+    refId: valueOrUndefined(meta.get('refId')),
+    name: valueOrUndefined(meta.get('name')),
+    meta: metaJson ? JSON.parse(metaJson) : undefined,
+    table,
+  };
+}
+
+function toArrowVector(field: Field): ArrowVector {
+  // OR: Float64Vector.from([1, 2, 3]));
+
+  let type: DataType;
+  if (field.type === FieldType.number) {
+    type = new Float64();
+  } else if (field.type === FieldType.time) {
+    type = new TimestampMillisecond();
+  } else if (field.type === FieldType.boolean) {
+    type = new Bool();
+  } else if (field.type === FieldType.string) {
+    type = new Utf8();
+  } else {
+    type = new Utf8();
+  }
+  const builder = Builder.new({ type, nullValues: [null] });
+  field.values.toArray().forEach(builder.append.bind(builder));
+  return builder.finish().toVector();
+}
+
+export function grafanaDataFrameToArrowTable(data: DataFrame): Table {
+  const table = Table.new(
+    data.fields.map(field => {
+      const column = Column.new(field.name, toArrowVector(field));
+      if (field.labels) {
+        column.metadata.set('labels', JSON.stringify(field.labels));
+      }
+      if (field.config) {
+        column.metadata.set('config', JSON.stringify(field.config));
+      }
+      return column;
+    })
+  );
+  const metadata = table.schema.metadata;
+  if (data.name) {
+    metadata.set('name', data.name);
+  }
+  if (data.refId) {
+    metadata.set('refId', data.refId);
+  }
+  if (data.meta) {
+    metadata.set('meta', JSON.stringify(data.meta));
+  }
+  return table;
+}
+
+export function resultsToDataFrames(rsp: any): DataFrame[] {
+  const frames: DataFrame[] = [];
+  for (const res of Object.values(rsp.results)) {
+    for (const b of (res as any).dataframes) {
+      const t = base64StringToArrowTable(b as string);
+      frames.push(arrowTableToDataFrame(t));
+    }
+  }
+  return frames;
+}

packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap

@@ -9,33 +9,20 @@ Array [
         "labels": undefined,
         "name": "Time",
         "type": "time",
-        "values": Int32Array [
+        "values": Array [
-          882710016,
+          1569334575000,
-          365389179,
+          1569334580000,
-          1587742720,
+          1569334585000,
-          365389180,
+          1569334590000,
-          -2002191872,
+          1569334595000,
-          365389181,
+          1569334600000,
-          -1297159168,
+          1569334605000,
-          365389182,
+          1569334610000,
-          -592126464,
+          1569334615000,
-          365389183,
+          1569334620000,
-          112906240,
+          1569334625000,
-          365389185,
+          1569334630000,
-          817938944,
+          1569334635000,
-          365389186,
-          1522971648,
-          365389187,
-          -2066962944,
-          365389188,
-          -1361930240,
-          365389189,
-          -656897536,
-          365389190,
-          48135168,
-          365389192,
-          753167872,
-          365389193,
         ],
       },
       Object {
@@ -43,7 +30,7 @@ Array [
         "labels": undefined,
         "name": "",
         "type": "number",
-        "values": Float64Array [
+        "values": Array [
           3,
           3,
           3,
@@ -71,33 +58,20 @@ Array [
         "labels": undefined,
         "name": "Time",
         "type": "time",
-        "values": Int32Array [
+        "values": Array [
-          882710016,
+          1569334575000,
-          365389179,
+          1569334580000,
-          1587742720,
+          1569334585000,
-          365389180,
+          1569334590000,
-          -2002191872,
+          1569334595000,
-          365389181,
+          1569334600000,
-          -1297159168,
+          1569334605000,
-          365389182,
+          1569334610000,
-          -592126464,
+          1569334615000,
-          365389183,
+          1569334620000,
-          112906240,
+          1569334625000,
-          365389185,
+          1569334630000,
-          817938944,
+          1569334635000,
-          365389186,
-          1522971648,
-          365389187,
-          -2066962944,
-          365389188,
-          -1361930240,
-          365389189,
-          -656897536,
-          365389190,
-          48135168,
-          365389192,
-          753167872,
-          365389193,
         ],
       },
       Object {
@@ -105,7 +79,7 @@ Array [
         "labels": undefined,
         "name": "GB-series",
         "type": "number",
-        "values": Float64Array [
+        "values": Array [
           0,
           0,
           0,

packages/grafana-data/src/dataframe/arrow/ArrowDataFrame.ts

@@ -1,76 +0,0 @@
-import { DataFrame, FieldType, Field, Vector } from '../../types';
-import { Table, ArrowType } from 'apache-arrow';
-
-export interface ArrowDataFrame extends DataFrame {
-  table: Table;
-}
-
-export function base64StringToArrowTable(text: string): Table {
-  const b64 = atob(text);
-  const arr = Uint8Array.from(b64, c => {
-    return c.charCodeAt(0);
-  });
-  return Table.from(arr);
-}
-
-function valueOrUndefined(val?: string) {
-  return val ? val : undefined;
-}
-
-export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
-  const fields: Field[] = [];
-
-  for (let i = 0; i < table.numCols; i++) {
-    const col = table.getColumnAt(i);
-    if (col) {
-      const schema = table.schema.fields[i];
-      let type = FieldType.other;
-      const values: Vector<any> = col;
-      switch ((schema.typeId as unknown) as ArrowType) {
-        case ArrowType.Decimal:
-        case ArrowType.Int:
-        case ArrowType.FloatingPoint: {
-          type = FieldType.number;
-          break;
-        }
-        case ArrowType.Bool: {
-          type = FieldType.boolean;
-          break;
-        }
-        case ArrowType.Timestamp: {
-          type = FieldType.time;
-          break;
-        }
-        default:
-          console.log('UNKNOWN Type:', schema);
-      }
-      // console.log(' field>', schema.metadata);
-
-      fields.push({
-        name: col.name,
-        type,
-        config: {}, // TODO, pull from metadata
-        values,
-      });
-    }
-  }
-  const meta = table.schema.metadata;
-  return {
-    fields,
-    length: table.length,
-    refId: valueOrUndefined(meta.get('refId')),
-    name: valueOrUndefined(meta.get('name')),
-    table,
-  };
-}
-
-export function resultsToDataFrames(rsp: any): DataFrame[] {
-  const frames: DataFrame[] = [];
-  for (const res of Object.values(rsp.results)) {
-    for (const b of (res as any).dataframes) {
-      const t = base64StringToArrowTable(b as string);
-      frames.push(arrowTableToDataFrame(t));
-    }
-  }
-  return frames;
-}

packages/grafana-data/src/dataframe/index.ts

@@ -4,10 +4,4 @@ export * from './CircularDataFrame';
 export * from './MutableDataFrame';
 export * from './processDataFrame';
 export * from './dimensions';
-
+export * from './ArrowDataFrame';
-// NOTE: We can not export arrow in the global scope because it will crash phantomjs
-// In core, this is loaded async.  In plugins you can import using:
-//
-// import { resultsToDataFrames } from '@grafana/data/dataframe/arrow/ArrowDataFrame'
-//
-// export * from './arrow/ArrowDataFrame';

packages/grafana-data/src/dataframe/processDataFrame.ts

@@ -442,11 +442,22 @@ export function getDataFrameRow(data: DataFrame, row: number): any[] {
  */
 export function toDataFrameDTO(data: DataFrame): DataFrameDTO {
   const fields: FieldDTO[] = data.fields.map(f => {
+    let values = f.values.toArray();
+    if (!Array.isArray(values)) {
+      // Apache arrow will pack objects into typed arrays
+      // Float64Array, etc
+      // TODO: Float64Array could be used directly
+      values = [];
+      for (let i = 0; i < f.values.length; i++) {
+        values.push(f.values.get(i));
+      }
+    }
+
     return {
       name: f.name,
       type: f.type,
       config: f.config,
-      values: f.values.toArray(),
+      values,
       labels: f.labels,
     };
   });

public/app/features/expressions/ExpressionDatasource.ts

@@ -69,7 +69,7 @@ export class ExpressionDatasourceApi extends DataSourceApi<ExpressionQuery> {
    */
   async toDataQueryResponse(rsp: any): Promise<DataQueryResponse> {
     const { resultsToDataFrames } = await import(
-      /* webpackChunkName: "apache-arrow-util" */ '@grafana/data/src/dataframe/arrow/ArrowDataFrame'
+      /* webpackChunkName: "apache-arrow-util" */ '@grafana/data/src/dataframe/ArrowDataFrame'
     );
     return { data: resultsToDataFrames(rsp) };
   }