DataFrame: add utilities to @grafana/data that support apache arrow (#20813)

package.json

@@ -205,7 +205,6 @@
     "angular-native-dragdrop": "1.2.2",
     "angular-route": "1.6.6",
     "angular-sanitize": "1.6.6",
-    "apache-arrow": "0.15.0",
     "baron": "3.0.3",
     "brace": "0.10.0",
     "calculate-size": "1.1.1",

packages/grafana-data/package.json

@@ -29,6 +29,7 @@
     "@types/pretty-format": "20.0.1",
     "@types/react": "16.8.16",
     "@types/sinon": "^7.0.11",
+    "apache-arrow": "0.15.1",
     "lodash": "4.17.15",
     "pretty-format": "24.9.0",
     "rollup": "1.6.0",

packages/grafana-data/src/dataframe/arrow/ArrowDataFrame.test.ts

@@ -1,5 +1,5 @@
-import { gelResponseToDataFrames } from './util';
-import { toDataFrameDTO } from '@grafana/data';
+import { resultsToDataFrames } from './ArrowDataFrame';
+import { toDataFrameDTO } from '../processDataFrame';
 
 /* tslint:disable */
 const resp = {
@@ -19,22 +19,8 @@ const resp = {
 /* tslint:enable */
 
 describe('GEL Utils', () => {
-  // test('should parse sample GEL output', () => {
-  //   const frames = gelResponseToDataFrames(resp);
-  //   const frame = frames[0];
-  //   expect(frame.name).toEqual('BBB');
-  //   expect(frame.fields.length).toEqual(2);
-  //   expect(frame.length).toEqual(resp.Frames[0].fields[0].values.length);
-
-  //   const timeField = frame.fields[0];
-  //   expect(timeField.name).toEqual('Time');
-
-  //   // The whole response
-  //   expect(frames).toMatchSnapshot();
-  // });
-
   test('should parse output with dataframe', () => {
-    const frames = gelResponseToDataFrames(resp);
+    const frames = resultsToDataFrames(resp);
     for (const frame of frames) {
       console.log('Frame', frame.refId);
       for (const field of frame.fields) {

packages/grafana-data/src/dataframe/arrow/ArrowDataFrame.ts

@@ -1,6 +1,10 @@
-import { DataFrame, FieldType, Field, Vector } from '@grafana/data';
+import { DataFrame, FieldType, Field, Vector } from '../../types';
 import { Table, ArrowType } from 'apache-arrow';
 
+export interface ArrowDataFrame extends DataFrame {
+  table: Table;
+}
+
 export function base64StringToArrowTable(text: string): Table {
   const b64 = atob(text);
   const arr = Uint8Array.from(b64, c => {
@@ -13,7 +17,7 @@ function valueOrUndefined(val?: string) {
   return val ? val : undefined;
 }
 
-export function arrowTableToDataFrame(table: Table): DataFrame {
+export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
   const fields: Field[] = [];
 
   for (let i = 0; i < table.numCols; i++) {
@@ -56,10 +60,11 @@ export function arrowTableToDataFrame(table: Table): DataFrame {
     length: table.length,
     refId: valueOrUndefined(meta.get('refId')),
     name: valueOrUndefined(meta.get('name')),
+    table,
   };
 }
 
-export function gelResponseToDataFrames(rsp: any): DataFrame[] {
+export function resultsToDataFrames(rsp: any): DataFrame[] {
   const frames: DataFrame[] = [];
   for (const res of Object.values(rsp.results)) {
     for (const b of (res as any).dataframes) {

public/app/features/expressions/ExpressionDatasource.ts

@@ -68,8 +68,10 @@ export class ExpressionDatasourceApi extends DataSourceApi<ExpressionQuery> {
    * This makes the arrow libary loading async.
    */
   async toDataQueryResponse(rsp: any): Promise<DataQueryResponse> {
-    const { gelResponseToDataFrames } = await import(/* webpackChunkName: "apache-arrow-util" */ './util');
-    return { data: gelResponseToDataFrames(rsp) };
+    const { resultsToDataFrames } = await import(
+      /* webpackChunkName: "apache-arrow-util" */ '@grafana/data/src/dataframe/arrow/ArrowDataFrame'
+    );
+    return { data: resultsToDataFrames(rsp) };
   }
 
   testDatasource() {

yarn.lock

@@ -4425,10 +4425,10 @@ anymatch@^2.0.0:
     micromatch "^3.1.4"
     normalize-path "^2.1.1"
 
-apache-arrow@0.15.0:
-  version "0.15.0"
-  resolved "https://registry.yarnpkg.com/apache-arrow/-/apache-arrow-0.15.0.tgz#aa34ba635c5e73579566be4d25008c412dae31b9"
-  integrity sha512-TLF7Bq7hPolQgFUsKVvYk2Qq7axg8k8yQ4J1uCLitkiUotdYACqN/j3ZE03cVJvSnXRKmqS2TCIjN4h7rOJKEQ==
+apache-arrow@0.15.1:
+  version "0.15.1"
+  resolved "https://registry.yarnpkg.com/apache-arrow/-/apache-arrow-0.15.1.tgz#136c03e18c3fa2617b41999608e7e685b0966147"
+  integrity sha512-3H+sC789nWn8JDnMwfd2j19NJ4gMcdtbpp2Haa22wBoDGUbbA5FgD2OqfE9Mr4yPlJZFWVJDw7C1hgJo2UolxA==
   dependencies:
     "@types/flatbuffers" "^1.9.1"
     "@types/node" "^12.0.4"