Arrow: toArray() on nullable values should include null values (#29520)

packages/grafana-data/src/dataframe/ArrowDataFrame.ts

@@ -55,6 +55,9 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
         case ArrowType.Decimal:
         case ArrowType.FloatingPoint: {
           type = FieldType.number;
+          if (col.nullCount) {
+            values = new WrappedColumn(col);
+          }
           break;
         }
         case ArrowType.Int: {
@@ -64,6 +67,9 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
         }
         case ArrowType.Bool: {
           type = FieldType.boolean;
+          if (col.nullCount) {
+            values = new WrappedColumn(col);
+          }
           break;
         }
         case ArrowType.Timestamp: {
@@ -202,3 +208,20 @@ class NumberColumn extends FunctionalVector<number> {
     return Number(v);
   }
 }
+
+// The `toArray()` arrow function will return a native ArrayBuffer -- this works fine
+// if there are no null values, but the native arrays do not support nulls.  This
+// class simply wraps the vector so `toArray` creates a new array
+class WrappedColumn extends FunctionalVector {
+  constructor(private col: Column) {
+    super();
+  }
+
+  get length() {
+    return this.col.length;
+  }
+
+  get(index: number): number {
+    return this.col.get(index);
+  }
+}

packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap

@@ -265,7 +265,7 @@ Object {
       "labels": undefined,
       "name": "float32_values",
       "type": "number",
-      "values": Array [
+      "values": Float32Array [
         1.401298464324817e-45,
         1.401298464324817e-45,
         1,

packages/grafana-data/src/dataframe/processDataFrame.ts

@@ -24,6 +24,7 @@ import { SortedVector } from '../vector/SortedVector';
 import { ArrayDataFrame } from './ArrayDataFrame';
 import { getFieldDisplayName } from '../field/fieldState';
 import { fieldIndexComparer } from '../field/fieldComparers';
+import { vectorToArray } from '../vector/vectorToArray';
 
 function convertTableToDataFrame(table: TableData): DataFrame {
   const fields = table.columns.map(c => {
@@ -444,16 +445,10 @@ export function getDataFrameRow(data: DataFrame, row: number): any[] {
 export function toDataFrameDTO(data: DataFrame): DataFrameDTO {
   const fields: FieldDTO[] = data.fields.map(f => {
     let values = f.values.toArray();
-    if (!Array.isArray(values)) {
+    // The byte buffers serialize like objects
-      // Apache arrow will pack objects into typed arrays
+    if (values instanceof Float64Array) {
-      // Float64Array, etc
+      values = vectorToArray(f.values);
-      // TODO: Float64Array could be used directly
-      values = [];
-      for (let i = 0; i < f.values.length; i++) {
-        values.push(f.values.get(i));
-      }
     }
-
     return {
       name: f.name,
       type: f.type,

packages/grafana-runtime/src/utils/queryResponse.test.ts

@@ -37,7 +37,7 @@ const emptyResults = {
 
 /* eslint-enable */
 
-describe('GEL Utils', () => {
+describe('Query Response parser', () => {
   test('should parse output with dataframe', () => {
     const res = toDataQueryResponse(resp);
     const frames = res.data;