Arrow DataFrame: cast BigInt/UInt to numbers (#25811)

packages/grafana-data/src/dataframe/ArrayDataFrame.test.ts

@@ -7,13 +7,15 @@ describe('Array DataFrame', () => {
     { name: 'first', value: 1, time: 123 },
     { name: 'second', value: 2, time: 456, extra: 'here' },
     { name: 'third', value: 3, time: 789 },
+    { name: '4th (NaN)', value: NaN, time: 1000 },
+    { name: '5th (Null)', value: null, time: 1100 },
   ];
 
   const frame = new ArrayDataFrame(input);
   frame.name = 'Hello';
   frame.refId = 'Z';
   frame.setFieldType('phantom', FieldType.string, v => '🦥');
-  const field = frame.fields.find(f => f.name == 'value');
+  const field = frame.fields.find(f => f.name === 'value');
   field!.config.unit = 'kwh';
 
   test('Should support functional methods', () => {
@@ -48,6 +50,8 @@ describe('Array DataFrame', () => {
               "first",
               "second",
               "third",
+              "4th (NaN)",
+              "5th (Null)",
             ],
           },
           Object {
@@ -61,6 +65,8 @@ describe('Array DataFrame', () => {
               1,
               2,
               3,
+              NaN,
+              null,
             ],
           },
           Object {
@@ -72,6 +78,8 @@ describe('Array DataFrame', () => {
               123,
               456,
               789,
+              1000,
+              1100,
             ],
           },
           Object {
@@ -83,6 +91,8 @@ describe('Array DataFrame', () => {
               "🦥",
               "🦥",
               "🦥",
+              "🦥",
+              "🦥",
             ],
           },
         ],

packages/grafana-data/src/dataframe/ArrowDataFrame.ts

@@ -1,4 +1,5 @@
 import { DataFrame, FieldType, Field, Vector } from '../types';
+import { FunctionalVector } from '../vector/FunctionalVector';
 
 import {
   Table,
@@ -48,14 +49,18 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
     if (col) {
       const schema = table.schema.fields[i];
       let type = FieldType.other;
-      const values: Vector<any> = col;
+      let values: Vector<any> = col;
       switch ((schema.typeId as unknown) as ArrowType) {
         case ArrowType.Decimal:
-        case ArrowType.Int:
         case ArrowType.FloatingPoint: {
           type = FieldType.number;
           break;
         }
+        case ArrowType.Int: {
+          type = FieldType.number;
+          values = new NumberColumn(col); // Cast to number
+          break;
+        }
         case ArrowType.Bool: {
           type = FieldType.boolean;
           break;
@@ -73,7 +78,7 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
       }
 
       fields.push({
-        name: stripFieldNamePrefix(col.name),
+        name: col.name,
         type,
         values,
         config: parseOptionalMeta(col.metadata.get('config')) || {},
@@ -92,17 +97,6 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
   };
 }
 
-// fieldNamePrefixSep is the delimiter used with fieldNamePrefix.
-const fieldNamePrefixSep = '🦥: ';
-
-function stripFieldNamePrefix(name: string): string {
-  const idx = name.indexOf(fieldNamePrefixSep);
-  if (idx > 0) {
-    return name.substring(idx + fieldNamePrefixSep.length);
-  }
-  return name;
-}
-
 function toArrowVector(field: Field): ArrowVector {
   // OR: Float64Vector.from([1, 2, 3]));
 
@@ -129,17 +123,10 @@ export function grafanaDataFrameToArrowTable(data: DataFrame): Table {
   if (table instanceof Table) {
     return table as Table;
   }
-  // Make sure the names are unique
-  const names = new Set<string>();
 
   table = Table.new(
     data.fields.map((field, index) => {
-      let name = field.name;
+      const column = Column.new(field.name, toArrowVector(field));
-      if (names.has(field.name)) {
-        name = `${index}${fieldNamePrefixSep}${field.name}`;
-      }
-      names.add(name);
-      const column = Column.new(name, toArrowVector(field));
       if (field.labels) {
         column.metadata.set('labels', JSON.stringify(field.labels));
       }
@@ -161,3 +148,26 @@ export function grafanaDataFrameToArrowTable(data: DataFrame): Table {
   }
   return table;
 }
+
+class NumberColumn extends FunctionalVector<number> {
+  constructor(private col: Column) {
+    super();
+  }
+
+  get length() {
+    return this.col.length;
+  }
+
+  get(index: number): number {
+    const v = this.col.get(index);
+    if (v === null || isNaN(v)) {
+      return v;
+    }
+
+    // The conversion operations are always silent, never give errors,
+    // but if the bigint is too huge and won’t fit the number type,
+    // then extra bits will be cut off, so we should be careful doing such conversion.
+    // See https://javascript.info/bigint
+    return Number(v);
+  }
+}

packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap

@@ -21,6 +21,7 @@ Object {
     Object {
       "config": Object {
         "decimals": 2,
+        "displayName": "Grafana ❤️ (Previous should be heart emoji) 🦥 (Previous should be sloth emoji)",
         "filterable": false,
         "links": Array [
           Object {
@@ -33,7 +34,6 @@ Object {
         "min": null,
         "noValue": "😤",
         "nullValueMode": "null",
-        "title": "Grafana ❤️ (Previous should be heart emoji) 🦥 (Previous should be sloth emoji)",
       },
       "labels": Object {
         "aLabelKey": "aLabelValue",
@@ -136,11 +136,11 @@ Object {
       "name": "int64_values",
       "type": "number",
       "values": Array [
-        "\\"-9223372036854775808\\"",
+        -9223372036854776000,
-        "\\"-9007199254740991\\"",
+        -9007199254740991,
-        "\\"1\\"",
+        1,
-        "\\"9007199254740991\\"",
+        9007199254740991,
-        "\\"9223372036854775807\\"",
+        9223372036854776000,
       ],
     },
     Object {
@@ -149,11 +149,11 @@ Object {
       "name": "nullable_int64_values",
       "type": "number",
       "values": Array [
-        "\\"-9223372036854775808\\"",
+        -9223372036854776000,
-        "\\"-9007199254740991\\"",
+        -9007199254740991,
         null,
-        "\\"9007199254740991\\"",
+        9007199254740991,
-        "\\"9223372036854775807\\"",
+        9223372036854776000,
       ],
     },
     Object {
@@ -240,11 +240,11 @@ Object {
       "name": "uint64_values",
       "type": "number",
       "values": Array [
-        "\\"0\\"",
+        0,
-        "\\"0\\"",
+        0,
-        "\\"1\\"",
+        1,
-        "\\"9007199254740991\\"",
+        9007199254740991,
-        "\\"18446744073709551615\\"",
+        18446744073709552000,
       ],
     },
     Object {
@@ -253,11 +253,11 @@ Object {
       "name": "nullable_uint64_values",
       "type": "number",
       "values": Array [
-        "\\"0\\"",
+        0,
-        "\\"0\\"",
+        0,
         null,
-        "\\"9007199254740991\\"",
+        9007199254740991,
-        "\\"18446744073709551615\\"",
+        18446744073709552000,
       ],
     },
     Object {
@@ -351,6 +351,19 @@ Object {
         9223372036854.775,
       ],
     },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "timestamps",
+      "type": "time",
+      "values": Array [
+        0,
+        1568039445000,
+        1568039450000,
+        9007199254.740992,
+        9223372036854.775,
+      ],
+    },
     Object {
       "config": Object {},
       "labels": undefined,
@@ -366,13 +379,9 @@ Object {
     },
   ],
   "meta": Object {
-    "limit": 4242,
+    "custom": Object {
-    "searchWords": Array [
+      "Hi": "there",
-      "Grafana",
+    },
-      "❤️",
-      " 🦥 ",
-      "test",
-    ],
   },
   "name": "many_types",
   "refId": "A",