DataFrames: add arrow test and capture metadata parsing errors (#21524)

packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts

@@ -1,6 +1,10 @@
+import fs from 'fs';
+import path from 'path';
+
 import { resultsToDataFrames, grafanaDataFrameToArrowTable, arrowTableToDataFrame } from './ArrowDataFrame';
 import { toDataFrameDTO, toDataFrame } from './processDataFrame';
 import { FieldType } from '../types';
+import { Table } from 'apache-arrow';
 
 /* tslint:disable */
 const resp = {
@@ -59,4 +63,12 @@ describe('Read/Write arrow Table to DataFrame', () => {
     const after = JSON.stringify(toDataFrameDTO(arrowTableToDataFrame(table)), null, 2);
     expect(after).toEqual(before);
   });
+
+  test('should parse output with dataframe', () => {
+    const fullpath = path.resolve(__dirname, './__snapshots__/all_types.golden.arrow');
+    const arrow = fs.readFileSync(fullpath);
+    const table = Table.from([arrow]);
+    const frame = arrowTableToDataFrame(table);
+    expect(toDataFrameDTO(frame)).toMatchSnapshot();
+  });
 });

packages/grafana-data/src/dataframe/ArrowDataFrame.ts

@@ -1,4 +1,4 @@
-import { DataFrame, FieldType, Field, Vector, FieldConfig, Labels } from '../types';
+import { DataFrame, FieldType, Field, Vector } from '../types';
 import {
   Table,
   ArrowType,
@@ -28,6 +28,17 @@ function valueOrUndefined(val?: string) {
   return val ? val : undefined;
 }
 
+function parseOptionalMeta(str?: string): any {
+  if (str && str.length && str !== '{}') {
+    try {
+      return JSON.parse(str);
+    } catch (err) {
+      console.warn('Error reading JSON from arrow metadata: ', str);
+    }
+  }
+  return undefined;
+}
+
 export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
   const fields: Field[] = [];
 
@@ -59,35 +70,23 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
         default:
           console.log('UNKNOWN Type:', schema);
       }
-      const labelsJson = col.metadata.get('labels');
-      const configJson = col.metadata.get('config');
-
-      let config: FieldConfig = {};
-      let labels: Labels | undefined = undefined;
-      if (labelsJson) {
-        labels = JSON.parse(labelsJson);
-      }
-      if (configJson) {
-        config = JSON.parse(configJson);
-      }
 
       fields.push({
         name: col.name,
         type,
-        config,
         values,
-        labels,
+        config: parseOptionalMeta(col.metadata.get('config')) || {},
+        labels: parseOptionalMeta(col.metadata.get('labels')),
       });
     }
   }
   const meta = table.schema.metadata;
-  const metaJson = valueOrUndefined(meta.get('meta'));
   return {
     fields,
     length: table.length,
     refId: valueOrUndefined(meta.get('refId')),
     name: valueOrUndefined(meta.get('name')),
-    meta: metaJson ? JSON.parse(metaJson) : undefined,
+    meta: parseOptionalMeta(meta.get('meta')),
     table,
   };
 }

packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap

@@ -102,3 +102,325 @@ Array [
   },
 ]
 `;
+
+exports[`Read/Write arrow Table to DataFrame should parse output with dataframe 1`] = `
+Object {
+  "fields": Array [
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "string_values",
+      "type": "string",
+      "values": Array [
+        "Grafana",
+        "❤️",
+        "Transforms",
+      ],
+    },
+    Object {
+      "config": Object {
+        "decimals": 2,
+        "filterable": false,
+        "links": Array [
+          Object {
+            "targetBlank": true,
+            "title": "Donate - The Sloth Conservation Foundation",
+            "url": "https://slothconservation.com/how-to-help/donate/",
+          },
+        ],
+        "max": null,
+        "min": null,
+        "noValue": "😤",
+        "nullValueMode": "null",
+        "title": "Grafana ❤️ (Previous should be heart emoji) 🦥 (Previous should be sloth emoji)",
+      },
+      "labels": undefined,
+      "name": "nullable_string_values",
+      "type": "string",
+      "values": Array [
+        "🦥",
+        null,
+        "update your unicode/font if no sloth, is 2019.",
+      ],
+    },
+    Object {
+      "config": Object {
+        "max": 1,
+        "min": 0,
+      },
+      "labels": undefined,
+      "name": "int8_values",
+      "type": "number",
+      "values": Array [
+        -128,
+        1,
+        127,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_int8_values",
+      "type": "number",
+      "values": Array [
+        -128,
+        null,
+        127,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "int16_values",
+      "type": "number",
+      "values": Array [
+        -32768,
+        1,
+        32767,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_int16_values",
+      "type": "number",
+      "values": Array [
+        -32768,
+        null,
+        32767,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "int32_values",
+      "type": "number",
+      "values": Array [
+        -2147483648,
+        1,
+        2147483647,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_int32_values",
+      "type": "number",
+      "values": Array [
+        -2147483648,
+        null,
+        2147483647,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "int64_values",
+      "type": "number",
+      "values": Array [
+        "\\"-9223372036854775808\\"",
+        "\\"1\\"",
+        "\\"9223372036854775807\\"",
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_int64_values",
+      "type": "number",
+      "values": Array [
+        "\\"-9223372036854775808\\"",
+        null,
+        "\\"9223372036854775807\\"",
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "uint8_values",
+      "type": "number",
+      "values": Array [
+        0,
+        1,
+        255,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_uint8_values",
+      "type": "number",
+      "values": Array [
+        0,
+        null,
+        255,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "uint16_values",
+      "type": "number",
+      "values": Array [
+        0,
+        1,
+        65535,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_uint16_values",
+      "type": "number",
+      "values": Array [
+        0,
+        null,
+        65535,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "uint32_values",
+      "type": "number",
+      "values": Array [
+        0,
+        1,
+        4294967295,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_uint32_values",
+      "type": "number",
+      "values": Array [
+        0,
+        null,
+        4294967295,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "uint64_values",
+      "type": "number",
+      "values": Array [
+        "\\"0\\"",
+        "\\"1\\"",
+        "\\"18446744073709551615\\"",
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_uint64_values",
+      "type": "number",
+      "values": Array [
+        "\\"0\\"",
+        null,
+        "\\"18446744073709551615\\"",
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "float32_values",
+      "type": "number",
+      "values": Array [
+        1.401298464324817e-45,
+        1,
+        3.4028234663852886e+38,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_float32_values",
+      "type": "number",
+      "values": Array [
+        1.401298464324817e-45,
+        null,
+        3.4028234663852886e+38,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "float64_values",
+      "type": "number",
+      "values": Array [
+        5e-324,
+        1,
+        1.7976931348623157e+308,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_float64_values",
+      "type": "number",
+      "values": Array [
+        5e-324,
+        null,
+        1.7976931348623157e+308,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "bool_values",
+      "type": "boolean",
+      "values": Array [
+        true,
+        true,
+        false,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_bool_values",
+      "type": "boolean",
+      "values": Array [
+        true,
+        null,
+        false,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "timestamps",
+      "type": "time",
+      "values": Array [
+        1568039445000,
+        1568039450000,
+        1568039455000,
+      ],
+    },
+    Object {
+      "config": Object {},
+      "labels": undefined,
+      "name": "nullable_timestamps",
+      "type": "time",
+      "values": Array [
+        1568039445000,
+        null,
+        1568039455000,
+      ],
+    },
+  ],
+  "meta": Object {
+    "limit": 4242,
+    "searchWords": Array [
+      "Grafana",
+      "❤️",
+      " 🦥 ",
+      "test",
+    ],
+  },
+  "name": "many_types",
+  "refId": "A",
+}
+`;