API: return query results as JSON rather than base64 encoded Arrow (#32303)

packages/grafana-data/src/dataframe/DataFrameJSON.ts

@@ -194,3 +194,29 @@ export function dataFrameFromJSON(dto: DataFrameJSON): DataFrame {
     length,
   };
 }
+
+/**
+ * This converts DataFrame to a json representation with distinct schema+data
+ *
+ * @alpha
+ */
+export function dataFrameToJSON(frame: DataFrame): DataFrameJSON {
+  const data: DataFrameData = {
+    values: [],
+  };
+  const schema: DataFrameSchema = {
+    refId: frame.refId,
+    meta: frame.meta,
+    name: frame.name,
+    fields: frame.fields.map((f) => {
+      const { values, ...sfield } = f;
+      data.values.push(values.toArray());
+      return sfield;
+    }),
+  };
+
+  return {
+    schema,
+    data,
+  };
+}

packages/grafana-runtime/src/utils/queryResponse.test.ts

@@ -1,7 +1,6 @@
 import { DataQuery, toDataFrameDTO, DataFrame } from '@grafana/data';
 import { toDataQueryResponse } from './queryResponse';
 
-/* eslint-disable */
 const resp = {
   data: {
     results: {
@@ -38,8 +37,6 @@ const emptyResults = {
   data: { results: { '': { refId: '' } } },
 };
 
-/* eslint-enable */
-
 describe('Query Response parser', () => {
   test('should parse output with dataframe', () => {
     const res = toDataQueryResponse(resp);
@@ -235,7 +232,7 @@ describe('Query Response parser', () => {
     const queries: DataQuery[] = [{ refId: 'A' }, { refId: 'B' }];
 
     const ids = (toDataQueryResponse(resp, queries).data as DataFrame[]).map((f) => f.refId);
-    expect(ids).toEqual(['A', 'B', 'X']);
+    expect(ids).toEqual(['A', 'B']);
   });
 
   test('resultWithError', () => {

packages/grafana-runtime/src/utils/queryResponse.ts

@@ -12,6 +12,8 @@ import {
   MetricFindValue,
   FieldType,
   DataQuery,
+  DataFrameJSON,
+  dataFrameFromJSON,
 } from '@grafana/data';
 import { FetchResponse } from '../services';
 
@@ -25,8 +27,10 @@ import { FetchResponse } from '../services';
 export interface DataResponse {
   error?: string;
   refId?: string;
-  // base64 encoded arrow tables
-  dataframes?: string[];
+  frames?: DataFrameJSON[];
+
+  // Legacy TSDB format...
+  dataframes?: string[]; // base64 encoded arrow tables
   series?: TimeSeries[];
   tables?: TableData[];
 }
@@ -59,9 +63,7 @@ export function toDataQueryResponse(
   // If the response isn't in a correct shape we just ignore the data and pass empty DataQueryResponse.
   if ((res as FetchResponse).data?.results) {
     const results = (res as FetchResponse).data.results;
-    const resultIDs = Object.keys(results);
-    const refIDs = queries ? queries.map((q) => q.refId) : resultIDs;
-    const usedResultIDs = new Set<string>(resultIDs);
+    const refIDs = queries?.length ? queries.map((q) => q.refId) : Object.keys(results);
     const data: DataResponse[] = [];
 
     for (const refId of refIDs) {
@@ -70,23 +72,9 @@ export function toDataQueryResponse(
         continue;
       }
       dr.refId = refId;
-      usedResultIDs.delete(refId);
       data.push(dr);
     }
 
-    // Add any refIds that do not match the query targets
-    if (usedResultIDs.size) {
-      for (const refId of usedResultIDs) {
-        const dr = results[refId];
-        if (!dr) {
-          continue;
-        }
-        dr.refId = refId;
-        usedResultIDs.delete(refId);
-        data.push(dr);
-      }
-    }
-
     for (const dr of data) {
       if (dr.error) {
         if (!rsp.error) {
@@ -98,6 +86,17 @@ export function toDataQueryResponse(
         }
       }
 
+      if (dr.frames?.length) {
+        for (const js of dr.frames) {
+          const df = dataFrameFromJSON(js);
+          if (!df.refId) {
+            df.refId = dr.refId;
+          }
+          rsp.data.push(df);
+        }
+        continue; // the other tests are legacy
+      }
+
       if (dr.series?.length) {
         for (const s of dr.series) {
           if (!s.refId) {