DataFrameJSON: add string enums inflation of field values (#54938)

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>

.betterer.results

@@ -109,14 +109,11 @@ exports[`better eslint`] = {
     ],
     "packages/grafana-data/src/dataframe/DataFrameJSON.ts:5381": [
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "1"],
+      [0, 0, 0, "Do not use any type assertions.", "1"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "2"],
+      [0, 0, 0, "Do not use any type assertions.", "2"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "3"],
+      [0, 0, 0, "Do not use any type assertions.", "3"],
       [0, 0, 0, "Do not use any type assertions.", "4"],
-      [0, 0, 0, "Do not use any type assertions.", "5"],
+      [0, 0, 0, "Unexpected any. Specify a different type.", "5"]
-      [0, 0, 0, "Unexpected any. Specify a different type.", "6"],
-      [0, 0, 0, "Do not use any type assertions.", "7"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "8"]
     ],
     "packages/grafana-data/src/dataframe/DataFrameView.test.ts:5381": [
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
@@ -3708,6 +3705,9 @@ exports[`better eslint`] = {
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "1"]
     ],
+    "public/app/features/dashboard/components/DebugWizard/randomizer.ts:5381": [
+      [0, 0, 0, "Do not use any type assertions.", "0"]
+    ],
     "public/app/features/dashboard/components/Inspector/PanelInspectActions.tsx:5381": [
       [0, 0, 0, "Do not use any type assertions.", "0"],
       [0, 0, 0, "Do not use any type assertions.", "1"],
@@ -7110,10 +7110,10 @@ exports[`better eslint`] = {
       [0, 0, 0, "Unexpected any. Specify a different type.", "4"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "5"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "6"],
-      [0, 0, 0, "Do not use any type assertions.", "7"],
+      [0, 0, 0, "Unexpected any. Specify a different type.", "7"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "8"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "9"],
+      [0, 0, 0, "Do not use any type assertions.", "9"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "10"],
+      [0, 0, 0, "Do not use any type assertions.", "10"],
       [0, 0, 0, "Do not use any type assertions.", "11"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "12"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "13"],

packages/grafana-data/src/dataframe/DataFrameJSON.test.ts

@@ -83,5 +83,59 @@ describe('DataFrame JSON', () => {
         }
       `);
     });
+
+    it('should inflate values from enums and switch to string field type', () => {
+      const json: DataFrameJSON = {
+        schema: {
+          fields: [
+            { name: 'time', type: FieldType.time },
+            { name: 'value', type: FieldType.number },
+          ],
+        },
+        data: {
+          values: [
+            [100, 200, 300, 400],
+            [1, 0, 2, 1],
+          ],
+          enums: [
+            null, // nothing to replace, but keeps the index
+            ['foo', 'bar', 'baz'],
+          ],
+        },
+      };
+
+      const frame = dataFrameFromJSON(json);
+      expect(frame).toMatchInlineSnapshot(`
+        Object {
+          "fields": Array [
+            Object {
+              "config": Object {},
+              "entities": Object {},
+              "name": "time",
+              "type": "time",
+              "values": Array [
+                100,
+                200,
+                300,
+                400,
+              ],
+            },
+            Object {
+              "config": Object {},
+              "entities": Object {},
+              "name": "value",
+              "type": "string",
+              "values": Array [
+                "bar",
+                "foo",
+                "baz",
+                "bar",
+              ],
+            },
+          ],
+          "length": 4,
+        }
+      `);
+    });
   });
 });

packages/grafana-data/src/dataframe/DataFrameJSON.ts

@@ -20,6 +20,8 @@ export interface DataFrameJSON {
   data?: DataFrameData;
 }
 
+type FieldValues = unknown[];
+
 /**
  * @alpha
  */
@@ -27,7 +29,7 @@ export interface DataFrameData {
   /**
    * A columnar store that matches fields defined by schema.
    */
-  values: any[][];
+  values: FieldValues[];
 
   /**
    * Since JSON cannot encode NaN, Inf, -Inf, and undefined, these entities
@@ -48,10 +50,12 @@ export interface DataFrameData {
   factors?: number[];
 
   /**
-   * Holds enums per field so we can encode recurring values as ints
+   * Holds enums per field so we can encode recurring string values as ints
    * e.g. ["foo", "foo", "baz", "foo"] -> ["foo", "baz"] + [0,0,1,0]
+   *
+   * NOTE: currently only decoding is implemented
    */
-  enums?: any[][];
+  enums?: Array<string[] | null>;
 }
 
 /**
@@ -117,10 +121,7 @@ const ENTITY_MAP: Record<keyof FieldValueEntityLookup, any> = {
 /**
  * @internal use locally
  */
-export function decodeFieldValueEntities(lookup: FieldValueEntityLookup, values: any[]) {
+export function decodeFieldValueEntities(lookup: FieldValueEntityLookup, values: FieldValues) {
-  if (!lookup || !values) {
-    return;
-  }
   for (const key in lookup) {
     const repl = ENTITY_MAP[key as keyof FieldValueEntityLookup];
     for (const idx of lookup[key as keyof FieldValueEntityLookup]!) {
@@ -131,7 +132,16 @@ export function decodeFieldValueEntities(lookup: FieldValueEntityLookup, values:
   }
 }
 
-function guessFieldType(name: string, values: any[]): FieldType {
+/**
+ * @internal use locally
+ */
+export function decodeFieldValueEnums(lookup: string[], values: FieldValues) {
+  for (let i = 0; i < values.length; i++) {
+    values[i] = lookup[values[i] as number];
+  }
+}
+
+function guessFieldType(name: string, values: FieldValues): FieldType {
   for (const v of values) {
     if (v != null) {
       return guessFieldTypeFromNameAndValue(name, v);
@@ -157,6 +167,7 @@ export function dataFrameFromJSON(dto: DataFrameJSON): DataFrame {
   const fields = schema.fields.map((f, index) => {
     let buffer = data ? data.values[index] : [];
     let origLen = buffer.length;
+    let type = f.type;
 
     if (origLen !== length) {
       buffer.length = length;
@@ -164,17 +175,24 @@ export function dataFrameFromJSON(dto: DataFrameJSON): DataFrame {
       buffer.fill(undefined, origLen);
     }
 
-    let entities: FieldValueEntityLookup | undefined | null;
+    let entities = data?.entities?.[index];
 
-    if ((entities = data && data.entities && data.entities[index])) {
+    if (entities) {
       decodeFieldValueEntities(entities, buffer);
     }
 
-    // TODO: expand arrays further using bases,factors,enums
+    let enums = data?.enums?.[index];
+
+    if (enums) {
+      decodeFieldValueEnums(enums, buffer);
+      type = FieldType.string;
+    }
+
+    // TODO: expand arrays further using bases,factors
 
     return {
       ...f,
-      type: f.type ?? guessFieldType(f.name, buffer),
+      type: type ?? guessFieldType(f.name, buffer),
       config: f.config ?? {},
       values: new ArrayVector(buffer),
       // the presence of this prop is an optimization signal & lookup for consumers

public/app/features/dashboard/components/DebugWizard/randomizer.ts

@@ -66,7 +66,8 @@ export function randomizeData(data: DataFrameJSON[], opts: Randomize): DataFrame
       if (opts.values) {
         schema.fields.forEach((f, idx) => {
           if (f.type === FieldType.string && data) {
-            const v = data.values[idx].map((v) => rand(v));
+            // eslint-ignore-next-line
+            const v = data.values[idx].map((v) => rand(v as string));
             data.values[idx] = v;
           }
         });

public/app/features/live/data/StreamingDataFrame.ts

@@ -312,7 +312,7 @@ export class StreamingDataFrame implements DataFrame {
         this.fields = values.map((vals, idx) => {
           let name = `Field ${idx}`;
           let type = guessFieldTypeFromValue(vals[0]);
-          const isTime = idx === 0 && type === FieldType.number && vals[0] > 1600016688632;
+          const isTime = idx === 0 && type === FieldType.number && (vals as number[])[0] > 1600016688632;
           if (isTime) {
             type = FieldType.time;
             name = 'Time';

public/app/plugins/datasource/postgres/datasource.ts

@@ -82,7 +82,7 @@ export class PostgresDatasource extends DataSourceWithBackend<PostgresQuery, Pos
     return {
       refId: target.refId,
       datasource: this.getRef(),
-      rawSql: queryModel.render(this.interpolateVariable as any),
+      rawSql: queryModel.render(this.interpolateVariable),
       format: target.format,
     };
   }
@@ -189,7 +189,7 @@ export class PostgresDatasource extends DataSourceWithBackend<PostgresQuery, Pos
     const results = value.data.results['meta'];
     if (results.frames) {
       // This returns number
-      return results.frames[0].data?.values[0][0].toString();
+      return (results.frames[0].data?.values[0] as number[])[0].toString();
     }
     return '';
   }
@@ -200,7 +200,7 @@ export class PostgresDatasource extends DataSourceWithBackend<PostgresQuery, Pos
     );
     const results = value.data.results['meta'];
     if (results.frames) {
-      return results.frames[0].data?.values[0][0];
+      return results.frames[0].data?.values[0][0] as string[];
     }
     return undefined;
   }