DataFrame: convert from row based to a columnar value format (#18391)

packages/grafana-ui/src/components/SingleStatShared/FieldDisplayEditor.tsx

@@ -9,7 +9,7 @@ import { StatsPicker } from '../StatsPicker/StatsPicker';
 // Types
 import { FieldDisplayOptions, DEFAULT_FIELD_DISPLAY_VALUES_LIMIT } from '../../utils/fieldDisplay';
 import Select from '../Select/Select';
-import { Field, ReducerID, toNumberString, toIntegerOrUndefined, SelectableValue } from '@grafana/data';
+import { ReducerID, toNumberString, toIntegerOrUndefined, SelectableValue, FieldConfig } from '@grafana/data';
 
 const showOptions: Array<SelectableValue<boolean>> = [
   {
@@ -40,7 +40,7 @@ export class FieldDisplayEditor extends PureComponent<Props> {
     this.props.onChange({ ...this.props.value, calcs });
   };
 
-  onDefaultsChange = (value: Partial<Field>) => {
+  onDefaultsChange = (value: FieldConfig) => {
     this.props.onChange({ ...this.props.value, defaults: value });
   };
 

packages/grafana-ui/src/components/SingleStatShared/FieldPropertiesEditor.tsx

@@ -7,7 +7,7 @@ import { FormLabel } from '../FormLabel/FormLabel';
 import { UnitPicker } from '../UnitPicker/UnitPicker';
 
 // Types
-import { toIntegerOrUndefined, Field, SelectableValue, toFloatOrUndefined, toNumberString } from '@grafana/data';
+import { toIntegerOrUndefined, SelectableValue, FieldConfig, toFloatOrUndefined, toNumberString } from '@grafana/data';
 
 import { VAR_SERIES_NAME, VAR_FIELD_NAME, VAR_CALC, VAR_CELL_PREFIX } from '../../utils/fieldDisplay';
 
@@ -15,8 +15,8 @@ const labelWidth = 6;
 
 export interface Props {
   showMinMax: boolean;
-  value: Partial<Field>;
-  onChange: (value: Partial<Field>, event?: React.SyntheticEvent<HTMLElement>) => void;
+  value: FieldConfig;
+  onChange: (value: FieldConfig, event?: React.SyntheticEvent<HTMLElement>) => void;
 }
 
 export const FieldPropertiesEditor: React.FC<Props> = ({ value, onChange, showMinMax }) => {

packages/grafana-ui/src/components/Table/Table.story.tsx

@@ -5,7 +5,7 @@ import { getTheme } from '../../themes';
 
 import { migratedTestTable, migratedTestStyles, simpleTable } from './examples';
 import { ScopedVars, GrafanaThemeType } from '../../types/index';
-import { DataFrame } from '@grafana/data';
+import { DataFrame, FieldType, ArrayVector } from '@grafana/data';
 import { withFullSizeStory } from '../../utils/storybook/withFullSizeStory';
 import { number, boolean } from '@storybook/addon-knobs';
 
@@ -33,14 +33,19 @@ export function columnIndexToLeter(column: number) {
 export function makeDummyTable(columnCount: number, rowCount: number): DataFrame {
   return {
     fields: Array.from(new Array(columnCount), (x, i) => {
+      const colId = columnIndexToLeter(i);
+      const values = new ArrayVector<string>();
+      for (let i = 0; i < rowCount; i++) {
+        values.buffer.push(colId + (i + 1));
+      }
       return {
-        name: columnIndexToLeter(i),
+        name: colId,
+        type: FieldType.string,
+        config: {},
+        values,
       };
     }),
-    rows: Array.from(new Array(rowCount), (x, rowId) => {
-      const suffix = (rowId + 1).toString();
-      return Array.from(new Array(columnCount), (x, colId) => columnIndexToLeter(colId) + suffix);
-    }),
+    length: rowCount,
   };
 }
 

packages/grafana-ui/src/components/Table/Table.tsx

@@ -12,7 +12,7 @@ import {
 } from 'react-virtualized';
 import { Themeable } from '../../types/theme';
 
-import { stringToJsRegex, DataFrame, sortDataFrame } from '@grafana/data';
+import { stringToJsRegex, DataFrame, sortDataFrame, getDataFrameRow, ArrayVector, FieldType } from '@grafana/data';
 
 import {
   TableCellBuilder,
@@ -107,7 +107,7 @@ export class Table extends Component<Props, State> {
 
     if (dataChanged || rotate !== prevProps.rotate) {
       const { width, minColumnWidth } = this.props;
-      this.rotateWidth = Math.max(width / data.rows.length, minColumnWidth);
+      this.rotateWidth = Math.max(width / data.length, minColumnWidth);
     }
 
     // Update the data when data or sort changes
@@ -146,7 +146,7 @@ export class Table extends Component<Props, State> {
       return {
         header: title,
         width: columnWidth,
-        builder: getCellBuilder(col, style, this.props),
+        builder: getCellBuilder(col.config || {}, style, this.props),
       };
     });
   }
@@ -185,9 +185,9 @@ export class Table extends Component<Props, State> {
     if (row < 0) {
       this.doSort(column);
     } else {
-      const values = this.state.data.rows[row];
-      const value = values[column];
-      console.log('CLICK', value, row);
+      const field = this.state.data.fields[columnIndex];
+      const value = field.values.get(rowIndex);
+      console.log('CLICK', value, field.name);
     }
   };
 
@@ -201,6 +201,9 @@ export class Table extends Component<Props, State> {
     if (!col) {
       col = {
         name: '??' + columnIndex + '???',
+        config: {},
+        values: new ArrayVector(),
+        type: FieldType.other,
       };
     }
 
@@ -226,7 +229,7 @@ export class Table extends Component<Props, State> {
     const { data } = this.state;
 
     const isHeader = row < 0;
-    const rowData = isHeader ? data.fields : data.rows[row];
+    const rowData = isHeader ? data.fields : getDataFrameRow(data, row); // TODO! improve
     const value = rowData ? rowData[column] : '';
     const builder = isHeader ? this.headerBuilder : this.getTableCellBuilder(column);
 
@@ -258,7 +261,7 @@ export class Table extends Component<Props, State> {
     }
 
     let columnCount = data.fields.length;
-    let rowCount = data.rows.length + (showHeader ? 1 : 0);
+    let rowCount = data.length + (showHeader ? 1 : 0);
 
     let fixedColumnCount = Math.min(fixedColumns, columnCount);
     let fixedRowCount = showHeader && fixedHeader ? 1 : 0;

packages/grafana-ui/src/components/Table/TableCellBuilder.tsx

@@ -6,7 +6,7 @@ import { Table, Props } from './Table';
 import { ValueFormatter, getValueFormat, getColorFromHexRgbOrName } from '../../utils/index';
 import { GrafanaTheme } from '../../types/theme';
 import { InterpolateFunction } from '../../types/panel';
-import { Field, dateTime } from '@grafana/data';
+import { Field, dateTime, FieldConfig } from '@grafana/data';
 
 export interface TableCellBuilderOptions {
   value: any;
@@ -73,7 +73,7 @@ export interface ColumnStyle {
 // private replaceVariables: InterpolateFunction,
 // private fmt?:ValueFormatter) {
 
-export function getCellBuilder(schema: Field, style: ColumnStyle | null, props: Props): TableCellBuilder {
+export function getCellBuilder(schema: FieldConfig, style: ColumnStyle | null, props: Props): TableCellBuilder {
   if (!style) {
     return simpleCellBuilder;
   }
@@ -153,7 +153,7 @@ class CellBuilderWithStyle {
     private mapper: ValueMapper,
     private style: ColumnStyle,
     private theme: GrafanaTheme,
-    private column: Field,
+    private schema: FieldConfig,
     private replaceVariables: InterpolateFunction,
     private fmt?: ValueFormatter
   ) {}
@@ -244,7 +244,7 @@ class CellBuilderWithStyle {
     }
 
     // ??? I don't think this will still work!
-    if (this.column.filterable) {
+    if (this.schema.filterable) {
       cellClasses.push('table-panel-cell-filterable');
       value = (
         <>

packages/grafana-ui/src/components/Table/TableInputCSV.tsx

@@ -71,10 +71,10 @@ export class TableInputCSV extends React.PureComponent<Props, State> {
         />
         {data && (
           <footer>
-            {data.map((series, index) => {
+            {data.map((frame, index) => {
               return (
                 <span key={index}>
-                  Rows:{series.rows.length}, Columns:{series.fields.length} &nbsp;
+                  Rows:{frame.length}, Columns:{frame.fields.length} &nbsp;
                   <i className="fa fa-check-circle" />
                 </span>
               );

packages/grafana-ui/src/components/Table/examples.ts

@@ -1,12 +1,12 @@
-import { DataFrame } from '@grafana/data';
+import { toDataFrame } from '@grafana/data';
 import { ColumnStyle } from './TableCellBuilder';
 import { getColorDefinitionByName } from '../../utils/namedColorsPalette';
 
 const SemiDarkOrange = getColorDefinitionByName('semi-dark-orange');
 
-export const migratedTestTable = {
+export const migratedTestTable = toDataFrame({
   type: 'table',
-  fields: [
+  columns: [
     { name: 'Time' },
     { name: 'Value' },
     { name: 'Colored' },
@@ -22,7 +22,7 @@ export const migratedTestTable = {
     { name: 'RangeMappingColored' },
   ],
   rows: [[1388556366666, 1230, 40, undefined, '', '', 'my.host.com', 'host1', ['value1', 'value2'], 1, 2, 1, 2]],
-} as DataFrame;
+});
 
 export const migratedTestStyles: ColumnStyle[] = [
   {

packages/grafana-ui/src/types/datasource.ts

@@ -1,5 +1,14 @@
 import { ComponentType, ComponentClass } from 'react';
-import { TimeRange, RawTimeRange, TableData, TimeSeries, DataFrame, LogRowModel, LoadingState } from '@grafana/data';
+import {
+  TimeRange,
+  RawTimeRange,
+  TableData,
+  TimeSeries,
+  DataFrame,
+  LogRowModel,
+  LoadingState,
+  DataFrameDTO,
+} from '@grafana/data';
 import { PluginMeta, GrafanaPlugin } from './plugin';
 import { PanelData } from './panel';
 
@@ -286,7 +295,7 @@ export interface ExploreStartPageProps {
  */
 export type LegacyResponseData = TimeSeries | TableData | any;
 
-export type DataQueryResponseData = DataFrame | LegacyResponseData;
+export type DataQueryResponseData = DataFrameDTO | LegacyResponseData;
 
 export type DataStreamObserver = (event: DataStreamState) => void;
 

packages/grafana-ui/src/utils/displayValue.test.ts

@@ -1,6 +1,6 @@
-import { MappingType, ValueMapping, DisplayValue } from '@grafana/data';
+import { MappingType, ValueMapping, DisplayProcessor, DisplayValue } from '@grafana/data';
 
-import { getDisplayProcessor, getColorFromThreshold, DisplayProcessor, getDecimalsForValue } from './displayValue';
+import { getDisplayProcessor, getColorFromThreshold, getDecimalsForValue } from './displayValue';
 
 function assertSame(input: any, processors: DisplayProcessor[], match: DisplayValue) {
   processors.forEach(processor => {

packages/grafana-ui/src/utils/displayValue.ts

@@ -1,6 +1,14 @@
 // Libraries
 import _ from 'lodash';
-import { Threshold, getMappedValue, Field, DecimalInfo, DisplayValue, DecimalCount } from '@grafana/data';
+import {
+  Threshold,
+  getMappedValue,
+  FieldConfig,
+  DisplayProcessor,
+  DecimalInfo,
+  DisplayValue,
+  DecimalCount,
+} from '@grafana/data';
 
 // Utils
 import { getValueFormat } from './valueFormats/valueFormats';
@@ -9,13 +17,8 @@ import { getColorFromHexRgbOrName } from './namedColorsPalette';
 // Types
 import { GrafanaTheme, GrafanaThemeType } from '../types';
 
-export type DisplayProcessor = (value: any) => DisplayValue;
-
 export interface DisplayValueOptions {
-  field?: Partial<Field>;
-
-  // Alternative to empty string
-  noValue?: string;
+  field?: FieldConfig;
 
   // Context
   isUtc?: boolean;
@@ -62,7 +65,11 @@ export function getDisplayProcessor(options?: DisplayValueOptions): DisplayProce
       }
 
       if (!text) {
-        text = options.noValue ? options.noValue : '';
+        if (field && field.noValue) {
+          text = field.noValue;
+        } else {
+          text = ''; // No data?
+        }
       }
       return { text, numeric, color };
     };

packages/grafana-ui/src/utils/fieldDisplay.test.ts

@@ -1,5 +1,5 @@
 import { getFieldProperties, getFieldDisplayValues, GetFieldDisplayValuesOptions } from './fieldDisplay';
-import { FieldType, ReducerID, Threshold } from '@grafana/data';
+import { ReducerID, Threshold, DataFrameHelper } from '@grafana/data';
 import { GrafanaThemeType } from '../types/theme';
 import { getTheme } from '../themes/index';
 
@@ -34,19 +34,14 @@ describe('FieldDisplay', () => {
   // Simple test dataset
   const options: GetFieldDisplayValuesOptions = {
     data: [
-      {
+      new DataFrameHelper({
         name: 'Series Name',
         fields: [
-          { name: 'Field 1', type: FieldType.string },
-          { name: 'Field 2', type: FieldType.number },
-          { name: 'Field 3', type: FieldType.number },
+          { name: 'Field 1', values: ['a', 'b', 'c'] },
+          { name: 'Field 2', values: [1, 3, 5] },
+          { name: 'Field 3', values: [2, 4, 6] },
         ],
-        rows: [
-          ['a', 1, 2], // 0
-          ['b', 3, 4], // 1
-          ['c', 5, 6], // 2
-        ],
-      },
+      }),
     ],
     replaceVariables: (value: string) => {
       return value; // Return it unchanged
@@ -140,7 +135,7 @@ describe('FieldDisplay', () => {
         {
           name: 'No data',
           fields: [],
-          rows: [],
+          length: 0,
         },
       ],
       replaceVariables: (value: string) => {

packages/grafana-ui/src/utils/fieldDisplay.ts

@@ -2,9 +2,8 @@ import {
   ReducerID,
   reduceField,
   FieldType,
-  NullValueMode,
   DataFrame,
-  Field,
+  FieldConfig,
   DisplayValue,
   GraphSeriesValue,
 } from '@grafana/data';
@@ -21,8 +20,8 @@ export interface FieldDisplayOptions {
   limit?: number; // if showing all values limit
   calcs: string[]; // when !values, pick one value for the whole field
 
-  defaults: Partial<Field>; // Use these values unless otherwise stated
-  override: Partial<Field>; // Set these values regardless of the source
+  defaults: FieldConfig; // Use these values unless otherwise stated
+  override: FieldConfig; // Set these values regardless of the source
 }
 
 export const VAR_SERIES_NAME = '__series_name';
@@ -60,7 +59,8 @@ function getTitleTemplate(title: string | undefined, stats: string[], data?: Dat
 }
 
 export interface FieldDisplay {
-  field: Field;
+  name: string; // NOT title!
+  field: FieldConfig;
   display: DisplayValue;
   sparkline?: GraphSeriesValue[][];
 }
@@ -109,45 +109,50 @@ export const getFieldDisplayValues = (options: GetFieldDisplayValuesOptions): Fi
       }
 
       for (let i = 0; i < series.fields.length && !hitLimit; i++) {
-        const field = getFieldProperties(defaults, series.fields[i], override);
+        const field = series.fields[i];
 
         // Show all number fields
         if (field.type !== FieldType.number) {
           continue;
         }
+        const config = getFieldProperties(defaults, field.config || {}, override);
 
-        if (!field.name) {
-          field.name = `Field[${s}]`; // it is a copy, so safe to edit
+        let name = field.name;
+        if (!name) {
+          name = `Field[${s}]`;
         }
 
-        scopedVars[VAR_FIELD_NAME] = { text: 'Field', value: field.name };
+        scopedVars[VAR_FIELD_NAME] = { text: 'Field', value: name };
 
         const display = getDisplayProcessor({
-          field,
+          field: config,
           theme: options.theme,
         });
 
-        const title = field.title ? field.title : defaultTitle;
+        const title = config.title ? config.title : defaultTitle;
 
         // Show all number fields
         if (fieldOptions.values) {
           const usesCellValues = title.indexOf(VAR_CELL_PREFIX) >= 0;
 
-          for (const row of series.rows) {
+          for (let j = 0; j < field.values.length; j++) {
             // Add all the row variables
             if (usesCellValues) {
-              for (let j = 0; j < series.fields.length; j++) {
-                scopedVars[VAR_CELL_PREFIX + j] = {
-                  value: row[j],
-                  text: toString(row[j]),
+              for (let k = 0; k < series.fields.length; k++) {
+                const f = series.fields[k];
+                const v = f.values.get(j);
+                scopedVars[VAR_CELL_PREFIX + k] = {
+                  value: v,
+                  text: toString(v),
                 };
               }
             }
 
-            const displayValue = display(row[i]);
+            const displayValue = display(field.values.get(j));
             displayValue.title = replaceVariables(title, scopedVars);
             values.push({
-              field,
+              name,
+              field: config,
               display: displayValue,
             });
 
@@ -158,10 +163,8 @@ export const getFieldDisplayValues = (options: GetFieldDisplayValuesOptions): Fi
           }
         } else {
           const results = reduceField({
-            series,
-            fieldIndex: i,
+            field,
             reducers: calcs, // The stats to calculate
-            nullValueMode: NullValueMode.Null,
           });
 
           // Single sparkline for a field
@@ -169,10 +172,8 @@ export const getFieldDisplayValues = (options: GetFieldDisplayValuesOptions): Fi
             timeColumn < 0
               ? undefined
               : getFlotPairs({
-                  rows: series.rows,
-                  xIndex: timeColumn,
-                  yIndex: i,
-                  nullValueMode: NullValueMode.Null,
+                  xField: series.fields[timeColumn],
+                  yField: series.fields[i],
                 });
 
           for (const calc of calcs) {
@@ -180,7 +181,8 @@ export const getFieldDisplayValues = (options: GetFieldDisplayValuesOptions): Fi
             const displayValue = display(results[calc]);
             displayValue.title = replaceVariables(title, scopedVars);
             values.push({
-              field,
+              name,
+              field: config,
               display: displayValue,
               sparkline: points,
             });
@@ -192,9 +194,9 @@ export const getFieldDisplayValues = (options: GetFieldDisplayValuesOptions): Fi
 
   if (values.length === 0) {
     values.push({
+      name: 'No data',
       field: {
         ...defaults,
-        name: 'No Data',
       },
       display: {
         numeric: 0,
@@ -222,7 +224,7 @@ const numericFieldProps: any = {
  * For numeric values, only valid numbers will be applied
  * for units, 'none' will be skipped
  */
-export function applyFieldProperties(field: Field, props?: Partial<Field>): Field {
+export function applyFieldProperties(field: FieldConfig, props?: FieldConfig): FieldConfig {
   if (!props) {
     return field;
   }
@@ -250,14 +252,11 @@ export function applyFieldProperties(field: Field, props?: Partial<Field>): Fiel
       copy[key] = val;
     }
   }
-  return copy as Field;
+  return copy as FieldConfig;
 }
 
-type PartialField = Partial<Field>;
-
-export function getFieldProperties(...props: PartialField[]): Field {
-  let field = props[0] as Field;
-
+export function getFieldProperties(...props: FieldConfig[]): FieldConfig {
+  let field = props[0] as FieldConfig;
   for (let i = 1; i < props.length; i++) {
     field = applyFieldProperties(field, props[i]);
   }

packages/grafana-ui/src/utils/flotPairs.test.ts

@@ -1,10 +1,19 @@
 import { getFlotPairs } from './flotPairs';
+import { DataFrameHelper } from '@grafana/data';
 
 describe('getFlotPairs', () => {
-  const rows = [[1, 100, 'a'], [2, 200, 'b'], [3, 300, 'c']];
-
+  const series = new DataFrameHelper({
+    fields: [
+      { name: 'a', values: [1, 2, 3] },
+      { name: 'b', values: [100, 200, 300] },
+      { name: 'c', values: ['a', 'b', 'c'] },
+    ],
+  });
   it('should get X and y', () => {
-    const pairs = getFlotPairs({ rows, xIndex: 0, yIndex: 1 });
+    const pairs = getFlotPairs({
+      xField: series.fields[0],
+      yField: series.fields[1],
+    });
 
     expect(pairs.length).toEqual(3);
     expect(pairs[0].length).toEqual(2);
@@ -13,7 +22,10 @@ describe('getFlotPairs', () => {
   });
 
   it('should work with strings', () => {
-    const pairs = getFlotPairs({ rows, xIndex: 0, yIndex: 2 });
+    const pairs = getFlotPairs({
+      xField: series.fields[0],
+      yField: series.fields[2],
+    });
 
     expect(pairs.length).toEqual(3);
     expect(pairs[0].length).toEqual(2);

packages/grafana-ui/src/utils/flotPairs.ts

@@ -1,22 +1,28 @@
 // Types
-import { NullValueMode, GraphSeriesValue } from '@grafana/data';
+import { NullValueMode, GraphSeriesValue, Field } from '@grafana/data';
 
 export interface FlotPairsOptions {
-  rows: any[][];
-  xIndex: number;
-  yIndex: number;
+  xField: Field;
+  yField: Field;
   nullValueMode?: NullValueMode;
 }
 
-export function getFlotPairs({ rows, xIndex, yIndex, nullValueMode }: FlotPairsOptions): GraphSeriesValue[][] {
+export function getFlotPairs({ xField, yField, nullValueMode }: FlotPairsOptions): GraphSeriesValue[][] {
+  const vX = xField.values;
+  const vY = yField.values;
+  const length = vX.length;
+  if (vY.length !== length) {
+    throw new Error('Unexpected field length');
+  }
+
   const ignoreNulls = nullValueMode === NullValueMode.Ignore;
   const nullAsZero = nullValueMode === NullValueMode.AsZero;
 
   const pairs: any[][] = [];
 
-  for (let i = 0; i < rows.length; i++) {
-    const x = rows[i][xIndex];
-    let y = rows[i][yIndex];
+  for (let i = 0; i < length; i++) {
+    const x = vX.get(i);
+    let y = vY.get(i);
 
     if (y === null) {
       if (ignoreNulls) {