DataFrame: split DataFrameHelper into MutableDataFrame and FieldCache (#18795)

* add appending utility

* add appending utility

* update comment

* rename to mutable

* move mutable functions out of DataFrameHelper

* move mutable functions out of DataFrameHelper

* move mutable functions out of DataFrameHelper

* turn DataFrameHelper into FieldCache

* guess time from name

* graph the numbers

* return the timeField, not just the index

* just warn on duplicate field names

* only use a parser if the input is a string

* append init all fields to the same length

* typo

* only parse string if value is a string

* DataFrame: test fixes

* Switch to null for missing values

* Fixed tests

public/app/plugins/datasource/elasticsearch/elastic_response.ts

@@ -2,7 +2,7 @@ import _ from 'lodash';
 import flatten from 'app/core/utils/flatten';
 import * as queryDef from './query_def';
 import TableModel from 'app/core/table_model';
-import { DataFrame, toDataFrame, FieldType, DataFrameHelper } from '@grafana/data';
+import { DataFrame, toDataFrame, FieldType, MutableDataFrame } from '@grafana/data';
 import { DataQueryResponse } from '@grafana/ui';
 import { ElasticsearchAggregation } from './types';
 
@@ -464,7 +464,7 @@ export class ElasticResponse {
 
       if (docs.length > 0) {
         propNames = propNames.sort();
-        const series = new DataFrameHelper({ fields: [] });
+        const series = new MutableDataFrame({ fields: [] });
 
         series.addField({
           name: this.targets[0].timeField,
@@ -513,7 +513,7 @@ export class ElasticResponse {
 
         // Add a row for each document
         for (const doc of docs) {
-          series.appendRowFrom(doc);
+          series.add(doc);
         }
 
         dataFrame.push(series);

public/app/plugins/datasource/elasticsearch/specs/elastic_response.test.ts

@@ -1,5 +1,5 @@
+import { DataFrameView, KeyValue, MutableDataFrame } from '@grafana/data';
 import { ElasticResponse } from '../elastic_response';
-import { DataFrameHelper, DataFrameView, KeyValue } from '@grafana/data';
 
 describe('ElasticResponse', () => {
   let targets;
@@ -859,7 +859,7 @@ describe('ElasticResponse', () => {
 
     it('should return histogram aggregation and documents', () => {
       expect(result.data.length).toBe(2);
-      const logResults = result.data[0] as DataFrameHelper;
+      const logResults = result.data[0] as MutableDataFrame;
       const fields = logResults.fields.map(f => {
         return {
           name: f.name,
@@ -874,16 +874,15 @@ describe('ElasticResponse', () => {
       let rows = new DataFrameView(logResults);
       for (let i = 0; i < rows.length; i++) {
         const r = rows.get(i);
-        const row = [r._id, r._type, r._index, r._source];
-        expect(row).toContain(response.responses[0].hits.hits[i]._id);
-        expect(row).toContain(response.responses[0].hits.hits[i]._type);
-        expect(row).toContain(response.responses[0].hits.hits[i]._index);
-        expect(row).toContain(JSON.stringify(response.responses[0].hits.hits[i]._source, undefined, 2));
+        expect(r._id).toEqual(response.responses[0].hits.hits[i]._id);
+        expect(r._type).toEqual(response.responses[0].hits.hits[i]._type);
+        expect(r._index).toEqual(response.responses[0].hits.hits[i]._index);
+        expect(r._source).toEqual(response.responses[0].hits.hits[i]._source);
       }
 
       // Make a map from the histogram results
       const hist: KeyValue<number> = {};
-      const histogramResults = new DataFrameHelper(result.data[1]);
+      const histogramResults = new MutableDataFrame(result.data[1]);
       rows = new DataFrameView(histogramResults);
       for (let i = 0; i < rows.length; i++) {
         const row = rows.get(i);

public/app/plugins/datasource/input/InputConfigEditor.tsx

@@ -5,7 +5,7 @@ import React, { PureComponent } from 'react';
 import { InputOptions } from './types';
 
 import { DataSourcePluginOptionsEditorProps, DataSourceSettings, TableInputCSV } from '@grafana/ui';
-import { DataFrame, DataFrameHelper } from '@grafana/data';
+import { DataFrame, MutableDataFrame } from '@grafana/data';
 import { dataFrameToCSV } from './utils';
 
 type InputSettings = DataSourceSettings<InputOptions>;
@@ -32,7 +32,7 @@ export class InputConfigEditor extends PureComponent<Props, State> {
   onSeriesParsed = (data: DataFrame[], text: string) => {
     const { options, onOptionsChange } = this.props;
     if (!data) {
-      data = [new DataFrameHelper()];
+      data = [new MutableDataFrame()];
     }
     // data is a property on 'jsonData'
     const jsonData = {

public/app/plugins/datasource/input/InputDatasource.test.ts

@@ -1,6 +1,6 @@
 import InputDatasource, { describeDataFrame } from './InputDatasource';
 import { InputQuery, InputOptions } from './types';
-import { readCSV, DataFrame, DataFrameHelper } from '@grafana/data';
+import { readCSV, DataFrame, MutableDataFrame } from '@grafana/data';
 import { DataSourceInstanceSettings, PluginMeta } from '@grafana/ui';
 import { getQueryOptions } from 'test/helpers/getQueryOptions';
 
@@ -38,7 +38,7 @@ describe('InputDatasource', () => {
     expect(describeDataFrame(null)).toEqual('');
     expect(
       describeDataFrame([
-        new DataFrameHelper({
+        new MutableDataFrame({
           name: 'x',
           fields: [{ name: 'a' }],
         }),

public/app/plugins/datasource/input/InputQueryEditor.tsx

@@ -6,7 +6,7 @@ import { InputDatasource, describeDataFrame } from './InputDatasource';
 import { InputQuery, InputOptions } from './types';
 
 import { FormLabel, Select, QueryEditorProps, TableInputCSV } from '@grafana/ui';
-import { DataFrame, toCSV, SelectableValue, DataFrameHelper } from '@grafana/data';
+import { DataFrame, toCSV, SelectableValue, MutableDataFrame } from '@grafana/data';
 
 import { dataFrameToCSV } from './utils';
 
@@ -41,7 +41,7 @@ export class InputQueryEditor extends PureComponent<Props, State> {
       }
       data = [...datasource.data];
       if (!data) {
-        data = [new DataFrameHelper()];
+        data = [new MutableDataFrame()];
       }
       this.setState({ text: toCSV(data) });
     }
@@ -53,7 +53,7 @@ export class InputQueryEditor extends PureComponent<Props, State> {
     const { query, onChange, onRunQuery } = this.props;
     this.setState({ text });
     if (!data) {
-      data = [new DataFrameHelper()];
+      data = [new MutableDataFrame()];
     }
     onChange({ ...query, data });
     onRunQuery();

public/app/plugins/datasource/loki/result_transformer.ts

@@ -1,7 +1,7 @@
 import { LokiLogsStream } from './types';
-import { parseLabels, FieldType, Labels, DataFrameHelper } from '@grafana/data';
+import { parseLabels, FieldType, Labels, MutableDataFrame } from '@grafana/data';
 
-export function logStreamToDataFrame(stream: LokiLogsStream, refId?: string): DataFrameHelper {
+export function logStreamToDataFrame(stream: LokiLogsStream, refId?: string): MutableDataFrame {
   let labels: Labels = stream.parsedLabels;
   if (!labels && stream.labels) {
     labels = parseLabels(stream.labels);
@@ -14,7 +14,7 @@ export function logStreamToDataFrame(stream: LokiLogsStream, refId?: string): Da
     lines.push(entry.line);
   }
 
-  return new DataFrameHelper({
+  return new MutableDataFrame({
     refId,
     labels,
     fields: [

public/app/plugins/datasource/testdata/StreamHandler.ts

@@ -7,7 +7,7 @@ import {
   LoadingState,
   LogLevel,
   CSVReader,
-  DataFrameHelper,
+  MutableDataFrame,
   CircularVector,
   DataFrame,
 } from '@grafana/data';
@@ -182,7 +182,7 @@ export class SignalWorker extends StreamWorker {
     const vals = new CircularVector({ capacity: maxRows });
     this.values = [times, vals];
 
-    const data = new DataFrameHelper({
+    const data = new MutableDataFrame({
       fields: [
         { name: 'Time', type: FieldType.time, values: times }, // The time field
         { name: 'Value', type: FieldType.number, values: vals },
@@ -351,7 +351,7 @@ export class LogsWorker extends StreamWorker {
     const lines = new CircularVector({ capacity: maxRows });
 
     this.values = [times, lines];
-    this.data = new DataFrameHelper({
+    this.data = new MutableDataFrame({
       fields: [
         { name: 'Time', type: FieldType.time, values: times },
         { name: 'Line', type: FieldType.string, values: lines },

public/app/plugins/panel/graph/data_processor.ts

@@ -1,6 +1,6 @@
 import _ from 'lodash';
 import { colors, getColorFromHexRgbOrName } from '@grafana/ui';
-import { TimeRange, FieldType, Field, DataFrame, DataFrameHelper } from '@grafana/data';
+import { TimeRange, FieldType, Field, DataFrame, getTimeField } from '@grafana/data';
 import TimeSeries from 'app/core/time_series2';
 import config from 'app/core/config';
 
@@ -21,15 +21,17 @@ export class DataProcessor {
     }
 
     for (const series of dataList) {
-      const data = new DataFrameHelper(series);
-      const time = data.getFirstFieldOfType(FieldType.time);
-
-      if (!time) {
+      const { timeField } = getTimeField(series);
+      if (!timeField) {
         continue;
       }
 
       const seriesName = series.name ? series.name : series.refId;
-      for (const field of data.getFields(FieldType.number)) {
+      for (const field of series.fields) {
+        if (field.type !== FieldType.number) {
+          continue;
+        }
+
         let name = field.config && field.config.title ? field.config.title : field.name;
 
         if (seriesName && dataList.length > 0 && name !== seriesName) {
@@ -37,8 +39,8 @@ export class DataProcessor {
         }
 
         const datapoints = [];
-        for (let r = 0; r < data.length; r++) {
-          datapoints.push([field.values.get(r), time.values.get(r)]);
+        for (let r = 0; r < series.length; r++) {
+          datapoints.push([field.values.get(r), timeField.values.get(r)]);
         }
 
         list.push(this.toTimeSeries(field, name, datapoints, list.length, range));

public/app/plugins/panel/graph2/getGraphSeriesModel.ts

@@ -1,5 +1,5 @@
 import { colors, getFlotPairs, getColorFromHexRgbOrName, getDisplayProcessor, PanelData } from '@grafana/ui';
-import { NullValueMode, reduceField, DataFrameHelper, FieldType, DisplayValue, GraphSeriesXY } from '@grafana/data';
+import { NullValueMode, reduceField, FieldType, DisplayValue, GraphSeriesXY, getTimeField } from '@grafana/data';
 
 import { SeriesOptions, GraphOptions } from './types';
 import { GraphLegendEditorLegendOptions } from './GraphLegendEditor';
@@ -19,16 +19,19 @@ export const getGraphSeriesModel = (
   });
 
   for (const series of data.series) {
-    const data = new DataFrameHelper(series);
-    const timeColumn = data.getFirstFieldOfType(FieldType.time);
-    if (!timeColumn) {
+    const { timeField } = getTimeField(series);
+    if (!timeField) {
       continue;
     }
 
-    for (const field of data.getFields(FieldType.number)) {
+    for (const field of series.fields) {
+      if (field.type !== FieldType.number) {
+        continue;
+      }
+
       // Use external calculator just to make sure it works :)
       const points = getFlotPairs({
-        xField: timeColumn,
+        xField: timeField,
         yField: field,
         nullValueMode: NullValueMode.Null,
       });