IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

DataFrame: convert from row based to a columnar value format (#18391)

Browse Source
This commit is contained in:
Ryan McKinley
2019-08-15 09:18:51 -07:00
committed by GitHub
parent 350b9a9494
commit e59bae55d9
63 changed files with 1856 additions and 995 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
packages/grafana-data/src/types/data.ts
View File

@@ -1,6 +1,3 @@
import { Threshold } from './threshold';
import { ValueMapping } from './valueMapping';
export enum LoadingState {
NotStarted = 'NotStarted',
Loading = 'Loading',
@@ -9,14 +6,6 @@ export enum LoadingState {
Error = 'Error',
}
export enum FieldType {
time = 'time', // or date
number = 'number',
string = 'string',
boolean = 'boolean',
other = 'other', // Object, Array, etc
}
export interface QueryResultMeta {
[key: string]: any;
@@ -42,34 +31,10 @@ export interface QueryResultBase {
meta?: QueryResultMeta;
}
export interface Field {
name: string; // The column name
title?: string; // The display value for this field. This supports template variables blank is auto
type?: FieldType;
filterable?: boolean;
unit?: string;
decimals?: number | null; // Significant digits (for display)
min?: number | null;
max?: number | null;
// Convert input values into a display value
mappings?: ValueMapping[];
// Must be sorted by 'value', first value is always -Infinity
thresholds?: Threshold[];
}
export interface Labels {
[key: string]: string;
}
export interface DataFrame extends QueryResultBase {
name?: string;
fields: Field[];
rows: any[][];
labels?: Labels;
}
export interface Column {
text: string; // For a Column, the 'text' is the field name
filterable?: boolean;

110
packages/grafana-data/src/types/dataFrame.ts Normal file
View File

@@ -0,0 +1,110 @@
import { Threshold } from './threshold';
import { ValueMapping } from './valueMapping';
import { QueryResultBase, Labels, NullValueMode } from './data';
import { FieldCalcs } from '../utils/index';
import { DisplayProcessor } from './displayValue';
export enum FieldType {
time = 'time', // or date
number = 'number',
string = 'string',
boolean = 'boolean',
other = 'other', // Object, Array, etc
}
/**
* Every property is optional
*
* Plugins may extend this with additional properties. Somethign like series overrides
*/
export interface FieldConfig {
title?: string; // The display value for this field. This supports template variables blank is auto
filterable?: boolean;
// Numeric Options
unit?: string;
decimals?: number | null; // Significant digits (for display)
min?: number | null;
max?: number | null;
// Convert input values into a display string
mappings?: ValueMapping[];
// Must be sorted by 'value', first value is always -Infinity
thresholds?: Threshold[];
// Used when reducing field values
nullValueMode?: NullValueMode;
// Alternative to empty string
noValue?: string;
}
export interface Vector<T = any> {
length: number;
/**
* Access the value by index (Like an array)
*/
get(index: number): T;
/**
* Get the resutls as an array.
*/
toArray(): T[];
/**
* Return the values as a simple array for json serialization
*/
toJSON(): any; // same results as toArray()
}
export interface Field<T = any> {
name: string; // The column name
type: FieldType;
config: FieldConfig;
values: Vector<T>; // `buffer` when JSON
/**
* Cache of reduced values
*/
calcs?: FieldCalcs;
/**
* Convert text to the field value
*/
parse?: (value: any) => T;
/**
* Convert a value for display
*/
display?: DisplayProcessor;
}
export interface DataFrame extends QueryResultBase {
name?: string;
fields: Field[]; // All fields of equal length
labels?: Labels;
// The number of rows
length: number;
}
/**
* Like a field, but properties are optional and values may be a simple array
*/
export interface FieldDTO<T = any> {
name: string; // The column name
type?: FieldType;
config?: FieldConfig;
values?: Vector<T> | T[]; // toJSON will always be T[], input could be either
}
/**
* Like a DataFrame, but fields may be a FieldDTO
*/
export interface DataFrameDTO extends QueryResultBase {
name?: string;
labels?: Labels;
fields: Array<FieldDTO | Field>;
}

2
packages/grafana-data/src/types/displayValue.ts
View File

@@ -1,3 +1,5 @@
export type DisplayProcessor = (value: any) => DisplayValue;
export interface DisplayValue {
text: string; // Show in the UI
numeric: number; // Use isNaN to check if it is a real number

1
packages/grafana-data/src/types/index.ts
View File

@@ -1,4 +1,5 @@
export * from './data';
export * from './dataFrame';
export * from './dataLink';
export * from './logs';
export * from './navModel';

149
packages/grafana-data/src/utils/__snapshots__/csv.test.ts.snap
View File

@@ -4,42 +4,54 @@ exports[`read csv should get X and y 1`] = `
Object {
"fields": Array [
Object {
"name": "Column 1",
"type": "number",
"config": Object {},
"name": "Field 1",
"type": "string",
"values": Array [
"",
"2",
"5",
"",
],
},
Object {
"name": "Column 2",
"config": Object {},
"name": "Field 2",
"type": "number",
"values": Array [
1,
3,
6,
NaN,
],
},
Object {
"name": "Column 3",
"config": Object {},
"name": "Field 3",
"type": "number",
"values": Array [
null,
4,
NaN,
NaN,
],
},
Object {
"config": Object {},
"name": "Field 4",
"type": "number",
"values": Array [
null,
null,
null,
7,
],
},
],
"rows": Array [
Array [
2,
3,
4,
null,
],
Array [
5,
6,
null,
null,
],
Array [
null,
null,
null,
7,
],
],
"labels": undefined,
"meta": undefined,
"name": undefined,
"refId": undefined,
}
`;
@@ -47,30 +59,37 @@ exports[`read csv should read csv from local file system 1`] = `
Object {
"fields": Array [
Object {
"config": Object {},
"name": "a",
"type": "number",
"values": Array [
10,
40,
],
},
Object {
"config": Object {},
"name": "b",
"type": "number",
"values": Array [
20,
50,
],
},
Object {
"config": Object {},
"name": "c",
"type": "number",
"values": Array [
30,
60,
],
},
],
"rows": Array [
Array [
10,
20,
30,
],
Array [
40,
50,
60,
],
],
"labels": undefined,
"meta": undefined,
"name": undefined,
"refId": undefined,
}
`;
@@ -78,42 +97,48 @@ exports[`read csv should read csv with headers 1`] = `
Object {
"fields": Array [
Object {
"config": Object {
"unit": "ms",
},
"name": "a",
"type": "number",
"unit": "ms",
"values": Array [
10,
40,
40,
40,
],
},
Object {
"config": Object {
"unit": "lengthm",
},
"name": "b",
"type": "string",
"unit": "lengthm",
"type": "number",
"values": Array [
20,
50,
500,
50,
],
},
Object {
"config": Object {
"unit": "s",
},
"name": "c",
"type": "boolean",
"unit": "s",
"values": Array [
true,
false,
false,
true,
],
},
],
"rows": Array [
Array [
10,
"20",
true,
],
Array [
40,
"50",
false,
],
Array [
40,
"500",
false,
],
Array [
40,
"50",
true,
],
],
"labels": undefined,
"meta": undefined,
"name": undefined,
"refId": undefined,
}
`;

37
packages/grafana-data/src/utils/csv.test.ts
View File

@@ -1,7 +1,9 @@
import { readCSV, toCSV, CSVHeaderStyle } from './csv';
import { getDataFrameRow } from './processDataFrame';
// Test with local CSV files
const fs = require('fs');
import fs from 'fs';
import { toDataFrameDTO } from './processDataFrame';
describe('read csv', () => {
it('should get X and y', () => {
@@ -11,14 +13,31 @@ describe('read csv', () => {
const series = data[0];
expect(series.fields.length).toBe(4);
expect(series.rows.length).toBe(3);
const rows = 4;
expect(series.length).toBe(rows);
// Make sure everythign it padded properly
for (const row of series.rows) {
expect(row.length).toBe(series.fields.length);
for (const field of series.fields) {
expect(field.values.length).toBe(rows);
}
expect(series).toMatchSnapshot();
const dto = toDataFrameDTO(series);
expect(dto).toMatchSnapshot();
});
it('should read single string OK', () => {
const text = 'a,b,c';
const data = readCSV(text);
expect(data.length).toBe(1);
const series = data[0];
expect(series.fields.length).toBe(3);
expect(series.length).toBe(0);
expect(series.fields[0].name).toEqual('a');
expect(series.fields[1].name).toEqual('b');
expect(series.fields[2].name).toEqual('c');
});
it('should read csv from local file system', () => {
@@ -28,7 +47,7 @@ describe('read csv', () => {
const csv = fs.readFileSync(path, 'utf8');
const data = readCSV(csv);
expect(data.length).toBe(1);
expect(data[0]).toMatchSnapshot();
expect(toDataFrameDTO(data[0])).toMatchSnapshot();
});
it('should read csv with headers', () => {
@@ -38,7 +57,7 @@ describe('read csv', () => {
const csv = fs.readFileSync(path, 'utf8');
const data = readCSV(csv);
expect(data.length).toBe(1);
expect(data[0]).toMatchSnapshot();
expect(toDataFrameDTO(data[0])).toMatchSnapshot();
});
});
@@ -54,7 +73,7 @@ describe('write csv', () => {
const data = readCSV(csv);
const out = toCSV(data, { headerStyle: CSVHeaderStyle.full });
expect(data.length).toBe(1);
expect(data[0].rows[0]).toEqual(firstRow);
expect(getDataFrameRow(data[0], 0)).toEqual(firstRow);
expect(data[0].fields.length).toBe(3);
expect(norm(out)).toBe(norm(csv));
@@ -65,7 +84,7 @@ describe('write csv', () => {
const f = readCSV(shorter);
const fields = f[0].fields;
expect(fields.length).toBe(3);
expect(f[0].rows[0]).toEqual(firstRow);
expect(getDataFrameRow(f[0], 0)).toEqual(firstRow);
expect(fields.map(f => f.name).join(',')).toEqual('a,b,c'); // the names
});
});

217
packages/grafana-data/src/utils/csv.ts
View File

@@ -4,8 +4,9 @@ import defaults from 'lodash/defaults';
import isNumber from 'lodash/isNumber';
// Types
import { DataFrame, Field, FieldType } from '../types';
import { DataFrame, Field, FieldType, FieldConfig } from '../types';
import { guessFieldTypeFromValue } from './processDataFrame';
import { DataFrameHelper } from './dataFrameHelper';
export enum CSVHeaderStyle {
full,
@@ -28,9 +29,9 @@ export interface CSVParseCallbacks {
* This can return a modified table to force any
* Column configurations
*/
onHeader: (table: DataFrame) => void;
onHeader: (fields: Field[]) => void;
// Called after each row is read and
// Called after each row is read
onRow: (row: any[]) => void;
}
@@ -49,16 +50,13 @@ enum ParseState {
ReadingRows,
}
type FieldParser = (value: string) => any;
export class CSVReader {
config: CSVConfig;
callback?: CSVParseCallbacks;
field: FieldParser[];
series: DataFrame;
state: ParseState;
data: DataFrame[];
data: DataFrameHelper[];
current: DataFrameHelper;
constructor(options?: CSVOptions) {
if (!options) {
@@ -67,12 +65,8 @@ export class CSVReader {
this.config = options.config || {};
this.callback = options.callback;
this.field = [];
this.current = new DataFrameHelper({ fields: [] });
this.state = ParseState.Starting;
this.series = {
fields: [],
rows: [],
};
this.data = [];
}
@@ -92,37 +86,42 @@ export class CSVReader {
const idx = first.indexOf('#', 2);
if (idx > 0) {
const k = first.substr(1, idx - 1);
const isName = 'name' === k;
// Simple object used to check if headers match
const headerKeys: Field = {
name: '#',
type: FieldType.number,
const headerKeys: FieldConfig = {
unit: '#',
};
// Check if it is a known/supported column
if (headerKeys.hasOwnProperty(k)) {
if (isName || headerKeys.hasOwnProperty(k)) {
// Starting a new table after reading rows
if (this.state === ParseState.ReadingRows) {
this.series = {
fields: [],
rows: [],
};
this.data.push(this.series);
this.current = new DataFrameHelper({ fields: [] });
this.data.push(this.current);
}
padColumnWidth(this.series.fields, line.length);
const fields: any[] = this.series.fields; // cast to any so we can lookup by key
const v = first.substr(idx + 1);
fields[0][k] = v;
for (let j = 1; j < fields.length; j++) {
fields[j][k] = line[j];
if (isName) {
this.current.addFieldFor(undefined, v);
for (let j = 1; j < line.length; j++) {
this.current.addFieldFor(undefined, line[j]);
}
} else {
const { fields } = this.current;
for (let j = 0; j < fields.length; j++) {
if (!fields[j].config) {
fields[j].config = {};
}
const disp = fields[j].config as any; // any lets name lookup
disp[k] = j === 0 ? v : line[j];
}
}
this.state = ParseState.InHeader;
continue;
}
} else if (this.state === ParseState.Starting) {
this.series.fields = makeFieldsFor(line);
this.state = ParseState.InHeader;
continue;
}
@@ -133,67 +132,48 @@ export class CSVReader {
if (this.state === ParseState.Starting) {
const type = guessFieldTypeFromValue(first);
if (type === FieldType.string) {
this.series.fields = makeFieldsFor(line);
for (const s of line) {
this.current.addFieldFor(undefined, s);
}
this.state = ParseState.InHeader;
continue;
}
this.series.fields = makeFieldsFor(new Array(line.length));
this.series.fields[0].type = type;
this.state = ParseState.InHeader; // fall through to read rows
}
}
if (this.state === ParseState.InHeader) {
padColumnWidth(this.series.fields, line.length);
this.state = ParseState.ReadingRows;
// Add the current results to the data
if (this.state !== ParseState.ReadingRows) {
// anything???
}
if (this.state === ParseState.ReadingRows) {
// Make sure colum structure is valid
if (line.length > this.series.fields.length) {
padColumnWidth(this.series.fields, line.length);
if (this.callback) {
this.callback.onHeader(this.series);
} else {
// Expand all rows with nulls
for (let x = 0; x < this.series.rows.length; x++) {
const row = this.series.rows[x];
while (row.length < line.length) {
row.push(null);
}
}
}
}
this.state = ParseState.ReadingRows;
const row: any[] = [];
for (let j = 0; j < line.length; j++) {
const v = line[j];
if (v) {
if (!this.field[j]) {
this.field[j] = makeFieldParser(v, this.series.fields[j]);
}
row.push(this.field[j](v));
} else {
row.push(null);
}
// Make sure colum structure is valid
if (line.length > this.current.fields.length) {
const { fields } = this.current;
for (let f = fields.length; f < line.length; f++) {
this.current.addFieldFor(line[f]);
}
if (this.callback) {
// Send the header after we guess the type
if (this.series.rows.length === 0) {
this.callback.onHeader(this.series);
this.series.rows.push(row); // Only add the first row
}
this.callback.onRow(row);
} else {
this.series.rows.push(row);
this.callback.onHeader(this.current.fields);
}
}
this.current.appendRow(line);
if (this.callback) {
// // Send the header after we guess the type
// if (this.series.rows.length === 0) {
// this.callback.onHeader(this.series);
// }
this.callback.onRow(line);
}
}
};
readCSV(text: string): DataFrame[] {
this.data = [this.series];
readCSV(text: string): DataFrameHelper[] {
this.current = new DataFrameHelper({ fields: [] });
this.data = [this.current];
const papacfg = {
...this.config,
@@ -204,61 +184,11 @@ export class CSVReader {
} as ParseConfig;
Papa.parse(text, papacfg);
return this.data;
}
}
function makeFieldParser(value: string, field: Field): FieldParser {
if (!field.type) {
if (field.name === 'time' || field.name === 'Time') {
field.type = FieldType.time;
} else {
field.type = guessFieldTypeFromValue(value);
}
}
if (field.type === FieldType.number) {
return (value: string) => {
return parseFloat(value);
};
}
// Will convert anything that starts with "T" to true
if (field.type === FieldType.boolean) {
return (value: string) => {
return !(value[0] === 'F' || value[0] === 'f' || value[0] === '0');
};
}
// Just pass the string back
return (value: string) => value;
}
/**
* Creates a field object for each string in the list
*/
function makeFieldsFor(line: string[]): Field[] {
const fields: Field[] = [];
for (let i = 0; i < line.length; i++) {
const v = line[i] ? line[i] : 'Column ' + (i + 1);
fields.push({ name: v });
}
return fields;
}
/**
* Makes sure the colum has valid entries up the the width
*/
function padColumnWidth(fields: Field[], width: number) {
if (fields.length < width) {
for (let i = fields.length; i < width; i++) {
fields.push({
name: 'Field ' + (i + 1),
});
}
}
}
type FieldWriter = (value: any) => string;
function writeValue(value: any, config: CSVConfig): string {
@@ -295,15 +225,26 @@ function makeFieldWriter(field: Field, config: CSVConfig): FieldWriter {
}
function getHeaderLine(key: string, fields: Field[], config: CSVConfig): string {
const isName = 'name' === key;
const isType = 'type' === key;
for (const f of fields) {
if (f.hasOwnProperty(key)) {
const display = f.config;
if (isName || isType || (display && display.hasOwnProperty(key))) {
let line = '#' + key + '#';
for (let i = 0; i < fields.length; i++) {
if (i > 0) {
line = line + config.delimiter;
}
const v = (fields[i] as any)[key];
let v: any = fields[i].name;
if (isType) {
v = fields[i].type;
} else if (isName) {
// already name
} else {
v = (fields[i].config as any)[key];
}
if (v) {
line = line + writeValue(v, config);
}
@@ -329,7 +270,7 @@ export function toCSV(data: DataFrame[], config?: CSVConfig): string {
});
for (const series of data) {
const { rows, fields } = series;
const { fields } = series;
if (config.headerStyle === CSVHeaderStyle.full) {
csv =
csv +
@@ -346,20 +287,22 @@ export function toCSV(data: DataFrame[], config?: CSVConfig): string {
}
csv += config.newline;
}
const writers = fields.map(field => makeFieldWriter(field, config!));
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
for (let j = 0; j < row.length; j++) {
if (j > 0) {
csv = csv + config.delimiter;
}
const length = fields[0].values.length;
if (length > 0) {
const writers = fields.map(field => makeFieldWriter(field, config!));
for (let i = 0; i < length; i++) {
for (let j = 0; j < fields.length; j++) {
if (j > 0) {
csv = csv + config.delimiter;
}
const v = row[j];
if (v !== null) {
csv = csv + writers[j](v);
const v = fields[j].values.get(i);
if (v !== null) {
csv = csv + writers[j](v);
}
}
csv = csv + config.newline;
}
csv = csv + config.newline;
}
csv = csv + config.newline;
}

89
packages/grafana-data/src/utils/dataFrameHelper.test.ts Normal file
View File

@@ -0,0 +1,89 @@
import { FieldType, DataFrameDTO, FieldDTO } from '../types/index';
import { DataFrameHelper } from './dataFrameHelper';
describe('dataFrameHelper', () => {
const frame: DataFrameDTO = {
fields: [
{ name: 'time', type: FieldType.time, values: [100, 200, 300] },
{ name: 'name', type: FieldType.string, values: ['a', 'b', 'c'] },
{ name: 'value', type: FieldType.number, values: [1, 2, 3] },
{ name: 'value', type: FieldType.number, values: [4, 5, 6] },
],
};
const ext = new DataFrameHelper(frame);
it('Should get a valid count for the fields', () => {
expect(ext.length).toEqual(3);
});
it('Should get the first field with a duplicate name', () => {
const field = ext.getFieldByName('value');
expect(field!.name).toEqual('value');
expect(field!.values.toJSON()).toEqual([1, 2, 3]);
});
});
describe('FieldCache', () => {
it('when creating a new FieldCache from fields should be able to query cache', () => {
const fields: FieldDTO[] = [
{ name: 'time', type: FieldType.time },
{ name: 'string', type: FieldType.string },
{ name: 'number', type: FieldType.number },
{ name: 'boolean', type: FieldType.boolean },
{ name: 'other', type: FieldType.other },
{ name: 'undefined' },
];
const fieldCache = new DataFrameHelper({ fields });
const allFields = fieldCache.getFields();
expect(allFields).toHaveLength(6);
const expectedFieldNames = ['time', 'string', 'number', 'boolean', 'other', 'undefined'];
expect(allFields.map(f => f.name)).toEqual(expectedFieldNames);
expect(fieldCache.hasFieldOfType(FieldType.time)).toBeTruthy();
expect(fieldCache.hasFieldOfType(FieldType.string)).toBeTruthy();
expect(fieldCache.hasFieldOfType(FieldType.number)).toBeTruthy();
expect(fieldCache.hasFieldOfType(FieldType.boolean)).toBeTruthy();
expect(fieldCache.hasFieldOfType(FieldType.other)).toBeTruthy();
expect(fieldCache.getFields(FieldType.time).map(f => f.name)).toEqual([expectedFieldNames[0]]);
expect(fieldCache.getFields(FieldType.string).map(f => f.name)).toEqual([expectedFieldNames[1]]);
expect(fieldCache.getFields(FieldType.number).map(f => f.name)).toEqual([expectedFieldNames[2]]);
expect(fieldCache.getFields(FieldType.boolean).map(f => f.name)).toEqual([expectedFieldNames[3]]);
expect(fieldCache.getFields(FieldType.other).map(f => f.name)).toEqual([
expectedFieldNames[4],
expectedFieldNames[5],
]);
expect(fieldCache.fields[0].name).toEqual(expectedFieldNames[0]);
expect(fieldCache.fields[1].name).toEqual(expectedFieldNames[1]);
expect(fieldCache.fields[2].name).toEqual(expectedFieldNames[2]);
expect(fieldCache.fields[3].name).toEqual(expectedFieldNames[3]);
expect(fieldCache.fields[4].name).toEqual(expectedFieldNames[4]);
expect(fieldCache.fields[5].name).toEqual(expectedFieldNames[5]);
expect(fieldCache.fields[6]).toBeUndefined();
expect(fieldCache.getFirstFieldOfType(FieldType.time)!.name).toEqual(expectedFieldNames[0]);
expect(fieldCache.getFirstFieldOfType(FieldType.string)!.name).toEqual(expectedFieldNames[1]);
expect(fieldCache.getFirstFieldOfType(FieldType.number)!.name).toEqual(expectedFieldNames[2]);
expect(fieldCache.getFirstFieldOfType(FieldType.boolean)!.name).toEqual(expectedFieldNames[3]);
expect(fieldCache.getFirstFieldOfType(FieldType.other)!.name).toEqual(expectedFieldNames[4]);
expect(fieldCache.hasFieldNamed('tim')).toBeFalsy();
expect(fieldCache.hasFieldNamed('time')).toBeTruthy();
expect(fieldCache.hasFieldNamed('string')).toBeTruthy();
expect(fieldCache.hasFieldNamed('number')).toBeTruthy();
expect(fieldCache.hasFieldNamed('boolean')).toBeTruthy();
expect(fieldCache.hasFieldNamed('other')).toBeTruthy();
expect(fieldCache.hasFieldNamed('undefined')).toBeTruthy();
expect(fieldCache.getFieldByName('time')!.name).toEqual(expectedFieldNames[0]);
expect(fieldCache.getFieldByName('string')!.name).toEqual(expectedFieldNames[1]);
expect(fieldCache.getFieldByName('number')!.name).toEqual(expectedFieldNames[2]);
expect(fieldCache.getFieldByName('boolean')!.name).toEqual(expectedFieldNames[3]);
expect(fieldCache.getFieldByName('other')!.name).toEqual(expectedFieldNames[4]);
expect(fieldCache.getFieldByName('undefined')!.name).toEqual(expectedFieldNames[5]);
expect(fieldCache.getFieldByName('null')).toBeUndefined();
});
});

232
packages/grafana-data/src/utils/dataFrameHelper.ts Normal file
View File

@@ -0,0 +1,232 @@
import { Field, FieldType, DataFrame, Vector, FieldDTO, DataFrameDTO } from '../types/dataFrame';
import { Labels, QueryResultMeta } from '../types/data';
import { guessFieldTypeForField, guessFieldTypeFromValue } from './processDataFrame';
import { ArrayVector } from './vector';
import isArray from 'lodash/isArray';
export class DataFrameHelper implements DataFrame {
refId?: string;
meta?: QueryResultMeta;
name?: string;
fields: Field[];
labels?: Labels;
length = 0; // updated so it is the length of all fields
private fieldByName: { [key: string]: Field } = {};
private fieldByType: { [key: string]: Field[] } = {};
constructor(data?: DataFrame | DataFrameDTO) {
if (!data) {
data = { fields: [] }; //
}
this.refId = data.refId;
this.meta = data.meta;
this.name = data.name;
this.labels = data.labels;
this.fields = [];
for (let i = 0; i < data.fields.length; i++) {
this.addField(data.fields[i]);
}
}
addFieldFor(value: any, name?: string): Field {
if (!name) {
name = `Field ${this.fields.length + 1}`;
}
return this.addField({
name,
type: guessFieldTypeFromValue(value),
});
}
/**
* Reverse the direction of all fields
*/
reverse() {
for (const f of this.fields) {
if (isArray(f.values)) {
const arr = f.values as any[];
arr.reverse();
}
}
}
private updateTypeIndex(field: Field) {
// Make sure it has a type
if (field.type === FieldType.other) {
const t = guessFieldTypeForField(field);
if (t) {
field.type = t;
}
}
if (!this.fieldByType[field.type]) {
this.fieldByType[field.type] = [];
}
this.fieldByType[field.type].push(field);
}
addField(f: Field | FieldDTO): Field {
const type = f.type || FieldType.other;
const values =
!f.values || isArray(f.values)
? new ArrayVector(f.values as any[] | undefined) // array or empty
: (f.values as Vector);
// And a name
let name = f.name;
if (!name) {
if (type === FieldType.time) {
name = `Time ${this.fields.length + 1}`;
} else {
name = `Column ${this.fields.length + 1}`;
}
}
const field: Field = {
name,
type,
config: f.config || {},
values,
};
this.updateTypeIndex(field);
if (this.fieldByName[field.name]) {
console.warn('Duplicate field names in DataFrame: ', field.name);
} else {
this.fieldByName[field.name] = field;
}
// Make sure the lengths all match
if (field.values.length !== this.length) {
if (field.values.length > this.length) {
// Add `null` to all other values
const newlen = field.values.length;
for (const fx of this.fields) {
const arr = fx.values as ArrayVector;
while (fx.values.length !== newlen) {
arr.buffer.push(null);
}
}
this.length = field.values.length;
} else {
const arr = field.values as ArrayVector;
while (field.values.length !== this.length) {
arr.buffer.push(null);
}
}
}
this.fields.push(field);
return field;
}
/**
* This will add each value to the corresponding column
*/
appendRow(row: any[]) {
for (let i = this.fields.length; i < row.length; i++) {
this.addFieldFor(row[i]);
}
// The first line may change the field types
if (this.length < 1) {
this.fieldByType = {};
for (let i = 0; i < this.fields.length; i++) {
const f = this.fields[i];
if (!f.type || f.type === FieldType.other) {
f.type = guessFieldTypeFromValue(row[i]);
}
this.updateTypeIndex(f);
}
}
for (let i = 0; i < this.fields.length; i++) {
const f = this.fields[i];
let v = row[i];
if (!f.parse) {
f.parse = makeFieldParser(v, f);
}
v = f.parse(v);
const arr = f.values as ArrayVector;
arr.buffer.push(v); // may be undefined
}
this.length++;
}
/**
* Add any values that match the field names
*/
appendRowFrom(obj: { [key: string]: any }) {
for (const f of this.fields) {
const v = obj[f.name];
if (!f.parse) {
f.parse = makeFieldParser(v, f);
}
const arr = f.values as ArrayVector;
arr.buffer.push(f.parse(v)); // may be undefined
}
this.length++;
}
getFields(type?: FieldType): Field[] {
if (!type) {
return [...this.fields]; // All fields
}
const fields = this.fieldByType[type];
if (fields) {
return [...fields];
}
return [];
}
hasFieldOfType(type: FieldType): boolean {
const types = this.fieldByType[type];
return types && types.length > 0;
}
getFirstFieldOfType(type: FieldType): Field | undefined {
const arr = this.fieldByType[type];
if (arr && arr.length > 0) {
return arr[0];
}
return undefined;
}
hasFieldNamed(name: string): boolean {
return !!this.fieldByName[name];
}
/**
* Returns the first field with the given name.
*/
getFieldByName(name: string): Field | undefined {
return this.fieldByName[name];
}
}
function makeFieldParser(value: string, field: Field): (value: string) => any {
if (!field.type) {
if (field.name === 'time' || field.name === 'Time') {
field.type = FieldType.time;
} else {
field.type = guessFieldTypeFromValue(value);
}
}
if (field.type === FieldType.number) {
return (value: string) => {
return parseFloat(value);
};
}
// Will convert anything that starts with "T" to true
if (field.type === FieldType.boolean) {
return (value: string) => {
return !(value[0] === 'F' || value[0] === 'f' || value[0] === '0');
};
}
// Just pass the string back
return (value: string) => value;
}

76
packages/grafana-data/src/utils/dataFrameView.test.ts Normal file
View File

@@ -0,0 +1,76 @@
import { FieldType, DataFrameDTO } from '../types/index';
import { DataFrameHelper } from './dataFrameHelper';
import { DataFrameView } from './dataFrameView';
import { DateTime } from './moment_wrapper';
interface MySpecialObject {
time: DateTime;
name: string;
value: number;
more: string; // MISSING
}
describe('dataFrameView', () => {
const frame: DataFrameDTO = {
fields: [
{ name: 'time', type: FieldType.time, values: [100, 200, 300] },
{ name: 'name', type: FieldType.string, values: ['a', 'b', 'c'] },
{ name: 'value', type: FieldType.number, values: [1, 2, 3] },
],
};
const ext = new DataFrameHelper(frame);
const vector = new DataFrameView<MySpecialObject>(ext);
it('Should get a typed vector', () => {
expect(vector.length).toEqual(3);
const first = vector.get(0);
expect(first.time).toEqual(100);
expect(first.name).toEqual('a');
expect(first.value).toEqual(1);
expect(first.more).toBeUndefined();
});
it('Should support the spread operator', () => {
expect(vector.length).toEqual(3);
const first = vector.get(0);
const copy = { ...first };
expect(copy.time).toEqual(100);
expect(copy.name).toEqual('a');
expect(copy.value).toEqual(1);
expect(copy.more).toBeUndefined();
});
it('Should support array indexes', () => {
expect(vector.length).toEqual(3);
const first = vector.get(0) as any;
expect(first[0]).toEqual(100);
expect(first[1]).toEqual('a');
expect(first[2]).toEqual(1);
expect(first[3]).toBeUndefined();
});
it('Should advertise the property names for each field', () => {
expect(vector.length).toEqual(3);
const first = vector.get(0);
const keys = Object.keys(first);
expect(keys).toEqual(['time', 'name', 'value']);
});
it('has a weird side effect that the object values change after interation', () => {
expect(vector.length).toEqual(3);
// Get the first value
const first = vector.get(0);
expect(first.name).toEqual('a');
// Then get the second one
const second = vector.get(1);
// the values for 'first' have changed
expect(first.name).toEqual('b');
expect(first.name).toEqual(second.name);
});
});

67
packages/grafana-data/src/utils/dataFrameView.ts Normal file
View File

@@ -0,0 +1,67 @@
import { DataFrame, Vector } from '../types/index';
/**
* This abstraction will present the contents of a DataFrame as if
* it were a well typed javascript object Vector.
*
* NOTE: The contents of the object returned from `view.get(index)`
* are optimized for use in a loop. All calls return the same object
* but the index has changed.
*
* For example, the three objects:
* const first = view.get(0);
* const second = view.get(1);
* const third = view.get(2);
* will point to the contents at index 2
*
* If you need three different objects, consider something like:
* const first = { ... view.get(0) };
* const second = { ... view.get(1) };
* const third = { ... view.get(2) };
*/
export class DataFrameView<T = any> implements Vector<T> {
private index = 0;
private obj: T;
constructor(private data: DataFrame) {
const obj = ({} as unknown) as T;
for (let i = 0; i < data.fields.length; i++) {
const field = data.fields[i];
const getter = () => {
return field.values.get(this.index);
};
if (!(obj as any).hasOwnProperty(field.name)) {
Object.defineProperty(obj, field.name, {
enumerable: true, // Shows up as enumerable property
get: getter,
});
}
Object.defineProperty(obj, i, {
enumerable: false, // Don't enumerate array index
get: getter,
});
}
this.obj = obj;
}
get length() {
return this.data.length;
}
get(idx: number) {
this.index = idx;
return this.obj;
}
toArray(): T[] {
const arr: T[] = [];
for (let i = 0; i < this.data.length; i++) {
arr.push({ ...this.get(i) });
}
return arr;
}
toJSON(): T[] {
return this.toArray();
}
}

71
packages/grafana-data/src/utils/fieldCache.test.ts
View File

@@ -1,71 +0,0 @@
import { FieldType } from '../types/index';
import { FieldCache } from './fieldCache';
describe('FieldCache', () => {
it('when creating a new FieldCache from fields should be able to query cache', () => {
const fields = [
{ name: 'time', type: FieldType.time },
{ name: 'string', type: FieldType.string },
{ name: 'number', type: FieldType.number },
{ name: 'boolean', type: FieldType.boolean },
{ name: 'other', type: FieldType.other },
{ name: 'undefined' },
];
const fieldCache = new FieldCache(fields);
const allFields = fieldCache.getFields();
expect(allFields).toHaveLength(6);
const expectedFields = [
{ ...fields[0], index: 0 },
{ ...fields[1], index: 1 },
{ ...fields[2], index: 2 },
{ ...fields[3], index: 3 },
{ ...fields[4], index: 4 },
{ ...fields[5], type: FieldType.other, index: 5 },
];
expect(allFields).toMatchObject(expectedFields);
expect(fieldCache.hasFieldOfType(FieldType.time)).toBeTruthy();
expect(fieldCache.hasFieldOfType(FieldType.string)).toBeTruthy();
expect(fieldCache.hasFieldOfType(FieldType.number)).toBeTruthy();
expect(fieldCache.hasFieldOfType(FieldType.boolean)).toBeTruthy();
expect(fieldCache.hasFieldOfType(FieldType.other)).toBeTruthy();
expect(fieldCache.getFields(FieldType.time)).toMatchObject([expectedFields[0]]);
expect(fieldCache.getFields(FieldType.string)).toMatchObject([expectedFields[1]]);
expect(fieldCache.getFields(FieldType.number)).toMatchObject([expectedFields[2]]);
expect(fieldCache.getFields(FieldType.boolean)).toMatchObject([expectedFields[3]]);
expect(fieldCache.getFields(FieldType.other)).toMatchObject([expectedFields[4], expectedFields[5]]);
expect(fieldCache.getFieldByIndex(0)).toMatchObject(expectedFields[0]);
expect(fieldCache.getFieldByIndex(1)).toMatchObject(expectedFields[1]);
expect(fieldCache.getFieldByIndex(2)).toMatchObject(expectedFields[2]);
expect(fieldCache.getFieldByIndex(3)).toMatchObject(expectedFields[3]);
expect(fieldCache.getFieldByIndex(4)).toMatchObject(expectedFields[4]);
expect(fieldCache.getFieldByIndex(5)).toMatchObject(expectedFields[5]);
expect(fieldCache.getFieldByIndex(6)).toBeNull();
expect(fieldCache.getFirstFieldOfType(FieldType.time)).toMatchObject(expectedFields[0]);
expect(fieldCache.getFirstFieldOfType(FieldType.string)).toMatchObject(expectedFields[1]);
expect(fieldCache.getFirstFieldOfType(FieldType.number)).toMatchObject(expectedFields[2]);
expect(fieldCache.getFirstFieldOfType(FieldType.boolean)).toMatchObject(expectedFields[3]);
expect(fieldCache.getFirstFieldOfType(FieldType.other)).toMatchObject(expectedFields[4]);
expect(fieldCache.hasFieldNamed('tim')).toBeFalsy();
expect(fieldCache.hasFieldNamed('time')).toBeTruthy();
expect(fieldCache.hasFieldNamed('string')).toBeTruthy();
expect(fieldCache.hasFieldNamed('number')).toBeTruthy();
expect(fieldCache.hasFieldNamed('boolean')).toBeTruthy();
expect(fieldCache.hasFieldNamed('other')).toBeTruthy();
expect(fieldCache.hasFieldNamed('undefined')).toBeTruthy();
expect(fieldCache.getFieldByName('time')).toMatchObject(expectedFields[0]);
expect(fieldCache.getFieldByName('string')).toMatchObject(expectedFields[1]);
expect(fieldCache.getFieldByName('number')).toMatchObject(expectedFields[2]);
expect(fieldCache.getFieldByName('boolean')).toMatchObject(expectedFields[3]);
expect(fieldCache.getFieldByName('other')).toMatchObject(expectedFields[4]);
expect(fieldCache.getFieldByName('undefined')).toMatchObject(expectedFields[5]);
expect(fieldCache.getFieldByName('null')).toBeNull();
});
});

76
packages/grafana-data/src/utils/fieldCache.ts
View File

@@ -1,76 +0,0 @@
import { Field, FieldType } from '../types/index';
export interface IndexedField extends Field {
index: number;
}
export class FieldCache {
private fields: Field[];
private fieldIndexByName: { [key: string]: number };
private fieldIndexByType: { [key: string]: number[] };
constructor(fields?: Field[]) {
this.fields = [];
this.fieldIndexByName = {};
this.fieldIndexByType = {};
this.fieldIndexByType[FieldType.time] = [];
this.fieldIndexByType[FieldType.string] = [];
this.fieldIndexByType[FieldType.number] = [];
this.fieldIndexByType[FieldType.boolean] = [];
this.fieldIndexByType[FieldType.other] = [];
if (fields) {
for (let n = 0; n < fields.length; n++) {
const field = fields[n];
this.addField(field);
}
}
}
addField(field: Field) {
this.fields.push({
type: FieldType.other,
...field,
});
const index = this.fields.length - 1;
this.fieldIndexByName[field.name] = index;
this.fieldIndexByType[field.type || FieldType.other].push(index);
}
hasFieldOfType(type: FieldType): boolean {
return this.fieldIndexByType[type] && this.fieldIndexByType[type].length > 0;
}
getFields(type?: FieldType): IndexedField[] {
const fields: IndexedField[] = [];
for (let index = 0; index < this.fields.length; index++) {
const field = this.fields[index];
if (!type || field.type === type) {
fields.push({ ...field, index });
}
}
return fields;
}
getFieldByIndex(index: number): IndexedField | null {
return this.fields[index] ? { ...this.fields[index], index } : null;
}
getFirstFieldOfType(type: FieldType): IndexedField | null {
return this.hasFieldOfType(type)
? { ...this.fields[this.fieldIndexByType[type][0]], index: this.fieldIndexByType[type][0] }
: null;
}
hasFieldNamed(name: string): boolean {
return this.fieldIndexByName[name] !== undefined;
}
getFieldByName(name: string): IndexedField | null {
return this.hasFieldNamed(name)
? { ...this.fields[this.fieldIndexByName[name]], index: this.fieldIndexByName[name] }
: null;
}
}

90
packages/grafana-data/src/utils/fieldReducer.test.ts
View File

@@ -1,20 +1,32 @@
import { fieldReducers, ReducerID, reduceField } from './fieldReducer';
import _ from 'lodash';
import { DataFrame } from '../types/data';
import { Field, FieldType } from '../types/index';
import { DataFrameHelper } from './dataFrameHelper';
import { ArrayVector } from './vector';
import { guessFieldTypeFromValue } from './processDataFrame';
/**
* Run a reducer and get back the value
*/
function reduce(series: DataFrame, fieldIndex: number, id: string): any {
return reduceField({ series, fieldIndex, reducers: [id] })[id];
function reduce(field: Field, id: string): any {
return reduceField({ field, reducers: [id] })[id];
}
function createField<T>(name: string, values?: T[], type?: FieldType): Field<T> {
const arr = new ArrayVector(values);
return {
name,
config: {},
type: type ? type : guessFieldTypeFromValue(arr.get(0)),
values: arr,
};
}
describe('Stats Calculators', () => {
const basicTable = {
fields: [{ name: 'a' }, { name: 'b' }, { name: 'c' }],
rows: [[10, 20, 30], [20, 30, 40]],
};
const basicTable = new DataFrameHelper({
fields: [{ name: 'a', values: [10, 20] }, { name: 'b', values: [20, 30] }, { name: 'c', values: [30, 40] }],
});
it('should load all standard stats', () => {
for (const id of Object.keys(ReducerID)) {
@@ -38,8 +50,7 @@ describe('Stats Calculators', () => {
it('should calculate basic stats', () => {
const stats = reduceField({
series: basicTable,
fieldIndex: 0,
field: basicTable.fields[0],
reducers: ['first', 'last', 'mean'],
});
@@ -54,9 +65,9 @@ describe('Stats Calculators', () => {
});
it('should support a single stat also', () => {
basicTable.fields[0].calcs = undefined; // clear the cache
const stats = reduceField({
series: basicTable,
fieldIndex: 0,
field: basicTable.fields[0],
reducers: ['first'],
});
@@ -67,8 +78,7 @@ describe('Stats Calculators', () => {
it('should get non standard stats', () => {
const stats = reduceField({
series: basicTable,
fieldIndex: 0,
field: basicTable.fields[0],
reducers: [ReducerID.distinctCount, ReducerID.changeCount],
});
@@ -78,8 +88,7 @@ describe('Stats Calculators', () => {
it('should calculate step', () => {
const stats = reduceField({
series: { fields: [{ name: 'A' }], rows: [[100], [200], [300], [400]] },
fieldIndex: 0,
field: createField('x', [100, 200, 300, 400]),
reducers: [ReducerID.step, ReducerID.delta],
});
@@ -88,53 +97,38 @@ describe('Stats Calculators', () => {
});
it('consistenly check allIsNull/allIsZero', () => {
const empty = {
fields: [{ name: 'A' }],
rows: [],
};
const allNull = ({
fields: [{ name: 'A' }],
rows: [null, null, null, null],
} as unknown) as DataFrame;
const allNull2 = {
fields: [{ name: 'A' }],
rows: [[null], [null], [null], [null]],
};
const allZero = {
fields: [{ name: 'A' }],
rows: [[0], [0], [0], [0]],
};
const empty = createField('x');
const allNull = createField('x', [null, null, null, null]);
const allUndefined = createField('x', [undefined, undefined, undefined, undefined]);
const allZero = createField('x', [0, 0, 0, 0]);
expect(reduce(empty, 0, ReducerID.allIsNull)).toEqual(true);
expect(reduce(allNull, 0, ReducerID.allIsNull)).toEqual(true);
expect(reduce(allNull2, 0, ReducerID.allIsNull)).toEqual(true);
expect(reduce(empty, ReducerID.allIsNull)).toEqual(true);
expect(reduce(allNull, ReducerID.allIsNull)).toEqual(true);
expect(reduce(allUndefined, ReducerID.allIsNull)).toEqual(true);
expect(reduce(empty, 0, ReducerID.allIsZero)).toEqual(false);
expect(reduce(allNull, 0, ReducerID.allIsZero)).toEqual(false);
expect(reduce(allNull2, 0, ReducerID.allIsZero)).toEqual(false);
expect(reduce(allZero, 0, ReducerID.allIsZero)).toEqual(true);
expect(reduce(empty, ReducerID.allIsZero)).toEqual(false);
expect(reduce(allNull, ReducerID.allIsZero)).toEqual(false);
expect(reduce(allZero, ReducerID.allIsZero)).toEqual(true);
});
it('consistent results for first/last value with null', () => {
const info = [
{
rows: [[null], [200], [null]], // first/last value is null
data: [null, 200, null], // first/last value is null
result: 200,
},
{
rows: [[null], [null], [null]], // All null
data: [null, null, null], // All null
result: undefined,
},
{
rows: [], // Empty row
data: [undefined, undefined, undefined], // Empty row
result: undefined,
},
];
const fields = [{ name: 'A' }];
const stats = reduceField({
series: { rows: info[0].rows, fields },
fieldIndex: 0,
field: createField('x', info[0].data),
reducers: [ReducerID.first, ReducerID.last, ReducerID.firstNotNull, ReducerID.lastNotNull], // uses standard path
});
expect(stats[ReducerID.first]).toEqual(null);
@@ -146,21 +140,19 @@ describe('Stats Calculators', () => {
for (const input of info) {
for (const reducer of reducers) {
const v1 = reduceField({
series: { rows: input.rows, fields },
fieldIndex: 0,
field: createField('x', input.data),
reducers: [reducer, ReducerID.mean], // uses standard path
})[reducer];
const v2 = reduceField({
series: { rows: input.rows, fields },
fieldIndex: 0,
field: createField('x', input.data),
reducers: [reducer], // uses optimized path
})[reducer];
if (v1 !== v2 || v1 !== input.result) {
const msg =
`Invalid ${reducer} result for: ` +
input.rows.join(', ') +
input.data.join(', ') +
` Expected: ${input.result}` + // configured
` Recieved: Multiple: ${v1}, Single: ${v2}`;
expect(msg).toEqual(null);

116
packages/grafana-data/src/utils/fieldReducer.ts
View File

@@ -1,7 +1,7 @@
// Libraries
import isNumber from 'lodash/isNumber';
import { DataFrame, NullValueMode } from '../types';
import { NullValueMode, Field } from '../types';
import { Registry, RegistryItem } from './registry';
export enum ReducerID {
@@ -33,7 +33,7 @@ export interface FieldCalcs {
}
// Internal function
type FieldReducer = (data: DataFrame, fieldIndex: number, ignoreNulls: boolean, nullAsZero: boolean) => FieldCalcs;
type FieldReducer = (field: Field, ignoreNulls: boolean, nullAsZero: boolean) => FieldCalcs;
export interface FieldReducerInfo extends RegistryItem {
// Internal details
@@ -43,52 +43,76 @@ export interface FieldReducerInfo extends RegistryItem {
}
interface ReduceFieldOptions {
series: DataFrame;
fieldIndex: number;
field: Field;
reducers: string[]; // The stats to calculate
nullValueMode?: NullValueMode;
}
/**
* @returns an object with a key for each selected stat
*/
export function reduceField(options: ReduceFieldOptions): FieldCalcs {
const { series, fieldIndex, reducers, nullValueMode } = options;
const { field, reducers } = options;
if (!reducers || reducers.length < 1) {
if (!field || !reducers || reducers.length < 1) {
return {};
}
if (field.calcs) {
// Find the values we need to calculate
const missing: string[] = [];
for (const s of reducers) {
if (!field.calcs.hasOwnProperty(s)) {
missing.push(s);
}
}
if (missing.length < 1) {
return {
...field.calcs,
};
}
}
const queue = fieldReducers.list(reducers);
// Return early for empty series
// This lets the concrete implementations assume at least one row
if (!series.rows || series.rows.length < 1) {
const calcs = {} as FieldCalcs;
const data = field.values;
if (data.length < 1) {
const calcs = { ...field.calcs } as FieldCalcs;
for (const reducer of queue) {
calcs[reducer.id] = reducer.emptyInputResult !== null ? reducer.emptyInputResult : null;
}
return calcs;
return (field.calcs = calcs);
}
const { nullValueMode } = field.config;
const ignoreNulls = nullValueMode === NullValueMode.Ignore;
const nullAsZero = nullValueMode === NullValueMode.AsZero;
// Avoid calculating all the standard stats if possible
if (queue.length === 1 && queue[0].reduce) {
return queue[0].reduce(series, fieldIndex, ignoreNulls, nullAsZero);
const values = queue[0].reduce(field, ignoreNulls, nullAsZero);
field.calcs = {
...field.calcs,
...values,
};
return values;
}
// For now everything can use the standard stats
let values = doStandardCalcs(series, fieldIndex, ignoreNulls, nullAsZero);
let values = doStandardCalcs(field, ignoreNulls, nullAsZero);
for (const reducer of queue) {
if (!values.hasOwnProperty(reducer.id) && reducer.reduce) {
values = {
...values,
...reducer.reduce(series, fieldIndex, ignoreNulls, nullAsZero),
...reducer.reduce(field, ignoreNulls, nullAsZero),
};
}
}
field.calcs = {
...field.calcs,
...values,
};
return values;
}
@@ -200,7 +224,7 @@ export const fieldReducers = new Registry<FieldReducerInfo>(() => [
},
]);
function doStandardCalcs(data: DataFrame, fieldIndex: number, ignoreNulls: boolean, nullAsZero: boolean): FieldCalcs {
function doStandardCalcs(field: Field, ignoreNulls: boolean, nullAsZero: boolean): FieldCalcs {
const calcs = {
sum: 0,
max: -Number.MAX_VALUE,
@@ -223,9 +247,10 @@ function doStandardCalcs(data: DataFrame, fieldIndex: number, ignoreNulls: boole
// Just used for calcutations -- not exposed as a stat
previousDeltaUp: true,
} as FieldCalcs;
const data = field.values;
for (let i = 0; i < data.rows.length; i++) {
let currentValue = data.rows[i] ? data.rows[i][fieldIndex] : null;
for (let i = 0; i < data.length; i++) {
let currentValue = data.get(i);
if (i === 0) {
calcs.first = currentValue;
}
@@ -260,7 +285,7 @@ function doStandardCalcs(data: DataFrame, fieldIndex: number, ignoreNulls: boole
if (calcs.lastNotNull! > currentValue) {
// counter reset
calcs.previousDeltaUp = false;
if (i === data.rows.length - 1) {
if (i === data.length - 1) {
// reset on last
calcs.delta += currentValue;
}
@@ -326,18 +351,14 @@ function doStandardCalcs(data: DataFrame, fieldIndex: number, ignoreNulls: boole
return calcs;
}
function calculateFirst(data: DataFrame, fieldIndex: number, ignoreNulls: boolean, nullAsZero: boolean): FieldCalcs {
return { first: data.rows[0][fieldIndex] };
function calculateFirst(field: Field, ignoreNulls: boolean, nullAsZero: boolean): FieldCalcs {
return { first: field.values.get(0) };
}
function calculateFirstNotNull(
data: DataFrame,
fieldIndex: number,
ignoreNulls: boolean,
nullAsZero: boolean
): FieldCalcs {
for (let idx = 0; idx < data.rows.length; idx++) {
const v = data.rows[idx][fieldIndex];
function calculateFirstNotNull(field: Field, ignoreNulls: boolean, nullAsZero: boolean): FieldCalcs {
const data = field.values;
for (let idx = 0; idx < data.length; idx++) {
const v = data.get(idx);
if (v != null) {
return { firstNotNull: v };
}
@@ -345,19 +366,16 @@ function calculateFirstNotNull(
return { firstNotNull: undefined };
}
function calculateLast(data: DataFrame, fieldIndex: number, ignoreNulls: boolean, nullAsZero: boolean): FieldCalcs {
return { last: data.rows[data.rows.length - 1][fieldIndex] };
function calculateLast(field: Field, ignoreNulls: boolean, nullAsZero: boolean): FieldCalcs {
const data = field.values;
return { last: data.get(data.length - 1) };
}
function calculateLastNotNull(
data: DataFrame,
fieldIndex: number,
ignoreNulls: boolean,
nullAsZero: boolean
): FieldCalcs {
let idx = data.rows.length - 1;
function calculateLastNotNull(field: Field, ignoreNulls: boolean, nullAsZero: boolean): FieldCalcs {
const data = field.values;
let idx = data.length - 1;
while (idx >= 0) {
const v = data.rows[idx--][fieldIndex];
const v = data.get(idx--);
if (v != null) {
return { lastNotNull: v };
}
@@ -365,17 +383,13 @@ function calculateLastNotNull(
return { lastNotNull: undefined };
}
function calculateChangeCount(
data: DataFrame,
fieldIndex: number,
ignoreNulls: boolean,
nullAsZero: boolean
): FieldCalcs {
function calculateChangeCount(field: Field, ignoreNulls: boolean, nullAsZero: boolean): FieldCalcs {
const data = field.values;
let count = 0;
let first = true;
let last: any = null;
for (let i = 0; i < data.rows.length; i++) {
let currentValue = data.rows[i][fieldIndex];
for (let i = 0; i < data.length; i++) {
let currentValue = data.get(i);
if (currentValue === null) {
if (ignoreNulls) {
continue;
@@ -394,15 +408,11 @@ function calculateChangeCount(
return { changeCount: count };
}
function calculateDistinctCount(
data: DataFrame,
fieldIndex: number,
ignoreNulls: boolean,
nullAsZero: boolean
): FieldCalcs {
function calculateDistinctCount(field: Field, ignoreNulls: boolean, nullAsZero: boolean): FieldCalcs {
const data = field.values;
const distinct = new Set<any>();
for (let i = 0; i < data.rows.length; i++) {
let currentValue = data.rows[i][fieldIndex];
for (let i = 0; i < data.length; i++) {
let currentValue = data.get(i);
if (currentValue === null) {
if (ignoreNulls) {
continue;

4
packages/grafana-data/src/utils/index.ts
View File

@@ -8,9 +8,11 @@ export * from './logs';
export * from './labels';
export * from './labels';
export * from './object';
export * from './fieldCache';
export * from './moment_wrapper';
export * from './thresholds';
export * from './dataFrameHelper';
export * from './dataFrameView';
export * from './vector';
export { getMappedValue } from './valueMappings';

24
packages/grafana-data/src/utils/logs.ts
View File

@@ -1,5 +1,6 @@
import { LogLevel } from '../types/logs';
import { DataFrame, FieldType } from '../types/data';
import { DataFrame, FieldType } from '../types/index';
import { ArrayVector } from './vector';
/**
* Returns the log level of a log line.
@@ -33,12 +34,23 @@ export function getLogLevelFromKey(key: string): LogLevel {
}
export function addLogLevelToSeries(series: DataFrame, lineIndex: number): DataFrame {
const levels = new ArrayVector<LogLevel>();
const lines = series.fields[lineIndex];
for (let i = 0; i < lines.values.length; i++) {
const line = lines.values.get(lineIndex);
levels.buffer.push(getLogLevel(line));
}
return {
...series, // Keeps Tags, RefID etc
fields: [...series.fields, { name: 'LogLevel', type: FieldType.string }],
rows: series.rows.map(row => {
const line = row[lineIndex];
return [...row, getLogLevel(line)];
}),
fields: [
...series.fields,
{
name: 'LogLevel',
type: FieldType.string,
values: levels,
config: {},
},
],
};
}

83
packages/grafana-data/src/utils/processDataFrame.test.ts
View File

@@ -5,9 +5,11 @@ import {
toDataFrame,
guessFieldTypes,
guessFieldTypeFromValue,
sortDataFrame,
} from './processDataFrame';
import { FieldType, TimeSeries, DataFrame, TableData } from '../types/data';
import { FieldType, TimeSeries, TableData, DataFrameDTO } from '../types/index';
import { dateTime } from './moment_wrapper';
import { DataFrameHelper } from './dataFrameHelper';
describe('toDataFrame', () => {
it('converts timeseries to series', () => {
@@ -17,7 +19,15 @@ describe('toDataFrame', () => {
};
let series = toDataFrame(input1);
expect(series.fields[0].name).toBe(input1.target);
expect(series.rows).toBe(input1.datapoints);
const v0 = series.fields[0].values;
const v1 = series.fields[1].values;
expect(v0.length).toEqual(2);
expect(v1.length).toEqual(2);
expect(v0.get(0)).toEqual(100);
expect(v0.get(1)).toEqual(200);
expect(v1.get(0)).toEqual(1);
expect(v1.get(1)).toEqual(2);
// Should fill a default name if target is empty
const input2 = {
@@ -39,12 +49,23 @@ describe('toDataFrame', () => {
});
it('keeps dataFrame unchanged', () => {
const input = {
fields: [{ text: 'A' }, { text: 'B' }, { text: 'C' }],
const input = toDataFrame({
datapoints: [[100, 1], [200, 2]],
});
expect(input.length).toEqual(2);
// If the object is alreay a DataFrame, it should not change
const again = toDataFrame(input);
expect(again).toBe(input);
});
it('migrate from 6.3 style rows', () => {
const oldDataFrame = {
fields: [{ name: 'A' }, { name: 'B' }, { name: 'C' }],
rows: [[100, 'A', 1], [200, 'B', 2], [300, 'C', 3]],
};
const series = toDataFrame(input);
expect(series).toBe(input);
const data = toDataFrame(oldDataFrame);
expect(data.length).toBe(oldDataFrame.rows.length);
});
it('Guess Colum Types from value', () => {
@@ -68,14 +89,18 @@ describe('toDataFrame', () => {
});
it('Guess Colum Types from series', () => {
const series = {
fields: [{ name: 'A (number)' }, { name: 'B (strings)' }, { name: 'C (nulls)' }, { name: 'Time' }],
rows: [[123, null, null, '2000'], [null, 'Hello', null, 'XXX']],
};
const series = new DataFrameHelper({
fields: [
{ name: 'A (number)', values: [123, null] },
{ name: 'B (strings)', values: [null, 'Hello'] },
{ name: 'C (nulls)', values: [null, null] },
{ name: 'Time', values: ['2000', 1967] },
],
});
const norm = guessFieldTypes(series);
expect(norm.fields[0].type).toBe(FieldType.number);
expect(norm.fields[1].type).toBe(FieldType.string);
expect(norm.fields[2].type).toBeUndefined();
expect(norm.fields[2].type).toBe(FieldType.other);
expect(norm.fields[3].type).toBe(FieldType.time); // based on name
});
});
@@ -103,6 +128,7 @@ describe('SerisData backwards compatibility', () => {
const series = toDataFrame(table);
expect(isTableData(table)).toBeTruthy();
expect(isDataFrame(series)).toBeTruthy();
expect(series.fields[0].config.unit).toEqual('ms');
const roundtrip = toLegacyResponseData(series) as TimeSeries;
expect(isTableData(roundtrip)).toBeTruthy();
@@ -110,23 +136,46 @@ describe('SerisData backwards compatibility', () => {
});
it('converts DataFrame to TableData to series and back again', () => {
const series: DataFrame = {
const json: DataFrameDTO = {
refId: 'Z',
meta: {
somethign: 8,
},
fields: [
{ name: 'T', type: FieldType.time }, // first
{ name: 'N', type: FieldType.number, filterable: true },
{ name: 'S', type: FieldType.string, filterable: true },
{ name: 'T', type: FieldType.time, values: [1, 2, 3] },
{ name: 'N', type: FieldType.number, config: { filterable: true }, values: [100, 200, 300] },
{ name: 'S', type: FieldType.string, config: { filterable: true }, values: ['1', '2', '3'] },
],
rows: [[1, 100, '1'], [2, 200, '2'], [3, 300, '3']],
};
const series = toDataFrame(json);
const table = toLegacyResponseData(series) as TableData;
expect(table.meta).toBe(series.meta);
expect(table.refId).toBe(series.refId);
expect(table.meta).toEqual(series.meta);
const names = table.columns.map(c => c.text);
expect(names).toEqual(['T', 'N', 'S']);
});
});
describe('sorted DataFrame', () => {
const frame = toDataFrame({
fields: [
{ name: 'fist', type: FieldType.time, values: [1, 2, 3] },
{ name: 'second', type: FieldType.string, values: ['a', 'b', 'c'] },
{ name: 'third', type: FieldType.number, values: [2000, 3000, 1000] },
],
});
it('Should sort numbers', () => {
const sorted = sortDataFrame(frame, 0, true);
expect(sorted.length).toEqual(3);
expect(sorted.fields[0].values.toJSON()).toEqual([3, 2, 1]);
expect(sorted.fields[1].values.toJSON()).toEqual(['c', 'b', 'a']);
});
it('Should sort strings', () => {
const sorted = sortDataFrame(frame, 1, true);
expect(sorted.length).toEqual(3);
expect(sorted.fields[0].values.toJSON()).toEqual([3, 2, 1]);
expect(sorted.fields[1].values.toJSON()).toEqual(['c', 'b', 'a']);
});
});

279
packages/grafana-data/src/utils/processDataFrame.ts
View File

@@ -4,61 +4,122 @@ import isString from 'lodash/isString';
import isBoolean from 'lodash/isBoolean';
// Types
import { DataFrame, Field, TimeSeries, FieldType, TableData, Column, GraphSeriesXY } from '../types/index';
import {
DataFrame,
Field,
FieldConfig,
TimeSeries,
FieldType,
TableData,
Column,
GraphSeriesXY,
TimeSeriesValue,
FieldDTO,
DataFrameDTO,
} from '../types/index';
import { isDateTime } from './moment_wrapper';
import { ArrayVector, SortedVector } from './vector';
import { DataFrameHelper } from './dataFrameHelper';
function convertTableToDataFrame(table: TableData): DataFrame {
const fields = table.columns.map(c => {
const { text, ...disp } = c;
return {
name: text, // rename 'text' to the 'name' field
config: (disp || {}) as FieldConfig,
values: new ArrayVector(),
type: FieldType.other,
};
});
// Fill in the field values
for (const row of table.rows) {
for (let i = 0; i < fields.length; i++) {
fields[i].values.buffer.push(row[i]);
}
}
for (const f of fields) {
const t = guessFieldTypeForField(f);
if (t) {
f.type = t;
}
}
return {
// rename the 'text' to 'name' field
fields: table.columns.map(c => {
const { text, ...field } = c;
const f = field as Field;
f.name = text;
return f;
}),
rows: table.rows,
fields,
refId: table.refId,
meta: table.meta,
name: table.name,
length: fields[0].values.length,
};
}
function convertTimeSeriesToDataFrame(timeSeries: TimeSeries): DataFrame {
return {
name: timeSeries.target,
fields: [
{
name: timeSeries.target || 'Value',
type: FieldType.number,
const fields = [
{
name: timeSeries.target || 'Value',
type: FieldType.number,
config: {
unit: timeSeries.unit,
},
{
name: 'Time',
type: FieldType.time,
values: new ArrayVector<TimeSeriesValue>(),
},
{
name: 'Time',
type: FieldType.time,
config: {
unit: 'dateTimeAsIso',
},
],
rows: timeSeries.datapoints,
values: new ArrayVector<number>(),
},
];
for (const point of timeSeries.datapoints) {
fields[0].values.buffer.push(point[0]);
fields[1].values.buffer.push(point[1]);
}
return {
name: timeSeries.target,
labels: timeSeries.tags,
refId: timeSeries.refId,
meta: timeSeries.meta,
fields,
length: timeSeries.datapoints.length,
};
}
/**
* This is added temporarily while we convert the LogsModel
* to DataFrame. See: https://github.com/grafana/grafana/issues/18528
*/
function convertGraphSeriesToDataFrame(graphSeries: GraphSeriesXY): DataFrame {
const x = new ArrayVector();
const y = new ArrayVector();
for (let i = 0; i < graphSeries.data.length; i++) {
const row = graphSeries.data[i];
x.buffer.push(row[0]);
y.buffer.push(row[1]);
}
return {
name: graphSeries.label,
fields: [
{
name: graphSeries.label || 'Value',
type: FieldType.number,
config: {},
values: x,
},
{
name: 'Time',
type: FieldType.time,
unit: 'dateTimeAsIso',
config: {
unit: 'dateTimeAsIso',
},
values: y,
},
],
rows: graphSeries.data,
length: x.buffer.length,
};
}
@@ -102,20 +163,18 @@ export function guessFieldTypeFromValue(v: any): FieldType {
/**
* Looks at the data to guess the column type. This ignores any existing setting
*/
export function guessFieldTypeFromSeries(series: DataFrame, index: number): FieldType | undefined {
const column = series.fields[index];
export function guessFieldTypeForField(field: Field): FieldType | undefined {
// 1. Use the column name to guess
if (column.name) {
const name = column.name.toLowerCase();
if (field.name) {
const name = field.name.toLowerCase();
if (name === 'date' || name === 'time') {
return FieldType.time;
}
}
// 2. Check the first non-null value
for (let i = 0; i < series.rows.length; i++) {
const v = series.rows[i][index];
for (let i = 0; i < field.values.length; i++) {
const v = field.values.get(i);
if (v !== null) {
return guessFieldTypeFromValue(v);
}
@@ -135,14 +194,14 @@ export const guessFieldTypes = (series: DataFrame): DataFrame => {
// Somethign is missing a type return a modified copy
return {
...series,
fields: series.fields.map((field, index) => {
if (field.type) {
fields: series.fields.map(field => {
if (field.type && field.type !== FieldType.other) {
return field;
}
// Replace it with a calculated version
// Calculate a reasonable schema value
return {
...field,
type: guessFieldTypeFromSeries(series, index),
type: guessFieldTypeForField(field) || FieldType.other,
};
}),
};
@@ -158,7 +217,22 @@ export const isDataFrame = (data: any): data is DataFrame => data && data.hasOwn
export const toDataFrame = (data: any): DataFrame => {
if (data.hasOwnProperty('fields')) {
return data as DataFrame;
// @deprecated -- remove in 6.5
if (data.hasOwnProperty('rows')) {
const v = new DataFrameHelper(data as DataFrameDTO);
const rows = data.rows as any[][];
for (let i = 0; i < rows.length; i++) {
v.appendRow(rows[i]);
}
// TODO: deprection warning
return v;
}
// DataFrameDTO does not have length
if (data.hasOwnProperty('length')) {
return data as DataFrame;
}
return new DataFrameHelper(data as DataFrameDTO);
}
if (data.hasOwnProperty('datapoints')) {
return convertTimeSeriesToDataFrame(data);
@@ -174,52 +248,129 @@ export const toDataFrame = (data: any): DataFrame => {
throw new Error('Unsupported data format');
};
export const toLegacyResponseData = (series: DataFrame): TimeSeries | TableData => {
const { fields, rows } = series;
export const toLegacyResponseData = (frame: DataFrame): TimeSeries | TableData => {
const { fields } = frame;
const length = fields[0].values.length;
const rows: any[][] = [];
for (let i = 0; i < length; i++) {
const row: any[] = [];
for (let j = 0; j < fields.length; j++) {
row.push(fields[j].values.get(i));
}
rows.push(row);
}
if (fields.length === 2) {
const type = guessFieldTypeFromSeries(series, 1);
let type = fields[1].type;
if (!type) {
type = guessFieldTypeForField(fields[1]) || FieldType.other;
}
if (type === FieldType.time) {
return {
alias: fields[0].name || series.name,
target: fields[0].name || series.name,
alias: fields[0].name || frame.name,
target: fields[0].name || frame.name,
datapoints: rows,
unit: fields[0].unit,
refId: series.refId,
meta: series.meta,
unit: fields[0].config ? fields[0].config.unit : undefined,
refId: frame.refId,
meta: frame.meta,
} as TimeSeries;
}
}
return {
columns: fields.map(f => {
const { name, ...column } = f;
(column as Column).text = name;
return column as Column;
const { name, config } = f;
if (config) {
// keep unit etc
const { ...column } = config;
(column as Column).text = name;
return column as Column;
}
return { text: name };
}),
refId: series.refId,
meta: series.meta,
refId: frame.refId,
meta: frame.meta,
rows,
};
};
export function sortDataFrame(data: DataFrame, sortIndex?: number, reverse = false): DataFrame {
if (isNumber(sortIndex)) {
const copy = {
...data,
rows: [...data.rows].sort((a, b) => {
a = a[sortIndex];
b = b[sortIndex];
// Sort null or undefined separately from comparable values
return +(a == null) - +(b == null) || +(a > b) || -(a < b);
}),
};
if (reverse) {
copy.rows.reverse();
}
return copy;
const field = data.fields[sortIndex!];
if (!field) {
return data;
}
return data;
// Natural order
const index: number[] = [];
for (let i = 0; i < data.length; i++) {
index.push(i);
}
const values = field.values;
// Numeric Comparison
let compare = (a: number, b: number) => {
const vA = values.get(a);
const vB = values.get(b);
return vA - vB; // works for numbers!
};
// String Comparison
if (field.type === FieldType.string) {
compare = (a: number, b: number) => {
const vA: string = values.get(a);
const vB: string = values.get(b);
return vA.localeCompare(vB);
};
}
// Run the sort function
index.sort(compare);
if (reverse) {
index.reverse();
}
// Return a copy that maps sorted values
return {
...data,
fields: data.fields.map(f => {
return {
...f,
values: new SortedVector(f.values, index),
};
}),
};
}
/**
* Wrapper to get an array from each field value
*/
export function getDataFrameRow(data: DataFrame, row: number): any[] {
const values: any[] = [];
for (const field of data.fields) {
values.push(field.values.get(row));
}
return values;
}
/**
* Returns a copy that does not include functions
*/
export function toDataFrameDTO(data: DataFrame): DataFrameDTO {
const fields: FieldDTO[] = data.fields.map(f => {
return {
name: f.name,
type: f.type,
config: f.config,
values: f.values.toJSON(),
};
});
return {
fields,
refId: data.refId,
meta: data.meta,
name: data.name,
labels: data.labels,
};
}

43
packages/grafana-data/src/utils/vector.test.ts Normal file
View File

@@ -0,0 +1,43 @@
import { ConstantVector, ScaledVector, ArrayVector, CircularVector } from './vector';
describe('Check Proxy Vector', () => {
it('should support constant values', () => {
const value = 3.5;
const v = new ConstantVector(value, 7);
expect(v.length).toEqual(7);
expect(v.get(0)).toEqual(value);
expect(v.get(1)).toEqual(value);
// Now check all of them
for (let i = 0; i < 10; i++) {
expect(v.get(i)).toEqual(value);
}
});
it('should support multiply operations', () => {
const source = new ArrayVector([1, 2, 3, 4]);
const scale = 2.456;
const v = new ScaledVector(source, scale);
expect(v.length).toEqual(source.length);
// expect(v.push(10)).toEqual(source.length); // not implemented
for (let i = 0; i < 10; i++) {
expect(v.get(i)).toEqual(source.get(i) * scale);
}
});
});
describe('Check Circular Vector', () => {
it('should support constant values', () => {
const buffer = [3, 2, 1, 0];
const v = new CircularVector(buffer);
expect(v.length).toEqual(4);
expect(v.toJSON()).toEqual([3, 2, 1, 0]);
v.append(4);
expect(v.toJSON()).toEqual([4, 3, 2, 1]);
v.append(5);
expect(v.toJSON()).toEqual([5, 4, 3, 2]);
});
});

133
packages/grafana-data/src/utils/vector.ts Normal file
View File

@@ -0,0 +1,133 @@
import { Vector } from '../types/dataFrame';
export function vectorToArray<T>(v: Vector<T>): T[] {
const arr: T[] = [];
for (let i = 0; i < v.length; i++) {
arr[i] = v.get(i);
}
return arr;
}
export class ArrayVector<T = any> implements Vector<T> {
buffer: T[];
constructor(buffer?: T[]) {
this.buffer = buffer ? buffer : [];
}
get length() {
return this.buffer.length;
}
get(index: number): T {
return this.buffer[index];
}
toArray(): T[] {
return this.buffer;
}
toJSON(): T[] {
return this.buffer;
}
}
export class ConstantVector<T = any> implements Vector<T> {
constructor(private value: T, private len: number) {}
get length() {
return this.len;
}
get(index: number): T {
return this.value;
}
toArray(): T[] {
const arr: T[] = [];
for (let i = 0; i < this.length; i++) {
arr[i] = this.value;
}
return arr;
}
toJSON(): T[] {
return this.toArray();
}
}
export class ScaledVector implements Vector<number> {
constructor(private source: Vector<number>, private scale: number) {}
get length(): number {
return this.source.length;
}
get(index: number): number {
return this.source.get(index) * this.scale;
}
toArray(): number[] {
return vectorToArray(this);
}
toJSON(): number[] {
return vectorToArray(this);
}
}
export class CircularVector<T = any> implements Vector<T> {
buffer: T[];
index: number;
length: number;
constructor(buffer: T[]) {
this.length = buffer.length;
this.buffer = buffer;
this.index = 0;
}
append(value: T) {
let idx = this.index - 1;
if (idx < 0) {
idx = this.length - 1;
}
this.buffer[idx] = value;
this.index = idx;
}
get(index: number): T {
return this.buffer[(index + this.index) % this.length];
}
toArray(): T[] {
return vectorToArray(this);
}
toJSON(): T[] {
return vectorToArray(this);
}
}
/**
* Values are returned in the order defined by the input parameter
*/
export class SortedVector<T = any> implements Vector<T> {
constructor(private source: Vector<T>, private order: number[]) {}
get length(): number {
return this.source.length;
}
get(index: number): T {
return this.source.get(this.order[index]);
}
toArray(): T[] {
return vectorToArray(this);
}
toJSON(): T[] {
return vectorToArray(this);
}
}