IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

DataFrame: round trip metadata to arrow Table (#21277)

Browse Source
This commit is contained in:
Ryan McKinley 2020-01-03 08:12:19 -08:00 committed by GitHub
parent cb3d91b53c
commit e550572033
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 224 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/grafana-data/rollup.config.ts
View File

@ -20,7 +20,7 @@ const buildCjsPackage = ({ env }) => {
globals: {},
},
],
external: ['lodash'], // Use Lodash from grafana
external: ['lodash', 'apache-arrow'], // Use Lodash & arrow from grafana
plugins: [
commonjs({
include: /node_modules/,

31
packages/grafana-data/src/dataframe/arrow/ArrowDataFrame.test.ts → packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts
View File

@ -1,5 +1,6 @@
import { resultsToDataFrames } from './ArrowDataFrame';
import { toDataFrameDTO } from '../processDataFrame';
import { resultsToDataFrames, grafanaDataFrameToArrowTable, arrowTableToDataFrame } from './ArrowDataFrame';
import { toDataFrameDTO, toDataFrame } from './processDataFrame';
import { FieldType } from '../types';
/* tslint:disable */
const resp = {
@ -33,3 +34,29 @@ describe('GEL Utils', () => {
expect(norm).toMatchSnapshot();
});
});
describe('Read/Write arrow Table to DataFrame', () => {
test('should parse output with dataframe', () => {
const frame = toDataFrame({
name: 'Hello',
refId: 'XYZ',
meta: {
aaa: 'xyz',
anything: 'xxx',
},
fields: [
{ name: 'time', config: {}, type: FieldType.time, values: [1, 2, 3] },
{ name: 'value', config: { min: 0, max: 50, unit: 'somthing' }, type: FieldType.number, values: [1, 2, 3] },
{ name: 'str', config: {}, type: FieldType.string, values: ['a', 'b', 'c'] },
],
});
const table = grafanaDataFrameToArrowTable(frame);
expect(table.length).toEqual(frame.length);
// Now back to DataFrame
const before = JSON.stringify(toDataFrameDTO(frame), null, 2);
const after = JSON.stringify(toDataFrameDTO(arrowTableToDataFrame(table)), null, 2);
expect(after).toEqual(before);
});
});

150
packages/grafana-data/src/dataframe/ArrowDataFrame.ts Normal file
View File

@ -0,0 +1,150 @@
import { DataFrame, FieldType, Field, Vector, FieldConfig, Labels } from '../types';
import {
Table,
ArrowType,
Builder,
Vector as ArrowVector,
Float64,
DataType,
Utf8,
TimestampMillisecond,
Bool,
Column,
} from 'apache-arrow';
export interface ArrowDataFrame extends DataFrame {
table: Table;
}
export function base64StringToArrowTable(text: string): Table {
const b64 = atob(text);
const arr = Uint8Array.from(b64, c => {
return c.charCodeAt(0);
});
return Table.from(arr);
}
function valueOrUndefined(val?: string) {
return val ? val : undefined;
}
export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
const fields: Field[] = [];
for (let i = 0; i < table.numCols; i++) {
const col = table.getColumnAt(i);
if (col) {
const schema = table.schema.fields[i];
let type = FieldType.other;
const values: Vector<any> = col;
switch ((schema.typeId as unknown) as ArrowType) {
case ArrowType.Decimal:
case ArrowType.Int:
case ArrowType.FloatingPoint: {
type = FieldType.number;
break;
}
case ArrowType.Bool: {
type = FieldType.boolean;
break;
}
case ArrowType.Timestamp: {
type = FieldType.time;
break;
}
case ArrowType.Utf8: {
type = FieldType.string;
break;
}
default:
console.log('UNKNOWN Type:', schema);
}
const labelsJson = col.metadata.get('labels');
const configJson = col.metadata.get('config');
let config: FieldConfig = {};
let labels: Labels | undefined = undefined;
if (labelsJson) {
labels = JSON.parse(labelsJson);
}
if (configJson) {
config = JSON.parse(configJson);
}
fields.push({
name: col.name,
type,
config,
values,
labels,
});
}
}
const meta = table.schema.metadata;
const metaJson = valueOrUndefined(meta.get('meta'));
return {
fields,
length: table.length,
refId: valueOrUndefined(meta.get('refId')),
name: valueOrUndefined(meta.get('name')),
meta: metaJson ? JSON.parse(metaJson) : undefined,
table,
};
}
function toArrowVector(field: Field): ArrowVector {
// OR: Float64Vector.from([1, 2, 3]));
let type: DataType;
if (field.type === FieldType.number) {
type = new Float64();
} else if (field.type === FieldType.time) {
type = new TimestampMillisecond();
} else if (field.type === FieldType.boolean) {
type = new Bool();
} else if (field.type === FieldType.string) {
type = new Utf8();
} else {
type = new Utf8();
}
const builder = Builder.new({ type, nullValues: [null] });
field.values.toArray().forEach(builder.append.bind(builder));
return builder.finish().toVector();
}
export function grafanaDataFrameToArrowTable(data: DataFrame): Table {
const table = Table.new(
data.fields.map(field => {
const column = Column.new(field.name, toArrowVector(field));
if (field.labels) {
column.metadata.set('labels', JSON.stringify(field.labels));
}
if (field.config) {
column.metadata.set('config', JSON.stringify(field.config));
}
return column;
})
);
const metadata = table.schema.metadata;
if (data.name) {
metadata.set('name', data.name);
}
if (data.refId) {
metadata.set('refId', data.refId);
}
if (data.meta) {
metadata.set('meta', JSON.stringify(data.meta));
}
return table;
}
export function resultsToDataFrames(rsp: any): DataFrame[] {
const frames: DataFrame[] = [];
for (const res of Object.values(rsp.results)) {
for (const b of (res as any).dataframes) {
const t = base64StringToArrowTable(b as string);
frames.push(arrowTableToDataFrame(t));
}
}
return frames;
}

86
packages/grafana-data/src/dataframe/arrow/__snapshots__/ArrowDataFrame.test.ts.snap → packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap
View File

@ -9,33 +9,20 @@ Array [
"labels": undefined,
"name": "Time",
"type": "time",
"values": Int32Array [
882710016,
365389179,
1587742720,
365389180,
-2002191872,
365389181,
-1297159168,
365389182,
-592126464,
365389183,
112906240,
365389185,
817938944,
365389186,
1522971648,
365389187,
-2066962944,
365389188,
-1361930240,
365389189,
-656897536,
365389190,
48135168,
365389192,
753167872,
365389193,
"values": Array [
1569334575000,
1569334580000,
1569334585000,
1569334590000,
1569334595000,
1569334600000,
1569334605000,
1569334610000,
1569334615000,
1569334620000,
1569334625000,
1569334630000,
1569334635000,
],
},
Object {
@ -43,7 +30,7 @@ Array [
"labels": undefined,
"name": "",
"type": "number",
"values": Float64Array [
"values": Array [
3,
3,
3,
@ -71,33 +58,20 @@ Array [
"labels": undefined,
"name": "Time",
"type": "time",
"values": Int32Array [
882710016,
365389179,
1587742720,
365389180,
-2002191872,
365389181,
-1297159168,
365389182,
-592126464,
365389183,
112906240,
365389185,
817938944,
365389186,
1522971648,
365389187,
-2066962944,
365389188,
-1361930240,
365389189,
-656897536,
365389190,
48135168,
365389192,
753167872,
365389193,
"values": Array [
1569334575000,
1569334580000,
1569334585000,
1569334590000,
1569334595000,
1569334600000,
1569334605000,
1569334610000,
1569334615000,
1569334620000,
1569334625000,
1569334630000,
1569334635000,
],
},
Object {
@ -105,7 +79,7 @@ Array [
"labels": undefined,
"name": "GB-series",
"type": "number",
"values": Float64Array [
"values": Array [
0,
0,
0,

76
packages/grafana-data/src/dataframe/arrow/ArrowDataFrame.ts
View File

@ -1,76 +0,0 @@
import { DataFrame, FieldType, Field, Vector } from '../../types';
import { Table, ArrowType } from 'apache-arrow';
export interface ArrowDataFrame extends DataFrame {
table: Table;
}
export function base64StringToArrowTable(text: string): Table {
const b64 = atob(text);
const arr = Uint8Array.from(b64, c => {
return c.charCodeAt(0);
});
return Table.from(arr);
}
function valueOrUndefined(val?: string) {
return val ? val : undefined;
}
export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
const fields: Field[] = [];
for (let i = 0; i < table.numCols; i++) {
const col = table.getColumnAt(i);
if (col) {
const schema = table.schema.fields[i];
let type = FieldType.other;
const values: Vector<any> = col;
switch ((schema.typeId as unknown) as ArrowType) {
case ArrowType.Decimal:
case ArrowType.Int:
case ArrowType.FloatingPoint: {
type = FieldType.number;
break;
}
case ArrowType.Bool: {
type = FieldType.boolean;
break;
}
case ArrowType.Timestamp: {
type = FieldType.time;
break;
}
default:
console.log('UNKNOWN Type:', schema);
}
// console.log(' field>', schema.metadata);
fields.push({
name: col.name,
type,
config: {}, // TODO, pull from metadata
values,
});
}
}
const meta = table.schema.metadata;
return {
fields,
length: table.length,
refId: valueOrUndefined(meta.get('refId')),
name: valueOrUndefined(meta.get('name')),
table,
};
}
export function resultsToDataFrames(rsp: any): DataFrame[] {
const frames: DataFrame[] = [];
for (const res of Object.values(rsp.results)) {
for (const b of (res as any).dataframes) {
const t = base64StringToArrowTable(b as string);
frames.push(arrowTableToDataFrame(t));
}
}
return frames;
}

8
packages/grafana-data/src/dataframe/index.ts
View File

@ -4,10 +4,4 @@ export * from './CircularDataFrame';
export * from './MutableDataFrame';
export * from './processDataFrame';
export * from './dimensions';
// NOTE: We can not export arrow in the global scope because it will crash phantomjs
// In core, this is loaded async. In plugins you can import using:
//
// import { resultsToDataFrames } from '@grafana/data/dataframe/arrow/ArrowDataFrame'
//
// export * from './arrow/ArrowDataFrame';
export * from './ArrowDataFrame';

13
packages/grafana-data/src/dataframe/processDataFrame.ts
View File

@ -442,11 +442,22 @@ export function getDataFrameRow(data: DataFrame, row: number): any[] {
*/
export function toDataFrameDTO(data: DataFrame): DataFrameDTO {
const fields: FieldDTO[] = data.fields.map(f => {
let values = f.values.toArray();
if (!Array.isArray(values)) {
// Apache arrow will pack objects into typed arrays
// Float64Array, etc
// TODO: Float64Array could be used directly
values = [];
for (let i = 0; i < f.values.length; i++) {
values.push(f.values.get(i));
}
}
return {
name: f.name,
type: f.type,
config: f.config,
values: f.values.toArray(),
values,
labels: f.labels,
};
});

2
public/app/features/expressions/ExpressionDatasource.ts
View File

@ -69,7 +69,7 @@ export class ExpressionDatasourceApi extends DataSourceApi<ExpressionQuery> {
*/
async toDataQueryResponse(rsp: any): Promise<DataQueryResponse> {
const { resultsToDataFrames } = await import(
/* webpackChunkName: "apache-arrow-util" */ '@grafana/data/src/dataframe/arrow/ArrowDataFrame'
/* webpackChunkName: "apache-arrow-util" */ '@grafana/data/src/dataframe/ArrowDataFrame'
);
return { data: resultsToDataFrames(rsp) };
}