IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

DataFrame: add utilities to @grafana/data that support apache arrow (#20813)

Browse Source
This commit is contained in:
Ryan McKinley
2019-12-02 12:39:50 -08:00
committed by GitHub
parent 74ac73716f
commit e216044c75
7 changed files with 20 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/grafana-data/package.json
View File

@@ -29,6 +29,7 @@
"@types/pretty-format": "20.0.1",
"@types/react": "16.8.16",
"@types/sinon": "^7.0.11",
"apache-arrow": "0.15.1",
"lodash": "4.17.15",
"pretty-format": "24.9.0",
"rollup": "1.6.0",

35
packages/grafana-data/src/dataframe/arrow/ArrowDataFrame.test.ts Normal file
View File

@@ -0,0 +1,35 @@
import { resultsToDataFrames } from './ArrowDataFrame';
import { toDataFrameDTO } from '../processDataFrame';
/* tslint:disable */
const resp = {
results: {
'': {
refId: '',
dataframes: [
'QVJST1cxAACsAQAAEAAAAAAACgAOAAwACwAEAAoAAAAUAAAAAAAAAQMACgAMAAAACAAEAAoAAAAIAAAAUAAAAAIAAAAoAAAABAAAAOD+//8IAAAADAAAAAIAAABHQwAABQAAAHJlZklkAAAAAP///wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAACAAAAlAAAAAQAAACG////FAAAAGAAAABgAAAAAAADAWAAAAACAAAALAAAAAQAAABQ////CAAAABAAAAAGAAAAbnVtYmVyAAAEAAAAdHlwZQAAAAB0////CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAAAAABm////AAACAAAAAAAAABIAGAAUABMAEgAMAAAACAAEABIAAAAUAAAAbAAAAHQAAAAAAAoBdAAAAAIAAAA0AAAABAAAANz///8IAAAAEAAAAAQAAAB0aW1lAAAAAAQAAAB0eXBlAAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAFRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAVGltZQAAAAC8AAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAA0AAAAAAAAAAUAAAAAAAAAwMACgAYAAwACAAEAAoAAAAUAAAAWAAAAA0AAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABoAAAAAAAAAGgAAAAAAAAAAAAAAAAAAABoAAAAAAAAAGgAAAAAAAAAAAAAAAIAAAANAAAAAAAAAAAAAAAAAAAADQAAAAAAAAAAAAAAAAAAAAAAAAAAFp00e2XHFQAIo158ZccVAPqoiH1lxxUA7K6yfmXHFQDetNx/ZccVANC6BoFlxxUAwsAwgmXHFQC0xlqDZccVAKbMhIRlxxUAmNKuhWXHFQCK2NiGZccVAHzeAohlxxUAbuQsiWXHFQAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRAAAAAAAAAFEAAAAAAAAAUQAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRAAAAAAAAAFEAAAAAAAAAUQAAAAAAAAAhAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADgAAAAAAAMAAQAAALgBAAAAAAAAwAAAAAAAAADQAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAABQAAAAAgAAACgAAAAEAAAA4P7//wgAAAAMAAAAAgAAAEdDAAAFAAAAcmVmSWQAAAAA////CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAIAAACUAAAABAAAAIb///8UAAAAYAAAAGAAAAAAAAMBYAAAAAIAAAAsAAAABAAAAFD///8IAAAAEAAAAAYAAABudW1iZXIAAAQAAAB0eXBlAAAAAHT///8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAAAAAAGb///8AAAIAAAAAAAAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABsAAAAdAAAAAAACgF0AAAAAgAAADQAAAAEAAAA3P///wgAAAAQAAAABAAAAHRpbWUAAAAABAAAAHR5cGUAAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAVGltZQAAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAADAAQAAABUaW1lAAAAANgBAABBUlJPVzE=',
'QVJST1cxAAC8AQAAEAAAAAAACgAOAAwACwAEAAoAAAAUAAAAAAAAAQMACgAMAAAACAAEAAoAAAAIAAAAUAAAAAIAAAAoAAAABAAAAND+//8IAAAADAAAAAIAAABHQgAABQAAAHJlZklkAAAA8P7//wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAACAAAApAAAAAQAAAB2////FAAAAGgAAABoAAAAAAADAWgAAAACAAAALAAAAAQAAABA////CAAAABAAAAAGAAAAbnVtYmVyAAAEAAAAdHlwZQAAAABk////CAAAABQAAAAJAAAAR0Itc2VyaWVzAAAABAAAAG5hbWUAAAAAAAAAAF7///8AAAIACQAAAEdCLXNlcmllcwASABgAFAATABIADAAAAAgABAASAAAAFAAAAGwAAAB0AAAAAAAKAXQAAAACAAAANAAAAAQAAADc////CAAAABAAAAAEAAAAdGltZQAAAAAEAAAAdHlwZQAAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAAvAAAABQAAAAAAAAADAAWABQAEwAMAAQADAAAANAAAAAAAAAAFAAAAAAAAAMDAAoAGAAMAAgABAAKAAAAFAAAAFgAAAANAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAaAAAAAAAAABoAAAAAAAAAAAAAAAAAAAAaAAAAAAAAABoAAAAAAAAAAAAAAACAAAADQAAAAAAAAAAAAAAAAAAAA0AAAAAAAAAAAAAAAAAAAAAAAAAABadNHtlxxUACKNefGXHFQD6qIh9ZccVAOyusn5lxxUA3rTcf2XHFQDQugaBZccVAMLAMIJlxxUAtMZag2XHFQCmzISEZccVAJjSroVlxxUAitjYhmXHFQB83gKIZccVAG7kLIllxxUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAABAAAAAMABQAEgAMAAgABAAMAAAAEAAAACwAAAA4AAAAAAADAAEAAADIAQAAAAAAAMAAAAAAAAAA0AAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAUAAAAAIAAAAoAAAABAAAAND+//8IAAAADAAAAAIAAABHQgAABQAAAHJlZklkAAAA8P7//wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAACAAAApAAAAAQAAAB2////FAAAAGgAAABoAAAAAAADAWgAAAACAAAALAAAAAQAAABA////CAAAABAAAAAGAAAAbnVtYmVyAAAEAAAAdHlwZQAAAABk////CAAAABQAAAAJAAAAR0Itc2VyaWVzAAAABAAAAG5hbWUAAAAAAAAAAF7///8AAAIACQAAAEdCLXNlcmllcwASABgAFAATABIADAAAAAgABAASAAAAFAAAAGwAAAB0AAAAAAAKAXQAAAACAAAANAAAAAQAAADc////CAAAABAAAAAEAAAAdGltZQAAAAAEAAAAdHlwZQAAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAA6AEAAEFSUk9XMQ==',
],
series: [] as any[],
tables: null as any,
frames: null as any,
},
},
};
/* tslint:enable */
describe('GEL Utils', () => {
test('should parse output with dataframe', () => {
const frames = resultsToDataFrames(resp);
for (const frame of frames) {
console.log('Frame', frame.refId);
for (const field of frame.fields) {
console.log(' > ', field.name, field.labels);
console.log(' (values)= ', field.values.toArray());
}
}
const norm = frames.map(f => toDataFrameDTO(f));
expect(norm).toMatchSnapshot();
});
});

76
packages/grafana-data/src/dataframe/arrow/ArrowDataFrame.ts Normal file
View File

@@ -0,0 +1,76 @@
import { DataFrame, FieldType, Field, Vector } from '../../types';
import { Table, ArrowType } from 'apache-arrow';
export interface ArrowDataFrame extends DataFrame {
table: Table;
}
export function base64StringToArrowTable(text: string): Table {
const b64 = atob(text);
const arr = Uint8Array.from(b64, c => {
return c.charCodeAt(0);
});
return Table.from(arr);
}
function valueOrUndefined(val?: string) {
return val ? val : undefined;
}
export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
const fields: Field[] = [];
for (let i = 0; i < table.numCols; i++) {
const col = table.getColumnAt(i);
if (col) {
const schema = table.schema.fields[i];
let type = FieldType.other;
const values: Vector<any> = col;
switch ((schema.typeId as unknown) as ArrowType) {
case ArrowType.Decimal:
case ArrowType.Int:
case ArrowType.FloatingPoint: {
type = FieldType.number;
break;
}
case ArrowType.Bool: {
type = FieldType.boolean;
break;
}
case ArrowType.Timestamp: {
type = FieldType.time;
break;
}
default:
console.log('UNKNOWN Type:', schema);
}
// console.log(' field>', schema.metadata);
fields.push({
name: col.name,
type,
config: {}, // TODO, pull from metadata
values,
});
}
}
const meta = table.schema.metadata;
return {
fields,
length: table.length,
refId: valueOrUndefined(meta.get('refId')),
name: valueOrUndefined(meta.get('name')),
table,
};
}
export function resultsToDataFrames(rsp: any): DataFrame[] {
const frames: DataFrame[] = [];
for (const res of Object.values(rsp.results)) {
for (const b of (res as any).dataframes) {
const t = base64StringToArrowTable(b as string);
frames.push(arrowTableToDataFrame(t));
}
}
return frames;
}

130
packages/grafana-data/src/dataframe/arrow/__snapshots__/ArrowDataFrame.test.ts.snap Normal file
View File

@@ -0,0 +1,130 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`GEL Utils should parse output with dataframe 1`] = `
Array [
Object {
"fields": Array [
Object {
"config": Object {},
"labels": undefined,
"name": "Time",
"type": "time",
"values": Int32Array [
882710016,
365389179,
1587742720,
365389180,
-2002191872,
365389181,
-1297159168,
365389182,
-592126464,
365389183,
112906240,
365389185,
817938944,
365389186,
1522971648,
365389187,
-2066962944,
365389188,
-1361930240,
365389189,
-656897536,
365389190,
48135168,
365389192,
753167872,
365389193,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "",
"type": "number",
"values": Float64Array [
3,
3,
3,
5,
5,
5,
3,
3,
3,
5,
5,
5,
3,
],
},
],
"meta": undefined,
"name": undefined,
"refId": "GC",
},
Object {
"fields": Array [
Object {
"config": Object {},
"labels": undefined,
"name": "Time",
"type": "time",
"values": Int32Array [
882710016,
365389179,
1587742720,
365389180,
-2002191872,
365389181,
-1297159168,
365389182,
-592126464,
365389183,
112906240,
365389185,
817938944,
365389186,
1522971648,
365389187,
-2066962944,
365389188,
-1361930240,
365389189,
-656897536,
365389190,
48135168,
365389192,
753167872,
365389193,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "GB-series",
"type": "number",
"values": Float64Array [
0,
0,
0,
2,
2,
2,
0,
0,
0,
2,
2,
2,
0,
],
},
],
"meta": undefined,
"name": undefined,
"refId": "GB",
},
]
`;