IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2026-09-05 04:40:13 -05:00
Code Issues Packages Projects Releases Wiki Activity

DataFrame: add utilities to @grafana/data that support apache arrow (#20813)

Browse Source
This commit is contained in:
Ryan McKinley
2019-12-02 12:39:50 -08:00
committed by GitHub
parent 74ac73716f
commit e216044c75
7 changed files with 20 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
public/app/features/expressions/ExpressionDatasource.ts
+4 -2
View File
@@ -68,8 +68,10 @@ export class ExpressionDatasourceApi extends DataSourceApi<ExpressionQuery> {
* This makes the arrow libary loading async.
*/
async toDataQueryResponse(rsp: any): Promise<DataQueryResponse> {
const { gelResponseToDataFrames } = await import(/* webpackChunkName: "apache-arrow-util" */ './util');
return { data: gelResponseToDataFrames(rsp) };
const { resultsToDataFrames } = await import(
/* webpackChunkName: "apache-arrow-util" */ '@grafana/data/src/dataframe/arrow/ArrowDataFrame'
);
return { data: resultsToDataFrames(rsp) };
}
testDatasource() {
public/app/features/expressions/__snapshots__/util.test.ts.snap
-130
View File
@@ -1,130 +0,0 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`GEL Utils should parse output with dataframe 1`] = `
Array [
Object {
"fields": Array [
Object {
"config": Object {},
"labels": undefined,
"name": "Time",
"type": "time",
"values": Int32Array [
882710016,
365389179,
1587742720,
365389180,
-2002191872,
365389181,
-1297159168,
365389182,
-592126464,
365389183,
112906240,
365389185,
817938944,
365389186,
1522971648,
365389187,
-2066962944,
365389188,
-1361930240,
365389189,
-656897536,
365389190,
48135168,
365389192,
753167872,
365389193,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "",
"type": "number",
"values": Float64Array [
3,
3,
3,
5,
5,
5,
3,
3,
3,
5,
5,
5,
3,
],
},
],
"meta": undefined,
"name": undefined,
"refId": "GC",
},
Object {
"fields": Array [
Object {
"config": Object {},
"labels": undefined,
"name": "Time",
"type": "time",
"values": Int32Array [
882710016,
365389179,
1587742720,
365389180,
-2002191872,
365389181,
-1297159168,
365389182,
-592126464,
365389183,
112906240,
365389185,
817938944,
365389186,
1522971648,
365389187,
-2066962944,
365389188,
-1361930240,
365389189,
-656897536,
365389190,
48135168,
365389192,
753167872,
365389193,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "GB-series",
"type": "number",
"values": Float64Array [
0,
0,
0,
2,
2,
2,
0,
0,
0,
2,
2,
2,
0,
],
},
],
"meta": undefined,
"name": undefined,
"refId": "GB",
},
]
`;
public/app/features/expressions/util.test.ts
-49
View File
@@ -1,49 +0,0 @@
import { gelResponseToDataFrames } from './util';
import { toDataFrameDTO } from '@grafana/data';
/* tslint:disable */
const resp = {
results: {
'': {
refId: '',
dataframes: [
'QVJST1cxAACsAQAAEAAAAAAACgAOAAwACwAEAAoAAAAUAAAAAAAAAQMACgAMAAAACAAEAAoAAAAIAAAAUAAAAAIAAAAoAAAABAAAAOD+//8IAAAADAAAAAIAAABHQwAABQAAAHJlZklkAAAAAP///wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAACAAAAlAAAAAQAAACG////FAAAAGAAAABgAAAAAAADAWAAAAACAAAALAAAAAQAAABQ////CAAAABAAAAAGAAAAbnVtYmVyAAAEAAAAdHlwZQAAAAB0////CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAAAAABm////AAACAAAAAAAAABIAGAAUABMAEgAMAAAACAAEABIAAAAUAAAAbAAAAHQAAAAAAAoBdAAAAAIAAAA0AAAABAAAANz///8IAAAAEAAAAAQAAAB0aW1lAAAAAAQAAAB0eXBlAAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAFRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAVGltZQAAAAC8AAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAA0AAAAAAAAAAUAAAAAAAAAwMACgAYAAwACAAEAAoAAAAUAAAAWAAAAA0AAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABoAAAAAAAAAGgAAAAAAAAAAAAAAAAAAABoAAAAAAAAAGgAAAAAAAAAAAAAAAIAAAANAAAAAAAAAAAAAAAAAAAADQAAAAAAAAAAAAAAAAAAAAAAAAAAFp00e2XHFQAIo158ZccVAPqoiH1lxxUA7K6yfmXHFQDetNx/ZccVANC6BoFlxxUAwsAwgmXHFQC0xlqDZccVAKbMhIRlxxUAmNKuhWXHFQCK2NiGZccVAHzeAohlxxUAbuQsiWXHFQAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRAAAAAAAAAFEAAAAAAAAAUQAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRAAAAAAAAAFEAAAAAAAAAUQAAAAAAAAAhAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADgAAAAAAAMAAQAAALgBAAAAAAAAwAAAAAAAAADQAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAABQAAAAAgAAACgAAAAEAAAA4P7//wgAAAAMAAAAAgAAAEdDAAAFAAAAcmVmSWQAAAAA////CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAIAAACUAAAABAAAAIb///8UAAAAYAAAAGAAAAAAAAMBYAAAAAIAAAAsAAAABAAAAFD///8IAAAAEAAAAAYAAABudW1iZXIAAAQAAAB0eXBlAAAAAHT///8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAAAAAAGb///8AAAIAAAAAAAAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABsAAAAdAAAAAAACgF0AAAAAgAAADQAAAAEAAAA3P///wgAAAAQAAAABAAAAHRpbWUAAAAABAAAAHR5cGUAAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAVGltZQAAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAADAAQAAABUaW1lAAAAANgBAABBUlJPVzE=',
'QVJST1cxAAC8AQAAEAAAAAAACgAOAAwACwAEAAoAAAAUAAAAAAAAAQMACgAMAAAACAAEAAoAAAAIAAAAUAAAAAIAAAAoAAAABAAAAND+//8IAAAADAAAAAIAAABHQgAABQAAAHJlZklkAAAA8P7//wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAACAAAApAAAAAQAAAB2////FAAAAGgAAABoAAAAAAADAWgAAAACAAAALAAAAAQAAABA////CAAAABAAAAAGAAAAbnVtYmVyAAAEAAAAdHlwZQAAAABk////CAAAABQAAAAJAAAAR0Itc2VyaWVzAAAABAAAAG5hbWUAAAAAAAAAAF7///8AAAIACQAAAEdCLXNlcmllcwASABgAFAATABIADAAAAAgABAASAAAAFAAAAGwAAAB0AAAAAAAKAXQAAAACAAAANAAAAAQAAADc////CAAAABAAAAAEAAAAdGltZQAAAAAEAAAAdHlwZQAAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAAvAAAABQAAAAAAAAADAAWABQAEwAMAAQADAAAANAAAAAAAAAAFAAAAAAAAAMDAAoAGAAMAAgABAAKAAAAFAAAAFgAAAANAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAaAAAAAAAAABoAAAAAAAAAAAAAAAAAAAAaAAAAAAAAABoAAAAAAAAAAAAAAACAAAADQAAAAAAAAAAAAAAAAAAAA0AAAAAAAAAAAAAAAAAAAAAAAAAABadNHtlxxUACKNefGXHFQD6qIh9ZccVAOyusn5lxxUA3rTcf2XHFQDQugaBZccVAMLAMIJlxxUAtMZag2XHFQCmzISEZccVAJjSroVlxxUAitjYhmXHFQB83gKIZccVAG7kLIllxxUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAABAAAAAMABQAEgAMAAgABAAMAAAAEAAAACwAAAA4AAAAAAADAAEAAADIAQAAAAAAAMAAAAAAAAAA0AAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAUAAAAAIAAAAoAAAABAAAAND+//8IAAAADAAAAAIAAABHQgAABQAAAHJlZklkAAAA8P7//wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAACAAAApAAAAAQAAAB2////FAAAAGgAAABoAAAAAAADAWgAAAACAAAALAAAAAQAAABA////CAAAABAAAAAGAAAAbnVtYmVyAAAEAAAAdHlwZQAAAABk////CAAAABQAAAAJAAAAR0Itc2VyaWVzAAAABAAAAG5hbWUAAAAAAAAAAF7///8AAAIACQAAAEdCLXNlcmllcwASABgAFAATABIADAAAAAgABAASAAAAFAAAAGwAAAB0AAAAAAAKAXQAAAACAAAANAAAAAQAAADc////CAAAABAAAAAEAAAAdGltZQAAAAAEAAAAdHlwZQAAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAA6AEAAEFSUk9XMQ==',
],
series: [] as any[],
tables: null as any,
frames: null as any,
},
},
};
/* tslint:enable */
describe('GEL Utils', () => {
// test('should parse sample GEL output', () => {
// const frames = gelResponseToDataFrames(resp);
// const frame = frames[0];
// expect(frame.name).toEqual('BBB');
// expect(frame.fields.length).toEqual(2);
// expect(frame.length).toEqual(resp.Frames[0].fields[0].values.length);
// const timeField = frame.fields[0];
// expect(timeField.name).toEqual('Time');
// // The whole response
// expect(frames).toMatchSnapshot();
// });
test('should parse output with dataframe', () => {
const frames = gelResponseToDataFrames(resp);
for (const frame of frames) {
console.log('Frame', frame.refId);
for (const field of frame.fields) {
console.log(' > ', field.name, field.labels);
console.log(' (values)= ', field.values.toArray());
}
}
const norm = frames.map(f => toDataFrameDTO(f));
expect(norm).toMatchSnapshot();
});
});
public/app/features/expressions/util.ts
-71
View File
@@ -1,71 +0,0 @@
import { DataFrame, FieldType, Field, Vector } from '@grafana/data';
import { Table, ArrowType } from 'apache-arrow';
export function base64StringToArrowTable(text: string): Table {
const b64 = atob(text);
const arr = Uint8Array.from(b64, c => {
return c.charCodeAt(0);
});
return Table.from(arr);
}
function valueOrUndefined(val?: string) {
return val ? val : undefined;
}
export function arrowTableToDataFrame(table: Table): DataFrame {
const fields: Field[] = [];
for (let i = 0; i < table.numCols; i++) {
const col = table.getColumnAt(i);
if (col) {
const schema = table.schema.fields[i];
let type = FieldType.other;
const values: Vector<any> = col;
switch ((schema.typeId as unknown) as ArrowType) {
case ArrowType.Decimal:
case ArrowType.Int:
case ArrowType.FloatingPoint: {
type = FieldType.number;
break;
}
case ArrowType.Bool: {
type = FieldType.boolean;
break;
}
case ArrowType.Timestamp: {
type = FieldType.time;
break;
}
default:
console.log('UNKNOWN Type:', schema);
}
// console.log(' field>', schema.metadata);
fields.push({
name: col.name,
type,
config: {}, // TODO, pull from metadata
values,
});
}
}
const meta = table.schema.metadata;
return {
fields,
length: table.length,
refId: valueOrUndefined(meta.get('refId')),
name: valueOrUndefined(meta.get('name')),
};
}
export function gelResponseToDataFrames(rsp: any): DataFrame[] {
const frames: DataFrame[] = [];
for (const res of Object.values(rsp.results)) {
for (const b of (res as any).dataframes) {
const t = base64StringToArrowTable(b as string);
frames.push(arrowTableToDataFrame(t));
}
}
return frames;
}