diff --git a/packages/grafana-data/rollup.config.ts b/packages/grafana-data/rollup.config.ts index a2d6da109d9..9a862a3eb25 100644 --- a/packages/grafana-data/rollup.config.ts +++ b/packages/grafana-data/rollup.config.ts @@ -20,7 +20,7 @@ const buildCjsPackage = ({ env }) => { globals: {}, }, ], - external: ['lodash'], // Use Lodash from grafana + external: ['lodash', 'apache-arrow'], // Use Lodash & arrow from grafana plugins: [ commonjs({ include: /node_modules/, diff --git a/packages/grafana-data/src/dataframe/arrow/ArrowDataFrame.test.ts b/packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts similarity index 79% rename from packages/grafana-data/src/dataframe/arrow/ArrowDataFrame.test.ts rename to packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts index 015a927bf81..9137198c95c 100644 --- a/packages/grafana-data/src/dataframe/arrow/ArrowDataFrame.test.ts +++ b/packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts @@ -1,5 +1,6 @@ -import { resultsToDataFrames } from './ArrowDataFrame'; -import { toDataFrameDTO } from '../processDataFrame'; +import { resultsToDataFrames, grafanaDataFrameToArrowTable, arrowTableToDataFrame } from './ArrowDataFrame'; +import { toDataFrameDTO, toDataFrame } from './processDataFrame'; +import { FieldType } from '../types'; /* tslint:disable */ const resp = { @@ -33,3 +34,29 @@ describe('GEL Utils', () => { expect(norm).toMatchSnapshot(); }); }); + +describe('Read/Write arrow Table to DataFrame', () => { + test('should parse output with dataframe', () => { + const frame = toDataFrame({ + name: 'Hello', + refId: 'XYZ', + meta: { + aaa: 'xyz', + anything: 'xxx', + }, + fields: [ + { name: 'time', config: {}, type: FieldType.time, values: [1, 2, 3] }, + { name: 'value', config: { min: 0, max: 50, unit: 'somthing' }, type: FieldType.number, values: [1, 2, 3] }, + { name: 'str', config: {}, type: FieldType.string, values: ['a', 'b', 'c'] }, + ], + }); + + const table = grafanaDataFrameToArrowTable(frame); + expect(table.length).toEqual(frame.length); + + // Now back to DataFrame + const before = JSON.stringify(toDataFrameDTO(frame), null, 2); + const after = JSON.stringify(toDataFrameDTO(arrowTableToDataFrame(table)), null, 2); + expect(after).toEqual(before); + }); +}); diff --git a/packages/grafana-data/src/dataframe/ArrowDataFrame.ts b/packages/grafana-data/src/dataframe/ArrowDataFrame.ts new file mode 100644 index 00000000000..2963f3d9136 --- /dev/null +++ b/packages/grafana-data/src/dataframe/ArrowDataFrame.ts @@ -0,0 +1,150 @@ +import { DataFrame, FieldType, Field, Vector, FieldConfig, Labels } from '../types'; +import { + Table, + ArrowType, + Builder, + Vector as ArrowVector, + Float64, + DataType, + Utf8, + TimestampMillisecond, + Bool, + Column, +} from 'apache-arrow'; + +export interface ArrowDataFrame extends DataFrame { + table: Table; +} + +export function base64StringToArrowTable(text: string): Table { + const b64 = atob(text); + const arr = Uint8Array.from(b64, c => { + return c.charCodeAt(0); + }); + return Table.from(arr); +} + +function valueOrUndefined(val?: string) { + return val ? val : undefined; +} + +export function arrowTableToDataFrame(table: Table): ArrowDataFrame { + const fields: Field[] = []; + + for (let i = 0; i < table.numCols; i++) { + const col = table.getColumnAt(i); + if (col) { + const schema = table.schema.fields[i]; + let type = FieldType.other; + const values: Vector = col; + switch ((schema.typeId as unknown) as ArrowType) { + case ArrowType.Decimal: + case ArrowType.Int: + case ArrowType.FloatingPoint: { + type = FieldType.number; + break; + } + case ArrowType.Bool: { + type = FieldType.boolean; + break; + } + case ArrowType.Timestamp: { + type = FieldType.time; + break; + } + case ArrowType.Utf8: { + type = FieldType.string; + break; + } + default: + console.log('UNKNOWN Type:', schema); + } + const labelsJson = col.metadata.get('labels'); + const configJson = col.metadata.get('config'); + + let config: FieldConfig = {}; + let labels: Labels | undefined = undefined; + if (labelsJson) { + labels = JSON.parse(labelsJson); + } + if (configJson) { + config = JSON.parse(configJson); + } + + fields.push({ + name: col.name, + type, + config, + values, + labels, + }); + } + } + const meta = table.schema.metadata; + const metaJson = valueOrUndefined(meta.get('meta')); + return { + fields, + length: table.length, + refId: valueOrUndefined(meta.get('refId')), + name: valueOrUndefined(meta.get('name')), + meta: metaJson ? JSON.parse(metaJson) : undefined, + table, + }; +} + +function toArrowVector(field: Field): ArrowVector { + // OR: Float64Vector.from([1, 2, 3])); + + let type: DataType; + if (field.type === FieldType.number) { + type = new Float64(); + } else if (field.type === FieldType.time) { + type = new TimestampMillisecond(); + } else if (field.type === FieldType.boolean) { + type = new Bool(); + } else if (field.type === FieldType.string) { + type = new Utf8(); + } else { + type = new Utf8(); + } + const builder = Builder.new({ type, nullValues: [null] }); + field.values.toArray().forEach(builder.append.bind(builder)); + return builder.finish().toVector(); +} + +export function grafanaDataFrameToArrowTable(data: DataFrame): Table { + const table = Table.new( + data.fields.map(field => { + const column = Column.new(field.name, toArrowVector(field)); + if (field.labels) { + column.metadata.set('labels', JSON.stringify(field.labels)); + } + if (field.config) { + column.metadata.set('config', JSON.stringify(field.config)); + } + return column; + }) + ); + const metadata = table.schema.metadata; + if (data.name) { + metadata.set('name', data.name); + } + if (data.refId) { + metadata.set('refId', data.refId); + } + if (data.meta) { + metadata.set('meta', JSON.stringify(data.meta)); + } + return table; +} + +export function resultsToDataFrames(rsp: any): DataFrame[] { + const frames: DataFrame[] = []; + for (const res of Object.values(rsp.results)) { + for (const b of (res as any).dataframes) { + const t = base64StringToArrowTable(b as string); + frames.push(arrowTableToDataFrame(t)); + } + } + return frames; +} diff --git a/packages/grafana-data/src/dataframe/arrow/__snapshots__/ArrowDataFrame.test.ts.snap b/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap similarity index 50% rename from packages/grafana-data/src/dataframe/arrow/__snapshots__/ArrowDataFrame.test.ts.snap rename to packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap index c0eb2137761..ae779698246 100644 --- a/packages/grafana-data/src/dataframe/arrow/__snapshots__/ArrowDataFrame.test.ts.snap +++ b/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap @@ -9,33 +9,20 @@ Array [ "labels": undefined, "name": "Time", "type": "time", - "values": Int32Array [ - 882710016, - 365389179, - 1587742720, - 365389180, - -2002191872, - 365389181, - -1297159168, - 365389182, - -592126464, - 365389183, - 112906240, - 365389185, - 817938944, - 365389186, - 1522971648, - 365389187, - -2066962944, - 365389188, - -1361930240, - 365389189, - -656897536, - 365389190, - 48135168, - 365389192, - 753167872, - 365389193, + "values": Array [ + 1569334575000, + 1569334580000, + 1569334585000, + 1569334590000, + 1569334595000, + 1569334600000, + 1569334605000, + 1569334610000, + 1569334615000, + 1569334620000, + 1569334625000, + 1569334630000, + 1569334635000, ], }, Object { @@ -43,7 +30,7 @@ Array [ "labels": undefined, "name": "", "type": "number", - "values": Float64Array [ + "values": Array [ 3, 3, 3, @@ -71,33 +58,20 @@ Array [ "labels": undefined, "name": "Time", "type": "time", - "values": Int32Array [ - 882710016, - 365389179, - 1587742720, - 365389180, - -2002191872, - 365389181, - -1297159168, - 365389182, - -592126464, - 365389183, - 112906240, - 365389185, - 817938944, - 365389186, - 1522971648, - 365389187, - -2066962944, - 365389188, - -1361930240, - 365389189, - -656897536, - 365389190, - 48135168, - 365389192, - 753167872, - 365389193, + "values": Array [ + 1569334575000, + 1569334580000, + 1569334585000, + 1569334590000, + 1569334595000, + 1569334600000, + 1569334605000, + 1569334610000, + 1569334615000, + 1569334620000, + 1569334625000, + 1569334630000, + 1569334635000, ], }, Object { @@ -105,7 +79,7 @@ Array [ "labels": undefined, "name": "GB-series", "type": "number", - "values": Float64Array [ + "values": Array [ 0, 0, 0, diff --git a/packages/grafana-data/src/dataframe/arrow/ArrowDataFrame.ts b/packages/grafana-data/src/dataframe/arrow/ArrowDataFrame.ts deleted file mode 100644 index 725e7133c03..00000000000 --- a/packages/grafana-data/src/dataframe/arrow/ArrowDataFrame.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { DataFrame, FieldType, Field, Vector } from '../../types'; -import { Table, ArrowType } from 'apache-arrow'; - -export interface ArrowDataFrame extends DataFrame { - table: Table; -} - -export function base64StringToArrowTable(text: string): Table { - const b64 = atob(text); - const arr = Uint8Array.from(b64, c => { - return c.charCodeAt(0); - }); - return Table.from(arr); -} - -function valueOrUndefined(val?: string) { - return val ? val : undefined; -} - -export function arrowTableToDataFrame(table: Table): ArrowDataFrame { - const fields: Field[] = []; - - for (let i = 0; i < table.numCols; i++) { - const col = table.getColumnAt(i); - if (col) { - const schema = table.schema.fields[i]; - let type = FieldType.other; - const values: Vector = col; - switch ((schema.typeId as unknown) as ArrowType) { - case ArrowType.Decimal: - case ArrowType.Int: - case ArrowType.FloatingPoint: { - type = FieldType.number; - break; - } - case ArrowType.Bool: { - type = FieldType.boolean; - break; - } - case ArrowType.Timestamp: { - type = FieldType.time; - break; - } - default: - console.log('UNKNOWN Type:', schema); - } - // console.log(' field>', schema.metadata); - - fields.push({ - name: col.name, - type, - config: {}, // TODO, pull from metadata - values, - }); - } - } - const meta = table.schema.metadata; - return { - fields, - length: table.length, - refId: valueOrUndefined(meta.get('refId')), - name: valueOrUndefined(meta.get('name')), - table, - }; -} - -export function resultsToDataFrames(rsp: any): DataFrame[] { - const frames: DataFrame[] = []; - for (const res of Object.values(rsp.results)) { - for (const b of (res as any).dataframes) { - const t = base64StringToArrowTable(b as string); - frames.push(arrowTableToDataFrame(t)); - } - } - return frames; -} diff --git a/packages/grafana-data/src/dataframe/index.ts b/packages/grafana-data/src/dataframe/index.ts index 9b3d2a69820..1626c3294fd 100644 --- a/packages/grafana-data/src/dataframe/index.ts +++ b/packages/grafana-data/src/dataframe/index.ts @@ -4,10 +4,4 @@ export * from './CircularDataFrame'; export * from './MutableDataFrame'; export * from './processDataFrame'; export * from './dimensions'; - -// NOTE: We can not export arrow in the global scope because it will crash phantomjs -// In core, this is loaded async. In plugins you can import using: -// -// import { resultsToDataFrames } from '@grafana/data/dataframe/arrow/ArrowDataFrame' -// -// export * from './arrow/ArrowDataFrame'; +export * from './ArrowDataFrame'; diff --git a/packages/grafana-data/src/dataframe/processDataFrame.ts b/packages/grafana-data/src/dataframe/processDataFrame.ts index 808cf9eed2f..87e746e770d 100644 --- a/packages/grafana-data/src/dataframe/processDataFrame.ts +++ b/packages/grafana-data/src/dataframe/processDataFrame.ts @@ -442,11 +442,22 @@ export function getDataFrameRow(data: DataFrame, row: number): any[] { */ export function toDataFrameDTO(data: DataFrame): DataFrameDTO { const fields: FieldDTO[] = data.fields.map(f => { + let values = f.values.toArray(); + if (!Array.isArray(values)) { + // Apache arrow will pack objects into typed arrays + // Float64Array, etc + // TODO: Float64Array could be used directly + values = []; + for (let i = 0; i < f.values.length; i++) { + values.push(f.values.get(i)); + } + } + return { name: f.name, type: f.type, config: f.config, - values: f.values.toArray(), + values, labels: f.labels, }; }); diff --git a/public/app/features/expressions/ExpressionDatasource.ts b/public/app/features/expressions/ExpressionDatasource.ts index ceda8cdfe6a..6044451fe56 100644 --- a/public/app/features/expressions/ExpressionDatasource.ts +++ b/public/app/features/expressions/ExpressionDatasource.ts @@ -69,7 +69,7 @@ export class ExpressionDatasourceApi extends DataSourceApi { */ async toDataQueryResponse(rsp: any): Promise { const { resultsToDataFrames } = await import( - /* webpackChunkName: "apache-arrow-util" */ '@grafana/data/src/dataframe/arrow/ArrowDataFrame' + /* webpackChunkName: "apache-arrow-util" */ '@grafana/data/src/dataframe/ArrowDataFrame' ); return { data: resultsToDataFrames(rsp) }; }