diff --git a/packages/grafana-data/package.json b/packages/grafana-data/package.json index 2596589a47c..7fde3e400bc 100644 --- a/packages/grafana-data/package.json +++ b/packages/grafana-data/package.json @@ -24,7 +24,6 @@ "dependencies": { "@braintree/sanitize-url": "4.0.0", "@types/d3-interpolate": "^1.3.1", - "apache-arrow": "0.16.0", "eventemitter3": "4.0.7", "lodash": "4.17.21", "marked": "2.0.1", diff --git a/packages/grafana-data/rollup.config.ts b/packages/grafana-data/rollup.config.ts index 10818c69bc6..830f86c88b4 100644 --- a/packages/grafana-data/rollup.config.ts +++ b/packages/grafana-data/rollup.config.ts @@ -21,7 +21,7 @@ const buildCjsPackage = ({ env }) => { globals: {}, }, ], - external: ['lodash', 'rxjs', 'apache-arrow'], // Use Lodash, rxjs & arrow from grafana + external: ['lodash', 'rxjs'], // Use Lodash, rxjs from grafana plugins: [ json({ include: ['../../node_modules/moment-timezone/data/packed/latest.json'], diff --git a/packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts b/packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts deleted file mode 100644 index 96efaccfce5..00000000000 --- a/packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts +++ /dev/null @@ -1,85 +0,0 @@ -import fs from 'fs'; -import path from 'path'; - -import { grafanaDataFrameToArrowTable, arrowTableToDataFrame } from './ArrowDataFrame'; -import { toDataFrameDTO, toDataFrame } from './processDataFrame'; -import { FieldType, DataFrame } from '../types'; -import { Table } from 'apache-arrow'; -import { ArrayVector } from '../vector'; - -describe('Read/Write arrow Table to DataFrame', () => { - test('should parse output with dataframe', () => { - const frame = toDataFrame({ - name: 'Hello', - refId: 'XYZ', - meta: { - aaa: 'xyz', - anything: 'xxx', - }, - fields: [ - { name: 'time', config: {}, type: FieldType.time, values: [1, 2, 3] }, - { name: 'value', config: { min: 0, max: 50, unit: 'something' }, type: FieldType.number, values: [1, 2, 3] }, - { name: 'str', config: {}, type: FieldType.string, values: ['a', 'b', 'c'] }, - ], - }); - - const table = grafanaDataFrameToArrowTable(frame); - expect(table.length).toEqual(frame.length); - - // Now back to DataFrame - const before = JSON.stringify(toDataFrameDTO(frame), null, 2); - const after = JSON.stringify(toDataFrameDTO(arrowTableToDataFrame(table)), null, 2); - expect(after).toEqual(before); - }); - - test('should support duplicate field names', () => { - const frame = toDataFrame({ - name: 'Hello', - refId: 'XYZ', - fields: [ - { name: 'time', config: {}, type: FieldType.time, values: [1, 2, 3] }, - { name: 'a', values: [1, 2, 3] }, - { name: 'a', values: ['a', 'b', 'c'] }, - ], - }); - - const table = grafanaDataFrameToArrowTable(frame, true); - expect(table.length).toEqual(frame.length); - - // Now back to DataFrame - const before = JSON.stringify(toDataFrameDTO(frame), null, 2); - const after = JSON.stringify(toDataFrameDTO(arrowTableToDataFrame(table)), null, 2); - expect(after).toEqual(before); - }); - - test('should read all types', () => { - const fullpath = path.resolve(__dirname, './__snapshots__/all_types.golden.arrow'); - const arrow = fs.readFileSync(fullpath); - const table = Table.from([arrow]); - const frame = arrowTableToDataFrame(table); - expect(toDataFrameDTO(frame)).toMatchSnapshot(); - }); - - test('Export arrow table names', () => { - const simple: DataFrame = { - name: 'hello', - fields: [ - { - name: 'fname', - labels: { - a: 'AAA', - b: 'BBB', - }, - config: {}, - type: FieldType.number, - values: new ArrayVector([1, 2]), - }, - ], - length: 2, - }; - const t1 = grafanaDataFrameToArrowTable(simple); - const t2 = grafanaDataFrameToArrowTable(simple, true); - expect(t1.getColumnAt(0)?.name).toEqual('fname {a="AAA", b="BBB"}'); - expect(t2.getColumnAt(0)?.name).toEqual('fname'); - }); -}); diff --git a/packages/grafana-data/src/dataframe/ArrowDataFrame.ts b/packages/grafana-data/src/dataframe/ArrowDataFrame.ts deleted file mode 100644 index 298fe76700c..00000000000 --- a/packages/grafana-data/src/dataframe/ArrowDataFrame.ts +++ /dev/null @@ -1,248 +0,0 @@ -import { DataFrame, FieldType, Field, Vector } from '../types'; -import { FunctionalVector } from '../vector/FunctionalVector'; - -import { - Table, - ArrowType, - Builder, - Vector as ArrowVector, - Float64, - DataType, - Utf8, - TimestampMillisecond, - Bool, - Column, -} from 'apache-arrow'; -import { getFieldDisplayName } from '../field'; - -export interface ArrowDataFrame extends DataFrame { - table: Table; -} - -export function base64StringToArrowTable(text: string): Table { - const b64 = atob(text); - const arr = Uint8Array.from(b64, (c) => { - return c.charCodeAt(0); - }); - return Table.from(arr); -} - -function valueOrUndefined(val?: string) { - return val ? val : undefined; -} - -function parseOptionalMeta(str?: string): any { - if (str && str.length && str !== '{}') { - try { - return JSON.parse(str); - } catch (err) { - console.warn('Error reading JSON from arrow metadata: ', str); - } - } - return undefined; -} - -/** - * Parse arrow table to basically a standard dataFrame. Use together with base64StringToArrowTable to get data frame - * from backend data source response. - * - * @public - */ -export function arrowTableToDataFrame(table: Table): ArrowDataFrame { - const fields: Field[] = []; - - for (let i = 0; i < table.numCols; i++) { - const col = table.getColumnAt(i); - if (col) { - const schema = table.schema.fields[i]; - let type = FieldType.other; - let values: Vector = col; - switch ((schema.typeId as unknown) as ArrowType) { - case ArrowType.Decimal: - case ArrowType.FloatingPoint: { - type = FieldType.number; - if (col.nullCount) { - values = new WrappedColumn(col); - } - break; - } - case ArrowType.Int: { - type = FieldType.number; - values = new NumberColumn(col); // Cast to number - break; - } - case ArrowType.Bool: { - type = FieldType.boolean; - if (col.nullCount) { - values = new WrappedColumn(col); - } - break; - } - case ArrowType.Timestamp: { - type = FieldType.time; - values = new NumberColumn(col); // Cast to number - break; - } - case ArrowType.Utf8: { - type = FieldType.string; - break; - } - default: - console.error('UNKNOWN Type:', schema); - } - - fields.push({ - name: col.name, - type, - values, - config: parseOptionalMeta(col.metadata.get('config')) || {}, - labels: parseOptionalMeta(col.metadata.get('labels')), - }); - } - } - const meta = table.schema.metadata; - return { - fields, - length: table.length, - refId: valueOrUndefined(meta.get('refId')), - name: valueOrUndefined(meta.get('name')), - meta: parseOptionalMeta(meta.get('meta')), - table, - }; -} - -function toArrowVector(field: Field): ArrowVector { - // OR: Float64Vector.from([1, 2, 3])); - - let type: DataType; - if (field.type === FieldType.number) { - type = new Float64(); - } else if (field.type === FieldType.time) { - type = new TimestampMillisecond(); - } else if (field.type === FieldType.boolean) { - type = new Bool(); - } else if (field.type === FieldType.string) { - type = new Utf8(); - } else { - type = new Utf8(); - } - const builder = Builder.new({ type, nullValues: [null] }); - field.values.toArray().forEach(builder.append.bind(builder)); - return builder.finish().toVector(); -} - -/** - * @param keepOriginalNames - by default, the exported Table will get names that match the - * display within grafana. This typically includes any labels defined in the metadata. - * - * When using this function to round-trip data, be sure to set `keepOriginalNames=true` - */ -export function grafanaDataFrameToArrowTable(data: DataFrame, keepOriginalNames?: boolean): Table { - // Return the original table - let table = (data as any).table; - if (table instanceof Table && table.numCols === data.fields.length) { - if (!keepOriginalNames) { - table = updateArrowTableNames(table, data); - } - if (table) { - return table as Table; - } - } - - table = Table.new( - data.fields.map((field, index) => { - let name = field.name; - // when used directly as an arrow table the name should match the arrow schema - if (!keepOriginalNames) { - name = getFieldDisplayName(field, data); - } - const column = Column.new(name, toArrowVector(field)); - if (field.labels) { - column.metadata.set('labels', JSON.stringify(field.labels)); - } - if (field.config) { - column.metadata.set('config', JSON.stringify(field.config)); - } - return column; - }) - ); - const metadata = table.schema.metadata; - if (data.name) { - metadata.set('name', data.name); - } - if (data.refId) { - metadata.set('refId', data.refId); - } - if (data.meta) { - metadata.set('meta', JSON.stringify(data.meta)); - } - return table; -} - -/** - * Turns arrow table into a base64 encoded string. This is symmetrical to base64StringToArrowTable and can be used - * to simulate response from backend. - * - * @public - */ -export function arrowTableToBase64String(table: Table): string { - const binstring = Array.prototype.map - .call(table.serialize() as Uint8Array, (ch: number) => { - return String.fromCharCode(ch); - }) - .join(''); - return btoa(binstring); -} - -function updateArrowTableNames(table: Table, frame: DataFrame): Table | undefined { - const cols: Column[] = []; - for (let i = 0; i < table.numCols; i++) { - const col = table.getColumnAt(i); - if (!col) { - return undefined; - } - const name = getFieldDisplayName(frame.fields[i], frame); - cols.push(Column.new(col.field.clone({ name: name }), ...col.chunks)); - } - return Table.new(cols); -} - -class NumberColumn extends FunctionalVector { - constructor(private col: Column) { - super(); - } - - get length() { - return this.col.length; - } - - get(index: number): number { - const v = this.col.get(index); - if (v === null || isNaN(v)) { - return v; - } - - // The conversion operations are always silent, never give errors, - // but if the bigint is too huge and won’t fit the number type, - // then extra bits will be cut off, so we should be careful doing such conversion. - // See https://javascript.info/bigint - return Number(v); - } -} - -// The `toArray()` arrow function will return a native ArrayBuffer -- this works fine -// if there are no null values, but the native arrays do not support nulls. This -// class simply wraps the vector so `toArray` creates a new array -class WrappedColumn extends FunctionalVector { - constructor(private col: Column) { - super(); - } - - get length() { - return this.col.length; - } - - get(index: number): number { - return this.col.get(index); - } -} diff --git a/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap b/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap deleted file mode 100644 index 4b7e781ba0d..00000000000 --- a/packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap +++ /dev/null @@ -1,389 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`Read/Write arrow Table to DataFrame should read all types 1`] = ` -Object { - "fields": Array [ - Object { - "config": Object {}, - "labels": Object { - "aLabelKey": "aLabelValue", - }, - "name": "string_values", - "type": "string", - "values": Array [ - "Go Min", - "JS Min (for >= 64)", - "0 / nil / misc", - "JS Max (for >= 64)", - "Go Max", - ], - }, - Object { - "config": Object { - "decimals": 2, - "displayName": "Grafana ❤️ (Previous should be heart emoji) 🦥 (Previous should be sloth emoji)", - "filterable": false, - "links": Array [ - Object { - "targetBlank": true, - "title": "Donate - The Sloth Conservation Foundation", - "url": "https://slothconservation.com/how-to-help/donate/", - }, - ], - "max": null, - "min": null, - "noValue": "😤", - "nullValueMode": "null", - }, - "labels": Object { - "aLabelKey": "aLabelValue", - "bLabelKey": "bLabelValue", - }, - "name": "nullable_string_values", - "type": "string", - "values": Array [ - "Grafana", - "❤️", - null, - "🦥", - "update your unicode/font if no sloth, is 2019.", - ], - }, - Object { - "config": Object { - "max": 1, - "min": 0, - }, - "labels": undefined, - "name": "int8_values", - "type": "number", - "values": Array [ - -128, - -128, - 0, - 127, - 127, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "nullable_int8_values", - "type": "number", - "values": Array [ - -128, - -128, - null, - 127, - 127, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "int16_values", - "type": "number", - "values": Array [ - -32768, - -32768, - 0, - 32767, - 32767, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "nullable_int16_values", - "type": "number", - "values": Array [ - -32768, - -32768, - null, - 32767, - 32767, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "int32_values", - "type": "number", - "values": Array [ - -2147483648, - -2147483648, - 1, - 2147483647, - 2147483647, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "nullable_int32_values", - "type": "number", - "values": Array [ - -2147483648, - -2147483648, - null, - 2147483647, - 2147483647, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "int64_values", - "type": "number", - "values": Array [ - -9223372036854776000, - -9007199254740991, - 1, - 9007199254740991, - 9223372036854776000, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "nullable_int64_values", - "type": "number", - "values": Array [ - -9223372036854776000, - -9007199254740991, - null, - 9007199254740991, - 9223372036854776000, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "uint8_values", - "type": "number", - "values": Array [ - 0, - 0, - 1, - 255, - 255, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "nullable_uint8_values", - "type": "number", - "values": Array [ - 0, - 0, - null, - 255, - 255, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "uint16_values", - "type": "number", - "values": Array [ - 0, - 0, - 1, - 65535, - 65535, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "nullable_uint16_values", - "type": "number", - "values": Array [ - 0, - 0, - null, - 65535, - 65535, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "uint32_values", - "type": "number", - "values": Array [ - 0, - 0, - 1, - 4294967295, - 4294967295, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "nullable_uint32_values", - "type": "number", - "values": Array [ - 0, - 0, - null, - 4294967295, - 4294967295, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "uint64_values", - "type": "number", - "values": Array [ - 0, - 0, - 1, - 9007199254740991, - 18446744073709552000, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "nullable_uint64_values", - "type": "number", - "values": Array [ - 0, - 0, - null, - 9007199254740991, - 18446744073709552000, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "float32_values", - "type": "number", - "values": Float32Array [ - 1.401298464324817e-45, - 1.401298464324817e-45, - 1, - 3.4028234663852886e+38, - 3.4028234663852886e+38, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "nullable_float32_values", - "type": "number", - "values": Array [ - 1.401298464324817e-45, - 1.401298464324817e-45, - null, - 3.4028234663852886e+38, - 3.4028234663852886e+38, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "float64_values", - "type": "number", - "values": Array [ - 5e-324, - -9007199254740991, - 1, - 9007199254740991, - 1.7976931348623157e+308, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "nullable_float64_values", - "type": "number", - "values": Array [ - 5e-324, - -9007199254740991, - null, - 1.7976931348623157e+308, - 9007199254740991, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "bool_values", - "type": "boolean", - "values": Array [ - true, - false, - true, - true, - false, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "nullable_bool_values", - "type": "boolean", - "values": Array [ - true, - false, - null, - true, - false, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "timestamps", - "type": "time", - "values": Array [ - 0, - 1568039445000, - 1568039450000, - 9007199254.740992, - 9223372036854.775, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "timestamps", - "type": "time", - "values": Array [ - 0, - 1568039445000, - 1568039450000, - 9007199254.740992, - 9223372036854.775, - ], - }, - Object { - "config": Object {}, - "labels": undefined, - "name": "nullable_timestamps", - "type": "time", - "values": Array [ - 0, - 1568039445000, - null, - 9007199254.740992, - 9223372036854.775, - ], - }, - ], - "meta": Object { - "custom": Object { - "Hi": "there", - }, - }, - "name": "many_types", - "refId": "A", -} -`; diff --git a/packages/grafana-data/src/dataframe/index.ts b/packages/grafana-data/src/dataframe/index.ts index 2d6dc3ae095..95b7cf951c4 100644 --- a/packages/grafana-data/src/dataframe/index.ts +++ b/packages/grafana-data/src/dataframe/index.ts @@ -4,7 +4,6 @@ export * from './CircularDataFrame'; export * from './MutableDataFrame'; export * from './processDataFrame'; export * from './dimensions'; -export * from './ArrowDataFrame'; export * from './ArrayDataFrame'; export * from './DataFrameJSON'; export * from './StreamingDataFrame'; diff --git a/packages/grafana-runtime/src/utils/queryResponse.test.ts b/packages/grafana-runtime/src/utils/queryResponse.test.ts index ca3c99cc500..f04647fccb4 100644 --- a/packages/grafana-runtime/src/utils/queryResponse.test.ts +++ b/packages/grafana-runtime/src/utils/queryResponse.test.ts @@ -1,37 +1,79 @@ import { DataQuery, toDataFrameDTO, DataFrame } from '@grafana/data'; -import { toDataQueryResponse } from './queryResponse'; +import { FetchResponse } from 'src/services'; +import { BackendDataSourceResponse, toDataQueryResponse } from './queryResponse'; -const resp = { +const resp = ({ data: { results: { A: { - refId: 'A', - dataframes: [ - 'QVJST1cxAAD/////cAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAFAAAAACAAAAKAAAAAQAAAAg////CAAAAAwAAAABAAAAQQAAAAUAAAByZWZJZAAAAED///8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAAgAAAHwAAAAEAAAAnv///xQAAABAAAAAQAAAAAAAAwFAAAAAAQAAAAQAAACM////CAAAABQAAAAIAAAAQS1zZXJpZXMAAAAABAAAAG5hbWUAAAAAAAAAAIb///8AAAIACAAAAEEtc2VyaWVzAAASABgAFAATABIADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAKAUwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAHRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAdGltZQAAAAAAAAAA/////7gAAAAUAAAAAAAAAAwAFgAUABMADAAEAAwAAABgAAAAAAAAABQAAAAAAAADAwAKABgADAAIAAQACgAAABQAAABYAAAABgAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAMAAAAAAAAAAAAAAAAgAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAAAAAAAAAAAQMC/OcElXhZAOAEFxCVeFkCwQtDGJV4WQCiEm8klXhZAoMVmzCVeFkAYBzLPJV4WAAAAAAAA8D8AAAAAAAA0QAAAAAAAgFZAAAAAAAAAPkAAAAAAAAAUQAAAAAAAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADgAAAAAAAMAAQAAAIABAAAAAAAAwAAAAAAAAABgAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAABQAAAAAgAAACgAAAAEAAAAIP///wgAAAAMAAAAAQAAAEEAAAAFAAAAcmVmSWQAAABA////CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAIAAAB8AAAABAAAAJ7///8UAAAAQAAAAEAAAAAAAAMBQAAAAAEAAAAEAAAAjP///wgAAAAUAAAACAAAAEEtc2VyaWVzAAAAAAQAAABuYW1lAAAAAAAAAACG////AAACAAgAAABBLXNlcmllcwAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABEAAAATAAAAAAACgFMAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAAB0aW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAHRpbWUAAAAAmAEAAEFSUk9XMQ==', + frames: [ + { + schema: { + refId: 'A', + fields: [ + { name: 'time', type: 'time', typeInfo: { frame: 'time.Time', nullable: true } }, + { name: 'A-series', type: 'number', typeInfo: { frame: 'float64', nullable: true } }, + ], + }, + data: { + values: [ + [1611767228473, 1611767240473, 1611767252473, 1611767264473, 1611767276473, 1611767288473], + [1, 20, 90, 30, 5, 0], + ], + }, + }, ], }, B: { - refId: 'B', - dataframes: [ - 'QVJST1cxAAD/////cAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAFAAAAACAAAAKAAAAAQAAAAg////CAAAAAwAAAABAAAAQgAAAAUAAAByZWZJZAAAAED///8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAAgAAAHwAAAAEAAAAnv///xQAAABAAAAAQAAAAAAAAwFAAAAAAQAAAAQAAACM////CAAAABQAAAAIAAAAQi1zZXJpZXMAAAAABAAAAG5hbWUAAAAAAAAAAIb///8AAAIACAAAAEItc2VyaWVzAAASABgAFAATABIADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAKAUwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAHRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAdGltZQAAAAAAAAAA/////7gAAAAUAAAAAAAAAAwAFgAUABMADAAEAAwAAABgAAAAAAAAABQAAAAAAAADAwAKABgADAAIAAQACgAAABQAAABYAAAABgAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAMAAAAAAAAAAAAAAAAgAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAAAAAAAAAAAQMC/OcElXhZAOAEFxCVeFkCwQtDGJV4WQCiEm8klXhZAoMVmzCVeFkAYBzLPJV4WAAAAAAAA8D8AAAAAAAA0QAAAAAAAgFZAAAAAAAAAPkAAAAAAAAAUQAAAAAAAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADgAAAAAAAMAAQAAAIABAAAAAAAAwAAAAAAAAABgAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAABQAAAAAgAAACgAAAAEAAAAIP///wgAAAAMAAAAAQAAAEIAAAAFAAAAcmVmSWQAAABA////CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAIAAAB8AAAABAAAAJ7///8UAAAAQAAAAEAAAAAAAAMBQAAAAAEAAAAEAAAAjP///wgAAAAUAAAACAAAAEItc2VyaWVzAAAAAAQAAABuYW1lAAAAAAAAAACG////AAACAAgAAABCLXNlcmllcwAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABEAAAATAAAAAAACgFMAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAAB0aW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAHRpbWUAAAAAmAEAAEFSUk9XMQ==', + frames: [ + { + schema: { + refId: 'B', + fields: [ + { name: 'time', type: 'time', typeInfo: { frame: 'time.Time', nullable: true } }, + { name: 'B-series', type: 'number', typeInfo: { frame: 'float64', nullable: true } }, + ], + }, + data: { + values: [ + [1611767228473, 1611767240473, 1611767252473, 1611767264473, 1611767276473, 1611767288473], + [1, 20, 90, 30, 5, 0], + ], + }, + }, ], }, }, }, -}; +} as any) as FetchResponse; -const resWithError = { +const resWithError = ({ data: { results: { A: { error: 'Hello Error', - dataframes: [ - 'QVJST1cxAAD/////WAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAJwAAAADAAAATAAAACgAAAAEAAAAPP///wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABc////CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAHz///8IAAAANAAAACoAAAB7Im5vdGljZXMiOlt7InNldmVyaXR5IjoyLCJ0ZXh0IjoiVGV4dCJ9XX0AAAQAAABtZXRhAAAAAAEAAAAYAAAAAAASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABMAAAAAAAAA0wAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABwAAAG51bWJlcnMABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAgAHAAAAbnVtYmVycwAAAAAA/////4gAAAAUAAAAAAAAAAwAFgAUABMADAAEAAwAAAAQAAAAAAAAABQAAAAAAAADAwAKABgADAAIAAQACgAAABQAAAA4AAAAAgAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAEAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAA8D8AAAAAAAAIQBAAAAAMABQAEgAMAAgABAAMAAAAEAAAACwAAAA4AAAAAAADAAEAAABoAQAAAAAAAJAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAnAAAAAMAAABMAAAAKAAAAAQAAAA8////CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAFz///8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAfP///wgAAAA0AAAAKgAAAHsibm90aWNlcyI6W3sic2V2ZXJpdHkiOjIsInRleHQiOiJUZXh0In1dfQAABAAAAG1ldGEAAAAAAQAAABgAAAAAABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEwAAAAAAAADTAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAHAAAAbnVtYmVycwAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAACAAcAAABudW1iZXJzAIABAABBUlJPVzE=', + frames: [ + { + schema: { + fields: [{ name: 'numbers', type: 'number' }], + meta: { + notices: [ + { + severity: 2, + text: 'Text', + }, + ], + }, + }, + data: { + values: [[1, 3]], + }, + }, ], }, }, }, -}; +} as any) as FetchResponse; const emptyResults = { data: { results: { '': { refId: '' } } }, diff --git a/packages/grafana-runtime/src/utils/queryResponse.ts b/packages/grafana-runtime/src/utils/queryResponse.ts index 8e3220c5b93..499de5df7e1 100644 --- a/packages/grafana-runtime/src/utils/queryResponse.ts +++ b/packages/grafana-runtime/src/utils/queryResponse.ts @@ -1,7 +1,5 @@ import { DataQueryResponse, - arrowTableToDataFrame, - base64StringToArrowTable, KeyValue, LoadingState, DataQueryError, @@ -30,7 +28,6 @@ export interface DataResponse { frames?: DataFrameJSON[]; // Legacy TSDB format... - dataframes?: string[]; // base64 encoded arrow tables series?: TimeSeries[]; tables?: TableData[]; } @@ -114,22 +111,6 @@ export function toDataQueryResponse( rsp.data.push(toDataFrame(s)); } } - - if (dr.dataframes) { - for (const b64 of dr.dataframes) { - try { - const t = base64StringToArrowTable(b64); - const f = arrowTableToDataFrame(t); - if (!f.refId) { - f.refId = dr.refId; - } - rsp.data.push(f); - } catch (err) { - rsp.state = LoadingState.Error; - rsp.error = toDataQueryError(err); - } - } - } } } diff --git a/pkg/services/ngalert/api/api.go b/pkg/services/ngalert/api/api.go index 71e47be9108..3423f8bfb88 100644 --- a/pkg/services/ngalert/api/api.go +++ b/pkg/services/ngalert/api/api.go @@ -13,7 +13,6 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/middleware" "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/services/datasourceproxy" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/ngalert/eval" @@ -131,14 +130,9 @@ func (api *API) conditionEvalEndpoint(c *models.ReqContext, cmd ngmodels.EvalAle } frame := evalResults.AsDataFrame() - df := plugins.NewDecodedDataFrames([]*data.Frame{&frame}) - instances, err := df.Encoded() - if err != nil { - return response.Error(400, "Failed to encode result dataframes", err) - } - return response.JSON(200, util.DynMap{ - "instances": instances, + return response.JSONStreaming(200, util.DynMap{ + "instances": []*data.Frame{&frame}, }) } @@ -162,17 +156,8 @@ func (api *API) alertDefinitionEvalEndpoint(c *models.ReqContext) response.Respo } frame := evalResults.AsDataFrame() - df := plugins.NewDecodedDataFrames([]*data.Frame{&frame}) - if err != nil { - return response.Error(400, "Failed to instantiate Dataframes from the decoded frames", err) - } - - instances, err := df.Encoded() - if err != nil { - return response.Error(400, "Failed to encode result dataframes", err) - } - return response.JSON(200, util.DynMap{ - "instances": instances, + return response.JSONStreaming(200, util.DynMap{ + "instances": []*data.Frame{&frame}, }) } diff --git a/pkg/tsdb/testdatasource/scenarios.go b/pkg/tsdb/testdatasource/scenarios.go index 983b270481a..d5f7a5cb61b 100644 --- a/pkg/tsdb/testdatasource/scenarios.go +++ b/pkg/tsdb/testdatasource/scenarios.go @@ -2,6 +2,7 @@ package testdatasource import ( "context" + "encoding/base64" "encoding/json" "fmt" "math" @@ -150,7 +151,7 @@ Timestamps will line up evenly on timeStepSeconds (For example, 60 seconds means p.registerScenario(&Scenario{ ID: string(arrowQuery), Name: "Load Apache Arrow Data", - handler: p.handleClientSideScenario, + handler: p.handleArrowScenario, }) p.registerScenario(&Scenario{ @@ -484,6 +485,30 @@ func (p *testDataPlugin) handleClientSideScenario(ctx context.Context, req *back return backend.NewQueryDataResponse(), nil } +func (p *testDataPlugin) handleArrowScenario(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { + resp := backend.NewQueryDataResponse() + + for _, q := range req.Queries { + model, err := simplejson.NewJson(q.JSON) + if err != nil { + return nil, err + } + + respD := resp.Responses[q.RefID] + frame, err := doArrowQuery(q, model) + if err != nil { + return nil, err + } + if frame == nil { + continue + } + respD.Frames = append(respD.Frames, frame) + resp.Responses[q.RefID] = respD + } + + return resp, nil +} + func (p *testDataPlugin) handleExponentialHeatmapBucketDataScenario(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { resp := backend.NewQueryDataResponse() @@ -875,6 +900,18 @@ func randomHeatmapData(query backend.DataQuery, fnBucketGen func(index int) floa return frame } +func doArrowQuery(query backend.DataQuery, model *simplejson.Json) (*data.Frame, error) { + encoded := model.Get("stringInput").MustString("") + if encoded == "" { + return nil, nil + } + arrow, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return nil, err + } + return data.UnmarshalArrowFrame(arrow) +} + func newSeriesForQuery(query backend.DataQuery, model *simplejson.Json, index int) *data.Frame { alias := model.Get("alias").MustString("") suffix := "" diff --git a/public/app/features/alerting/state/actions.ts b/public/app/features/alerting/state/actions.ts index e8baafb608d..0832914a7e3 100644 --- a/public/app/features/alerting/state/actions.ts +++ b/public/app/features/alerting/state/actions.ts @@ -1,8 +1,8 @@ import { AppEvents, applyFieldOverrides, - arrowTableToDataFrame, - base64StringToArrowTable, + dataFrameFromJSON, + DataFrameJSON, DataSourceApi, dateMath, } from '@grafana/data'; @@ -187,11 +187,11 @@ export function evaluateAlertDefinition(): ThunkResult { return async (dispatch, getStore) => { const { alertDefinition } = getStore().alertDefinition; - const response: { instances: string[] } = await getBackendSrv().get( + const response: { instances: DataFrameJSON[] } = await getBackendSrv().get( `/api/alert-definitions/eval/${alertDefinition.uid}` ); - const handledResponse = handleBase64Response(response.instances); + const handledResponse = handleJSONResponse(response.instances); dispatch(setInstanceData(handledResponse)); appEvents.emit(AppEvents.alertSuccess, ['Alert definition tested successfully']); @@ -203,12 +203,12 @@ export function evaluateNotSavedAlertDefinition(): ThunkResult { const { alertDefinition, getQueryOptions } = getStore().alertDefinition; const defaultDataSource = await getDataSourceSrv().get(null); - const response: { instances: string[] } = await getBackendSrv().post('/api/alert-definitions/eval', { + const response: { instances: DataFrameJSON[] } = await getBackendSrv().post('/api/alert-definitions/eval', { condition: alertDefinition.condition, data: buildDataQueryModel(getQueryOptions(), defaultDataSource), }); - const handledResponse = handleBase64Response(response.instances); + const handledResponse = handleJSONResponse(response.instances); dispatch(setInstanceData(handledResponse)); appEvents.emit(AppEvents.alertSuccess, ['Alert definition tested successfully']); }; @@ -231,10 +231,9 @@ async function buildAlertDefinition(state: AlertDefinitionState) { }; } -function handleBase64Response(frames: string[]) { +function handleJSONResponse(frames: DataFrameJSON[]) { const dataFrames = frames.map((instance) => { - const table = base64StringToArrowTable(instance); - return arrowTableToDataFrame(table); + return dataFrameFromJSON(instance); }); return applyFieldOverrides({ diff --git a/public/app/plugins/datasource/cloudwatch/specs/datasource.test.ts b/public/app/plugins/datasource/cloudwatch/specs/datasource.test.ts index 9c0111e5e6d..b8382e62105 100644 --- a/public/app/plugins/datasource/cloudwatch/specs/datasource.test.ts +++ b/public/app/plugins/datasource/cloudwatch/specs/datasource.test.ts @@ -84,13 +84,58 @@ describe('CloudWatchDatasource', () => { const response = { results: { A: { - dataframes: [ - 'QVJST1cxAAD/////GAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAFgAAAACAAAAKAAAAAQAAAB8////CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAJz///8IAAAAFAAAAAkAAABsb2dHcm91cHMAAAAEAAAAbmFtZQAAAAABAAAAGAAAAAAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABMAAAAUAAAAAAABQFMAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAGAAAAAwAAABsb2dHcm91cE5hbWUAAAAABAAAAG5hbWUAAAAAAAAAAAQABAAEAAAADAAAAGxvZ0dyb3VwTmFtZQAAAAD/////mAAAABQAAAAAAAAADAAWABQAEwAMAAQADAAAAGAGAAAAAAAAFAAAAAAAAAMDAAoAGAAMAAgABAAKAAAAFAAAAEgAAAAhAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAiAAAAAAAAACIAAAAAAAAANgFAAAAAAAAAAAAAAEAAAAhAAAAAAAAAAAAAAAAAAAAAAAAADIAAABiAAAAkQAAALwAAADuAAAAHwEAAFQBAACHAQAAtQEAAOoBAAAbAgAASgIAAHQCAAClAgAA1QIAABADAABEAwAAdgMAAKMDAADXAwAACQQAAEAEAAB3BAAAlwQAAK0EAAC8BAAA+wQAAEIFAABhBQAAeAUAAJIFAAC0BQAA1gUAAC9hd3MvY29udGFpbmVyaW5zaWdodHMvZGV2MzAzLXdvcmtzaG9wL2FwcGxpY2F0aW9uL2F3cy9jb250YWluZXJpbnNpZ2h0cy9kZXYzMDMtd29ya3Nob3AvZGF0YXBsYW5lL2F3cy9jb250YWluZXJpbnNpZ2h0cy9kZXYzMDMtd29ya3Nob3AvZmxvd2xvZ3MvYXdzL2NvbnRhaW5lcmluc2lnaHRzL2RldjMwMy13b3Jrc2hvcC9ob3N0L2F3cy9jb250YWluZXJpbnNpZ2h0cy9kZXYzMDMtd29ya3Nob3AvcGVyZm9ybWFuY2UvYXdzL2NvbnRhaW5lcmluc2lnaHRzL2RldjMwMy13b3Jrc2hvcC9wcm9tZXRoZXVzL2F3cy9jb250YWluZXJpbnNpZ2h0cy9lY29tbWVyY2Utc29ja3Nob3AvYXBwbGljYXRpb24vYXdzL2NvbnRhaW5lcmluc2lnaHRzL2Vjb21tZXJjZS1zb2Nrc2hvcC9kYXRhcGxhbmUvYXdzL2NvbnRhaW5lcmluc2lnaHRzL2Vjb21tZXJjZS1zb2Nrc2hvcC9ob3N0L2F3cy9jb250YWluZXJpbnNpZ2h0cy9lY29tbWVyY2Utc29ja3Nob3AvcGVyZm9ybWFuY2UvYXdzL2NvbnRhaW5lcmluc2lnaHRzL3dhdGNoZGVtby1wZXJmL2FwcGxpY2F0aW9uL2F3cy9jb250YWluZXJpbnNpZ2h0cy93YXRjaGRlbW8tcGVyZi9kYXRhcGxhbmUvYXdzL2NvbnRhaW5lcmluc2lnaHRzL3dhdGNoZGVtby1wZXJmL2hvc3QvYXdzL2NvbnRhaW5lcmluc2lnaHRzL3dhdGNoZGVtby1wZXJmL3BlcmZvcm1hbmNlL2F3cy9jb250YWluZXJpbnNpZ2h0cy93YXRjaGRlbW8tcGVyZi9wcm9tZXRoZXVzL2F3cy9jb250YWluZXJpbnNpZ2h0cy93YXRjaGRlbW8tcHJvZC11cy1lYXN0LTEvcGVyZm9ybWFuY2UvYXdzL2NvbnRhaW5lcmluc2lnaHRzL3dhdGNoZGVtby1zdGFnaW5nL2FwcGxpY2F0aW9uL2F3cy9jb250YWluZXJpbnNpZ2h0cy93YXRjaGRlbW8tc3RhZ2luZy9kYXRhcGxhbmUvYXdzL2NvbnRhaW5lcmluc2lnaHRzL3dhdGNoZGVtby1zdGFnaW5nL2hvc3QvYXdzL2NvbnRhaW5lcmluc2lnaHRzL3dhdGNoZGVtby1zdGFnaW5nL3BlcmZvcm1hbmNlL2F3cy9lY3MvY29udGFpbmVyaW5zaWdodHMvYnVnYmFzaC1lYzIvcGVyZm9ybWFuY2UvYXdzL2Vjcy9jb250YWluZXJpbnNpZ2h0cy9lY3MtZGVtb3dvcmtzaG9wL3BlcmZvcm1hbmNlL2F3cy9lY3MvY29udGFpbmVyaW5zaWdodHMvZWNzLXdvcmtzaG9wLWRldi9wZXJmb3JtYW5jZS9hd3MvZWtzL2RldjMwMy13b3Jrc2hvcC9jbHVzdGVyL2F3cy9ldmVudHMvY2xvdWR0cmFpbC9hd3MvZXZlbnRzL2Vjcy9hd3MvbGFtYmRhL2N3c3luLW15Y2FuYXJ5LWZhYzk3ZGVkLWYxMzQtNDk5YS05ZDcxLTRjM2JlMWY2MzE4Mi9hd3MvbGFtYmRhL2N3c3luLXdhdGNoLWxpbmtjaGVja3MtZWY3ZWYyNzMtNWRhMi00NjYzLWFmNTQtZDJmNTJkNTViMDYwL2Vjcy9lY3MtY3dhZ2VudC1kYWVtb24tc2VydmljZS9lY3MvZWNzLWRlbW8tbGltaXRUYXNrQ2xvdWRUcmFpbC9EZWZhdWx0TG9nR3JvdXBjb250YWluZXItaW5zaWdodHMtcHJvbWV0aGV1cy1iZXRhY29udGFpbmVyLWluc2lnaHRzLXByb21ldGhldXMtZGVtbwAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAMAAQAAACgBAAAAAAAAoAAAAAAAAABgBgAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAWAAAAAIAAAAoAAAABAAAAHz///8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAAnP///wgAAAAUAAAACQAAAGxvZ0dyb3VwcwAAAAQAAABuYW1lAAAAAAEAAAAYAAAAAAASABgAFAATABIADAAAAAgABAASAAAAFAAAAEwAAABQAAAAAAAFAUwAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAYAAAADAAAAGxvZ0dyb3VwTmFtZQAAAAAEAAAAbmFtZQAAAAAAAAAABAAEAAQAAAAMAAAAbG9nR3JvdXBOYW1lAAAAAEgBAABBUlJPVzE=', + frames: [ + { + schema: { + name: 'logGroups', + refId: 'A', + fields: [{ name: 'logGroupName', type: 'string', typeInfo: { frame: 'string', nullable: true } }], + }, + data: { + values: [ + [ + '/aws/containerinsights/dev303-workshop/application', + '/aws/containerinsights/dev303-workshop/dataplane', + '/aws/containerinsights/dev303-workshop/flowlogs', + '/aws/containerinsights/dev303-workshop/host', + '/aws/containerinsights/dev303-workshop/performance', + '/aws/containerinsights/dev303-workshop/prometheus', + '/aws/containerinsights/ecommerce-sockshop/application', + '/aws/containerinsights/ecommerce-sockshop/dataplane', + '/aws/containerinsights/ecommerce-sockshop/host', + '/aws/containerinsights/ecommerce-sockshop/performance', + '/aws/containerinsights/watchdemo-perf/application', + '/aws/containerinsights/watchdemo-perf/dataplane', + '/aws/containerinsights/watchdemo-perf/host', + '/aws/containerinsights/watchdemo-perf/performance', + '/aws/containerinsights/watchdemo-perf/prometheus', + '/aws/containerinsights/watchdemo-prod-us-east-1/performance', + '/aws/containerinsights/watchdemo-staging/application', + '/aws/containerinsights/watchdemo-staging/dataplane', + '/aws/containerinsights/watchdemo-staging/host', + '/aws/containerinsights/watchdemo-staging/performance', + '/aws/ecs/containerinsights/bugbash-ec2/performance', + '/aws/ecs/containerinsights/ecs-demoworkshop/performance', + '/aws/ecs/containerinsights/ecs-workshop-dev/performance', + '/aws/eks/dev303-workshop/cluster', + '/aws/events/cloudtrail', + '/aws/events/ecs', + '/aws/lambda/cwsyn-mycanary-fac97ded-f134-499a-9d71-4c3be1f63182', + '/aws/lambda/cwsyn-watch-linkchecks-ef7ef273-5da2-4663-af54-d2f52d55b060', + '/ecs/ecs-cwagent-daemon-service', + '/ecs/ecs-demo-limitTask', + 'CloudTrail/DefaultLogGroup', + 'container-insights-prometheus-beta', + 'container-insights-prometheus-demo', + ], + ], + }, + }, ], - refId: 'A', }, }, }; + const { ds } = getTestContext({ response }); const expectedLogGroups = [ '/aws/containerinsights/dev303-workshop/application', diff --git a/public/app/plugins/datasource/cloudwatch/types.ts b/public/app/plugins/datasource/cloudwatch/types.ts index 136a3999bbc..2804e33ca56 100644 --- a/public/app/plugins/datasource/cloudwatch/types.ts +++ b/public/app/plugins/datasource/cloudwatch/types.ts @@ -176,7 +176,6 @@ export interface TSDBQueryResult { refId: string; series: TSDBTimeSeries[]; tables: Array>; - dataframes: number[][]; error?: string; meta?: any; diff --git a/public/app/plugins/datasource/postgres/specs/datasource.test.ts b/public/app/plugins/datasource/postgres/specs/datasource.test.ts index d6da46a3038..bd56d8d2293 100644 --- a/public/app/plugins/datasource/postgres/specs/datasource.test.ts +++ b/public/app/plugins/datasource/postgres/specs/datasource.test.ts @@ -96,7 +96,6 @@ describe('PostgreSQLDatasource', () => { }, ], tables: null, - dataframes: null, }, }, }; @@ -167,7 +166,6 @@ describe('PostgreSQLDatasource', () => { rows: [[1599643351085, 'America', 30.226249741223704]], }, ], - dataframes: null, }, }, }; diff --git a/public/app/plugins/datasource/tempo/datasource.test.ts b/public/app/plugins/datasource/tempo/datasource.test.ts index c89c78f13d6..8a950bd62c3 100644 --- a/public/app/plugins/datasource/tempo/datasource.test.ts +++ b/public/app/plugins/datasource/tempo/datasource.test.ts @@ -1,11 +1,4 @@ -import { - arrowTableToBase64String, - DataFrame, - DataSourceInstanceSettings, - grafanaDataFrameToArrowTable, - MutableDataFrame, - PluginType, -} from '@grafana/data'; +import { DataFrame, dataFrameToJSON, DataSourceInstanceSettings, MutableDataFrame, PluginType } from '@grafana/data'; import { Observable, of } from 'rxjs'; import { createFetchResponse } from 'test/helpers/createFetchResponse'; import { TempoDatasource } from './datasource'; @@ -48,7 +41,7 @@ function setupBackendSrv(frame: DataFrame) { createFetchResponse({ results: { refid1: { - dataframes: [encode(frame)], + frames: [dataFrameToJSON(frame)], }, }, }) @@ -57,11 +50,6 @@ function setupBackendSrv(frame: DataFrame) { } as any); } -function encode(frame: DataFrame) { - const table = grafanaDataFrameToArrowTable(frame); - return arrowTableToBase64String(table); -} - const defaultSettings: DataSourceInstanceSettings = { id: 0, uid: '0', diff --git a/public/app/plugins/datasource/testdata/datasource.ts b/public/app/plugins/datasource/testdata/datasource.ts index 6e20f5cbbc3..75619598a21 100644 --- a/public/app/plugins/datasource/testdata/datasource.ts +++ b/public/app/plugins/datasource/testdata/datasource.ts @@ -4,8 +4,6 @@ import { delay } from 'rxjs/operators'; import { AnnotationEvent, ArrayDataFrame, - arrowTableToDataFrame, - base64StringToArrowTable, DataFrame, DataQueryRequest, DataQueryResponse, @@ -22,7 +20,6 @@ import { getLiveMeasurementsObserver, getTemplateSrv, TemplateSrv, - toDataQueryError, } from '@grafana/runtime'; import { queryMetricTree } from './metricTree'; import { runStream } from './runStreams'; @@ -61,9 +58,6 @@ export class TestDataDataSource extends DataSourceWithBackend { case 'grafana_api': streams.push(runGrafanaAPI(target, options)); break; - case 'arrow': - streams.push(runArrowFile(target, options)); - break; case 'annotations': streams.push(this.annotationDataTopicTest(target, options)); break; @@ -176,24 +170,6 @@ export class TestDataDataSource extends DataSourceWithBackend { } } -function runArrowFile(target: TestDataQuery, req: DataQueryRequest): Observable { - let data: DataFrame[] = []; - if (target.stringInput && target.stringInput.length > 10) { - try { - const table = base64StringToArrowTable(target.stringInput); - const frame = arrowTableToDataFrame(table); - frame.refId = target.refId; - data = [frame]; - } catch (e) { - console.warn('Error reading saved arrow', e); - const error = toDataQueryError(e); - error.refId = target.refId; - return of({ state: LoadingState.Error, error, data }); - } - } - return of({ state: LoadingState.Done, data, key: req.requestId + target.refId }); -} - function runGrafanaAPI(target: TestDataQuery, req: DataQueryRequest): Observable { const url = `/api/${target.stringInput}`; return from( diff --git a/scripts/webpack/webpack.common.js b/scripts/webpack/webpack.common.js index 652c0c96eb4..5320264d278 100644 --- a/scripts/webpack/webpack.common.js +++ b/scripts/webpack/webpack.common.js @@ -24,15 +24,7 @@ function shouldExclude(filename) { return false; } - const packagesToProcessbyBabel = [ - 'debug', - 'lru-cache', - 'yallist', - 'apache-arrow', - 'react-hook-form', - 'rc-trigger', - 'monaco-editor', - ]; + const packagesToProcessbyBabel = ['debug', 'lru-cache', 'yallist', 'react-hook-form', 'rc-trigger', 'monaco-editor']; for (const package of packagesToProcessbyBabel) { if (filename.indexOf(`node_modules/${package}`) > 0) { return false; diff --git a/yarn.lock b/yarn.lock index f36fb551999..19a2e43c02d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6085,11 +6085,6 @@ resolved "https://registry.yarnpkg.com/@types/file-saver/-/file-saver-2.0.1.tgz#e18eb8b069e442f7b956d313f4fadd3ef887354e" integrity sha512-g1QUuhYVVAamfCifK7oB7G3aIl4BbOyzDOqVyUfEr4tfBKrXfeH+M+Tg7HKCXSrbzxYdhyCP7z9WbKo0R2hBCw== -"@types/flatbuffers@^1.9.1": - version "1.9.1" - resolved "https://registry.yarnpkg.com/@types/flatbuffers/-/flatbuffers-1.9.1.tgz#1910bebfc15c8f67a287fae07bfc061f94e9d291" - integrity sha512-TC3X0Nkj5wgvuY217VkodBtjbD3Yr0JNApDY1GW9IU5Mzm5ie1IJErqe4vRm+wy08IRz3bemaDATrdEw1CJlVQ== - "@types/fs-extra@^8.0.1", "@types/fs-extra@^8.1.0": version "8.1.0" resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-8.1.0.tgz#1114834b53c3914806cd03b3304b37b3bd221a4d" @@ -6881,11 +6876,6 @@ "@types/react" "*" "@types/react-test-renderer" "*" -"@types/text-encoding-utf-8@^1.0.1": - version "1.0.1" - resolved "https://registry.yarnpkg.com/@types/text-encoding-utf-8/-/text-encoding-utf-8-1.0.1.tgz#908d884af1114e5d8df47597b1e04f833383d23d" - integrity sha512-GpIEYaS+yNfYqpowLLziiY42pyaL+lThd/wMh6tTubaKuG4IRkXqqyxK7Nddn3BvpUg2+go3Gv/jbXvAFMRjiQ== - "@types/through@*": version "0.0.29" resolved "https://registry.yarnpkg.com/@types/through/-/through-0.0.29.tgz#72943aac922e179339c651fa34a4428a4d722f93" @@ -7987,22 +7977,6 @@ anymatch@^3.0.3, anymatch@~3.1.1: normalize-path "^3.0.0" picomatch "^2.0.4" -apache-arrow@0.16.0: - version "0.16.0" - resolved "https://registry.yarnpkg.com/apache-arrow/-/apache-arrow-0.16.0.tgz#7ee7a6397d1a2d6349aed90c6ce5b92362e79881" - integrity sha512-hiabMZb2XgHiNK6f7C/2x/fyGS85PoCdkeMhJDyZipaJy1il1BJMDDEa3VLvM6mdxsG61pY1zev6zWetOj8Eog== - dependencies: - "@types/flatbuffers" "^1.9.1" - "@types/node" "^12.0.4" - "@types/text-encoding-utf-8" "^1.0.1" - command-line-args "5.0.2" - command-line-usage "5.0.5" - flatbuffers "1.11.0" - json-bignum "^0.0.3" - pad-left "^2.1.0" - text-encoding-utf-8 "^1.0.2" - tslib "^1.9.3" - app-root-dir@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/app-root-dir/-/app-root-dir-1.0.2.tgz#38187ec2dea7577fff033ffcb12172692ff6e118" @@ -8043,14 +8017,6 @@ argparse@^1.0.7, argparse@~1.0.9: dependencies: sprintf-js "~1.0.2" -argv-tools@^0.1.1: - version "0.1.2" - resolved "https://registry.yarnpkg.com/argv-tools/-/argv-tools-0.1.2.tgz#fc4918a70775b8cc5f8296fa0cfea137bd8a8229" - integrity sha512-wxqoymY0BEu9NblZVQiOTOAiJUjPhaa/kbNMjC2h6bnrmUSgnxKgWJo3lzXvi3bHJRwXyqK/dHzMlZVRT89Cxg== - dependencies: - array-back "^2.0.0" - find-replace "^2.0.1" - aria-query@^4.2.2: version "4.2.2" resolved "https://registry.yarnpkg.com/aria-query/-/aria-query-4.2.2.tgz#0d2ca6c9aceb56b8977e9fed6aed7e15bbd2f83b" @@ -8074,13 +8040,6 @@ arr-union@^3.1.0: resolved "https://registry.yarnpkg.com/arr-union/-/arr-union-3.1.0.tgz#e39b09aea9def866a8f206e288af63919bae39c4" integrity sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ= -array-back@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/array-back/-/array-back-2.0.0.tgz#6877471d51ecc9c9bfa6136fb6c7d5fe69748022" - integrity sha512-eJv4pLLufP3g5kcZry0j6WXpIbzYw9GUB4mVJZno9wfwiBxbizTnHCw3VJb07cBihbFX48Y7oSrW9y+gt4glyw== - dependencies: - typical "^2.6.1" - array-differ@^2.0.3: version "2.1.0" resolved "https://registry.yarnpkg.com/array-differ/-/array-differ-2.1.0.tgz#4b9c1c3f14b906757082925769e8ab904f4801b1" @@ -10093,27 +10052,6 @@ command-exists@^1.2.8: resolved "https://registry.yarnpkg.com/command-exists/-/command-exists-1.2.8.tgz#715acefdd1223b9c9b37110a149c6392c2852291" integrity sha512-PM54PkseWbiiD/mMsbvW351/u+dafwTJ0ye2qB60G1aGQP9j3xK2gmMDc+R34L3nDtx4qMCitXT75mkbkGJDLw== -command-line-args@5.0.2: - version "5.0.2" - resolved "https://registry.yarnpkg.com/command-line-args/-/command-line-args-5.0.2.tgz#c4e56b016636af1323cf485aa25c3cb203dfbbe4" - integrity sha512-/qPcbL8zpqg53x4rAaqMFlRV4opN3pbla7I7k9x8kyOBMQoGT6WltjN6sXZuxOXw6DgdK7Ad+ijYS5gjcr7vlA== - dependencies: - argv-tools "^0.1.1" - array-back "^2.0.0" - find-replace "^2.0.1" - lodash.camelcase "^4.3.0" - typical "^2.6.1" - -command-line-usage@5.0.5: - version "5.0.5" - resolved "https://registry.yarnpkg.com/command-line-usage/-/command-line-usage-5.0.5.tgz#5f25933ffe6dedd983c635d38a21d7e623fda357" - integrity sha512-d8NrGylA5oCXSbGoKz05FkehDAzSmIm4K03S5VDh4d5lZAtTWfc3D1RuETtuQCn8129nYfJfDdF7P/lwcz1BlA== - dependencies: - array-back "^2.0.0" - chalk "^2.4.1" - table-layout "^0.4.3" - typical "^2.6.1" - commander@2, commander@^2.18.0, commander@^2.19.0, commander@^2.20.0, commander@^2.7.1, commander@^2.8.1, commander@^2.9.0: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" @@ -11561,7 +11499,7 @@ deep-equal@^1.0.1: object-keys "^1.1.1" regexp.prototype.flags "^1.2.0" -deep-extend@^0.6.0, deep-extend@~0.6.0: +deep-extend@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac" integrity sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA== @@ -13589,14 +13527,6 @@ find-cache-dir@^3.3.1: make-dir "^3.0.2" pkg-dir "^4.1.0" -find-replace@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/find-replace/-/find-replace-2.0.1.tgz#6d9683a7ca20f8f9aabeabad07e4e2580f528550" - integrity sha512-LzDo3Fpa30FLIBsh6DCDnMN1KW2g4QKkqKmejlImgWY67dDFPX/x9Kh/op/GK522DchQXEvDi/wD48HKW49XOQ== - dependencies: - array-back "^2.0.0" - test-value "^3.0.0" - find-root@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/find-root/-/find-root-1.1.0.tgz#abcfc8ba76f708c42a97b3d685b7e9450bfb9ce4" @@ -13674,11 +13604,6 @@ flat@^4.1.0: dependencies: is-buffer "~2.0.3" -flatbuffers@1.11.0: - version "1.11.0" - resolved "https://registry.yarnpkg.com/flatbuffers/-/flatbuffers-1.11.0.tgz#90a47e584dd7851ad7a913f5a0ee99c1d76ce59f" - integrity sha512-0PqFKtXI4MjxomI7jO4g5XfLPm/15g2R+5WGCHBGYGh0ihQiypnHlJ6bMmkkrAe0GzZ4d7PDAfCONKIPUxNF+A== - flatted@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.1.1.tgz#c4b489e80096d9df1dfc97c79871aea7c617c469" @@ -16954,11 +16879,6 @@ jsesc@^2.5.1: resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== -json-bignum@^0.0.3: - version "0.0.3" - resolved "https://registry.yarnpkg.com/json-bignum/-/json-bignum-0.0.3.tgz#41163b50436c773d82424dbc20ed70db7604b8d7" - integrity sha1-QRY7UENsdz2CQk28IO1w23YEuNc= - json-markup@^1.1.0: version "1.1.3" resolved "https://registry.yarnpkg.com/json-markup/-/json-markup-1.1.3.tgz#0a3531ee7a531cef6e22564ca166987a4d22b56e" @@ -17472,11 +17392,6 @@ lodash._reinterpolate@^3.0.0: resolved "https://registry.yarnpkg.com/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz#0ccf2d89166af03b3663c796538b75ac6e114d9d" integrity sha1-DM8tiRZq8Ds2Y8eWU4t1rG4RTZ0= -lodash.camelcase@^4.3.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6" - integrity sha1-soqmKIorn8ZRA1x3EfZathkDMaY= - lodash.capitalize@^4.1.0: version "4.2.1" resolved "https://registry.yarnpkg.com/lodash.capitalize/-/lodash.capitalize-4.2.1.tgz#f826c9b4e2a8511d84e3aca29db05e1a4f3b72a9" @@ -17556,11 +17471,6 @@ lodash.once@^4.1.1: resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac" integrity sha1-DdOXEhPHxW34gJd9UEyI+0cal6w= -lodash.padend@^4.6.1: - version "4.6.1" - resolved "https://registry.yarnpkg.com/lodash.padend/-/lodash.padend-4.6.1.tgz#53ccba047d06e158d311f45da625f4e49e6f166e" - integrity sha1-U8y6BH0G4VjTEfRdpiX05J5vFm4= - lodash.set@^4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/lodash.set/-/lodash.set-4.3.2.tgz#d8757b1da807dde24816b0d6a84bea1a76230b23" @@ -19620,13 +19530,6 @@ package-json@^4.0.0: registry-url "^3.0.3" semver "^5.1.0" -pad-left@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/pad-left/-/pad-left-2.1.0.tgz#16e6a3b2d44a8e138cb0838cc7cb403a4fc9e994" - integrity sha1-FuajstRKjhOMsIOMx8tAOk/J6ZQ= - dependencies: - repeat-string "^1.5.4" - pako@^0.2.6: version "0.2.9" resolved "https://registry.yarnpkg.com/pako/-/pako-0.2.9.tgz#f3f7522f4ef782348da8161bad9ecfd51bf83a75" @@ -22409,11 +22312,6 @@ redent@^3.0.0: indent-string "^4.0.0" strip-indent "^3.0.0" -reduce-flatten@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/reduce-flatten/-/reduce-flatten-1.0.1.tgz#258c78efd153ddf93cb561237f61184f3696e327" - integrity sha1-JYx479FT3fk8tWEjf2EYTzaW4yc= - redux-logger@3.0.6: version "3.0.6" resolved "https://registry.yarnpkg.com/redux-logger/-/redux-logger-3.0.6.tgz#f7555966f3098f3c88604c449cf0baf5778274bf" @@ -24733,17 +24631,6 @@ systemjs@0.20.19: resolved "https://registry.yarnpkg.com/systemjs/-/systemjs-0.20.19.tgz#c2b9e79c19f4bea53a19b1ed3f974ffb463be949" integrity sha512-H/rKwNEEyej/+IhkmFNmKFyJul8tbH/muiPq5TyNoVTwsGhUjRsN3NlFnFQUvFXA3+GQmsXkCNXU6QKPl779aw== -table-layout@^0.4.3: - version "0.4.5" - resolved "https://registry.yarnpkg.com/table-layout/-/table-layout-0.4.5.tgz#d906de6a25fa09c0c90d1d08ecd833ecedcb7378" - integrity sha512-zTvf0mcggrGeTe/2jJ6ECkJHAQPIYEwDoqsiqBjI24mvRmQbInK5jq33fyypaCBxX08hMkfmdOqj6haT33EqWw== - dependencies: - array-back "^2.0.0" - deep-extend "~0.6.0" - lodash.padend "^4.6.1" - typical "^2.6.1" - wordwrapjs "^3.0.0" - table@^3.7.8: version "3.8.3" resolved "https://registry.yarnpkg.com/table/-/table-3.8.3.tgz#2bbc542f0fda9861a755d3947fefd8b3f513855f" @@ -24938,14 +24825,6 @@ test-exclude@^6.0.0: glob "^7.1.4" minimatch "^3.0.4" -test-value@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/test-value/-/test-value-3.0.0.tgz#9168c062fab11a86b8d444dd968bb4b73851ce92" - integrity sha512-sVACdAWcZkSU9x7AOmJo5TqE+GyNJknHaHsMrR6ZnhjVlVN9Yx6FjHrsKZ3BjIpPCT68zYesPWkakrNupwfOTQ== - dependencies: - array-back "^2.0.0" - typical "^2.6.1" - "tether-drop@https://github.com/torkelo/drop": version "1.5.0" resolved "https://github.com/torkelo/drop#fc83ca88db0076fbf6359cbe1743a9ef0f1ee6e1" @@ -24957,11 +24836,6 @@ tether@1.4.7, tether@^1.1.0: resolved "https://registry.yarnpkg.com/tether/-/tether-1.4.7.tgz#d56a818590d8fe72e387f77a67f93ab96d8e1fb2" integrity sha512-Z0J1aExjoFU8pybVkQAo/vD2wfSO63r+XOPfWQMC5qtf1bI7IWqNk4MiyBcgvvnY8kqnY06dVdvwTK2S3PU/Fw== -text-encoding-utf-8@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/text-encoding-utf-8/-/text-encoding-utf-8-1.0.2.tgz#585b62197b0ae437e3c7b5d0af27ac1021e10d13" - integrity sha512-8bw4MY9WjdsD2aMtO0OzOCY3pXGYNx2d2FfHRVUKkiCPDWjKuOlhLVASS+pD7VkLTVjW268LYJHwsnPFlBpbAg== - text-extensions@^1.0.0: version "1.9.0" resolved "https://registry.yarnpkg.com/text-extensions/-/text-extensions-1.9.0.tgz#1853e45fee39c945ce6f6c36b2d659b5aabc2a26" @@ -25475,11 +25349,6 @@ typescript@~3.9.7: resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.7.tgz#98d600a5ebdc38f40cb277522f12dc800e9e25fa" integrity sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw== -typical@^2.6.1: - version "2.6.1" - resolved "https://registry.yarnpkg.com/typical/-/typical-2.6.1.tgz#5c080e5d661cbbe38259d2e70a3c7253e873881d" - integrity sha1-XAgOXWYcu+OCWdLnCjxyU+hziB0= - ua-parser-js@^0.7.18: version "0.7.20" resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.20.tgz#7527178b82f6a62a0f243d1f94fd30e3e3c21098" @@ -26496,14 +26365,6 @@ wordwrap@^1.0.0, wordwrap@~1.0.0: resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" integrity sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus= -wordwrapjs@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/wordwrapjs/-/wordwrapjs-3.0.0.tgz#c94c372894cadc6feb1a66bff64e1d9af92c5d1e" - integrity sha512-mO8XtqyPvykVCsrwj5MlOVWvSnCdT+C+QVbm6blradR7JExAhbkZ7hZ9A+9NUtwzSqrlUo9a67ws0EiILrvRpw== - dependencies: - reduce-flatten "^1.0.1" - typical "^2.6.1" - worker-farm@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/worker-farm/-/worker-farm-1.7.0.tgz#26a94c5391bbca926152002f69b84a4bf772e5a8"