diff --git a/package.json b/package.json index 5d19e8cf0d2..555fb46d6a8 100644 --- a/package.json +++ b/package.json @@ -416,8 +416,7 @@ "uplot": "1.6.31", "uuid": "11.0.5", "visjs-network": "4.25.0", - "whatwg-fetch": "3.6.20", - "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz" + "whatwg-fetch": "3.6.20" }, "resolutions": { "underscore": "1.13.7", diff --git a/public/app/core/utils/sheet.test.ts b/public/app/core/utils/sheet.test.ts deleted file mode 100644 index 139dc02dc1a..00000000000 --- a/public/app/core/utils/sheet.test.ts +++ /dev/null @@ -1,192 +0,0 @@ -import { utils } from 'xlsx'; - -import { DataFrame } from '@grafana/data'; - -import { workSheetToFrame } from './sheet'; - -describe('sheets', () => { - it('should handle an empty sheet', () => { - const emptySheet = utils.aoa_to_sheet([]); - const frame = workSheetToFrame(emptySheet); - - expect(frame.name).toBeUndefined(); - expect(frame.fields).toHaveLength(0); - expect(frame.length).toBe(0); - }); - - it('will use first row as names', () => { - const sheet = utils.aoa_to_sheet([ - ['Number', 'String', 'Bool', 'Date', 'Object'], - [1, 'A', true, Date.UTC(2020, 1, 1), { hello: 'world' }], - [2, 'B', false, Date.UTC(2020, 1, 2), { hello: 'world' }], - ]); - const frame = workSheetToFrame(sheet); - - expect(toSnapshotFrame(frame)).toMatchInlineSnapshot(` - [ - { - "name": "Number", - "type": "number", - "values": [ - 1, - 2, - ], - }, - { - "name": "String", - "type": "string", - "values": [ - "A", - "B", - ], - }, - { - "name": "Bool", - "type": "boolean", - "values": [ - true, - false, - ], - }, - { - "name": "Date", - "type": "number", - "values": [ - 1580515200000, - 1580601600000, - ], - }, - { - "name": "Object", - "type": "string", - "values": [ - undefined, - undefined, - ], - }, - ] - `); - }); - - it('will use calculated data when cells are typed', () => { - const sheet = utils.aoa_to_sheet([ - [1, 'A', true, Date.UTC(2020, 1, 1), { hello: 'world' }], - [2, 'B', false, Date.UTC(2020, 1, 2), { hello: 'world' }], - [3, 'C', true, Date.UTC(2020, 1, 3), { hello: 'world' }], - ]); - const frame = workSheetToFrame(sheet); - - expect(toSnapshotFrame(frame)).toMatchInlineSnapshot(` - [ - { - "name": "A", - "type": "number", - "values": [ - 1, - 2, - 3, - ], - }, - { - "name": "B", - "type": "string", - "values": [ - "A", - "B", - "C", - ], - }, - { - "name": "C", - "type": "boolean", - "values": [ - true, - false, - true, - ], - }, - { - "name": "D", - "type": "number", - "values": [ - 1580515200000, - 1580601600000, - 1580688000000, - ], - }, - { - "name": "E", - "type": "string", - "values": [ - undefined, - undefined, - undefined, - ], - }, - ] - `); - }); - - it('is OK with nulls and undefineds, and misalignment', () => { - const sheet = utils.aoa_to_sheet([ - [null, 'A', true], - [2, 'B', null, Date.UTC(2020, 1, 2), { hello: 'world' }], - [3, 'C', true, undefined, { hello: 'world' }], - ]); - const frame = workSheetToFrame(sheet); - - expect(toSnapshotFrame(frame)).toMatchInlineSnapshot(` - [ - { - "name": "A", - "type": "number", - "values": [ - undefined, - 2, - 3, - ], - }, - { - "name": "B", - "type": "string", - "values": [ - "A", - "B", - "C", - ], - }, - { - "name": "C", - "type": "boolean", - "values": [ - true, - undefined, - true, - ], - }, - { - "name": "D", - "type": "number", - "values": [ - undefined, - 1580601600000, - undefined, - ], - }, - { - "name": "E", - "type": "string", - "values": [ - undefined, - undefined, - undefined, - ], - }, - ] - `); - }); -}); - -function toSnapshotFrame(frame: DataFrame) { - return frame.fields.map((f) => ({ name: f.name, values: f.values, type: f.type })); -} diff --git a/public/app/core/utils/sheet.ts b/public/app/core/utils/sheet.ts deleted file mode 100644 index a97d31c2b33..00000000000 --- a/public/app/core/utils/sheet.ts +++ /dev/null @@ -1,172 +0,0 @@ -import { read, utils, WorkSheet, WorkBook, Range, ColInfo, CellObject, ExcelDataType } from 'xlsx'; - -import { DataFrame, FieldType } from '@grafana/data'; - -export function readSpreadsheet(file: ArrayBuffer): DataFrame[] { - return workBookToFrames(read(file, { type: 'buffer' })); -} - -export function workBookToFrames(wb: WorkBook): DataFrame[] { - return wb.SheetNames.map((name) => workSheetToFrame(wb.Sheets[name], name)); -} - -export function workSheetToFrame(sheet: WorkSheet, name?: string): DataFrame { - const columns = sheetAsColumns(sheet); - if (!columns?.length) { - return { - fields: [], - name: name, - length: 0, - }; - } - - return { - fields: columns.map((c, idx) => { - let type = FieldType.string; - let values: unknown[] = []; - switch (c.type ?? 's') { - case 'b': - type = FieldType.boolean; - values = c.data.map((v) => (v?.v == null ? v?.v : Boolean(v.v))); - break; - - case 'n': - type = FieldType.number; - values = c.data.map((v) => (v?.v == null ? v?.v : +v.v)); - break; - - case 'd': - type = FieldType.time; - values = c.data.map((v) => (v?.v == null ? v?.v : +v.v)); // ??? - break; - - default: - type = FieldType.string; - values = c.data.map((v) => (v?.v == null ? v?.v : utils.format_cell(v))); - break; - } - - return { - name: c.name, - config: {}, // TODO? we could apply decimal formatting from worksheet - type, - values, - }; - }), - name: name, - length: columns[0].data.length, - }; -} - -interface ColumnData { - index: number; - name: string; - info?: ColInfo; - data: CellObject[]; - type?: ExcelDataType; -} - -function sheetAsColumns(sheet: WorkSheet): ColumnData[] | null { - const r = sheet['!ref']; - if (!r) { - return null; - } - const columnInfo = sheet['!cols']; - const cols: ColumnData[] = []; - const range = safe_decode_range(r); - const types = new Set(); - let firstRowIsHeader = true; - - for (let c = range.s.c; c <= range.e.c; ++c) { - types.clear(); - const info = columnInfo?.[c] ?? {}; - if (info.hidden) { - continue; // skip the column - } - const field: ColumnData = { - index: c, - name: utils.encode_col(c), - data: [], - info, - }; - const pfix = utils.encode_col(c); - for (let r = range.s.r; r <= range.e.r; ++r) { - const cell = sheet[pfix + utils.encode_row(r)]; - if (cell) { - if (field.data.length) { - types.add(cell.t); - } else if (cell.t !== 's') { - firstRowIsHeader = false; - } - } - field.data.push(cell); - } - cols.push(field); - if (types.size === 1) { - field.type = Array.from(types)[0]; - } - } - - if (firstRowIsHeader) { - return cols.map((c) => { - const first = c.data[0]; - if (first?.v) { - c.name = utils.format_cell(first); - } - c.data = c.data.slice(1); - return c; - }); - } - return cols; -} - -/** - * Copied from Apache 2 licensed sheetjs: - * https://git.sheetjs.com/sheetjs/sheetjs/src/branch/master/xlsx.flow.js#L4338 - */ -function safe_decode_range(range: string): Range { - let o = { s: { c: 0, r: 0 }, e: { c: 0, r: 0 } }; - let idx = 0, - i = 0, - cc = 0; - let len = range.length; - for (idx = 0; i < len; ++i) { - if ((cc = range.charCodeAt(i) - 64) < 1 || cc > 26) { - break; - } - idx = 26 * idx + cc; - } - o.s.c = --idx; - - for (idx = 0; i < len; ++i) { - if ((cc = range.charCodeAt(i) - 48) < 0 || cc > 9) { - break; - } - idx = 10 * idx + cc; - } - o.s.r = --idx; - - if (i === len || cc !== 10) { - o.e.c = o.s.c; - o.e.r = o.s.r; - return o; - } - ++i; - - for (idx = 0; i !== len; ++i) { - if ((cc = range.charCodeAt(i) - 64) < 1 || cc > 26) { - break; - } - idx = 26 * idx + cc; - } - o.e.c = --idx; - - for (idx = 0; i !== len; ++i) { - if ((cc = range.charCodeAt(i) - 48) < 0 || cc > 9) { - break; - } - idx = 10 * idx + cc; - } - o.e.r = --idx; - return o; -} diff --git a/public/app/features/dataframe-import/constants.ts b/public/app/features/dataframe-import/constants.ts index e32027a5e46..9753ce71f08 100644 --- a/public/app/features/dataframe-import/constants.ts +++ b/public/app/features/dataframe-import/constants.ts @@ -2,10 +2,6 @@ import { Accept } from 'react-dropzone'; export const acceptedFiles: Accept = { 'text/plain': ['.csv', '.txt'], - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': ['.xlsx'], - 'application/vnd.ms-excel': ['.xls'], - 'application/vnd.apple.numbers': ['.numbers'], - 'application/vnd.oasis.opendocument.spreadsheet': ['.ods'], 'application/json': ['.json'], }; diff --git a/public/app/features/dataframe-import/utils.test.ts b/public/app/features/dataframe-import/utils.test.ts deleted file mode 100644 index 20631933136..00000000000 --- a/public/app/features/dataframe-import/utils.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { formatFileTypes } from './utils'; - -describe('Dataframe import / Utils', () => { - describe('formatFileTypes', () => { - it('should nicely format file extensions', () => { - expect( - formatFileTypes({ - 'text/plain': ['.csv', '.txt'], - 'application/json': ['.json'], - }) - ).toBe('.csv, .txt or .json'); - }); - - it('should remove duplicates', () => { - expect( - formatFileTypes({ - 'text/plain': ['.csv', '.txt'], - 'application/json': ['.json', '.txt'], - }) - ).toBe('.csv, .txt or .json'); - }); - - it('should nicely format a single file type extension', () => { - expect( - formatFileTypes({ - 'text/plain': ['.txt'], - }) - ).toBe('.txt'); - }); - - it('should nicely format two file type extension', () => { - expect( - formatFileTypes({ - 'text/plain': ['.txt', '.csv'], - }) - ).toBe('.txt or .csv'); - }); - }); -}); diff --git a/public/app/features/dataframe-import/utils.ts b/public/app/features/dataframe-import/utils.ts index 83bb58383a5..c190cd1d538 100644 --- a/public/app/features/dataframe-import/utils.ts +++ b/public/app/features/dataframe-import/utils.ts @@ -1,55 +1,31 @@ -import { Accept } from 'react-dropzone'; import { Observable } from 'rxjs'; -import { toDataFrame } from '@grafana/data'; +import { readCSV, toDataFrame } from '@grafana/data'; import { FileImportResult } from './types'; -function getFileExtensions(acceptedFiles: Accept) { - const fileExtentions = new Set(); - Object.keys(acceptedFiles).forEach((v) => { - acceptedFiles[v].forEach((extension) => { - fileExtentions.add(extension); - }); - }); - return fileExtentions; -} - -export function formatFileTypes(acceptedFiles: Accept) { - const fileExtentions = Array.from(getFileExtensions(acceptedFiles)); - if (fileExtentions.length === 1) { - return fileExtentions[0]; - } - return `${fileExtentions.slice(0, -1).join(', ')} or ${fileExtentions.slice(-1)}`; -} - export function filesToDataframes(files: File[]): Observable { return new Observable((subscriber) => { let completedFiles = 0; - import('app/core/utils/sheet') - .then((sheet) => { - files.forEach((file) => { - const reader = new FileReader(); - reader.readAsArrayBuffer(file); - reader.onload = () => { - const result = reader.result; - if (result && result instanceof ArrayBuffer) { - if (file.type === 'application/json') { - const decoder = new TextDecoder('utf-8'); - const json = JSON.parse(decoder.decode(result)); - subscriber.next({ dataFrames: [toDataFrame(json)], file: file }); - } else { - subscriber.next({ dataFrames: sheet.readSpreadsheet(result), file: file }); - } - if (++completedFiles >= files.length) { - subscriber.complete(); - } - } - }; - }); - }) - .catch(() => { - throw 'Failed to load sheets module'; - }); + files.forEach((file) => { + const reader = new FileReader(); + reader.readAsArrayBuffer(file); + reader.onload = () => { + const result = reader.result; + if (result && result instanceof ArrayBuffer) { + const decoder = new TextDecoder('utf-8'); + const fileString = decoder.decode(result); + if (file.type === 'application/json') { + const json = JSON.parse(fileString); + subscriber.next({ dataFrames: [toDataFrame(json)], file: file }); + } else { + subscriber.next({ dataFrames: readCSV(fileString), file: file }); + } + if (++completedFiles >= files.length) { + subscriber.complete(); + } + } + }; + }); }); } diff --git a/yarn.lock b/yarn.lock index 1c4fee5ed1d..2ebb6dc6940 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18096,7 +18096,6 @@ __metadata: webpack-merge: "npm:6.0.1" webpackbar: "npm:^7.0.0" whatwg-fetch: "npm:3.6.20" - xlsx: "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz" yaml: "npm:^2.0.0" yargs: "npm:^17.5.1" dependenciesMeta: @@ -31772,15 +31771,6 @@ __metadata: languageName: node linkType: hard -"xlsx@https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz": - version: 0.20.2 - resolution: "xlsx@https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz" - bin: - xlsx: ./bin/xlsx.njs - checksum: 10/2d8e0644888f90fa9145ea74ed90b844154ce89c4f0e4f92fcce3f224fa71654da99aa48d99d65ba86eb0632a4858ba2dea7eef8b54fd8bd23954a09d1884aa1 - languageName: node - linkType: hard - "xml-but-prettier@npm:^1.0.1": version: 1.0.1 resolution: "xml-but-prettier@npm:1.0.1"