Panel edit: Remove use of SheetJS (#100161)

Remove SheetJS

package.json

@@ -416,8 +416,7 @@
     "uplot": "1.6.31",
     "uuid": "11.0.5",
     "visjs-network": "4.25.0",
-    "whatwg-fetch": "3.6.20",
-    "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz"
+    "whatwg-fetch": "3.6.20"
   },
   "resolutions": {
     "underscore": "1.13.7",

public/app/core/utils/sheet.test.ts

@@ -1,192 +0,0 @@
-import { utils } from 'xlsx';
-
-import { DataFrame } from '@grafana/data';
-
-import { workSheetToFrame } from './sheet';
-
-describe('sheets', () => {
-  it('should handle an empty sheet', () => {
-    const emptySheet = utils.aoa_to_sheet([]);
-    const frame = workSheetToFrame(emptySheet);
-
-    expect(frame.name).toBeUndefined();
-    expect(frame.fields).toHaveLength(0);
-    expect(frame.length).toBe(0);
-  });
-
-  it('will use first row as names', () => {
-    const sheet = utils.aoa_to_sheet([
-      ['Number', 'String', 'Bool', 'Date', 'Object'],
-      [1, 'A', true, Date.UTC(2020, 1, 1), { hello: 'world' }],
-      [2, 'B', false, Date.UTC(2020, 1, 2), { hello: 'world' }],
-    ]);
-    const frame = workSheetToFrame(sheet);
-
-    expect(toSnapshotFrame(frame)).toMatchInlineSnapshot(`
-      [
-        {
-          "name": "Number",
-          "type": "number",
-          "values": [
-            1,
-            2,
-          ],
-        },
-        {
-          "name": "String",
-          "type": "string",
-          "values": [
-            "A",
-            "B",
-          ],
-        },
-        {
-          "name": "Bool",
-          "type": "boolean",
-          "values": [
-            true,
-            false,
-          ],
-        },
-        {
-          "name": "Date",
-          "type": "number",
-          "values": [
-            1580515200000,
-            1580601600000,
-          ],
-        },
-        {
-          "name": "Object",
-          "type": "string",
-          "values": [
-            undefined,
-            undefined,
-          ],
-        },
-      ]
-    `);
-  });
-
-  it('will use calculated data when cells are typed', () => {
-    const sheet = utils.aoa_to_sheet([
-      [1, 'A', true, Date.UTC(2020, 1, 1), { hello: 'world' }],
-      [2, 'B', false, Date.UTC(2020, 1, 2), { hello: 'world' }],
-      [3, 'C', true, Date.UTC(2020, 1, 3), { hello: 'world' }],
-    ]);
-    const frame = workSheetToFrame(sheet);
-
-    expect(toSnapshotFrame(frame)).toMatchInlineSnapshot(`
-      [
-        {
-          "name": "A",
-          "type": "number",
-          "values": [
-            1,
-            2,
-            3,
-          ],
-        },
-        {
-          "name": "B",
-          "type": "string",
-          "values": [
-            "A",
-            "B",
-            "C",
-          ],
-        },
-        {
-          "name": "C",
-          "type": "boolean",
-          "values": [
-            true,
-            false,
-            true,
-          ],
-        },
-        {
-          "name": "D",
-          "type": "number",
-          "values": [
-            1580515200000,
-            1580601600000,
-            1580688000000,
-          ],
-        },
-        {
-          "name": "E",
-          "type": "string",
-          "values": [
-            undefined,
-            undefined,
-            undefined,
-          ],
-        },
-      ]
-    `);
-  });
-
-  it('is OK with nulls and undefineds, and misalignment', () => {
-    const sheet = utils.aoa_to_sheet([
-      [null, 'A', true],
-      [2, 'B', null, Date.UTC(2020, 1, 2), { hello: 'world' }],
-      [3, 'C', true, undefined, { hello: 'world' }],
-    ]);
-    const frame = workSheetToFrame(sheet);
-
-    expect(toSnapshotFrame(frame)).toMatchInlineSnapshot(`
-      [
-        {
-          "name": "A",
-          "type": "number",
-          "values": [
-            undefined,
-            2,
-            3,
-          ],
-        },
-        {
-          "name": "B",
-          "type": "string",
-          "values": [
-            "A",
-            "B",
-            "C",
-          ],
-        },
-        {
-          "name": "C",
-          "type": "boolean",
-          "values": [
-            true,
-            undefined,
-            true,
-          ],
-        },
-        {
-          "name": "D",
-          "type": "number",
-          "values": [
-            undefined,
-            1580601600000,
-            undefined,
-          ],
-        },
-        {
-          "name": "E",
-          "type": "string",
-          "values": [
-            undefined,
-            undefined,
-            undefined,
-          ],
-        },
-      ]
-    `);
-  });
-});
-
-function toSnapshotFrame(frame: DataFrame) {
-  return frame.fields.map((f) => ({ name: f.name, values: f.values, type: f.type }));
-}

public/app/core/utils/sheet.ts

@@ -1,172 +0,0 @@
-import { read, utils, WorkSheet, WorkBook, Range, ColInfo, CellObject, ExcelDataType } from 'xlsx';
-
-import { DataFrame, FieldType } from '@grafana/data';
-
-export function readSpreadsheet(file: ArrayBuffer): DataFrame[] {
-  return workBookToFrames(read(file, { type: 'buffer' }));
-}
-
-export function workBookToFrames(wb: WorkBook): DataFrame[] {
-  return wb.SheetNames.map((name) => workSheetToFrame(wb.Sheets[name], name));
-}
-
-export function workSheetToFrame(sheet: WorkSheet, name?: string): DataFrame {
-  const columns = sheetAsColumns(sheet);
-  if (!columns?.length) {
-    return {
-      fields: [],
-      name: name,
-      length: 0,
-    };
-  }
-
-  return {
-    fields: columns.map((c, idx) => {
-      let type = FieldType.string;
-      let values: unknown[] = [];
-      switch (c.type ?? 's') {
-        case 'b':
-          type = FieldType.boolean;
-          values = c.data.map((v) => (v?.v == null ? v?.v : Boolean(v.v)));
-          break;
-
-        case 'n':
-          type = FieldType.number;
-          values = c.data.map((v) => (v?.v == null ? v?.v : +v.v));
-          break;
-
-        case 'd':
-          type = FieldType.time;
-          values = c.data.map((v) => (v?.v == null ? v?.v : +v.v)); // ???
-          break;
-
-        default:
-          type = FieldType.string;
-          values = c.data.map((v) => (v?.v == null ? v?.v : utils.format_cell(v)));
-          break;
-      }
-
-      return {
-        name: c.name,
-        config: {}, // TODO? we could apply decimal formatting from worksheet
-        type,
-        values,
-      };
-    }),
-    name: name,
-    length: columns[0].data.length,
-  };
-}
-
-interface ColumnData {
-  index: number;
-  name: string;
-  info?: ColInfo;
-  data: CellObject[];
-  type?: ExcelDataType;
-}
-
-function sheetAsColumns(sheet: WorkSheet): ColumnData[] | null {
-  const r = sheet['!ref'];
-  if (!r) {
-    return null;
-  }
-  const columnInfo = sheet['!cols'];
-  const cols: ColumnData[] = [];
-  const range = safe_decode_range(r);
-  const types = new Set<ExcelDataType>();
-  let firstRowIsHeader = true;
-
-  for (let c = range.s.c; c <= range.e.c; ++c) {
-    types.clear();
-    const info = columnInfo?.[c] ?? {};
-    if (info.hidden) {
-      continue; // skip the column
-    }
-    const field: ColumnData = {
-      index: c,
-      name: utils.encode_col(c),
-      data: [],
-      info,
-    };
-    const pfix = utils.encode_col(c);
-    for (let r = range.s.r; r <= range.e.r; ++r) {
-      const cell = sheet[pfix + utils.encode_row(r)];
-      if (cell) {
-        if (field.data.length) {
-          types.add(cell.t);
-        } else if (cell.t !== 's') {
-          firstRowIsHeader = false;
-        }
-      }
-      field.data.push(cell);
-    }
-    cols.push(field);
-    if (types.size === 1) {
-      field.type = Array.from(types)[0];
-    }
-  }
-
-  if (firstRowIsHeader) {
-    return cols.map((c) => {
-      const first = c.data[0];
-      if (first?.v) {
-        c.name = utils.format_cell(first);
-      }
-      c.data = c.data.slice(1);
-      return c;
-    });
-  }
-  return cols;
-}
-
-/**
- * Copied from Apache 2 licensed sheetjs:
- * https://git.sheetjs.com/sheetjs/sheetjs/src/branch/master/xlsx.flow.js#L4338
- */
-function safe_decode_range(range: string): Range {
-  let o = { s: { c: 0, r: 0 }, e: { c: 0, r: 0 } };
-  let idx = 0,
-    i = 0,
-    cc = 0;
-  let len = range.length;
-  for (idx = 0; i < len; ++i) {
-    if ((cc = range.charCodeAt(i) - 64) < 1 || cc > 26) {
-      break;
-    }
-    idx = 26 * idx + cc;
-  }
-  o.s.c = --idx;
-
-  for (idx = 0; i < len; ++i) {
-    if ((cc = range.charCodeAt(i) - 48) < 0 || cc > 9) {
-      break;
-    }
-    idx = 10 * idx + cc;
-  }
-  o.s.r = --idx;
-
-  if (i === len || cc !== 10) {
-    o.e.c = o.s.c;
-    o.e.r = o.s.r;
-    return o;
-  }
-  ++i;
-
-  for (idx = 0; i !== len; ++i) {
-    if ((cc = range.charCodeAt(i) - 64) < 1 || cc > 26) {
-      break;
-    }
-    idx = 26 * idx + cc;
-  }
-  o.e.c = --idx;
-
-  for (idx = 0; i !== len; ++i) {
-    if ((cc = range.charCodeAt(i) - 48) < 0 || cc > 9) {
-      break;
-    }
-    idx = 10 * idx + cc;
-  }
-  o.e.r = --idx;
-  return o;
-}

public/app/features/dataframe-import/constants.ts

@@ -2,10 +2,6 @@ import { Accept } from 'react-dropzone';
 
 export const acceptedFiles: Accept = {
   'text/plain': ['.csv', '.txt'],
-  'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': ['.xlsx'],
-  'application/vnd.ms-excel': ['.xls'],
-  'application/vnd.apple.numbers': ['.numbers'],
-  'application/vnd.oasis.opendocument.spreadsheet': ['.ods'],
   'application/json': ['.json'],
 };
 

public/app/features/dataframe-import/utils.test.ts

@@ -1,39 +0,0 @@
-import { formatFileTypes } from './utils';
-
-describe('Dataframe import / Utils', () => {
-  describe('formatFileTypes', () => {
-    it('should nicely format file extensions', () => {
-      expect(
-        formatFileTypes({
-          'text/plain': ['.csv', '.txt'],
-          'application/json': ['.json'],
-        })
-      ).toBe('.csv, .txt or .json');
-    });
-
-    it('should remove duplicates', () => {
-      expect(
-        formatFileTypes({
-          'text/plain': ['.csv', '.txt'],
-          'application/json': ['.json', '.txt'],
-        })
-      ).toBe('.csv, .txt or .json');
-    });
-
-    it('should nicely format a single file type extension', () => {
-      expect(
-        formatFileTypes({
-          'text/plain': ['.txt'],
-        })
-      ).toBe('.txt');
-    });
-
-    it('should nicely format two file type extension', () => {
-      expect(
-        formatFileTypes({
-          'text/plain': ['.txt', '.csv'],
-        })
-      ).toBe('.txt or .csv');
-    });
-  });
-});

public/app/features/dataframe-import/utils.ts

@@ -1,45 +1,25 @@
-import { Accept } from 'react-dropzone';
 import { Observable } from 'rxjs';
 
-import { toDataFrame } from '@grafana/data';
+import { readCSV, toDataFrame } from '@grafana/data';
 
 import { FileImportResult } from './types';
 
-function getFileExtensions(acceptedFiles: Accept) {
-  const fileExtentions = new Set<string>();
-  Object.keys(acceptedFiles).forEach((v) => {
-    acceptedFiles[v].forEach((extension) => {
-      fileExtentions.add(extension);
-    });
-  });
-  return fileExtentions;
-}
-
-export function formatFileTypes(acceptedFiles: Accept) {
-  const fileExtentions = Array.from(getFileExtensions(acceptedFiles));
-  if (fileExtentions.length === 1) {
-    return fileExtentions[0];
-  }
-  return `${fileExtentions.slice(0, -1).join(', ')} or ${fileExtentions.slice(-1)}`;
-}
-
 export function filesToDataframes(files: File[]): Observable<FileImportResult> {
   return new Observable<FileImportResult>((subscriber) => {
     let completedFiles = 0;
-    import('app/core/utils/sheet')
-      .then((sheet) => {
     files.forEach((file) => {
       const reader = new FileReader();
       reader.readAsArrayBuffer(file);
       reader.onload = () => {
         const result = reader.result;
         if (result && result instanceof ArrayBuffer) {
-              if (file.type === 'application/json') {
           const decoder = new TextDecoder('utf-8');
-                const json = JSON.parse(decoder.decode(result));
+          const fileString = decoder.decode(result);
+          if (file.type === 'application/json') {
+            const json = JSON.parse(fileString);
             subscriber.next({ dataFrames: [toDataFrame(json)], file: file });
           } else {
-                subscriber.next({ dataFrames: sheet.readSpreadsheet(result), file: file });
+            subscriber.next({ dataFrames: readCSV(fileString), file: file });
           }
           if (++completedFiles >= files.length) {
             subscriber.complete();
@@ -47,9 +27,5 @@ export function filesToDataframes(files: File[]): Observable<FileImportResult> {
         }
       };
     });
-      })
-      .catch(() => {
-        throw 'Failed to load sheets module';
-      });
   });
 }

yarn.lock

@@ -18096,7 +18096,6 @@ __metadata:
     webpack-merge: "npm:6.0.1"
     webpackbar: "npm:^7.0.0"
     whatwg-fetch: "npm:3.6.20"
-    xlsx: "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz"
     yaml: "npm:^2.0.0"
     yargs: "npm:^17.5.1"
   dependenciesMeta:
@@ -31772,15 +31771,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"xlsx@https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz":
-  version: 0.20.2
-  resolution: "xlsx@https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz"
-  bin:
-    xlsx: ./bin/xlsx.njs
-  checksum: 10/2d8e0644888f90fa9145ea74ed90b844154ce89c4f0e4f92fcce3f224fa71654da99aa48d99d65ba86eb0632a4858ba2dea7eef8b54fd8bd23954a09d1884aa1
-  languageName: node
-  linkType: hard
-
 "xml-but-prettier@npm:^1.0.1":
   version: 1.0.1
   resolution: "xml-but-prettier@npm:1.0.1"