diff --git a/public/app/core/specs/file_export.jest.ts b/public/app/core/specs/file_export.jest.ts index bbb894094ff..82097227b97 100644 --- a/public/app/core/specs/file_export.jest.ts +++ b/public/app/core/specs/file_export.jest.ts @@ -30,17 +30,17 @@ describe('file_export', () => { it('should export points in proper order', () => { let text = fileExport.convertSeriesListToCsv(ctx.seriesList, ctx.timeFormat); const expectedText = - 'Series;Time;Value\n' + - 'series_1;1500026100;1\n' + - 'series_1;1500026200;2\n' + - 'series_1;1500026300;null\n' + - 'series_1;1500026400;null\n' + - 'series_1;1500026500;null\n' + - 'series_1;1500026600;6\n' + - 'series_2;1500026100;11\n' + - 'series_2;1500026200;12\n' + - 'series_2;1500026300;13\n' + - 'series_2;1500026500;15\n'; + '"Series";"Time";"Value"\r\n' + + '"series_1";"1500026100";1\r\n' + + '"series_1";"1500026200";2\r\n' + + '"series_1";"1500026300";null\r\n' + + '"series_1";"1500026400";null\r\n' + + '"series_1";"1500026500";null\r\n' + + '"series_1";"1500026600";6\r\n' + + '"series_2";"1500026100";11\r\n' + + '"series_2";"1500026200";12\r\n' + + '"series_2";"1500026300";13\r\n' + + '"series_2";"1500026500";15'; expect(text).toBe(expectedText); }); @@ -50,15 +50,79 @@ describe('file_export', () => { it('should export points in proper order', () => { let text = fileExport.convertSeriesListToCsvColumns(ctx.seriesList, ctx.timeFormat); const expectedText = - 'Time;series_1;series_2\n' + - '1500026100;1;11\n' + - '1500026200;2;12\n' + - '1500026300;null;13\n' + - '1500026400;null;null\n' + - '1500026500;null;15\n' + - '1500026600;6;null\n'; + '"Time";"series_1";"series_2"\r\n' + + '"1500026100";1;11\r\n' + + '"1500026200";2;12\r\n' + + '"1500026300";null;13\r\n' + + '"1500026400";null;null\r\n' + + '"1500026500";null;15\r\n' + + '"1500026600";6;null'; expect(text).toBe(expectedText); }); }); + + describe('when exporting table data to csv', () => { + + it('should properly escape special characters and quote all string values', () => { + const inputTable = { + columns: [ + { title: 'integer_value' }, + { text: 'string_value' }, + { title: 'float_value' }, + { text: 'boolean_value' }, + ], + rows: [ + [123, 'some_string', 1.234, true], + [0o765, 'some string with " in the middle', 1e-2, false], + [0o765, 'some string with "" in the middle', 1e-2, false], + [0o765, 'some string with """ in the middle', 1e-2, false], + [0o765, '"some string with " at the beginning', 1e-2, false], + [0o765, 'some string with " at the end"', 1e-2, false], + [0x123, 'some string with \n in the middle', 10.01, false], + [0b1011, 'some string with ; in the middle', -12.34, true], + [123, 'some string with ;; in the middle', -12.34, true], + ], + }; + + const returnedText = fileExport.convertTableDataToCsv(inputTable, false); + + const expectedText = + '"integer_value";"string_value";"float_value";"boolean_value"\r\n' + + '123;"some_string";1.234;true\r\n' + + '501;"some string with "" in the middle";0.01;false\r\n' + + '501;"some string with """" in the middle";0.01;false\r\n' + + '501;"some string with """""" in the middle";0.01;false\r\n' + + '501;"""some string with "" at the beginning";0.01;false\r\n' + + '501;"some string with "" at the end""";0.01;false\r\n' + + '291;"some string with \n in the middle";10.01;false\r\n' + + '11;"some string with ; in the middle";-12.34;true\r\n' + + '123;"some string with ;; in the middle";-12.34;true'; + + expect(returnedText).toBe(expectedText); + }); + + it('should decode HTML encoded characters', function() { + const inputTable = { + columns: [ + { text: 'string_value' }, + ], + rows: [ + ['"&ä'], + ['"some html"'], + ['some text'] + ], + }; + + const returnedText = fileExport.convertTableDataToCsv(inputTable, false); + + const expectedText = + '"string_value"\r\n' + + '"""&รค"\r\n' + + '"""some html"""\r\n' + + '"some text"'; + + expect(returnedText).toBe(expectedText); + }); + }); }); diff --git a/public/app/core/utils/file_export.ts b/public/app/core/utils/file_export.ts index 670326fc068..f25d340a0be 100644 --- a/public/app/core/utils/file_export.ts +++ b/public/app/core/utils/file_export.ts @@ -1,59 +1,108 @@ -import _ from 'lodash'; +import { isBoolean, isNumber, sortedUniq, sortedIndexOf, unescape as htmlUnescaped } from 'lodash'; import moment from 'moment'; import { saveAs } from 'file-saver'; +import { isNullOrUndefined } from 'util'; const DEFAULT_DATETIME_FORMAT = 'YYYY-MM-DDTHH:mm:ssZ'; const POINT_TIME_INDEX = 1; const POINT_VALUE_INDEX = 0; +const END_COLUMN = ';'; +const END_ROW = '\r\n'; +const QUOTE = '"'; +const EXPORT_FILENAME = 'grafana_data_export.csv'; + +function csvEscaped(text) { + if (!text) { + return text; + } + + return text.split(QUOTE).join(QUOTE + QUOTE); +} + +const domParser = new DOMParser(); +function htmlDecoded(text) { + if (!text) { + return text; + } + + const regexp = /&[^;]+;/g; + function htmlDecoded(value) { + const parsedDom = domParser.parseFromString(value, 'text/html'); + return parsedDom.body.textContent; + } + return text.replace(regexp, htmlDecoded).replace(regexp, htmlDecoded); +} + +function formatSpecialHeader(useExcelHeader) { + return useExcelHeader ? `sep=${END_COLUMN}${END_ROW}` : ''; +} + +function formatRow(row, addEndRowDelimiter = true) { + let text = ''; + for (let i = 0; i < row.length; i += 1) { + if (isBoolean(row[i]) || isNullOrUndefined(row[i])) { + text += row[i]; + } else if (isNumber(row[i])) { + text += row[i].toLocaleString(); + } else { + text += `${QUOTE}${csvEscaped(htmlUnescaped(htmlDecoded(row[i])))}${QUOTE}`; + } + + if (i < row.length - 1) { + text += END_COLUMN; + } + } + return addEndRowDelimiter ? text + END_ROW : text; +} + export function convertSeriesListToCsv(seriesList, dateTimeFormat = DEFAULT_DATETIME_FORMAT, excel = false) { - var text = (excel ? 'sep=;\n' : '') + 'Series;Time;Value\n'; - _.each(seriesList, function(series) { - _.each(series.datapoints, function(dp) { - text += - series.alias + ';' + moment(dp[POINT_TIME_INDEX]).format(dateTimeFormat) + ';' + dp[POINT_VALUE_INDEX] + '\n'; - }); - }); + let text = formatSpecialHeader(excel) + formatRow(['Series', 'Time', 'Value']); + for (let seriesIndex = 0; seriesIndex < seriesList.length; seriesIndex += 1) { + for (let i = 0; i < seriesList[seriesIndex].datapoints.length; i += 1) { + text += formatRow( + [ + seriesList[seriesIndex].alias, + moment(seriesList[seriesIndex].datapoints[i][POINT_TIME_INDEX]).format(dateTimeFormat), + seriesList[seriesIndex].datapoints[i][POINT_VALUE_INDEX], + ], + i < seriesList[seriesIndex].datapoints.length - 1 || seriesIndex < seriesList.length - 1 + ); + } + } return text; } export function exportSeriesListToCsv(seriesList, dateTimeFormat = DEFAULT_DATETIME_FORMAT, excel = false) { - var text = convertSeriesListToCsv(seriesList, dateTimeFormat, excel); - saveSaveBlob(text, 'grafana_data_export.csv'); + let text = convertSeriesListToCsv(seriesList, dateTimeFormat, excel); + saveSaveBlob(text, EXPORT_FILENAME); } export function convertSeriesListToCsvColumns(seriesList, dateTimeFormat = DEFAULT_DATETIME_FORMAT, excel = false) { - let text = (excel ? 'sep=;\n' : '') + 'Time;'; // add header - _.each(seriesList, function(series) { - text += series.alias + ';'; - }); - text = text.substring(0, text.length - 1); - text += '\n'; - + let text = + formatSpecialHeader(excel) + + formatRow( + ['Time'].concat( + seriesList.map(function(val) { + return val.alias; + }) + ) + ); // process data seriesList = mergeSeriesByTime(seriesList); - var dataArr = [[]]; - var sIndex = 1; - _.each(seriesList, function(series) { - var cIndex = 0; - dataArr.push([]); - _.each(series.datapoints, function(dp) { - dataArr[0][cIndex] = moment(dp[POINT_TIME_INDEX]).format(dateTimeFormat); - dataArr[sIndex][cIndex] = dp[POINT_VALUE_INDEX]; - cIndex++; - }); - sIndex++; - }); // make text - for (var i = 0; i < dataArr[0].length; i++) { - text += dataArr[0][i] + ';'; - for (var j = 1; j < dataArr.length; j++) { - text += dataArr[j][i] + ';'; - } - text = text.substring(0, text.length - 1); - text += '\n'; + for (let i = 0; i < seriesList[0].datapoints.length; i += 1) { + const timestamp = moment(seriesList[0].datapoints[i][POINT_TIME_INDEX]).format(dateTimeFormat); + text += formatRow( + [timestamp].concat( + seriesList.map(function(series) { + return series.datapoints[i][POINT_VALUE_INDEX]; + }) + ), + i < seriesList[0].datapoints.length - 1 + ); } return text; @@ -71,15 +120,15 @@ function mergeSeriesByTime(seriesList) { timestamps.push(seriesPoints[j][POINT_TIME_INDEX]); } } - timestamps = _.sortedUniq(timestamps.sort()); + timestamps = sortedUniq(timestamps.sort()); for (let i = 0; i < seriesList.length; i++) { let seriesPoints = seriesList[i].datapoints; - let seriesTimestamps = _.map(seriesPoints, p => p[POINT_TIME_INDEX]); + let seriesTimestamps = seriesPoints.map(p => p[POINT_TIME_INDEX]); let extendedSeries = []; let pointIndex; for (let j = 0; j < timestamps.length; j++) { - pointIndex = _.sortedIndexOf(seriesTimestamps, timestamps[j]); + pointIndex = sortedIndexOf(seriesTimestamps, timestamps[j]); if (pointIndex !== -1) { extendedSeries.push(seriesPoints[pointIndex]); } else { @@ -93,27 +142,26 @@ function mergeSeriesByTime(seriesList) { export function exportSeriesListToCsvColumns(seriesList, dateTimeFormat = DEFAULT_DATETIME_FORMAT, excel = false) { let text = convertSeriesListToCsvColumns(seriesList, dateTimeFormat, excel); - saveSaveBlob(text, 'grafana_data_export.csv'); + saveSaveBlob(text, EXPORT_FILENAME); +} + +export function convertTableDataToCsv(table, excel = false) { + let text = formatSpecialHeader(excel); + // add headline + text += formatRow(table.columns.map(val => val.title || val.text)); + // process data + for (let i = 0; i < table.rows.length; i += 1) { + text += formatRow(table.rows[i], i < table.rows.length - 1); + } + return text; } export function exportTableDataToCsv(table, excel = false) { - var text = excel ? 'sep=;\n' : ''; - // add header - _.each(table.columns, function(column) { - text += (column.title || column.text) + ';'; - }); - text += '\n'; - // process data - _.each(table.rows, function(row) { - _.each(row, function(value) { - text += value + ';'; - }); - text += '\n'; - }); - saveSaveBlob(text, 'grafana_data_export.csv'); + let text = convertTableDataToCsv(table, excel); + saveSaveBlob(text, EXPORT_FILENAME); } export function saveSaveBlob(payload, fname) { - var blob = new Blob([payload], { type: 'text/csv;charset=utf-8' }); + let blob = new Blob([payload], { type: 'text/csv;charset=utf-8;header=present;' }); saveAs(blob, fname); }