diff --git a/docs/_static/benchmarks_files/benchmark-data.csv b/docs/_static/benchmarks_files/OV-benchmark-data.csv similarity index 99% rename from docs/_static/benchmarks_files/benchmark-data.csv rename to docs/_static/benchmarks_files/OV-benchmark-data.csv index bdfae7ce98e..5cbb8b7c24f 100644 --- a/docs/_static/benchmarks_files/benchmark-data.csv +++ b/docs/_static/benchmarks_files/OV-benchmark-data.csv @@ -508,4 +508,4 @@ yolo_v4,OV-2022.3-8991,core,Intel® Core™ i7-1165G7 CPU-only,11.067,,3.259,0. yolo_v4,OV-2022.3-8991,core-iGPU,Intel® Core™ i7-1165G7 iGPU-only,25.048,,7.384,0.053,0.895,$469 ,28,1,$469 ,28,39.1492,,, yolo_v4,OV-2022.3-8991,core-CPU+iGPU,Intel® Core™ i7-1165G7 CPU+iGPU,29.658,,8.32,0.063,1.059,$469 ,28,1,$469 ,28,,,, yolo_v4,OV-2022.3-8991,accel,Intel® Flex-170 GPU,454.49,56.78,,0.236,3.03,"$1,925 ",150,1,"$1,925 ",150,6.969,,, -end_rec,,,,,,,,,,,,,,,,, +end_rec,,,,,,,,,,,,,,,,, \ No newline at end of file diff --git a/docs/_static/benchmarks_files/OVMS-benchmark-data.csv b/docs/_static/benchmarks_files/OVMS-benchmark-data.csv index 2ccf0811cc6..54f4c796ea2 100644 --- a/docs/_static/benchmarks_files/OVMS-benchmark-data.csv +++ b/docs/_static/benchmarks_files/OVMS-benchmark-data.csv @@ -102,4 +102,4 @@ Yolo_V4,OV-2022.3-8991,core,Intel® Core™ i7-8700T CPU-only,4.60,4.71,2.45,2. Yolo_V4,OV-2022.3-8991,core,Intel® Core™ i9-10920X CPU-only,36.33,40.21,10.52,10.90 Yolo_V4,OV-2022.3-8991,xeon,Intel® Xeon® Gold 6238 CPU-only,81.88,95.46,26.43,27.57 Yolo_V4,OV-2022.3-8991,xeon,Intel® Xeon® 8260 CPU-only,96.58,111.93,30.48,34.50 -end_rec,,,,,,, +end_rec,,,,,,, \ No newline at end of file diff --git a/docs/_static/js/graphs.js b/docs/_static/js/graphs.js index 469507aa435..ef15086f09f 100644 --- a/docs/_static/js/graphs.js +++ b/docs/_static/js/graphs.js @@ -1,14 +1,17 @@ -// general output config +// =================== GENERAL OUTPUT CONFIG ========================= + const chartDisclaimers = { Value: 'Value: Performance/(No_of_sockets * Price_of_CPU_dGPU), where prices are in USD as of December 2022.', Efficiency: 'Efficiency: Performance/(No_of_sockets * TDP_of_CPU_dGPU), where total power dissipation (TDP) is in Watt as of December 2022.' } -const defaultSelections = { +const OVdefaultSelections = { platforms: {name: 'platform', data: [ 'Intel® Core™ i9-12900K CPU-only', + 'Intel® Core™ i9-13900K CPU-only', 'Intel® Core™ i5-10500TE CPU-only', + 'Intel® Core™ i5-13600K CPU-only', 'Intel® Core™ i5-8500 CPU-only', 'Intel® Core™ i7-8700T CPU-only', 'Intel® Core™ i9-10900TE CPU-only', @@ -28,6 +31,30 @@ const defaultSelections = { pracision: {name: 'precision', data: ['INT8', 'FP32']} } +const OVMSdefaultSelections = { + platforms: {name: 'platform', + data: [ + 'Intel® Core™ i3-10100 CPU-only', + 'Intel® Core™ i5-8500 CPU-only', + 'Intel® Core™ i7-8700T CPU-only', + 'Intel® Core™ i9-10920X CPU-only', + ] + }, + models: {name: 'networkmodel', + data: [ + 'bert-small-uncased-whole-word-masking-squad-0002', + 'mobilenet-ssd ', + 'resnet-50', + 'yolo_v3_tiny' + ] + }, + parameters: {name: 'kpi', data: ['Throughput']}, + pracision: {name: 'precision', data: ['OV-INT8 (reference)', 'INT8']} +} + +// ==================================================== + + class Filter { // param: GraphData[], networkModels[] @@ -67,9 +94,11 @@ class Filter { }); } } + + class ExcelDataTransformer { - static transform(csvdata) { + static transform(csvdata, version) { const entries = csvdata.filter((entry) => { return !entry.includes('begin_rec') && !entry.includes('end_rec'); }); @@ -77,17 +106,20 @@ class ExcelDataTransformer { // else generate return entries.map((entry) => { + if (version == 'ovms') + return new GraphData(new OVMSExcelData(entry)); return new GraphData(new ExcelData(entry)); }); } } + class ExcelData { constructor(csvdataline) { if (!csvdataline) { return; } - this.networkModel = csvdataline[0]; + this.networkModel = csvdataline[0].toLowerCase(); this.release = csvdataline[1]; this.ieType = csvdataline[2]; this.platformName = csvdataline[3]; @@ -103,21 +135,17 @@ class ExcelData { this.tdpPerSocket = csvdataline[13]; this.latency = csvdataline[14]; } - networkModel = ''; - release = ''; - ieType = ''; - platformName = ''; - throughputInt8 = ''; - throughputFP16 = ''; - throughputFP32 = ''; - value = ''; - efficiency = ''; - price = ''; - tdp = ''; - sockets = ''; - pricePerSocket = ''; - tdpPerSocket = ''; - latency = ''; +} + + +class OVMSExcelData extends ExcelData { + constructor(csvdataline) { + super(csvdataline); + this.throughputOVMSInt8 = csvdataline[5]; + this.throughputInt8 = csvdataline[4]; + this.throughputOVMSFP32 = csvdataline[7]; + this.throughputFP32 = csvdataline[6]; + } } @@ -131,7 +159,13 @@ class GraphData { this.ieType = excelData.ieType; this.platformName = excelData.platformName; this.kpi = new KPI( - new Precision(excelData.throughputInt8, excelData.throughputFP16, excelData.throughputFP32), + { + 'ovmsint8': excelData.throughputOVMSInt8, + 'ovmsfp32': excelData.throughputOVMSFP32, + 'int8': excelData.throughputInt8, + 'fp16': excelData.throughputFP16, + 'fp32': excelData.throughputFP32 + }, excelData.value, excelData.efficiency, excelData.latency); @@ -142,18 +176,9 @@ class GraphData { this.tdpPerSocket = excelData.tdpPerSocket; this.latency = excelData.latency; } - networkModel = ''; - platformName = ''; - release = ''; - ieType = ''; - kpi = new KPI(); - price = ''; - tdp = ''; - sockets = ''; - pricePerSocket = ''; - tdpPerSocket = ''; } + class KPI { constructor(precisions, value, efficiency, latency) { this.throughput = precisions; @@ -161,22 +186,8 @@ class KPI { this.efficiency = efficiency; this.latency = latency; } - throughput = new Precision(); - value = ''; - efficiency = ''; - latency = ''; } -class Precision { - constructor(int8, fp16, fp32) { - this.int8 = int8; - this.fp16 = fp16; - this.fp32 = fp32; - } - int8 = ''; - fp16 = ''; - fp32 = ''; -} class Modal { static getIeTypeLabel(ietype) { @@ -196,10 +207,14 @@ class Modal { static getCoreTypesLabels() { return ['CPU', 'iGPU', 'CPU+iGPU']; } - static getKpisLabels() { + static getKpisLabels(version) { + if (version == 'ovms') + return ['Throughput']; return ['Throughput', 'Value', 'Efficiency', 'Latency']; } - static getPrecisionsLabels() { + static getPrecisionsLabels(version) { + if (version == 'ovms') + return ['OV-INT8 (reference)', 'INT8', 'OV-FP32 (reference)', 'FP32']; return ['INT8', 'FP16', 'FP32']; } static getCoreTypes(labels) { @@ -219,6 +234,10 @@ class Modal { static getPrecisions(labels) { return labels.map((label) => { switch (label) { + case 'OV-INT8 (reference)': + return 'ovmsint8'; + case 'OV-FP32 (reference)': + return 'ovmsfp32'; case 'INT8': return 'int8'; case 'FP16': @@ -232,6 +251,7 @@ class Modal { } } + class Graph { constructor(data) { this.data = data; @@ -311,6 +331,10 @@ class Graph { static getPrecisionConfig(precision) { switch (precision) { + case 'ovmsint8': + return { data: null, color: '#FF8F51', label: 'FPS (OV Ref. INT8)' }; + case 'ovmsfp32': + return { data: null, color: '#B24501', label: 'FPS (OV Ref. FP32)' }; case 'int8': return { data: null, color: '#00C7FD', label: 'FPS (INT8)' }; case 'fp16': @@ -338,6 +362,7 @@ class Graph { } } + class ChartDisplay { constructor(mode, numberOfCharts) { this.mode = mode; @@ -345,9 +370,11 @@ class ChartDisplay { } } + $(document).ready(function () { - $('.ov-toolkit-benchmark-results').on('click', showModal); + $('.ov-toolkit-benchmark-results').on('click', () => showModal('ov')); + $('.ovms-toolkit-benchmark-results').on('click', () => showModal('ovms')); function clickBuildGraphs(graph, networkModels, ietype, platforms, kpis, precisions) { renderData(graph, networkModels, ietype, platforms, kpis, precisions); @@ -378,21 +405,19 @@ $(document).ready(function () { } function hideModal() { - $('#graphModal').hide(); + $('#graphModal').remove(); $('body').css('overflow', 'auto'); } - function showModal() { + function showModal(version) { $('body').css('overflow', 'hidden'); - if ($('#graphModal').length) { - $('#graphModal').show(); - return; - } - const dataPath = '_static/benchmarks_files/benchmark-data.csv'; + let dataPath = '_static/benchmarks_files/OV-benchmark-data.csv'; + if (version == 'ovms') + dataPath = '_static/benchmarks_files/OVMS-benchmark-data.csv'; Papa.parse(dataPath, { download: true, - complete: renderModal + complete: (result) => renderModal(result, version) }); } @@ -446,10 +471,10 @@ $(document).ready(function () { $('#build-graphs-btn').prop('disabled', true); } - function renderModal(result) { + function renderModal(result, version) { // remove header from csv line result.data.shift(); - var graph = new Graph(ExcelDataTransformer.transform(result.data)); + var graph = new Graph(ExcelDataTransformer.transform(result.data, version)); var networkModels = Graph.getNetworkModels(graph.data); var ieTypes = Graph.getIeTypes(graph.data); @@ -471,7 +496,7 @@ $(document).ready(function () { modal.find('.models-column-one').append(selectAllModelsButton).append(models.slice(0, models.length / 2)); modal.find('.models-column-two').append(models.slice(models.length / 2)); - const precisions = Modal.getPrecisionsLabels().map((precision) => createCheckMark(precision, 'precision')); + const precisions = Modal.getPrecisionsLabels(version).map((precision) => createCheckMark(precision, 'precision')); modal.find('.precisions-column').append(precisions); selectAllCheckboxes(precisions); disableAllCheckboxes(precisions); @@ -494,13 +519,13 @@ $(document).ready(function () { modal.find('.ietype-column').append(types); modal.find('.ietype-column input').first().prop('checked', true); - const kpiLabels = Modal.getKpisLabels().map((kpi) => createCheckMark(kpi, 'kpi')); + const kpiLabels = Modal.getKpisLabels(version).map((kpi) => createCheckMark(kpi, 'kpi')); modal.find('.kpi-column').append(kpiLabels); $('body').prepend(modal); - renderClientPlatforms(graph.data, modal, true); - preselectDefaultSettings(graph.data, modal); + renderClientPlatforms(graph.data, modal, version, true); + preselectDefaultSettings(graph.data, modal, version); $('.clear-all-btn').on('click', clearAll); $('#build-graphs-btn').on('click', () => { @@ -514,7 +539,7 @@ $(document).ready(function () { selectAllCheckboxes(models); else deSelectAllCheckboxes(models); }); - modal.find('.ietype-column input').on('click', () => renderClientPlatforms(graph.data, modal, true)); + modal.find('.ietype-column input').on('click', () => renderClientPlatforms(graph.data, modal, version, true)); modal.find('.kpi-column input').on('click', validateThroughputSelection); modal.find('input').on('click', validateSelections); }); @@ -538,14 +563,15 @@ $(document).ready(function () { validateSelections(); } - function preselectDefaultSettings(data, modal) { + function preselectDefaultSettings(data, modal, version) { + const defaultSelections = (version == 'ov') ? OVdefaultSelections : OVMSdefaultSelections; if (defaultSelections.platformFilters) { const filters = modal.find('.selectable-box-container').children('.selectable-box'); filters.removeClass('selected'); defaultSelections.platformFilters.data.forEach(selection => { filters.filter(`[data-${defaultSelections.platformFilters.name}="${selection}"]`).addClass('selected'); }); - renderClientPlatforms(data, modal); + renderClientPlatforms(data, modal, version); } clearAll(); for (setting in defaultSelections) { @@ -600,9 +626,11 @@ $(document).ready(function () { return Array.from(optionMap.values()); } - function renderClientPlatforms(data, modal, preselectEveryItem) { + function renderClientPlatforms(data, modal, version, preselectEveryItem) { if (getSelectedIeType() === 'core') { showCoreSelectorTypes(Modal.getCoreTypesLabels(), data, modal); + if (version === 'ovms') + hideCoreSelectorTypes(); } else { hideCoreSelectorTypes();