diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py index 4419c4ffeb8..1245ae8a0fb 100644 --- a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py @@ -126,6 +126,7 @@ def collect_statistic(root: ET.Element, is_conformance_mode: bool): pass_rate_avg[device.tag] = round(float(pass_rate_avg[device.tag]), 1) general_pass_rate[device.tag] = general_passed_tests[device.tag] * 100 / general_test_count[device.tag] general_pass_rate[device.tag] = round(float(general_pass_rate[device.tag]), 1) + trusted_ops[device.tag] = round(float(trusted_ops[device.tag] * 100 / len(results[device.tag])), 1) logger.info("Test number comparison between devices is started") for op in op_res: diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/filters.js b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/filters.js new file mode 100644 index 00000000000..af832efa3cf --- /dev/null +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/filters.js @@ -0,0 +1,209 @@ +deviceList = []; +$(document).ready(function () { + var opsets = {}; + LoadOpsetNumbers(); + LoadDevices(); + + $("#filters").submit(function (event) { + event.preventDefault(); + filterTable(); + }); + $('#reset').click(function () { + $('#opsetNumber').val(0); + $('#operationName').val(''); + $('#status').prop("disabled", true).val(0); + $('#devices').val(0); + $('#references').val(0); + filterTable(); + }); + $('#devices').on('change', function () { + if (this.value == 0) { + $('#status').prop("disabled", true).val(0); + } else { + $('#status').prop("disabled", false); + }; + }); + +}); + +function LoadOpsetNumbers() { + var data = []; + + $('#data th[scope="row"]').each(function () { + + num = $(this).text().split("-")[1]; + if (data.indexOf(num) < 0) { + data.push(num); + } + }); + data.sort(); + data = $.map(data, function (item) { + return ""; + }); + $("#opsetNumber").html(''); + $("#opsetNumber").append(data.join("")); +} + +function LoadDevices() { + var data = []; + + $('.table-dark.device').each(function () { + if (data.indexOf($(this).text()) < 0) { + data.push($(this).text()); + } + }); + data.sort(); + deviceList = data; + data = $.map(data, function (item) { + return ""; + }); + $("#devices").html(''); + $("#devices").append(data.join("")); +} + +function filterTable() { + device = $("#devices").val(); + if (device == 0) { + $("#report td.value, #report td.table-secondary, #report td.table-primary, #report th.table-dark.device").show(); + } else { + $("#report td.value, #report td.table-secondary, #report td.table-primary, #report th.table-dark.device").filter(function () { + $(this).toggle($(this).hasClass(device)) + }); + } + opsetNumber = $("#opsetNumber").val(); + operationName = $('#operationName').val().trim(); + status = $('#status').val(); + references = $('#references').val(); + + $("#report #data tr").show(); + $('#report').show(); + $('#message').hide(); + if (opsetNumber != 0) { + $("#report #data tr").filter(function () { + $(this).toggle(checkVersion($(this), opsetNumber)); + }); + } + + if (operationName) { + $("#report #data tr:not(:hidden)").filter(function () { + $(this).toggle($(this).find('th').text().split("-")[0].toLowerCase().indexOf(operationName.toLowerCase()) > -1); + }); + } + + if (references != 0) { + if (references == 'nv') { + $("#report #data tr:not(:hidden)").filter(function () { + $(this).toggle($(this).find('th').hasClass("colorRed")) + }); + } else { + $("#report #data tr:not(:hidden)").filter(function () { + $(this).toggle(!$(this).find('th').hasClass("colorRed")); + }); + } + } + if (status != 0) { + if (status == 'p') { + $("#report #data tr:not(:hidden)").filter(function () { + $(this).toggle($(this).find('.value:visible[crashed="0"][failed="0"][skipped="0"]').length > 0) + }); + } else if (status == 'f') { + $("#report #data tr:not(:hidden)").filter(function () { + $(this).toggle($(this).find('.value:visible[passed="0"][crashed="0"][skipped="0"]').length > 0) + }); + } else if (status == 'c') { + $("#report #data tr:not(:hidden)").filter(function () { + $(this).toggle($(this).find('.value:visible[passed="0"][failed="0"][skipped="0"]').length > 0) + }); + } else if (status == 's') { + $("#report #data tr:not(:hidden)").filter(function () { + $(this).toggle($(this).find('.value:visible[passed="0"][failed="0"][crashed="0"]').length > 0) + }); + } else { // No tests + + $("#report #data tr:not(:hidden)").filter(function () { + $(this).toggle($(this).find('.table-secondary:visible').length > 0) + }); + } + } + + + if ($("#report #data tr").length == $("#report #data tr:hidden").length) { + $('#report').hide(); + $('#message').show(); + } else { + calculateStatistics(device); + } +} + +function checkVersion(element, opsetNumber) { + var name = element.find('th').text().split("-")[0]; + var version = element.find('th').text().split("-")[1]; + if (version > opsetNumber) { + return false; + } else { + var versions = []; + $('#report #data tr th[name^="' + name + '-"]').each(function () { + if ($(this).text().split('-')[1] <= opsetNumber) { + versions.push(+$(this).text().split('-')[1]); + } + }); + return version == Math.max.apply(null, versions); + } +} + +function calculateStatistics() { + if (device != 0) { + calculateColumnStatistics(device); + } else { + deviceList.map((el) => calculateColumnStatistics(el)) + } +} + +function calculateColumnStatistics(device) { + // total + total = $("#report #data tr:not(:hidden)").length; + $('#statistic .table-primary[scope="row"] i').text(total); + // trusted op + count_trasted_op = $("#report #data tr:not(:hidden) ." + device + ".value[value^='100'][crashed='0'][failed='0'][skipped='0']").length; + all_operations = $("#report #data tr:not(:hidden) .value." + device).length; + if (!all_operations) { + trasted_op = "---"; + } else { + trasted_op = (count_trasted_op * 100 / all_operations).toFixed(1) + ' %'; + } + $('#statistic .table-primary.' + device + '.trusted-ops').text(trasted_op); + $('#statistic .table-primary.' + device + '.test_total').text(all_operations || 0); + + // tested op_counter + tested_op_count = 0; + passed_tested_op_count = 0; + $("#report #data tr:not(:hidden) ." + device + ".value span").each(function () { + text = $(this).text().split(':')[1]; + if ($(this).hasClass('green')) { + passed_tested_op_count += +text; + } + tested_op_count += +text; + }); + + // General Pass Rate + if (tested_op_count == 0) { + $('#statistic .table-primary.' + device + '.general_pass_rate').text('---'); + + } else { + general_pass_rate = (passed_tested_op_count * 100 / tested_op_count).toFixed(1) + ' %'; + $('#statistic .table-primary.' + device + '.general_pass_rate').text(general_pass_rate); + } + $('#statistic .table-primary.' + device + '.tested-ops_count').text(tested_op_count); + + // AVG Pass Rate + sum_pass_rate = 0; + $("#report #data tr:not(:hidden) ." + device + ".value").each(function () { + sum_pass_rate += +$(this).attr('value'); + }); + if (all_operations == 0) { + $('#statistic .table-primary.' + device + '.avg_pass_rate').text('---'); + } else { + avg_pass_rate = (sum_pass_rate / all_operations).toFixed(1) + ' %'; + $('#statistic .table-primary.' + device + '.avg_pass_rate').text(avg_pass_rate); + } +} diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html index 7d6f751f917..12bb09fcd81 100644 --- a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html @@ -1,5 +1,6 @@ + @@ -7,106 +8,151 @@ - - Report - - - - - - + + Report + -
-

Operations coverage summary: {{report_tag}} {{ timestamp }}

-
- - - - - - - - - - - -
"Operation_name"-"opset_version" Not verified Ngraph references
Collected statistic info
- - - - - {% for d in devices -%} - - {% endfor %} - - - - - - {% for d in devices -%} - - {% endfor %} - - - - {% for d in devices -%} - - {% endfor %} - - - - {% for d in devices -%} - - {% endfor %} - - - - {% for d in devices -%} - - {% endfor %} - - - - {% for d in devices -%} - - {% endfor %} - - - - {% for op in ordered_ops -%} - - {% if op in verified_operations -%} - - {% else -%} - - {% endif -%} - {% for d in devices -%} - {% if op in results[d] -%} - - {% else -%} - - {% endif -%} + +
+

Operations coverage summary: {{report_tag}} {{ timestamp }}

+
+
+ Acosh-4Not verified Ngraph references +
- {% endfor %} -
- {% endfor -%} +
+ Collected statistic info +
+
+ N/ANo Tests +
+
+ Status: + P:85Passed + F:0Failed + S:2Skipped + C:0Crashed +
+ + + +
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
-
-
Operation{{ d }}
Total: {{ordered_ops|length}}{{results[d]|length}}
Trusted op counter (passrate=100%):{{trusted_ops[d]}}
Tested op counter:{{general_test_count[d]}}
AVG passrate per op (=sum_pass_rates/covered_ops_num):{{pass_rate_avg[d]}}%
General passrate (=passed_tests/all_tests):{{general_pass_rate[d]}}%
{{ op }}{{ op }} - {{ results[d][op].passrate }}% (p:{{ results[d][op].passed }}, - f:{{ results[d][op].failed }},s:{{ results[d][op].skipped }}, - c:{{ results[d][op].crashed }}) - No tests
+ +
+ + +
+ + + + + + + + + + + {% for d in devices -%} + + {% endfor %} + + + + + + {% for d in devices -%} + + {% endfor %} + + + + {% for d in devices -%} + + {% endfor %} + + + + {% for d in devices -%} + + {% endfor %} + + + + {% for d in devices -%} + + {% endfor %} + + + + {% for d in devices -%} + + {% endfor %} + + + + {% for op in ordered_ops -%} + + + + {% for d in devices -%} + {% if op in results[d] -%} + + {% else -%} + + {% endif -%} + + {% endfor %} + + {% endfor -%} + + +
Operation{{ d }}
Total: {{ordered_ops|length}}{{results[d]|length}}
Trusted op (passrate=100%):{{trusted_ops[d]}} %
Tested op counter:{{general_test_count[d]}}
AVG passrate per op (=sum_pass_rates/covered_ops_num):{{pass_rate_avg[d]}} %
General passrate (=passed_tests/all_tests):{{general_pass_rate[d]}} %
{{ + op }} + {{ results[d][op].passrate }} %
+ P:{{ results[d][op].passed }} + F:{{ results[d][op].failed }} + S:{{ results[d][op].skipped }} + C:{{ results[d][op].crashed }} + 		N/A
+ + diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/style.css b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/style.css new file mode 100644 index 00000000000..11831d83b83 --- /dev/null +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/style.css @@ -0,0 +1,88 @@ +body { + font-size: 14px; +} +.table td, .table th { + padding: .5em .75em; +} +.table td span { + font-size: 0.8em; +} +.value { + font-weight: 500; +} +.value span { + display:inline-block; + font-weight: 400; + padding: 1px 5px; + border-radius: 2px; + cursor: default; +} + .green { + background: #0080002e; +} +.red { + background: #ff000038; +} +.grey { + background: #8080803d; +} +.dark { + background: #8b000040; +} +.filters { + background: #FFF; + padding: 5px 10px; + position: sticky; + top: 0; +} +.filters form { + display: flex; + background: #efefef; + padding: 10px; + border-radius: 5px; +} +form div{ + margin-right: 10px; +} +form button { + align-self: center; + margin-top: 26px; + margin-left: 20px; +} +.main { + margin: 10px; +} +.legend { + display: flex; +} +.legend div { + display:flex; + align-items: center; + margin-right: 20px; +} +.legend span{ + display: inline-block; + padding: 3px 5px; + min-height: 25px; + min-width: 25px; +} +.colorRed { + color:#cf1d1d; + font-weight: bold; +} +.form-group { + margin-bottom: 0; +} +#message { + font-weight: 500; + font-size: 20px; + margin: 20px; + text-align: center; + color: #cf1d1d; +} +.table-dark:hover { + background: #212529!important; +} +h2 { + margin-bottom: 2rem; +}