Added filters block to report template (#6201)

inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py

@@ -126,6 +126,7 @@ def collect_statistic(root: ET.Element, is_conformance_mode: bool):
         pass_rate_avg[device.tag] = round(float(pass_rate_avg[device.tag]), 1)
         general_pass_rate[device.tag] = general_passed_tests[device.tag] * 100 / general_test_count[device.tag]
         general_pass_rate[device.tag] = round(float(general_pass_rate[device.tag]), 1)
+        trusted_ops[device.tag] = round(float(trusted_ops[device.tag] * 100 / len(results[device.tag])), 1)
 
     logger.info("Test number comparison between devices is started")
     for op in op_res:

inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/filters.js

@@ -0,0 +1,209 @@
+deviceList = [];
+$(document).ready(function () {
+    var opsets = {};
+    LoadOpsetNumbers();
+    LoadDevices();
+
+    $("#filters").submit(function (event) {
+        event.preventDefault();
+        filterTable();
+    });
+    $('#reset').click(function () {
+        $('#opsetNumber').val(0);
+        $('#operationName').val('');
+        $('#status').prop("disabled", true).val(0);
+        $('#devices').val(0);
+        $('#references').val(0);
+        filterTable();
+    });
+    $('#devices').on('change', function () {
+        if (this.value == 0) {
+            $('#status').prop("disabled", true).val(0);
+        } else {
+            $('#status').prop("disabled", false);
+        };
+    });
+
+});
+
+function LoadOpsetNumbers() {
+    var data = [];
+
+    $('#data th[scope="row"]').each(function () {
+
+        num = $(this).text().split("-")[1];
+        if (data.indexOf(num) < 0) {
+            data.push(num);
+        }
+    });
+    data.sort();
+    data = $.map(data, function (item) {
+        return "<option value=" + item + ">" + item + "</option>";
+    });
+    $("#opsetNumber").html('<option value="0">All</option>');
+    $("#opsetNumber").append(data.join(""));
+}
+
+function LoadDevices() {
+    var data = [];
+
+    $('.table-dark.device').each(function () {
+        if (data.indexOf($(this).text()) < 0) {
+            data.push($(this).text());
+        }
+    });
+    data.sort();
+    deviceList = data;
+    data = $.map(data, function (item) {
+        return "<option value=" + item + ">" + item + "</option>";
+    });
+    $("#devices").html('<option value="0">All</option>');
+    $("#devices").append(data.join(""));
+}
+
+function filterTable() {
+    device = $("#devices").val();
+    if (device == 0) {
+        $("#report td.value, #report td.table-secondary, #report td.table-primary, #report th.table-dark.device").show();
+    } else {
+        $("#report td.value, #report td.table-secondary, #report td.table-primary, #report th.table-dark.device").filter(function () {
+            $(this).toggle($(this).hasClass(device))
+        });
+    }
+    opsetNumber = $("#opsetNumber").val();
+    operationName = $('#operationName').val().trim();
+    status = $('#status').val();
+    references = $('#references').val();
+
+    $("#report #data tr").show();
+    $('#report').show();
+    $('#message').hide();
+    if (opsetNumber != 0) {
+        $("#report #data tr").filter(function () {
+            $(this).toggle(checkVersion($(this), opsetNumber));
+        });
+    }
+
+    if (operationName) {
+        $("#report #data tr:not(:hidden)").filter(function () {
+            $(this).toggle($(this).find('th').text().split("-")[0].toLowerCase().indexOf(operationName.toLowerCase()) > -1);
+        });
+    }
+
+    if (references != 0) {
+        if (references == 'nv') {
+            $("#report #data tr:not(:hidden)").filter(function () {
+                $(this).toggle($(this).find('th').hasClass("colorRed"))
+            });
+        } else {
+            $("#report #data tr:not(:hidden)").filter(function () {
+                $(this).toggle(!$(this).find('th').hasClass("colorRed"));
+            });
+        }
+    }
+    if (status != 0) {
+        if (status == 'p') {
+            $("#report #data tr:not(:hidden)").filter(function () {
+                $(this).toggle($(this).find('.value:visible[crashed="0"][failed="0"][skipped="0"]').length > 0)
+            });
+        } else if (status == 'f') {
+            $("#report #data tr:not(:hidden)").filter(function () {
+                $(this).toggle($(this).find('.value:visible[passed="0"][crashed="0"][skipped="0"]').length > 0)
+            });
+        } else if (status == 'c') {
+            $("#report #data tr:not(:hidden)").filter(function () {
+                $(this).toggle($(this).find('.value:visible[passed="0"][failed="0"][skipped="0"]').length > 0)
+            });
+        } else if (status == 's') {
+            $("#report #data tr:not(:hidden)").filter(function () {
+                $(this).toggle($(this).find('.value:visible[passed="0"][failed="0"][crashed="0"]').length > 0)
+            });
+        } else { // No tests
+
+            $("#report #data tr:not(:hidden)").filter(function () {
+                $(this).toggle($(this).find('.table-secondary:visible').length > 0)
+            });
+        }
+    }
+
+
+    if ($("#report #data tr").length == $("#report #data tr:hidden").length) {
+        $('#report').hide();
+        $('#message').show();
+    } else {
+        calculateStatistics(device);
+    }
+}
+
+function checkVersion(element, opsetNumber) {
+    var name = element.find('th').text().split("-")[0];
+    var version = element.find('th').text().split("-")[1];
+    if (version > opsetNumber) {
+        return false;
+    } else {
+        var versions = [];
+        $('#report #data tr th[name^="' + name + '-"]').each(function () {
+            if ($(this).text().split('-')[1] <= opsetNumber) {
+                versions.push(+$(this).text().split('-')[1]);
+            }
+        });
+        return version == Math.max.apply(null, versions);
+    }
+}
+
+function calculateStatistics() {
+    if (device != 0) {
+        calculateColumnStatistics(device);
+    } else {
+        deviceList.map((el) => calculateColumnStatistics(el))
+    }
+}
+
+function calculateColumnStatistics(device) {
+    // total
+    total = $("#report #data tr:not(:hidden)").length;
+    $('#statistic .table-primary[scope="row"] i').text(total);
+    // trusted op
+    count_trasted_op = $("#report #data tr:not(:hidden) ." + device + ".value[value^='100'][crashed='0'][failed='0'][skipped='0']").length;
+    all_operations = $("#report #data tr:not(:hidden) .value." + device).length;
+    if (!all_operations) {
+        trasted_op = "---";
+    } else {
+        trasted_op = (count_trasted_op * 100 / all_operations).toFixed(1) + ' %';
+    }
+    $('#statistic .table-primary.' + device + '.trusted-ops').text(trasted_op);
+    $('#statistic .table-primary.' + device + '.test_total').text(all_operations || 0);
+
+    // tested op_counter
+    tested_op_count = 0;
+    passed_tested_op_count = 0;
+    $("#report #data tr:not(:hidden) ." + device + ".value span").each(function () {
+        text = $(this).text().split(':')[1];
+        if ($(this).hasClass('green')) {
+            passed_tested_op_count += +text;
+        }
+        tested_op_count += +text;
+    });
+
+    // General Pass Rate
+    if (tested_op_count == 0) {
+        $('#statistic .table-primary.' + device + '.general_pass_rate').text('---');
+
+    } else {
+        general_pass_rate = (passed_tested_op_count * 100 / tested_op_count).toFixed(1) + ' %';
+        $('#statistic .table-primary.' + device + '.general_pass_rate').text(general_pass_rate);
+    }
+    $('#statistic .table-primary.' + device + '.tested-ops_count').text(tested_op_count);
+
+    // AVG Pass Rate
+    sum_pass_rate = 0;
+    $("#report #data tr:not(:hidden) ." + device + ".value").each(function () {
+        sum_pass_rate += +$(this).attr('value');
+    });
+    if (all_operations == 0) {
+        $('#statistic .table-primary.' + device + '.avg_pass_rate').text('---');
+    } else {
+        avg_pass_rate = (sum_pass_rate / all_operations).toFixed(1) + ' %';
+        $('#statistic .table-primary.' + device + '.avg_pass_rate').text(avg_pass_rate);
+    }
+}

inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html

@@ -1,5 +1,6 @@
 <!doctype html>
 <html lang="en">
+
 <head>
     <!-- Required meta tags -->
     <meta charset="utf-8">
@@ -7,106 +8,151 @@
 
     <!-- Bootstrap CSS -->
     <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css"
-          integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
+        integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
-    <link rel="stylesheet" type="text/css" href="extensions/filter-control/bootstrap-table-filter-control.css">
+    <link rel="stylesheet" href="template/style.css" />
-    <title>Report</title>
+    <script src="https://code.jquery.com/jquery-3.2.1.slim.min.js"
-</head>
-<body>
-<script src="extensions/filter-control/bootstrap-table-filter-control.js"></script>
-<!-- Optional JavaScript -->
-<!-- jQuery first, then Popper.js, then Bootstrap JS -->
-<!--
-<script src="https://code.jquery.com/jquery-3.2.1.slim.min.js"
         integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN"
         crossorigin="anonymous"></script>
-<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js"
+    <script src="template/filters.js"></script>
-        integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q"
+    <title>Report</title>
-        crossorigin="anonymous"></script>
+</head>
-<script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js"
-        integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl"
-        crossorigin="anonymous"></script>
--->
 
-<hr class="my-4">
+<body>
-<h2>Operations coverage summary: {{report_tag}} {{ timestamp }}</h2>
+    <div class="main">
-<hr class="my-4">
+        <h2>Operations coverage summary: {{report_tag}} {{ timestamp }}</h2>
-<table class="table table-hover" id="legend">
+        <div class="legend">
-    <thead>
+            <div>
-    <tr>
+                <span class="border colorRed">Acosh-4</span><span>Not verified Ngraph references</span>
-        <th scope="col"><span style="color: red">"Operation_name"-"opset_version"</span></th>
+            </div>
-        <th><span style="color: black"> Not verified Ngraph references</span></th>
-    </tr>
-    <tr>
-        <th class="table-secondary" scope="col"></th>
-        <th>Collected statistic info</th>
-    </tr>
-    </thead>
-</table>
-<table class="table table-hover" id="report">
-    <thead>
-    <tr>
-        <th class="table-primary" scope="col" style="position: sticky; top: 0">Operation</th>
-        {% for d in devices -%}
-        <th class="table-primary" style="position: sticky; top: 0">{{ d }}</th>
-        {% endfor %}
-    </tr>
-    </thead>
-    <tbody>
-        <tr>
-        <th class="table-secondary" scope="row">Total: {{ordered_ops|length}}</th>
-        {% for d in devices -%}
-        <td class="table-secondary" >{{results[d]|length}}</td>
-        {% endfor %}
-    </tr>
-        <tr>
-        <th class="table-secondary" scope="row">Trusted op counter (passrate=100%):</th>
-        {% for d in devices -%}
-        <td class="table-secondary" >{{trusted_ops[d]}}</td>
-        {% endfor %}
-    </tr>
-        <tr>
-        <th class="table-secondary" scope="row">Tested op counter:</th>
-        {% for d in devices -%}
-        <td class="table-secondary" >{{general_test_count[d]}}</td>
-        {% endfor %}
-    </tr>
-        <tr>
-        <th class="table-secondary" scope="row">AVG passrate per op (=sum_pass_rates/covered_ops_num):</th>
-        {% for d in devices -%}
-        <td class="table-secondary" >{{pass_rate_avg[d]}}%</td>
-        {% endfor %}
-    </tr>
-        <tr>
-        <th class="table-secondary" scope="row">General passrate (=passed_tests/all_tests):</th>
-        {% for d in devices -%}
-        <td class="table-secondary" >{{general_pass_rate[d]}}%</td>
-        {% endfor %}
-    </tr>
-    </tbody>
-    <tbody>
-    {% for op in ordered_ops -%}
-    <tr>
-        {% if op in verified_operations -%}
-            <th scope="row">{{ op }}</th>
-        {% else -%}
-        <th scope="row"><span style="color: red">{{ op }}</span></th>
-        {% endif -%}
-        {% for d in devices -%}
-        {% if op in results[d] -%}
-            <td>
-            {{ results[d][op].passrate }}% (<span style="color: green">p:{{ results[d][op].passed }}</span>,
-               <span style="color: red">f:{{ results[d][op].failed }}</span>,s:{{ results[d][op].skipped }}</span>,
-               <span style="color: red">c:{{ results[d][op].crashed }}</span>)
-            </td>
-        {% else -%}
-        <td class="table-warning">No tests</td>
-        {% endif -%}
 
-        {% endfor %}
+            <div>
-    </tr>
+                <span class="table-primary border"></span><span>Collected statistic info</span>
-    {% endfor -%}
+            </div>
+            <div>
+                <span class="table-secondary border">N/A</span><span>No Tests</span>
+            </div>
+            <div>
+                <span><b>Status:</b></span>
+                <span class="green">P:85</span><span>Passed</span>
+                <span class="red">F:0</span><span>Failed</span>
+                <span class="grey">S:2</span><span>Skipped</span>
+                <span class="dark">C:0</span><span>Crashed</span>
+            </div>
+        </div>
+    </div>
+    <!-- Filters block -->
+    <div class="filters">
+        <form id="filters">
+            <div class="form-group">
+                <label for="operationName"><b>Operation Name</b></label>
+                <input id="operationName" type="text" class="form-control" />
+            </div>
+            <div class="form-group">
+                <label for="opsetNumber"><b>Opset Number</b></label>
+                <select id="opsetNumber" class="form-control"></select>
+            </div>
+            <div class="form-group">
+                <label for="references"><b>Ngraph references</b></label>
+                <select id="references" class="form-control">
+                    <option value="0">All</option>
+                    <option value="v">Verified</option>
+                    <option value="nv">Not verified</option>
+                </select>
+            </div>
+            <div class="form-group">
+                <label for="devices"><b>Devices</b></label>
+                <select id="devices" class="form-control"></select>
+            </div>
 
-    </tbody>
+
-</table>
+            <div class="form-group">
+                <label for="status"><b>Status</b></label>
+                <select id="status" class="form-control" disabled>
+                    <option value="0">All</option>
+                    <option value="p">100% Passed</option>
+                    <option value="f">100% Failed</option>
+                    <option value="s">100% Skipped</option>
+                    <option value="c">100% Crashed</option>
+                    <option value="na">No tests</option>
+                </select>
+            </div>
+            <button type="submit" class="btn btn-primary">Apply filters</button>
+            <button type="button" class="btn btn-secondary" id="reset">Reset filters</button>
+        </form>
+    </div>
+
+    <!-- Results -->
+    <table class="table table-hover" id="report">
+        <thead>
+            <tr>
+                <th class="table-dark" scope="col" style="position: sticky; top: 97px">Operation</th>
+                {% for d in devices -%}
+                <th class="table-dark device {{ d }}" style="position: sticky; top: 97px">{{ d }}</th>
+                {% endfor %}
+            </tr>
+        </thead>
+        <tbody id="statistic">
+            <tr>
+                <th class="table-primary" scope="row">Total: <i>{{ordered_ops|length}}</i></th>
+                {% for d in devices -%}
+                <td class="table-primary {{ d }} test_total">{{results[d]|length}}</td>
+                {% endfor %}
+            </tr>
+            <tr>
+                <th class="table-primary" scope="row">Trusted op (passrate=100%):</th>
+                {% for d in devices -%}
+                <td class="table-primary  {{ d }} trusted-ops">{{trusted_ops[d]}} %</td>
+                {% endfor %}
+            </tr>
+            <tr>
+                <th class="table-primary" scope="row">Tested op counter:</th>
+                {% for d in devices -%}
+                <td class="table-primary  {{ d }} tested-ops_count">{{general_test_count[d]}}</td>
+                {% endfor %}
+            </tr>
+            <tr>
+                <th class="table-primary" scope="row">AVG passrate per op (=sum_pass_rates/covered_ops_num):</th>
+                {% for d in devices -%}
+                <td class="table-primary  {{ d }} avg_pass_rate">{{pass_rate_avg[d]}} %</td>
+                {% endfor %}
+            </tr>
+            <tr>
+                <th class="table-primary" scope="row">General passrate (=passed_tests/all_tests):</th>
+                {% for d in devices -%}
+                <td class="table-primary  {{ d }} general_pass_rate">{{general_pass_rate[d]}} %</td>
+                {% endfor %}
+            </tr>
+        </tbody>
+        <tbody id="data">
+            {% for op in ordered_ops -%}
+            <tr>
+                <th scope="row" {% if op not in verified_operations -%} class="colorRed" {% endif -%} name="{{ op }}">{{
+                    op }}</th>
+
+                {% for d in devices -%}
+                {% if op in results[d] -%}
+                <td class="value {{ d }}" passed="{{ results[d][op].passed }}" failed="{{ results[d][op].failed }}"
+                    skipped="{{ results[d][op].skipped }}" crashed="{{ results[d][op].crashed }}"
+                    value="{{ results[d][op].passrate }}">
+                    {{ results[d][op].passrate }} %<br />
+                    <span class="green" title="Passed">P:{{ results[d][op].passed }}</span>
+                    <span class="red" title="Failed">F:{{ results[d][op].failed }}</span>
+                    <span class="grey" title="Skipped">S:{{ results[d][op].skipped }}</span>
+                    <span class="dark" title="Crashed">C:{{ results[d][op].crashed }}</span>
+                </td>
+                {% else -%}
+                <td class="table-secondary {{ d }}">N/A</td>
+                {% endif -%}
+
+                {% endfor %}
+            </tr>
+            {% endfor -%}
+
+        </tbody>
+    </table>
+    <div id="message" style="display:none">
+        There is no data related to selected filters. Please set new filters.
+    </div>
 </body>
+
 </html>

inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/style.css

@@ -0,0 +1,88 @@
+body {
+    font-size: 14px;
+}
+.table td, .table th {
+    padding: .5em .75em;
+}
+.table td span {
+    font-size: 0.8em;
+}
+.value {
+    font-weight: 500;
+}
+.value span {
+   display:inline-block;
+   font-weight: 400;
+   padding: 1px 5px;
+   border-radius: 2px;
+   cursor: default;
+}
+ .green {
+    background: #0080002e;
+}
+.red {
+    background: #ff000038;
+}
+.grey {
+    background: #8080803d;
+}
+.dark {
+    background: #8b000040;
+}
+.filters {
+    background: #FFF;
+    padding: 5px 10px;
+    position: sticky;
+    top: 0;
+}
+.filters form {
+    display: flex;
+    background: #efefef;
+    padding: 10px;
+    border-radius: 5px;
+}
+form div{
+    margin-right: 10px;
+}
+form button {
+    align-self: center;
+    margin-top: 26px;
+    margin-left: 20px;
+}
+.main {
+    margin: 10px;
+}
+.legend {
+    display: flex;
+}
+.legend div {
+    display:flex;
+    align-items: center;
+    margin-right: 20px;
+}
+.legend span{
+    display: inline-block;
+    padding: 3px 5px;
+    min-height: 25px;
+    min-width: 25px;
+}
+.colorRed {
+    color:#cf1d1d;
+    font-weight: bold;
+}
+.form-group {
+    margin-bottom: 0;
+}
+#message {
+    font-weight: 500;
+    font-size: 20px;
+    margin: 20px;
+    text-align: center;
+    color: #cf1d1d;
+}
+.table-dark:hover {
+    background: #212529!important;
+}
+h2 {
+    margin-bottom: 2rem;
+}