[DOCS] conformance update (#18732)
conformance table fix Add in ONNX layers
This commit is contained in:
Karol Blaszczak
GitHub
parent
5833f32c22
commit
bfe3faca1e
File diff suppressed because it is too large
Load Diff
@ -161,20 +161,10 @@ function filterTable() {
|
||||
}
|
||||
|
||||
function checkVersion(element, opsetNumber) {
|
||||
var name = element.find('th').text().split("-")[0];
|
||||
var version = Number(element.find('th').text().split("-")[1]);
|
||||
var name = element.find('th')[0].getAttribute('name');
|
||||
var opsets = JSON.parse(name)[element.find('th').text()];
|
||||
var realOpsetNumber = Number(opsetNumber);
|
||||
if (version > realOpsetNumber) {
|
||||
return false;
|
||||
} else {
|
||||
var versions = [];
|
||||
$('#report #data tr th[name^="' + name + '-"]').each(function () {
|
||||
if (Number($(this).text().split('-')[1]) <= realOpsetNumber) {
|
||||
versions.push(Number(+$(this).text().split('-')[1]));
|
||||
}
|
||||
});
|
||||
return version == Math.max.apply(null, versions);
|
||||
}
|
||||
return opsets.indexOf(realOpsetNumber) != -1;
|
||||
}
|
||||
|
||||
function calculateStatistics() {
|
||||
@ -195,10 +185,10 @@ function calculateColumnStatistics(device) {
|
||||
if (!all_operations) {
|
||||
trusted_op = "---";
|
||||
} else {
|
||||
trusted_op = (count_trusted_op * 100 / all_operations).toFixed(1) + ' %';
|
||||
trusted_op = (count_trusted_op * 100 / all_operations).toFixed(2) + ' %';
|
||||
}
|
||||
$('#statistic .table-primary.' + device + '.trusted-ops').text(trusted_op);
|
||||
$('#statistic .table-primary.' + device + '.test_total').text(all_operations || 0);
|
||||
// $('#statistic .table-primary.' + device + '.test_total').text(all_operations || 0);
|
||||
|
||||
// tested op_counter
|
||||
tested_op_count = 0;
|
||||
@ -218,7 +208,7 @@ function calculateColumnStatistics(device) {
|
||||
$('#statistic .table-primary.' + device + '.general_pass_rate').text('---');
|
||||
|
||||
} else {
|
||||
general_pass_rate = (passed_tested_op_count * 100 / tested_op_count).toFixed(1) + ' %';
|
||||
general_pass_rate = (passed_tested_op_count * 100 / tested_op_count).toFixed(2) + ' %';
|
||||
$('#statistic .table-primary.' + device + '.general_pass_rate').text(general_pass_rate);
|
||||
}
|
||||
$('#statistic .table-primary.' + device + '.tested-ops_count').text(tested_op_count);
|
||||
@ -231,9 +221,9 @@ function calculateColumnStatistics(device) {
|
||||
}
|
||||
});
|
||||
if (all_operations == 0) {
|
||||
$('#statistic .table-primary.' + device + '.avg_pass_rate').text('---');
|
||||
$('#statistic .table-primary.' + device + '.rel_pass_rate').text('---');
|
||||
} else {
|
||||
avg_pass_rate = (sum_pass_rate / all_operations).toFixed(1) + ' %';
|
||||
$('#statistic .table-primary.' + device + '.avg_pass_rate').text(avg_pass_rate);
|
||||
rel_pass_rate = (sum_pass_rate / all_operations).toFixed(2) + ' %';
|
||||
$('#statistic .table-primary.' + device + '.rel_pass_rate').text(rel_pass_rate);
|
||||
}
|
||||
}
|
||||
|
||||
@ -7,98 +7,121 @@
|
||||
<!-- Bootstrap CSS -->
|
||||
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css"
|
||||
integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
|
||||
<link rel="stylesheet" href="template/style.css" />
|
||||
<title>{% block title %}highlight_table{% endblock %}</title>
|
||||
</head>
|
||||
<body>
|
||||
{% block content%}
|
||||
<h1 class="ml-3 mt-3">Highlights</h1>
|
||||
<ul class="list-group list-group-flush">
|
||||
{% for test_mode in expected_test_mode %}
|
||||
<li class="list-group-item">
|
||||
{% if expected_test_mode|length > 1 %}
|
||||
<h3><span class="mr-3">●</span>{{ test_mode }}</h3>
|
||||
<h1 class="ml-3 mt-3">Conformance Highlights</h1>
|
||||
{% if report_tag != "" or report_version != "" %}
|
||||
<h3 class="ml-3 mt-3">Tag: {{report_tag}} | Version: {{report_version}}</h3>
|
||||
{% endif %}
|
||||
<table class="table table-hover">
|
||||
{% if current_commit != "" %}
|
||||
<h3 class="ml-3 mt-3">Current state: {{current_commit}}</h3>
|
||||
{% endif %}
|
||||
{% if prev_commit != "" %}
|
||||
<h3 class="ml-3 mt-3">Previous state: {{prev_commit}}</h3>
|
||||
{% endif %}
|
||||
{% if ops_info|length == 0 and api_info|length == 0 %}
|
||||
<h3 style="text-align: center;" class="mt-10">Unfortunately report is empty. Data for analysis wasn't found.</h3>
|
||||
{% else %}
|
||||
{% for test_mode in expected_test_mode %}
|
||||
<h3 class="ml-3" style="margin-top: 30px;">
|
||||
{% if expected_test_mode|length > 1 %}
|
||||
<span class="mr-3">●</span>{{ test_mode }}
|
||||
{% endif %}
|
||||
</h3>
|
||||
<table class="table table-hover table-bordered" style="margin-left: auto; margin-right: auto; width: 98%; font-size: 12px;">
|
||||
<thead>
|
||||
<tr>
|
||||
<th class="table-dark">Plugins</th>
|
||||
<th class="diagonal"><span class="th-devices">Devices</span><span class="th-sw-plugin">Statistic</span></th>
|
||||
{% for device in devices %}
|
||||
<th class="table-dark">{{ device }}</th>
|
||||
<th colspan="2" class="table-dark" style="text-align: center; vertical-align: middle !important;">{{ device }}</th>
|
||||
{% endfor %}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tbody style="text-align: center;">
|
||||
<tr>
|
||||
<td class="table-primary">Total ops pass:</td>
|
||||
<td class="table-primary" style="text-align: left; width: 20%;">Total ops pass (total pass)/(all ops amount):</td>
|
||||
{% for device in devices %}
|
||||
<td>
|
||||
{% if device in ops_info[test_mode] %}
|
||||
<td colspan="2" title="{{ops_info[test_mode][device]['title']}}">
|
||||
<!-- 10(+3)/(205)(-10) -->
|
||||
{{ ops_info[test_mode][device]['totalPass'] }}
|
||||
<span>{{ ops_info[test_mode][device]['totalPass'] }}</span>
|
||||
{% if ops_info[test_mode][device]['diffTotalPass'] > 0 %}
|
||||
(<span class="text-success font-weight-bold">+{{ ops_info[test_mode][device]['diffTotalPass'] }}</span>)
|
||||
{% elif ops_info[test_mode][device]['diffTotalPass'] < 0 %}
|
||||
(<span class="text-danger font-weight-bold">{{ ops_info[test_mode][device]['diffTotalPass'] }}</span>)
|
||||
{% endif %}
|
||||
/{{ ops_info[test_mode][device]['totalAmount'] }}
|
||||
/<span>{{ ops_info[test_mode][device]['totalAmount'] }}</span>
|
||||
{% if ops_info[test_mode][device]['diffTotalAmount'] > 0 %}
|
||||
(<span class="text-success font-weight-bold">+{{ ops_info[test_mode][device]['diffTotalAmount'] }}</span>)
|
||||
{% elif ops_info[test_mode][device]['diffTotalAmount'] < 0 %}
|
||||
(<span class="text-danger font-weight-bold">{{ ops_info[test_mode][device]['diffTotalAmount'] }}</span>)
|
||||
{% endif %}
|
||||
{% else %}
|
||||
NOT RUN
|
||||
{% endif %}
|
||||
</td>
|
||||
{% else %}
|
||||
<td colspan="2" title="Opset conformanace wasn't run on {{device}}">
|
||||
NOT RUN
|
||||
</td>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="table-primary">Passrate:</td>
|
||||
<td class="table-primary" style="text-align: left; width: 20%;">Passrate:</td>
|
||||
{% for device in devices %}
|
||||
<td>
|
||||
{% if device in general_pass_rate[test_mode] %}
|
||||
<td title="{{general_pass_rate[test_mode][device]['title']}}">
|
||||
<span>Total:</span>
|
||||
{{ general_pass_rate[test_mode][device]['current'] }}
|
||||
{% if general_pass_rate[test_mode][device]['prev'] > 0 %}
|
||||
(<span class="text-success font-weight-bold">+{{ general_pass_rate[test_mode][device]['prev'] }}</span>)
|
||||
{% elif general_pass_rate[test_mode][device]['prev'] < 0 %}
|
||||
(<span class="text-danger font-weight-bold">{{ general_pass_rate[test_mode][device]['prev'] }}</span>)
|
||||
{% if general_pass_rate[test_mode][device]['diff'] > 0 %}
|
||||
(<span class="text-success font-weight-bold" >+{{ general_pass_rate[test_mode][device]['diff'] }}</span>)
|
||||
{% elif general_pass_rate[test_mode][device]['diff'] < 0 %}
|
||||
(<span class="text-danger font-weight-bold">{{ general_pass_rate[test_mode][device]['diff'] }}</span>)
|
||||
{% endif %}
|
||||
%
|
||||
{% else %}
|
||||
NOT RUN
|
||||
{% endif %}
|
||||
</td>
|
||||
<td title="{{general_pass_rate[test_mode][device]['rel_title']}}">
|
||||
<span>Rel:</span>
|
||||
{{ general_pass_rate[test_mode][device]['rel_current'] }}
|
||||
{% if general_pass_rate[test_mode][device]['rel_diff'] > 0 %}
|
||||
(<span class="text-success font-weight-bold">+{{ general_pass_rate[test_mode][device]['rel_diff'] }}</span>)
|
||||
{% elif general_pass_rate[test_mode][device]['rel_diff'] < 0 %}
|
||||
(<span class="text-danger font-weight-bold">{{ general_pass_rate[test_mode][device]['rel_diff'] }}</span>)
|
||||
{% endif %}
|
||||
%
|
||||
</td>
|
||||
{% else %}
|
||||
<td colspan="2" title="Opset conformanace wasn't run on {{device}}">NOT RUN</td>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</tr>
|
||||
|
||||
</tbody>
|
||||
</table>
|
||||
</li>
|
||||
{% endfor %}
|
||||
{% if api_info.keys()|length > 0 %}
|
||||
<li class="list-group-item">
|
||||
<h3><span class="mr-3">●</span> API </h3>
|
||||
<table class="table table-hover">
|
||||
<h3 class="ml-3" style="margin-top: 30px;"><span class="mr-3">●</span> API </h3>
|
||||
<table class="table table-hover table-bordered" style="margin-left: auto; margin-right: auto; width: 98%; font-size: 12px;">
|
||||
<thead>
|
||||
<tr>
|
||||
<th class="table-dark">Plugins</th>
|
||||
<th class="table-dark">SW Plugins</th>
|
||||
<th class="table-dark" style="text-align: left; vertical-align: middle !important;">Entity</th>
|
||||
<th class="diagonal"><span class="th-devices">Devices</span><span class="th-sw-plugin">Plugins</span></th>
|
||||
{% for device in devices %}
|
||||
<th class="table-dark">{{ device }}</th>
|
||||
<th class="table-dark" colspan="2" style="text-align: center; vertical-align: middle !important;">{{ device }}</th>
|
||||
{% endfor %}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tbody style="text-align: center;">
|
||||
{% for test_type in api_info %}
|
||||
<td class="table-primary" rowspan="{{ api_info[test_type].keys()|length + 1 }}">{{ test_type }}</td>
|
||||
|
||||
<td class="table-primary" style="text-align: left;" rowspan="{{ api_info[test_type].keys()|length + 1 }}" style="vertical-align: middle !important;">{{ test_type }}</td>
|
||||
{% for sw_plugin in sw_plugins %}
|
||||
<tr>
|
||||
<td>{{sw_plugin}}</td>
|
||||
<td style="text-align: left;">{{sw_plugin}}</td>
|
||||
{% for device in devices %}
|
||||
{% if device in api_info[test_type][sw_plugin] %}
|
||||
<td>
|
||||
{{ api_info[test_type][sw_plugin][device]['passrate'] }}
|
||||
<td title="{{api_info[test_type][sw_plugin][device]['title']}}">
|
||||
<span>Total:</span>
|
||||
<span>{{ api_info[test_type][sw_plugin][device]['passrate'] }}</span>
|
||||
{% if api_info[test_type][sw_plugin][device]['diff'] > 0 %}
|
||||
(<span class="text-success font-weight-bold">+{{ api_info[test_type][sw_plugin][device]['diff'] }}</span>)
|
||||
{% elif api_info[test_type][sw_plugin][device]['diff'] < 0 %}
|
||||
@ -106,8 +129,18 @@
|
||||
{% endif %}
|
||||
%
|
||||
</td>
|
||||
<td title="{{api_info[test_type][sw_plugin][device]['rel_title']}}">
|
||||
<span>Rel:</span>
|
||||
<span>{{ api_info[test_type][sw_plugin][device]['rel_passrate'] }}</span>
|
||||
{% if api_info[test_type][sw_plugin][device]['rel_diff'] > 0 %}
|
||||
(<span class="text-success font-weight-bold">+{{ api_info[test_type][sw_plugin][device]['rel_diff'] }}</span>)
|
||||
{% elif api_info[test_type][sw_plugin][device]['rel_diff'] < 0 %}
|
||||
(<span class="text-danger font-weight-bold">{{ api_info[test_type][sw_plugin][device]['rel_diff'] }}</span>)
|
||||
{% endif %}
|
||||
%
|
||||
</td>
|
||||
{% else %}
|
||||
<td>NOT RUN</td>
|
||||
<td colspan="2" style="text-align: center;" title="API conformanace wasn't run on {{device}} {{sw_plugin}}">NOT RUN</td>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</tr>
|
||||
@ -115,9 +148,9 @@
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</li>
|
||||
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</ul>
|
||||
{% endblock %}
|
||||
</body>
|
||||
</html>
|
||||
@ -28,6 +28,12 @@
|
||||
<div>
|
||||
<span class="table-secondary border">N/A</span><span>No Tests</span>
|
||||
</div>
|
||||
<div>
|
||||
<span><b>Passrates are based on relative weights each subgraphs! You can check absolute value in `General passrate` row!</b></span>
|
||||
</div>
|
||||
<div>
|
||||
<span><b>Relative Passrate could be from 0 to 200% now! Relative weight of operation is calculated once for current set of models, but there are more then one tests per model. </b></span>
|
||||
</div>
|
||||
<div>
|
||||
<span><b>Status:</b></span>
|
||||
<span class="green">P:85</span><span>Passed</span>
|
||||
@ -111,15 +117,9 @@
|
||||
{% endfor %}
|
||||
</tr>
|
||||
<tr>
|
||||
<th class="table-primary" scope="row">Tested op counter:</th>
|
||||
<th class="table-primary" scope="row">AVG passrate (=sum_pass_rates/covered_ops_num):</th>
|
||||
{% for d in devices -%}
|
||||
<td class="table-primary {{ d }} tested-ops_count">{{general_test_count[d]}}</td>
|
||||
{% endfor %}
|
||||
</tr>
|
||||
<tr>
|
||||
<th class="table-primary" scope="row">AVG passrate per op (=sum_pass_rates/covered_ops_num):</th>
|
||||
{% for d in devices -%}
|
||||
<td class="table-primary {{ d }} avg_pass_rate">{{pass_rate_avg[d]}} {% if pass_rate_avg[d] != "NOT RUN" -%}%{% endif -%}</td>
|
||||
<td class="table-primary {{ d }} rel_pass_rate">{{pass_rate_avg_rel[d]}} {% if pass_rate_avg_rel[d] != "NOT RUN" -%}%{% endif -%}</td>
|
||||
{% endfor %}
|
||||
</tr>
|
||||
<tr>
|
||||
@ -130,9 +130,10 @@
|
||||
</tr>
|
||||
</tbody>
|
||||
<tbody id="data">
|
||||
{% for op in ordered_ops -%}
|
||||
{% for op, opsets in ordered_ops.items() -%}
|
||||
<tr>
|
||||
<th scope="row" name="{{ op }}">{{ op }}</th>
|
||||
<th scope="row" name='{ "{{ op }}": {{ opsets }} }'>{{ op }}</th>
|
||||
|
||||
|
||||
{% for d in devices -%}
|
||||
{% if op in results[d] -%}
|
||||
@ -143,7 +144,7 @@
|
||||
passed="{{ results[d][op].passed }}" failed="{{ results[d][op].failed }}"
|
||||
skipped="{{ results[d][op].skipped }}" crashed="{{ results[d][op].crashed }}"
|
||||
hanged="{{ results[d][op].hanged }}"
|
||||
value="{% if (results[d][op].passed != '0' or results[d][op].failed != '0' or results[d][op].crashed != '0' or results[d][op].skipped) != '0' -%}{{ results[d][op].passrate }}{% else -%}---{% endif -%}"
|
||||
value="{% if (results[d][op].passed != '0' or results[d][op].failed != '0' or results[d][op].crashed != '0' or results[d][op].skipped) != '0' -%}{{ results[d][op].relative_passrate }}{% else -%}---{% endif -%}"
|
||||
title="{% if results[d][op].implemented == 'true' -%}
|
||||
{{op}} is implemented in {{d}} plugin
|
||||
{% else -%}
|
||||
@ -151,7 +152,7 @@
|
||||
{% endif -%}">
|
||||
<span class="{% if results[d][op].implemented == 'true' -%} checkmark {% else -%} check {% endif -%}"></span>
|
||||
{% if (results[d][op].passed != '0' or results[d][op].failed != '0' or results[d][op].crashed != '0' or results[d][op].skipped != '0' or results[d][op].hanged != '0') -%}
|
||||
{{ results[d][op].passrate }} % <br />
|
||||
{{ results[d][op].relative_passrate }} % <br />
|
||||
{% else -%}
|
||||
---<br />
|
||||
{% endif -%}
|
||||
|
||||
@ -617,3 +617,111 @@ h2 {
|
||||
.not_run:hover {
|
||||
background: #8b000040;
|
||||
}
|
||||
|
||||
.scope_stable {
|
||||
display: flex !important;
|
||||
align-items: center;
|
||||
width: 270px;
|
||||
height: 100px;
|
||||
}
|
||||
|
||||
.switch {
|
||||
position: relative;
|
||||
display: inline-block;
|
||||
width: 200px;
|
||||
height: 38px;
|
||||
background-color: #ccc;
|
||||
border-radius: 20px;
|
||||
}
|
||||
|
||||
/* hide HTML checkbox */
|
||||
.switch input {
|
||||
opacity: 0;
|
||||
width: 0;
|
||||
height: 0;
|
||||
}
|
||||
|
||||
.slider {
|
||||
border-radius: 20px;
|
||||
position: absolute;
|
||||
height: 30px;
|
||||
width: 80px;
|
||||
left: 5px;
|
||||
bottom: 4px;
|
||||
background-color: white;
|
||||
font-weight: bold;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
-webkit-transition: .4s;
|
||||
transition: .4s;
|
||||
}
|
||||
|
||||
input:checked + .slider {
|
||||
-webkit-transform: translateX(110px);
|
||||
-ms-transform: translateX(110px);
|
||||
transform: translateX(110px);
|
||||
}
|
||||
|
||||
.mandatory {
|
||||
border-radius: 20px;
|
||||
position: absolute;
|
||||
height: 30px;
|
||||
width: 80px;
|
||||
left: 5px;
|
||||
bottom: 4px;
|
||||
background-color: rgb(173, 166, 166);
|
||||
opacity: 0.5;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.optional {
|
||||
border-radius: 20px;
|
||||
position: absolute;
|
||||
height: 30px;
|
||||
width: 80px;
|
||||
left: 115px;
|
||||
bottom: 4px;
|
||||
background-color: rgb(173, 166, 166);
|
||||
opacity: 0.5;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.toggle_arrow {
|
||||
display: flex;
|
||||
font-size: 20px;
|
||||
justify-content: center;
|
||||
opacity: 0.5;
|
||||
}
|
||||
|
||||
.diagonal {
|
||||
position: relative;
|
||||
background: linear-gradient(to right top, #000000 0%,#000000 46.9%,#ffffff 47%,#ffffff 51%,#000000 51.1%,#000000 100%) !important;
|
||||
background-color: transparent;
|
||||
background-color: none !important;
|
||||
}
|
||||
|
||||
.th-sw-plugin {
|
||||
color: white;
|
||||
}
|
||||
|
||||
.th-devices {
|
||||
color: white;
|
||||
display: block;
|
||||
text-align: end;
|
||||
}
|
||||
|
||||
.device {
|
||||
vertical-align: middle !important;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.entity {
|
||||
vertical-align: middle !important;
|
||||
width: 20%;
|
||||
}
|
||||
|
||||
|
||||
@ -9,9 +9,6 @@
|
||||
This page lists operations supported by OpenVINO Framework Frontend.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
.. tab-set::
|
||||
|
||||
.. tab-item:: PyTorch
|
||||
@ -262,6 +259,7 @@ This page lists operations supported by OpenVINO Framework Frontend.
|
||||
Abs
|
||||
Acos
|
||||
Acosh
|
||||
Add
|
||||
And
|
||||
ArgMin
|
||||
ArgMax
|
||||
|
||||
Loading…
Reference in New Issue
Block a user