IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[DOCS] conformance update (#18732)

Browse Source 
conformance table
fix Add in ONNX layers
This commit is contained in:
Karol Blaszczak 2023-07-24 11:43:33 +02:00 committed by GitHub
parent 5833f32c22
commit bfe3faca1e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 12349 additions and 4691 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16692
docs/_static/download/operation_conformance_table_files/opset_report_omz_static.html vendored
View File

File diff suppressed because it is too large Load Diff

28
docs/_static/download/operation_conformance_table_files/template/filters.js vendored
View File

@ -161,20 +161,10 @@ function filterTable() {
} }
function checkVersion(element, opsetNumber) { function checkVersion(element, opsetNumber) {
var name = element.find('th').text().split("-")[0]; var name = element.find('th')[0].getAttribute('name');
var version = Number(element.find('th').text().split("-")[1]); var opsets = JSON.parse(name)[element.find('th').text()];
var realOpsetNumber = Number(opsetNumber); var realOpsetNumber = Number(opsetNumber);
if (version > realOpsetNumber) { return opsets.indexOf(realOpsetNumber) != -1;
return false;
} else {
var versions = [];
$('#report #data tr th[name^="' + name + '-"]').each(function () {
if (Number($(this).text().split('-')[1]) <= realOpsetNumber) {
versions.push(Number(+$(this).text().split('-')[1]));
}
});
return version == Math.max.apply(null, versions);
}
} }
function calculateStatistics() { function calculateStatistics() {
@ -195,10 +185,10 @@ function calculateColumnStatistics(device) {
if (!all_operations) { if (!all_operations) {
trusted_op = "---"; trusted_op = "---";
} else { } else {
trusted_op = (count_trusted_op * 100 / all_operations).toFixed(1) + ' %'; trusted_op = (count_trusted_op * 100 / all_operations).toFixed(2) + ' %';
} }
$('#statistic .table-primary.' + device + '.trusted-ops').text(trusted_op); $('#statistic .table-primary.' + device + '.trusted-ops').text(trusted_op);
$('#statistic .table-primary.' + device + '.test_total').text(all_operations || 0); // $('#statistic .table-primary.' + device + '.test_total').text(all_operations || 0);
// tested op_counter // tested op_counter
tested_op_count = 0; tested_op_count = 0;
@ -218,7 +208,7 @@ function calculateColumnStatistics(device) {
$('#statistic .table-primary.' + device + '.general_pass_rate').text('---'); $('#statistic .table-primary.' + device + '.general_pass_rate').text('---');
} else { } else {
general_pass_rate = (passed_tested_op_count * 100 / tested_op_count).toFixed(1) + ' %'; general_pass_rate = (passed_tested_op_count * 100 / tested_op_count).toFixed(2) + ' %';
$('#statistic .table-primary.' + device + '.general_pass_rate').text(general_pass_rate); $('#statistic .table-primary.' + device + '.general_pass_rate').text(general_pass_rate);
} }
$('#statistic .table-primary.' + device + '.tested-ops_count').text(tested_op_count); $('#statistic .table-primary.' + device + '.tested-ops_count').text(tested_op_count);
@ -231,9 +221,9 @@ function calculateColumnStatistics(device) {
} }
}); });
if (all_operations == 0) { if (all_operations == 0) {
$('#statistic .table-primary.' + device + '.avg_pass_rate').text('---'); $('#statistic .table-primary.' + device + '.rel_pass_rate').text('---');
} else { } else {
avg_pass_rate = (sum_pass_rate / all_operations).toFixed(1) + ' %'; rel_pass_rate = (sum_pass_rate / all_operations).toFixed(2) + ' %';
$('#statistic .table-primary.' + device + '.avg_pass_rate').text(avg_pass_rate); $('#statistic .table-primary.' + device + '.rel_pass_rate').text(rel_pass_rate);
} }
} }

179
docs/_static/download/operation_conformance_table_files/template/highlight_tables_template.html vendored
View File

@ -7,98 +7,121 @@
<!-- Bootstrap CSS --> <!-- Bootstrap CSS -->
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css"
integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous"> integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
<link rel="stylesheet" href="template/style.css" />
<title>{% block title %}highlight_table{% endblock %}</title> <title>{% block title %}highlight_table{% endblock %}</title>
</head> </head>
<body> <body>
{% block content%} {% block content%}
<h1 class="ml-3 mt-3">Highlights</h1> <h1 class="ml-3 mt-3">Conformance Highlights</h1>
<ul class="list-group list-group-flush"> {% if report_tag != "" or report_version != "" %}
<h3 class="ml-3 mt-3">Tag: {{report_tag}} | Version: {{report_version}}</h3>
{% endif %}
{% if current_commit != "" %}
<h3 class="ml-3 mt-3">Current state: {{current_commit}}</h3>
{% endif %}
{% if prev_commit != "" %}
<h3 class="ml-3 mt-3">Previous state: {{prev_commit}}</h3>
{% endif %}
{% if ops_info|length == 0 and api_info|length == 0 %}
<h3 style="text-align: center;" class="mt-10">Unfortunately report is empty. Data for analysis wasn't found.</h3>
{% else %}
{% for test_mode in expected_test_mode %} {% for test_mode in expected_test_mode %}
<li class="list-group-item"> <h3 class="ml-3" style="margin-top: 30px;">
{% if expected_test_mode|length > 1 %} {% if expected_test_mode|length > 1 %}
<h3><span class="mr-3">&#9679</span>{{ test_mode }}</h3> <span class="mr-3">&#9679</span>{{ test_mode }}
{% endif %} {% endif %}
<table class="table table-hover"> </h3>
<thead> <table class="table table-hover table-bordered" style="margin-left: auto; margin-right: auto; width: 98%; font-size: 12px;">
<tr> <thead>
<th class="table-dark">Plugins</th> <tr>
{% for device in devices %} <th class="diagonal"><span class="th-devices">Devices</span><span class="th-sw-plugin">Statistic</span></th>
<th class="table-dark">{{ device }}</th> {% for device in devices %}
{% endfor %} <th colspan="2" class="table-dark" style="text-align: center; vertical-align: middle !important;">{{ device }}</th>
</tr> {% endfor %}
</thead> </tr>
<tbody> </thead>
<tr> <tbody style="text-align: center;">
<td class="table-primary">Total ops pass:</td> <tr>
{% for device in devices %} <td class="table-primary" style="text-align: left; width: 20%;">Total ops pass (total pass)/(all ops amount):</td>
<td> {% for device in devices %}
{% if device in ops_info[test_mode] %} {% if device in ops_info[test_mode] %}
<!-- 10(+3)/(205)(-10) --> <td colspan="2" title="{{ops_info[test_mode][device]['title']}}">
{{ ops_info[test_mode][device]['totalPass'] }} <!-- 10(+3)/(205)(-10) -->
{% if ops_info[test_mode][device]['diffTotalPass'] > 0 %} <span>{{ ops_info[test_mode][device]['totalPass'] }}</span>
(<span class="text-success font-weight-bold">+{{ ops_info[test_mode][device]['diffTotalPass'] }}</span>) {% if ops_info[test_mode][device]['diffTotalPass'] > 0 %}
{% elif ops_info[test_mode][device]['diffTotalPass'] < 0 %} (<span class="text-success font-weight-bold">+{{ ops_info[test_mode][device]['diffTotalPass'] }}</span>)
(<span class="text-danger font-weight-bold">{{ ops_info[test_mode][device]['diffTotalPass'] }}</span>) {% elif ops_info[test_mode][device]['diffTotalPass'] < 0 %}
{% endif %} (<span class="text-danger font-weight-bold">{{ ops_info[test_mode][device]['diffTotalPass'] }}</span>)
/{{ ops_info[test_mode][device]['totalAmount'] }} {% endif %}
{% if ops_info[test_mode][device]['diffTotalAmount'] > 0 %} /<span>{{ ops_info[test_mode][device]['totalAmount'] }}</span>
(<span class="text-success font-weight-bold">+{{ ops_info[test_mode][device]['diffTotalAmount'] }}</span>) {% if ops_info[test_mode][device]['diffTotalAmount'] > 0 %}
{% elif ops_info[test_mode][device]['diffTotalAmount'] < 0 %} (<span class="text-success font-weight-bold">+{{ ops_info[test_mode][device]['diffTotalAmount'] }}</span>)
(<span class="text-danger font-weight-bold">{{ ops_info[test_mode][device]['diffTotalAmount'] }}</span>) {% elif ops_info[test_mode][device]['diffTotalAmount'] < 0 %}
{% endif %} (<span class="text-danger font-weight-bold">{{ ops_info[test_mode][device]['diffTotalAmount'] }}</span>)
{% else %}
NOT RUN
{% endif %} {% endif %}
</td> </td>
{% endfor %} {% else %}
</tr> <td colspan="2" title="Opset conformanace wasn't run on {{device}}">
<tr> NOT RUN
<td class="table-primary">Passrate:</td>
{% for device in devices %}
<td>
{% if device in general_pass_rate[test_mode] %}
{{ general_pass_rate[test_mode][device]['current'] }}
{% if general_pass_rate[test_mode][device]['prev'] > 0 %}
(<span class="text-success font-weight-bold">+{{ general_pass_rate[test_mode][device]['prev'] }}</span>)
{% elif general_pass_rate[test_mode][device]['prev'] < 0 %}
(<span class="text-danger font-weight-bold">{{ general_pass_rate[test_mode][device]['prev'] }}</span>)
{% endif %}
%
{% else %}
NOT RUN
{% endif %}
</td> </td>
{% endfor %} {% endif %}
</tr> {% endfor %}
</tr>
</tbody> <tr>
</table> <td class="table-primary" style="text-align: left; width: 20%;">Passrate:</td>
</li> {% for device in devices %}
{% if device in general_pass_rate[test_mode] %}
<td title="{{general_pass_rate[test_mode][device]['title']}}">
<span>Total:</span>
{{ general_pass_rate[test_mode][device]['current'] }}
{% if general_pass_rate[test_mode][device]['diff'] > 0 %}
(<span class="text-success font-weight-bold" >+{{ general_pass_rate[test_mode][device]['diff'] }}</span>)
{% elif general_pass_rate[test_mode][device]['diff'] < 0 %}
(<span class="text-danger font-weight-bold">{{ general_pass_rate[test_mode][device]['diff'] }}</span>)
{% endif %}
%
</td>
<td title="{{general_pass_rate[test_mode][device]['rel_title']}}">
<span>Rel:</span>
{{ general_pass_rate[test_mode][device]['rel_current'] }}
{% if general_pass_rate[test_mode][device]['rel_diff'] > 0 %}
(<span class="text-success font-weight-bold">+{{ general_pass_rate[test_mode][device]['rel_diff'] }}</span>)
{% elif general_pass_rate[test_mode][device]['rel_diff'] < 0 %}
(<span class="text-danger font-weight-bold">{{ general_pass_rate[test_mode][device]['rel_diff'] }}</span>)
{% endif %}
%
</td>
{% else %}
<td colspan="2" title="Opset conformanace wasn't run on {{device}}">NOT RUN</td>
{% endif %}
{% endfor %}
</tr>
</tbody>
</table>
{% endfor %} {% endfor %}
{% if api_info.keys()|length > 0 %} {% if api_info.keys()|length > 0 %}
<li class="list-group-item"> <h3 class="ml-3" style="margin-top: 30px;"><span class="mr-3">&#9679</span> API </h3>
<h3><span class="mr-3">&#9679</span> API </h3> <table class="table table-hover table-bordered" style="margin-left: auto; margin-right: auto; width: 98%; font-size: 12px;">
<table class="table table-hover">
<thead> <thead>
<tr> <tr>
<th class="table-dark">Plugins</th> <th class="table-dark" style="text-align: left; vertical-align: middle !important;">Entity</th>
<th class="table-dark">SW Plugins</th> <th class="diagonal"><span class="th-devices">Devices</span><span class="th-sw-plugin">Plugins</span></th>
{% for device in devices %} {% for device in devices %}
<th class="table-dark">{{ device }}</th> <th class="table-dark" colspan="2" style="text-align: center; vertical-align: middle !important;">{{ device }}</th>
{% endfor %} {% endfor %}
</tr> </tr>
</thead> </thead>
<tbody> <tbody style="text-align: center;">
{% for test_type in api_info %} {% for test_type in api_info %}
<td class="table-primary" rowspan="{{ api_info[test_type].keys()|length + 1 }}">{{ test_type }}</td> <td class="table-primary" style="text-align: left;" rowspan="{{ api_info[test_type].keys()|length + 1 }}" style="vertical-align: middle !important;">{{ test_type }}</td>
{% for sw_plugin in sw_plugins %} {% for sw_plugin in sw_plugins %}
<tr> <tr>
<td>{{sw_plugin}}</td> <td style="text-align: left;">{{sw_plugin}}</td>
{% for device in devices %} {% for device in devices %}
{% if device in api_info[test_type][sw_plugin] %} {% if device in api_info[test_type][sw_plugin] %}
<td> <td title="{{api_info[test_type][sw_plugin][device]['title']}}">
{{ api_info[test_type][sw_plugin][device]['passrate'] }} <span>Total:</span>
<span>{{ api_info[test_type][sw_plugin][device]['passrate'] }}</span>
{% if api_info[test_type][sw_plugin][device]['diff'] > 0 %} {% if api_info[test_type][sw_plugin][device]['diff'] > 0 %}
(<span class="text-success font-weight-bold">+{{ api_info[test_type][sw_plugin][device]['diff'] }}</span>) (<span class="text-success font-weight-bold">+{{ api_info[test_type][sw_plugin][device]['diff'] }}</span>)
{% elif api_info[test_type][sw_plugin][device]['diff'] < 0 %} {% elif api_info[test_type][sw_plugin][device]['diff'] < 0 %}
@ -106,8 +129,18 @@
{% endif %} {% endif %}
% %
</td> </td>
<td title="{{api_info[test_type][sw_plugin][device]['rel_title']}}">
<span>Rel:</span>
<span>{{ api_info[test_type][sw_plugin][device]['rel_passrate'] }}</span>
{% if api_info[test_type][sw_plugin][device]['rel_diff'] > 0 %}
(<span class="text-success font-weight-bold">+{{ api_info[test_type][sw_plugin][device]['rel_diff'] }}</span>)
{% elif api_info[test_type][sw_plugin][device]['rel_diff'] < 0 %}
(<span class="text-danger font-weight-bold">{{ api_info[test_type][sw_plugin][device]['rel_diff'] }}</span>)
{% endif %}
%
</td>
{% else %} {% else %}
<td>NOT RUN</td> <td colspan="2" style="text-align: center;" title="API conformanace wasn't run on {{device}} {{sw_plugin}}">NOT RUN</td>
{% endif %} {% endif %}
{% endfor %} {% endfor %}
</tr> </tr>
@ -115,9 +148,9 @@
{% endfor %} {% endfor %}
</tbody> </tbody>
</table> </table>
</li>
{% endif %} {% endif %}
</ul> {% endif %}
{% endblock %} {% endblock %}
</body> </body>
</html> </html>

27
docs/_static/download/operation_conformance_table_files/template/report_template.html vendored
View File

@ -28,6 +28,12 @@
<div> <div>
<span class="table-secondary border">N/A</span><span>No Tests</span> <span class="table-secondary border">N/A</span><span>No Tests</span>
</div> </div>
<div>
<span><b>Passrates are based on relative weights each subgraphs! You can check absolute value in `General passrate` row!</b></span>
</div>
<div>
<span><b>Relative Passrate could be from 0 to 200% now! Relative weight of operation is calculated once for current set of models, but there are more then one tests per model. </b></span>
</div>
<div> <div>
<span><b>Status:</b></span> <span><b>Status:</b></span>
<span class="green">P:85</span><span>Passed</span> <span class="green">P:85</span><span>Passed</span>
@ -111,15 +117,9 @@
{% endfor %} {% endfor %}
</tr> </tr>
<tr> <tr>
<th class="table-primary" scope="row">Tested op counter:</th> <th class="table-primary" scope="row">AVG passrate (=sum_pass_rates/covered_ops_num):</th>
{% for d in devices -%} {% for d in devices -%}
<td class="table-primary {{ d }} tested-ops_count">{{general_test_count[d]}}</td> <td class="table-primary {{ d }} rel_pass_rate">{{pass_rate_avg_rel[d]}} {% if pass_rate_avg_rel[d] != "NOT RUN" -%}%{% endif -%}</td>
{% endfor %}
</tr>
<tr>
<th class="table-primary" scope="row">AVG passrate per op (=sum_pass_rates/covered_ops_num):</th>
{% for d in devices -%}
<td class="table-primary {{ d }} avg_pass_rate">{{pass_rate_avg[d]}} {% if pass_rate_avg[d] != "NOT RUN" -%}%{% endif -%}</td>
{% endfor %} {% endfor %}
</tr> </tr>
<tr> <tr>
@ -130,9 +130,10 @@
</tr> </tr>
</tbody> </tbody>
<tbody id="data"> <tbody id="data">
{% for op in ordered_ops -%} {% for op, opsets in ordered_ops.items() -%}
<tr> <tr>
<th scope="row" name="{{ op }}">{{ op }}</th> <th scope="row" name='{ "{{ op }}": {{ opsets }} }'>{{ op }}</th>
{% for d in devices -%} {% for d in devices -%}
{% if op in results[d] -%} {% if op in results[d] -%}
@ -143,7 +144,7 @@
passed="{{ results[d][op].passed }}" failed="{{ results[d][op].failed }}" passed="{{ results[d][op].passed }}" failed="{{ results[d][op].failed }}"
skipped="{{ results[d][op].skipped }}" crashed="{{ results[d][op].crashed }}" skipped="{{ results[d][op].skipped }}" crashed="{{ results[d][op].crashed }}"
hanged="{{ results[d][op].hanged }}" hanged="{{ results[d][op].hanged }}"
value="{% if (results[d][op].passed != '0' or results[d][op].failed != '0' or results[d][op].crashed != '0' or results[d][op].skipped) != '0' -%}{{ results[d][op].passrate }}{% else -%}---{% endif -%}" value="{% if (results[d][op].passed != '0' or results[d][op].failed != '0' or results[d][op].crashed != '0' or results[d][op].skipped) != '0' -%}{{ results[d][op].relative_passrate }}{% else -%}---{% endif -%}"
title="{% if results[d][op].implemented == 'true' -%} title="{% if results[d][op].implemented == 'true' -%}
{{op}} is implemented in {{d}} plugin {{op}} is implemented in {{d}} plugin
{% else -%} {% else -%}
@ -151,7 +152,7 @@
{% endif -%}"> {% endif -%}">
<span class="{% if results[d][op].implemented == 'true' -%} checkmark {% else -%} check {% endif -%}"></span> <span class="{% if results[d][op].implemented == 'true' -%} checkmark {% else -%} check {% endif -%}"></span>
{% if (results[d][op].passed != '0' or results[d][op].failed != '0' or results[d][op].crashed != '0' or results[d][op].skipped != '0' or results[d][op].hanged != '0') -%} {% if (results[d][op].passed != '0' or results[d][op].failed != '0' or results[d][op].crashed != '0' or results[d][op].skipped != '0' or results[d][op].hanged != '0') -%}
{{ results[d][op].passrate }} % <br /> {{ results[d][op].relative_passrate }} % <br />
{% else -%} {% else -%}
---<br /> ---<br />
{% endif -%} {% endif -%}
@ -185,4 +186,4 @@
</div> </div>
</body> </body>
</html> </html>

110
docs/_static/download/operation_conformance_table_files/template/style.css vendored
View File

@ -616,4 +616,112 @@ h2 {
.not_run:hover { .not_run:hover {
background: #8b000040; background: #8b000040;
} }
.scope_stable {
display: flex !important;
align-items: center;
width: 270px;
height: 100px;
}
.switch {
position: relative;
display: inline-block;
width: 200px;
height: 38px;
background-color: #ccc;
border-radius: 20px;
}
/* hide HTML checkbox */
.switch input {
opacity: 0;
width: 0;
height: 0;
}
.slider {
border-radius: 20px;
position: absolute;
height: 30px;
width: 80px;
left: 5px;
bottom: 4px;
background-color: white;
font-weight: bold;
display: flex;
align-items: center;
justify-content: center;
-webkit-transition: .4s;
transition: .4s;
}
input:checked + .slider {
-webkit-transform: translateX(110px);
-ms-transform: translateX(110px);
transform: translateX(110px);
}
.mandatory {
border-radius: 20px;
position: absolute;
height: 30px;
width: 80px;
left: 5px;
bottom: 4px;
background-color: rgb(173, 166, 166);
opacity: 0.5;
display: flex;
align-items: center;
justify-content: center;
}
.optional {
border-radius: 20px;
position: absolute;
height: 30px;
width: 80px;
left: 115px;
bottom: 4px;
background-color: rgb(173, 166, 166);
opacity: 0.5;
display: flex;
align-items: center;
justify-content: center;
}
.toggle_arrow {
display: flex;
font-size: 20px;
justify-content: center;
opacity: 0.5;
}
.diagonal {
position: relative;
background: linear-gradient(to right top, #000000 0%,#000000 46.9%,#ffffff 47%,#ffffff 51%,#000000 51.1%,#000000 100%) !important;
background-color: transparent;
background-color: none !important;
}
.th-sw-plugin {
color: white;
}
.th-devices {
color: white;
display: block;
text-align: end;
}
.device {
vertical-align: middle !important;
font-weight: bold;
}
.entity {
vertical-align: middle !important;
width: 20%;
}

4
docs/resources/supported_operations_framework_frontend.md
View File

@ -9,9 +9,6 @@
This page lists operations supported by OpenVINO Framework Frontend. This page lists operations supported by OpenVINO Framework Frontend.
.. tab-set:: .. tab-set::
.. tab-item:: PyTorch .. tab-item:: PyTorch
@ -262,6 +259,7 @@ This page lists operations supported by OpenVINO Framework Frontend.
Abs Abs
Acos Acos
Acosh Acosh
Add
And And
ArgMin ArgMin
ArgMax ArgMax