From d55e67736b252d20f9b9f1b929f4f9555d9fcef9 Mon Sep 17 00:00:00 2001 From: Irina Efode Date: Tue, 7 Dec 2021 22:17:00 +0300 Subject: [PATCH] [IE TESTS][IE CONFORMANCE] Implement infra to check operation implementation status in plugin (#8606) * Init * ini2 * init 3 * s * Link error * Init run * Provide BinaryEltwiseExamples * Report * Update merge script * Remove extra * Filter + interface * sssd * Statistic * Fix Win * Update summary.cpp --- .../src/op_impl_check/op_impl_check.cpp | 21 ++++++ .../include/op_impl_check/op_impl_check.hpp | 41 ++++++++++ .../include/op_impl_check/single_op_graph.hpp | 35 +++++++++ .../src/op_impl_check/om_impl_check.cpp | 67 +++++++++++++++++ .../src/op_impl_check/single_op_graph.cpp | 75 +++++++++++++++++++ .../shared_test_classes/base/ov_subgraph.hpp | 2 +- .../layer_test_utils/summary.hpp | 13 ++++ .../layer_tests_summary/merge_xmls.py | 6 ++ .../layer_tests_summary/summarize.py | 24 ++++-- .../layer_tests_summary/template/filters.js | 38 ++++++---- .../template/report_template.html | 59 ++++++++++----- .../layer_tests_summary/template/style.css | 70 ++++++++++++++++- .../layer_tests_summary/utils/utils.py | 7 +- .../src/layer_test_utils/summary.cpp | 59 +++++++++++++++ 14 files changed, 471 insertions(+), 46 deletions(-) create mode 100644 inference-engine/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/op_impl_check/op_impl_check.cpp create mode 100644 inference-engine/tests/functional/plugin/shared/include/op_impl_check/op_impl_check.hpp create mode 100644 inference-engine/tests/functional/plugin/shared/include/op_impl_check/single_op_graph.hpp create mode 100644 inference-engine/tests/functional/plugin/shared/src/op_impl_check/om_impl_check.cpp create mode 100644 inference-engine/tests/functional/plugin/shared/src/op_impl_check/single_op_graph.cpp diff --git a/inference-engine/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/op_impl_check/op_impl_check.cpp b/inference-engine/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/op_impl_check/op_impl_check.cpp new file mode 100644 index 00000000000..8412d98c288 --- /dev/null +++ b/inference-engine/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/op_impl_check/op_impl_check.cpp @@ -0,0 +1,21 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "op_impl_check/op_impl_check.hpp" +#include "op_impl_check/single_op_graph.hpp" +#include "conformance.hpp" + +namespace ConformanceTests { +using namespace ov::test::subgraph; + +namespace { +INSTANTIATE_TEST_SUITE_P(conformance, + OpImplCheckTest, + ::testing::Combine( + ::testing::ValuesIn(createFunctions()), + ::testing::Values(targetDevice), + ::testing::Values(std::map())), + OpImplCheckTest::getTestCaseName); +} // namespace +} // namespace ConformanceTests \ No newline at end of file diff --git a/inference-engine/tests/functional/plugin/shared/include/op_impl_check/op_impl_check.hpp b/inference-engine/tests/functional/plugin/shared/include/op_impl_check/op_impl_check.hpp new file mode 100644 index 00000000000..a4145d4a4a4 --- /dev/null +++ b/inference-engine/tests/functional/plugin/shared/include/op_impl_check/op_impl_check.hpp @@ -0,0 +1,41 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "gtest/gtest.h" + +#include "common_test_utils/test_common.hpp" +#include "common_test_utils/common_utils.hpp" + +#include "functional_test_utils/layer_test_utils/summary.hpp" +#include "functional_test_utils/ov_plugin_cache.hpp" + +namespace ov { +namespace test { +namespace subgraph { + +using OpImplParams = std::tuple< + std::pair>, // Function to check + std::string, // Target Device + std::map>; // Plugin Config + +class OpImplCheckTest : public testing::WithParamInterface, + public CommonTestUtils::TestsCommon { +protected: + LayerTestsUtils::Summary& summary = LayerTestsUtils::Summary::getInstance(); + std::shared_ptr core = ov::test::utils::PluginCache::get().core(); + std::shared_ptr function; + std::string targetDevice; + std::map configuration; + +public: + void run(); + void SetUp() override; + static std::string getTestCaseName(const testing::TestParamInfo &obj); +}; + +} // namespace subgraph +} // namespace test +} // namespace ov \ No newline at end of file diff --git a/inference-engine/tests/functional/plugin/shared/include/op_impl_check/single_op_graph.hpp b/inference-engine/tests/functional/plugin/shared/include/op_impl_check/single_op_graph.hpp new file mode 100644 index 00000000000..c5b69d8a545 --- /dev/null +++ b/inference-engine/tests/functional/plugin/shared/include/op_impl_check/single_op_graph.hpp @@ -0,0 +1,35 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +namespace ov { +namespace test { +namespace subgraph { + +using OpGenerator = std::map()>>; +OpGenerator getOpGeneratorMap(); + +static const std::vector>> createFunctions() { + std::vector>> res; + auto opsets = LayerTestsUtils::Summary::getInstance().getOpSets(); + auto opGenerator = getOpGeneratorMap(); + std::set opsInfo; + for (const auto& opset : opsets) { + const auto &type_info_set = opset.get_type_info_set(); + opsInfo.insert(type_info_set.begin(), type_info_set.end()); + } + + for (const auto& type_info : opsInfo) { + res.push_back({type_info, opGenerator.find(type_info)->second()}); + } + return res; +} + +} // namespace subgraph +} // namespace test +} // namespace ov \ No newline at end of file diff --git a/inference-engine/tests/functional/plugin/shared/src/op_impl_check/om_impl_check.cpp b/inference-engine/tests/functional/plugin/shared/src/op_impl_check/om_impl_check.cpp new file mode 100644 index 00000000000..21e823409c2 --- /dev/null +++ b/inference-engine/tests/functional/plugin/shared/src/op_impl_check/om_impl_check.cpp @@ -0,0 +1,67 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include +#ifdef _WIN32 +#include +#endif + +#include "op_impl_check/op_impl_check.hpp" + +namespace ov { +namespace test { +namespace subgraph { + +void OpImplCheckTest::run() { + if (function == nullptr) { + GTEST_FAIL() << "Target function is empty!"; + } + auto crashHandler = [](int errCode) { + auto& s = LayerTestsUtils::Summary::getInstance(); + s.saveReport(); + std::cerr << "Unexpected application crash with code: " << errCode << std::endl; + std::abort(); + }; + signal(SIGSEGV, crashHandler); + + summary.setDeviceName(targetDevice); + try { + auto executableNetwork = core->compile_model(function, targetDevice, configuration); + summary.updateOPsImplStatus(function, true); + } catch (...) { + summary.updateOPsImplStatus(function, false); + GTEST_FAIL() << "Error in the LoadNetwork!"; + } +} + +void OpImplCheckTest::SetUp() { + std::pair> funcInfo; + std::tie(funcInfo, targetDevice, configuration) = this->GetParam(); + function = funcInfo.second; +} + +std::string OpImplCheckTest::getTestCaseName(const testing::TestParamInfo &obj) { + std::pair> funcInfo; + std::string targetDevice; + std::map config; + std::tie(funcInfo, targetDevice, config) = obj.param; + + std::ostringstream result; + std::string friendlyName = funcInfo.first.name + std::string("_") + funcInfo.first.get_version(); + result << "Function=" << friendlyName << "_"; + result << "Device=" << targetDevice << "_"; + result << "Config=("; + for (const auto& configItem : config) { + result << configItem.first << "=" << configItem.second << "_"; + } + result << ")"; + return result.str(); +} + +TEST_P(OpImplCheckTest, checkPluginImplementation) { + run(); +} + +} // namespace subgraph +} // namespace test +} // namespace ov \ No newline at end of file diff --git a/inference-engine/tests/functional/plugin/shared/src/op_impl_check/single_op_graph.cpp b/inference-engine/tests/functional/plugin/shared/src/op_impl_check/single_op_graph.cpp new file mode 100644 index 00000000000..03231e5c2fc --- /dev/null +++ b/inference-engine/tests/functional/plugin/shared/src/op_impl_check/single_op_graph.cpp @@ -0,0 +1,75 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include + +namespace ov { +namespace test { +namespace subgraph { + +namespace { +std::shared_ptr generate(const std::shared_ptr &node) { + return nullptr; +} + +std::shared_ptr generateBinaryEltwise(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 2}, + {1, 2}}); + std::shared_ptr eltwiseNode; + if (ov::is_type(node)) { + eltwiseNode = std::make_shared(params.front(), params.back()); + } else if (ov::is_type(node)) { + eltwiseNode = std::make_shared(params.front(), params.back()); + } else if (ov::is_type(node)) { + eltwiseNode = std::make_shared(params.front(), params.back()); + } else if (ov::is_type(node)) { + eltwiseNode = std::make_shared(params.front(), params.back()); + } else if (ov::is_type(node)) { + eltwiseNode = std::make_shared(params.front(), params.back()); + } else if (ov::is_type(node)) { + eltwiseNode = std::make_shared(params.front(), params.back()); + } else if (ov::is_type(node)) { + eltwiseNode = std::make_shared(params.front(), params.back()); + } else if (ov::is_type(node)) { + eltwiseNode = std::make_shared(params.front(), params.back()); + } else if (ov::is_type(node)) { + eltwiseNode = std::make_shared(params.front(), params.back()); + } else { + return nullptr; + } + + ngraph::ResultVector results{std::make_shared(eltwiseNode)}; + return std::make_shared(results, params, "BinaryEltwiseGraph"); +} +} // namespace + +template +std::shared_ptr generateGraph() { + std::shared_ptr node = std::shared_ptr(new T); + if (ov::is_type(node)) { + return generateBinaryEltwise(node); + } + return generate(node); +} + +OpGenerator getOpGeneratorMap() { + static OpGenerator opGeneratorMap{ +#define _OPENVINO_OP_REG(NAME, NAMESPACE) {NAMESPACE::NAME::get_type_info_static(), generateGraph}, +#include "openvino/opsets/opset1_tbl.hpp" +#include "openvino/opsets/opset2_tbl.hpp" +#include "openvino/opsets/opset3_tbl.hpp" +#include "openvino/opsets/opset4_tbl.hpp" +#include "openvino/opsets/opset5_tbl.hpp" +#include "openvino/opsets/opset6_tbl.hpp" +#include "openvino/opsets/opset7_tbl.hpp" +#include "openvino/opsets/opset8_tbl.hpp" +#undef _OPENVINO_OP_REG + }; + return opGeneratorMap; +} + +} // namespace subgraph +} // namespace test +} // namespace ov \ No newline at end of file diff --git a/inference-engine/tests/functional/shared_test_classes/include/shared_test_classes/base/ov_subgraph.hpp b/inference-engine/tests/functional/shared_test_classes/include/shared_test_classes/base/ov_subgraph.hpp index 38796be27e7..ec5daa12143 100644 --- a/inference-engine/tests/functional/shared_test_classes/include/shared_test_classes/base/ov_subgraph.hpp +++ b/inference-engine/tests/functional/shared_test_classes/include/shared_test_classes/base/ov_subgraph.hpp @@ -61,7 +61,7 @@ protected: constexpr static const double disable_threshold = std::numeric_limits::max(); double abs_threshold = disable_threshold, rel_threshold = disable_threshold; - LayerTestsUtils::Summary& summary = LayerTestsUtils::Summary::getInstance();; + LayerTestsUtils::Summary& summary = LayerTestsUtils::Summary::getInstance(); private: std::vector calculate_refs(); diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/layer_test_utils/summary.hpp b/inference-engine/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/layer_test_utils/summary.hpp index ba1002a2f5a..45f897a5731 100644 --- a/inference-engine/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/layer_test_utils/summary.hpp +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/layer_test_utils/summary.hpp @@ -36,6 +36,7 @@ struct PassRate { unsigned long failed = 0; unsigned long skipped = 0; unsigned long crashed = 0; + bool isImplemented = false; PassRate() = default; @@ -44,6 +45,13 @@ struct PassRate { failed = f; skipped = s; crashed = c; + if (!isImplemented && passed > 0) { + isImplemented = true; + } + } + + void setImplementationStatus(bool implStatus) { + isImplemented = implStatus; } float getPassrate() const { @@ -87,10 +95,15 @@ public: std::map getOPsStats() { return opsStats; } void updateOPsStats(const std::shared_ptr &function, const PassRate::Statuses &status); + void updateOPsImplStatus(const std::shared_ptr &function, const bool implStatus); void updateOPsStats(const ngraph::NodeTypeInfo &op, const PassRate::Statuses &status); + void updateOPsImplStatus(const ngraph::NodeTypeInfo &op, const bool implStatus); static Summary &getInstance(); + std::vector getOpSets() { + return opsets; + } // #define IE_TEST_DEBUG diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/merge_xmls.py b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/merge_xmls.py index 7ee9c8dbb3b..5e323f6dab7 100644 --- a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/merge_xmls.py +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/merge_xmls.py @@ -49,6 +49,12 @@ def aggregate_test_results(results: ET.SubElement, xml_reports: list): for attr_name in device_results.find(op.tag).attrib: if attr_name == "passrate": continue + if attr_name == "implemented": + xml_value = op.attrib.get(attr_name) == "true" + aggregated_value = entry.attrib.get(attr_name) == "true" + str_value = "true" if xml_value or aggregated_value else "false" + device_results.find(entry.tag).set(attr_name, str_value) + continue xml_value = int(op.attrib.get(attr_name)) aggregated_value = int(entry.attrib.get(attr_name)) device_results.find(entry.tag).set(attr_name, str(xml_value + aggregated_value)) diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py index bc1db07f863..cc231b6f66f 100644 --- a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py @@ -69,6 +69,9 @@ def merge_xmls(xml_paths: list): for attr_name in device_results.find(op_result.tag).attrib: if attr_name == "passrate": continue + # TODO + if attr_name == "implemented": + continue total_tests_count_xml += int(op_result.attrib.get(attr_name)) total_tests_count_summary += int(current_op_res.attrib.get(attr_name)) if total_tests_count_xml > total_tests_count_summary: @@ -97,6 +100,7 @@ def collect_statistic(root: ET.Element, is_conformance_mode: bool): op_res = dict() results = dict() + covered_ops = dict() for device in root.find("results"): results[device.tag] = {op.tag: op.attrib for op in device} @@ -104,16 +108,20 @@ def collect_statistic(root: ET.Element, is_conformance_mode: bool): general_test_count[device.tag] = 0 general_passed_tests[device.tag] = 0 trusted_ops[device.tag] = 0 + covered_ops[device.tag] = 0 for op in results[device.tag]: + op_test_cnt = int(results[device.tag][op]["passed"]) + int(results[device.tag][op]["failed"]) + \ + int(results[device.tag][op]["crashed"]) + int(results[device.tag][op]["skipped"]) + if op_test_cnt == 0: + continue + covered_ops[device.tag] += 1 pass_rate = round(float(results[device.tag][op]["passrate"]), 1) results[device.tag][op]["passrate"] = pass_rate pass_rate_avg[device.tag] += pass_rate if pass_rate == 100.: trusted_ops[device.tag] += 1 - device_general_test_count = \ - int(results[device.tag][op]["passed"]) + int(results[device.tag][op]["failed"]) +\ - int(results[device.tag][op]["crashed"]) + int(results[device.tag][op]["skipped"]) + device_general_test_count = op_test_cnt general_test_count[device.tag] += device_general_test_count general_passed_tests[device.tag] += int(results[device.tag][op]["passed"]) @@ -123,9 +131,9 @@ def collect_statistic(root: ET.Element, is_conformance_mode: bool): op_res.update({op: {device.tag: device_general_test_count}}) pass_rate_avg[device.tag] /= len(results[device.tag]) pass_rate_avg[device.tag] = round(float(pass_rate_avg[device.tag]), 1) - general_pass_rate[device.tag] = general_passed_tests[device.tag] * 100 / general_test_count[device.tag] + general_pass_rate[device.tag] = 0 if general_test_count[device.tag] == 0 else (general_passed_tests[device.tag] * 100 / general_test_count[device.tag]) general_pass_rate[device.tag] = round(float(general_pass_rate[device.tag]), 1) - trusted_ops[device.tag] = round(float(trusted_ops[device.tag] * 100 / len(results[device.tag])), 1) + trusted_ops[device.tag] = round(float(trusted_ops[device.tag] * 100 / covered_ops[device.tag]), 1) if device.tag in covered_ops and covered_ops[device.tag] != 0 else 0 logger.info("Test number comparison between devices is started") for op in op_res: @@ -145,14 +153,14 @@ def collect_statistic(root: ET.Element, is_conformance_mode: bool): devices = results.keys() logger.info("Statistic collecting is completed") - return devices, results, general_pass_rate, pass_rate_avg, general_test_count, trusted_ops + return devices, results, general_pass_rate, pass_rate_avg, general_test_count, trusted_ops, covered_ops def create_summary(summary_root: ET.Element, output_folder: os.path, report_tag: str, is_conformance_mode: bool, output_filename='report'): if is_conformance_mode: utils.update_conformance_test_counters(summary_root, logger) - device_list, results, general_pass_rate, pass_rate_avg, general_test_count, trusted_ops = \ + device_list, results, general_pass_rate, pass_rate_avg, general_test_count, trusted_ops, covered_ops = \ collect_statistic(summary_root, is_conformance_mode) timestamp = summary_root.attrib["timestamp"] @@ -168,7 +176,7 @@ def create_summary(summary_root: ET.Element, output_folder: os.path, report_tag: res_summary = template.render(ordered_ops=op_list, devices=device_list, results=results, timestamp=timestamp, general_pass_rate=general_pass_rate, pass_rate_avg=pass_rate_avg, - trusted_ops=trusted_ops, + trusted_ops=trusted_ops, covered_ops=covered_ops, general_test_count=general_test_count, report_tag=report_tag) report_path = os.path.join(output_folder, f'{output_filename}.html') diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/filters.js b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/filters.js index dcad4ddb6a1..4cf45cc49f2 100644 --- a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/filters.js +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/filters.js @@ -14,7 +14,7 @@ $(document).ready(function () { $('#operationName').val(''); $('#status').prop("disabled", true).val(''); $('#devices').val(0); - $('#references').val(0); + $('#implementation').val(0); $("#status").chosen("destroy"); $("#status").chosen({max_selected_options: 6}); filterTable(); @@ -79,7 +79,7 @@ function filterTable() { opsetNumber = $("#opsetNumber").val(); operationName = $('#operationName').val().trim(); status = $('#status').val(); - references = $('#references').val(); + implementation = $('#implementation').val(); $("#report #data tr").show(); $('#report').show(); @@ -96,14 +96,18 @@ function filterTable() { }); } - if (references != 0) { - if (references == 'nv') { + if (implementation != 0) { + if (implementation == 'ni') { $("#report #data tr:not(:hidden)").filter(function () { - $(this).toggle($(this).find('th').hasClass("colorRed")) + $(this).toggle($(this).find('td').hasClass("not_impl")) + }); + } else if (implementation == 'i') { + $("#report #data tr:not(:hidden)").filter(function () { + $(this).toggle($(this).find('td').hasClass("impl")); }); } else { $("#report #data tr:not(:hidden)").filter(function () { - $(this).toggle(!$(this).find('th').hasClass("colorRed")); + $(this).toggle(!$(this).find('td').hasClass("not_impl") && !$(this).find('td').hasClass("impl")); }); } } @@ -173,14 +177,14 @@ function calculateColumnStatistics(device) { total = $("#report #data tr:not(:hidden)").length; $('#statistic .table-primary[scope="row"] i').text(total); // trusted op - count_trasted_op = $("#report #data tr:not(:hidden) ." + device + ".value[value^='100'][crashed='0'][failed='0'][skipped='0']").length; - all_operations = $("#report #data tr:not(:hidden) .value." + device).length; + count_trusted_op = $("#report #data tr:not(:hidden) ." + device + ".value[value^='100'][crashed='0'][failed='0'][skipped='0']").length; + all_operations = $("#report #data tr:not(:hidden) .value[value!='N/A'][value!='---']." + device).length; if (!all_operations) { - trasted_op = "---"; + trusted_op = "---"; } else { - trasted_op = (count_trasted_op * 100 / all_operations).toFixed(1) + ' %'; + trusted_op = (count_trusted_op * 100 / all_operations).toFixed(1) + ' %'; } - $('#statistic .table-primary.' + device + '.trusted-ops').text(trasted_op); + $('#statistic .table-primary.' + device + '.trusted-ops').text(trusted_op); $('#statistic .table-primary.' + device + '.test_total').text(all_operations || 0); // tested op_counter @@ -188,10 +192,12 @@ function calculateColumnStatistics(device) { passed_tested_op_count = 0; $("#report #data tr:not(:hidden) ." + device + ".value span").each(function () { text = $(this).text().split(':')[1]; - if ($(this).hasClass('green')) { - passed_tested_op_count += +text; + if (text) { + if ($(this).hasClass('green')) { + passed_tested_op_count += +text; + } + tested_op_count += +text; } - tested_op_count += +text; }); // General Pass Rate @@ -207,7 +213,9 @@ function calculateColumnStatistics(device) { // AVG Pass Rate sum_pass_rate = 0; $("#report #data tr:not(:hidden) ." + device + ".value").each(function () { - sum_pass_rate += +$(this).attr('value'); + if ($(this).attr('value') != 'N/A' && $(this).attr('value') != '---') { + sum_pass_rate += +$(this).attr('value'); + } }); if (all_operations == 0) { $('#statistic .table-primary.' + device + '.avg_pass_rate').text('---'); diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html index ed860be450b..49ca088cc6c 100644 --- a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html @@ -35,6 +35,11 @@ S:2Skipped C:0Crashed +
+ Plugin operation implementation status: +
Implemented
+
Not implemented
+
@@ -48,18 +53,19 @@ -
- - -
+
+ + +
@@ -92,7 +98,7 @@ Total: {{ordered_ops|length}} {% for d in devices -%} - {{results[d]|length}} + {% if d in covered_ops -%} {{covered_ops[d]}} {% else -%} 0 {% endif -%} {% endfor %} @@ -127,17 +133,36 @@ {% for d in devices -%} {% if op in results[d] -%} - - {{ results[d][op].passrate }} %
- P:{{ results[d][op].passed }} - F:{{ results[d][op].failed }} - S:{{ results[d][op].skipped }} - C:{{ results[d][op].crashed }} + value="{% if (results[d][op].passed != '0' or results[d][op].failed != '0' or results[d][op].crashed != '0' or results[d][op].skipped) != '0' -%}{{ results[d][op].passrate }}{% else -%}N/A{% endif -%}" + title="{% if results[d][op].implemented == 'true' -%} + {{op}} is implemented in {{d}} plugin + {% else -%} + {{op}} is not implemented in {{d}} plugin + {% endif -%}"> + {% if (results[d][op].passed != '0' or results[d][op].failed != '0' or results[d][op].crashed != '0' or results[d][op].skipped) != '0' -%} + {{ results[d][op].passrate }} %
+ {% else -%} + ---
+ {% endif -%} +
+
+ {% if (results[d][op].passed != '0' or results[d][op].failed != '0' or results[d][op].crashed != '0' or results[d][op].skipped) != '0' -%} + P:{{ results[d][op].passed }} + F:{{ results[d][op].failed }} + S:{{ results[d][op].skipped }} + C:{{ results[d][op].crashed }} + {% else -%} + {% endif -%} +
+
+
+ {% else -%} - N/A + N/A {% endif -%} {% endfor %} diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/style.css b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/style.css index 9df47ccfd87..9a15630b256 100644 --- a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/style.css +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/style.css @@ -67,10 +67,6 @@ form button { min-height: 25px; min-width: 25px; } -.colorRed { - color:#cf1d1d; - font-weight: bold; -} .form-group { margin-bottom: 0; } @@ -535,3 +531,69 @@ h2 { background-position: -12px 2px; } + + .check { + width: 16px; + height: 16px; + background: #be2d2d; + text-indent: -1000; + border-radius: 8px; + position: relative; + } + + .check::before { + display: block; + border-bottom: 2px solid #FFF; + position: absolute; + z:index: 10; + width: 8px; + top:7px; + left: 4px; + content: ""; + } + + .checkmark { + display: block; + width: 16px; + height: 16px; + position: relative; + } + + .checkmark::before { + position: absolute; + display: block; + width: 2px; + height: 16px; + background-color: green; + left: 10px; + content: ''; + top: 0px; + z-index: 20; + transform: rotate(45deg); + } + + .checkmark::after { + position: absolute; + width: 2px; + content: ''; + height: 7px; + background-color: green; + left:3px; + display: block; + top: 7px; + z-index: 20; + transform: rotate(-45deg); + } + + .flex { + display: flex; + justify-content: space-between; + } + + .not_impl:hover { + background: #ffdee1; + } + + .impl:hover { + background: #e0fde6; + } diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/utils/utils.py b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/utils/utils.py index b2cfd20c96c..071ad68bcef 100644 --- a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/utils/utils.py +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/utils/utils.py @@ -19,10 +19,15 @@ def update_passrates(results: ET.SubElement): for attrib in op.attrib: if attrib == "passrate": continue + if attrib == "implemented": + continue if attrib == "passed": passed_tests = int(op.attrib.get(attrib)) total_tests += int(op.attrib.get(attrib)) - passrate = float(passed_tests * 100 / total_tests) if passed_tests < total_tests else 100 + if total_tests == 0: + passrate = 0 + else: + passrate = float(passed_tests * 100 / total_tests) if passed_tests < total_tests else 100 op.set("passrate", str(round(passrate, 1))) diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/src/layer_test_utils/summary.cpp b/inference-engine/tests/ie_test_utils/functional_test_utils/src/layer_test_utils/summary.cpp index 449d8b29491..24c5740ed1b 100644 --- a/inference-engine/tests/ie_test_utils/functional_test_utils/src/layer_test_utils/summary.cpp +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/src/layer_test_utils/summary.cpp @@ -53,6 +53,9 @@ void Summary::updateOPsStats(const ngraph::NodeTypeInfo &op, const PassRate::Sta auto &passrate = it->second; switch (status) { case PassRate::PASSED: + if (!passrate.isImplemented) { + passrate.isImplemented = true; + } passrate.passed++; passrate.crashed--; break; @@ -85,6 +88,18 @@ void Summary::updateOPsStats(const ngraph::NodeTypeInfo &op, const PassRate::Sta } } +void Summary::updateOPsImplStatus(const ngraph::NodeTypeInfo &op, const bool implStatus) { + auto it = opsStats.find(op); + if (it != opsStats.end()) { + if (!it->second.isImplemented && implStatus) { + it->second.isImplemented = true; + } + } else { + opsStats[op] = PassRate(0, 0, 0, 0); + opsStats[op].isImplemented = implStatus; + } +} + std::string Summary::getOpVersion(const ngraph::NodeTypeInfo &type_info) { for (size_t i = 0; i < opsets.size(); i++) { if (opsets[i].contains_type(type_info)) { @@ -120,6 +135,9 @@ std::map Summary::getOpStatisticFromReport() { } void Summary::updateOPsStats(const std::shared_ptr &function, const PassRate::Statuses &status) { + if (function->get_parameters().empty()) { + return; + } bool isFunctionalGraph = false; for (const auto &op : function->get_ordered_ops()) { if (!ngraph::is_type(op) && @@ -151,6 +169,41 @@ void Summary::updateOPsStats(const std::shared_ptr &function, } } +void Summary::updateOPsImplStatus(const std::shared_ptr &function, const bool implStatus) { + if (function->get_parameters().empty()) { + return; + } + bool isFunctionalGraph = false; + for (const auto &op : function->get_ordered_ops()) { + if (!ngraph::is_type(op) && + !ngraph::is_type(op) && + !ngraph::is_type(op)) { + isFunctionalGraph = true; + break; + } + } + + for (const auto &op : function->get_ordered_ops()) { + if ((ngraph::is_type(op) || + ngraph::is_type(op) || + ngraph::is_type(op)) && isFunctionalGraph) { + continue; + } else if (ngraph::is_type(op)) { + updateOPsImplStatus(op->get_type_info(), implStatus); + auto ti = ngraph::as_type_ptr(op); + auto ti_body = ti->get_function(); + updateOPsImplStatus(ti_body, implStatus); + } else if (ngraph::is_type(op)) { + updateOPsImplStatus(op->get_type_info(), implStatus); + auto loop = ngraph::as_type_ptr(op); + auto loop_body = loop->get_function(); + updateOPsImplStatus(loop_body, implStatus); + } else { + updateOPsImplStatus(op->get_type_info(), implStatus); + } + } +} + #ifdef IE_TEST_DEBUG void Summary::saveDebugReport(const char* className, const char* opName, unsigned long passed, unsigned long failed, unsigned long skipped, unsigned long crashed) { @@ -233,6 +286,7 @@ void Summary::saveReport() { std::string name = std::string(it.first.name) + "-" + getOpVersion(it.first); opList.insert(name); pugi::xml_node entry = currentDeviceNode.append_child(name.c_str()); + entry.append_attribute("implemented").set_value(it.second.isImplemented); entry.append_attribute("passed").set_value(it.second.passed); entry.append_attribute("failed").set_value(it.second.failed); entry.append_attribute("skipped").set_value(it.second.skipped); @@ -246,6 +300,7 @@ void Summary::saveReport() { pugi::xml_node entry; if (opList.find(item.first) == opList.end()) { entry = currentDeviceNode.append_child(item.first.c_str()); + entry.append_attribute("implemented").set_value(item.second.isImplemented); entry.append_attribute("passed").set_value(item.second.passed); entry.append_attribute("failed").set_value(item.second.failed); entry.append_attribute("skipped").set_value(item.second.skipped); @@ -253,12 +308,16 @@ void Summary::saveReport() { entry.append_attribute("passrate").set_value(item.second.getPassrate()); } else { entry = currentDeviceNode.child(item.first.c_str()); + auto implStatus = entry.attribute("implemented").value() == std::string("true") ? true : false; auto p = std::stoi(entry.attribute("passed").value()) + item.second.passed; auto f = std::stoi(entry.attribute("failed").value()) + item.second.failed; auto s = std::stoi(entry.attribute("skipped").value()) + item.second.skipped; auto c = std::stoi(entry.attribute("crashed").value()) + item.second.crashed; PassRate obj(p, f, s, c); + (implStatus || obj.isImplemented) + ? entry.attribute("implemented").set_value(true) + : entry.attribute("implemented").set_value(false); entry.attribute("passed").set_value(obj.passed); entry.attribute("failed").set_value(obj.failed); entry.attribute("skipped").set_value(obj.skipped);