From 8c7a0800b6ffb341c3c2d612803648b50f6d1280 Mon Sep 17 00:00:00 2001 From: Irina Efode Date: Fri, 28 Apr 2023 16:28:27 +0400 Subject: [PATCH] [CONFORMANCE] Add rel_weights reporting for QM tests (#17198) --- .../conformance_infra/src/read_ir_test/read_ir.cpp | 10 +++++----- .../layer_tests_summary/merge_xmls.py | 2 +- .../layer_tests_summary/run_conformance.py | 7 +++---- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp index 246051b7f7e..bbdd9013e9a 100644 --- a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp @@ -112,19 +112,19 @@ void ReadIRTest::query_model() { s.setDeviceName(targetDevice); if (FuncTestUtils::SkipTestsConfig::currentTestIsDisabled()) { - s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::SKIPPED); + s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::SKIPPED, rel_influence_coef); GTEST_SKIP() << "Disabled test due to configuration" << std::endl; } else { - s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::CRASHED); + s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::CRASHED, rel_influence_coef); } try { SubgraphBaseTest::query_model(); - s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::PASSED); + s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::PASSED, rel_influence_coef); } catch (...) { - s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::FAILED); + s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::FAILED, rel_influence_coef); } } else if (jmpRes == CommonTestUtils::JMP_STATUS::alarmErr) { - s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::HANGED); + s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::HANGED, rel_influence_coef); IE_THROW() << "Crash happens"; } else if (jmpRes == CommonTestUtils::JMP_STATUS::anyError) { IE_THROW() << "Crash happens"; diff --git a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/merge_xmls.py b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/merge_xmls.py index 3b71556b493..4e506e9d324 100644 --- a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/merge_xmls.py +++ b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/merge_xmls.py @@ -87,7 +87,7 @@ def aggregate_test_results(aggregated_results: SubElement, xml_reports: list, else: aggregated_device_results.append(xml_results_entry) continue - if report_type == "OP": + if report_type == OP_CONFORMANCE or report_type == OP_CONFORMANCE.lower(): update_result_node(xml_results_entry, aggregated_results_entry) else: for xml_real_device_entry in xml_results_entry: diff --git a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/run_conformance.py b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/run_conformance.py index 5c7b1dfbc24..ed2eb74ae8a 100644 --- a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/run_conformance.py +++ b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/run_conformance.py @@ -180,8 +180,8 @@ class Conformance: # API Conformance contains both report type merge_xml([parallel_report_dir], report_dir, final_report_name, self._type, True) if self._type == constants.API_CONFORMANCE: - final_op_report_name = f'report_{constants.OP_CONFORMANCE.lower()}' - merge_xml([parallel_report_dir], report_dir, final_op_report_name, constants.OP_CONFORMANCE.lower()) + final_report_name = f'report_{constants.OP_CONFORMANCE.lower()}' + merge_xml([parallel_report_dir], report_dir, final_report_name, constants.OP_CONFORMANCE.lower(), True) logger.info(f"Conformance is successful. XML reportwas saved to {report_dir}") return (os.path.join(report_dir, final_report_name + ".xml"), report_dir) @@ -235,8 +235,7 @@ class Conformance: logger.error(f"Directory {self._model_path} does not exist") exit(-1) xml_report, report_dir = self.__run_conformance() - if self._type == "OP": - self.__summarize(xml_report, report_dir) + self.__summarize(xml_report, report_dir) if __name__ == "__main__": args = parse_arguments()