IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CONFORMANCE] Add rel_weights reporting for QM tests (#17198)

Browse Source
This commit is contained in:
Irina Efode 2023-04-28 16:28:27 +04:00 committed by GitHub
parent 9248ecadee
commit 8c7a0800b6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 9 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp
View File

@ -112,19 +112,19 @@ void ReadIRTest::query_model() {
s.setDeviceName(targetDevice); s.setDeviceName(targetDevice);
if (FuncTestUtils::SkipTestsConfig::currentTestIsDisabled()) { if (FuncTestUtils::SkipTestsConfig::currentTestIsDisabled()) {
s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::SKIPPED); s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::SKIPPED, rel_influence_coef);
GTEST_SKIP() << "Disabled test due to configuration" << std::endl; GTEST_SKIP() << "Disabled test due to configuration" << std::endl;
} else { } else {
s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::CRASHED); s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::CRASHED, rel_influence_coef);
} }
try { try {
SubgraphBaseTest::query_model(); SubgraphBaseTest::query_model();
s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::PASSED); s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::PASSED, rel_influence_coef);
} catch (...) { } catch (...) {
s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::FAILED); s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::FAILED, rel_influence_coef);
} }
} else if (jmpRes == CommonTestUtils::JMP_STATUS::alarmErr) { } else if (jmpRes == CommonTestUtils::JMP_STATUS::alarmErr) {
s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::HANGED); s.updateOPsStats(functionRefs, ov::test::utils::PassRate::Statuses::HANGED, rel_influence_coef);
IE_THROW() << "Crash happens"; IE_THROW() << "Crash happens";
} else if (jmpRes == CommonTestUtils::JMP_STATUS::anyError) { } else if (jmpRes == CommonTestUtils::JMP_STATUS::anyError) {
IE_THROW() << "Crash happens"; IE_THROW() << "Crash happens";

2
src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/merge_xmls.py
View File

@ -87,7 +87,7 @@ def aggregate_test_results(aggregated_results: SubElement, xml_reports: list,
else: else:
aggregated_device_results.append(xml_results_entry) aggregated_device_results.append(xml_results_entry)
continue continue
if report_type == "OP": if report_type == OP_CONFORMANCE or report_type == OP_CONFORMANCE.lower():
update_result_node(xml_results_entry, aggregated_results_entry) update_result_node(xml_results_entry, aggregated_results_entry)
else: else:
for xml_real_device_entry in xml_results_entry: for xml_real_device_entry in xml_results_entry:

7
src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/run_conformance.py
View File

@ -180,8 +180,8 @@ class Conformance:
# API Conformance contains both report type # API Conformance contains both report type
merge_xml([parallel_report_dir], report_dir, final_report_name, self._type, True) merge_xml([parallel_report_dir], report_dir, final_report_name, self._type, True)
if self._type == constants.API_CONFORMANCE: if self._type == constants.API_CONFORMANCE:
final_op_report_name = f'report_{constants.OP_CONFORMANCE.lower()}' final_report_name = f'report_{constants.OP_CONFORMANCE.lower()}'
merge_xml([parallel_report_dir], report_dir, final_op_report_name, constants.OP_CONFORMANCE.lower()) merge_xml([parallel_report_dir], report_dir, final_report_name, constants.OP_CONFORMANCE.lower(), True)
logger.info(f"Conformance is successful. XML reportwas saved to {report_dir}") logger.info(f"Conformance is successful. XML reportwas saved to {report_dir}")
return (os.path.join(report_dir, final_report_name + ".xml"), report_dir) return (os.path.join(report_dir, final_report_name + ".xml"), report_dir)
@ -235,8 +235,7 @@ class Conformance:
logger.error(f"Directory {self._model_path} does not exist") logger.error(f"Directory {self._model_path} does not exist")
exit(-1) exit(-1)
xml_report, report_dir = self.__run_conformance() xml_report, report_dir = self.__run_conformance()
if self._type == "OP": self.__summarize(xml_report, report_dir)
self.__summarize(xml_report, report_dir)
if __name__ == "__main__": if __name__ == "__main__":
args = parse_arguments() args = parse_arguments()