From 0232042061a9dd892d98194619238cad283f00a7 Mon Sep 17 00:00:00 2001 From: Irina Efode Date: Mon, 7 Aug 2023 19:42:01 +0400 Subject: [PATCH] [CONFORMANCE] Change URL to conformance IRs and add list of failed models (#19023) --- .../layer_tests_summary/data/models.lst | 2 +- .../layer_tests_summary/run_parallel.py | 24 +++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/tests/test_utils/functional_test_utils/layer_tests_summary/data/models.lst b/src/tests/test_utils/functional_test_utils/layer_tests_summary/data/models.lst index 21b24521160..99c5151ca2f 100644 --- a/src/tests/test_utils/functional_test_utils/layer_tests_summary/data/models.lst +++ b/src/tests/test_utils/functional_test_utils/layer_tests_summary/data/models.lst @@ -1 +1 @@ -https://storage.openvinotoolkit.org/test_data/conformance_ir/2023.0.0-9637-dda19231c89.tar \ No newline at end of file +https://storage.openvinotoolkit.org/test_data/conformance_ir/conformance_ir.tar \ No newline at end of file diff --git a/src/tests/test_utils/functional_test_utils/layer_tests_summary/run_parallel.py b/src/tests/test_utils/functional_test_utils/layer_tests_summary/run_parallel.py index 466ab3646ef..060e37ba5fc 100644 --- a/src/tests/test_utils/functional_test_utils/layer_tests_summary/run_parallel.py +++ b/src/tests/test_utils/functional_test_utils/layer_tests_summary/run_parallel.py @@ -10,6 +10,7 @@ from hashlib import sha256 from pathlib import Path from shutil import rmtree, copyfile from tarfile import open as tar_open +import defusedxml.ElementTree as ET if not constants.IS_WIN: from signal import SIGKILL @@ -657,10 +658,17 @@ class TestParallelRunner: for priority, name in fix_priority: csv_writer.writerow([name, priority]) if "IR=" in name: - ir_hashes.append(name[name.find('IR=')+3:name.find('_Device=')]) + ir_hash = name[name.find('IR=')+3:name.find('_Device=')] + if os.path.isfile(ir_hash): + _, tail = os.path.split(ir_hash) + ir_hash, _ = os.path.splitext(tail) + ir_hashes.append(ir_hash) + logger.info(f"Fix priorities list is saved to: {fix_priority_path}") # Find all irs for failed tests failed_ir_dir = os.path.join(self._working_dir, f'{self._device}_failed_ir') + failed_models_file_path = os.path.join(self._working_dir, f'failed_models.lst') + failed_models = set() for conformance_ir_filelist in self._conformance_ir_filelists: with open(conformance_ir_filelist, 'r') as file: for conformance_ir in file.readlines(): @@ -684,6 +692,12 @@ class TestParallelRunner: copyfile(xml_file, failed_ir_xml) copyfile(bin_file, failed_ir_bin) copyfile(meta_file, failed_ir_meta) + + meta_root = ET.parse(failed_ir_meta).getroot() + for unique_model in meta_root.find("models"): + for path in unique_model: + for unique_path in path: + failed_models.add(unique_path.attrib["path"]) # api conformance has no failed irs if os.path.exists(failed_ir_dir): output_file_name = failed_ir_dir + '.tar' @@ -691,7 +705,13 @@ class TestParallelRunner: tar.add(failed_ir_dir, arcname=os.path.basename(failed_ir_dir)) logger.info(f"All Conformance IRs for failed tests are saved to: {output_file_name}") rmtree(failed_ir_dir) - + if len(failed_models) > 0: + with open(failed_models_file_path, "w") as failed_models_file: + failed_models_list = list() + for item in failed_models: + failed_models_list.append(f"{item}\n") + failed_models_file.writelines(failed_models_list) + failed_models_file.close() disabled_tests_path = os.path.join(logs_dir, "disabled_tests.log") with open(disabled_tests_path, "w") as disabled_tests_file: for i in range(len(self._disabled_tests)):