diff --git a/tests/conditional_compilation/conftest.py b/tests/conditional_compilation/conftest.py index 7f76ec13c22..85e2c333d66 100644 --- a/tests/conditional_compilation/conftest.py +++ b/tests/conditional_compilation/conftest.py @@ -6,7 +6,6 @@ # pylint: disable=line-too-long """Pytest configuration for compilation tests.""" - import logging import sys from inspect import getsourcefile @@ -19,8 +18,8 @@ import yaml import pytest from path_utils import expand_env_vars # pylint: disable=import-error -from test_utils import make_build, validate_path_arg, write_session_info, SESSION_INFO_FILE # pylint: disable=import-error - +from test_utils import make_build, validate_path_arg, write_session_info, \ + SESSION_INFO_FILE # pylint: disable=import-error log = logging.getLogger() @@ -71,15 +70,19 @@ def pytest_generate_tests(metafunc): test_cases = yaml.safe_load(file) for test in test_cases: - extra_args = {} - model_path = test["model"]["path"] - if "marks" in test: - extra_args["marks"] = test["marks"] + model_list = [] + test_id_list = [] + for models in test: + extra_args = {} + model_path = models["model"]["path"] + if "marks" in test: + extra_args["marks"] = test["marks"] + model_list.append(expand_env_vars(model_path)) + test_id_list.append(model_path.split("/")[- 1]) + ids = ids + ['-'.join(test_id_list)] + params.append(pytest.param('-'.join(test_id_list), model_list), **extra_args) - test_id = model_path.replace("$", "").replace("{", "").replace("}", "") - params.append(pytest.param(test_id, Path(expand_env_vars(model_path)), **extra_args)) - ids = ids + [test_id] - metafunc.parametrize("test_id, model", params, ids=ids) + metafunc.parametrize("test_id, models", params, ids=ids) @pytest.fixture(scope="session") diff --git a/tests/conditional_compilation/test_cc.py b/tests/conditional_compilation/test_cc.py index aab828ad5a4..423b9524426 100644 --- a/tests/conditional_compilation/test_cc.py +++ b/tests/conditional_compilation/test_cc.py @@ -10,27 +10,30 @@ import glob import logging import os import sys +from pathlib import Path + import numpy as np import pytest - from proc_utils import cmd_exec # pylint: disable=import-error -from test_utils import get_lib_sizes, infer_tool, make_build, run_infer # pylint: disable=import-error +from test_utils import get_lib_sizes, infer_tool, make_build, run_infer # pylint: disable=import-error log = logging.getLogger() @pytest.mark.dependency(name="cc_collect") -def test_cc_collect(test_id, model, openvino_ref, test_info, +def test_cc_collect(test_id, models, openvino_ref, test_info, save_session_info, sea_runtool, collector_dir, artifacts): # pylint: disable=unused-argument """Test conditional compilation statistics collection :param test_info: custom `test_info` field of built-in `request` pytest fixture. contain a dictionary to store test metadata. """ out = artifacts / test_id + infer_out_dir = out / "inference_result" test_info["test_id"] = test_id + # cleanup old data if any - prev_result = glob.glob(f"{out}.pid*.csv") + prev_result = glob.glob(f"{out / test_id}.pid*.csv") for path in prev_result: os.remove(path) # run use case @@ -38,32 +41,32 @@ def test_cc_collect(test_id, model, openvino_ref, test_info, [ sys.executable, str(sea_runtool), - f"--output={out}", + f"--output={out / test_id}", f"--bindir={collector_dir}", "!", sys.executable, infer_tool, - f"-m={model}", + *[f"-m={model}" for model in models], "-d=CPU", - f"-r={out}", + f"-r={infer_out_dir}" ] ) - out_csv = glob.glob(f"{out}.pid*.csv") + out_csv = glob.glob(f"{out / test_id}.pid*.csv") test_info["out_csv"] = out_csv assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}" - assert len(out_csv) == 1, f'Multiple or none "{out}.pid*.csv" files' + assert len(out_csv) == 1, f'Multiple or none "{out / test_id}.pid*.csv" files' @pytest.mark.dependency(depends=["cc_collect"]) -def test_minimized_pkg(test_id, model, openvino_root_dir, artifacts): # pylint: disable=unused-argument +def test_minimized_pkg(test_id, models, openvino_root_dir, artifacts): # pylint: disable=unused-argument """Build and install OpenVINO package with collected conditional compilation statistics.""" out = artifacts / test_id install_prefix = out / "install_pkg" build_dir = openvino_root_dir / "build_minimized" - out_csv = glob.glob(f"{out}.pid*.csv") - assert len(out_csv) == 1, f'Multiple or none "{out}.pid*.csv" files' + out_csv = glob.glob(f"{out / test_id}.pid*.csv") + assert len(out_csv) == 1, f'Multiple or none "{out / test_id}.pid*.csv" files' log.info("Building minimized build at %s", build_dir) @@ -78,38 +81,47 @@ def test_minimized_pkg(test_id, model, openvino_root_dir, artifacts): # pylint: @pytest.mark.dependency(depends=["cc_collect", "minimized_pkg"]) -def test_infer(test_id, model, artifacts): +def test_infer(test_id, models, artifacts): """Test inference with conditional compiled binaries.""" out = artifacts / test_id minimized_pkg = out / "install_pkg" - return_code, output = run_infer(model, f"{out}_cc.npz", minimized_pkg) + infer_out_dir_cc = out / "inference_result_cc/" + + return_code, output = run_infer(models, infer_out_dir_cc, minimized_pkg) assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}" @pytest.mark.dependency(depends=["cc_collect", "minimized_pkg"]) -def test_verify(test_id, model, openvino_ref, artifacts, tolerance=1e-6): # pylint: disable=too-many-arguments +def test_verify(test_id, models, openvino_ref, artifacts, tolerance=1e-6): # pylint: disable=too-many-arguments """Test verifying that inference results are equal.""" out = artifacts / test_id minimized_pkg = out / "install_pkg" - out_file = f"{out}.npz" - out_file_cc = f"{out}_cc.npz" - return_code, output = run_infer(model, out_file, openvino_ref) + + infer_out_dir_cc = out / "inference_result_cc/" + infer_out_dir = out / "inference_result/" + + return_code, output = run_infer(models, infer_out_dir, openvino_ref) assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}" - return_code, output = run_infer(model, out_file_cc, minimized_pkg) + return_code, output = run_infer(models, infer_out_dir_cc, minimized_pkg) assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}" - reference_results = dict(np.load(out_file)) - inference_results = dict(np.load(out_file_cc)) - assert sorted(reference_results.keys()) == sorted( - inference_results.keys() - ), "Results have different number of layers" - for layer in reference_results.keys(): - assert np.allclose( - reference_results[layer], inference_results[layer], tolerance - ), "Reference and inference results differ" + + for model in models: + out_file = f"{infer_out_dir / Path(model).name}.npz" + out_file_cc = f"{infer_out_dir_cc / Path(model).name}.npz" + + reference_results = dict(np.load(out_file)) + inference_results = dict(np.load(out_file_cc)) + assert sorted(reference_results.keys()) == sorted( + inference_results.keys() + ), "Results have different number of layers" + for layer in reference_results.keys(): + assert np.allclose( + reference_results[layer], inference_results[layer], tolerance + ), "Reference and inference results differ" @pytest.mark.dependency(depends=["cc_collect", "minimized_pkg"]) -def test_libs_size(test_id, model, openvino_ref, artifacts): # pylint: disable=unused-argument +def test_libs_size(test_id, models, openvino_ref, artifacts): # pylint: disable=unused-argument """Test if libraries haven't increased in size after conditional compilation.""" libraries = ["inference_engine_transformations", "MKLDNNPlugin", "ngraph"] minimized_pkg = artifacts / test_id / "install_pkg" diff --git a/tests/conditional_compilation/test_config.yml b/tests/conditional_compilation/test_config.yml index a9f8a463d7a..bc56db783b0 100644 --- a/tests/conditional_compilation/test_config.yml +++ b/tests/conditional_compilation/test_config.yml @@ -1,15 +1,31 @@ # Using models from https://github.com/openvinotoolkit/testdata # $find models -wholename "*.xml" -- model: - path: ${TESTDATA}/models/mobilenet_v2_1.4_224/mobilenet_v2_1.4_224_i8.xml -- model: - path: ${TESTDATA}/models/mobilenet_v2_1.0_224/mobilenet_v2_1.0_224_i8.xml -- model: - path: ${TESTDATA}/models/inception_v3/inception_v3_i8.xml -#- model: -# path: ${TESTDATA}/models/resnet_v1_50/resnet_v1_50_i8.xml -- model: - path: ${TESTDATA}/models/test_model/test_model_fp16.xml -- model: - path: ${TESTDATA}/models/test_model/test_model_fp32.xml +- + - model: + path: ${TESTDATA}/models/mobilenet_v2_1.4_224/mobilenet_v2_1.4_224_i8.xml +- + - model: + path: ${TESTDATA}/models/mobilenet_v2_1.0_224/mobilenet_v2_1.0_224_i8.xml +- + - model: + path: ${TESTDATA}/models/test_model/test_model_fp16.xml +- + - model: + path: ${TESTDATA}/models/test_model/test_model_fp32.xml +- + - model: + path: ${TESTDATA}/models/resnet_v1_50/resnet_v1_50_i8.xml +- + - model: + path: ${TESTDATA}/models/inception_v3/inception_v3_i8.xml +- + - model: + path: ${TESTDATA}/models/test_model/test_model_fp16.xml + - model: + path: ${TESTDATA}/models/test_model/test_model_fp32.xml +- + - model: + path: ${TESTDATA}/models/mobilenet_v2_1.4_224/mobilenet_v2_1.4_224_i8.xml + - model: + path: ${TESTDATA}/models/mobilenet_v2_1.0_224/mobilenet_v2_1.0_224_i8.xml diff --git a/tests/conditional_compilation/test_utils.py b/tests/conditional_compilation/test_utils.py index 499a760bbab..9fa503efa83 100644 --- a/tests/conditional_compilation/test_utils.py +++ b/tests/conditional_compilation/test_utils.py @@ -59,13 +59,16 @@ def write_session_info(path: Path = Path(getsourcefile(lambda: 0)).parent / SESS json.dump(data, json_file, indent=4) -def run_infer(model, out_file, install_dir): +def run_infer(models, out_dir, install_dir): """ Function running inference """ + out_dir.mkdir(parents=True, exist_ok=True) return_code, output = cmd_exec( [sys.executable, infer_tool, - "-d=CPU", f"-m={model}", f"-r={out_file}" + "-d=CPU", + *[f"-m={model}" for model in models], + f"-r={out_dir}" ], env=get_openvino_environment(install_dir), ) diff --git a/tests/conditional_compilation/tools/infer_tool.py b/tests/conditional_compilation/tools/infer_tool.py index 9d733b6911c..54bafd78d1e 100644 --- a/tests/conditional_compilation/tools/infer_tool.py +++ b/tests/conditional_compilation/tools/infer_tool.py @@ -9,8 +9,10 @@ """ import argparse import logging as log -import sys import os +import sys +from pathlib import Path + import numpy as np from openvino.inference_engine import IECore @@ -58,9 +60,9 @@ def cli_parser(): :return: ir path, device and output folder path variables. """ parser = argparse.ArgumentParser(description='Arguments for python API inference') - parser.add_argument('-m', dest='ir_path', required=True, help='Path to XML file of IR') + parser.add_argument('-m', dest='ir_path', required=True, help='Path to XML file of IR', action="append") parser.add_argument('-d', dest='device', required=True, help='Target device to infer on') - parser.add_argument('-r', dest='out_path', required=True, + parser.add_argument('-r', dest='out_path', required=True, type=Path, help='Dumps results to the output file') parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', help='Increase output verbosity') @@ -75,9 +77,13 @@ def cli_parser(): if __name__ == "__main__": ir_path, device, out_path = cli_parser() - results = infer(ir_path=ir_path, device=device) - np.savez(out_path, **results) - log.info("Path for inference results: {}".format(out_path)) - log.debug("Inference results:") - log.debug(results) - log.debug("SUCCESS!") + + for model in ir_path: + result = infer(ir_path=model, device=device) + + np.savez(out_path / f"{Path(model).name}.npz", **result) + + log.info("Path for inference results: {}".format(out_path)) + log.debug("Inference results:") + log.debug(result) + log.debug("SUCCESS!")