Change logic for using multiple models in cc tests (#6618)

* Change logic for using multiple models in cc tests

* removed extra key

* moved multiple infer to main function

* moved multiple infer to main function

* change test_config.yml and work with this file

* change test_config.yml and work with this file

* removed extra key

* changed separator symbol

* removed extra param from output

* Rollback comments in test_config.yml

* removed extra key in yml config file, change logic to save .npz for all model seperate instead of common npz file, change logic in parser arg

* removed extra key in yml config file, change logic to save .npz for all model seperate instead of common npz file, change logic in parser arg

* changed save path for infer result, deleted extra param

* updated test_config.yml

* rollback old way to get bin_path

* changed work with save path for inference result

* add empty line in the end for config file

* rollback line order

* removed extra param allow_pickle=True in loading .npz file

* removed mkdir in run_infer

* uncomment resnet model

* added empty line

* changed save folder for cc result

tests/conditional_compilation/conftest.py

@@ -6,7 +6,6 @@
 # pylint: disable=line-too-long
 
 """Pytest configuration for compilation tests."""
-
 import logging
 import sys
 from inspect import getsourcefile
@@ -19,8 +18,8 @@ import yaml
 import pytest
 
 from path_utils import expand_env_vars  # pylint: disable=import-error
-from test_utils import make_build, validate_path_arg, write_session_info, SESSION_INFO_FILE  # pylint: disable=import-error
+from test_utils import make_build, validate_path_arg, write_session_info, \
-
+    SESSION_INFO_FILE  # pylint: disable=import-error
 
 log = logging.getLogger()
 
@@ -71,15 +70,19 @@ def pytest_generate_tests(metafunc):
         test_cases = yaml.safe_load(file)
 
     for test in test_cases:
-        extra_args = {}
+        model_list = []
-        model_path = test["model"]["path"]
+        test_id_list = []
-        if "marks" in test:
+        for models in test:
-            extra_args["marks"] = test["marks"]
+            extra_args = {}
+            model_path = models["model"]["path"]
+            if "marks" in test:
+                extra_args["marks"] = test["marks"]
+            model_list.append(expand_env_vars(model_path))
+            test_id_list.append(model_path.split("/")[- 1])
+        ids = ids + ['-'.join(test_id_list)]
+        params.append(pytest.param('-'.join(test_id_list), model_list), **extra_args)
 
-        test_id = model_path.replace("$", "").replace("{", "").replace("}", "")
+    metafunc.parametrize("test_id, models", params, ids=ids)
-        params.append(pytest.param(test_id, Path(expand_env_vars(model_path)), **extra_args))
-        ids = ids + [test_id]
-    metafunc.parametrize("test_id, model", params, ids=ids)
 
 
 @pytest.fixture(scope="session")

tests/conditional_compilation/test_cc.py

@@ -10,27 +10,30 @@ import glob
 import logging
 import os
 import sys
+from pathlib import Path
+
 import numpy as np
 import pytest
-
 from proc_utils import cmd_exec  # pylint: disable=import-error
-from test_utils import get_lib_sizes, infer_tool, make_build, run_infer  # pylint: disable=import-error
 
+from test_utils import get_lib_sizes, infer_tool, make_build, run_infer  # pylint: disable=import-error
 
 log = logging.getLogger()
 
 
 @pytest.mark.dependency(name="cc_collect")
-def test_cc_collect(test_id, model, openvino_ref, test_info,
+def test_cc_collect(test_id, models, openvino_ref, test_info,
                     save_session_info, sea_runtool, collector_dir, artifacts):  # pylint: disable=unused-argument
     """Test conditional compilation statistics collection
     :param test_info: custom `test_info` field of built-in `request` pytest fixture.
                       contain a dictionary to store test metadata.
     """
     out = artifacts / test_id
+    infer_out_dir = out / "inference_result"
     test_info["test_id"] = test_id
+
     # cleanup old data if any
-    prev_result = glob.glob(f"{out}.pid*.csv")
+    prev_result = glob.glob(f"{out / test_id}.pid*.csv")
     for path in prev_result:
         os.remove(path)
     # run use case
@@ -38,32 +41,32 @@ def test_cc_collect(test_id, model, openvino_ref, test_info,
         [
             sys.executable,
             str(sea_runtool),
-            f"--output={out}",
+            f"--output={out / test_id}",
             f"--bindir={collector_dir}",
             "!",
             sys.executable,
             infer_tool,
-            f"-m={model}",
+            *[f"-m={model}" for model in models],
             "-d=CPU",
-            f"-r={out}",
+            f"-r={infer_out_dir}"
         ]
     )
-    out_csv = glob.glob(f"{out}.pid*.csv")
+    out_csv = glob.glob(f"{out / test_id}.pid*.csv")
     test_info["out_csv"] = out_csv
 
     assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}"
-    assert len(out_csv) == 1, f'Multiple or none "{out}.pid*.csv" files'
+    assert len(out_csv) == 1, f'Multiple or none "{out / test_id}.pid*.csv" files'
 
 
 @pytest.mark.dependency(depends=["cc_collect"])
-def test_minimized_pkg(test_id, model, openvino_root_dir, artifacts):  # pylint: disable=unused-argument
+def test_minimized_pkg(test_id, models, openvino_root_dir, artifacts):  # pylint: disable=unused-argument
     """Build and install OpenVINO package with collected conditional compilation statistics."""
     out = artifacts / test_id
     install_prefix = out / "install_pkg"
     build_dir = openvino_root_dir / "build_minimized"
 
-    out_csv = glob.glob(f"{out}.pid*.csv")
+    out_csv = glob.glob(f"{out / test_id}.pid*.csv")
-    assert len(out_csv) == 1, f'Multiple or none "{out}.pid*.csv" files'
+    assert len(out_csv) == 1, f'Multiple or none "{out / test_id}.pid*.csv" files'
 
     log.info("Building minimized build at %s", build_dir)
 
@@ -78,38 +81,47 @@ def test_minimized_pkg(test_id, model, openvino_root_dir, artifacts):  # pylint:
 
 
 @pytest.mark.dependency(depends=["cc_collect", "minimized_pkg"])
-def test_infer(test_id, model, artifacts):
+def test_infer(test_id, models, artifacts):
     """Test inference with conditional compiled binaries."""
     out = artifacts / test_id
     minimized_pkg = out / "install_pkg"
-    return_code, output = run_infer(model, f"{out}_cc.npz", minimized_pkg)
+    infer_out_dir_cc =  out / "inference_result_cc/"
+
+    return_code, output = run_infer(models, infer_out_dir_cc, minimized_pkg)
     assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}"
 
 
 @pytest.mark.dependency(depends=["cc_collect", "minimized_pkg"])
-def test_verify(test_id, model, openvino_ref, artifacts, tolerance=1e-6):  # pylint: disable=too-many-arguments
+def test_verify(test_id, models, openvino_ref, artifacts, tolerance=1e-6):  # pylint: disable=too-many-arguments
     """Test verifying that inference results are equal."""
     out = artifacts / test_id
     minimized_pkg = out / "install_pkg"
-    out_file = f"{out}.npz"
+
-    out_file_cc = f"{out}_cc.npz"
+    infer_out_dir_cc = out / "inference_result_cc/"
-    return_code, output = run_infer(model, out_file, openvino_ref)
+    infer_out_dir = out / "inference_result/"
+
+    return_code, output = run_infer(models, infer_out_dir, openvino_ref)
     assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}"
-    return_code, output = run_infer(model, out_file_cc, minimized_pkg)
+    return_code, output = run_infer(models, infer_out_dir_cc, minimized_pkg)
     assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}"
-    reference_results = dict(np.load(out_file))
+
-    inference_results = dict(np.load(out_file_cc))
+    for model in models:
-    assert sorted(reference_results.keys()) == sorted(
+        out_file = f"{infer_out_dir / Path(model).name}.npz"
-        inference_results.keys()
+        out_file_cc = f"{infer_out_dir_cc / Path(model).name}.npz"
-    ), "Results have different number of layers"
+
-    for layer in reference_results.keys():
+        reference_results = dict(np.load(out_file))
-        assert np.allclose(
+        inference_results = dict(np.load(out_file_cc))
-            reference_results[layer], inference_results[layer], tolerance
+        assert sorted(reference_results.keys()) == sorted(
-        ), "Reference and inference results differ"
+            inference_results.keys()
+        ), "Results have different number of layers"
+        for layer in reference_results.keys():
+            assert np.allclose(
+                reference_results[layer], inference_results[layer], tolerance
+            ), "Reference and inference results differ"
 
 
 @pytest.mark.dependency(depends=["cc_collect", "minimized_pkg"])
-def test_libs_size(test_id, model, openvino_ref, artifacts):  # pylint: disable=unused-argument
+def test_libs_size(test_id, models, openvino_ref, artifacts):  # pylint: disable=unused-argument
     """Test if libraries haven't increased in size after conditional compilation."""
     libraries = ["inference_engine_transformations", "MKLDNNPlugin", "ngraph"]
     minimized_pkg = artifacts / test_id / "install_pkg"

tests/conditional_compilation/test_config.yml

@@ -1,15 +1,31 @@
 # Using models from https://github.com/openvinotoolkit/testdata
 # $find models -wholename "*.xml"
 
-- model:
+-
-    path: ${TESTDATA}/models/mobilenet_v2_1.4_224/mobilenet_v2_1.4_224_i8.xml
+    - model:
-- model:
+        path: ${TESTDATA}/models/mobilenet_v2_1.4_224/mobilenet_v2_1.4_224_i8.xml
-    path: ${TESTDATA}/models/mobilenet_v2_1.0_224/mobilenet_v2_1.0_224_i8.xml
+-
-- model:
+    - model:
-    path: ${TESTDATA}/models/inception_v3/inception_v3_i8.xml
+        path: ${TESTDATA}/models/mobilenet_v2_1.0_224/mobilenet_v2_1.0_224_i8.xml
-#- model:
+-
-#    path: ${TESTDATA}/models/resnet_v1_50/resnet_v1_50_i8.xml
+    - model:
-- model:
+        path: ${TESTDATA}/models/test_model/test_model_fp16.xml
-    path: ${TESTDATA}/models/test_model/test_model_fp16.xml
+-
-- model:
+    - model:
-    path: ${TESTDATA}/models/test_model/test_model_fp32.xml
+        path: ${TESTDATA}/models/test_model/test_model_fp32.xml
+-
+    - model:
+        path: ${TESTDATA}/models/resnet_v1_50/resnet_v1_50_i8.xml
+-
+    - model:
+        path: ${TESTDATA}/models/inception_v3/inception_v3_i8.xml
+-
+    - model:
+        path: ${TESTDATA}/models/test_model/test_model_fp16.xml
+    - model:
+        path: ${TESTDATA}/models/test_model/test_model_fp32.xml
+-
+    - model:
+        path: ${TESTDATA}/models/mobilenet_v2_1.4_224/mobilenet_v2_1.4_224_i8.xml
+    - model:
+        path: ${TESTDATA}/models/mobilenet_v2_1.0_224/mobilenet_v2_1.0_224_i8.xml

tests/conditional_compilation/test_utils.py

@@ -59,13 +59,16 @@ def write_session_info(path: Path = Path(getsourcefile(lambda: 0)).parent / SESS
         json.dump(data, json_file, indent=4)
 
 
-def run_infer(model, out_file, install_dir):
+def run_infer(models, out_dir, install_dir):
     """ Function running inference
     """
+    out_dir.mkdir(parents=True, exist_ok=True)
     return_code, output = cmd_exec(
         [sys.executable,
          infer_tool,
-         "-d=CPU", f"-m={model}", f"-r={out_file}"
+         "-d=CPU",
+         *[f"-m={model}" for model in models],
+         f"-r={out_dir}"
          ],
         env=get_openvino_environment(install_dir),
     )

tests/conditional_compilation/tools/infer_tool.py

@@ -9,8 +9,10 @@
 """
 import argparse
 import logging as log
-import sys
 import os
+import sys
+from pathlib import Path
+
 import numpy as np
 from openvino.inference_engine import IECore
 
@@ -58,9 +60,9 @@ def cli_parser():
     :return: ir path, device and output folder path variables.
     """
     parser = argparse.ArgumentParser(description='Arguments for python API inference')
-    parser.add_argument('-m', dest='ir_path', required=True, help='Path to XML file of IR')
+    parser.add_argument('-m', dest='ir_path', required=True, help='Path to XML file of IR',  action="append")
     parser.add_argument('-d', dest='device', required=True, help='Target device to infer on')
-    parser.add_argument('-r', dest='out_path', required=True,
+    parser.add_argument('-r', dest='out_path', required=True, type=Path,
                         help='Dumps results to the output file')
     parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
                         help='Increase output verbosity')
@@ -75,9 +77,13 @@ def cli_parser():
 
 if __name__ == "__main__":
     ir_path, device, out_path = cli_parser()
-    results = infer(ir_path=ir_path, device=device)
+
-    np.savez(out_path, **results)
+    for model in ir_path:
-    log.info("Path for inference results: {}".format(out_path))
+        result = infer(ir_path=model, device=device)
-    log.debug("Inference results:")
+
-    log.debug(results)
+        np.savez(out_path / f"{Path(model).name}.npz", **result)
-    log.debug("SUCCESS!")
+
+        log.info("Path for inference results: {}".format(out_path))
+        log.debug("Inference results:")
+        log.debug(result)
+        log.debug("SUCCESS!")