Design test config and integrate in CC tests (#4051)

tests/conditional_compilation/conftest.py

@@ -7,7 +7,7 @@
 """ Pytest configuration for compilation tests.
 
 Sample usage:
-python3 -m pytest --artifacts ./compiled --models_root=<path to openvinotoolkit/testdata repository> \
+python3 -m pytest --artifacts ./compiled --test_conf=<path to test config> \
     --sea_runtool=./IntelSEAPI/runtool/sea_runtool.py \
     --benchmark_app=./bin/benchmark_app test_collect.py
 """
@@ -17,32 +17,30 @@ from inspect import getsourcefile
 from pathlib import Path
 
 import pytest
+import yaml
 
 # add ../lib to imports
 sys.path.insert(
     0, str((Path(getsourcefile(lambda: 0)) / ".." / ".." / "lib").resolve(strict=True))
 )
 
-# Using models from https://github.com/openvinotoolkit/testdata
-# $find models -wholename "*.xml"
-TESTS = [
-    {"path": "models/mobilenet_v2_1.4_224/mobilenet_v2_1.4_224_i8.xml"},
-    {"path": "models/mobilenet_v2_1.0_224/mobilenet_v2_1.0_224_i8.xml"},
-    {"path": "models/inception_v3/inception_v3_i8.xml"},
-    {"path": "models/resnet_v1_50/resnet_v1_50_i8.xml"},
-    {"path": "models/test_model/test_model_fp16.xml"},
-    {"path": "models/test_model/test_model_fp32.xml"},
-]
+from path_utils import expand_env_vars  # pylint: disable=import-error
 
 
 def pytest_addoption(parser):
     """ Define extra options for pytest options
     """
     parser.addoption(
-        "--models_root", required=True, type=Path, help="Path to models root directory"
+        "--test_conf",
+        type=Path,
+        default=Path(__file__).parent / "test_config.yml",
+        help="Path to models root directory"
     )
     parser.addoption(
-        "--sea_runtool", required=True, type=Path, help="Path to sea_runtool.py"
+        "--sea_runtool",
+        required=True,
+        type=Path,
+        help="Path to sea_runtool.py"
     )
     parser.addoption(
         "--benchmark_app",
@@ -65,14 +63,17 @@ def pytest_generate_tests(metafunc):
     params = []
     ids = []
 
-    for test in TESTS:
+    with open(metafunc.config.getoption('test_conf'), "r") as file:
+        test_cases = yaml.safe_load(file)
+
+    for test in test_cases:
         extra_args = {}
-        path = test["path"]
+        model_path = test["model"]["path"]
         if "marks" in test:
             extra_args["marks"] = test["marks"]
 
-        params.append(pytest.param(Path(path), **extra_args))
-        ids = ids + [path]
+        params.append(pytest.param(Path(expand_env_vars(model_path)), **extra_args))
+        ids = ids + [model_path]
     metafunc.parametrize("model", params, ids=ids)
 
 
@@ -88,12 +89,6 @@ def benchmark_app(request):
     return request.config.getoption("benchmark_app")
 
 
-@pytest.fixture(scope="session")
-def models_root(request):
-    """Fixture function for command-line option."""
-    return request.config.getoption("models_root")
-
-
 @pytest.fixture(scope="session")
 def artifacts(request):
     """Fixture function for command-line option."""

tests/conditional_compilation/test_collect.py

@@ -11,7 +11,7 @@ import os
 from proc_utils import cmd_exec  # pylint: disable=import-error
 
 
-def test_cc_collect(model, sea_runtool, benchmark_app, models_root, artifacts):
+def test_cc_collect(model, sea_runtool, benchmark_app, artifacts):
     """ Test conditional compilation statistics collection
     """
     out = artifacts / model.parent / model.stem
@@ -29,7 +29,7 @@ def test_cc_collect(model, sea_runtool, benchmark_app, models_root, artifacts):
             "!",
             str(benchmark_app),
             "-d=CPU",
-            f"-m={models_root / model}",
+            f"-m={model}",
             "-niter=1",
             "-nireq=1",
         ]

tests/conditional_compilation/test_config.yml

@@ -0,0 +1,15 @@
+# Using models from https://github.com/openvinotoolkit/testdata
+# $find models -wholename "*.xml"
+
+- model:
+    path: ${TESTDATA}/models/mobilenet_v2_1.4_224/mobilenet_v2_1.4_224_i8.xml
+- model:
+    path: ${TESTDATA}/models/mobilenet_v2_1.0_224/mobilenet_v2_1.0_224_i8.xml
+- model:
+    path: ${TESTDATA}/models/inception_v3/inception_v3_i8.xml
+- model:
+    path: ${TESTDATA}/models/resnet_v1_50/resnet_v1_50_i8.xml
+- model:
+    path: ${TESTDATA}/models/test_model/test_model_fp16.xml
+- model:
+    path: ${TESTDATA}/models/test_model/test_model_fp32.xml

tests/conditional_compilation/test_infer.py

@@ -8,10 +8,10 @@
 from proc_utils import cmd_exec  # pylint: disable=import-error
 
 
-def test_infer(model, models_root, benchmark_app):
+def test_infer(model, benchmark_app):
     """ Test inference with conditional compiled binaries
     """
     returncode, _ = cmd_exec(
-        [str(benchmark_app), "-d=CPU", f"-m={models_root / model}", "-niter=1", "-nireq=1"]
+        [str(benchmark_app), "-d=CPU", f"-m={model}", "-niter=1", "-nireq=1"]
     )
     assert returncode == 0, f"Command exited with non-zero status {returncode}"

tests/lib/path_utils.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+# Copyright (C) 2021 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+""" Common utilities for working with processes.
+"""
+
+import os
+
+
+def expand_env_vars(obj):
+    """Expand environment variables in provided object."""
+
+    if isinstance(obj, list):
+        for i, value in enumerate(obj):
+            obj[i] = expand_env_vars(value)
+    elif isinstance(obj, dict):
+        for name, value in obj.items():
+            obj[name] = expand_env_vars(value)
+    else:
+        obj = os.path.expandvars(obj)
+    return obj