From d48d2109dc68684780511ae028f999a59392693f Mon Sep 17 00:00:00 2001
From: Vitaliy Urusovskij <vitaliy.urusovskij@intel.com>
Date: Mon, 21 Sep 2020 21:33:42 +0300
Subject: [PATCH] Test timetest MVP (#2345)

---
 tests/time_tests/README.md                    |  7 +-
 tests/time_tests/scripts/requirements.txt     |  1 +
 tests/time_tests/test_runner/conftest.py      | 98 +++++++++++++++++++
 tests/time_tests/test_runner/requirements.txt |  3 +
 tests/time_tests/test_runner/test_config.yml  |  8 ++
 tests/time_tests/test_runner/test_timetest.py | 43 +++++++-
 tests/time_tests/test_runner/utils.py         | 20 ++++
 7 files changed, 178 insertions(+), 2 deletions(-)
 create mode 100644 tests/time_tests/scripts/requirements.txt
 create mode 100644 tests/time_tests/test_runner/conftest.py
 create mode 100644 tests/time_tests/test_runner/requirements.txt
 create mode 100644 tests/time_tests/test_runner/test_config.yml
 create mode 100644 tests/time_tests/test_runner/utils.py

diff --git a/tests/time_tests/README.md b/tests/time_tests/README.md
index 52be0f00531..a68b2ff4814 100644
--- a/tests/time_tests/README.md
+++ b/tests/time_tests/README.md
@@ -21,6 +21,11 @@ cmake .. -DInferenceEngineDeveloperPackage_DIR=$(realpath ../../../build) && mak
 
 2. Run test:
 ``` bash
-./scripts/run_timetest.py ../../../bin/intel64/Release/timetest_infer -m model.xml -d CPU
+./scripts/run_timetest.py ../../bin/intel64/Release/timetest_infer -m model.xml -d CPU
 ```
 
+2. Run several configurations using `pytest`:
+``` bash
+export PYTHONPATH=./:$PYTHONPATH
+pytest ./test_runner/test_timetest.py --exe ../../bin/intel64/Release/timetest_infer
+```
diff --git a/tests/time_tests/scripts/requirements.txt b/tests/time_tests/scripts/requirements.txt
new file mode 100644
index 00000000000..3101697065f
--- /dev/null
+++ b/tests/time_tests/scripts/requirements.txt
@@ -0,0 +1 @@
+PyYAML==5.3.1
\ No newline at end of file
diff --git a/tests/time_tests/test_runner/conftest.py b/tests/time_tests/test_runner/conftest.py
new file mode 100644
index 00000000000..02de0f2c505
--- /dev/null
+++ b/tests/time_tests/test_runner/conftest.py
@@ -0,0 +1,98 @@
+# Copyright (C) 2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+"""
+Basic high-level plugin file for pytest.
+
+See [Writing plugins](https://docs.pytest.org/en/latest/writing_plugins.html)
+for more information.
+
+This plugin adds the following command-line options:
+
+* `--test_conf` - Path to test configuration file. Used to parametrize tests.
+  Format: YAML file.
+* `--exe` - Path to a timetest binary to execute.
+* `--niter` - Number of times to run executable.
+"""
+
+# pylint:disable=import-error
+import pytest
+from pathlib import Path
+import yaml
+
+from test_runner.utils import expand_env_vars
+
+
+# -------------------- CLI options --------------------
+
+def pytest_addoption(parser):
+    """Specify command-line options for all plugins"""
+    parser.addoption(
+        "--test_conf",
+        type=Path,
+        help="Path to test config",
+        default=Path(__file__).parent / "test_config.yml"
+    )
+    parser.addoption(
+        "--exe",
+        required=True,
+        dest="executable",
+        type=Path,
+        help="Path to a timetest binary to execute",
+    )
+    parser.addoption(
+        "--niter",
+        type=int,
+        help="Number of iterations to run executable and aggregate results",
+        default=3
+    )
+    # TODO: add support of --mo, --omz etc. required for OMZ support
+
+
+@pytest.fixture(scope="session")
+def test_conf(request):
+    """Fixture function for command-line option."""
+    return request.config.getoption('test_conf')
+
+
+@pytest.fixture(scope="session")
+def executable(request):
+    """Fixture function for command-line option."""
+    return request.config.getoption('executable')
+
+
+@pytest.fixture(scope="session")
+def niter(request):
+    """Fixture function for command-line option."""
+    return request.config.getoption('niter')
+
+# -------------------- CLI options --------------------
+
+
+def pytest_generate_tests(metafunc):
+    """Pytest hook for test generation.
+
+    Generate parameterized tests from discovered modules and test config
+    parameters.
+    """
+    with open(metafunc.config.getoption('test_conf'), "r") as file:
+        test_cases = expand_env_vars(yaml.safe_load(file))
+    if test_cases:
+        metafunc.parametrize("instance", test_cases)
+
+
+def pytest_make_parametrize_id(config, val, argname):
+    """Pytest hook for user-friendly test name representation"""
+
+    def get_dict_values(d):
+        """Unwrap dictionary to get all values of nested dictionaries"""
+        if isinstance(d, dict):
+            for v in d.values():
+                yield from get_dict_values(v)
+        else:
+            yield d
+
+    keys = val.keys()
+    values = list(get_dict_values(val))
+
+    return "-".join(["_".join([key, val]) for key, val in zip(keys, values)])
diff --git a/tests/time_tests/test_runner/requirements.txt b/tests/time_tests/test_runner/requirements.txt
new file mode 100644
index 00000000000..13426ece5e1
--- /dev/null
+++ b/tests/time_tests/test_runner/requirements.txt
@@ -0,0 +1,3 @@
+pytest==4.0.1
+attrs==19.1.0   # required for pytest==4.0.1 to resolve compatibility issues
+PyYAML==5.3.1
\ No newline at end of file
diff --git a/tests/time_tests/test_runner/test_config.yml b/tests/time_tests/test_runner/test_config.yml
new file mode 100644
index 00000000000..bfcb5652646
--- /dev/null
+++ b/tests/time_tests/test_runner/test_config.yml
@@ -0,0 +1,8 @@
+- device:
+    name: CPU
+  model:
+    path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml # TODO: add link to `test_data` repo model
+- device:
+    name: GPU
+  model:
+    path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml
\ No newline at end of file
diff --git a/tests/time_tests/test_runner/test_timetest.py b/tests/time_tests/test_runner/test_timetest.py
index 2064a9b248a..82858ec2522 100644
--- a/tests/time_tests/test_runner/test_timetest.py
+++ b/tests/time_tests/test_runner/test_timetest.py
@@ -1 +1,42 @@
-#TODO: fill
\ No newline at end of file
+# Copyright (C) 2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Main entry-point to run timetests tests.
+
+Default run:
+$ pytest test_timetest.py
+
+Options[*]:
+--test_conf     Path to test config
+--exe           Path to timetest binary to execute
+--niter         Number of times to run executable
+
+[*] For more information see conftest.py
+"""
+
+from pathlib import Path
+import logging
+
+from scripts.run_timetest import run_timetest
+
+
+def test_timetest(instance, executable, niter):
+    """Parameterized test.
+
+    :param instance: test instance
+    :param executable: timetest executable to run
+    :param niter: number of times to run executable
+    """
+    # Prepare model to get model_path
+    model_path = instance["model"].get("path")
+    assert model_path, "Model path is empty"
+
+    # Run executable
+    exe_args = {
+        "executable": Path(executable),
+        "model": Path(model_path),
+        "device": instance["device"]["name"],
+        "niter": niter
+    }
+    retcode, aggr_stats = run_timetest(exe_args, log=logging)
+    assert retcode == 0, "Run of executable failed"
diff --git a/tests/time_tests/test_runner/utils.py b/tests/time_tests/test_runner/utils.py
new file mode 100644
index 00000000000..d1767d8c77e
--- /dev/null
+++ b/tests/time_tests/test_runner/utils.py
@@ -0,0 +1,20 @@
+# Copyright (C) 2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Utility module."""
+
+import os
+
+
+def expand_env_vars(obj):
+    """Expand environment variables in provided object."""
+
+    if isinstance(obj, list):
+        for i, value in enumerate(obj):
+            obj[i] = expand_env_vars(value)
+    elif isinstance(obj, dict):
+        for name, value in obj.items():
+            obj[name] = expand_env_vars(value)
+    else:
+        obj = os.path.expandvars(obj)
+    return obj