From 81da815dd8f864dce710cc09df2b4b8180967eb7 Mon Sep 17 00:00:00 2001 From: Andrey Somsikov Date: Thu, 28 Jan 2021 13:27:07 +0300 Subject: [PATCH] Add conditional compilation tests (#3996) --- tests/conditional_compilation/conftest.py | 100 ++++++++++++++++++ tests/conditional_compilation/test_collect.py | 40 +++++++ tests/conditional_compilation/test_infer.py | 17 +++ tests/lib/__init__.py | 0 tests/lib/proc_utils.py | 43 ++++++++ 5 files changed, 200 insertions(+) create mode 100644 tests/conditional_compilation/conftest.py create mode 100644 tests/conditional_compilation/test_collect.py create mode 100644 tests/conditional_compilation/test_infer.py create mode 100644 tests/lib/__init__.py create mode 100644 tests/lib/proc_utils.py diff --git a/tests/conditional_compilation/conftest.py b/tests/conditional_compilation/conftest.py new file mode 100644 index 00000000000..3a0ae137029 --- /dev/null +++ b/tests/conditional_compilation/conftest.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# Copyright (C) 2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# pylint: disable=line-too-long + +""" Pytest configuration for compilation tests. + +Sample usage: +python3 -m pytest --artifacts ./compiled --models_root= \ + --sea_runtool=./IntelSEAPI/runtool/sea_runtool.py \ + --benchmark_app=./bin/benchmark_app test_collect.py +""" + +import sys +from inspect import getsourcefile +from pathlib import Path + +import pytest + +# add ../lib to imports +sys.path.insert( + 0, str((Path(getsourcefile(lambda: 0)) / ".." / ".." / "lib").resolve(strict=True)) +) + +# Using models from https://github.com/openvinotoolkit/testdata +# $find models -wholename "*.xml" +TESTS = [ + {"path": "models/mobilenet_v2_1.4_224/mobilenet_v2_1.4_224_i8.xml"}, + {"path": "models/mobilenet_v2_1.0_224/mobilenet_v2_1.0_224_i8.xml"}, + {"path": "models/inception_v3/inception_v3_i8.xml"}, + {"path": "models/resnet_v1_50/resnet_v1_50_i8.xml"}, + {"path": "models/test_model/test_model_fp16.xml"}, + {"path": "models/test_model/test_model_fp32.xml"}, +] + + +def pytest_addoption(parser): + """ Define extra options for pytest options + """ + parser.addoption( + "--models_root", required=True, type=Path, help="Path to models root directory" + ) + parser.addoption( + "--sea_runtool", required=True, type=Path, help="Path to sea_runtool.py" + ) + parser.addoption( + "--benchmark_app", + required=True, + type=Path, + help="Path to the benchmark_app tool", + ) + parser.addoption( + "-A", + "--artifacts", + required=True, + type=Path, + help="Artifacts directory where tests write output or read input", + ) + + +def pytest_generate_tests(metafunc): + """ Generate tests depending on command line options + """ + params = [] + ids = [] + + for test in TESTS: + extra_args = {} + path = test["path"] + if "marks" in test: + extra_args["marks"] = test["marks"] + + params.append(pytest.param(Path(path), **extra_args)) + ids = ids + [path] + metafunc.parametrize("model", params, ids=ids) + + +@pytest.fixture(scope="session") +def sea_runtool(request): + """Fixture function for command-line option.""" + return request.config.getoption("sea_runtool") + + +@pytest.fixture(scope="session") +def benchmark_app(request): + """Fixture function for command-line option.""" + return request.config.getoption("benchmark_app") + + +@pytest.fixture(scope="session") +def models_root(request): + """Fixture function for command-line option.""" + return request.config.getoption("models_root") + + +@pytest.fixture(scope="session") +def artifacts(request): + """Fixture function for command-line option.""" + return request.config.getoption("artifacts") diff --git a/tests/conditional_compilation/test_collect.py b/tests/conditional_compilation/test_collect.py new file mode 100644 index 00000000000..4088c2f7d22 --- /dev/null +++ b/tests/conditional_compilation/test_collect.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# Copyright (C) 2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +""" Test conditional compilation statistics collection. +""" + +import glob +import os + +from proc_utils import cmd_exec # pylint: disable=import-error + + +def test_cc_collect(model, sea_runtool, benchmark_app, models_root, artifacts): + """ Test conditional compilation statistics collection + """ + out = artifacts / model.parent / model.stem + # cleanup old data if any + prev_results = glob.glob(f"{out}.pid*.csv") + for path in prev_results: + os.remove(path) + # run use case + returncode, _ = cmd_exec( + [ + "python", + str(sea_runtool), + f"-o={out}", + "-f=stat", + "!", + str(benchmark_app), + "-d=CPU", + f"-m={models_root / model}", + "-niter=1", + "-nireq=1", + ] + ) + assert returncode == 0, f"Command exited with non-zero status {returncode}" + assert ( + len(glob.glob(f"{out}.pid*.csv")) == 1 + ), f'Multiple or none "{out}.pid*.csv" files' diff --git a/tests/conditional_compilation/test_infer.py b/tests/conditional_compilation/test_infer.py new file mode 100644 index 00000000000..b7d66f72015 --- /dev/null +++ b/tests/conditional_compilation/test_infer.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# Copyright (C) 2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +""" Test inference with conditional compiled binaries. +""" + +from proc_utils import cmd_exec # pylint: disable=import-error + + +def test_infer(model, models_root, benchmark_app): + """ Test inference with conditional compiled binaries + """ + returncode, _ = cmd_exec( + [str(benchmark_app), "-d=CPU", f"-m={models_root / model}", "-niter=1", "-nireq=1"] + ) + assert returncode == 0, f"Command exited with non-zero status {returncode}" diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/lib/proc_utils.py b/tests/lib/proc_utils.py new file mode 100644 index 00000000000..e207369a513 --- /dev/null +++ b/tests/lib/proc_utils.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# Copyright (C) 2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +""" Common utilities for working with processes. +""" + +import logging +import subprocess + + +def cmd_exec(args, log=None, verbose=True): + """ Run cmd using subprocess with logging and other improvements + """ + if log is None: + log = logging.getLogger() + log_out = log.info if verbose else log.debug + + log_out( # pylint: disable=logging-fstring-interpolation + f'========== cmd: {" ".join(args)}' + ) + + proc = subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + universal_newlines=True, + ) + output = [] + for line in iter(proc.stdout.readline, ""): + log_out(line.strip("\n")) + output.append(line) + if line or proc.poll() is None: + continue + break + outs = proc.communicate()[0] + + if outs: + log_out(outs.strip("\n")) + output.append(outs) + log_out("========== Completed. Exit code: %d", proc.returncode) + return proc.returncode, "".join(output)