IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change logic for using multiple models in cc tests (#6618)

Browse Source 
* Change logic for using multiple models in cc tests

* removed extra key

* moved multiple infer to main function

* moved multiple infer to main function

* change test_config.yml and work with this file

* change test_config.yml and work with this file

* removed extra key

* changed separator symbol

* removed extra param from output

* Rollback comments in test_config.yml

* removed extra key in yml config file, change logic to save .npz for all model seperate instead of common npz file, change logic in parser arg

* removed extra key in yml config file, change logic to save .npz for all model seperate instead of common npz file, change logic in parser arg

* changed save path for infer result, deleted extra param

* updated test_config.yml

* rollback old way to get bin_path

* changed work with save path for inference result

* add empty line in the end for config file

* rollback line order

* removed extra param allow_pickle=True in loading .npz file

* removed mkdir in run_infer

* uncomment resnet model

* added empty line

* changed save folder for cc result
This commit is contained in:
Anastasiya Koryachikhina 2021-07-30 10:30:38 +03:00 committed by GitHub
parent bc06279825
commit 4d7f6c54ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 103 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
tests/conditional_compilation/conftest.py
View File

@ -6,7 +6,6 @@
# pylint: disable=line-too-long
"""Pytest configuration for compilation tests."""
import logging
import sys
from inspect import getsourcefile
@ -19,8 +18,8 @@ import yaml
import pytest
from path_utils import expand_env_vars # pylint: disable=import-error
from test_utils import make_build, validate_path_arg, write_session_info, SESSION_INFO_FILE # pylint: disable=import-error
from test_utils import make_build, validate_path_arg, write_session_info, \
SESSION_INFO_FILE # pylint: disable=import-error
log = logging.getLogger()
@ -71,15 +70,19 @@ def pytest_generate_tests(metafunc):
test_cases = yaml.safe_load(file)
for test in test_cases:
extra_args = {}
model_path = test["model"]["path"]
if "marks" in test:
extra_args["marks"] = test["marks"]
model_list = []
test_id_list = []
for models in test:
extra_args = {}
model_path = models["model"]["path"]
if "marks" in test:
extra_args["marks"] = test["marks"]
model_list.append(expand_env_vars(model_path))
test_id_list.append(model_path.split("/")[- 1])
ids = ids + ['-'.join(test_id_list)]
params.append(pytest.param('-'.join(test_id_list), model_list), **extra_args)
test_id = model_path.replace("$", "").replace("{", "").replace("}", "")
params.append(pytest.param(test_id, Path(expand_env_vars(model_path)), **extra_args))
ids = ids + [test_id]
metafunc.parametrize("test_id, model", params, ids=ids)
metafunc.parametrize("test_id, models", params, ids=ids)
@pytest.fixture(scope="session")

70
tests/conditional_compilation/test_cc.py
View File

@ -10,27 +10,30 @@ import glob
import logging
import os
import sys
from pathlib import Path
import numpy as np
import pytest
from proc_utils import cmd_exec # pylint: disable=import-error
from test_utils import get_lib_sizes, infer_tool, make_build, run_infer # pylint: disable=import-error
from test_utils import get_lib_sizes, infer_tool, make_build, run_infer # pylint: disable=import-error
log = logging.getLogger()
@pytest.mark.dependency(name="cc_collect")
def test_cc_collect(test_id, model, openvino_ref, test_info,
def test_cc_collect(test_id, models, openvino_ref, test_info,
save_session_info, sea_runtool, collector_dir, artifacts): # pylint: disable=unused-argument
"""Test conditional compilation statistics collection
:param test_info: custom `test_info` field of built-in `request` pytest fixture.
contain a dictionary to store test metadata.
"""
out = artifacts / test_id
infer_out_dir = out / "inference_result"
test_info["test_id"] = test_id
# cleanup old data if any
prev_result = glob.glob(f"{out}.pid*.csv")
prev_result = glob.glob(f"{out / test_id}.pid*.csv")
for path in prev_result:
os.remove(path)
# run use case
@ -38,32 +41,32 @@ def test_cc_collect(test_id, model, openvino_ref, test_info,
[
sys.executable,
str(sea_runtool),
f"--output={out}",
f"--output={out / test_id}",
f"--bindir={collector_dir}",
"!",
sys.executable,
infer_tool,
f"-m={model}",
*[f"-m={model}" for model in models],
"-d=CPU",
f"-r={out}",
f"-r={infer_out_dir}"
]
)
out_csv = glob.glob(f"{out}.pid*.csv")
out_csv = glob.glob(f"{out / test_id}.pid*.csv")
test_info["out_csv"] = out_csv
assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}"
assert len(out_csv) == 1, f'Multiple or none "{out}.pid*.csv" files'
assert len(out_csv) == 1, f'Multiple or none "{out / test_id}.pid*.csv" files'
@pytest.mark.dependency(depends=["cc_collect"])
def test_minimized_pkg(test_id, model, openvino_root_dir, artifacts): # pylint: disable=unused-argument
def test_minimized_pkg(test_id, models, openvino_root_dir, artifacts): # pylint: disable=unused-argument
"""Build and install OpenVINO package with collected conditional compilation statistics."""
out = artifacts / test_id
install_prefix = out / "install_pkg"
build_dir = openvino_root_dir / "build_minimized"
out_csv = glob.glob(f"{out}.pid*.csv")
assert len(out_csv) == 1, f'Multiple or none "{out}.pid*.csv" files'
out_csv = glob.glob(f"{out / test_id}.pid*.csv")
assert len(out_csv) == 1, f'Multiple or none "{out / test_id}.pid*.csv" files'
log.info("Building minimized build at %s", build_dir)
@ -78,38 +81,47 @@ def test_minimized_pkg(test_id, model, openvino_root_dir, artifacts): # pylint:
@pytest.mark.dependency(depends=["cc_collect", "minimized_pkg"])
def test_infer(test_id, model, artifacts):
def test_infer(test_id, models, artifacts):
"""Test inference with conditional compiled binaries."""
out = artifacts / test_id
minimized_pkg = out / "install_pkg"
return_code, output = run_infer(model, f"{out}_cc.npz", minimized_pkg)
infer_out_dir_cc = out / "inference_result_cc/"
return_code, output = run_infer(models, infer_out_dir_cc, minimized_pkg)
assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}"
@pytest.mark.dependency(depends=["cc_collect", "minimized_pkg"])
def test_verify(test_id, model, openvino_ref, artifacts, tolerance=1e-6): # pylint: disable=too-many-arguments
def test_verify(test_id, models, openvino_ref, artifacts, tolerance=1e-6): # pylint: disable=too-many-arguments
"""Test verifying that inference results are equal."""
out = artifacts / test_id
minimized_pkg = out / "install_pkg"
out_file = f"{out}.npz"
out_file_cc = f"{out}_cc.npz"
return_code, output = run_infer(model, out_file, openvino_ref)
infer_out_dir_cc = out / "inference_result_cc/"
infer_out_dir = out / "inference_result/"
return_code, output = run_infer(models, infer_out_dir, openvino_ref)
assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}"
return_code, output = run_infer(model, out_file_cc, minimized_pkg)
return_code, output = run_infer(models, infer_out_dir_cc, minimized_pkg)
assert return_code == 0, f"Command exited with non-zero status {return_code}:\n {output}"
reference_results = dict(np.load(out_file))
inference_results = dict(np.load(out_file_cc))
assert sorted(reference_results.keys()) == sorted(
inference_results.keys()
), "Results have different number of layers"
for layer in reference_results.keys():
assert np.allclose(
reference_results[layer], inference_results[layer], tolerance
), "Reference and inference results differ"
for model in models:
out_file = f"{infer_out_dir / Path(model).name}.npz"
out_file_cc = f"{infer_out_dir_cc / Path(model).name}.npz"
reference_results = dict(np.load(out_file))
inference_results = dict(np.load(out_file_cc))
assert sorted(reference_results.keys()) == sorted(
inference_results.keys()
), "Results have different number of layers"
for layer in reference_results.keys():
assert np.allclose(
reference_results[layer], inference_results[layer], tolerance
), "Reference and inference results differ"
@pytest.mark.dependency(depends=["cc_collect", "minimized_pkg"])
def test_libs_size(test_id, model, openvino_ref, artifacts): # pylint: disable=unused-argument
def test_libs_size(test_id, models, openvino_ref, artifacts): # pylint: disable=unused-argument
"""Test if libraries haven't increased in size after conditional compilation."""
libraries = ["inference_engine_transformations", "MKLDNNPlugin", "ngraph"]
minimized_pkg = artifacts / test_id / "install_pkg"

40
tests/conditional_compilation/test_config.yml
View File

@ -1,15 +1,31 @@
# Using models from https://github.com/openvinotoolkit/testdata
# $find models -wholename "*.xml"
- model:
path: ${TESTDATA}/models/mobilenet_v2_1.4_224/mobilenet_v2_1.4_224_i8.xml
- model:
path: ${TESTDATA}/models/mobilenet_v2_1.0_224/mobilenet_v2_1.0_224_i8.xml
- model:
path: ${TESTDATA}/models/inception_v3/inception_v3_i8.xml
#- model:
# path: ${TESTDATA}/models/resnet_v1_50/resnet_v1_50_i8.xml
- model:
path: ${TESTDATA}/models/test_model/test_model_fp16.xml
- model:
path: ${TESTDATA}/models/test_model/test_model_fp32.xml
-
- model:
path: ${TESTDATA}/models/mobilenet_v2_1.4_224/mobilenet_v2_1.4_224_i8.xml
-
- model:
path: ${TESTDATA}/models/mobilenet_v2_1.0_224/mobilenet_v2_1.0_224_i8.xml
-
- model:
path: ${TESTDATA}/models/test_model/test_model_fp16.xml
-
- model:
path: ${TESTDATA}/models/test_model/test_model_fp32.xml
-
- model:
path: ${TESTDATA}/models/resnet_v1_50/resnet_v1_50_i8.xml
-
- model:
path: ${TESTDATA}/models/inception_v3/inception_v3_i8.xml
-
- model:
path: ${TESTDATA}/models/test_model/test_model_fp16.xml
- model:
path: ${TESTDATA}/models/test_model/test_model_fp32.xml
-
- model:
path: ${TESTDATA}/models/mobilenet_v2_1.4_224/mobilenet_v2_1.4_224_i8.xml
- model:
path: ${TESTDATA}/models/mobilenet_v2_1.0_224/mobilenet_v2_1.0_224_i8.xml

7
tests/conditional_compilation/test_utils.py
View File

@ -59,13 +59,16 @@ def write_session_info(path: Path = Path(getsourcefile(lambda: 0)).parent / SESS
json.dump(data, json_file, indent=4)
def run_infer(model, out_file, install_dir):
def run_infer(models, out_dir, install_dir):
""" Function running inference
"""
out_dir.mkdir(parents=True, exist_ok=True)
return_code, output = cmd_exec(
[sys.executable,
infer_tool,
"-d=CPU", f"-m={model}", f"-r={out_file}"
"-d=CPU",
*[f"-m={model}" for model in models],
f"-r={out_dir}"
],
env=get_openvino_environment(install_dir),
)

24
tests/conditional_compilation/tools/infer_tool.py
View File

@ -9,8 +9,10 @@
"""
import argparse
import logging as log
import sys
import os
import sys
from pathlib import Path
import numpy as np
from openvino.inference_engine import IECore
@ -58,9 +60,9 @@ def cli_parser():
:return: ir path, device and output folder path variables.
"""
parser = argparse.ArgumentParser(description='Arguments for python API inference')
parser.add_argument('-m', dest='ir_path', required=True, help='Path to XML file of IR')
parser.add_argument('-m', dest='ir_path', required=True, help='Path to XML file of IR', action="append")
parser.add_argument('-d', dest='device', required=True, help='Target device to infer on')
parser.add_argument('-r', dest='out_path', required=True,
parser.add_argument('-r', dest='out_path', required=True, type=Path,
help='Dumps results to the output file')
parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
help='Increase output verbosity')
@ -75,9 +77,13 @@ def cli_parser():
if __name__ == "__main__":
ir_path, device, out_path = cli_parser()
results = infer(ir_path=ir_path, device=device)
np.savez(out_path, **results)
log.info("Path for inference results: {}".format(out_path))
log.debug("Inference results:")
log.debug(results)
log.debug("SUCCESS!")
for model in ir_path:
result = infer(ir_path=model, device=device)
np.savez(out_path / f"{Path(model).name}.npz", **result)
log.info("Path for inference results: {}".format(out_path))
log.debug("Inference results:")
log.debug(result)
log.debug("SUCCESS!")