IntenseWebs/openvino
3
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Stress, Memory tests update to comply with new OMZ tools layout (#7886)

Browse Source 
* work with omz: change hardcoded paths for memory_tests, fix paths for memcheck get_testdata.py script

* change paths to omz tools

* revert mo changes

* remove mo from arguments

* remove extra var from cmd string

* remove is_db_used check

* add timeout to proc_exec

* merge master
This commit is contained in:
Victor Kuznetsov authored and GitHub committed 2021-10-13 10:42:00 +03:00
1 parent db527fff41
commit 6d9a7a108e
5 files changed
+63 -70
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/stress_tests/scripts/get_testdata.py
+14 -16
View File
@@ -55,8 +55,7 @@ class VirtualEnv:
def create(self):
"""Creates virtual environment."""
cmd = '{executable} -m venv {venv}'.format(executable=sys.executable,
venv=self.get_venv_dir())
cmd = '{executable} -m venv {venv}'.format(executable=sys.executable, venv=self.get_venv_dir())
run_in_subprocess(cmd)
self.is_created = True
@@ -113,7 +112,6 @@ def main():
parser.add_argument('--omz_repo', required=False,
help='Path to Open Model Zoo (OMZ) repository. It will be used to skip cloning step.')
parser.add_argument('--mo_tool', type=Path,
default=Path(abs_path('../../../model-optimizer/mo.py')).resolve(),
help='Path to Model Optimizer (MO) runner. Required for OMZ converter.py only.')
parser.add_argument('--omz_models_out_dir', type=Path,
default=abs_path('../_omz_out/models'),
@@ -147,11 +145,11 @@ def main():
if not args.no_venv:
Venv = VirtualEnv("./.stress_venv")
requirements = [
omz_path / "tools" / "downloader" / "requirements.in",
args.mo_tool.parent / "requirements.txt",
args.mo_tool.parent / "requirements_dev.txt",
omz_path / "tools" / "downloader" / "requirements-caffe2.in",
omz_path / "tools" / "downloader" / "requirements-pytorch.in"
omz_path / "tools" / "model_tools" / "requirements.in",
omz_path / "tools" / "model_tools" / "requirements-caffe2.in",
omz_path / "tools" / "model_tools" / "requirements-pytorch.in"
]
Venv.create_n_install_requirements(*requirements)
python_executable = Venv.get_venv_executable()
@@ -166,7 +164,7 @@ def main():
model_name = model_rec.attrib["name"]
precision = model_rec.attrib["precision"]
info_dumper_path = omz_path / "tools" / "downloader" / "info_dumper.py"
info_dumper_path = omz_path / "tools" / "model_tools" / "info_dumper.py"
cmd = '"{executable}" "{info_dumper_path}" --name {model_name}'.format(executable=sys.executable,
info_dumper_path=info_dumper_path,
model_name=model_name)
@@ -194,28 +192,28 @@ def main():
args.omz_irs_out_dir / model_rec.attrib["subdirectory"] / precision / (model_rec.attrib["name"] + ".xml"))
# prepare models
downloader_path = omz_path / "tools" / "downloader" / "downloader.py"
downloader_path = omz_path / "tools" / "model_tools" / "downloader.py"
cmd = '{downloader_path} --name {model_name}' \
' --precisions={precision}' \
' --num_attempts {num_attempts}' \
' --output_dir {models_dir}' \
' --cache_dir {cache_dir}'.format(downloader_path=downloader_path, model_name=model_name,
precision=precision, num_attempts=OMZ_NUM_ATTEMPTS,
models_dir=args.omz_models_out_dir, cache_dir=args.omz_cache_dir)
' --cache_dir {cache_dir}'.format(downloader_path=downloader_path, precision=precision,
models_dir=args.omz_models_out_dir,
num_attempts=OMZ_NUM_ATTEMPTS, model_name=model_name,
cache_dir=args.omz_cache_dir)
run_in_subprocess(cmd, check_call=not args.skip_omz_errors)
# convert models to IRs
converter_path = omz_path / "tools" / "downloader" / "converter.py"
converter_path = omz_path / "tools" / "model_tools" / "converter.py"
# NOTE: remove --precisions if both precisions (FP32 & FP16) required
cmd = '{executable} {converter_path} --name {model_name}' \
' -p {executable}' \
' -p "{executable}"' \
' --precisions={precision}' \
' --output_dir {irs_dir}' \
' --download_dir {models_dir}' \
' --mo {mo_tool}'.format(executable=python_executable, precision=precision,
converter_path=converter_path,
model_name=model_name, irs_dir=args.omz_irs_out_dir,
' --mo {mo_tool}'.format(executable=python_executable, converter_path=converter_path,
precision=precision, model_name=model_name, irs_dir=args.omz_irs_out_dir,
models_dir=args.omz_models_out_dir, mo_tool=args.mo_tool)
run_in_subprocess(cmd, check_call=not args.skip_omz_errors)
tests/stress_tests/scripts/run_memcheck.py
+22 -22
View File
@@ -16,15 +16,11 @@ import subprocess
import sys
from glob import glob
from compare_memcheck_2_runs import compare_memcheck_2_runs, \
get_memcheck_records, get_db_memcheck_records
from compare_memcheck_2_runs import compare_memcheck_2_runs, get_memcheck_records, get_db_memcheck_records
# Database arguments
from memcheck_upload import DATABASE, DB_COLLECTIONS
from memcheck_upload import create_memcheck_records, \
upload_memcheck_records, \
create_memcheck_report, \
metadata_from_manifest, \
info_from_test_config
from memcheck_upload import create_memcheck_records, upload_memcheck_records, create_memcheck_report, \
metadata_from_manifest, info_from_test_config
def run(args, log=None, verbose=True):
@@ -74,7 +70,7 @@ def main():
binary_args = []
for idx, arg in enumerate(sys.argv):
if arg == '--':
binary_args = sys.argv[idx+1:]
binary_args = sys.argv[idx + 1:]
sys.argv = sys.argv[:idx]
break
@@ -94,15 +90,21 @@ def main():
description='Run memcheck tests',
usage='%(prog)s [options] binary -- [additional args]',
parents=[init_parser])
parser.add_argument('binary', help='test binary to execute')
parser.add_argument('--gtest_parallel', help='path to gtest-parallel to use',
parser.add_argument('binary',
help='test binary to execute')
parser.add_argument('--gtest_parallel',
help='path to gtest-parallel to use',
default='gtest_parallel')
parser.add_argument('--timeout', help='timeout for tests run within gtest-parallel')
parser.add_argument('--timeout',
help='timeout for tests run within gtest-parallel',
default='')
parser.add_argument('-d', '--output_dir',
required=args.timeline_report or args.upload or args.compare,
help='output directory for test logs')
parser.add_argument('-w', '--workers', help='number of gtest-parallel workers to spawn')
help='output directory for test logs',
default='')
parser.add_argument('-w', '--workers',
help='number of gtest-parallel workers to spawn',
default='')
parser.add_argument('--db_url',
required=args.timeline_report or args.upload or
(args.compare and not os.path.isdir(args.compare)),
@@ -122,8 +124,7 @@ def main():
parser.add_argument('--ref_db_collection',
required=args.compare and not os.path.isdir(args.compare),
help=f'use collection name in {DATABASE} database to query'
f' reference data',
help=f'use collection name in {DATABASE} database to query reference data',
choices=DB_COLLECTIONS)
parser.add_argument('--comparison_report',
required=args.compare,
@@ -145,12 +146,11 @@ def main():
args.output_dir)
sys.exit(1)
return_code, _ = run([sys.executable, args.gtest_parallel] +
(['--output_dir', f'{args.output_dir}'] if args.output_dir else []) +
(['--workers', f'{args.workers}'] if args.workers else []) +
(['--timeout', f'{args.timeout}'] if args.timeout else []) +
[args.binary] +
['--'] + binary_args)
return_code, _ = run([sys.executable, args.gtest_parallel,
'--output_dir', f'{args.output_dir}',
'--workers', f'{args.workers}',
'--timeout', f'{args.timeout}',
args.binary, '--'] + binary_args)
if args.upload or args.timeline_report or args.compare:
# prepare commit information