From 0ef1fb8a097e91145c694bef85dc50c4341e92a9 Mon Sep 17 00:00:00 2001 From: Vitaliy Urusovskij Date: Fri, 30 Oct 2020 00:37:02 +0300 Subject: [PATCH] Update MemCheckTests OMZ part: get_testdata.py and configs (#2722) --- .../desktop_references_config.xml | 321 +++++++++++++++++- .../nightly_configs/desktop_test_config.xml | 136 -------- tests/stress_tests/scripts/get_testdata.py | 150 ++++---- 3 files changed, 378 insertions(+), 229 deletions(-) diff --git a/tests/stress_tests/.automation/memcheck_tests/nightly_configs/desktop_references_config.xml b/tests/stress_tests/.automation/memcheck_tests/nightly_configs/desktop_references_config.xml index 2f7ef805efe..549f04af833 100644 --- a/tests/stress_tests/.automation/memcheck_tests/nightly_configs/desktop_references_config.xml +++ b/tests/stress_tests/.automation/memcheck_tests/nightly_configs/desktop_references_config.xml @@ -1,6 +1,323 @@ - - + # References were collected from DB with next query: {"target_branch": "releases/2020/4", "commit_date": + "2020-06-15 13:21:41+00:00"} + # and modified on FACTOR = 1.3 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + # values from {"target_branch": "releases/2020/2", "commit_date": "2020-05-14 11:19:36+00:00"} and *= 1.3 + # values from {"target_branch": "releases/2020/2", "commit_date": "2020-05-14 11:19:36+00:00"} and *= 1.3 + # values from {"target_branch": "releases/2020/2", "commit_date": "2020-05-14 11:19:36+00:00"} and *= 1.3 + # values from {"target_branch": "releases/2020/2", "commit_date": "2020-05-14 11:19:36+00:00"} and *= 1.3 + # values from {"target_branch": "releases/2020/2", "commit_date": "2020-05-14 11:19:36+00:00"} and *= 1.3 + # values from {"target_branch": "releases/2020/2", "commit_date": "2020-05-14 11:19:36+00:00"} and *= 1.3 + # values from {"target_branch": "releases/2020/2", "commit_date": "2020-05-14 11:19:36+00:00"} and *= 1.3 + # values from {"target_branch": "releases/2020/2", "commit_date": "2020-05-14 11:19:36+00:00"} and *= 1.3 + # values from {"target_branch": "releases/2020/2", "commit_date": "2020-05-14 11:19:36+00:00"} and *= 1.3 + # values from {"target_branch": "releases/2020/2", "commit_date": "2020-05-14 11:19:36+00:00"} and *= 1.3 + # values from {"target_branch": "releases/2020/2", "commit_date": "2020-05-14 11:19:36+00:00"} and *= 1.3 + # values from {"target_branch": "releases/2020/2", "commit_date": "2020-05-14 11:19:36+00:00"} and *= 1.3 \ No newline at end of file diff --git a/tests/stress_tests/.automation/memcheck_tests/nightly_configs/desktop_test_config.xml b/tests/stress_tests/.automation/memcheck_tests/nightly_configs/desktop_test_config.xml index 4d7fc7d7058..5ed61a76729 100644 --- a/tests/stress_tests/.automation/memcheck_tests/nightly_configs/desktop_test_config.xml +++ b/tests/stress_tests/.automation/memcheck_tests/nightly_configs/desktop_test_config.xml @@ -5,14 +5,9 @@ - - - - - @@ -23,49 +18,14 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -73,111 +33,15 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/tests/stress_tests/scripts/get_testdata.py b/tests/stress_tests/scripts/get_testdata.py index 4afe39e5d2e..785013e249f 100755 --- a/tests/stress_tests/scripts/get_testdata.py +++ b/tests/stress_tests/scripts/get_testdata.py @@ -11,7 +11,6 @@ Usage: ./scrips/get_testdata.py import argparse import logging as log -import multiprocessing import os import shutil import subprocess @@ -26,7 +25,6 @@ log.basicConfig(format="{file}: [ %(levelname)s ] %(message)s".format(file=os.pa # Parameters OMZ_NUM_ATTEMPTS = 6 -DOWNLOADER_JOBS_NUM = 4 def abs_path(relative_path): @@ -120,18 +118,6 @@ def main(): # constants PRECISION = "FP32" - # parse models from test config - test_conf_obj = ET.parse(str(args.test_conf)) - test_conf_root = test_conf_obj.getroot() - models_from_cfg = [] - for model_rec in test_conf_root.find("models"): - if "source" in model_rec.attrib and "name" in model_rec.attrib: - if model_rec.attrib["source"] == "omz": - models_from_cfg.append(model_rec.attrib["name"]) - else: - log.warning("Record from test config '{}' doesn't contain attributes 'source' or 'name'" - .format(args.test_conf.resolve())) - # prepare Open Model Zoo if args.omz_repo: omz_path = Path(args.omz_repo).resolve() @@ -144,69 +130,6 @@ def main(): ' https://github.com/opencv/open_model_zoo {omz_path}'.format(omz_path=omz_path) run_in_subprocess(cmd) - # prepare models list - downloader_path = omz_path / "tools" / "downloader" / "downloader.py" - models_list_path = args.omz_models_out_dir / "models_list.txt" - - cmd = '{downloader_path} --print_all'.format(downloader_path=downloader_path) - models_available = subprocess.check_output(cmd, shell=True, universal_newlines=True).split("\n") - models_to_run = set(models_from_cfg).intersection(models_available) - - os.makedirs(str(args.omz_models_out_dir), exist_ok=True) - log.info("List of models from {models_list_path} used for downloader.py and converter.py: " - "{models_to_run}".format(models_list_path=models_list_path, models_to_run=",".join(models_to_run))) - with open(str(models_list_path), "w") as file: - file.writelines([name + "\n" for name in models_to_run]) - if set(models_from_cfg) - models_to_run: - log.warning("List of models defined in config but not available in OMZ: {}" - .format(",".join(set(models_from_cfg) - models_to_run))) - - # update test config with Open Model Zoo info - info_dumper_path = omz_path / "tools" / "downloader" / "info_dumper.py" - cmd = "{executable} {info_dumper_path} --list {models_list_path}"\ - .format(executable=sys.executable, info_dumper_path=info_dumper_path, - models_list_path=models_list_path) - out = subprocess.check_output(cmd, shell=True, universal_newlines=True) - models_info = json.loads(out) - - fields_to_add = ["framework", "subdirectory"] - models_root = test_conf_root.find("models") - for model_rec in list(models_root): # convert iterator to list to prevent incorrect removing of records - if model_rec.attrib.get("source") == "omz" and "name" in model_rec.attrib: - if model_rec.attrib["name"] not in models_to_run: - # remove models from test config which aren't available in OMZ - models_root.remove(model_rec) - else: - # Open Model Zoo ensures name uniqueness of every model - models_info_rec = next(iter([rec for rec in models_info if rec["name"] == model_rec.attrib["name"]])) - info_to_add = {key: models_info_rec[key] for key in fields_to_add} - model_rec.attrib.update(info_to_add) - model_rec.attrib["precision"] = PRECISION - model_rec.attrib["path"] = str( - Path(model_rec.attrib["subdirectory"]) / PRECISION / (model_rec.attrib["name"] + ".xml")) - model_rec.attrib["full_path"] = str( - args.omz_irs_out_dir / model_rec.attrib["subdirectory"] / PRECISION / (model_rec.attrib["name"] + ".xml")) - test_conf_obj.write(args.test_conf) - - # prepare models - cmd = '{downloader_path} --list {models_list_path}' \ - ' --num_attempts {num_attempts}' \ - ' --output_dir {models_dir}' \ - ' --cache_dir {cache_dir}' \ - ' --jobs {jobs_num}'.format(downloader_path=downloader_path, models_list_path=models_list_path, - num_attempts=OMZ_NUM_ATTEMPTS, - models_dir=args.omz_models_out_dir, - cache_dir=args.omz_cache_dir, - jobs_num=DOWNLOADER_JOBS_NUM) - - run_in_subprocess(cmd, check_call=not args.skip_omz_errors) - - # Open Model Zoo doesn't copy downloaded IRs to converter.py output folder where IRs should be stored. - # Do it manually to have only one folder with IRs - for ir_src_path in args.omz_models_out_dir.rglob("*.xml"): - ir_dst_path = args.omz_irs_out_dir / os.path.relpath(ir_src_path, args.omz_models_out_dir) - shutil.copytree(ir_src_path.parent, ir_dst_path.parent) - # prepare virtual environment and install requirements python_executable = sys.executable if not args.no_venv: @@ -221,20 +144,65 @@ def main(): Venv.create_n_install_requirements(*requirements) python_executable = Venv.get_venv_executable() - # convert models to IRs - converter_path = omz_path / "tools" / "downloader" / "converter.py" - # NOTE: remove --precision if both precisions (FP32 & FP16) required - cmd = '{executable} {converter_path} --list {models_list_path}' \ - ' -p {executable}' \ - ' --precision={PRECISION}' \ - ' --output_dir {irs_dir}' \ - ' --download_dir {models_dir}' \ - ' --mo {mo_tool} --jobs {workers_num}'.format(executable=python_executable, PRECISION=PRECISION, - converter_path=converter_path, - models_list_path=models_list_path, irs_dir=args.omz_irs_out_dir, - models_dir=args.omz_models_out_dir, mo_tool=args.mo_tool, - workers_num=multiprocessing.cpu_count()) - run_in_subprocess(cmd, check_call=not args.skip_omz_errors) + # parse models from test config + test_conf_obj = ET.parse(str(args.test_conf)) + test_conf_root = test_conf_obj.getroot() + for model_rec in test_conf_root.find("models"): + if "name" not in model_rec.attrib or model_rec.attrib.get("source") != "omz": + continue + model_name = model_rec.attrib["name"] + + info_dumper_path = omz_path / "tools" / "downloader" / "info_dumper.py" + cmd = "{executable} {info_dumper_path} --name {model_name}"\ + .format(executable=sys.executable, info_dumper_path=info_dumper_path, + model_name=model_name) + out = subprocess.check_output(cmd, shell=True, universal_newlines=True) + model_info = json.loads(out)[0] + + # update model record from test config with Open Model Zoo info + fields_to_add = ["framework", "subdirectory"] + info_to_add = {key: model_info[key] for key in fields_to_add} + model_rec.attrib.update(info_to_add) + model_rec.attrib["precision"] = PRECISION + model_rec.attrib["path"] = str( + Path(model_rec.attrib["subdirectory"]) / PRECISION / (model_rec.attrib["name"] + ".xml")) + model_rec.attrib["full_path"] = str( + args.omz_irs_out_dir / model_rec.attrib["subdirectory"] / PRECISION / (model_rec.attrib["name"] + ".xml")) + + # prepare models + downloader_path = omz_path / "tools" / "downloader" / "downloader.py" + cmd = '{downloader_path} --name {model_name}' \ + ' --precisions={PRECISION}' \ + ' --num_attempts {num_attempts}' \ + ' --output_dir {models_dir}' \ + ' --cache_dir {cache_dir}'.format(downloader_path=downloader_path, model_name=model_name, + PRECISION=PRECISION, num_attempts=OMZ_NUM_ATTEMPTS, + models_dir=args.omz_models_out_dir, cache_dir=args.omz_cache_dir) + + run_in_subprocess(cmd, check_call=not args.skip_omz_errors) + + # convert models to IRs + converter_path = omz_path / "tools" / "downloader" / "converter.py" + # NOTE: remove --precisions if both precisions (FP32 & FP16) required + cmd = '{executable} {converter_path} --name {model_name}' \ + ' -p {executable}' \ + ' --precisions={PRECISION}' \ + ' --output_dir {irs_dir}' \ + ' --download_dir {models_dir}' \ + ' --mo {mo_tool}'.format(executable=python_executable, PRECISION=PRECISION, + converter_path=converter_path, + model_name=model_name, irs_dir=args.omz_irs_out_dir, + models_dir=args.omz_models_out_dir, mo_tool=args.mo_tool) + run_in_subprocess(cmd, check_call=not args.skip_omz_errors) + + # rewrite test config with updated records + test_conf_obj.write(args.test_conf) + + # Open Model Zoo doesn't copy downloaded IRs to converter.py output folder where IRs should be stored. + # Do it manually to have only one folder with IRs + for ir_src_path in args.omz_models_out_dir.rglob("*.xml"): + ir_dst_path = args.omz_irs_out_dir / os.path.relpath(ir_src_path, args.omz_models_out_dir) + shutil.copytree(ir_src_path.parent, ir_dst_path.parent) if __name__ == "__main__":