diff --git a/tests/memory_tests/CMakeLists.txt b/tests/memory_tests/CMakeLists.txt index fdf0ebe0dd5..d2ca2f7eccd 100644 --- a/tests/memory_tests/CMakeLists.txt +++ b/tests/memory_tests/CMakeLists.txt @@ -4,18 +4,27 @@ cmake_minimum_required(VERSION 3.13) -if (CMAKE_BUILD_TYPE STREQUAL "") - message(STATUS "CMAKE_BUILD_TYPE not defined, 'Release' will be used") - set(CMAKE_BUILD_TYPE "Release") +set (CMAKE_CXX_STANDARD 11) +set (CMAKE_CXX_EXTENSIONS OFF) +set (CMAKE_CXX_STANDARD_REQUIRED ON) +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS}") endif() +set (CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the build type") + +project(memory_tests) + # Define directory where artifacts will be placed set(OUTPUT_ROOT ${CMAKE_CURRENT_SOURCE_DIR}) set(OpenVINO_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../") -project(memory_tests) - -find_package(InferenceEngine REQUIRED) +# Search OpenVINO Inference Engine installed +find_package(OpenVINO REQUIRED) add_subdirectory(src) + +install(DIRECTORY test_runner/ DESTINATION tests/memory_tests/test_runner COMPONENT tests EXCLUDE_FROM_ALL) +install(DIRECTORY .automation/ DESTINATION tests/memory_tests/test_runner/.automation COMPONENT tests EXCLUDE_FROM_ALL) +install(DIRECTORY scripts/ DESTINATION tests/memory_tests/scripts COMPONENT tests EXCLUDE_FROM_ALL) diff --git a/tests/memory_tests/src/memory_tests/CMakeLists.txt b/tests/memory_tests/src/memory_tests/CMakeLists.txt index 20ac7b01d2c..ca1252d2824 100644 --- a/tests/memory_tests/src/memory_tests/CMakeLists.txt +++ b/tests/memory_tests/src/memory_tests/CMakeLists.txt @@ -15,7 +15,7 @@ foreach(test_source ${tests}) get_filename_component(test_name ${test_source} NAME_WE) add_executable(${test_name} ${test_source}) - target_link_libraries(${test_name} PRIVATE IE::inference_engine memory_tests_helper tests_shared_lib) + target_link_libraries(${test_name} PRIVATE memory_tests_helper tests_shared_lib) add_dependencies(memory_tests ${test_name}) diff --git a/tests/memory_tests/test_runner/conftest.py b/tests/memory_tests/test_runner/conftest.py index 3e17afc7a92..d0ba97059d9 100644 --- a/tests/memory_tests/test_runner/conftest.py +++ b/tests/memory_tests/test_runner/conftest.py @@ -18,6 +18,7 @@ This plugin adds the following command-line options: import hashlib import json import logging +import tempfile # pylint:disable=import-error import os import sys @@ -33,11 +34,10 @@ from jsonschema import validate, ValidationError UTILS_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "utils") sys.path.insert(0, str(UTILS_DIR)) -from plugins.conftest import * from path_utils import check_positive_int from proc_utils import cmd_exec from platform_utils import get_os_name, get_os_version, get_cpu_info -from utils import metadata_from_manifest, DATABASES, DB_COLLECTIONS +from utils import metadata_from_manifest, DATABASES, DB_COLLECTIONS, upload_data MEMORY_TESTS_DIR = os.path.dirname(os.path.dirname(__file__)) sys.path.append(MEMORY_TESTS_DIR) @@ -56,8 +56,6 @@ def abs_path(relative_path): # -------------------- CLI options -------------------- - - def pytest_addoption(parser): """Specify command-line options for all plugins""" test_args_parser = parser.getgroup("test run") @@ -177,11 +175,19 @@ def executable(request): def niter(request): """Fixture function for command-line option.""" return request.config.getoption('niter') - - # -------------------- CLI options -------------------- +@pytest.fixture(scope="function") +def temp_dir(pytestconfig): + """Create temporary directory for test purposes. + It will be cleaned up after every test run. + """ + temp_dir = tempfile.TemporaryDirectory() + yield Path(temp_dir.name) + temp_dir.cleanup() + + @pytest.fixture(scope="function") def omz_models_conversion(instance, request): """ @@ -204,8 +210,9 @@ def omz_models_conversion(instance, request): model_precision = instance["instance"]["model"]["precision"] # get full model info - cmd = f'"{sys.executable}" "{info_dumper_path}" --name {model_name}' - _, info = cmd_exec([cmd], shell=True, log=logging) + cmd = [f'{sys.executable}', f'{info_dumper_path}', '--name', f'{model_name}'] + return_code, info = cmd_exec(cmd, log=logging) + assert return_code == 0, "Getting information about OMZ models has failed!" model_info = json.loads(info)[0] @@ -213,29 +220,27 @@ def omz_models_conversion(instance, request): logging.error(f"Please specify precision for the model " f"{model_name} from the list: {model_info['precisions']}") - model_path = Path(model_info["subdirectory"]) / model_precision / (model_name + ".xml") + model_out_path = Path(omz_models_out_dir / model_info["subdirectory"]) / model_precision / ( + model_name + ".xml") model_full_path = omz_irs_out_dir / model_info["subdirectory"] / model_precision / (model_name + ".xml") # prepare models and convert models to IRs - cmd = f'{sys.executable} {downloader_path}' \ - f' --name {model_name}' \ - f' --precisions={model_precision}' \ - f' --num_attempts {OMZ_NUM_ATTEMPTS}' \ - f' --output_dir {omz_models_out_dir}' \ - f' --cache_dir {cache_dir}' - cmd_exec([cmd], shell=True, log=logging) + cmd = [f'{sys.executable}', f'{downloader_path}', '--name', f'{model_name}', + '--precisions', f'{model_precision}', '--num_attempts', f'{OMZ_NUM_ATTEMPTS}', + '--output_dir', f'{omz_models_out_dir}', '--cache_dir', f'{cache_dir}'] - cmd = f'{sys.executable} {converter_path}' \ - f' --name {model_name}' \ - f' -p {sys.executable}' \ - f' --precisions={model_precision}' \ - f' --output_dir {omz_irs_out_dir}' \ - f' --download_dir {omz_models_out_dir}' \ - f' --mo {mo_path}' - cmd_exec([cmd], shell=True, log=logging) + return_code, _ = cmd_exec(cmd, log=logging) + assert return_code == 0, "Downloading OMZ models has failed!" + + cmd = [f'{sys.executable}', f'{converter_path}', '--name', f'{model_name}', '-p', f'{sys.executable}', + '--precisions', f'{model_precision}', '--output_dir', f'{omz_irs_out_dir}', + '--download_dir', f'{omz_models_out_dir}', '--mo', f'{mo_path}'] + + return_code, _ = cmd_exec(cmd, log=logging) + assert return_code == 0, "Converting OMZ models has failed!" instance["instance"]["model"]["framework"] = model_info["framework"] - instance["instance"]["model"]["path"] = model_path + instance["instance"]["model"]["path"] = model_out_path instance["instance"]["model"]["full_path"] = model_full_path @@ -294,6 +299,8 @@ def prepare_db_info(request, instance, executable, niter, manifest_metadata): yield return + instance["db"] = {} + # add db_metadata db_meta_path = request.config.getoption("db_metadata") if db_meta_path: @@ -317,10 +324,10 @@ def prepare_db_info(request, instance, executable, niter, manifest_metadata): "references": instance["instance"].get("references", {}), # upload actual references that were used "ref_factor": REFS_FACTOR, } - info['_id'] = hashlib.sha256( - ''.join([str(info[key]) for key in FIELDS_FOR_ID]).encode()).hexdigest() - instance["db"] = info + info['_id'] = hashlib.sha256(''.join([str(info[key]) for key in FIELDS_FOR_ID]).encode()).hexdigest() + # add metadata + instance["db"].update(info) # add manifest metadata instance["db"].update(manifest_metadata) @@ -433,7 +440,7 @@ def prepare_timeline_report(pytestconfig): env = jinja2.Environment( loader=jinja2.FileSystemLoader( - searchpath=Path().absolute() / 'memory-template'), + searchpath=Path().absolute() / 'memory_template'), autoescape=False) template = env.get_template('timeline_report.html') template.stream(records=records, timelines=timelines).dump(report_path) @@ -539,7 +546,7 @@ def pytest_runtest_makereport(item, call): instance["db"]["status"] = "passed" instance["db"]["results"] = instance["results"] instance["db"]["raw_results"] = instance["raw_results"] - logging.info("Upload data to {}/{}.{}. Data: {}".format(db_url, db_name, db_collection, instance["db"])) - # TODO: upload to new DB (memcheck -> memory_tests) - # upload_data(data, db_url, db_name, db_collection) + logging.info(f"Upload data to {db_url}/{db_name}.{db_collection}. " + f"Data: {instance['db']}") + upload_data(instance["db"], db_url, db_name, db_collection) diff --git a/tests/memory_tests/test_runner/test_memorytest.py b/tests/memory_tests/test_runner/test_memorytest.py index 324b329fdb4..c5e18067b75 100644 --- a/tests/memory_tests/test_runner/test_memorytest.py +++ b/tests/memory_tests/test_runner/test_memorytest.py @@ -41,7 +41,15 @@ def test(instance, executable, niter, temp_dir, omz_models_conversion, validate_ :param omz_models_conversion: custom pytest fixture. Should be declared as test argument to be enabled """ # Prepare model to get model_path - model_path = instance["instance"]["model"].get("full_path") + model_path = '' + cache_model_path = instance["instance"]["model"].get("path") + irs_model_path = instance["instance"]["model"].get("full_path") + + if os.path.isfile(irs_model_path): + model_path = irs_model_path + elif os.path.isfile(cache_model_path): + model_path = cache_model_path + assert model_path, "Model path is empty" model_path = Path(expand_env_vars(model_path)) diff --git a/tests/utils/utils.py b/tests/utils/utils.py index a0ccb3d7410..e05b23c52d1 100644 --- a/tests/utils/utils.py +++ b/tests/utils/utils.py @@ -9,7 +9,7 @@ import yaml from pymongo import MongoClient # constants -DATABASES = ['timetests', 'memcheck'] +DATABASES = ['timetests', 'memorytests'] DB_COLLECTIONS = ["commit", "nightly", "weekly"] PRODUCT_NAME = 'dldt' # product name from build manifest