[Memory_tests] Work with DB (#7492)

* uncomment upload db func

* upd CMakeLists.txt

* fix template path

* fix CmakeLists

* fix imports

* fix work with cmd and with models our_dir

* add quotation marks

* fix quotation marks

* add base config for test

* upd work with db instance

* revert changes with test config

tests/memory_tests/CMakeLists.txt

@@ -4,18 +4,27 @@
 
 cmake_minimum_required(VERSION 3.13)
 
-if (CMAKE_BUILD_TYPE STREQUAL "")
+set (CMAKE_CXX_STANDARD 11)
-    message(STATUS "CMAKE_BUILD_TYPE not defined, 'Release' will be used")
+set (CMAKE_CXX_EXTENSIONS OFF)
-    set(CMAKE_BUILD_TYPE "Release")
+set (CMAKE_CXX_STANDARD_REQUIRED ON)
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    set (CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS}")
 endif()
 
+set (CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the build type")
+
+project(memory_tests)
+
 # Define directory where artifacts will be placed
 set(OUTPUT_ROOT ${CMAKE_CURRENT_SOURCE_DIR})
 
 set(OpenVINO_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../")
 
-project(memory_tests)
+# Search OpenVINO Inference Engine installed
-
+find_package(OpenVINO REQUIRED)
-find_package(InferenceEngine REQUIRED)
 
 add_subdirectory(src)
+
+install(DIRECTORY test_runner/ DESTINATION tests/memory_tests/test_runner COMPONENT tests EXCLUDE_FROM_ALL)
+install(DIRECTORY .automation/ DESTINATION tests/memory_tests/test_runner/.automation COMPONENT tests EXCLUDE_FROM_ALL)
+install(DIRECTORY scripts/ DESTINATION tests/memory_tests/scripts COMPONENT tests EXCLUDE_FROM_ALL)

tests/memory_tests/src/memory_tests/CMakeLists.txt

@@ -15,7 +15,7 @@ foreach(test_source ${tests})
     get_filename_component(test_name ${test_source} NAME_WE)
     add_executable(${test_name} ${test_source})
 
-    target_link_libraries(${test_name} PRIVATE IE::inference_engine memory_tests_helper tests_shared_lib)
+    target_link_libraries(${test_name} PRIVATE memory_tests_helper tests_shared_lib)
 
     add_dependencies(memory_tests ${test_name})
 

tests/memory_tests/test_runner/conftest.py

@@ -18,6 +18,7 @@ This plugin adds the following command-line options:
 import hashlib
 import json
 import logging
+import tempfile
 # pylint:disable=import-error
 import os
 import sys
@@ -33,11 +34,10 @@ from jsonschema import validate, ValidationError
 UTILS_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "utils")
 sys.path.insert(0, str(UTILS_DIR))
 
-from plugins.conftest import *
 from path_utils import check_positive_int
 from proc_utils import cmd_exec
 from platform_utils import get_os_name, get_os_version, get_cpu_info
-from utils import metadata_from_manifest, DATABASES, DB_COLLECTIONS
+from utils import metadata_from_manifest, DATABASES, DB_COLLECTIONS, upload_data
 
 MEMORY_TESTS_DIR = os.path.dirname(os.path.dirname(__file__))
 sys.path.append(MEMORY_TESTS_DIR)
@@ -56,8 +56,6 @@ def abs_path(relative_path):
 
 
 # -------------------- CLI options --------------------
-
-
 def pytest_addoption(parser):
     """Specify command-line options for all plugins"""
     test_args_parser = parser.getgroup("test run")
@@ -177,11 +175,19 @@ def executable(request):
 def niter(request):
     """Fixture function for command-line option."""
     return request.config.getoption('niter')
-
-
 # -------------------- CLI options --------------------
 
 
+@pytest.fixture(scope="function")
+def temp_dir(pytestconfig):
+    """Create temporary directory for test purposes.
+    It will be cleaned up after every test run.
+    """
+    temp_dir = tempfile.TemporaryDirectory()
+    yield Path(temp_dir.name)
+    temp_dir.cleanup()
+
+
 @pytest.fixture(scope="function")
 def omz_models_conversion(instance, request):
     """
@@ -204,8 +210,9 @@ def omz_models_conversion(instance, request):
             model_precision = instance["instance"]["model"]["precision"]
 
             # get full model info
-            cmd = f'"{sys.executable}" "{info_dumper_path}" --name {model_name}'
+            cmd = [f'{sys.executable}', f'{info_dumper_path}', '--name', f'{model_name}']
-            _, info = cmd_exec([cmd], shell=True, log=logging)
+            return_code, info = cmd_exec(cmd, log=logging)
+            assert return_code == 0, "Getting information about OMZ models has failed!"
 
             model_info = json.loads(info)[0]
 
@@ -213,29 +220,27 @@ def omz_models_conversion(instance, request):
                 logging.error(f"Please specify precision for the model "
                               f"{model_name} from the list: {model_info['precisions']}")
 
-            model_path = Path(model_info["subdirectory"]) / model_precision / (model_name + ".xml")
+            model_out_path = Path(omz_models_out_dir / model_info["subdirectory"]) / model_precision / (
+                    model_name + ".xml")
             model_full_path = omz_irs_out_dir / model_info["subdirectory"] / model_precision / (model_name + ".xml")
 
             # prepare models and convert models to IRs
-            cmd = f'{sys.executable} {downloader_path}' \
+            cmd = [f'{sys.executable}', f'{downloader_path}', '--name', f'{model_name}',
-                  f' --name {model_name}' \
+                   '--precisions', f'{model_precision}', '--num_attempts', f'{OMZ_NUM_ATTEMPTS}',
-                  f' --precisions={model_precision}' \
+                   '--output_dir', f'{omz_models_out_dir}', '--cache_dir', f'{cache_dir}']
-                  f' --num_attempts {OMZ_NUM_ATTEMPTS}' \
-                  f' --output_dir {omz_models_out_dir}' \
-                  f' --cache_dir {cache_dir}'
-            cmd_exec([cmd], shell=True, log=logging)
 
-            cmd = f'{sys.executable} {converter_path}' \
+            return_code, _ = cmd_exec(cmd, log=logging)
-                  f' --name {model_name}' \
+            assert return_code == 0, "Downloading OMZ models has failed!"
-                  f' -p {sys.executable}' \
+
-                  f' --precisions={model_precision}' \
+            cmd = [f'{sys.executable}', f'{converter_path}', '--name', f'{model_name}', '-p', f'{sys.executable}',
-                  f' --output_dir {omz_irs_out_dir}' \
+                   '--precisions', f'{model_precision}', '--output_dir', f'{omz_irs_out_dir}',
-                  f' --download_dir {omz_models_out_dir}' \
+                   '--download_dir', f'{omz_models_out_dir}', '--mo', f'{mo_path}']
-                  f' --mo {mo_path}'
+
-            cmd_exec([cmd], shell=True, log=logging)
+            return_code, _ = cmd_exec(cmd, log=logging)
+            assert return_code == 0, "Converting OMZ models has failed!"
 
             instance["instance"]["model"]["framework"] = model_info["framework"]
-            instance["instance"]["model"]["path"] = model_path
+            instance["instance"]["model"]["path"] = model_out_path
             instance["instance"]["model"]["full_path"] = model_full_path
 
 
@@ -294,6 +299,8 @@ def prepare_db_info(request, instance, executable, niter, manifest_metadata):
         yield
         return
 
+    instance["db"] = {}
+
     # add db_metadata
     db_meta_path = request.config.getoption("db_metadata")
     if db_meta_path:
@@ -317,10 +324,10 @@ def prepare_db_info(request, instance, executable, niter, manifest_metadata):
         "references": instance["instance"].get("references", {}),  # upload actual references that were used
         "ref_factor": REFS_FACTOR,
     }
-    info['_id'] = hashlib.sha256(
+    info['_id'] = hashlib.sha256(''.join([str(info[key]) for key in FIELDS_FOR_ID]).encode()).hexdigest()
-        ''.join([str(info[key]) for key in FIELDS_FOR_ID]).encode()).hexdigest()
-    instance["db"] = info
 
+    # add metadata
+    instance["db"].update(info)
     # add manifest metadata
     instance["db"].update(manifest_metadata)
 
@@ -433,7 +440,7 @@ def prepare_timeline_report(pytestconfig):
 
         env = jinja2.Environment(
             loader=jinja2.FileSystemLoader(
-                searchpath=Path().absolute() / 'memory-template'),
+                searchpath=Path().absolute() / 'memory_template'),
             autoescape=False)
         template = env.get_template('timeline_report.html')
         template.stream(records=records, timelines=timelines).dump(report_path)
@@ -539,7 +546,7 @@ def pytest_runtest_makereport(item, call):
                 instance["db"]["status"] = "passed"
         instance["db"]["results"] = instance["results"]
         instance["db"]["raw_results"] = instance["raw_results"]
-        logging.info("Upload data to {}/{}.{}. Data: {}".format(db_url, db_name, db_collection, instance["db"]))
 
-        # TODO: upload to new DB (memcheck -> memory_tests)
+        logging.info(f"Upload data to {db_url}/{db_name}.{db_collection}. "
-        # upload_data(data, db_url, db_name, db_collection)
+                     f"Data: {instance['db']}")
+        upload_data(instance["db"], db_url, db_name, db_collection)

tests/memory_tests/test_runner/test_memorytest.py

@@ -41,7 +41,15 @@ def test(instance, executable, niter, temp_dir, omz_models_conversion, validate_
     :param omz_models_conversion: custom pytest fixture. Should be declared as test argument to be enabled
     """
     # Prepare model to get model_path
-    model_path = instance["instance"]["model"].get("full_path")
+    model_path = ''
+    cache_model_path = instance["instance"]["model"].get("path")
+    irs_model_path = instance["instance"]["model"].get("full_path")
+
+    if os.path.isfile(irs_model_path):
+        model_path = irs_model_path
+    elif os.path.isfile(cache_model_path):
+        model_path = cache_model_path
+
     assert model_path, "Model path is empty"
     model_path = Path(expand_env_vars(model_path))
 

tests/utils/utils.py

@@ -9,7 +9,7 @@ import yaml
 from pymongo import MongoClient
 
 # constants
-DATABASES = ['timetests', 'memcheck']
+DATABASES = ['timetests', 'memorytests']
 DB_COLLECTIONS = ["commit", "nightly", "weekly"]
 PRODUCT_NAME = 'dldt'   # product name from build manifest