Template device testing (#3521)

* Added template plugin testing on public CI * Updated tests config * Added DEVICE_TEMPLATE constant to tests * Updated tests * Disable template plugin by default * Fixes * Fixed HETERO tests
2020-12-09 17:13:32 +03:00 · 2020-12-09 17:13:32 +03:00 · e4260cdc3f
commit e4260cdc3f
parent 17df09967d
30 changed files with 145 additions and 101 deletions
--- a/.ci/azure/linux.yml
+++ b/.ci/azure/linux.yml
@ -90,7 +90,16 @@ jobs:
  - task: CMake@1
    inputs:
      # CMake must get Python 3.x version by default
-      cmakeArgs: -GNinja -DVERBOSE_BUILD=ON -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DENABLE_PYTHON=ON -DPYTHON_EXECUTABLE=/usr/bin/python3.6 -DENABLE_TESTS=ON -DIE_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)/modules $(REPO_DIR)
+      cmakeArgs: >
+        -GNinja
+        -DVERBOSE_BUILD=ON
+        -DENABLE_TEMPLATE_PLUGIN=ON
+        -DCMAKE_BUILD_TYPE=$(BUILD_TYPE)
+        -DENABLE_PYTHON=ON
+        -DPYTHON_EXECUTABLE=/usr/bin/python3.6
+        -DENABLE_TESTS=ON
+        -DIE_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)/modules
+        $(REPO_DIR)
      workingDirectory: $(BUILD_DIR)

  - script: ninja
@ -132,6 +141,10 @@ jobs:
    displayName: 'IE FuncTests'
    continueOnError: false

+  - script: $(BIN_DIR)/templateFuncTests --gtest_filter=*smoke* --gtest_output=xml:TEST-templateFuncTests.xml
+    displayName: 'TEMPLATE FuncTests'
+    continueOnError: false
+
  - script: $(BIN_DIR)/cpuFuncTests --gtest_filter=*smoke* --gtest_print_time=1 --gtest_output=xml:TEST-cpuFuncTests.xml
    displayName: 'CPU FuncTests'
    continueOnError: false
--- a/.ci/azure/windows.yml
+++ b/.ci/azure/windows.yml
@ -90,7 +90,7 @@ jobs:

  - script: |
      set PATH=$(WORK_DIR)\ninja-win;%PATH%
-      call "$(MSVS_VARS_PATH)" && cmake -GNinja -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DENABLE_TESTS=ON -DIE_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)\modules -DCMAKE_C_COMPILER:PATH="$(MSVC_COMPILER_PATH)" -DCMAKE_CXX_COMPILER:PATH="$(MSVC_COMPILER_PATH)" $(REPO_DIR)
+      call "$(MSVS_VARS_PATH)" && cmake -GNinja -DENABLE_TEMPLATE_PLUGIN=ON -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DENABLE_TESTS=ON -DIE_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)\modules -DCMAKE_C_COMPILER:PATH="$(MSVC_COMPILER_PATH)" -DCMAKE_CXX_COMPILER:PATH="$(MSVC_COMPILER_PATH)" $(REPO_DIR)
    workingDirectory: $(BUILD_DIR)
    displayName: 'CMake'

@ -154,6 +154,12 @@ jobs:
    displayName: 'IE FuncTests'
    continueOnError: false

+  - script: |
+      set PATH=$(REPO_DIR)\inference-engine\temp\tbb\bin;%PATH%
+      $(BIN_DIR)\templateFuncTests --gtest_output=xml:TEST-templateFuncTests.xml
+    displayName: 'TEMPLATE FuncTests'
+    continueOnError: false
+
  - script: |
      set PATH=$(REPO_DIR)\inference-engine\temp\tbb\bin;%PATH%
      $(BIN_DIR)\cpuFuncTests --gtest_filter=*smoke* --gtest_print_time=1 --gtest_output=xml:TEST-cpuFuncTests.xml
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -120,7 +120,6 @@ function(build_ngraph)
 endfunction()

 file(REMOVE "${CMAKE_BINARY_DIR}/openvino_targets_developer.cmake")
-
 unset(OpenVINODeveloperPackageTargets CACHE)

 function(openvino_developer_export_targets)
@ -145,11 +144,8 @@ function(openvino_developer_export_targets)
 endfunction()

 add_subdirectory(openvino)
-
 build_ngraph()
-
 add_subdirectory(inference-engine)
-
 add_subdirectory(model-optimizer)
 add_subdirectory(docs)

--- a/cmake/features.cmake
+++ b/cmake/features.cmake
@ -58,6 +58,8 @@ ie_dependent_option(ENABLE_CPPLINT "Enable cpplint checks during the build" ON "

 ie_dependent_option(ENABLE_CPPLINT_REPORT "Build cpplint report instead of failing the build" OFF "ENABLE_CPPLINT" OFF)

+ie_option(ENABLE_TEMPLATE_PLUGIN "Register template plugin into plugins.xml" OFF)
+
 ie_option(ENABLE_CLANG_FORMAT "Enable clang-format checks during the build" ON)

 ie_option_enum(SELECTIVE_BUILD "Enable OpenVINO conditional compilation or statistics collection. \
--- a/docs/CMakeLists.txt
+++ b/docs/CMakeLists.txt
@ -224,9 +224,9 @@ function(build_docs)

    # added linkcheker

-    if(EXISTS "${LINKCHECKER}")
+    if(EXISTS "${LINKCHECKER_PY}")
        add_custom_target(docs_check
-                            COMMAND ${Python3_EXECUTABLE} "${LINKCHECKER}" -v "${DOCS_BUILD_DIR}/html/"
+                            COMMAND ${Python3_EXECUTABLE} "${LINKCHECKER_PY}" -v "${DOCS_BUILD_DIR}/html/"
                            COMMENT "Check links in generated documentation"
                            WORKING_DIRECTORY "${DOCS_BUILD_DIR}"
                            VERBATIM)
--- a/docs/template_plugin/src/CMakeLists.txt
+++ b/docs/template_plugin/src/CMakeLists.txt
@ -33,8 +33,10 @@ target_link_libraries(${TARGET_NAME} PRIVATE
 set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO})

 # ATTENTION: uncomment to register a plugin in the plugins.xml file
-# ie_register_plugins(MAIN_TARGET ${TARGET_NAME}
-#                     POSSIBLE_PLUGINS ${TARGET_NAME})
+if(ENABLE_TEMPLATE_PLUGIN)
+    ie_register_plugins(MAIN_TARGET ${TARGET_NAME}
+                        POSSIBLE_PLUGINS ${TARGET_NAME})
+endif()
 # [cmake:plugin]

 # ATTENTION: uncomment to install component
--- a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/config.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/config.cpp
@ -29,14 +29,14 @@ const std::vector<std::map<std::string, std::string>> inconfigs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, IncorrectConfigTests,
                        ::testing::Combine(
                                ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                ::testing::ValuesIn(inconfigs)),
                        IncorrectConfigTests::getTestCaseName);

 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, IncorrectConfigAPITests,
                        ::testing::Combine(
                                ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                ::testing::ValuesIn(inconfigs)),
                        IncorrectConfigAPITests::getTestCaseName);

@ -44,14 +44,14 @@ INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, IncorrectConfigAPITests,
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, CorrectConfigAPITests,
                        ::testing::Combine(
                                ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                ::testing::ValuesIn(configs)),
                        CorrectConfigAPITests::getTestCaseName);

 INSTANTIATE_TEST_CASE_P(smoke_Multi_BehaviorTests, CorrectConfigTests,
                        ::testing::Combine(
                                ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                ::testing::ValuesIn(configs)),
                        CorrectConfigAPITests::getTestCaseName);

--- a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/core_integration.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/core_integration.cpp
@ -17,52 +17,52 @@ namespace {
 //

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassBasicTestP, IEClassBasicTestP,
-        ::testing::Values(std::make_pair("templatePlugin", "TEMPLATE")));
+        smoke_IEClassBasicTestP, IEClassBasicTestP,
+        ::testing::Values(std::make_pair("templatePlugin", CommonTestUtils::DEVICE_TEMPLATE)));

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassNetworkTestP, IEClassNetworkTestP,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassNetworkTestP, IEClassNetworkTestP,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

 //
 // IE Class GetMetric
 //

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetMetricTest, IEClassGetMetricTest_SUPPORTED_CONFIG_KEYS,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassGetMetricTest, IEClassGetMetricTest_SUPPORTED_CONFIG_KEYS,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetMetricTest, IEClassGetMetricTest_SUPPORTED_METRICS,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassGetMetricTest, IEClassGetMetricTest_SUPPORTED_METRICS,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetMetricTest, IEClassGetMetricTest_AVAILABLE_DEVICES,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassGetMetricTest, IEClassGetMetricTest_AVAILABLE_DEVICES,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetMetricTest, IEClassGetMetricTest_FULL_DEVICE_NAME,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassGetMetricTest, IEClassGetMetricTest_FULL_DEVICE_NAME,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetMetricTest, IEClassGetMetricTest_OPTIMIZATION_CAPABILITIES,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassGetMetricTest, IEClassGetMetricTest_OPTIMIZATION_CAPABILITIES,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetMetricTest, IEClassGetMetricTest_RANGE_FOR_ASYNC_INFER_REQUESTS,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassGetMetricTest, IEClassGetMetricTest_RANGE_FOR_ASYNC_INFER_REQUESTS,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetMetricTest, IEClassGetMetricTest_ThrowUnsupported,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassGetMetricTest, IEClassGetMetricTest_ThrowUnsupported,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetConfigTest, IEClassGetConfigTest_ThrowUnsupported,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassGetConfigTest, IEClassGetConfigTest_ThrowUnsupported,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetAvailableDevices, IEClassGetAvailableDevices,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassGetAvailableDevices, IEClassGetAvailableDevices,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));


 //
@ -71,7 +71,7 @@ INSTANTIATE_TEST_CASE_P(

 using IEClassSetConfigTestHETERO = IEClassNetworkTest;

-TEST_F(IEClassSetConfigTestHETERO, nightly_SetConfigNoThrow) {
+TEST_F(IEClassSetConfigTestHETERO, smoke_SetConfigNoThrow) {
    {
        Core ie;
        Parameter p;
@ -112,15 +112,15 @@ TEST_F(IEClassSetConfigTestHETERO, nightly_SetConfigNoThrow) {
 //

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetConfigTest, IEClassGetConfigTest,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassGetConfigTest, IEClassGetConfigTest,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

 using IEClassGetConfigTestTEMPLATE = IEClassNetworkTest;

-TEST_F(IEClassGetConfigTestTEMPLATE, nightly_GetConfigNoThrow) {
+TEST_F(IEClassGetConfigTestTEMPLATE, smoke_GetConfigNoThrow) {
    Core ie;
    Parameter p;
-    std::string deviceName = "TEMPLATE";
+    std::string deviceName = CommonTestUtils::DEVICE_TEMPLATE;

    ASSERT_NO_THROW(p = ie.GetMetric(deviceName, METRIC_KEY(SUPPORTED_CONFIG_KEYS)));
    std::vector<std::string> configValues = p;
@ -144,47 +144,47 @@ TEST_F(IEClassGetConfigTestTEMPLATE, nightly_GetConfigNoThrow) {
 //

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS,
-        ::testing::Values("TEMPLATE", "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
+        smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_SUPPORTED_METRICS,
-        ::testing::Values("TEMPLATE", "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
+        smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_SUPPORTED_METRICS,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_NETWORK_NAME,
-        ::testing::Values("TEMPLATE", "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
+        smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_NETWORK_NAME,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_OPTIMAL_NUMBER_OF_INFER_REQUESTS,
-        ::testing::Values("TEMPLATE", "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
+        smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_OPTIMAL_NUMBER_OF_INFER_REQUESTS,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassExecutableNetworkGetMetricTest_ThrowsUnsupported, IEClassExecutableNetworkGetMetricTest,
-        ::testing::Values("TEMPLATE", "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
+        smoke_IEClassExecutableNetworkGetMetricTest_ThrowsUnsupported, IEClassExecutableNetworkGetMetricTest,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
 //
 // Executable Network GetConfig / SetConfig
 //

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassExecutableNetworkGetConfigTest, IEClassExecutableNetworkGetConfigTest,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassExecutableNetworkGetConfigTest, IEClassExecutableNetworkGetConfigTest,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassExecutableNetworkSetConfigTest, IEClassExecutableNetworkSetConfigTest,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassExecutableNetworkSetConfigTest, IEClassExecutableNetworkSetConfigTest,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

 // IE Class Query network

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassQueryNetworkTest, IEClassQueryNetworkTest,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassQueryNetworkTest, IEClassQueryNetworkTest,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

 // IE Class Load network

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassLoadNetworkTest, IEClassLoadNetworkTest,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassLoadNetworkTest, IEClassLoadNetworkTest,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

 //
 // Hetero Executable Network GetMetric
@ -193,20 +193,20 @@ INSTANTIATE_TEST_CASE_P(
 #ifdef ENABLE_MKL_DNN

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_METRICS,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_METRICS,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_NETWORK_NAME,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_NETWORK_NAME,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_TARGET_FALLBACK,
-        ::testing::Values("TEMPLATE"));
+        smoke_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_TARGET_FALLBACK,
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));

 #endif  // ENABLE_MKL_DNN
 } // namespace
--- a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/cpp_holders.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/cpp_holders.cpp
@ -22,7 +22,7 @@ const std::vector<std::vector<int >> orders = {

 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, HoldersTest,
        ::testing::Combine(
-        ::testing::Values("TEMPLATE"),
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
        ::testing::ValuesIn(orders)),
        HoldersTest::getTestCaseName);

--- a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/exec_graph_info.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/exec_graph_info.cpp
@ -22,7 +22,7 @@ const std::vector<std::map<std::string, std::string>> configs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, ExecGraphTests,
                        ::testing::Combine(
                                ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                ::testing::ValuesIn(configs)),
                        ExecGraphTests::getTestCaseName);

--- a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request.cpp
@ -22,7 +22,7 @@ const std::vector<std::map<std::string, std::string>> configs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, InferRequestTests,
                        ::testing::Combine(
                                ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                ::testing::ValuesIn(configs)),
                        InferRequestTests::getTestCaseName);

--- a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request_callback.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request_callback.cpp
@ -22,7 +22,7 @@ const std::vector<std::map<std::string, std::string>> configs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, CallbackTests,
        ::testing::Combine(
            ::testing::ValuesIn(netPrecisions),
-            ::testing::Values("TEMPLATE"),
+            ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
            ::testing::ValuesIn(configs)),
        CallbackTests::getTestCaseName);
 }  // namespace
--- a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request_config.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request_config.cpp
@ -22,7 +22,7 @@ const std::vector<std::map<std::string, std::string>> configs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, InferConfigTests,
                        ::testing::Combine(
                                ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                ::testing::ValuesIn(configs)),
                        InferConfigTests::getTestCaseName);

--- a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request_input.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request_input.cpp
@ -22,7 +22,7 @@ const std::vector<std::map<std::string, std::string>> configs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, InferRequestInputTests,
                        ::testing::Combine(
                                ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                ::testing::ValuesIn(configs)),
                        InferRequestInputTests::getTestCaseName);

--- a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request_output.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request_output.cpp
@ -22,7 +22,7 @@ const std::vector<std::map<std::string, std::string>> configs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, InferRequestOutputTests,
                        ::testing::Combine(
                                ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                ::testing::ValuesIn(configs)),
                        InferRequestOutputTests::getTestCaseName);

--- a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/layout.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/layout.cpp
@ -29,7 +29,7 @@ const std::vector<std::vector<size_t>> inputShapes = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, LayoutTest,
                        ::testing::Combine(
                                ::testing::Values(InferenceEngine::Precision::FP32),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                ::testing::ValuesIn(configs),
                                ::testing::ValuesIn(Layout),
                                ::testing::ValuesIn(inputShapes)),
--- a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/preprocessing.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/preprocessing.cpp
@ -24,7 +24,7 @@ INSTANTIATE_TEST_CASE_P(PreprocessingPrecisionConvertTestsViaSetInput, Preproces
                                ::testing::ValuesIn(inputPrecisions),
                                ::testing::Values(1, 2, 3, 4, 5),   // Number of input tensor channels
                                ::testing::Values(true),            // Use SetInput
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                ::testing::ValuesIn(configs)),
                        PreprocessingPrecisionConvertTest::getTestCaseName);

@ -33,7 +33,7 @@ INSTANTIATE_TEST_CASE_P(PreprocessingPrecisionConvertTestsViaGetBlob, Preprocess
                                ::testing::ValuesIn(inputPrecisions),
                                ::testing::Values(4, 5),       // Number of input tensor channels (blob_copy only supports 4d and 5d tensors)
                                ::testing::Values(false),      // use GetBlob
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                ::testing::ValuesIn(configs)),
                        PreprocessingPrecisionConvertTest::getTestCaseName);

--- a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/set_preprocess.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/set_preprocess.cpp
@ -22,7 +22,7 @@ const std::vector<std::map<std::string, std::string>> configs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, PreprocessTest,
                        ::testing::Combine(
                                ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                ::testing::ValuesIn(configs)),
                        PreprocessTest::getTestCaseName);

--- a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/test_plugin.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/test_plugin.cpp
@ -20,21 +20,21 @@ const std::vector<std::map<std::string, std::string>> configs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, BehaviorTests,
                        ::testing::Combine(
                                ::testing::Values(InferenceEngine::Precision::FP32),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                ::testing::ValuesIn(configs)),
                        BehaviorTests::getTestCaseName);

 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, BehaviorTestInput,
                        ::testing::Combine(
                                ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                ::testing::ValuesIn(configs)),
                        BehaviorTestInput::getTestCaseName);

 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, BehaviorTestOutput,
                        ::testing::Combine(
                                ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                ::testing::ValuesIn(configs)),
                        BehaviorTestOutput::getTestCaseName);

--- a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/version.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/version.cpp
@ -15,7 +15,7 @@ const std::vector<std::map<std::string, std::string>> configs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, VersionTest,
                        ::testing::Combine(
                                ::testing::Values(InferenceEngine::Precision::FP32),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                ::testing::ValuesIn(configs)),
                        VersionTest::getTestCaseName);

--- a/docs/template_plugin/tests/functional/shared_tests_instances/hetero/query_network.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/hetero/query_network.cpp
@ -15,7 +15,7 @@ auto ConvBias = ngraph::builder::subgraph::makeConvBias();

 INSTANTIATE_TEST_CASE_P(smoke_FullySupportedTopologies, QueryNetworkTest,
                        ::testing::Combine(
-                                ::testing::Values("TEMPLATE", "HETERO:TEMPLATE", "MULTI:TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "HETERO:TEMPLATE", "MULTI:TEMPLATE"),
                                ::testing::Values(ConvBias)),
                        QueryNetworkTest::getTestCaseName);
 }  // namespace
--- a/docs/template_plugin/tests/functional/shared_tests_instances/single_layer_tests/convolution.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/single_layer_tests/convolution.cpp
@ -65,7 +65,7 @@ INSTANTIATE_TEST_CASE_P(Convolution2D_ExplicitPadding, ConvolutionLayerTest,
                                ::testing::Values(InferenceEngine::Layout::ANY),
                                ::testing::Values(InferenceEngine::Layout::ANY),
                                ::testing::Values(std::vector<size_t >({1, 3, 30, 30})),
-                                ::testing::Values("TEMPLATE")),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE)),
                        ConvolutionLayerTest::getTestCaseName);
 // ! [test_convolution:instantiate]

@ -78,7 +78,7 @@ INSTANTIATE_TEST_CASE_P(Convolution2D_AutoPadValid, ConvolutionLayerTest,
                                ::testing::Values(InferenceEngine::Layout::ANY),
                                ::testing::Values(InferenceEngine::Layout::ANY),
                                ::testing::Values(std::vector<size_t >({1, 3, 30, 30})),
-                                ::testing::Values("TEMPLATE")),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE)),
                        ConvolutionLayerTest::getTestCaseName);

 /* ============= 3D Convolution ============= */
@ -121,7 +121,7 @@ INSTANTIATE_TEST_CASE_P(Convolution3D_ExplicitPadding, ConvolutionLayerTest,
                                ::testing::Values(InferenceEngine::Layout::ANY),
                                ::testing::Values(InferenceEngine::Layout::ANY),
                                ::testing::Values(std::vector<size_t >({1, 3, 10, 10, 10})),
-                                ::testing::Values("TEMPLATE")),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE)),
                        ConvolutionLayerTest::getTestCaseName);

 INSTANTIATE_TEST_CASE_P(Convolution3D_AutoPadValid, ConvolutionLayerTest,
@ -133,7 +133,7 @@ INSTANTIATE_TEST_CASE_P(Convolution3D_AutoPadValid, ConvolutionLayerTest,
                                ::testing::Values(InferenceEngine::Layout::ANY),
                                ::testing::Values(InferenceEngine::Layout::ANY),
                                ::testing::Values(std::vector<size_t >({1, 3, 10, 10, 10})),
-                                ::testing::Values("TEMPLATE")),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE)),
                        ConvolutionLayerTest::getTestCaseName);

 }  // namespace
--- a/docs/template_plugin/tests/functional/shared_tests_instances/single_layer_tests/reshape.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/single_layer_tests/reshape.cpp
@ -24,7 +24,7 @@ INSTANTIATE_TEST_CASE_P(ReshapeCheckDynBatch, ReshapeLayerTest,
                ::testing::Values(InferenceEngine::Layout::ANY),
                ::testing::Values(std::vector<size_t>({30, 30, 30, 30})),
                ::testing::Values(std::vector<size_t>({30, 30, 30, 30})),
-                ::testing::Values("TEMPLATE"),
+                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                ::testing::Values(std::map<std::string, std::string>({}))),
                ReshapeLayerTest::getTestCaseName);

@ -38,7 +38,7 @@ INSTANTIATE_TEST_CASE_P(ReshapeCheck, ReshapeLayerTest,
                ::testing::Values(InferenceEngine::Layout::ANY),
                ::testing::Values(std::vector<size_t>({10, 10, 10, 10})),
                ::testing::Values(std::vector<size_t>({10, 0, 100})),
-                ::testing::Values("TEMPLATE"),
+                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                ::testing::Values(std::map<std::string, std::string>({}))),
                ReshapeLayerTest::getTestCaseName);
 }  // namespace
--- a/docs/template_plugin/tests/functional/shared_tests_instances/single_layer_tests/softmax.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/single_layer_tests/softmax.cpp
@ -37,7 +37,7 @@ const auto params2D = testing::Combine(
    testing::Values(InferenceEngine::Layout::ANY),
    testing::ValuesIn(inputShapes2D),
    testing::ValuesIn(axis2D),
-    testing::Values("TEMPLATE"),
+    testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
    testing::Values(std::map<std::string, std::string>())
 );

@ -64,7 +64,7 @@ const auto params4D = testing::Combine(
    testing::Values(InferenceEngine::Layout::ANY),
    testing::ValuesIn(inputShapes4D),
    testing::ValuesIn(axis4D),
-    testing::Values("TEMPLATE"),
+    testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
    testing::Values(std::map<std::string, std::string>())
 );

--- a/docs/template_plugin/tests/functional/shared_tests_instances/single_layer_tests/split.cpp
+++ b/docs/template_plugin/tests/functional/shared_tests_instances/single_layer_tests/split.cpp
@ -22,7 +22,7 @@ INSTANTIATE_TEST_CASE_P(NumSplitsCheck, SplitLayerTest,
                                ::testing::Values(InferenceEngine::Layout::ANY),
                                ::testing::Values(std::vector<size_t>({30, 30, 30, 30})),
                                ::testing::Values(std::vector<size_t>({})),
-                                ::testing::Values("TEMPLATE")),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE)),
                        SplitLayerTest::getTestCaseName);

 }  // namespace
--- a/docs/template_plugin/tests/functional/skip_tests_config.cpp
+++ b/docs/template_plugin/tests/functional/skip_tests_config.cpp
@ -11,8 +11,20 @@ std::vector<std::string> disabledTestPatterns() {
    return {
        ".*ExclusiveAsyncRequests.*",
        ".*reusableCPUStreamsExecutor.*",
-        ".*registerPlugin.*",
-        ".*IEClassGetAvailableDevices.*",
-        R"(.*SplitLayerTest.*numSplits\=30.*)"
+        R"(.*SplitLayerTest.*numSplits\=30.*)",
+        // CVS-44775: for all cases below
+        ".*Hetero.*",
+        ".*QueryNetwork.*",
+        ".*SetAffinityWithKSO.*",
+        ".*queryNetworkResultContainAllAndOnlyInputLayers.*",
+        R"(.*IEClassExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS.*)",
+        R"(.*IEClassExecutableNetworkGetMetricTest_SUPPORTED_METRICS.*/2)",
+        R"(.*IEClassExecutableNetworkGetMetricTest_NETWORK_NAME.*/2)",
+        R"(.*IEClassExecutableNetworkGetMetricTest_OPTIMAL_NUMBER_OF_INFER_REQUESTS.*/2)",
+        ".*LoadNetworkActualHeteroDeviceNoThrow.*",
+        ".*LoadNetworkActualHeteroDevice2NoThrow.*",
+        ".*IEClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS.*",
+        // CVS-44774
+        ".*PreprocessTest.*",
    };
 }
--- a/inference-engine/tests/functional/plugin/shared/include/behavior/exec_graph_info.hpp
+++ b/inference-engine/tests/functional/plugin/shared/include/behavior/exec_graph_info.hpp
@ -59,7 +59,9 @@ TEST_P(ExecGraphTests, CheckExecGraphInfoBeforeExecution) {
    // Create CNNNetwork from ngrpah::Function
    InferenceEngine::CNNNetwork cnnNet(function);
    InferenceEngine::CNNNetwork execGraph;
-    if (targetDevice != CommonTestUtils::DEVICE_MULTI && targetDevice != CommonTestUtils::DEVICE_GNA) {
+    if (targetDevice != CommonTestUtils::DEVICE_MULTI &&
+        targetDevice != CommonTestUtils::DEVICE_TEMPLATE &&
+        targetDevice != CommonTestUtils::DEVICE_GNA) {
        // Load CNNNetwork to target plugins
        auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
        ASSERT_NO_THROW(execGraph = execNet.GetExecGraphInfo());
@ -146,7 +148,9 @@ TEST_P(ExecGraphTests, CheckExecGraphInfoAfterExecution) {
    // Create CNNNetwork from ngrpah::Function
    InferenceEngine::CNNNetwork cnnNet(function);
    InferenceEngine::CNNNetwork execGraph;
-    if (targetDevice != CommonTestUtils::DEVICE_MULTI && targetDevice != CommonTestUtils::DEVICE_GNA) {
+    if (targetDevice != CommonTestUtils::DEVICE_MULTI &&
+        targetDevice != CommonTestUtils::DEVICE_TEMPLATE &&
+        targetDevice != CommonTestUtils::DEVICE_GNA) {
        // Load CNNNetwork to target plugins
        auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
        ASSERT_NO_THROW(execGraph = execNet.GetExecGraphInfo());
@ -250,7 +254,9 @@ TEST_P(ExecGraphTests, CheckExecGraphInfoSerialization) {
    // Create CNNNetwork from ngrpah::Function
    InferenceEngine::CNNNetwork cnnNet(function);
    InferenceEngine::CNNNetwork execGraph;
-    if (targetDevice != CommonTestUtils::DEVICE_MULTI && targetDevice != CommonTestUtils::DEVICE_GNA) {
+    if (targetDevice != CommonTestUtils::DEVICE_MULTI &&
+        targetDevice != CommonTestUtils::DEVICE_TEMPLATE &&
+        targetDevice != CommonTestUtils::DEVICE_GNA) {
        // Load CNNNetwork to target plugins
        auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
        ASSERT_NO_THROW(execGraph = execNet.GetExecGraphInfo());
--- a/inference-engine/tests/functional/plugin/shared/src/hetero/query_network.cpp
+++ b/inference-engine/tests/functional/plugin/shared/src/hetero/query_network.cpp
@ -23,6 +23,8 @@ std::string QueryNetworkTest::getTestCaseName(const ::testing::TestParamInfo<Que
 }

 TEST_P(QueryNetworkTest, queryNetworkResultContainAllAndOnlyInputLayers) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED();
    auto& param = GetParam();
    auto queryNetworkResult = PluginCache::get().ie()->QueryNetwork(cnnNetwork, std::get<Plugin>(param));
    ASSERT_NE(nullptr, cnnNetwork.getFunction());
--- a/inference-engine/tests/functional/plugin/shared/src/hetero/synthetic.cpp
+++ b/inference-engine/tests/functional/plugin/shared/src/hetero/synthetic.cpp
@ -110,8 +110,10 @@ void HeteroSyntheticTest::SetUp() {
 }

 void HeteroSyntheticTest::TearDown() {
-    for (auto&& pluginName : _registredPlugins) {
-        PluginCache::get().ie()->UnregisterPlugin(pluginName);
+    if (!FuncTestUtils::SkipTestsConfig::currentTestIsDisabled()) {
+        for (auto&& pluginName : _registredPlugins) {
+            PluginCache::get().ie()->UnregisterPlugin(pluginName);
+        }
    }
 }

@ -144,7 +146,9 @@ TEST_P(HeteroSyntheticTest, someLayersToMajorPluginOthersToFallback) {
    auto affinities = SetUpAffinity();
    SCOPED_TRACE(affinities);
    Run();
-    ASSERT_NE(nullptr, cnnNetwork.getFunction());
+    if (!FuncTestUtils::SkipTestsConfig::currentTestIsDisabled()) {
+        ASSERT_NE(nullptr, cnnNetwork.getFunction());
+    }
 }

 }  //  namespace HeteroTests
--- a/inference-engine/tests/ie_test_utils/common_test_utils/test_constants.hpp
+++ b/inference-engine/tests/ie_test_utils/common_test_utils/test_constants.hpp
@ -13,6 +13,7 @@ const char DEVICE_FPGA[] = "FPGA";
 const char DEVICE_MYRIAD[] = "MYRIAD";
 const char DEVICE_KEEMBAY[] = "VPUX";
 const char DEVICE_MULTI[] = "MULTI";
+const char DEVICE_TEMPLATE[] = "TEMPLATE";
 const char DEVICE_HETERO[] = "HETERO";

 #ifdef _WIN32