Template device testing (#3521)

* Added template plugin testing on public CI

* Updated tests config

* Added DEVICE_TEMPLATE constant to tests

* Updated tests

* Disable template plugin by default

* Fixes

* Fixed HETERO tests

.ci/azure/linux.yml

@@ -90,7 +90,16 @@ jobs:
   - task: CMake@1
     inputs:
       # CMake must get Python 3.x version by default
-      cmakeArgs: -GNinja -DVERBOSE_BUILD=ON -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DENABLE_PYTHON=ON -DPYTHON_EXECUTABLE=/usr/bin/python3.6 -DENABLE_TESTS=ON -DIE_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)/modules $(REPO_DIR)
+      cmakeArgs: >
+        -GNinja
+        -DVERBOSE_BUILD=ON
+        -DENABLE_TEMPLATE_PLUGIN=ON
+        -DCMAKE_BUILD_TYPE=$(BUILD_TYPE)
+        -DENABLE_PYTHON=ON
+        -DPYTHON_EXECUTABLE=/usr/bin/python3.6
+        -DENABLE_TESTS=ON
+        -DIE_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)/modules
+        $(REPO_DIR)
       workingDirectory: $(BUILD_DIR)
 
   - script: ninja
@@ -132,6 +141,10 @@ jobs:
     displayName: 'IE FuncTests'
     continueOnError: false
 
+  - script: $(BIN_DIR)/templateFuncTests --gtest_filter=*smoke* --gtest_output=xml:TEST-templateFuncTests.xml
+    displayName: 'TEMPLATE FuncTests'
+    continueOnError: false
+
   - script: $(BIN_DIR)/cpuFuncTests --gtest_filter=*smoke* --gtest_print_time=1 --gtest_output=xml:TEST-cpuFuncTests.xml
     displayName: 'CPU FuncTests'
     continueOnError: false

.ci/azure/windows.yml

@@ -90,7 +90,7 @@ jobs:
 
   - script: |
       set PATH=$(WORK_DIR)\ninja-win;%PATH%
-      call "$(MSVS_VARS_PATH)" && cmake -GNinja -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DENABLE_TESTS=ON -DIE_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)\modules -DCMAKE_C_COMPILER:PATH="$(MSVC_COMPILER_PATH)" -DCMAKE_CXX_COMPILER:PATH="$(MSVC_COMPILER_PATH)" $(REPO_DIR)
+      call "$(MSVS_VARS_PATH)" && cmake -GNinja -DENABLE_TEMPLATE_PLUGIN=ON -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DENABLE_TESTS=ON -DIE_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)\modules -DCMAKE_C_COMPILER:PATH="$(MSVC_COMPILER_PATH)" -DCMAKE_CXX_COMPILER:PATH="$(MSVC_COMPILER_PATH)" $(REPO_DIR)
     workingDirectory: $(BUILD_DIR)
     displayName: 'CMake'
 
@@ -154,6 +154,12 @@ jobs:
     displayName: 'IE FuncTests'
     continueOnError: false
 
+  - script: |
+      set PATH=$(REPO_DIR)\inference-engine\temp\tbb\bin;%PATH%
+      $(BIN_DIR)\templateFuncTests --gtest_output=xml:TEST-templateFuncTests.xml
+    displayName: 'TEMPLATE FuncTests'
+    continueOnError: false
+
   - script: |
       set PATH=$(REPO_DIR)\inference-engine\temp\tbb\bin;%PATH%
       $(BIN_DIR)\cpuFuncTests --gtest_filter=*smoke* --gtest_print_time=1 --gtest_output=xml:TEST-cpuFuncTests.xml

CMakeLists.txt

@@ -120,7 +120,6 @@ function(build_ngraph)
 endfunction()
 
 file(REMOVE "${CMAKE_BINARY_DIR}/openvino_targets_developer.cmake")
-
 unset(OpenVINODeveloperPackageTargets CACHE)
 
 function(openvino_developer_export_targets)
@@ -145,11 +144,8 @@ function(openvino_developer_export_targets)
 endfunction()
 
 add_subdirectory(openvino)
-
 build_ngraph()
-
 add_subdirectory(inference-engine)
-
 add_subdirectory(model-optimizer)
 add_subdirectory(docs)
 

cmake/features.cmake

@@ -58,6 +58,8 @@ ie_dependent_option(ENABLE_CPPLINT "Enable cpplint checks during the build" ON "
 
 ie_dependent_option(ENABLE_CPPLINT_REPORT "Build cpplint report instead of failing the build" OFF "ENABLE_CPPLINT" OFF)
 
+ie_option(ENABLE_TEMPLATE_PLUGIN "Register template plugin into plugins.xml" OFF)
+
 ie_option(ENABLE_CLANG_FORMAT "Enable clang-format checks during the build" ON)
 
 ie_option_enum(SELECTIVE_BUILD "Enable OpenVINO conditional compilation or statistics collection. \

docs/CMakeLists.txt

@@ -224,9 +224,9 @@ function(build_docs)
 
     # added linkcheker
 
-    if(EXISTS "${LINKCHECKER}")
+    if(EXISTS "${LINKCHECKER_PY}")
         add_custom_target(docs_check
-                            COMMAND ${Python3_EXECUTABLE} "${LINKCHECKER}" -v "${DOCS_BUILD_DIR}/html/"
+                            COMMAND ${Python3_EXECUTABLE} "${LINKCHECKER_PY}" -v "${DOCS_BUILD_DIR}/html/"
                             COMMENT "Check links in generated documentation"
                             WORKING_DIRECTORY "${DOCS_BUILD_DIR}"
                             VERBATIM)

docs/template_plugin/src/CMakeLists.txt

@@ -33,8 +33,10 @@ target_link_libraries(${TARGET_NAME} PRIVATE
 set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO})
 
 # ATTENTION: uncomment to register a plugin in the plugins.xml file
-# ie_register_plugins(MAIN_TARGET ${TARGET_NAME}
+if(ENABLE_TEMPLATE_PLUGIN)
-#                     POSSIBLE_PLUGINS ${TARGET_NAME})
+    ie_register_plugins(MAIN_TARGET ${TARGET_NAME}
+                        POSSIBLE_PLUGINS ${TARGET_NAME})
+endif()
 # [cmake:plugin]
 
 # ATTENTION: uncomment to install component

docs/template_plugin/tests/functional/shared_tests_instances/behavior/config.cpp

@@ -29,14 +29,14 @@ const std::vector<std::map<std::string, std::string>> inconfigs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, IncorrectConfigTests,
                         ::testing::Combine(
                                 ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                 ::testing::ValuesIn(inconfigs)),
                         IncorrectConfigTests::getTestCaseName);
 
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, IncorrectConfigAPITests,
                         ::testing::Combine(
                                 ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                 ::testing::ValuesIn(inconfigs)),
                         IncorrectConfigAPITests::getTestCaseName);
 
@@ -44,14 +44,14 @@ INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, IncorrectConfigAPITests,
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, CorrectConfigAPITests,
                         ::testing::Combine(
                                 ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                 ::testing::ValuesIn(configs)),
                         CorrectConfigAPITests::getTestCaseName);
 
 INSTANTIATE_TEST_CASE_P(smoke_Multi_BehaviorTests, CorrectConfigTests,
                         ::testing::Combine(
                                 ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                 ::testing::ValuesIn(configs)),
                         CorrectConfigAPITests::getTestCaseName);
 

docs/template_plugin/tests/functional/shared_tests_instances/behavior/core_integration.cpp

@@ -17,52 +17,52 @@ namespace {
 //
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassBasicTestP, IEClassBasicTestP,
+        smoke_IEClassBasicTestP, IEClassBasicTestP,
-        ::testing::Values(std::make_pair("templatePlugin", "TEMPLATE")));
+        ::testing::Values(std::make_pair("templatePlugin", CommonTestUtils::DEVICE_TEMPLATE)));
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassNetworkTestP, IEClassNetworkTestP,
+        smoke_IEClassNetworkTestP, IEClassNetworkTestP,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 //
 // IE Class GetMetric
 //
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetMetricTest, IEClassGetMetricTest_SUPPORTED_CONFIG_KEYS,
+        smoke_IEClassGetMetricTest, IEClassGetMetricTest_SUPPORTED_CONFIG_KEYS,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetMetricTest, IEClassGetMetricTest_SUPPORTED_METRICS,
+        smoke_IEClassGetMetricTest, IEClassGetMetricTest_SUPPORTED_METRICS,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetMetricTest, IEClassGetMetricTest_AVAILABLE_DEVICES,
+        smoke_IEClassGetMetricTest, IEClassGetMetricTest_AVAILABLE_DEVICES,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetMetricTest, IEClassGetMetricTest_FULL_DEVICE_NAME,
+        smoke_IEClassGetMetricTest, IEClassGetMetricTest_FULL_DEVICE_NAME,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetMetricTest, IEClassGetMetricTest_OPTIMIZATION_CAPABILITIES,
+        smoke_IEClassGetMetricTest, IEClassGetMetricTest_OPTIMIZATION_CAPABILITIES,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetMetricTest, IEClassGetMetricTest_RANGE_FOR_ASYNC_INFER_REQUESTS,
+        smoke_IEClassGetMetricTest, IEClassGetMetricTest_RANGE_FOR_ASYNC_INFER_REQUESTS,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetMetricTest, IEClassGetMetricTest_ThrowUnsupported,
+        smoke_IEClassGetMetricTest, IEClassGetMetricTest_ThrowUnsupported,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetConfigTest, IEClassGetConfigTest_ThrowUnsupported,
+        smoke_IEClassGetConfigTest, IEClassGetConfigTest_ThrowUnsupported,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetAvailableDevices, IEClassGetAvailableDevices,
+        smoke_IEClassGetAvailableDevices, IEClassGetAvailableDevices,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 
 //
@@ -71,7 +71,7 @@ INSTANTIATE_TEST_CASE_P(
 
 using IEClassSetConfigTestHETERO = IEClassNetworkTest;
 
-TEST_F(IEClassSetConfigTestHETERO, nightly_SetConfigNoThrow) {
+TEST_F(IEClassSetConfigTestHETERO, smoke_SetConfigNoThrow) {
     {
         Core ie;
         Parameter p;
@@ -112,15 +112,15 @@ TEST_F(IEClassSetConfigTestHETERO, nightly_SetConfigNoThrow) {
 //
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassGetConfigTest, IEClassGetConfigTest,
+        smoke_IEClassGetConfigTest, IEClassGetConfigTest,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 using IEClassGetConfigTestTEMPLATE = IEClassNetworkTest;
 
-TEST_F(IEClassGetConfigTestTEMPLATE, nightly_GetConfigNoThrow) {
+TEST_F(IEClassGetConfigTestTEMPLATE, smoke_GetConfigNoThrow) {
     Core ie;
     Parameter p;
-    std::string deviceName = "TEMPLATE";
+    std::string deviceName = CommonTestUtils::DEVICE_TEMPLATE;
 
     ASSERT_NO_THROW(p = ie.GetMetric(deviceName, METRIC_KEY(SUPPORTED_CONFIG_KEYS)));
     std::vector<std::string> configValues = p;
@@ -144,47 +144,47 @@ TEST_F(IEClassGetConfigTestTEMPLATE, nightly_GetConfigNoThrow) {
 //
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS,
+        smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS,
-        ::testing::Values("TEMPLATE", "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_SUPPORTED_METRICS,
+        smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_SUPPORTED_METRICS,
-        ::testing::Values("TEMPLATE", "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_NETWORK_NAME,
+        smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_NETWORK_NAME,
-        ::testing::Values("TEMPLATE", "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_OPTIMAL_NUMBER_OF_INFER_REQUESTS,
+        smoke_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_OPTIMAL_NUMBER_OF_INFER_REQUESTS,
-        ::testing::Values("TEMPLATE", "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassExecutableNetworkGetMetricTest_ThrowsUnsupported, IEClassExecutableNetworkGetMetricTest,
+        smoke_IEClassExecutableNetworkGetMetricTest_ThrowsUnsupported, IEClassExecutableNetworkGetMetricTest,
-        ::testing::Values("TEMPLATE", "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "MULTI:TEMPLATE", "HETERO:TEMPLATE"));
 //
 // Executable Network GetConfig / SetConfig
 //
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassExecutableNetworkGetConfigTest, IEClassExecutableNetworkGetConfigTest,
+        smoke_IEClassExecutableNetworkGetConfigTest, IEClassExecutableNetworkGetConfigTest,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassExecutableNetworkSetConfigTest, IEClassExecutableNetworkSetConfigTest,
+        smoke_IEClassExecutableNetworkSetConfigTest, IEClassExecutableNetworkSetConfigTest,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 // IE Class Query network
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassQueryNetworkTest, IEClassQueryNetworkTest,
+        smoke_IEClassQueryNetworkTest, IEClassQueryNetworkTest,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 // IE Class Load network
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassLoadNetworkTest, IEClassLoadNetworkTest,
+        smoke_IEClassLoadNetworkTest, IEClassLoadNetworkTest,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 //
 // Hetero Executable Network GetMetric
@@ -193,20 +193,20 @@ INSTANTIATE_TEST_CASE_P(
 #ifdef ENABLE_MKL_DNN
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS,
+        smoke_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_METRICS,
+        smoke_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_METRICS,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_NETWORK_NAME,
+        smoke_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_NETWORK_NAME,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 INSTANTIATE_TEST_CASE_P(
-        nightly_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_TARGET_FALLBACK,
+        smoke_IEClassHeteroExecutableNetworlGetMetricTest, IEClassHeteroExecutableNetworkGetMetricTest_TARGET_FALLBACK,
-        ::testing::Values("TEMPLATE"));
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE));
 
 #endif  // ENABLE_MKL_DNN
 } // namespace

docs/template_plugin/tests/functional/shared_tests_instances/behavior/cpp_holders.cpp

@@ -22,7 +22,7 @@ const std::vector<std::vector<int >> orders = {
 
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, HoldersTest,
         ::testing::Combine(
-        ::testing::Values("TEMPLATE"),
+        ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
         ::testing::ValuesIn(orders)),
         HoldersTest::getTestCaseName);
 

docs/template_plugin/tests/functional/shared_tests_instances/behavior/exec_graph_info.cpp

@@ -22,7 +22,7 @@ const std::vector<std::map<std::string, std::string>> configs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, ExecGraphTests,
                         ::testing::Combine(
                                 ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                 ::testing::ValuesIn(configs)),
                         ExecGraphTests::getTestCaseName);
 

docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request.cpp

@@ -22,7 +22,7 @@ const std::vector<std::map<std::string, std::string>> configs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, InferRequestTests,
                         ::testing::Combine(
                                 ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                 ::testing::ValuesIn(configs)),
                         InferRequestTests::getTestCaseName);
 

docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request_callback.cpp

@@ -22,7 +22,7 @@ const std::vector<std::map<std::string, std::string>> configs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, CallbackTests,
         ::testing::Combine(
             ::testing::ValuesIn(netPrecisions),
-            ::testing::Values("TEMPLATE"),
+            ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
             ::testing::ValuesIn(configs)),
         CallbackTests::getTestCaseName);
 }  // namespace

docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request_config.cpp

@@ -22,7 +22,7 @@ const std::vector<std::map<std::string, std::string>> configs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, InferConfigTests,
                         ::testing::Combine(
                                 ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                 ::testing::ValuesIn(configs)),
                         InferConfigTests::getTestCaseName);
 

docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request_input.cpp

@@ -22,7 +22,7 @@ const std::vector<std::map<std::string, std::string>> configs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, InferRequestInputTests,
                         ::testing::Combine(
                                 ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                 ::testing::ValuesIn(configs)),
                         InferRequestInputTests::getTestCaseName);
 

docs/template_plugin/tests/functional/shared_tests_instances/behavior/infer_request_output.cpp

@@ -22,7 +22,7 @@ const std::vector<std::map<std::string, std::string>> configs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, InferRequestOutputTests,
                         ::testing::Combine(
                                 ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                 ::testing::ValuesIn(configs)),
                         InferRequestOutputTests::getTestCaseName);
 

docs/template_plugin/tests/functional/shared_tests_instances/behavior/layout.cpp

@@ -29,7 +29,7 @@ const std::vector<std::vector<size_t>> inputShapes = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, LayoutTest,
                         ::testing::Combine(
                                 ::testing::Values(InferenceEngine::Precision::FP32),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                 ::testing::ValuesIn(configs),
                                 ::testing::ValuesIn(Layout),
                                 ::testing::ValuesIn(inputShapes)),

docs/template_plugin/tests/functional/shared_tests_instances/behavior/preprocessing.cpp

@@ -24,7 +24,7 @@ INSTANTIATE_TEST_CASE_P(PreprocessingPrecisionConvertTestsViaSetInput, Preproces
                                 ::testing::ValuesIn(inputPrecisions),
                                 ::testing::Values(1, 2, 3, 4, 5),   // Number of input tensor channels
                                 ::testing::Values(true),            // Use SetInput
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                 ::testing::ValuesIn(configs)),
                         PreprocessingPrecisionConvertTest::getTestCaseName);
 
@@ -33,7 +33,7 @@ INSTANTIATE_TEST_CASE_P(PreprocessingPrecisionConvertTestsViaGetBlob, Preprocess
                                 ::testing::ValuesIn(inputPrecisions),
                                 ::testing::Values(4, 5),       // Number of input tensor channels (blob_copy only supports 4d and 5d tensors)
                                 ::testing::Values(false),      // use GetBlob
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                 ::testing::ValuesIn(configs)),
                         PreprocessingPrecisionConvertTest::getTestCaseName);
 

docs/template_plugin/tests/functional/shared_tests_instances/behavior/set_preprocess.cpp

@@ -22,7 +22,7 @@ const std::vector<std::map<std::string, std::string>> configs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, PreprocessTest,
                         ::testing::Combine(
                                 ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                 ::testing::ValuesIn(configs)),
                         PreprocessTest::getTestCaseName);
 

docs/template_plugin/tests/functional/shared_tests_instances/behavior/test_plugin.cpp

@@ -20,21 +20,21 @@ const std::vector<std::map<std::string, std::string>> configs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, BehaviorTests,
                         ::testing::Combine(
                                 ::testing::Values(InferenceEngine::Precision::FP32),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                 ::testing::ValuesIn(configs)),
                         BehaviorTests::getTestCaseName);
 
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, BehaviorTestInput,
                         ::testing::Combine(
                                 ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                 ::testing::ValuesIn(configs)),
                         BehaviorTestInput::getTestCaseName);
 
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, BehaviorTestOutput,
                         ::testing::Combine(
                                 ::testing::ValuesIn(netPrecisions),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                 ::testing::ValuesIn(configs)),
                         BehaviorTestOutput::getTestCaseName);
 

docs/template_plugin/tests/functional/shared_tests_instances/behavior/version.cpp

@@ -15,7 +15,7 @@ const std::vector<std::map<std::string, std::string>> configs = {
 INSTANTIATE_TEST_CASE_P(smoke_BehaviorTests, VersionTest,
                         ::testing::Combine(
                                 ::testing::Values(InferenceEngine::Precision::FP32),
-                                ::testing::Values("TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                                 ::testing::ValuesIn(configs)),
                         VersionTest::getTestCaseName);
 

docs/template_plugin/tests/functional/shared_tests_instances/hetero/query_network.cpp

@@ -15,7 +15,7 @@ auto ConvBias = ngraph::builder::subgraph::makeConvBias();
 
 INSTANTIATE_TEST_CASE_P(smoke_FullySupportedTopologies, QueryNetworkTest,
                         ::testing::Combine(
-                                ::testing::Values("TEMPLATE", "HETERO:TEMPLATE", "MULTI:TEMPLATE"),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE, "HETERO:TEMPLATE", "MULTI:TEMPLATE"),
                                 ::testing::Values(ConvBias)),
                         QueryNetworkTest::getTestCaseName);
 }  // namespace

docs/template_plugin/tests/functional/shared_tests_instances/single_layer_tests/convolution.cpp

@@ -65,7 +65,7 @@ INSTANTIATE_TEST_CASE_P(Convolution2D_ExplicitPadding, ConvolutionLayerTest,
                                 ::testing::Values(InferenceEngine::Layout::ANY),
                                 ::testing::Values(InferenceEngine::Layout::ANY),
                                 ::testing::Values(std::vector<size_t >({1, 3, 30, 30})),
-                                ::testing::Values("TEMPLATE")),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE)),
                         ConvolutionLayerTest::getTestCaseName);
 // ! [test_convolution:instantiate]
 
@@ -78,7 +78,7 @@ INSTANTIATE_TEST_CASE_P(Convolution2D_AutoPadValid, ConvolutionLayerTest,
                                 ::testing::Values(InferenceEngine::Layout::ANY),
                                 ::testing::Values(InferenceEngine::Layout::ANY),
                                 ::testing::Values(std::vector<size_t >({1, 3, 30, 30})),
-                                ::testing::Values("TEMPLATE")),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE)),
                         ConvolutionLayerTest::getTestCaseName);
 
 /* ============= 3D Convolution ============= */
@@ -121,7 +121,7 @@ INSTANTIATE_TEST_CASE_P(Convolution3D_ExplicitPadding, ConvolutionLayerTest,
                                 ::testing::Values(InferenceEngine::Layout::ANY),
                                 ::testing::Values(InferenceEngine::Layout::ANY),
                                 ::testing::Values(std::vector<size_t >({1, 3, 10, 10, 10})),
-                                ::testing::Values("TEMPLATE")),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE)),
                         ConvolutionLayerTest::getTestCaseName);
 
 INSTANTIATE_TEST_CASE_P(Convolution3D_AutoPadValid, ConvolutionLayerTest,
@@ -133,7 +133,7 @@ INSTANTIATE_TEST_CASE_P(Convolution3D_AutoPadValid, ConvolutionLayerTest,
                                 ::testing::Values(InferenceEngine::Layout::ANY),
                                 ::testing::Values(InferenceEngine::Layout::ANY),
                                 ::testing::Values(std::vector<size_t >({1, 3, 10, 10, 10})),
-                                ::testing::Values("TEMPLATE")),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE)),
                         ConvolutionLayerTest::getTestCaseName);
 
 }  // namespace

docs/template_plugin/tests/functional/shared_tests_instances/single_layer_tests/reshape.cpp

@@ -24,7 +24,7 @@ INSTANTIATE_TEST_CASE_P(ReshapeCheckDynBatch, ReshapeLayerTest,
                 ::testing::Values(InferenceEngine::Layout::ANY),
                 ::testing::Values(std::vector<size_t>({30, 30, 30, 30})),
                 ::testing::Values(std::vector<size_t>({30, 30, 30, 30})),
-                ::testing::Values("TEMPLATE"),
+                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                 ::testing::Values(std::map<std::string, std::string>({}))),
                 ReshapeLayerTest::getTestCaseName);
 
@@ -38,7 +38,7 @@ INSTANTIATE_TEST_CASE_P(ReshapeCheck, ReshapeLayerTest,
                 ::testing::Values(InferenceEngine::Layout::ANY),
                 ::testing::Values(std::vector<size_t>({10, 10, 10, 10})),
                 ::testing::Values(std::vector<size_t>({10, 0, 100})),
-                ::testing::Values("TEMPLATE"),
+                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
                 ::testing::Values(std::map<std::string, std::string>({}))),
                 ReshapeLayerTest::getTestCaseName);
 }  // namespace

docs/template_plugin/tests/functional/shared_tests_instances/single_layer_tests/softmax.cpp

@@ -37,7 +37,7 @@ const auto params2D = testing::Combine(
     testing::Values(InferenceEngine::Layout::ANY),
     testing::ValuesIn(inputShapes2D),
     testing::ValuesIn(axis2D),
-    testing::Values("TEMPLATE"),
+    testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
     testing::Values(std::map<std::string, std::string>())
 );
 
@@ -64,7 +64,7 @@ const auto params4D = testing::Combine(
     testing::Values(InferenceEngine::Layout::ANY),
     testing::ValuesIn(inputShapes4D),
     testing::ValuesIn(axis4D),
-    testing::Values("TEMPLATE"),
+    testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
     testing::Values(std::map<std::string, std::string>())
 );
 

docs/template_plugin/tests/functional/shared_tests_instances/single_layer_tests/split.cpp

@@ -22,7 +22,7 @@ INSTANTIATE_TEST_CASE_P(NumSplitsCheck, SplitLayerTest,
                                 ::testing::Values(InferenceEngine::Layout::ANY),
                                 ::testing::Values(std::vector<size_t>({30, 30, 30, 30})),
                                 ::testing::Values(std::vector<size_t>({})),
-                                ::testing::Values("TEMPLATE")),
+                                ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE)),
                         SplitLayerTest::getTestCaseName);
 
 }  // namespace

docs/template_plugin/tests/functional/skip_tests_config.cpp

@@ -11,8 +11,20 @@ std::vector<std::string> disabledTestPatterns() {
     return {
         ".*ExclusiveAsyncRequests.*",
         ".*reusableCPUStreamsExecutor.*",
-        ".*registerPlugin.*",
+        R"(.*SplitLayerTest.*numSplits\=30.*)",
-        ".*IEClassGetAvailableDevices.*",
+        // CVS-44775: for all cases below
-        R"(.*SplitLayerTest.*numSplits\=30.*)"
+        ".*Hetero.*",
+        ".*QueryNetwork.*",
+        ".*SetAffinityWithKSO.*",
+        ".*queryNetworkResultContainAllAndOnlyInputLayers.*",
+        R"(.*IEClassExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS.*)",
+        R"(.*IEClassExecutableNetworkGetMetricTest_SUPPORTED_METRICS.*/2)",
+        R"(.*IEClassExecutableNetworkGetMetricTest_NETWORK_NAME.*/2)",
+        R"(.*IEClassExecutableNetworkGetMetricTest_OPTIMAL_NUMBER_OF_INFER_REQUESTS.*/2)",
+        ".*LoadNetworkActualHeteroDeviceNoThrow.*",
+        ".*LoadNetworkActualHeteroDevice2NoThrow.*",
+        ".*IEClassHeteroExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS.*",
+        // CVS-44774
+        ".*PreprocessTest.*",
     };
 }

inference-engine/tests/functional/plugin/shared/include/behavior/exec_graph_info.hpp

@@ -59,7 +59,9 @@ TEST_P(ExecGraphTests, CheckExecGraphInfoBeforeExecution) {
     // Create CNNNetwork from ngrpah::Function
     InferenceEngine::CNNNetwork cnnNet(function);
     InferenceEngine::CNNNetwork execGraph;
-    if (targetDevice != CommonTestUtils::DEVICE_MULTI && targetDevice != CommonTestUtils::DEVICE_GNA) {
+    if (targetDevice != CommonTestUtils::DEVICE_MULTI &&
+        targetDevice != CommonTestUtils::DEVICE_TEMPLATE &&
+        targetDevice != CommonTestUtils::DEVICE_GNA) {
         // Load CNNNetwork to target plugins
         auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
         ASSERT_NO_THROW(execGraph = execNet.GetExecGraphInfo());
@@ -146,7 +148,9 @@ TEST_P(ExecGraphTests, CheckExecGraphInfoAfterExecution) {
     // Create CNNNetwork from ngrpah::Function
     InferenceEngine::CNNNetwork cnnNet(function);
     InferenceEngine::CNNNetwork execGraph;
-    if (targetDevice != CommonTestUtils::DEVICE_MULTI && targetDevice != CommonTestUtils::DEVICE_GNA) {
+    if (targetDevice != CommonTestUtils::DEVICE_MULTI &&
+        targetDevice != CommonTestUtils::DEVICE_TEMPLATE &&
+        targetDevice != CommonTestUtils::DEVICE_GNA) {
         // Load CNNNetwork to target plugins
         auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
         ASSERT_NO_THROW(execGraph = execNet.GetExecGraphInfo());
@@ -250,7 +254,9 @@ TEST_P(ExecGraphTests, CheckExecGraphInfoSerialization) {
     // Create CNNNetwork from ngrpah::Function
     InferenceEngine::CNNNetwork cnnNet(function);
     InferenceEngine::CNNNetwork execGraph;
-    if (targetDevice != CommonTestUtils::DEVICE_MULTI && targetDevice != CommonTestUtils::DEVICE_GNA) {
+    if (targetDevice != CommonTestUtils::DEVICE_MULTI &&
+        targetDevice != CommonTestUtils::DEVICE_TEMPLATE &&
+        targetDevice != CommonTestUtils::DEVICE_GNA) {
         // Load CNNNetwork to target plugins
         auto execNet = ie->LoadNetwork(cnnNet, targetDevice, configuration);
         ASSERT_NO_THROW(execGraph = execNet.GetExecGraphInfo());

inference-engine/tests/functional/plugin/shared/src/hetero/query_network.cpp

@@ -23,6 +23,8 @@ std::string QueryNetworkTest::getTestCaseName(const ::testing::TestParamInfo<Que
 }
 
 TEST_P(QueryNetworkTest, queryNetworkResultContainAllAndOnlyInputLayers) {
+    // Skip test according to plugin specific disabledTestPatterns() (if any)
+    SKIP_IF_CURRENT_TEST_IS_DISABLED();
     auto& param = GetParam();
     auto queryNetworkResult = PluginCache::get().ie()->QueryNetwork(cnnNetwork, std::get<Plugin>(param));
     ASSERT_NE(nullptr, cnnNetwork.getFunction());

inference-engine/tests/functional/plugin/shared/src/hetero/synthetic.cpp

@@ -110,8 +110,10 @@ void HeteroSyntheticTest::SetUp() {
 }
 
 void HeteroSyntheticTest::TearDown() {
-    for (auto&& pluginName : _registredPlugins) {
+    if (!FuncTestUtils::SkipTestsConfig::currentTestIsDisabled()) {
-        PluginCache::get().ie()->UnregisterPlugin(pluginName);
+        for (auto&& pluginName : _registredPlugins) {
+            PluginCache::get().ie()->UnregisterPlugin(pluginName);
+        }
     }
 }
 
@@ -144,7 +146,9 @@ TEST_P(HeteroSyntheticTest, someLayersToMajorPluginOthersToFallback) {
     auto affinities = SetUpAffinity();
     SCOPED_TRACE(affinities);
     Run();
-    ASSERT_NE(nullptr, cnnNetwork.getFunction());
+    if (!FuncTestUtils::SkipTestsConfig::currentTestIsDisabled()) {
+        ASSERT_NE(nullptr, cnnNetwork.getFunction());
+    }
 }
 
 }  //  namespace HeteroTests

inference-engine/tests/ie_test_utils/common_test_utils/test_constants.hpp

@@ -13,6 +13,7 @@ const char DEVICE_FPGA[] = "FPGA";
 const char DEVICE_MYRIAD[] = "MYRIAD";
 const char DEVICE_KEEMBAY[] = "VPUX";
 const char DEVICE_MULTI[] = "MULTI";
+const char DEVICE_TEMPLATE[] = "TEMPLATE";
 const char DEVICE_HETERO[] = "HETERO";
 
 #ifdef _WIN32