From bd04dc1ecf43d1a295c3a5bc3f2df674a18d5c6f Mon Sep 17 00:00:00 2001 From: "Wang, Yang" Date: Sat, 25 Jun 2022 10:17:30 +0800 Subject: [PATCH] fix compiled model failed issue when set config with ov device properties (#11793) * 1. Enable IE Core filter to promote the secondary properties to first level for hardware device. 2. Enable IE Core filter to pass the secondary properties to AUTO plguin. 3. Enable AUTO Plugin to parse secondary properties to first level and pass them to corresponding target hardware device. Signed-off-by: Wang, Yang * 1. Enable MULTI Plugin to support secondary properties. Signed-off-by: Wang, Yang * 1. Enable HETERO Plugin to support secondary priorities. Signed-off-by: Wang, Yang * Update. Signed-off-by: Wang, Yang * Catch the EXPECT_CALL with AVAILABLE_DEVICES argument inputting to GetMetric. Signed-off-by: Wang, Yang * Revert the logic of handling secondary properties for MULTI and HETERO device. Signed-off-by: Wang, Yang * Update. Signed-off-by: Wang, Yang * Remove the secondary property flattening logic because this logic has been implemented within AUTO plugin. Signed-off-by: Wang, Yang * 1. update flatten logic when secondary properties is specified. 2. add the test case with secondary properties for CPU. Signed-off-by: Wang, Yang * add the test case with secondary properties for GPU plugin. Signed-off-by: Wang, Yang * Update. Signed-off-by: Wang, Yang * Update. Signed-off-by: Wang, Yang * Add debug message to fix the test case failure issue. Signed-off-by: Wang, Yang * Add more debug info. Signed-off-by: Wang, Yang * Update. 1. For IE Core, 1st level property overides the 2nd level property. 2. For AUTO plugin, add available device list to check if the secondary properties is vaild. Signed-off-by: Wang, Yang * Add CUDA and ARM. Signed-off-by: Wang, Yang * Update device name for ARM Plugin and add device name for HPU plugin. Signed-off-by: Wang, Yang Co-authored-by: Chen Peter --- src/inference/src/ie_core.cpp | 25 +++++-- src/plugins/auto/plugin.cpp | 21 ++++-- src/plugins/auto/plugin.hpp | 4 +- src/plugins/intel_gpu/src/plugin/plugin.cpp | 2 +- .../behavior/ov_plugin/core_integration.cpp | 68 ++++++++++++++++++- .../behavior/ov_plugin/core_integration.cpp | 44 ++++++++++++ .../behavior/ov_plugin/core_integration.hpp | 25 +++++++ 7 files changed, 172 insertions(+), 17 deletions(-) diff --git a/src/inference/src/ie_core.cpp b/src/inference/src/ie_core.cpp index 012749a2a10..c8b01b25dea 100644 --- a/src/inference/src/ie_core.cpp +++ b/src/inference/src/ie_core.cpp @@ -127,14 +127,27 @@ void allowNotImplemented(F&& f) { ov::AnyMap flatten_sub_properties(const std::string& device, const ov::AnyMap& properties) { ov::AnyMap result = properties; - for (auto&& property : properties) { - auto parsed = parseDeviceNameIntoConfig(property.first); + for (auto item = result.begin(); item != result.end();) { + auto parsed = parseDeviceNameIntoConfig(item->first); + if (!item->second.is()) { + item++; + continue; + } if (device.find(parsed._deviceName) != std::string::npos) { - if (property.second.is()) { - for (auto&& sub_property : property.second.as()) { - result[sub_property.first] = sub_property.second; - } + // 1. flatten the scondary property for target device + for (auto&& sub_property : item->second.as()) { + // 1.1 1st level property overides 2nd level property + if (result.find(sub_property.first) != result.end()) + continue; + result[sub_property.first] = sub_property.second; } + item = result.erase(item); + } else if (device != "AUTO" && device != "MULTI" && device != "HETERO") { + // 2. remove the secondary property setting for other hard ware device + item = result.erase(item); + } else { + // 3. keep the secondary property for the other virtual devices + item++; } } return result; diff --git a/src/plugins/auto/plugin.cpp b/src/plugins/auto/plugin.cpp index 98c68a26a8f..16cd15be542 100644 --- a/src/plugins/auto/plugin.cpp +++ b/src/plugins/auto/plugin.cpp @@ -77,6 +77,8 @@ namespace { std::mutex MultiDeviceInferencePlugin::_mtx; std::map> MultiDeviceInferencePlugin::_priorityMap; +std::set MultiDeviceInferencePlugin::_availableDevices = + std::set{"CPU", "GPU", "GNA", "TEMPLATE", "MYRAID", "HDDL", "VPUX", "MULTI", "HETERO", "CUDA", "HPU_GOYA"}; std::vector MultiDeviceInferencePlugin::ParseMetaDevices(const std::string& priorities, const std::map & config) const { @@ -338,7 +340,6 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons auto workMode = fullConfig.find(CONFIG_KEY_INTERNAL(MULTI_WORK_MODE_AS_AUTO)); bool workModeAuto = workMode != fullConfig.end() && workMode->second == InferenceEngine::PluginConfigParams::YES; auto priorities = fullConfig.find(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES); - // if workMode is AUTO if (workModeAuto) { // check the configure and check if need to set PerfCounters configure to device @@ -347,9 +348,10 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, "MultiDeviceInferencePlugin::LoadNetworkImpl::AutoMode"); auto autoSContext = std::make_shared(); std::map filterConfig; + auto strDevices = GetDeviceList(fullConfig); + // keep the secondary priorities when the config key is one of the available hardware devices CheckConfig(fullConfig, autoSContext, filterConfig); // filter the device that supports filter configure - auto strDevices = GetDeviceList(fullConfig); auto metaDevices = ParseMetaDevices(strDevices, fullConfig); auto supportDevicesByConfig = FilterDevice(metaDevices, filterConfig); if (supportDevicesByConfig.size() == 0) { @@ -431,8 +433,8 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons auto tmpiter = fullConfig.find(CONFIG_KEY(ALLOW_AUTO_BATCHING)); if (tmpiter != fullConfig.end()) p.config.insert({tmpiter->first, tmpiter->second}); - const auto &deviceName = p.deviceName; - const auto &deviceConfig = p.config; + const auto& deviceName = p.deviceName; + const auto& deviceConfig = p.config; SoExecutableNetworkInternal exec_net; if (modelPath.empty()) { exec_net = GetCore()->LoadNetwork(network, deviceName, deviceConfig); @@ -769,9 +771,9 @@ std::string MultiDeviceInferencePlugin::GetDeviceList(const std::map& config, - AutoScheduleContext::Ptr& context, std::map& filterConfig) { + AutoScheduleContext::Ptr& context, + std::map& filterConfig) { // TODO need to optimize this code, too much duplicated code - const auto perf_hints_configs = PerfHintsConfig::SupportedKeys(); for (auto&& kvp : config) { if (kvp.first == ov::enable_profiling) { @@ -840,7 +842,12 @@ void MultiDeviceInferencePlugin::CheckConfig(const std::map_performanceHint = kvp.second; } - } else if (supported_configKeys.end() == std::find(supported_configKeys.begin(), supported_configKeys.end(), kvp.first)) { + } else if (_availableDevices.end() != + std::find(_availableDevices.begin(), _availableDevices.end(), kvp.first)) { + // keep secondary prperties for HW or virtual device + continue; + } else if (supported_configKeys.end() == + std::find(supported_configKeys.begin(), supported_configKeys.end(), kvp.first)) { IE_THROW() << "Unsupported config key: " << kvp.first; } else if (kvp.first.find("AUTO_") == 0) { continue; diff --git a/src/plugins/auto/plugin.hpp b/src/plugins/auto/plugin.hpp index a2be5877c7f..8e4f511940f 100644 --- a/src/plugins/auto/plugin.hpp +++ b/src/plugins/auto/plugin.hpp @@ -65,7 +65,8 @@ private: InferenceEngine::CNNNetwork network, const std::map& config, const std::string &networkPrecision = METRIC_VALUE(FP32)); - static void CheckConfig(const std::map& config, AutoScheduleContext::Ptr& context, + static void CheckConfig(const std::map& config, + AutoScheduleContext::Ptr& context, std::map& filterConfig); std::vector FilterDevice(const std::vector& metaDevices, const std::map& config); @@ -73,6 +74,7 @@ private: InferenceEngine::CNNNetwork network); static std::mutex _mtx; static std::map> _priorityMap; + static std::set _availableDevices; }; } // namespace MultiDevicePlugin diff --git a/src/plugins/intel_gpu/src/plugin/plugin.cpp b/src/plugins/intel_gpu/src/plugin/plugin.cpp index 85737c3eaa8..28de1ee9a66 100644 --- a/src/plugins/intel_gpu/src/plugin/plugin.cpp +++ b/src/plugins/intel_gpu/src/plugin/plugin.cpp @@ -615,7 +615,7 @@ Parameter Plugin::GetConfig(const std::string& name, const std::map multiConfigs = { - {ov::device::priorities(CommonTestUtils::DEVICE_CPU)} + {ov::device::priorities(CommonTestUtils::DEVICE_CPU)} }; +const std::vector configsWithSecondaryProperties = { + {ov::device::properties("CPU", + ov::enable_profiling(true), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::properties("CPU", + ov::enable_profiling(true), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), + ov::device::properties("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}}; + +const std::vector multiConfigsWithSecondaryProperties = { + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), + ov::device::properties("CPU", + ov::enable_profiling(true), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), + ov::device::properties("CPU", + ov::enable_profiling(true), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), + ov::device::properties("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}}; + +const std::vector autoConfigsWithSecondaryProperties = { + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), + ov::device::properties("AUTO", + ov::enable_profiling(true), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), + ov::device::properties("CPU", + ov::enable_profiling(true), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), + ov::device::properties("CPU", + ov::enable_profiling(true), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), + ov::device::properties("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}, + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), + ov::device::properties("AUTO", + ov::enable_profiling(false), + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)), + ov::device::properties("CPU", + ov::enable_profiling(true), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), + ov::device::properties("AUTO", + ov::enable_profiling(false), + ov::device::priorities(CommonTestUtils::DEVICE_GPU), + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)), + ov::device::properties("CPU", + ov::enable_profiling(true), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), + ov::device::properties("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}}; + INSTANTIATE_TEST_SUITE_P( smoke_OVClassSetDevicePriorityConfigTest, OVClassSetDevicePriorityConfigTest, ::testing::Combine(::testing::Values("MULTI", "AUTO"), @@ -204,9 +255,22 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values("CPU")); // IE Class Load network +INSTANTIATE_TEST_SUITE_P(smoke_CPU_OVClassLoadNetworkWithCorrectSecondaryPropertiesTest, + OVClassLoadNetworkWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values("CPU", "AUTO:CPU", "MULTI:CPU"), + ::testing::ValuesIn(configsWithSecondaryProperties))); + +INSTANTIATE_TEST_SUITE_P(smoke_Multi_OVClassLoadNetworkWithSecondaryPropertiesTest, + OVClassLoadNetworkWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values("MULTI"), + ::testing::ValuesIn(multiConfigsWithSecondaryProperties))); + +INSTANTIATE_TEST_SUITE_P(smoke_AUTO_OVClassLoadNetworkWithSecondaryPropertiesTest, + OVClassLoadNetworkWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values("AUTO"), + ::testing::ValuesIn(autoConfigsWithSecondaryProperties))); INSTANTIATE_TEST_SUITE_P( smoke_OVClassLoadNetworkTest, OVClassLoadNetworkTest, ::testing::Values("CPU")); } // namespace - diff --git a/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/core_integration.cpp b/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/core_integration.cpp index 42b04fd611c..34bfa1481de 100644 --- a/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/core_integration.cpp +++ b/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/core_integration.cpp @@ -692,10 +692,31 @@ const std::vector gpuCorrectConfigs = { } }; +const std::vector gpuCorrectConfigsWithSecondaryProperties = { + {ov::device::properties(CommonTestUtils::DEVICE_GPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), + ov::hint::allow_auto_batching(false))}, + {ov::device::properties(CommonTestUtils::DEVICE_GPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), + ov::hint::allow_auto_batching(false)), + ov::device::properties(CommonTestUtils::DEVICE_CPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY), + ov::hint::allow_auto_batching(false))}}; + INSTANTIATE_TEST_SUITE_P(smoke_OVClassLoadNetworkWithCorrectPropertiesAutoBatchingTest, OVClassLoadNetworkWithCorrectPropertiesTest, ::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_GPU), ::testing::ValuesIn(gpuCorrectConfigs))); +INSTANTIATE_TEST_SUITE_P(smoke_OVClassLoadNetworkWithCorrectSecondaryPropertiesTest, + OVClassLoadNetworkWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_GPU), + ::testing::ValuesIn(gpuCorrectConfigsWithSecondaryProperties))); + +INSTANTIATE_TEST_SUITE_P(smoke_AUTO_OVClassLoadNetworkWithCorrectSecondaryPropertiesTest, + OVClassLoadNetworkWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values("AUTO:GPU", "MULTI:GPU"), + ::testing::ValuesIn(gpuCorrectConfigsWithSecondaryProperties))); + const std::vector autoCorrectConfigs = { { ov::device::priorities(CommonTestUtils::DEVICE_GPU), @@ -709,10 +730,33 @@ const std::vector autoCorrectConfigs = { } }; +const std::vector autoCorrectConfigsWithSecondaryProperties = { + {ov::device::priorities(CommonTestUtils::DEVICE_GPU), + ov::device::properties(CommonTestUtils::DEVICE_AUTO, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), + ov::hint::allow_auto_batching(false))}, + {ov::device::priorities(CommonTestUtils::DEVICE_GPU), + ov::device::properties(CommonTestUtils::DEVICE_GPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), + ov::hint::allow_auto_batching(false))}, + {ov::device::priorities(CommonTestUtils::DEVICE_GPU), + ov::device::properties(CommonTestUtils::DEVICE_GPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), + ov::hint::allow_auto_batching(false)), + ov::device::properties(CommonTestUtils::DEVICE_CPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY), + ov::hint::allow_auto_batching(false))}}; + INSTANTIATE_TEST_SUITE_P(smoke_Auto_OVClassLoadNetworkWithCorrectPropertiesAutoBatchingTest, OVClassLoadNetworkWithCorrectPropertiesTest, ::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_MULTI, CommonTestUtils::DEVICE_AUTO), ::testing::ValuesIn(autoCorrectConfigs))); +INSTANTIATE_TEST_SUITE_P(smoke_Auto_OVClassLoadNetworkWithCorrectSecondaryPropertiesTest, + OVClassLoadNetworkWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_MULTI, + CommonTestUtils::DEVICE_AUTO), + ::testing::ValuesIn(autoCorrectConfigsWithSecondaryProperties))); + const std::vector batchCorrectConfigs = { {} }; diff --git a/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp b/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp index 9c7b85934fa..399ca7337b2 100644 --- a/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp +++ b/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp @@ -1063,6 +1063,31 @@ TEST_P(OVClassLoadNetworkTest, LoadNetworkHETEROAndDeviceIDThrows) { } } +// +// LoadNetwork with AUTO on MULTI combinations particular device +// +TEST_P(OVClassLoadNetworkTest, LoadNetworkMULTIwithAUTONoThrow) { + ov::Core ie = createCoreWithTemplate(); + if (supportsDeviceID(ie, deviceName) && supportsAvaliableDevices(ie, deviceName)) { + std::string devices; + auto availableDevices = ie.get_property(deviceName, ov::available_devices); + for (auto&& device : availableDevices) { + devices += deviceName + '.' + device; + if (&device != &(availableDevices.back())) { + devices += ','; + } + } + OV_ASSERT_NO_THROW( + ie.compile_model(actualNetwork, + CommonTestUtils::DEVICE_MULTI, + ov::device::properties(CommonTestUtils::DEVICE_AUTO, ov::device::priorities(devices)), + ov::device::properties(CommonTestUtils::DEVICE_MULTI, + ov::device::priorities(CommonTestUtils::DEVICE_AUTO, deviceName)))); + } else { + GTEST_SKIP(); + } +} + // // LoadNetwork with HETERO on MULTI combinations particular device //