diff --git a/src/inference/src/ie_core.cpp b/src/inference/src/ie_core.cpp index 012749a2a10..c8b01b25dea 100644 --- a/src/inference/src/ie_core.cpp +++ b/src/inference/src/ie_core.cpp @@ -127,14 +127,27 @@ void allowNotImplemented(F&& f) { ov::AnyMap flatten_sub_properties(const std::string& device, const ov::AnyMap& properties) { ov::AnyMap result = properties; - for (auto&& property : properties) { - auto parsed = parseDeviceNameIntoConfig(property.first); + for (auto item = result.begin(); item != result.end();) { + auto parsed = parseDeviceNameIntoConfig(item->first); + if (!item->second.is()) { + item++; + continue; + } if (device.find(parsed._deviceName) != std::string::npos) { - if (property.second.is()) { - for (auto&& sub_property : property.second.as()) { - result[sub_property.first] = sub_property.second; - } + // 1. flatten the scondary property for target device + for (auto&& sub_property : item->second.as()) { + // 1.1 1st level property overides 2nd level property + if (result.find(sub_property.first) != result.end()) + continue; + result[sub_property.first] = sub_property.second; } + item = result.erase(item); + } else if (device != "AUTO" && device != "MULTI" && device != "HETERO") { + // 2. remove the secondary property setting for other hard ware device + item = result.erase(item); + } else { + // 3. keep the secondary property for the other virtual devices + item++; } } return result; diff --git a/src/plugins/auto/plugin.cpp b/src/plugins/auto/plugin.cpp index 98c68a26a8f..16cd15be542 100644 --- a/src/plugins/auto/plugin.cpp +++ b/src/plugins/auto/plugin.cpp @@ -77,6 +77,8 @@ namespace { std::mutex MultiDeviceInferencePlugin::_mtx; std::map> MultiDeviceInferencePlugin::_priorityMap; +std::set MultiDeviceInferencePlugin::_availableDevices = + std::set{"CPU", "GPU", "GNA", "TEMPLATE", "MYRAID", "HDDL", "VPUX", "MULTI", "HETERO", "CUDA", "HPU_GOYA"}; std::vector MultiDeviceInferencePlugin::ParseMetaDevices(const std::string& priorities, const std::map & config) const { @@ -338,7 +340,6 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons auto workMode = fullConfig.find(CONFIG_KEY_INTERNAL(MULTI_WORK_MODE_AS_AUTO)); bool workModeAuto = workMode != fullConfig.end() && workMode->second == InferenceEngine::PluginConfigParams::YES; auto priorities = fullConfig.find(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES); - // if workMode is AUTO if (workModeAuto) { // check the configure and check if need to set PerfCounters configure to device @@ -347,9 +348,10 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, "MultiDeviceInferencePlugin::LoadNetworkImpl::AutoMode"); auto autoSContext = std::make_shared(); std::map filterConfig; + auto strDevices = GetDeviceList(fullConfig); + // keep the secondary priorities when the config key is one of the available hardware devices CheckConfig(fullConfig, autoSContext, filterConfig); // filter the device that supports filter configure - auto strDevices = GetDeviceList(fullConfig); auto metaDevices = ParseMetaDevices(strDevices, fullConfig); auto supportDevicesByConfig = FilterDevice(metaDevices, filterConfig); if (supportDevicesByConfig.size() == 0) { @@ -431,8 +433,8 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons auto tmpiter = fullConfig.find(CONFIG_KEY(ALLOW_AUTO_BATCHING)); if (tmpiter != fullConfig.end()) p.config.insert({tmpiter->first, tmpiter->second}); - const auto &deviceName = p.deviceName; - const auto &deviceConfig = p.config; + const auto& deviceName = p.deviceName; + const auto& deviceConfig = p.config; SoExecutableNetworkInternal exec_net; if (modelPath.empty()) { exec_net = GetCore()->LoadNetwork(network, deviceName, deviceConfig); @@ -769,9 +771,9 @@ std::string MultiDeviceInferencePlugin::GetDeviceList(const std::map& config, - AutoScheduleContext::Ptr& context, std::map& filterConfig) { + AutoScheduleContext::Ptr& context, + std::map& filterConfig) { // TODO need to optimize this code, too much duplicated code - const auto perf_hints_configs = PerfHintsConfig::SupportedKeys(); for (auto&& kvp : config) { if (kvp.first == ov::enable_profiling) { @@ -840,7 +842,12 @@ void MultiDeviceInferencePlugin::CheckConfig(const std::map_performanceHint = kvp.second; } - } else if (supported_configKeys.end() == std::find(supported_configKeys.begin(), supported_configKeys.end(), kvp.first)) { + } else if (_availableDevices.end() != + std::find(_availableDevices.begin(), _availableDevices.end(), kvp.first)) { + // keep secondary prperties for HW or virtual device + continue; + } else if (supported_configKeys.end() == + std::find(supported_configKeys.begin(), supported_configKeys.end(), kvp.first)) { IE_THROW() << "Unsupported config key: " << kvp.first; } else if (kvp.first.find("AUTO_") == 0) { continue; diff --git a/src/plugins/auto/plugin.hpp b/src/plugins/auto/plugin.hpp index a2be5877c7f..8e4f511940f 100644 --- a/src/plugins/auto/plugin.hpp +++ b/src/plugins/auto/plugin.hpp @@ -65,7 +65,8 @@ private: InferenceEngine::CNNNetwork network, const std::map& config, const std::string &networkPrecision = METRIC_VALUE(FP32)); - static void CheckConfig(const std::map& config, AutoScheduleContext::Ptr& context, + static void CheckConfig(const std::map& config, + AutoScheduleContext::Ptr& context, std::map& filterConfig); std::vector FilterDevice(const std::vector& metaDevices, const std::map& config); @@ -73,6 +74,7 @@ private: InferenceEngine::CNNNetwork network); static std::mutex _mtx; static std::map> _priorityMap; + static std::set _availableDevices; }; } // namespace MultiDevicePlugin diff --git a/src/plugins/intel_gpu/src/plugin/plugin.cpp b/src/plugins/intel_gpu/src/plugin/plugin.cpp index 85737c3eaa8..28de1ee9a66 100644 --- a/src/plugins/intel_gpu/src/plugin/plugin.cpp +++ b/src/plugins/intel_gpu/src/plugin/plugin.cpp @@ -615,7 +615,7 @@ Parameter Plugin::GetConfig(const std::string& name, const std::map multiConfigs = { - {ov::device::priorities(CommonTestUtils::DEVICE_CPU)} + {ov::device::priorities(CommonTestUtils::DEVICE_CPU)} }; +const std::vector configsWithSecondaryProperties = { + {ov::device::properties("CPU", + ov::enable_profiling(true), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::properties("CPU", + ov::enable_profiling(true), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), + ov::device::properties("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}}; + +const std::vector multiConfigsWithSecondaryProperties = { + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), + ov::device::properties("CPU", + ov::enable_profiling(true), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), + ov::device::properties("CPU", + ov::enable_profiling(true), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), + ov::device::properties("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}}; + +const std::vector autoConfigsWithSecondaryProperties = { + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), + ov::device::properties("AUTO", + ov::enable_profiling(true), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), + ov::device::properties("CPU", + ov::enable_profiling(true), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), + ov::device::properties("CPU", + ov::enable_profiling(true), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), + ov::device::properties("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}, + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), + ov::device::properties("AUTO", + ov::enable_profiling(false), + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)), + ov::device::properties("CPU", + ov::enable_profiling(true), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), + ov::device::properties("AUTO", + ov::enable_profiling(false), + ov::device::priorities(CommonTestUtils::DEVICE_GPU), + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)), + ov::device::properties("CPU", + ov::enable_profiling(true), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), + ov::device::properties("GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}}; + INSTANTIATE_TEST_SUITE_P( smoke_OVClassSetDevicePriorityConfigTest, OVClassSetDevicePriorityConfigTest, ::testing::Combine(::testing::Values("MULTI", "AUTO"), @@ -204,9 +255,22 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values("CPU")); // IE Class Load network +INSTANTIATE_TEST_SUITE_P(smoke_CPU_OVClassLoadNetworkWithCorrectSecondaryPropertiesTest, + OVClassLoadNetworkWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values("CPU", "AUTO:CPU", "MULTI:CPU"), + ::testing::ValuesIn(configsWithSecondaryProperties))); + +INSTANTIATE_TEST_SUITE_P(smoke_Multi_OVClassLoadNetworkWithSecondaryPropertiesTest, + OVClassLoadNetworkWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values("MULTI"), + ::testing::ValuesIn(multiConfigsWithSecondaryProperties))); + +INSTANTIATE_TEST_SUITE_P(smoke_AUTO_OVClassLoadNetworkWithSecondaryPropertiesTest, + OVClassLoadNetworkWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values("AUTO"), + ::testing::ValuesIn(autoConfigsWithSecondaryProperties))); INSTANTIATE_TEST_SUITE_P( smoke_OVClassLoadNetworkTest, OVClassLoadNetworkTest, ::testing::Values("CPU")); } // namespace - diff --git a/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/core_integration.cpp b/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/core_integration.cpp index 42b04fd611c..34bfa1481de 100644 --- a/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/core_integration.cpp +++ b/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/core_integration.cpp @@ -692,10 +692,31 @@ const std::vector gpuCorrectConfigs = { } }; +const std::vector gpuCorrectConfigsWithSecondaryProperties = { + {ov::device::properties(CommonTestUtils::DEVICE_GPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), + ov::hint::allow_auto_batching(false))}, + {ov::device::properties(CommonTestUtils::DEVICE_GPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), + ov::hint::allow_auto_batching(false)), + ov::device::properties(CommonTestUtils::DEVICE_CPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY), + ov::hint::allow_auto_batching(false))}}; + INSTANTIATE_TEST_SUITE_P(smoke_OVClassLoadNetworkWithCorrectPropertiesAutoBatchingTest, OVClassLoadNetworkWithCorrectPropertiesTest, ::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_GPU), ::testing::ValuesIn(gpuCorrectConfigs))); +INSTANTIATE_TEST_SUITE_P(smoke_OVClassLoadNetworkWithCorrectSecondaryPropertiesTest, + OVClassLoadNetworkWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_GPU), + ::testing::ValuesIn(gpuCorrectConfigsWithSecondaryProperties))); + +INSTANTIATE_TEST_SUITE_P(smoke_AUTO_OVClassLoadNetworkWithCorrectSecondaryPropertiesTest, + OVClassLoadNetworkWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values("AUTO:GPU", "MULTI:GPU"), + ::testing::ValuesIn(gpuCorrectConfigsWithSecondaryProperties))); + const std::vector autoCorrectConfigs = { { ov::device::priorities(CommonTestUtils::DEVICE_GPU), @@ -709,10 +730,33 @@ const std::vector autoCorrectConfigs = { } }; +const std::vector autoCorrectConfigsWithSecondaryProperties = { + {ov::device::priorities(CommonTestUtils::DEVICE_GPU), + ov::device::properties(CommonTestUtils::DEVICE_AUTO, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), + ov::hint::allow_auto_batching(false))}, + {ov::device::priorities(CommonTestUtils::DEVICE_GPU), + ov::device::properties(CommonTestUtils::DEVICE_GPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), + ov::hint::allow_auto_batching(false))}, + {ov::device::priorities(CommonTestUtils::DEVICE_GPU), + ov::device::properties(CommonTestUtils::DEVICE_GPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), + ov::hint::allow_auto_batching(false)), + ov::device::properties(CommonTestUtils::DEVICE_CPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY), + ov::hint::allow_auto_batching(false))}}; + INSTANTIATE_TEST_SUITE_P(smoke_Auto_OVClassLoadNetworkWithCorrectPropertiesAutoBatchingTest, OVClassLoadNetworkWithCorrectPropertiesTest, ::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_MULTI, CommonTestUtils::DEVICE_AUTO), ::testing::ValuesIn(autoCorrectConfigs))); +INSTANTIATE_TEST_SUITE_P(smoke_Auto_OVClassLoadNetworkWithCorrectSecondaryPropertiesTest, + OVClassLoadNetworkWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_MULTI, + CommonTestUtils::DEVICE_AUTO), + ::testing::ValuesIn(autoCorrectConfigsWithSecondaryProperties))); + const std::vector batchCorrectConfigs = { {} }; diff --git a/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp b/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp index 9c7b85934fa..399ca7337b2 100644 --- a/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp +++ b/src/tests/functional/plugin/shared/include/behavior/ov_plugin/core_integration.hpp @@ -1063,6 +1063,31 @@ TEST_P(OVClassLoadNetworkTest, LoadNetworkHETEROAndDeviceIDThrows) { } } +// +// LoadNetwork with AUTO on MULTI combinations particular device +// +TEST_P(OVClassLoadNetworkTest, LoadNetworkMULTIwithAUTONoThrow) { + ov::Core ie = createCoreWithTemplate(); + if (supportsDeviceID(ie, deviceName) && supportsAvaliableDevices(ie, deviceName)) { + std::string devices; + auto availableDevices = ie.get_property(deviceName, ov::available_devices); + for (auto&& device : availableDevices) { + devices += deviceName + '.' + device; + if (&device != &(availableDevices.back())) { + devices += ','; + } + } + OV_ASSERT_NO_THROW( + ie.compile_model(actualNetwork, + CommonTestUtils::DEVICE_MULTI, + ov::device::properties(CommonTestUtils::DEVICE_AUTO, ov::device::priorities(devices)), + ov::device::properties(CommonTestUtils::DEVICE_MULTI, + ov::device::priorities(CommonTestUtils::DEVICE_AUTO, deviceName)))); + } else { + GTEST_SKIP(); + } +} + // // LoadNetwork with HETERO on MULTI combinations particular device //