From d7f47aa122855175a669e8187137435dedc3e7c4 Mon Sep 17 00:00:00 2001 From: Wang Wangwang Date: Fri, 17 Feb 2023 11:15:07 +0800 Subject: [PATCH] =?UTF-8?q?Provide=20ENABLE=5FSTARTUP=5FFALLBACK=20propert?= =?UTF-8?q?y=20to=20enable/disable=20CPU=20as=20acc=E2=80=A6=20(#14503)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Provide ENABLE_STARTUP_FALLBACK property to enable/disable CPU as acceleration * Add more test cases to ENABLE_STARTUP_FALLBACK property * Remove unnecessary config * Remove plugin.hpp * Update test case by PR#14754 * Update copyright Co-authored-by: River Li * Simplify using namespace --------- Co-authored-by: Chen Peter Co-authored-by: River Li --- .../openvino/runtime/auto/properties.hpp | 5 + src/plugins/auto/auto_schedule.cpp | 2 +- src/plugins/auto/common.hpp | 1 + src/plugins/auto/plugin.cpp | 5 + src/plugins/auto/utils/config.hpp | 17 +- .../behavior/ov_plugin/properties_tests.cpp | 5 +- .../behavior/ov_plugin/properties_tests.cpp | 4 +- .../auto_startup_fallback_properties_test.cpp | 147 ++++++++++++++++++ 8 files changed, 181 insertions(+), 5 deletions(-) create mode 100644 src/tests/unit/auto/auto_startup_fallback_properties_test.cpp diff --git a/src/inference/include/openvino/runtime/auto/properties.hpp b/src/inference/include/openvino/runtime/auto/properties.hpp index e73469b7ad5..9a45a93467e 100644 --- a/src/inference/include/openvino/runtime/auto/properties.hpp +++ b/src/inference/include/openvino/runtime/auto/properties.hpp @@ -18,5 +18,10 @@ namespace intel_auto { */ static constexpr Property device_bind_buffer{"DEVICE_BIND_BUFFER"}; +/** + * @brief auto/multi device setting that enable/disable CPU as acceleration (or helper device) at the beginning + */ +static constexpr Property enable_startup_fallback{"ENABLE_STARTUP_FALLBACK"}; + } // namespace intel_auto } // namespace ov \ No newline at end of file diff --git a/src/plugins/auto/auto_schedule.cpp b/src/plugins/auto/auto_schedule.cpp index 243a1d985fb..645b27ba69e 100644 --- a/src/plugins/auto/auto_schedule.cpp +++ b/src/plugins/auto/auto_schedule.cpp @@ -160,7 +160,7 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) { bool isActualDevCPU = _loadContext[ACTUALDEVICE].deviceInfo.deviceName.find("CPU") !=std::string::npos && !isCumulative; // if Actual device is CPU or perf_hint is cumulative, disabled _loadContext[CPU], only use _loadContext[ACTUALDEVICE] - if (isActualDevCPU || isCumulative) { + if (isActualDevCPU || isCumulative || !_autoSContext->_startupfallback) { _loadContext[CPU].isEnabled = false; } else { const auto CPUIter = std::find_if(_autoSContext->_devicePriorities.begin(), _autoSContext->_devicePriorities.end(), diff --git a/src/plugins/auto/common.hpp b/src/plugins/auto/common.hpp index 1a271c2c2b9..58c13ae9ab1 100644 --- a/src/plugins/auto/common.hpp +++ b/src/plugins/auto/common.hpp @@ -123,6 +123,7 @@ public: bool _needPerfCounters; bool _batchingDisabled = {false}; bool _bindBuffer = false; + bool _startupfallback = true; virtual ~MultiScheduleContext() = default; }; diff --git a/src/plugins/auto/plugin.cpp b/src/plugins/auto/plugin.cpp index 04fdf913d4e..ffcef3822a9 100644 --- a/src/plugins/auto/plugin.cpp +++ b/src/plugins/auto/plugin.cpp @@ -246,6 +246,8 @@ InferenceEngine::Parameter MultiDeviceInferencePlugin::GetConfig(const std::stri return ov::util::from_string(val, ov::log::level); } else if (name == ov::device::priorities) { return ov::util::from_string(val, ov::device::priorities); + } else if (name == ov::intel_auto::enable_startup_fallback) { + return val == PluginConfigParams::YES ? true : false; } else { return val; } @@ -453,6 +455,9 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons auto tmpiter = fullConfig.find(ov::intel_auto::device_bind_buffer.name()); if (tmpiter != fullConfig.end() && tmpiter->second == PluginConfigParams::YES) autoSContext->_bindBuffer = true; + auto tmpiter_enableStartupFallback = fullConfig.find(ov::intel_auto::enable_startup_fallback.name()); + if (tmpiter_enableStartupFallback != fullConfig.end() && tmpiter_enableStartupFallback->second == PluginConfigParams::NO) + autoSContext->_startupfallback = false; return std::make_shared(autoSContext, std::make_shared()); } OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, "MultiDeviceInferencePlugin::LoadNetworkImpl:MultiMode"); diff --git a/src/plugins/auto/utils/config.hpp b/src/plugins/auto/utils/config.hpp index 3d1d8c5ca93..a14bcdb6870 100644 --- a/src/plugins/auto/utils/config.hpp +++ b/src/plugins/auto/utils/config.hpp @@ -24,6 +24,7 @@ struct PluginConfig { _devicePriority(""), _modelPriority(1), _deviceBindBuffer(false), + _enableStartupFallback(true), _logLevel("LOG_NONE") { adjustKeyMapValues(); } @@ -38,6 +39,7 @@ struct PluginConfig { res.push_back(ov::log::level.name()); res.push_back(ov::intel_auto::device_bind_buffer.name()); res.push_back(ov::auto_batch_timeout.name()); + res.push_back(ov::intel_auto::enable_startup_fallback.name()); return res; }(); auto multi_supported_configKeys = supported_configKeys; @@ -65,7 +67,8 @@ struct PluginConfig { RW_property(ov::hint::performance_mode.name()), RW_property(ov::hint::num_requests.name()), RW_property(ov::intel_auto::device_bind_buffer.name()), - RW_property(ov::cache_dir.name())}; + RW_property(ov::cache_dir.name()), + RW_property(ov::intel_auto::enable_startup_fallback.name())}; std::vector supportedProperties; supportedProperties.reserve(roProperties.size() + rwProperties.size()); supportedProperties.insert(supportedProperties.end(), roProperties.begin(), roProperties.end()); @@ -178,6 +181,12 @@ struct PluginConfig { } else if (kvp.first == ov::cache_dir.name()) { _cacheDir = kvp.second; _isSetCacheDir = true; + } else if (kvp.first == ov::intel_auto::enable_startup_fallback.name()) { + if (kvp.second == PluginConfigParams::YES) _enableStartupFallback = true; + else if (kvp.second == PluginConfigParams::NO) _enableStartupFallback = false; + else + IE_THROW() << "Unsupported config value: " << kvp.second + << " for key: " << kvp.first; } else { if (pluginName.find("AUTO") != std::string::npos || !supportHWProprety) // AUTO and MULTI just only accept its own properites and secondary property when calling @@ -261,7 +270,10 @@ struct PluginConfig { _keyConfigMap[ov::intel_auto::device_bind_buffer.name()] = PluginConfigParams::YES; else _keyConfigMap[ov::intel_auto::device_bind_buffer.name()] = PluginConfigParams::NO; - + if (_enableStartupFallback) + _keyConfigMap[ov::intel_auto::enable_startup_fallback.name()] = PluginConfigParams::YES; + else + _keyConfigMap[ov::intel_auto::enable_startup_fallback.name()] = PluginConfigParams::NO; _keyConfigMap[ov::auto_batch_timeout.name()] = _batchTimeout; _keyConfigMap[ov::log::level.name()] = _logLevel; @@ -281,6 +293,7 @@ struct PluginConfig { std::string _devicePriority; int _modelPriority; bool _deviceBindBuffer; + bool _enableStartupFallback; std::string _logLevel; PerfHintsConfig _perfHintsConfig; // Add this flag to check if user app sets hint with none value that is equal to the default value of hint. diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp index 5fe84640337..c8c5e15ff05 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp @@ -28,7 +28,9 @@ const std::vector multi_Auto_properties = { {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)}, {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT)}, {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::intel_auto::device_bind_buffer("YES")}, - {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::intel_auto::device_bind_buffer("NO")} + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::intel_auto::device_bind_buffer("NO")}, + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::intel_auto::enable_startup_fallback("YES")}, + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::intel_auto::enable_startup_fallback("NO")} }; INSTANTIATE_TEST_SUITE_P(smoke_AutoMultiBehaviorTests, OVPropertiesTests, @@ -81,6 +83,7 @@ const std::vector default_properties = { {ov::hint::allow_auto_batching(true)}, {ov::auto_batch_timeout("1000")}, {ov::intel_auto::device_bind_buffer(false)}, + {ov::intel_auto::enable_startup_fallback(true)}, {ov::device::priorities("")} }; INSTANTIATE_TEST_SUITE_P(smoke_AutoBehaviorTests, OVPropertiesDefaultTests, diff --git a/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp b/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp index 0bb86d73b42..584d511e7f3 100644 --- a/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp +++ b/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp @@ -33,7 +33,9 @@ auto auto_multi_properties = []() { {ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT)}, {ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::intel_auto::device_bind_buffer("YES")}, - {ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::intel_auto::device_bind_buffer("NO")}}; + {ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::intel_auto::device_bind_buffer("NO")}, + {ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::intel_auto::enable_startup_fallback("YES")}, + {ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::intel_auto::enable_startup_fallback("NO")}}; }; const std::vector multi_properties = { diff --git a/src/tests/unit/auto/auto_startup_fallback_properties_test.cpp b/src/tests/unit/auto/auto_startup_fallback_properties_test.cpp new file mode 100644 index 00000000000..d738354bc4a --- /dev/null +++ b/src/tests/unit/auto/auto_startup_fallback_properties_test.cpp @@ -0,0 +1,147 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include +#include "unit_test_utils/mocks/cpp_interfaces/interface/mock_icore.hpp" +#include "unit_test_utils/mocks/mock_iinfer_request.hpp" +#include "unit_test_utils/mocks/cpp_interfaces/impl/mock_inference_plugin_internal.hpp" +#include "unit_test_utils/mocks/cpp_interfaces/interface/mock_iexecutable_network_internal.hpp" +#include "unit_test_utils/mocks/cpp_interfaces/interface/mock_ivariable_state_internal.hpp" +#include "unit_test_utils/mocks/cpp_interfaces/interface/mock_iinference_plugin.hpp" +#include +#include +#include +#include +#include +#include "plugin/mock_auto_device_plugin.hpp" +#include "mock_common.hpp" +#include + +using ::testing::_; +using ::testing::StrEq; +using ::testing::Return; +using ::testing::NiceMock; +using Config = std::map; +using namespace MockMultiDevice; + +using ConfigParams = std::tuple; + +// define a matcher if all the elements of subMap are contained in the map. +MATCHER_P(MapContains, subMap, "Check if all the elements of the subMap are contained in the map.") { + if (subMap.empty()) + return true; + for (auto& item : subMap) { + auto key = item.first; + auto value = item.second; + auto dest = arg.find(key); + if (dest == arg.end()) { + return false; + } else if (dest->second != value) { + return false; + } + } + return true; +} +class AutoStartupFallback : public ::testing::TestWithParam { +public: + std::shared_ptr function; + InferenceEngine::CNNNetwork cnnNet; + std::shared_ptr> core; + std::shared_ptr> plugin; + + //mock exeNetwork helper + ov::SoPtr mockExeNetwork; + // config for Auto device + std::map config; + std::vector metaDevices; + std::shared_ptr> inferReqInternal; + size_t optimalNum; + +public: + void TearDown() override { + core.reset(); + plugin.reset(); + //mockIExeNet.reset(); + mockExeNetwork = {}; + config.clear(); + metaDevices.clear(); + inferReqInternal.reset(); + } + + void SetUp() override { + // prepare mockExeNetwork + auto mockIExeNet = std::make_shared>(); + mockExeNetwork = {mockIExeNet, {}}; + // prepare mockicore and cnnNetwork for loading + core = std::make_shared>(); + NiceMock* mock_multi = new NiceMock(); + plugin.reset(mock_multi); + function = ngraph::builder::subgraph::makeConvPoolRelu(); + cnnNet = InferenceEngine::CNNNetwork(function); + // replace core with mock Icore + plugin->SetCore(core); + // mock execNetwork can work + inferReqInternal = std::make_shared>(); + ON_CALL(*mockIExeNet.get(), CreateInferRequest()).WillByDefault(Return(inferReqInternal)); + IE_SET_METRIC(OPTIMAL_NUMBER_OF_INFER_REQUESTS, optimalNum, 1); + ON_CALL(*mockIExeNet.get(), GetMetric(StrEq(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)))) + .WillByDefault(Return(optimalNum)); + IE_SET_METRIC(SUPPORTED_CONFIG_KEYS, supportConfigs, {}); + ON_CALL(*core, GetMetric(_, StrEq(METRIC_KEY(SUPPORTED_CONFIG_KEYS)), _)) + .WillByDefault(Return(supportConfigs)); + ON_CALL(*core, GetConfig(_, StrEq(GPU_CONFIG_KEY(MAX_NUM_THREADS)))) + .WillByDefault(Return(12)); + } +}; + +TEST_P(AutoStartupFallback, releaseResource) { + // get Parameter + bool startup_fallback; + Config config; + std::tie(startup_fallback, config) = this->GetParam(); + // test auto plugin + plugin->SetName("AUTO"); + + ON_CALL(*core, LoadNetwork(::testing::Matcher(_), + ::testing::Matcher(_), + ::testing::Matcher(_))).WillByDefault(Return(mockExeNetwork)); + + metaDevices = {{CommonTestUtils::DEVICE_CPU, {}, -1}, {CommonTestUtils::DEVICE_GPU, {}, -1}}; + // DeviceInformation devInfo; + ON_CALL(*plugin, ParseMetaDevices(_, _)).WillByDefault(Return(metaDevices)); + ON_CALL(*plugin, GetValidDevice) + .WillByDefault([this](const std::vector& metaDevices, const std::string& netPrecision) { + std::list devices(metaDevices.begin(), metaDevices.end()); + return devices; + }); + ON_CALL(*plugin, SelectDevice(_, _, _)).WillByDefault(Return(metaDevices[1])); + + EXPECT_CALL( + *core, + LoadNetwork(::testing::Matcher(_), + ::testing::Matcher(CommonTestUtils::DEVICE_GPU), + ::testing::Matcher&>(_))) + .Times(1); + if (startup_fallback) { + std::map test_map = {{"PERFORMANCE_HINT", "LATENCY"}}; + EXPECT_CALL( + *core, + LoadNetwork(::testing::Matcher(_), + ::testing::Matcher(CommonTestUtils::DEVICE_CPU), + ::testing::Matcher&>(MapContains(test_map)))) + .Times(1); + } + + ASSERT_NO_THROW(plugin->LoadExeNetworkImpl(cnnNet, config)); +} + +const std::vector testConfigs = {ConfigParams {true, {{"ENABLE_STARTUP_FALLBACK", "YES"}}}, + ConfigParams {false, {{"ENABLE_STARTUP_FALLBACK", "NO"}}} + }; + +INSTANTIATE_TEST_SUITE_P(smoke_Auto_StartupFallback, + AutoStartupFallback, + ::testing::ValuesIn(testConfigs));