IntenseWebs/openvino
3
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Provide ENABLE_STARTUP_FALLBACK property to enable/disable CPU as acc… (#14503)

Browse Source 
* Provide ENABLE_STARTUP_FALLBACK property to enable/disable CPU as acceleration

* Add more test cases to ENABLE_STARTUP_FALLBACK property

* Remove unnecessary config

* Remove plugin.hpp

* Update test case by PR#14754

* Update copyright

Co-authored-by: River Li <river.li@intel.com>

* Simplify using namespace

---------

Co-authored-by: Chen Peter <peter.chen@intel.com>
Co-authored-by: River Li <river.li@intel.com>
This commit is contained in:
Wang WangwangRiver LiChen Peter
 authored and GitHub committed 2023-02-17 11:15:07 +08:00
1 parent 829257510a
commit d7f47aa122
8 files changed
+181 -5
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/inference/include/openvino/runtime/auto/properties.hpp
+5
View File
@@ -18,5 +18,10 @@ namespace intel_auto {
*/
static constexpr Property<bool> device_bind_buffer{"DEVICE_BIND_BUFFER"};
/**
* @brief auto/multi device setting that enable/disable CPU as acceleration (or helper device) at the beginning
*/
static constexpr Property<bool> enable_startup_fallback{"ENABLE_STARTUP_FALLBACK"};
} // namespace intel_auto
} // namespace ov
src/plugins/auto/auto_schedule.cpp
+1 -1
View File
@@ -160,7 +160,7 @@ void AutoSchedule::init(const ScheduleContext::Ptr& sContext) {
bool isActualDevCPU =
_loadContext[ACTUALDEVICE].deviceInfo.deviceName.find("CPU") !=std::string::npos && !isCumulative;
// if Actual device is CPU or perf_hint is cumulative, disabled _loadContext[CPU], only use _loadContext[ACTUALDEVICE]
if (isActualDevCPU || isCumulative) {
if (isActualDevCPU || isCumulative || !_autoSContext->_startupfallback) {
_loadContext[CPU].isEnabled = false;
} else {
const auto CPUIter = std::find_if(_autoSContext->_devicePriorities.begin(), _autoSContext->_devicePriorities.end(),
src/plugins/auto/common.hpp
+1
View File
@@ -123,6 +123,7 @@ public:
bool _needPerfCounters;
bool _batchingDisabled = {false};
bool _bindBuffer = false;
bool _startupfallback = true;
virtual ~MultiScheduleContext() = default;
};
src/plugins/auto/plugin.cpp
+5
View File
@@ -246,6 +246,8 @@ InferenceEngine::Parameter MultiDeviceInferencePlugin::GetConfig(const std::stri
return ov::util::from_string(val, ov::log::level);
} else if (name == ov::device::priorities) {
return ov::util::from_string(val, ov::device::priorities);
} else if (name == ov::intel_auto::enable_startup_fallback) {
return val == PluginConfigParams::YES ? true : false;
} else {
return val;
}
@@ -453,6 +455,9 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
auto tmpiter = fullConfig.find(ov::intel_auto::device_bind_buffer.name());
if (tmpiter != fullConfig.end() && tmpiter->second == PluginConfigParams::YES)
autoSContext->_bindBuffer = true;
auto tmpiter_enableStartupFallback = fullConfig.find(ov::intel_auto::enable_startup_fallback.name());
if (tmpiter_enableStartupFallback != fullConfig.end() && tmpiter_enableStartupFallback->second == PluginConfigParams::NO)
autoSContext->_startupfallback = false;
return std::make_shared<AutoExecutableNetwork>(autoSContext, std::make_shared<AutoSchedule>());
}
OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, "MultiDeviceInferencePlugin::LoadNetworkImpl:MultiMode");
src/plugins/auto/utils/config.hpp
+15 -2
View File
@@ -24,6 +24,7 @@ struct PluginConfig {
_devicePriority(""),
_modelPriority(1),
_deviceBindBuffer(false),
_enableStartupFallback(true),
_logLevel("LOG_NONE") {
adjustKeyMapValues();
}
@@ -38,6 +39,7 @@ struct PluginConfig {
res.push_back(ov::log::level.name());
res.push_back(ov::intel_auto::device_bind_buffer.name());
res.push_back(ov::auto_batch_timeout.name());
res.push_back(ov::intel_auto::enable_startup_fallback.name());
return res;
}();
auto multi_supported_configKeys = supported_configKeys;
@@ -65,7 +67,8 @@ struct PluginConfig {
RW_property(ov::hint::performance_mode.name()),
RW_property(ov::hint::num_requests.name()),
RW_property(ov::intel_auto::device_bind_buffer.name()),
RW_property(ov::cache_dir.name())};
RW_property(ov::cache_dir.name()),
RW_property(ov::intel_auto::enable_startup_fallback.name())};
std::vector<ov::PropertyName> supportedProperties;
supportedProperties.reserve(roProperties.size() + rwProperties.size());
supportedProperties.insert(supportedProperties.end(), roProperties.begin(), roProperties.end());
@@ -178,6 +181,12 @@ struct PluginConfig {
} else if (kvp.first == ov::cache_dir.name()) {
_cacheDir = kvp.second;
_isSetCacheDir = true;
} else if (kvp.first == ov::intel_auto::enable_startup_fallback.name()) {
if (kvp.second == PluginConfigParams::YES) _enableStartupFallback = true;
else if (kvp.second == PluginConfigParams::NO) _enableStartupFallback = false;
else
IE_THROW() << "Unsupported config value: " << kvp.second
<< " for key: " << kvp.first;
} else {
if (pluginName.find("AUTO") != std::string::npos || !supportHWProprety)
// AUTO and MULTI just only accept its own properites and secondary property when calling
@@ -261,7 +270,10 @@ struct PluginConfig {
_keyConfigMap[ov::intel_auto::device_bind_buffer.name()] = PluginConfigParams::YES;
else
_keyConfigMap[ov::intel_auto::device_bind_buffer.name()] = PluginConfigParams::NO;
if (_enableStartupFallback)
_keyConfigMap[ov::intel_auto::enable_startup_fallback.name()] = PluginConfigParams::YES;
else
_keyConfigMap[ov::intel_auto::enable_startup_fallback.name()] = PluginConfigParams::NO;
_keyConfigMap[ov::auto_batch_timeout.name()] = _batchTimeout;
_keyConfigMap[ov::log::level.name()] = _logLevel;
@@ -281,6 +293,7 @@ struct PluginConfig {
std::string _devicePriority;
int _modelPriority;
bool _deviceBindBuffer;
bool _enableStartupFallback;
std::string _logLevel;
PerfHintsConfig _perfHintsConfig;
// Add this flag to check if user app sets hint with none value that is equal to the default value of hint.
src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp
+4 -1
View File
@@ -28,7 +28,9 @@ const std::vector<ov::AnyMap> multi_Auto_properties = {
{ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)},
{ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT)},
{ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::intel_auto::device_bind_buffer("YES")},
{ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::intel_auto::device_bind_buffer("NO")}
{ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::intel_auto::device_bind_buffer("NO")},
{ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::intel_auto::enable_startup_fallback("YES")},
{ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::intel_auto::enable_startup_fallback("NO")}
};
INSTANTIATE_TEST_SUITE_P(smoke_AutoMultiBehaviorTests, OVPropertiesTests,
@@ -81,6 +83,7 @@ const std::vector<ov::AnyMap> default_properties = {
{ov::hint::allow_auto_batching(true)},
{ov::auto_batch_timeout("1000")},
{ov::intel_auto::device_bind_buffer(false)},
{ov::intel_auto::enable_startup_fallback(true)},
{ov::device::priorities("")}
};
INSTANTIATE_TEST_SUITE_P(smoke_AutoBehaviorTests, OVPropertiesDefaultTests,
src/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp
+3 -1
View File
@@ -33,7 +33,9 @@ auto auto_multi_properties = []() {
{ov::device::priorities(CommonTestUtils::DEVICE_GPU),
ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT)},
{ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::intel_auto::device_bind_buffer("YES")},
{ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::intel_auto::device_bind_buffer("NO")}};
{ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::intel_auto::device_bind_buffer("NO")},
{ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::intel_auto::enable_startup_fallback("YES")},
{ov::device::priorities(CommonTestUtils::DEVICE_GPU), ov::intel_auto::enable_startup_fallback("NO")}};
};
const std::vector<ov::AnyMap> multi_properties = {
src/tests/unit/auto/auto_startup_fallback_properties_test.cpp
+147
View File
@@ -0,0 +1,147 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <ie_metric_helpers.hpp>
#include <common_test_utils/test_constants.hpp>
#include "unit_test_utils/mocks/cpp_interfaces/interface/mock_icore.hpp"
#include "unit_test_utils/mocks/mock_iinfer_request.hpp"
#include "unit_test_utils/mocks/cpp_interfaces/impl/mock_inference_plugin_internal.hpp"
#include "unit_test_utils/mocks/cpp_interfaces/interface/mock_iexecutable_network_internal.hpp"
#include "unit_test_utils/mocks/cpp_interfaces/interface/mock_ivariable_state_internal.hpp"
#include "unit_test_utils/mocks/cpp_interfaces/interface/mock_iinference_plugin.hpp"
#include <ie_core.hpp>
#include <multi-device/multi_device_config.hpp>
#include <ngraph_functions/subgraph_builders.hpp>
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include "plugin/mock_auto_device_plugin.hpp"
#include "mock_common.hpp"
#include <thread>
using ::testing::_;
using ::testing::StrEq;
using ::testing::Return;
using ::testing::NiceMock;
using Config = std::map<std::string, std::string>;
using namespace MockMultiDevice;
using ConfigParams = std::tuple<bool,
Config>;
// define a matcher if all the elements of subMap are contained in the map.
MATCHER_P(MapContains, subMap, "Check if all the elements of the subMap are contained in the map.") {
if (subMap.empty())
return true;
for (auto& item : subMap) {
auto key = item.first;
auto value = item.second;
auto dest = arg.find(key);
if (dest == arg.end()) {
return false;
} else if (dest->second != value) {
return false;
}
}
return true;
}
class AutoStartupFallback : public ::testing::TestWithParam<ConfigParams> {
public:
std::shared_ptr<ngraph::Function> function;
InferenceEngine::CNNNetwork cnnNet;
std::shared_ptr<NiceMock<MockICore>> core;
std::shared_ptr<NiceMock<MockMultiDeviceInferencePlugin>> plugin;
//mock exeNetwork helper
ov::SoPtr<IExecutableNetworkInternal> mockExeNetwork;
// config for Auto device
std::map<std::string, std::string> config;
std::vector<DeviceInformation> metaDevices;
std::shared_ptr<NiceMock<MockIInferRequestInternal>> inferReqInternal;
size_t optimalNum;
public:
void TearDown() override {
core.reset();
plugin.reset();
//mockIExeNet.reset();
mockExeNetwork = {};
config.clear();
metaDevices.clear();
inferReqInternal.reset();
}
void SetUp() override {
// prepare mockExeNetwork
auto mockIExeNet = std::make_shared<NiceMock<MockIExecutableNetworkInternal>>();
mockExeNetwork = {mockIExeNet, {}};
// prepare mockicore and cnnNetwork for loading
core = std::make_shared<NiceMock<MockICore>>();
NiceMock<MockMultiDeviceInferencePlugin>* mock_multi = new NiceMock<MockMultiDeviceInferencePlugin>();
plugin.reset(mock_multi);
function = ngraph::builder::subgraph::makeConvPoolRelu();
cnnNet = InferenceEngine::CNNNetwork(function);
// replace core with mock Icore
plugin->SetCore(core);
// mock execNetwork can work
inferReqInternal = std::make_shared<NiceMock<MockIInferRequestInternal>>();
ON_CALL(*mockIExeNet.get(), CreateInferRequest()).WillByDefault(Return(inferReqInternal));
IE_SET_METRIC(OPTIMAL_NUMBER_OF_INFER_REQUESTS, optimalNum, 1);
ON_CALL(*mockIExeNet.get(), GetMetric(StrEq(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS))))
.WillByDefault(Return(optimalNum));
IE_SET_METRIC(SUPPORTED_CONFIG_KEYS, supportConfigs, {});
ON_CALL(*core, GetMetric(_, StrEq(METRIC_KEY(SUPPORTED_CONFIG_KEYS)), _))
.WillByDefault(Return(supportConfigs));
ON_CALL(*core, GetConfig(_, StrEq(GPU_CONFIG_KEY(MAX_NUM_THREADS))))
.WillByDefault(Return(12));
}
};
TEST_P(AutoStartupFallback, releaseResource) {
// get Parameter
bool startup_fallback;
Config config;
std::tie(startup_fallback, config) = this->GetParam();
// test auto plugin
plugin->SetName("AUTO");
ON_CALL(*core, LoadNetwork(::testing::Matcher<const InferenceEngine::CNNNetwork&>(_),
::testing::Matcher<const std::string&>(_),
::testing::Matcher<const Config&>(_))).WillByDefault(Return(mockExeNetwork));
metaDevices = {{CommonTestUtils::DEVICE_CPU, {}, -1}, {CommonTestUtils::DEVICE_GPU, {}, -1}};
// DeviceInformation devInfo;
ON_CALL(*plugin, ParseMetaDevices(_, _)).WillByDefault(Return(metaDevices));
ON_CALL(*plugin, GetValidDevice)
.WillByDefault([this](const std::vector<DeviceInformation>& metaDevices, const std::string& netPrecision) {
std::list<DeviceInformation> devices(metaDevices.begin(), metaDevices.end());
return devices;
});
ON_CALL(*plugin, SelectDevice(_, _, _)).WillByDefault(Return(metaDevices[1]));
EXPECT_CALL(
*core,
LoadNetwork(::testing::Matcher<const InferenceEngine::CNNNetwork&>(_),
::testing::Matcher<const std::string&>(CommonTestUtils::DEVICE_GPU),
::testing::Matcher<const std::map<std::string, std::string>&>(_)))
.Times(1);
if (startup_fallback) {
std::map<std::string, std::string> test_map = {{"PERFORMANCE_HINT", "LATENCY"}};
EXPECT_CALL(
*core,
LoadNetwork(::testing::Matcher<const InferenceEngine::CNNNetwork&>(_),
::testing::Matcher<const std::string&>(CommonTestUtils::DEVICE_CPU),
::testing::Matcher<const std::map<std::string, std::string>&>(MapContains(test_map))))
.Times(1);
}
ASSERT_NO_THROW(plugin->LoadExeNetworkImpl(cnnNet, config));
}
const std::vector<ConfigParams> testConfigs = {ConfigParams {true, {{"ENABLE_STARTUP_FALLBACK", "YES"}}},
ConfigParams {false, {{"ENABLE_STARTUP_FALLBACK", "NO"}}}
};
INSTANTIATE_TEST_SUITE_P(smoke_Auto_StartupFallback,
AutoStartupFallback,
::testing::ValuesIn(testConfigs));