enable batching plugin through auto (#9520)
* initial commit Signed-off-by: fishbell <bell.song@intel.com> * remove debug code Signed-off-by: fishbell <bell.song@intel.com> * fix bug with CPU,GPU Signed-off-by: fishbell <bell.song@intel.com> * refine code and test case for auto batching Signed-off-by: fishbell <bell.song@intel.com> * optimize the batching query Signed-off-by: fishbell <bell.song@intel.com> * refine test case Signed-off-by: fishbell <bell.song@intel.com> * optimize if/else Signed-off-by: fishbell <bell.song@intel.com> * fix sporadic crash Signed-off-by: fishbell <bell.song@intel.com> * refine logic Signed-off-by: fishbell <bell.song@intel.com> * refine case Signed-off-by: fishbell <bell.song@intel.com> * temp solution to disable batching in auto Signed-off-by: fishbell <bell.song@intel.com> * add debug logs Signed-off-by: fishbell <bell.song@intel.com> * merge with #9724 Signed-off-by: fishbell <bell.song@intel.com> * refine the logic Signed-off-by: fishbell <bell.song@intel.com> * accurate debug message Signed-off-by: fishbell <bell.song@intel.com>
This commit is contained in:
song, bell authored and 
GitHub committed
1 parent
fe5c16ba6a
commit
2b9e726040
5 files changed
+80
-35
No files matched your search
@@ -697,21 +697,45 @@ InferenceEngine::Parameter MultiDeviceExecutableNetwork::GetMetric(const std::st
|
||||
std::unique_lock<std::mutex> lock(_confMutex);
|
||||
auto deviceInfo = _loadContext[ACTUALDEVICE].deviceInfo;
|
||||
lock.unlock();
|
||||
if (deviceInfo.deviceName.find("GPU") != std::string::npos) {
|
||||
unsigned int optimalBatchSize = 0;
|
||||
unsigned int requests = 0;
|
||||
std::map<std::string, InferenceEngine::Parameter> options;
|
||||
options["MODEL_PTR"] = std::const_pointer_cast<ngraph::Function>(_network.getFunction());
|
||||
try {
|
||||
optimalBatchSize = _core->GetMetric(deviceInfo.deviceName,
|
||||
METRIC_KEY(OPTIMAL_BATCH_SIZE), options).as<unsigned int>();
|
||||
LOG_DEBUG("[AUTOPLUGIN]BATCHING:%s:%ld", "optimal batch size", optimalBatchSize);
|
||||
} catch (...) {
|
||||
LOG_DEBUG("[AUTOPLUGIN]BATCHING:%s", "metric OPTIMAL_BATCH_SIZE not supported");
|
||||
}
|
||||
if (optimalBatchSize > 1) {
|
||||
const auto& mode = deviceInfo.config.find(CONFIG_KEY(PERFORMANCE_HINT));
|
||||
if (mode != deviceInfo.config.end() && mode->second == CONFIG_VALUE(THROUGHPUT)) {
|
||||
std::map<std::string, InferenceEngine::Parameter> options;
|
||||
options["MODEL_PTR"] = _network.getFunction(); // CNNntework
|
||||
try {
|
||||
auto optimalBatchSize = _core->GetMetric(deviceInfo.deviceName,
|
||||
METRIC_KEY(OPTIMAL_BATCH_SIZE), options).as<unsigned int>();
|
||||
auto rangeOfStreams = _core->GetMetric(deviceInfo.deviceName,
|
||||
METRIC_KEY(RANGE_FOR_STREAMS), options).as<std::tuple<unsigned int, unsigned int>>();
|
||||
real = (std::max)(real, std::get<1>(rangeOfStreams) * optimalBatchSize);
|
||||
} catch (const InferenceEngine::Exception &iie) {
|
||||
LOG_WARNING("[AUTOPLUGIN]get optimal infer requset num for GPU auto-batch failed :%s", iie.what());
|
||||
}
|
||||
try {
|
||||
// for benchmark through AUTO:CPU,GPU
|
||||
// SetConfig directly set to CPU/GPU in this case
|
||||
auto bThroughputEnabledInPlugin =
|
||||
_core->GetConfig(deviceInfo.deviceName, CONFIG_KEY(PERFORMANCE_HINT)).as<std::string>() == CONFIG_VALUE(THROUGHPUT);
|
||||
if (bThroughputEnabledInPlugin ||
|
||||
(mode != deviceInfo.config.end() && mode->second == CONFIG_VALUE(THROUGHPUT))) {
|
||||
// check if app have set preferred value
|
||||
auto res =
|
||||
_core->GetConfig(deviceInfo.deviceName, CONFIG_KEY(PERFORMANCE_HINT_NUM_REQUESTS)).as<std::string>();
|
||||
requests = PerfHintsConfig::CheckPerformanceHintRequestValue(res);
|
||||
const auto& reqs = deviceInfo.config.find(CONFIG_KEY(PERFORMANCE_HINT_NUM_REQUESTS));
|
||||
if (reqs != deviceInfo.config.end())
|
||||
requests = static_cast<unsigned int>(PerfHintsConfig::CheckPerformanceHintRequestValue(reqs->second));
|
||||
LOG_DEBUG("[AUTOPLUGIN]BATCHING:%s:%ld", "user requested size", requests);
|
||||
if (!requests) { // no limitations from user
|
||||
auto rangeOfStreams = _core->GetMetric(deviceInfo.deviceName,
|
||||
METRIC_KEY(RANGE_FOR_STREAMS), options).as<std::tuple<unsigned int, unsigned int>>();
|
||||
requests = optimalBatchSize * std::get<1>(rangeOfStreams);
|
||||
LOG_DEBUG("[AUTOPLUGIN]BATCHING:%s:%ld", "deduced size:", requests);
|
||||
}
|
||||
}
|
||||
} catch (const InferenceEngine::Exception &iie) {
|
||||
LOG_WARNING("[AUTOPLUGIN]get optimal infer requset num for auto-batch failed :%s", iie.what());
|
||||
}
|
||||
real = (std::max)(real, (std::max)(requests, optimalBatchSize));
|
||||
}
|
||||
}
|
||||
unsigned int res = (std::max)(8u, real);
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#include "threading/ie_itask_executor.hpp"
|
||||
#include "threading/ie_executor_manager.hpp"
|
||||
#include "ie_icore.hpp"
|
||||
#include <ie_performance_hints.hpp>
|
||||
|
||||
#ifdef MULTIUNITTEST
|
||||
#define MOCKTESTMACRO virtual
|
||||
|
||||
@@ -294,7 +294,6 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
|
||||
strDevices += iter->deviceName;
|
||||
strDevices += ((iter + 1) == supportDevices.end()) ? "" : ",";
|
||||
}
|
||||
|
||||
return std::make_shared<MultiDeviceExecutableNetwork>(modelPath, network, supportDevices, strDevices, this, context, context.needPerfCounters);
|
||||
}
|
||||
OV_ITT_SCOPED_TASK(itt::domains::MULTIPlugin, "MultiDeviceInferencePlugin::LoadNetworkImpl:MultiMode");
|
||||
|
||||
@@ -31,6 +31,7 @@ using ::testing::Eq;
|
||||
using ::testing::ReturnRef;
|
||||
using ::testing::AtLeast;
|
||||
using ::testing::AnyNumber;
|
||||
using ::testing::ContainsRegex;
|
||||
using Config = std::map<std::string, std::string>;
|
||||
using namespace MockMultiDevice;
|
||||
|
||||
@@ -148,6 +149,14 @@ public:
|
||||
IE_SET_METRIC(OPTIMAL_NUMBER_OF_INFER_REQUESTS, optimalNum, 2);
|
||||
ON_CALL(*mockIExeNet.get(), GetMetric(StrEq(METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS))))
|
||||
.WillByDefault(Return(optimalNum));
|
||||
IE_SET_METRIC(OPTIMAL_BATCH_SIZE, optimalBatchSize, 8);
|
||||
ON_CALL(*core, GetMetric(_, StrEq(METRIC_KEY(OPTIMAL_BATCH_SIZE)), _))
|
||||
.WillByDefault(Return(optimalBatchSize));
|
||||
EXPECT_CALL(*core, GetMetric(_, StrEq(METRIC_KEY(OPTIMAL_BATCH_SIZE)), _)).Times(AnyNumber());
|
||||
IE_SET_METRIC(RANGE_FOR_STREAMS, rangeStreamsSize, {1u, 2u});
|
||||
ON_CALL(*core, GetMetric(_, StrEq(METRIC_KEY(RANGE_FOR_STREAMS)), _))
|
||||
.WillByDefault(Return(rangeStreamsSize));
|
||||
EXPECT_CALL(*core, GetMetric(_, StrEq(METRIC_KEY(RANGE_FOR_STREAMS)), _)).Times(AnyNumber());
|
||||
IE_SET_METRIC(SUPPORTED_CONFIG_KEYS, supportConfigs, {});
|
||||
ON_CALL(*core, GetMetric(_, StrEq(METRIC_KEY(SUPPORTED_CONFIG_KEYS)), _))
|
||||
.WillByDefault(Return(supportConfigs));
|
||||
|
||||
@@ -34,6 +34,7 @@ using ::testing::ReturnRef;
|
||||
using ::testing::AtLeast;
|
||||
using ::testing::AnyNumber;
|
||||
using ::testing::InvokeWithoutArgs;
|
||||
using ::testing::ContainsRegex;
|
||||
using Config = std::map<std::string, std::string>;
|
||||
using namespace MockMultiDevice;
|
||||
|
||||
@@ -45,7 +46,8 @@ using ConfigParams = std::tuple<
|
||||
unsigned int, // gpu OPTIMAL_NUMBER_OF_INFER_REQUESTS
|
||||
int, // gpu infer requet num of customer want
|
||||
bool, // if gpu sleep, cpu device will load slow
|
||||
unsigned int // expect OPTIMAL_NUMBER_OF_INFER_REQUESTS
|
||||
unsigned int, // expect OPTIMAL_NUMBER_OF_INFER_REQUESTS
|
||||
int // gpu PERFORMANCE_HINT_NUM_REQUESTS
|
||||
>;
|
||||
class ExecNetworkGetMetric : public ::testing::TestWithParam<ConfigParams> {
|
||||
public:
|
||||
@@ -79,8 +81,9 @@ public:
|
||||
bool cpuSleep;
|
||||
bool gpuSleep;
|
||||
bool isThroughput;
|
||||
int gpuPerfHintNum;
|
||||
std::tie(isThroughput, cpuOptimalNum, cpuCustomerNum, cpuSleep,
|
||||
gpuOptimalNum, gpuCustomerNum, gpuSleep, expectOptimalNum) = obj.param;
|
||||
gpuOptimalNum, gpuCustomerNum, gpuSleep, expectOptimalNum, gpuPerfHintNum) = obj.param;
|
||||
std::ostringstream result;
|
||||
result << "cpuOptimalNum_" << cpuOptimalNum << "cpuCustomerNum_" << cpuCustomerNum;
|
||||
result << "gpuOptimalNum_" << gpuOptimalNum << "gpuCustomerNum_" << gpuCustomerNum;
|
||||
@@ -90,6 +93,7 @@ public:
|
||||
} else {
|
||||
result << "__isThroughput" << "false";
|
||||
}
|
||||
result << "_gpuPerfHintNum_" << gpuPerfHintNum;
|
||||
if (cpuSleep) {
|
||||
result << "_cpuSleep_" << "true";
|
||||
} else {
|
||||
@@ -137,7 +141,6 @@ public:
|
||||
// remove annoying ON CALL message
|
||||
EXPECT_CALL(*gpuMockIPluginPtr, LoadNetwork(MatcherCast<const CNNNetwork&>(_), _)).Times(1);
|
||||
gpuMockExeNetwork = gpuMockPlugin.LoadNetwork(CNNNetwork{}, {});
|
||||
|
||||
// prepare mockicore and cnnNetwork for loading
|
||||
core = std::shared_ptr<MockICore>(new MockICore());
|
||||
auto* origin_plugin = new MockMultiDeviceInferencePlugin();
|
||||
@@ -160,10 +163,6 @@ public:
|
||||
config.insert({CONFIG_KEY_INTERNAL(MULTI_WORK_MODE_AS_AUTO), InferenceEngine::PluginConfigParams::YES});
|
||||
config.insert({InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES,
|
||||
CommonTestUtils::DEVICE_CPU + std::string(",") + CommonTestUtils::DEVICE_GPU});
|
||||
|
||||
ON_CALL(*(HLogger), print(_)).WillByDefault([&](std::stringstream& stream) {
|
||||
std::cout << stream.str() << std::endl;
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
@@ -176,8 +175,9 @@ TEST_P(ExecNetworkGetMetric, OPTIMAL_NUMBER_OF_INFER_REQUESTS) {
|
||||
bool cpuSleep;
|
||||
bool gpuSleep;
|
||||
bool isThroughput;
|
||||
int gpuPerfHintNum;
|
||||
std::tie(isThroughput, cpuOptimalNum, cpuCustomerNum, cpuSleep,
|
||||
gpuOptimalNum, gpuCustomerNum, gpuSleep, expectOptimalNum) = this->GetParam();
|
||||
gpuOptimalNum, gpuCustomerNum, gpuSleep, expectOptimalNum, gpuPerfHintNum) = this->GetParam();
|
||||
if (isThroughput) {
|
||||
metaDevices.push_back({CommonTestUtils::DEVICE_CPU, {{CONFIG_KEY(PERFORMANCE_HINT),
|
||||
InferenceEngine::PluginConfigParams::THROUGHPUT}}, cpuCustomerNum, ""});
|
||||
@@ -189,9 +189,18 @@ TEST_P(ExecNetworkGetMetric, OPTIMAL_NUMBER_OF_INFER_REQUESTS) {
|
||||
.WillByDefault(RETURN_MOCK_VALUE(optimalBatchNum));
|
||||
ON_CALL(*core.get(), GetMetric(StrEq(CommonTestUtils::DEVICE_GPU), StrEq(METRIC_KEY(RANGE_FOR_STREAMS)), _))
|
||||
.WillByDefault(RETURN_MOCK_VALUE(rangeOfStreams));
|
||||
ON_CALL(*core.get(), GetConfig(StrEq(CommonTestUtils::DEVICE_GPU), StrEq(CONFIG_KEY(PERFORMANCE_HINT))))
|
||||
.WillByDefault(Return(CONFIG_VALUE(THROUGHPUT)));
|
||||
EXPECT_CALL(*core.get(), GetConfig(StrEq(CommonTestUtils::DEVICE_GPU), StrEq(CONFIG_KEY(PERFORMANCE_HINT)))).Times(AnyNumber());
|
||||
ON_CALL(*core.get(), GetConfig(StrEq(CommonTestUtils::DEVICE_GPU), StrEq(CONFIG_KEY(PERFORMANCE_HINT_NUM_REQUESTS))))
|
||||
.WillByDefault(Return(std::to_string(gpuPerfHintNum)));
|
||||
EXPECT_CALL(*core.get(), GetConfig(StrEq(CommonTestUtils::DEVICE_GPU), StrEq(CONFIG_KEY(PERFORMANCE_HINT_NUM_REQUESTS)))).Times(AnyNumber());
|
||||
} else {
|
||||
metaDevices.push_back({CommonTestUtils::DEVICE_CPU, {}, cpuCustomerNum, ""});
|
||||
metaDevices.push_back({CommonTestUtils::DEVICE_GPU, {}, gpuCustomerNum, ""});
|
||||
ON_CALL(*core.get(), GetConfig(StrEq(CommonTestUtils::DEVICE_GPU), StrEq(CONFIG_KEY(PERFORMANCE_HINT))))
|
||||
.WillByDefault(Return(CONFIG_VALUE(LATENCY)));
|
||||
EXPECT_CALL(*core.get(), GetConfig(StrEq(CommonTestUtils::DEVICE_GPU), StrEq(CONFIG_KEY(PERFORMANCE_HINT)))).Times(AnyNumber());
|
||||
}
|
||||
ON_CALL(*plugin, SelectDevice(_, _, _)).WillByDefault(Return(metaDevices[1]));
|
||||
ON_CALL(*plugin, ParseMetaDevices(_, _)).WillByDefault(Return(metaDevices));
|
||||
@@ -270,19 +279,22 @@ TEST_P(ExecNetworkGetMetric, OPTIMAL_NUMBER_OF_INFER_REQUESTS) {
|
||||
// expectOptimalNum of Auto ExecNetwork}
|
||||
//
|
||||
const std::vector<ConfigParams> testConfigs = {
|
||||
ConfigParams {false, 1, -1, false, 2, -1, true, 8},
|
||||
ConfigParams {false, 1, -1, false, 10, -1, true, 8},
|
||||
ConfigParams {false, 12, -1, false, 2, -1, true, 12},
|
||||
ConfigParams {false, 12, -1, false, 10, -1, true, 12},
|
||||
ConfigParams {false, 1, -1, true, 2, -1, false, 8},
|
||||
ConfigParams {false, 1, -1, true, 10, -1, false, 10},
|
||||
ConfigParams {false, 6, -1, true, 2, -1, false, 8},
|
||||
ConfigParams {false, 6, -1, true, 10, -1, false, 10},
|
||||
ConfigParams {false, 6, 4, false, 2, 3, true, 8},
|
||||
ConfigParams {false, 6, 4, false, 10, 3, true, 8},
|
||||
ConfigParams {false, 1, 4, true, 2, 3, false, 8},
|
||||
ConfigParams {false, 1, 4, true, 10, 3, false, 10},
|
||||
ConfigParams {true, 1, 4, false, 10, 3, true, 512}
|
||||
ConfigParams {false, 1, -1, false, 2, -1, true, 8, 0},
|
||||
ConfigParams {false, 1, -1, false, 10, -1, true, 8, 0},
|
||||
ConfigParams {false, 12, -1, false, 2, -1, true, 12, 0},
|
||||
ConfigParams {false, 12, -1, false, 10, -1, true, 12, 0},
|
||||
ConfigParams {false, 1, -1, true, 2, -1, false, 8, 0},
|
||||
ConfigParams {false, 1, -1, true, 10, -1, false, 10, 0},
|
||||
ConfigParams {false, 6, -1, true, 2, -1, false, 8, 0},
|
||||
ConfigParams {false, 6, -1, true, 10, -1, false, 10, 0},
|
||||
ConfigParams {false, 6, 4, false, 2, 3, true, 8, 0},
|
||||
ConfigParams {false, 6, 4, false, 10, 3, true, 8, 0},
|
||||
ConfigParams {false, 1, 4, true, 2, 3, false, 8, 0},
|
||||
ConfigParams {false, 1, 4, true, 10, 3, false, 10, 0},
|
||||
ConfigParams {true, 1, 4, false, 10, 3, true, 512, 0},
|
||||
ConfigParams {true, 1, 4, false, 10, 3, true, 256, -1},
|
||||
ConfigParams {true, 1, 4, false, 10, 3, true, 512, 512},
|
||||
ConfigParams {true, 1, 4, false, 10, 3, true, 256, 256},
|
||||
};
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, ExecNetworkGetMetric,
|
||||
|
||||
Reference in new issue
Block a user