IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

AUTO_BATCH_TIMEOUT and tests

Browse Source
This commit is contained in:
myshevts 2021-12-09 19:40:49 +03:00
parent 88ba3de4d7
commit 16e4881e5f
10 changed files with 193 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
inference-engine/tests/functional/plugin/gpu/shared_tests_instances/auto_batching/auto_batching_tests.cpp
View File

@ -3,10 +3,10 @@
//
#include <auto_batching/auto_batching_tests.hpp>
const std::vector<size_t> num_streams{ 1, 2 };
const std::vector<size_t> num_streams{ 2 };
const std::vector<bool> get_vs_set{ true, false };
const std::vector<size_t> num_requests{ 1, 8, 16, 64 };
const std::vector<size_t> num_batch{ 1, 4, 8, 16, 32, 64, 128, 256 };
const std::vector<size_t> num_batch{ 1, 8, 32, 256 };
using namespace AutoBatchingTests;
namespace AutoBatchingTests {

11
inference-engine/tests/functional/plugin/gpu/shared_tests_instances/behavior/executable_network/exec_net_base.cpp
View File

@ -52,6 +52,10 @@ const std::vector<std::map<std::string, std::string>> autoConfig = {
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_GPU}},
};
const std::vector<std::map<std::string, std::string>> autoBatchConfig = {
{{CONFIG_KEY(AUTO_BATCH) , CommonTestUtils::DEVICE_GPU}},
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, ExecNetSetPrecision,
::testing::Combine(
::testing::ValuesIn(netPrecisions),
@ -72,4 +76,11 @@ INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, ExecNetSetPrecision,
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(autoConfig)),
ExecNetSetPrecision::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_AutoBatch_BehaviorTests, ExecNetSetPrecision,
::testing::Combine(
::testing::ValuesIn(netPrecisions),
::testing::Values(CommonTestUtils::DEVICE_BATCH),
::testing::ValuesIn(autoBatchConfig)),
ExecNetSetPrecision::getTestCaseName);
} // namespace

10
inference-engine/tests/functional/plugin/gpu/shared_tests_instances/behavior/executable_network/get_metric.cpp
View File

@ -22,27 +22,27 @@ namespace {
INSTANTIATE_TEST_SUITE_P(
nightly_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_OPTIMAL_NUMBER_OF_INFER_REQUESTS,
::testing::Values("GPU", "MULTI:GPU", "HETERO:GPU", "AUTO:GPU,CPU")
::testing::Values("GPU", "MULTI:GPU", "HETERO:GPU", "AUTO:GPU,CPU", "BATCH:GPU")
);
INSTANTIATE_TEST_SUITE_P(
nightly_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_SUPPORTED_CONFIG_KEYS,
::testing::Values("GPU", "MULTI:GPU", "HETERO:GPU", "AUTO:GPU,CPU")
::testing::Values("GPU", "MULTI:GPU", "HETERO:GPU", "AUTO:GPU,CPU", "BATCH:GPU")
);
INSTANTIATE_TEST_SUITE_P(
nightly_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_SUPPORTED_METRICS,
::testing::Values("GPU", "MULTI:GPU", "HETERO:GPU", "AUTO:GPU,CPU")
::testing::Values("GPU", "MULTI:GPU", "HETERO:GPU", "AUTO:GPU,CPU", "BATCH:GPU")
);
INSTANTIATE_TEST_SUITE_P(
nightly_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_NETWORK_NAME,
::testing::Values("GPU", "MULTI:GPU", "HETERO:GPU", "AUTO:GPU,CPU")
::testing::Values("GPU", "MULTI:GPU", "HETERO:GPU", "AUTO:GPU,CPU", "BATCH:GPU")
);
INSTANTIATE_TEST_SUITE_P(
nightly_IEClassExecutableNetworkGetMetricTest, IEClassExecutableNetworkGetMetricTest_ThrowsUnsupported,
::testing::Values("GPU", "MULTI:GPU", "HETERO:GPU", "AUTO:GPU,CPU")
::testing::Values("GPU", "MULTI:GPU", "HETERO:GPU", "AUTO:GPU,CPU", "BATCH:GPU")
);
//

10
inference-engine/tests/functional/plugin/gpu/shared_tests_instances/behavior/ov_plugin/core_integration.cpp
View File

@ -30,11 +30,11 @@ INSTANTIATE_TEST_SUITE_P(nightly_OVClassNetworkTestP, OVClassNetworkTestP, ::tes
INSTANTIATE_TEST_SUITE_P(nightly_OVClassGetMetricTest,
OVClassGetMetricTest_SUPPORTED_CONFIG_KEYS,
::testing::Values("GPU", "MULTI", "HETERO", "AUTO"));
::testing::Values("GPU", "MULTI", "HETERO", "AUTO", "BATCH"));
INSTANTIATE_TEST_SUITE_P(nightly_OVClassGetMetricTest,
OVClassGetMetricTest_SUPPORTED_METRICS,
::testing::Values("GPU", "MULTI", "HETERO", "AUTO"));
::testing::Values("GPU", "MULTI", "HETERO", "AUTO", "BATCH"));
INSTANTIATE_TEST_SUITE_P(nightly_OVClassGetMetricTest,
OVClassGetMetricTest_AVAILABLE_DEVICES,
@ -42,7 +42,7 @@ INSTANTIATE_TEST_SUITE_P(nightly_OVClassGetMetricTest,
INSTANTIATE_TEST_SUITE_P(nightly_OVClassGetMetricTest,
OVClassGetMetricTest_FULL_DEVICE_NAME,
::testing::Values("GPU", "MULTI", "HETERO", "AUTO"));
::testing::Values("GPU", "MULTI", "HETERO", "AUTO", "BATCH"));
INSTANTIATE_TEST_SUITE_P(nightly_OVClassGetMetricTest,
OVClassGetMetricTest_OPTIMIZATION_CAPABILITIES,
@ -62,11 +62,11 @@ INSTANTIATE_TEST_SUITE_P(nightly_OVClassGetMetricTest,
INSTANTIATE_TEST_SUITE_P(nightly_OVClassGetMetricTest,
OVClassGetMetricTest_ThrowUnsupported,
::testing::Values("GPU", "MULTI", "HETERO", "AUTO"));
::testing::Values("GPU", "MULTI", "HETERO", "AUTO", "BATCH"));
INSTANTIATE_TEST_SUITE_P(nightly_OVClassGetConfigTest,
OVClassGetConfigTest_ThrowUnsupported,
::testing::Values("GPU", "MULTI", "HETERO", "AUTO"));
::testing::Values("GPU", "MULTI", "HETERO", "AUTO", "BATCH"));
INSTANTIATE_TEST_SUITE_P(nightly_OVClassGetAvailableDevices, OVClassGetAvailableDevices, ::testing::Values("GPU"));

66
inference-engine/tests/functional/plugin/gpu/shared_tests_instances/behavior/plugin/config.cpp
View File

@ -91,6 +91,31 @@ namespace {
CommonTestUtils::DEVICE_GPU + std::string(",") + CommonTestUtils::DEVICE_CPU},
{InferenceEngine::PluginConfigParams::KEY_LOG_LEVEL, "NAN"}}
};
const std::vector<std::map<std::string, std::string>> auto_batch_inconfigs = {
{{CONFIG_KEY(AUTO_BATCH), CommonTestUtils::DEVICE_GPU},
{CONFIG_KEY(AUTO_BATCH_TIMEOUT), "-1"}},
{{CONFIG_KEY(AUTO_BATCH), CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, "DOESN'T EXIST"}},
{{CONFIG_KEY(AUTO_BATCH), CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, "DOESN'T EXIST"}},
{{CONFIG_KEY(AUTO_BATCH) , CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "-1"}},
{{CONFIG_KEY(AUTO_BATCH) , CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, "ON"}},
{{CONFIG_KEY(AUTO_BATCH) , CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_CONFIG_FILE, "unknown_file"}},
{{CONFIG_KEY(AUTO_BATCH) , CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_DUMP_KERNELS, "ON"}},
{{CONFIG_KEY(AUTO_BATCH) , CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_TUNING_MODE, "TUNING_UNKNOWN_MODE"}},
{{CONFIG_KEY(AUTO_BATCH) , CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_DEVICE_ID, "DEVICE_UNKNOWN"}},
};
IE_SUPPRESS_DEPRECATED_END
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, IncorrectConfigTests,
@ -112,6 +137,12 @@ namespace {
IncorrectConfigTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_AutoBatch_BehaviorTests, IncorrectConfigTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_BATCH),
::testing::ValuesIn(auto_batch_inconfigs)),
IncorrectConfigTests::getTestCaseName);
const std::vector<std::map<std::string, std::string>> conf = {
{}
};
@ -154,17 +185,6 @@ namespace {
};
IE_SUPPRESS_DEPRECATED_END
const std::vector<std::map<std::string, std::string>> multiconf = {
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_GPU}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "1"}}
};
const std::vector<std::map<std::string, std::string>> autoConfigs = {
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_GPU}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_GPU},
@ -204,6 +224,19 @@ namespace {
{InferenceEngine::PluginConfigParams::KEY_LOG_LEVEL, InferenceEngine::PluginConfigParams::LOG_TRACE}}
};
const std::vector<std::map<std::string, std::string>> auto_batch_configs = {
{{CONFIG_KEY(AUTO_BATCH) , CommonTestUtils::DEVICE_GPU}},
{{CONFIG_KEY(AUTO_BATCH) , CommonTestUtils::DEVICE_GPU},
{CONFIG_KEY(AUTO_BATCH_TIMEOUT) , "1"}},
{{CONFIG_KEY(AUTO_BATCH) , CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT}},
{{CONFIG_KEY(AUTO_BATCH) , CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY}},
{{CONFIG_KEY(AUTO_BATCH) , CommonTestUtils::DEVICE_GPU},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "1"}},
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, DefaultValuesConfigTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_GPU),
@ -229,4 +262,15 @@ namespace {
IncorrectConfigAPITests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_AutoBatch_BehaviorTests, IncorrectConfigAPITests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_BATCH),
::testing::ValuesIn(auto_batch_inconfigs)),
IncorrectConfigAPITests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_AutoBatch_BehaviorTests, CorrectConfigTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_BATCH),
::testing::ValuesIn(auto_batch_configs)),
CorrectConfigTests::getTestCaseName);
} // namespace

2
inference-engine/tests/functional/plugin/shared/include/auto_batching/auto_batching_tests.hpp
View File

@ -74,6 +74,8 @@ protected:
config[CONFIG_KEY(GPU_THROUGHPUT_STREAMS)] = std::to_string(num_streams);
if (device_name.find("CPU") != std::string::npos)
config[CONFIG_KEY(CPU_THROUGHPUT_STREAMS)] = std::to_string(num_streams);
// minimize timeout to reduce test time
config[CONFIG_KEY(AUTO_BATCH_TIMEOUT)] = std::to_string(1);
auto exec_net_ref = ie.LoadNetwork(net, std::string(CommonTestUtils::DEVICE_BATCH) + ":" +
device_name + "(" + std::to_string(num_batch) + ")",
config);

4
src/inference/include/ie/ie_plugin_config.hpp
View File

@ -256,11 +256,11 @@ DECLARE_CONFIG_VALUE(YES);
DECLARE_CONFIG_VALUE(NO);
/**
* @brief Auto-batching configuration: device + batch size
* @brief Auto-batching configuration, string for the device + batch size, e.g. "GPU(4)"
*/
DECLARE_CONFIG_KEY(AUTO_BATCH);
/**
* @brief Auto-batching configuration: timeout
* @brief Auto-batching configuration: string with timeout (in ms), e.g. "100"
*/
DECLARE_CONFIG_KEY(AUTO_BATCH_TIMEOUT);

37
src/inference/src/ie_core.cpp
View File

@ -534,18 +534,18 @@ public:
// do not reshape/re-batch originally batched networks!
const InputsDataMap inputInfo = network.getInputsInfo();
ICNNNetwork::InputShapes shapes = network.getInputShapes();
for (const InputsDataMap::value_type &item : inputInfo) {
for (const InputsDataMap::value_type& item : inputInfo) {
auto layout = item.second->getTensorDesc().getLayout();
if (layout == InferenceEngine::Layout::NC || layout == InferenceEngine::Layout::NCDHW
|| layout == InferenceEngine::Layout::NCHW || layout == InferenceEngine::Layout::NHWC
|| layout == InferenceEngine::Layout::NDHWC) {
if (1 != shapes[item.first][0]) // do not reshape/re-batch originally batched networks
if (layout == InferenceEngine::Layout::NC || layout == InferenceEngine::Layout::NCDHW ||
layout == InferenceEngine::Layout::NCHW || layout == InferenceEngine::Layout::NHWC ||
layout == InferenceEngine::Layout::NDHWC) {
if (1 != shapes[item.first][0]) // do not reshape/re-batch originally batched networks
IE_THROW(NotImplemented)
<< "Auto-batching does not reshape/re-batch originally batched networks!";
<< "Auto-batching does not reshape/re-batch originally batched networks!";
}
auto deviceNameWithoutBatch =
!deviceNameWithBatchSize.empty() ? DeviceIDParser::getBatchDevice(deviceNameWithBatchSize)
: deviceName;
auto deviceNameWithoutBatch = !deviceNameWithBatchSize.empty()
? DeviceIDParser::getBatchDevice(deviceNameWithBatchSize)
: deviceName;
unsigned int requests = 0;
unsigned int optimalBatchSize = 0;
if (deviceNameWithBatchSize.empty()) {
@ -554,15 +554,14 @@ public:
std::map<std::string, ie::Parameter> options;
options["MODEL_PTR"] = &network;
optimalBatchSize = GetCPPPluginByName(DeviceIDParser(deviceNameWithoutBatch).getDeviceName())
.get_metric(METRIC_KEY(OPTIMAL_BATCH), options)
.as<unsigned int>();
.get_metric(METRIC_KEY(OPTIMAL_BATCH), options)
.as<unsigned int>();
auto res =
GetConfig(deviceNameWithoutBatch,
CONFIG_KEY(PERFORMANCE_HINT_NUM_REQUESTS)).as<std::string>();
GetConfig(deviceNameWithoutBatch, CONFIG_KEY(PERFORMANCE_HINT_NUM_REQUESTS)).as<std::string>();
requests = PerfHintsConfig::CheckPerformanceHintRequestValue(res);
const auto &reqs = config_with_batch.find(CONFIG_KEY(PERFORMANCE_HINT_NUM_REQUESTS));
const auto& reqs = config_with_batch.find(CONFIG_KEY(PERFORMANCE_HINT_NUM_REQUESTS));
if (reqs != config_with_batch.end())
requests = (unsigned int) PerfHintsConfig::CheckPerformanceHintRequestValue(reqs->second);
requests = (unsigned int)PerfHintsConfig::CheckPerformanceHintRequestValue(reqs->second);
if (requests) {
std::cout << "!!!!!!!!!!!!!!!Detected reqs_limitation: " << requests << std::endl;
optimalBatchSize = std::max(1u, std::min(requests, optimalBatchSize));
@ -570,8 +569,8 @@ public:
}
auto function = network.getFunction();
bool bDetectionOutput = false;
for (auto &&node : function->get_ops()) {
auto isDetectionOutputParent = [](decltype(node) &nd) {
for (auto&& node : function->get_ops()) {
auto isDetectionOutputParent = [](decltype(node)& nd) {
for (size_t n = 0; n < nd->get_input_size(); n++) {
if (!std::strcmp("DetectionOutput", nd->get_input_node_ptr(n)->get_type_info().name))
return true;
@ -591,8 +590,8 @@ public:
}
if (optimalBatchSize > 1 || !deviceNameWithBatchSize.empty()) {
auto batchConfig = deviceNameWithBatchSize.empty()
? deviceNameWithoutBatch + "(" + std::to_string(optimalBatchSize) + ")"
: deviceNameWithBatchSize;
? deviceNameWithoutBatch + "(" + std::to_string(optimalBatchSize) + ")"
: deviceNameWithBatchSize;
if (bDetectionOutput) {
deviceName = "HETERO:BATCH," + deviceNameWithoutBatch;
std::cout << "HETERO code path!!!!" << std::endl;

123
src/plugins/auto_batch/auto_batch.cpp
View File

@ -23,6 +23,8 @@
namespace AutoBatchPlugin {
using namespace InferenceEngine;
std::vector<std::string> supported_configKeys = { CONFIG_KEY(AUTO_BATCH), CONFIG_KEY(AUTO_BATCH_TIMEOUT) };
template <Precision::ePrecision precision>
Blob::Ptr create_shared_blob_on_top_of_batched_blob(Blob::Ptr batched_blob, size_t batch_id, size_t batch_num) {
typedef typename PrecisionTrait<precision>::value_type TYPE;
@ -250,6 +252,9 @@ AutoBatchExecutableNetwork::AutoBatchExecutableNetwork(
_needPerfCounters{needPerfCounters} {
// WA for gcc 4.8 ( fails compilation with member init-list)
_device = networkDevice;
auto time_out = config.find(CONFIG_KEY(AUTO_BATCH_TIMEOUT));
if (time_out != config.end())
_timeOut = ParseTimeoutValue(time_out->second.as<std::string>());
}
AutoBatchExecutableNetwork::~AutoBatchExecutableNetwork() {
@ -260,6 +265,14 @@ AutoBatchExecutableNetwork::~AutoBatchExecutableNetwork() {
_workerRequests.clear();
}
unsigned int AutoBatchExecutableNetwork::ParseTimeoutValue(const std::string& s) {
auto val = std::stoi(s);
if (val < 0)
IE_THROW(ParameterMismatch) << "Value for the " << CONFIG_KEY(AUTO_BATCH_TIMEOUT) << " should be unsigned int";
std::cout << "AutoBatching timeout is set to " << s << " ms" << std::endl;
return val;
}
std::shared_ptr<InferenceEngine::RemoteContext> AutoBatchExecutableNetwork::GetContext() const {
return _network->GetContext();
}
@ -289,7 +302,7 @@ InferenceEngine::IInferRequestInternal::Ptr AutoBatchExecutableNetwork::CreateIn
workerRequestPtr->_thread = std::thread([workerRequestPtr, this] {
while (1) {
std::unique_lock<std::mutex> lock(workerRequestPtr->_mutex);
auto status = workerRequestPtr->_cond.wait_for(lock, std::chrono::milliseconds(1000));
auto status = workerRequestPtr->_cond.wait_for(lock, std::chrono::milliseconds(_timeOut));
// as we pop the tasks from the queue only here
// it is ok to call unsafe_size (as the _tasks can only grow in parallel)
const int sz = workerRequestPtr->_tasks.unsafe_size();
@ -353,15 +366,27 @@ InferenceEngine::IInferRequestInternal::Ptr AutoBatchExecutableNetwork::CreateIn
}
void AutoBatchExecutableNetwork::SetConfig(const std::map<std::string, InferenceEngine::Parameter> &config) {
// TODO
IE_THROW(NotImplemented);
auto timeout = config.find(CONFIG_KEY(AUTO_BATCH_TIMEOUT));
if (timeout == config.end() || config.size() > 1) {
IE_THROW() << "The only config that can be changed on the fly for the AutoBatching the is the " <<
CONFIG_KEY(AUTO_BATCH_TIMEOUT);
} else {
_timeOut = ParseTimeoutValue(timeout->second.as<std::string>());
}
}
InferenceEngine::Parameter AutoBatchExecutableNetwork::GetConfig(const std::string &name) const {
auto res = _config.find(name);
if (res != _config.end()) {
return res->second;
auto it = _config.find(name);
if (it != _config.end()) {
return it->second;
} else {
// find config key among networks config keys
auto param = _network->GetMetric(METRIC_KEY(SUPPORTED_CONFIG_KEYS));
for (auto &&configKey : param.as<std::vector<std::string>>()) {
if (configKey == name) {
return _network->GetConfig(configKey);
}
}
IE_THROW(NotFound) << name <<" not found in the ExecutableNetwork config";
}
}
@ -391,8 +416,7 @@ InferenceEngine::Parameter AutoBatchExecutableNetwork::GetMetric(const std::stri
METRIC_KEY(SUPPORTED_CONFIG_KEYS)
});
} else if (name == METRIC_KEY(SUPPORTED_CONFIG_KEYS)) {
std::vector<std::string> configKeys = { CONFIG_KEY(AUTO_BATCH) };
IE_SET_METRIC_RETURN(SUPPORTED_CONFIG_KEYS, configKeys);
IE_SET_METRIC_RETURN(SUPPORTED_CONFIG_KEYS, {CONFIG_KEY(AUTO_BATCH_TIMEOUT)}); // only timeout can be changed on the fly
} else {
IE_THROW() <<"Unsupported Network metric: " << name;
}
@ -425,9 +449,26 @@ std::map<std::string, std::string> AutoBatchInferencePlugin::GetSupportedConfig(
return supportedConfig;
}
DeviceInformation AutoBatchInferencePlugin::ParseBatchDevice(const std::string& deviceWithBatch ) {
auto && d = deviceWithBatch;
auto openingBracket = d.find_first_of('(');
auto closingBracket = d.find_first_of(')', openingBracket);
auto deviceName = d.substr(0, openingBracket);
int batch = 1;
if (closingBracket != std::string::npos && openingBracket < closingBracket) {
batch = std::stol(d.substr(openingBracket + 1, closingBracket - 1));
if (batch <= 0) {
IE_THROW() <<"Batch value for '" << deviceName << "' must be > 0, while " << batch
<< "is passed";
}
}
return { deviceName, {{}}, batch };
}
DeviceInformation AutoBatchInferencePlugin::ParseMetaDevice(const std::string& devicesBatchCfg,
const std::map<std::string, std::string> & config) const {
DeviceInformation metaDevice;
auto getDeviceConfig = [&] (const DeviceName & deviceWithID) {
DeviceIDParser deviceParser(deviceWithID);
std::string deviceName = deviceParser.getDeviceName();
@ -442,26 +483,8 @@ DeviceInformation AutoBatchInferencePlugin::ParseMetaDevice(const std::string& d
return GetSupportedConfig(tconfig, deviceName);
};
auto && d = devicesBatchCfg;
{
auto openingBracket = d.find_first_of('(');
auto closingBracket = d.find_first_of(')', openingBracket);
auto deviceName = d.substr(0, openingBracket);
int batch = 1;
if (closingBracket != std::string::npos && openingBracket < closingBracket) {
batch = std::stol(d.substr(openingBracket + 1, closingBracket - 1));
if (batch <= 0) {
IE_THROW() <<"Batch value for '" << deviceName << "' must be > 0, while " << batch
<< "is passed";
}
}
// create meta device
auto cfg = getDeviceConfig(deviceName);
metaDevice = { deviceName, cfg, batch };
}
auto metaDevice = ParseBatchDevice(devicesBatchCfg);
metaDevice.config = getDeviceConfig(metaDevice.deviceName);
return metaDevice;
}
@ -481,19 +504,39 @@ RemoteContext::Ptr AutoBatchInferencePlugin::CreateContext(const InferenceEngine
Parameter AutoBatchInferencePlugin::GetConfig(const std::string& name,
const std::map<std::string, Parameter> & options) const {
if (name == CONFIG_KEY(AUTO_BATCH)) {
auto it = _config.find(CONFIG_KEY(AUTO_BATCH));
if (supported_configKeys.end() != std::find(supported_configKeys.begin(), supported_configKeys.end(), name)) {
auto it = _config.find(name);
if (it == _config.end()) {
IE_THROW() <<"Value for KEY_AUTO_BATCH is not set";
IE_THROW() << "Value for " << name << " is not set";
} else {
return { it->second };
}
} else {
IE_THROW() <<"Unsupported config key: " << name;
IE_THROW() << "Unsupported config key: " << name;
}
}
void AutoBatchInferencePlugin::SetConfig(const std::map<std::string, std::string> & config) {
void AutoBatchInferencePlugin::CheckConfig(const std::map<std::string, std::string>& config) {
for (auto &&kvp : config) {
const auto name = kvp.first;
const auto val = kvp.second;
if (supported_configKeys.end() == std::find(supported_configKeys.begin(), supported_configKeys.end(), name))
IE_THROW() << "Unsupported config key: " << name;
if (name == CONFIG_KEY(AUTO_BATCH)) {
ParseBatchDevice(val);
} else if (name == CONFIG_KEY(AUTO_BATCH_TIMEOUT)) {
try {
std::stoi(val);
} catch (const std::exception& e) {
IE_THROW(ParameterMismatch) << " Expecting unsigned int value for " << CONFIG_KEY(AUTO_BATCH_TIMEOUT)
<< " got " << val;
}
}
}
}
void AutoBatchInferencePlugin::SetConfig(const std::map<std::string, std::string>& config) {
CheckConfig(config);
for (auto && kvp : config) {
_config[kvp.first] = kvp.second;
}
@ -517,14 +560,12 @@ InferenceEngine::Parameter AutoBatchInferencePlugin::GetMetric(const std::string
} else if (name == METRIC_KEY(FULL_DEVICE_NAME)) {
IE_SET_METRIC_RETURN(FULL_DEVICE_NAME, _pluginName);
} else if (name == METRIC_KEY(SUPPORTED_CONFIG_KEYS)) {
std::vector<std::string> configKeys = PerfHintsConfig::SupportedKeys();
IE_SET_METRIC_RETURN(SUPPORTED_CONFIG_KEYS, configKeys);
IE_SET_METRIC_RETURN(SUPPORTED_CONFIG_KEYS, supported_configKeys);
} else {
IE_THROW(NotFound) << "Unsupported metric key " << name;
}
}
IExecutableNetworkInternal::Ptr AutoBatchInferencePlugin::LoadExeNetworkImpl(const InferenceEngine::CNNNetwork&network,
const std::map<std::string, std::string>& config) {
return LoadNetworkImpl(network, nullptr, config);
@ -575,10 +616,12 @@ InferenceEngine::IExecutableNetworkInternal::Ptr AutoBatchInferencePlugin::LoadN
metaDevice.batchForDevice = std::min(metaDevice.batchForDevice, closest);
}
}
// device settings + auto-batch settings
// auto-batch settings
std::unordered_map<std::string, InferenceEngine::Parameter> networkConfig;
networkConfig.insert(*device_batch);
networkConfig.insert(deviceConfig.begin(), deviceConfig.end());
for (auto c : fullConfig) {
if (supported_configKeys.end() != std::find(supported_configKeys.begin(), supported_configKeys.end(), c.first))
networkConfig.insert(c);
}
InferenceEngine::SoExecutableNetworkInternal executableNetworkWithBatch;
if (metaDevice.batchForDevice > 1) {

13
src/plugins/auto_batch/auto_batch.hpp
View File

@ -66,6 +66,8 @@ public:
std::shared_ptr<InferenceEngine::RemoteContext> GetContext() const override;
virtual ~AutoBatchExecutableNetwork();
protected:
static unsigned int ParseTimeoutValue(const std::string&);
std::atomic_bool _terminate = {false};
DeviceInformation _device;
InferenceEngine::SoExecutableNetworkInternal _network;
@ -74,6 +76,7 @@ public:
std::unordered_map<std::string, InferenceEngine::Parameter> _config;
bool _needPerfCounters = false;
std::atomic_size_t _numRequestsCreated = {0};
std::atomic_int _timeOut = {1000}; // in ms
};
class AutoBatchInferRequest : public InferenceEngine::IInferRequestInternal {
@ -126,20 +129,24 @@ public:
const std::map<std::string, std::string>& config) override;
void SetConfig(const std::map<std::string, std::string>& config) override;
void CheckConfig(const std::map<std::string, std::string>& config);
InferenceEngine::Parameter GetConfig(const std::string& name,
const std::map<std::string, InferenceEngine::Parameter> & options) const override;
InferenceEngine::QueryNetworkResult QueryNetwork(const InferenceEngine::CNNNetwork& network,
const std::map<std::string, std::string>& config) const override;
InferenceEngine::Parameter GetMetric(const std::string& name,
const std::map<std::string, InferenceEngine::Parameter>& options) const override;
DeviceInformation ParseMetaDevice(const std::string & devicesBatchCfg,
const std::map<std::string, std::string> & config) const;
InferenceEngine::RemoteContext::Ptr CreateContext(const InferenceEngine::ParamMap&) override;
protected:
DeviceInformation ParseMetaDevice(const std::string & devicesBatchCfg,
const std::map<std::string, std::string> & config) const;
std::map<std::string, std::string> GetSupportedConfig(const std::map<std::string, std::string>& config,
const DeviceName & deviceName) const;
static DeviceInformation ParseBatchDevice(const std::string& deviceWithBatch);
InferenceEngine::IExecutableNetworkInternal::Ptr LoadNetworkImpl(
const InferenceEngine::CNNNetwork& network,
const std::shared_ptr<InferenceEngine::RemoteContext> context,