IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

allowing the auto-batching only with tput hint to let more conventional tests pass

Browse Source
This commit is contained in:
myshevts 2021-10-21 18:32:32 +03:00
parent e11c6cb0ad
commit 7560a9319e
2 changed files with 13 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
inference-engine/src/auto_batch/auto_batch.cpp
View File

@ -193,7 +193,7 @@ void AutoBatchInferRequest::InferImpl() {
std::unique_lock<std::mutex> lock(_workerInferRequest->_mutex);
int sz = _workerInferRequest->_tasks.unsafe_size();
if (sz == _workerInferRequest->_batchSize) {
printf("!!! BATCH : %ld \n", _workerInferRequest->_tasks.unsafe_size());
// printf("!!! BATCH : %ld \n", _workerInferRequest->_tasks.unsafe_size());
std::pair<AutoBatchAsyncInferRequest*, InferenceEngine::Task> t;
for (int c = 0; c < _workerInferRequest->_batchSize; c++) {
if (_workerInferRequest->_tasks.try_pop(t)) {

18
inference-engine/src/inference_engine/src/ie_core.cpp
View File

@ -1204,12 +1204,18 @@ ExecutableNetwork Core::LoadNetwork(const CNNNetwork& network,
const std::string& deviceName,
const std::map<std::string, std::string>& config) {
std::map<std::string, std::string> config_with_batch = config;
// const auto& mode = config.find(KEY_PERFORMANCE_HINT);
// if (mode != config.end() && mode->second ==CONFIG_VALUE(THROUGHPUT) && deviceName.find("GPU") !=
// std::string::npos)
if (deviceName.find("GPU") != std::string::npos)
config_with_batch[CONFIG_KEY(ALLOW_AUTO_BATCHING)] = CONFIG_VALUE(YES);
auto device = ov::runtime::parseDeviceNameIntoConfig(deviceName);
if (device._deviceName.find("GPU") != std::string::npos) {
bool bThroughputEnabledInPlugin = false;
try {
bThroughputEnabledInPlugin =
GetConfig(device._deviceName, CONFIG_KEY(PERFORMANCE_HINT)).as<std::string>() == CONFIG_VALUE(THROUGHPUT);
} catch (...) {
}
const auto &mode = config.find(CONFIG_KEY(PERFORMANCE_HINT));
if (bThroughputEnabledInPlugin || (mode != config.end() && mode->second == CONFIG_VALUE(THROUGHPUT)))
config_with_batch[CONFIG_KEY(ALLOW_AUTO_BATCHING)] = CONFIG_VALUE(YES);
}
auto exec = _impl->LoadNetwork(network, deviceName, config_with_batch);
return {exec._so, exec._ptr};
}