IntenseWebs/openvino
3
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Auto batch smart reshape strict testing (once we moved to dim tracking) (#10253)

Browse Source 
* fixed perf-counters

* explicit auto-batching params that should guarantee the auto-batching is triggered ( to avoid fallback to no-batching when the selected batch1 size is just 1)

* makeConvPoolReluNoReshapes and using that whenever applicable to gaurantee the auto-batching is required (not important for things like plugin/executable-network config tests, but important for the inference-requests)

* getDefaultNGraphFunctionForTheDevice moved to the ov_behavior_test_utils.hpp
This commit is contained in:
Maxim Shevtsov authored and GitHub committed 2022-02-12 02:00:34 +03:00
1 parent 653ed4a34c
commit e3cc4833f4
24 files changed
+133 -40
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/plugins/auto_batch/auto_batch.cpp
+21 -17
View File
@@ -345,23 +345,27 @@ AutoBatchAsyncInferRequest::AutoBatchAsyncInferRequest(
};
AutoBatchAsyncInferRequest* _this = nullptr;
};
_pipeline = {{/*TaskExecutor*/ std::make_shared<ThisRequestExecutor>(this), /*task*/ [this, needPerfCounters] {
if (this->_inferRequest->_exceptionPtr) // if the exception happened in the batch1 fallback
std::rethrow_exception(this->_inferRequest->_exceptionPtr);
auto& batchReq = this->_inferRequest->_myBatchedRequestWrapper;
if (batchReq._exceptionPtr) // when the batchN execution failed
std::rethrow_exception(batchReq._exceptionPtr);
// in the case of non-batched execution the blobs were set explicitly
if (AutoBatchInferRequest::eExecutionFlavor::BATCH_EXECUTED ==
this->_inferRequest->_wasBatchedRequestUsed)
this->_inferRequest->CopyOutputsIfNeeded();
if (needPerfCounters) {
try {
this->_inferRequest->_perfMap = batchReq._inferRequestBatched->GetPerformanceCounts();
} catch (...) {
}
}
}}};
_pipeline = {
{/*TaskExecutor*/ std::make_shared<ThisRequestExecutor>(this), /*task*/ [this, needPerfCounters] {
if (this->_inferRequest->_exceptionPtr) // if the exception happened in the batch1 fallback
std::rethrow_exception(this->_inferRequest->_exceptionPtr);
auto& batchReq = this->_inferRequest->_myBatchedRequestWrapper;
if (batchReq._exceptionPtr) // when the batchN execution failed
std::rethrow_exception(batchReq._exceptionPtr);
// in the case of non-batched execution the blobs were set explicitly
if (AutoBatchInferRequest::eExecutionFlavor::BATCH_EXECUTED == this->_inferRequest->_wasBatchedRequestUsed)
this->_inferRequest->CopyOutputsIfNeeded();
if (needPerfCounters) {
try {
if (AutoBatchInferRequest::eExecutionFlavor::BATCH_EXECUTED ==
this->_inferRequest->_wasBatchedRequestUsed)
this->_inferRequest->_perfMap = batchReq._inferRequestBatched->GetPerformanceCounts();
else
this->_inferRequest->_perfMap = this->_inferRequestWithoutBatch->GetPerformanceCounts();
} catch (...) {
}
}
}}};
}
void AutoBatchAsyncInferRequest::Infer_ThreadUnsafe() {