IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CPU] Zero-copy optimizations for model outputs (#18476)

Browse Source 
- Implement zero-copy output between plugin graph and infer request, thus eliminate memory copy overhead and optimize performance
- Implement double buffer for InferRequest outputs
This commit is contained in:
cecilia peng
2023-07-25 10:33:48 +04:00
committed by GitHub
parent fdfafbb7d2
commit 7fbd3a7ebf
25 changed files with 1239 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/inference/dev_api/openvino/runtime/make_tensor.hpp
+3 -3
View File
@@ -71,9 +71,9 @@ ov::SoPtr<ITensor> make_tensor(const std::shared_ptr<InferenceEngine::Blob>& ten
const InferenceEngine::Blob* get_hardware_blob(const InferenceEngine::Blob* blob);
InferenceEngine::Blob* get_hardware_blob(InferenceEngine::Blob* blob);
std::shared_ptr<InferenceEngine::Blob> tensor_to_blob(const ov::SoPtr<ITensor>& tensor,
bool unwrap = true,
InferenceEngine::TensorDesc desc = {});
OPENVINO_RUNTIME_API std::shared_ptr<InferenceEngine::Blob> tensor_to_blob(const ov::SoPtr<ITensor>& tensor,
bool unwrap = true,
InferenceEngine::TensorDesc desc = {});
/** @endcond */
IE_SUPPRESS_DEPRECATED_END
src/inference/src/dev/isync_infer_request.cpp
+3 -2
View File
@@ -260,8 +260,9 @@ void ov::ISyncInferRequest::check_tensor(const ov::Output<const ov::Node>& port,
" expecting ",
port.get_shape(),
".");
OPENVINO_ASSERT(std::dynamic_pointer_cast<ov::IRemoteTensor>(tensor._ptr) || tensor->data() != nullptr,
"Tensor data equal nullptr!");
OPENVINO_ASSERT(
std::dynamic_pointer_cast<ov::IRemoteTensor>(tensor._ptr) || tensor->data() != nullptr || is_dynamic,
"Tensor data equal nullptr!");
}
void ov::ISyncInferRequest::allocate_tensor(
src/inference/src/ie_layouts.cpp
+3 -1
View File
@@ -298,7 +298,9 @@ BlockingDesc::BlockingDesc(const SizeVector& blocked_dims,
this->offsetPaddingToData = dimOffsets;
// check that strides are valid
{
if (!std::any_of(blocked_dims.begin(), blocked_dims.end(), [](const size_t dim) {
return dim == 0ul;
})) {
size_t denseStride = 1;
for (size_t i = 1; i <= strides.size(); i++) {