diff --git a/src/plugins/intel_gpu/include/intel_gpu/plugin/infer_request.hpp b/src/plugins/intel_gpu/include/intel_gpu/plugin/infer_request.hpp index ddbdcefa82d..01a2b2fb935 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/plugin/infer_request.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/plugin/infer_request.hpp @@ -57,6 +57,8 @@ public: void enable_external_queue() { m_useExternalQueue = true; } private: + // This blob is used for outputs processing if output data type convertion or padding handling is needed + InferenceEngine::Blob::Ptr intermediate_output_blob = nullptr; InferenceEngine::BlobMap _deviceOutputs; std::map inputsMap; std::map outputsMap; @@ -73,12 +75,14 @@ private: void prepare_input(const cldnn::primitive_id &inputName, InferenceEngine::Blob::Ptr &inputBlob, std::vector& dependencies); void prepare_output(const cldnn::primitive_id& outputName, InferenceEngine::Blob::Ptr& outputBlob); + void allocate_dev_mem_if_needed(InferenceEngine::BlobMap& device_mems, InferenceEngine::Blob::Ptr& user_blob, + const cldnn::primitive_id& blob_name, const cldnn::layout& layout); InferenceEngine::Blob::Ptr create_host_blob(const InferenceEngine::TensorDesc& desc, bool is_dynamic); InferenceEngine::Blob::Ptr create_device_blob(const InferenceEngine::TensorDesc& desc); void copy_output_data(cldnn::memory::ptr outputMemory, InferenceEngine::Blob::Ptr bptr); - void copy_input_data(std::shared_ptr network, const cldnn::primitive_id &inputName, + void copy_input_data(std::shared_ptr network, const cldnn::primitive_id& inputName, const cldnn::layout& inputLayout, const InferenceEngine::Blob &inputBlob); InferenceEngine::Blob::Ptr create_shared_device_blob(const InferenceEngine::TensorDesc& desc, const cldnn::layout& layout, void* usm_host_mem); diff --git a/src/plugins/intel_gpu/include/intel_gpu/runtime/engine.hpp b/src/plugins/intel_gpu/include/intel_gpu/runtime/engine.hpp index d6940667fb6..176ae7d4470 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/runtime/engine.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/runtime/engine.hpp @@ -137,6 +137,8 @@ public: /// Returns service stream which can be used during program build and optimizations virtual stream& get_program_stream() const = 0; + virtual allocation_type detect_usm_allocation_type(const void* memory) const = 0; + #ifdef ENABLE_ONEDNN_FOR_GPU /// Returns onednn engine object which shares device and context with current engine virtual dnnl::engine& get_onednn_engine() const = 0; diff --git a/src/plugins/intel_gpu/include/intel_gpu/runtime/memory.hpp b/src/plugins/intel_gpu/include/intel_gpu/runtime/memory.hpp index 269a7877bcc..c2e8d61ac1b 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/runtime/memory.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/runtime/memory.hpp @@ -71,6 +71,9 @@ struct memory { virtual event::ptr copy_from(stream& /* stream */, const memory& /* other */) = 0; virtual event::ptr copy_from(stream& /* stream */, const void* /* host_ptr */) = 0; + virtual event::ptr copy_to(stream& stream, memory& other) { return other.copy_from(stream, *this); } + virtual event::ptr copy_to(stream& /* stream */, void* /* host_ptr */) = 0; + #ifdef ENABLE_ONEDNN_FOR_GPU virtual dnnl::memory get_onednn_memory(dnnl::memory::desc /* desc */, int64_t offset = 0) { throw std::runtime_error("[CLDNN] Can't convert memory object to onednn"); @@ -108,6 +111,9 @@ struct simple_attached_memory : memory { event::ptr copy_from(stream& /* stream */, const memory& /* other */) override { return nullptr; }; event::ptr copy_from(stream& /* stream */, const void* /* host_ptr */) override { return nullptr; } + event::ptr copy_to(stream& /* stream */, memory& /* other */) override { return nullptr; }; + event::ptr copy_to(stream& /* stream */, void* /* host_ptr */) override { return nullptr; } + private: void* _pointer; }; diff --git a/src/plugins/intel_gpu/src/plugin/infer_request.cpp b/src/plugins/intel_gpu/src/plugin/infer_request.cpp index bad0b6708fc..8854440f52d 100644 --- a/src/plugins/intel_gpu/src/plugin/infer_request.cpp +++ b/src/plugins/intel_gpu/src/plugin/infer_request.cpp @@ -23,6 +23,8 @@ using namespace InferenceEngine; namespace { +const char str_device_input_unsupported_blob[] = "Device input is of an unsupported blob type."; +const char str_device_output_unsupported_blob[] = "Device output is of an unsupported blob type."; const char str_input_not_allocated[] = "Input data was not allocated."; const char str_output_not_allocated[] = "Output data was not allocated."; @@ -46,28 +48,24 @@ void convertAndCopy(const InferenceEngine::Blob* src, dst_t* dst) { } template -void copyResultToOutputBlob(cldnn::memory::ptr src, Blob::Ptr dst, cldnn::stream& stream) { - size_t n = dst->size(); - - auto layout = src->get_layout(); - auto size = layout.get_tensor(); +void copy_result_to_output_blob(InferenceEngine::Blob::Ptr src, InferenceEngine::Blob::Ptr dst, const cldnn::layout& src_layout) { + auto locked_src = src->buffer(); + auto src_ptr = locked_src.as(); + OPENVINO_ASSERT(src_ptr, "[GPU] Invalid source blob"); auto locked_dst = dst->buffer(); auto dst_ptr = locked_dst.as(); - if (dst_ptr == nullptr) { - IE_THROW() << "Invalid output blob"; - } - cldnn::mem_lock src_lock{ src, stream }; - src_dt* src_ptr = src_lock.data(); + OPENVINO_ASSERT(dst_ptr, "[GPU] Invalid output blob"); - if (layout.data_padding) { + if (src_layout.data_padding) { + auto size = src_layout.get_tensor(); for (int64_t b = 0; b < size.batch[0]; b++) { for (int64_t f = 0; f < size.feature[0]; f++) { for (int64_t w = 0; w < size.spatial[3]; w++) { for (int64_t z = 0; z < size.spatial[2]; z++) { for (int64_t y = 0; y < size.spatial[1]; y++) { for (int64_t x = 0; x < size.spatial[0]; x++) { - *dst_ptr++ = src_ptr[layout.get_linear_offset(cldnn::tensor(b, f, x, y, z, w))]; + *dst_ptr++ = src_ptr[src_layout.get_linear_offset(cldnn::tensor(b, f, x, y, z, w))]; } } } @@ -75,6 +73,7 @@ void copyResultToOutputBlob(cldnn::memory::ptr src, Blob::Ptr dst, cldnn::stream } } } else { + size_t n = dst->size(); for (size_t i = 0; i < n; i++) { dst_ptr[i] = src_ptr[i]; } @@ -252,7 +251,7 @@ void InferRequest::set_input(const std::string& name, const Blob::Ptr& data) { } void InferRequest::set_output(const std::string& name, const Blob::Ptr& data) { - auto remote_ptr = data->as(); + auto remote_ptr = data->as(); bool is_remote = remote_ptr != nullptr; auto node = findOutputByNodeName(name); @@ -620,21 +619,59 @@ Blob::Ptr InferRequest::create_shared_device_blob(const InferenceEngine::TensorD void InferRequest::copy_output_data(cldnn::memory::ptr src, Blob::Ptr dst) { OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "InferRequest::copy_output_data"); + auto is_convert_needed = [](const Precision& prc) { + const std::vector convert_needed = { Precision::I16, Precision::U16, Precision::FP64, + Precision::U32, Precision::U64 }; + const std::vector convert_not_needed = { Precision::FP32, Precision::FP16, Precision::I64, Precision::I32, + Precision::I8, Precision::U8, Precision::BOOL }; + + if (std::find(convert_needed.begin(), convert_needed.end(), prc) != convert_needed.end()) + return true; + else if (std::find(convert_not_needed.begin(), convert_not_needed.end(), prc) != convert_not_needed.end()) + return false; + else + OPENVINO_ASSERT(false, "[GPU] Plugin does not support output ", prc, " precision"); + }; + + const auto convert_needed = is_convert_needed(dst->getTensorDesc().getPrecision()); + const auto src_layout = src->get_layout(); auto& stream = m_graph->GetNetwork()->get_stream(); - switch (dst->getTensorDesc().getPrecision()) { - case Precision::FP64: copyResultToOutputBlob(src, dst, stream); break; - case Precision::FP32: copyResultToOutputBlob(src, dst, stream); break; - case Precision::FP16: copyResultToOutputBlob(src, dst, stream); break; - case Precision::I64: copyResultToOutputBlob(src, dst, stream); break; - case Precision::I32: copyResultToOutputBlob(src, dst, stream); break; - case Precision::I16: copyResultToOutputBlob(src, dst, stream); break; - case Precision::I8: copyResultToOutputBlob(src, dst, stream); break; - case Precision::U16: copyResultToOutputBlob(src, dst, stream); break; - case Precision::U32: copyResultToOutputBlob(src, dst, stream); break; - case Precision::U64: copyResultToOutputBlob(src, dst, stream); break; - case Precision::U8: copyResultToOutputBlob(src, dst, stream); break; - case Precision::BOOL: copyResultToOutputBlob(src, dst, stream); break; - default: IE_THROW(NotImplemented) << "The plugin does not support output " << dst->getTensorDesc().getPrecision() << " precision"; + + if (convert_needed || src_layout.data_padding) { + if (!intermediate_output_blob || intermediate_output_blob->byteSize() < src_layout.bytes_count()) { + auto desc = TensorDesc(Precision::U8, + SizeVector{src_layout.bytes_count()}, + InferenceEngine::Layout::C); + intermediate_output_blob = create_host_blob(desc, src_layout.is_dynamic()); + } + + OPENVINO_ASSERT(intermediate_output_blob, "[GPU] Intermediate blob for outputs precessing is not allocated"); + + auto event = src->copy_to(stream, intermediate_output_blob->buffer()); + event->wait(); + + switch (dst->getTensorDesc().getPrecision()) { + #define CASE(PRC, SRC_DT, DST_DT) \ + case PRC: copy_result_to_output_blob (intermediate_output_blob, dst, src_layout); break; + CASE(Precision::FP64, float, double) + CASE(Precision::FP32, float, float) + CASE(Precision::FP16, uint16_t, uint16_t) + CASE(Precision::I64, int64_t, int64_t) + CASE(Precision::I32, int32_t, int32_t) + CASE(Precision::I16, float, int16_t) + CASE(Precision::I8, int8_t, int8_t) + CASE(Precision::U16, float, uint16_t) + CASE(Precision::U32, int32_t, uint32_t) + CASE(Precision::U64, int32_t, uint64_t) + CASE(Precision::U8, uint8_t, uint8_t) + CASE(Precision::BOOL, int8_t, int8_t) + #undef CASE + default: break; + } + } else { + auto dst_ptr = dst->buffer().as(); + auto event = src->copy_to(stream, dst_ptr); + event->wait(); } } @@ -714,11 +751,8 @@ void InferRequest::allocate_inputs() { if (!is_nv12_input) { auto litr = inputLayouts.find(name); - if (litr == inputLayouts.end()) { - IE_THROW() << "Input layout for " << name << " is not found"; - } - - auto input_layout = litr->second; + OPENVINO_ASSERT(litr != inputLayouts.end(), "[GPU] Input layout for ", name, " is not found"); + const auto input_layout = litr->second; GPU_DEBUG_GET_INSTANCE(debug_config); GPU_DEBUG_IF(debug_config->verbose >= 2) { @@ -732,15 +766,10 @@ void InferRequest::allocate_inputs() { _deviceInputs[name] = create_device_blob(desc_fp32); } else { _inputs[name] = create_host_blob(desc, input_layout.is_dynamic()); - if (input_layout.is_static()) { - if (m_graph->GetEngine()->use_unified_shared_memory()) { - // For USM case we create host blob using custom USM host allocator - // and then create shared device blob on top of this buffer - auto host_blob = _inputs[name]; - _deviceInputs[name] = create_shared_device_blob(desc, input_layout, host_blob->buffer().as()); - } else { - _deviceInputs[name] = create_device_blob(desc); - } + // Pre-allocate device input only if USM is not supported; in other case it will be allocated + // in prepare_input() function later + if (input_layout.is_static() && !m_graph->GetEngine()->use_unified_shared_memory()) { + _deviceInputs[name] = create_device_blob(desc); } } } @@ -782,15 +811,10 @@ void InferRequest::allocate_outputs() { _deviceOutputs[no.first] = create_device_blob(device_blob_desc); } else { _outputs[no.first] = create_host_blob(desc, output_layout.is_dynamic()); - if (output_layout.is_static()) { - if (m_graph->GetEngine()->use_unified_shared_memory()) { - // For USM case we create host blob using custom USM host allocator - // and then create shared device blob on top of this buffer - auto host_blob = _outputs[no.first]; - _deviceOutputs[no.first] = create_shared_device_blob(desc, output_layout, host_blob->buffer().as()); - } else { - _deviceOutputs[no.first] = create_device_blob(desc); - } + // Pre-allocate device output only if USM is not supported; in other case it will be allocated + // in prepare_output() function later + if (output_layout.is_static() && !m_graph->GetEngine()->use_unified_shared_memory()) { + _deviceOutputs[no.first] = create_device_blob(desc); } } } @@ -813,14 +837,53 @@ std::map InferRequest::GetPerformanceCo } } +void InferRequest::allocate_dev_mem_if_needed(InferenceEngine::BlobMap& device_mems, InferenceEngine::Blob::Ptr& user_blob, + const cldnn::primitive_id& blob_name, const cldnn::layout& layout) { + const auto input_ptr = static_cast(user_blob->cbuffer()); + const auto alloc_type = m_graph->GetEngine()->detect_usm_allocation_type(input_ptr); + const auto is_usm_host = alloc_type == cldnn::allocation_type::usm_host; + const auto has_device_blob = device_mems.find(blob_name) != device_mems.end(); + bool can_skip_allocation = false; + + if (has_device_blob) { + auto impl = getBlobImpl(device_mems[blob_name]->as()); + + OPENVINO_ASSERT(impl, str_device_output_unsupported_blob); + OPENVINO_ASSERT(impl->is_allocated(), str_input_not_allocated); + + auto impl_mem = impl->getMemory(); + auto src_ptr = user_blob->cbuffer().as(); + // If device mem already exists, we can reuse blob if buffer has usm_host type and points to the same memory, + // so we don't need to allocate new memory + can_skip_allocation |= same_host_mem(impl_mem, src_ptr); + // Or if blob has any type except usm_host - in that case explicit copy will be performed anyway + can_skip_allocation |= impl_mem->get_allocation_type() != cldnn::allocation_type::usm_host; + } + + if (!can_skip_allocation) { + if (is_usm_host) { + // For USM case we create host blob using custom USM host allocator + // and then create shared device blob on top of this buffer + device_mems[blob_name] = create_shared_device_blob(user_blob->getTensorDesc(), layout, user_blob->buffer().as()); + } else { + device_mems[blob_name] = create_device_blob(user_blob->getTensorDesc()); + } + } +} + void InferRequest::prepare_input(const cldnn::primitive_id& inputName, Blob::Ptr& inputBlob, std::vector& dependencies) { OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "InferRequest::prepare_input"); auto inputLayoutItr = m_graph->GetInputLayouts().find(inputName); - if (inputLayoutItr == m_graph->GetInputLayouts().end()) { - IE_THROW() << "Input name mismatch."; - } + OPENVINO_ASSERT(inputLayoutItr != m_graph->GetInputLayouts().end(), "[GPU] Input name mismatch"); + auto input_layout = inputLayoutItr->second; + auto& prec = inputBlob->getTensorDesc().getPrecision(); + auto remote_ptr = inputBlob->as(); + auto& stream = m_graph->GetNetwork()->get_stream(); + const bool is_dev_input = remote_ptr != nullptr; + const bool can_use_usm = m_graph->GetEngine()->use_unified_shared_memory(); + if (input_layout.is_dynamic()) { bool has_device_blob = _deviceInputs.find(inputName) != _deviceInputs.end(); bool should_allocate_device_blob = !has_device_blob; @@ -836,15 +899,13 @@ void InferRequest::prepare_input(const cldnn::primitive_id& inputName, Blob::Ptr } else { _deviceInputs[inputName] = reinterpret_device_blob(_deviceInputs[inputName], inputBlob->getTensorDesc()); } + } else if (input_layout.is_static() && !is_dev_input && can_use_usm) { + allocate_dev_mem_if_needed(_deviceInputs, inputBlob, inputName, input_layout); } OPENVINO_ASSERT(_deviceInputs.find(inputName) != _deviceInputs.end(), "[GPU] Couldn't find device blob allocated for ", inputName, " input"); auto reqBlob = _deviceInputs.at(inputName)->as(); auto _nw_ptr = m_graph->GetNetwork(); - cldnn::primitive_id internalName = "parameter:" + inputName; - const auto& prec = inputBlob->getTensorDesc().getPrecision(); - auto remote_ptr = inputBlob->as(); - auto& stream = m_graph->GetNetwork()->get_stream(); - bool is_dev_input = remote_ptr != nullptr; + const cldnn::primitive_id internalName = "parameter:" + inputName; switch (prec) { case Precision::FP64: @@ -875,6 +936,7 @@ void InferRequest::prepare_input(const cldnn::primitive_id& inputName, Blob::Ptr } if (!is_dev_input) { + // TODO: Remove this checks once 95363 issue is solved if (prec == Precision::I16 || prec == Precision::U16 || prec == Precision::FP64) { // GPU plugin doesn't support I16 input precision, // so have to convert input data to fp32 precision @@ -912,14 +974,26 @@ void InferRequest::prepare_input(const cldnn::primitive_id& inputName, Blob::Ptr void InferRequest::prepare_output(const cldnn::primitive_id& outputName, Blob::Ptr& outputBlob) { OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "InferRequest::prepare_output"); + const auto output_id = outputsMap.at(outputName); + const auto output_layout = m_graph->GetNetwork()->get_node_output_layout(output_id); + const bool is_static = output_layout.is_static(); + const bool can_use_usm = m_graph->GetEngine()->use_unified_shared_memory(); + auto remote_ptr = outputBlob->as(); + const bool is_dev_input = remote_ptr != nullptr; + + if (is_static && can_use_usm && !is_dev_input) { + allocate_dev_mem_if_needed(_deviceOutputs, outputBlob, outputName, output_layout); + } + + OPENVINO_ASSERT(!is_static || _deviceOutputs.find(outputName) != _deviceOutputs.end(), + "[GPU] Couldn't find device blob allocated for ", outputName, " output"); // Missing output in _deviceOutputs means that the network is dynamic and outputs couldn't be pre-allocated if (_deviceOutputs.find(outputName) == _deviceOutputs.end()) return; Blob::Ptr reqBlob = _deviceOutputs.at(outputName); cldnn::primitive_id internalName = outputsMap[outputName]; auto _nw_ptr = m_graph->GetNetwork(); - auto remote_ptr = outputBlob->as(); - auto output_blob_ptr = (reqBlob != outputBlob && remote_ptr != nullptr) + auto output_blob_ptr = (reqBlob != outputBlob && is_dev_input) ? remote_ptr : reqBlob->as(); auto impl = getBlobImpl(output_blob_ptr); diff --git a/src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.cpp b/src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.cpp index f7930b868fd..e6a06941b75 100644 --- a/src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.cpp +++ b/src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.cpp @@ -89,6 +89,11 @@ const cl::UsmHelper& ocl_engine::get_usm_helper() const { return *_usm_helper; } +allocation_type ocl_engine::detect_usm_allocation_type(const void* memory) const { + return use_unified_shared_memory() ? ocl::gpu_usm::detect_allocation_type(this, memory) + : allocation_type::unknown; +} + memory::ptr ocl_engine::allocate_memory(const layout& layout, allocation_type type, bool reset) { OPENVINO_ASSERT(!layout.is_dynamic(), "[GPU] Can't allocate memory for dynamic layout"); diff --git a/src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.hpp b/src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.hpp index ca4dd3c7139..2999c1be157 100644 --- a/src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.hpp +++ b/src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.hpp @@ -32,6 +32,7 @@ public: void* get_user_context() const override; allocation_type get_default_allocation_type() const override { return allocation_type::cl_mem; } + allocation_type detect_usm_allocation_type(const void* memory) const override; const cl::Context& get_cl_context() const; const cl::Device& get_cl_device() const; diff --git a/src/plugins/intel_gpu/src/runtime/ocl/ocl_memory.cpp b/src/plugins/intel_gpu/src/runtime/ocl/ocl_memory.cpp index aa95b958b02..e20a237b6ad 100644 --- a/src/plugins/intel_gpu/src/runtime/ocl/ocl_memory.cpp +++ b/src/plugins/intel_gpu/src/runtime/ocl/ocl_memory.cpp @@ -110,6 +110,15 @@ event::ptr gpu_buffer::copy_from(stream& stream, const void* host_ptr) { return ev; } +event::ptr gpu_buffer::copy_to(stream& stream, void* host_ptr) { + auto& cl_stream = downcast(stream); + auto ev = stream.create_base_event(); + cl::Event ev_ocl = downcast(ev.get())->get(); + cl_stream.get_cl_queue().enqueueReadBuffer(_buffer, false, 0, size(), host_ptr, nullptr, &ev_ocl); + + return ev; +} + #ifdef ENABLE_ONEDNN_FOR_GPU dnnl::memory gpu_buffer::get_onednn_memory(dnnl::memory::desc desc, int64_t offset) { auto onednn_engine = _engine->get_onednn_engine(); @@ -237,11 +246,19 @@ shared_mem_params gpu_image2d::get_internal_params() const { } event::ptr gpu_image2d::copy_from(stream& /* stream */, const memory& /* other */) { - throw std::runtime_error("[clDNN] copy_from is not implemented for gpu_image2d"); + throw std::runtime_error("[GPU] copy_from is not implemented for gpu_image2d"); } event::ptr gpu_image2d::copy_from(stream& /* stream */, const void* /* host_ptr */) { - throw std::runtime_error("[clDNN] copy_from is not implemented for gpu_image2d"); + throw std::runtime_error("[GPU] copy_from is not implemented for gpu_image2d"); +} + +event::ptr gpu_image2d::copy_to(stream& /* stream */, memory& /* other */) { + throw std::runtime_error("[GPU] copy_to is not implemented for gpu_image2d"); +} + +event::ptr gpu_image2d::copy_to(stream& /* stream */, void* /* host_ptr */) { + throw std::runtime_error("[GPU] copy_to is not implemented for gpu_image2d"); } gpu_media_buffer::gpu_media_buffer(ocl_engine* engine, @@ -384,15 +401,24 @@ event::ptr gpu_usm::copy_from(stream& stream, const memory& other) { event::ptr gpu_usm::copy_from(stream& stream, const void* host_ptr) { auto& cl_stream = downcast(stream); - auto ev = stream.create_base_event(); auto dst_ptr = get_buffer().get(); cl_stream.get_usm_helper().enqueue_memcpy(cl_stream.get_cl_queue(), dst_ptr, host_ptr, _bytes_count, true); + return stream.create_user_event(true); +} - return ev; +event::ptr gpu_usm::copy_to(stream& stream, void* host_ptr) { + auto& cl_stream = downcast(stream); + auto src_ptr = get_buffer().get(); + cl_stream.get_usm_helper().enqueue_memcpy(cl_stream.get_cl_queue(), + host_ptr, + src_ptr, + _bytes_count, + true); + return stream.create_user_event(true); } #ifdef ENABLE_ONEDNN_FOR_GPU @@ -420,20 +446,28 @@ shared_mem_params gpu_usm::get_internal_params() const { }; } -allocation_type gpu_usm::detect_allocation_type(ocl_engine* engine, const cl::UsmMemory& buffer) { - auto cl_alloc_type = engine->get_usm_helper().get_usm_allocation_type(buffer.get()); +allocation_type gpu_usm::detect_allocation_type(const ocl_engine* engine, const void* mem_ptr) { + auto cl_alloc_type = engine->get_usm_helper().get_usm_allocation_type(mem_ptr); - allocation_type res = allocation_type::unknown; + allocation_type res; switch (cl_alloc_type) { case CL_MEM_TYPE_DEVICE_INTEL: res = allocation_type::usm_device; break; case CL_MEM_TYPE_HOST_INTEL: res = allocation_type::usm_host; break; case CL_MEM_TYPE_SHARED_INTEL: res = allocation_type::usm_shared; break; - default: throw std::runtime_error("[GPU] Unsupported USM alloc type: " + std::to_string(cl_alloc_type)); + default: res = allocation_type::unknown; } return res; } +allocation_type gpu_usm::detect_allocation_type(const ocl_engine* engine, const cl::UsmMemory& buffer) { + auto alloc_type = detect_allocation_type(engine, buffer.get()); + OPENVINO_ASSERT(alloc_type == allocation_type::usm_device || + alloc_type == allocation_type::usm_host || + alloc_type == allocation_type::usm_shared, "[GPU] Unsupported USM alloc type: " + to_string(alloc_type)); + return alloc_type; +} + std::vector ocl_surfaces_lock::get_handles(std::vector mem) const { std::vector res; for (auto& m : mem) { diff --git a/src/plugins/intel_gpu/src/runtime/ocl/ocl_memory.hpp b/src/plugins/intel_gpu/src/runtime/ocl/ocl_memory.hpp index ecbe9257bc4..fc9e40b64d5 100644 --- a/src/plugins/intel_gpu/src/runtime/ocl/ocl_memory.hpp +++ b/src/plugins/intel_gpu/src/runtime/ocl/ocl_memory.hpp @@ -42,6 +42,9 @@ struct gpu_buffer : public lockable_gpu_mem, public memory { event::ptr copy_from(stream& stream, const memory& other) override; event::ptr copy_from(stream& stream, const void* host_ptr) override; + + event::ptr copy_to(stream& /* stream */, void* /* other */) override; + #ifdef ENABLE_ONEDNN_FOR_GPU dnnl::memory get_onednn_memory(dnnl::memory::desc /* desc */, int64_t offset = 0) override; #endif @@ -67,6 +70,9 @@ struct gpu_image2d : public lockable_gpu_mem, public memory { event::ptr copy_from(stream& /* stream */, const memory& /* other */) override; event::ptr copy_from(stream& /* stream */, const void* /* other */) override; + event::ptr copy_to(stream& /* stream */, memory& /* other */) override; + event::ptr copy_to(stream& /* stream */, void* /* other */) override; + protected: cl::Image2D _buffer; size_t _width; @@ -116,15 +122,18 @@ struct gpu_usm : public lockable_gpu_mem, public memory { event::ptr copy_from(stream& stream, const memory& other) override; event::ptr copy_from(stream& stream, const void* host_ptr) override; + event::ptr copy_to(stream& stream, void* host_ptr) override; #ifdef ENABLE_ONEDNN_FOR_GPU dnnl::memory get_onednn_memory(dnnl::memory::desc /* desc */, int64_t offset = 0) override; #endif + static allocation_type detect_allocation_type(const ocl_engine* engine, const void* mem_ptr); + protected: cl::UsmMemory _buffer; cl::UsmMemory _host_buffer; - static allocation_type detect_allocation_type(ocl_engine* engine, const cl::UsmMemory& buffer); + static allocation_type detect_allocation_type(const ocl_engine* engine, const cl::UsmMemory& buffer); }; struct ocl_surfaces_lock : public surfaces_lock { diff --git a/src/tests/functional/plugin/gpu/concurrency/gpu_concurrency_tests.cpp b/src/tests/functional/plugin/gpu/concurrency/gpu_concurrency_tests.cpp index 0400286e04a..ce9b5cee6da 100644 --- a/src/tests/functional/plugin/gpu/concurrency/gpu_concurrency_tests.cpp +++ b/src/tests/functional/plugin/gpu/concurrency/gpu_concurrency_tests.cpp @@ -108,3 +108,166 @@ INSTANTIATE_TEST_SUITE_P(smoke_RemoteTensor, OVConcurrencyTest, ::testing::Combine(::testing::ValuesIn(num_streams), ::testing::ValuesIn(num_requests)), OVConcurrencyTest::getTestCaseName); + +TEST(canSwapTensorsBetweenInferRequests, inputs) { + std::vector> ref; + std::vector input_tensors; + auto fn = ngraph::builder::subgraph::makeSplitMultiConvConcat(); + + auto ie = ov::Core(); + auto compiled_model = ie.compile_model(fn, CommonTestUtils::DEVICE_GPU); + + const int infer_requests_num = 2; + ov::InferRequest infer_request1 = compiled_model.create_infer_request(); + ov::InferRequest infer_request2 = compiled_model.create_infer_request(); + + input_tensors.push_back(infer_request1.get_input_tensor()); + input_tensors.push_back(infer_request2.get_input_tensor()); + + auto calc_ref_results = [&](const ov::Tensor& tensor){ + const auto tensor_size = tensor.get_byte_size(); + const auto in_blob_buf = static_cast(tensor.data()); + std::vector inData(in_blob_buf, in_blob_buf + tensor_size); + auto ref_out_data = ngraph::helpers::interpreterFunction(fn, {inData}).front().second; + ref.push_back(ref_out_data); + }; + + auto compare_results = [&](ov::Tensor& result, const uint8_t* refResult) { + auto thr = FuncTestUtils::GetComparisonThreshold(InferenceEngine::Precision::FP32); + ASSERT_EQ(ov::shape_size(fn->get_output_shape(0)), result.get_size()); + FuncTestUtils::compareRawBuffers(result.data(), + reinterpret_cast(refResult), ov::shape_size(fn->get_output_shape(0)), + ov::shape_size(fn->get_output_shape(0)), + thr); + }; + + for (size_t i = 0; i < infer_requests_num; i++) { + FuncTestUtils::fill_tensor(input_tensors[i], 10, -5, 1, i); + calc_ref_results(input_tensors[i]); + } + + // Swap tensors between IRs + infer_request1.set_input_tensor(input_tensors[1]); + infer_request2.set_input_tensor(input_tensors[0]); + + const int niter_limit = 10; + int iter1 = 0, iter2 = 0; + infer_request1.set_callback([&](std::exception_ptr exception_ptr) mutable { + if (exception_ptr) { + std::rethrow_exception(exception_ptr); + } else { + iter1++; + ov::Tensor output_tensor = infer_request1.get_output_tensor(); + compare_results(output_tensor, ref[iter1 % 2].data()); + if (iter1 < niter_limit) { + infer_request1.set_input_tensor(input_tensors[(iter1 + 1) % 2]); + infer_request1.start_async(); + } + } + }); + + infer_request2.set_callback([&](std::exception_ptr exception_ptr) mutable { + if (exception_ptr) { + std::rethrow_exception(exception_ptr); + } else { + iter2++; + ov::Tensor output_tensor = infer_request2.get_output_tensor(); + compare_results(output_tensor, ref[(iter2 + 1) % 2].data()); + if (iter2 < niter_limit) { + infer_request2.set_input_tensor(input_tensors[iter2 % 2]); + infer_request2.start_async(); + } + } + }); + + infer_request1.start_async(); + infer_request2.start_async(); + + for (size_t i = 0; i < niter_limit; i++) { + infer_request1.wait(); + infer_request2.wait(); + } +} + +TEST(canSwapTensorsBetweenInferRequests, outputs) { + std::vector> ref; + std::vector input_tensors; + std::vector output_tensors; + auto fn = ngraph::builder::subgraph::makeSplitMultiConvConcat(); + + auto ie = ov::Core(); + auto compiled_model = ie.compile_model(fn, CommonTestUtils::DEVICE_GPU); + + const int infer_requests_num = 2; + ov::InferRequest infer_request1 = compiled_model.create_infer_request(); + ov::InferRequest infer_request2 = compiled_model.create_infer_request(); + + input_tensors.push_back(infer_request1.get_input_tensor()); + input_tensors.push_back(infer_request2.get_input_tensor()); + output_tensors.push_back(infer_request1.get_output_tensor()); + output_tensors.push_back(infer_request2.get_output_tensor()); + + auto calc_ref_results = [&](const ov::Tensor& tensor){ + const auto tensor_size = tensor.get_byte_size(); + const auto in_blob_buf = static_cast(tensor.data()); + std::vector inData(in_blob_buf, in_blob_buf + tensor_size); + auto ref_out_data = ngraph::helpers::interpreterFunction(fn, {inData}).front().second; + ref.push_back(ref_out_data); + }; + + auto compare_results = [&](ov::Tensor& result, const uint8_t* refResult) { + auto thr = FuncTestUtils::GetComparisonThreshold(InferenceEngine::Precision::FP32); + ASSERT_EQ(ov::shape_size(fn->get_output_shape(0)), result.get_size()); + FuncTestUtils::compareRawBuffers(result.data(), + reinterpret_cast(refResult), ov::shape_size(fn->get_output_shape(0)), + ov::shape_size(fn->get_output_shape(0)), + thr); + }; + + for (size_t i = 0; i < infer_requests_num; i++) { + FuncTestUtils::fill_tensor(input_tensors[i], 10, -5, 1, i); + calc_ref_results(input_tensors[i]); + } + + // Swap tensors between IRs + infer_request1.set_output_tensor(output_tensors[1]); + infer_request2.set_output_tensor(output_tensors[0]); + + const int niter_limit = 10; + int iter1 = 0, iter2 = 0; + infer_request1.set_callback([&](std::exception_ptr exception_ptr) mutable { + if (exception_ptr) { + std::rethrow_exception(exception_ptr); + } else { + iter1++; + ov::Tensor output_tensor = infer_request1.get_output_tensor(); + compare_results(output_tensor, ref[iter1 % 2].data()); + if (iter1 < niter_limit) { + infer_request1.set_output_tensor(output_tensors[(iter1 + 1) % 2]); + infer_request1.start_async(); + } + } + }); + + infer_request2.set_callback([&](std::exception_ptr exception_ptr) mutable { + if (exception_ptr) { + std::rethrow_exception(exception_ptr); + } else { + iter2++; + ov::Tensor output_tensor = infer_request2.get_output_tensor(); + compare_results(output_tensor, ref[(iter2 + 1) % 2].data()); + if (iter2 < niter_limit) { + infer_request2.set_output_tensor(output_tensors[iter2 % 2]); + infer_request2.start_async(); + } + } + }); + + infer_request1.start_async(); + infer_request2.start_async(); + + for (size_t i = 0; i < niter_limit; i++) { + infer_request1.wait(); + infer_request2.wait(); + } +} diff --git a/src/tests/ie_test_utils/common_test_utils/data_utils.hpp b/src/tests/ie_test_utils/common_test_utils/data_utils.hpp index 6eec22874f5..da2820145d9 100644 --- a/src/tests/ie_test_utils/common_test_utils/data_utils.hpp +++ b/src/tests/ie_test_utils/common_test_utils/data_utils.hpp @@ -12,6 +12,7 @@ #include "openvino/runtime/tensor.hpp" #include #include +#include #include #include @@ -29,7 +30,7 @@ inline void fill_data(float *data, size_t size, size_t duty_ratio = 10) { } /** - * @brief Create vector of floats with length of vec_len, with values ranging from min to max, + * @brief Create vector of floats with length of vec_len, with values ranging from min to max, * with initial seed equal to variable seed with default of 0 */ inline std::vector generate_float_numbers(std::size_t vec_len, float min, float max, int seed = 0) { @@ -253,6 +254,52 @@ void inline fill_random_unique_sequence(T* rawBlobDataPtr, std::copy(elems.begin(), elems.end(), rawBlobDataPtr); } +/** @brief Fill tensor with random data. + * + * @param tensor Target tensor + * @param range Values range + * @param start_from Value from which range should start + * @param k Resolution of floating point numbers. + * - With k = 1 every random number will be basically integer number. + * - With k = 2 numbers resolution will 1/2 so outputs only .0 or .50 + * - With k = 4 numbers resolution will 1/4 so outputs only .0 .25 .50 0.75 and etc. + */ +template +void inline fill_tensor_random(ov::Tensor& tensor, const uint32_t range = 10, int32_t start_from = 0, + const int32_t k = 1, const int seed = 1) { + using T = typename ov::element_type_traits
::value_type; + auto *rawBlobDataPtr = static_cast(tensor.data()); + if (DT == ov::element::u4 || DT == ov::element::i4 || + DT == ov::element::u1) { + fill_data_random(rawBlobDataPtr, tensor.get_byte_size(), range, start_from, k, seed); + } else { + fill_data_random(rawBlobDataPtr, tensor.get_size(), range, start_from, k, seed); + } +} + +template +void inline +fill_tensor_random_float(ov::Tensor& tensor, const uint32_t range, int32_t start_from, const int32_t k, + const int seed = 1) { + using T = typename ov::element_type_traits
::value_type; + std::default_random_engine random(seed); + // 1/k is the resolution of the floating point numbers + std::uniform_int_distribution distribution(k * start_from, k * (start_from + range)); + + auto *rawBlobDataPtr = static_cast(tensor.data()); + for (size_t i = 0; i < tensor.get_size(); i++) { + auto value = static_cast(distribution(random)); + value /= static_cast(k); + if (DT == ov::element::Type_t::f16) { + rawBlobDataPtr[i] = static_cast(ngraph::float16(value).to_bits()); + } else if (DT == ov::element::Type_t::bf16) { + rawBlobDataPtr[i] = static_cast(ngraph::bfloat16(value).to_bits()); + } else { + rawBlobDataPtr[i] = static_cast(value); + } + } +} + /** @brief Fill blob with random data. * * @param blob Target blob diff --git a/src/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/blob_utils.hpp b/src/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/blob_utils.hpp index bdfd8231cb9..d527c549f5a 100644 --- a/src/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/blob_utils.hpp +++ b/src/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/blob_utils.hpp @@ -572,6 +572,37 @@ inline ov::Tensor create_and_fill_tensor( return tensor; } +void inline fill_tensor(ov::Tensor& tensor, + uint32_t range = 10, + int32_t start_from = 0, + const int32_t k = 1, + const int seed = 1) { + auto element_type = tensor.get_element_type(); + switch (element_type) { +#define CASE(X) case X: CommonTestUtils::fill_tensor_random(tensor, range, start_from, k, seed); break; +#define CASE_FLOAT(X) case X: CommonTestUtils::fill_tensor_random_float(tensor, range, start_from, k, seed); break; + CASE_FLOAT(ov::element::f64) + CASE_FLOAT(ov::element::f32) + CASE_FLOAT(ov::element::f16) + CASE_FLOAT(ov::element::bf16) + CASE(ov::element::u1) + CASE(ov::element::u4) + CASE(ov::element::u8) + CASE(ov::element::u32) + CASE(ov::element::u16) + CASE(ov::element::u64) + CASE(ov::element::i4) + CASE(ov::element::i8) + CASE(ov::element::i16) + CASE(ov::element::i32) + CASE(ov::element::i64) +#undef CASE +#undef CASE_FLOAT + default: + IE_THROW() << "Wrong precision specified: " << element_type; + } +} + inline InferenceEngine::Blob::Ptr createAndFillBlobConsistently( const InferenceEngine::TensorDesc &td, const uint32_t range,