[GPU] Move device memory allocation from preprocessing stage to runtime for usm memory type (#13697)

* [GPU] Move device memory allocation from preprocessing stage to runtime to fix incorrect behaviour in case of tensor swapping

* [GPU] Apply review comments

* Replace source of dynamic layout cheking

src/plugins/intel_gpu/include/intel_gpu/plugin/infer_request.hpp

@@ -57,6 +57,8 @@ public:
     void enable_external_queue() { m_useExternalQueue = true; }
 
 private:
+    // This blob is used for outputs processing if output data type convertion or padding handling is needed
+    InferenceEngine::Blob::Ptr intermediate_output_blob = nullptr;
     InferenceEngine::BlobMap _deviceOutputs;
     std::map<std::string, cldnn::primitive_id> inputsMap;
     std::map<std::string, cldnn::primitive_id> outputsMap;
@@ -73,12 +75,14 @@ private:
     void prepare_input(const cldnn::primitive_id &inputName, InferenceEngine::Blob::Ptr &inputBlob,
                        std::vector<cldnn::event::ptr>& dependencies);
     void prepare_output(const cldnn::primitive_id& outputName, InferenceEngine::Blob::Ptr& outputBlob);
+    void allocate_dev_mem_if_needed(InferenceEngine::BlobMap& device_mems, InferenceEngine::Blob::Ptr& user_blob,
+                                    const cldnn::primitive_id& blob_name, const cldnn::layout& layout);
 
     InferenceEngine::Blob::Ptr create_host_blob(const InferenceEngine::TensorDesc& desc, bool is_dynamic);
     InferenceEngine::Blob::Ptr create_device_blob(const InferenceEngine::TensorDesc& desc);
 
     void copy_output_data(cldnn::memory::ptr outputMemory, InferenceEngine::Blob::Ptr bptr);
-    void copy_input_data(std::shared_ptr<cldnn::network> network, const cldnn::primitive_id &inputName,
+    void copy_input_data(std::shared_ptr<cldnn::network> network, const cldnn::primitive_id& inputName,
                          const cldnn::layout& inputLayout, const InferenceEngine::Blob &inputBlob);
 
     InferenceEngine::Blob::Ptr create_shared_device_blob(const InferenceEngine::TensorDesc& desc, const cldnn::layout& layout, void* usm_host_mem);

src/plugins/intel_gpu/include/intel_gpu/runtime/engine.hpp

@@ -137,6 +137,8 @@ public:
     /// Returns service stream which can be used during program build and optimizations
     virtual stream& get_program_stream() const = 0;
 
+    virtual allocation_type detect_usm_allocation_type(const void* memory) const = 0;
+
 #ifdef ENABLE_ONEDNN_FOR_GPU
     /// Returns onednn engine object which shares device and context with current engine
     virtual dnnl::engine& get_onednn_engine() const = 0;

src/plugins/intel_gpu/include/intel_gpu/runtime/memory.hpp

@@ -71,6 +71,9 @@ struct memory {
     virtual event::ptr copy_from(stream& /* stream */, const memory& /* other */) = 0;
     virtual event::ptr copy_from(stream& /* stream */, const void* /* host_ptr */) = 0;
 
+    virtual event::ptr copy_to(stream& stream, memory& other) { return other.copy_from(stream, *this); }
+    virtual event::ptr copy_to(stream& /* stream */, void* /* host_ptr */) = 0;
+
 #ifdef ENABLE_ONEDNN_FOR_GPU
     virtual dnnl::memory get_onednn_memory(dnnl::memory::desc /* desc */, int64_t offset = 0) {
         throw std::runtime_error("[CLDNN] Can't convert memory object to onednn");
@@ -108,6 +111,9 @@ struct simple_attached_memory : memory {
     event::ptr copy_from(stream& /* stream */, const memory& /* other */) override { return nullptr; };
     event::ptr copy_from(stream& /* stream */, const void* /* host_ptr */) override { return nullptr; }
 
+    event::ptr copy_to(stream& /* stream */, memory& /* other */) override { return nullptr; };
+    event::ptr copy_to(stream& /* stream */, void* /* host_ptr */) override { return nullptr; }
+
 private:
     void* _pointer;
 };

src/plugins/intel_gpu/src/plugin/infer_request.cpp

@@ -23,6 +23,8 @@ using namespace InferenceEngine;
 
 namespace {
 
+const char str_device_input_unsupported_blob[] = "Device input is of an unsupported blob type.";
+const char str_device_output_unsupported_blob[] = "Device output is of an unsupported blob type.";
 const char str_input_not_allocated[] = "Input data was not allocated.";
 const char str_output_not_allocated[] = "Output data was not allocated.";
 
@@ -46,28 +48,24 @@ void convertAndCopy(const InferenceEngine::Blob* src, dst_t* dst) {
 }
 
 template<typename src_dt, typename dst_dt>
-void copyResultToOutputBlob(cldnn::memory::ptr src, Blob::Ptr dst, cldnn::stream& stream) {
-    size_t n = dst->size();
-
-    auto layout = src->get_layout();
-    auto size = layout.get_tensor();
+void copy_result_to_output_blob(InferenceEngine::Blob::Ptr src, InferenceEngine::Blob::Ptr dst, const cldnn::layout& src_layout) {
+    auto locked_src = src->buffer();
+    auto src_ptr = locked_src.as<src_dt*>();
+    OPENVINO_ASSERT(src_ptr, "[GPU] Invalid source blob");
 
     auto locked_dst = dst->buffer();
     auto dst_ptr = locked_dst.as<dst_dt*>();
-    if (dst_ptr == nullptr) {
-        IE_THROW() << "Invalid output blob";
-    }
-    cldnn::mem_lock<src_dt> src_lock{ src, stream };
-    src_dt* src_ptr = src_lock.data();
+    OPENVINO_ASSERT(dst_ptr, "[GPU] Invalid output blob");
 
-    if (layout.data_padding) {
+    if (src_layout.data_padding) {
+        auto size = src_layout.get_tensor();
         for (int64_t b = 0; b < size.batch[0]; b++) {
             for (int64_t f = 0; f < size.feature[0]; f++) {
                 for (int64_t w = 0; w < size.spatial[3]; w++) {
                     for (int64_t z = 0; z < size.spatial[2]; z++) {
                         for (int64_t y = 0; y < size.spatial[1]; y++) {
                             for (int64_t x = 0; x < size.spatial[0]; x++) {
-                                *dst_ptr++ = src_ptr[layout.get_linear_offset(cldnn::tensor(b, f, x, y, z, w))];
+                                *dst_ptr++ = src_ptr[src_layout.get_linear_offset(cldnn::tensor(b, f, x, y, z, w))];
                             }
                         }
                     }
@@ -75,6 +73,7 @@ void copyResultToOutputBlob(cldnn::memory::ptr src, Blob::Ptr dst, cldnn::stream
             }
         }
     } else {
+        size_t n = dst->size();
         for (size_t i = 0; i < n; i++) {
             dst_ptr[i] = src_ptr[i];
         }
@@ -252,7 +251,7 @@ void InferRequest::set_input(const std::string& name, const Blob::Ptr& data) {
 }
 
 void InferRequest::set_output(const std::string& name, const Blob::Ptr& data) {
-        auto remote_ptr = data->as<gpu::ClBlob>();
+    auto remote_ptr = data->as<gpu::ClBlob>();
     bool is_remote = remote_ptr != nullptr;
 
     auto node = findOutputByNodeName(name);
@@ -620,21 +619,59 @@ Blob::Ptr InferRequest::create_shared_device_blob(const InferenceEngine::TensorD
 
 void InferRequest::copy_output_data(cldnn::memory::ptr src, Blob::Ptr dst) {
     OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "InferRequest::copy_output_data");
+    auto is_convert_needed = [](const Precision& prc) {
+        const std::vector<Precision> convert_needed = { Precision::I16, Precision::U16, Precision::FP64,
+                                                        Precision::U32, Precision::U64 };
+        const std::vector<Precision> convert_not_needed = { Precision::FP32, Precision::FP16, Precision::I64, Precision::I32,
+                                                            Precision::I8, Precision::U8, Precision::BOOL };
+
+        if (std::find(convert_needed.begin(), convert_needed.end(), prc) != convert_needed.end())
+            return true;
+        else if (std::find(convert_not_needed.begin(), convert_not_needed.end(), prc) != convert_not_needed.end())
+            return false;
+        else
+            OPENVINO_ASSERT(false, "[GPU] Plugin does not support output ", prc, " precision");
+    };
+
+    const auto convert_needed = is_convert_needed(dst->getTensorDesc().getPrecision());
+    const auto src_layout = src->get_layout();
     auto& stream = m_graph->GetNetwork()->get_stream();
-    switch (dst->getTensorDesc().getPrecision()) {
-    case Precision::FP64: copyResultToOutputBlob<float, double>(src, dst, stream);    break;
-    case Precision::FP32: copyResultToOutputBlob<float, float>(src, dst, stream);    break;
-    case Precision::FP16: copyResultToOutputBlob<uint16_t, uint16_t>(src, dst, stream); break;
-    case Precision::I64:  copyResultToOutputBlob<int64_t, int64_t>(src, dst, stream);  break;
-    case Precision::I32:  copyResultToOutputBlob<int32_t, int32_t>(src, dst, stream);  break;
-    case Precision::I16:  copyResultToOutputBlob<float, int16_t>(src, dst, stream);  break;
-    case Precision::I8:   copyResultToOutputBlob<int8_t, int8_t>(src, dst, stream);  break;
-    case Precision::U16:  copyResultToOutputBlob<float, uint16_t>(src, dst, stream);  break;
-    case Precision::U32:  copyResultToOutputBlob<int32_t, uint32_t>(src, dst, stream);  break;
-    case Precision::U64:  copyResultToOutputBlob<int32_t, uint64_t>(src, dst, stream);  break;
-    case Precision::U8:   copyResultToOutputBlob<uint8_t, uint8_t>(src, dst, stream);  break;
-    case Precision::BOOL: copyResultToOutputBlob<int8_t, int8_t>(src, dst, stream);  break;
-    default: IE_THROW(NotImplemented) << "The plugin does not support output " << dst->getTensorDesc().getPrecision() << " precision";
+
+    if (convert_needed || src_layout.data_padding) {
+        if (!intermediate_output_blob || intermediate_output_blob->byteSize() < src_layout.bytes_count()) {
+            auto desc = TensorDesc(Precision::U8,
+                                   SizeVector{src_layout.bytes_count()},
+                                   InferenceEngine::Layout::C);
+            intermediate_output_blob = create_host_blob(desc, src_layout.is_dynamic());
+        }
+
+        OPENVINO_ASSERT(intermediate_output_blob, "[GPU] Intermediate blob for outputs precessing is not allocated");
+
+        auto event = src->copy_to(stream, intermediate_output_blob->buffer());
+        event->wait();
+
+        switch (dst->getTensorDesc().getPrecision()) {
+        #define CASE(PRC, SRC_DT, DST_DT) \
+            case PRC: copy_result_to_output_blob<SRC_DT, DST_DT> (intermediate_output_blob, dst, src_layout); break;
+        CASE(Precision::FP64, float,    double)
+        CASE(Precision::FP32, float,    float)
+        CASE(Precision::FP16, uint16_t, uint16_t)
+        CASE(Precision::I64,  int64_t,  int64_t)
+        CASE(Precision::I32,  int32_t,  int32_t)
+        CASE(Precision::I16,  float,    int16_t)
+        CASE(Precision::I8,   int8_t,   int8_t)
+        CASE(Precision::U16,  float,    uint16_t)
+        CASE(Precision::U32,  int32_t,  uint32_t)
+        CASE(Precision::U64,  int32_t,  uint64_t)
+        CASE(Precision::U8,   uint8_t,  uint8_t)
+        CASE(Precision::BOOL, int8_t,   int8_t)
+        #undef CASE
+        default: break;
+        }
+    } else {
+        auto dst_ptr = dst->buffer().as<void*>();
+        auto event = src->copy_to(stream, dst_ptr);
+        event->wait();
     }
 }
 
@@ -714,11 +751,8 @@ void InferRequest::allocate_inputs() {
 
         if (!is_nv12_input) {
             auto litr = inputLayouts.find(name);
-            if (litr == inputLayouts.end()) {
-                IE_THROW() << "Input layout for " << name << " is not found";
-            }
-
-            auto input_layout = litr->second;
+            OPENVINO_ASSERT(litr != inputLayouts.end(), "[GPU] Input layout for ", name, " is not found");
+            const auto input_layout = litr->second;
 
             GPU_DEBUG_GET_INSTANCE(debug_config);
             GPU_DEBUG_IF(debug_config->verbose >= 2) {
@@ -732,15 +766,10 @@ void InferRequest::allocate_inputs() {
                     _deviceInputs[name] = create_device_blob(desc_fp32);
             } else {
                 _inputs[name] = create_host_blob(desc, input_layout.is_dynamic());
-                if (input_layout.is_static()) {
-                    if (m_graph->GetEngine()->use_unified_shared_memory()) {
-                        // For USM case we create host blob using custom USM host allocator
-                        // and then create shared device blob on top of this buffer
-                        auto host_blob = _inputs[name];
-                        _deviceInputs[name] = create_shared_device_blob(desc, input_layout, host_blob->buffer().as<void*>());
-                    } else {
-                        _deviceInputs[name] = create_device_blob(desc);
-                    }
+                // Pre-allocate device input only if USM is not supported; in other case it will be allocated
+                // in prepare_input() function later
+                if (input_layout.is_static() && !m_graph->GetEngine()->use_unified_shared_memory()) {
+                    _deviceInputs[name] = create_device_blob(desc);
                 }
             }
         }
@@ -782,15 +811,10 @@ void InferRequest::allocate_outputs() {
                 _deviceOutputs[no.first] = create_device_blob(device_blob_desc);
         } else {
             _outputs[no.first] = create_host_blob(desc, output_layout.is_dynamic());
-            if (output_layout.is_static()) {
-                if (m_graph->GetEngine()->use_unified_shared_memory()) {
-                    // For USM case we create host blob using custom USM host allocator
-                    // and then create shared device blob on top of this buffer
-                    auto host_blob = _outputs[no.first];
-                    _deviceOutputs[no.first] = create_shared_device_blob(desc, output_layout, host_blob->buffer().as<void*>());
-                } else {
-                    _deviceOutputs[no.first] = create_device_blob(desc);
-                }
+            // Pre-allocate device output only if USM is not supported; in other case it will be allocated
+            // in prepare_output() function later
+            if (output_layout.is_static() && !m_graph->GetEngine()->use_unified_shared_memory()) {
+                _deviceOutputs[no.first] = create_device_blob(desc);
             }
         }
     }
@@ -813,14 +837,53 @@ std::map<std::string, InferenceEngineProfileInfo> InferRequest::GetPerformanceCo
     }
 }
 
+void InferRequest::allocate_dev_mem_if_needed(InferenceEngine::BlobMap& device_mems, InferenceEngine::Blob::Ptr& user_blob,
+                                              const cldnn::primitive_id& blob_name, const cldnn::layout& layout) {
+    const auto input_ptr = static_cast<const void*>(user_blob->cbuffer());
+    const auto alloc_type = m_graph->GetEngine()->detect_usm_allocation_type(input_ptr);
+    const auto is_usm_host = alloc_type == cldnn::allocation_type::usm_host;
+    const auto has_device_blob = device_mems.find(blob_name) != device_mems.end();
+    bool can_skip_allocation = false;
+
+    if (has_device_blob) {
+        auto impl = getBlobImpl(device_mems[blob_name]->as<gpu::ClBlob>());
+
+        OPENVINO_ASSERT(impl, str_device_output_unsupported_blob);
+        OPENVINO_ASSERT(impl->is_allocated(), str_input_not_allocated);
+
+        auto impl_mem = impl->getMemory();
+        auto src_ptr = user_blob->cbuffer().as<uint8_t*>();
+        // If device mem already exists, we can reuse blob if buffer has usm_host type and points to the same memory,
+        // so we don't need to allocate new memory
+        can_skip_allocation |= same_host_mem(impl_mem, src_ptr);
+        // Or if blob has any type except usm_host - in that case explicit copy will be performed anyway
+        can_skip_allocation |= impl_mem->get_allocation_type() != cldnn::allocation_type::usm_host;
+    }
+
+    if (!can_skip_allocation) {
+        if (is_usm_host) {
+            // For USM case we create host blob using custom USM host allocator
+            // and then create shared device blob on top of this buffer
+            device_mems[blob_name] = create_shared_device_blob(user_blob->getTensorDesc(), layout, user_blob->buffer().as<void*>());
+        } else {
+            device_mems[blob_name] = create_device_blob(user_blob->getTensorDesc());
+        }
+    }
+}
+
 void InferRequest::prepare_input(const cldnn::primitive_id& inputName, Blob::Ptr& inputBlob,
                                       std::vector<cldnn::event::ptr>& dependencies) {
     OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "InferRequest::prepare_input");
     auto inputLayoutItr = m_graph->GetInputLayouts().find(inputName);
-    if (inputLayoutItr == m_graph->GetInputLayouts().end()) {
-        IE_THROW() << "Input name mismatch.";
-    }
+    OPENVINO_ASSERT(inputLayoutItr != m_graph->GetInputLayouts().end(), "[GPU] Input name mismatch");
+
     auto input_layout = inputLayoutItr->second;
+    auto& prec = inputBlob->getTensorDesc().getPrecision();
+    auto remote_ptr = inputBlob->as<gpu::ClBlob>();
+    auto& stream = m_graph->GetNetwork()->get_stream();
+    const bool is_dev_input = remote_ptr != nullptr;
+    const bool can_use_usm = m_graph->GetEngine()->use_unified_shared_memory();
+
     if (input_layout.is_dynamic()) {
         bool has_device_blob = _deviceInputs.find(inputName) != _deviceInputs.end();
         bool should_allocate_device_blob = !has_device_blob;
@@ -836,15 +899,13 @@ void InferRequest::prepare_input(const cldnn::primitive_id& inputName, Blob::Ptr
         } else {
             _deviceInputs[inputName] = reinterpret_device_blob(_deviceInputs[inputName], inputBlob->getTensorDesc());
         }
+    } else if (input_layout.is_static() && !is_dev_input && can_use_usm) {
+        allocate_dev_mem_if_needed(_deviceInputs, inputBlob, inputName, input_layout);
     }
     OPENVINO_ASSERT(_deviceInputs.find(inputName) != _deviceInputs.end(), "[GPU] Couldn't find device blob allocated for ", inputName, " input");
     auto reqBlob = _deviceInputs.at(inputName)->as<gpu::ClBlob>();
     auto _nw_ptr = m_graph->GetNetwork();
-    cldnn::primitive_id internalName = "parameter:" + inputName;
-    const auto& prec = inputBlob->getTensorDesc().getPrecision();
-    auto remote_ptr = inputBlob->as<gpu::ClBlob>();
-    auto& stream = m_graph->GetNetwork()->get_stream();
-    bool is_dev_input = remote_ptr != nullptr;
+    const cldnn::primitive_id internalName = "parameter:" + inputName;
 
     switch (prec) {
         case Precision::FP64:
@@ -875,6 +936,7 @@ void InferRequest::prepare_input(const cldnn::primitive_id& inputName, Blob::Ptr
             }
 
             if (!is_dev_input) {
+                // TODO: Remove this checks once 95363 issue is solved
                 if (prec == Precision::I16 || prec == Precision::U16 || prec == Precision::FP64) {
                     // GPU plugin doesn't support I16 input precision,
                     // so have to convert input data to fp32 precision
@@ -912,14 +974,26 @@ void InferRequest::prepare_input(const cldnn::primitive_id& inputName, Blob::Ptr
 
 void InferRequest::prepare_output(const cldnn::primitive_id& outputName, Blob::Ptr& outputBlob) {
     OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "InferRequest::prepare_output");
+    const auto output_id = outputsMap.at(outputName);
+    const auto output_layout = m_graph->GetNetwork()->get_node_output_layout(output_id);
+    const bool is_static = output_layout.is_static();
+    const bool can_use_usm = m_graph->GetEngine()->use_unified_shared_memory();
+    auto remote_ptr = outputBlob->as<gpu::ClBlob>();
+    const bool is_dev_input = remote_ptr != nullptr;
+
+    if (is_static && can_use_usm && !is_dev_input) {
+        allocate_dev_mem_if_needed(_deviceOutputs, outputBlob, outputName, output_layout);
+    }
+
+    OPENVINO_ASSERT(!is_static || _deviceOutputs.find(outputName) != _deviceOutputs.end(),
+                    "[GPU] Couldn't find device blob allocated for ", outputName, " output");
     // Missing output in _deviceOutputs means that the network is dynamic and outputs couldn't be pre-allocated
     if (_deviceOutputs.find(outputName) == _deviceOutputs.end())
         return;
     Blob::Ptr reqBlob = _deviceOutputs.at(outputName);
     cldnn::primitive_id internalName = outputsMap[outputName];
     auto _nw_ptr = m_graph->GetNetwork();
-    auto remote_ptr = outputBlob->as<gpu::ClBlob>();
-    auto output_blob_ptr = (reqBlob != outputBlob && remote_ptr != nullptr)
+    auto output_blob_ptr = (reqBlob != outputBlob && is_dev_input)
         ? remote_ptr
         : reqBlob->as<gpu::ClBlob>();
     auto impl = getBlobImpl(output_blob_ptr);

src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.cpp

@@ -89,6 +89,11 @@ const cl::UsmHelper& ocl_engine::get_usm_helper() const {
     return *_usm_helper;
 }
 
+allocation_type ocl_engine::detect_usm_allocation_type(const void* memory) const {
+    return use_unified_shared_memory() ? ocl::gpu_usm::detect_allocation_type(this, memory)
+                                       : allocation_type::unknown;
+}
+
 memory::ptr ocl_engine::allocate_memory(const layout& layout, allocation_type type, bool reset) {
     OPENVINO_ASSERT(!layout.is_dynamic(), "[GPU] Can't allocate memory for dynamic layout");
 

src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.hpp

@@ -32,6 +32,7 @@ public:
     void* get_user_context() const override;
 
     allocation_type get_default_allocation_type() const override { return allocation_type::cl_mem; }
+    allocation_type detect_usm_allocation_type(const void* memory) const override;
 
     const cl::Context& get_cl_context() const;
     const cl::Device& get_cl_device() const;

src/plugins/intel_gpu/src/runtime/ocl/ocl_memory.cpp

@@ -110,6 +110,15 @@ event::ptr gpu_buffer::copy_from(stream& stream, const void* host_ptr) {
     return ev;
 }
 
+event::ptr gpu_buffer::copy_to(stream& stream, void* host_ptr) {
+    auto& cl_stream = downcast<ocl_stream>(stream);
+    auto ev = stream.create_base_event();
+    cl::Event ev_ocl = downcast<ocl_event>(ev.get())->get();
+    cl_stream.get_cl_queue().enqueueReadBuffer(_buffer, false, 0, size(), host_ptr, nullptr, &ev_ocl);
+
+    return ev;
+}
+
 #ifdef ENABLE_ONEDNN_FOR_GPU
 dnnl::memory gpu_buffer::get_onednn_memory(dnnl::memory::desc desc, int64_t offset) {
     auto onednn_engine = _engine->get_onednn_engine();
@@ -237,11 +246,19 @@ shared_mem_params gpu_image2d::get_internal_params() const {
 }
 
 event::ptr gpu_image2d::copy_from(stream& /* stream */, const memory& /* other */) {
-    throw std::runtime_error("[clDNN] copy_from is not implemented for gpu_image2d");
+    throw std::runtime_error("[GPU] copy_from is not implemented for gpu_image2d");
 }
 
 event::ptr gpu_image2d::copy_from(stream& /* stream */, const void* /* host_ptr */) {
-    throw std::runtime_error("[clDNN] copy_from is not implemented for gpu_image2d");
+    throw std::runtime_error("[GPU] copy_from is not implemented for gpu_image2d");
+}
+
+event::ptr gpu_image2d::copy_to(stream& /* stream */, memory& /* other */) {
+    throw std::runtime_error("[GPU] copy_to is not implemented for gpu_image2d");
+}
+
+event::ptr gpu_image2d::copy_to(stream& /* stream */, void* /* host_ptr */) {
+    throw std::runtime_error("[GPU] copy_to is not implemented for gpu_image2d");
 }
 
 gpu_media_buffer::gpu_media_buffer(ocl_engine* engine,
@@ -384,15 +401,24 @@ event::ptr gpu_usm::copy_from(stream& stream, const memory& other) {
 
 event::ptr gpu_usm::copy_from(stream& stream, const void* host_ptr) {
     auto& cl_stream = downcast<ocl_stream>(stream);
-    auto ev = stream.create_base_event();
     auto dst_ptr = get_buffer().get();
     cl_stream.get_usm_helper().enqueue_memcpy(cl_stream.get_cl_queue(),
                                               dst_ptr,
                                               host_ptr,
                                               _bytes_count,
                                               true);
+    return stream.create_user_event(true);
+}
 
-    return ev;
+event::ptr gpu_usm::copy_to(stream& stream, void* host_ptr) {
+    auto& cl_stream = downcast<ocl_stream>(stream);
+    auto src_ptr = get_buffer().get();
+    cl_stream.get_usm_helper().enqueue_memcpy(cl_stream.get_cl_queue(),
+                                              host_ptr,
+                                              src_ptr,
+                                              _bytes_count,
+                                              true);
+    return stream.create_user_event(true);
 }
 
 #ifdef ENABLE_ONEDNN_FOR_GPU
@@ -420,20 +446,28 @@ shared_mem_params gpu_usm::get_internal_params() const {
     };
 }
 
-allocation_type gpu_usm::detect_allocation_type(ocl_engine* engine, const cl::UsmMemory& buffer) {
-    auto cl_alloc_type = engine->get_usm_helper().get_usm_allocation_type(buffer.get());
+allocation_type gpu_usm::detect_allocation_type(const ocl_engine* engine, const void* mem_ptr) {
+    auto cl_alloc_type = engine->get_usm_helper().get_usm_allocation_type(mem_ptr);
 
-    allocation_type res = allocation_type::unknown;
+    allocation_type res;
     switch (cl_alloc_type) {
         case CL_MEM_TYPE_DEVICE_INTEL: res = allocation_type::usm_device; break;
         case CL_MEM_TYPE_HOST_INTEL: res = allocation_type::usm_host; break;
         case CL_MEM_TYPE_SHARED_INTEL: res = allocation_type::usm_shared; break;
-        default: throw std::runtime_error("[GPU] Unsupported USM alloc type: " + std::to_string(cl_alloc_type));
+        default: res = allocation_type::unknown;
     }
 
     return res;
 }
 
+allocation_type gpu_usm::detect_allocation_type(const ocl_engine* engine, const cl::UsmMemory& buffer) {
+    auto alloc_type = detect_allocation_type(engine, buffer.get());
+    OPENVINO_ASSERT(alloc_type == allocation_type::usm_device ||
+                    alloc_type == allocation_type::usm_host ||
+                    alloc_type == allocation_type::usm_shared, "[GPU] Unsupported USM alloc type: " + to_string(alloc_type));
+    return alloc_type;
+}
+
 std::vector<cl_mem> ocl_surfaces_lock::get_handles(std::vector<memory::ptr> mem) const {
     std::vector<cl_mem> res;
     for (auto& m : mem) {

src/plugins/intel_gpu/src/runtime/ocl/ocl_memory.hpp

@@ -42,6 +42,9 @@ struct gpu_buffer : public lockable_gpu_mem, public memory {
 
     event::ptr copy_from(stream& stream, const memory& other) override;
     event::ptr copy_from(stream& stream, const void* host_ptr) override;
+
+    event::ptr copy_to(stream& /* stream */, void* /* other */) override;
+
 #ifdef ENABLE_ONEDNN_FOR_GPU
     dnnl::memory get_onednn_memory(dnnl::memory::desc /* desc */, int64_t offset = 0) override;
 #endif
@@ -67,6 +70,9 @@ struct gpu_image2d : public lockable_gpu_mem, public memory {
     event::ptr copy_from(stream& /* stream */, const memory& /* other */) override;
     event::ptr copy_from(stream& /* stream */, const void* /* other */) override;
 
+    event::ptr copy_to(stream& /* stream */, memory& /* other */) override;
+    event::ptr copy_to(stream& /* stream */, void* /* other */) override;
+
 protected:
     cl::Image2D _buffer;
     size_t _width;
@@ -116,15 +122,18 @@ struct gpu_usm : public lockable_gpu_mem, public memory {
     event::ptr copy_from(stream& stream, const memory& other) override;
     event::ptr copy_from(stream& stream, const void* host_ptr) override;
 
+    event::ptr copy_to(stream& stream, void* host_ptr) override;
 #ifdef ENABLE_ONEDNN_FOR_GPU
     dnnl::memory get_onednn_memory(dnnl::memory::desc /* desc */, int64_t offset = 0) override;
 #endif
 
+    static allocation_type detect_allocation_type(const ocl_engine* engine, const void* mem_ptr);
+
 protected:
     cl::UsmMemory _buffer;
     cl::UsmMemory _host_buffer;
 
-    static allocation_type detect_allocation_type(ocl_engine* engine, const cl::UsmMemory& buffer);
+    static allocation_type detect_allocation_type(const ocl_engine* engine, const cl::UsmMemory& buffer);
 };
 
 struct ocl_surfaces_lock : public surfaces_lock {

src/tests/functional/plugin/gpu/concurrency/gpu_concurrency_tests.cpp

@@ -108,3 +108,166 @@ INSTANTIATE_TEST_SUITE_P(smoke_RemoteTensor, OVConcurrencyTest,
     ::testing::Combine(::testing::ValuesIn(num_streams),
         ::testing::ValuesIn(num_requests)),
     OVConcurrencyTest::getTestCaseName);
+
+TEST(canSwapTensorsBetweenInferRequests, inputs) {
+    std::vector<std::vector<uint8_t>> ref;
+    std::vector<ov::Tensor> input_tensors;
+    auto fn = ngraph::builder::subgraph::makeSplitMultiConvConcat();
+
+    auto ie = ov::Core();
+    auto compiled_model = ie.compile_model(fn, CommonTestUtils::DEVICE_GPU);
+
+    const int infer_requests_num = 2;
+    ov::InferRequest infer_request1 = compiled_model.create_infer_request();
+    ov::InferRequest infer_request2 = compiled_model.create_infer_request();
+
+    input_tensors.push_back(infer_request1.get_input_tensor());
+    input_tensors.push_back(infer_request2.get_input_tensor());
+
+    auto calc_ref_results = [&](const ov::Tensor& tensor){
+        const auto tensor_size = tensor.get_byte_size();
+        const auto in_blob_buf = static_cast<uint8_t*>(tensor.data());
+        std::vector<uint8_t> inData(in_blob_buf, in_blob_buf + tensor_size);
+        auto ref_out_data = ngraph::helpers::interpreterFunction(fn, {inData}).front().second;
+        ref.push_back(ref_out_data);
+    };
+
+    auto compare_results = [&](ov::Tensor& result, const uint8_t* refResult) {
+        auto thr = FuncTestUtils::GetComparisonThreshold(InferenceEngine::Precision::FP32);
+        ASSERT_EQ(ov::shape_size(fn->get_output_shape(0)), result.get_size());
+        FuncTestUtils::compareRawBuffers(result.data<float>(),
+                                        reinterpret_cast<const float *>(refResult), ov::shape_size(fn->get_output_shape(0)),
+                                        ov::shape_size(fn->get_output_shape(0)),
+                                        thr);
+    };
+
+    for (size_t i = 0; i < infer_requests_num; i++) {
+        FuncTestUtils::fill_tensor(input_tensors[i], 10, -5, 1, i);
+        calc_ref_results(input_tensors[i]);
+    }
+
+    // Swap tensors between IRs
+    infer_request1.set_input_tensor(input_tensors[1]);
+    infer_request2.set_input_tensor(input_tensors[0]);
+
+    const int niter_limit = 10;
+    int iter1 = 0, iter2 = 0;
+    infer_request1.set_callback([&](std::exception_ptr exception_ptr) mutable {
+        if (exception_ptr) {
+            std::rethrow_exception(exception_ptr);
+        } else {
+            iter1++;
+            ov::Tensor output_tensor = infer_request1.get_output_tensor();
+            compare_results(output_tensor, ref[iter1 % 2].data());
+            if (iter1 < niter_limit) {
+                infer_request1.set_input_tensor(input_tensors[(iter1 + 1) % 2]);
+                infer_request1.start_async();
+            }
+        }
+    });
+
+    infer_request2.set_callback([&](std::exception_ptr exception_ptr) mutable {
+        if (exception_ptr) {
+            std::rethrow_exception(exception_ptr);
+        } else {
+            iter2++;
+            ov::Tensor output_tensor = infer_request2.get_output_tensor();
+            compare_results(output_tensor, ref[(iter2 + 1) % 2].data());
+            if (iter2 < niter_limit) {
+                infer_request2.set_input_tensor(input_tensors[iter2 % 2]);
+                infer_request2.start_async();
+            }
+        }
+    });
+
+    infer_request1.start_async();
+    infer_request2.start_async();
+
+    for (size_t i = 0; i < niter_limit; i++) {
+        infer_request1.wait();
+        infer_request2.wait();
+    }
+}
+
+TEST(canSwapTensorsBetweenInferRequests, outputs) {
+    std::vector<std::vector<uint8_t>> ref;
+    std::vector<ov::Tensor> input_tensors;
+    std::vector<ov::Tensor> output_tensors;
+    auto fn = ngraph::builder::subgraph::makeSplitMultiConvConcat();
+
+    auto ie = ov::Core();
+    auto compiled_model = ie.compile_model(fn, CommonTestUtils::DEVICE_GPU);
+
+    const int infer_requests_num = 2;
+    ov::InferRequest infer_request1 = compiled_model.create_infer_request();
+    ov::InferRequest infer_request2 = compiled_model.create_infer_request();
+
+    input_tensors.push_back(infer_request1.get_input_tensor());
+    input_tensors.push_back(infer_request2.get_input_tensor());
+    output_tensors.push_back(infer_request1.get_output_tensor());
+    output_tensors.push_back(infer_request2.get_output_tensor());
+
+    auto calc_ref_results = [&](const ov::Tensor& tensor){
+        const auto tensor_size = tensor.get_byte_size();
+        const auto in_blob_buf = static_cast<uint8_t*>(tensor.data());
+        std::vector<uint8_t> inData(in_blob_buf, in_blob_buf + tensor_size);
+        auto ref_out_data = ngraph::helpers::interpreterFunction(fn, {inData}).front().second;
+        ref.push_back(ref_out_data);
+    };
+
+    auto compare_results = [&](ov::Tensor& result, const uint8_t* refResult) {
+        auto thr = FuncTestUtils::GetComparisonThreshold(InferenceEngine::Precision::FP32);
+        ASSERT_EQ(ov::shape_size(fn->get_output_shape(0)), result.get_size());
+        FuncTestUtils::compareRawBuffers(result.data<float>(),
+                                        reinterpret_cast<const float *>(refResult), ov::shape_size(fn->get_output_shape(0)),
+                                        ov::shape_size(fn->get_output_shape(0)),
+                                        thr);
+    };
+
+    for (size_t i = 0; i < infer_requests_num; i++) {
+        FuncTestUtils::fill_tensor(input_tensors[i], 10, -5, 1, i);
+        calc_ref_results(input_tensors[i]);
+    }
+
+    // Swap tensors between IRs
+    infer_request1.set_output_tensor(output_tensors[1]);
+    infer_request2.set_output_tensor(output_tensors[0]);
+
+    const int niter_limit = 10;
+    int iter1 = 0, iter2 = 0;
+    infer_request1.set_callback([&](std::exception_ptr exception_ptr) mutable {
+        if (exception_ptr) {
+            std::rethrow_exception(exception_ptr);
+        } else {
+            iter1++;
+            ov::Tensor output_tensor = infer_request1.get_output_tensor();
+            compare_results(output_tensor, ref[iter1 % 2].data());
+            if (iter1 < niter_limit) {
+                infer_request1.set_output_tensor(output_tensors[(iter1 + 1) % 2]);
+                infer_request1.start_async();
+            }
+        }
+    });
+
+    infer_request2.set_callback([&](std::exception_ptr exception_ptr) mutable {
+        if (exception_ptr) {
+            std::rethrow_exception(exception_ptr);
+        } else {
+            iter2++;
+            ov::Tensor output_tensor = infer_request2.get_output_tensor();
+            compare_results(output_tensor, ref[(iter2 + 1) % 2].data());
+            if (iter2 < niter_limit) {
+                infer_request2.set_output_tensor(output_tensors[iter2 % 2]);
+                infer_request2.start_async();
+            }
+        }
+    });
+
+    infer_request1.start_async();
+    infer_request2.start_async();
+
+    for (size_t i = 0; i < niter_limit; i++) {
+        infer_request1.wait();
+        infer_request2.wait();
+    }
+}

src/tests/ie_test_utils/common_test_utils/data_utils.hpp

@@ -12,6 +12,7 @@
 #include "openvino/runtime/tensor.hpp"
 #include <ngraph/type/bfloat16.hpp>
 #include <ngraph/type/float16.hpp>
+#include <ngraph/type/element_type_traits.hpp>
 
 #include <ie_blob.h>
 #include <random>
@@ -29,7 +30,7 @@ inline void fill_data(float *data, size_t size, size_t duty_ratio = 10) {
 }
 
 /**
- * @brief Create vector of floats with length of vec_len, with values ranging from min to max, 
+ * @brief Create vector of floats with length of vec_len, with values ranging from min to max,
  * with initial seed equal to variable seed with default of 0
  */
 inline std::vector<float> generate_float_numbers(std::size_t vec_len, float min, float max, int seed = 0) {
@@ -253,6 +254,52 @@ void inline fill_random_unique_sequence(T* rawBlobDataPtr,
     std::copy(elems.begin(), elems.end(), rawBlobDataPtr);
 }
 
+/** @brief Fill tensor with random data.
+ *
+ * @param tensor Target tensor
+ * @param range Values range
+ * @param start_from Value from which range should start
+ * @param k Resolution of floating point numbers.
+ * - With k = 1 every random number will be basically integer number.
+ * - With k = 2 numbers resolution will 1/2 so outputs only .0 or .50
+ * - With k = 4 numbers resolution will 1/4 so outputs only .0 .25 .50 0.75 and etc.
+ */
+template<ov::element::Type_t DT>
+void inline fill_tensor_random(ov::Tensor& tensor, const uint32_t range = 10, int32_t start_from = 0,
+                               const int32_t k = 1, const int seed = 1) {
+    using T = typename ov::element_type_traits<DT>::value_type;
+    auto *rawBlobDataPtr = static_cast<T*>(tensor.data());
+    if (DT == ov::element::u4 || DT == ov::element::i4 ||
+        DT == ov::element::u1) {
+        fill_data_random(rawBlobDataPtr, tensor.get_byte_size(), range, start_from, k, seed);
+    } else {
+        fill_data_random(rawBlobDataPtr, tensor.get_size(), range, start_from, k, seed);
+    }
+}
+
+template<ov::element::Type_t DT>
+void inline
+fill_tensor_random_float(ov::Tensor& tensor, const uint32_t range, int32_t start_from, const int32_t k,
+                         const int seed = 1) {
+    using T = typename ov::element_type_traits<DT>::value_type;
+    std::default_random_engine random(seed);
+    // 1/k is the resolution of the floating point numbers
+    std::uniform_int_distribution<int32_t> distribution(k * start_from, k * (start_from + range));
+
+    auto *rawBlobDataPtr = static_cast<T*>(tensor.data());
+    for (size_t i = 0; i < tensor.get_size(); i++) {
+        auto value = static_cast<float>(distribution(random));
+        value /= static_cast<float>(k);
+        if (DT == ov::element::Type_t::f16) {
+            rawBlobDataPtr[i] = static_cast<T>(ngraph::float16(value).to_bits());
+        } else if (DT == ov::element::Type_t::bf16) {
+            rawBlobDataPtr[i] = static_cast<T>(ngraph::bfloat16(value).to_bits());
+        } else {
+            rawBlobDataPtr[i] = static_cast<T>(value);
+        }
+    }
+}
+
 /** @brief Fill blob with random data.
  *
  * @param blob Target blob

src/tests/ie_test_utils/functional_test_utils/include/functional_test_utils/blob_utils.hpp

@@ -572,6 +572,37 @@ inline ov::Tensor create_and_fill_tensor(
     return tensor;
 }
 
+void inline fill_tensor(ov::Tensor& tensor,
+                        uint32_t range = 10,
+                        int32_t start_from = 0,
+                        const int32_t k = 1,
+                        const int seed = 1) {
+    auto element_type = tensor.get_element_type();
+    switch (element_type) {
+#define CASE(X) case X: CommonTestUtils::fill_tensor_random<X>(tensor, range, start_from, k, seed); break;
+#define CASE_FLOAT(X) case X: CommonTestUtils::fill_tensor_random_float<X>(tensor, range, start_from, k, seed); break;
+    CASE_FLOAT(ov::element::f64)
+    CASE_FLOAT(ov::element::f32)
+    CASE_FLOAT(ov::element::f16)
+    CASE_FLOAT(ov::element::bf16)
+    CASE(ov::element::u1)
+    CASE(ov::element::u4)
+    CASE(ov::element::u8)
+    CASE(ov::element::u32)
+    CASE(ov::element::u16)
+    CASE(ov::element::u64)
+    CASE(ov::element::i4)
+    CASE(ov::element::i8)
+    CASE(ov::element::i16)
+    CASE(ov::element::i32)
+    CASE(ov::element::i64)
+#undef CASE
+#undef CASE_FLOAT
+    default:
+        IE_THROW() << "Wrong precision specified: " << element_type;
+    }
+}
+
 inline InferenceEngine::Blob::Ptr createAndFillBlobConsistently(
     const InferenceEngine::TensorDesc &td,
     const uint32_t range,