[GPU] Fix gpu memory handling (#19930)

* [GPU] Added more debug print for prepare input/output
Skip enqueuemempcy for zero count tensor

* Fix mem alloc limitation

* Simplify memory check condition

* Fix not to wait for null event

* Additional fixes
- apply review comments (disable 0 size memcpy for other mem types) && return user event instead of nullptr
- fixd get_max_memory to return cpu memory instead of global memorysize (Because in windows, max_global_memory returns shared gpu memory instead of physical memory size)

* Fixed functest failure

src/plugins/intel_gpu/include/intel_gpu/runtime/engine.hpp

@@ -125,6 +125,9 @@ public:
     /// Returns the size of the larger of the GPU memory and CPU memory.
     uint64_t get_max_memory_size() const;
 
+    /// Returns the size of CPU memory.
+    uint64_t get_host_memory_size() const;
+
     /// Create stream object for current engine
     virtual stream_ptr create_stream(const ExecutionConfig& config) const = 0;
 

src/plugins/intel_gpu/src/plugin/sync_infer_request.cpp

@@ -676,6 +676,9 @@ std::vector<cldnn::event::ptr> SyncInferRequest::prepare_input(const std::string
     auto element_type = user_tensor->get_element_type();
     auto remote_ptr = std::dynamic_pointer_cast<RemoteTensorImpl>(user_tensor);
     bool is_remote = remote_ptr != nullptr;
+    GPU_DEBUG_TRACE_DETAIL << "Prepare input for " << name << " ( is_remote ? " << is_remote << ")" << std::endl;
+    GPU_DEBUG_TRACE_DETAIL << "    port shape       : " << pshape.to_string() << std::endl;
+    GPU_DEBUG_TRACE_DETAIL << "    user_tensor shape: " << user_tensor->get_shape().to_string() << std::endl;
 
     auto network = m_graph->get_network();
     auto& engine = m_graph->get_engine();
@@ -709,6 +712,7 @@ std::vector<cldnn::event::ptr> SyncInferRequest::prepare_input(const std::string
             if (device_tensor->get_original_memory()->size() < user_tensor->get_byte_size()) {
                 auto& shape_predictor = network->get_shape_predictor();
                 auto actual_shape = predict_shape(name, user_tensor->get_shape(), device_tensor_et, shape_predictor);
+                GPU_DEBUG_TRACE_DETAIL << "    actual memory shape: " << actual_shape.to_string() << std::endl;
                 auto new_tensor = create_device_tensor(actual_shape, device_tensor_et, false);
                 new_tensor->set_shape(user_tensor->get_shape());
                 m_plugin_inputs[name] = { new_tensor, TensorOwner::PLUGIN };
@@ -759,7 +763,7 @@ std::vector<cldnn::event::ptr> SyncInferRequest::prepare_input(const std::string
     const cldnn::primitive_id internal_name = "parameter:" + name;
     network->set_input_data(internal_name, memory);
 
-    if (ret_event)
+    if (ret_event && !ret_event->is_set())
         return { ret_event };
     else
         return {};
@@ -776,6 +780,10 @@ std::vector<cldnn::event::ptr> SyncInferRequest::prepare_output(const std::strin
     auto remote_ptr = std::dynamic_pointer_cast<RemoteTensorImpl>(user_tensor);
     bool is_remote = remote_ptr != nullptr;
 
+    GPU_DEBUG_TRACE_DETAIL << "Prepare output for " << name << std::endl;
+    GPU_DEBUG_TRACE_DETAIL << "    port shape       : " << pshape.to_string() << std::endl;
+    GPU_DEBUG_TRACE_DETAIL << "    user_tensor shape: " << user_tensor->get_shape().to_string() << std::endl;
+
     if (user_tensor->get_size() > 0) {
         OPENVINO_ASSERT(pshape.compatible(ov::PartialShape(user_tensor->get_shape())),
                         "[GPU] The output tensor size is not equal to model port shape, can't handle output tensor with name: ",

src/plugins/intel_gpu/src/runtime/engine.cpp

@@ -82,10 +82,18 @@ bool engine::use_unified_shared_memory() const {
 }
 
 uint64_t engine::get_max_memory_size() const {
-    static uint64_t max_device_mem = (std::max)(get_device_info().max_global_mem_size, static_cast<uint64_t>(get_cpu_ram_size()));
+    static uint64_t max_device_mem = get_host_memory_size();
+    const auto& dev_type = get_device_info().dev_type;
+    if (dev_type == device_type::discrete_gpu) {
+        max_device_mem += get_device_info().max_global_mem_size;
+    }
     return max_device_mem;
 }
 
+uint64_t engine::get_host_memory_size() const {
+    return static_cast<uint64_t>(get_cpu_ram_size());
+}
+
 bool engine::supports_allocation(allocation_type type) const {
     if (memory_capabilities::is_usm_type(type) && !use_unified_shared_memory())
         return false;

src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.cpp

@@ -125,23 +125,29 @@ allocation_type ocl_engine::detect_usm_allocation_type(const void* memory) const
                                        : allocation_type::unknown;
 }
 
+bool ocl_engine::check_allocatable(const layout& layout, allocation_type type) const {
+    OPENVINO_ASSERT(supports_allocation(type) || type == allocation_type::cl_mem, "[GPU] Unsupported allocation type: ", type);
+    auto used_mem = get_used_device_memory(allocation_type::usm_device) + get_used_device_memory(allocation_type::usm_host);
+#ifdef __unix__
+    // Prevent from being killed by Ooo Killer of Linux
+    OPENVINO_ASSERT(layout.bytes_count() + used_mem <= get_max_memory_size(),
+            "[GPU] Exceeded max size of memory allocation: ",
+            "Required ", layout.bytes_count(), " bytes, already occupied : ", used_mem, " bytes, ",
+            "but available memory size is ", get_max_memory_size(), " bytes");
+#else
+    if (layout.bytes_count() + used_mem > get_max_memory_size()) {
+        GPU_DEBUG_COUT << "[Warning] [GPU] Exceeded max size of memory allocation: " << "Required " << layout.bytes_count() << " bytes, already occupied : "
+                       << used_mem << " bytes, but available memory size is " << get_max_memory_size() << " bytes" << std::endl;
+        GPU_DEBUG_COUT << "Please note that performance might drop due to memory swap." << std::endl;
+    }
+#endif
+    return true;
+}
+
 memory::ptr ocl_engine::allocate_memory(const layout& layout, allocation_type type, bool reset) {
     OPENVINO_ASSERT(!layout.is_dynamic() || layout.has_upper_bound(), "[GPU] Can't allocate memory for dynamic layout");
 
-    OPENVINO_ASSERT(layout.bytes_count() <= get_device_info().max_alloc_mem_size,
-                    "[GPU] Exceeded max size of memory object allocation: ",
-                    "Requested ", layout.bytes_count(), " bytes "
-                    "but max alloc size is ", get_device_info().max_alloc_mem_size, " bytes");
-
-    auto used_mem = get_used_device_memory(allocation_type::usm_device) + get_used_device_memory(allocation_type::usm_host);
-    OPENVINO_ASSERT(layout.bytes_count() + used_mem <= get_max_memory_size(),
-                    "[GPU] Exceeded max size of memory allocation: ",
-                    "Required ", (layout.bytes_count() + used_mem), " bytes "
-                    "but memory size is ", get_max_memory_size(), " bytes");
-
-    OPENVINO_ASSERT(supports_allocation(type) || type == allocation_type::cl_mem,
-                    "[GPU] Unsupported allocation type: ", type);
-
+    check_allocatable(layout, type);
     try {
         memory::ptr res = nullptr;
         if (layout.format.is_image_2d()) {
@@ -153,7 +159,10 @@ memory::ptr ocl_engine::allocate_memory(const layout& layout, allocation_type ty
         }
 
         if (reset || res->is_memory_reset_needed(layout)) {
-            get_service_stream().wait_for_events({res->fill(get_service_stream())});
+            auto ev = res->fill(get_service_stream());
+            if (ev) {
+                get_service_stream().wait_for_events({ev});
+            }
         }
 
         return res;

src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.hpp

@@ -28,6 +28,7 @@ public:
     memory_ptr reinterpret_handle(const layout& new_layout, shared_mem_params params) override;
     memory_ptr reinterpret_buffer(const memory& memory, const layout& new_layout) override;
     bool is_the_same_buffer(const memory& mem1, const memory& mem2) override;
+    bool check_allocatable(const layout& layout, allocation_type type) const;
 
     void* get_user_context() const override;
 

src/plugins/intel_gpu/src/runtime/ocl/ocl_memory.cpp

@@ -64,10 +64,18 @@ void gpu_buffer::unlock(const stream& stream) {
 }
 
 event::ptr gpu_buffer::fill(stream& stream) {
+    if (_bytes_count == 0) {
+        GPU_DEBUG_TRACE_DETAIL << "Skip EnqueueMemcpy for 0 size tensor" << std::endl;
+        return stream.create_user_event(true);
+    }
     return fill(stream, 0);
 }
 
 event::ptr gpu_buffer::fill(stream& stream, unsigned char pattern) {
+    if (_bytes_count == 0) {
+        GPU_DEBUG_TRACE_DETAIL << "Skip EnqueueMemcpy for 0 size tensor" << std::endl;
+        return stream.create_user_event(true);
+    }
     auto& cl_stream = downcast<ocl_stream>(stream);
     auto ev = stream.create_base_event();
     cl::Event& ev_ocl = downcast<ocl_event>(ev.get())->get();
@@ -89,6 +97,10 @@ shared_mem_params gpu_buffer::get_internal_params() const {
 }
 
 event::ptr gpu_buffer::copy_from(stream& stream, const memory& other, bool blocking) {
+    if (_bytes_count == 0) {
+        GPU_DEBUG_TRACE_DETAIL << "Skip EnqueueMemcpy for 0 size tensor" << std::endl;
+        return stream.create_user_event(true);
+    }
     switch (other.get_allocation_type()) {
     case allocation_type::usm_host:
     case allocation_type::usm_shared:
@@ -118,6 +130,10 @@ event::ptr gpu_buffer::copy_from(stream& stream, const memory& other, bool block
 }
 
 event::ptr gpu_buffer::copy_from(stream& stream, const void* host_ptr, bool blocking) {
+    if (_bytes_count == 0) {
+        GPU_DEBUG_TRACE_DETAIL << "Skip EnqueueMemcpy for 0 size tensor" << std::endl;
+        return stream.create_user_event(true);
+    }
     auto& cl_stream = downcast<ocl_stream>(stream);
     auto ev = blocking ? stream.create_user_event(true) : stream.create_base_event();
     cl::Event* ev_ocl = blocking ? nullptr : &downcast<ocl_event>(ev.get())->get();
@@ -127,6 +143,10 @@ event::ptr gpu_buffer::copy_from(stream& stream, const void* host_ptr, bool bloc
 }
 
 event::ptr gpu_buffer::copy_to(stream& stream, void* host_ptr, bool blocking) {
+    if (_bytes_count == 0) {
+        GPU_DEBUG_TRACE_DETAIL << "Skip EnqueueMemcpy for 0 size tensor" << std::endl;
+        return stream.create_user_event(true);
+    }
     auto& cl_stream = downcast<ocl_stream>(stream);
     auto ev = blocking ? stream.create_user_event(true) : stream.create_base_event();
     cl::Event* ev_ocl = blocking ? nullptr : &downcast<ocl_event>(ev.get())->get();
@@ -211,10 +231,18 @@ gpu_image2d::gpu_image2d(ocl_engine* engine,
 }
 
 event::ptr gpu_image2d::fill(stream& stream) {
+    if (_bytes_count == 0) {
+        GPU_DEBUG_TRACE_DETAIL << "Skip EnqueueMemcpy for 0 size tensor" << std::endl;
+        return stream.create_user_event(true);
+    }
     return fill(stream, 0);
 }
 
 event::ptr gpu_image2d::fill(stream& stream, unsigned char pattern) {
+    if (_bytes_count == 0) {
+        GPU_DEBUG_TRACE_DETAIL << "Skip EnqueueMemcpy for 0 size tensor" << std::endl;
+        return stream.create_user_event(true);
+    }
     auto& cl_stream = downcast<ocl_stream>(stream);
     auto ev = stream.create_base_event();
     cl::Event& ev_ocl = downcast<ocl_event>(ev.get())->get();
@@ -268,6 +296,10 @@ shared_mem_params gpu_image2d::get_internal_params() const {
 }
 
 event::ptr gpu_image2d::copy_from(stream& stream, const memory& other, bool blocking) {
+    if (_bytes_count == 0) {
+        GPU_DEBUG_TRACE_DETAIL << "Skip EnqueueMemcpy for 0 size tensor" << std::endl;
+        return stream.create_user_event(true);
+    }
     auto& cl_stream = downcast<const ocl_stream>(stream);
     auto& casted = downcast<const gpu_image2d>(other);
     auto ev = stream.create_base_event();
@@ -283,6 +315,10 @@ event::ptr gpu_image2d::copy_from(stream& stream, const memory& other, bool bloc
 }
 
 event::ptr gpu_image2d::copy_from(stream& stream, const void* host_ptr, bool blocking) {
+    if (_bytes_count == 0) {
+        GPU_DEBUG_TRACE_DETAIL << "Skip EnqueueMemcpy for 0 size tensor" << std::endl;
+        return stream.create_user_event(true);
+    }
     auto& cl_stream = downcast<ocl_stream>(stream);
     auto ev = blocking ? stream.create_user_event(true) : stream.create_base_event();
     cl::Event* ev_ocl = blocking ? nullptr : &downcast<ocl_event>(ev.get())->get();
@@ -293,6 +329,10 @@ event::ptr gpu_image2d::copy_from(stream& stream, const void* host_ptr, bool blo
 }
 
 event::ptr gpu_image2d::copy_to(stream& stream, memory& other, bool blocking) {
+    if (_bytes_count == 0) {
+        GPU_DEBUG_TRACE_DETAIL << "Skip EnqueueMemcpy for 0 size tensor" << std::endl;
+        return stream.create_user_event(true);
+    }
     auto& cl_stream = downcast<const ocl_stream>(stream);
     auto& casted = downcast<const gpu_image2d>(other);
     auto ev = stream.create_base_event();
@@ -308,6 +348,10 @@ event::ptr gpu_image2d::copy_to(stream& stream, memory& other, bool blocking) {
 }
 
 event::ptr gpu_image2d::copy_to(stream& stream, void* host_ptr, bool blocking) {
+    if (_bytes_count == 0) {
+        GPU_DEBUG_TRACE_DETAIL << "Skip EnqueueMemcpy for 0 size tensor" << std::endl;
+        return stream.create_user_event(true);
+    }
     auto& cl_stream = downcast<ocl_stream>(stream);
     auto ev = blocking ? stream.create_user_event(true) : stream.create_base_event();
     cl::Event* ev_ocl = blocking ? nullptr : &downcast<ocl_event>(ev.get())->get();
@@ -414,6 +458,10 @@ void gpu_usm::unlock(const stream& /* stream */) {
 }
 
 event::ptr gpu_usm::fill(stream& stream, unsigned char pattern) {
+    if (_bytes_count == 0) {
+        GPU_DEBUG_TRACE_DETAIL << "Skip gpu_usm::fill for 0 size tensor" << std::endl;
+        return stream.create_user_event(true);
+    }
     auto& cl_stream = downcast<ocl_stream>(stream);
     auto ev = stream.create_base_event();
     cl::Event& ev_ocl = downcast<ocl_event>(ev.get())->get();
@@ -435,10 +483,18 @@ event::ptr gpu_usm::fill(stream& stream) {
     // ev->wait();
 
     // [WA]
+    if (_bytes_count == 0) {
+        GPU_DEBUG_TRACE_DETAIL << "Skip EnqueueMemcpy for 0 size tensor" << std::endl;
+        return stream.create_user_event(true);
+    }
     return fill(stream, 0);
 }
 
 event::ptr gpu_usm::copy_from(stream& stream, const memory& other, bool blocking) {
+    if (_bytes_count == 0) {
+        GPU_DEBUG_TRACE_DETAIL << "Skip EnqueueMemcpy for 0 size tensor" << std::endl;
+        return stream.create_user_event(true);
+    }
     auto& cl_stream = downcast<const ocl_stream>(stream);
     auto ev = blocking ? stream.create_user_event(true) : stream.create_base_event();
     cl::Event* ev_ocl = blocking ? nullptr : &downcast<ocl_event>(ev.get())->get();
@@ -462,6 +518,10 @@ event::ptr gpu_usm::copy_from(stream& stream, const memory& other, bool blocking
 }
 
 event::ptr gpu_usm::copy_from(stream& stream, const void* host_ptr, bool blocking) {
+    if (_bytes_count == 0) {
+        GPU_DEBUG_TRACE_DETAIL << "Skip EnqueueMemcpy for 0 size tensor" << std::endl;
+        return stream.create_user_event(true);
+    }
     auto& cl_stream = downcast<ocl_stream>(stream);
     auto dst_ptr = get_buffer().get();
     auto ev = blocking ? stream.create_user_event(true) : stream.create_base_event();
@@ -477,6 +537,10 @@ event::ptr gpu_usm::copy_from(stream& stream, const void* host_ptr, bool blockin
 }
 
 event::ptr gpu_usm::copy_to(stream& stream, void* host_ptr, bool blocking) {
+    if (_bytes_count == 0) {
+        GPU_DEBUG_TRACE_DETAIL << "Skip EnqueueMemcpy for 0 size tensor" << std::endl;
+        return stream.create_user_event(true);
+    }
     auto& cl_stream = downcast<ocl_stream>(stream);
     auto ev = blocking ? stream.create_user_event(true) : stream.create_base_event();
     cl::Event* ev_ocl = blocking ? nullptr : &downcast<ocl_event>(ev.get())->get();