[GPU] Memory preallocation optimization (#18181)

* [GPU] Use real layout for cpu impls instead of memory's

* [GPU] Add memory tracking and pre allocation mechanism

* Tests and minor code refactoring

* Apply review comments

* Remove unused include

src/plugins/intel_gpu/include/intel_gpu/graph/network.hpp

@@ -13,6 +13,7 @@
 #include "intel_gpu/runtime/event.hpp"
 #include "intel_gpu/runtime/stream.hpp"
 #include "intel_gpu/runtime/lru_cache.hpp"
+#include "intel_gpu/runtime/shape_predictor.hpp"
 
 #include <map>
 #include <vector>
@@ -242,6 +243,8 @@ public:
 
     const ExecutionConfig& get_config() const { return _config; }
 
+    ShapePredictor& get_shape_predictor() { return *_shape_predictor; }
+
 private:
     using output_chains_map = std::map<primitive_id, std::vector<std::shared_ptr<primitive_inst>>>;
     uint32_t net_id = 0;
@@ -274,6 +277,8 @@ private:
     std::unordered_map<primitive_id, event::ptr> _events;
     output_chains_map _output_chains;
 
+    std::unique_ptr<ShapePredictor> _shape_predictor;
+
     void build_exec_order();
     void allocate_primitive_instance(program_node const& node);
     void transfer_memory_to_device(std::shared_ptr<primitive_inst> instance, program_node const& node);

src/plugins/intel_gpu/include/intel_gpu/runtime/debug_configuration.hpp

@@ -121,6 +121,18 @@ public:
     static const debug_configuration *get_instance();
     bool is_dumped_layer(const std::string& layerName, bool is_output = false) const;
     bool is_target_iteration(int64_t iteration) const;
+
+    struct memory_preallocation_params {
+        bool is_initialized = false;
+
+        // Iterations mode preallocation
+        size_t next_iters_preallocation_count = 0;
+        size_t max_per_iter_size = 0;
+        size_t max_per_dim_diff = 0;
+
+        // Percentage mode preallocation
+        float buffers_preallocation_ratio = 0.0f;
+    } mem_preallocation_params;
 };
 
 }  // namespace cldnn

src/plugins/intel_gpu/include/intel_gpu/runtime/internal_properties.hpp

@@ -55,6 +55,7 @@ static constexpr Property<bool, PropertyMutability::RW> enable_lp_transformation
 static constexpr Property<size_t, PropertyMutability::RW> max_dynamic_batch{"DYN_BATCH_LIMIT"};
 static constexpr Property<bool, PropertyMutability::RW> exclusive_async_requests{"EXCLUSIVE_ASYNC_REQUESTS"};
 static constexpr Property<bool, PropertyMutability::RW> nv12_two_inputs{"GPU_NV12_TWO_INPUTS"};
+static constexpr Property<float, PropertyMutability::RW> buffers_preallocation_ratio{"GPU_BUFFERS_PREALLOCATION_RATIO"};
 
 }  // namespace intel_gpu
 }  // namespace ov

src/plugins/intel_gpu/include/intel_gpu/runtime/shape_predictor.hpp

@@ -0,0 +1,73 @@
+// Copyright (C) 2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include "layout.hpp"
+
+#include <deque>
+
+namespace cldnn {
+
+class engine;
+
+struct ShapePredictor {
+public:
+    ShapePredictor(const engine* engine, float buffers_preallocation_ratio)
+        : _engine(engine)
+        , _buffers_preallocation_ratio(buffers_preallocation_ratio) {
+        static_assert(_max_deque_size >= 2, "[GPU] Deque is supposed to contain at least 2 elements for prediction");
+    }
+
+    ShapePredictor(const engine* engine,
+                   size_t next_iters_preallocation_count,
+                   size_t max_per_iter_size,
+                   size_t max_per_dim_diff,
+                   float buffers_preallocation_ratio)
+        : _engine(engine)
+        , _next_iters_preallocation_count(next_iters_preallocation_count)
+        , _max_per_iter_size(max_per_iter_size)
+        , _max_per_dim_diff(max_per_dim_diff)
+        , _buffers_preallocation_ratio(buffers_preallocation_ratio) {
+        static_assert(_max_deque_size >= 2, "[GPU] Deque is supposed to contain at least 2 elements for prediction");
+    }
+
+
+/// \brief Predicts the next possible shapes sizes based on history collected by previous
+///        predict_preallocation_shape() calls.
+///        This function works in two modes: by default it tries to predict shape for the next
+///        `_next_iters_preallocation_count` iterations, in case if per-iteration buffer size is less than
+///        `_max_per_iter_size` and difference between shapes is less than `_max_per_dim_diff`; the second
+///        operation mode is percentage preallocation - this mode can be configured with
+///        ov::intel_gpu::buffers_preallocation_ratio property, it increases buffer size by
+///        `_buffers_preallocation_ratio` value unconditionally.
+/// \param id Primitive id.
+/// \param current_shape Primitive's shape on current iteration.
+/// \param dt_size Primitive's data_type size.
+/// \param can_reuse_buffer Specifies if current memory buffer is enough to store data.
+/// \return The result of shape size prediction as std::pair<bool, ov::Shape>, where the first element
+///         says if shape is successfully predicted and can be preallocated, and the second element is ov::Shape itself.
+    std::pair<bool, ov::Shape> predict_preallocation_shape(const std::string& id,
+                                                           const ov::Shape& current_shape,
+                                                           size_t dt_size,
+                                                           bool can_reuse_buffer);
+    bool can_preallocate(size_t desired_buffer_size);
+
+private:
+    void add_shape(const std::string& id, const ov::Shape& shape);
+
+    static constexpr size_t _max_deque_size = 3;
+    std::map<std::string, std::deque<ov::Shape>> _shapes_info;
+    const engine* _engine;
+
+    // Iterations mode preallocation
+    const size_t _next_iters_preallocation_count = 10;
+    const size_t _max_per_iter_size = 16 * 1024; // 16KB => maximum preallocation size is 16KB * 10iters = 160KB
+    const size_t _max_per_dim_diff = 2;
+
+    // Percentage mode preallocation
+    const float _buffers_preallocation_ratio = 1.0f;
+};
+
+}  // namespace cldnn

src/plugins/intel_gpu/src/graph/impls/cpu/activation.cpp

@@ -96,13 +96,15 @@ struct activation_impl : public typed_primitive_impl<activation> {
         ov::TensorVector input_host_tensors;
         ov::TensorVector output_host_tensors;
 
+        auto params = instance.get_impl_params();
+
         std::vector<memory::ptr> input_mem_ptrs;
         for (size_t i = 0; i < instance.dependencies().size(); i++)
             input_mem_ptrs.push_back(instance.dep_memory_ptr(i));
 
         // TODO: consider to re-implement lock/unlock in more exception-safetest way
         for (size_t i = 0; i < input_mem_ptrs.size(); i++)
-            input_host_tensors.push_back(make_tensor(input_mem_ptrs[i]->get_layout(), input_mem_ptrs[i]->lock(stream, mem_lock_type::read)));
+            input_host_tensors.push_back(make_tensor(params->input_layouts[i], input_mem_ptrs[i]->lock(stream, mem_lock_type::read)));
 
         // Most of the evaluate functions expect same data type for all inputs, so we need to convert params from float
         typename data_type_to_type<DT>::type param_a = static_cast<typename data_type_to_type<DT>::type>(additional_params.a);
@@ -123,7 +125,7 @@ struct activation_impl : public typed_primitive_impl<activation> {
         auto output_mem_ptr = instance.output_memory_ptr();
 
         cldnn::mem_lock<uint8_t, mem_lock_type::write> output_lock(output_mem_ptr, stream);
-        output_host_tensors.push_back(make_tensor(output_mem_ptr->get_layout(), output_lock.data()));
+        output_host_tensors.push_back(make_tensor(params->output_layouts[0], output_lock.data()));
 
         OPENVINO_ASSERT(op->evaluate(output_host_tensors, input_host_tensors),
                         "[GPU] Couldn't execute activation primitive with id ", instance.id());

src/plugins/intel_gpu/src/graph/impls/cpu/broadcast.cpp

@@ -66,6 +66,8 @@ struct broadcast_impl : public typed_primitive_impl<broadcast> {
 
         auto ev = stream.create_user_event(false);
 
+        auto params = instance.get_impl_params();
+
         ov::TensorVector input_host_tensors;
         ov::TensorVector output_host_tensors;
 
@@ -82,7 +84,7 @@ struct broadcast_impl : public typed_primitive_impl<broadcast> {
             input_mem_ptrs.push_back(instance.dep_memory_ptr(i));
 
         for (size_t i = 0; i < input_mem_ptrs.size(); i++)
-            input_host_tensors.push_back(make_tensor(input_mem_ptrs[i]->get_layout(), input_mem_ptrs[i]->lock(stream, mem_lock_type::read)));
+            input_host_tensors.push_back(make_tensor(params->input_layouts[i], input_mem_ptrs[i]->lock(stream, mem_lock_type::read)));
 
         if (instance.dependencies().size() < 2) {
             OPENVINO_ASSERT(!target_shape.empty(), "[GPU] Unexpected empty target_shape for broadcast operation with id ", instance.id());
@@ -97,7 +99,7 @@ struct broadcast_impl : public typed_primitive_impl<broadcast> {
         auto output_mem_ptr = instance.output_memory_ptr();
 
         cldnn::mem_lock<uint8_t, mem_lock_type::read> output_lock(output_mem_ptr, stream);
-        output_host_tensors.push_back(make_tensor(output_mem_ptr->get_layout(), output_lock.data()));
+        output_host_tensors.push_back(make_tensor(params->output_layouts[0], output_lock.data()));
 
         OPENVINO_ASSERT(op->evaluate(output_host_tensors, input_host_tensors),
                         "[GPU] Couldn't execute broadcast primitive with id ", instance.id());

src/plugins/intel_gpu/src/graph/impls/cpu/concat.cpp

@@ -57,6 +57,8 @@ struct concatenation_impl : public typed_primitive_impl<concatenation> {
 
         auto ev = stream.create_user_event(false);
 
+        auto params = instance.get_impl_params();
+
         ov::TensorVector input_host_tensors;
         ov::TensorVector output_host_tensors;
 
@@ -76,9 +78,9 @@ struct concatenation_impl : public typed_primitive_impl<concatenation> {
         cldnn::mem_lock<uint8_t, mem_lock_type::read> output_lock(output_mem_ptr, stream);
 
         for (size_t i = 0; i < input_mem_ptrs.size(); i++)
-            input_host_tensors.push_back(make_tensor(input_mem_ptrs[i]->get_layout(), input_mem_ptrs[i]->lock(stream, mem_lock_type::read)));
+            input_host_tensors.push_back(make_tensor(params->input_layouts[i], input_mem_ptrs[i]->lock(stream, mem_lock_type::read)));
 
-        output_host_tensors.push_back(make_tensor(output_mem_ptr->get_layout(), output_lock.data()));
+        output_host_tensors.push_back(make_tensor(params->output_layouts[0], output_lock.data()));
 
         if (!op) {
             op = std::make_shared<ov::op::v0::Concat>();

src/plugins/intel_gpu/src/graph/impls/cpu/crop.cpp

@@ -82,8 +82,8 @@ struct crop_impl : public typed_primitive_impl<crop> {
         auto stop_tensor = ov::Tensor(ov::element::i64, {stop_vec.size()}, stop_vec.data());
         auto steps_tensor = ov::Tensor(ov::element::i64, {steps_vec.size()}, steps_vec.data());
 
-        auto input_tensor = make_tensor(input_mem_ptr->get_layout(), input_lock.data());
-        auto output_tensor = make_tensor(output_mem_ptr->get_layout(), output_lock.data());
+        auto input_tensor = make_tensor(params->input_layouts[0], input_lock.data());
+        auto output_tensor = make_tensor(params->output_layouts[0], output_lock.data());
 
         input_host_tensors.push_back(input_tensor);
         input_host_tensors.push_back(start_tensor);

src/plugins/intel_gpu/src/graph/impls/cpu/eltwise.cpp

@@ -83,6 +83,8 @@ struct eltwise_impl : public typed_primitive_impl<eltwise> {
 
         auto ev = stream.create_user_event(false);
 
+        auto params = instance.get_impl_params();
+
         ov::TensorVector input_host_tensors;
         ov::TensorVector output_host_tensors;
 
@@ -176,9 +178,9 @@ struct eltwise_impl : public typed_primitive_impl<eltwise> {
         cldnn::mem_lock<uint8_t, mem_lock_type::write> output_lock(output_mem_ptr, stream);
 
         for (size_t i = 0; i < input_mem_ptrs.size(); i++)
-            input_host_tensors.push_back(make_tensor(input_mem_ptrs[i]->get_layout(), input_mem_ptrs[i]->lock(stream, mem_lock_type::read)));
+            input_host_tensors.push_back(make_tensor(params->input_layouts[i], input_mem_ptrs[i]->lock(stream, mem_lock_type::read)));
 
-        output_host_tensors.push_back(make_tensor(output_mem_ptr->get_layout(), output_lock.data()));
+        output_host_tensors.push_back(make_tensor(params->output_layouts[0], output_lock.data()));
 
         OPENVINO_ASSERT(op->evaluate(output_host_tensors, input_host_tensors),
                         "[GPU] Couldn't execute eltwise primitive with id ", instance.id());

src/plugins/intel_gpu/src/graph/impls/cpu/gather.cpp

@@ -60,6 +60,7 @@ struct gather_impl : public typed_primitive_impl<gather> {
         }
 
         auto ev = stream.create_user_event(false);
+        auto params = instance.get_impl_params();
 
         ov::TensorVector input_host_tensors;
         ov::TensorVector output_host_tensors;
@@ -78,11 +79,11 @@ struct gather_impl : public typed_primitive_impl<gather> {
         cldnn::mem_lock<uint8_t, mem_lock_type::read> output_lock(output_mem_ptr, stream);
 
         for (size_t i = 0; i < input_mem_ptrs.size(); i++)
-            input_host_tensors.push_back(make_tensor(input_mem_ptrs[i]->get_layout(), input_mem_ptrs[i]->lock(stream, mem_lock_type::read)));
+            input_host_tensors.push_back(make_tensor(params->input_layouts[i], input_mem_ptrs[i]->lock(stream, mem_lock_type::read)));
 
         auto axis_tensor = ov::Tensor(ov::element::i64, ov::Shape{1}, static_cast<void*>(&axis));
 
-        output_host_tensors.push_back(make_tensor(output_mem_ptr->get_layout(), output_lock.data()));
+        output_host_tensors.push_back(make_tensor(params->output_layouts[0], output_lock.data()));
         input_host_tensors.push_back(axis_tensor);
 
         OPENVINO_ASSERT(op->evaluate(output_host_tensors, input_host_tensors),

src/plugins/intel_gpu/src/graph/impls/cpu/range.cpp

@@ -42,7 +42,9 @@ struct range_impl : public typed_primitive_impl<range> {
         for (auto e : events) {
             e->wait();
         }
+
         auto ev = stream.create_user_event(false);
+        auto params = instance.get_impl_params();
 
         ov::HostTensorVector input_host_tensors;
         ov::HostTensorVector output_host_tensors;
@@ -56,12 +58,12 @@ struct range_impl : public typed_primitive_impl<range> {
         cldnn::mem_lock<uint8_t, mem_lock_type::read> output_lock(output_mem_ptr, stream);
 
         for (size_t i = 0; i < input_mem_ptrs.size(); i++)
-            input_host_tensors.push_back(make_host_tensor(input_mem_ptrs[i]->get_layout(), input_mem_ptrs[i]->lock(stream, mem_lock_type::read)));
+            input_host_tensors.push_back(make_host_tensor(params->input_layouts[i], input_mem_ptrs[i]->lock(stream, mem_lock_type::read)));
 
-        output_host_tensors.push_back(make_host_tensor(output_mem_ptr->get_layout(), output_lock.data()));
+        output_host_tensors.push_back(make_host_tensor(params->output_layouts[0], output_lock.data()));
 
         if (!op) {
-            const auto output_dt = instance.get_impl_params()->get_output_layout().data_type;
+            const auto output_dt = params->get_output_layout().data_type;
 
             op = std::make_shared<ov::op::v4::Range>();
             op->set_output_type(data_type_to_element_type(output_dt));

src/plugins/intel_gpu/src/graph/impls/cpu/reorder.cpp

@@ -44,6 +44,7 @@ struct reorder_impl : public typed_primitive_impl<reorder> {
         }
 
         auto ev = stream.create_user_event(false);
+        auto params = instance.get_impl_params();
 
         if (instance.get_impl_params()->input_layouts[0].format != instance.get_impl_params()->input_layouts[0].format)
             OPENVINO_THROW("[GPU] Unsupported reorder case: input and output type are different");
@@ -57,8 +58,8 @@ struct reorder_impl : public typed_primitive_impl<reorder> {
         cldnn::mem_lock<uint8_t, mem_lock_type::read> input_lock(input_mem_ptr, stream);
         cldnn::mem_lock<uint8_t, mem_lock_type::read> output_lock(output_mem_ptr, stream);
 
-        input_host_tensors.push_back(make_tensor(input_mem_ptr->get_layout(), input_lock.data()));
-        output_host_tensors.push_back(make_tensor(output_mem_ptr->get_layout(), output_lock.data()));
+        input_host_tensors.push_back(make_tensor(params->input_layouts[0], input_lock.data()));
+        output_host_tensors.push_back(make_tensor(params->output_layouts[0], output_lock.data()));
 
         if (!op) {
             op = std::make_shared<ov::op::v0::Convert>();

src/plugins/intel_gpu/src/graph/impls/cpu/scatter_update.cpp

@@ -56,6 +56,7 @@ struct scatter_update_impl : public typed_primitive_impl<scatter_update> {
         }
 
         auto ev = stream.create_user_event(false);
+        auto params = instance.get_impl_params();
 
         ov::TensorVector input_host_tensors;
         ov::TensorVector output_host_tensors;
@@ -71,11 +72,11 @@ struct scatter_update_impl : public typed_primitive_impl<scatter_update> {
         cldnn::mem_lock<uint8_t, mem_lock_type::read> output_lock(output_mem_ptr, stream);
 
         for (size_t i = 0; i < input_mem_ptrs.size(); i++)
-            input_host_tensors.push_back(make_tensor(input_mem_ptrs[i]->get_layout(), input_mem_ptrs[i]->lock(stream, mem_lock_type::read)));
+            input_host_tensors.push_back(make_tensor(params->input_layouts[i], input_mem_ptrs[i]->lock(stream, mem_lock_type::read)));
 
         input_host_tensors.push_back(axis_tensor);
 
-        output_host_tensors.push_back(make_tensor(output_mem_ptr->get_layout(), output_lock.data()));
+        output_host_tensors.push_back(make_tensor(params->output_layouts[0], output_lock.data()));
 
         if (!op) {
             op = std::make_shared<ov::op::v3::ScatterUpdate>();

src/plugins/intel_gpu/src/graph/impls/cpu/strided_slice.cpp

@@ -88,6 +88,7 @@ struct strided_slice_impl : public typed_primitive_impl<strided_slice> {
         }
 
         auto ev = stream.create_user_event(false);
+        auto params = instance.get_impl_params();
 
         ov::TensorVector input_host_tensors;
         ov::TensorVector output_host_tensors;
@@ -120,11 +121,11 @@ struct strided_slice_impl : public typed_primitive_impl<strided_slice> {
         auto input_mem_ptr = instance.dep_memory_ptr(0);
         auto output_mem_ptr = instance.output_memory_ptr();
 
-        input_host_tensors.push_back(make_tensor(input_mem_ptr->get_layout(), input_mem_ptr->lock(stream, mem_lock_type::read)));
+        input_host_tensors.push_back(make_tensor(params->input_layouts[0], input_mem_ptr->lock(stream, mem_lock_type::read)));
         input_host_tensors.push_back(begin_host_tensor);
         input_host_tensors.push_back(end_host_tensor);
         input_host_tensors.push_back(strides_host_tensor);
-        output_host_tensors.push_back(make_tensor(output_mem_ptr->get_layout(), output_mem_ptr->lock(stream, mem_lock_type::write)));
+        output_host_tensors.push_back(make_tensor(params->output_layouts[0], output_mem_ptr->lock(stream, mem_lock_type::write)));
 
         if (!op) {
             op = std::make_shared<ov::op::v1::StridedSlice>();

src/plugins/intel_gpu/src/graph/impls/cpu/tile.cpp

@@ -55,6 +55,7 @@ struct tile_impl : public typed_primitive_impl<tile> {
         }
 
         auto ev = stream.create_user_event(false);
+        auto params = instance.get_impl_params();
 
         ov::TensorVector input_host_tensors;
         ov::TensorVector output_host_tensors;
@@ -68,7 +69,7 @@ struct tile_impl : public typed_primitive_impl<tile> {
             input_mem_ptrs.push_back(instance.dep_memory_ptr(i));
 
         for (size_t i = 0; i < input_mem_ptrs.size(); i++)
-            input_host_tensors.push_back(make_tensor(input_mem_ptrs[i]->get_layout(), input_mem_ptrs[i]->lock(stream, mem_lock_type::read)));
+            input_host_tensors.push_back(make_tensor(params->input_layouts[i], input_mem_ptrs[i]->lock(stream, mem_lock_type::read)));
 
         if (instance.dependencies().size() == 1) {
             if (repeats.empty())
@@ -81,7 +82,7 @@ struct tile_impl : public typed_primitive_impl<tile> {
         auto output_mem_ptr = instance.output_memory_ptr();
 
         cldnn::mem_lock<int32_t, mem_lock_type::read> output_lock(output_mem_ptr, stream);
-        output_host_tensors.push_back(make_tensor(output_mem_ptr->get_layout(), output_lock.data()));
+        output_host_tensors.push_back(make_tensor(params->output_layouts[0], output_lock.data()));
 
         OPENVINO_ASSERT(op->evaluate(output_host_tensors, input_host_tensors),
                         "[GPU] Couldn't execute tile primitive with id ", instance.id());

src/plugins/intel_gpu/src/graph/include/primitive_inst.h

@@ -276,6 +276,7 @@ protected:
     layout _node_output_layout;
 
     bool update_shape_done_by_other = false;
+    bool allocation_done_by_other = false;
     std::unique_ptr<kernel_impl_params> _impl_params;
     std::unique_ptr<primitive_impl> _impl;
     std::unique_ptr<primitive_impl> _dynamic_impl = nullptr;

src/plugins/intel_gpu/src/graph/network.cpp

@@ -319,7 +319,8 @@ network::network(program::ptr program, const ExecutionConfig& config, stream::pt
     , _internal(is_internal)
     , _is_primary_stream(is_primary_stream)
     , _enable_profiling(config.get_property(ov::enable_profiling))
-    , _reset_arguments(true) {
+    , _reset_arguments(true)
+    , _shape_predictor(new ShapePredictor(&program->get_engine(), config.get_property(ov::intel_gpu::buffers_preallocation_ratio))) {
     if (!_internal) {
         net_id = get_unique_net_id();
     }
@@ -329,6 +330,15 @@ network::network(program::ptr program, const ExecutionConfig& config, stream::pt
         wait_for_the_turn();
     }
 
+    GPU_DEBUG_IF(debug_config->mem_preallocation_params.is_initialized) {
+        auto& mem_preallocation_params = debug_config->mem_preallocation_params;
+        _shape_predictor.reset(new ShapePredictor(&program->get_engine(),
+                                                  mem_preallocation_params.next_iters_preallocation_count,
+                                                  mem_preallocation_params.max_per_iter_size,
+                                                  mem_preallocation_params.max_per_dim_diff,
+                                                  mem_preallocation_params.buffers_preallocation_ratio));
+    }
+
     calculate_weights_cache_capacity();
     allocate_primitives();
     configure_primitives_second_output();
@@ -371,9 +381,20 @@ network::network(cldnn::BinaryInputBuffer& ib, const ExecutionConfig& config, st
     , _internal(false)
     , _is_primary_stream(is_primary_stream)
     , _reset_arguments(true)
-    , _local_net_id(local_net_id) {
+    , _local_net_id(local_net_id)
+    , _shape_predictor(new ShapePredictor(&engine, config.get_property(ov::intel_gpu::buffers_preallocation_ratio))) {
     net_id = get_unique_net_id();
 
+    GPU_DEBUG_GET_INSTANCE(debug_config);
+    GPU_DEBUG_IF(debug_config->mem_preallocation_params.is_initialized) {
+        auto& mem_preallocation_params = debug_config->mem_preallocation_params;
+        _shape_predictor.reset(new ShapePredictor(&engine,
+                                                  mem_preallocation_params.next_iters_preallocation_count,
+                                                  mem_preallocation_params.max_per_iter_size,
+                                                  mem_preallocation_params.max_per_dim_diff,
+                                                  mem_preallocation_params.buffers_preallocation_ratio));
+    }
+
     kernels_cache kernels_cache(get_engine(), config, 0, nullptr, {""});
     ib >> kernels_cache;
 
@@ -735,6 +756,7 @@ void network::set_input_data(const primitive_id& id, memory::ptr data) {
 
     // Wait for previous execution completion
     reset_execution(true);
+
     input->set_data(data);
 }
 

src/plugins/intel_gpu/src/graph/primitive_inst.cpp

@@ -335,7 +335,10 @@ event::ptr primitive_inst::realloc_if_needed() {
     if (_node->get_users().size() == 1 && _node->get_users().front()->is_type<concatenation>()) {
         auto concat_inst = _network.get_primitive(get_users().front()->id());
         if (concat_inst->can_be_optimized()) {
-            concat_inst->realloc_if_needed();
+            if (!concat_inst->allocation_done_by_other) {
+                concat_inst->realloc_if_needed();
+                concat_inst->allocation_done_by_other = true;
+            }
             this->_outputs[0] = concat_inst->_outputs[0];
             GPU_DEBUG_TRACE_DETAIL << id() << ": use concat user's memory " << this->_outputs[0]->buffer_ptr() << std::endl;
             return ev;
@@ -362,6 +365,22 @@ event::ptr primitive_inst::realloc_if_needed() {
 
     bool can_reuse_buffer = _outputs[0] && actual_layout.count() <= max_output_layout_size;
 
+    // Handle runtime dynamic concat optimization
+    if (_node->is_type<concatenation>() && can_be_optimized() && allocation_done_by_other) {
+        allocation_done_by_other = false;
+        return ev;
+    }
+
+    auto current_shape = actual_layout.get_shape();
+    auto& sp = get_network().get_shape_predictor();
+    auto dt_size = data_type_traits::size_of(actual_layout.data_type);
+    auto prealloc_info = sp.predict_preallocation_shape(id(), current_shape, dt_size, can_reuse_buffer);
+    if (prealloc_info.first && sp.can_preallocate(ov::shape_size(prealloc_info.second) * dt_size)) {
+        auto new_layout = actual_layout;
+        new_layout.set_partial_shape(prealloc_info.second);
+        updated_params.output_layouts[0] = new_layout;
+    }
+
     if (can_reuse_buffer) {
         GPU_DEBUG_TRACE_DETAIL << id() << ": reuse previously allocated output buffer" << std::endl;
         if (_outputs[0]->get_layout() != actual_layout) {

src/plugins/intel_gpu/src/plugin/infer_request.cpp

@@ -873,7 +873,7 @@ void InferRequest::allocate_dev_mem_if_needed(InferenceEngine::BlobMap& device_m
 }
 
 void InferRequest::prepare_input(const cldnn::primitive_id& inputName, Blob::Ptr& inputBlob,
-                                      std::vector<cldnn::event::ptr>& dependencies) {
+                                 std::vector<cldnn::event::ptr>& dependencies) {
     OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "InferRequest::prepare_input");
     auto inputLayoutItr = m_graph->GetInputLayouts().find(inputName);
     OPENVINO_ASSERT(inputLayoutItr != m_graph->GetInputLayouts().end(), "[GPU] Input name mismatch");
@@ -898,6 +898,7 @@ void InferRequest::prepare_input(const cldnn::primitive_id& inputName, Blob::Ptr
         }
     };
 
+    auto _nw_ptr = m_graph->GetNetwork();
     if (input_layout.is_dynamic()) {
         bool has_device_blob = _deviceInputs.find(inputName) != _deviceInputs.end();
         bool should_allocate_device_blob = !has_device_blob;
@@ -909,8 +910,25 @@ void InferRequest::prepare_input(const cldnn::primitive_id& inputName, Blob::Ptr
             }
         }
 
+        auto& sp = _nw_ptr->get_shape_predictor();
+        const auto& tensor_desc = inputBlob->getTensorDesc();
+        auto dt_size = cldnn::data_type_traits::size_of(DataTypeFromPrecision(tensor_desc.getPrecision()));
+        auto current_shape = ov::Shape(tensor_desc.getDims());
+        auto prealloc_info = sp.predict_preallocation_shape(inputName, current_shape, dt_size, !should_allocate_device_blob);
+
         if (should_allocate_device_blob) {
-            _deviceInputs[inputName] = create_device_blob(inputBlob->getTensorDesc());
+            auto preallocation_shape = prealloc_info.second;
+            auto can_preallocate_buffer = prealloc_info.first &&
+                                          sp.can_preallocate(ov::shape_size(preallocation_shape) * dt_size);
+
+            if (can_preallocate_buffer) {
+                auto new_tensor_desc = tensor_desc;
+                new_tensor_desc.setDims(preallocation_shape);
+                auto device_blob = create_device_blob(new_tensor_desc);
+                _deviceInputs[inputName] = reinterpret_device_blob(device_blob, inputBlob->getTensorDesc());
+            } else {
+                _deviceInputs[inputName] = create_device_blob(tensor_desc);
+            }
         } else {
             _deviceInputs[inputName] = reinterpret_device_blob(_deviceInputs[inputName], inputBlob->getTensorDesc());
         }
@@ -919,7 +937,6 @@ void InferRequest::prepare_input(const cldnn::primitive_id& inputName, Blob::Ptr
     }
     OPENVINO_ASSERT(_deviceInputs.find(inputName) != _deviceInputs.end(), "[GPU] Couldn't find device blob allocated for ", inputName, " input");
     auto reqBlob = _deviceInputs.at(inputName)->as<gpu::ClBlob>();
-    auto _nw_ptr = m_graph->GetNetwork();
     const cldnn::primitive_id internalName = "parameter:" + inputName;
 
     switch (prec) {

src/plugins/intel_gpu/src/runtime/debug_configuration.cpp

@@ -132,6 +132,10 @@ static void print_help_messages() {
                               " For primitives fc, gemm, do, reduce, concat are supported. Separated by space.");
     message_list.emplace_back("OV_GPU_MaxKernelsPerBatch", "Maximum number of kernels in a batch during compiling kernels");
     message_list.emplace_back("OV_GPU_DumpIteration", "Dump n-th execution of network, separated by space.");
+    message_list.emplace_back("OV_GPU_MemPreallocationOptions", "Controls buffer pre-allocation feature. Expects 4 values separated by space in"
+                              "the following order: number of iterations for pre-allocation(int), max size of single iteration in bytes(int), "
+                              "max per-dim allowed diff(int), unconditional buffers preallocation ratio(float). For example for disabling memory"
+                              "preallocation at all, you can use OV_GPU_MemPreallocationOptions='0 0 0 1.0'");
 
     auto max_name_length_item = std::max_element(message_list.begin(), message_list.end(),
         [](std::pair<std::string, std::string>& a, std::pair<std::string, std::string>& b){
@@ -197,6 +201,8 @@ debug_configuration::debug_configuration()
     get_gpu_debug_env_var("MaxKernelsPerBatch", max_kernels_per_batch);
     std::string dump_iteration_str;
     get_gpu_debug_env_var("DumpIteration", dump_iteration_str);
+    std::string mem_preallocation_params_str;
+    get_gpu_debug_env_var("MemPreallocationOptions", mem_preallocation_params_str);
 
     if (help > 0) {
         print_help_messages();
@@ -236,6 +242,32 @@ debug_configuration::debug_configuration()
         }
     }
 
+    if (mem_preallocation_params_str.size() > 0) {
+        mem_preallocation_params_str = " " + mem_preallocation_params_str + " ";
+        std::istringstream ss(mem_preallocation_params_str);
+        std::vector<std::string> params;
+        std::string param;
+        while (ss >> param)
+            params.push_back(param);
+
+        bool correct_params = params.size() == 4;
+        if (correct_params) {
+            try {
+                mem_preallocation_params.next_iters_preallocation_count = std::stol(params[0]);
+                mem_preallocation_params.max_per_iter_size = std::stol(params[1]);
+                mem_preallocation_params.max_per_dim_diff = std::stol(params[2]);
+                mem_preallocation_params.buffers_preallocation_ratio = std::stof(params[3]);
+            } catch(const std::exception& ex) {
+                correct_params = false;
+            }
+        }
+
+        if (!correct_params)
+            GPU_DEBUG_COUT_ << "OV_GPU_MemPreallocationOptions were ignored, because they cannot be parsed.\n";
+
+        mem_preallocation_params.is_initialized = correct_params;
+    }
+
     if (after_proc_str.length() > 0) {
 #ifdef _WIN32
         GPU_DEBUG_COUT_ << "Warning: OV_GPU_AfterProc is supported only on linux" << std::endl;

src/plugins/intel_gpu/src/runtime/execution_config.cpp

@@ -70,7 +70,8 @@ void ExecutionConfig::set_default() {
         std::make_tuple(ov::intel_gpu::force_implementations, ImplForcingMap{}),
         std::make_tuple(ov::intel_gpu::partial_build_program, false),
         std::make_tuple(ov::intel_gpu::allow_new_shape_infer, false),
-        std::make_tuple(ov::intel_gpu::use_only_static_kernels_for_dynamic_shape, false));
+        std::make_tuple(ov::intel_gpu::use_only_static_kernels_for_dynamic_shape, false),
+        std::make_tuple(ov::intel_gpu::buffers_preallocation_ratio, 1.1f));
 }
 
 void ExecutionConfig::register_property_impl(const std::pair<std::string, ov::Any>& property, PropertyVisibility visibility, BaseValidator::Ptr validator) {

src/plugins/intel_gpu/src/runtime/shape_predictor.cpp

@@ -0,0 +1,135 @@
+// Copyright (C) 2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "intel_gpu/runtime/shape_predictor.hpp"
+#include "intel_gpu/runtime/engine.hpp"
+
+namespace cldnn {
+
+static ov::Shape operator*(const ov::Shape& s1, const ov::Shape& s2) {
+    OPENVINO_ASSERT(s1.size() == s2.size(), "[GPU] Rank mismatch: ", s1, " and ", s2);
+
+    std::vector<size_t> result;
+    for (size_t i = 0; i < s1.size(); i++) {
+        result.push_back(s1[i] * s2[i]);
+    }
+    return result;
+}
+
+static ov::Shape operator+(const ov::Shape& s1, const ov::Shape& s2) {
+    OPENVINO_ASSERT(s1.size() == s2.size(), "[GPU] Rank mismatch: ", s1, " and ", s2);
+
+    std::vector<size_t> result;
+    for (size_t i = 0; i < s1.size(); i++)
+        result.push_back(s1[i] + s2[i]);
+
+    return result;
+}
+
+static ov::Shape operator-(const ov::Shape& s1, const ov::Shape& s2) {
+    OPENVINO_ASSERT(s1.size() == s2.size(), "[GPU] Rank mismatch: ", s1, " and ", s2);
+
+    std::vector<size_t> result;
+    for (size_t i = 0; i < s1.size(); i++) {
+        if (s1[i] < s2[i])
+            return std::vector<size_t>();
+
+        result.push_back(s1[i] - s2[i]);
+    }
+
+    return result;
+}
+
+void ShapePredictor::add_shape(const std::string& id, const ov::Shape& shape) {
+    auto& shapes = _shapes_info[id];
+    if (shapes.size() >= _max_deque_size)
+        shapes.pop_front();
+
+    shapes.push_back(shape);
+}
+
+bool ShapePredictor::can_preallocate(size_t desired_buffer_size) {
+    const auto memory_threshold = 0.90f;
+    auto device_mem_usage = _engine->get_used_device_memory(cldnn::allocation_type::usm_device);
+
+    return device_mem_usage + desired_buffer_size < _engine->get_device_info().max_global_mem_size * memory_threshold;
+}
+
+std::pair<bool, ov::Shape> ShapePredictor::predict_preallocation_shape(const std::string& id,
+                                                                       const ov::Shape& current_shape,
+                                                                       size_t dt_size,
+                                                                       bool can_reuse_buffer) {
+    add_shape(id, current_shape);
+
+    // Save shape information and exit without pre-allocation suggestion if current
+    // buffer can be reused
+    if (can_reuse_buffer)
+        return {false, {}};
+
+    // Check if there is enough data for prediction
+    auto& shapes = _shapes_info[id];
+    const auto shapes_num = shapes.size();
+
+    // Number of shapes used for iterations mode predictions
+    const auto min_shapes_num = _max_deque_size;
+
+    if (shapes_num >= min_shapes_num) {
+        std::vector<ov::Shape> diffs;
+
+        for (size_t i = 0; i < min_shapes_num - 1; ++i) {
+            auto result = shapes[shapes_num - i - 1] - shapes[shapes_num - i - 2];
+            if (result.empty())
+                break;
+            diffs.push_back(result);
+        }
+
+        bool can_use_iterations_preallocation = diffs.size() == min_shapes_num - 1;
+        for (size_t i = 1; i < diffs.size(); ++i) {
+            if (diffs[0] != diffs[i]) {
+                can_use_iterations_preallocation = false;
+                break;
+            }
+        }
+
+        if (can_use_iterations_preallocation)
+            can_use_iterations_preallocation = !all_zeroes(diffs[0]);
+
+        // Allow iterations preallocation only for per-dimension diff less than
+        // '_max_per_dim_diff' and iteration size less than `_max_per_iter_size`
+        // to avoid huge unexpected memory preallocations
+        if (can_use_iterations_preallocation) {
+            for (size_t i = 0; i < diffs[0].size(); ++i) {
+                if (diffs[0][i] > _max_per_dim_diff) {
+                    can_use_iterations_preallocation = false;
+                    break;
+                }
+            }
+
+            ov::Shape single_iter_shape;
+            for (size_t i = 0; i < current_shape.size(); ++i)
+                single_iter_shape.push_back(diffs[0][i] == 0 ? current_shape[i] : 1);
+
+            if (ov::shape_size(single_iter_shape) * dt_size > _max_per_iter_size)
+                can_use_iterations_preallocation = false;
+        }
+
+        if (can_use_iterations_preallocation) {
+            // Apply preallocation for the next N iterations
+            ov::Shape mul_shape(diffs[0].size(), _next_iters_preallocation_count);
+            auto preallocation_shape = diffs[0] * mul_shape;
+            auto new_shape = current_shape + preallocation_shape;
+            return {true, new_shape};
+        } else if (_buffers_preallocation_ratio > 1.0f) {
+            // Apply percentage buffer preallocation
+            auto current_shape_size = ov::shape_size(current_shape);
+            ov::Shape new_shape_size(current_shape.size(), 1);
+            new_shape_size[0] = static_cast<size_t>(current_shape_size * _buffers_preallocation_ratio);
+            return {true, new_shape_size};
+        }
+    }
+
+    return {false, {}};
+}
+
+}  // namespace cldnn

src/plugins/intel_gpu/tests/unit/module_tests/shape_predictor_test.cpp

@@ -0,0 +1,66 @@
+// Copyright (C) 2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "test_utils.h"
+
+#include "intel_gpu/runtime/shape_predictor.hpp"
+
+using namespace cldnn;
+using namespace ::tests;
+
+struct shape_predictor_test_params {
+    std::vector<ov::Shape> in_shapes;
+    ov::Shape expected_predicted_shape;
+    float buffers_preallocation_ratio;
+    bool can_reuse_buffer;
+};
+
+class shape_predictor_tests : public testing::TestWithParam<shape_predictor_test_params> {};
+TEST_P(shape_predictor_tests, prediction) {
+    auto p = GetParam();
+    auto& in_shapes = p.in_shapes;
+    auto& expected_predicted_shape = p.expected_predicted_shape;
+    auto& engine = get_test_engine();
+
+    ShapePredictor sp(&engine, p.buffers_preallocation_ratio);
+    std::pair<bool, ov::Shape> result;
+    const auto dt_size = 4;
+
+    for (auto& shape : in_shapes)
+        result = sp.predict_preallocation_shape("dummy_name", shape, dt_size, p.can_reuse_buffer);
+
+    ASSERT_TRUE(result.first == !expected_predicted_shape.empty());
+    ASSERT_EQ(result.second, expected_predicted_shape);
+}
+
+INSTANTIATE_TEST_SUITE_P(smoke, shape_predictor_tests,
+    testing::ValuesIn(std::vector<shape_predictor_test_params>{
+        // Preallocation for next N iterations tests
+        {{{1,1}, {1,1}, {1,1}}, {}, 1.0f, false},
+        {{{1,1}, {1,21}, {1,31}}, {}, 1.0f, false},
+        {{{1,3}, {1,2}, {1,1}}, {}, 1.0f, false},
+        {{{1,1}, {1,2}, {1,3}}, {1,13}, 1.0f, false},
+        {{{1,1}, {1,2}, {1,3}}, {1,13}, 1.1f, false},
+        {{{1,1,1}, {1,2,2}, {1,3,3}}, {1,13,13}, 1.0f, false},
+        {{{1,1,1}, {1,2,2}, {1,3,3}}, {1,13,13}, 1.1f, false},
+        {{{1,1,1}, {1,3,2}, {1,7,3}}, {}, 1.0f, false},
+        {{{1,1,1}, {1,1,3}, {1,1,5}}, {1,1,25}, 1.0f, false},
+        {{{1,1,1}, {1,1,3}, {1,1,5}}, {1,1,25}, 1.1f, false},
+        {{{1,1,1}, {1,1,3}, {1,1,5}}, {1,1,25}, 1.0f, false},
+        {{{1,1,1}, {1,1,3}, {1,1,5}}, {1,1,25}, 1.1f, false},
+        {{{1,1}, {1,1}, {1,1}, {1,1}, {1,1}, {1,1}}, {}, 1.0f, false},
+        {{{1,10}, {1,1}, {1,2}, {1,3}}, {1,13}, 1.0f, false},
+        {{{1,10}, {1,1}, {1,2}, {1,3}}, {1,13}, 1.1f, false},
+        {{{1,3,480,720}, {3,3,480,720}, {5,3,480,720}}, {}, 1.0f, false},
+        {{{1,1}, {1,1}, {1,1}}, {}, 1.0f, true},
+        {{{1,1}, {1,2}, {1,3}}, {}, 1.0f, true},
+
+        // Percentage preallocation tests
+        {{{1,1}, {1,1}, {1,1}}, {1,1}, 1.1f, false},
+        {{{1,1}, {1,128}, {1,256}}, {281, 1}, 1.1f, false},
+        {{{1,3,128}, {1,3,112}, {1,3,418}, {1,3,512}}, {1689,1,1}, 1.1f, false},
+        {{{1,1}, {1,1}, {1,1}}, {}, 1.1f, true},
+        {{{1,1}, {1,128}, {1,256}}, {}, 1.1f, true},
+        {{{1,3,128}, {1,3,112}, {1,3,418}, {1,3,512}}, {}, 1.1f, true},
+    }));

src/plugins/intel_gpu/tests/unit/test_cases/concatenation_gpu_test.cpp

@@ -160,7 +160,7 @@ void start_concat_test_dynamic(impl_types impl_type) {
         ASSERT_EQ(outputs.begin()->first, "concat");
 
         auto output_memory = outputs.at("concat").get_memory();
-        auto output_layout = output_memory->get_layout();
+        auto output_layout = outputs.at("concat").get_layout();
         cldnn::mem_lock<float> output_ptr(output_memory, get_test_stream());
 
         ov::PartialShape expected_shape = layout0.get_partial_shape();