[GPU] Moved memory pool to network (#6599)

inference-engine/thirdparty/clDNN/api/cldnn/runtime/engine.hpp

@@ -15,6 +15,7 @@
 #include <set>
 #include <utility>
 #include <string>
+#include <atomic>
 
 #define CLDNN_THREADING_SEQ 0
 #define CLDNN_THREADING_TBB 1
@@ -39,15 +40,6 @@ public:
     /// Returns runtime type used in the engine
     virtual runtime_types runtime_type() const = 0;
 
-    /// Create memory object with specified @p layout and allocation @p type for primitive with @p id
-    /// Underlying memory handle can be reused with other primitives from memory pool based on @p dependencies
-    memory_ptr get_memory_from_pool(const layout& layout,
-                                    primitive_id id,
-                                    uint32_t network_id,
-                                    std::set<primitive_id> dependencies,
-                                    allocation_type type,
-                                    bool reusable = true);
-
     /// Create memory object attached to the buffer allocated by user.
     /// @param ptr  The pointer to user allocated buffer.
     /// @note Size (in bytes) of the buffer should be equal to @p layout.bytes_count()
@@ -101,9 +93,6 @@ public:
     /// Returns device object associated with the engine
     const device::ptr get_device() const;
 
-    /// Returns memory pool for the engine
-    memory_pool& get_memory_pool();
-
     /// Returns user context handle which was used to create the engine
     virtual void* get_user_context() const = 0;
 
@@ -113,6 +102,12 @@ public:
     /// Returns the amount of GPU memory currently used by the engine
     uint64_t get_used_device_memory() const;
 
+    /// Adds @p bytes count to currently used memory size
+    void add_memory_used(uint64_t bytes);
+
+    /// Subtracts @p bytes count from currently used memory size
+    void subtract_memory_used(uint64_t bytes);
+
     /// Returns true if USM is enabled in engine config and device/driver supports required features
     bool use_unified_shared_memory() const;
 
@@ -144,11 +139,11 @@ public:
 protected:
     /// Create engine for given @p device and @p configuration
     engine(const device::ptr device, const engine_configuration& configuration);
-
-    // TODO: Consider moving memory pool to cldnn::network
-    std::unique_ptr<memory_pool> _memory_pool;
     const device::ptr _device;
     engine_configuration _configuration;
+
+    std::atomic<uint64_t> memory_usage = {0};
+    std::atomic<uint64_t> peak_memory_usage = {0};
 };
 
 }  // namespace cldnn

inference-engine/thirdparty/clDNN/api/cldnn/runtime/memory_caps.hpp

@@ -8,6 +8,7 @@
 #include <string>
 #include <vector>
 #include <algorithm>
+#include <ostream>
 
 namespace cldnn {
 
@@ -19,6 +20,18 @@ enum class allocation_type {
     usm_device,  // Accessible only by device. Not migratable.
 };
 
+inline std::ostream& operator<<(std::ostream& out, const allocation_type& alloc_type) {
+    switch (alloc_type) {
+        case allocation_type::cl_mem:     out << "cl_mem";     break;
+        case allocation_type::usm_host:   out << "usm_host";   break;
+        case allocation_type::usm_shared: out << "usm_shared"; break;
+        case allocation_type::usm_device: out << "usm_device"; break;
+        default: out << "unknown"; break;
+    }
+
+    return out;
+}
+
 class memory_capabilities {
 public:
     memory_capabilities(std::vector<allocation_type> supported_allocation_types) : _caps(supported_allocation_types) {}

inference-engine/thirdparty/clDNN/api/cldnn/runtime/memory_pool.hpp

@@ -97,8 +97,6 @@ class memory_pool {
     std::map<layout, std::list<memory_record>, padded_pool_comparer> _padded_pool;
     std::multimap<uint64_t, memory_record> _no_reusable_pool;
     engine* _engine;
-    std::atomic<uint64_t> _temp_memory_used;
-    std::atomic<uint64_t> _max_peak_memory_used;
 
 public:
     explicit memory_pool(engine& engine);
@@ -127,11 +125,6 @@ public:
     void clear_pool();
     void clear_pool_for_network(uint32_t network_id);
     void release_memory(memory* memory, const primitive_id& id, uint32_t network_id);
-
-    uint64_t get_temp_memory_used() const { return _temp_memory_used; }
-    uint64_t get_max_peak_device_memory_used() const { return _max_peak_memory_used; }
-    void add_memory_used(size_t value);
-    void subtract_memory_used(size_t value);
 };
 
 }  // namespace cldnn

inference-engine/thirdparty/clDNN/runtime/engine.cpp

@@ -20,8 +20,7 @@
 namespace cldnn {
 
 engine::engine(const device::ptr device, const engine_configuration& configuration)
-: _memory_pool(new memory_pool(*this))
-, _device(device)
+: _device(device)
 , _configuration(configuration) {}
 
 device_info engine::get_device_info() const {
@@ -72,17 +71,6 @@ allocation_type engine::get_lockable_preffered_memory_allocation_type(bool is_im
     throw std::runtime_error("[clDNN internal error] Could not find proper allocation type!");
 }
 
-memory::ptr engine::get_memory_from_pool(const layout& layout,
-                                         primitive_id id,
-                                         uint32_t network_id,
-                                         std::set<primitive_id> dependencies,
-                                         allocation_type type,
-                                         bool reusable) {
-    if (_configuration.use_memory_pool)
-        return _memory_pool->get_memory(layout, id, network_id, dependencies, type, reusable);
-    return _memory_pool->get_memory(layout, type);
-}
-
 memory::ptr engine::attach_memory(const layout& layout, void* ptr) {
     return std::make_shared<simple_attached_memory>(layout, ptr);
 }
@@ -131,16 +119,23 @@ memory_ptr engine::share_surface(const layout& layout, shared_surface surf, uint
 }
 #endif  // _WIN32
 
-memory_pool& engine::get_memory_pool() {
-    return *_memory_pool.get();
-}
-
 uint64_t engine::get_max_used_device_memory() const {
-    return _memory_pool->get_max_peak_device_memory_used();
+    return peak_memory_usage.load();
 }
 
 uint64_t engine::get_used_device_memory() const {
-    return _memory_pool->get_temp_memory_used();
+    return memory_usage.load();
+}
+
+void engine::add_memory_used(size_t bytes) {
+    memory_usage += bytes;
+    if (memory_usage > peak_memory_usage) {
+        peak_memory_usage = memory_usage.load();
+    }
+}
+
+void engine::subtract_memory_used(size_t bytes) {
+    memory_usage -= bytes;
 }
 
 std::shared_ptr<cldnn::engine> engine::create(engine_types engine_type,

inference-engine/thirdparty/clDNN/runtime/memory.cpp

@@ -5,6 +5,7 @@
 #include "cldnn/runtime/memory.hpp"
 #include "cldnn/runtime/engine.hpp"
 #include "cldnn/runtime/stream.hpp"
+#include "cldnn/runtime/debug_configuration.hpp"
 
 #include "ocl/ocl_memory.hpp"
 
@@ -17,12 +18,29 @@
 namespace cldnn {
 
 memory::memory(engine* engine, const layout& layout, allocation_type type, bool reused)
-    : _engine(engine), _layout(layout), _bytes_count(_layout.bytes_count()), _type(type), _reused(reused) {}
+    : _engine(engine), _layout(layout), _bytes_count(_layout.bytes_count()), _type(type), _reused(reused) {
+    if (!_reused && _engine) {
+        _engine->add_memory_used(_bytes_count);
+    }
+
+    GPU_DEBUG_GET_INSTANCE(debug_config);
+    GPU_DEBUG_IF(debug_config->verbose >= 1) {
+        GPU_DEBUG_COUT << "Allocate " << _bytes_count << " bytes of " << type << " allocation type"
+                       << " (current=" << _engine->get_used_device_memory() << ";"
+                       << " max=" << _engine->get_max_used_device_memory() << ")" << std::endl;
+    }
+}
 
 memory::~memory() {
     if (!_reused && _engine) {
-        // TODO: Make memory usage tracker static in memory class
-        _engine->get_memory_pool().subtract_memory_used(_bytes_count);
+        _engine->subtract_memory_used(_bytes_count);
+    }
+
+    GPU_DEBUG_GET_INSTANCE(debug_config);
+    GPU_DEBUG_IF(debug_config->verbose >= 1) {
+        GPU_DEBUG_COUT << "Free " << _bytes_count << " bytes"
+                       << " (current=" << _engine->get_used_device_memory() << ";"
+                       << " max=" << _engine->get_max_used_device_memory() << ")" << std::endl;
     }
 }
 

inference-engine/thirdparty/clDNN/runtime/memory_pool.cpp

@@ -24,10 +24,6 @@ memory_record::memory_record(memory_set users,
     : _users(users), _memory(memory), _network_id(net_id), _type(type) {}
 
 memory::ptr memory_pool::alloc_memory(const layout& layout, allocation_type type) {
-    if (_max_peak_memory_used > _engine->get_device_info().max_global_mem_size) {
-        throw std::runtime_error("exceeded global device memory");
-    }
-
     return _engine->allocate_memory(layout, type);
 }
 
@@ -298,26 +294,6 @@ void memory_pool::clear_pool_for_network(uint32_t network_id) {
     }
 }
 
-memory_pool::memory_pool(engine& engine) : _engine(&engine), _temp_memory_used(0), _max_peak_memory_used(0) { }
-
-void memory_pool::add_memory_used(size_t value) {
-    // std::cerr << "ADD MEM: " << value
-    //           << " max: " << _engine->get_device_info().max_global_mem_size
-    //           << " peak: " << _max_peak_memory_used
-    //           << " tmp: " << _temp_memory_used << std::endl;
-
-    _temp_memory_used += value;
-    if (_temp_memory_used > _max_peak_memory_used) {
-        _max_peak_memory_used = _temp_memory_used.load();
-    }
-}
-
-void memory_pool::subtract_memory_used(size_t value) {
-    _temp_memory_used -= value;
-    // std::cerr << "FREE MEM: " << value
-    //           << " max: " << _engine->get_device_info().max_global_mem_size
-    //           << " peak: " << _max_peak_memory_used
-    //           << " tmp: " << _temp_memory_used << std::endl;
-}
+memory_pool::memory_pool(engine& engine) : _engine(&engine) { }
 
 }  // namespace cldnn

inference-engine/thirdparty/clDNN/runtime/ocl/ocl_engine.cpp

@@ -73,8 +73,6 @@ memory::ptr ocl_engine::allocate_memory(const layout& layout, allocation_type ty
         throw std::runtime_error("exceeded max size of memory object allocation");
     }
 
-    _memory_pool->add_memory_used(layout.bytes_count());
-
     try {
         memory::ptr res = nullptr;
         if (layout.format.is_image_2d()) {

inference-engine/thirdparty/clDNN/src/include/network_impl.h

@@ -102,10 +102,19 @@ public:
     bool is_internal() const { return _internal; }
     bool is_primary_stream() { return _is_primary_stream; }
 
+    /// Create memory object with specified @p layout and allocation @p type for primitive with @p id
+    /// Underlying memory handle can be reused with other primitives from memory pool based on @p dependencies
+    memory_ptr get_memory_from_pool(const layout& layout,
+                                    primitive_id id,
+                                    std::set<primitive_id> dependencies,
+                                    allocation_type type,
+                                    bool reusable = true);
+
 private:
     uint32_t net_id = 0;
     program_impl::ptr _program;
     stream::ptr _stream;
+    std::unique_ptr<memory_pool> _memory_pool;
     bool _internal;
     bool _is_primary_stream;
     bool _reset_arguments;

inference-engine/thirdparty/clDNN/src/network.cpp

@@ -284,7 +284,12 @@ Network_impl will always have net_id = 0 when it will be cldnn internal micronet
 opt pass).
 */
 network_impl::network_impl(program_impl::ptr program, stream::ptr stream, bool is_internal, bool is_primary_stream)
-    : _program(program), _stream(stream), _internal(is_internal), _is_primary_stream(is_primary_stream), _reset_arguments(true) {
+    : _program(program)
+    , _stream(stream)
+    , _memory_pool(new memory_pool(program->get_engine()))
+    , _internal(is_internal)
+    , _is_primary_stream(is_primary_stream)
+    , _reset_arguments(true) {
     static std::atomic<uint32_t> id_gen{0};
     if (!_internal) {
         net_id = ++id_gen;
@@ -298,7 +303,7 @@ network_impl::network_impl(program_impl::ptr program, stream::ptr stream, bool i
 }
 
 network_impl::~network_impl() {
-    get_engine().get_memory_pool().clear_pool_for_network(net_id);
+    _memory_pool->clear_pool_for_network(net_id);
 }
 
 network_impl::ptr network_impl::allocate_network(stream::ptr stream, program_impl::ptr program, bool is_internal, bool is_primary_stream) {
@@ -711,11 +716,20 @@ void network_impl::transfer_memory_to_device(std::shared_ptr<primitive_inst> ins
 
     if (alloc_type == allocation_type::usm_host || alloc_type == allocation_type::usm_shared) {
         // Allocate and transfer memory
-        auto& mem_pool = inst_mem.get_engine()->get_memory_pool();
         auto device_mem = inst_mem.get_engine()->allocate_memory(inst_mem.get_layout(), allocation_type::usm_device, false);
         device_mem->copy_from(get_stream(), inst_mem);
-        mem_pool.release_memory(&inst_mem, node.id(), get_id());
+        _memory_pool->release_memory(&inst_mem, node.id(), get_id());
         instance->set_output_memory(device_mem);
     }
 }
+
+memory::ptr network_impl::get_memory_from_pool(const layout& layout,
+                                               primitive_id id,
+                                               std::set<primitive_id> dependencies,
+                                               allocation_type type,
+                                               bool reusable) {
+    if (get_engine().configuration().use_memory_pool)
+        return _memory_pool->get_memory(layout, id, get_id(), dependencies, type, reusable);
+    return _memory_pool->get_memory(layout, type);
+}
 }  // namespace cldnn

inference-engine/thirdparty/clDNN/src/primitive_inst.cpp

@@ -149,7 +149,6 @@ primitive_inst::primitive_inst(network_impl& network, program_node const& node,
 
 memory::ptr primitive_inst::allocate_output() {
     auto layout = _node.get_output_layout();
-    auto net_id = get_network_id();
     auto& engine = get_network().get_engine();
 
     // For outputs, cpu prim we want to have lockable alloc type
@@ -163,12 +162,11 @@ memory::ptr primitive_inst::allocate_output() {
                                                      : allocation_type::usm_device;
 
     if (!_network.is_internal() && (_node.can_be_optimized() || _node.is_type<generic_layer>())) {
-        return engine.get_memory_from_pool(layout,
-                                           _node.id(),
-                                           net_id,
-                                           _node.get_memory_dependencies(),
-                                           alloc_type,
-                                           false);
+        return _network.get_memory_from_pool(layout,
+                                             _node.id(),
+                                             _node.get_memory_dependencies(),
+                                             alloc_type,
+                                             false);
     } else if (_network.is_internal() && _node.is_output() && _node.is_type<generic_layer>() &&
                engine.supports_allocation(allocation_type::usm_device)) {
         return engine.allocate_memory(layout, allocation_type::usm_device, false);
@@ -179,12 +177,11 @@ memory::ptr primitive_inst::allocate_output() {
     } else if (_network.is_internal() || (!_node.can_share_buffer()) || _node.can_be_optimized() || _node.is_output()) {
         return engine.allocate_memory(layout, alloc_type);
     } else {
-        return engine.get_memory_from_pool(layout,
-                                           _node.id(),
-                                           net_id,
-                                           _node.get_memory_dependencies(),
-                                           alloc_type,
-                                           true);
+        return _network.get_memory_from_pool(layout,
+                                             _node.id(),
+                                             _node.get_memory_dependencies(),
+                                             alloc_type,
+                                             true);
     }
 }