[GPU] Peak memory usage optimizations (#14799)

src/plugins/intel_gpu/include/intel_gpu/runtime/debug_configuration.hpp

@@ -42,6 +42,9 @@ enum class LogLevel : int8_t {
 
 #ifdef GPU_DEBUG_CONFIG
 #define GPU_DEBUG_IF(cond) if (cond)
+#define GPU_DEBUG_CODE(...) __VA_ARGS__
+#define GPU_DEBUG_DEFINE_MEM_LOGGER(stage) \
+    cldnn::instrumentation::mem_usage_logger mem_logger{stage, cldnn::debug_configuration::get_instance()->verbose >= 2};
 #define GPU_DEBUG_PROFILED_STAGE(stage) \
     auto stage_prof = cldnn::instrumentation::profiled_stage<primitive_inst>(\
         !cldnn::debug_configuration::get_instance()->dump_profiling_data.empty(), *this, stage)
@@ -52,6 +55,8 @@ enum class LogLevel : int8_t {
 #define GPU_DEBUG_LOG_RAW(min_verbose_level) GPU_DEBUG_LOG_RAW_INT(static_cast<std::underlying_type<ov::intel_gpu::LogLevel>::type>(min_verbose_level))
 #else
 #define GPU_DEBUG_IF(cond) if (0)
+#define GPU_DEBUG_CODE(...)
+#define GPU_DEBUG_DEFINE_MEM_LOGGER(stage)
 #define GPU_DEBUG_PROFILED_STAGE(stage)
 #define GPU_DEBUG_PROFILED_STAGE_CACHE_HIT(val)
 #define GPU_DEBUG_LOG_RAW(min_verbose_level) if (0) std::cout << cldnn::debug_configuration::prefix

src/plugins/intel_gpu/include/intel_gpu/runtime/profiling.hpp

@@ -9,6 +9,20 @@
 #include <vector>
 #include <string>
 
+#if defined(_WIN32)
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#ifndef NOGDI
+#define NOGDI
+#endif
+
+#include <windows.h>
+#include "Psapi.h"
+#elif !defined(__APPLE__) && !defined(__MACOSX)
+#include <fstream>
+#endif
+
 #include "layout.hpp"
 #include "utils.hpp"
 #include "debug_configuration.hpp"
@@ -163,6 +177,83 @@ private:
     bool cache_hit = false;
 };
 
+class mem_usage_logger {
+public:
+    struct memory_footprint {
+        memory_footprint() : rss(0), peak_rss(0) {}
+        memory_footprint(int64_t rss, int64_t peak_rss) : rss(rss), peak_rss(peak_rss) {}
+        int64_t rss;
+        int64_t peak_rss;
+    };
+
+    mem_usage_logger(const std::string& stage_name, bool lifetime_logging_mode = true, bool print_mem_usage = true)
+        : _stage_name(stage_name)
+        , _lifetime_logging_mode(lifetime_logging_mode)
+        , _print_mem_usage(print_mem_usage) {
+        if (_lifetime_logging_mode)
+            start_logging();
+    }
+
+    ~mem_usage_logger() {
+        if (_lifetime_logging_mode)
+            stop_logging();
+        if (_print_mem_usage && _is_active)
+            print_mem_usage_info();
+    }
+
+    void start_logging() {
+        _is_active = true;
+        _before = get_memory_footprint();
+    }
+
+    void stop_logging() {
+        _after = get_memory_footprint();
+    }
+
+    memory_footprint get_elapsed_mem_usage() {
+        return memory_footprint{ _after.rss - _before.rss, _after.peak_rss - _before.peak_rss };
+    }
+
+    void print_mem_usage_info() {
+        auto mem_usage = get_elapsed_mem_usage();
+        GPU_DEBUG_LOG << "Memory usage for " << _stage_name << ": " << mem_usage.rss << " KB (current RSS: "
+                      << _after.rss << " KB; peak RSS: " << _after.peak_rss << " KB)" << std::endl;
+    }
+
+private:
+    memory_footprint get_memory_footprint() {
+        memory_footprint footprint;
+#if defined(_WIN32)
+        PROCESS_MEMORY_COUNTERS pmc;
+        GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc));
+        footprint.rss = (int64_t)(pmc.WorkingSetSize/1024);
+        footprint.peak_rss = (int64_t)(pmc.PeakWorkingSetSize/1024);
+#elif !defined(__APPLE__) && !defined(__MACOSX)
+        std::ifstream status("/proc/self/status");
+        if (!status.is_open())
+            return footprint;
+
+        std::string line, title;
+        while (std::getline(status, line)) {
+            std::istringstream iss(line);
+            iss >> title;
+            if (title == "VmHWM:")
+                iss >> footprint.peak_rss;
+            else if (title == "VmRSS:")
+                iss >> footprint.rss;
+        }
+#endif
+        return footprint;
+    }
+
+    std::string _stage_name = {};
+    bool _lifetime_logging_mode = false;
+    bool _print_mem_usage = false;
+    bool _is_active = false;
+    memory_footprint _before;
+    memory_footprint _after;
+};
+
 /// @}
 /// @}
 }  // namespace instrumentation

src/plugins/intel_gpu/src/graph/impls/ocl/primitive_base.hpp

@@ -329,6 +329,12 @@ protected:
         }
         return kernel_strings;
     }
+
+    void reset_kernels_source() override {
+        for (size_t i = 0; i < _kernel_data.kernels.size(); ++i) {
+            _kernel_data.kernels[i].code.kernelString.reset();
+        }
+    }
 };
 
 }  // namespace ocl

src/plugins/intel_gpu/src/graph/include/primitive_inst.h

@@ -65,6 +65,7 @@ struct primitive_impl {
         return {};
     }
     virtual std::vector<std::shared_ptr<cldnn::kernel_string>> get_kernels_source() { return {}; }
+    virtual void reset_kernels_source() {}
     virtual void set_kernels(std::vector<kernel::ptr>) {}
     virtual std::vector<kernel::ptr> get_kernels() const { return {}; }
     virtual void set_kernel_ids(std::vector<kernel_id> kernel_ids) {}

src/plugins/intel_gpu/src/graph/network.cpp

@@ -305,6 +305,7 @@ network::network(program::ptr program, stream::ptr stream, bool is_internal, boo
     add_default_output_chains();
 
     if (is_dynamic()) {
+        GPU_DEBUG_DEFINE_MEM_LOGGER("dynamic_network_initialization");
         _kernels_cache = std::unique_ptr<kernels_cache>(new kernels_cache(program->get_engine(), program->get_id(),
                                                                         kernel_selector::KernelBase::get_db().get_batch_header_str()));
         _impls_cache = std::unique_ptr<ImplementationsCache>(new ImplementationsCache(_impls_cache_capacity));
@@ -529,6 +530,7 @@ network::ptr network::build_network(engine& engine,
 }
 
 void network::validate_primitives() {
+    GPU_DEBUG_DEFINE_MEM_LOGGER("validate_primitives");
     for (auto const& prim : _exec_order) {
         bool valid = prim->validate();
         CLDNN_ERROR_NOT_EQUAL(prim->id(), "validate", valid, "", true, "has not a valid instance.");
@@ -582,6 +584,7 @@ void network::set_input_data(const primitive_id& id, memory::ptr data) {
 }
 
 void network::add_default_output_chains() {
+    GPU_DEBUG_DEFINE_MEM_LOGGER("add_default_output_chains");
     for (auto& output : _outputs) {
         add_output_chain(output);
     }
@@ -760,6 +763,7 @@ layout network::get_node_output_layout(const primitive_id& output_id) const {
 }
 
 void network::allocate_primitives() {
+    GPU_DEBUG_DEFINE_MEM_LOGGER("allocate_primitives");
     std::vector<std::shared_ptr<program_node>> nodes_to_allocate{};
     auto& po = _program->get_processing_order();
     for (auto node : po) {
@@ -822,6 +826,7 @@ void network::allocate_primitives() {
 }
 
 void network::configure_primitives_second_output() {
+    GPU_DEBUG_DEFINE_MEM_LOGGER("configure_primitives_second_output");
     std::map<cldnn::memory::ptr, std::vector<const cldnn::program_node*>> mutable_datas_ptrs;
     for (auto& inst : _primitives) {
         auto& node = inst.second->get_node();
@@ -857,12 +862,14 @@ void network::configure_primitives_second_output() {
 }
 
 void network::build_insts_deps() {
+    GPU_DEBUG_DEFINE_MEM_LOGGER("build_insts_deps");
     for (auto& inst : _primitives) {
         inst.second->build_deps();
     }
 }
 
 void network::build_exec_order() {
+    GPU_DEBUG_DEFINE_MEM_LOGGER("build_exec_order");
     for (auto& node : _program->get_processing_order()) {
         if (!node->is_type<data>() && !(node->is_type<mutable_data>() && node->get_dependencies().empty())) {
             add_to_exec_order(node->id());

src/plugins/intel_gpu/src/graph/nodes_ordering.cpp

@@ -46,6 +46,7 @@ void program::nodes_ordering::calc_processing_order(program& p) {
     output: BFS topological order.
     */
 void program::nodes_ordering::calculate_BFS_processing_order() {
+    GPU_DEBUG_DEFINE_MEM_LOGGER("calculate_BFS_processing_order");
     std::map<program_node*, int> distances;
     for (auto itr : _processing_order) {
         distances[itr] = -1;

src/plugins/intel_gpu/src/graph/pass_manager.cpp

@@ -49,6 +49,7 @@ void pass_manager::run(program& p, base_pass& pass) {
     using Time = std::chrono::high_resolution_clock;
 
     GPU_DEBUG_LOG << "Run pass " << pass.get_name() << std::endl;
+    GPU_DEBUG_DEFINE_MEM_LOGGER(pass.get_name());
     auto start = Time::now();
     pass.run(p);
     auto stop = Time::now();

src/plugins/intel_gpu/src/graph/primitive_inst.cpp

@@ -802,11 +802,14 @@ memory::ptr primitive_inst::allocate_output(engine& _engine, memory_pool& pool,
     auto use_lockable_memory = is_output_buffer(_node) || is_cpu || is_any_user_cpu(_node.get_users()) ||
                                !_engine.supports_allocation(allocation_type::usm_device);
     const auto& lockable_mem_type = _engine.get_lockable_preferred_memory_allocation_type(layout.format.is_image_2d());
-    const auto& alloc_type = use_lockable_memory ? lockable_mem_type
-        : usm_device_allocatable ? allocation_type::usm_device : lockable_mem_type;
+    auto alloc_type = use_lockable_memory ? lockable_mem_type
+                    : usm_device_allocatable ? allocation_type::usm_device : lockable_mem_type;
 
     if ((is_internal && (_node.can_be_optimized() || _node.is_type<generic_layer>())) || (memory_reuse_by_user == false)) {
         GPU_DEBUG_LOG << "[" << _node.id() << ": output]" << std::endl;
+        // Use usm_device memory for weights reordering
+        if (is_internal && _node.is_type<generic_layer>() && _engine.supports_allocation(allocation_type::usm_device))
+            alloc_type = allocation_type::usm_device;
         return get_memory_from_pool(_engine,
                 layout,
                 _node.id(),

src/plugins/intel_gpu/src/graph/program.cpp

@@ -174,18 +174,23 @@ void program::init_primitives() {
 }
 
 void program::compile() {
+    GPU_DEBUG_DEFINE_MEM_LOGGER("compile");
     _kernels_cache->build_all();
 }
 
 void program::init_kernels() {
+    GPU_DEBUG_DEFINE_MEM_LOGGER("init_kernels");
     for (auto& n : get_processing_order()) {
-        if (n->get_selected_impl())
+        if (n->get_selected_impl()) {
             n->get_selected_impl()->init_kernels(*_kernels_cache);
+            n->get_selected_impl()->reset_kernels_source();
+        }
     }
 }
 
 void program::load_tuning_cache() {
     OV_ITT_SCOPED_TASK(itt::domains::CLDNN, "ProgramImpl::LoadTuningCache");
+    GPU_DEBUG_DEFINE_MEM_LOGGER("ProgramImpl::LoadTuningCache");
     try {
         tuning_cache = kernel_selector::CreateTuningCacheFromFile(get_engine().configuration().tuning_cache_path);
     } catch (...) {
@@ -239,6 +244,7 @@ program_node const& program::get_node(primitive_id const& id) const {
 
 // TODO: Remove once we will get full support for input/output padding in all primitive implementations.
 bool program::analyze_output_size_handling_need() {
+    GPU_DEBUG_DEFINE_MEM_LOGGER("analyze_output_size_handling_need");
     bool handling_needed = false;
 
     // Calculate output size and compare with specified.
@@ -700,6 +706,7 @@ void program::mark_if_data_flow(program_node& node) {
 }
 
 void program::transfer_memory_to_device() {
+    GPU_DEBUG_DEFINE_MEM_LOGGER("transfer_memory_to_device");
     OV_ITT_SCOPED_TASK(itt::domains::CLDNN, "ProgramImpl::TransferMemory");
     if (!get_engine().supports_allocation(allocation_type::usm_device))
         return;
@@ -734,6 +741,7 @@ void program::transfer_memory_to_device() {
 }
 
 void program::cleanup() {
+    GPU_DEBUG_DEFINE_MEM_LOGGER("cleanup");
     for (auto& node : processing_order)
         node->get_output_layout();
 

src/plugins/intel_gpu/src/plugin/plugin.cpp

@@ -108,6 +108,7 @@ void Plugin::TransformNetwork(std::shared_ptr<ov::Model>& model, const Config& c
 InferenceEngine::CNNNetwork Plugin::CloneAndTransformNetwork(const InferenceEngine::CNNNetwork& network,
                                                              const Config& config) const {
     OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "Plugin::CloneAndTransformNetwork");
+    GPU_DEBUG_DEFINE_MEM_LOGGER("Plugin::CloneAndTransformNetwork");
     CNNNetwork clonedNetwork = InferenceEngine::details::cloneNetwork(network);
 
     auto nGraphFunc = clonedNetwork.getFunction();

src/plugins/intel_gpu/src/plugin/program.cpp

@@ -2,6 +2,10 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+#if defined(__unix__) && !defined(__ANDROID__)
+#include <malloc.h>
+#endif
+
 #include "intel_gpu/plugin/program.hpp"
 #include "ngraph/ops.hpp"
 #include "ov_ops/nms_ie_internal.hpp"
@@ -305,6 +309,14 @@ void Program::CleanupBuild() {
     m_topology.reset();
     m_networkInputs.clear();
     m_networkOutputs.clear();
+    #if defined(__unix__) && !defined(__ANDROID__)
+    //  NOTE: In linux, without malloc_trim, an amount of the memory used by compilation is not being returned to system thought they are freed.
+    //  (It is at least 500 MB when we perform parallel compilation)
+    //  It is observed that freeing the memory manually with malloc_trim saves significant amount of the memory.
+    //  Also, this is not happening in Windows.
+    //  So, added malloc_trim for linux build until we figure out a better solution.
+    malloc_trim(0);
+    #endif
 }
 
 std::shared_ptr<cldnn::program> Program::BuildProgram(const std::vector<std::shared_ptr<ngraph::Node>>& ops,
@@ -327,8 +339,11 @@ std::shared_ptr<cldnn::program> Program::BuildProgram(const std::vector<std::sha
         options.set_option(cldnn::build_option::partial_build_program(true));
     }
     PrepareBuild(networkInputs, networkOutputs);
-    for (const auto& op : ops) {
-        CreateSingleLayerPrimitive(*m_topology, op);
+    {
+        GPU_DEBUG_DEFINE_MEM_LOGGER("CreateSingleLayerPrimitives");
+        for (const auto& op : ops) {
+            CreateSingleLayerPrimitive(*m_topology, op);
+        }
     }
     if (createTopologyOnly) {
         return {};