From c360de0f07587ed5559f8d374da334f9483d1e48 Mon Sep 17 00:00:00 2001 From: Sergey Shlyapnikov Date: Wed, 15 Nov 2023 08:47:36 +0400 Subject: [PATCH] [GPU] Optimize Condition operation by better integrating its subnetwork primitives with the main network (#21062) * [GPU] Optimize Condition operation by better integrating its subnetwork primitives with the main network * Reuse main network's stream in sub-networks --- src/plugins/intel_gpu/src/graph/condition.cpp | 6 ++--- .../src/graph/impls/common/condition.cpp | 22 ++++++++++++------- .../intel_gpu/src/runtime/ocl/ocl_stream.cpp | 4 +++- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/condition.cpp b/src/plugins/intel_gpu/src/graph/condition.cpp index c564623fcd0..5b7470e8887 100644 --- a/src/plugins/intel_gpu/src/graph/condition.cpp +++ b/src/plugins/intel_gpu/src/graph/condition.cpp @@ -219,8 +219,8 @@ Condition primitive is reusing memory with the input. */ condition_inst::typed_primitive_inst(network& network, condition_node const& node) : parent(network, node), - _net_true(network::allocate_network(node.get_program().get_engine(), node.get_branch_true().inner_program)), - _net_false(network::allocate_network(node.get_program().get_engine(), node.get_branch_false().inner_program)) { + _net_true(network::allocate_network(network.get_stream_ptr(), node.get_branch_true().inner_program)), + _net_false(network::allocate_network(network.get_stream_ptr(), node.get_branch_false().inner_program)) { this->set_inner_networks({_net_true, _net_false}); } @@ -257,7 +257,7 @@ void condition_inst::postprocess_output_memory(network::ptr executed_net, cldnn: for (auto out_mem_map : branch.output_map) { auto out_mem_idx = out_mem_map.first; auto inner_out_id = out_mem_map.second; - auto mem_ptr = executed_net->get_output(inner_out_id).get_memory(); + auto mem_ptr = executed_net->get_output_memory(inner_out_id); if (mem_ptr) { auto layout = _impl_params->get_output_layout(out_mem_idx); GPU_DEBUG_LOG << "Reshape output from " << mem_ptr->get_layout().to_short_string() diff --git a/src/plugins/intel_gpu/src/graph/impls/common/condition.cpp b/src/plugins/intel_gpu/src/graph/impls/common/condition.cpp index 84e2811c420..d72c8d70832 100644 --- a/src/plugins/intel_gpu/src/graph/impls/common/condition.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/common/condition.cpp @@ -30,13 +30,15 @@ struct condition_impl : typed_primitive_impl { } event::ptr execute_impl(const std::vector& events, condition_inst& instance) override { - for (auto& a : events) { - a->wait(); - } - auto ev = instance.get_network().get_stream().create_user_event(false); + // Wait for condition statement event only, and pass all other events to sub-network directly + if (!events.empty()) + events[0]->wait(); + + auto& stream = instance.get_network().get_stream(); + auto ev = stream.create_user_event(false); set_node_params(instance.get_node()); - auto pred = condition_inst::get_pred_from_memory(instance.pred_memory_ptr(), instance.get_network().get_stream()); + auto pred = condition_inst::get_pred_from_memory(instance.pred_memory_ptr(), stream); network::ptr executed_net = pred ? instance.get_net_true() : instance.get_net_false(); auto branch = pred ? instance.get_branch_true() : instance.get_branch_false(); executed_net->set_shape_predictor(instance.get_network().get_shape_predictor()); @@ -62,7 +64,7 @@ struct condition_impl : typed_primitive_impl { } } - executed_net->execute({}); + auto sub_net_results = executed_net->execute(events); // Update output layout of impl_param in condition_inst instance.update_output_layout(); @@ -70,8 +72,12 @@ struct condition_impl : typed_primitive_impl { // Set output memory of condition_inst to inner network output memory after inner network execution instance.postprocess_output_memory(executed_net, branch); - ev->set(); - return ev; + std::vector output_events; + for (auto& output : sub_net_results) + if (output.second.get_event() != nullptr) + output_events.push_back(output.second.get_event()); + + return stream.group_events(output_events); } static std::unique_ptr create(const condition_node& arg, const kernel_impl_params&) { diff --git a/src/plugins/intel_gpu/src/runtime/ocl/ocl_stream.cpp b/src/plugins/intel_gpu/src/runtime/ocl/ocl_stream.cpp index 72af490c075..71942563ba8 100644 --- a/src/plugins/intel_gpu/src/runtime/ocl/ocl_stream.cpp +++ b/src/plugins/intel_gpu/src/runtime/ocl/ocl_stream.cpp @@ -61,7 +61,9 @@ cl_int set_kernel_arg(ocl_kernel_type& kernel, uint32_t idx, cldnn::memory::cptr return kernel.setArg(idx, buf); } else if (memory_capabilities::is_usm_type(mem->get_allocation_type())) { auto buf = std::dynamic_pointer_cast(mem)->get_buffer(); - GPU_DEBUG_TRACE_DETAIL << "kernel: " << kernel.get() << " set arg (usm) " << idx << " mem: " << buf.get() << " size: " << mem->size() << std::endl; + auto mem_type = std::dynamic_pointer_cast(mem)->get_allocation_type(); + GPU_DEBUG_TRACE_DETAIL << "kernel: " << kernel.get() << " set arg (" << mem_type << ") " << idx + << " mem: " << buf.get() << " size: " << mem->size() << std::endl; return kernel.setArgUsm(idx, buf); } else { auto buf = std::dynamic_pointer_cast(mem)->get_buffer();