From c4138258459b509055b47dcfcf4a7df9bdbff607 Mon Sep 17 00:00:00 2001 From: Andrew Kwangwoong Park Date: Sat, 10 Jun 2023 08:07:01 +0900 Subject: [PATCH] [GPU] Fuse type conversion only reorders to the prev nodes (#17881) * Fuse convert reorder to prev MVN/Concat node Signed-off-by: Andrew Park * Add dynamic TCs for ov_gpu_unit_test Signed-off-by: Andrew Park * Add descriptions for changes Signed-off-by: Andrew Park * Fix kernel selection failure Signed-off-by: Andrew Park * Add is_type_conversion_only function for reorder_node Signed-off-by: Andrew Park --------- Signed-off-by: Andrew Park --- .../intel_gpu/src/graph/concatenation.cpp | 6 ++ .../remove_redundant_reorders.cpp | 13 ++- .../src/graph/include/reorder_inst.h | 13 +++ .../intel_gpu/src/graph/layout_optimizer.cpp | 7 ++ .../concatenation_kernel_base.cpp | 5 ++ .../concatenation/concatenation_kernel_ref.h | 5 ++ .../concatenation_kernel_simple_ref.h | 5 ++ .../kernels/mvn/mvn_kernel_bfyx_opt.h | 3 +- .../kernels/mvn/mvn_kernel_ref.h | 3 +- .../intel_gpu/src/plugin/ops/result.cpp | 17 ++-- .../remove_redundant_reorders_tests.cpp | 85 ++++++++++++++++++- 11 files changed, 151 insertions(+), 11 deletions(-) diff --git a/src/plugins/intel_gpu/src/graph/concatenation.cpp b/src/plugins/intel_gpu/src/graph/concatenation.cpp index 31d410a4d07..06df7fe4128 100644 --- a/src/plugins/intel_gpu/src/graph/concatenation.cpp +++ b/src/plugins/intel_gpu/src/graph/concatenation.cpp @@ -23,6 +23,9 @@ layout concatenation_inst::calc_output_layout(concatenation_node const& node, ke auto result_sizes = input_layout.get_dims(); auto output_dt = desc->output_data_types[0].value_or(input_layout.data_type); + if (impl_param.has_fused_primitives()) { + output_dt = impl_param.get_fused_output_layout().data_type; + } auto axis_index = desc->axis; @@ -48,6 +51,9 @@ std::vector concatenation_inst::calc_output_layouts(const concatenation_ auto input_layout = impl_param.get_input_layout(); auto output_dt = desc->output_data_types[0].value_or(input_layout.data_type); + if (impl_param.has_fused_primitives()) { + output_dt = impl_param.get_fused_output_layout().data_type; + } auto output_format = input_layout.format; for (size_t i = 0; i < desc->input.size(); ++i) { if (impl_param.get_input_layout(i).format == format::b_fs_yx_fsv16) diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp index 8e343c2cfb0..8a160a4f1dc 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp @@ -17,6 +17,7 @@ #include "concatenation_inst.h" #include "region_yolo_inst.h" #include "fully_connected_inst.h" +#include "mvn_inst.h" #include #include @@ -413,7 +414,7 @@ void remove_redundant_reorders::run(program& p) { continue; bool same_data_type = input.get_output_layout().data_type == output_layout.data_type; - bool allowed_dt_conversion_fuse = (input.is_type() || input.is_type() || + bool allowed_dt_conversion_fuse = (input.is_type() || input.is_type() || input.is_type() || input.is_type() || input.is_type() || input.is_type() || input.is_type()); if (!same_data_type && !allowed_dt_conversion_fuse) continue; @@ -424,6 +425,16 @@ void remove_redundant_reorders::run(program& p) { auto old_output_layout_of_input = input.get_output_layout(); input.set_output_layout(output_layout, false); if (input.type()->does_possible_implementation_exist(input)) { + // Add fused_primitive_desc of reorder to the previous node which propagates original output layout during shape inference + if (input.is_type() || input.is_type()) { + fused_primitive_desc local_desc(node.get_primitive()); + local_desc.f_param = node.get_fuse_params(); + local_desc.total_num_deps = node.get_dependencies().size(); + local_desc.input_layout = old_output_layout_of_input; + local_desc.output_layout = output_layout; + input.add_fused_primitive(local_desc); + } + node.can_be_optimized(true); p.add_optimized_primitive_info(node.id()); diff --git a/src/plugins/intel_gpu/src/graph/include/reorder_inst.h b/src/plugins/intel_gpu/src/graph/include/reorder_inst.h index 53816428227..d7b8e3f3be5 100644 --- a/src/plugins/intel_gpu/src/graph/include/reorder_inst.h +++ b/src/plugins/intel_gpu/src/graph/include/reorder_inst.h @@ -41,6 +41,19 @@ public: void set_input_layout(layout const& lo) { input_layout = lo; } + bool is_type_conversion_only() const { + if (this->has_fused_primitives() || has_mean() || !typed_desc()->subtract_per_feature.empty()) + return false; + auto in_layout = input().get_output_layout(); + auto out_layout = this->get_output_layout(); + auto check_common_layout = in_layout.data_type != out_layout.data_type && + in_layout.format == out_layout.format && + in_layout.data_padding == out_layout.data_padding; + return typed_desc()->truncate && + ((this->is_dynamic() && check_common_layout && in_layout.get_partial_shape().rank() == out_layout.get_partial_shape().rank()) || + (!this->is_dynamic() && check_common_layout && in_layout.get_partial_shape() == out_layout.get_partial_shape())); + } + std::shared_ptr get_fuse_params() const override { return std::make_shared(input_layout, get_output_layout()); } diff --git a/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp b/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp index 173345b24b3..2669d3c918f 100644 --- a/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp +++ b/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp @@ -414,6 +414,13 @@ bool layout_optimizer::can_fuse_reorder(program_node& prev, program_node& next, } bool layout_optimizer::can_fuse_reorder_to_prev(program_node& prev, reorder_node& node, format fmt_prev, format fmt_next) { + // Because mvn and concatenation kernel can work cross-layout, if reorder only performs type conversion, + // fusing reorder to the previous node can be done even if it is a dynamic shape case + if ((prev.is_type() || prev.is_type()) && + (format::is_simple_data_format(fmt_prev) && format::is_simple_data_format(fmt_next)) && + node.is_type_conversion_only()) + return true; + if (prev.is_dynamic() || (!node.get_users().empty() && node.get_users().front()->is_dynamic())) return false; diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/concatenation/concatenation_kernel_base.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/concatenation/concatenation_kernel_base.cpp index ed9e60908c3..9bff04ad9ed 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/concatenation/concatenation_kernel_base.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/concatenation/concatenation_kernel_base.cpp @@ -39,6 +39,11 @@ bool ConcatenationKernelBase::Validate(const Params& p, const optional_params&) const concatenation_params& params = static_cast(p); + for (auto& fused_op : params.fused_ops) { + if (!IsFusedPrimitiveSupported(fused_op)) + return false; + } + if (GetConcatChannelIndex(params) == -1) { return false; } diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/concatenation/concatenation_kernel_ref.h b/src/plugins/intel_gpu/src/kernel_selector/kernels/concatenation/concatenation_kernel_ref.h index 4720d6363de..14ac815a5fc 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/concatenation/concatenation_kernel_ref.h +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/concatenation/concatenation_kernel_ref.h @@ -19,5 +19,10 @@ public: protected: JitConstants GetJitConstants(const concatenation_params& params) const override; + std::vector GetSupportedFusedOps() const override { + return { + FusedOpType::REORDER + }; + } }; } // namespace kernel_selector diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/concatenation/concatenation_kernel_simple_ref.h b/src/plugins/intel_gpu/src/kernel_selector/kernels/concatenation/concatenation_kernel_simple_ref.h index 73b869aeaa7..33e626660a5 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/concatenation/concatenation_kernel_simple_ref.h +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/concatenation/concatenation_kernel_simple_ref.h @@ -20,5 +20,10 @@ public: protected: ParamsKey GetSupportedKey() const override; + std::vector GetSupportedFusedOps() const override { + return { + FusedOpType::REORDER + }; + } }; } // namespace kernel_selector diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/mvn/mvn_kernel_bfyx_opt.h b/src/plugins/intel_gpu/src/kernel_selector/kernels/mvn/mvn_kernel_bfyx_opt.h index 3dbae71c0e4..d40fd562a61 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/mvn/mvn_kernel_bfyx_opt.h +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/mvn/mvn_kernel_bfyx_opt.h @@ -24,7 +24,8 @@ private: return { FusedOpType::ACTIVATION, FusedOpType::QUANTIZE, - FusedOpType::ELTWISE + FusedOpType::ELTWISE, + FusedOpType::REORDER }; } DispatchData SetDefault(const mvn_params& params) const override; diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/mvn/mvn_kernel_ref.h b/src/plugins/intel_gpu/src/kernel_selector/kernels/mvn/mvn_kernel_ref.h index 03101f428f3..3f84658b5ab 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/mvn/mvn_kernel_ref.h +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/mvn/mvn_kernel_ref.h @@ -25,7 +25,8 @@ protected: return { FusedOpType::ACTIVATION, FusedOpType::QUANTIZE, - FusedOpType::ELTWISE + FusedOpType::ELTWISE, + FusedOpType::REORDER }; } std::string GetKernelName(const mvn_params&) const override; diff --git a/src/plugins/intel_gpu/src/plugin/ops/result.cpp b/src/plugins/intel_gpu/src/plugin/ops/result.cpp index d533d1fa411..4fa67f0cf64 100644 --- a/src/plugins/intel_gpu/src/plugin/ops/result.cpp +++ b/src/plugins/intel_gpu/src/plugin/ops/result.cpp @@ -68,24 +68,27 @@ static void CreateResultOp(Program& p, const std::shared_ptrget_input_element_type(0)); Precision precision = outputData->getPrecision(); + auto outputDataType = DataTypeFromPrecision(precision); cldnn::input_info outputID = inputs[0]; - if (p.use_new_shape_infer() - // Note:: Currently Split/Variadic Split are divided to multiple crops - && !ngraph::is_type(prev) - && !ngraph::is_type(prev)) { + // Even for result op, if reorder only performs type conversion, reorder is created in truncation mode + if (inputDataType != outputDataType) { auto reorder_primitive = cldnn::reorder(outLayerName, outputID, out_format, - DataTypeFromPrecision(precision)); + outputDataType, + std::vector(), + cldnn::reorder_mean_mode::subtract, + cldnn::padding(), + true); p.add_primitive(*op, reorder_primitive, {originalOutName}); - } else { auto reorder_primitive = cldnn::reorder(outLayerName, outputID, out_format, - DataTypeFromPrecision(precision)); + outputDataType); p.add_primitive(*op, reorder_primitive, {originalOutName}); } p.outputDims[originalOutName] = outputDesc.getDims(); diff --git a/src/plugins/intel_gpu/tests/unit/passes/remove_redundant_reorders_tests.cpp b/src/plugins/intel_gpu/tests/unit/passes/remove_redundant_reorders_tests.cpp index d4aeb33962e..95a0aa05f72 100644 --- a/src/plugins/intel_gpu/tests/unit/passes/remove_redundant_reorders_tests.cpp +++ b/src/plugins/intel_gpu/tests/unit/passes/remove_redundant_reorders_tests.cpp @@ -14,10 +14,11 @@ #include "softmax_inst.h" #include "reduce_inst.h" #include "fully_connected_inst.h" -#include "convolution_inst.h" #include "permute_inst.h" #include "reshape_inst.h" #include "activation_inst.h" +#include "mvn_inst.h" +#include "concatenation_inst.h" #include "pass_manager.h" #include "to_string_utils.h" @@ -240,3 +241,85 @@ TEST(remove_redundant_reorders, remove_fused) { network network(engine, topology, config); ASSERT_TRUE(has_node(*prog, "reorder2")); } + +TEST(remove_redundant_reorders, fuse_reorder_to_prev_mvn_dyn) { + auto& engine = get_test_engine(); + auto weights = engine.allocate_memory({ ov::PartialShape{ 1024, 256 }, data_types::f16, format::bfyx }); + auto in_layout = layout{ov::PartialShape{ ov::Dimension::dynamic(), ov::Dimension::dynamic(), 256 }, data_types::f32, format::bfyx}; + auto input = engine.allocate_memory({ ov::PartialShape{ 1, 33, 256 }, data_types::f32, format::bfyx }); + + topology topology; + topology.add(data("weights", weights)); + topology.add(input_layout("input", in_layout)); + topology.add(mvn("mvn", input_info("input"), true, 1e-10f, true, { 2 })); + topology.add(reorder("reorder", input_info("mvn"), format::any, data_types::f16, + std::vector(), reorder_mean_mode::subtract, padding(), true)); + topology.add(fully_connected("fc", input_info("reorder"), { "weights" }, "", data_types::f16, padding(), 3, 2)); + + ExecutionConfig config = get_test_default_config(engine); + config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + config.set_property(ov::intel_gpu::optimize_data(true)); + auto prog = program::build_program(engine, topology, config, false, true); + + layout_optimizer lo(true); + bool optimize_data = config.get_property(ov::intel_gpu::optimize_data); + program_wrapper::apply_opt_pass(*prog, lo, optimize_data); + + ASSERT_NE(prog, nullptr); + ASSERT_FALSE(has_node_with_type(*prog)); + + network network(engine, topology, config); + network.set_input_data("input", input); + + EXPECT_NO_THROW(network.execute()); + + auto& mvn_node = prog->get_node("mvn"); + auto mvn_layout = mvn_node.get_output_layout(); + + ASSERT_EQ(mvn_layout.data_type, data_types::f16); +} + +TEST(remove_redundant_reorders, fuse_reorder_to_prev_concat_dyn) { + auto& engine = get_test_engine(); + auto in_layout1 = layout{ov::PartialShape{ ov::Dimension::dynamic(), 32, ov::Dimension::dynamic(), 80 }, data_types::f16, format::bfyx}; + auto in_layout2 = layout{ov::PartialShape{ ov::Dimension::dynamic(), 32, ov::Dimension::dynamic(), 80 }, data_types::f16, format::bfyx}; + auto input1 = engine.allocate_memory({ ov::PartialShape{ 2, 32, 30, 80 }, data_types::f16, format::bfyx }); + auto input2 = engine.allocate_memory({ ov::PartialShape{ 2, 32, 30, 80 }, data_types::f16, format::bfyx }); + + topology topology; + topology.add(input_layout("input1", in_layout1)); + topology.add(input_layout("input2", in_layout2)); + topology.add(reshape("reshape1", input_info("input1"), false, {0}, + ov::PartialShape{ 1, ov::Dimension::dynamic(), 32, ov::Dimension::dynamic(), 80 }, + reshape::reshape_mode::unsqueeze)); + topology.add(reshape("reshape2", input_info("input2"), false, {0}, + ov::PartialShape{ 1, ov::Dimension::dynamic(), 32, ov::Dimension::dynamic(), 80 }, + reshape::reshape_mode::unsqueeze)); + topology.add(concatenation("concat", { input_info("reshape1"), input_info("reshape2") }, 0, data_types::f16)); + topology.add(reorder("reorder", input_info("concat"), format::any, data_types::f32, + std::vector(), reorder_mean_mode::subtract, padding(), true)); + topology.add(softmax("softmax", input_info("reorder"), 1)); + + ExecutionConfig config = get_test_default_config(engine); + config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + config.set_property(ov::intel_gpu::optimize_data(true)); + auto prog = program::build_program(engine, topology, config, false, true); + + layout_optimizer lo(true); + bool optimize_data = config.get_property(ov::intel_gpu::optimize_data); + program_wrapper::apply_opt_pass(*prog, lo, optimize_data); + + ASSERT_NE(prog, nullptr); + ASSERT_FALSE(has_node_with_type(*prog)); + + network network(engine, topology, config); + network.set_input_data("input1", input1); + network.set_input_data("input2", input2); + + EXPECT_NO_THROW(network.execute()); + + auto& concat_node = prog->get_node("concat"); + auto concat_layout = concat_node.get_output_layout(); + + ASSERT_EQ(concat_layout.data_type, data_types::f32); +}