[GPU] Fuse type conversion only reorders to the prev nodes (#17881)

* Fuse convert reorder to prev MVN/Concat node

Signed-off-by: Andrew Park <andrew.park@intel.com>

* Add dynamic TCs for ov_gpu_unit_test

Signed-off-by: Andrew Park <andrew.park@intel.com>

* Add descriptions for changes

Signed-off-by: Andrew Park <andrew.park@intel.com>

* Fix kernel selection failure

Signed-off-by: Andrew Park <andrew.park@intel.com>

* Add is_type_conversion_only function for reorder_node

Signed-off-by: Andrew Park <andrew.park@intel.com>

---------

Signed-off-by: Andrew Park <andrew.park@intel.com>

src/plugins/intel_gpu/src/graph/concatenation.cpp

@@ -23,6 +23,9 @@ layout concatenation_inst::calc_output_layout(concatenation_node const& node, ke
     auto result_sizes = input_layout.get_dims();
 
     auto output_dt = desc->output_data_types[0].value_or(input_layout.data_type);
+    if (impl_param.has_fused_primitives()) {
+        output_dt = impl_param.get_fused_output_layout().data_type;
+    }
 
     auto axis_index = desc->axis;
 
@@ -48,6 +51,9 @@ std::vector<layout> concatenation_inst::calc_output_layouts(const concatenation_
     auto input_layout = impl_param.get_input_layout();
 
     auto output_dt = desc->output_data_types[0].value_or(input_layout.data_type);
+    if (impl_param.has_fused_primitives()) {
+        output_dt = impl_param.get_fused_output_layout().data_type;
+    }
     auto output_format = input_layout.format;
     for (size_t i = 0; i < desc->input.size(); ++i) {
         if (impl_param.get_input_layout(i).format == format::b_fs_yx_fsv16)

src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp

@@ -17,6 +17,7 @@
 #include "concatenation_inst.h"
 #include "region_yolo_inst.h"
 #include "fully_connected_inst.h"
+#include "mvn_inst.h"
 
 #include <vector>
 #include <list>
@@ -413,7 +414,7 @@ void remove_redundant_reorders::run(program& p) {
                 continue;
 
             bool same_data_type = input.get_output_layout().data_type == output_layout.data_type;
-            bool allowed_dt_conversion_fuse = (input.is_type<one_hot>() || input.is_type<permute>() ||
+            bool allowed_dt_conversion_fuse = (input.is_type<one_hot>() || input.is_type<permute>() || input.is_type<mvn>() || input.is_type<concatenation>() ||
                                                input.is_type<depth_to_space>() || input.is_type<region_yolo>() || input.is_type<detection_output>());
             if (!same_data_type && !allowed_dt_conversion_fuse)
                 continue;
@@ -424,6 +425,16 @@ void remove_redundant_reorders::run(program& p) {
             auto old_output_layout_of_input = input.get_output_layout();
             input.set_output_layout(output_layout, false);
             if (input.type()->does_possible_implementation_exist(input)) {
+                // Add fused_primitive_desc of reorder to the previous node which propagates original output layout during shape inference
+                if (input.is_type<mvn>() || input.is_type<concatenation>()) {
+                    fused_primitive_desc local_desc(node.get_primitive());
+                    local_desc.f_param = node.get_fuse_params();
+                    local_desc.total_num_deps = node.get_dependencies().size();
+                    local_desc.input_layout = old_output_layout_of_input;
+                    local_desc.output_layout = output_layout;
+                    input.add_fused_primitive(local_desc);
+                }
+
                 node.can_be_optimized(true);
                 p.add_optimized_primitive_info(node.id());
 

src/plugins/intel_gpu/src/graph/include/reorder_inst.h

@@ -41,6 +41,19 @@ public:
 
     void set_input_layout(layout const& lo) { input_layout = lo; }
 
+    bool is_type_conversion_only() const {
+        if (this->has_fused_primitives() || has_mean() || !typed_desc()->subtract_per_feature.empty())
+            return false;
+        auto in_layout = input().get_output_layout();
+        auto out_layout = this->get_output_layout();
+        auto check_common_layout = in_layout.data_type != out_layout.data_type &&
+                                   in_layout.format == out_layout.format &&
+                                   in_layout.data_padding == out_layout.data_padding;
+        return typed_desc()->truncate &&
+               ((this->is_dynamic() && check_common_layout && in_layout.get_partial_shape().rank() == out_layout.get_partial_shape().rank()) ||
+                (!this->is_dynamic() && check_common_layout && in_layout.get_partial_shape() == out_layout.get_partial_shape()));
+    }
+
     std::shared_ptr<NodeFuseParams> get_fuse_params() const override {
         return std::make_shared<ReorderFuseParams>(input_layout, get_output_layout());
     }

src/plugins/intel_gpu/src/graph/layout_optimizer.cpp

@@ -414,6 +414,13 @@ bool layout_optimizer::can_fuse_reorder(program_node& prev, program_node& next,
 }
 
 bool layout_optimizer::can_fuse_reorder_to_prev(program_node& prev, reorder_node& node, format fmt_prev, format fmt_next) {
+    // Because mvn and concatenation kernel can work cross-layout, if reorder only performs type conversion,
+    // fusing reorder to the previous node can be done even if it is a dynamic shape case
+    if ((prev.is_type<mvn>() || prev.is_type<concatenation>()) &&
+        (format::is_simple_data_format(fmt_prev) && format::is_simple_data_format(fmt_next)) &&
+        node.is_type_conversion_only())
+        return true;
+
     if (prev.is_dynamic() || (!node.get_users().empty() && node.get_users().front()->is_dynamic()))
         return false;
 

src/plugins/intel_gpu/src/kernel_selector/kernels/concatenation/concatenation_kernel_base.cpp

@@ -39,6 +39,11 @@ bool ConcatenationKernelBase::Validate(const Params& p, const optional_params&)
 
     const concatenation_params& params = static_cast<const concatenation_params&>(p);
 
+    for (auto& fused_op : params.fused_ops) {
+        if (!IsFusedPrimitiveSupported(fused_op))
+            return false;
+    }
+
     if (GetConcatChannelIndex(params) == -1) {
         return false;
     }

src/plugins/intel_gpu/src/kernel_selector/kernels/concatenation/concatenation_kernel_ref.h

@@ -19,5 +19,10 @@ public:
 
 protected:
     JitConstants GetJitConstants(const concatenation_params& params) const override;
+    std::vector<FusedOpType> GetSupportedFusedOps() const override {
+        return {
+            FusedOpType::REORDER
+        };
+    }
 };
 }  // namespace kernel_selector

src/plugins/intel_gpu/src/kernel_selector/kernels/concatenation/concatenation_kernel_simple_ref.h

@@ -20,5 +20,10 @@ public:
 
 protected:
     ParamsKey GetSupportedKey() const override;
+    std::vector<FusedOpType> GetSupportedFusedOps() const override {
+        return {
+            FusedOpType::REORDER
+        };
+    }
 };
 }  // namespace kernel_selector

src/plugins/intel_gpu/src/kernel_selector/kernels/mvn/mvn_kernel_bfyx_opt.h

@@ -24,7 +24,8 @@ private:
         return {
             FusedOpType::ACTIVATION,
             FusedOpType::QUANTIZE,
-            FusedOpType::ELTWISE
+            FusedOpType::ELTWISE,
+            FusedOpType::REORDER
         };
     }
     DispatchData SetDefault(const mvn_params& params) const override;

src/plugins/intel_gpu/src/kernel_selector/kernels/mvn/mvn_kernel_ref.h

@@ -25,7 +25,8 @@ protected:
         return {
             FusedOpType::ACTIVATION,
             FusedOpType::QUANTIZE,
-            FusedOpType::ELTWISE
+            FusedOpType::ELTWISE,
+            FusedOpType::REORDER
         };
     }
     std::string GetKernelName(const mvn_params&) const override;

src/plugins/intel_gpu/src/plugin/ops/result.cpp

@@ -68,24 +68,27 @@ static void CreateResultOp(Program& p, const std::shared_ptr<ngraph::op::v0::Res
     }
 
     auto outLayerName = layer_type_name_ID(op);
+    auto inputDataType = cldnn::element_type_to_data_type(op->get_input_element_type(0));
     Precision precision = outputData->getPrecision();
+    auto outputDataType = DataTypeFromPrecision(precision);
     cldnn::input_info outputID = inputs[0];
 
-    if (p.use_new_shape_infer()
-        // Note:: Currently Split/Variadic Split are divided to multiple crops
-        && !ngraph::is_type<ngraph::op::v1::Split>(prev)
-        && !ngraph::is_type<ngraph::op::v1::VariadicSplit>(prev)) {
+    // Even for result op, if reorder only performs type conversion, reorder is created in truncation mode
+    if (inputDataType != outputDataType) {
         auto reorder_primitive = cldnn::reorder(outLayerName,
                                                 outputID,
                                                 out_format,
-                                                DataTypeFromPrecision(precision));
+                                                outputDataType,
+                                                std::vector<float>(),
+                                                cldnn::reorder_mean_mode::subtract,
+                                                cldnn::padding(),
+                                                true);
         p.add_primitive(*op, reorder_primitive, {originalOutName});
-
     } else {
         auto reorder_primitive = cldnn::reorder(outLayerName,
                                                 outputID,
                                                 out_format,
-                                                DataTypeFromPrecision(precision));
+                                                outputDataType);
         p.add_primitive(*op, reorder_primitive, {originalOutName});
     }
     p.outputDims[originalOutName] = outputDesc.getDims();

src/plugins/intel_gpu/tests/unit/passes/remove_redundant_reorders_tests.cpp

@@ -14,10 +14,11 @@
 #include "softmax_inst.h"
 #include "reduce_inst.h"
 #include "fully_connected_inst.h"
-#include "convolution_inst.h"
 #include "permute_inst.h"
 #include "reshape_inst.h"
 #include "activation_inst.h"
+#include "mvn_inst.h"
+#include "concatenation_inst.h"
 #include "pass_manager.h"
 #include "to_string_utils.h"
 
@@ -240,3 +241,85 @@ TEST(remove_redundant_reorders, remove_fused) {
     network network(engine, topology, config);
     ASSERT_TRUE(has_node(*prog, "reorder2"));
 }
+
+TEST(remove_redundant_reorders, fuse_reorder_to_prev_mvn_dyn) {
+    auto& engine = get_test_engine();
+    auto weights = engine.allocate_memory({ ov::PartialShape{ 1024, 256 }, data_types::f16, format::bfyx });
+    auto in_layout = layout{ov::PartialShape{ ov::Dimension::dynamic(), ov::Dimension::dynamic(), 256 }, data_types::f32, format::bfyx};
+    auto input = engine.allocate_memory({ ov::PartialShape{ 1, 33, 256 }, data_types::f32, format::bfyx });
+
+    topology topology;
+    topology.add(data("weights", weights));
+    topology.add(input_layout("input", in_layout));
+    topology.add(mvn("mvn", input_info("input"), true, 1e-10f, true, { 2 }));
+    topology.add(reorder("reorder", input_info("mvn"), format::any, data_types::f16,
+                         std::vector<float>(), reorder_mean_mode::subtract, padding(), true));
+    topology.add(fully_connected("fc", input_info("reorder"), { "weights" }, "", data_types::f16, padding(), 3, 2));
+
+    ExecutionConfig config = get_test_default_config(engine);
+    config.set_property(ov::intel_gpu::allow_new_shape_infer(true));
+    config.set_property(ov::intel_gpu::optimize_data(true));
+    auto prog = program::build_program(engine, topology, config, false, true);
+
+    layout_optimizer lo(true);
+    bool optimize_data = config.get_property(ov::intel_gpu::optimize_data);
+    program_wrapper::apply_opt_pass<remove_redundant_reorders>(*prog, lo, optimize_data);
+
+    ASSERT_NE(prog, nullptr);
+    ASSERT_FALSE(has_node_with_type<reorder>(*prog));
+
+    network network(engine, topology, config);
+    network.set_input_data("input", input);
+
+    EXPECT_NO_THROW(network.execute());
+
+    auto& mvn_node = prog->get_node("mvn");
+    auto mvn_layout = mvn_node.get_output_layout();
+
+    ASSERT_EQ(mvn_layout.data_type, data_types::f16);
+}
+
+TEST(remove_redundant_reorders, fuse_reorder_to_prev_concat_dyn) {
+    auto& engine = get_test_engine();
+    auto in_layout1 = layout{ov::PartialShape{ ov::Dimension::dynamic(), 32, ov::Dimension::dynamic(), 80 }, data_types::f16, format::bfyx};
+    auto in_layout2 = layout{ov::PartialShape{ ov::Dimension::dynamic(), 32, ov::Dimension::dynamic(), 80 }, data_types::f16, format::bfyx};
+    auto input1 = engine.allocate_memory({ ov::PartialShape{ 2, 32, 30, 80 }, data_types::f16, format::bfyx });
+    auto input2 = engine.allocate_memory({ ov::PartialShape{ 2, 32, 30, 80 }, data_types::f16, format::bfyx });
+
+    topology topology;
+    topology.add(input_layout("input1", in_layout1));
+    topology.add(input_layout("input2", in_layout2));
+    topology.add(reshape("reshape1", input_info("input1"), false, {0},
+                         ov::PartialShape{ 1, ov::Dimension::dynamic(), 32, ov::Dimension::dynamic(), 80 },
+                         reshape::reshape_mode::unsqueeze));
+    topology.add(reshape("reshape2", input_info("input2"), false, {0},
+                         ov::PartialShape{ 1, ov::Dimension::dynamic(), 32, ov::Dimension::dynamic(), 80 },
+                         reshape::reshape_mode::unsqueeze));
+    topology.add(concatenation("concat", { input_info("reshape1"), input_info("reshape2") }, 0, data_types::f16));
+    topology.add(reorder("reorder", input_info("concat"), format::any, data_types::f32,
+                         std::vector<float>(), reorder_mean_mode::subtract, padding(), true));
+    topology.add(softmax("softmax", input_info("reorder"), 1));
+
+    ExecutionConfig config = get_test_default_config(engine);
+    config.set_property(ov::intel_gpu::allow_new_shape_infer(true));
+    config.set_property(ov::intel_gpu::optimize_data(true));
+    auto prog = program::build_program(engine, topology, config, false, true);
+
+    layout_optimizer lo(true);
+    bool optimize_data = config.get_property(ov::intel_gpu::optimize_data);
+    program_wrapper::apply_opt_pass<remove_redundant_reorders>(*prog, lo, optimize_data);
+
+    ASSERT_NE(prog, nullptr);
+    ASSERT_FALSE(has_node_with_type<reorder>(*prog));
+
+    network network(engine, topology, config);
+    network.set_input_data("input1", input1);
+    network.set_input_data("input2", input2);
+
+    EXPECT_NO_THROW(network.execute());
+
+    auto& concat_node = prog->get_node("concat");
+    auto concat_layout = concat_node.get_output_layout();
+
+    ASSERT_EQ(concat_layout.data_type, data_types::f32);
+}