From bc15596c9e4b660353dc23969111a697e0400b4a Mon Sep 17 00:00:00 2001 From: "Min, Byungil" Date: Mon, 20 Mar 2023 14:58:09 +0900 Subject: [PATCH] Remove redundant reorder (#15661) + Reorder 1d data + Reorder which only changes format Signed-off-by: Min, Byungil --- .../include/intel_gpu/runtime/format.hpp | 4 + .../remove_redundant_reorders.cpp | 1 - src/plugins/intel_gpu/src/runtime/layout.cpp | 43 ++++-- .../tests/test_cases/reorder_gpu_test.cpp | 138 +++++++++++++++++- 4 files changed, 174 insertions(+), 12 deletions(-) diff --git a/src/plugins/intel_gpu/include/intel_gpu/runtime/format.hpp b/src/plugins/intel_gpu/include/intel_gpu/runtime/format.hpp index 3b0366d37d7..f7443424e6c 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/runtime/format.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/runtime/format.hpp @@ -341,6 +341,10 @@ struct format { static bool is_image(type fmt) { return (is_image_2d(fmt)); } /// @brief Checks if @p format is blocked format static bool is_blocked(type fmt) { return !(block_sizes(fmt).empty()); } + /// @brief Checks if @p format is blocked format which has single inner block + static bool is_single_blocked(type fmt) { return (block_sizes(fmt).size() == 1); } + /// @brief Checks if @p format is blocked format which has multiple inner blocks + static bool is_multi_blocked(type fmt) { return (block_sizes(fmt).size() > 1); } /// @brief Checks if @p format is nv12 format static bool is_nv12(type fmt) { return (fmt == nv12); } diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp index 8bde129a836..9eeca090fb7 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp @@ -23,7 +23,6 @@ using namespace cldnn; #define LOG_NODE_REMOVAL(id) GPU_DEBUG_LOG_PASS << "Remove node: " << (id) << std::endl; #define LOG_NODE_REPLACEMENT(id) GPU_DEBUG_LOG_PASS << "Replace node: " << (id) << std::endl; - remove_redundant_reorders::remove_redundant_reorders(layout_optimizer& lo_ref, bool enable_reorder_fusing, bool update_implementations, bool remove_output_reorders) : base_pass("remove_redundant_reorders"), lo(lo_ref), enable_reorder_fusing(enable_reorder_fusing), update_implementations(update_implementations), diff --git a/src/plugins/intel_gpu/src/runtime/layout.cpp b/src/plugins/intel_gpu/src/runtime/layout.cpp index ee825df8ec5..79e6925f0cd 100644 --- a/src/plugins/intel_gpu/src/runtime/layout.cpp +++ b/src/plugins/intel_gpu/src/runtime/layout.cpp @@ -9,6 +9,7 @@ #include namespace cldnn { +static inline bool check_redundant_1d_along_feature(layout const& l1, layout const& l2); namespace { // pair.first tells whether l1 and l2 are absolutely identical // pair.second tells whether l1 and l2 can be reinterpreted to each other without need of reordering @@ -26,10 +27,10 @@ std::pair are_layouts_identical(layout const& l1, layout const& l2) auto l1_size = l1.get_tensor(); auto l2_size = l2.get_tensor(); - int64_t offset_last_element_l1 = l1.get_linear_offset(l1_size - tensor{1}); - int64_t offset_last_element_l2 = l2.get_linear_offset(l2_size - tensor{1}); if (l1 == l2) return {true, true}; + if (check_redundant_1d_along_feature(l1, l2)) + return {false, true}; if (l1.data_type != l2.data_type) return {false, false}; // Reorders between bfyx, bfzyx, bfwzyx can pe reinterpeted as reshape when @@ -73,13 +74,6 @@ std::pair are_layouts_identical(layout const& l1, layout const& l2) check_format(format::bs_fs_zyx_bsv32_fsv32)) return {false, false}; - // If data is actually 1d along f and dense, the layouts are identical - if (l1.data_type == l2.data_type && l1_size == l2_size && !l1_pad && !l2_pad && l1_size.batch[0] == 1 && - ((l1.format.spatial_num() == 2 && l1_size.spatial[0] == 1 && l1_size.spatial[1] == 1) || - ((l1.format.spatial_num() == 3 && l1_size.spatial[0] == 1 && l1_size.spatial[1] == 1 && l1_size.spatial[2] == 1))) && - (offset_last_element_l1 + 1 == l1_size.feature[0] && offset_last_element_l2 + 1 == l2_size.feature[0])) - return {false, true}; - auto l1_pitch = l1.get_pitches(); auto l2_pitch = l2.get_pitches(); @@ -634,4 +628,35 @@ ov::PartialShape layout::transform(cldnn::format new_fmt) const { return ov::PartialShape(new_shape); } +// Check a reorder is 1d along feature axis. Or feature size fits to inner block size of feature axis +static inline bool check_redundant_1d_along_feature(layout const& l1, layout const& l2) { + // No padding, double blocked format and different data_type + if (!l1.data_padding && !l2.data_padding && !format::is_multi_blocked(l1.format) && !format::is_multi_blocked(l2.format) && + l2.data_type == l1.data_type && l2.count() == l1.count()) { + auto l1_inner_blk = format::is_single_blocked(l1.format) ? format::traits(l1.format).block_sizes.at(0).second : 1; + auto l2_inner_blk = format::is_single_blocked(l2.format) ? format::traits(l2.format).block_sizes.at(0).second : 1; + auto max_inner_blk = std::max(l1_inner_blk, l2_inner_blk); + if (static_cast(l2.feature()) == l1.count() && l2.feature() == l1.feature() && + (l2.feature() % max_inner_blk == 0)) { + return true; + } + + // Acceptable if a feature size of l2 'byxf' fits to l1's inner block size of 'b_fs_yx_fsv' + if ((l2.format == format::byxf && (l1.format == format::b_fs_yx_fsv16 || l1.format == format::b_fs_yx_fsv32) && + l2.feature() == l1_inner_blk) || + (l1.format == format::byxf && (l2.format == format::b_fs_yx_fsv16 || l2.format == format::b_fs_yx_fsv32) && + l1.feature() == l2_inner_blk)) { + // each spatial axis should be same + for (size_t i = 0 ; i < l2.get_spatial_rank() ; i++) { + if (l2.spatial(i) != l1.spatial(i)) + return false; + } + + return true; + } + } + + return false; +} + } // namespace cldnn diff --git a/src/plugins/intel_gpu/tests/test_cases/reorder_gpu_test.cpp b/src/plugins/intel_gpu/tests/test_cases/reorder_gpu_test.cpp index 328d9baae4f..5619d415928 100644 --- a/src/plugins/intel_gpu/tests/test_cases/reorder_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/test_cases/reorder_gpu_test.cpp @@ -2552,8 +2552,6 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_REORDER, tests::generic_test::custom_param_name_functor()); - - struct reorder_test_param { tensor in_shape; tensor out_shape; @@ -2588,6 +2586,18 @@ public: executed_prims = network_test->get_executed_primitive_ids(); } + std::shared_ptr execute_and_query(T& p, primitive_id prim_id) { + auto input_prim = this->get_mem(get_input_layout(p)); + + cldnn::network::ptr network_test = get_network(this->engine, this->topology_test, this->config, get_test_stream_ptr(), false); + + network_test->set_input_data("input", input_prim); + + executed_prims = network_test->get_executed_primitive_ids(); + + return network_test->get_primitive(prim_id); + } + bool check_optimized_out(T& p, primitive_id target_id) { for (auto& prim : executed_prims) if (prim == target_id) @@ -2761,6 +2771,129 @@ INSTANTIATE_TEST_SUITE_P(reorder_gpu_testing, testing_removal_reorder, })); +struct redundant_reorder_test_param { + tensor in_shape; + tensor out_shape; + tensor kernel; + ov::Strides stride; + ov::CoordinateDiff pad; + data_types data_type; + format input_format; + data_types weights_type; + format weights_format; + data_types default_type; + format default_format; + bool opt_out; +}; + +class testing_removal_1d_reorder : public ReorderTest {}; +TEST_P(testing_removal_1d_reorder, removal_reorder_1d_along_f_mixed_format) { + auto p = GetParam(); + + std::vector pad = { 0, 0, static_cast(p.pad[1]), static_cast(p.pad[0]) }; + layout in_layout{ p.data_type, p.input_format, p.in_shape, padding{pad} }; + + create_topologies(input_layout("input", in_layout), + data("weights", get_mem(get_weights_layout(p))), + data("bias1", get_mem(get_bias_layout(p))), + reorder("reorder_bias1", input_info("bias1"), format::b_fs_yx_fsv32, data_types::f16), + convolution("conv_prim", input_info("input"), {"weights"}, std::vector{}, 1, p.stride, p.pad), + reorder("reorder_conv", input_info("conv_prim"), format::b_fs_yx_fsv32, data_types::f16), + eltwise("add_bias1", { input_info("reorder_conv"), input_info("reorder_bias1") }, eltwise_mode::sum), + reorder("reorder_bfyx", input_info("add_bias1"), p.default_format, data_types::f16) + ); + + const auto& target_reorder = execute_and_query(p, "reorder_conv"); + ASSERT_EQ(check_optimized_out(p, "reorder_conv"), false); + ASSERT_EQ(target_reorder->can_be_optimized(), true); +} + +// Negative : reorder is padded +TEST_P(testing_removal_1d_reorder, padded_reorder_1d_along_f_mixed_format) { + auto p = GetParam(); + + std::vector pad = { 0, 0, static_cast(p.pad[1]), static_cast(p.pad[0]) }; + layout in_layout{ p.data_type, p.input_format, p.in_shape, padding{pad} }; + + layout reorder_layout(data_types::f16, format::b_fs_yx_fsv32, p.out_shape, padding({0, 0, 1, 1}, 0)); + + create_topologies(input_layout("input", in_layout), + data("weights", get_mem(get_weights_layout(p))), + data("bias1", get_mem(get_bias_layout(p))), + reorder("reorder_bias1", input_info("bias1"), format::b_fs_yx_fsv32, data_types::f16), + convolution("conv_prim", input_info("input"), {"weights"}, std::vector{}, 1, p.stride, p.pad), + reorder("reorder_conv", input_info("conv_prim"), reorder_layout), + eltwise("add_bias1", { input_info("reorder_conv"), input_info("reorder_bias1") }, eltwise_mode::sum), + reorder("reorder_bfyx", input_info("add_bias1"), p.default_format, data_types::f16) + ); + + const auto& target_reorder = execute_and_query(p, "reorder_conv"); + ASSERT_EQ(check_optimized_out(p, "reorder_conv"), false); + ASSERT_EQ(target_reorder->can_be_optimized(), false); +} + +INSTANTIATE_TEST_SUITE_P(reorder_gpu_testing_1d_removal, testing_removal_1d_reorder, + ::testing::ValuesIn(std::vector{ + redundant_reorder_test_param{{1, 32, 1, 1}, {1, 32, 1, 1}, {1, 1, 1, 1}, {1, 1}, {0, 0}, + data_types::f16, format::b_fs_yx_fsv16, data_types::f16, format::goiyx, data_types::f16, format::b_fs_yx_fsv16, false}, + })); + +class testing_removal_feature_aligned_reorder : public ReorderTest {}; +TEST_P(testing_removal_feature_aligned_reorder, removal_reorder_aligned_mixed_format) { + auto p = GetParam(); + + std::vector pad = { 0, 0, static_cast(p.pad[1]), static_cast(p.pad[0]) }; + layout in_layout{ p.data_type, p.input_format, p.in_shape, padding{pad} }; + + create_topologies(input_layout("input", in_layout), + data("bias1", get_mem(get_bias_layout(p))), + reorder("reorder_bias1", input_info("bias1"), format::b_fs_yx_fsv32, data_types::f16), + reorder("reorder_input", input_info("input"), format::b_fs_yx_fsv32, data_types::f16), + eltwise("add_bias1", { input_info("reorder_input"), input_info("reorder_bias1") }, eltwise_mode::sum), + reorder("reorder_bfyx", input_info("add_bias1"), p.default_format, data_types::f16) + ); + + const auto& target_reorder = execute_and_query(p, "reorder_input"); + ASSERT_EQ(check_optimized_out(p, "reorder_input"), false); + ASSERT_EQ(target_reorder->can_be_optimized(), p.opt_out); +} + +// Negative : reorder is padded +TEST_P(testing_removal_feature_aligned_reorder, padded_reorder_aligned_mixed_format) { + auto p = GetParam(); + + std::vector pad = { 0, 0, static_cast(p.pad[1]), static_cast(p.pad[0]) }; + layout in_layout{ p.data_type, p.input_format, p.in_shape, padding{pad} }; + + layout reorder_layout(data_types::f16, format::b_fs_yx_fsv32, p.out_shape, padding({0, 0, 1, 1}, 0)); + + create_topologies(input_layout("input", in_layout), + data("bias1", get_mem(get_bias_layout(p))), + reorder("reorder_bias1", input_info("bias1"), format::b_fs_yx_fsv32, data_types::f16), + reorder("reorder_input", input_info("input"), reorder_layout), + eltwise("add_bias1", { input_info("reorder_input"), input_info("reorder_bias1") }, eltwise_mode::sum), + reorder("reorder_bfyx", input_info("add_bias1"), p.default_format, data_types::f16) + ); + + const auto& target_reorder = execute_and_query(p, "reorder_input"); + ASSERT_EQ(check_optimized_out(p, "reorder_input"), false); + ASSERT_EQ(target_reorder->can_be_optimized(), false); +} + +INSTANTIATE_TEST_SUITE_P(reorder_gpu_testing_1d_removal, testing_removal_feature_aligned_reorder, + ::testing::ValuesIn(std::vector{ + redundant_reorder_test_param{{1, 32, 8, 8}, {1, 32, 8, 8}, {1, 1, 1, 1}, {1, 1}, {0, 0}, + data_types::f16, format::byxf, data_types::f16, format::goiyx, data_types::f16, format::b_fs_yx_fsv16, true}, + redundant_reorder_test_param{{1, 32, 1, 1}, {1, 32, 1, 1}, {1, 1, 1, 1}, {1, 1}, {0, 0}, + data_types::f16, format::byxf, data_types::f16, format::goiyx, data_types::f16, format::b_fs_yx_fsv16, true}, + redundant_reorder_test_param{{1, 64, 8, 8}, {1, 64, 8, 8}, {1, 1, 1, 1}, {1, 1}, {0, 0}, + data_types::f16, format::byxf, data_types::f16, format::goiyx, data_types::f16, format::b_fs_yx_fsv16, false}, + redundant_reorder_test_param{{1, 1, 1, 32}, {1, 1, 1, 32}, {1, 1, 1, 1}, {1, 1}, {0, 0}, + data_types::f16, format::byxf, data_types::f16, format::goiyx, data_types::f16, format::b_fs_yx_fsv16, false}, + redundant_reorder_test_param{{1, 1, 1, 32}, {1, 1, 1, 32}, {1, 1, 1, 1}, {1, 1}, {0, 0}, + data_types::f16, format::b_fs_yx_fsv16, data_types::f16, format::goiyx, data_types::f16, format::b_fs_yx_fsv16, false}, + })); + #ifdef ENABLE_ONEDNN_FOR_GPU TEST(reorder_onednn_gpu, basic_convert_int8) { auto& engine = get_test_engine(); @@ -2957,6 +3090,7 @@ TEST_P(testing_removal_reorder, removal_reorder_1d_along_f_cached) { ASSERT_EQ(check_optimized_out(p, "reorder_bias1"), true); } #endif + TEST_P(testing_removal_reorder, only_remove_reorder_shallow_depth_input_cached) { auto p = GetParam(); layout reorder_layout(data_types::u8, format::b_fs_yx_fsv32, p.in_shape, padding({0, }, 0));