From 1ba8005688f00e6421f2ec45aaeee0d4a0d85c09 Mon Sep 17 00:00:00 2001 From: Roman Lyamin Date: Tue, 26 Sep 2023 18:43:57 +0400 Subject: [PATCH] [GPU] Disabling redundant copying of constant weights (#18949) --- ...decompression_convert_constant_folding.hpp | 11 +++ .../rt_info/keep_const_precision.hpp | 35 ++++++++ .../rt_info/keep_fp16_const.hpp | 35 -------- .../src/transformations/convert_precision.cpp | 6 +- ...decompression_convert_constant_folding.cpp | 49 +++++++++- .../rt_info/keep_const_precision.cpp | 20 +++++ .../rt_info/keep_fp16_const.cpp | 20 ----- ...stants_precision_and_add_converts_test.cpp | 86 ++++++++++++++++++ .../include/intel_gpu/graph/program.hpp | 30 ++++--- .../include/intel_gpu/runtime/format.hpp | 16 +++- .../graph_optimizer/post_optimize_weights.cpp | 13 +-- .../prepare_primitive_fusing.cpp | 89 +++++++++++++++++++ .../prepare_primitive_fusing_through.cpp | 2 +- .../remove_redundant_reorders.cpp | 49 +++------- .../graph_optimizer/reorder_transfer.cpp | 56 ++++++++++++ .../impls/ocl/kernel_selector_helper.cpp | 7 ++ .../intel_gpu/src/graph/impls/ocl/reorder.cpp | 2 + .../src/graph/include/pass_manager.h | 9 ++ .../src/graph/include/reorder_inst.h | 29 +++--- src/plugins/intel_gpu/src/graph/program.cpp | 11 ++- .../intel_gpu/src/kernel_selector/jitter.cpp | 10 ++- .../kernel_selector_common.cpp | 1 + .../kernel_selector/kernel_selector_utils.cpp | 3 + .../kernels/reorder/reorder_weights_opt.cpp | 5 ++ .../src/kernel_selector/tensor_type.cpp | 1 + .../src/kernel_selector/tensor_type.h | 2 + .../transformations/convert_matmul_to_fc.cpp | 8 +- .../src/plugin/transformations_pipeline.cpp | 17 ++-- src/plugins/intel_gpu/src/runtime/format.cpp | 44 +++++++-- src/plugins/intel_gpu/src/runtime/layout.cpp | 2 + .../dynamic/matmul_weights_decompression.cpp | 5 +- .../tests/unit/module_tests/format_test.cpp | 87 ++++++++++++++++++ .../passes/prepare_primitive_fusing_test.cpp | 76 ++++++++++++++++ .../tests/unit/passes/reorder_transfer.cpp | 48 ++++++++++ .../unit/test_cases/hash_key_gpu_test.cpp | 2 +- .../tests/unit/test_utils/test_utils.h | 21 +++++ 36 files changed, 754 insertions(+), 153 deletions(-) create mode 100644 src/common/transformations/include/transformations/rt_info/keep_const_precision.hpp delete mode 100644 src/common/transformations/include/transformations/rt_info/keep_fp16_const.hpp create mode 100644 src/common/transformations/src/transformations/rt_info/keep_const_precision.cpp delete mode 100644 src/common/transformations/src/transformations/rt_info/keep_fp16_const.cpp create mode 100644 src/common/transformations/tests/common_optimizations/keep_constants_precision_and_add_converts_test.cpp create mode 100644 src/plugins/intel_gpu/src/graph/graph_optimizer/reorder_transfer.cpp create mode 100644 src/plugins/intel_gpu/tests/unit/passes/reorder_transfer.cpp diff --git a/src/common/transformations/include/transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp b/src/common/transformations/include/transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp index 29d335b0db1..7576abfac15 100644 --- a/src/common/transformations/include/transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp +++ b/src/common/transformations/include/transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp @@ -14,6 +14,7 @@ namespace pass { class TRANSFORMATIONS_API EnableDecompressionConvertConstantFolding; class TRANSFORMATIONS_API DisableDecompressionConvertConstantFolding; class TRANSFORMATIONS_API KeepConstAndDecompression; +class TRANSFORMATIONS_API KeepConstantsPrecisionAndAddConverts; } // namespace pass } // namespace ov @@ -47,3 +48,13 @@ public: OPENVINO_RTTI("KeepConstAndDecompression", "0"); KeepConstAndDecompression(); }; + +/** + * @ingroup ie_transformation_common_api + * @brief Prevents Consts precision conversion and adds Convert with disabled ConstantFolding + */ +class ov::pass::KeepConstantsPrecisionAndAddConverts : public MatcherPass { +public: + OPENVINO_RTTI("KeepConstantsPrecisionAndAddConverts", "0"); + KeepConstantsPrecisionAndAddConverts(); +}; diff --git a/src/common/transformations/include/transformations/rt_info/keep_const_precision.hpp b/src/common/transformations/include/transformations/rt_info/keep_const_precision.hpp new file mode 100644 index 00000000000..af9d7002387 --- /dev/null +++ b/src/common/transformations/include/transformations/rt_info/keep_const_precision.hpp @@ -0,0 +1,35 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/core/node.hpp" +#include "openvino/core/runtime_attribute.hpp" +#include "transformations_visibility.hpp" + +namespace ov { + +TRANSFORMATIONS_API void enable_keep_const_precision(const std::shared_ptr& node); + +TRANSFORMATIONS_API void disable_keep_const_precision(const std::shared_ptr& node); + +TRANSFORMATIONS_API bool is_keep_const_precision(const std::shared_ptr& node); + +/** + * @ingroup ie_runtime_attr_api + * @brief KeepConstPrecision class represents runtime info attribute that marks a Constant + * as prohibitted to fuse precision in ConvertPrecision + */ +class TRANSFORMATIONS_API KeepConstPrecision : public RuntimeAttribute { +public: + OPENVINO_RTTI("keep_const_precision", "0"); + + KeepConstPrecision() = default; + + bool is_copyable() const override { + return false; + } +}; + +} // namespace ov diff --git a/src/common/transformations/include/transformations/rt_info/keep_fp16_const.hpp b/src/common/transformations/include/transformations/rt_info/keep_fp16_const.hpp deleted file mode 100644 index f38ca72f9a0..00000000000 --- a/src/common/transformations/include/transformations/rt_info/keep_fp16_const.hpp +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include "openvino/core/node.hpp" -#include "openvino/core/runtime_attribute.hpp" -#include "transformations_visibility.hpp" - -namespace ov { - -TRANSFORMATIONS_API void enable_keep_fp16_const(const std::shared_ptr& node); - -TRANSFORMATIONS_API void disable_keep_fp16_const(const std::shared_ptr& node); - -TRANSFORMATIONS_API bool is_keep_fp16_const(const std::shared_ptr& node); - -/** - * @ingroup ie_runtime_attr_api - * @brief DisableFP16Compression class represents runtime info attribute that marks operation - * as prohibitted to convert to FP16 as part of Compressed Only format. - */ -class TRANSFORMATIONS_API KeepFP16Const : public RuntimeAttribute { -public: - OPENVINO_RTTI("keep_fp16_const", "0"); - - KeepFP16Const() = default; - - bool is_copyable() const override { - return false; - } -}; - -} // namespace ov diff --git a/src/common/transformations/src/transformations/convert_precision.cpp b/src/common/transformations/src/transformations/convert_precision.cpp index 65387086310..c5e2b13381c 100644 --- a/src/common/transformations/src/transformations/convert_precision.cpp +++ b/src/common/transformations/src/transformations/convert_precision.cpp @@ -26,7 +26,7 @@ #include "transformations/fp16_compression/mark_subgraphs_to_keep_in_mixed_precision.hpp" #include "transformations/rt_info/decompression.hpp" #include "transformations/rt_info/disable_fp16_compression.hpp" -#include "transformations/rt_info/keep_fp16_const.hpp" +#include "transformations/rt_info/keep_const_precision.hpp" #include "transformations/utils/utils.hpp" using namespace ov; @@ -1125,8 +1125,8 @@ std::shared_ptr convert_low_precisions_int(std::shared_ptr& node, const precisions_map& precisions, const std::vector>& consumers) { - // Consts marked with disable_constant_folding should be kept in f16 until they reach the plugin - if (is_keep_fp16_const(node)) + // Consts marked with is_keep_const_precision should be kept in their own precision until they reach the plugin + if (is_keep_const_precision(node)) return false; auto from = node->get_element_type(); diff --git a/src/common/transformations/src/transformations/fp16_compression/mark_decompression_convert_constant_folding.cpp b/src/common/transformations/src/transformations/fp16_compression/mark_decompression_convert_constant_folding.cpp index de03d931ac4..26505dee527 100644 --- a/src/common/transformations/src/transformations/fp16_compression/mark_decompression_convert_constant_folding.cpp +++ b/src/common/transformations/src/transformations/fp16_compression/mark_decompression_convert_constant_folding.cpp @@ -5,14 +5,16 @@ #include "transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp" #include "itt.hpp" +#include "openvino/core/rt_info.hpp" #include "openvino/op/constant.hpp" #include "openvino/op/convert.hpp" #include "openvino/op/matmul.hpp" #include "openvino/pass/pattern/op/wrap_type.hpp" #include "transformations/rt_info/decompression.hpp" #include "transformations/rt_info/disable_constant_folding.hpp" +#include "transformations/rt_info/disable_fp16_compression.hpp" #include "transformations/rt_info/is_shape_subgraph.hpp" -#include "transformations/rt_info/keep_fp16_const.hpp" +#include "transformations/rt_info/keep_const_precision.hpp" using namespace ov; @@ -67,10 +69,53 @@ pass::KeepConstAndDecompression::KeepConstAndDecompression() { if (!is_type(node->input_value(0).get_node_shared_ptr())) return false; - enable_keep_fp16_const(node->input_value(0).get_node_shared_ptr()); + enable_keep_const_precision(node->input_value(0).get_node_shared_ptr()); return false; }; auto m = std::make_shared(node_pattern, matcher_name); register_matcher(m, callback); } + +pass::KeepConstantsPrecisionAndAddConverts::KeepConstantsPrecisionAndAddConverts() { + MATCHER_SCOPE(KeepConstantsPrecisionAndAddConverts); + auto const_pattern = pattern::wrap_type(); + + matcher_pass_callback callback = [=](pattern::Matcher& m) { + auto const_node = m.get_match_root(); + + if (transformation_callback(const_node)) { + return false; + } + + enable_keep_const_precision(const_node); + + const auto& constant_target_inputs = const_node->get_output_target_inputs(0); + const auto& next_node = constant_target_inputs.begin()->get_node()->shared_from_this(); + if (is_type(next_node)) { + disable_constant_folding(next_node); + if (is_decompression(next_node)) { + unmark_as_decompression(next_node); + } + return true; + } + + auto convert = std::make_shared(const_node, const_node->get_element_type()); + convert->set_friendly_name(const_node->get_friendly_name()); + + std::string postfix = const_node->get_element_type() == ov::element::f32 ? "compression" : "decompression"; + const_node->set_friendly_name(const_node->get_friendly_name() + "_postponed_" + postfix); + + ov::copy_runtime_info(const_node, convert); + disable_constant_folding(convert); + + for (const auto& target_input : constant_target_inputs) { + target_input.replace_source_output(convert); + } + + return true; + }; + + auto m = std::make_shared(const_pattern, matcher_name); + this->register_matcher(m, callback); +} diff --git a/src/common/transformations/src/transformations/rt_info/keep_const_precision.cpp b/src/common/transformations/src/transformations/rt_info/keep_const_precision.cpp new file mode 100644 index 00000000000..8752631e74b --- /dev/null +++ b/src/common/transformations/src/transformations/rt_info/keep_const_precision.cpp @@ -0,0 +1,20 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/rt_info/keep_const_precision.hpp" + +void ov::enable_keep_const_precision(const std::shared_ptr& node) { + auto& rt_info = node->get_rt_info(); + rt_info[KeepConstPrecision::get_type_info_static()] = KeepConstPrecision{}; +} + +void ov::disable_keep_const_precision(const std::shared_ptr& node) { + auto& rt_info = node->get_rt_info(); + rt_info.erase(KeepConstPrecision::get_type_info_static()); +} + +bool ov::is_keep_const_precision(const std::shared_ptr& node) { + const auto& rt_info = node->get_rt_info(); + return rt_info.count(KeepConstPrecision::get_type_info_static()); +} diff --git a/src/common/transformations/src/transformations/rt_info/keep_fp16_const.cpp b/src/common/transformations/src/transformations/rt_info/keep_fp16_const.cpp deleted file mode 100644 index 41fce20bd7e..00000000000 --- a/src/common/transformations/src/transformations/rt_info/keep_fp16_const.cpp +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "transformations/rt_info/keep_fp16_const.hpp" - -void ov::enable_keep_fp16_const(const std::shared_ptr& node) { - auto& rt_info = node->get_rt_info(); - rt_info[KeepFP16Const::get_type_info_static()] = KeepFP16Const{}; -} - -void ov::disable_keep_fp16_const(const std::shared_ptr& node) { - auto& rt_info = node->get_rt_info(); - rt_info.erase(KeepFP16Const::get_type_info_static()); -} - -bool ov::is_keep_fp16_const(const std::shared_ptr& node) { - const auto& rt_info = node->get_rt_info(); - return rt_info.count(KeepFP16Const::get_type_info_static()); -} diff --git a/src/common/transformations/tests/common_optimizations/keep_constants_precision_and_add_converts_test.cpp b/src/common/transformations/tests/common_optimizations/keep_constants_precision_and_add_converts_test.cpp new file mode 100644 index 00000000000..ae6076e81c8 --- /dev/null +++ b/src/common/transformations/tests/common_optimizations/keep_constants_precision_and_add_converts_test.cpp @@ -0,0 +1,86 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include + +#include "common_test_utils/ov_test_utils.hpp" +#include "openvino/opsets/opset1.hpp" +#include "transformations/convert_precision.hpp" +#include "transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp" +#include "transformations/utils/utils.hpp" + +using namespace testing; +using namespace ov; +using namespace ov::opset1; +using const_node_ptr = const std::shared_ptr; + +TEST_F(TransformationTestsF, KeepConstantsPrecisionAndAddConvertsTestBase) { + { + auto input = std::make_shared(element::f32, Shape{3, 2, 2}); + auto weights = Constant::create(element::f32, Shape{1, 2, 2}, {1}); + auto matmul = std::make_shared(input, weights); + + model = std::make_shared(NodeVector{matmul}, ParameterVector{input}); + + manager.register_pass(); + manager.get_pass_config()->set_callback( + [](const_node_ptr& node) -> bool { + auto next_node = node->get_output_target_inputs(0).begin()->get_node(); + if (is_type(next_node)) { + next_node = next_node->get_output_target_inputs(0).begin()->get_node(); + } + return !is_type(next_node); + }); + + const precisions_map precisions = {{element::f32, element::f16}}; + const type_to_fuse_map empty_fuse_map = {}; + const bool keep_precision_sensitive_in_fp32_1 = true; + manager.register_pass(precisions, empty_fuse_map, keep_precision_sensitive_in_fp32_1); + } + { + auto input = std::make_shared(element::f16, Shape{3, 2, 2}); + auto weights = Constant::create(element::f32, Shape{1, 2, 2}, {1}); + auto convert_weights = std::make_shared(weights, element::f16); + auto matmul = std::make_shared(input, convert_weights); + + model_ref = std::make_shared(NodeVector{matmul}, ParameterVector{input}); + } +} + +TEST_F(TransformationTestsF, KeepConstantsPrecisionAndAddConvertsTestWithCompressedConvert) { + { + auto input = std::make_shared(element::f16, Shape{3, 2, 2}); + auto weights = Constant::create(element::f32, Shape{1, 2, 2}, {1}); + auto convert_weights = std::make_shared(weights, element::f16); + mark_as_decompression(convert_weights); + auto matmul = std::make_shared(input, convert_weights); + + model = std::make_shared(NodeVector{matmul}, ParameterVector{input}); + + manager.register_pass(); + manager.get_pass_config()->set_callback( + [](const_node_ptr& node) -> bool { + auto next_node = node->get_output_target_inputs(0).begin()->get_node(); + if (is_type(next_node)) { + next_node = next_node->get_output_target_inputs(0).begin()->get_node(); + } + return !is_type(next_node); + }); + + const precisions_map precisions = {{element::f32, element::f16}}; + const type_to_fuse_map empty_fuse_map = {}; + const bool keep_precision_sensitive_in_fp32_1 = true; + manager.register_pass(precisions, empty_fuse_map, keep_precision_sensitive_in_fp32_1); + } + { + auto input = std::make_shared(element::f16, Shape{3, 2, 2}); + auto weights = Constant::create(element::f32, Shape{1, 2, 2}, {1}); + auto convert_weights = std::make_shared(weights, element::f16); + auto matmul = std::make_shared(input, convert_weights); + + model_ref = std::make_shared(NodeVector{matmul}, ParameterVector{input}); + } +} diff --git a/src/plugins/intel_gpu/include/intel_gpu/graph/program.hpp b/src/plugins/intel_gpu/include/intel_gpu/graph/program.hpp index dab52844e7e..1a196ea49e8 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/graph/program.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/graph/program.hpp @@ -35,20 +35,22 @@ class ICompilationContext; struct program { using ptr = std::shared_ptr; using cptr = std::shared_ptr; - friend class calculate_prior_boxes; // to be removed when possible - friend class graph_initializations; // to be removed when possible - friend class prepare_padding; // to be removed when possible - friend class propagate_constants; // to be removed when possible - friend class pre_replace_deconv; // to be removed when possible - friend class prepare_primitive_fusing; // to be removed when possible - friend class prepare_quantization; // to be removed when possible - friend class prepare_conv_eltw_fusing; // to be removed when possible - friend class reorder_inputs; // to be removed when possible - friend class remove_redundant_reorders; // to be removed when possible - friend class post_optimize_weights; // to be removed when possible - friend class program_wrapper; // this class is intended to extend the interface of program for - // the usage within tests_core_internal project only - friend class prepare_primitive_fusing_through; // to be removed when possible + friend class calculate_prior_boxes; // to be removed when possible + friend class graph_initializations; // to be removed when possible + friend class prepare_padding; // to be removed when possible + friend class propagate_constants; // to be removed when possible + friend class pre_replace_deconv; // to be removed when possible + friend class prepare_primitive_fusing; // to be removed when possible + friend class prepare_quantization; // to be removed when possible + friend class prepare_conv_eltw_fusing; // to be removed when possible + friend class reorder_inputs; // to be removed when possible + friend class remove_redundant_reorders; // to be removed when possible + friend class post_optimize_weights; // to be removed when possible + friend class prepare_primitive_fusing_through; // to be removed when possible + friend class reorder_transfer; // to be removed when possible + friend class fuse_constant_transposes; // to be removed when possible + friend class program_wrapper; // this class is intended to extend the interface of program for + // the usage within tests_core_internal project only public: struct nodes_ordering { public: diff --git a/src/plugins/intel_gpu/include/intel_gpu/runtime/format.hpp b/src/plugins/intel_gpu/include/intel_gpu/runtime/format.hpp index a8186b9d62f..fb6bfaa3fc8 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/runtime/format.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/runtime/format.hpp @@ -80,6 +80,7 @@ struct format { bfvuwzyx, ///< 8d tensor yxfb, ///< batch first, feature and than spatials byxf, ///< used in bitmaps, input from user i.e b images of RGB format + fbyx, fyxb, ///< format not used inside clDNN, but supported in reorder as extension bzyxf, byfx, ///< To be used when onednn gemm allows permute fusing in transformer network. Not for normal use from cldnn. @@ -341,8 +342,9 @@ struct format { return (fmt == yxfb || fmt == byxf || fmt == byfx || fmt == bxfy || fmt == bfyx || fmt == fyxb || - fmt == bfzyx || fmt == bfwzyx || - fmt == bfuwzyx || fmt == bfvuwzyx); + fmt == fbyx || fmt == bfzyx || + fmt == bfwzyx || fmt == bfuwzyx || + fmt == bfvuwzyx); } static format get_default_format(size_t rank, bool is_weights = false, bool is_grouped = false); @@ -352,6 +354,14 @@ struct format { static const std::vector> per_axis_block_size(format fmt); + static format find_format(const std::vector& order, + const std::vector>& block_sizes, + bool is_weights = false, + bool is_grouped = false, + bool is_image_2d = false, + bool is_winograd = false, + bool is_nv12 = false); + /// @brief Checks if @p format is of grouped type static bool is_grouped(type fmt) { return group_num(fmt) != 0; } /// @brief Checks if @p format is of image type @@ -373,6 +383,8 @@ struct format { size_t spatial_num() const { return traits(value).spatial_num; } /// @brief Returns number of group dimensions. size_t group_num() const { return traits(value).group_num; } + /// @brief Returns an order of dimensions. + const std::vector& dims_order() const { return traits(value)._order; } /// @brief Returns an order of dimensions in form of string. const std::string& order() const { return traits(value).order; } /// @brief Returns an internal orders of dimensions form of string. diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/post_optimize_weights.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/post_optimize_weights.cpp index 175a6eef54b..4f48148c395 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/post_optimize_weights.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/post_optimize_weights.cpp @@ -79,16 +79,19 @@ void post_optimize_weights::optimize_weights(T& node, program& p) { if (weights_reorder_params != nullptr) { bool can_be_fused = prev_node.is_type() && + prev_node.as().is_simple_reorder() && prev_node.get_users().size() == 1 && - prev_node.get_dependencies().size() == 1 && - !prev_node.has_fused_primitives() && - !prev_node.as().has_mean() && - prev_node.as().get_primitive()->subtract_per_feature.empty(); + prev_node.get_dependencies().size() == 1; if (can_be_fused) { // Need to update input data_type for correct merging format reorder with precision reorder - data_types input_dtype = prev_node.get_input_layouts()[0].data_type; auto updated_input_layout = weights_reorder_params->get_input_layout(); + data_types input_dtype = prev_node.get_input_layout().data_type; updated_input_layout.data_type = input_dtype; + + // Need to update input format in case of fusing weights constant with transpose + format input_fmt = prev_node.get_input_layout().format; + updated_input_layout.format = from_weights_layout(to_weights_layout(input_fmt, false)); + weights_reorder_params->set_input_layout(updated_input_layout); auto weights_reorder = _rf.get_weights_reorder(prev_node.get_primitive()->input[0].pid, weights_reorder_params); diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp index f7b3fd2a08f..4d89826ac54 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp @@ -60,6 +60,7 @@ void prepare_primitive_fusing::run(program& p) { fuse_sigmoid_mul_to_swish(p); fuse_bias(p); fuse_simple_primitives(p); + fuse_constant_transposes(p); optimize_fused_ops(p); } @@ -1186,6 +1187,94 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { p.get_processing_order().calc_processing_order(p); } +void prepare_primitive_fusing::fuse_constant_transposes(program& p) { + std::function get_weightable_node = + [&get_weightable_node](const program_node* node) -> const program_node* { + if (node->get_users().empty()) + return nullptr; + + const auto* next_node = node->get_users().front(); + + if (next_node->is_type() || + next_node->is_type() || + next_node->is_type() || + next_node->is_type() || + next_node->is_type()) { + size_t weights_offset = next_node->get_primitive()->input_size(); + return &next_node->get_dependency(weights_offset) == node ? next_node + : nullptr; + } + + if (node->is_constant() && node->get_users().size() == 1) + return get_weightable_node(next_node); + + return nullptr; + }; + + auto convert_data_format_by_order = [](format fmt, const std::vector& order) -> format { + const auto& old_order = fmt.dims_order(); + auto new_order = old_order; + + for (size_t i = 0; i < order.size(); ++i) { + new_order[i] = old_order[order[i]]; + } + + return format::find_format(new_order, fmt.block_sizes()); + }; + + auto itr = p.get_processing_order().begin(); + while (itr != p.get_processing_order().end()) { + auto& node = *itr++; + + if (!node->is_type()) + continue; + + auto& permute_node = node->as(); + + auto weightable_node = get_weightable_node(&permute_node); + + if (weightable_node == nullptr || !permute_node.get_dependency(0).is_type()) + continue; + + auto& prev_const = permute_node.get_dependency(0).as(); + + if (prev_const.get_users().size() != 1) + continue; + + auto permute_order = permute_node.get_primitive()->permute_order; + // Assumption that fc weights will be reshaped to 2d + if (permute_order.size() != 2 && weightable_node->is_type()) { + if (permute_order == std::vector{0, 2, 1} || + permute_order == std::vector{0, 1, 3, 2}) { + permute_order = {1, 0}; + } else { + continue; + } + } + + format new_fmt = format::any; + try { + new_fmt = convert_data_format_by_order(prev_const.get_output_layout().format, permute_order); + } catch(ov::Exception&) { + continue; + } + + layout updated_const_layout = prev_const.get_output_layout(); + updated_const_layout.format = new_fmt; + updated_const_layout.set_partial_shape(permute_node.get_output_pshape()); + + p.extract_and_remove(permute_node); + + const auto& new_mem = p.get_engine().reinterpret_buffer(prev_const.get_attached_memory(), updated_const_layout); + + auto new_const_prim = std::make_shared(prev_const.id() + "_fused_with_transpose", new_mem); + auto& new_const_node = p.get_or_create(new_const_prim); + + p.replace(prev_const, new_const_node); + new_const_node.recalc_output_layout(false); + } +} + void prepare_primitive_fusing::optimize_fused_ops(program& p) { auto itr = p.get_processing_order().begin(); while (itr != p.get_processing_order().end()) { diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing_through.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing_through.cpp index c85e4cbd8de..899369aa5c7 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing_through.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing_through.cpp @@ -96,7 +96,7 @@ void prepare_primitive_fusing_through::run(program& p) { input_node = &node->get_dependency(0); } else if (node->is_type()) { - if (node->get_dependencies().size() !=2) + if (node->get_dependencies().size() != 2) continue; size_t second_input_idx = 0; diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp index 7ff8390cb9f..362cf5bf63d 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp @@ -59,13 +59,7 @@ void remove_redundant_reorders::run(program& p) { auto& input = node.input(); auto output_layout = node.get_output_layout(); - if (node.is_output()) - continue; - - if (node.has_mean() || !node.get_primitive()->subtract_per_feature.empty()) - continue; - - if (node.has_fused_primitives()) + if (!node.is_simple_reorder() || node.is_output()) continue; std::function has_quantize_user; @@ -149,12 +143,11 @@ void remove_redundant_reorders::run(program& p) { auto& r_dep_node = dep_node.as(); - bool remove_dep = r_dep_node.get_users().size() == 1 && - !r_dep_node.has_mean() && - r_dep_node.get_primitive()->subtract_per_feature.empty() && - !r_dep_node.is_output() && - !r_dep_node.has_fused_primitives() && - !r_dep_node.get_primitive()->has_surface_input(); + bool remove_dep = r_dep_node.is_simple_reorder() && + r_dep_node.get_users().size() == 1 && + !r_dep_node.is_output() && + !r_dep_node.get_primitive()->has_surface_input() && + !r_node.get_primitive()->weights_reorder_params; // for chains like // fp32 -> reorder -> u8 -> reorder -> fp32 @@ -164,13 +157,10 @@ void remove_redundant_reorders::run(program& p) { continue; } - bool remove_current = - r_dep_node.get_users().size() == 1 && - !r_dep_node.is_output() && - !r_node.has_mean() && - r_node.get_primitive()->subtract_per_feature.empty() && - !r_node.has_fused_primitives() && - !r_node.get_primitive()->has_surface_input(); + bool remove_current = r_node.is_simple_reorder() && + r_dep_node.get_users().size() == 1 && + !r_dep_node.is_output() && + !r_node.get_primitive()->has_surface_input(); if (remove_dep) { // for chains like @@ -271,13 +261,9 @@ void remove_redundant_reorders::run(program& p) { r_node.is_output() && (r_node.get_dependency(0).is_output() || r_node.get_dependency(0).is_type() || r_node.get_dependency(0).can_be_optimized() || r_node.get_dependency(0).get_users().size() != 1) : r_node.is_output(); - if (r_node.has_mean() || - !r_node.get_primitive()->subtract_per_feature.empty() || + if (!r_node.is_simple_reorder() || no_output_optimization || - r_node.has_fused_primitives() || - r_node.get_primitive()->has_surface_input() || - (r_node.get_primitive()->weights_reorder_params && - r_node.get_primitive()->weights_reorder_params->should_be_transposed())) + r_node.get_primitive()->has_surface_input()) continue; auto o_layout = r_node.get_output_layout(); @@ -406,10 +392,7 @@ void remove_redundant_reorders::run(program& p) { auto& input = node.input(); auto output_layout = node.get_output_layout(); - if (node.has_mean() || !node.get_primitive()->subtract_per_feature.empty()) - continue; - - if (node.has_fused_primitives()) + if (!node.is_simple_reorder()) continue; if (input.get_users().size() != 1) @@ -614,11 +597,7 @@ void remove_redundant_reorders::run(program& p) { if (!r_node.is_in_data_flow() || r_node.get_dependencies().size() != 1) continue; - if (r_node.has_mean() || - !r_node.get_primitive()->subtract_per_feature.empty()) - continue; - - if (r_node.has_fused_primitives()) + if (!r_node.is_simple_reorder()) continue; // Remove reorder for Convolution bfyx -> fs_b_yx_fsv32 diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/reorder_transfer.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/reorder_transfer.cpp new file mode 100644 index 00000000000..daa9dbd89ad --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/reorder_transfer.cpp @@ -0,0 +1,56 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "pass_manager.h" +#include "reorder_inst.h" +#include "permute_inst.h" +#include "program_helpers.h" + +using namespace cldnn; + +void reorder_transfer::run(program& p) { + auto itr = p.get_processing_order().begin(); + while (itr != p.get_processing_order().end()) { + auto& node = *itr++; + + if (!node->is_type()) + continue; + + auto& reorder_node = node->as(); + + bool is_simple_type_conversion_reorder = reorder_node.is_constant() && + !reorder_node.is_output() && + reorder_node.get_users().size() == 1 && + reorder_node.get_dependencies().size() == 1 && + reorder_node.is_type_conversion_only(); + if (!is_simple_type_conversion_reorder) + continue; + + auto transfer_through_node = [](cldnn::program_node* node) -> bool { // Conditions can be extended to other ops + return node->is_type() && + node->get_users().size() == 1 && + node->get_dependencies().size() == 1; + }; + + auto change_output_dtype = [](cldnn::program_node* node, cldnn::data_types dtype) { + layout new_layout = node->get_output_layout(); + new_layout.data_type = dtype; + node->set_output_layout(new_layout, false); + }; + + auto* supposed_new_prev = reorder_node.get_users().front(); + cldnn::program_node* new_prev = nullptr; + while (transfer_through_node(supposed_new_prev)) { + change_output_dtype(supposed_new_prev, reorder_node.get_input_layout().data_type); + new_prev = supposed_new_prev; + supposed_new_prev = supposed_new_prev->get_users().front(); + } + + if (new_prev != nullptr) { + auto& new_next = new_prev->get_users().front(); + p.move_node(reorder_node, *new_prev, *new_next); + reorder_node.recalc_output_layout(false); + } + } +} diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp index 5cc70002deb..5fd4a17728e 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/kernel_selector_helper.cpp @@ -211,6 +211,8 @@ kernel_selector::data_layout to_data_layout(format f) { return kernel_selector::data_layout::byfx; case format::bxfy: return kernel_selector::data_layout::bxfy; + case format::fbyx: + return kernel_selector::data_layout::fbyx; case format::fyxb: return kernel_selector::data_layout::fyxb; case format::b_fs_yx_fsv2: @@ -310,6 +312,8 @@ cldnn::format from_data_layout(kernel_selector::data_layout l) { return cldnn::format::byfx; case kernel_selector::data_layout::bxfy: return cldnn::format::bxfy; + case kernel_selector::data_layout::fbyx: + return cldnn::format::fbyx; case kernel_selector::data_layout::fyxb: return cldnn::format::fyxb; case kernel_selector::data_layout::b_fs_yx_fsv2: @@ -382,6 +386,7 @@ kernel_selector::weights_layout to_weights_layout(format f, bool is_grouped) { case format::bfyx: case format::oiyx: return kernel_selector::weights_layout::oiyx; + case format::fbyx: case format::ioyx: return kernel_selector::weights_layout::ioyx; case format::iyxo: @@ -390,8 +395,10 @@ kernel_selector::weights_layout to_weights_layout(format f, bool is_grouped) { case format::oyxi: case format::byxf: return kernel_selector::weights_layout::oyxi; + case format::oyix: case format::byfx: return kernel_selector::weights_layout::oyix; + case format::oxiy: case format::bxfy: return kernel_selector::weights_layout::oxiy; case format::yxfb: diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/reorder.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/reorder.cpp index af825934d08..976d6e38a58 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/reorder.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/reorder.cpp @@ -126,6 +126,8 @@ public: const auto& weights_params = prim->weights_reorder_params; auto& kernel_selector = kernel_selector::ReorderWeightsKernelSelector::Instance(); + OPENVINO_ASSERT(weights_params != nullptr, "[GPU] Attempt to create reorder weights without weights params"); + OPENVINO_ASSERT(impl_param.get_input_layout().bytes_count() == weights_params->get_input_layout().bytes_count(), "[GPU] Input layout doesn't match required reorder weights layout"); diff --git a/src/plugins/intel_gpu/src/graph/include/pass_manager.h b/src/plugins/intel_gpu/src/graph/include/pass_manager.h index e507014f233..d6409eab31e 100644 --- a/src/plugins/intel_gpu/src/graph/include/pass_manager.h +++ b/src/plugins/intel_gpu/src/graph/include/pass_manager.h @@ -196,6 +196,7 @@ private: void fuse_bias(program &p); void fuse_reorders(program& p); void fuse_simple_primitives(program &p); + void fuse_constant_transposes(program &p); void optimize_fused_ops(program &p); void remove_redundant_reshape(program &p); layout_optimizer& _lo; @@ -406,4 +407,12 @@ public: void run(program& p) override; }; +class reorder_transfer : public base_pass { +public: + reorder_transfer() : base_pass("reorder_transfer") {} + +private: + void run(program& p) override; +}; + } // namespace cldnn diff --git a/src/plugins/intel_gpu/src/graph/include/reorder_inst.h b/src/plugins/intel_gpu/src/graph/include/reorder_inst.h index b1637032ffb..00e48ec8715 100644 --- a/src/plugins/intel_gpu/src/graph/include/reorder_inst.h +++ b/src/plugins/intel_gpu/src/graph/include/reorder_inst.h @@ -41,16 +41,25 @@ public: void set_input_layout(layout const& lo) { input_layout = lo; } bool is_type_conversion_only() const { - if (this->has_fused_primitives() || has_mean() || !typed_desc()->subtract_per_feature.empty()) - return false; - auto in_layout = input().get_output_layout(); - auto out_layout = this->get_output_layout(); - auto check_common_layout = in_layout.data_type != out_layout.data_type && - in_layout.format == out_layout.format && - in_layout.data_padding == out_layout.data_padding; - return typed_desc()->truncate && - ((this->is_dynamic() && check_common_layout && in_layout.get_partial_shape().rank() == out_layout.get_partial_shape().rank()) || - (!this->is_dynamic() && check_common_layout && in_layout.get_partial_shape() == out_layout.get_partial_shape())); + auto in_layout = get_input_layout(); + auto out_layout = get_output_layout(); + bool only_precision_changed = in_layout.data_type != out_layout.data_type && + in_layout.format == out_layout.format && + in_layout.data_padding == out_layout.data_padding; + if (is_dynamic()) { + only_precision_changed &= in_layout.get_partial_shape().rank() == out_layout.get_partial_shape().rank(); + } else { + only_precision_changed &= in_layout.get_partial_shape() == out_layout.get_partial_shape(); + } + + return only_precision_changed && is_simple_reorder() && typed_desc()->truncate; + } + + bool is_simple_reorder() const { + return !has_fused_primitives() && + !has_mean() && + get_primitive()->subtract_per_feature.empty() && + !get_primitive()->weights_reorder_params; } std::shared_ptr get_fuse_params() const override { diff --git a/src/plugins/intel_gpu/src/graph/program.cpp b/src/plugins/intel_gpu/src/graph/program.cpp index ab454e6bc6b..4fcb120cdd4 100644 --- a/src/plugins/intel_gpu/src/graph/program.cpp +++ b/src/plugins/intel_gpu/src/graph/program.cpp @@ -555,6 +555,8 @@ void program::pre_optimize_graph(bool is_internal) { apply_opt_pass(lo); + apply_opt_pass(); + #ifdef GPU_DEBUG_CONFIG GPU_DEBUG_IF(!debug_config->disable_primitive_fusing) { #else @@ -608,7 +610,12 @@ void program::post_optimize_graph(bool is_internal) { reorder_factory rf; layout_optimizer lo; set_layout_optimizer_attributes(lo); - apply_opt_pass(rf); + + bool optimize_data = _config.get_property(ov::intel_gpu::optimize_data); + + if (!is_internal) { + apply_opt_pass(rf); + } apply_opt_pass(lo, false, true); // TODO: do we need it at this place also? @@ -623,7 +630,7 @@ void program::post_optimize_graph(bool is_internal) { apply_opt_pass(); } - if (_config.get_property(ov::intel_gpu::optimize_data)) + if (optimize_data) apply_opt_pass(lo, false, true, true); // pass to remove output reorders while all others graph optimizations were done // update inner program input/output primitive mappings diff --git a/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp b/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp index 92dbe560ef6..20d8e63ae29 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/jitter.cpp @@ -663,6 +663,11 @@ class WeightTensorJitConstant : public TensorBaseTJitConstant; if (l == WeightsLayout::oiyx || l == WeightsLayout::ioyx || + l == WeightsLayout::oyxi || + l == WeightsLayout::oyix || + l == WeightsLayout::oxiy || + l == WeightsLayout::iyxo || + l == WeightsLayout::yxio || l == WeightsLayout::iozyx || l == WeightsLayout::oizyx || l == WeightsLayout::goiyx || @@ -974,7 +979,10 @@ JitDefinitions WeightTensorJitConstant::GetDefinitions() const { bool is_common_4d_layout = is_common_nd_layout(base_4d_channels, layout); if (is_common_4d_layout) { index_macro_name = _name + "_GET_INDEX(o, i, y, x)"; - if (layout == WeightsLayout::oiyx || layout == WeightsLayout::ioyx) + if (layout == WeightsLayout::oiyx || layout == WeightsLayout::ioyx || + layout == WeightsLayout::oyxi || layout == WeightsLayout::oyix || + layout == WeightsLayout::oxiy || layout == WeightsLayout::iyxo || + layout == WeightsLayout::yxio) index_func_val = called_func_name + "(" + _name + ", 0, o, i, 0, y, x)"; else if (layout == WeightsLayout::os_is_yx_isv16_osv16) index_func_val = called_func_name + "(" + _name + ", 0, o, i, 0, y, x, 16)"; diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_common.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_common.cpp index dfae8451980..49e3e2a1f2e 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_common.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_common.cpp @@ -92,6 +92,7 @@ std::string toString(DataLayout l) { case kernel_selector::DataLayout::byxf: return "BYXF"; case kernel_selector::DataLayout::byfx: return "BYFX"; case kernel_selector::DataLayout::bxfy: return "BXFY"; + case kernel_selector::DataLayout::fbyx: return "FBYX"; case kernel_selector::DataLayout::fyxb: return "FYXB"; case kernel_selector::DataLayout::b_fs_yx_fsv2: return "B_FS_YX_FSV2"; case kernel_selector::DataLayout::b_fs_yx_fsv4: return "B_FS_YX_FSV4"; diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_utils.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_utils.cpp index 59a003c554e..a2e0226367d 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_utils.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernel_selector_utils.cpp @@ -289,6 +289,9 @@ std::vector GetOptimalLocalWorkGroupSizes(std::vector gws, const case DataLayout::bxfy: layout_order = { y, f, x, b, z, w, u, v }; break; + case DataLayout::fbyx: + layout_order = { x, y, b, f, z, w, u, v }; + break; case DataLayout::fyxb: layout_order = { b, x, y, f, z, w, u, v }; break; diff --git a/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_weights_opt.cpp b/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_weights_opt.cpp index 9a9333c757b..6ff7daa3484 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_weights_opt.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/kernels/reorder/reorder_weights_opt.cpp @@ -20,6 +20,11 @@ ParamsKey ReorderWeightsOpt::GetSupportedKey() const { k.EnableOutputWeightsType(WeightsType::INT32); k.EnableInputWeightsLayout(WeightsLayout::oiyx); k.EnableInputWeightsLayout(WeightsLayout::ioyx); + k.EnableInputWeightsLayout(WeightsLayout::oyxi); + k.EnableInputWeightsLayout(WeightsLayout::oyix); + k.EnableInputWeightsLayout(WeightsLayout::oxiy); + k.EnableInputWeightsLayout(WeightsLayout::iyxo); + k.EnableInputWeightsLayout(WeightsLayout::yxio); k.EnableInputWeightsLayout(WeightsLayout::oizyx); k.EnableInputWeightsLayout(WeightsLayout::iozyx); k.EnableInputWeightsLayout(WeightsLayout::goiyx); diff --git a/src/plugins/intel_gpu/src/kernel_selector/tensor_type.cpp b/src/plugins/intel_gpu/src/kernel_selector/tensor_type.cpp index e0e57ede499..545f01c6f38 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/tensor_type.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/tensor_type.cpp @@ -23,6 +23,7 @@ DataTensor::DataChannelArray DataTensor::dataChannelArray {{ { DataLayout::byxf, { 1, 2, -1, -1, -1, -1, 0, 3 } }, { DataLayout::byfx, { 0, 2, -1, -1, -1, -1, 1, 3 } }, { DataLayout::bxfy, { 2, 0, -1, -1, -1, -1, 1, 3 } }, + { DataLayout::fbyx, { 0, 1, -1, -1, -1, -1, 3, 2 } }, { DataLayout::fyxb, { 1, 2, -1, -1, -1, -1, 3, 0 } }, { DataLayout::b_fs_yx_fsv2, { 0, 1, -1, -1, -1, -1, 2, 3 } }, { DataLayout::b_fs_yx_fsv4, { 0, 1, -1, -1, -1, -1, 2, 3 } }, diff --git a/src/plugins/intel_gpu/src/kernel_selector/tensor_type.h b/src/plugins/intel_gpu/src/kernel_selector/tensor_type.h index 8da7dec329c..85cec1fe17c 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/tensor_type.h +++ b/src/plugins/intel_gpu/src/kernel_selector/tensor_type.h @@ -32,6 +32,7 @@ enum DataLayout { yxfb, // 3D+batch byxf, // 3D+batch fyxb, // 3D+batch + fbyx, // 3D+batch bfxy, // 3D+batch byfx, bxfy, @@ -318,6 +319,7 @@ inline bool SimpleLayout(DataLayout l) { case DataLayout::byxf: case DataLayout::byfx: case DataLayout::bxfy: + case DataLayout::fbyx: case DataLayout::fyxb: case DataLayout::bfxy: case DataLayout::bfzyx: diff --git a/src/plugins/intel_gpu/src/plugin/transformations/convert_matmul_to_fc.cpp b/src/plugins/intel_gpu/src/plugin/transformations/convert_matmul_to_fc.cpp index 5d4531a5496..a30c88e7d14 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations/convert_matmul_to_fc.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations/convert_matmul_to_fc.cpp @@ -46,12 +46,8 @@ ConvertMatMulToFullyConnected::ConvertMatMulToFullyConnected() { auto fc_input_b = pattern_map.at(weights_m); bool is_convert = false; if (auto convert_node = std::dynamic_pointer_cast(fc_input_b.get_node_shared_ptr())) { - if (is_decompression(convert_node)) { - is_convert = true; - fc_input_b = convert_node->get_input_node_shared_ptr(0); - } else { - return false; - } + is_convert = true; + fc_input_b = convert_node->get_input_node_shared_ptr(0); } auto shape_a = fc_input_a.get_partial_shape(); diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index 6aff07cfdd4..14122656fc5 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -50,6 +50,7 @@ #include "transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp" #include "transformations/fp16_compression/convert_compression_only_to_legacy.hpp" +#include "transformations/fp16_compression/mark_decompression_convert_constant_folding.hpp" #include "transformations/common_optimizations/common_optimizations.hpp" #include "transformations/common_optimizations/lin_op_sequence_fusion.hpp" #include "transformations/common_optimizations/weights_dequantize_to_fake_quantize.hpp" @@ -58,7 +59,6 @@ #include "transformations/common_optimizations/transpose_sinking.hpp" #include "transformations/common_optimizations/softmax_fusion.hpp" #include "transformations/common_optimizations/mvn_fusion.hpp" -#include "transformations/common_optimizations/compress_float_constants.hpp" #include "transformations/op_conversions/convert_depth_to_space.hpp" #include "transformations/op_conversions/convert_space_to_depth.hpp" @@ -220,14 +220,17 @@ void TransformationsPipeline::apply(std::shared_ptr func) { // decompose MVNs that sre not supported in GPU, so that they will be marked as precision sensitive in ConvertPrecision manager.register_pass(); - auto is_matmul_output = [](const_node_ptr &node) -> bool { - const auto outputs = node->get_output_target_inputs(0); - return !is_type(outputs.begin()->get_node()); - }; + manager.register_pass(); + pass_config->set_callback( + [](const_node_ptr& node) -> bool { + auto next_node = node->get_output_target_inputs(0).begin()->get_node(); + if (is_type(next_node)) { + next_node = next_node->get_output_target_inputs(0).begin()->get_node(); + } + return !is_type(next_node); + }); - manager.register_pass(); manager.register_pass(ov::element::TypeVector{ov::element::u8}, true); - pass_config->set_callback(is_matmul_output); const bool keep_precision_sensitive_in_fp32_1 = true; const bool convert_input_output_precision = false; diff --git a/src/plugins/intel_gpu/src/runtime/format.cpp b/src/plugins/intel_gpu/src/runtime/format.cpp index a3c6f766bca..af268e94daa 100644 --- a/src/plugins/intel_gpu/src/runtime/format.cpp +++ b/src/plugins/intel_gpu/src/runtime/format.cpp @@ -25,6 +25,7 @@ static const std::map format_traits_map { FMT_TRAITS(yxfb, 1, 1, 2, 0, {2, 3, 1, 0}, "yxfb", "bfxy", {}), FMT_TRAITS(byxf, 1, 1, 2, 0, {0, 2, 3, 1}, "byxf", "bfxy", {}), FMT_TRAITS(bfyx, 1, 1, 2, 0, {0, 1, 2, 3}, "bfyx", "bfxy", {}), + FMT_TRAITS(fbyx, 1, 1, 2, 0, {1, 0, 2, 3}, "fbyx", "bfxy", {}), FMT_TRAITS(fyxb, 1, 1, 2, 0, {1, 2, 3, 0}, "fyxb", "bfxy", {}), FMT_TRAITS(byfx, 1, 1, 2, 0, {0, 2, 1, 3}, "byfx", "bfxy", {}), FMT_TRAITS(bxfy, 1, 1, 2, 0, {0, 3, 1, 2}, "bxfy", "bfxy", {}), @@ -45,7 +46,7 @@ static const std::map format_traits_map { FMT_TRAITS(bfuwzyx, 1, 1, 5, 0, {0, 1, 2, 3, 4, 5, 6}, "bfuwzyx", "bfxyzwu", {}), FMT_TRAITS(bfvuwzyx, 1, 1, 6, 0, {0, 1, 2, 3, 4, 5, 6, 7}, "bfvuwzyx", "bfxyzwuv", {}), FMT_TRAITS(fs_b_yx_fsv32, 1, 1, 2, 0, {1, 0, 2, 3}, "fbyx", "bfxy", {{1, 32}}), - FMT_TRAITS(b_fs_yx_32fp, 1, 1, 2, 0, {0, 1, 2, 3}, "bfyx", "bfxy", {}), + FMT_TRAITS(b_fs_yx_32fp, 1, 1, 2, 0, {0, 1, 2, 3}, "bfyx", "bfxy", {{1, 32}}), FMT_TRAITS(b_fs_zyx_fsv16, 1, 1, 3, 0, {0, 1, 2, 3, 4}, "bfzyx", "bfxyz", {{1, 16}}), FMT_TRAITS(bs_fs_zyx_bsv16_fsv32, 1, 1, 3, 0, {0, 1, 2, 3, 4}, "bfzyx", "bfxyz", {{0, 16 }, {1, 32}}), FMT_TRAITS(bs_fs_zyx_bsv16_fsv16, 1, 1, 3, 0, {0, 1, 2, 3, 4}, "bfzyx", "bfxyz", {{0, 16 }, {1, 16}}), @@ -93,9 +94,7 @@ static const std::map format_traits_map { FMT_TRAITS(image_2d_weights_c4_fyx_b, 1, 1, 2, 0, {0, 1, 2, 3}, "oiyx", "oixy", {}), FMT_TRAITS(image_2d_weights_c1_b_fyx, 1, 1, 2, 0, {0, 1, 2, 3}, "oiyx", "oixy", {}), FMT_TRAITS(lstm_weights_dio, 1, 1, 2, 0, {0, 1, 3, 2}, "oixy", "oixy", {}), - FMT_TRAITS(os_is_yx_isa8_osv8_isv4, 1, 1, 2, 0, {0, 1, 2, 3}, "oiyx", "oixy", {{1, 8}, {0, 8}, {1, 4}}), FMT_TRAITS(os_is_yx_isa8_osv16_isv4, 1, 1, 2, 0, {0, 1, 2, 3}, "oiyx", "oixy", {{1, 8}, {0, 16}, {1, 4}}), - FMT_TRAITS(os_is_yx_isa8_osv8_isv4_swizzled_by_4, 1, 1, 2, 0, {0, 1, 2, 3}, "oiyx", "oixy", {}), FMT_TRAITS(os_is_yx_osa4_isa8_osv8_isv2, 1, 1, 2, 0, {0, 1, 2, 3}, "oiyx", "oixy", {{0, 4}, {1, 8}, {0, 8}, {1, 2}}), FMT_TRAITS(os_is_yx_osa4_isa8_osv8_isv4, 1, 1, 2, 0, {0, 1, 2, 3}, "oiyx", "oixy", {{0, 4}, {1, 8}, {0, 8}, {1, 4}}), FMT_TRAITS(os_is_zyx_osa4_isa8_osv8_isv2, 1, 1, 3, 0, {0, 1, 2, 3, 4}, "oizyx", "oixyz", {{0, 4}, {1, 8}, {0, 8}, {1, 2}}), @@ -112,9 +111,9 @@ static const std::map format_traits_map { FMT_TRAITS(is_os_yx_isa2_osa8_isv8_osv2, 1, 1, 2, 0, {1, 0, 2, 3}, "ioyx", "oixy", {{1, 2}, {0, 8}, {1, 8}, {0, 2}}), FMT_TRAITS(is_os_yx_isa4_osa8_isv8_osv4, 1, 1, 2, 0, {1, 0, 2, 3}, "ioyx", "oixy", {{1, 4}, {0, 8}, {1, 8}, {0, 4}}), FMT_TRAITS(is_o_yx_isv32, 1, 1, 2, 0, {1, 0, 2, 3}, "oyxi", "oixy", {{1, 32}}), - FMT_TRAITS(is_o32_yx_isv32_swizzled_by_4, 1, 1, 2, 0, {0, 1, 2, 3}, "oyxi", "oixy", {}), - FMT_TRAITS(os_is_y_x8_osv8_isv4, 1, 1, 2, 0, {0, 1, 2, 3}, "oyxi", "oixy", {}), - FMT_TRAITS(os_is_y_x8_osv8_isv4_swizzled_by_4, 1, 1, 2, 0, {0, 1, 2, 3}, "oyxi", "oixy", {}), + FMT_TRAITS(is_o32_yx_isv32_swizzled_by_4, 1, 1, 2, 0, {0, 1, 2, 3}, "oyxi", "oixy", {{0, 32}, {1, 32}}), + FMT_TRAITS(os_is_y_x8_osv8_isv4, 1, 1, 2, 0, {0, 1, 2, 3}, "oyxi", "oixy", {{0, 8}, {1, 4}, {2, 8}}), + FMT_TRAITS(os_is_y_x8_osv8_isv4_swizzled_by_4, 1, 1, 2, 0, {0, 1, 2, 3}, "oyxi", "oixy", {{0, 8}, {1, 4}, {2, 8}}), FMT_TRAITS(os_is_yx_osv16_isv4, 1, 1, 2, 0, {0, 1, 2, 3}, "oiyx", "oixy", {{0, 16}, {1, 4}}), FMT_TRAITS(os_is_yx_osv8_isv4, 1, 1, 2, 0, {0, 1, 2, 3}, "oiyx", "oixy", {{0, 8}, {1, 4}}), FMT_TRAITS(os_is_zyx_osv8_isv4, 1, 1, 3, 0, {0, 1, 2, 3, 4}, "oizyx", "oixyz", {{0, 8}, {1, 4}}), @@ -124,7 +123,7 @@ static const std::map format_traits_map { FMT_TRAITS(os_is_yx_osv32_isv4_swizzled_by_2, 1, 1, 2, 0, {0, 1, 2, 3}, "oiyx", "oixy", {{0, 32}, {1, 4}}), FMT_TRAITS(os_is_yx_osv32_isv4, 1, 1, 2, 0, {0, 1, 2, 3}, "oiyx", "oixy", {{0, 32}, {1, 4}}), FMT_TRAITS(os_is_zyx_osv32_isv4, 1, 1, 3, 0, {0, 1, 2, 3, 4}, "oizyx", "oixyz", {{0, 32}, {1, 4}}), - FMT_TRAITS(os_is_yx_osv32_isv32p, 1, 1, 1, 0, {0, 1, 2, 3}, "oiyx", "oixy", {}), + FMT_TRAITS(os_is_yx_osv32_isv32p, 1, 1, 1, 0, {0, 1, 2, 3}, "oiyx", "oixy", {{0, 32}, {1, 32}}), FMT_TRAITS(os_is_zyx_isv16_osv16, 1, 1, 3, 0, {0, 1, 2, 3, 4}, "oizyx", "oixyz", {{1, 16}, {0, 16}}), FMT_TRAITS(is_os_zyx_isv16_osv16, 1, 1, 3, 0, {1, 0, 2, 3, 4}, "iozyx", "oixyz", {{1, 16}, {0, 16}}), FMT_TRAITS(is_os_yx_isv16_osv16, 1, 1, 2, 0, {1, 0, 2, 3}, "ioyx", "oixy", {{1, 16}, {0, 16}}), @@ -140,6 +139,7 @@ static const std::map format_traits_map { FMT_TRAITS(is_os_yx_isa8_osv8_isv4, 1, 1, 2, 0, {1, 0, 2, 3}, "ioyx", "oixy", {{1, 8}, {0, 8}, {1, 4}}), FMT_TRAITS(is_os_yx_osa8_isv16_osv4, 1, 1, 2, 0, {1, 0, 2, 3}, "ioyx", "oixy", {{0, 8}, {1, 16}, {0, 4}}), FMT_TRAITS(os_is_yx_isa8_osv8_isv4, 1, 1, 2, 0, {0, 1, 2, 3}, "oiyx", "oixy", {{1, 8}, {0, 8}, {1, 4}}), + FMT_TRAITS(os_is_yx_isa8_osv8_isv4_swizzled_by_4, 1, 1, 2, 0, {0, 1, 2, 3}, "oiyx", "oixy", {{1, 32}, {0, 32}}), FMT_TRAITS(os_is_yx_isa8_osv8_isv2, 1, 1, 2, 0, {0, 1, 2, 3}, "oiyx", "oixy", {{1, 8}, {0, 8}, {1, 2}}), FMT_TRAITS(os_is_osv32_isv32_swizzled_by_4, 1, 1, 0, 0, {0, 1, 2, 3}, "oiyx", "oixy", {{0, 32}, {1, 32}}), FMT_TRAITS(os_is_zyx_isv8_osv16_isv2, 1, 1, 3, 0, {0, 1, 2, 3, 4}, "oizyx", "oixyz", {{1, 8}, {0, 16}, {1, 2}}), @@ -344,4 +344,34 @@ const std::vector> format::per_axis_block_size(format fmt return sizes_for_dims; } + +format format::find_format(const std::vector& order, + const std::vector>& block_sizes, + bool is_weights, + bool is_grouped, + bool is_image_2d, + bool is_winograd, + bool is_nv12) { + auto is_suitable_traits = [&](const std::pair& traits) -> bool { + return traits.second._order == order && + traits.second.block_sizes == block_sizes && + format::is_weights_format(traits.first) == is_weights && + format::is_grouped(traits.first) == is_grouped && + format::is_image_2d(traits.first) == is_image_2d && + format::is_winograd(traits.first) == is_winograd && + format::is_nv12(traits.first) == is_nv12; + }; + + std::vector finded_formats; + for (auto& traits : format_traits_map) { + if (is_suitable_traits(traits)) + finded_formats.emplace_back(traits.first); + } + + OPENVINO_ASSERT(!finded_formats.empty(), "[GPU] Cannot find a format with the specified parameters"); + OPENVINO_ASSERT(finded_formats.size() == 1, "[GPU] Cannot find a format. Specified parameters are ambiguous"); + + return finded_formats.front(); +} + } // namespace cldnn diff --git a/src/plugins/intel_gpu/src/runtime/layout.cpp b/src/plugins/intel_gpu/src/runtime/layout.cpp index e00aa0ff16c..0ca5e60de61 100644 --- a/src/plugins/intel_gpu/src/runtime/layout.cpp +++ b/src/plugins/intel_gpu/src/runtime/layout.cpp @@ -138,6 +138,8 @@ static format to_weights_format(format f, bool is_grouped) { switch (f) { case format::bfyx: return format::oiyx; + case format::fbyx: + return format::ioyx; case format::fyxb: return format::iyxo; case format::byxf: diff --git a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp index 01f7f43749b..5eaa69d4e62 100644 --- a/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp +++ b/src/plugins/intel_gpu/tests/functional/subgraph_tests/dynamic/matmul_weights_decompression.cpp @@ -194,11 +194,9 @@ const std::vector activations_precisions = {ov::element:: const std::vector weights_precisions = {ov::element::u8}; const std::vector> input_shapes_basic = { {{{-1, -1, -1}, {{1, 4, 16}, {10, 16, 16}}}, {{}, {{16, 32}}}}, - {{{}, {{1, 4, 16}}}, {{}, {{1, 16, 32}}}}, {{{}, {{10, 40, 496}}}, {{}, {{1, 496, 240}}}}, {{{}, {{1, 4, 48}}}, {{}, {{48, 256}}}}, {{{}, {{11, 339, 377}}}, {{}, {{377, 335}}}}, - {{{-1, -1, -1}, {{10, 40, 480}, {11, 40, 480}}}, {{}, {{1, 480, 256}}}}, {{{}, {{1, 4, 32}}}, {{}, {{32, 256}}}}, {{{}, {{1, 4, 512}}}, {{}, {{512, 256}}}}, {{{}, {{1, 16, 32}}}, {{}, {{32, 64}}}}, @@ -220,7 +218,10 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeights_basic, const std::vector> input_shapes_corner_cases_basic = { {{{-1, -1, -1}, {{1, 4, 16}}}, {{}, {{1, 16, 32}}}}, + {{{}, {{1, 4, 16}}}, {{}, {{1, 16, 32}}}}, {{{-1, -1, -1}, {{1, 4, 16}}}, {{}, {{16, 32}}}}, + {{{-1, -1, -1, -1}, {{1, 1, 4, 16}}}, {{}, {{1, 1, 16, 32}}}}, + {{{}, {{1, 1, 4, 16}}}, {{}, {{1, 1, 16, 32}}}}, }; const std::vector> input_shapes_corner_cases_big = { {{{-1, -1, -1}, {{10, 40, 480}, {11, 40, 480}}}, {{}, {{1, 480, 256}}}}, diff --git a/src/plugins/intel_gpu/tests/unit/module_tests/format_test.cpp b/src/plugins/intel_gpu/tests/unit/module_tests/format_test.cpp index 2fbdc2fb3c2..98169c608f2 100644 --- a/src/plugins/intel_gpu/tests/unit/module_tests/format_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/module_tests/format_test.cpp @@ -148,3 +148,90 @@ INSTANTIATE_TEST_SUITE_P(smoke, axes_test_format, {format::gs_oizyx_gsv16, {{8, 16}}, {{8, 16}}}, }), axes_test_format::PrintToString); + +struct find_format_test_params { + std::vector dims_order; + std::vector> block_sizes; + bool is_weights; + bool is_grouped; + bool is_image_2d; + bool is_winograd; + bool is_nv12; + format expected_format; +}; + +class find_format_test : public testing::TestWithParam { +public: + static std::string PrintToString(testing::TestParamInfo param_info) { + auto& p = param_info.param; + + std::string res = "order = ["; + for (size_t i = 0; i < p.dims_order.size() - 1; ++i) { + res += std::to_string(p.dims_order[i]) + ", "; + } + res += std::to_string(p.dims_order.back()) + "], "; + + res += "block_sizes = ["; + for (auto block : p.block_sizes) { + res += "{ " + std::to_string(block.first) + ", " + std::to_string(block.second) + "}"; + } + res += "], "; + + res += "is_weights = " + std::to_string(p.is_weights) + ", " + + "is_grouped = " + std::to_string(p.is_grouped) + ", " + + "is_image_2d = " + std::to_string(p.is_image_2d) + ", " + + "is_winograd = " + std::to_string(p.is_winograd) + ", " + + "is_nv12 = " + std::to_string(p.is_nv12) + ", " + + "expected_format = " + param_info.param.expected_format.to_string(); + + return res; + } + }; + +TEST_P(find_format_test, simple_test) { + auto p = GetParam(); + + if (p.expected_format == format::any) { + ASSERT_ANY_THROW(format::find_format(p.dims_order, p.block_sizes, + p.is_weights, p.is_grouped, p.is_image_2d, p.is_winograd, p.is_nv12)); + } else { + ASSERT_EQ(format::find_format(p.dims_order, p.block_sizes, + p.is_weights, p.is_grouped, p.is_image_2d, p.is_winograd, p.is_nv12), p.expected_format); + } +} + +INSTANTIATE_TEST_SUITE_P(smoke, find_format_test, + testing::ValuesIn(std::vector{ + // Dims order Block sizes is_weights is_grouped is_image_2d is_winograd is_nv12 Expected format + {{0, 1, 2, 3}, {}, false, false, false, false, false, format::bfyx}, + {{1, 0, 2, 3}, {}, false, false, false, false, false, format::fbyx}, + {{2, 3, 1, 0}, {}, false, false, false, false, false, format::yxfb}, + {{0, 2, 3, 1}, {}, false, false, false, false, false, format::byxf}, + {{1, 2, 3, 0}, {}, false, false, false, false, false, format::fyxb}, + {{0, 2, 1, 3}, {}, false, false, false, false, false, format::byfx}, + {{0, 3, 1, 2}, {}, false, false, false, false, false, format::bxfy}, + {{0, 1, 2, 3}, {{1, 16}}, false, false, false, false, false, format::b_fs_yx_fsv16}, + {{0, 1}, {{0, 8}, {1, 8}}, false, false, false, false, false, format::bs_fs_fsv8_bsv8}, + {{0, 1, 2, 3, 4, 5, 6}, {}, false, false, false, false, false, format::bfuwzyx}, + {{0, 1, 2, 3}, {}, false, false, true, false, true, format::nv12}, + {{0, 1, 2, 3}, {}, false, false, true, false, false, format::image_2d_rgba}, + {{0, 1, 2, 3}, {}, true, false, false, false, false, format::oiyx}, + {{1, 0, 2, 3}, {}, true, false, false, false, false, format::ioyx}, + {{1, 2, 3, 0}, {}, true, false, false, false, false, format::iyxo}, + {{0, 2, 3, 1}, {}, true, false, false, false, false, format::oyxi}, + {{0, 2, 1, 3}, {}, true, false, false, false, false, format::oyix}, + {{0, 3, 1, 2}, {}, true, false, false, false, false, format::oxiy}, + {{2, 3, 1, 0}, {}, true, false, false, false, false, format::yxio}, + {{0, 1, 2, 3}, {{0, 16}}, true, false, false, false, false, format::os_iyx_osv16}, + {{0, 1, 2, 3}, {}, true, false, false, true, false, format::winograd_2x3_s1_weights}, + {{0, 1, 3, 2}, {}, true, false, false, false, false, format::lstm_weights_dio}, + {{0, 1, 2, 3}, {{1, 8}, {0, 8}, {1, 4}}, true, false, false, false, false, format::os_is_yx_isa8_osv8_isv4}, + {{0, 1, 2, 3, 4}, {}, true, true, false, false, false, format::goiyx}, + {{0, 2, 1, 3, 4}, {{1, 16}, {0, 16}}, true, true, false, false, false, format::g_is_os_yx_isv16_osv16}, + + // Expected error throw + {{0, 0, 2, 3}, {}, false, false, false, false, false, format::any}, + {{0, 1, 2, 3}, {{1, 1}}, false, false, false, false, false, format::any}, + {{0, 1, 2, 3}, {}, false, true, false, false, false, format::any} + }), + find_format_test::PrintToString); diff --git a/src/plugins/intel_gpu/tests/unit/passes/prepare_primitive_fusing_test.cpp b/src/plugins/intel_gpu/tests/unit/passes/prepare_primitive_fusing_test.cpp index 448dce28a28..8f2c2cab502 100644 --- a/src/plugins/intel_gpu/tests/unit/passes/prepare_primitive_fusing_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/passes/prepare_primitive_fusing_test.cpp @@ -517,3 +517,79 @@ TEST(prepare_primitive_fusing, eltwise_fusing_residual_connection) { net.execute(); ASSERT_TRUE(conv_inst->has_unfused_subgraph()); } + +TEST(prepare_primitive_fusing, fuse_constant_transposes_removal_check) { + auto& engine = get_test_engine(); + + auto input = engine.allocate_memory({ { 2, 32 }, data_types::f16, format::bfyx }); + auto weights = engine.allocate_memory({{ 32, 2 }, data_types::f32, format::bfyx }); + + topology topology( + input_layout("input", input->get_layout()), + data("weights", weights), + permute("permute", input_info("weights"), {1, 0}), + reorder("reorder_dt", input_info("permute"), format::bfyx, data_types::f16), + fully_connected("fc", input_info("input"), { "reorder_dt" }, "", data_types::f16) + ); + + ExecutionConfig config = get_test_default_config(engine); + config.set_property(ov::intel_gpu::optimize_data(true)); + config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + + auto prog = program::build_program(engine, topology, config, false, true); + + layout_optimizer lo(true); + program_wrapper::apply_opt_pass(*prog, lo); + + ASSERT_TRUE(!has_node(*prog, "permute")); + ASSERT_EQ(prog->get_node("weights").get_output_layout().format, format::fbyx); +} + +TEST(prepare_primitive_fusing, fuse_constant_transposes_accuracy_test) { + auto& engine = get_test_engine(); + + auto input = engine.allocate_memory({ { 2, 32 }, data_types::f16, format::bfyx }); + auto weights = engine.allocate_memory({{ 32, 2 }, data_types::f32, format::bfyx }); + + tests::random_generator rg(GET_SUITE_NAME); + auto input_data = rg.generate_random_2d(2, 32, -1, 1); + auto weights_data = rg.generate_random_2d(32, 2, -1, 1); + + set_values(input, flatten_2d(format::bfyx, input_data)); + set_values(weights, flatten_2d(format::bfyx, weights_data)); + + topology topology( + input_layout("input", input->get_layout()), + data("weights", weights), + reorder("reorder_dt", input_info("weights"), format::bfyx, data_types::f16, + std::vector(), reorder_mean_mode::subtract, padding(), true), + permute("permute", input_info("reorder_dt"), {1, 0}), + fully_connected("fc", input_info("input"), { "permute" }, "", data_types::f16) + ); + + ExecutionConfig config = get_test_default_config(engine); + config.set_property(ov::intel_gpu::optimize_data(true)); + config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + + cldnn::network network(engine, topology, config); + network.set_input_data("input", input); + + auto outputs = network.execute(); + auto output = outputs.at("fc").get_memory(); + cldnn::mem_lock output_ptr(output, get_test_stream()); + + ExecutionConfig config_ref = get_test_default_config(engine); + config_ref.set_property(ov::intel_gpu::optimize_data(false)); + config_ref.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + + cldnn::network network_ref(engine, topology, config_ref); + network_ref.set_input_data("input", input); + + auto outputs_ref = network_ref.execute(); + auto output_ref = outputs_ref.at("fc").get_memory(); + cldnn::mem_lock output_ptr_ref(output_ref, get_test_stream()); + + for (size_t i = 0; i < output_ptr_ref.size(); ++i) { + ASSERT_EQ(output_ptr[i], output_ptr_ref[i]); + } +} diff --git a/src/plugins/intel_gpu/tests/unit/passes/reorder_transfer.cpp b/src/plugins/intel_gpu/tests/unit/passes/reorder_transfer.cpp new file mode 100644 index 00000000000..7731da37ef1 --- /dev/null +++ b/src/plugins/intel_gpu/tests/unit/passes/reorder_transfer.cpp @@ -0,0 +1,48 @@ +// Copyright (C) 2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "test_utils.h" +#include "program_wrapper.h" +#include "fully_connected_inst.h" +#include "permute_inst.h" + +using namespace cldnn; +using namespace ::tests; + +TEST(reorder_transfer, transfer_per_permute) { + auto& engine = get_test_engine(); + + auto input = engine.allocate_memory({ { 2, 32 }, data_types::f16, format::bfyx }); + auto weights = engine.allocate_memory({{ 32, 2 }, data_types::f32, format::bfyx }); + + topology topology( + input_layout("input", input->get_layout()), + data("weights", weights), + reorder("reorder_dt", input_info("weights"), format::bfyx, data_types::f16, + std::vector(), reorder_mean_mode::subtract, padding(), true), + permute("permute", input_info("reorder_dt"), {1, 0}), + fully_connected("fc", input_info("input"), { "permute" }, "", data_types::f16) + ); + + ExecutionConfig config = get_test_default_config(engine); + config.set_property(ov::intel_gpu::optimize_data(true)); + config.set_property(ov::intel_gpu::allow_new_shape_infer(true)); + auto prog = program::build_program(engine, topology, config, false, true); + + for (auto& node : prog->get_processing_order()) { + if (!node->is_type()) + node->get_output_layouts(); + } + + program_wrapper::apply_opt_pass(*prog); + auto& processing_order = prog->get_processing_order(); + + auto reorder_node = std::find(processing_order.begin(), processing_order.end(), &prog->get_node("reorder_dt")); + size_t reorder_dist = std::distance(processing_order.begin(), reorder_node); + + auto permute_node = std::find(processing_order.begin(), processing_order.end(), &prog->get_node("permute")); + size_t permute_dist = std::distance(processing_order.begin(), permute_node); + + ASSERT_TRUE(reorder_dist > permute_dist); +} diff --git a/src/plugins/intel_gpu/tests/unit/test_cases/hash_key_gpu_test.cpp b/src/plugins/intel_gpu/tests/unit/test_cases/hash_key_gpu_test.cpp index a40469eb8a8..367277fa0e0 100644 --- a/src/plugins/intel_gpu/tests/unit/test_cases/hash_key_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/unit/test_cases/hash_key_gpu_test.cpp @@ -176,7 +176,7 @@ public: const auto params_hash = prim_inst->get_impl_params()->hash(); ASSERT_EQ(primitive_hash, 16293979194373117693UL); - ASSERT_EQ(params_hash, 4771142562684430881UL); + ASSERT_EQ(params_hash, 550629972043680951UL); } void test_reshape_basic(bool is_caching_test) { diff --git a/src/plugins/intel_gpu/tests/unit/test_utils/test_utils.h b/src/plugins/intel_gpu/tests/unit/test_utils/test_utils.h index 5266e27dc93..dcc1e256c7e 100644 --- a/src/plugins/intel_gpu/tests/unit/test_utils/test_utils.h +++ b/src/plugins/intel_gpu/tests/unit/test_utils/test_utils.h @@ -118,6 +118,27 @@ using VVVVVF = std::vector>; // split of bfyx filters template using VVVVVVF = std::vector>; // split of bfyx filters +template +inline VF flatten_2d(cldnn::format input_format, VVF &data) { + size_t a = data.size(); + size_t b = data[0].size(); + + VF vec(a * b, (T)(0.0f)); + size_t idx = 0; + + switch (input_format.value) { + case cldnn::format::bfyx: + for (size_t bi = 0; bi < a; ++bi) + for (size_t fi = 0; fi < b; ++fi) + vec[idx++] = data[bi][fi]; + break; + + default: + assert(0); + } + return vec; +} + template inline VF flatten_4d(cldnn::format input_format, VVVVF &data) { size_t a = data.size();