From aea04e275cdf1f90701f6912b6c0abbd5d8ef598 Mon Sep 17 00:00:00 2001 From: "Min, Byungil" Date: Fri, 3 Jun 2022 18:36:27 +0900 Subject: [PATCH] [GPU] Enable onednn reduction (#11570) * It has better performance by using reduction kernel instead of pooling kernel in oneDNN for reduction layer. * Stop using global pooling instead of reduce primitive * Use oneDNN reduction if its mode is supported by optimized onedNN kernel * activation pow is supported * Use clDNN reduce if 3d or redundant reduce, tensor size mismatch * Updated thirdparty onednn_gpu Signed-off-by: Min, Byungil Co-authored-by: Wei Tang Co-authored-by: Chen Kurt --- .../add_onednn_optimization_attributes.cpp | 1 + .../prepare_primitive_fusing.cpp | 14 ++- .../graph/impls/onednn/reduction_onednn.cpp | 114 ++++++++++++++++++ .../src/graph/impls/onednn/register.cpp | 1 + .../src/graph/impls/onednn/register.hpp | 1 + .../src/graph/impls/onednn/utils.cpp | 3 + .../intel_gpu/src/graph/layout_optimizer.cpp | 46 +++++++ .../intel_gpu/src/graph/program_node.cpp | 10 +- .../src/plugin/transformations_pipeline.cpp | 12 +- src/plugins/intel_gpu/thirdparty/onednn_gpu | 2 +- 10 files changed, 197 insertions(+), 7 deletions(-) create mode 100644 src/plugins/intel_gpu/src/graph/impls/onednn/reduction_onednn.cpp diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/add_onednn_optimization_attributes.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/add_onednn_optimization_attributes.cpp index b6a7a9de7ac..1dfc07822f1 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/add_onednn_optimization_attributes.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/add_onednn_optimization_attributes.cpp @@ -9,6 +9,7 @@ #ifdef ENABLE_ONEDNN_FOR_GPU #include "fully_connected_inst.h" +#include "reduce_inst.h" #include #endif diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp index 3b0552c9611..f4fbbbdadb4 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp @@ -612,8 +612,20 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { return true; }; - auto reduce_supports_fusings = [](reduce_node& node) -> bool { + auto reduce_supports_fusings = [&](reduce_node& node) -> bool { auto keep_dims = node.as().get_primitive()->keep_dims; + auto axes = node.as().get_primitive()->axes; + + // If reduce tensor size is small, it sets not to fuse eltwise which leads to select oneDNN reference reduction + // Because oneDNN optimized kernel does NOT support eltwise fusing + if (p.get_engine().get_device_info().supports_immad && node.get_output_layout().get_dims().size() <= 4 && + ((find(axes.begin(), axes.end(), reduce::along_x) != axes.end() && + node.input().get_output_layout().spatial(0) > 16) || + (find(axes.begin(), axes.end(), reduce::along_y) != axes.end() && + node.input().get_output_layout().spatial(1) > 16) || + (find(axes.begin(), axes.end(), reduce::along_f) != axes.end() && + node.input().get_output_layout().feature() > 16))) + return false; if (keep_dims) return true; diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/reduction_onednn.cpp b/src/plugins/intel_gpu/src/graph/impls/onednn/reduction_onednn.cpp new file mode 100644 index 00000000000..77694356773 --- /dev/null +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/reduction_onednn.cpp @@ -0,0 +1,114 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "reduce_inst.h" +#include "primitive_onednn_base.h" +#include "impls/implementation_map.hpp" + +#include "kernel_selector_common.h" +#include "kernel_base.h" + +#include + +#include +#include +namespace cldnn { +namespace onednn { +struct reduction_onednn : typed_primitive_onednn_impl { + using parent = typed_primitive_onednn_impl; + using parent::parent; + +protected: + std::unique_ptr clone() const override { + return make_unique(*this); + } + + static std::shared_ptr get_reduction_descriptor(const reduce_node& arg) { + auto prim = arg.get_primitive(); + auto& input = arg.get_dependency(0); + auto input_md = onednn::layout_to_memory_desc(input.get_output_layout()); + auto output_md = onednn::layout_to_memory_desc(arg.get_output_layout()); + + float p = 0.f; + float eps = 0.f; + dnnl::algorithm alg; + switch (prim->mode) { + case reduce_mode::mean: alg = dnnl::algorithm::reduction_mean; break; + case reduce_mode::max: alg = dnnl::algorithm::reduction_max; break; + case reduce_mode::min: alg = dnnl::algorithm::reduction_min; break; + case reduce_mode::sum: alg = dnnl::algorithm::reduction_sum; break; + case reduce_mode::prod: alg = dnnl::algorithm::reduction_mul; break; + case reduce_mode::sum_square: + alg = dnnl::algorithm::reduction_norm_lp_power_p_sum; + p = 2.0f; + break; + case reduce_mode::l1: + alg = dnnl::algorithm::reduction_norm_lp_sum; + p = 1.0f; + break; + case reduce_mode::l2: + alg = dnnl::algorithm::reduction_norm_lp_sum; + p = 2.0f; + break; + default: throw std::runtime_error("unsupported reduce mode"); + } + + return std::make_shared( + alg, + input_md, + output_md, + p, + eps); + } + +public: + static primitive_impl* create(const reduce_node& arg) { + auto& engine = arg.get_program().get_engine(); + auto desc = get_reduction_descriptor(arg); + auto attr = arg.get_onednn_primitive_attributes(); + dnnl::primitive_desc prim_desc{&desc->data, attr.get(), engine.get_onednn_engine(), nullptr}; + + return new reduction_onednn(arg, desc, attr, prim_desc); + } +}; + +namespace detail { + +attach_reduction_onednn::attach_reduction_onednn() { + implementation_map::add(impl_types::onednn, reduction_onednn::create, { + std::make_tuple(data_types::f32, format::bfyx), + std::make_tuple(data_types::f16, format::bfyx), + std::make_tuple(data_types::u8, format::bfyx), + std::make_tuple(data_types::i8, format::bfyx), + + std::make_tuple(data_types::f32, format::b_fs_yx_fsv16), + std::make_tuple(data_types::f16, format::b_fs_yx_fsv16), + std::make_tuple(data_types::u8, format::b_fs_yx_fsv16), + std::make_tuple(data_types::i8, format::b_fs_yx_fsv16), + + std::make_tuple(data_types::f32, format::b_fs_yx_fsv32), + std::make_tuple(data_types::f16, format::b_fs_yx_fsv32), + std::make_tuple(data_types::u8, format::b_fs_yx_fsv32), + std::make_tuple(data_types::i8, format::b_fs_yx_fsv32), + + std::make_tuple(data_types::f32, format::bs_fs_yx_bsv16_fsv16), + std::make_tuple(data_types::f16, format::bs_fs_yx_bsv16_fsv16), + std::make_tuple(data_types::u8, format::bs_fs_yx_bsv16_fsv16), + std::make_tuple(data_types::i8, format::bs_fs_yx_bsv16_fsv16), + + std::make_tuple(data_types::f32, format::bs_fs_yx_bsv32_fsv16), + std::make_tuple(data_types::f16, format::bs_fs_yx_bsv32_fsv16), + std::make_tuple(data_types::u8, format::bs_fs_yx_bsv32_fsv16), + std::make_tuple(data_types::i8, format::bs_fs_yx_bsv32_fsv16), + + std::make_tuple(data_types::f32, format::bs_fs_yx_bsv32_fsv32), + std::make_tuple(data_types::f16, format::bs_fs_yx_bsv32_fsv32), + std::make_tuple(data_types::u8, format::bs_fs_yx_bsv32_fsv32), + std::make_tuple(data_types::i8, format::bs_fs_yx_bsv32_fsv32), + }); +} + +} // namespace detail +} // namespace onednn +} // namespace cldnn diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/register.cpp b/src/plugins/intel_gpu/src/graph/impls/onednn/register.cpp index bd33b447234..a1b3a35d3af 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/register.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/register.cpp @@ -17,6 +17,7 @@ void register_implementations() { REGISTER_ONEDNN_IMPL(eltwise); REGISTER_ONEDNN_IMPL(gemm); REGISTER_ONEDNN_IMPL(pooling); + REGISTER_ONEDNN_IMPL(reduction); REGISTER_ONEDNN_IMPL(reorder); REGISTER_ONEDNN_IMPL(fully_connected);} diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/register.hpp b/src/plugins/intel_gpu/src/graph/impls/onednn/register.hpp index d24d628ba78..34dbf01c8f9 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/register.hpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/register.hpp @@ -23,6 +23,7 @@ REGISTER_ONEDNN_IMPL(concatenation); REGISTER_ONEDNN_IMPL(eltwise); REGISTER_ONEDNN_IMPL(gemm); REGISTER_ONEDNN_IMPL(pooling); +REGISTER_ONEDNN_IMPL(reduction); REGISTER_ONEDNN_IMPL(reorder); REGISTER_ONEDNN_IMPL(fully_connected); diff --git a/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp b/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp index 3eb4458a0d8..127fe496ec6 100644 --- a/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp @@ -471,6 +471,7 @@ cldnn::format find_format(dnnl::memory::desc desc, bool is_grouped) { } } +// Currently, usage of alpha and beta between cldnn::pow and dnnl::eltwise::pow is different : d = pow(src, a) / d = a * pow(src, b) dnnl::algorithm convert_activation_func(cldnn::activation_func func) { switch (func) { case cldnn::activation_func::relu: return dnnl::algorithm::eltwise_relu; @@ -485,6 +486,8 @@ dnnl::algorithm convert_activation_func(cldnn::activation_func func) { case cldnn::activation_func::logistic: return dnnl::algorithm::eltwise_logistic; case cldnn::activation_func::clamp: return dnnl::algorithm::eltwise_clip; case cldnn::activation_func::hyperbolic_tan: return dnnl::algorithm::eltwise_tanh; + case cldnn::activation_func::pow: return dnnl::algorithm::eltwise_pow; + case cldnn::activation_func::sqrt: return dnnl::algorithm::eltwise_sqrt; default: throw std::runtime_error("Unsupported activation func for onednn primitive " + std::to_string(static_cast(func))); } } diff --git a/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp b/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp index adc50890a64..577801c7d1a 100644 --- a/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp +++ b/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp @@ -16,6 +16,7 @@ #include "gemm_inst.h" #include "eltwise_inst.h" #include "pooling_inst.h" +#include "reduce_inst.h" #include "one_hot_inst.h" #include "permute_inst.h" #include "gemm_inst.h" @@ -759,6 +760,43 @@ bool layout_optimizer::deconvolution_b_fs_yx_fsv16_opt(layout const &input_layou return false; } +static bool is_node_for_onednn(reduce_node const& node) { + auto reduce_prim = node.get_primitive(); + // oneDNN reduction currently does not support logical_and, logical_or, log_sum and log_sum_exp. + switch (reduce_prim->mode) { + case reduce_mode::mean: + case reduce_mode::max: + case reduce_mode::min: + case reduce_mode::sum: + case reduce_mode::prod: + case reduce_mode::sum_square: + case reduce_mode::l1: + case reduce_mode::l2: + break; + default: + return false; + } + + auto& input = node.input(); + auto input_layout = input.get_output_layout(); + auto reduce_axes = reduce_prim->axes; + // redundant reduce is not acceptable on oneDNN reduction + if (node.get_output_layout() == input_layout) { + return false; + } + // tensor size mismatch of unreduced axis is not supported + for (size_t idx = 0 ; idx < input_layout.get_dims().size() ; idx++) { + bool reduced_axis = std::find(reduce_axes.begin(), reduce_axes.end(), idx) != reduce_axes.end(); + if (!reduced_axis && (input_layout.get_dim(idx) != node.get_output_layout().get_dim(idx))) + return false; + } + if (input_layout.get_dims().size() > 4) { + return false; + } + + return true; +} + static bool is_node_for_onednn(deconvolution_node const& node) { auto prim = node.get_primitive(); auto input_layout = node.get_dependency(0).get_output_layout(); @@ -1376,6 +1414,14 @@ impl_types layout_optimizer::get_preferred_impl_type(program_node& node, format preferred_impl = impl_types::ocl; if (output_fmt == format::bfyx && output_dt == data_types::f32) preferred_impl = impl_types::ocl; + } else if (node.is_type()) { + if (!_optimization_attributes.use_onednn_impls) + return impl_types::ocl; + + if (is_node_for_onednn(node.as())) + return impl_types::onednn; + else + return impl_types::ocl; } else if (node.is_type() || node.is_type() || node.is_type()) { if (!_optimization_attributes.use_onednn_impls) return impl_types::ocl; diff --git a/src/plugins/intel_gpu/src/graph/program_node.cpp b/src/plugins/intel_gpu/src/graph/program_node.cpp index b2bc0b873f9..ba9ccfae4c1 100644 --- a/src/plugins/intel_gpu/src/graph/program_node.cpp +++ b/src/plugins/intel_gpu/src/graph/program_node.cpp @@ -12,6 +12,7 @@ #include "quantize_inst.h" #include "reorder_inst.h" #include "pooling_inst.h" +#include "reduce_inst.h" #include #endif // ENABLE_ONEDNN_FOR_GPU @@ -888,7 +889,12 @@ void program_node::init_onednn_primitive_attributes() { update_onednn_post_op_list(onednn_post_op_type::eltwise_clip, empty_mem); } else { dnnl::algorithm alg = onednn::convert_activation_func(fused_desc->activation_function); - post_ops.append_eltwise(1.0f, alg, fused_desc->additional_params.a, fused_desc->additional_params.b); + // Usage of alpha and beta between cldnn::pow and dnnl::eltwise::pow is different : d = pow(src, a) / d = a * pow(src, b) + if (alg == dnnl::algorithm::eltwise_pow) + post_ops.append_eltwise(1.0f, alg, 1.0f, fused_desc->additional_params.a); + else + post_ops.append_eltwise(1.0f, alg, fused_desc->additional_params.a, fused_desc->additional_params.b); + update_onednn_post_op_list(onednn_post_op_type::eltwise_act, empty_mem); } } else if (node->is_type()) { @@ -914,7 +920,7 @@ void program_node::init_onednn_primitive_attributes() { } else { if (in.spatial(0) > 1 || in.spatial(1) > 1 || in.batch() > 1) throw std::runtime_error("Unsupported eltwise mode for fused onednn op"); - if (idx == 0 && !has_out_scales(attrs) && !is_type()) { + if (idx == 0 && !has_out_scales(attrs) && !is_type() && !is_type()) { int mask = in.count() > 1 ? 2 : 0; attrs->set_output_scales(mask, {DNNL_RUNTIME_F32_VAL}); update_onednn_post_op_list(onednn_post_op_type::scale, dep_idx); diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index d4cf3f6bfd3..ac7a2ef247a 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -190,20 +190,26 @@ void TransformationsPipeline::apply(std::shared_ptr func) { return rank <= 5lu; }); - pass_config->set_callback( + if (device_info.supports_immad) { + pass_config->disable(); + pass_config->disable(); + pass_config->disable(); + } else { + pass_config->set_callback( [](const_node_ptr &node) -> bool { return disableReduceDecomposition(node); }); - pass_config->set_callback( + pass_config->set_callback( [](const_node_ptr &node) -> bool { return disableReduceDecomposition(node); }); - pass_config->set_callback( + pass_config->set_callback( [](const_node_ptr &node) -> bool { return disableReduceDecomposition(node); }); + } auto isCellPrimitiveSupported = [](const_node_ptr &node) -> bool { if (std::dynamic_pointer_cast(node)) { diff --git a/src/plugins/intel_gpu/thirdparty/onednn_gpu b/src/plugins/intel_gpu/thirdparty/onednn_gpu index f8de0c93e9f..efbf9b5e8c1 160000 --- a/src/plugins/intel_gpu/thirdparty/onednn_gpu +++ b/src/plugins/intel_gpu/thirdparty/onednn_gpu @@ -1 +1 @@ -Subproject commit f8de0c93e9ff53a7d0a41b97aabc85e828020881 +Subproject commit efbf9b5e8c12666314f3484ce279cee0a1a91a44