[GPU] Enable onednn reduction (#11570)

* It has better performance by using reduction kernel instead of pooling kernel in oneDNN for reduction layer.
* Stop using global pooling instead of reduce primitive
* Use oneDNN reduction if its mode is supported by optimized onedNN kernel
* activation pow is supported
* Use clDNN reduce if 3d or redundant reduce, tensor size mismatch
* Updated thirdparty onednn_gpu

Signed-off-by: Min, Byungil <byungil.min@intel.com>

Co-authored-by: Wei Tang <wei1.tang@intel.com>
Co-authored-by: Chen Kurt <kurt.chen@intel.com>

src/plugins/intel_gpu/src/graph/graph_optimizer/add_onednn_optimization_attributes.cpp

@@ -9,6 +9,7 @@
 
 #ifdef ENABLE_ONEDNN_FOR_GPU
 #include "fully_connected_inst.h"
+#include "reduce_inst.h"
 #include <impls/onednn/utils.hpp>
 #endif
 

src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp

@@ -612,8 +612,20 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) {
             return true;
         };
 
-        auto reduce_supports_fusings = [](reduce_node& node) -> bool {
+        auto reduce_supports_fusings = [&](reduce_node& node) -> bool {
             auto keep_dims = node.as<reduce>().get_primitive()->keep_dims;
+            auto axes = node.as<reduce>().get_primitive()->axes;
+
+            // If reduce tensor size is small, it sets not to fuse eltwise which leads to select oneDNN reference reduction
+            // Because oneDNN optimized kernel does NOT support eltwise fusing
+            if (p.get_engine().get_device_info().supports_immad && node.get_output_layout().get_dims().size() <= 4 &&
+                ((find(axes.begin(), axes.end(), reduce::along_x) != axes.end() &&
+                node.input().get_output_layout().spatial(0) > 16) ||
+                (find(axes.begin(), axes.end(), reduce::along_y) != axes.end() &&
+                node.input().get_output_layout().spatial(1) > 16) ||
+                (find(axes.begin(), axes.end(), reduce::along_f) != axes.end() &&
+                node.input().get_output_layout().feature() > 16)))
+                return false;
 
             if (keep_dims)
                 return true;

src/plugins/intel_gpu/src/graph/impls/onednn/reduction_onednn.cpp

@@ -0,0 +1,114 @@
+// Copyright (C) 2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "reduce_inst.h"
+#include "primitive_onednn_base.h"
+#include "impls/implementation_map.hpp"
+
+#include "kernel_selector_common.h"
+#include "kernel_base.h"
+
+#include <oneapi/dnnl/dnnl.hpp>
+
+#include <algorithm>
+#include <memory>
+namespace cldnn {
+namespace onednn {
+struct reduction_onednn : typed_primitive_onednn_impl<reduce, dnnl::reduction::desc> {
+    using parent = typed_primitive_onednn_impl<reduce, dnnl::reduction::desc>;
+    using parent::parent;
+
+protected:
+    std::unique_ptr<primitive_impl> clone() const override {
+        return make_unique<reduction_onednn>(*this);
+    }
+
+    static std::shared_ptr<dnnl::reduction::desc> get_reduction_descriptor(const reduce_node& arg) {
+        auto prim = arg.get_primitive();
+        auto& input = arg.get_dependency(0);
+        auto input_md = onednn::layout_to_memory_desc(input.get_output_layout());
+        auto output_md = onednn::layout_to_memory_desc(arg.get_output_layout());
+
+        float p = 0.f;
+        float eps = 0.f;
+        dnnl::algorithm alg;
+        switch (prim->mode) {
+            case reduce_mode::mean: alg = dnnl::algorithm::reduction_mean; break;
+            case reduce_mode::max: alg = dnnl::algorithm::reduction_max; break;
+            case reduce_mode::min: alg = dnnl::algorithm::reduction_min; break;
+            case reduce_mode::sum: alg = dnnl::algorithm::reduction_sum; break;
+            case reduce_mode::prod: alg = dnnl::algorithm::reduction_mul; break;
+            case reduce_mode::sum_square:
+                alg = dnnl::algorithm::reduction_norm_lp_power_p_sum;
+                p = 2.0f;
+                break;
+            case reduce_mode::l1:
+                alg = dnnl::algorithm::reduction_norm_lp_sum;
+                p = 1.0f;
+                break;
+            case reduce_mode::l2:
+                alg = dnnl::algorithm::reduction_norm_lp_sum;
+                p = 2.0f;
+                break;
+            default: throw std::runtime_error("unsupported reduce mode");
+        }
+
+        return std::make_shared<dnnl::reduction::desc>(
+            alg,
+            input_md,
+            output_md,
+            p,
+            eps);
+    }
+
+public:
+    static primitive_impl* create(const reduce_node& arg) {
+        auto& engine = arg.get_program().get_engine();
+        auto desc = get_reduction_descriptor(arg);
+        auto attr = arg.get_onednn_primitive_attributes();
+        dnnl::primitive_desc prim_desc{&desc->data, attr.get(), engine.get_onednn_engine(), nullptr};
+
+        return new reduction_onednn(arg, desc, attr, prim_desc);
+    }
+};
+
+namespace detail {
+
+attach_reduction_onednn::attach_reduction_onednn() {
+    implementation_map<reduce>::add(impl_types::onednn, reduction_onednn::create, {
+        std::make_tuple(data_types::f32, format::bfyx),
+        std::make_tuple(data_types::f16, format::bfyx),
+        std::make_tuple(data_types::u8, format::bfyx),
+        std::make_tuple(data_types::i8, format::bfyx),
+
+        std::make_tuple(data_types::f32, format::b_fs_yx_fsv16),
+        std::make_tuple(data_types::f16, format::b_fs_yx_fsv16),
+        std::make_tuple(data_types::u8, format::b_fs_yx_fsv16),
+        std::make_tuple(data_types::i8, format::b_fs_yx_fsv16),
+
+        std::make_tuple(data_types::f32, format::b_fs_yx_fsv32),
+        std::make_tuple(data_types::f16, format::b_fs_yx_fsv32),
+        std::make_tuple(data_types::u8, format::b_fs_yx_fsv32),
+        std::make_tuple(data_types::i8, format::b_fs_yx_fsv32),
+
+        std::make_tuple(data_types::f32, format::bs_fs_yx_bsv16_fsv16),
+        std::make_tuple(data_types::f16, format::bs_fs_yx_bsv16_fsv16),
+        std::make_tuple(data_types::u8, format::bs_fs_yx_bsv16_fsv16),
+        std::make_tuple(data_types::i8, format::bs_fs_yx_bsv16_fsv16),
+
+        std::make_tuple(data_types::f32, format::bs_fs_yx_bsv32_fsv16),
+        std::make_tuple(data_types::f16, format::bs_fs_yx_bsv32_fsv16),
+        std::make_tuple(data_types::u8, format::bs_fs_yx_bsv32_fsv16),
+        std::make_tuple(data_types::i8, format::bs_fs_yx_bsv32_fsv16),
+
+        std::make_tuple(data_types::f32, format::bs_fs_yx_bsv32_fsv32),
+        std::make_tuple(data_types::f16, format::bs_fs_yx_bsv32_fsv32),
+        std::make_tuple(data_types::u8, format::bs_fs_yx_bsv32_fsv32),
+        std::make_tuple(data_types::i8, format::bs_fs_yx_bsv32_fsv32),
+    });
+}
+
+}  // namespace detail
+}  // namespace onednn
+}  // namespace cldnn

src/plugins/intel_gpu/src/graph/impls/onednn/register.cpp

@@ -17,6 +17,7 @@ void register_implementations() {
     REGISTER_ONEDNN_IMPL(eltwise);
     REGISTER_ONEDNN_IMPL(gemm);
     REGISTER_ONEDNN_IMPL(pooling);
+    REGISTER_ONEDNN_IMPL(reduction);
     REGISTER_ONEDNN_IMPL(reorder);
     REGISTER_ONEDNN_IMPL(fully_connected);}
 

src/plugins/intel_gpu/src/graph/impls/onednn/register.hpp

@@ -23,6 +23,7 @@ REGISTER_ONEDNN_IMPL(concatenation);
 REGISTER_ONEDNN_IMPL(eltwise);
 REGISTER_ONEDNN_IMPL(gemm);
 REGISTER_ONEDNN_IMPL(pooling);
+REGISTER_ONEDNN_IMPL(reduction);
 REGISTER_ONEDNN_IMPL(reorder);
 REGISTER_ONEDNN_IMPL(fully_connected);
 

src/plugins/intel_gpu/src/graph/impls/onednn/utils.cpp

@@ -471,6 +471,7 @@ cldnn::format find_format(dnnl::memory::desc desc, bool is_grouped) {
     }
 }
 
+// Currently, usage of alpha and beta between cldnn::pow and dnnl::eltwise::pow is different : d = pow(src, a) / d = a * pow(src, b)
 dnnl::algorithm convert_activation_func(cldnn::activation_func func) {
     switch (func) {
         case cldnn::activation_func::relu: return dnnl::algorithm::eltwise_relu;
@@ -485,6 +486,8 @@ dnnl::algorithm convert_activation_func(cldnn::activation_func func) {
         case cldnn::activation_func::logistic: return dnnl::algorithm::eltwise_logistic;
         case cldnn::activation_func::clamp: return dnnl::algorithm::eltwise_clip;
         case cldnn::activation_func::hyperbolic_tan: return dnnl::algorithm::eltwise_tanh;
+        case cldnn::activation_func::pow: return dnnl::algorithm::eltwise_pow;
+        case cldnn::activation_func::sqrt: return dnnl::algorithm::eltwise_sqrt;
         default: throw std::runtime_error("Unsupported activation func for onednn primitive " + std::to_string(static_cast<int>(func)));
     }
 }

src/plugins/intel_gpu/src/graph/layout_optimizer.cpp

@@ -16,6 +16,7 @@
 #include "gemm_inst.h"
 #include "eltwise_inst.h"
 #include "pooling_inst.h"
+#include "reduce_inst.h"
 #include "one_hot_inst.h"
 #include "permute_inst.h"
 #include "gemm_inst.h"
@@ -759,6 +760,43 @@ bool layout_optimizer::deconvolution_b_fs_yx_fsv16_opt(layout const &input_layou
     return false;
 }
 
+static bool is_node_for_onednn(reduce_node const& node) {
+    auto reduce_prim = node.get_primitive();
+    // oneDNN reduction currently does not support logical_and, logical_or, log_sum and log_sum_exp.
+    switch (reduce_prim->mode) {
+        case reduce_mode::mean:
+        case reduce_mode::max:
+        case reduce_mode::min:
+        case reduce_mode::sum:
+        case reduce_mode::prod:
+        case reduce_mode::sum_square:
+        case reduce_mode::l1:
+        case reduce_mode::l2:
+            break;
+        default:
+            return false;
+    }
+
+    auto& input = node.input();
+    auto input_layout = input.get_output_layout();
+    auto reduce_axes = reduce_prim->axes;
+    // redundant reduce is not acceptable on oneDNN reduction
+    if (node.get_output_layout() == input_layout) {
+        return false;
+    }
+    // tensor size mismatch of unreduced axis is not supported
+    for (size_t idx = 0 ; idx < input_layout.get_dims().size() ; idx++) {
+        bool reduced_axis = std::find(reduce_axes.begin(), reduce_axes.end(), idx) != reduce_axes.end();
+        if (!reduced_axis && (input_layout.get_dim(idx) != node.get_output_layout().get_dim(idx)))
+            return false;
+    }
+    if (input_layout.get_dims().size() > 4) {
+        return false;
+    }
+
+    return true;
+}
+
 static bool is_node_for_onednn(deconvolution_node const& node) {
     auto prim = node.get_primitive();
     auto input_layout = node.get_dependency(0).get_output_layout();
@@ -1376,6 +1414,14 @@ impl_types layout_optimizer::get_preferred_impl_type(program_node& node, format
             preferred_impl = impl_types::ocl;
         if (output_fmt == format::bfyx && output_dt == data_types::f32)
             preferred_impl = impl_types::ocl;
+    } else if (node.is_type<reduce>()) {
+        if (!_optimization_attributes.use_onednn_impls)
+            return impl_types::ocl;
+
+        if (is_node_for_onednn(node.as<reduce>()))
+            return impl_types::onednn;
+        else
+            return impl_types::ocl;
     } else if (node.is_type<pooling>() || node.is_type<convolution>() || node.is_type<deconvolution>()) {
         if (!_optimization_attributes.use_onednn_impls)
             return impl_types::ocl;

src/plugins/intel_gpu/src/graph/program_node.cpp

@@ -12,6 +12,7 @@
 #include "quantize_inst.h"
 #include "reorder_inst.h"
 #include "pooling_inst.h"
+#include "reduce_inst.h"
 #include <impls/onednn/utils.hpp>
 #endif // ENABLE_ONEDNN_FOR_GPU
 
@@ -888,7 +889,12 @@ void program_node::init_onednn_primitive_attributes() {
                 update_onednn_post_op_list(onednn_post_op_type::eltwise_clip, empty_mem);
             } else {
                 dnnl::algorithm alg = onednn::convert_activation_func(fused_desc->activation_function);
-                post_ops.append_eltwise(1.0f, alg, fused_desc->additional_params.a, fused_desc->additional_params.b);
+                // Usage of alpha and beta between cldnn::pow and dnnl::eltwise::pow is different : d = pow(src, a) / d = a * pow(src, b)
+                if (alg == dnnl::algorithm::eltwise_pow)
+                    post_ops.append_eltwise(1.0f, alg, 1.0f, fused_desc->additional_params.a);
+                else
+                    post_ops.append_eltwise(1.0f, alg, fused_desc->additional_params.a, fused_desc->additional_params.b);
+
                 update_onednn_post_op_list(onednn_post_op_type::eltwise_act, empty_mem);
             }
         } else if (node->is_type<eltwise>()) {
@@ -914,7 +920,7 @@ void program_node::init_onednn_primitive_attributes() {
             } else {
                 if (in.spatial(0) > 1 || in.spatial(1) > 1 || in.batch() > 1)
                     throw std::runtime_error("Unsupported eltwise mode for fused onednn op");
-                if (idx == 0 && !has_out_scales(attrs) && !is_type<pooling>()) {
+                if (idx == 0 && !has_out_scales(attrs) && !is_type<pooling>() && !is_type<reduce>()) {
                     int mask = in.count() > 1 ? 2 : 0;
                     attrs->set_output_scales(mask, {DNNL_RUNTIME_F32_VAL});
                     update_onednn_post_op_list(onednn_post_op_type::scale, dep_idx);

src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp

@@ -190,20 +190,26 @@ void TransformationsPipeline::apply(std::shared_ptr<ov::Model> func) {
                     return rank <= 5lu;
                 });
 
-        pass_config->set_callback<ngraph::pass::ConvertReduceSumToPooling>(
+        if (device_info.supports_immad) {
+            pass_config->disable<ngraph::pass::ConvertReduceSumToPooling>();
+            pass_config->disable<ngraph::pass::ConvertReduceMeanToPooling>();
+            pass_config->disable<ngraph::pass::ConvertReduceMaxToPooling>();
+        } else {
+            pass_config->set_callback<ngraph::pass::ConvertReduceSumToPooling>(
             [](const_node_ptr &node) -> bool {
                 return disableReduceDecomposition<ngraph::opset1::ReduceSum>(node);
             });
 
-        pass_config->set_callback<ngraph::pass::ConvertReduceMeanToPooling>(
+            pass_config->set_callback<ngraph::pass::ConvertReduceMeanToPooling>(
             [](const_node_ptr &node) -> bool {
                 return disableReduceDecomposition<ngraph::opset1::ReduceMean>(node);
             });
 
-        pass_config->set_callback<ngraph::pass::ConvertReduceMaxToPooling>(
+            pass_config->set_callback<ngraph::pass::ConvertReduceMaxToPooling>(
             [](const_node_ptr &node) -> bool {
                 return disableReduceDecomposition<ngraph::opset1::ReduceMax>(node);
             });
+        }
 
         auto isCellPrimitiveSupported = [](const_node_ptr &node) -> bool {
             if (std::dynamic_pointer_cast<const ngraph::opset6::RNNCell>(node)) {