[GPU] Onednn integration for pooling (#7753)

* [GPU] Onednn integration for pooling primitive

* layout_optimizer for pooling

* Padding code merge for onednn

Co-authored-by: Kim, Mingyu <mingyu.kim@intel.com>

inference-engine/thirdparty/clDNN/src/graph_optimizer/add_required_reorders.cpp

@@ -51,9 +51,28 @@ void add_required_reorders::run(program& p) {
             continue;  // only nodes with dependencies
         if (usr->is_type<data>())
             continue;
-        if (usr->type()->does_an_implementation_exist(*usr))
-            continue;
 
+        if (usr->type()->does_an_implementation_exist(*usr)) {
+            if (usr->get_preferred_impl_type() != impl_types::onednn) {
+                continue;
+            } else {
+                // oneDNN doesn't support padded memory, so add reorder directly if needed
+                for (size_t i = 0; i < usr->get_dependencies().size(); i++) {
+                    auto& input = usr->get_dependency(i);
+                    if (!input.is_in_data_flow() || input.is_constant())
+                        continue;
+
+                    if (static_cast<bool>(input.get_output_layout().data_padding)) {
+                        cldnn::layout layout_wo_padding = input.get_output_layout();
+                        layout_wo_padding.data_padding = cldnn::padding{};
+                        auto new_reorder = std::make_shared<reorder>(input.id() + "_padding_reorder_" + usr->id(), input.id(), layout_wo_padding);
+                        auto& new_reorder_node = p.get_or_create(new_reorder);
+                        p.add_intermediate(new_reorder_node, *usr, i);
+                    }
+                }
+                continue;
+            }
+        }
         bool correct_layout_selected = false;
         bool weights_data = (usr->is_type<convolution>() || usr->is_type<deconvolution>() ||
                              usr->is_type<deformable_conv>() || usr->is_type<fully_connected>());

inference-engine/thirdparty/clDNN/src/graph_optimizer/prepare_padding.cpp

@@ -17,6 +17,10 @@ void prepare_padding::run(program& p) {
     if (output_size_handling_enabled) {
         // Prepare upper padding for primitives that support output_size parameter.
         for (const auto& node : p.get_processing_order()) {
+            // Padded offsets aren't supported by onednn kernels
+            if (node->get_preferred_impl_type() == impl_types::onednn)
+                continue;
+
             if (node->is_type<convolution>()) {
                 auto& prim_node = node->as<convolution>();
                 const auto& prim = prim_node.get_primitive();
@@ -28,6 +32,7 @@ void prepare_padding::run(program& p) {
                 if (format == format::b_fs_zyx_fsv16 ||
                     format == format::bs_fs_zyx_bsv16_fsv16 ||
                     format == format::bs_fs_yx_bsv16_fsv16 ||
+                    format == format::bs_fs_yx_bsv32_fsv32 ||
                     format == format::b_fs_zyx_fsv32)
                     continue;
 
@@ -147,6 +152,13 @@ void prepare_padding::run(program& p) {
         if (conv_input_node.is_output() || conv_input_node.is_type<data>())
             continue;
 
+        // Padded offsets aren't supported by onednn kernels
+        if (conv_input_node.get_preferred_impl_type() == impl_types::onednn)
+            continue;
+
+        if (node.get_preferred_impl_type() == impl_types::onednn)
+            continue;
+
         // Calculating input padding needed for convolution
         auto& filter_node = node.as<convolution>().weights(0);
         auto filter_prim = filter_node.get_primitive();

inference-engine/thirdparty/clDNN/src/graph_optimizer/reorder_inputs.cpp

@@ -58,7 +58,7 @@ std::map<program_node*, format::type> get_preferred_formats(program& p, layout_o
             continue;
 
         auto ex = lo.get_preferred_format(*n);
-        auto impl = lo.get_preferred_impl_type(*n);
+        auto impl = lo.get_preferred_impl_type(*n, ex);
         fmt_map[n] = ex;
 
         n->set_preferred_impl_type(impl);

inference-engine/thirdparty/clDNN/src/impls/onednn/pooling_onednn.cpp

@@ -15,10 +15,103 @@
 namespace cldnn {
 namespace onednn {
 
+struct pooling_onednn : typed_primitive_onednn_impl<pooling, dnnl::pooling_forward::desc> {
+    using parent = typed_primitive_onednn_impl<pooling, dnnl::pooling_forward::desc>;
+    using parent::parent;
+
+protected:
+    std::unique_ptr<primitive_impl> clone() const override {
+        return make_unique<pooling_onednn>(*this);
+    }
+
+    static std::shared_ptr<dnnl::pooling_forward::desc> get_pooling_descriptor(const pooling_node& arg) {
+        auto prim = arg.get_primitive();
+
+        auto& input = arg.get_dependency(0);
+        auto spatials_rank = cldnn::format::spatial_num(input.get_output_layout().format);
+
+        auto stride = onednn::convert_spatials(prim->stride, spatials_rank);
+        auto kernel = onednn::convert_spatials(prim->size, spatials_rank);
+        auto pad_l = onednn::convert_spatials(prim->input_offset, spatials_rank);
+        auto pad_r = onednn::convert_spatials(prim->pad_end, spatials_rank);
+
+        auto input_md = onednn::layout_to_memory_desc(input.get_output_layout());
+        auto output_md = onednn::layout_to_memory_desc(arg.get_output_layout());
+
+        if (prim->global_pooling) {
+            for (size_t i = 0; i < kernel.size(); i++)
+                kernel[i] = input_md.dims()[2 + i];
+        }
+
+        for (size_t i = 0; i < kernel.size(); i++) {
+            pad_l[i] = -pad_l[i];
+            pad_r[i] = (output_md.dims()[2 + i] - 1) * stride[i] - input_md.dims()[2 + i] + kernel[i] - pad_l[i];
+        }
+
+        dnnl::algorithm alg;
+        switch (prim->mode) {
+            case pooling_mode::average: alg = dnnl::algorithm::pooling_avg; break;
+            case pooling_mode::max: alg = dnnl::algorithm::pooling_max; break;
+            case pooling_mode::average_no_padding: alg = dnnl::algorithm::pooling_avg_exclude_padding; break;
+            default: throw std::runtime_error("unsupported pool mode");
+        }
+
+        return std::make_shared<dnnl::pooling_forward::desc>(
+            dnnl::prop_kind::forward_inference,
+            alg,
+            input_md,
+            output_md,
+            stride,
+            kernel,
+            pad_l,
+            pad_r);
+    }
+
+public:
+    static primitive_impl* create(const pooling_node& arg) {
+        auto& engine = arg.get_program().get_engine();
+        auto desc = get_pooling_descriptor(arg);
+        auto attr = get_primitive_attributes(arg);
+        dnnl::primitive_desc prim_desc{&desc->data, attr.get(), engine.get_onednn_engine(), nullptr};
+
+        return new pooling_onednn(arg, desc, attr, prim_desc);
+    }
+};
 
 namespace detail {
 
 attach_pooling_onednn::attach_pooling_onednn() {
+    implementation_map<pooling>::add(impl_types::onednn, pooling_onednn::create, {
+        std::make_tuple(data_types::f32, format::bfyx),
+        std::make_tuple(data_types::f16, format::bfyx),
+        std::make_tuple(data_types::u8, format::bfyx),
+        std::make_tuple(data_types::i8, format::bfyx),
+
+        std::make_tuple(data_types::f32, format::b_fs_yx_fsv16),
+        std::make_tuple(data_types::f16, format::b_fs_yx_fsv16),
+        std::make_tuple(data_types::u8, format::b_fs_yx_fsv16),
+        std::make_tuple(data_types::i8, format::b_fs_yx_fsv16),
+
+        std::make_tuple(data_types::f32, format::b_fs_yx_fsv32),
+        std::make_tuple(data_types::f16, format::b_fs_yx_fsv32),
+        std::make_tuple(data_types::u8, format::b_fs_yx_fsv32),
+        std::make_tuple(data_types::i8, format::b_fs_yx_fsv32),
+
+        std::make_tuple(data_types::f32, format::bs_fs_yx_bsv16_fsv16),
+        std::make_tuple(data_types::f16, format::bs_fs_yx_bsv16_fsv16),
+        std::make_tuple(data_types::u8, format::bs_fs_yx_bsv16_fsv16),
+        std::make_tuple(data_types::i8, format::bs_fs_yx_bsv16_fsv16),
+
+        std::make_tuple(data_types::f32, format::bs_fs_yx_bsv32_fsv16),
+        std::make_tuple(data_types::f16, format::bs_fs_yx_bsv32_fsv16),
+        std::make_tuple(data_types::u8, format::bs_fs_yx_bsv32_fsv16),
+        std::make_tuple(data_types::i8, format::bs_fs_yx_bsv32_fsv16),
+
+        std::make_tuple(data_types::f32, format::bs_fs_yx_bsv32_fsv32),
+        std::make_tuple(data_types::f16, format::bs_fs_yx_bsv32_fsv32),
+        std::make_tuple(data_types::u8, format::bs_fs_yx_bsv32_fsv32),
+        std::make_tuple(data_types::i8, format::bs_fs_yx_bsv32_fsv32),
+    });
 }
 
 }  // namespace detail

inference-engine/thirdparty/clDNN/src/include/layout_optimizer.h

@@ -171,7 +171,7 @@ public:
     explicit layout_optimizer(bool output_size_handling_enabled = true);
 
     format get_preferred_format(program_node& node);
-    impl_types get_preferred_impl_type(program_node& node);
+    impl_types get_preferred_impl_type(program_node& node, format preferred_format);
 
     bool is_format_supported(program_node& node, format::type fmt);
 

inference-engine/thirdparty/clDNN/src/layout_optimizer.cpp

@@ -874,7 +874,7 @@ layout layout_optimizer::get_expected_layout(layout const& current_layout,
     return layout(expected_data_type, expected_format, expected_tensor);
 }
 
-impl_types layout_optimizer::get_preferred_impl_type(program_node& node) {
+impl_types layout_optimizer::get_preferred_impl_type(program_node& node, format preferred_format) {
     impl_types preferred_impl = impl_types::any;
     if (!_forcing_map.empty() && _forcing_map.count(node.id()) != 0) {
         preferred_impl = _forcing_map.at(node.id()).second;
@@ -937,6 +937,107 @@ impl_types layout_optimizer::get_preferred_impl_type(program_node& node) {
         if (input_layout.format.dimension() != output_layout.format.dimension()) {
             preferred_impl = impl_types::ocl;
         }
+    } else if (node.is_type<pooling>()) {
+        if (!_optimization_attributes.use_onednn_impls)
+            return impl_types::ocl;
+
+        std::vector<format> onednn_optimized_formats = {
+            format::b_fs_zyx_fsv16,
+            format::b_fs_zyx_fsv32,
+            format::b_fs_yx_fsv32,
+            format::b_fs_yx_fsv16,
+            format::bs_fs_yx_bsv16_fsv16,
+            format::bs_fs_zyx_bsv16_fsv16,
+            format::bs_fs_yx_bsv32_fsv16,
+            format::bs_fs_yx_bsv32_fsv32,
+            format::bs_fs_yx_bsv4_fsv4,
+            format::bs_fs_yx_bsv4_fsv2,
+            format::bs_fs_zyx_bsv4_fsv4,
+            format::bs_fs_zyx_bsv4_fsv2,
+        };
+
+        impl_types impl_candidate = impl_types::onednn;
+
+        // Unexpected layout
+        if (std::find(onednn_optimized_formats.begin(), onednn_optimized_formats.end(), preferred_format) == onednn_optimized_formats.end()) {
+            impl_candidate = impl_types::ocl;
+        }
+
+        // if (node.is_type<convolution>()) {
+        //     // onednn doesn't have good support for groups with fsv16 fmt
+        //     auto& conv = node.as<convolution>();
+        //     auto input_layout = conv.input().get_output_layout();
+        //     bool fp16_input = input_layout.data_type == data_types::f16;
+        //     bool has_groups = conv.get_primitive()->groups > 1;
+        //     bool is_depthwise = conv.get_primitive()->groups == input_layout.size.feature[0];
+        //     bool first_conv = input_layout.size.feature[0] <= 4;
+        //     bool enable_onednn_dw_fp16_conv = fp16_input && is_depthwise;
+        //     if (((has_groups && !enable_onednn_dw_fp16_conv) || first_conv) &&
+        //         (conv.get_output_layout().format == format::b_fs_yx_fsv16)) {
+        //         impl_candidate = impl_types::ocl;
+        //     }
+        // }
+
+        // if (node.is_type<deconvolution>()) {
+        //     auto& deconv = node.as<deconvolution>();
+        //     auto input_layout = deconv.input().get_output_layout();
+        //     bool valid_ic = input_layout.size.feature[0] >= 16;
+        //     bool valid_groups = deconv.get_primitive()->groups == 1;
+        //     bool valid_post_ops = get_post_ops_count(node) <= 10;
+        //     bool valid_batch = input_layout.size.batch[0] < 16;  // oneDNNs optimized kernel doesn't support big batches yet
+        //     bool valid_params = valid_ic && valid_groups && valid_post_ops && valid_batch;
+        //     if (!valid_params)
+        //         impl_candidate = impl_types::ocl;
+        // }
+
+        // // [WA] onednn doesn't support > 32 post-ops. Remove once onednn improve post-ops for GPU.
+        // if (get_post_ops_count(node) > 32) {
+        //    impl_candidate = impl_types::ocl;
+        // }
+
+        // if (!data_types_are_suitable_for_onednn(node)) {
+        //     impl_candidate = impl_types::ocl;
+        // }
+
+        // for (auto& fo : node.get_fused_primitives()) {
+        //     if (fo.node->is_type<eltwise>()) {
+        //         auto in_layout = node.get_dependency(fo.dep_start_idx).get_output_layout();
+        //         auto out_layout = node.get_output_layout();
+        //         auto in_dt = in_layout.data_type;
+        //         auto out_dt = out_layout.data_type;
+        //         if ((out_layout.count() == in_layout.count()) &&
+        //             (data_type_traits::is_floating_point(in_dt) || data_type_traits::is_floating_point(out_dt)) && in_dt != out_dt) {
+        //             impl_candidate = impl_types::ocl;
+        //             break;
+        //         }
+        //     } else if (fo.node->is_type<activation>()) {
+        //         // Some activations aren't implemented in oneDNN
+        //         auto activation_prim = fo.node->as<activation>().get_primitive();
+        //         if (activation_prim->activation_function == activation_func::negative ||
+        //             activation_prim->activation_function == activation_func::negation ||
+        //             activation_prim->activation_function == activation_func::sign)
+        //             impl_candidate = impl_types::ocl;
+        //     }
+        // }
+
+        // // oneDNN doesn't support asymmetric weights quantization
+        // if (node.is_type<convolution>() && node.as<convolution>().weights_zero_points_term())
+        //     impl_candidate = impl_types::ocl;
+
+        // // OneDNN doesn't support sum post ops for deconvolutions
+        // if (node.is_type<deconvolution>() && impl_candidate == impl_types::onednn) {
+        //     for (auto& fused_op : node.get_fused_primitives()) {
+        //         if (fused_op.node->is_type<eltwise>() && fused_op.deps.size() == 1) {
+        //             auto eltw_in_layout = node.get_dependency(fused_op.dep_start_idx).get_output_layout();
+        //             if (fused_op.node->as<eltwise>().get_primitive()->needs_onednn_sum_post_op(eltw_in_layout)) {
+        //                 impl_candidate = impl_types::ocl;
+        //                 break;
+        //             }
+        //         }
+        //     }
+        // }
+
+        preferred_impl = impl_candidate;
     }
 
     return preferred_impl;
@@ -956,6 +1057,15 @@ format layout_optimizer::get_preferred_format(program_node& node) {
         auto& bconv_node = node.as<binary_convolution>();
         auto weights_layout = bconv_node.weights(0).get_output_layout();
         expected = get_expected_layout(output_layout, bconv_node, weights_layout).format;
+    } else if (node.is_type<pooling>() && _optimization_attributes.use_onednn_impls) {
+         auto in_layout = node.get_dependency(0).get_output_layout();
+         auto out_layout = node.get_output_layout();
+
+         if (out_layout.size.batch[0] % 16 == 0 || out_layout.size.batch[0] == 8) {
+             if (!data_type_traits::is_floating_point(in_layout.data_type) && in_layout.data_type != out_layout.data_type) {
+                 expected = format::b_fs_yx_fsv16;
+             }
+         }
     } else if (node.is_type<detection_output>()) {
         expected = get_expected_layout(
             output_layout,

inference-engine/thirdparty/clDNN/tests/test_cases/pooling_gpu_test.cpp

@@ -3765,3 +3765,55 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_POOLING,
                         ::testing::Combine(::testing::ValuesIn(pooling_test::generate_generic_test_params()),
                                            ::testing::ValuesIn(pooling_test::generate_specific_test_params())),
                         tests::generic_test::custom_param_name_functor());
+
+
+#ifdef ENABLE_ONEDNN_FOR_GPU
+TEST(pooling_forward_gpu_onednn, basic_max_pooling_int8) {
+    auto& engine = get_onednn_test_engine();
+    layout in_layout = { type_to_data_type<float>::value, format::byxf, { 1, 1, 3, 3 } };
+    layout out_layout = { type_to_data_type<float>::value, format::byxf, { 1, 1, 1, 1 } };
+    layout byte_layout = { type_to_data_type<int8_t>::value, format::bfyx, { 1, 1, 3, 3 } };
+    std::initializer_list<float> input_f = { 1.0f, -2.5f, 3.1f, -4.0f, 5.03f, -6.99f, 7.0f, -8.0f, 9.5f };
+    std::list<float> final_results = { 10.0f };
+
+    // Allocate memory for input image.
+    auto input_memory = engine.allocate_memory(in_layout);
+    set_values(input_memory, input_f);
+
+    // Create input_layout description
+    // "input" - is the primitive id inside topology
+    input_layout input("input", in_layout);
+
+    topology topology(
+        // 1. input layout primitive.
+        input,
+        // 2. reorder primitive with id "reorder_input"
+        reorder("reorder_input", input, byte_layout),
+        pooling("pool1", "reorder_input", pooling_mode::max, { 1, 1, 3, 3 }, {1, 1, 1, 1}),
+        reorder("reorder2", "pool1", out_layout)
+    );
+
+    build_options options_target;
+    options_target.set_option(build_option::outputs({ "reorder2"}));
+    implementation_desc impl = {format::bfyx, std::string(""), impl_types::onednn};
+    options_target.set_option(build_option::force_implementations({{"pool1", impl}}));
+
+    network network(
+        engine,
+        topology,
+        options_target);
+
+    network.set_input_data("input", input_memory);
+
+    auto outputs = network.execute();
+
+    auto interm = outputs.at("reorder2").get_memory();
+    cldnn::mem_lock<float> interm_ptr(interm, get_test_stream());
+    unsigned int cntr = 0;
+    for (const auto& exp : final_results)
+    {
+        EXPECT_EQ(exp, interm_ptr[cntr++]);
+    }
+}
+
+#endif   // ENABLE_ONEDNN_FOR_GPU