diff --git a/inference-engine/thirdparty/clDNN/src/graph_optimizer/add_onednn_optimization_attributes.cpp b/inference-engine/thirdparty/clDNN/src/graph_optimizer/add_onednn_optimization_attributes.cpp index bb7b396467c..5b8e65015b7 100644 --- a/inference-engine/thirdparty/clDNN/src/graph_optimizer/add_onednn_optimization_attributes.cpp +++ b/inference-engine/thirdparty/clDNN/src/graph_optimizer/add_onednn_optimization_attributes.cpp @@ -7,12 +7,34 @@ #include "pass_manager.h" #include "program_node.h" +#ifdef ENABLE_ONEDNN_FOR_GPU +#include "fully_connected_inst.h" +#include +#endif + using namespace cldnn; void add_onednn_optimization_attributes::run(program& p) { #ifdef ENABLE_ONEDNN_FOR_GPU for (auto& node : p.get_processing_order()) { if (node->get_preferred_impl_type() == impl_types::onednn) { + if (node->is_type()) { + auto fc_prim = node->as().get_primitive(); + + // Reshape fused ops tensors for OneDNN FC if needed + if (fc_prim->input_size == 3) { + for (auto& fused_prim : node->get_fused_primitives()) { + auto fused_node = fused_prim.node; + if (fused_node->is_type()) { + auto& dependency = node->get_dependency(fused_prim.dep_start_idx); + auto original_layout = dependency.get_output_layout(); + onednn::combine_bf_with_first_spatial_dim(original_layout); + dependency.set_output_layout(original_layout, false); + } + } + } + } + node->init_onednn_primitive_attributes(); } } diff --git a/inference-engine/thirdparty/clDNN/src/impls/onednn/fully_connected_onednn.cpp b/inference-engine/thirdparty/clDNN/src/impls/onednn/fully_connected_onednn.cpp index 2796070a51b..81a0fcf32a3 100644 --- a/inference-engine/thirdparty/clDNN/src/impls/onednn/fully_connected_onednn.cpp +++ b/inference-engine/thirdparty/clDNN/src/impls/onednn/fully_connected_onednn.cpp @@ -128,20 +128,6 @@ public: static primitive_impl* create(const fully_connected_node& arg) { auto& engine = arg.get_program().get_engine(); auto desc = get_fully_connected_descriptor(arg); - auto prim = arg.get_primitive(); - - if (prim->input_size == 3) { - for (auto& fused_node : arg.get_fused_primitives()) { - auto node = fused_node.node; - if (node->is_type()) { - auto& dependency = arg.get_dependency(fused_node.dep_start_idx); - auto original_layout = dependency.get_output_layout(); - onednn::combine_bf_with_first_spatial_dim(original_layout); - dependency.set_output_layout(original_layout, false); - } - } - } - auto attr = arg.get_onednn_primitive_attributes(); dnnl::primitive_desc prim_desc{&desc->data, attr.get(), engine.get_onednn_engine(), nullptr}; diff --git a/inference-engine/thirdparty/clDNN/tests/test_cases/fusings_gpu_test.cpp b/inference-engine/thirdparty/clDNN/tests/test_cases/fusings_gpu_test.cpp index 35e4fe25e08..48c42e4eae5 100644 --- a/inference-engine/thirdparty/clDNN/tests/test_cases/fusings_gpu_test.cpp +++ b/inference-engine/thirdparty/clDNN/tests/test_cases/fusings_gpu_test.cpp @@ -608,6 +608,7 @@ public: #define CASE_FC_U8S8_3D_1 {2, 32, 1, 3}, {2, 32, 1, 16}, {16, 3, 1, 1}, tensor{1}, tensor{0}, tensor{1}, 1, data_types::u8, format::bfyx, data_types::i8, format::oiyx, data_types::f32, format::bfyx #define CASE_FC_U8S8_3D_2 {1, 1, 1, 3}, {1, 1, 1, 32}, {32, 3, 1, 1}, tensor{1}, tensor{0}, tensor{1}, 1, data_types::u8, format::bfyx, data_types::i8, format::oiyx, data_types::f32, format::bfyx #define CASE_FC_U8S8_3D_3 {2, 3, 1, 1}, {2, 3, 1, 15}, {15, 1, 1, 1}, tensor{1}, tensor{0}, tensor{1}, 1, data_types::u8, format::bfyx, data_types::i8, format::oiyx, data_types::f32, format::bfyx +#define CASE_FC_U8S8_3D_4 {1, 512, 1, 1024}, {1, 384, 1, 1024}, {1024, 1024, 1, 1}, tensor{1}, tensor{0}, tensor{1}, 1, data_types::u8, format::bfyx, data_types::i8, format::oiyx, data_types::f32, format::bfyx #define CASE_NORMALIZE_I8_1 {1, 2, 3, 3}, data_types::u8, format::bfyx, data_types::f32, format::bfyx @@ -9258,7 +9259,7 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, gather_elements_activation_scale_eltwise, })); #ifdef ENABLE_ONEDNN_FOR_GPU -class ConvFusingTestOneDNN : public WeightsPrimitiveFusingTest { +class WeightsPrimitiveFusingTestOneDNN : public WeightsPrimitiveFusingTest { public: void execute(bc_test_params& p) { // Onednn post operation has issue in a machine that does not support imad. @@ -9299,7 +9300,7 @@ public: } }; -class conv_int8_eltwise_onednn : public ConvFusingTestOneDNN {}; +class conv_int8_eltwise_onednn : public WeightsPrimitiveFusingTestOneDNN {}; TEST_P(conv_int8_eltwise_onednn, u8_eltwise_sum_out) { auto p = GetParam(); @@ -9364,7 +9365,7 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, conv_int8_eltwise_onednn, bc_test_params{CASE_CONV3D_S8S8_5, 3, 4}, })); -class conv_fp32_activation_abs_onednn : public ConvFusingTestOneDNN {}; +class conv_fp32_activation_abs_onednn : public WeightsPrimitiveFusingTestOneDNN {}; TEST_P(conv_fp32_activation_abs_onednn, basic) { auto p = GetParam(); create_topologies(input_layout("input", get_input_layout(p)), @@ -9387,7 +9388,7 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, conv_fp32_activation_abs_onednn, bc_test_params{CASE_CONV_FP16_4, 2, 3}, })); -class conv_fp32_activation_mish_onednn : public ConvFusingTestOneDNN {}; +class conv_fp32_activation_mish_onednn : public WeightsPrimitiveFusingTestOneDNN {}; TEST_P(conv_fp32_activation_mish_onednn, basic) { auto p = GetParam(); create_topologies(input_layout("input", get_input_layout(p)), @@ -9410,7 +9411,7 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, conv_fp32_activation_mish_onednn, bc_test_params{CASE_CONV_FP16_4, 2, 3}, })); -class conv_fp32_activation_swish_onednn : public ConvFusingTestOneDNN {}; +class conv_fp32_activation_swish_onednn : public WeightsPrimitiveFusingTestOneDNN {}; TEST_P(conv_fp32_activation_swish_onednn, basic) { auto p = GetParam(); create_topologies(input_layout("input", get_input_layout(p)), @@ -9433,7 +9434,7 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, conv_fp32_activation_swish_onednn, bc_test_params{CASE_CONV_FP16_4, 2, 3}, })); -class conv_fp32_activation_hswish_onednn : public ConvFusingTestOneDNN {}; +class conv_fp32_activation_hswish_onednn : public WeightsPrimitiveFusingTestOneDNN {}; TEST_P(conv_fp32_activation_hswish_onednn, basic) { auto p = GetParam(); create_topologies(input_layout("input", get_input_layout(p)), @@ -9456,7 +9457,7 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, conv_fp32_activation_hswish_onednn, bc_test_params{CASE_CONV_FP16_4, 2, 3}, })); -class conv_fp32_activation_exp_onednn : public ConvFusingTestOneDNN {}; +class conv_fp32_activation_exp_onednn : public WeightsPrimitiveFusingTestOneDNN {}; TEST_P(conv_fp32_activation_exp_onednn, basic) { auto p = GetParam(); create_topologies(input_layout("input", get_input_layout(p)), @@ -9479,7 +9480,7 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, conv_fp32_activation_exp_onednn, bc_test_params{CASE_CONV_FP16_4, 2, 3}, })); -class conv_int8_quantize_u8_onednn : public ConvFusingTestOneDNN {}; +class conv_int8_quantize_u8_onednn : public WeightsPrimitiveFusingTestOneDNN {}; TEST_P(conv_int8_quantize_u8_onednn, per_channel) { auto p = GetParam(); create_topologies(input_layout("input", get_input_layout(p)), @@ -9526,7 +9527,7 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, conv_int8_quantize_u8_onednn, bc_test_params{CASE_CONV_S8S8_3, 2, 3}, })); -class conv_int8_activation_eltwise_quantize_onednn : public ConvFusingTestOneDNN {}; +class conv_int8_activation_eltwise_quantize_onednn : public WeightsPrimitiveFusingTestOneDNN {}; TEST_P(conv_int8_activation_eltwise_quantize_onednn, bsv32_fsv32) { auto p = GetParam(); layout eltwise_layout = get_output_layout(p); @@ -9578,7 +9579,7 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, conv_int8_activation_eltwise_quantize_oned bc_test_params{CASE_CONV_S8S8_15, 2, 5}, })); -class conv_int8_scale_shift_swish_onednn : public ConvFusingTestOneDNN {}; +class conv_int8_scale_shift_swish_onednn : public WeightsPrimitiveFusingTestOneDNN {}; TEST_P(conv_int8_scale_shift_swish_onednn, bsv32_fsv32) { auto p = GetParam(); create_topologies(input_layout("input", get_input_layout(p)), @@ -9617,7 +9618,7 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, conv_int8_scale_shift_swish_onednn, bc_test_params{CASE_CONV_S8S8_15, 2, 7}, })); -class conv_int8_eltwise_scale_onednn : public ConvFusingTestOneDNN {}; +class conv_int8_eltwise_scale_onednn : public WeightsPrimitiveFusingTestOneDNN {}; TEST_P(conv_int8_eltwise_scale_onednn, u8_eltwise_prod_out_reuse) { auto p = GetParam(); @@ -9667,7 +9668,7 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, conv_int8_eltwise_scale_onednn, // Limitations: no // DNNL_VERBOSE log without optimization: attr-post-ops:eltwise_linear:12.75:127.5+eltwise_linear:1:-128 // DNNL_VERBOSE log with optimization: attr-post-ops:eltwise_linear:12.75:-0.5 -class post_ops_optimizations_onednn_eltw_linear_eltw_linear : public ConvFusingTestOneDNN {}; +class post_ops_optimizations_onednn_eltw_linear_eltw_linear : public WeightsPrimitiveFusingTestOneDNN {}; TEST_P(post_ops_optimizations_onednn_eltw_linear_eltw_linear, basic) { auto p = GetParam(); create_topologies(input_layout("input", get_input_layout(p)), @@ -9718,7 +9719,7 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, post_ops_optimizations_onednn_eltw_linear_ // Limitations: beta = 0 in eltw_linear // DNNL_VERBOSE log without optimization: attr-post-ops:eltwise_linear:12.75:127.5+eltwise_round+eltwise_linear:2.00784+eltwise_clip:0:512 // DNNL_VERBOSE log with optimization: attr-post-ops:eltwise_linear:12.75:127.5+eltwise_round:0:0:2.00784+eltwise_clip:0:512 -class post_ops_optimizations_onednn_eltw_non_linear_eltw_linear : public ConvFusingTestOneDNN {}; +class post_ops_optimizations_onednn_eltw_non_linear_eltw_linear : public WeightsPrimitiveFusingTestOneDNN {}; TEST_P(post_ops_optimizations_onednn_eltw_non_linear_eltw_linear, basic) { auto p = GetParam(); create_topologies(input_layout("input", get_input_layout(p)), @@ -9769,7 +9770,7 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, post_ops_optimizations_onednn_eltw_non_lin // Limitations: alpha = 1 and scale = 1 in eltw_linear; binary_add is a constant compile-time buffer // DNNL_VERBOSE log without optimization: attr-oscale:2 attr-post-ops:binary_add:f32:2+eltwise_linear:1:-127+eltwise_clip:-127:127 // DNNL_VERBOSE log with optimization: attr-oscale:2 attr-post-ops:binary_add:f32:2+eltwise_clip:-127:127 -class post_ops_optimizations_onednn_binary_add_eltw_linear : public ConvFusingTestOneDNN {}; +class post_ops_optimizations_onednn_binary_add_eltw_linear : public WeightsPrimitiveFusingTestOneDNN {}; TEST_P(post_ops_optimizations_onednn_binary_add_eltw_linear, basic) { auto p = GetParam(); create_topologies(input_layout("input", get_input_layout(p)), @@ -9820,7 +9821,7 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, post_ops_optimizations_onednn_binary_add_e // Limitations: beta = 0 in eltw_linear; binary_mul is a constant compile-time buffer // DNNL_VERBOSE log without optimization: attr-oscale:2 attr-post-ops:binary_mul:f32:2+eltwise_linear:2.01575+eltwise_clip:0:512 // DNNL_VERBOSE log with optimization: attr-oscale:2 attr-post-ops:binary_mul:f32:2+eltwise_clip:0:512 -class post_ops_optimizations_onednn_binary_mul_eltw_linear : public ConvFusingTestOneDNN {}; +class post_ops_optimizations_onednn_binary_mul_eltw_linear : public WeightsPrimitiveFusingTestOneDNN {}; TEST_P(post_ops_optimizations_onednn_binary_mul_eltw_linear, basic) { auto p = GetParam(); create_topologies(input_layout("input", get_input_layout(p)), @@ -9871,7 +9872,7 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, post_ops_optimizations_onednn_binary_mul_e // Limitations: beta = 0 in eltw_linear // DNNL_VERBOSE log without optimization: attr-oscale:2 attr-post-ops:eltwise_linear:2.01575+eltwise_clip:0:512 // DNNL_VERBOSE log with optimization: attr-oscale:2 attr-post-ops:eltwise_clip:0:512 -class post_ops_optimizations_onednn_oscale_eltw_linear : public ConvFusingTestOneDNN {}; +class post_ops_optimizations_onednn_oscale_eltw_linear : public WeightsPrimitiveFusingTestOneDNN {}; TEST_P(post_ops_optimizations_onednn_oscale_eltw_linear, basic) { auto p = GetParam(); create_topologies(input_layout("input", get_input_layout(p)), @@ -9920,7 +9921,7 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, post_ops_optimizations_onednn_oscale_eltw_ // Limitations: beta = 0 in eltw_linear // DNNL_VERBOSE log without optimization: attr-post-ops:eltwise_relu+sum:1:0:u8+eltwise_linear:12.7+eltwise_clip:0:127 // DNNL_VERBOSE log with optimization: attr-post-ops:eltwise_relu:0:0:12.7+sum:12.7:0:u8+eltwise_clip:0:127 -class post_ops_optimizations_onednn_eltw_any_sum_eltw_linear : public ConvFusingTestOneDNN {}; +class post_ops_optimizations_onednn_eltw_any_sum_eltw_linear : public WeightsPrimitiveFusingTestOneDNN {}; TEST_P(post_ops_optimizations_onednn_eltw_any_sum_eltw_linear, basic) { auto p = GetParam(); create_topologies(input_layout("input", get_input_layout(p)), @@ -9970,7 +9971,7 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, post_ops_optimizations_onednn_eltw_any_sum // Input range uses in 2 cases: not per-tensor output range or out_lo > out_hi // Here's out_lo > out_hi and no optimizations // DNNL_VERBOSE log: attr-post-ops:eltwise_linear:12.75:127.5+eltwise_round+eltwise_linear:-1:127 -class post_ops_optimizations_input_range : public ConvFusingTestOneDNN {}; +class post_ops_optimizations_input_range : public WeightsPrimitiveFusingTestOneDNN {}; TEST_P(post_ops_optimizations_input_range, basic) { auto p = GetParam(); create_topologies(input_layout("input", get_input_layout(p)), @@ -10015,6 +10016,33 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, post_ops_optimizations_input_range, bc_test_params{CASE_CONV_S8S8_14, 2, 3}, bc_test_params{CASE_CONV_S8S8_15, 2, 3}, })); + +class fc_int8_inputs_fused_fp32_sum : public WeightsPrimitiveFusingTestOneDNN {}; +TEST_P(fc_int8_inputs_fused_fp32_sum, basic) { + auto p = GetParam(); + auto shift_layout = layout{ p.default_type, p.default_format, tensor{1, 1, 1, p.kernel.batch[0]} }; + + create_topologies(input_layout("input", get_input_layout(p)), + data("weights", get_mem(get_fc_weights_layout(p))), + data("bias", get_mem(get_fc_bias_layout(p))), + data("shift_data", get_mem(shift_layout, 1)), + fully_connected("fc_prim", "input", "weights", "bias", cldnn::data_types::f32, "", padding(), get_fc_output_dim_size(p)), + eltwise("shift", {"fc_prim", "shift_data"}, eltwise_mode::sum, cldnn::data_types::f32), + crop("crop", "shift", get_output_layout(p).size, {0, 0, 0, 0}), + reorder("reorder_bfyx", "crop", p.default_format, data_types::f32) + ); + + tolerance = 1e-5f; + execute(p); +} + +INSTANTIATE_TEST_SUITE_P(fusings_gpu, fc_int8_inputs_fused_fp32_sum, ::testing::ValuesIn(std::vector{ + // OneDNN has issue with small shapes - ticket 7064 + // bc_test_params{ CASE_FC_U8S8_3D_1, 2, 4 }, + // bc_test_params{ CASE_FC_U8S8_3D_2, 2, 4 }, + bc_test_params{ CASE_FC_U8S8_3D_4, 2, 4 }, + +})); #endif