From 74a608592a05b5846df3aba28d3f274905cae79a Mon Sep 17 00:00:00 2001 From: Chen Xu Date: Tue, 1 Aug 2023 19:56:36 +0800 Subject: [PATCH] [CPU] Fix Reduce node bug regarding fusing with low precision (#18835) --- src/plugins/intel_cpu/src/nodes/reduce.cpp | 14 +++---- src/plugins/intel_cpu/src/nodes/reduce.h | 2 +- .../instances/x64/reduce.cpp | 42 +++++++++++++++++++ 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/src/plugins/intel_cpu/src/nodes/reduce.cpp b/src/plugins/intel_cpu/src/nodes/reduce.cpp index 5bab74b7fbb..8c964caf598 100644 --- a/src/plugins/intel_cpu/src/nodes/reduce.cpp +++ b/src/plugins/intel_cpu/src/nodes/reduce.cpp @@ -2154,7 +2154,7 @@ void Reduce::createPrimitive() { auto reduce_jcp = jcp; reduce_jcp.dst_dt = fuse_low_precision ? DnnlExtensionUtils::IEPrecisionToDataType(intermediate_prec) : jcp.dst_dt; - jcp.dst_data_size = DnnlExtensionUtils::sizeOfDataType(reduce_jcp.dst_dt); + reduce_jcp.dst_data_size = DnnlExtensionUtils::sizeOfDataType(reduce_jcp.dst_dt); create_reduce_kernel(reduce_kernel, reduce_jcp); // set_use_aux_kernel being false means this is a dynamic case, and prepareParams() hasn't been invoked yet. @@ -2255,7 +2255,7 @@ void Reduce::reduce_type(const uint8_t *in_ptr, uint8_t *out_ptr) { } void Reduce::reduce_PLN(const uint8_t *in_ptr, uint8_t *out_ptr) { - output_info_reassign(out_ptr); + output_info_reassign(&out_ptr); init_dst_data(out_ptr, dst_size); if (ReduceN && !ReduceC && !ReduceD && !ReduceH && !ReduceW) { @@ -2472,7 +2472,7 @@ void Reduce::reduce_PLN(const uint8_t *in_ptr, uint8_t *out_ptr) { void Reduce::reduce_BLK(const uint8_t *in_ptr, uint8_t *out_ptr) { size_t ICB = div_up(IC, blk_size); size_t OCB = div_up(OC, blk_size); - output_info_reassign(out_ptr); + output_info_reassign(&out_ptr); init_dst_data(out_ptr, dst_size); for (size_t ib = 0; ib < IB; ib++) { @@ -2552,7 +2552,7 @@ void Reduce::reduce_BLK(const uint8_t *in_ptr, uint8_t *out_ptr) { void Reduce::reduce_BLK_concern_padding(const uint8_t *in_ptr, uint8_t *out_ptr) { size_t ICB = div_up(IC, blk_size); size_t OCB = div_up(OC, blk_size); - output_info_reassign(out_ptr); + output_info_reassign(&out_ptr); init_dst_data(out_ptr, dst_size); auto reduceSkipPadding = [&](const uint8_t *in_ptr_ncd, uint8_t *out_ptr_ncd, size_t ic) { @@ -2720,10 +2720,10 @@ inline void Reduce::reduce_kernel_restore() { } } -inline void Reduce::output_info_reassign(uint8_t *out_ptr) { +inline void Reduce::output_info_reassign(uint8_t **out_ptr) { if (fuse_low_precision) { - tmp_ptr = out_ptr; - out_ptr = static_cast(&intermediate_buf[0]); + tmp_ptr = *out_ptr; + *out_ptr = static_cast(&intermediate_buf[0]); tmp_prec = output_prec; output_prec = intermediate_prec; tmp_data_size = dst_data_size; diff --git a/src/plugins/intel_cpu/src/nodes/reduce.h b/src/plugins/intel_cpu/src/nodes/reduce.h index 2f07cb196a7..08890a21186 100644 --- a/src/plugins/intel_cpu/src/nodes/reduce.h +++ b/src/plugins/intel_cpu/src/nodes/reduce.h @@ -117,7 +117,7 @@ private: inline void reduce_kernel_post_process(uint8_t *out_ptr); inline void reduce_kernel_reassign(); inline void reduce_kernel_restore(); - inline void output_info_reassign(uint8_t *out_ptr); + inline void output_info_reassign(uint8_t **out_ptr); inline void output_info_restore(uint8_t **out_ptr); inline void init_dst_data(uint8_t *out_ptr, size_t dst_size); inline void create_hybrid_working_memory(); diff --git a/src/plugins/intel_cpu/tests/functional/single_layer_tests/instances/x64/reduce.cpp b/src/plugins/intel_cpu/tests/functional/single_layer_tests/instances/x64/reduce.cpp index a4198f5b15d..e267c938f54 100644 --- a/src/plugins/intel_cpu/tests/functional/single_layer_tests/instances/x64/reduce.cpp +++ b/src/plugins/intel_cpu/tests/functional/single_layer_tests/instances/x64/reduce.cpp @@ -6,6 +6,7 @@ #include "shared_test_classes/single_layer/reduce_ops.hpp" #include "test_utils/cpu_test_utils.hpp" #include "test_utils/fusing_test_utils.hpp" +#include "lpt_ngraph_functions/common/builders.hpp" using namespace InferenceEngine; using namespace CPUTestUtils; @@ -110,6 +111,22 @@ const std::vector reductionTypesFusing = { ngraph::helpers::ReductionType::L2, }; +// This custom subgraph is used to test post-ops fusing case with U8/I8 precision on output, +// since Transpose prevents dequantization part to be fused back into Reduce +const auto fusingFakeQuantizeTranspose = fusingSpecificParams{std::make_shared(std::vector{ + {[](postNodeConfig& cfg){ + auto localPrc = cfg.input->get_element_type(); + ngraph::Shape newShape(cfg.input->get_output_partial_shape(0).size(), 1); + const auto fakeQuantize = ngraph::builder::makeFakeQuantize(cfg.input, localPrc, 256, newShape); + std::vector order(newShape.size()); + std::iota(order.begin(), order.end(), 0); + auto last = order[order.size() - 1]; + order.pop_back(); + order.insert(order.begin(), last); + const auto transpose = ngraph::builder::subgraph::Transpose(order); + return ngraph::builder::subgraph::makeTranspose(fakeQuantize, transpose); + }, "FakeQuantize(PerTensor)"}}), {"FakeQuantize"}}; + const std::vector fusingParamsSet { /* activations */ fusingSwish, @@ -132,6 +149,10 @@ const std::vector fusingParamsSet_KeepNoDims { fusingScaleShift }; +const std::vector fusingParamsSet_LowPrecision { + fusingFakeQuantizeTranspose +}; + /* ================================ 1.1 No fusion - Arithmetic ================================ */ const auto params_OneAxis = testing::Combine( testing::Combine( @@ -492,6 +513,20 @@ const auto params_MultiAxis_5D_fusing = testing::Combine( testing::ValuesIn(fusingParamsSet), testing::ValuesIn(additionalConfig())); +const auto params_LowPrecision_fusing = testing::Combine( + testing::Combine( + testing::ValuesIn(axesNDFusing), + testing::Values(ov::test::utils::OpType::VECTOR), + testing::Values(true), + testing::ValuesIn(reductionTypesFusing), + testing::ValuesIn(inpOutPrc()), + testing::Values(ElementType::undefined), + testing::Values(ElementType::undefined), + testing::ValuesIn(inputShapes_dyn)), + testing::ValuesIn(filterCPUSpecificParams(cpuParams_4D)), + testing::ValuesIn(fusingParamsSet_LowPrecision), + testing::ValuesIn(additionalConfig())); + INSTANTIATE_TEST_SUITE_P( smoke_Reduce_OneAxis_fusing_CPU, ReduceCPULayerTest, @@ -513,6 +548,13 @@ INSTANTIATE_TEST_SUITE_P( ReduceCPULayerTest::getTestCaseName ); +INSTANTIATE_TEST_SUITE_P( + smoke_Reduce_LowPrecision_fusing_CPU, + ReduceCPULayerTest, + params_LowPrecision_fusing, + ReduceCPULayerTest::getTestCaseName +); + /* ================================ 2.2 Fusion - KeepNoDims ================================ */ const auto params_OneAxis_fusing_KeepNoDims = testing::Combine( testing::Combine(