From 7834dba545bc3a3db8cb20ff2035e13f36737720 Mon Sep 17 00:00:00 2001 From: Bo Liu Date: Wed, 29 Jun 2022 18:14:30 +0800 Subject: [PATCH] fix CPU Plugin deformable conv Node output incorrect issues with uneven dilations (#11940) --- src/core/tests/frontend/paddle/op_fuzzy.cpp | 16 + .../gen_scripts/generate_deformable_conv.py | 318 ++++++++++++++++++ src/plugins/intel_cpu/src/nodes/def_conv.cpp | 4 +- .../deformable_convolution.cpp | 23 ++ 4 files changed, 359 insertions(+), 2 deletions(-) create mode 100644 src/core/tests/frontend/paddle/test_models/gen_scripts/generate_deformable_conv.py diff --git a/src/core/tests/frontend/paddle/op_fuzzy.cpp b/src/core/tests/frontend/paddle/op_fuzzy.cpp index 2c6527091a3..7fc91fdacb6 100644 --- a/src/core/tests/frontend/paddle/op_fuzzy.cpp +++ b/src/core/tests/frontend/paddle/op_fuzzy.cpp @@ -78,6 +78,22 @@ static const std::vector models{ std::string("cumsum_i64"), std::string("cumsum_f32"), std::string("cumsum_f64"), + std::string("deformable_conv_default"), + std::string("deformable_conv_with_bias"), + std::string("deformable_conv_with_deformable_groups"), + std::string("deformable_conv_with_dilation"), + std::string("deformable_conv_with_dilation_list"), + std::string("deformable_conv_with_dilation_tuple"), + std::string("deformable_conv_with_groups"), + // (CVS-86585: about e-5 level accuracy diff) + // std::string("deformable_conv_with_mask"), + // std::string("deformable_conv_with_mask_bias"), + std::string("deformable_conv_with_pad"), + std::string("deformable_conv_with_pad_list"), + std::string("deformable_conv_with_pad_tuple"), + std::string("deformable_conv_with_stride"), + std::string("deformable_conv_with_stride_list"), + std::string("deformable_conv_with_stride_tuple"), std::string("depthwise_conv2d_convolution"), std::string("depthwise_conv2d_transpose_convolution"), std::string("dropout"), diff --git a/src/core/tests/frontend/paddle/test_models/gen_scripts/generate_deformable_conv.py b/src/core/tests/frontend/paddle/test_models/gen_scripts/generate_deformable_conv.py new file mode 100644 index 00000000000..2080d489d5c --- /dev/null +++ b/src/core/tests/frontend/paddle/test_models/gen_scripts/generate_deformable_conv.py @@ -0,0 +1,318 @@ +# +# pool2d paddle model generator +# +import numpy as np +import sys +from typing import List + +from save_model import saveModel + + +# helpers +def pdpd_attr_to_list(s, p, d): + if isinstance(s, int): + strides = [s, ]*2 + elif isinstance(s, tuple): + strides = list(s) + elif isinstance(s, list): + strides = s + else: + raise ValueError('unknown type of strides!') + assert len(strides) == 2, 'len(strides) must be 2!' + + if isinstance(d, int): + dilations = [d, ]*2 + elif isinstance(d, tuple): + dilations = list(d) + elif isinstance(d, list): + dilations = d + else: + raise ValueError('unknown type of dilations!') + assert len(dilations) == 2, 'len(dilations) must be 2!' + + if isinstance(p, int): + padding = [p, ]*2 + elif isinstance(p, tuple): + padding = list(p) + elif isinstance(p, list): + padding = p + else: + raise ValueError('unknown type of padding!') + assert len(padding) == 2, 'len(padding) must be 2!' + + return strides, padding, dilations + + +def deformable_conv(name: str, x, weight, offset, mask, bias, stride=1, padding=0, dilation=1, deformable_groups=1, groups=1): + import paddle as pdpd + pdpd.enable_static() + + with pdpd.static.program_guard(pdpd.static.Program(), pdpd.static.Program()): + node_x = pdpd.static.data(name='x', shape=x.shape, dtype=x.dtype) + node_offset = pdpd.static.data( + name='offset', shape=offset.shape, dtype=offset.dtype) + node_weight = pdpd.static.data( + name='weight', shape=weight.shape, dtype=weight.dtype) + if mask is not None: + node_mask = pdpd.static.data( + name='mask', shape=mask.shape, dtype=mask.dtype) + if bias is not None: + node_bias = pdpd.static.data( + name='bias', shape=bias.shape, dtype=bias.dtype) + + pdpd.static.nn.deform_conv2d + + node_out = pdpd.vision.ops.deform_conv2d(node_x, + node_offset, + node_weight, + bias=node_bias if bias is not None else None, + stride=stride, # int|list|tuple + padding=padding, # int|list|tuple + dilation=dilation, # int|list|tuple + deformable_groups=deformable_groups, # int + groups=groups, # int + mask=node_mask if mask is not None else None) + + cpu = pdpd.static.cpu_places(1) + exe = pdpd.static.Executor(cpu[0]) + # startup program will call initializer to initialize the parameters. + exe.run(pdpd.static.default_startup_program()) + + feed_dict = {'x': x, 'offset': offset, 'weight': weight} + inputs_list = [x, offset, weight] + if mask is not None: + feed_dict['mask'] = mask + inputs_list.append(mask) + if bias is not None: + feed_dict['bias'] = bias + inputs_list.append(bias) + outs = exe.run( + feed=feed_dict, + fetch_list=node_out) + + # Save inputs in order of ngraph function, to facilite Fuzzy test, + # which accepts inputs and outputs in this order as well. + saveModel(name, exe, feedkeys=list(feed_dict.keys()), fetchlist=[node_out], + inputs=inputs_list, outputs=outs, target_dir=sys.argv[1] if len(sys.argv) > 1 else '.') + + +def get_output_size(input_size, out_channels, kernel_size, stride, padding, dilation): + # calculate output shape of conv2d + def out_size(in_size, pad_size, dilation_size, kernel_size, + stride_size): + return (in_size + 2 * pad_size - + (dilation_size * (kernel_size - 1) + 1)) / stride_size + 1 + + stride, padding, dilation = pdpd_attr_to_list(stride, padding, dilation) + + out_h = int( + out_size(input_size[2], padding[0], dilation[0], + kernel_size[0], stride[0])) + assert out_h > 0 + + out_w = int( + out_size(input_size[3], padding[1], dilation[1], + kernel_size[1], stride[1])) + assert out_w > 0 + + return [input_size[0], out_channels, out_h, out_w] + + +# ref: https://www.paddlepaddle.org.cn/documentation/docs/en/api/paddle/vision/ops/deform_conv2d_en.html +# PaddlePaddle conv attributes padding, strides, dilation can optionally be int|lit|tuple, +# while they are list of int in ngraph. +def generator(input_size=[2, 8, 4, 4], # NCHW + out_channels=4, + kernel_size=[3, 3], # spatial_kernel + padding=[0, 0], # int|lit|tuple + stride=[1, 1], # int|lit|tuple + dilation=[1, 1], # int|lit|tuple + groups=1, deformable_groups=1, # int + no_bias=True, no_mask=True, dtype='float32'): + # np.random.seed(1) + if isinstance(kernel_size, int): + kernel_size = [kernel_size, ] * 2 + else: + assert len(kernel_size) == 2 + + assert np.mod(input_size[1], groups) == 0 + f_c = input_size[1] // groups + filter_size = [out_channels, f_c] + kernel_size # weight + + output_size = get_output_size(input_size, out_channels, kernel_size, + stride, padding, dilation) # output + + offset_c = 2 * deformable_groups * filter_size[ + 2] * filter_size[3] + mask_c = deformable_groups * filter_size[ + 2] * filter_size[3] + offset_size = [ + input_size[0], offset_c, output_size[2], output_size[3] # offset + ] + mask_size = [ + input_size[0], mask_c, output_size[2], output_size[3] # mask + ] + + # data + test_x = np.random.random(size=input_size).astype(dtype) + + weight = np.random.random(size=filter_size).astype(dtype) + + offset = 10 * \ + np.random.uniform(size=offset_size).astype( + 'int').astype(dtype) # TODO: negative, fractioned + + mask = 10 * \ + np.random.random(size=mask_size).astype(dtype) if not no_mask else None + + bias = np.random.uniform(-1, 1, + size=(filter_size[0],)).astype(dtype) if not no_bias else None + + return [test_x, weight, offset, mask, bias] + + +def TEST1(): + data_x, data_weight, data_offset, data_mask, data_bias = generator() + deformable_conv('deformable_conv_default', data_x, + data_weight, data_offset, data_mask, data_bias) + + +def TestWithPad(): + padding = 1 + data_x, data_weight, data_offset, data_mask, data_bias = generator(input_size=[1, 1, 4, 4], + kernel_size=[3, 3], out_channels=1, + padding=padding) + deformable_conv('deformable_conv_with_pad', data_x, + data_weight, data_offset, data_mask, data_bias, padding=padding) + + # data_mask = None + # data_bias = None + + # data_x = np.random.random((1, 1, 3, 5)).astype('float32') + # data_weight = np.random.random((1, 1, 3, 3)).astype('float32') + # data_offset = np.random.random((1, 18, 1, 1)).astype('float32') + + # deformable_conv('deformable_conv_with_pad', data_x, + # data_weight, data_offset, data_mask, data_bias, dilation=[1,2]) + + +def TestWithPadTuple(): + padding = (3, 3) + data_x, data_weight, data_offset, data_mask, data_bias = generator( + padding=padding) + deformable_conv('deformable_conv_with_pad_tuple', data_x, + data_weight, data_offset, data_mask, data_bias, padding=padding) + + +def TestWithPadList(): + padding = [3, 3] + data_x, data_weight, data_offset, data_mask, data_bias = generator( + padding=padding) + deformable_conv('deformable_conv_with_pad_list', data_x, + data_weight, data_offset, data_mask, data_bias, padding=padding) + + +def TestWithStride(): + stride = 2 + data_x, data_weight, data_offset, data_mask, data_bias = generator( + stride=stride) + deformable_conv('deformable_conv_with_stride', data_x, + data_weight, data_offset, data_mask, data_bias, stride=stride) + + +def TestWithStrideTuple(): + stride = [2, 3] + data_x, data_weight, data_offset, data_mask, data_bias = generator( + stride=stride) + deformable_conv('deformable_conv_with_stride_tuple', data_x, + data_weight, data_offset, data_mask, data_bias, stride=stride) + + +def TestWithStrideList(): + stride = [3, 2] + data_x, data_weight, data_offset, data_mask, data_bias = generator( + stride=stride) + deformable_conv('deformable_conv_with_stride_list', data_x, + data_weight, data_offset, data_mask, data_bias, stride=stride) + + +def TestWithDilation(): + dilation = 2 + data_x, data_weight, data_offset, data_mask, data_bias = generator( + input_size=[1, 1, 7, 7], dilation=dilation) + deformable_conv('deformable_conv_with_dilation', data_x, + data_weight, data_offset, data_mask, data_bias, dilation=dilation) + + +def TestWithDilationTuple(): + dilation = (2, 3) + data_x, data_weight, data_offset, data_mask, data_bias = generator( + input_size=[1, 1, 7, 7], dilation=dilation) + deformable_conv('deformable_conv_with_dilation_tuple', data_x, + data_weight, data_offset, data_mask, data_bias, dilation=dilation) + + +def TestWithDilationList(): + dilation = [3, 2] + data_x, data_weight, data_offset, data_mask, data_bias = generator( + input_size=[1, 1, 7, 7], dilation=dilation) + deformable_conv('deformable_conv_with_dilation_list', data_x, + data_weight, data_offset, data_mask, data_bias, dilation=dilation) + + +def TestWithGroup(): + data_x, data_weight, data_offset, data_mask, data_bias = generator( + groups=2) + deformable_conv('deformable_conv_with_groups', data_x, + data_weight, data_offset, data_mask, data_bias, groups=2) + + +def TestWithDeformableGroups(): + data_x, data_weight, data_offset, data_mask, data_bias = generator( + deformable_groups=2) + deformable_conv('deformable_conv_with_deformable_groups', data_x, + data_weight, data_offset, data_mask, data_bias, deformable_groups=2) + + +def TestWithMask(): + data_x, data_weight, data_offset, data_mask, data_bias = generator( + no_mask=False) + deformable_conv('deformable_conv_with_mask', data_x, + data_weight, data_offset, data_mask, data_bias) + + +def TestWithBias(): + data_x, data_weight, data_offset, data_mask, data_bias = generator( + no_bias=False) + deformable_conv('deformable_conv_with_bias', data_x, + data_weight, data_offset, data_mask, data_bias) + + +def TestWithMaskBias(): + data_x, data_weight, data_offset, data_mask, data_bias = generator( + no_mask=False, no_bias=False) + deformable_conv('deformable_conv_with_mask_bias', data_x, + data_weight, data_offset, data_mask, data_bias) + + +if __name__ == "__main__": + TEST1() + + TestWithPad() + TestWithPadTuple() + TestWithPadList() + + TestWithStride() + TestWithStrideTuple() + TestWithStrideList() + + TestWithDilation() + TestWithDilationTuple() + TestWithDilationList() + + TestWithGroup() + TestWithDeformableGroups() + + TestWithMask() + TestWithBias() + TestWithMaskBias() diff --git a/src/plugins/intel_cpu/src/nodes/def_conv.cpp b/src/plugins/intel_cpu/src/nodes/def_conv.cpp index 53a836ca218..6d811408413 100644 --- a/src/plugins/intel_cpu/src/nodes/def_conv.cpp +++ b/src/plugins/intel_cpu/src/nodes/def_conv.cpp @@ -700,8 +700,8 @@ DeformableConvolution::DeformableConvolution(const std::shared_ptr } auto& dilations = defConvNodeBase->get_dilations(); - for (int i = 1; i <= dilations.size(); i++) { - defConvAttr.dilation.push_back(dilations[dilations.size() - i] - 1); + for (int i = 0; i < dilations.size(); i++) { + defConvAttr.dilation.push_back(dilations[i] - 1); } defConvAttr.padL = defConvNodeBase->get_pads_begin(); diff --git a/src/tests/functional/plugin/cpu/single_layer_tests/deformable_convolution.cpp b/src/tests/functional/plugin/cpu/single_layer_tests/deformable_convolution.cpp index 9086506b040..81ddd3447ca 100644 --- a/src/tests/functional/plugin/cpu/single_layer_tests/deformable_convolution.cpp +++ b/src/tests/functional/plugin/cpu/single_layer_tests/deformable_convolution.cpp @@ -306,6 +306,14 @@ const auto addSpParams = ::testing::Combine( ::testing::Values(std::vector {1, 1}) // dilations ); +const auto addSpParamsDilationUneven = ::testing::Combine( + ::testing::ValuesIn(padTypes), // pad. type + ::testing::Values(std::vector({0, 0})), // pad. begin + ::testing::Values(std::vector({0, 0})), // pad. end + ::testing::Values(std::vector {1, 1}), // strides + ::testing::Values(std::vector {2, 1}) // dilations +); + const std::vector> spatParams1 = { {1}, // batch {34, 34}, // in. spat. shape @@ -330,6 +338,12 @@ const std::vector> spatParams4 = { {2, 1}, // off. spat. shape {2, 2} // ker. spat. shape }; +const std::vector> spatParamsDilationUneven = { + {1}, // batch + {3, 2}, // in. spat. shape + {1, 1}, // off. spat. shape + {2, 2} // ker. spat. shape +}; const std::vector> channelParamsSingleGr = { {1}, // gr. 2,4 {1, 2}, // def. gr. 1,2 @@ -579,6 +593,14 @@ const auto params9 = ::testing::Combine( ::testing::ValuesIn(netPrecisions), ::testing::Values(CommonTestUtils::DEVICE_CPU)), ::testing::ValuesIn(filterCPUInfoForDevice(false))); +const auto params10 = ::testing::Combine( + ::testing::Combine( + addSpParamsDilationUneven, + ::testing::ValuesIn(static_shapes_to_test_representation(buildStaticParams(spatParamsDilationUneven, channelParamsSingleGr))), + defConvSpecificParams, + ::testing::ValuesIn(netPrecisions), + ::testing::Values(CommonTestUtils::DEVICE_CPU)), + ::testing::ValuesIn(filterCPUInfoForDevice(false))); INSTANTIATE_TEST_SUITE_P(DefConvLayoutTest1, DefConvLayerCPUTest, params1, DefConvLayerCPUTest::getTestCaseName); INSTANTIATE_TEST_SUITE_P(DefConvLayoutTest2, DefConvLayerCPUTest, params2, DefConvLayerCPUTest::getTestCaseName); @@ -589,5 +611,6 @@ INSTANTIATE_TEST_SUITE_P(DefConvLayoutTest6, DefConvLayerCPUTest, params6, DefCo INSTANTIATE_TEST_SUITE_P(DefConvLayoutTest7, DefConvLayerCPUTest, params7, DefConvLayerCPUTest::getTestCaseName); INSTANTIATE_TEST_SUITE_P(DefConvLayoutTest8, DefConvLayerCPUTest, params8, DefConvLayerCPUTest::getTestCaseName); INSTANTIATE_TEST_SUITE_P(DefConvLayoutTest9, DefConvLayerCPUTest, params9, DefConvLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(DefConvLayoutTest10, DefConvLayerCPUTest, params10, DefConvLayerCPUTest::getTestCaseName); } // namespace } // namespace CPULayerTestsDefinitions