diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_def_conv_node.cpp b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_def_conv_node.cpp index 060624bdab0..969cf85d130 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_def_conv_node.cpp +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_def_conv_node.cpp @@ -3,10 +3,6 @@ // #include "mkldnn_def_conv_node.h" -#include "mkldnn_reorder_node.h" -#include "mkldnn_input_node.h" - -#include "mkldnn_eltwise_node.h" #include #include #include @@ -31,6 +27,8 @@ template struct jit_uni_def_conv_kernel_f32 : public jit_uni_def_conv_kernel, public jit_generator { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_def_conv_kernel_f32) + constexpr static int sampledPointsPerPixel = MKLDNNDeformableConvolutionNode::sampledPointsPerPixel; + explicit jit_uni_def_conv_kernel_f32(jit_def_conv_params jcp) : jit_uni_def_conv_kernel(jcp), jit_generator() {} void create_ker() override { @@ -42,13 +40,20 @@ struct jit_uni_def_conv_kernel_f32 : public jit_uni_def_conv_kernel, public jit_ this->preamble(); mov(reg_input, ptr[this->param1 + GET_OFF(src)]); - mov(reg_def_off, ptr[this->param1 + GET_OFF(off)]); + mov(reg_sampled_wei, ptr[this->param1 + GET_OFF(sampledWei)]); + mov(reg_sampled_offs, ptr[this->param1 + GET_OFF(sampledCoords)]); + mov(reg_kernel, ptr[this->param1 + GET_OFF(filt)]); if (jcp_.with_bias) mov(reg_bias, ptr[this->param1 + GET_OFF(bias)]); mov(reg_output, ptr[this->param1 + GET_OFF(dst)]); - mov(reg_input_buffer, ptr[this->param1 + GET_OFF(buf)]); - mov(reg_oh_pos, ptr[param1 + GET_OFF(oh_pos)]); + mov(reg_input_buffer_temp, ptr[this->param1 + GET_OFF(buf)]); + mov(oh_pos_temp, ptr[param1 + GET_OFF(oh_pos)]); + + // need to save temporary to prevent using of %rdi during GET_OFF(...) + mov(reg_oh_pos, oh_pos_temp); + // prevents mismatching param1 == %rcx (on windows) and reg_input_buffer + mov(reg_input_buffer, reg_input_buffer_temp); ow_loop(); @@ -69,11 +74,11 @@ private: using reg8_t = const Xbyak::Reg8; reg64_t reg_input = r8; - reg64_t reg_def_off = r9; + reg64_t reg_sampled_wei = r9; reg64_t reg_kernel = r10; reg64_t reg_bias = r11; reg64_t reg_output = r12; - reg64_t reg_oh_pos = r13; + reg64_t reg_oh_pos = rdi; reg64_t aux_reg_bias = rsi; reg64_t reg_ow_pos = rdx; reg64_t aux_reg_output = reg_ow_pos; @@ -82,16 +87,20 @@ private: reg64_t aux2_reg_input = reg_kernel; reg64_t reg_ic_iter = rbx; reg64_t reg_oc_work = reg_ic_iter; - reg64_t aux_reg_def_off = reg_bias; - reg64_t reg_input_buffer = abi_not_param1; + reg64_t aux_reg_sampled_wei = reg_bias; + reg64_t reg_input_buffer = rcx; reg64_t aux_reg_input_buffer = r14; reg32_t reg_tmp_32 = r15d; reg64_t reg_tmp_64 = r15; reg64_t reg_table = rbp; reg64_t aux_reg_kernel = reg_table; reg64_t aux2_reg_kernel = r15; + reg64_t oh_pos_temp = aux2_reg_kernel; reg64_t aux2_reg_input_buffer = aux_reg_bias; + reg64_t reg_sampled_offs = aux2_reg_input_buffer; reg64_t aux3_reg_input_buffer = reg_input; + reg64_t aux_reg_sampled_offs = r13; + reg64_t reg_input_buffer_temp = aux_reg_sampled_offs; Xbyak::Opmask ktail_mask = Xbyak::Opmask(2); @@ -117,9 +126,10 @@ private: jg(ow_tail, T_NEAR); oc_loop(jcp_.ur_w); - add(reg_input, jcp_.ur_w * jcp_.stride_w * jcp_.ic * jcp_.typesize_in); - add(reg_def_off, jcp_.ur_w * jcp_.typesize_off); + add(reg_sampled_wei, jcp_.ur_w * jcp_.kh * jcp_.kw * sampledPointsPerPixel * jcp_.typesize_sampled_wei); // type = float + add(reg_sampled_offs, jcp_.ur_w * jcp_.kh * jcp_.kw * sampledPointsPerPixel * jcp_.typesize_sampled_offsets); // type = int + add(reg_output, jcp_.ur_w * jcp_.oc * jcp_.typesize_out); add(reg_ow_pos, jcp_.ur_w); @@ -189,7 +199,6 @@ private: if (isa == cpu::x64::sse41 && ow > 0) { uni_vmovups(vmm_ker, ptr[aux2_reg_kernel + ker_off * jcp_.typesize_in]); } - uni_vfmadd231ps(vmm_acc, vmm_ker, vmm_src); } } @@ -205,7 +214,6 @@ private: for (int ocb = 0; ocb < oc_blocks_step; ocb++) { for (int ow = 0; ow < ow_step; ow++) { Vmm vmm_acc = get_vmm_acc(r * jcp_.ur_w * jcp_.nb_oc_blocking + ocb * ow_step + ow); - uni_vpxor(vmm_acc, vmm_acc, vmm_acc); } } @@ -219,10 +227,10 @@ private: push(reg_oc_work); push(aux_reg_bias); + push(reg_sampled_offs); mov(aux2_reg_kernel, aux_reg_kernel); mov(aux2_reg_input_buffer, reg_input_buffer); - mov(reg_ic_iter, jcp_.ic); init_accums(ow_step, oc_blocks_step, oc_step); @@ -232,7 +240,6 @@ private: jl(ic_tail, T_NEAR); apply_filter(ow_step, oc_blocks_step, oc_step, jcp_.ic_block); - add(aux2_reg_input_buffer, jcp_.ic_block * jcp_.typesize_in); add(aux2_reg_kernel, jcp_.kh * jcp_.kw * jcp_.ic_block * jcp_.oc_block * jcp_.typesize_in); sub(reg_ic_iter, jcp_.ic_block); @@ -245,17 +252,20 @@ private: } } + pop(reg_sampled_offs); pop(aux_reg_bias); pop(reg_oc_work); } - void interpolate_input(int ow_step) { + void prepare_buffer(int ow_step) { Label dg_loop; Label dg_loop_end; mov(reg_table, l_table); - mov(aux_reg_def_off, reg_def_off); + mov(aux_reg_sampled_wei, reg_sampled_wei); + mov(aux_reg_sampled_offs, reg_sampled_offs); mov(aux_reg_input, reg_input); + push(reg_sampled_offs); mov(aux2_reg_input_buffer, aux_reg_input_buffer); xor_(reg_dg_iter, reg_dg_iter); @@ -267,15 +277,9 @@ private: for (int ow = 0; ow < ow_step; ow++) { for (int kh = 0; kh < jcp_.kh; kh++) { for (int kw = 0; kw < jcp_.kw; kw++) { - Label init_with_zeros; Label ic_loop_main; Label ic_loop_tail; - Label ic_loop_zeros; Label loop_end; - Label h_sec_opt; - Label h_sec_opt_exit; - Label w_sec_opt; - Label w_sec_opt_exit; mov(aux2_reg_input, aux_reg_input); add(aux2_reg_input, (ow * jcp_.stride_w * jcp_.ic) * jcp_.typesize_in); @@ -283,200 +287,51 @@ private: mov(aux3_reg_input_buffer, aux2_reg_input_buffer); add(aux3_reg_input_buffer, (ow * jcp_.kh * jcp_.kw * jcp_.ic) * jcp_.typesize_in); - Xmm xmm_tmp = Xmm(0); - - Xmm xmm_map_h = Xmm(2); - Xmm xmm_ih_in = Xmm(4); - Xmm xmm_ih_im = Xmm(1); - Xmm xmm_cur_height = xmm_ih_im; - Xmm xmm_h_low = xmm_ih_in; - Xmm xmm_h_high = xmm_cur_height; - Xmm xmm_lh = xmm_map_h; - Xmm xmm_hh = Xmm(3); - - Xmm xmm_map_w = Xmm(6); - Xmm xmm_iw_in = Xmm(8); - Xmm xmm_iw_im = Xmm(5); - Xmm xmm_cur_width = xmm_iw_im; - Xmm xmm_w_low = xmm_iw_in; - Xmm xmm_w_high = xmm_cur_width; - Xmm xmm_lw = xmm_map_w; - Xmm xmm_hw = Xmm(7); - Xmm xmm_v1_off = Xmm(9); Xmm xmm_v2_off = Xmm(10); Xmm xmm_v3_off = Xmm(11); Xmm xmm_v4_off = Xmm(12); - Xmm xmm_w1 = xmm_h_low; - Xmm xmm_w2 = xmm_h_high; - Xmm xmm_w3 = xmm_w_low; - Xmm xmm_w4 = xmm_w_high; + Xmm xmm_w1 = Xmm(4); + Xmm xmm_w2 = Xmm(1); + Xmm xmm_w3 = Xmm(8); + Xmm xmm_w4 = Xmm(5); - Xmm xmm_v1 = xmm_lh; - Xmm xmm_v2 = xmm_hh; - Xmm xmm_v3 = xmm_lw; - Xmm xmm_v4 = xmm_hw; + Xmm xmm_v1 = Xmm(2); + Xmm xmm_v2 = Xmm(3);; + Xmm xmm_v3 = Xmm(6); + Xmm xmm_v4 = Xmm(7); - Vmm vmm_w1 = Vmm(xmm_h_low.getIdx()); - Vmm vmm_w2 = Vmm(xmm_h_high.getIdx()); - Vmm vmm_w3 = Vmm(xmm_w_low.getIdx()); - Vmm vmm_w4 = Vmm(xmm_w_high.getIdx()); + Vmm vmm_w1 = Vmm(xmm_w1.getIdx()); + Vmm vmm_w2 = Vmm(xmm_w2.getIdx()); + Vmm vmm_w3 = Vmm(xmm_w3.getIdx()); + Vmm vmm_w4 = Vmm(xmm_w4.getIdx()); - Vmm vmm_v1 = Vmm(xmm_lh.getIdx()); - Vmm vmm_v2 = Vmm(xmm_hh.getIdx()); - Vmm vmm_v3 = Vmm(xmm_lw.getIdx()); - Vmm vmm_v4 = Vmm(xmm_hw.getIdx()); + Vmm vmm_v1 = Vmm(xmm_v1.getIdx()); + Vmm vmm_v2 = Vmm(xmm_v2.getIdx()); + Vmm vmm_v3 = Vmm(xmm_v3.getIdx()); + Vmm vmm_v4 = Vmm(xmm_v4.getIdx()); - size_t def_off_h = ((2 * (kh * jcp_.kw + kw) + 0) * jcp_.oh * jcp_.ow) + ow; - mov(reg_tmp_32, ptr[aux_reg_def_off + def_off_h * jcp_.typesize_off]); - movq(xmm_tmp, reg_tmp_64); - mov(reg_tmp_32, cpu::x64::float2int(static_cast((kh * (jcp_.dilate_h + 1))))); - movq(xmm_map_h, reg_tmp_64); - addss(xmm_map_h, xmm_tmp); + // offsets computation + size_t ind_off_hh = sampledPointsPerPixel * (((size_t) kh * jcp_.kw + kw) + ow * (jcp_.kh * jcp_.kw)); + size_t ind_off_hl = ind_off_hh + 1; + size_t ind_off_lh = ind_off_hl + 1; + size_t ind_off_ll = ind_off_lh + 1; - mov(reg_tmp_32, jcp_.stride_h); - imul(reg_tmp_32, reg_oh_pos); - sub(reg_tmp_32, jcp_.t_pad); - movq(xmm_ih_in, reg_tmp_64); + movq(xmm_v1_off, qword[aux_reg_sampled_offs + ind_off_ll * jcp_.typesize_sampled_offsets]); + movq(xmm_v2_off, qword[aux_reg_sampled_offs + ind_off_hl * jcp_.typesize_sampled_offsets]); + movq(xmm_v3_off, qword[aux_reg_sampled_offs + ind_off_lh * jcp_.typesize_sampled_offsets]); + movq(xmm_v4_off, qword[aux_reg_sampled_offs + ind_off_hh * jcp_.typesize_sampled_offsets]); - cvtsi2ss(xmm_ih_im, reg_tmp_32); - addss(xmm_ih_im, xmm_map_h); - - movss(xmm_tmp, xmm_ih_im); - cmpss(xmm_tmp, table_val(0), 1); - movq(reg_tmp_64, xmm_tmp); - cmp(reg_tmp_32, 0); - jne(init_with_zeros, T_NEAR); - - cmpss(xmm_ih_im, table_val(1), 1); - movq(reg_tmp_64, xmm_ih_im); - cmp(reg_tmp_32, 0); - je(init_with_zeros, T_NEAR); - - - size_t def_off_w = ((2 * (kh * jcp_.kw + kw) + 1) * jcp_.oh * jcp_.ow) + ow; - mov(reg_tmp_32, ptr[aux_reg_def_off + def_off_w * jcp_.typesize_off]); - movq(xmm_tmp, reg_tmp_64); - mov(reg_tmp_32, cpu::x64::float2int(static_cast((kw * (jcp_.dilate_w + 1))))); - movq(xmm_map_w, reg_tmp_64); - addss(xmm_map_w, xmm_tmp); - - mov(reg_tmp_32, jcp_.stride_w); - imul(reg_tmp_32, reg_ow_pos); - sub(reg_tmp_32, jcp_.l_pad - ow * jcp_.stride_w); - movq(xmm_iw_in, reg_tmp_64); - - cvtsi2ss(xmm_iw_im, reg_tmp_32); - addss(xmm_iw_im, xmm_map_w); - - movss(xmm_tmp, xmm_iw_im); - cmpss(xmm_tmp, table_val(0), 1); - movq(reg_tmp_64, xmm_tmp); - cmp(reg_tmp_32, 0); - jne(init_with_zeros, T_NEAR); - - cmpss(xmm_iw_im, table_val(2), 1); - movq(reg_tmp_64, xmm_iw_im); - cmp(reg_tmp_32, 0); - je(init_with_zeros, T_NEAR); - - - movd(xmm_cur_height, table_val(3)); - psubd(xmm_cur_height, xmm_ih_in); - - roundps(xmm_h_low, xmm_map_h, 1); - cvtps2dq(xmm_h_low, xmm_h_low); - - movups(xmm_tmp, xmm_cur_height); - pcmpgtd(xmm_tmp, xmm_h_low); - - movq(reg_tmp_64, xmm_tmp); - cmp(reg_tmp_32, 0); - jne(h_sec_opt, T_NEAR); - - movups(xmm_h_low, xmm_cur_height); - movups(xmm_h_high, xmm_h_low); - jmp(h_sec_opt_exit); - - L(h_sec_opt); - - movups(xmm_h_high, xmm_h_low); - paddd(xmm_h_high, table_val(5)); - - L(h_sec_opt_exit); - - cvtdq2ps(xmm_tmp, xmm_h_low); - subss(xmm_lh, xmm_tmp); - movss(xmm_hh, table_val(5)); - cvtdq2ps(xmm_hh, xmm_hh); - subss(xmm_hh, xmm_lh); - - - movd(xmm_cur_width, table_val(4)); - psubd(xmm_cur_width, xmm_iw_in); - - roundps(xmm_w_low, xmm_map_w, 1); - cvtps2dq(xmm_w_low, xmm_w_low); - - movups(xmm_tmp, xmm_cur_width); - pcmpgtd(xmm_tmp, xmm_w_low); - - movq(reg_tmp_64, xmm_tmp); - cmp(reg_tmp_32, 0); - jne(w_sec_opt, T_NEAR); - - movups(xmm_w_low, xmm_cur_width); - movups(xmm_w_high, xmm_w_low); - jmp(w_sec_opt_exit); - - L(w_sec_opt); - - movups(xmm_w_high, xmm_w_low); - paddd(xmm_w_high, table_val(5)); - - L(w_sec_opt_exit); - - cvtdq2ps(xmm_tmp, xmm_w_low); - subss(xmm_lw, xmm_tmp); - movss(xmm_hw, table_val(5)); - cvtdq2ps(xmm_hw, xmm_hw); - subss(xmm_hw, xmm_lw); - - - movups(xmm_v1_off, table_val(2)); - cvtps2dq(xmm_v1_off, xmm_v1_off); - movups(xmm_v3_off, xmm_v1_off); - - pmulld(xmm_v1_off, xmm_h_low); - movups(xmm_v2_off, xmm_v1_off); - paddd(xmm_v1_off, xmm_w_low); - paddd(xmm_v2_off, xmm_w_high); - - pmulld(xmm_v3_off, xmm_h_high); - movups(xmm_v4_off, xmm_v3_off); - paddd(xmm_v3_off, xmm_w_low); - paddd(xmm_v4_off, xmm_w_high); - - - movss(xmm_w1, xmm_hh); - mulss(xmm_w1, xmm_hw); - uni_vbroadcastss(vmm_w1, xmm_w1); - - movss(xmm_w2, xmm_hh); - mulss(xmm_w2, xmm_lw); - uni_vbroadcastss(vmm_w2, xmm_w2); - - movss(xmm_w3, xmm_lh); - mulss(xmm_w3, xmm_hw); - uni_vbroadcastss(vmm_w3, xmm_w3); - - movss(xmm_w4, xmm_lh); - mulss(xmm_w4, xmm_lw); - uni_vbroadcastss(vmm_w4, xmm_w4); + // w's computation + uni_vbroadcastss(vmm_w1, dword[aux_reg_sampled_wei + ind_off_ll * jcp_.typesize_sampled_wei]); + uni_vbroadcastss(vmm_w2, dword[aux_reg_sampled_wei + ind_off_hl * jcp_.typesize_sampled_wei]); + uni_vbroadcastss(vmm_w3, dword[aux_reg_sampled_wei + ind_off_lh * jcp_.typesize_sampled_wei]); + uni_vbroadcastss(vmm_w4, dword[aux_reg_sampled_wei + ind_off_hh * jcp_.typesize_sampled_wei]); int simd_w = vlen / jcp_.typesize_in; mov(reg_ic_iter, ic_per_def_group); + L(ic_loop_main); { cmp(reg_ic_iter, simd_w); @@ -529,7 +384,6 @@ private: jl(loop_end, T_NEAR); size_t input_buffer_off = (size_t) kh * jcp_.kw * jcp_.ic + kw * jcp_.ic; - pmovsxdq(xmm_v1_off, xmm_v1_off); movq(reg_tmp_64, xmm_v1_off); imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in); @@ -568,32 +422,14 @@ private: sub(reg_ic_iter, 1); jmp(ic_loop_tail, T_NEAR); } - jmp(loop_end, T_NEAR); - - L(init_with_zeros); - - mov(reg_ic_iter, 0); - L(ic_loop_zeros); - { - cmp(reg_ic_iter, ic_per_def_group); - je(loop_end, T_NEAR); - - size_t input_buffer_off = (size_t) kh * jcp_.kw * jcp_.ic + kw * jcp_.ic; - - pxor(xmm_tmp, xmm_tmp); - movss(ptr[aux3_reg_input_buffer + input_buffer_off * jcp_.typesize_in], xmm_tmp); - add(aux3_reg_input_buffer, jcp_.typesize_in); - inc(reg_ic_iter); - jmp(ic_loop_zeros, T_NEAR); - } - L(loop_end); } } } - add(aux_reg_def_off, 2 * jcp_.kh * jcp_.kw * jcp_.oh * jcp_.ow * jcp_.typesize_off); + add(aux_reg_sampled_wei, sampledPointsPerPixel * jcp_.kh * jcp_.kw * jcp_.oh * jcp_.ow * jcp_.typesize_sampled_wei); + add(aux_reg_sampled_offs, sampledPointsPerPixel * jcp_.kh * jcp_.kw * jcp_.oh * jcp_.ow * jcp_.typesize_sampled_offsets); add(aux_reg_input, ic_per_def_group * jcp_.typesize_in); add(aux2_reg_input_buffer, ic_per_def_group * jcp_.typesize_in); inc(reg_dg_iter); @@ -601,6 +437,7 @@ private: } L(dg_loop_end); + pop(reg_sampled_offs); } void store_output(int ow_step, int oc_blocks_step, int oc_step) { @@ -614,7 +451,6 @@ private: for (int ow = 0; ow < ow_step; ow++) { Vmm vmm_acc = get_vmm_acc(r * jcp_.ur_w * jcp_.nb_oc_blocking + ocb * ow_step + ow); - uni_vaddps(vmm_acc, vmm_acc, Vmm(0)); } } @@ -637,12 +473,10 @@ private: if (isa == avx512_common) { size_t out_off = (size_t) ow * jcp_.oc; - uni_vmovups(ptr[aux_reg_output + out_off * jcp_.typesize_out], vmm_dst | ktail_mask); } else { for (int oc = 0; oc < tail_size; oc++) { size_t out_off = (size_t) ow * jcp_.oc + oc + r * (jcp_.oc_block / 2); - movq(reg_tmp_64, xmm_dst); mov(ptr[aux_reg_output + out_off * jcp_.typesize_out], reg_tmp_32); @@ -663,8 +497,7 @@ private: for (int ocb = 0; ocb < oc_blocks_step; ocb++) { for (int ow = 0; ow < ow_step; ow++) { Vmm vmm_acc = get_vmm_acc(r * jcp_.ur_w * jcp_.nb_oc_blocking + ocb * ow_step + ow); - size_t out_off = (size_t) ow * jcp_.oc + ocb * jcp_.oc_block + r * (jcp_.oc_block / 2); - + size_t out_off = (size_t) ow * jcp_.oc * jcp_.ngroups + ocb * jcp_.oc_block + r * (jcp_.oc_block / 2); uni_vmovups(ptr[aux_reg_output + out_off * jcp_.typesize_out], vmm_acc); } } @@ -684,19 +517,20 @@ private: push(reg_input); push(reg_kernel); - interpolate_input(ow_step); + prepare_buffer(ow_step); pop(reg_kernel); pop(reg_input); pop(reg_bias); pop(reg_output); + push(reg_sampled_offs); push(reg_ow_pos); + push(aux2_reg_kernel); mov(aux_reg_kernel, reg_kernel); mov(aux_reg_output, reg_output); mov(aux_reg_bias, reg_bias); - mov(reg_oc_work, jcp_.oc); L(oc_unrolled_loop); { @@ -736,7 +570,9 @@ private: } } + pop(aux2_reg_kernel); pop(reg_ow_pos); + pop(reg_sampled_offs); } }; @@ -759,8 +595,7 @@ bool MKLDNNDeformableConvolutionNode::isSupportedOperation(const std::shared_ptr } MKLDNNDeformableConvolutionNode::MKLDNNDeformableConvolutionNode(const std::shared_ptr& op, - const mkldnn::engine& eng, MKLDNNWeightsSharing::Ptr &cache) - : MKLDNNNode(op, eng, cache) { + const mkldnn::engine& eng, MKLDNNWeightsSharing::Ptr &cache) : MKLDNNNode(op, eng, cache) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { IE_THROW(NotImplemented) << errorMessage; @@ -793,29 +628,23 @@ MKLDNNDeformableConvolutionNode::MKLDNNDeformableConvolutionNode(const std::shar } else { with_bilinear_pad = false; } - enforceRef = (op->get_type_info() == ngraph::op::v8::DeformableConvolution::type_info); } void MKLDNNDeformableConvolutionNode::getSupportedDescriptors() { std::string errorPrefix = "DeformableConvolution layer with name '" + getName() + "' "; - if (getParentEdges().size() != 3 && getParentEdges().size() != 4) IE_THROW() << errorPrefix << "has incorrect number of input edges"; if (getChildEdges().empty()) IE_THROW() << errorPrefix << "has incorrect number of output edges"; - if (getInputShapeAtPort(0).getRank() != 4) { IE_THROW() << "Deformable convolution layer. Unsupported mode. Only 4D blobs are supported as input."; } - if (getInputShapeAtPort(1).getRank() != 4) { IE_THROW() << errorPrefix << "doesn't support 1st input with rank: " << getInputShapeAtPort(1).getRank(); } - if (getInputShapeAtPort(2).getRank() != 4) { IE_THROW() << errorPrefix << "doesn't support 2nd input with rank: " << getInputShapeAtPort(2).getRank(); } - if (getOutputShapeAtPort(0).getRank() != 4) { IE_THROW() << errorPrefix << "doesn't support output with rank: " << getOutputShapeAtPort(0).getRank(); } @@ -845,6 +674,12 @@ void MKLDNNDeformableConvolutionNode::initSupportedPrimitiveDescriptors() { config.outConfs[0].inPlace = -1; impl_desc_type impl_type; + const int simd_w = mayiuse(cpu::x64::avx512_common) ? 16 : 8; + if (group != 1 || (((getInputShapeAtPort(0).getStaticDims()[1] / group) % simd_w != 0) + || ((getOutputShapeAtPort(0).getStaticDims()[1] / group) % simd_w != 0))) { + enforceRef = true; + } + if (enforceRef) { impl_type = impl_desc_type::ref; } else if (mayiuse(cpu::x64::avx512_common)) { @@ -861,14 +696,11 @@ void MKLDNNDeformableConvolutionNode::initSupportedPrimitiveDescriptors() { // optimized implementation auto dataFormat = memory::format_tag::nhwc; auto offFormat = memory::format_tag::nchw; - auto weiFormat = group > 1 ? mayiuse(avx512_common) ? memory::format_tag::gOIhw16i16o : memory::format_tag::gOIhw8i8o - : mayiuse(avx512_common) ? memory::format_tag::OIhw16i16o : memory::format_tag::OIhw8i8o; - + auto weiFormat = mayiuse(avx512_common) ? memory::format_tag::OIhw16i16o : memory::format_tag::OIhw8i8o; config.inConfs[0].desc = std::make_shared(getInputShapeAtPort(0), memory::data_type::f32, dataFormat); config.inConfs[1].desc = std::make_shared(getInputShapeAtPort(1), memory::data_type::f32, offFormat); - auto& wDims = getInputShapeAtPort(2).getStaticDims(); if (group > 1 && wDims.size() != 5) { auto new_dims = InferenceEngine::SizeVector({group, div_up(wDims[0], group)}); @@ -882,7 +714,6 @@ void MKLDNNDeformableConvolutionNode::initSupportedPrimitiveDescriptors() { memory::data_type::f32, weiFormat); } - if (inputsNumber > 3) { config.inConfs[3].desc = std::make_shared(getInputShapeAtPort(3), memory::data_type::f32, memory::format_tag::nchw); @@ -908,6 +739,133 @@ void MKLDNNDeformableConvolutionNode::initSupportedPrimitiveDescriptors() { } } +void MKLDNNDeformableConvolutionNode::prepareSamplingWeights( + const std::vector& src_strides, const float* offsets, const std::vector& off_strides, + const float* modulation, const std::vector& modulation_strides) { + const int MB = jcp.mb; + const int OH = jcp.oh; + const int OW = jcp.ow; + + const int KH = jcp.kh; + const int KW = jcp.kw; + const int ker_size = KH * KW; + + const int DG = jcp.dg; + + const int IH = jcp.ih; + const int IW = jcp.iw; + + const int KSH = jcp.stride_h; + const int KSW = jcp.stride_w; + + const int KDH = jcp.dilate_h; + const int KDW = jcp.dilate_w; + + const int padT = jcp.t_pad; + const int padL = jcp.l_pad; + + const bool with_bi_pad = jcp.with_bi_pad; + + // prepare weights and indices + sampledCoordsVector.resize(MB * DG * KH * KW * OH * OW * sampledPointsPerPixel); + interpWeightsVector.resize(MB * DG * KH * KW * OH * OW * sampledPointsPerPixel); + auto precompKer = [&](int mb, int dg, int oh, int ow) { + int sampledCoordIndex = (mb * DG * OH * OW + dg * OH * OW + oh * OW + ow) * KH * KW * sampledPointsPerPixel; + const int h_in = oh * KSH - padT; + const int w_in = ow * KSW - padL; + + const int waOffsetH = (enforceRef ? 0 : h_in); + const int waOffsetW = (enforceRef ? 0 : w_in); + + const float *data_offset_ptr = offsets + mb * off_strides[0] + (dg * 2 * KH * KW) * off_strides[1]; + const float *modulation_offset_ptr = nullptr; + if (modulation != nullptr) { + modulation_offset_ptr = modulation + mb * modulation_strides[0] + (dg * ker_size) * modulation_strides[1]; + } + + for (int kh = 0; kh < KH; kh++) { + for (int kw = 0; kw < KW; kw++) { + const size_t data_offset_h_index = 2 * ((size_t) kh * KW + kw) * off_strides[1] + oh * off_strides[2] + ow * off_strides[3]; + const size_t data_offset_w_index = (2 * ((size_t) kh * KW + kw) + 1) * off_strides[1] + oh * off_strides[2] + ow * off_strides[3]; + const float offset_h = data_offset_ptr[data_offset_h_index]; + const float offset_w = data_offset_ptr[data_offset_w_index]; + float map_h = h_in + kh * (KDH + 1) + offset_h; + float map_w = w_in + kw * (KDW + 1) + offset_w; + bool skip_compute; + if (with_bilinear_pad) { + skip_compute = !(static_cast(map_w) > -1 && + static_cast(map_w) < IW && + static_cast(map_h) > -1 && + static_cast(map_h) < IH); + } else { + skip_compute = !(map_w >= 0 && map_w < IW && + map_h >= 0 && map_h < IH); + } + if (!skip_compute) { + // modulations precomp. + float modulation_scalar = 1.0f; + + if (modulation_offset_ptr != nullptr) { + size_t modulation_index = (kh * KW + kw) * modulation_strides[1] + oh * modulation_strides[2] + ow * modulation_strides[3]; + modulation_scalar = modulation_offset_ptr[modulation_index]; + } + // interpolation precomp. + const int cur_h_end = IH; + const int cur_w_end = IW; + int h_low = with_bi_pad ? static_cast(floorf(map_h)) : + std::max(static_cast(floorf(map_h)), 0); + int w_low = with_bi_pad ? static_cast(floorf(map_w)) : + std::max(static_cast(floorf(map_w)), 0); + int h_high = with_bi_pad ? h_low + 1 : std::min(static_cast(ceilf(map_h)), cur_h_end - 1); + int w_high = with_bi_pad ? w_low + 1 : std::min(static_cast(ceilf(map_w)), cur_w_end - 1); + + float lh = map_h - h_low; + float lw = map_w - w_low; + float hh = 1 - lh, hw = 1 - lw; + + int h_ind_low = std::max(h_low, 0) - waOffsetH; + int h_ind_high = std::min(h_high, cur_h_end - 1) - waOffsetH; + int w_ind_low = std::max(w_low, 0) - waOffsetW; + int w_ind_high = std::min(w_high, cur_w_end - 1) - waOffsetW; + + hh = (h_low >= 0 ? hh : 0); + hw = (w_low >= 0 ? hw : 0); + lh = (h_high < cur_h_end ? lh : 0); + lw = (w_high < cur_w_end ? lw : 0); + + const int h_off_low = h_ind_low * src_strides[2] / src_strides[3]; + const int h_off_high = h_ind_high * src_strides[2] / src_strides[3]; + const int w_off_low = w_ind_low; + const int w_off_high = w_ind_high; + sampledCoordsVector[sampledCoordIndex] = h_off_high + w_off_high; + sampledCoordsVector[sampledCoordIndex + 1] = h_off_high + w_off_low; + sampledCoordsVector[sampledCoordIndex + 2] = h_off_low + w_off_high; + sampledCoordsVector[sampledCoordIndex + 3] = h_off_low + w_off_low; + + float w22 = hh * hw * modulation_scalar, w21 = hh * lw * modulation_scalar, + w12 = lh * hw * modulation_scalar, w11 = lh * lw * modulation_scalar; + + interpWeightsVector[sampledCoordIndex] = w11; + interpWeightsVector[sampledCoordIndex + 1] = w12; + interpWeightsVector[sampledCoordIndex + 2] = w21; + interpWeightsVector[sampledCoordIndex + 3] = w22; + } else { + sampledCoordsVector[sampledCoordIndex] = 0; + interpWeightsVector[sampledCoordIndex] = 0; + interpWeightsVector[sampledCoordIndex + 1] = 0; + interpWeightsVector[sampledCoordIndex + 2] = 0; + interpWeightsVector[sampledCoordIndex + 3] = 0; + } + sampledCoordIndex += sampledPointsPerPixel; + } + } + }; + + parallel_nd(MB, DG, OH, OW, [&](int mb, int dg, int oh, int ow) { + precompKer(mb, dg, oh, ow); + }); +} + void MKLDNNDeformableConvolutionNode::createPrimitive() { auto selectedPrimitiveDescriptor = getSelectedPrimitiveDescriptor(); if (!selectedPrimitiveDescriptor) @@ -958,8 +916,9 @@ void MKLDNNDeformableConvolutionNode::createPrimitive() { jcp.typesize_in = sizeof(float); jcp.typesize_off = sizeof(float); + jcp.typesize_sampled_wei = sizeof(float); + jcp.typesize_sampled_offsets = sizeof(int); jcp.typesize_out = sizeof(float); - jcp.typesize_modulation = sizeof(float); jcp.ur_w = mayiuse(cpu::x64::avx512_common) ? 6 : 3; jcp.nb_oc_blocking = !mayiuse(cpu::x64::avx2) ? 2 : 4; @@ -980,126 +939,68 @@ void MKLDNNDeformableConvolutionNode::createPrimitive() { def_conv_kernel->create_ker(); } -void MKLDNNDeformableConvolutionNode::executeReference(const float* src, const float* offsets, const float* weights, float* dst, - const std::vector& src_strides, const std::vector& off_strides, - const std::vector& wei_strides, const std::vector& dst_strides, - const float* modulation, const std::vector& modulation_strides) { - const bool with_groups = jcp.ngroups > 1; +void MKLDNNDeformableConvolutionNode::executeReference(const float* src, const float* weights, float* dst, const std::vector& src_strides, + const std::vector& wei_strides, const std::vector& dst_strides) { const int G = jcp.ngroups; const int MB = jcp.mb; const int OH = jcp.oh; const int OW = jcp.ow; - const int IH = jcp.ih; - const int IW = jcp.iw; const int OC = jcp.oc; const int IC = jcp.ic; const int KH = jcp.kh; const int KW = jcp.kw; - - const int KSH = jcp.stride_h; - const int KSW = jcp.stride_w; - - const int KDH = jcp.dilate_h; - const int KDW = jcp.dilate_w; - - const int padT = jcp.t_pad; - const int padL = jcp.l_pad; + const int ker_size = KH * KW; const int DG = jcp.dg; + const int DGHW = DG * OH * OW; + const int HW = OH * OW; + const int channel_per_deformable_group = (IC * G) / DG; - - const bool with_bi_pad = jcp.with_bi_pad; - auto ker = [=](int g, int mb, int oc, int oh, int ow) { + const size_t group_wei_stride = wei_strides[0] * OC; + auto compKer = [=](int g, int mb, int oc, int oh, int ow) { float d = 0; - const int h_in = oh * KSH - padT; - const int w_in = ow * KSW - padL; - for (int ic = 0; ic < IC; ic++) { const float *data_im_ptr = src + mb * src_strides[0] + (g * IC + ic) * src_strides[1]; const int deformable_group_index = (IC * g + ic) / channel_per_deformable_group; - const float *data_offset_ptr = offsets + mb * off_strides[0] + (deformable_group_index * 2 * KH * KW) * off_strides[1]; - const float *modulation_offset_ptr = nullptr; - if (modulation != nullptr) { - modulation_offset_ptr = modulation + mb * modulation_strides[0] + (deformable_group_index * KH * KW) * modulation_strides[1]; - } - - for (int kh = 0; kh < KH; kh++) { - for (int kw = 0; kw < KW; kw++) { - const size_t data_offset_h_index = 2 * (kh * KW + kw) * off_strides[1] + oh * off_strides[2] + ow * off_strides[3]; - const size_t data_offset_w_index = (2 * (kh * KW + kw) + 1) * off_strides[1] + oh * off_strides[2] + ow * off_strides[3]; - const float offset_h = data_offset_ptr[data_offset_h_index]; - const float offset_w = data_offset_ptr[data_offset_w_index]; - float map_h = h_in + kh * (KDH + 1) + offset_h; - float map_w = w_in + kw * (KDW + 1) + offset_w; - bool skip_compute; - if (with_bilinear_pad) { - skip_compute = !(static_cast(map_w) > -1 && - static_cast(map_w) < IW && - static_cast(map_h) > -1 && - static_cast(map_h) < IH); + int sampledCoordIndex = (mb * DGHW + deformable_group_index * HW + oh * OW + ow) * ker_size * sampledPointsPerPixel; + size_t weiIndex = (size_t) g * group_wei_stride + oc * wei_strides[0] + ic * wei_strides[1]; + for (int kh_off = 0; kh_off < KH * wei_strides[2]; kh_off += wei_strides[2]) { + for (int kw_off = 0; kw_off < KW * wei_strides[3]; kw_off += wei_strides[3]) { + // check if current addendum marked as equal zero + if (sampledCoordsVector[sampledCoordIndex] != -1) { + const int v11 = sampledCoordsVector[sampledCoordIndex]; + const int v12 = sampledCoordsVector[sampledCoordIndex + 1]; + const int v21 = sampledCoordsVector[sampledCoordIndex + 2]; + const int v22 = sampledCoordsVector[sampledCoordIndex + 3]; + float val = interpWeightsVector[sampledCoordIndex++] * data_im_ptr[v11]; // v11 + val += interpWeightsVector[sampledCoordIndex++] * data_im_ptr[v12]; // v12 + val += interpWeightsVector[sampledCoordIndex++] * data_im_ptr[v21]; // v21 + val += interpWeightsVector[sampledCoordIndex++] * data_im_ptr[v22]; // v22 + d += val * weights[weiIndex + kh_off + kw_off]; } else { - skip_compute = !(map_w >= 0 && - map_w < IW && - map_h >= 0 && - map_h < IH); - } - if (!skip_compute) { - const int cur_h_end = IH; - const int cur_w_end = IW; - int h_low = with_bi_pad ? static_cast(floorf(map_h)) : - std::max(static_cast(floorf(map_h)), 0); - int w_low = with_bi_pad ? static_cast(floorf(map_w)) : - std::max(static_cast(floorf(map_w)), 0); - const int cur_h_start = h_low; - const int cur_w_start = w_low; - int h_high = with_bi_pad ? h_low + 1 : std::min(static_cast(ceilf(map_h)), cur_h_end - 1); - int w_high = with_bi_pad ? w_low + 1 : std::min(static_cast(ceilf(map_w)), cur_w_end - 1); - - float lh = map_h - h_low; - float lw = map_w - w_low; - float hh = 1 - lh, hw = 1 - lw; - - float v1 = (cur_w_start >= 0 && cur_h_start >= 0) ? data_im_ptr[h_low * src_strides[2] + w_low * src_strides[3]] : 0.0f; - float v2 = (w_high < cur_w_end && cur_h_start >= 0) ? data_im_ptr[h_low * src_strides[2] + w_high * src_strides[3]] : 0.0f; - float v3 = (cur_w_start >= 0 && h_high < cur_h_end) ? data_im_ptr[h_high * src_strides[2] + w_low * src_strides[3]] : 0.0f; - float v4 = (w_high < cur_w_end && h_high < cur_h_end) ? data_im_ptr[h_high * src_strides[2] + w_high * src_strides[3]] : 0.0f; - float w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; - - float val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); - - float modulation_scalar = 1.0f; - - if (modulation_offset_ptr != nullptr) { - size_t modulation_index = (kh * KW + kw) * modulation_strides[1] + oh * modulation_strides[2] + ow * modulation_strides[3]; - modulation_scalar = modulation_offset_ptr[modulation_index]; - } - - const float weight = with_groups ? weights[(g + oc / G) * wei_strides[0] + ic * wei_strides[1] + kh * wei_strides[2] + - kw * wei_strides[3]] - : weights[oc * wei_strides[0] + ic * wei_strides[1] + kh * wei_strides[2] + kw * wei_strides[3]]; - d += val * weight * modulation_scalar; + sampledCoordIndex += sampledPointsPerPixel; } } } } - return d; }; parallel_nd(G, MB, OC, OH, OW, - [&](int g, int mb, int oc, int oh, int ow) { - dst[mb * dst_strides[0] + (g * OC + oc) * dst_strides[1] + oh * dst_strides[2] + ow * dst_strides[3]] = ker(g, mb, oc, oh, ow); - }); + [&](int g, int mb, int oc, int oh, int ow) { + dst[mb * dst_strides[0] + (g * OC + oc) * dst_strides[1] + oh * dst_strides[2] + ow * dst_strides[3]] = compKer(g, mb, oc, oh, ow); + }); } -void MKLDNNDeformableConvolutionNode::executeOptimized(const float* src, const float* offsets, const float* weights, float* dst, - const std::vector& src_strides, const std::vector& off_strides, + +void MKLDNNDeformableConvolutionNode::executeOptimized(const float* src, const float* weights, float* dst, + const std::vector& src_strides, const std::vector& dst_strides) { size_t buffer_size = (size_t)jcp.nthr * jcp.ur_w * jcp.kh * jcp.kw * jcp.ic * jcp.typesize_in; std::vector input_buffer(buffer_size, 0); - float* input_buffer_ptr = &input_buffer[0]; + float* input_buffer_ptr = input_buffer.data(); parallel_for3d(jcp.mb, jcp.ngroups, jcp.oh, [&](size_t n, size_t g, size_t oh) { auto ithr = parallel_get_thread_num(); @@ -1111,10 +1012,10 @@ void MKLDNNDeformableConvolutionNode::executeOptimized(const float* src, const f par_conv.src = &src[n * src_strides[0] + _ic*jcp.ic_block * src_strides[1] + (oh * jcp.stride_h - jcp.t_pad) * src_strides[2] - jcp.l_pad * src_strides[3]]; - par_conv.off = &offsets[n * off_strides[0] + oh * off_strides[2]]; - par_conv.filt = weights; - par_conv.dst = &dst[n * dst_strides[0] + _oc*jcp.oc_block * dst_strides[1] + oh * dst_strides[2]]; - + par_conv.sampledWei = &interpWeightsVector[(n * jcp.dg * jcp.oh + oh) * jcp.kh * jcp.kw * jcp.ow * sampledPointsPerPixel]; + par_conv.sampledCoords = &sampledCoordsVector[(n * jcp.dg * jcp.oh + oh) * jcp.kh * jcp.kw * jcp.ow * sampledPointsPerPixel]; + par_conv.filt = &weights[g * jcp.nb_oc * jcp.nb_ic * jcp.kh * jcp.kw * jcp.ic_block * jcp.oc_block]; + par_conv.dst = &dst[n * dst_strides[0] + _oc * jcp.oc_block * dst_strides[1] + oh * dst_strides[2]]; par_conv.buf = input_buffer_ptr + ithr * jcp.ur_w * jcp.kh * jcp.kw * jcp.ic; par_conv.oh_pos = oh; @@ -1158,7 +1059,6 @@ void MKLDNNDeformableConvolutionNode::execute(mkldnn::stream strm) { dst_strides[dst_block_desc->getOrder()[i]] = dst_block_desc->getStrides()[i]; } - auto off_strides = getParentEdgeAt(1)->getMemory().GetDescWithType()->getStrides(); auto wei_strides = getParentEdgeAt(2)->getMemory().GetDescWithType()->getStrides(); InferenceEngine::SizeVector modulation_strides; @@ -1166,11 +1066,12 @@ void MKLDNNDeformableConvolutionNode::execute(mkldnn::stream strm) { modulation_strides = getParentEdgeAt(3)->getMemory().GetDescWithType()->getStrides(); } + prepareSamplingWeights(src_strides, offsets, off_strides, modulation, modulation_strides); if (def_conv_kernel) { - executeOptimized(src, offsets, weights, dst, src_strides, off_strides, dst_strides); + executeOptimized(src, weights, dst, src_strides, dst_strides); } else { - executeReference(src, offsets, weights, dst, src_strides, off_strides, wei_strides, dst_strides, modulation, modulation_strides); + executeReference(src, weights, dst, src_strides, wei_strides, dst_strides); } } diff --git a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_def_conv_node.h b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_def_conv_node.h index 29af41af4da..28f5295949d 100644 --- a/inference-engine/src/mkldnn_plugin/nodes/mkldnn_def_conv_node.h +++ b/inference-engine/src/mkldnn_plugin/nodes/mkldnn_def_conv_node.h @@ -30,7 +30,8 @@ struct jit_def_conv_params { int ur_w_tail; int typesize_in; int typesize_off; - int typesize_modulation; + int typesize_sampled_wei; + int typesize_sampled_offsets; int typesize_bia; int typesize_out; bool with_bias; @@ -41,8 +42,8 @@ struct jit_def_conv_params { struct jit_def_conv_call_args { const void *src; - const void *off; - const void *modulation; + const void *sampledWei; + const void *sampledCoords; const void *filt; const void *bias; const void *dst; @@ -80,6 +81,7 @@ public: return false; } bool enforceRef = false; + constexpr static int sampledPointsPerPixel = 4; // count of sampling points ({top|bottom}, {left|right}) InferenceEngine::Precision getRuntimePrecision() const override; @@ -94,14 +96,18 @@ private: jit_def_conv_params jcp = {}; + std::vector sampledCoordsVector; + std::vector interpWeightsVector; + std::shared_ptr def_conv_kernel = nullptr; - void executeReference(const float* src, const float* offsets, const float* weights, float* dst, - const std::vector& src_strides, const std::vector& off_strides, - const std::vector& wei_strides, const std::vector& dst_strides, - const float* modulation = nullptr, const std::vector& modulation_strides = {}); - void executeOptimized(const float* src, const float* offsets, const float* weights, float* dst, - const std::vector& src_strides, const std::vector& off_strides, const std::vector& dst_strides); + void prepareSamplingWeights(const std::vector& src_strides, const float* offsets, const std::vector& off_strides, + const float* modulation = nullptr, const std::vector& modulation_strides = {}); + + void executeReference(const float* src, const float* weights, float* dst, const std::vector& src_strides, + const std::vector& wei_strides, const std::vector& dst_strides); + void executeOptimized(const float* src, const float* weights, float* dst, + const std::vector& src_strides, const std::vector& dst_strides); }; } // namespace MKLDNNPlugin diff --git a/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/deformable_convolution.cpp b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/deformable_convolution.cpp index 3155580319d..1bb3a669a0c 100644 --- a/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/deformable_convolution.cpp +++ b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/deformable_convolution.cpp @@ -128,5 +128,53 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values(std::vector({1, 3, 30, 30})), ::testing::Values(CommonTestUtils::DEVICE_CPU)), DeformableConvolutionLayerTest::getTestCaseName); - +/* ============= Multiple groups case ============= */ +INSTANTIATE_TEST_SUITE_P( + smoke_DeformableConvolution2D_MultipleGroups, DeformableConvolutionLayerTest, + ::testing::Combine( + ::testing::Combine( + ::testing::ValuesIn(std::vector> {{1, 16, 2, 2}}), // offsets + ::testing::ValuesIn(std::vector> {{2, 2, 2, 2}}), // ker. + ::testing::ValuesIn(strides), + ::testing::Values(std::vector({0, 0})), + ::testing::Values(std::vector({0, 0})), + ::testing::ValuesIn(dilations), + ::testing::ValuesIn(std::vector {2}), // gr. + ::testing::ValuesIn(std::vector {2}), // def. gr. + ::testing::ValuesIn(numOutChannels), + ::testing::Values(ngraph::op::PadType::EXPLICIT), + ::testing::ValuesIn(with_bilinear_interpolation_pad), + ::testing::ValuesIn(with_modulated_scalar)), + ::testing::ValuesIn(netPrecisions), + ::testing::Values(InferenceEngine::Precision::UNSPECIFIED), + ::testing::Values(InferenceEngine::Precision::UNSPECIFIED), + ::testing::Values(InferenceEngine::Layout::ANY), + ::testing::Values(InferenceEngine::Layout::ANY), + ::testing::Values(std::vector({1, 4, 3, 3})), + ::testing::Values(CommonTestUtils::DEVICE_CPU)), + DeformableConvolutionLayerTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P( + smoke_DeformableConvolution2D_MultipleGroups_2, DeformableConvolutionLayerTest, + ::testing::Combine( + ::testing::Combine( + ::testing::ValuesIn(std::vector> {{1, 18, 66, 66}}), // offsets + ::testing::ValuesIn(std::vector> {{4, 2, 3, 3}}), // ker. + ::testing::ValuesIn(strides), + ::testing::Values(std::vector({0, 0})), + ::testing::Values(std::vector({0, 0})), + ::testing::ValuesIn(dilations), + ::testing::ValuesIn(std::vector {4}), // gr. + ::testing::ValuesIn(std::vector {1}), // def. gr. + ::testing::ValuesIn(numOutChannels), + ::testing::Values(ngraph::op::PadType::EXPLICIT), + ::testing::ValuesIn(with_bilinear_interpolation_pad), + ::testing::ValuesIn(with_modulated_scalar)), + ::testing::ValuesIn(netPrecisions), + ::testing::Values(InferenceEngine::Precision::UNSPECIFIED), + ::testing::Values(InferenceEngine::Precision::UNSPECIFIED), + ::testing::Values(InferenceEngine::Layout::ANY), + ::testing::Values(InferenceEngine::Layout::ANY), + ::testing::Values(std::vector({1, 8, 68, 68})), + ::testing::Values(CommonTestUtils::DEVICE_CPU)), + DeformableConvolutionLayerTest::getTestCaseName); } // namespace diff --git a/inference-engine/tests/functional/plugin/cpu/single_layer_tests/deformable_convolution.cpp b/inference-engine/tests/functional/plugin/cpu/single_layer_tests/deformable_convolution.cpp new file mode 100644 index 00000000000..5b7ea755f5b --- /dev/null +++ b/inference-engine/tests/functional/plugin/cpu/single_layer_tests/deformable_convolution.cpp @@ -0,0 +1,357 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "test_utils/cpu_test_utils.hpp" + +#include "ngraph_functions/builders.hpp" +#include "ngraph_functions/utils/ngraph_helpers.hpp" + +using namespace InferenceEngine; +using namespace CPUTestUtils; + +namespace CPULayerTestsDefinitions { +enum OffsetType {ZERO, NATURAL, REAL_POSITIVE, REAL_NEGATIVE, REAL_MISC}; + +typedef std::tuple< + bool, // with_bilinear_interpolation_pad + bool, // with_modulation + OffsetType // type of def. offsets + > DefConvSpecificParams; + +typedef std::tuple< + size_t, // batches + std::vector, // input spatial shape + std::vector, // offsets spatial shape + std::vector, // kernel spatial shape + ngraph::op::PadType, // pad. type + std::vector, // pad. begin + std::vector, // pad. end + std::vector, // strides + std::vector // dilations + > SpatialParams; + +typedef std::tuple< + size_t, // groups + size_t, // deformable groups + size_t, // input channels per group + size_t // output channels per group + > ChannelParams; + +typedef std::tuple< + SpatialParams, + ChannelParams, + DefConvSpecificParams, + InferenceEngine::Precision, // Net precision + LayerTestsUtils::TargetDevice // Device name + > DefConvLayerTestParams; + +typedef std::tuple< + CPULayerTestsDefinitions::DefConvLayerTestParams, + CPUSpecificParams> DefConvLayerCPUTestParamsSet; + +class DefConvLayerCPUTest : public testing::WithParamInterface, + virtual public LayerTestsUtils::LayerTestsCommon, public CPUTestsBase { +public: + OffsetType offsetType; + static std::string getTestCaseName(testing::TestParamInfo obj) { + CPULayerTestsDefinitions::DefConvLayerTestParams basicParamsSet; + std::string td; + Precision netPr; + InferenceEngine::Precision inPrc, outPrc; + ChannelParams chParams; + SpatialParams spParams; + CPULayerTestsDefinitions::DefConvSpecificParams dcSpecificParams; + CPUSpecificParams cpuParams; + std::tie(basicParamsSet, cpuParams) = obj.param; + std::tie(spParams, chParams, dcSpecificParams, netPr, td) = basicParamsSet; + inPrc = outPrc = netPr; + ngraph::op::PadType padType; + size_t batch; + InferenceEngine::SizeVector offsets, filter, stride, dilation; + std::vector padBegin, padEnd; + InferenceEngine::SizeVector inSpatShape, offSpatShape, kerSpatShape; + std::tie(batch, inSpatShape, offSpatShape, kerSpatShape, + padType, padBegin, padEnd, stride, dilation) = spParams; + size_t groups, deformableGroups, inGrCh, outGrCh; + std::tie(groups, deformableGroups, inGrCh, outGrCh) = chParams; + bool withBilinearInterpolationPad, withModulation; + OffsetType offType; + std::tie(withBilinearInterpolationPad, withModulation, offType) = dcSpecificParams; + std::ostringstream result; + result << "DefConvTest("; + result << std::to_string(obj.index) << ")_"; + result << "IS=(" << batch << "_" << groups * inGrCh << "_" << inSpatShape[0] << "_" << inSpatShape[1] << ")_"; + result << "OS=(" << batch << "_" << groups * outGrCh << "_" << offSpatShape[0] << "_" << offSpatShape[1] << ")_"; + result << "K" << CommonTestUtils::vec2str(kerSpatShape) << "_"; + result << "S" << CommonTestUtils::vec2str(stride) << "_"; + result << "PB" << CommonTestUtils::vec2str(padBegin) << "_"; + result << "PE" << CommonTestUtils::vec2str(padEnd) << "_"; + result << "D=" << CommonTestUtils::vec2str(dilation) << "_"; + result << "O=" << groups * outGrCh << "_"; + result << "AP=" << padType << "_"; + result << "netPRC=" << netPr.name() << "_"; + result << "inPRC=" << inPrc.name() << "_"; + result << "outPRC=" << outPrc.name() << "_"; + result << "withBilPad=" << withBilinearInterpolationPad << "_"; + result << "withMod=" << withModulation << "_"; + result << "offsetType=" << offType << "_"; + result << "trgDev=" << td; + result << CPUTestsBase::getTestCaseName(cpuParams); + return result.str(); + } +protected: + void GenerateInputs() override { + for (const auto &input : cnnNetwork.getInputsInfo()) { + const auto info = input.second.get(); + const auto &name = info->name(); + InferenceEngine::Blob::Ptr blob; + if (name == "a_data") { + blob = GenerateInput(*info); + } else if (name == "b_offset_vals") { + if (offsetType == OffsetType::NATURAL) { + blob = FuncTestUtils::createAndFillBlobFloat(info->getTensorDesc(), 10, 0, 1); + } else if (offsetType == OffsetType::ZERO) { + blob = FuncTestUtils::createAndFillBlobFloat(info->getTensorDesc(), 0, 1, 1); + } else if (offsetType == OffsetType::REAL_POSITIVE) { + blob = FuncTestUtils::createAndFillBlobFloat(info->getTensorDesc(), 2, 0, 100); + } else if (offsetType == OffsetType::REAL_NEGATIVE) { + blob = FuncTestUtils::createAndFillBlobFloat(info->getTensorDesc(), 2, -2, 100); + } else if (offsetType == OffsetType::REAL_MISC) { + blob = FuncTestUtils::createAndFillBlobFloat(info->getTensorDesc(), 4, -2, 100); + } else { + IE_THROW() << "Unexpected offset type"; + } + } else if (name == "c_filter_vals") { + blob = GenerateInput(*info); + } else if (name == "c_modulation_scalars") { + blob = FuncTestUtils::createAndFillBlobFloat(info->getTensorDesc(), 1, 0, 100); + } else { + IE_THROW() << "Unknown input of DeformableConvolution"; + } + inputs.push_back(blob); + } + } + void SetUp() override { + ChannelParams chParams; + SpatialParams spParams; + CPULayerTestsDefinitions::DefConvSpecificParams dcSpecificParams; + + std::vector inShape; + InferenceEngine::Precision netPrecision; + CPULayerTestsDefinitions::DefConvLayerTestParams basicParamsSet; + CPUSpecificParams cpuParams; + std::tie(basicParamsSet, cpuParams) = this->GetParam(); + std::tie(inFmts, outFmts, priority, selectedType) = cpuParams; + + std::tie(spParams, chParams, dcSpecificParams, netPrecision, targetDevice) = basicParamsSet; + + inPrc = outPrc = netPrecision; + inLayout = outLayout = InferenceEngine::Layout::ANY; + ngraph::op::PadType padType; + size_t batch; + InferenceEngine::SizeVector offsets, filter, stride, dilation; + std::vector padBegin, padEnd; + InferenceEngine::SizeVector inSpatShape, offSpatShape, kerSpatShape; + std::tie(batch, inSpatShape, offSpatShape, kerSpatShape, + padType, padBegin, padEnd, stride, dilation) = spParams; + + size_t groups, deformableGroups, inGrCh, outGrCh; + std::tie(groups, deformableGroups, inGrCh, outGrCh) = chParams; + bool withBilinearInterpolationPad, withModulation; + std::tie(withBilinearInterpolationPad, withModulation, offsetType) = dcSpecificParams; + + inShape = std::vector({batch, groups * inGrCh, inSpatShape[0], inSpatShape[1]}); + offsets = std::vector {batch, deformableGroups * kerSpatShape[0] * kerSpatShape[1] * 2, + offSpatShape[0], offSpatShape[1]}; + filter = std::vector {groups * outGrCh, inGrCh, kerSpatShape[0], kerSpatShape[1]}; + + auto ngPrc = FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc(netPrecision); + auto params = ngraph::builder::makeParams(ngPrc, {inShape, offsets, filter}); + auto paramOuts = ngraph::helpers::convert2OutputVector( + ngraph::helpers::castOps2Nodes(params)); + auto data = std::make_shared(ngPrc, ngraph::Shape(inShape)); + data->set_friendly_name("a_data"); + auto offset_vals = std::make_shared(ngPrc, ngraph::Shape(offsets)); + offset_vals->set_friendly_name("b_offset_vals"); + auto filter_vals = std::make_shared(ngPrc, ngraph::Shape(filter)); + filter_vals->set_friendly_name("c_filter_vals"); + ngraph::ParameterVector parameters{data, offset_vals, filter_vals}; + std::shared_ptr deformable_conv; + if (withModulation) { + auto modulation_shape = ngraph::Shape(offsets); + modulation_shape[1] = offsets[1] / 2; + auto modulation_scalars = std::make_shared(ngPrc, modulation_shape); + modulation_scalars->set_friendly_name("c_modulation_scalars"); + + deformable_conv = std::make_shared(data, offset_vals, filter_vals, modulation_scalars, stride, padBegin, + padEnd, dilation, padType, groups, deformableGroups, + withBilinearInterpolationPad); + parameters.push_back(modulation_scalars); + } else { + deformable_conv = std::make_shared(data, offset_vals, filter_vals, stride, padBegin, padEnd, dilation, + padType, groups, deformableGroups, withBilinearInterpolationPad); + } + + function = makeNgraphFunction(ngPrc, parameters, deformable_conv, "deformable_convolution"); + } +}; + +TEST_P(DefConvLayerCPUTest, CompareWithRefs) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + Run(); + CheckPluginRelatedResults(executableNetwork, "DeformableConvolution"); +} + +namespace { + +/* CPU PARAMS */ +std::vector filterCPUInfoForDevice(bool enforceRef = false) { + std::vector resCPUParams; + if (enforceRef) { + resCPUParams.push_back(CPUSpecificParams{{}, {}, {}, {"ref_FP32"}}); + } else if (with_cpu_x86_avx512f()) { + resCPUParams.push_back(CPUSpecificParams{{}, {}, {}, {"jit_avx512_FP32"}}); + } else if (with_cpu_x86_avx2()) { + resCPUParams.push_back(CPUSpecificParams{{}, {}, {}, {"jit_avx2_FP32"}}); + } else if (with_cpu_x86_sse42()) { + resCPUParams.push_back(CPUSpecificParams{{}, {}, {}, {"jit_sse42"}}); + } else { + resCPUParams.push_back(CPUSpecificParams{{}, {}, {}, {"ref_FP32"}}); + } + return resCPUParams; +} + +const std::vector netPrecisions = { + InferenceEngine::Precision::FP32 +}; + +const auto defConvSpecificParams = ::testing::Combine( + ::testing::ValuesIn(std::vector { + true, + false + }), // with_bilinear_interpolation_pad + ::testing::ValuesIn(std::vector { + true, + false + }), // with_modulation + ::testing::ValuesIn(std::vector { + OffsetType::NATURAL, + OffsetType::ZERO, + OffsetType::REAL_MISC, + OffsetType::REAL_POSITIVE, + OffsetType::REAL_NEGATIVE + }) // offset type +); + +std::vector padTypes = { + ngraph::op::PadType::EXPLICIT, + ngraph::op::PadType::VALID +}; + +const auto spParams1 = ::testing::Combine( + ::testing::Values(1), // batch + ::testing::Values(std::vector({68, 68})), // in. spat. shape + ::testing::Values(std::vector({66, 66})), // off. spat. shape + ::testing::Values(std::vector({3, 3})), // ker. spat. shape + ::testing::ValuesIn(padTypes), // pad. type + ::testing::Values(std::vector({0, 0})), // pad. begin + ::testing::Values(std::vector({0, 0})), // pad. end + ::testing::Values(std::vector {1, 1}), // strides + ::testing::Values(std::vector {1, 1}) // dilations +); + +const auto spParams2 = ::testing::Combine( + ::testing::Values(1), // batch + ::testing::Values(std::vector({3, 3})), // in. spat. shape + ::testing::Values(std::vector({2, 2})), // off. spat. shape + ::testing::Values(std::vector({2, 2})), // ker. spat. shape + ::testing::ValuesIn(padTypes), // pad. type + ::testing::Values(std::vector({0, 0})), // pad. begin + ::testing::Values(std::vector({0, 0})), // pad. end + ::testing::Values(std::vector {1, 1}), // strides + ::testing::Values(std::vector {1, 1}) // dilations +); + +const auto spParams3 = ::testing::Combine( + ::testing::Values(1), // batch + ::testing::Values(std::vector({5, 5})), // in. spat. shape + ::testing::Values(std::vector({4, 4})), // off. spat. shape + ::testing::Values(std::vector({2, 2})), // ker. spat. shape + ::testing::ValuesIn(padTypes), // pad. type + ::testing::Values(std::vector({0, 0})), // pad. begin + ::testing::Values(std::vector({0, 0})), // pad. end + ::testing::Values(std::vector {1, 1}), // strides + ::testing::Values(std::vector {1, 1}) // dilations +); +const auto spParams4 = ::testing::Combine( + ::testing::Values(1), // batch + ::testing::Values(std::vector({3, 2})), // in. spat. shape + ::testing::Values(std::vector({2, 1})), // off. spat. shape + ::testing::Values(std::vector({2, 2})), // ker. spat. shape + ::testing::ValuesIn(padTypes), // pad. type + ::testing::Values(std::vector({0, 0})), // pad. begin + ::testing::Values(std::vector({0, 0})), // pad. end + ::testing::Values(std::vector {1, 1}), // strides + ::testing::Values(std::vector {1, 1}) // dilations +); + +const auto chParamsSingleGr = ::testing::Combine( + ::testing::ValuesIn(std::vector {1}), // gr. 1 + ::testing::ValuesIn(std::vector {1, 2}), // def. gr. 1,2 + ::testing::ValuesIn(std::vector {16, 32}), // in. ch. per gr. + ::testing::ValuesIn(std::vector {16, 32})); // out. ch. per gr. + +const auto chParamsMulGr = ::testing::Combine( + ::testing::ValuesIn(std::vector {2, 4}), // gr. 2,4 + ::testing::ValuesIn(std::vector {1, 2}), // def. gr. 1,2 + ::testing::ValuesIn(std::vector {3, 7}), // in. ch. per gr. + ::testing::ValuesIn(std::vector {3, 7})); // out. ch. per gr. + +const auto params1 = ::testing::Combine( + ::testing::Combine( + spParams1, + chParamsSingleGr, + defConvSpecificParams, + ::testing::ValuesIn(netPrecisions), + ::testing::Values(CommonTestUtils::DEVICE_CPU)), + ::testing::ValuesIn(filterCPUInfoForDevice())); +const auto params2 = ::testing::Combine( + ::testing::Combine( + spParams2, + chParamsSingleGr, + defConvSpecificParams, + ::testing::ValuesIn(netPrecisions), + ::testing::Values(CommonTestUtils::DEVICE_CPU)), + ::testing::ValuesIn(filterCPUInfoForDevice())); +const auto params3 = ::testing::Combine( + ::testing::Combine( + spParams3, + chParamsSingleGr, + defConvSpecificParams, + ::testing::ValuesIn(netPrecisions), + ::testing::Values(CommonTestUtils::DEVICE_CPU)), + ::testing::ValuesIn(filterCPUInfoForDevice())); +const auto params4 = ::testing::Combine( + ::testing::Combine( + spParams4, + chParamsSingleGr, + defConvSpecificParams, + ::testing::ValuesIn(netPrecisions), + ::testing::Values(CommonTestUtils::DEVICE_CPU)), + ::testing::ValuesIn(filterCPUInfoForDevice())); +const auto params5 = ::testing::Combine( + ::testing::Combine( + spParams4, + chParamsMulGr, + defConvSpecificParams, + ::testing::ValuesIn(netPrecisions), + ::testing::Values(CommonTestUtils::DEVICE_CPU)), + ::testing::ValuesIn(filterCPUInfoForDevice(true))); +INSTANTIATE_TEST_SUITE_P(smoke_DefConvLayoutTest1, DefConvLayerCPUTest, params1, DefConvLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_DefConvLayoutTest2, DefConvLayerCPUTest, params2, DefConvLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_DefConvLayoutTest3, DefConvLayerCPUTest, params3, DefConvLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_DefConvLayoutTest4, DefConvLayerCPUTest, params4, DefConvLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_DefConvLayoutTest5, DefConvLayerCPUTest, params5, DefConvLayerCPUTest::getTestCaseName); +} // namespace +} // namespace CPULayerTestsDefinitions