From 5afb63b06b244c12b0ebf618bbb9431d0ac2df8b Mon Sep 17 00:00:00 2001 From: azhogov Date: Mon, 27 Dec 2021 15:18:54 +0300 Subject: [PATCH 1/6] Azure CI: Add yaml for creating Windows docker --- .ci/azure/windows_docker.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .ci/azure/windows_docker.yml diff --git a/.ci/azure/windows_docker.yml b/.ci/azure/windows_docker.yml new file mode 100644 index 00000000000..b481cb41a37 --- /dev/null +++ b/.ci/azure/windows_docker.yml @@ -0,0 +1,24 @@ +jobs: +- job: Win_docker + timeoutInMinutes: 60 + + pool: + name: WIN_DOCKER_VMSS_VENV_F4S_WU2 + + variables: + system.debug: true + + steps: + - script: | + powershell -command "Invoke-RestMethod -Headers @{\"Metadata\"=\"true\"} -Method GET -Uri http://169.254.169.254/metadata/instance/compute?api-version=2019-06-01 | format-custom" + where python3 + python3 --version + where python + python --version + wmic computersystem get TotalPhysicalMemory + wmic cpu list + wmic logicaldisk get description,name + wmic VOLUME list + set + displayName: 'System info' + From a83bcee4bd188493e2e2ff10a7048ff07497e6b7 Mon Sep 17 00:00:00 2001 From: Chenhu Wang Date: Mon, 27 Dec 2021 20:51:50 +0800 Subject: [PATCH 2/6] [CPU] NMS optimization (#8312) --- .../nodes/mkldnn_non_max_suppression_node.cpp | 835 ++++++++++++++++-- .../nodes/mkldnn_non_max_suppression_node.h | 74 +- 2 files changed, 810 insertions(+), 99 deletions(-) diff --git a/src/plugins/intel_cpu/src/nodes/mkldnn_non_max_suppression_node.cpp b/src/plugins/intel_cpu/src/nodes/mkldnn_non_max_suppression_node.cpp index 5cb090a6951..8fe284f7d82 100644 --- a/src/plugins/intel_cpu/src/nodes/mkldnn_non_max_suppression_node.cpp +++ b/src/plugins/intel_cpu/src/nodes/mkldnn_non_max_suppression_node.cpp @@ -15,8 +15,536 @@ #include #include "utils/general_utils.h" +#include "cpu/x64/jit_generator.hpp" +#include "emitters/jit_load_store_emitters.hpp" +#include + using namespace MKLDNNPlugin; using namespace InferenceEngine; +using namespace mkldnn; +using namespace mkldnn::impl; +using namespace mkldnn::impl::cpu::x64; +using namespace mkldnn::impl::utils; +using namespace Xbyak; + +#define GET_OFF(field) offsetof(jit_nms_args, field) + +template +struct jit_uni_nms_kernel_f32 : public jit_uni_nms_kernel, public jit_generator { + DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_nms_kernel_f32) + + explicit jit_uni_nms_kernel_f32(jit_nms_config_params jcp_) : jit_uni_nms_kernel(jcp_), jit_generator() {} + + void create_ker() override { + jit_generator::create_kernel(); + ker_ = (decltype(ker_))jit_ker(); + } + + void generate() override { + load_emitter.reset(new jit_load_emitter(this, isa, nullptr)); + store_emitter.reset(new jit_store_emitter(this, isa, nullptr)); + exp_injector.reset(new jit_uni_eltwise_injector_f32(this, mkldnn::impl::alg_kind::eltwise_exp, 0.f, 0.f, 1.0f)); + + this->preamble(); + + uni_vpxor(vmm_zero, vmm_zero, vmm_zero); + + load_pool_gpr_idxs = {static_cast(reg_load_store_mask.getIdx()), static_cast(reg_load_table.getIdx())}; + store_pool_gpr_idxs = {static_cast(reg_load_store_mask.getIdx())}; + store_pool_vec_idxs = {static_cast(vmm_zero.getIdx())}; + + mov(reg_boxes_coord0, ptr[reg_params + GET_OFF(selected_boxes_coord[0])]); + mov(reg_boxes_coord1, ptr[reg_params + GET_OFF(selected_boxes_coord[0]) + 1 * sizeof(size_t)]); + mov(reg_boxes_coord2, ptr[reg_params + GET_OFF(selected_boxes_coord[0]) + 2 * sizeof(size_t)]); + mov(reg_boxes_coord3, ptr[reg_params + GET_OFF(selected_boxes_coord[0]) + 3 * sizeof(size_t)]); + mov(reg_candidate_box, ptr[reg_params + GET_OFF(candidate_box)]); + mov(reg_candidate_status, ptr[reg_params + GET_OFF(candidate_status)]); + mov(reg_boxes_num, ptr[reg_params + GET_OFF(selected_boxes_num)]); + mov(reg_iou_threshold, ptr[reg_params + GET_OFF(iou_threshold)]); + // soft + mov(reg_score_threshold, ptr[reg_params + GET_OFF(score_threshold)]); + mov(reg_score, ptr[reg_params + GET_OFF(score)]); + mov(reg_scale, ptr[reg_params + GET_OFF(scale)]); + + // could use rcx(reg_table) and rdi(reg_temp) now as abi parse finished + mov(reg_table, l_table_constant); + if (mayiuse(cpu::x64::avx512_common)) { + kmovw(k_mask_one, word[reg_table + vlen]); + } + uni_vbroadcastss(vmm_iou_threshold, ptr[reg_iou_threshold]); + uni_vbroadcastss(vmm_score_threshold, ptr[reg_score_threshold]); + + uni_vbroadcastss(vmm_candidate_coord0, ptr[reg_candidate_box]); + uni_vbroadcastss(vmm_candidate_coord1, ptr[reg_candidate_box + 1 * sizeof(float)]); + uni_vbroadcastss(vmm_candidate_coord2, ptr[reg_candidate_box + 2 * sizeof(float)]); + uni_vbroadcastss(vmm_candidate_coord3, ptr[reg_candidate_box + 3 * sizeof(float)]); + + if (jcp.box_encode_type == NMSBoxEncodeType::CORNER) { + // box format: y1, x1, y2, x2 + uni_vminps(vmm_temp1, vmm_candidate_coord0, vmm_candidate_coord2); + uni_vmaxps(vmm_temp2, vmm_candidate_coord0, vmm_candidate_coord2); + uni_vmovups(vmm_candidate_coord0, vmm_temp1); + uni_vmovups(vmm_candidate_coord2, vmm_temp2); + + uni_vminps(vmm_temp1, vmm_candidate_coord1, vmm_candidate_coord3); + uni_vmaxps(vmm_temp2, vmm_candidate_coord1, vmm_candidate_coord3); + uni_vmovups(vmm_candidate_coord1, vmm_temp1); + uni_vmovups(vmm_candidate_coord3, vmm_temp2); + } else { + // box format: x_center, y_center, width, height --> y1, x1, y2, x2 + uni_vmulps(vmm_temp1, vmm_candidate_coord2, ptr[reg_table]); // width/2 + uni_vmulps(vmm_temp2, vmm_candidate_coord3, ptr[reg_table]); // height/2 + + uni_vaddps(vmm_temp3, vmm_candidate_coord0, vmm_temp1); // x_center + width/2 + uni_vmovups(vmm_candidate_coord3, vmm_temp3); + + uni_vaddps(vmm_temp3, vmm_candidate_coord1, vmm_temp2); // y_center + height/2 + uni_vmovups(vmm_candidate_coord2, vmm_temp3); + + uni_vsubps(vmm_temp3, vmm_candidate_coord0, vmm_temp1); // x_center - width/2 + uni_vsubps(vmm_temp4, vmm_candidate_coord1, vmm_temp2); // y_center - height/2 + + uni_vmovups(vmm_candidate_coord1, vmm_temp3); + uni_vmovups(vmm_candidate_coord0, vmm_temp4); + } + + // check from last to first + imul(reg_temp_64, reg_boxes_num, sizeof(float)); + add(reg_boxes_coord0, reg_temp_64); // y1 + add(reg_boxes_coord1, reg_temp_64); // x1 + add(reg_boxes_coord2, reg_temp_64); // y2 + add(reg_boxes_coord3, reg_temp_64); // x2 + + Xbyak::Label hard_nms_label; + Xbyak::Label nms_end_label; + + mov(reg_temp_32, ptr[reg_scale]); + test(reg_temp_32, reg_temp_32); + jz(hard_nms_label, T_NEAR); + + soft_nms(); + + jmp(nms_end_label, T_NEAR); + + L(hard_nms_label); + + hard_nms(); + + L(nms_end_label); + + this->postamble(); + + load_emitter->emit_data(); + store_emitter->emit_data(); + + prepare_table(); + exp_injector->prepare_table(); + } + +private: + using Vmm = typename conditional3::type; + uint32_t vlen = cpu_isa_traits::vlen; + + Xbyak::Reg64 reg_boxes_coord0 = r8; + Xbyak::Reg64 reg_boxes_coord1 = r9; + Xbyak::Reg64 reg_boxes_coord2 = r10; + Xbyak::Reg64 reg_boxes_coord3 = r11; + Xbyak::Reg64 reg_candidate_box = r12; + Xbyak::Reg64 reg_candidate_status = r13; + Xbyak::Reg64 reg_boxes_num = r14; + Xbyak::Reg64 reg_iou_threshold = r15; + // more for soft + Xbyak::Reg64 reg_score_threshold = rdx; + Xbyak::Reg64 reg_score = rbp; + Xbyak::Reg64 reg_scale = rsi; + + Xbyak::Reg64 reg_load_table = rax; + Xbyak::Reg64 reg_load_store_mask = rbx; + + // reuse + Xbyak::Label l_table_constant; + Xbyak::Reg64 reg_table = rcx; + Xbyak::Reg64 reg_temp_64 = rdi; + Xbyak::Reg32 reg_temp_32 = edi; + + Xbyak::Reg64 reg_params = abi_param1; + + std::unique_ptr load_emitter = nullptr; + std::unique_ptr store_emitter = nullptr; + + std::vector store_pool_gpr_idxs; + std::vector store_pool_vec_idxs; + std::vector load_pool_gpr_idxs; + + Vmm vmm_boxes_coord0 = Vmm(1); + Vmm vmm_boxes_coord1 = Vmm(2); + Vmm vmm_boxes_coord2 = Vmm(3); + Vmm vmm_boxes_coord3 = Vmm(4); + Vmm vmm_candidate_coord0 = Vmm(5); + Vmm vmm_candidate_coord1 = Vmm(6); + Vmm vmm_candidate_coord2 = Vmm(7); + Vmm vmm_candidate_coord3 = Vmm(8); + Vmm vmm_temp1 = Vmm(9); + Vmm vmm_temp2 = Vmm(10); + Vmm vmm_temp3 = Vmm(11); + Vmm vmm_temp4 = Vmm(12); + + Vmm vmm_iou_threshold = Vmm(13); + Vmm vmm_zero = Vmm(15); + + // soft + Vmm vmm_score_threshold = Vmm(14); + Vmm vmm_scale = Vmm(0); + + Xbyak::Opmask k_mask = Xbyak::Opmask(7); + Xbyak::Opmask k_mask_one = Xbyak::Opmask(6); + + std::shared_ptr> exp_injector; + + inline void hard_nms() { + int step = vlen / sizeof(float); + Xbyak::Label main_loop_label_hard; + Xbyak::Label main_loop_end_label_hard; + Xbyak::Label tail_loop_label_hard; + Xbyak::Label terminate_label_hard; + L(main_loop_label_hard); + { + cmp(reg_boxes_num, step); + jl(main_loop_end_label_hard, T_NEAR); + + sub(reg_boxes_coord0, step * sizeof(float)); + sub(reg_boxes_coord1, step * sizeof(float)); + sub(reg_boxes_coord2, step * sizeof(float)); + sub(reg_boxes_coord3, step * sizeof(float)); + + // iou result is in vmm_temp3 + iou(step); + + sub(reg_boxes_num, step); + + suppressed_by_iou(false); + + // if zero continue, else set result to suppressed and terminate + jz(main_loop_label_hard, T_NEAR); + + uni_vpextrd(ptr[reg_candidate_status], Xmm(vmm_zero.getIdx()), 0); + + jmp(terminate_label_hard, T_NEAR); + } + L(main_loop_end_label_hard); + + step = 1; + L(tail_loop_label_hard); + { + cmp(reg_boxes_num, 1); + jl(terminate_label_hard, T_NEAR); + + sub(reg_boxes_coord0, step * sizeof(float)); + sub(reg_boxes_coord1, step * sizeof(float)); + sub(reg_boxes_coord2, step * sizeof(float)); + sub(reg_boxes_coord3, step * sizeof(float)); + + // iou result is in vmm_temp3 + iou(step); + + sub(reg_boxes_num, step); + + suppressed_by_iou(true); + + jz(tail_loop_label_hard, T_NEAR); + + uni_vpextrd(ptr[reg_candidate_status], Xmm(vmm_zero.getIdx()), 0); + + jmp(terminate_label_hard, T_NEAR); + } + + L(terminate_label_hard); + } + + inline void soft_nms() { + uni_vbroadcastss(vmm_scale, ptr[reg_scale]); + + int step = vlen / sizeof(float); + Xbyak::Label main_loop_label; + Xbyak::Label main_loop_end_label; + Xbyak::Label tail_loop_label; + Xbyak::Label terminate_label; + + Xbyak::Label main_loop_label_soft; + Xbyak::Label tail_loop_label_soft; + L(main_loop_label); + { + cmp(reg_boxes_num, step); + jl(main_loop_end_label, T_NEAR); + + sub(reg_boxes_coord0, step * sizeof(float)); + sub(reg_boxes_coord1, step * sizeof(float)); + sub(reg_boxes_coord2, step * sizeof(float)); + sub(reg_boxes_coord3, step * sizeof(float)); + + // result(iou and weight) is in vmm_temp3 + iou(step); + sub(reg_boxes_num, step); + + // soft suppressed by iou_threshold + if (jcp.is_soft_suppressed_by_iou) { + suppressed_by_iou(false); + + // if zero continue soft suppression, else set result to suppressed and terminate + jz(main_loop_label_soft, T_NEAR); + + uni_vpextrd(ptr[reg_candidate_status], Xmm(vmm_zero.getIdx()), 0); + + jmp(terminate_label, T_NEAR); + + L(main_loop_label_soft); + } + + // weight: std::exp(scale * iou * iou) + soft_coeff(); + + // vector weights multiply + horizontal_mul(); + + uni_vbroadcastss(vmm_temp1, ptr[reg_score]); + + // new score in vmm3[0] + uni_vmulps(vmm_temp3, vmm_temp3, vmm_temp1); + // store new score + uni_vmovss(ptr[reg_score], vmm_temp3); + + // cmpps(_CMP_LE_OS) if new score is less or equal than score_threshold + suppressed_by_score(); + + jz(main_loop_label, T_NEAR); + + uni_vpextrd(ptr[reg_candidate_status], Xmm(vmm_zero.getIdx()), 0); + + jmp(terminate_label, T_NEAR); + } + L(main_loop_end_label); + + step = 1; + L(tail_loop_label); + { + cmp(reg_boxes_num, 1); + jl(terminate_label, T_NEAR); + + sub(reg_boxes_coord0, step * sizeof(float)); + sub(reg_boxes_coord1, step * sizeof(float)); + sub(reg_boxes_coord2, step * sizeof(float)); + sub(reg_boxes_coord3, step * sizeof(float)); + + iou(step); + sub(reg_boxes_num, step); + + // soft suppressed by iou_threshold + if (jcp.is_soft_suppressed_by_iou) { + suppressed_by_iou(true); + + jz(tail_loop_label_soft, T_NEAR); + + uni_vpextrd(ptr[reg_candidate_status], Xmm(vmm_zero.getIdx()), 0); + + jmp(terminate_label, T_NEAR); + + L(tail_loop_label_soft); + } + + soft_coeff(); + + uni_vbroadcastss(vmm_temp1, ptr[reg_score]); + + // vmm3[0] is valide, no need horizontal mul. + uni_vmulps(vmm_temp3, vmm_temp3, vmm_temp1); + + uni_vmovss(ptr[reg_score], vmm_temp3); + + // cmpps(_CMP_LE_OS) if new score is less or equal than score_threshold + suppressed_by_score(); + + jz(tail_loop_label, T_NEAR); + + uni_vpextrd(ptr[reg_candidate_status], Xmm(vmm_zero.getIdx()), 0); + + jmp(terminate_label, T_NEAR); + } + + L(terminate_label); + } + + inline void suppressed_by_iou(bool is_scalar) { + if (mayiuse(cpu::x64::avx512_common)) { + vcmpps(k_mask, vmm_temp3, vmm_iou_threshold, 0x0D); // _CMP_GE_OS. vcmpps w/ kmask only on V5 + if (is_scalar) + kandw(k_mask, k_mask, k_mask_one); + kortestw(k_mask, k_mask); // bitwise check if all zero + } else if (mayiuse(cpu::x64::avx)) { + // vex instructions with xmm on avx and ymm on avx2 + vcmpps(vmm_temp4, vmm_temp3, vmm_iou_threshold, 0x0D); // xmm and ymm only on V1. + if (is_scalar) { + uni_vpextrd(reg_temp_32, Xmm(vmm_temp4.getIdx()), 0); + test(reg_temp_32, reg_temp_32); + } else { + uni_vtestps(vmm_temp4, vmm_temp4); // vtestps: sign bit check if all zeros, ymm and xmm only on V1, N/A on V5 + } + } else { + // pure sse path, make sure don't spoil vmm_temp3, which may used in after soft-suppression + uni_vmovups(vmm_temp4, vmm_temp3); + cmpps(vmm_temp4, vmm_iou_threshold, 0x07); // order compare, 0 for unorders + + uni_vmovups(vmm_temp2, vmm_temp3); + cmpps(vmm_temp2, vmm_iou_threshold, 0x05); // _CMP_GE_US on sse, no direct _CMP_GE_OS supported. + + uni_vandps(vmm_temp4, vmm_temp4, vmm_temp2); + if (is_scalar) { + uni_vpextrd(reg_temp_32, Xmm(vmm_temp4.getIdx()), 0); + test(reg_temp_32, reg_temp_32); + } else { + uni_vtestps(vmm_temp4, vmm_temp4); // ptest: bitwise check if all zeros, on sse41 + } + } + } + + inline void suppressed_by_score() { + if (mayiuse(cpu::x64::avx512_common)) { + vcmpps(k_mask, vmm_temp3, vmm_score_threshold, 0x02); // vcmpps w/ kmask only on V5, w/o kmask version N/A on V5 + kandw(k_mask, k_mask, k_mask_one); + kortestw(k_mask, k_mask); // bitwise check if all zero + } else if (mayiuse(cpu::x64::avx)) { + vcmpps(vmm_temp4, vmm_temp3, vmm_score_threshold, 0x02); + uni_vpextrd(reg_temp_32, Xmm(vmm_temp4.getIdx()), 0); + test(reg_temp_32, reg_temp_32); + } else { + cmpps(vmm_temp3, vmm_score_threshold, 0x02); // _CMP_LE_OS on sse + uni_vpextrd(reg_temp_32, Xmm(vmm_temp3.getIdx()), 0); + test(reg_temp_32, reg_temp_32); + } + } + + inline void iou(int ele_num) { + auto load = [&](Xbyak::Reg64 reg_src, Vmm vmm_dst) { + load_emitter->emit_code({static_cast(reg_src.getIdx())}, {static_cast(vmm_dst.getIdx())}, + std::make_shared(Precision::FP32, Precision::FP32, ele_num), + {}, {load_pool_gpr_idxs}); + }; + load(reg_boxes_coord0, vmm_boxes_coord0); + load(reg_boxes_coord1, vmm_boxes_coord1); + load(reg_boxes_coord2, vmm_boxes_coord2); + load(reg_boxes_coord3, vmm_boxes_coord3); + + if (jcp.box_encode_type == NMSBoxEncodeType::CORNER) { + // box format: y1, x1, y2, x2 + uni_vminps(vmm_temp1, vmm_boxes_coord0, vmm_boxes_coord2); + uni_vmaxps(vmm_temp2, vmm_boxes_coord0, vmm_boxes_coord2); + uni_vmovups(vmm_boxes_coord0, vmm_temp1); + uni_vmovups(vmm_boxes_coord2, vmm_temp2); + + uni_vminps(vmm_temp1, vmm_boxes_coord1, vmm_boxes_coord3); + uni_vmaxps(vmm_temp2, vmm_boxes_coord1, vmm_boxes_coord3); + uni_vmovups(vmm_boxes_coord1, vmm_temp1); + uni_vmovups(vmm_boxes_coord3, vmm_temp2); + } else { + // box format: x_center, y_center, width, height --> y1, x1, y2, x2 + uni_vmulps(vmm_temp1, vmm_boxes_coord2, ptr[reg_table]); // width/2 + uni_vmulps(vmm_temp2, vmm_boxes_coord3, ptr[reg_table]); // height/2 + + uni_vaddps(vmm_temp3, vmm_boxes_coord0, vmm_temp1); // x_center + width/2 + uni_vmovups(vmm_boxes_coord3, vmm_temp3); + + uni_vaddps(vmm_temp3, vmm_boxes_coord1, vmm_temp2); // y_center + height/2 + uni_vmovups(vmm_boxes_coord2, vmm_temp3); + + uni_vsubps(vmm_temp3, vmm_boxes_coord0, vmm_temp1); // x_center - width/2 + uni_vsubps(vmm_temp4, vmm_boxes_coord1, vmm_temp2); // y_center - height/2 + + uni_vmovups(vmm_boxes_coord1, vmm_temp3); + uni_vmovups(vmm_boxes_coord0, vmm_temp4); + } + + uni_vsubps(vmm_temp1, vmm_boxes_coord2, vmm_boxes_coord0); + uni_vsubps(vmm_temp2, vmm_boxes_coord3, vmm_boxes_coord1); + uni_vmulps(vmm_temp1, vmm_temp1, vmm_temp2); // boxes area + + uni_vsubps(vmm_temp2, vmm_candidate_coord2, vmm_candidate_coord0); + uni_vsubps(vmm_temp3, vmm_candidate_coord3, vmm_candidate_coord1); + uni_vmulps(vmm_temp2, vmm_temp2, vmm_temp3); // candidate(bc) area // candidate area calculate once and check if 0 + + uni_vaddps(vmm_temp1, vmm_temp1, vmm_temp2); // areaI + areaJ to free vmm_temp2 + + // y of intersection + uni_vminps(vmm_temp3, vmm_boxes_coord2, vmm_candidate_coord2); // min(Ymax) + uni_vmaxps(vmm_temp4, vmm_boxes_coord0, vmm_candidate_coord0); // max(Ymin) + uni_vsubps(vmm_temp3, vmm_temp3, vmm_temp4); // min(Ymax) - max(Ymin) + uni_vmaxps(vmm_temp3, vmm_temp3, vmm_zero); + + // x of intersection + uni_vminps(vmm_temp4, vmm_boxes_coord3, vmm_candidate_coord3); // min(Xmax) + uni_vmaxps(vmm_temp2, vmm_boxes_coord1, vmm_candidate_coord1); // max(Xmin) + uni_vsubps(vmm_temp4, vmm_temp4, vmm_temp2); // min(Xmax) - max(Xmin) + uni_vmaxps(vmm_temp4, vmm_temp4, vmm_zero); + + // intersection_area + uni_vmulps(vmm_temp3, vmm_temp3, vmm_temp4); + + // iou: intersection_area / (areaI + areaJ - intersection_area); + uni_vsubps(vmm_temp1, vmm_temp1, vmm_temp3); + uni_vdivps(vmm_temp3, vmm_temp3, vmm_temp1); + } + + // std::exp(scale * iou * iou) + inline void soft_coeff() { + uni_vmulps(vmm_temp3, vmm_temp3, vmm_temp3); + uni_vmulps(vmm_temp3, vmm_temp3, vmm_scale); + exp_injector->compute_vector_range(vmm_temp3.getIdx(), vmm_temp3.getIdx() + 1); + } + + inline void horizontal_mul_xmm(const Xbyak::Xmm &xmm_weight, const Xbyak::Xmm &xmm_aux) { + uni_vmovshdup(xmm_aux, xmm_weight); // weight:1,2,3,4; aux:2,2,4,4 + uni_vmulps(xmm_weight, xmm_weight, xmm_aux); // weight:1*2,2*2,3*4,4*4 + uni_vmovhlps(xmm_aux, xmm_aux, xmm_weight); // aux:3*4,4*4,4,4 + uni_vmulps(xmm_weight, xmm_weight, xmm_aux); // weight:1*2*3*4,... + } + + // horizontal mul for vmm_weight(Vmm(3)), temp1 and temp2 as aux + inline void horizontal_mul() { + Xbyak::Xmm xmm_weight = Xbyak::Xmm(vmm_temp3.getIdx()); + Xbyak::Xmm xmm_temp1 = Xbyak::Xmm(vmm_temp1.getIdx()); + Xbyak::Xmm xmm_temp2 = Xbyak::Xmm(vmm_temp2.getIdx()); + if (isa == cpu::x64::sse41) { + horizontal_mul_xmm(xmm_weight, xmm_temp1); + } else if (isa == cpu::x64::avx2) { + Xbyak::Ymm ymm_weight = Xbyak::Ymm(vmm_temp3.getIdx()); + vextractf128(xmm_temp1, ymm_weight, 0); + vextractf128(xmm_temp2, ymm_weight, 1); + uni_vmulps(xmm_weight, xmm_temp1, xmm_temp2); + horizontal_mul_xmm(xmm_weight, xmm_temp1); + } else { + Xbyak::Zmm zmm_weight = Xbyak::Zmm(vmm_temp3.getIdx()); + vextractf32x4(xmm_temp1, zmm_weight, 0); + vextractf32x4(xmm_temp2, zmm_weight, 1); + uni_vmulps(xmm_temp1, xmm_temp1, xmm_temp2); + vextractf32x4(xmm_temp2, zmm_weight, 2); + vextractf32x4(xmm_weight, zmm_weight, 3); + uni_vmulps(xmm_weight, xmm_weight, xmm_temp2); + uni_vmulps(xmm_weight, xmm_weight, xmm_temp1); + horizontal_mul_xmm(xmm_weight, xmm_temp1); + } + } + + inline void prepare_table() { + auto broadcast_d = [&](int val) { + for (size_t d = 0; d < vlen / sizeof(int); ++d) { + dd(val); + } + }; + + align(64); + L(l_table_constant); + broadcast_d(0x3f000000); // 0.5f + dw(0x0001); + } +}; bool MKLDNNNonMaxSuppressionNode::isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept { try { @@ -42,7 +570,7 @@ bool MKLDNNNonMaxSuppressionNode::isSupportedOperation(const std::shared_ptr& op, const mkldnn::engine& eng, - MKLDNNWeightsSharing::Ptr &cache) : MKLDNNNode(op, eng, cache) { + MKLDNNWeightsSharing::Ptr &cache) : MKLDNNNode(op, eng, cache), isSoftSuppressedByIOU(true) { std::string errorMessage; if (!isSupportedOperation(op, errorMessage)) { IE_THROW(NotImplemented) << errorMessage; @@ -57,12 +585,12 @@ MKLDNNNonMaxSuppressionNode::MKLDNNNonMaxSuppressionNode(const std::shared_ptr(op)) { - boxEncodingType = static_cast(nms5->get_box_encoding()); - sort_result_descending = nms5->get_sort_result_descending(); + boxEncodingType = static_cast(nms5->get_box_encoding()); + sortResultDescending = nms5->get_sort_result_descending(); // TODO [DS NMS]: remove when nodes from models where nms is not last node in model supports DS } else if (const auto nmsIe = std::dynamic_pointer_cast(op)) { - boxEncodingType = nmsIe->m_center_point_box ? boxEncoding::CENTER : boxEncoding::CORNER; - sort_result_descending = nmsIe->m_sort_result_descending; + boxEncodingType = nmsIe->m_center_point_box ? NMSBoxEncodeType::CENTER : NMSBoxEncodeType::CORNER; + sortResultDescending = nmsIe->m_sort_result_descending; } else { const auto &typeInfo = op->get_type_info(); IE_THROW() << errorPrefix << " doesn't support NMS: " << typeInfo.name << " v" << typeInfo.version; @@ -125,32 +653,63 @@ void MKLDNNNonMaxSuppressionNode::initSupportedPrimitiveDescriptors() { outDataConf.emplace_back(LayoutType::ncsp, outPrecision); } - addSupportedPrimDesc(inDataConf, outDataConf, impl_desc_type::ref_any); + impl_desc_type impl_type; + if (mayiuse(cpu::x64::avx512_common)) { + impl_type = impl_desc_type::jit_avx512; + } else if (mayiuse(cpu::x64::avx2)) { + impl_type = impl_desc_type::jit_avx2; + } else if (mayiuse(cpu::x64::sse41)) { + impl_type = impl_desc_type::jit_sse42; + } else { + impl_type = impl_desc_type::ref; + } + + addSupportedPrimDesc(inDataConf, outDataConf, impl_type); + + // as only FP32 and ncsp is supported, and kernel is shape agnostic, we can create here. There is no need to recompilation. + createJitKernel(); } void MKLDNNNonMaxSuppressionNode::prepareParams() { - const auto& boxes_dims = isDynamicNode() ? getParentEdgesAtPort(NMS_BOXES)[0]->getMemory().getStaticDims() : + const auto& boxesDims = isDynamicNode() ? getParentEdgesAtPort(NMS_BOXES)[0]->getMemory().getStaticDims() : getInputShapeAtPort(NMS_BOXES).getStaticDims(); - const auto& scores_dims = isDynamicNode() ? getParentEdgesAtPort(NMS_SCORES)[0]->getMemory().getStaticDims() : + const auto& scoresDims = isDynamicNode() ? getParentEdgesAtPort(NMS_SCORES)[0]->getMemory().getStaticDims() : getInputShapeAtPort(NMS_SCORES).getStaticDims(); - num_batches = boxes_dims[0]; - num_boxes = boxes_dims[1]; - num_classes = scores_dims[1]; - if (num_batches != scores_dims[0]) - IE_THROW() << errorPrefix << " num_batches is different in 'boxes' and 'scores' inputs"; - if (num_boxes != scores_dims[2]) - IE_THROW() << errorPrefix << " num_boxes is different in 'boxes' and 'scores' inputs"; + numBatches = boxesDims[0]; + numBoxes = boxesDims[1]; + numClasses = scoresDims[1]; + if (numBatches != scoresDims[0]) + IE_THROW() << errorPrefix << " numBatches is different in 'boxes' and 'scores' inputs"; + if (numBoxes != scoresDims[2]) + IE_THROW() << errorPrefix << " numBoxes is different in 'boxes' and 'scores' inputs"; - numFiltBox.resize(num_batches); + numFiltBox.resize(numBatches); for (auto & i : numFiltBox) - i.resize(num_classes); + i.resize(numClasses); } bool MKLDNNNonMaxSuppressionNode::isExecutable() const { return isDynamicNode() || MKLDNNNode::isExecutable(); } +void MKLDNNNonMaxSuppressionNode::createJitKernel() { + auto jcp = jit_nms_config_params(); + jcp.box_encode_type = boxEncodingType; + jcp.is_soft_suppressed_by_iou = isSoftSuppressedByIOU; + + if (mayiuse(cpu::x64::avx512_common)) { + nms_kernel.reset(new jit_uni_nms_kernel_f32(jcp)); + } else if (mayiuse(cpu::x64::avx2)) { + nms_kernel.reset(new jit_uni_nms_kernel_f32(jcp)); + } else if (mayiuse(cpu::x64::sse41)) { + nms_kernel.reset(new jit_uni_nms_kernel_f32(jcp)); + } + + if (nms_kernel) + nms_kernel->create_ker(); +} + void MKLDNNNonMaxSuppressionNode::executeDynamicImpl(mkldnn::stream strm) { if (hasEmptyInputTensors() || (inputShapes.size() > NMS_MAXOUTPUTBOXESPERCLASS && reinterpret_cast(getParentEdgeAt(NMS_MAXOUTPUTBOXESPERCLASS)->getMemoryPtr()->GetPtr())[0] == 0)) { @@ -169,35 +728,35 @@ void MKLDNNNonMaxSuppressionNode::execute(mkldnn::stream strm) { const float *scores = reinterpret_cast(getParentEdgeAt(NMS_SCORES)->getMemoryPtr()->GetPtr()); if (inputShapes.size() > NMS_MAXOUTPUTBOXESPERCLASS) { - max_output_boxes_per_class = reinterpret_cast(getParentEdgeAt(NMS_MAXOUTPUTBOXESPERCLASS)->getMemoryPtr()->GetPtr())[0]; + maxOutputBoxesPerClass = reinterpret_cast(getParentEdgeAt(NMS_MAXOUTPUTBOXESPERCLASS)->getMemoryPtr()->GetPtr())[0]; } - max_output_boxes_per_class = std::min(max_output_boxes_per_class, num_boxes); + maxOutputBoxesPerClass = std::min(maxOutputBoxesPerClass, numBoxes); - if (max_output_boxes_per_class == 0) { + if (maxOutputBoxesPerClass == 0) { return; } if (inputShapes.size() > NMS_IOUTHRESHOLD) - iou_threshold = reinterpret_cast(getParentEdgeAt(NMS_IOUTHRESHOLD)->getMemoryPtr()->GetPtr())[0]; + iouThreshold = reinterpret_cast(getParentEdgeAt(NMS_IOUTHRESHOLD)->getMemoryPtr()->GetPtr())[0]; if (inputShapes.size() > NMS_SCORETHRESHOLD) - score_threshold = reinterpret_cast(getParentEdgeAt(NMS_SCORETHRESHOLD)->getMemoryPtr()->GetPtr())[0]; + scoreThreshold = reinterpret_cast(getParentEdgeAt(NMS_SCORETHRESHOLD)->getMemoryPtr()->GetPtr())[0]; if (inputShapes.size() > NMS_SOFTNMSSIGMA) - soft_nms_sigma = reinterpret_cast(getParentEdgeAt(NMS_SOFTNMSSIGMA)->getMemoryPtr()->GetPtr())[0]; + softNMSSigma = reinterpret_cast(getParentEdgeAt(NMS_SOFTNMSSIGMA)->getMemoryPtr()->GetPtr())[0]; scale = 0.0f; - if (soft_nms_sigma > 0.0) { - scale = -0.5f / soft_nms_sigma; + if (softNMSSigma > 0.0) { + scale = -0.5f / softNMSSigma; } auto boxesStrides = getParentEdgeAt(NMS_BOXES)->getMemory().GetDescWithType()->getStrides(); auto scoresStrides = getParentEdgeAt(NMS_SCORES)->getMemory().GetDescWithType()->getStrides(); - const auto maxNumberOfBoxes = max_output_boxes_per_class * num_batches * num_classes; + const auto maxNumberOfBoxes = maxOutputBoxesPerClass * numBatches * numClasses; std::vector filtBoxes(maxNumberOfBoxes); - if (soft_nms_sigma == 0.0f) { + if (softNMSSigma == 0.0f) { nmsWithoutSoftSigma(boxes, scores, boxesStrides, scoresStrides, filtBoxes); } else { nmsWithSoftSigma(boxes, scores, boxesStrides, scoresStrides, filtBoxes); @@ -205,9 +764,9 @@ void MKLDNNNonMaxSuppressionNode::execute(mkldnn::stream strm) { size_t startOffset = numFiltBox[0][0]; for (size_t b = 0; b < numFiltBox.size(); b++) { - size_t batchOffset = b*num_classes*max_output_boxes_per_class; + size_t batchOffset = b*numClasses*maxOutputBoxesPerClass; for (size_t c = (b == 0 ? 1 : 0); c < numFiltBox[b].size(); c++) { - size_t offset = batchOffset + c*max_output_boxes_per_class; + size_t offset = batchOffset + c*maxOutputBoxesPerClass; for (size_t i = 0; i < numFiltBox[b][c]; i++) { filtBoxes[startOffset + i] = filtBoxes[offset + i]; } @@ -218,7 +777,7 @@ void MKLDNNNonMaxSuppressionNode::execute(mkldnn::stream strm) { // need more particular comparator to get deterministic behaviour // escape situation when filtred boxes with same score have different position from launch to launch - if (sort_result_descending) { + if (sortResultDescending) { parallel_sort(filtBoxes.begin(), filtBoxes.end(), [](const filteredBoxes& l, const filteredBoxes& r) { return (l.score > r.score) || @@ -273,7 +832,7 @@ bool MKLDNNNonMaxSuppressionNode::created() const { float MKLDNNNonMaxSuppressionNode::intersectionOverUnion(const float *boxesI, const float *boxesJ) { float yminI, xminI, ymaxI, xmaxI, yminJ, xminJ, ymaxJ, xmaxJ; - if (boxEncodingType == boxEncoding::CENTER) { + if (boxEncodingType == NMSBoxEncodeType::CENTER) { // box format: x_center, y_center, width, height yminI = boxesI[1] - boxesI[3] / 2.f; xminI = boxesI[0] - boxesI[2] / 2.f; @@ -312,99 +871,205 @@ void MKLDNNNonMaxSuppressionNode::nmsWithSoftSigma(const float *boxes, const flo return l.score < r.score || ((l.score == r.score) && (l.idx > r.idx)); }; + // update score, if iou is 0, weight is 1, score does not change + // if is_soft_suppressed_by_iou is false, apply for all iou, including iou>iou_threshold, soft suppressed when score < score_threshold + // if is_soft_suppressed_by_iou is true, hard suppressed by iou_threshold, then soft suppress auto coeff = [&](float iou) { - const float weight = std::exp(scale * iou * iou); - return iou <= iou_threshold ? weight : 0.0f; + if (isSoftSuppressedByIOU && iou > iouThreshold) + return 0.0f; + return std::exp(scale * iou * iou); }; - parallel_for2d(num_batches, num_classes, [&](int batch_idx, int class_idx) { - std::vector fb; + parallel_for2d(numBatches, numClasses, [&](int batch_idx, int class_idx) { + std::vector selectedBoxes; const float *boxesPtr = boxes + batch_idx * boxesStrides[0]; const float *scoresPtr = scores + batch_idx * scoresStrides[0] + class_idx * scoresStrides[1]; - std::priority_queue, decltype(less)> sorted_boxes(less); - for (int box_idx = 0; box_idx < num_boxes; box_idx++) { - if (scoresPtr[box_idx] > score_threshold) + std::priority_queue, decltype(less)> sorted_boxes(less); // score, box_id, suppress_begin_index + for (int box_idx = 0; box_idx < numBoxes; box_idx++) { + if (scoresPtr[box_idx] > scoreThreshold) sorted_boxes.emplace(boxInfo({scoresPtr[box_idx], box_idx, 0})); } + size_t sortedBoxSize = sorted_boxes.size(); + size_t maxSeletedBoxNum = std::min(sortedBoxSize, maxOutputBoxesPerClass); + selectedBoxes.reserve(maxSeletedBoxNum); + if (maxSeletedBoxNum > 0) { + // include first directly + boxInfo candidateBox = sorted_boxes.top(); + sorted_boxes.pop(); + selectedBoxes.push_back({ candidateBox.score, batch_idx, class_idx, candidateBox.idx }); + if (maxSeletedBoxNum > 1) { + if (nms_kernel) { + std::vector boxCoord0(maxSeletedBoxNum, 0.0f); + std::vector boxCoord1(maxSeletedBoxNum, 0.0f); + std::vector boxCoord2(maxSeletedBoxNum, 0.0f); + std::vector boxCoord3(maxSeletedBoxNum, 0.0f); - fb.reserve(sorted_boxes.size()); - if (sorted_boxes.size() > 0) { - while (fb.size() < max_output_boxes_per_class && !sorted_boxes.empty()) { - boxInfo currBox = sorted_boxes.top(); - float origScore = currBox.score; - sorted_boxes.pop(); + boxCoord0[0] = boxesPtr[candidateBox.idx * 4]; + boxCoord1[0] = boxesPtr[candidateBox.idx * 4 + 1]; + boxCoord2[0] = boxesPtr[candidateBox.idx * 4 + 2]; + boxCoord3[0] = boxesPtr[candidateBox.idx * 4 + 3]; - bool box_is_selected = true; - for (int idx = static_cast(fb.size()) - 1; idx >= currBox.suppress_begin_index; idx--) { - float iou = intersectionOverUnion(&boxesPtr[currBox.idx * 4], &boxesPtr[fb[idx].box_index * 4]); - currBox.score *= coeff(iou); - if (iou >= iou_threshold) { - box_is_selected = false; - break; + auto arg = jit_nms_args(); + arg.iou_threshold = static_cast(&iouThreshold); + arg.score_threshold = static_cast(&scoreThreshold); + arg.scale = static_cast(&scale); + while (selectedBoxes.size() < maxOutputBoxesPerClass && !sorted_boxes.empty()) { + boxInfo candidateBox = sorted_boxes.top(); + float origScore = candidateBox.score; + sorted_boxes.pop(); + + int candidateStatus = NMSCandidateStatus::SELECTED; // 0 for suppressed, 1 for selected, 2 for updated + arg.score = static_cast(&candidateBox.score); + arg.selected_boxes_num = selectedBoxes.size() - candidateBox.suppress_begin_index; + arg.selected_boxes_coord[0] = static_cast(&boxCoord0[candidateBox.suppress_begin_index]); + arg.selected_boxes_coord[1] = static_cast(&boxCoord1[candidateBox.suppress_begin_index]); + arg.selected_boxes_coord[2] = static_cast(&boxCoord2[candidateBox.suppress_begin_index]); + arg.selected_boxes_coord[3] = static_cast(&boxCoord3[candidateBox.suppress_begin_index]); + arg.candidate_box = static_cast(&boxesPtr[candidateBox.idx * 4]); + arg.candidate_status = static_cast(&candidateStatus); + (*nms_kernel)(&arg); + + if (candidateStatus == NMSCandidateStatus::SUPPRESSED) { + continue; + } else { + if (candidateBox.score == origScore) { + selectedBoxes.push_back({ candidateBox.score, batch_idx, class_idx, candidateBox.idx }); + int selectedSize = selectedBoxes.size(); + boxCoord0[selectedSize - 1] = boxesPtr[candidateBox.idx * 4]; + boxCoord1[selectedSize - 1] = boxesPtr[candidateBox.idx * 4 + 1]; + boxCoord2[selectedSize - 1] = boxesPtr[candidateBox.idx * 4 + 2]; + boxCoord3[selectedSize - 1] = boxesPtr[candidateBox.idx * 4 + 3]; + } else { + candidateBox.suppress_begin_index = selectedBoxes.size(); + sorted_boxes.push(candidateBox); + } + } } - if (currBox.score <= score_threshold) - break; - } + } else { + while (selectedBoxes.size() < maxOutputBoxesPerClass && !sorted_boxes.empty()) { + boxInfo candidateBox = sorted_boxes.top(); + float origScore = candidateBox.score; + sorted_boxes.pop(); - currBox.suppress_begin_index = fb.size(); - if (box_is_selected) { - if (currBox.score == origScore) { - fb.push_back({ currBox.score, batch_idx, class_idx, currBox.idx }); - continue; - } - if (currBox.score > score_threshold) { - sorted_boxes.push(currBox); + int candidateStatus = NMSCandidateStatus::SELECTED; // 0 for suppressed, 1 for selected, 2 for updated + for (int selected_idx = static_cast(selectedBoxes.size()) - 1; selected_idx >= candidateBox.suppress_begin_index; selected_idx--) { + float iou = intersectionOverUnion(&boxesPtr[candidateBox.idx * 4], &boxesPtr[selectedBoxes[selected_idx].box_index * 4]); + + // when is_soft_suppressed_by_iou is true, score is decayed to zero and implicitely suppressed if iou > iou_threshold. + candidateBox.score *= coeff(iou); + // soft suppressed + if (candidateBox.score <= scoreThreshold) { + candidateStatus = NMSCandidateStatus::SUPPRESSED; + break; + } + } + + if (candidateStatus == NMSCandidateStatus::SUPPRESSED) { + continue; + } else { + if (candidateBox.score == origScore) { + selectedBoxes.push_back({ candidateBox.score, batch_idx, class_idx, candidateBox.idx }); + } else { + candidateBox.suppress_begin_index = selectedBoxes.size(); + sorted_boxes.push(candidateBox); + } + } } } } } - numFiltBox[batch_idx][class_idx] = fb.size(); - size_t offset = batch_idx*num_classes*max_output_boxes_per_class + class_idx*max_output_boxes_per_class; - for (size_t i = 0; i < fb.size(); i++) { - filtBoxes[offset + i] = fb[i]; + numFiltBox[batch_idx][class_idx] = selectedBoxes.size(); + size_t offset = batch_idx*numClasses*maxOutputBoxesPerClass + class_idx*maxOutputBoxesPerClass; + for (size_t i = 0; i < selectedBoxes.size(); i++) { + filtBoxes[offset + i] = selectedBoxes[i]; } }); } void MKLDNNNonMaxSuppressionNode::nmsWithoutSoftSigma(const float *boxes, const float *scores, const VectorDims &boxesStrides, const VectorDims &scoresStrides, std::vector &filtBoxes) { - int max_out_box = static_cast(max_output_boxes_per_class); - parallel_for2d(num_batches, num_classes, [&](int batch_idx, int class_idx) { + int max_out_box = static_cast(maxOutputBoxesPerClass); + parallel_for2d(numBatches, numClasses, [&](int batch_idx, int class_idx) { const float *boxesPtr = boxes + batch_idx * boxesStrides[0]; const float *scoresPtr = scores + batch_idx * scoresStrides[0] + class_idx * scoresStrides[1]; - std::vector> sorted_boxes; - for (int box_idx = 0; box_idx < num_boxes; box_idx++) { - if (scoresPtr[box_idx] > score_threshold) + std::vector> sorted_boxes; // score, box_idx + for (int box_idx = 0; box_idx < numBoxes; box_idx++) { + if (scoresPtr[box_idx] > scoreThreshold) sorted_boxes.emplace_back(std::make_pair(scoresPtr[box_idx], box_idx)); } int io_selection_size = 0; - if (sorted_boxes.size() > 0) { + size_t sortedBoxSize = sorted_boxes.size(); + if (sortedBoxSize > 0) { parallel_sort(sorted_boxes.begin(), sorted_boxes.end(), [](const std::pair& l, const std::pair& r) { return (l.first > r.first || ((l.first == r.first) && (l.second < r.second))); }); - int offset = batch_idx*num_classes*max_output_boxes_per_class + class_idx*max_output_boxes_per_class; + int offset = batch_idx*numClasses*maxOutputBoxesPerClass + class_idx*maxOutputBoxesPerClass; filtBoxes[offset + 0] = filteredBoxes(sorted_boxes[0].first, batch_idx, class_idx, sorted_boxes[0].second); io_selection_size++; - for (size_t box_idx = 1; (box_idx < sorted_boxes.size()) && (io_selection_size < max_out_box); box_idx++) { - bool box_is_selected = true; - for (int idx = io_selection_size - 1; idx >= 0; idx--) { - float iou = intersectionOverUnion(&boxesPtr[sorted_boxes[box_idx].second * 4], &boxesPtr[filtBoxes[offset + idx].box_index * 4]); - if (iou >= iou_threshold) { - box_is_selected = false; - break; - } - } + if (sortedBoxSize > 1) { + if (nms_kernel) { + std::vector boxCoord0(sortedBoxSize, 0.0f); + std::vector boxCoord1(sortedBoxSize, 0.0f); + std::vector boxCoord2(sortedBoxSize, 0.0f); + std::vector boxCoord3(sortedBoxSize, 0.0f); - if (box_is_selected) { - filtBoxes[offset + io_selection_size] = filteredBoxes(sorted_boxes[box_idx].first, batch_idx, class_idx, sorted_boxes[box_idx].second); - io_selection_size++; + boxCoord0[0] = boxesPtr[sorted_boxes[0].second * 4]; + boxCoord1[0] = boxesPtr[sorted_boxes[0].second * 4 + 1]; + boxCoord2[0] = boxesPtr[sorted_boxes[0].second * 4 + 2]; + boxCoord3[0] = boxesPtr[sorted_boxes[0].second * 4 + 3]; + + auto arg = jit_nms_args(); + arg.iou_threshold = static_cast(&iouThreshold); + arg.score_threshold = static_cast(&scoreThreshold); + arg.scale = static_cast(&scale); + // box start index do not change for hard supresion + arg.selected_boxes_coord[0] = static_cast(&boxCoord0[0]); + arg.selected_boxes_coord[1] = static_cast(&boxCoord1[0]); + arg.selected_boxes_coord[2] = static_cast(&boxCoord2[0]); + arg.selected_boxes_coord[3] = static_cast(&boxCoord3[0]); + + for (size_t candidate_idx = 1; (candidate_idx < sortedBoxSize) && (io_selection_size < max_out_box); candidate_idx++) { + int candidateStatus = NMSCandidateStatus::SELECTED; // 0 for suppressed, 1 for selected + arg.selected_boxes_num = io_selection_size; + arg.candidate_box = static_cast(&boxesPtr[sorted_boxes[candidate_idx].second * 4]); + arg.candidate_status = static_cast(&candidateStatus); + (*nms_kernel)(&arg); + if (candidateStatus == NMSCandidateStatus::SELECTED) { + boxCoord0[io_selection_size] = boxesPtr[sorted_boxes[candidate_idx].second * 4]; + boxCoord1[io_selection_size] = boxesPtr[sorted_boxes[candidate_idx].second * 4 + 1]; + boxCoord2[io_selection_size] = boxesPtr[sorted_boxes[candidate_idx].second * 4 + 2]; + boxCoord3[io_selection_size] = boxesPtr[sorted_boxes[candidate_idx].second * 4 + 3]; + filtBoxes[offset + io_selection_size] = + filteredBoxes(sorted_boxes[candidate_idx].first, batch_idx, class_idx, sorted_boxes[candidate_idx].second); + io_selection_size++; + } + } + } else { + for (size_t candidate_idx = 1; (candidate_idx < sortedBoxSize) && (io_selection_size < max_out_box); candidate_idx++) { + int candidateStatus = NMSCandidateStatus::SELECTED; // 0 for suppressed, 1 for selected + for (int selected_idx = io_selection_size - 1; selected_idx >= 0; selected_idx--) { + float iou = intersectionOverUnion(&boxesPtr[sorted_boxes[candidate_idx].second * 4], + &boxesPtr[filtBoxes[offset + selected_idx].box_index * 4]); + if (iou >= iouThreshold) { + candidateStatus = NMSCandidateStatus::SUPPRESSED; + break; + } + } + + if (candidateStatus == NMSCandidateStatus::SELECTED) { + filtBoxes[offset + io_selection_size] = + filteredBoxes(sorted_boxes[candidate_idx].first, batch_idx, class_idx, sorted_boxes[candidate_idx].second); + io_selection_size++; + } + } } } } + numFiltBox[batch_idx][class_idx] = io_selection_size; }); } diff --git a/src/plugins/intel_cpu/src/nodes/mkldnn_non_max_suppression_node.h b/src/plugins/intel_cpu/src/nodes/mkldnn_non_max_suppression_node.h index e30ffcdc65d..6500810033f 100644 --- a/src/plugins/intel_cpu/src/nodes/mkldnn_non_max_suppression_node.h +++ b/src/plugins/intel_cpu/src/nodes/mkldnn_non_max_suppression_node.h @@ -10,10 +10,56 @@ #include #include +#define BOX_COORD_NUM 4 + using namespace InferenceEngine; namespace MKLDNNPlugin { +enum class NMSBoxEncodeType { + CORNER, + CENTER +}; + +enum NMSCandidateStatus { + SUPPRESSED = 0, + SELECTED = 1, + UPDATED = 2 +}; + +struct jit_nms_config_params { + NMSBoxEncodeType box_encode_type; + bool is_soft_suppressed_by_iou; +}; + +struct jit_nms_args { + const void* selected_boxes_coord[BOX_COORD_NUM]; + size_t selected_boxes_num; + const void* candidate_box; + const void* iou_threshold; + void* candidate_status; + // for soft suppression, score *= scale * iou * iou; + const void* score_threshold; + const void* scale; + void* score; +}; + +struct jit_uni_nms_kernel { + void (*ker_)(const jit_nms_args *); + + void operator()(const jit_nms_args *args) { + assert(ker_); + ker_(args); + } + + explicit jit_uni_nms_kernel(jit_nms_config_params jcp_) : ker_(nullptr), jcp(jcp_) {} + virtual ~jit_uni_nms_kernel() {} + + virtual void create_ker() = 0; + + jit_nms_config_params jcp; +}; + class MKLDNNNonMaxSuppressionNode : public MKLDNNNode { public: MKLDNNNonMaxSuppressionNode(const std::shared_ptr& op, const mkldnn::engine& eng, MKLDNNWeightsSharing::Ptr &cache); @@ -73,23 +119,20 @@ private: NMS_VALIDOUTPUTS }; + NMSBoxEncodeType boxEncodingType = NMSBoxEncodeType::CORNER; + bool sortResultDescending = true; - enum class boxEncoding { - CORNER, - CENTER - }; - boxEncoding boxEncodingType = boxEncoding::CORNER; - bool sort_result_descending = true; + size_t numBatches = 0; + size_t numBoxes = 0; + size_t numClasses = 0; - size_t num_batches = 0; - size_t num_boxes = 0; - size_t num_classes = 0; - - size_t max_output_boxes_per_class = 0lu; - float iou_threshold = 0.0f; - float score_threshold = 0.0f; - float soft_nms_sigma = 0.0f; + size_t maxOutputBoxesPerClass = 0lu; + float iouThreshold = 0.0f; + float scoreThreshold = 0.0f; + float softNMSSigma = 0.0f; float scale = 1.f; + // control placeholder for NMS in new opset. + bool isSoftSuppressedByIOU = true; std::string errorPrefix; @@ -99,6 +142,9 @@ private: void checkPrecision(const Precision& prec, const std::vector& precList, const std::string& name, const std::string& type); void check1DInput(const Shape& shape, const std::vector& precList, const std::string& name, const size_t port); void checkOutput(const Shape& shape, const std::vector& precList, const std::string& name, const size_t port); + + void createJitKernel(); + std::shared_ptr nms_kernel; }; } // namespace MKLDNNPlugin From fa1b59b7be2a869bda84da68e43ac3a13bfd41c1 Mon Sep 17 00:00:00 2001 From: Eugeny Volosenkov Date: Mon, 27 Dec 2021 15:57:02 +0300 Subject: [PATCH 3/6] Fix incorrect working UnpackPackReverseInputChannels for centernet (#9201) * fix UnpackPackReverseInputChannels * Add UnpackPackReverseInputChannels test --- .../tf/UnpackPackReverseInputChannels.py | 2 +- .../tf/UnpackPackReverseInputChannels_test.py | 42 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 tools/mo/unit_tests/mo/front/tf/UnpackPackReverseInputChannels_test.py diff --git a/tools/mo/openvino/tools/mo/front/tf/UnpackPackReverseInputChannels.py b/tools/mo/openvino/tools/mo/front/tf/UnpackPackReverseInputChannels.py index 915bc26e64b..264315b5dea 100644 --- a/tools/mo/openvino/tools/mo/front/tf/UnpackPackReverseInputChannels.py +++ b/tools/mo/openvino/tools/mo/front/tf/UnpackPackReverseInputChannels.py @@ -66,7 +66,7 @@ class UnpackPackReverseInputChannels(FrontReplacementSubgraph): reverse_channels = ReverseChannels(graph, { 'name': pack.soft_get('name', pack.id) + '/ReverseChannels', - 'axis': int64_array(axis), 'order': int64_array([2, 0, 1])}).create_node() + 'axis': int64_array(axis), 'order': int64_array([2, 1, 0])}).create_node() pack.out_port(0).get_connection().set_source(reverse_channels.out_port(0)) unpack.in_port(0).get_connection().set_destination(reverse_channels.in_port(0)) diff --git a/tools/mo/unit_tests/mo/front/tf/UnpackPackReverseInputChannels_test.py b/tools/mo/unit_tests/mo/front/tf/UnpackPackReverseInputChannels_test.py new file mode 100644 index 00000000000..04f1927e312 --- /dev/null +++ b/tools/mo/unit_tests/mo/front/tf/UnpackPackReverseInputChannels_test.py @@ -0,0 +1,42 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import unittest + +from openvino.tools.mo.front.common.partial_infer.utils import int64_array +from openvino.tools.mo.front.tf.UnpackPackReverseInputChannels import UnpackPackReverseInputChannels +from openvino.tools.mo.utils.ir_engine.compare_graphs import compare_graphs + +from unit_tests.utils.graph import build_graph, regular_op_with_empty_data, result, connect_front + +nodes = { + **regular_op_with_empty_data('input', {'type': 'Parameter'}), + **regular_op_with_empty_data('unpack', {'op': 'AttributedSplit', 'axis': int64_array(0)}), + **regular_op_with_empty_data('pack', {'op': 'Pack', 'axis': int64_array(0)}), + **result(), + + **regular_op_with_empty_data('reverseChannels', + {'op': 'ReverseChannels', 'order': int64_array([2, 1, 0]), 'axis': int64_array(0), 'type': None}), +} + + +class UnpackPackReverseInputChannelsTest(unittest.TestCase): + def test_replace_to_reverse_channel(self): + graph = build_graph(nodes_attrs=nodes, edges=[ + *connect_front('input:0', '0:unpack'), + *connect_front('unpack:0', '2:pack'), + *connect_front('unpack:1', '1:pack'), + *connect_front('unpack:2', '0:pack'), + *connect_front('pack:0', '0:output'), + ], nodes_with_edges_only=True) + graph.stage = 'front' + + UnpackPackReverseInputChannels().find_and_replace_pattern(graph) + + graph_ref = build_graph(nodes_attrs=nodes, edges=[ + *connect_front('input:0', '0:reverseChannels'), + *connect_front('reverseChannels:0', '0:output'), + ], nodes_with_edges_only=True) + + (flag, resp) = compare_graphs(graph, graph_ref, 'output', check_op_attrs=True) + self.assertTrue(flag, resp) From c5824b84941f16d22dde5f1443ab7e7a0108dd75 Mon Sep 17 00:00:00 2001 From: Vladimir Zinoviev Date: Mon, 27 Dec 2021 17:07:20 +0300 Subject: [PATCH 4/6] [LPT] Assign + ReadValue transformation (#8690) * [LPT] Assign + ReadValue transformation * [LPT] Assign+ReadValue: applied review comments * fixed some misses --- .../backend/int_executable.cpp | 19 +- .../template_plugin/backend/opset_int_tbl.hpp | 2 + .../low_precision/assign_and_read_value.hpp | 27 +++ .../common/fake_quantize_dequantization.hpp | 1 + .../include/low_precision/fake_quantize.hpp | 5 +- .../include/low_precision/network_helper.hpp | 94 -------- .../src/assign_and_read_value.cpp | 132 ++++++++++++ .../src/fake_quantize.cpp | 22 +- .../src/fake_quantize_dequantization.cpp | 20 ++ .../src/low_precision.cpp | 2 + .../src/network_helper.cpp | 29 +-- .../assign_and_read_value_transformation.cpp | 200 ++++++++++++++++++ .../simple_low_precision_transformer.hpp | 4 + .../assign_and_read_value_transformation.cpp | 55 +++++ .../assign_and_read_value_transformation.cpp | 55 +++++ .../assign_and_read_value_transformation.hpp | 35 +++ .../assign_and_read_value_transformation.cpp | 49 +++++ .../assign_and_read_value_function.hpp | 46 ++++ .../src/assign_and_read_value_function.cpp | 185 ++++++++++++++++ 19 files changed, 847 insertions(+), 135 deletions(-) create mode 100644 src/common/low_precision_transformations/include/low_precision/assign_and_read_value.hpp create mode 100644 src/common/low_precision_transformations/src/assign_and_read_value.cpp create mode 100644 src/tests/functional/inference_engine/lp_transformations/assign_and_read_value_transformation.cpp create mode 100644 src/tests/functional/plugin/cpu/shared_tests_instances/low_precision_transformations/assign_and_read_value_transformation.cpp create mode 100644 src/tests/functional/plugin/gpu/shared_tests_instances/low_precision_transformations/assign_and_read_value_transformation.cpp create mode 100644 src/tests/functional/plugin/shared/include/low_precision_transformations/assign_and_read_value_transformation.hpp create mode 100644 src/tests/functional/plugin/shared/src/low_precision_transformations/assign_and_read_value_transformation.cpp create mode 100644 src/tests/ngraph_helpers/lpt_ngraph_functions/include/lpt_ngraph_functions/assign_and_read_value_function.hpp create mode 100644 src/tests/ngraph_helpers/lpt_ngraph_functions/src/assign_and_read_value_function.cpp diff --git a/docs/template_plugin/backend/int_executable.cpp b/docs/template_plugin/backend/int_executable.cpp index 4c119438e3c..0628a8371df 100644 --- a/docs/template_plugin/backend/int_executable.cpp +++ b/docs/template_plugin/backend/int_executable.cpp @@ -5,6 +5,7 @@ #include "int_executable.hpp" #include +#include #include "evaluates_map.hpp" #include "ngraph/except.hpp" @@ -87,6 +88,10 @@ bool runtime::interpreter::INTExecutable::call(const vector(op) != nullptr) { @@ -143,8 +148,20 @@ bool runtime::interpreter::INTExecutable::call(const vector(cloned_node)) { + auto variable = var_extension->get_variable(); + if (!variable_context.get_variable_value(variable)) { + auto h_tensor = std::make_shared(cloned_node->get_input_element_type(0), + cloned_node->get_input_shape(0)); + std::vector data(ov::shape_size(cloned_node->get_input_shape(0)), 0); + h_tensor->write(data.data(), data.size() * sizeof(float)); + variable_context.set_variable_value(variable, std::make_shared(h_tensor)); + } + } + // Call evaluate for cloned_node with static shapes - if (!cloned_node->evaluate(op_outputs, op_inputs)) { + if (!cloned_node->evaluate(op_outputs, op_inputs, eval_context)) { evaluate_node(cloned_node, op_outputs, op_inputs); } if (m_performance_counters_enabled) { diff --git a/docs/template_plugin/backend/opset_int_tbl.hpp b/docs/template_plugin/backend/opset_int_tbl.hpp index 287bf9a0d11..4d4383da233 100644 --- a/docs/template_plugin/backend/opset_int_tbl.hpp +++ b/docs/template_plugin/backend/opset_int_tbl.hpp @@ -85,6 +85,7 @@ NGRAPH_OP(NonMaxSuppression, op::v5) NGRAPH_OP(RNNSequence, op::v5) NGRAPH_OP(Round, op::v5) +NGRAPH_OP(Assign, ngraph::op::v6) NGRAPH_OP(CTCGreedyDecoderSeqLen, op::v6) NGRAPH_OP(ExperimentalDetectronDetectionOutput, op::v6) NGRAPH_OP(ExperimentalDetectronGenerateProposalsSingleImage, op::v6) @@ -93,6 +94,7 @@ NGRAPH_OP(ExperimentalDetectronROIFeatureExtractor, op::v6) NGRAPH_OP(ExperimentalDetectronTopKROIs, op::v6) NGRAPH_OP(GatherElements, op::v6) NGRAPH_OP(MVN, ngraph::op::v6) +NGRAPH_OP(ReadValue, ngraph::op::v6) NGRAPH_OP(DFT, op::v7) NGRAPH_OP(Einsum, op::v7) diff --git a/src/common/low_precision_transformations/include/low_precision/assign_and_read_value.hpp b/src/common/low_precision_transformations/include/low_precision/assign_and_read_value.hpp new file mode 100644 index 00000000000..b7ea91d099e --- /dev/null +++ b/src/common/low_precision_transformations/include/low_precision/assign_and_read_value.hpp @@ -0,0 +1,27 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include "layer_transformation.hpp" + +namespace ngraph { +namespace pass { +namespace low_precision { + +class LP_TRANSFORMATIONS_API AssignAndReadValueTransformation : public LayerTransformation { +public: + NGRAPH_RTTI_DECLARATION; + AssignAndReadValueTransformation(const std::shared_ptr function, const Params& params = Params()); + bool transform(TransformationContext& context, ngraph::pattern::Matcher& m) override; + bool canBeTransformed(const TransformationContext& context, std::shared_ptr op) const override; + bool isPrecisionPreserved(std::shared_ptr layer) const noexcept override; +private: + std::shared_ptr function; +}; + +} // namespace low_precision +} // namespace pass +} // namespace ngraph \ No newline at end of file diff --git a/src/common/low_precision_transformations/include/low_precision/common/fake_quantize_dequantization.hpp b/src/common/low_precision_transformations/include/low_precision/common/fake_quantize_dequantization.hpp index 0da82810c97..4bc50744c5a 100644 --- a/src/common/low_precision_transformations/include/low_precision/common/fake_quantize_dequantization.hpp +++ b/src/common/low_precision_transformations/include/low_precision/common/fake_quantize_dequantization.hpp @@ -33,6 +33,7 @@ public: bool multiplyHasZeroOrDenormal() const; bool isShared() const; bool isLowPrecision() const; + std::shared_ptr copyWithNewInput(const std::shared_ptr& input) const; static bool checkElementwise(const std::shared_ptr& elementwise); diff --git a/src/common/low_precision_transformations/include/low_precision/fake_quantize.hpp b/src/common/low_precision_transformations/include/low_precision/fake_quantize.hpp index 15975782ef0..6a3f84b6b4c 100644 --- a/src/common/low_precision_transformations/include/low_precision/fake_quantize.hpp +++ b/src/common/low_precision_transformations/include/low_precision/fake_quantize.hpp @@ -22,11 +22,10 @@ public: static bool checkElementwise(const std::shared_ptr& eltwise); -private: - std::shared_ptr fuseElementwise( + static std::shared_ptr fuseElementwise( TransformationContext& context, MatcherPass* matcherPass, - const std::shared_ptr& fakeQuantize) const; + const std::shared_ptr& fakeQuantize); }; } // namespace low_precision diff --git a/src/common/low_precision_transformations/include/low_precision/network_helper.hpp b/src/common/low_precision_transformations/include/low_precision/network_helper.hpp index 5f7d6a74113..569cd79d506 100644 --- a/src/common/low_precision_transformations/include/low_precision/network_helper.hpp +++ b/src/common/low_precision_transformations/include/low_precision/network_helper.hpp @@ -41,17 +41,12 @@ public: static std::vector> consumer_inputs(std::shared_ptr node); - // returns true if at least one child is not FQ - static bool notAllChildrensAreFQ(const NodeVector& layer); - // Collect and return a vector with all nodes that consumes any of the `node` output static std::vector> consumers(std::shared_ptr node); // return true if op is on a constant path static bool isConstantPath(const std::shared_ptr& op); - static Shape alignShapeForChannelDim(const Shape& shape, Rank rank); - template static std::shared_ptr setOutDataPrecisionForTypeRelaxed(std::shared_ptr operation, const element::Type& precision); @@ -215,87 +210,6 @@ public: const std::shared_ptr& dequantization, const std::shared_ptr& newNode); - static void replaceAttributeInNodes( - std::shared_ptr f, - const std::string& name, - const ov::Any& newAttribute, - const ov::Any& oldAttribute, - const std::shared_ptr& initialNode) { - std::set> visited; - std::deque> nodes; - nodes.emplace_back(initialNode); - - while (!nodes.empty()) { - auto node = nodes.front(); - nodes.pop_front(); - - if (visited.count(node) || ov::is_type(node)) { - continue; - } - - visited.insert(node); - - bool handleConnectedNodes = false; - if (NetworkHelper::isPrecisionPreserved(node) || ov::is_type(node)) { - auto& rt = node->get_rt_info(); - - if (node == initialNode) { - rt[name] = newAttribute; - handleConnectedNodes = true; - } else { - auto it = rt.find(name); - if (it != rt.end()) { - const auto currentAttribute = it->second; - if (oldAttribute == currentAttribute) { - rt[name] = newAttribute; - } - handleConnectedNodes = true; - } - } - } - - if (!handleConnectedNodes) { - continue; - } - - if (!ov::is_type(node)) { - for (size_t index = 0ul; index < node->get_input_size(); ++index) { - auto getInput = [](const std::shared_ptr& node, const size_t index) { - const auto dequantization = NetworkHelper::getDequantization(node, index); - if (!dequantization.empty() && - (ov::is_type(dequantization.data.get_node())) && - ov::is_type(dequantization.data.get_node()->get_input_node_ptr(0))) { - const auto input = dequantization.data.get_node()->input(0); - return input; - } - return node->input(index); - }; - - const auto& input = getInput(node, index); - const auto& input_node = input.get_source_output().get_node_shared_ptr(); - - //const auto& input_node = input.get_source_output().get_node_shared_ptr(); - if (visited.count(input_node) || ov::is_type(input_node)) { - continue; - } - - nodes.push_front(input_node); - } - } - - for (auto& output : node->outputs()) { - for (auto& input_value : output.get_target_inputs()) { - const auto& output_node = input_value.get_node()->shared_from_this(); - if (visited.count(output_node) || ov::is_type(output_node)) { - continue; - } - - nodes.push_front(output_node); - } - } - } - } - template static void reassign( const std::shared_ptr& sharedValue, @@ -370,14 +284,6 @@ std::shared_ptr make_op_pattern(const ngraph::NodeVector& args) { return std::make_shared(element::undefined, PartialShape{}, [](std::shared_ptr n) {return !!ov::as_type_ptr(n); }, args); } -template -std::shared_ptr make_op_label() { - return std::make_shared( - element::undefined, - PartialShape{}, - [](std::shared_ptr n) {return !!ov::as_type_ptr(n); }); -} - template std::shared_ptr fold(Args&&... args) { auto node = std::make_shared(std::forward(args)...); diff --git a/src/common/low_precision_transformations/src/assign_and_read_value.cpp b/src/common/low_precision_transformations/src/assign_and_read_value.cpp new file mode 100644 index 00000000000..472b8df8eb2 --- /dev/null +++ b/src/common/low_precision_transformations/src/assign_and_read_value.cpp @@ -0,0 +1,132 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "low_precision/assign_and_read_value.hpp" +#include + +#include +#include "low_precision/network_helper.hpp" +#include +#include +#include +#include "low_precision/fake_quantize.hpp" + +namespace ngraph { +namespace pass { +namespace low_precision { + +NGRAPH_RTTI_DEFINITION(ngraph::pass::low_precision::AssignAndReadValueTransformation, "AssignAndReadValueTransformation", 0); + +AssignAndReadValueTransformation::AssignAndReadValueTransformation(const std::shared_ptr function, const Params& params) : + LayerTransformation(params), function(function) { + auto assign3 = pattern::wrap_type({ pattern::wrap_type() }); + auto assign6 = pattern::wrap_type({ pattern::wrap_type() }); + + ngraph::graph_rewrite_callback callback = [=](pattern::Matcher& m) { + const auto& opsMap = m.get_pattern_value_map(); + auto op = m.get_match_root(); + auto assignIt = opsMap.find(assign3); + if (assignIt == opsMap.end()) { + assignIt = opsMap.find(assign6); + } + const auto assign = assignIt->second.get_node_shared_ptr(); + // check that we have ReadValue as the first dependency + if (assign->get_control_dependencies().empty()) { + return false; + } + + if (transformation_callback(op)) { + return false; + } + return transform(*context, m); + }; + + auto m = std::make_shared( + std::make_shared(OutputVector{ assign3, assign6 }), + "AssignAndReadValueTransformation"); + this->register_matcher(m, callback); +} + +bool AssignAndReadValueTransformation::transform(TransformationContext& context, ngraph::pattern::Matcher& m) { + if (!canBeTransformed(context, m.get_match_root())) { + return false; + } + + const auto oldAssign = m.get_match_root(); + const auto readValue = oldAssign->get_control_dependencies()[0]; + oldAssign->remove_control_dependency(readValue); + + const auto assign = NetworkHelper::separateInStandaloneBranch(oldAssign); + const auto dequantization = NetworkHelper::getDequantization(assign); + + auto oldVar = ov::as_type_ptr(readValue)->get_variable(); + auto variableInfo = oldVar->get_info(); + // set new precision for oldVar to update precision in newReadValue + oldVar->update({variableInfo.data_shape, dequantization.data.get_element_type(), variableInfo.variable_id}); + // transform ReadValue part + const auto newConstant = foldConvert(readValue->get_input_node_shared_ptr(0), dequantization.data.get_element_type()); + const auto newReadValue = readValue->copy_with_new_inputs({newConstant}); + const auto newDequantization = dequantization.copyWithNewInput(newReadValue); + replace_node(readValue, newDequantization); + + // transform Assign part + + const auto newAssign = assign->copy_with_new_inputs({dequantization.data}); + function->remove_sink(as_type_ptr(oldAssign)); + function->add_sinks({as_type_ptr(newAssign)}); + + NetworkHelper::copyInfo(assign, newAssign); + replace_node(assign, newAssign); + newAssign->add_control_dependency(newReadValue); + + // fuse dequantization multiply with FQ after ReadValue if possible + const auto nextLayers = newDequantization->get_output_target_inputs(0); + if (nextLayers.size() > 1) { + return true; + } + const auto fakeQuantize = as_type_ptr(nextLayers.begin()->get_node()->shared_from_this()); + + if (fakeQuantize == nullptr) { + return true; + } + auto fakeQuantizeInputs = fakeQuantize->input_values(); + + const auto inputLow = as_type_ptr(fakeQuantizeInputs[1].get_node_shared_ptr()); + const auto inputHigh = as_type_ptr(fakeQuantizeInputs[2].get_node_shared_ptr()); + + if (inputLow == nullptr || inputHigh == nullptr) { + return true; + } + + FakeQuantizeTransformation::fuseElementwise(context, this, fakeQuantize); + + return true; +} + +bool AssignAndReadValueTransformation::canBeTransformed(const TransformationContext& context, std::shared_ptr op) const { + if (!LayerTransformation::canBeTransformed(context, op)) { + return false; + } + + const auto readValue = std::dynamic_pointer_cast(op->get_control_dependencies()[0]); + if (!readValue) { + return false; + } + + // TODO: remove this limitation and change the transformation when this constant will be accepted to be non-zero + if (!NetworkHelper::isZeroConst(readValue->get_input_node_shared_ptr(0))) { + return false; + } + + const auto dequantization = NetworkHelper::getDequantization(op); + return dequantization.subtract == nullptr && dequantization.multiply != nullptr; +} + +bool AssignAndReadValueTransformation::isPrecisionPreserved(std::shared_ptr layer) const noexcept { + return false; +} + +} // namespace low_precision +} // namespace pass +} // namespace ngraph diff --git a/src/common/low_precision_transformations/src/fake_quantize.cpp b/src/common/low_precision_transformations/src/fake_quantize.cpp index deb7ec1ea28..130ab6f5103 100644 --- a/src/common/low_precision_transformations/src/fake_quantize.cpp +++ b/src/common/low_precision_transformations/src/fake_quantize.cpp @@ -129,17 +129,15 @@ bool FakeQuantizeTransformation::checkElementwise(const std::shared_ptr& e std::shared_ptr FakeQuantizeTransformation::fuseElementwise( TransformationContext& context, MatcherPass* matcherPass, - const std::shared_ptr& fakeQuantize) const { + const std::shared_ptr& fakeQuantize) { const std::shared_ptr eltwise = fakeQuantize->get_input_node_shared_ptr(0); - std::shared_ptr inputLowConst_f32 = foldConvert(fakeQuantize->input_value(1), deqPrecision); - std::shared_ptr inputHighConst_f32 = foldConvert(fakeQuantize->input_value(2), deqPrecision); + std::shared_ptr inputLowConst_f32 = foldConvert(fakeQuantize->input_value(1), element::f32); + std::shared_ptr inputHighConst_f32 = foldConvert(fakeQuantize->input_value(2), element::f32); std::shared_ptr constant = fq::getConstant(eltwise); if (ov::is_type(eltwise) && checkElementwise(eltwise)) { - const auto value = constant->get_output_element_type(0) == deqPrecision ? - constant : - foldConvert(constant, deqPrecision); + const auto value = foldConvert(constant, element::f32); const auto valueVec = ov::as_type_ptr(value)->cast_vector(); @@ -159,9 +157,7 @@ std::shared_ptr FakeQuantizeTransformation::fuseElementwis inputLowConst_f32 = fq::updateShape(inputLowConst_f32, fakeQuantize->get_output_partial_shape(0)); inputHighConst_f32 = fq::updateShape(inputHighConst_f32, fakeQuantize->get_output_partial_shape(0)); } else if (ov::is_type(eltwise) && checkElementwise(eltwise)) { - const auto value = constant->get_output_element_type(0) == deqPrecision ? - constant : - foldConvert(constant, deqPrecision); + const auto value = foldConvert(constant, element::f32); inputLowConst_f32 = fq::updateShape(fold(inputLowConst_f32, value), fakeQuantize->get_output_partial_shape(0)); inputHighConst_f32 = fq::updateShape(fold(inputHighConst_f32, value), fakeQuantize->get_output_partial_shape(0)); @@ -173,9 +169,7 @@ std::shared_ptr FakeQuantizeTransformation::fuseElementwis return nullptr; } - const auto value = constant->get_output_element_type(0) == deqPrecision ? - constant : - foldConvert(constant, deqPrecision); + const auto value = foldConvert(constant, element::f32); inputLowConst_f32 = fq::updateShape(fold(inputLowConst_f32, value), fakeQuantize->get_output_partial_shape(0)); inputHighConst_f32 = fq::updateShape(fold(inputHighConst_f32, value), fakeQuantize->get_output_partial_shape(0)); @@ -196,8 +190,8 @@ std::shared_ptr FakeQuantizeTransformation::fuseElementwis data->output(outputIdx), inputLowConst_f32, inputHighConst_f32, - foldConvert(fakeQuantize->input_value(3), deqPrecision), - foldConvert(fakeQuantize->input_value(4), deqPrecision) })); + foldConvert(fakeQuantize->input_value(3), element::f32), + foldConvert(fakeQuantize->input_value(4), element::f32) })); matcherPass->register_new_node(newFakeQuantize); diff --git a/src/common/low_precision_transformations/src/fake_quantize_dequantization.cpp b/src/common/low_precision_transformations/src/fake_quantize_dequantization.cpp index 46b56716081..fc6ae961a24 100644 --- a/src/common/low_precision_transformations/src/fake_quantize_dequantization.cpp +++ b/src/common/low_precision_transformations/src/fake_quantize_dequantization.cpp @@ -155,6 +155,26 @@ bool FakeQuantizeDequantization::checkElementwise(const std::shared_ptr FakeQuantizeDequantization::copyWithNewInput(const std::shared_ptr& input) const { + auto lastNode = input; + if (convert) { + lastNode = convert->copy_with_new_inputs({lastNode}); + } + if (subtract) { + std::shared_ptr input1 = nullptr; + if (subtractConvert) { + input1 = subtractConvert; + } else { + input1 = subtractConstant; + } + lastNode = subtract->copy_with_new_inputs({lastNode, input1}); + } + if (multiply) { + lastNode = multiply->copy_with_new_inputs({lastNode, multiplyConstant}); + } + return lastNode; +} + int FakeQuantizeDequantization::fillDequantizationParams( const std::shared_ptr& elementwise, std::shared_ptr& convert, diff --git a/src/common/low_precision_transformations/src/low_precision.cpp b/src/common/low_precision_transformations/src/low_precision.cpp index 715bd2ea0b5..5d21e3d4bd3 100644 --- a/src/common/low_precision_transformations/src/low_precision.cpp +++ b/src/common/low_precision_transformations/src/low_precision.cpp @@ -38,6 +38,7 @@ // general transformations #include "low_precision/add.hpp" +#include "low_precision/assign_and_read_value.hpp" #include "low_precision/avg_pool.hpp" #include "low_precision/clamp.hpp" #include "low_precision/convolution.hpp" @@ -207,6 +208,7 @@ bool ngraph::pass::low_precision::LowPrecision::run_on_model(const std::shared_p std::shared_ptr common = manager.register_pass(); common->add_matcher(params); + common->add_matcher(f, params); common->add_matcher(params); common->add_matcher(params); common->add_matcher(params); diff --git a/src/common/low_precision_transformations/src/network_helper.cpp b/src/common/low_precision_transformations/src/network_helper.cpp index 171f841f4cc..d7b5a9890d0 100644 --- a/src/common/low_precision_transformations/src/network_helper.cpp +++ b/src/common/low_precision_transformations/src/network_helper.cpp @@ -37,17 +37,6 @@ bool NetworkHelper::is_castable_to_one_of(NodeTypeInfo type, const std::unordere return false; } -bool NetworkHelper::notAllChildrensAreFQ(const NodeVector& childrens) { - // NOTE: This check was added for models that don't have FQ after AvgPool - // They will have transparent precision as it was in old LPT. - for (const auto& child : childrens) { - if (!ov::is_type(child)) { - return true; - } - } - return false; -} - // Collect and return a vector with all nodes that consumes any of the `node` output std::vector> NetworkHelper::consumer_inputs(std::shared_ptr node) { std::vector> result; @@ -199,15 +188,6 @@ size_t NetworkHelper::getGroupsCount(std::shared_ptr layer) { } } -// Assumin tensor in NC... layout, append necessary number of 1s to shape to align it to a give rank -Shape NetworkHelper::alignShapeForChannelDim(const Shape& shape, Rank rank) { - assert(shape.size() == 1); - assert(rank.is_static()); - Shape result = shape; - result.resize(rank.get_length() - 1, 1); - return result; -} - void NetworkHelper::removeLayer(std::shared_ptr layer) { ngraph::replace_output_update_name(layer->output(0), layer->input_value(0)); } @@ -1359,9 +1339,12 @@ FakeQuantizeDequantization NetworkHelper::getDequantization(const std::shared_pt const std::shared_ptr convert = ov::as_type_ptr(dataNode.get_node_shared_ptr()); if (convert != nullptr) { - if ((convert->input(0).get_element_type() != element::i8) && (convert->input(0).get_element_type() != element::u8) && - (convert->input(0).get_element_type() != element::i4) && (convert->input(0).get_element_type() != element::u4) && - (convert->output(0).get_element_type() != element::f32)) { + auto defaultPrecisions = LayerTransformation::getDefaultPrecisions(); + auto el_type = convert->input(0).get_element_type(); + auto foundIt = std::find(defaultPrecisions.begin(), defaultPrecisions.end(), el_type); + if (foundIt == defaultPrecisions.end() && + el_type != element::i4 && el_type != element::u4 && + el_type != element::f32 && el_type != element::f16) { return FakeQuantizeDequantization(dataNode, nullptr, subtract, subtractConvert, subtractConstant, multiply, multiplyConstant); } dataNode = convert->get_input_source_output(0); diff --git a/src/tests/functional/inference_engine/lp_transformations/assign_and_read_value_transformation.cpp b/src/tests/functional/inference_engine/lp_transformations/assign_and_read_value_transformation.cpp new file mode 100644 index 00000000000..0a9b125dea4 --- /dev/null +++ b/src/tests/functional/inference_engine/lp_transformations/assign_and_read_value_transformation.cpp @@ -0,0 +1,200 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "layer_transformation.hpp" + +#include +#include +#include + +#include +#include + +#include "common_test_utils/ngraph_test_utils.hpp" +#include "lpt_ngraph_functions/common/dequantization_operations.hpp" +#include "lpt_ngraph_functions/assign_and_read_value_function.hpp" +#include "simple_low_precision_transformer.hpp" +#include "low_precision/layer_transformation.hpp" + + +namespace { +using namespace testing; +using namespace ngraph::pass; +using namespace ngraph; + +class AssignTransformationTestValues { +public: + class Actual { + public: + std::vector constantValue; + ngraph::builder::subgraph::DequantizationOperations dequantization; + }; + + class Expected { + public: + std::vector constantValue; + ngraph::builder::subgraph::DequantizationOperations dequantizationBefore; + ngraph::builder::subgraph::DequantizationOperations dequantizationAfter; + }; + + TestTransformationParams params; + Actual actual; + Expected expected; + bool FQAfterReadValue; +}; + +typedef std::tuple < + ngraph::PartialShape, // input shape + element::Type, // input precision + element::Type, // precision before dequantization + size_t, // opset version + AssignTransformationTestValues // test values +> AssignTransformationParams; + +class AssignTransformation : public LayerTransformation, public testing::WithParamInterface { +public: + void SetUp() override { + const ngraph::PartialShape inputShape = std::get<0>(GetParam()); + const element::Type precision = std::get<1>(GetParam()); + const element::Type precisionBeforeDequantization = std::get<2>(GetParam()); + const size_t opsetVersion = std::get<3>(GetParam()); + const AssignTransformationTestValues testValues = std::get<4>(GetParam()); + low_precision::LayerTransformation::setDefaultPrecisions({ + ngraph::element::u8, ngraph::element::i8, + ngraph::element::u16, ngraph::element::i16, + ngraph::element::u32, ngraph::element::i32 + }); + actualFunction = ngraph::builder::subgraph::AssignAndReadValueFunction::getOriginal( + inputShape, + precision, + precisionBeforeDequantization, + opsetVersion, + testValues.FQAfterReadValue, + testValues.actual.constantValue, + testValues.actual.dequantization); + + SimpleLowPrecisionTransformer transformer; + transformer.add(actualFunction, testValues.params); + transformer.transform(actualFunction); + + referenceFunction = ngraph::builder::subgraph::AssignAndReadValueFunction::getReference( + inputShape, + precision, + precisionBeforeDequantization, + opsetVersion, + testValues.FQAfterReadValue, + testValues.expected.constantValue, + testValues.expected.dequantizationBefore, + testValues.expected.dequantizationAfter); + } + + static std::string getTestCaseName(testing::TestParamInfo obj) { + const ngraph::PartialShape inputShape = std::get<0>(obj.param); + const element::Type precision = std::get<1>(obj.param); + const element::Type precisionBeforeDequantization = std::get<2>(obj.param); + const size_t opsetVersion = std::get<3>(obj.param); + const AssignTransformationTestValues testValues = std::get<4>(obj.param); + + std::ostringstream result; + result << toString(testValues.params) << "_" << + inputShape << "_" << precision << "_" << + opsetVersion << "_" << testValues.FQAfterReadValue << "_" << + precisionBeforeDequantization << "_" << + testValues.actual.constantValue << "_" << + testValues.actual.dequantization; + return result.str(); + } +}; + +TEST_P(AssignTransformation, CompareFunctions) { + actualFunction->validate_nodes_and_infer_types(); + auto res = compare_functions(referenceFunction, actualFunction, true, true); + ASSERT_TRUE(res.first) << res.second; + + ASSERT_TRUE(LayerTransformation::allNamesAreUnique(actualFunction)) << "Not all names are unique"; +} + +namespace testValues1 { +const std::vector inputShapes = { + ngraph::PartialShape({ 1, 3, 224, 224 }), +}; + +const element::TypeVector precisions = { + element::f16, element::f32 +}; + +const element::TypeVector precisionsBeforeDequantizations = { + element::i8, element::u8, + element::i16, element::u16, + element::i32, element::u32, +}; + +const std::vector opsetVersions = { + 3, + 6 +}; + +const std::vector testValues = { + // general case, no subtract, FQ after ReadValue + { + LayerTransformation::createParamsU8I8(), + // ActualValues + { + {0}, + {{ngraph::element::f32}, {}, {3.f}} + }, + // ExpectedValues + { + {0}, + {{}, {}, {}}, + {{ngraph::element::f32}, {}, {3.f}} + }, + true + }, + // no FQ after ReadValue + { + LayerTransformation::createParamsU8I8(), + // ActualValues + { + {0}, + {{ngraph::element::f32}, {}, {3.f}} + }, + // ExpectedValues + { + {0}, + {{}, {}, {}}, + {{ngraph::element::f32}, {}, {3.f}} + }, + false + }, + // non-zero constant + { + LayerTransformation::createParamsU8I8(), + // ActualValues + { + {5}, + {{ngraph::element::f32}, {}, {3.f}} + }, + // ExpectedValues + { + {5}, + {{ngraph::element::f32}, {}, {3.f}}, + {} + }, + false + }, +}; + +INSTANTIATE_TEST_SUITE_P( + smoke_LPT, + AssignTransformation, + ::testing::Combine( + ::testing::ValuesIn(inputShapes), + ::testing::ValuesIn(precisions), + ::testing::ValuesIn(precisionsBeforeDequantizations), + ::testing::ValuesIn(opsetVersions), + ::testing::ValuesIn(testValues)), + AssignTransformation::getTestCaseName); +} // namespace testValues1 +} // namespace diff --git a/src/tests/functional/inference_engine/lp_transformations/simple_low_precision_transformer.hpp b/src/tests/functional/inference_engine/lp_transformations/simple_low_precision_transformer.hpp index c349056f6de..9f68b6d0542 100644 --- a/src/tests/functional/inference_engine/lp_transformations/simple_low_precision_transformer.hpp +++ b/src/tests/functional/inference_engine/lp_transformations/simple_low_precision_transformer.hpp @@ -24,6 +24,10 @@ public: void add(const TestTransformationParams& params) { commonGraphRewrite->add_matcher(TestTransformationParams::toParams(params)); } + template + void add(const std::shared_ptr function, const TestTransformationParams& params) { + commonGraphRewrite->add_matcher(function, TestTransformationParams::toParams(params)); + } void transform(std::shared_ptr& function); bool run_on_model(const std::shared_ptr& m) override; diff --git a/src/tests/functional/plugin/cpu/shared_tests_instances/low_precision_transformations/assign_and_read_value_transformation.cpp b/src/tests/functional/plugin/cpu/shared_tests_instances/low_precision_transformations/assign_and_read_value_transformation.cpp new file mode 100644 index 00000000000..0da67ab4379 --- /dev/null +++ b/src/tests/functional/plugin/cpu/shared_tests_instances/low_precision_transformations/assign_and_read_value_transformation.cpp @@ -0,0 +1,55 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include + +#include "low_precision_transformations/assign_and_read_value_transformation.hpp" + + +using namespace LayerTestsDefinitions; + +namespace { +const std::vector netPrecisions = { + ngraph::element::f32, + // ngraph::element::f16 +}; + +const std::vector opsetVersions = { + // 3, // no evaluate for opset 3 in ngraph + 6 +}; + +const std::vector trasformationParamValues = { + LayerTestsUtils::LayerTransformationParamsNGraphFactory::createParams(), + LayerTestsUtils::LayerTransformationParamsNGraphFactory::createParams().setUpdatePrecisions(false), +}; + +const std::vector params{ + // u8 + { + { 256ul, ngraph::Shape{ 1, 1, 1, 1 }, { 0.f }, { 25.5f }, { 0.f }, { 12.8f } }, + }, + // u16 + { + { 65536ul, ngraph::Shape{ 1, 1, 1, 1 }, { 0.f }, { 25.5f }, { 0.f }, { 12.8f } }, + }, + // u32 + { + { 4294967296ul, ngraph::Shape{ 1, 1, 1, 1 }, { 0.f }, { 25.5f }, { 0.f }, { 12.8f } }, + }, +}; + +INSTANTIATE_TEST_SUITE_P(smoke_LPT, AssignAndReadValueTransformation, + ::testing::Combine( + ::testing::ValuesIn(netPrecisions), + ::testing::Values(ngraph::PartialShape({ 1, 3, 16, 16 })), + ::testing::ValuesIn(opsetVersions), + ::testing::Values(CommonTestUtils::DEVICE_CPU), + ::testing::ValuesIn(trasformationParamValues), + ::testing::ValuesIn(params)), + AssignAndReadValueTransformation::getTestCaseName); + +} // namespace diff --git a/src/tests/functional/plugin/gpu/shared_tests_instances/low_precision_transformations/assign_and_read_value_transformation.cpp b/src/tests/functional/plugin/gpu/shared_tests_instances/low_precision_transformations/assign_and_read_value_transformation.cpp new file mode 100644 index 00000000000..390514c1884 --- /dev/null +++ b/src/tests/functional/plugin/gpu/shared_tests_instances/low_precision_transformations/assign_and_read_value_transformation.cpp @@ -0,0 +1,55 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include + +#include "low_precision_transformations/assign_and_read_value_transformation.hpp" + + +using namespace LayerTestsDefinitions; + +namespace { +const std::vector netPrecisions = { + ngraph::element::f32, + // ngraph::element::f16 +}; + +const std::vector opsetVersions = { + // 3, // no evaluate for opset 3 in ngraph + // 6 // not supported on GPU +}; + +const std::vector trasformationParamValues = { + LayerTestsUtils::LayerTransformationParamsNGraphFactory::createParams(), + LayerTestsUtils::LayerTransformationParamsNGraphFactory::createParams().setUpdatePrecisions(false), +}; + +const std::vector params{ + // u8 + { + { 256ul, ngraph::Shape{ 1, 1, 1, 1 }, { 0.f }, { 25.5f }, { 0.f }, { 12.8f } }, + }, + // u16 + { + { 65536ul, ngraph::Shape{ 1, 1, 1, 1 }, { 0.f }, { 25.5f }, { 0.f }, { 12.8f } }, + }, + // u32 + { + { 4294967296ul, ngraph::Shape{ 1, 1, 1, 1 }, { 0.f }, { 25.5f }, { 0.f }, { 12.8f } }, + }, +}; + +INSTANTIATE_TEST_SUITE_P(smoke_LPT, AssignAndReadValueTransformation, + ::testing::Combine( + ::testing::ValuesIn(netPrecisions), + ::testing::Values(ngraph::PartialShape({ 1, 3, 16, 16 })), + ::testing::ValuesIn(opsetVersions), + ::testing::Values(CommonTestUtils::DEVICE_GPU), + ::testing::ValuesIn(trasformationParamValues), + ::testing::ValuesIn(params)), + AssignAndReadValueTransformation::getTestCaseName); + +} // namespace diff --git a/src/tests/functional/plugin/shared/include/low_precision_transformations/assign_and_read_value_transformation.hpp b/src/tests/functional/plugin/shared/include/low_precision_transformations/assign_and_read_value_transformation.hpp new file mode 100644 index 00000000000..3a5adf331e1 --- /dev/null +++ b/src/tests/functional/plugin/shared/include/low_precision_transformations/assign_and_read_value_transformation.hpp @@ -0,0 +1,35 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "shared_test_classes/base/low_precision_transformations/layer_transformation.hpp" +#include "lpt_ngraph_functions/common/fake_quantize_on_data.hpp" +#include "lpt_ngraph_functions/common/dequantization_operations.hpp" + +namespace LayerTestsDefinitions { +class AssignAndReadValueTransformationParam { +public: + ngraph::builder::subgraph::FakeQuantizeOnData fakeQuantize; +}; + +typedef std::tuple < + ngraph::element::Type, // input precision + ngraph::PartialShape, // input shape + size_t, // opset version + std::string, // device + ngraph::pass::low_precision::LayerTransformation::Params, // transformation params + AssignAndReadValueTransformationParam // test params +> AssignAndReadValueTransformationParams; + +class AssignAndReadValueTransformation : + public testing::WithParamInterface, + public LayerTestsUtils::LayerTransformation { +public: + static std::string getTestCaseName(const testing::TestParamInfo& obj); +protected: + void SetUp() override; +}; + +} // namespace LayerTestsDefinitions diff --git a/src/tests/functional/plugin/shared/src/low_precision_transformations/assign_and_read_value_transformation.cpp b/src/tests/functional/plugin/shared/src/low_precision_transformations/assign_and_read_value_transformation.cpp new file mode 100644 index 00000000000..43123a36cd1 --- /dev/null +++ b/src/tests/functional/plugin/shared/src/low_precision_transformations/assign_and_read_value_transformation.cpp @@ -0,0 +1,49 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "low_precision_transformations/assign_and_read_value_transformation.hpp" +#include +#include +#include +#include + +#include "lpt_ngraph_functions/assign_and_read_value_function.hpp" + +namespace LayerTestsDefinitions { + +std::string AssignAndReadValueTransformation::getTestCaseName(const testing::TestParamInfo& obj) { + ngraph::element::Type netPrecision; + ngraph::PartialShape inputShape; + size_t opset; + std::string targetDevice; + ngraph::pass::low_precision::LayerTransformation::Params params; + AssignAndReadValueTransformationParam param;; + std::tie(netPrecision, inputShape, opset, targetDevice, params, param) = obj.param; + + std::ostringstream result; + result << getTestCaseNameByParams(netPrecision, inputShape, targetDevice, params) << "_" << + param.fakeQuantize << "_" << opset; + return result.str(); +} + +void AssignAndReadValueTransformation::SetUp() { + ngraph::element::Type netPrecision; + ngraph::PartialShape inputShape; + size_t opset; + ngraph::pass::low_precision::LayerTransformation::Params params; + AssignAndReadValueTransformationParam param; + std::tie(netPrecision, inputShape, opset, targetDevice, params, param) = this->GetParam(); + + function = ngraph::builder::subgraph::AssignAndReadValueFunction::getOriginal( + netPrecision, + inputShape, + param.fakeQuantize, + opset); +} + +TEST_P(AssignAndReadValueTransformation, CompareWithRefImpl) { + Run(); +}; + +} // namespace LayerTestsDefinitions diff --git a/src/tests/ngraph_helpers/lpt_ngraph_functions/include/lpt_ngraph_functions/assign_and_read_value_function.hpp b/src/tests/ngraph_helpers/lpt_ngraph_functions/include/lpt_ngraph_functions/assign_and_read_value_function.hpp new file mode 100644 index 00000000000..d5113da7b1b --- /dev/null +++ b/src/tests/ngraph_helpers/lpt_ngraph_functions/include/lpt_ngraph_functions/assign_and_read_value_function.hpp @@ -0,0 +1,46 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include "lpt_ngraph_functions/common/fake_quantize_on_data.hpp" +#include "lpt_ngraph_functions/common/dequantization_operations.hpp" + +namespace ngraph { +namespace builder { +namespace subgraph { + +class AssignAndReadValueFunction { +public: + static std::shared_ptr getOriginal( + const ngraph::PartialShape& inputShape, + const element::Type& inputPrecision, + const ngraph::element::Type precisionBeforeDequantization, + const size_t opsetVersion, + const bool FQAfterReadValue, + const std::vector& constantValue, + const ngraph::builder::subgraph::DequantizationOperations& dequantization); + + static std::shared_ptr getOriginal( + const ngraph::element::Type originalFunctionPrecision, + const ngraph::PartialShape& inputShape, + const ngraph::builder::subgraph::FakeQuantizeOnData fakeQuantize, + const size_t opsetVersion); + + static std::shared_ptr getReference( + const ngraph::PartialShape& inputShape, + const element::Type& inputPrecision, + const ngraph::element::Type precisionBeforeDequantization, + const size_t opsetVersion, + const bool FQAfterReadValue, + const std::vector& constantValue, + const ngraph::builder::subgraph::DequantizationOperations& dequantizationBefore, + const ngraph::builder::subgraph::DequantizationOperations& dequantizationAfter); +}; +} // namespace subgraph +} // namespace builder +} // namespace ngraph diff --git a/src/tests/ngraph_helpers/lpt_ngraph_functions/src/assign_and_read_value_function.cpp b/src/tests/ngraph_helpers/lpt_ngraph_functions/src/assign_and_read_value_function.cpp new file mode 100644 index 00000000000..67e50de1852 --- /dev/null +++ b/src/tests/ngraph_helpers/lpt_ngraph_functions/src/assign_and_read_value_function.cpp @@ -0,0 +1,185 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include +#include + + +#include +#include "ngraph_functions/subgraph_builders.hpp" +#include "openvino/op/util/variable.hpp" +#include + +#include "lpt_ngraph_functions/common/builders.hpp" +#include "lpt_ngraph_functions/assign_and_read_value_function.hpp" +#include "low_precision/network_helper.hpp" + +namespace ngraph { +namespace builder { +namespace subgraph { + +std::shared_ptr AssignAndReadValueFunction::getOriginal( + const ngraph::PartialShape& inputShape, + const element::Type& inputPrecision, + const ngraph::element::Type precisionBeforeDequantization, + const size_t opsetVersion, + const bool FQAfterReadValue, + const std::vector& constantValue, + const ngraph::builder::subgraph::DequantizationOperations& dequantization) { + const auto input = std::make_shared(inputPrecision, inputShape); + const auto defaultConstant = std::make_shared(inputPrecision, inputShape.get_shape(), constantValue); + const auto variable = std::make_shared(VariableInfo{inputShape.get_shape(), inputPrecision, "id"}); + std::shared_ptr readValue; + if (opsetVersion == 6) { + readValue = std::make_shared(defaultConstant, variable); + } else if (opsetVersion == 3) { + readValue = std::make_shared(defaultConstant, "id"); + } else { + throw std::runtime_error("Unknown opset version"); + } + std::shared_ptr lastNode = readValue; + if (FQAfterReadValue) { + lastNode = builder::subgraph::makeFakeQuantize( + lastNode, + element::f32, + FakeQuantizeOnData{256ul, Shape{}, {0}, {2.55f}, {0}, {2.55f}}); + } + const auto add = std::make_shared(lastNode, input); + const auto FQAfterAdd = builder::subgraph::makeFakeQuantizeTypeRelaxed( + add, + element::f32, + FakeQuantizeOnData{256ul, Shape{}, {0}, {2.55f}, {0}, {2.55f}, precisionBeforeDequantization}); + auto deqStructure = dequantization; + deqStructure.multiply.outPrecision = inputPrecision; + const auto dequantizationOp = makeDequantization(FQAfterAdd, deqStructure); + std::shared_ptr assign; + if (opsetVersion == 6) { + assign = std::make_shared(dequantizationOp, variable); + } else { + assign = std::make_shared(dequantizationOp, "id"); + } + assign->add_control_dependency(readValue); + add->set_friendly_name("output"); + + ngraph::ResultVector results{ std::make_shared(add) }; + ngraph::SinkVector sinks{ as_type_ptr(assign) }; + return std::make_shared(results, sinks, ngraph::ParameterVector{ input }, "AssignAndReadValueFunction"); +} + +std::shared_ptr AssignAndReadValueFunction::getOriginal( + const ngraph::element::Type precision, + const ngraph::PartialShape& inputShape, + const ngraph::builder::subgraph::FakeQuantizeOnData fakeQuantize, + const size_t opsetVersion) { + const auto input = std::make_shared(precision, inputShape); + const auto defaultConstant = std::make_shared(precision, inputShape.get_shape(), std::vector{0}); + const auto variable = std::make_shared(VariableInfo{inputShape.get_shape(), precision, "id"}); + std::shared_ptr readValue; + if (opsetVersion == 6) { + readValue = std::make_shared(defaultConstant, variable); + } else if (opsetVersion == 3) { + readValue = std::make_shared(defaultConstant, "id"); + } else { + throw std::runtime_error("Unknown opset version"); + } + std::shared_ptr lastNode = readValue; + lastNode = builder::subgraph::makeFakeQuantize( + lastNode, + element::f32, + FakeQuantizeOnData{256ul, Shape{}, {0}, {2.55f}, {0}, {2.55f}}); + const auto add = std::make_shared(lastNode, input); + const auto FQAfterAdd = fakeQuantize.empty() ? nullptr : + ngraph::builder::makeFakeQuantize( + add, + precision, + fakeQuantize.quantizationLevel, + fakeQuantize.constantShape, + fakeQuantize.inputLowValues, + fakeQuantize.inputHighValues, + fakeQuantize.outputLowValues, + fakeQuantize.outputHighValues); + std::shared_ptr assign; + if (opsetVersion == 6) { + assign = std::make_shared(FQAfterAdd, variable); + } else { + assign = std::make_shared(FQAfterAdd, "id"); + } + assign->add_control_dependency(readValue); + add->set_friendly_name("output"); + + ngraph::ResultVector results{ std::make_shared(add) }; + ngraph::SinkVector sinks{ as_type_ptr(assign) }; + return std::make_shared(results, sinks, ngraph::ParameterVector{ input }, "AssignAndReadValueFunction"); +} + +std::shared_ptr AssignAndReadValueFunction::getReference( + const ngraph::PartialShape& inputShape, + const element::Type& inputPrecision, + const ngraph::element::Type precisionBeforeDequantization, + const size_t opsetVersion, + const bool FQAfterReadValue, + const std::vector& constantValue, + const ngraph::builder::subgraph::DequantizationOperations& dequantizationBefore, + const ngraph::builder::subgraph::DequantizationOperations& dequantizationAfter) { + const auto input = std::make_shared(inputPrecision, inputShape); + auto constantPrecision = precisionBeforeDequantization; + if (constantValue != std::vector{0}) { + constantPrecision = inputPrecision; + } + const auto defaultConstant = std::make_shared(constantPrecision, inputShape.get_shape(), constantValue); + const auto variable = std::make_shared(VariableInfo{inputShape.get_shape(), constantPrecision, "id"}); + std::shared_ptr readValue; + if (opsetVersion == 6) { + readValue = std::make_shared(defaultConstant, variable); + } else if (opsetVersion == 3) { + readValue = std::make_shared(defaultConstant, "id"); + } else { + throw std::runtime_error("Unknown opset version"); + } + std::shared_ptr lastNode = readValue; + + auto deqStructureAfter = dequantizationAfter; + if (FQAfterReadValue) { + DequantizationOperations tempDequantization; + tempDequantization.convert = dequantizationAfter.convert; + tempDequantization.subtract = dequantizationAfter.subtract; + lastNode = makeDequantization(lastNode, tempDequantization); + } else { + deqStructureAfter.multiply.outPrecision = inputPrecision; + lastNode = makeDequantization(lastNode, deqStructureAfter); + } + + if (FQAfterReadValue) { + lastNode = builder::subgraph::makeFakeQuantizeTypeRelaxed( + lastNode, + element::f32, + FakeQuantizeOnData{256ul, Shape{}, {0}, {2.55f / dequantizationAfter.multiply.values[0]}, {0}, {2.55f}, inputPrecision}); + } + const auto add = std::make_shared(lastNode, input); + const auto FQAfterAdd = builder::subgraph::makeFakeQuantizeTypeRelaxed( + add, + element::f32, + FakeQuantizeOnData{256ul, Shape{}, {0}, {2.55f}, {0}, {2.55f}, precisionBeforeDequantization}); + + auto deqStructureBefore = dequantizationBefore; + deqStructureBefore.multiply.outPrecision = inputPrecision; + const auto dequantizationBeforeStructure = makeDequantization(FQAfterAdd, deqStructureBefore); + std::shared_ptr assign; + if (opsetVersion == 6) { + assign = std::make_shared(dequantizationBeforeStructure, variable); + } else { + assign = std::make_shared(dequantizationBeforeStructure, "id"); + } + assign->add_control_dependency(readValue); + add->set_friendly_name("output"); + + ngraph::ResultVector results{ std::make_shared(add) }; + ngraph::SinkVector sinks{ as_type_ptr(assign) }; + return std::make_shared(results, sinks, ngraph::ParameterVector{ input }, "AssignAndReadValueFunction"); +} + +} // namespace subgraph +} // namespace builder +} // namespace ngraph From 2538ae5da10f2679c4d3168aca0c9aba2d94c4c1 Mon Sep 17 00:00:00 2001 From: Artyom Anokhov Date: Mon, 27 Dec 2021 17:17:22 +0300 Subject: [PATCH 5/6] [Deployment Manager] Remove `Python common` component from configs (#9448) * Deployment Manager configs: Remove `Python common` component * Dpeloyment Manager configs: Fixed misprint * DM configs: Fixed JSON syntax * DM darwin.json: Removed double comma --- tools/deployment_manager/configs/darwin.json | 15 ++++----------- tools/deployment_manager/configs/linux.json | 15 ++++----------- tools/deployment_manager/configs/windows.json | 15 ++++----------- 3 files changed, 12 insertions(+), 33 deletions(-) diff --git a/tools/deployment_manager/configs/darwin.json b/tools/deployment_manager/configs/darwin.json index 501f15599c0..058d2fcede0 100644 --- a/tools/deployment_manager/configs/darwin.json +++ b/tools/deployment_manager/configs/darwin.json @@ -46,17 +46,10 @@ "runtime/lib/intel64/pcie-ma2x8x.mvcmd" ] }, - "python_common": { - "group": ["python"], - "dependencies" : ["ie_core"], - "files": [ - "python/requirements.txt" - ] - }, "python3.6": { "ui_name": "OpenVINO Python API for Python3.6", "group": ["python"], - "dependencies" : ["ie_core", "python_common"], + "dependencies" : ["ie_core"], "files": [ "python/python3.6" ] @@ -64,7 +57,7 @@ "python3.7": { "ui_name": "OpenVINO Python API for Python3.7", "group": ["python"], - "dependencies" : ["ie_core", "python_common"], + "dependencies" : ["ie_core"], "files": [ "python/python3.7" ] @@ -72,7 +65,7 @@ "python3.8": { "ui_name": "OpenVINO Python API for Python3.8", "group": ["python"], - "dependencies" : ["ie_core", "python_common"], + "dependencies" : ["ie_core"], "files": [ "python/python3.8" ] @@ -80,7 +73,7 @@ "python3.9": { "ui_name": "OpenVINO Python API for Python3.9", "group": ["python"], - "dependencies" : ["ie_core", "python_common"], + "dependencies" : ["ie_core"], "files": [ "python/python3.9" ] diff --git a/tools/deployment_manager/configs/linux.json b/tools/deployment_manager/configs/linux.json index e5b4889e34a..777241b4d19 100644 --- a/tools/deployment_manager/configs/linux.json +++ b/tools/deployment_manager/configs/linux.json @@ -85,17 +85,10 @@ "runtime/3rdparty/hddl" ] }, - "python_common": { - "group": ["python"], - "dependencies" : ["ie_core"], - "files": [ - "python/requirements.txt" - ] - }, "python3.6": { "ui_name": "OpenVINO Python API for Python3.6", "group": ["python"], - "dependencies" : ["ie_core", "python_common"], + "dependencies" : ["ie_core"], "files": [ "python/python3.6" ] @@ -103,7 +96,7 @@ "python3.7": { "ui_name": "OpenVINO Python API for Python3.7", "group": ["python"], - "dependencies" : ["ie_core", "python_common"], + "dependencies" : ["ie_core"], "files": [ "python/python3.7" ] @@ -111,7 +104,7 @@ "python3.8": { "ui_name": "OpenVINO Python API for Python3.8", "group": ["python"], - "dependencies" : ["ie_core", "python_common"], + "dependencies" : ["ie_core"], "files": [ "python/python3.8" ] @@ -119,7 +112,7 @@ "python3.9": { "ui_name": "OpenVINO Python API for Python3.9", "group": ["python"], - "dependencies" : ["ie_core", "python_common"], + "dependencies" : ["ie_core"], "files": [ "python/python3.9" ] diff --git a/tools/deployment_manager/configs/windows.json b/tools/deployment_manager/configs/windows.json index 1eb0498a652..9c16b5df8c6 100644 --- a/tools/deployment_manager/configs/windows.json +++ b/tools/deployment_manager/configs/windows.json @@ -74,17 +74,10 @@ "runtime/3rdparty/hddl" ] }, - "python_common": { - "group": ["python"], - "dependencies" : ["ie_core"], - "files": [ - "python/requirements.txt" - ] - }, "python3.6": { "ui_name": "OpenVINO Python API for Python3.6", "group": ["python"], - "dependencies" : ["ie_core", "python_common"], + "dependencies" : ["ie_core"], "files": [ "python/python3.6" ] @@ -92,7 +85,7 @@ "python3.7": { "ui_name": "OpenVINO Python API for Python3.7", "group": ["python"], - "dependencies" : ["ie_core", "python_common"], + "dependencies" : ["ie_core"], "files": [ "python/python3.7" ] @@ -100,7 +93,7 @@ "python3.8": { "ui_name": "OpenVINO Python API for Python3.8", "group": ["python"], - "dependencies" : ["ie_core", "python_common"], + "dependencies" : ["ie_core"], "files": [ "python/python3.8" ] @@ -108,7 +101,7 @@ "python3.9": { "ui_name": "OpenVINO Python API for Python3.9", "group": ["python"], - "dependencies" : ["ie_core", "python_common"], + "dependencies" : ["ie_core"], "files": [ "python/python3.9" ] From ab100573718464af36f233c66a191c394e3e8682 Mon Sep 17 00:00:00 2001 From: Vladimir Dudnik Date: Mon, 27 Dec 2021 17:32:40 +0300 Subject: [PATCH 6/6] update open_model_zoo submodule. remove deprecated public models, fix mo path in omz_converter (#9453) --- thirdparty/open_model_zoo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thirdparty/open_model_zoo b/thirdparty/open_model_zoo index f18c014ad41..e813a1f0ee0 160000 --- a/thirdparty/open_model_zoo +++ b/thirdparty/open_model_zoo @@ -1 +1 @@ -Subproject commit f18c014ad4173babaf5554aa7930a66701f80bf6 +Subproject commit e813a1f0ee071c48b8bb3f56ec85331be9b35c07