IntenseWebs/openvino
3
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CPU] optimize ExtractImagePatches (#4898)

Browse Source
This commit is contained in:
Ivan Novoselov authored and GitHub committed 2021-04-25 19:52:07 +03:00
1 parent b770be227b
commit 594c4882e6
5 files changed
+623 -228
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

inference-engine/src/mkldnn_plugin/nodes/extract_image_patches.cpp
+464 -216
View File
@@ -1,17 +1,15 @@
// Copyright (C) 2018-2021 Intel Corporation
// Copyright (C) 2020-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "list.hpp"
#include "base.hpp"
#include "extract_image_patches.hpp"
#include "caseless.hpp"
#include <cmath>
#include <string>
#include <vector>
#include <cassert>
#include <set>
#include "ie_parallel.hpp"
#include "list.hpp"
#include <cpu/x64/jit_generator.hpp>
#include <cstring>
#include <string>
#include <cmath>
namespace InferenceEngine {
namespace Extensions {
@@ -19,229 +17,479 @@ namespace Cpu {
using details::CaselessEq;
class ExtractImagePatchesImpl : public ExtLayerBase {
public:
explicit ExtractImagePatchesImpl(const CNNLayer* layer) {
try {
std::string errorPrefix = std::string("Layer ") + layer->type + " with name '" + layer->name + "' ";
if (details::CaselessEq<std::string>()("ExtractImagePatchesLayer", layer->type))
IE_THROW() << errorPrefix << "is not an instance of ExtractImagePatchesLayer class";
using namespace dnnl::impl::cpu;
using namespace dnnl::impl::cpu::x64;
using namespace dnnl::impl::utils;
using namespace Xbyak;
if (layer->insData.size() != 1 || layer->outData.size() != 1)
IE_THROW() << errorPrefix << "has incorrect number of input or output edges!"
<< " Input: " << layer->insData.size() << "; Output: " << layer->outData.size();
#define GET_OFF(field) offsetof(jit_extract_image_patches_args, field)
auto inData = layer->insData[0].lock();
if (inData == nullptr)
IE_THROW() << errorPrefix << "has nullable input data";
template <cpu_isa_t isa>
struct jit_extract_image_patches_kernel : public jit_uni_extract_image_patches_kernel, public jit_generator {
DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_extract_image_patches_kernel)
if (inData->getTensorDesc().getDims().size() != 4)
IE_THROW() << errorPrefix << "must have 4D input tensor. Actual: " << inData->getTensorDesc().getDims().size();
explicit jit_extract_image_patches_kernel(jit_extract_image_patches_params jpp) : jit_uni_extract_image_patches_kernel(jpp), jit_generator() {}
if (layer->outData[0]->getTensorDesc().getDims().size() != 4)
IE_THROW() << errorPrefix << "must have 4D output tensor. Actual: " << layer->outData[0]->getTensorDesc().getDims().size();
if (inData->getLayout() != NCHW)
IE_THROW() << errorPrefix << "has unsupported layout: " << inData->getLayout();
const auto precision = inData->getTensorDesc().getPrecision();
if (_supported_precisions_sizes.find(precision.size()) == _supported_precisions_sizes.end())
IE_THROW() << errorPrefix << "has unsupported precision: " << precision.name();
auto ksizes = layer->GetParamAsUInts("sizes");
auto strides = layer->GetParamAsUInts("strides");
auto rates = layer->GetParamAsUInts("rates");
_auto_pad = layer->GetParamAsString("auto_pad");
if (!CaselessEq<std::string>()(_auto_pad, "valid")
&& !CaselessEq<std::string>()(_auto_pad, "same_upper")
&& !CaselessEq<std::string>()(_auto_pad, "same_lower"))
IE_THROW() << errorPrefix << "has unsupported auto_pad value: " << _auto_pad;
if (ksizes.size() != 2 || strides.size() != 2 || rates.size() != 2)
IE_THROW() << errorPrefix << "must have the following attributes with shape {2}: sizes, strides, rates.";
_ksizes.clear();
_strides.clear();
_rates.clear();
for (size_t i = 0; i < ksizes.size(); i++)
_ksizes.push_back((int64_t)ksizes[i]);
for (size_t i = 0; i < strides.size(); i++)
_strides.push_back((int64_t)strides[i]);
for (size_t i = 0; i < rates.size(); i++)
_rates.push_back((int64_t)rates[i]);
LayerConfig config;
DataConfig inConfig;
inConfig.desc = inData->getTensorDesc();
config.inConfs.push_back(inConfig);
DataConfig outConfig;
outConfig.desc = layer->outData[0]->getTensorDesc();
outConfig.desc.setPrecision(inConfig.desc.getPrecision());
outConfig.desc.setLayout(inConfig.desc.getLayout());
config.outConfs.push_back(outConfig);
config.dynBatchSupport = false;
confs.push_back(config);
} catch (InferenceEngine::Exception &ex) {
errorMsg = ex.what();
}
void create_ker() override {
jit_generator::create_kernel();
ker_ = (decltype(ker_))jit_ker();
}
StatusCode execute(std::vector<Blob::Ptr>& inputs, std::vector<Blob::Ptr>& outputs, ResponseDesc *resp) noexcept override {
switch (inputs[0]->getTensorDesc().getPrecision().size()) {
case 1: {
process_data<PrecisionTrait<Precision::U8>::value_type>(inputs, outputs);
break;
}
case 2: {
process_data<PrecisionTrait<Precision::U16>::value_type>(inputs, outputs);
break;
}
case 4: {
process_data<PrecisionTrait<Precision::I32>::value_type>(inputs, outputs);
break;
}
case 8: {
process_data<PrecisionTrait<Precision::U64>::value_type>(inputs, outputs);
break;
}
default: {
if (resp) {
std::string errorMsg = "ExtractImagePatches layer does not support precision '"
+ std::string(inputs[0]->getTensorDesc().getPrecision().name()) + "'";
errorMsg.copy(resp->msg, sizeof(resp->msg) - 1);
}
return GENERAL_ERROR;
}
void generate() override {
this->preamble();
mov(reg_num_pads, ptr[reg_params + GET_OFF(h_lo_pad)]);
mov(reg_h_hi_pad, ptr[reg_params + GET_OFF(h_hi_pad)]);
mov(reg_w_lo_pad, ptr[reg_params + GET_OFF(w_lo_pad)]);
mov(reg_w_hi_pad, ptr[reg_params + GET_OFF(w_hi_pad)]);
mov(reg_src, ptr[reg_params + GET_OFF(src)]);
mov(reg_dst, ptr[reg_params + GET_OFF(dst)]);
mov(reg_src_incr, jpp.SH * jpp.IW * jpp.dtype_size);
mov(reg_aux64, reg_w_hi_pad);
mul_by_const(reg_aux64, reg_aux64_2, jpp.SW * jpp.dtype_size);
sub(reg_src_incr, reg_aux64);
mov(reg_aux64, reg_w_lo_pad);
mul_by_const(reg_aux64, reg_aux64_2, jpp.SW * jpp.dtype_size);
add(reg_src_incr, reg_aux64);
add(reg_src, reg_aux64);
mov(reg_ow_work_amount, reg_w_hi_pad);
sub(reg_ow_work_amount, reg_w_lo_pad);
uni_vpxor(vmm_zero, vmm_zero, vmm_zero);
if (mayiuse_gather) {
mov(reg_aux64, gather_index_table);
uni_vmovups(vmm_gather_index, ptr[reg_aux64]);
}
loop();
return OK;
}
this->postamble();
template<typename T>
void process_data(std::vector<Blob::Ptr>& inputs, std::vector<Blob::Ptr>& outputs) noexcept {
const T* src_data = inputs[0]->cbuffer().as<const T*>() +
inputs[0]->getTensorDesc().getBlockingDesc().getOffsetPadding();
T* dst_data = outputs[0]->buffer().as<T*>() +
outputs[0]->getTensorDesc().getBlockingDesc().getOffsetPadding();
const auto& inDims = inputs[0]->getTensorDesc().getDims();
const size_t inDimsSize = inDims.size();
const size_t BATCH = 0, CHANNEL = 1, HIGHT = 0, WIDTH = 1;
const int64_t IC = inDims[CHANNEL];
const int64_t IH = inDims[inDimsSize - 2];
const int64_t IW = inDims[inDimsSize - 1];
const auto& outDims = outputs[0]->getTensorDesc().getDims();
const size_t outDimsSize = outDims.size();
const int64_t OB = outDims[BATCH];
const int64_t OC = outDims[CHANNEL];
const int64_t OH = outDims[outDimsSize - 2];
const int64_t OW = outDims[outDimsSize - 1];
const int64_t KH = _ksizes[HIGHT];
const int64_t KW = _ksizes[WIDTH];
const int64_t SH = _strides[HIGHT];
const int64_t SW = _strides[WIDTH];
const int64_t RH = _rates[HIGHT];
const int64_t RW = _rates[WIDTH];
int64_t iwStep = KW + (RW - 1) * (KW - 1);
int64_t ihStep = KH + (RH - 1) * (KH - 1);
int64_t PL = 0, PT = 0;
if (!CaselessEq<std::string>()(_auto_pad, "valid")) {
int64_t PW = (std::ceil(1.f * IW/SW) - 1) * SW + iwStep - IW;
int64_t PH = (std::ceil(1.f * IH/SH) - 1) * SH + ihStep - IH;
if ((PW > 0) && (PW < iwStep)) {
if (PW % 2 == 1) {
if (CaselessEq<std::string>()(_auto_pad, "same_lower")) {
PL = (PW + 1) / 2;
} else if (CaselessEq<std::string>()(_auto_pad, "same_upper")) {
PL = (PW - 1) / 2;
}
} else {
PL = PW / 2;
}
}
if ((PH > 0) && (PH < ihStep)) {
if (PH % 2 == 1) {
if (CaselessEq<std::string>()(_auto_pad, "same_lower")) {
PT = (PH + 1) / 2;
} else if (CaselessEq<std::string>()(_auto_pad, "same_upper")) {
PT = (PH - 1) / 2;
}
} else {
PT = PH / 2;
}
}
}
const int64_t OH_OW = OH * OW;
const int64_t OC_OH_OW = OC * OH_OW;
const int64_t IH_IW = IH * IW;
const int64_t IC_IH_IW = IC * IH_IW;
const int64_t work_amount = OB;
auto thread_body = [&](const int ithr, const int nthr) {
int64_t start(0lu), end(0lu);
splitter(work_amount, nthr, ithr, start, end);
if (start >= end)
return;
for (int64_t ob = start; ob < end; ob++) {
const int64_t ibICIHIW = ob * IC_IH_IW;
const int64_t obOCOHOW = ob * OC_OH_OW;
for (int64_t oh = 0; oh < OH; oh++) {
const int64_t obOCOHOWohOW = obOCOHOW + oh * OW;
int64_t ih0 = oh * SH - PT;
for (int64_t ow = 0; ow < OW; ow++) {
const int64_t obOCOHOWohOWow = obOCOHOWohOW + ow;
int64_t iw0 = ow * SW - PL;
int64_t oc = 0;
for (int64_t kh = 0; kh < KH; kh++) {
int64_t ihKH = ih0 + kh * RH;
int64_t ibICIHIWihFHIW = ibICIHIW + ihKH * IW;
for (int64_t kw = 0; kw < KW; kw++) {
for (int64_t ic = 0; ic < IC; ic++, oc++) {
int64_t iwKW = iw0 + kw * RW;
int64_t dst_idx = obOCOHOWohOWow + oc * OH_OW;
if (ihKH < 0 || ihKH >= IH || iwKW < 0 || iwKW >= IW) {
dst_data[dst_idx] = T(0);
} else {
int64_t src_idx = ibICIHIWihFHIW + ic * IH_IW + iwKW;
dst_data[dst_idx] = src_data[src_idx];
}
}
}
}
}
}
}
};
parallel_nt(0, thread_body);
if (mayiuse_gather)
prepare_table();
}
private:
std::vector<int64_t> _ksizes;
std::vector<int64_t> _strides;
std::vector<int64_t> _rates;
std::string _auto_pad;
using Vmm = typename conditional3<isa == x64::sse41, Xbyak::Xmm, isa == x64::avx2, Xbyak::Ymm, Xbyak::Zmm>::type;
using reg64_t = const Xbyak::Reg64;
using reg32_t = const Xbyak::Reg32;
bool mayiuse_gather = (mayiuse(x64::avx2) || mayiuse(x64::avx512_common)) && (jpp.dtype_size == 4);
uint32_t vlen = cpu_isa_traits<isa>::vlen;
reg64_t reg_src = r8;
reg64_t reg_dst = r9;
reg64_t reg_oh_count = r10;
reg64_t reg_ow_count = r11;
reg64_t reg_num_pads = r12;
reg64_t reg_src_incr = r13;
reg64_t reg_aux64 = rax;
reg64_t reg_w_hi_pad = r14;
reg64_t reg_w_lo_pad = r15;
reg64_t reg_h_hi_pad = rbp;
reg64_t reg_aux64_2 = rbx;
reg64_t reg_ow_work_amount = rsi;
reg64_t reg_params = abi_param1;
static const std::set<size_t> _supported_precisions_sizes;
Vmm vmm = Vmm(0);
Xmm xmm = Xmm(0);
Vmm vmm_zero = Vmm(1); // reserved for pad
Xbyak::Xmm xmm_aux = Xbyak::Xmm(2);
Vmm vmm_gather_index = Vmm(3);
Vmm vmm_gather_mask = Vmm(4);
Opmask k_mask = Xbyak::Opmask(1);
Xbyak::Label gather_index_table;
inline void load_scalar(Vmm vmm_arg, const Xbyak::Address &op) {
Xbyak::Xmm xmm_src = Xmm(vmm_arg.getIdx());
switch (jpp.dtype_size) {
case 4: uni_vmovss(vmm_arg, op); break;
case 2: uni_vpinsrw(xmm_src, xmm_src, op, 0x0); break;
case 1: uni_vpinsrb(xmm_src, xmm_src, op, 0x0); break;
default: IE_THROW() << "The data type of size '" << jpp.dtype_size << "' is not supported.";
}
}
inline void store_scalar(const Xbyak::Address &op, Vmm vmm_arg) {
Xbyak::Xmm xmm_dst = Xmm(vmm_arg.getIdx());
switch (jpp.dtype_size) {
case 4: uni_vmovss(op, vmm_arg); break;
case 2: uni_vpextrw(op, xmm_dst, 0x0); break;
case 1: uni_vpextrb(op, xmm_dst, 0x0); break;
default: IE_THROW() << "The data type of size '" << jpp.dtype_size << "' is not supported.";
}
}
inline void pad_with_zeros(reg64_t &reg_num_pads_arg, reg64_t &reg_dst_arg) {
Xbyak::Label main, tail, exit;
L(main);
{
cmp(reg_num_pads_arg, jpp.block_size);
jl(tail);
uni_vmovups(ptr[reg_dst_arg], vmm_zero);
add(reg_dst_arg, jpp.dtype_size * jpp.block_size);
sub(reg_num_pads_arg, jpp.block_size);
jmp(main);
}
L(tail);
{
cmp(reg_num_pads_arg, 0);
jle(exit);
store_scalar(ptr[reg_dst_arg], vmm_zero);
add(reg_dst_arg, jpp.dtype_size);
dec(reg_num_pads_arg);
jmp(tail);
}
L(exit);
}
inline void custom_uni_vgatherdps(const Vmm &vmm_arg, reg64_t &mem_base, const Vmm &mem_offset, Vmm &vmm_mask) {
switch (isa) {
case x64::avx2:
uni_vpcmpeqd(vmm_mask, vmm_mask, vmm_mask);
vgatherdps(vmm_arg, ptr[mem_base + mem_offset], vmm_mask);
break;
case x64::avx512_common:
kxnord(k_mask, k_mask, k_mask);
vgatherdps(vmm_arg | k_mask, ptr[mem_base + mem_offset]);
break;
case x64::sse41:
emulate_gather(vmm_arg, mem_base);
break;
default: IE_THROW() << "Got unsupported instruction set.";
}
}
inline void gather_src2vmm(const Vmm &vmm_arg, reg64_t &mem_base) {
switch (jpp.dtype_size) {
case 4: custom_uni_vgatherdps(vmm, mem_base, vmm_gather_index, vmm_gather_mask); break;
case 2:
case 1: emulate_gather(vmm_arg, mem_base); break;
default: IE_THROW() << "The data type of size '" << jpp.dtype_size << "' is not supported.";
}
}
inline void emulate_gather(const Xbyak::Xmm &xmm_arg, reg64_t &mem_base, int xmm_offset = 0) {
const int xmm_size = 16; // bytes
const int xmm_block_size = xmm_size / jpp.dtype_size;
const int offset = xmm_offset * jpp.SW * jpp.dtype_size * xmm_block_size;
for (int i = 0; i < xmm_block_size; i++) {
Xbyak::Address addr = ptr[mem_base + i * jpp.SW * jpp.dtype_size + offset];
switch (jpp.dtype_size) {
case 4: uni_vpinsrd(xmm_arg, xmm_arg, addr, i); break;
case 2: uni_vpinsrw(xmm_arg, xmm_arg, addr, i); break;
case 1: uni_vpinsrb(xmm_arg, xmm_arg, addr, i); break;
default: IE_THROW() << "The data type of size '" << jpp.dtype_size << "' is not supported.";
}
}
}
inline void emulate_gather(const Xbyak::Ymm &ymm_arg, reg64_t &mem_base) {
Xbyak::Xmm low_xmm = Xbyak::Xmm(ymm_arg.getIdx());
emulate_gather(low_xmm, mem_base, 0);
emulate_gather(xmm_aux, mem_base, 1);
vinserti128(ymm_arg, ymm_arg, xmm_aux, 1);
}
inline void emulate_gather(const Xbyak::Zmm &zmm_arg, reg64_t &mem_base) {
Xbyak::Xmm low_xmm = Xbyak::Xmm(zmm_arg.getIdx());
emulate_gather(low_xmm, mem_base, 0);
for (int i = 1; i < 4; i++) {
emulate_gather(xmm_aux, mem_base, i);
vinserti64x2(zmm_arg, zmm_arg, xmm_aux, i);
}
}
void loop() {
mov(reg_oh_count, reg_h_hi_pad);
// reg_num_pads contains h_lo_pad at this point
sub(reg_oh_count, reg_num_pads);
Xbyak::Label ih_loop, ih_tail, ih_exit;
Xbyak::Label iw_loop, iw_tail, iw_exit;
if (jpp.need_padding) {
mul_by_const(reg_num_pads, reg_aux64, jpp.OW);
pad_with_zeros(reg_num_pads, reg_dst);
}
L(ih_loop);
{
cmp(reg_oh_count, 0);
jle(ih_exit, T_NEAR);
if (jpp.need_padding) {
mov(reg_num_pads, reg_w_lo_pad);
pad_with_zeros(reg_num_pads, reg_dst);
}
mov(reg_ow_count, reg_ow_work_amount);
L(iw_loop);
{
cmp(reg_ow_count, jpp.block_size);
jle(iw_tail, T_NEAR);
gather_src2vmm(vmm, reg_src);
add(reg_src, jpp.SW * jpp.dtype_size * jpp.block_size);
uni_vmovups(ptr[reg_dst], vmm);
add(reg_dst, jpp.dtype_size * jpp.block_size);
sub(reg_ow_count, jpp.block_size);
jmp(iw_loop);
}
L(iw_tail);
{
cmp(reg_ow_count, 0);
jle(iw_exit, T_NEAR);
load_scalar(vmm, ptr[reg_src]);
store_scalar(ptr[reg_dst], vmm);
dec(reg_ow_count);
add(reg_src, jpp.SW * jpp.dtype_size);
add(reg_dst, jpp.dtype_size);
jmp(iw_tail);
}
L(iw_exit);
if (jpp.need_padding) {
mov(reg_num_pads, jpp.OW);
sub(reg_num_pads, reg_w_hi_pad);
pad_with_zeros(reg_num_pads, reg_dst);
}
dec(reg_oh_count);
add(reg_src, reg_src_incr);
jmp(ih_loop, T_NEAR);
}
L(ih_exit);
if (jpp.need_padding) {
mov(reg_num_pads, jpp.OH);
sub(reg_num_pads, reg_h_hi_pad);
mul_by_const(reg_num_pads, reg_aux64, jpp.OW);
pad_with_zeros(reg_num_pads, reg_dst);
}
}
void prepare_table() {
align(64);
L(gather_index_table);
for (int32_t i = 0; i < vlen / sizeof(int32_t); i++)
dd(i * jpp.SW * jpp.dtype_size);
}
};
const std::set<size_t> ExtractImagePatchesImpl::_supported_precisions_sizes = {1, 2, 4, 8};
ExtractImagePatchesImpl::ExtractImagePatchesImpl(const CNNLayer* layer) {
try {
std::string errorPrefix = std::string("Layer ") + layer->type + " with name '" + layer->name + "' ";
if (details::CaselessEq<std::string>()("ExtractImagePatchesLayer", layer->type))
IE_THROW() << errorPrefix << "is not an instance of ExtractImagePatchesLayer class";
REG_FACTORY_FOR(ExtractImagePatchesImpl, ExtractImagePatches);
if (layer->insData.size() != 1 || layer->outData.size() != 1)
IE_THROW() << errorPrefix << "has incorrect number of input or output edges!"
<< " Input: " << layer->insData.size() << "; Output: " << layer->outData.size();
auto inData = layer->insData[0].lock();
if (inData == nullptr)
IE_THROW() << errorPrefix << "has nullable input data";
if (inData->getTensorDesc().getDims().size() != 4)
IE_THROW() << errorPrefix << "must have 4D input tensor. Actual: " << inData->getTensorDesc().getDims().size();
if (layer->outData[0]->getTensorDesc().getDims().size() != 4)
IE_THROW() << errorPrefix << "must have 4D output tensor. Actual: " << layer->outData[0]->getTensorDesc().getDims().size();
if (inData->getLayout() != NCHW)
IE_THROW() << errorPrefix << "has unsupported layout: " << inData->getLayout();
const auto precision = inData->getTensorDesc().getPrecision();
if (_supported_precisions_sizes.find(precision.size()) == _supported_precisions_sizes.end())
IE_THROW() << errorPrefix << "has unsupported precision: " << precision.name();
auto ksizes = layer->GetParamAsUInts("sizes");
auto strides = layer->GetParamAsUInts("strides");
auto rates = layer->GetParamAsUInts("rates");
std::string auto_pad = layer->GetParamAsString("auto_pad");
if (!CaselessEq<std::string>()(auto_pad, "valid")
&& !CaselessEq<std::string>()(auto_pad, "same_upper")
&& !CaselessEq<std::string>()(auto_pad, "same_lower"))
IE_THROW() << errorPrefix << "has unsupported auto_pad value: " << auto_pad;
if (ksizes.size() != 2 || strides.size() != 2 || rates.size() != 2)
IE_THROW() << errorPrefix << "must have the following attributes with shape {2}: sizes, strides, rates.";
_ksizes.clear();
_strides.clear();
_rates.clear();
for (const auto& x : ksizes) {
if (x < 0)
IE_THROW() << "Kernel sizes must be non-negative, got '" << x << "'.";
_ksizes.push_back(static_cast<size_t>(x));
}
for (const auto& x : strides) {
if (x < 0)
IE_THROW() << "Strides must be non-negative, got '" << x << "'.";
_strides.push_back(static_cast<size_t>(x));
}
for (const auto& x : rates) {
if (x < 0)
IE_THROW() << "Rates must be non-negative, got '" << x << "'.";
_rates.push_back(static_cast<size_t>(x));
}
SizeVector in_dims = inData->getTensorDesc().getDims();
_pad_left = 0;
_pad_top = 0;
jit_extract_image_patches_params jpp;
jpp.need_padding = false;
if (!CaselessEq<std::string>()(auto_pad, "valid")) {
const size_t iheight = in_dims[2];
const size_t iwidth = in_dims[3];
const int64_t ihStep = _ksizes[0] + (_rates[0] - 1) * (_ksizes[0] - 1);
const int64_t iwStep = _ksizes[1] + (_rates[1] - 1) * (_ksizes[1] - 1);
int64_t PW = (std::ceil(1.f * iwidth/_strides[1]) - 1) * _strides[1] + iwStep - iwidth;
int64_t PH = (std::ceil(1.f * iheight/_strides[0]) - 1) * _strides[0] + ihStep - iheight;
int64_t increment_sign = 0;
if (CaselessEq<std::string>()(auto_pad, "same_lower")) {
increment_sign = 1;
} else if (CaselessEq<std::string>()(auto_pad, "same_upper")) {
increment_sign = -1;
}
if ((PW > 0) && (PW < iwStep)) {
_pad_left = static_cast<size_t>((PW + increment_sign * (PW % 2)) / 2);
jpp.need_padding = true;
}
if ((PH > 0) && (PH < ihStep)) {
_pad_top = static_cast<size_t>((PH + increment_sign * (PH % 2)) / 2);
jpp.need_padding = true;
}
}
jpp.IW = in_dims[3];
SizeVector out_dims = layer->outData[0]->getTensorDesc().getDims();
jpp.OH = out_dims[2];
jpp.OW = out_dims[3];
jpp.KH = _ksizes[0];
jpp.KW = _ksizes[1];
jpp.SH = _strides[0];
jpp.SW = _strides[1];
jpp.dtype_size = layer->insData.front().lock()->getPrecision().size();
jpp.block_size = 1;
if (mayiuse(x64::avx512_common)) {
jpp.block_size = cpu_isa_traits<x64::avx512_common>::vlen / jpp.dtype_size;
extract_image_patches_kernel.reset(new jit_extract_image_patches_kernel<x64::avx512_common>(jpp));
} else if (mayiuse(x64::avx2)) {
jpp.block_size = cpu_isa_traits<x64::avx2>::vlen / jpp.dtype_size;
extract_image_patches_kernel.reset(new jit_extract_image_patches_kernel<x64::avx2>(jpp));
} else if (mayiuse(x64::sse41)) {
jpp.block_size = cpu_isa_traits<x64::sse41>::vlen / jpp.dtype_size;
extract_image_patches_kernel.reset(new jit_extract_image_patches_kernel<x64::sse41>(jpp));
}
if (extract_image_patches_kernel)
extract_image_patches_kernel->create_ker();
LayerConfig config;
DataConfig inConfig;
inConfig.desc = inData->getTensorDesc();
config.inConfs.push_back(inConfig);
DataConfig outConfig;
outConfig.desc = layer->outData[0]->getTensorDesc();
outConfig.desc.setPrecision(inConfig.desc.getPrecision());
outConfig.desc.setLayout(inConfig.desc.getLayout());
config.outConfs.push_back(outConfig);
config.dynBatchSupport = false;
confs.push_back(config);
} catch (InferenceEngine::Exception &ex) {
errorMsg = ex.what();
}
}
StatusCode ExtractImagePatchesImpl::execute(std::vector<Blob::Ptr>& inputs, std::vector<Blob::Ptr>& outputs, ResponseDesc *resp) noexcept {
const char *src_data = inputs[0]->cbuffer().as<const char *>() +
inputs[0]->getTensorDesc().getBlockingDesc().getOffsetPadding();
char *dst_data = outputs[0]->buffer().as<char *>() +
outputs[0]->getTensorDesc().getBlockingDesc().getOffsetPadding();
const size_t dtype_size = inputs[0]->getTensorDesc().getPrecision().size();
const auto& inDims = inputs[0]->getTensorDesc().getDims();
const size_t IC = inDims[1];
const size_t IH = inDims[2];
const size_t IW = inDims[3];
const auto& outDims = outputs[0]->getTensorDesc().getDims();
const size_t OB = outDims[0];
const size_t OH = outDims[2];
const size_t OW = outDims[3];
const size_t KH = _ksizes[0], KW = _ksizes[1];
const size_t SH = _strides[0], SW = _strides[1];
const size_t RH = _rates[0], RW = _rates[1];
const size_t PT = _pad_top, PL = _pad_left;
const std::vector<size_t> istrides = inputs[0]->getTensorDesc().getBlockingDesc().getStrides();
const std::vector<size_t> ostrides = outputs[0]->getTensorDesc().getBlockingDesc().getStrides();
const std::vector<size_t> ostrides_partial = {ostrides[0], KW * IC * ostrides[1], IC * ostrides[1], ostrides[1]};
if (extract_image_patches_kernel) {
parallel_for4d(OB, KH, KW, IC, [&](const size_t ob, const size_t kh, const size_t kw, const size_t ic) {
const int64_t ih_start = kh * RH - PT;
const int64_t iw_start = kw * RW - PL;
const size_t ih_lpad = ih_start >= 0 ? 0 : std::ceil(-1.f * ih_start / SH);
const size_t iw_lpad = iw_start >= 0 ? 0 : std::ceil(-1.f * iw_start / SW);
const size_t ih_hpad = std::ceil((IH - 1.f * ih_start) / SH) > OH ? OH : std::ceil((IH - 1.f * ih_start) / SH);
const size_t iw_hpad = std::ceil((IW - 1.f * iw_start) / SW) > OW ? OW : std::ceil((IW - 1.f * iw_start) / SW);
size_t dst_offset = ob * ostrides_partial[0] + kh * ostrides_partial[1] + kw * ostrides_partial[2] + ic * ostrides_partial[3];
size_t src_offset = ob * istrides[0] + ic * istrides[1] + ih_start * istrides[2] + iw_start + ih_lpad * SH * IW;
auto args = jit_extract_image_patches_args();
args.src = src_data + src_offset * dtype_size;
args.dst = dst_data + dst_offset * dtype_size;
args.h_lo_pad = ih_lpad;
args.h_hi_pad = ih_hpad;
args.w_lo_pad = iw_lpad;
args.w_hi_pad = iw_hpad;
(*extract_image_patches_kernel)(&args);
});
} else {
parallel_for4d(OB, KH, KW, IC, [&](const size_t ob, const size_t kh, const size_t kw, const size_t ic) {
const int64_t iw_start = kw * RW - PL;
const int64_t ih_start = kh * RH - PT;
const size_t ih_lpad = ih_start >= 0 ? 0 : std::ceil(- 1.f * ih_start / SH);
const size_t iw_lpad = iw_start >= 0 ? 0 : std::ceil(- 1.f * iw_start / SW);
const size_t ih_hpad = std::ceil((IH - 1.f * ih_start) / SH) > OH ? OH : std::ceil((IH + -1.f * ih_start) / SH);
const size_t iw_hpad = std::ceil((IW - 1.f * iw_start) / SW) > OW ? OW : std::ceil((IW - 1.f * iw_start) / SW);
char *my_dst_ptr = dst_data +
(ob * ostrides_partial[0] + kh * ostrides_partial[1] + kw * ostrides_partial[2] + ic * ostrides_partial[3]) * dtype_size;
const char *my_src_ptr = src_data + (ob * istrides[0] + ic * istrides[1] + ih_start * istrides[2] + iw_start) * dtype_size;
size_t num_bytes_to_set = ih_lpad * OW * dtype_size;
memset(my_dst_ptr, 0, num_bytes_to_set);
my_dst_ptr += num_bytes_to_set;
const char* src_ptr_h_stop = my_src_ptr + ih_hpad * SH * IW * dtype_size;
for (const char *src_h_ptr = my_src_ptr + ih_lpad * SH * IW * dtype_size;
src_h_ptr < src_ptr_h_stop; src_h_ptr += SH * IW * dtype_size) {
num_bytes_to_set = iw_lpad * dtype_size;
memset(my_dst_ptr, 0, num_bytes_to_set);
my_dst_ptr += num_bytes_to_set;
const char* src_ptr_w_stop = src_h_ptr + iw_hpad * SW * dtype_size;
for (const char* src_w_ptr = src_h_ptr + iw_lpad * SW * dtype_size;
src_w_ptr < src_ptr_w_stop; src_w_ptr += SW * dtype_size) {
num_bytes_to_set = dtype_size;
memcpy(my_dst_ptr, src_w_ptr, num_bytes_to_set);
my_dst_ptr += num_bytes_to_set;
}
num_bytes_to_set = (OW - iw_hpad) * dtype_size;
memset(my_dst_ptr, 0, num_bytes_to_set);
my_dst_ptr += num_bytes_to_set;
}
num_bytes_to_set = (OH - ih_hpad) * OW * dtype_size;
memset(my_dst_ptr, 0, num_bytes_to_set);
});
}
return OK;
}
const std::set<size_t> ExtractImagePatchesImpl::_supported_precisions_sizes = {1, 2, 4};
} // namespace Cpu
} // namespace Extensions
inference-engine/src/mkldnn_plugin/nodes/extract_image_patches.hpp
+62
View File
@@ -0,0 +1,62 @@
// Copyright (C) 2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "base.hpp"
#include <vector>
#include <set>
namespace InferenceEngine {
namespace Extensions {
namespace Cpu {
struct jit_extract_image_patches_params {
size_t IW;
size_t OH, OW;
size_t KH, KW;
size_t SH, SW;
size_t dtype_size;
size_t block_size;
bool need_padding;
};
struct jit_extract_image_patches_args {
uint64_t h_lo_pad;
uint64_t h_hi_pad;
uint64_t w_lo_pad;
uint64_t w_hi_pad;
const void* src;
void* dst;
};
struct jit_uni_extract_image_patches_kernel {
void (*ker_)(const jit_extract_image_patches_args *);
void operator()(const jit_extract_image_patches_args *args) { assert(ker_); ker_(args); }
jit_extract_image_patches_params jpp;
virtual void create_ker() = 0;
explicit jit_uni_extract_image_patches_kernel(jit_extract_image_patches_params jpp) : ker_(nullptr), jpp(jpp) {}
virtual ~jit_uni_extract_image_patches_kernel() {}
};
class ExtractImagePatchesImpl : public ExtLayerBase {
public:
explicit ExtractImagePatchesImpl(const CNNLayer*);
StatusCode execute(std::vector<Blob::Ptr>&, std::vector<Blob::Ptr>&, ResponseDesc*) noexcept override;
private:
std::vector<size_t> _ksizes;
std::vector<size_t> _strides;
std::vector<size_t> _rates;
size_t _pad_left;
size_t _pad_top;
std::shared_ptr<jit_uni_extract_image_patches_kernel> extract_image_patches_kernel;
static const std::set<size_t> _supported_precisions_sizes;
};
REG_FACTORY_FOR(ExtractImagePatchesImpl, ExtractImagePatches);
} // namespace Cpu
} // namespace Extensions
} // namespace InferenceEngine
inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/extract_image_patches.cpp
+4 -11
View File
@@ -1,4 +1,4 @@
// Copyright (C) 2018-2021 Intel Corporation
// Copyright (C) 2020-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
@@ -19,19 +19,12 @@ const std::vector<PadType> autoPads = {PadType::VALID, PadType::SAME_UPPER, PadT
const std::vector<InferenceEngine::Precision> netPrecisions = {
InferenceEngine::Precision::I8,
InferenceEngine::Precision::U8,
InferenceEngine::Precision::I16,
InferenceEngine::Precision::BF16,
InferenceEngine::Precision::I32,
InferenceEngine::Precision::FP32
InferenceEngine::Precision::FP32,
InferenceEngine::Precision::I64
};
const auto extractImagePatchesParamsSet = ::testing::Combine(
::testing::ValuesIn(inDataShape),
::testing::ValuesIn(kernels),
::testing::ValuesIn(strides),
::testing::ValuesIn(rates),
::testing::ValuesIn(autoPads)
);
INSTANTIATE_TEST_CASE_P(smoke_layers_CPU, ExtractImagePatchesTest,
::testing::Combine(
::testing::ValuesIn(inDataShape),
inference-engine/tests/functional/plugin/cpu/single_layer_tests/extract_image_patches.cpp
Executable
+92
View File
@@ -0,0 +1,92 @@
// Copyright (C) 2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "test_utils/cpu_test_utils.hpp"
#include "shared_test_classes/base/layer_test_utils.hpp"
#include "ngraph_functions/utils/ngraph_helpers.hpp"
#include <shared_test_classes/single_layer/extract_image_patches.hpp>
using namespace InferenceEngine;
using namespace CPUTestUtils;
namespace CPULayerTestsDefinitions {
using LayerTestsDefinitions::extractImagePatchesTuple;
typedef std::tuple<
extractImagePatchesTuple,
CPUSpecificParams> extractImagePatchesCPUTestParamsSet;
class ExtractImagePatchesLayerCPUTest : public testing::WithParamInterface<extractImagePatchesCPUTestParamsSet>,
virtual public LayerTestsUtils::LayerTestsCommon, public CPUTestsBase {
public:
static std::string getTestCaseName(testing::TestParamInfo<extractImagePatchesCPUTestParamsSet> obj) {
extractImagePatchesTuple basicParamsSet;
CPUSpecificParams cpuParams;
std::tie(basicParamsSet, cpuParams) = obj.param;
std::ostringstream result;
result << LayerTestsDefinitions::ExtractImagePatchesTest::getTestCaseName(testing::TestParamInfo<extractImagePatchesTuple>(
basicParamsSet, 0));
result << CPUTestsBase::getTestCaseName(cpuParams);
return result.str();
}
protected:
void SetUp() override {
extractImagePatchesTuple basicParamsSet;
CPUSpecificParams cpuParams;
std::tie(basicParamsSet, cpuParams) = this->GetParam();
std::tie(inFmts, outFmts, priority, selectedType) = cpuParams;
std::vector<size_t> inputShape, kernel, strides, rates;
ngraph::op::PadType pad_type;
InferenceEngine::Precision netPrecision;
std::tie(inputShape, kernel, strides, rates, pad_type, netPrecision, inPrc, outPrc, inLayout, targetDevice) = basicParamsSet;
selectedType = std::string("unknown_") + netPrecision.name();
auto ngPrc = FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc(netPrecision);
auto inputNode = std::make_shared<ngraph::opset6::Parameter>(ngPrc, ngraph::Shape(inputShape));
ngraph::ParameterVector params = {inputNode};
auto extImgPatches = std::make_shared<ngraph::opset6::ExtractImagePatches>(
inputNode, ngraph::Shape(kernel), ngraph::Strides(strides), ngraph::Shape(rates), pad_type);
ngraph::ResultVector results{std::make_shared<ngraph::opset6::Result>(extImgPatches)};
function = std::make_shared<ngraph::Function>(results, params, "ExtractImagePatches");
}
};
TEST_P(ExtractImagePatchesLayerCPUTest, CompareWithRefs) {
SKIP_IF_CURRENT_TEST_IS_DISABLED()
Run();
CheckPluginRelatedResults(executableNetwork, "ExtractImagePatches");
}
namespace {
const std::vector<std::vector<size_t>> inShapes = {{2, 3, 13, 37}};
const std::vector<std::vector<size_t>> kSizes = {{1, 5}, {3, 4}, {3, 1}};
const std::vector<std::vector<size_t>> strides = {{1, 2}, {2, 2}, {2, 1}};
const std::vector<std::vector<size_t>> rates = {{1, 3}, {3, 3}, {3, 1}};
const std::vector<ngraph::op::PadType> autoPads = {ngraph::op::PadType::VALID, ngraph::op::PadType::SAME_UPPER, ngraph::op::PadType::SAME_LOWER};
const std::vector<Precision> netPrecision = {Precision::I8, Precision::BF16, Precision::FP32};
const CPUSpecificParams CPUParams = emptyCPUSpec;
const auto Layer_params = ::testing::Combine(
::testing::ValuesIn(inShapes),
::testing::ValuesIn(kSizes),
::testing::ValuesIn(strides),
::testing::ValuesIn(rates),
::testing::ValuesIn(autoPads),
::testing::ValuesIn(netPrecision),
::testing::Values(InferenceEngine::Precision::UNSPECIFIED),
::testing::Values(InferenceEngine::Precision::UNSPECIFIED),
::testing::Values(InferenceEngine::Layout::ANY),
::testing::Values(CommonTestUtils::DEVICE_CPU));
INSTANTIATE_TEST_CASE_P(smoke_ExtractImagePatches_CPU, ExtractImagePatchesLayerCPUTest,
::testing::Combine(Layer_params, ::testing::Values(CPUParams)),
ExtractImagePatchesLayerCPUTest::getTestCaseName);
} // namespace
} // namespace CPULayerTestsDefinitions
inference-engine/tests/functional/shared_test_classes/include/shared_test_classes/single_layer/extract_image_patches.hpp
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (C) 2018-2021 Intel Corporation
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//