[CPU] ExtractImagePatches operation (#575)

This commit is contained in:
Nikolay Shchegolev 2020-05-27 15:46:49 +03:00 committed by GitHub
parent fd1cc08cd8
commit 27e8580b7d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 536 additions and 4 deletions

View File

@ -53,6 +53,7 @@ set(LAYERS
${CMAKE_CURRENT_SOURCE_DIR}/nodes/depth_to_space.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nodes/detectionoutput.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nodes/detectionoutput_onnx.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nodes/extract_image_patches.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nodes/fill.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nodes/gather.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nodes/gather_tree.cpp

View File

@ -0,0 +1,253 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "list.hpp"
#include "base.hpp"
#include "details/caseless.hpp"
#include <cmath>
#include <string>
#include <vector>
#include <cassert>
#include <set>
#include "ie_parallel.hpp"
namespace InferenceEngine {
namespace Extensions {
namespace Cpu {
using details::CaselessEq;
class ExtractImagePatchesImpl : public ExtLayerBase {
public:
explicit ExtractImagePatchesImpl(const CNNLayer* layer) {
try {
std::string errorPrefix = std::string("Layer ") + layer->type + " with name '" + layer->name + "' ";
if (details::CaselessEq<std::string>()("ExtractImagePatchesLayer", layer->type))
THROW_IE_EXCEPTION << errorPrefix << "is not an instance of ExtractImagePatchesLayer class";
if (layer->insData.size() != 1 || layer->outData.size() != 1)
THROW_IE_EXCEPTION << errorPrefix << "has incorrect number of input or output edges!"
<< " Input: " << layer->insData.size() << "; Output: " << layer->outData.size();
auto inData = layer->insData[0].lock();
if (inData == nullptr)
THROW_IE_EXCEPTION << errorPrefix << "has nullable input data";
if (inData->getTensorDesc().getDims().size() != 4)
THROW_IE_EXCEPTION << errorPrefix << "must have 4D input tensor. Actual: " << inData->getTensorDesc().getDims().size();
if (layer->outData[0]->getTensorDesc().getDims().size() != 4)
THROW_IE_EXCEPTION << errorPrefix << "must have 4D output tensor. Actual: " << layer->outData[0]->getTensorDesc().getDims().size();
if (inData->getLayout() != NCHW)
THROW_IE_EXCEPTION << errorPrefix << "has unsupported layout: " << inData->getLayout();
const auto precision = inData->getTensorDesc().getPrecision();
if (_supported_precisions_sizes.find(precision.size()) == _supported_precisions_sizes.end())
THROW_IE_EXCEPTION << errorPrefix << "has unsupported precision: " << precision.name();
auto ksizes = layer->GetParamAsUInts("sizes");
auto strides = layer->GetParamAsUInts("strides");
auto rates = layer->GetParamAsUInts("rates");
_auto_pad = layer->GetParamAsString("auto_pad");
if (!CaselessEq<std::string>()(_auto_pad, "valid")
&& !CaselessEq<std::string>()(_auto_pad, "same_upper")
&& !CaselessEq<std::string>()(_auto_pad, "same_lower"))
THROW_IE_EXCEPTION << errorPrefix << "has unsupported auto_pad value: " << _auto_pad;
if (ksizes.size() != 2 || strides.size() != 2 || rates.size() != 2)
THROW_IE_EXCEPTION << errorPrefix << "must have the following attributes with shape {2}: sizes, strides, rates.";
_ksizes.clear();
_strides.clear();
_rates.clear();
for (size_t i = 0; i < ksizes.size(); i++)
_ksizes.push_back((int64_t)ksizes[i]);
for (size_t i = 0; i < strides.size(); i++)
_strides.push_back((int64_t)strides[i]);
for (size_t i = 0; i < rates.size(); i++)
_rates.push_back((int64_t)rates[i]);
LayerConfig config;
DataConfig inConfig;
inConfig.desc = inData->getTensorDesc();
config.inConfs.push_back(inConfig);
DataConfig outConfig;
outConfig.desc = layer->outData[0]->getTensorDesc();
outConfig.desc.setPrecision(inConfig.desc.getPrecision());
outConfig.desc.setLayout(inConfig.desc.getLayout());
config.outConfs.push_back(outConfig);
config.dynBatchSupport = false;
confs.push_back(config);
} catch (InferenceEngine::details::InferenceEngineException &ex) {
errorMsg = ex.what();
}
}
StatusCode execute(std::vector<Blob::Ptr>& inputs, std::vector<Blob::Ptr>& outputs, ResponseDesc *resp) noexcept override {
switch (inputs[0]->getTensorDesc().getPrecision().size()) {
case 1: {
process_data<PrecisionTrait<Precision::U8>::value_type>(inputs, outputs);
break;
}
case 2: {
process_data<PrecisionTrait<Precision::U16>::value_type>(inputs, outputs);
break;
}
case 4: {
process_data<PrecisionTrait<Precision::I32>::value_type>(inputs, outputs);
break;
}
case 8: {
process_data<PrecisionTrait<Precision::U64>::value_type>(inputs, outputs);
break;
}
default: {
if (resp) {
std::string errorMsg = "ExtractImagePatches layer does not support precision '"
+ std::string(inputs[0]->getTensorDesc().getPrecision().name()) + "'";
errorMsg.copy(resp->msg, sizeof(resp->msg) - 1);
}
return GENERAL_ERROR;
}
}
return OK;
}
template<typename T>
void process_data(std::vector<Blob::Ptr>& inputs, std::vector<Blob::Ptr>& outputs) noexcept {
const T* src_data = inputs[0]->cbuffer().as<const T*>() +
inputs[0]->getTensorDesc().getBlockingDesc().getOffsetPadding();
T* dst_data = outputs[0]->buffer().as<T*>() +
outputs[0]->getTensorDesc().getBlockingDesc().getOffsetPadding();
const auto& inDims = inputs[0]->getTensorDesc().getDims();
const size_t inDimsSize = inDims.size();
const size_t BATCH = 0, CHANNEL = 1, HIGHT = 0, WIDTH = 1;
const int64_t IB = inDims[BATCH];
const int64_t IC = inDims[CHANNEL];
const int64_t IH = inDims[inDimsSize - 2];
const int64_t IW = inDims[inDimsSize - 1];
const auto& outDims = outputs[0]->getTensorDesc().getDims();
const size_t outDimsSize = outDims.size();
const int64_t OB = outDims[BATCH];
const int64_t OC = outDims[CHANNEL];
const int64_t OH = outDims[outDimsSize - 2];
const int64_t OW = outDims[outDimsSize - 1];
const int64_t KH = _ksizes[HIGHT];
const int64_t KW = _ksizes[WIDTH];
const int64_t SH = _strides[HIGHT];
const int64_t SW = _strides[WIDTH];
const int64_t RH = _rates[HIGHT];
const int64_t RW = _rates[WIDTH];
int64_t ihStart = 0;
int64_t iwStart = 0;
int64_t iwStep = KW + (RW - 1) * (KW - 1);
int64_t ihStep = KH + (RH - 1) * (KH - 1);
int64_t PL = 0, PT = 0;
if (!CaselessEq<std::string>()(_auto_pad, "valid")) {
int64_t PW = (std::ceil(1.f * IW/SW) - 1) * SW + iwStep - IW;
int64_t PH = (std::ceil(1.f * IH/SH) - 1) * SH + ihStep - IH;
if ((PW > 0) && (PW < iwStep)) {
if (PW % 2 == 1) {
if (CaselessEq<std::string>()(_auto_pad, "same_lower")) {
PL = (PW + 1) / 2;
} else if (CaselessEq<std::string>()(_auto_pad, "same_upper")) {
PL = (PW - 1) / 2;
}
} else {
PL = PW / 2;
}
}
if ((PH > 0) && (PH < ihStep)) {
if (PH % 2 == 1) {
if (CaselessEq<std::string>()(_auto_pad, "same_lower")) {
PT = (PH + 1) / 2;
} else if (CaselessEq<std::string>()(_auto_pad, "same_upper")) {
PT = (PH - 1) / 2;
}
} else {
PT = PH / 2;
}
}
}
const int64_t OH_OW = OH * OW;
const int64_t OC_OH_OW = OC * OH_OW;
const int64_t IH_IW = IH * IW;
const int64_t IC_IH_IW = IC * IH_IW;
const int64_t KH_KW = KH * KW;
const int64_t work_amount = OB;
auto thread_body = [&](const int ithr, const int nthr) {
int64_t start(0lu), end(0lu);
splitter(work_amount, nthr, ithr, start, end);
if (start >= end)
return;
for (int64_t ob = start; ob < end; ob++) {
const int64_t ibICIHIW = ob * IC_IH_IW;
const int64_t obOCOHOW = ob * OC_OH_OW;
for (int64_t oh = 0; oh < OH; oh++) {
const int64_t obOCOHOWohOW = obOCOHOW + oh * OW;
int64_t ih0 = oh * SH - PT;
for (int64_t ow = 0; ow < OW; ow++) {
const int64_t obOCOHOWohOWow = obOCOHOWohOW + ow;
int64_t iw0 = ow * SW - PL;
int64_t oc = 0;
for (int64_t kh = 0; kh < KH; kh++) {
int64_t ihKH = ih0 + kh * RH;
int64_t ibICIHIWihFHIW = ibICIHIW + ihKH * IW;
for (int64_t kw = 0; kw < KW; kw++) {
for (int64_t ic = 0; ic < IC; ic++, oc++) {
int64_t iwKW = iw0 + kw * RW;
int64_t dst_idx = obOCOHOWohOWow + oc * OH_OW;
if (ihKH < 0 || ihKH >= IH || iwKW < 0 || iwKW >= IW) {
dst_data[dst_idx] = T(0);
} else {
int64_t src_idx = ibICIHIWihFHIW + ic * IH_IW + iwKW;
dst_data[dst_idx] = src_data[src_idx];
}
}
}
}
}
}
}
};
parallel_nt(0, thread_body);
}
private:
std::vector<int64_t> _ksizes;
std::vector<int64_t> _strides;
std::vector<int64_t> _rates;
std::string _auto_pad;
static const std::set<size_t> _supported_precisions_sizes;
};
const std::set<size_t> ExtractImagePatchesImpl::_supported_precisions_sizes = {1, 2, 4, 8};
REG_FACTORY_FOR(ExtractImagePatchesImpl, ExtractImagePatches);
} // namespace Cpu
} // namespace Extensions
} // namespace InferenceEngine

View File

@ -32,6 +32,7 @@ MKLDNN_EXTENSION_NODE(MathImpl, Softplus);
MKLDNN_EXTENSION_NODE(MathImpl, Softsign);
MKLDNN_EXTENSION_NODE(MathImpl, Tan);
MKLDNN_EXTENSION_NODE(ExperimentalDetectronTopKROIsImpl, ExperimentalDetectronTopKROIs);
MKLDNN_EXTENSION_NODE(ExtractImagePatchesImpl, ExtractImagePatches);
MKLDNN_EXTENSION_NODE(ReverseSequenceImpl, ReverseSequence);
MKLDNN_EXTENSION_NODE(DetectionOutputImpl, DetectionOutput);
MKLDNN_EXTENSION_NODE(ArgMaxImpl, ArgMax);

View File

@ -0,0 +1,46 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <vector>
#include "single_layer_tests/extract_image_patches.hpp"
using namespace LayerTestsDefinitions;
using ngraph::op::PadType;
namespace {
const std::vector<std::vector<size_t>> inDataShape = {{1, 1, 10, 10}, {1, 3, 10, 10}};
const std::vector<std::vector<size_t>> kernels = {{2, 2}, {3, 3}, {4, 4}, {1, 3}, {4, 2}};
const std::vector<std::vector<size_t>> strides = {{3, 3}, {5, 5}, {9, 9}, {1, 3}, {6, 2}};
const std::vector<std::vector<size_t>> rates = {{1, 1}, {1, 2}, {2, 1}, {2, 2}};
const std::vector<PadType> autoPads = {PadType::VALID, PadType::SAME_UPPER, PadType::SAME_LOWER};
const std::vector<InferenceEngine::Precision> netPrecisions = {
InferenceEngine::Precision::I8,
InferenceEngine::Precision::U8,
InferenceEngine::Precision::I16,
InferenceEngine::Precision::I32,
InferenceEngine::Precision::FP32
};
const auto extractImagePatchesParamsSet = ::testing::Combine(
::testing::ValuesIn(inDataShape),
::testing::ValuesIn(kernels),
::testing::ValuesIn(strides),
::testing::ValuesIn(rates),
::testing::ValuesIn(autoPads)
);
INSTANTIATE_TEST_CASE_P(layers_CPU, ExtractImagePatchesTest,
::testing::Combine(
::testing::ValuesIn(inDataShape),
::testing::ValuesIn(kernels),
::testing::ValuesIn(strides),
::testing::ValuesIn(rates),
::testing::ValuesIn(autoPads),
::testing::ValuesIn(netPrecisions),
::testing::Values(CommonTestUtils::DEVICE_CPU)),
ExtractImagePatchesTest::getTestCaseName);
} // namespace

View File

@ -0,0 +1,34 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <memory>
#include <string>
#include <tuple>
#include <vector>
#include "functional_test_utils/layer_test_utils.hpp"
namespace LayerTestsDefinitions {
using extractImagePatchesTuple = typename std::tuple<
std::vector<size_t>, // input shape
std::vector<size_t>, // kernel size
std::vector<size_t>, // strides
std::vector<size_t>, // rates
ngraph::op::PadType, // pad type
InferenceEngine::Precision, // Network precision
LayerTestsUtils::TargetDevice>; // Device name
class ExtractImagePatchesTest : public testing::WithParamInterface<extractImagePatchesTuple>,
public LayerTestsUtils::LayerTestsCommon {
public:
static std::string getTestCaseName(const testing::TestParamInfo<extractImagePatchesTuple> &obj);
protected:
void SetUp() override;
};
} // namespace LayerTestsDefinitions

View File

@ -0,0 +1,57 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <algorithm>
#include <functional>
#include <memory>
#include <string>
#include <tuple>
#include <vector>
#include "single_layer_tests/extract_image_patches.hpp"
#include "functional_test_utils/layer_test_utils.hpp"
#include "ngraph_functions/builders.hpp"
namespace LayerTestsDefinitions {
std::string ExtractImagePatchesTest::getTestCaseName(const testing::TestParamInfo<extractImagePatchesTuple> &obj) {
std::vector<size_t> inputShape, kernel, strides, rates;
ngraph::op::PadType pad_type;
InferenceEngine::Precision netPrc;
std::string targetName;
std::tie(inputShape, kernel, strides, rates, pad_type, netPrc, targetName) = obj.param;
std::ostringstream result;
result << "IS=" << CommonTestUtils::vec2str(inputShape) << "_";
result << "netPRC=" << netPrc.name() << "_";
result << "K=" << CommonTestUtils::vec2str(kernel) << "_";
result << "S=" << CommonTestUtils::vec2str(strides) << "_";
result << "R=" << CommonTestUtils::vec2str(rates) << "_";
result << "P=" << pad_type << "_";
result << "targetDevice=" << targetName;
return result.str();
}
void ExtractImagePatchesTest::SetUp() {
std::vector<size_t> inputShape, kernel, strides, rates;
ngraph::op::PadType pad_type;
InferenceEngine::Precision netPrecision;
std::tie(inputShape, kernel, strides, rates, pad_type, netPrecision, targetDevice) = this->GetParam();
auto ngPrc = FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc(netPrecision);
auto inputNode = std::make_shared<ngraph::opset1::Parameter>(ngPrc, ngraph::Shape(inputShape));
ngraph::ParameterVector params = {inputNode};
auto extImgPatches = std::make_shared<ngraph::opset3::ExtractImagePatches>(
inputNode, ngraph::Shape(kernel), ngraph::Strides(strides), ngraph::Shape(rates), pad_type);
ngraph::ResultVector results{std::make_shared<ngraph::opset1::Result>(extImgPatches)};
function = std::make_shared<ngraph::Function>(results, params, "ExtractImagePatches");
}
TEST_P(ExtractImagePatchesTest, CompareWithRefs) {
Run();
};
} // namespace LayerTestsDefinitions

View File

@ -42,10 +42,6 @@ void op::v3::ExtractImagePatches::validate_and_infer_types()
{
const PartialShape input_Pshape = get_input_partial_shape(0);
NODE_VALIDATION_CHECK(this,
get_input_element_type(0).is_dynamic() ||
get_input_element_type(0).is_integral_number(),
"input tensor must be an integral number.");
NODE_VALIDATION_CHECK(this, input_Pshape.rank() == 4, "input tensor must be 4D tensor.");
NODE_VALIDATION_CHECK(this,

View File

@ -0,0 +1,122 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "ngraph/shape_util.hpp"
namespace ngraph {
namespace runtime {
namespace reference {
template <typename T, typename U>
void extractImagePatches(
const op::ExtractImagePatches* extImgPatches,
const T* input,
T* out,
const Shape& inShape,
const Shape& outShape) {
const size_t dimsSize = inShape.size();
const size_t BATCH = 0, CHANNEL = 1,
HIGHT = 0, WIDTH = 1;
const int64_t KH = extImgPatches->get_sizes()[HIGHT];
const int64_t KW = extImgPatches->get_sizes()[WIDTH];
const int64_t SH = extImgPatches->get_strides()[HIGHT];
const int64_t SW = extImgPatches->get_strides()[WIDTH];
const int64_t RH = extImgPatches->get_rates()[HIGHT];
const int64_t RW = extImgPatches->get_rates()[WIDTH];
const auto auto_pad = extImgPatches->get_auto_pad();
const int64_t IB = inShape[BATCH];
const int64_t IC = inShape[CHANNEL];
const int64_t IH = inShape[dimsSize - 2];
const int64_t IW = inShape[dimsSize - 1];
const int64_t OB = outShape[BATCH];
const int64_t OC = outShape[CHANNEL];
const int64_t OH = outShape[dimsSize - 2];
const int64_t OW = outShape[dimsSize - 1];
int64_t ihStart = 0;
int64_t iwStart = 0;
int64_t iwStep = KW + (RW - 1) * (KW - 1);
int64_t ihStep = KH + (RH - 1) * (KH - 1);
const int64_t OH_OW = OH * OW;
const int64_t OC_OH_OW = OC * OH_OW;
const int64_t OB_OC_OH_OW = OC_OH_OW * OB;
const int64_t IH_IW = IH * IW;
const int64_t IC_IH_IW = IC * IH_IW;
const int64_t IB_IC_IH_IW = IC_IH_IW * IB;
const int64_t KH_KW = KH * KW;
int64_t PL = 0, PT = 0;
if (auto_pad != op::PadType::VALID) {
int64_t PW = (std::ceil(1.f * IW/SW) - 1) * SW + iwStep - IW;
int64_t PH = (std::ceil(1.f * IH/SH) - 1) * SH + ihStep - IH;
if ((PW > 0) && (PW < iwStep)) {
if (PW % 2 == 1) {
if (auto_pad == op::PadType::SAME_LOWER) {
PL = (PW + 1) / 2;
} else if (auto_pad == op::PadType::SAME_UPPER) {
PL = (PW - 1) / 2;
}
} else {
PL = PW / 2;
}
}
if ((PH > 0) && (PH < ihStep)) {
if (PH % 2 == 1) {
if (auto_pad == op::PadType::SAME_LOWER) {
PT = (PH + 1) / 2;
} else if (auto_pad == op::PadType::SAME_UPPER) {
PT = (PH - 1) / 2;
}
} else {
PT = PH / 2;
}
}
}
for (int64_t ob = 0; ob < OB; ob++) {
const int64_t ib_ICIHIW = ob * IC_IH_IW;
const int64_t ob_OCOHOW = ob * OC_OH_OW;
for (int64_t oh = 0; oh < OH; oh++) {
const int64_t ob_OCOHOW_ohOW = ob_OCOHOW + oh * OW;
int64_t ih0 = oh * SH - PT;
for (int64_t ow = 0; ow < OW; ow++) {
const int64_t ob_OCOHOW_ohOW_ow = ob_OCOHOW_ohOW + ow;
int64_t iw0 = ow * SW - PL;
int64_t oc = 0;
for (int64_t kh = 0; kh < KH; kh++) {
int64_t ihKH = ih0 + kh * RH;
int64_t ib_ICIHIW_ihKH_IW = ib_ICIHIW + ihKH * IW;
for (int64_t kw = 0; kw < KW; kw++) {
for (int64_t ic = 0; ic < IC; ic++, oc++) {
int64_t iwKW = iw0 + kw * RW;
int64_t dst_idx = ob_OCOHOW_ohOW_ow + oc * OH_OW;
if (dst_idx >= OB_OC_OH_OW)
throw ngraph_error("ExtractImagePatches. Destination index is out of bounds.");
if (ihKH < 0 || ihKH >= IH || iwKW < 0 || iwKW >= IW) {
out[dst_idx] = T(0);
} else {
int64_t src_idx = ib_ICIHIW_ihKH_IW + ic * IH_IW + iwKW;
if (src_idx >= IB_IC_IH_IW)
throw ngraph_error("ExtractImagePatches. Source index is out of bounds.");
out[dst_idx] = input[src_idx];
}
}
}
}
}
}
}
} // extractImagePatches
} // reference
} // runtime
} // ngraph

View File

@ -492,6 +492,15 @@ namespace
EXPECT_FALSE(node.is_binary_elementwise_logical());
}
void op_is_ExtractImagePatches()
{
op::ExtractImagePatches node;
EXPECT_FALSE(node.is_unary_elementwise_arithmetic());
EXPECT_FALSE(node.is_binary_elementwise_arithmetic());
EXPECT_FALSE(node.is_binary_elementwise_comparison());
EXPECT_FALSE(node.is_binary_elementwise_logical());
}
void op_is_FakeQuantize()
{
op::FakeQuantize node;

View File

@ -57,6 +57,7 @@
#include "ngraph/runtime/reference/embedding_lookup.hpp"
#include "ngraph/runtime/reference/erf.hpp"
#include "ngraph/runtime/reference/exp.hpp"
#include "ngraph/runtime/reference/extract_image_patches.hpp"
#include "ngraph/runtime/reference/floor.hpp"
#include "ngraph/runtime/reference/gather.hpp"
#include "ngraph/runtime/reference/gather_nd.hpp"
@ -739,6 +740,17 @@ protected:
args[0]->get_data_ptr<const T>(), out[0]->get_data_ptr<T>(), element_count);
break;
}
case OP_TYPEID::ExtractImagePatches_v3:
{
const op::ExtractImagePatches* extImgPatches = static_cast<const op::ExtractImagePatches*>(&node);
reference::extractImagePatches<T, size_t>(
extImgPatches,
args[0]->get_data_ptr<const T>(),
out[0]->get_data_ptr<T>(),
extImgPatches->get_input_shape(0),
extImgPatches->get_shape());
break;
}
case OP_TYPEID::Exp:
{
size_t element_count = shape_size(node.get_output_shape(0));

View File

@ -27,6 +27,7 @@ NGRAPH_OP(LogicalNot, op::v1)
#undef ID_SUFFIX
#define ID_SUFFIX(NAME) NAME##_v3
NGRAPH_OP(ExtractImagePatches, op::v3)
NGRAPH_OP(ShapeOf, op::v3)
NGRAPH_OP(NonZero, op::v3)
#undef ID_SUFFIX