From f3ac97e9f453af65ef450e010647c1afaf0d4c86 Mon Sep 17 00:00:00 2001 From: Andrey Sokolov Date: Tue, 3 Nov 2020 11:56:55 +0300 Subject: [PATCH] [IE][VPU]: Interpolate - reuse "interp", "resample" layers (#2932) Reuse existing "interp", "resample" layers task: #-29955 --- .../include/vpu/frontend/frontend.hpp | 1 + .../include/vpu/stage_builder.hpp | 17 +++ .../src/frontend/frontend.cpp | 1 + .../graph_transformer/src/stages/interp.cpp | 26 +++- .../src/stages/interpolate.cpp | 85 ++++++++++++ .../graph_transformer/src/stages/resample.cpp | 41 ++++-- .../single_layer_tests/interpolate.cpp | 129 ++++++++++++++++++ 7 files changed, 285 insertions(+), 15 deletions(-) create mode 100644 inference-engine/src/vpu/graph_transformer/src/stages/interpolate.cpp create mode 100644 inference-engine/tests/functional/plugin/myriad/shared_tests_instances/single_layer_tests/interpolate.cpp diff --git a/inference-engine/src/vpu/graph_transformer/include/vpu/frontend/frontend.hpp b/inference-engine/src/vpu/graph_transformer/include/vpu/frontend/frontend.hpp index 2440941c8d4..0b85b299df3 100644 --- a/inference-engine/src/vpu/graph_transformer/include/vpu/frontend/frontend.hpp +++ b/inference-engine/src/vpu/graph_transformer/include/vpu/frontend/frontend.hpp @@ -125,6 +125,7 @@ public: void parseMTCNN(const Model& model, const ie::CNNLayerPtr& layer, const DataVector& inputs, const DataVector& outputs) const; void parsePad(const Model& model, const ie::CNNLayerPtr& layer, const DataVector& inputs, const DataVector& outputs) const; void parseResample(const Model& model, const ie::CNNLayerPtr& layer, const DataVector& inputs, const DataVector& outputs) const; + void parseInterpolate(const Model& model, const ie::CNNLayerPtr& layer, const DataVector& inputs, const DataVector& outputs) const; void parseRNN(const Model& model, const ie::CNNLayerPtr& layer, const DataVector& inputs, const DataVector& outputs) const; void parseGEMM(const Model& model, const ie::CNNLayerPtr& layer, const DataVector& inputs, const DataVector& outputs) const; void parseLog(const Model& model, const ie::CNNLayerPtr& layer, const DataVector& inputs, const DataVector& outputs) const; diff --git a/inference-engine/src/vpu/graph_transformer/include/vpu/stage_builder.hpp b/inference-engine/src/vpu/graph_transformer/include/vpu/stage_builder.hpp index b463f4acf82..8ca5898a0d0 100644 --- a/inference-engine/src/vpu/graph_transformer/include/vpu/stage_builder.hpp +++ b/inference-engine/src/vpu/graph_transformer/include/vpu/stage_builder.hpp @@ -321,6 +321,23 @@ public: const Data& indices, const Data& output, int32_t batch_dims); + + Stage addInterpStage( + const Model& model, + const std::string& name, + const ie::CNNLayerPtr& layer, + bool align_corners, + const Data& input, + const Data& output); + + Stage addResampleNearestStage( + const Model& model, + const std::string& name, + const ie::CNNLayerPtr& layer, + bool antialias, + float factor, + const Data& input, + const Data& output); }; } // namespace vpu diff --git a/inference-engine/src/vpu/graph_transformer/src/frontend/frontend.cpp b/inference-engine/src/vpu/graph_transformer/src/frontend/frontend.cpp index e885a7b542f..96c49223e41 100644 --- a/inference-engine/src/vpu/graph_transformer/src/frontend/frontend.cpp +++ b/inference-engine/src/vpu/graph_transformer/src/frontend/frontend.cpp @@ -86,6 +86,7 @@ FrontEnd::FrontEnd(StageBuilder::Ptr stageBuilder, const ie::ICore* core) {"ROIPooling", LAYER_PARSER(parseROIPooling)}, {"PSROIPooling", LAYER_PARSER(parsePSROIPooling)}, {"Interp", LAYER_PARSER(parseInterp)}, + {"Interpolate", LAYER_PARSER(parseInterpolate)}, {"Custom", LAYER_PARSER(parseCustom)}, {"MTCNN", LAYER_PARSER(parseMTCNN)}, {"LSTMCell", LAYER_PARSER(parseLSTMCell)}, diff --git a/inference-engine/src/vpu/graph_transformer/src/stages/interp.cpp b/inference-engine/src/vpu/graph_transformer/src/stages/interp.cpp index 144412ae4eb..280c2c5bc94 100644 --- a/inference-engine/src/vpu/graph_transformer/src/stages/interp.cpp +++ b/inference-engine/src/vpu/graph_transformer/src/stages/interp.cpp @@ -60,12 +60,28 @@ private: } // namespace -void FrontEnd::parseInterp(const Model& model, const ie::CNNLayerPtr& layer, const DataVector& inputs, const DataVector& outputs) const { - IE_ASSERT(inputs.size() == 1); - IE_ASSERT(outputs.size() == 1); +Stage StageBuilder::addInterpStage( + const Model& model, + const std::string& name, + const ie::CNNLayerPtr& layer, + bool align_corners, + const Data& input, + const Data& output) { + auto stage = model->addNewStage(layer->name, StageType::Interp, layer, {input}, {output}); + stage->attrs().set("align_corners", align_corners); - auto stage = model->addNewStage(layer->name, StageType::Interp, layer, inputs, outputs); - stage->attrs().set("align_corners", layer->GetParamAsInt("align_corners", 0)); + return stage; +} + +void FrontEnd::parseInterp(const Model& model, const ie::CNNLayerPtr& layer, const DataVector& inputs, const DataVector& outputs) const { + VPU_THROW_UNLESS(inputs.size() == 1, + "Interp stage with name {} must have only 1 input, " + "actually provided {}", layer->name, inputs.size()); + VPU_THROW_UNLESS(outputs.size() == 1, + "Interp stage with name {} must have only 1 output, " + "actually provided {}", layer->name, outputs.size()); + + _stageBuilder->addInterpStage(model, layer->name, layer, layer->GetParamAsInt("align_corners", 0), inputs[0], outputs[0]); } } // namespace vpu diff --git a/inference-engine/src/vpu/graph_transformer/src/stages/interpolate.cpp b/inference-engine/src/vpu/graph_transformer/src/stages/interpolate.cpp new file mode 100644 index 00000000000..158834df384 --- /dev/null +++ b/inference-engine/src/vpu/graph_transformer/src/stages/interpolate.cpp @@ -0,0 +1,85 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace InferenceEngine; + +namespace vpu { + +void FrontEnd::parseInterpolate(const Model& model, const ie::CNNLayerPtr& _layer, const DataVector& inputs, const DataVector& outputs) const { + VPU_THROW_UNLESS(inputs.size() <= 4 && inputs.size() >= 1, + "Interpolate stage with name {} must have no more than 4 inputs and no less than 1 input, actually provided {} inputs", + _layer->name, inputs.size()); + + VPU_THROW_UNLESS(outputs.size() == 1, + "Interpolate stage with name {} must have only 1 output, actually provided {} outputs", + _layer->name, outputs.size()); + + const auto interpolateMode = _layer->GetParamAsString("mode"); + + const auto input = inputs[0]; + const auto output = outputs[0]; + + // try to use existing resize layers + if (input->desc().dimsOrder() == DimsOrder::NCHW || input->desc().dimsOrder() == DimsOrder::NHWC || + input->desc().dimsOrder() == DimsOrder::CHW || input->desc().dimsOrder() == DimsOrder::HWC) { + const auto ic = input->desc().dim(Dim::C); + const auto in = (input->desc().numDims() == 3) ? 1 : input->desc().dim(Dim::N); + + const auto oc = output->desc().dim(Dim::C); + const auto on = (output->desc().numDims() == 3) ? 1 : output->desc().dim(Dim::N); + + auto padsBegin = _layer->GetParamAsInts("pads_begin", {}); + auto padsEnd = _layer->GetParamAsInts("pads_end", {}); + + const auto isPadZeros = [](const std::vector& pad) { + return std::all_of(pad.begin(), pad.end(), [](int i) { return i == 0; }); + }; + + if (ic == oc && in == 1 && on == 1 && isPadZeros(padsBegin) && isPadZeros(padsEnd)) { + ie::details::CaselessEq cmp; + if (cmp(interpolateMode, "nearest")) { + // current "Resample" supports the following "Interpolate" modes only: + // coordinate_transformation_mode = half_pixel; nearest_mode = round_prefer_ceil; + // other "Interpolate" modes are translated to the default ones + const auto antialias = _layer->GetParamAsBool("antialias", false); + _stageBuilder->addResampleNearestStage(model, + _layer->name, + _layer, + antialias, + -1.0f, + input, + output); + } else if (cmp(interpolateMode, "linear")) { + // current "Interp" supports modes "align_corners" and "asymmetric" only + // other "Interpolate" modes are translated to the default ones + const auto coordinate_transformation_mode = _layer->GetParamAsString("coordinate_transformation_mode", "half_pixel"); + + _stageBuilder->addInterpStage(model, + _layer->name, + _layer, + cmp(coordinate_transformation_mode, "align_corners"), + input, + output); + } else { + VPU_THROW_EXCEPTION << "Current Interpolate supports 'nearest' and 'linear' modes only; layer name = " << _layer->name; + } + } else { + VPU_THROW_EXCEPTION << "Current Interpolate does not support paddings, batches, and resize by channels; layer name = " << _layer->name; + } + } else { + VPU_THROW_EXCEPTION << "Current Interpolate supports (N)HWC, (N)CHW data orders only; layer name = " << _layer->name; + } +} + +} // namespace vpu diff --git a/inference-engine/src/vpu/graph_transformer/src/stages/resample.cpp b/inference-engine/src/vpu/graph_transformer/src/stages/resample.cpp index 9f947313b1a..d4206b5c937 100644 --- a/inference-engine/src/vpu/graph_transformer/src/stages/resample.cpp +++ b/inference-engine/src/vpu/graph_transformer/src/stages/resample.cpp @@ -71,20 +71,41 @@ private: } // namespace +Stage StageBuilder::addResampleNearestStage( + const Model& model, + const std::string& name, + const ie::CNNLayerPtr& layer, + bool antialias, + float factor, + const Data& input, + const Data& output) { + auto stage = model->addNewStage(layer->name, StageType::Resample, layer, {input}, {output}); + + stage->attrs().set("antialias", antialias); + stage->attrs().set("factor", factor); + stage->attrs().set("type", ResampleType::Nearest); + + return stage; +} + void FrontEnd::parseResample(const Model& model, const ie::CNNLayerPtr& layer, const DataVector& inputs, const DataVector& outputs) const { - IE_ASSERT(inputs.size() == 1); - IE_ASSERT(outputs.size() == 1); + VPU_THROW_UNLESS(inputs.size() == 1, + "Resample stage with name {} must have only 1 input, " + "actually provided {}", layer->name, inputs.size()); + VPU_THROW_UNLESS(outputs.size() == 1, + "Resample stage with name {} must have only 1 output, " + "actually provided {}", layer->name, outputs.size()); ie::details::CaselessEq cmp; - - auto stage = model->addNewStage(layer->name, StageType::Resample, layer, inputs, outputs); - - stage->attrs().set("antialias", layer->GetParamAsInt("antialias", 0)); - stage->attrs().set("factor", layer->GetParamAsFloat("factor", -1.0f)); - - auto method = layer->GetParamAsString("type", "caffe.ResampleParameter.NEAREST"); + const auto method = layer->GetParamAsString("type", "caffe.ResampleParameter.NEAREST"); if (cmp(method, "caffe.ResampleParameter.NEAREST")) { - stage->attrs().set("type", ResampleType::Nearest); + _stageBuilder->addResampleNearestStage(model, + layer->name, + layer, + layer->GetParamAsInt("antialias", 0), + layer->GetParamAsFloat("factor", -1), + inputs[0], + outputs[0]); } else { VPU_THROW_EXCEPTION << "Layer with name " << layer->name << " supports only caffe.ResampleParameter.NEAREST resample type"; } diff --git a/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/single_layer_tests/interpolate.cpp b/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/single_layer_tests/interpolate.cpp new file mode 100644 index 00000000000..f3f549135f0 --- /dev/null +++ b/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/single_layer_tests/interpolate.cpp @@ -0,0 +1,129 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "single_layer_tests/interpolate.hpp" +#include "common_test_utils/test_constants.hpp" +#include +#include "common/myriad_common_test_utils.hpp" +#include + +using namespace LayerTestsDefinitions; + +namespace { + +const std::vector netPrecisions = { + InferenceEngine::Precision::FP16, +}; + +const std::vector> inShapes = { + {1, 8, 38, 38}, +}; + +const std::vector> targetShapes = { + {1, 8, 38 * 2, 38 * 2}, +}; + +const std::vector modesWithoutNearest = { + ngraph::op::v4::Interpolate::InterpolateMode::linear, +}; + +const std::vector nearestMode = { + ngraph::op::v4::Interpolate::InterpolateMode::nearest, +}; + +const std::vector coordinateTransformModesNearest = { + ngraph::op::v4::Interpolate::CoordinateTransformMode::half_pixel, +}; + +const std::vector coordinateTransformModesWithoutNearest = { + ngraph::op::v4::Interpolate::CoordinateTransformMode::asymmetric, + ngraph::op::v4::Interpolate::CoordinateTransformMode::align_corners, +}; + +const std::vector nearestModes = { + ngraph::op::v4::Interpolate::NearestMode::simple, + ngraph::op::v4::Interpolate::NearestMode::round_prefer_floor, + ngraph::op::v4::Interpolate::NearestMode::floor, + ngraph::op::v4::Interpolate::NearestMode::ceil, + ngraph::op::v4::Interpolate::NearestMode::round_prefer_ceil, +}; + +const std::vector defaultNearestMode = { + ngraph::op::v4::Interpolate::NearestMode::round_prefer_ceil, +}; + +const std::vector> pads = { + {0, 0, 0, 0}, +}; + +const std::vector antialias = { + false, +}; + +const std::vector cubeCoefs = { + -0.75f, +}; + +const std::vector> defaultAxes = { + {0, 1, 2, 3} +}; + +const std::vector> defaultScales = { + {1.f, 1.f, 2.f, 2.f} +}; + +const std::vector shapeCalculationMode = { + ngraph::op::v4::Interpolate::ShapeCalcMode::sizes, + ngraph::op::v4::Interpolate::ShapeCalcMode::scales, +}; + +const auto interpolateCasesNearestMode = ::testing::Combine( + ::testing::ValuesIn(nearestMode), + ::testing::ValuesIn(shapeCalculationMode), + ::testing::ValuesIn(coordinateTransformModesNearest), + ::testing::ValuesIn(defaultNearestMode), + ::testing::ValuesIn(antialias), + ::testing::ValuesIn(pads), + ::testing::ValuesIn(pads), + ::testing::ValuesIn(cubeCoefs), + ::testing::ValuesIn(defaultAxes), + ::testing::ValuesIn(defaultScales)); + +const auto interpolateCasesWithoutNearestMode = ::testing::Combine( + ::testing::ValuesIn(modesWithoutNearest), + ::testing::ValuesIn(shapeCalculationMode), + ::testing::ValuesIn(coordinateTransformModesWithoutNearest), + ::testing::ValuesIn(defaultNearestMode), + ::testing::ValuesIn(antialias), + ::testing::ValuesIn(pads), + ::testing::ValuesIn(pads), + ::testing::ValuesIn(cubeCoefs), + ::testing::ValuesIn(defaultAxes), + ::testing::ValuesIn(defaultScales)); + +INSTANTIATE_TEST_CASE_P(smoke_Interpolate_nearest_mode, InterpolateLayerTest, ::testing::Combine( + interpolateCasesNearestMode, + ::testing::ValuesIn(netPrecisions), + ::testing::Values(InferenceEngine::Precision::UNSPECIFIED), + ::testing::Values(InferenceEngine::Precision::UNSPECIFIED), + ::testing::Values(InferenceEngine::Layout::ANY), + ::testing::Values(InferenceEngine::Layout::ANY), + ::testing::ValuesIn(inShapes), + ::testing::ValuesIn(targetShapes), + ::testing::Values(CommonTestUtils::DEVICE_MYRIAD)), + InterpolateLayerTest::getTestCaseName); + +INSTANTIATE_TEST_CASE_P(smoke_Interpolate_without_nearest, InterpolateLayerTest, ::testing::Combine( + interpolateCasesWithoutNearestMode, + ::testing::ValuesIn(netPrecisions), + ::testing::Values(InferenceEngine::Precision::UNSPECIFIED), + ::testing::Values(InferenceEngine::Precision::UNSPECIFIED), + ::testing::Values(InferenceEngine::Layout::ANY), + ::testing::Values(InferenceEngine::Layout::ANY), + ::testing::ValuesIn(inShapes), + ::testing::ValuesIn(targetShapes), + ::testing::Values(CommonTestUtils::DEVICE_MYRIAD)), + InterpolateLayerTest::getTestCaseName); + +} // namespace