From ac8a39da87cfd67d7a98ed8e6d3b71b7e06dd6e4 Mon Sep 17 00:00:00 2001 From: Gabriele Galiero Casay Date: Mon, 30 Nov 2020 04:59:31 +0100 Subject: [PATCH] Reference Implementation of ROIPooling op (#2903) * ROIPooling: Specification and op class alignment * ROIPooling: Add check to input tensor type to be aligned with spec * ROIPooling: Corrected spec description for input tensor shape and box coordinates * ROIPooling: Changed attributes pooled_h and pooled_w from Shape to plain int * Revert "ROIPooling: Changed attributes pooled_h and pooled_w from Shape to plain int" This reverts commit d49cfa8e536f6e617558db116621586e8fead7c3. * ROIPooling: Further specification changes * ROIPooling: Rename enum class ROIPoolingMethod methods * Fix style * ROIPooling: Draft reference implementation * ROIPooling: Adjust feature map element type to float for attribute unit test * ROIPooling: Add single layer test class * ROIPooling: Corrected output index to iterate through output tensor elements * ROIPooling: Added validation checks for input types in op constructor * ROIPooling: Add unit tests * ROIPooling: Attributes unit test changed to align with spec * ROIPooling: Add check for batch id in reference implementation and unit test * ROIPooling: Refactor single layer test class * ROIPooling: Add test for invalid pooling method * ROIPooling: Clean up unnecessary function declaration * ROIPooling: Remove duplicated default ROIPooling method in op constructors * ROIPooling: Add Infer method to generate suitable ROI data * ROIPooling: CPU single layer test instantiation for max method * ROIPooling: Remove enum class ROIPoolingMethod * Revert "ROIPooling: Clean up unnecessary function declaration" This reverts commit 074b540dea6ca82ef3a22bbfe9fe953ea8ba2587. * ROIPooling: Refactor single layer tests after removing enum class ROIPoolingMethod * ROIPooling: Add attribute checks in op constructor to align with spec and unit tests * Resolve CI failure: clang could not resolve static conversion from uint64_t to size_t * ROIPooling: Fix for output index calculation to loop through all ROIs * ROIPooling: Add unit test for bilinear interpolation method * ROIPooling: Add CPU single layer test instantiation for bilinear method * ROIPooling: Clean up unnecessary enum class for pooling method * ROIPooling: Add myriad single layer test instantiation * ROIPooling: Add F16 precision single layer tests for CPU plugin * ROIPooling: Add node validation check for string method attribute in constructor and unit tests * ROIPooling: Spec changes to improve understanding of the operation * ROIPooling: Fix for bilinear method when pooled size is 1x1 * ROIPooling: Add unit test for bilinear method and pooled size 1x1 * ROIPooling: Fix to broken format of specifications * ROIPooling: Disable Myriad single layer tests * ROIPooling: Handle dynamic dims and ranks for input tensors and unit tests * ROIPooling: Code clean up * ROIPooling: Address review comments * ROIPooling: Changed location for makeROIPooling helper method Co-authored-by: Kirill Molchanov --- docs/ops/detection/ROIPooling_1.md | 25 +- .../single_layer_tests/roi_pooling.cpp | 64 +++++ .../single_layer_tests/roi_pooling.cpp | 58 +++++ .../skip_tests_config.cpp | 2 + .../single_layer_tests/roi_pooling.hpp | 43 ++++ .../src/single_layer_tests/roi_pooling.cpp | 104 ++++++++ .../common_test_utils/data_utils.hpp | 29 +++ .../include/ngraph_functions/builders.hpp | 6 + .../ngraph_functions/utils/ngraph_helpers.hpp | 6 + .../ngraph_functions/src/roi_pooling.cpp | 30 +++ ngraph/core/include/ngraph/op/roi_pooling.hpp | 13 +- .../ngraph/runtime/reference/roi_pooling.hpp | 231 ++++++++++++++++++ ngraph/core/src/op/roi_pooling.cpp | 118 +++++++-- ngraph/test/CMakeLists.txt | 3 + ngraph/test/attributes.cpp | 6 +- ngraph/test/backend/roi_pooling.in.cpp | 216 ++++++++++++++++ ngraph/test/op_eval/roi_pooling.cpp | 64 +++++ ngraph/test/runtime/ie/unit_test.manifest | 3 + .../runtime/interpreter/int_executable.hpp | 14 ++ .../runtime/interpreter/opset_int_tbl.hpp | 1 + ngraph/test/type_prop/roi_pooling.cpp | 136 +++++++++++ ngraph/test/type_prop_layers.cpp | 2 +- 22 files changed, 1138 insertions(+), 36 deletions(-) create mode 100644 inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/roi_pooling.cpp create mode 100644 inference-engine/tests/functional/plugin/myriad/shared_tests_instances/single_layer_tests/roi_pooling.cpp create mode 100644 inference-engine/tests/functional/plugin/shared/include/single_layer_tests/roi_pooling.hpp create mode 100644 inference-engine/tests/functional/plugin/shared/src/single_layer_tests/roi_pooling.cpp create mode 100644 inference-engine/tests/ngraph_helpers/ngraph_functions/src/roi_pooling.cpp create mode 100644 ngraph/core/reference/include/ngraph/runtime/reference/roi_pooling.hpp create mode 100644 ngraph/test/backend/roi_pooling.in.cpp create mode 100644 ngraph/test/op_eval/roi_pooling.cpp create mode 100644 ngraph/test/type_prop/roi_pooling.cpp diff --git a/docs/ops/detection/ROIPooling_1.md b/docs/ops/detection/ROIPooling_1.md index 4ab319875dc..67a7f0a1e77 100644 --- a/docs/ops/detection/ROIPooling_1.md +++ b/docs/ops/detection/ROIPooling_1.md @@ -6,7 +6,18 @@ **Short description**: *ROIPooling* is a *pooling layer* used over feature maps of non-uniform input sizes and outputs a feature map of a fixed size. -**Detailed description**: [deepsense.io reference](https://blog.deepsense.ai/region-of-interest-pooling-explained/) +**Detailed description**: + +*ROIPooling* performs the following operations for each Region of Interest (ROI) over the input feature maps: +1. Produce box coordinates relative to the input feature map size, based on *method* attribute. +2. Calculate box height and width. +3. Divide the box into bins according to the pooled size attributes, `[pooled_h, pooled_w]`. +4. Apply maximum or bilinear interpolation pooling, for each bin, based on *method* attribute to produce output feature map element. + +The box height and width have different representation based on **method** attribute: + * *max*: Expressed in relative coordinates. The box height and width are calculated the following way: `roi_width = max(spatial_scale * (x_2 - x_1), 1.0)`, +`roi_height = max(spatial_scale * (y_2 - y_1), 1.0)`, so the malformed boxes are expressed as a box of size `1 x 1`. + * *bilinear*: Expressed in absolute coordinates and normalized to the `[0, 1]` interval. The box height and width are calculated the following way: `roi_width = (W - 1) * (x_2 - x_1)`, `roi_height = (H - 1) * (y_2 - y_1)`. **Attributes** @@ -44,13 +55,19 @@ **Inputs**: -* **1**: 4D input tensor of shape `[1, C, H, W]` with feature maps. Required. +* **1**: 4D input tensor of shape `[N, C, H, W]` with feature maps of type *T*. Required. + +* **2**: 2D input tensor of shape `[NUM_ROIS, 5]` describing region of interest box consisting of 5 element tuples of type *T*: `[batch_id, x_1, y_1, x_2, y_2]`. Required. +Batch indices must be in the range of `[0, N-1]`. -* **2**: 2D input tensor of shape `[NUM_ROIS, 5]` describing box consisting of 5 element tuples: `[batch_id, x_1, y_1, x_2, y_2]`. Required. **Outputs**: -* **1**: 4D output tensor of shape `[NUM_ROIS, C, pooled_h, pooled_w]` with feature maps. Required. +* **1**: 4D output tensor of shape `[NUM_ROIS, C, pooled_h, pooled_w]` with feature maps of type *T*. Required. + +**Types** + +* *T*: any supported floating point type. **Example** diff --git a/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/roi_pooling.cpp b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/roi_pooling.cpp new file mode 100644 index 00000000000..bc244cf5b57 --- /dev/null +++ b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/roi_pooling.cpp @@ -0,0 +1,64 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include "single_layer_tests/roi_pooling.hpp" +#include "common_test_utils/test_constants.hpp" + +using namespace LayerTestsDefinitions; + +const std::vector> inShapes = { + {1, 3, 8, 8}, + {3, 4, 50, 50} +}; + +const std::vector> pooledShapes_max = { + {1, 1}, + {2, 2}, + {3, 3}, + {6, 6} +}; + +const std::vector> pooledShapes_bilinear = { + {2, 2}, + {3, 3}, + {6, 6} +}; + +const std::vector> coordShapes = { + {1, 5}, + {3, 5}, + {5, 5} +}; + +const std::vector netPRCs = { + InferenceEngine::Precision::FP16, + InferenceEngine::Precision::FP32 +}; + +const std::vector spatial_scales = {0.625f, 1.f}; + +const auto test_ROIPooling_max = ::testing::Combine( + ::testing::ValuesIn(inShapes), + ::testing::ValuesIn(coordShapes), + ::testing::ValuesIn(pooledShapes_max), + ::testing::ValuesIn(spatial_scales), + ::testing::Values(ngraph::helpers::ROIPoolingTypes::ROI_MAX), + ::testing::ValuesIn(netPRCs), + ::testing::Values(CommonTestUtils::DEVICE_CPU) +); + +const auto test_ROIPooling_bilinear = ::testing::Combine( + ::testing::ValuesIn(inShapes), + ::testing::ValuesIn(coordShapes), + ::testing::ValuesIn(pooledShapes_bilinear), + ::testing::Values(spatial_scales[1]), + ::testing::Values(ngraph::helpers::ROIPoolingTypes::ROI_BILINEAR), + ::testing::ValuesIn(netPRCs), + ::testing::Values(CommonTestUtils::DEVICE_CPU) +); + +INSTANTIATE_TEST_CASE_P(smoke_TestsROIPooling_max, ROIPoolingLayerTest, test_ROIPooling_max, ROIPoolingLayerTest::getTestCaseName); +INSTANTIATE_TEST_CASE_P(smoke_TestsROIPooling_bilinear, ROIPoolingLayerTest, test_ROIPooling_bilinear, ROIPoolingLayerTest::getTestCaseName); diff --git a/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/single_layer_tests/roi_pooling.cpp b/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/single_layer_tests/roi_pooling.cpp new file mode 100644 index 00000000000..ffdae0ca7b2 --- /dev/null +++ b/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/single_layer_tests/roi_pooling.cpp @@ -0,0 +1,58 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include "single_layer_tests/roi_pooling.hpp" +#include "common_test_utils/test_constants.hpp" + +using namespace LayerTestsDefinitions; + +namespace { + +const std::vector> inShapes = { + {3, 4, 50, 50} +}; + +const std::vector> pooledShapes_max = { + {1, 1}, + {3, 3}, +}; + +const std::vector> pooledShapes_bilinear = { + {2, 2}, + {6, 6} +}; + +const std::vector> coordShapes = { + {1, 5}, + {3, 5}, +}; + +const std::vector spatial_scales = {0.625f, 1.f}; + +const auto test_ROIPooling_max = ::testing::Combine( + ::testing::ValuesIn(inShapes), + ::testing::ValuesIn(coordShapes), + ::testing::ValuesIn(pooledShapes_max), + ::testing::ValuesIn(spatial_scales), + ::testing::Values(ngraph::helpers::ROIPoolingTypes::ROI_MAX), + ::testing::Values(InferenceEngine::Precision::FP32), + ::testing::Values(CommonTestUtils::DEVICE_MYRIAD) +); + +const auto test_ROIPooling_bilinear = ::testing::Combine( + ::testing::ValuesIn(inShapes), + ::testing::ValuesIn(coordShapes), + ::testing::ValuesIn(pooledShapes_bilinear), + ::testing::ValuesIn(spatial_scales), + ::testing::Values(ngraph::helpers::ROIPoolingTypes::ROI_BILINEAR), + ::testing::Values(InferenceEngine::Precision::FP32), + ::testing::Values(CommonTestUtils::DEVICE_MYRIAD) +); + +INSTANTIATE_TEST_CASE_P(smoke_TestsROIPooling_max, ROIPoolingLayerTest, test_ROIPooling_max, ROIPoolingLayerTest::getTestCaseName); +INSTANTIATE_TEST_CASE_P(smoke_TestsROIPooling_bilinear, ROIPoolingLayerTest, test_ROIPooling_bilinear, ROIPoolingLayerTest::getTestCaseName); + +} // namespace diff --git a/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp b/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp index d4e280b0d3c..8d11db060a4 100644 --- a/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp +++ b/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp @@ -34,5 +34,7 @@ std::vector disabledTestPatterns() { R"(.*DSR_NonMaxSuppression.*NBoxes=(5|20|200).*)", // TODO: Issue: 42721 R"(.*(DSR_GatherND).*)", + // TODO: Issue 43781 + ".*ROIPoolingLayerTest.*" }; } diff --git a/inference-engine/tests/functional/plugin/shared/include/single_layer_tests/roi_pooling.hpp b/inference-engine/tests/functional/plugin/shared/include/single_layer_tests/roi_pooling.hpp new file mode 100644 index 00000000000..7f863b006fb --- /dev/null +++ b/inference-engine/tests/functional/plugin/shared/include/single_layer_tests/roi_pooling.hpp @@ -0,0 +1,43 @@ +// Copyright (C) 2020 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include + +#include "ngraph_functions/builders.hpp" +#include "ngraph_functions/utils/ngraph_helpers.hpp" + +#include "functional_test_utils/layer_test_utils.hpp" + +namespace LayerTestsDefinitions { + +using roiPoolingParamsTuple = std::tuple< + InferenceEngine::SizeVector, // Input shape + InferenceEngine::SizeVector, // Coords shape + std::vector, // Pooled shape {pooled_h, pooled_w} + float, // Spatial scale + ngraph::helpers::ROIPoolingTypes, // ROIPooling method + InferenceEngine::Precision, // Net precision + LayerTestsUtils::TargetDevice>; // Device name + +class ROIPoolingLayerTest : public testing::WithParamInterface, + virtual public LayerTestsUtils::LayerTestsCommon { +public: + static std::string getTestCaseName(testing::TestParamInfo obj); + void Infer() override; + +protected: + void SetUp() override; + +private: + ngraph::helpers::ROIPoolingTypes pool_method; + float spatial_scale; +}; + +} // namespace LayerTestsDefinitions diff --git a/inference-engine/tests/functional/plugin/shared/src/single_layer_tests/roi_pooling.cpp b/inference-engine/tests/functional/plugin/shared/src/single_layer_tests/roi_pooling.cpp new file mode 100644 index 00000000000..c2f7d584558 --- /dev/null +++ b/inference-engine/tests/functional/plugin/shared/src/single_layer_tests/roi_pooling.cpp @@ -0,0 +1,104 @@ +// Copyright (C) 2020 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include +#include +#include + +#include "common_test_utils/common_utils.hpp" +#include "functional_test_utils/skip_tests_config.hpp" +#include "functional_test_utils/layer_test_utils.hpp" + +#include "single_layer_tests/roi_pooling.hpp" + +using namespace InferenceEngine; +using namespace FuncTestUtils::PrecisionUtils; + +namespace LayerTestsDefinitions { + + std::string ROIPoolingLayerTest::getTestCaseName(testing::TestParamInfo obj) { + std::vector inputShape; + std::vector coordsShape; + std::vector poolShape; + float spatial_scale; + ngraph::helpers::ROIPoolingTypes pool_method; + InferenceEngine::Precision netPrecision; + std::string targetDevice; + std::tie(inputShape, coordsShape, poolShape, spatial_scale, pool_method, netPrecision, targetDevice) = obj.param; + + std::ostringstream result; + + result << "IS=" << CommonTestUtils::vec2str(inputShape) << "_"; + result << "CS=" << CommonTestUtils::vec2str(coordsShape) << "_"; + result << "PS=" << CommonTestUtils::vec2str(poolShape) << "_"; + result << "Scale=" << spatial_scale << "_"; + switch (pool_method) { + case ngraph::helpers::ROIPoolingTypes::ROI_MAX: + result << "Max_"; + break; + case ngraph::helpers::ROIPoolingTypes::ROI_BILINEAR: + result << "Bilinear_"; + break; + } + result << "netPRC=" << netPrecision.name() << "_"; + result << "trgDev=" << targetDevice; + return result.str(); + } + + void ROIPoolingLayerTest::Infer() { + inferRequest = executableNetwork.CreateInferRequest(); + inputs.clear(); + + auto feat_map_shape = cnnNetwork.getInputShapes().begin()->second; + const int height = pool_method == ngraph::helpers::ROIPoolingTypes::ROI_MAX ? feat_map_shape[2] / spatial_scale : 1; + const int width = pool_method == ngraph::helpers::ROIPoolingTypes::ROI_MAX ? feat_map_shape[3] / spatial_scale : 1; + + size_t it = 0; + for (const auto &input : cnnNetwork.getInputsInfo()) { + const auto &info = input.second; + Blob::Ptr blob; + + if (it == 1) { + blob = make_blob_with_precision(info->getTensorDesc()); + blob->allocate(); + CommonTestUtils::fill_data_roi(blob->buffer(), blob->size(), feat_map_shape[0] - 1, + height, width, 1.0f); + } else { + blob = GenerateInput(*info); + } + inferRequest.SetBlob(info->name(), blob); + inputs.push_back(blob); + it++; + } + inferRequest.Infer(); + } + + void ROIPoolingLayerTest::SetUp() { + InferenceEngine::SizeVector inputShape; + InferenceEngine::SizeVector coordsShape; + InferenceEngine::SizeVector poolShape; + InferenceEngine::Precision netPrecision; + float spatial_scale; + + std::tie(inputShape, coordsShape, poolShape, spatial_scale, pool_method, netPrecision, targetDevice) = this->GetParam(); + + auto ngPrc = FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc(netPrecision); + auto params = ngraph::builder::makeParams(ngPrc, {inputShape, coordsShape}); + auto paramOuts = ngraph::helpers::convert2OutputVector( + ngraph::helpers::castOps2Nodes(params)); + std::shared_ptr roi_pooling = ngraph::builder::makeROIPooling(paramOuts[0], + paramOuts[1], + poolShape, + spatial_scale, + pool_method); + ngraph::ResultVector results{std::make_shared(roi_pooling)}; + function = std::make_shared(results, params, "roi_pooling"); + } + + TEST_P(ROIPoolingLayerTest, CompareWithRefs) { + Run(); + } +} // namespace LayerTestsDefinitions diff --git a/inference-engine/tests/ie_test_utils/common_test_utils/data_utils.hpp b/inference-engine/tests/ie_test_utils/common_test_utils/data_utils.hpp index 670f027f883..379d6c57449 100644 --- a/inference-engine/tests/ie_test_utils/common_test_utils/data_utils.hpp +++ b/inference-engine/tests/ie_test_utils/common_test_utils/data_utils.hpp @@ -123,6 +123,35 @@ static void fill_data_bbox(float *data, size_t size, int height, int width, floa } } +static void fill_data_roi(float *data, size_t size, const uint32_t range, const int height, const int width, const float omega, const int seed = 1) { + std::default_random_engine random(seed); + std::uniform_int_distribution distribution(0, range); + float center_h = (height - 1.0f) / 2; + float center_w = (width - 1.0f) / 2; + for (size_t i = 0; i < size; i += 5) { + data[i] = static_cast(distribution(random)); + data[i + 1] = std::floor(center_w + width * 0.6f * sin(static_cast(i+1) * omega)); + data[i + 3] = std::floor(center_w + width * 0.6f * sin(static_cast(i+3) * omega)); + if (data[i + 3] < data[i + 1]) { + std::swap(data[i + 1], data[i + 3]); + } + if (data[i + 1] < 0) + data[i + 1] = 0; + if (data[i + 3] > width - 1) + data[i + 3] = static_cast(width - 1); + + data[i + 2] = std::floor(center_h + height * 0.6f * sin(static_cast(i+2) * omega)); + data[i + 4] = std::floor(center_h + height * 0.6f * sin(static_cast(i+4) * omega)); + if (data[i + 4] < data[i + 2]) { + std::swap(data[i + 2], data[i + 4]); + } + if (data[i + 2] < 0) + data[i + 2] = 0; + if (data[i + 4] > height - 1) + data[i + 4] = static_cast(height - 1); + } +} + /** @brief Fill blob with random data. * * @param blob Target blob diff --git a/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/builders.hpp b/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/builders.hpp index 13069b516a6..931da1dd21a 100644 --- a/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/builders.hpp +++ b/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/builders.hpp @@ -347,6 +347,12 @@ std::shared_ptr makePooling(const ngraph::Output &in, bool excludePad, const ngraph::helpers::PoolingTypes &poolType); +std::shared_ptr makeROIPooling(const Output& input, + const Output& coords, + const Shape& output_size, + const float spatial_scale, + const ngraph::helpers::ROIPoolingTypes& roi_pool_type); + std::shared_ptr makeScatterUpdate(const ngraph::Output &in, const element::Type& indicesType, const std::vector& indicesShape, diff --git a/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/utils/ngraph_helpers.hpp b/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/utils/ngraph_helpers.hpp index 69906db5701..c4e9b51cc52 100644 --- a/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/utils/ngraph_helpers.hpp +++ b/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/utils/ngraph_helpers.hpp @@ -79,6 +79,12 @@ enum PoolingTypes { MAX, AVG }; + +enum ROIPoolingTypes { + ROI_MAX, + ROI_BILINEAR +}; + enum ActivationTypes { None, Sigmoid, diff --git a/inference-engine/tests/ngraph_helpers/ngraph_functions/src/roi_pooling.cpp b/inference-engine/tests/ngraph_helpers/ngraph_functions/src/roi_pooling.cpp new file mode 100644 index 00000000000..3ae3532c8dc --- /dev/null +++ b/inference-engine/tests/ngraph_helpers/ngraph_functions/src/roi_pooling.cpp @@ -0,0 +1,30 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +// + +#include +#include + +#include "ngraph_functions/builders.hpp" + +namespace ngraph { +namespace builder { + +std::shared_ptr makeROIPooling(const Output& input, + const Output& coords, + const Shape& output_size, + const float spatial_scale, + const ngraph::helpers::ROIPoolingTypes& roi_pool_type) { + switch (roi_pool_type) { + case helpers::ROIPoolingTypes::ROI_MAX: + return std::make_shared(input, coords, output_size, spatial_scale, "max"); + case helpers::ROIPoolingTypes::ROI_BILINEAR: + return std::make_shared(input, coords, output_size, spatial_scale, "bilinear"); + default: + throw std::runtime_error("Incorrect type of ROIPooling operation"); + } +} + +} // namespace builder +} // namespace ngraph diff --git a/ngraph/core/include/ngraph/op/roi_pooling.hpp b/ngraph/core/include/ngraph/op/roi_pooling.hpp index e1d7073ea78..0c45f2e4b7d 100644 --- a/ngraph/core/include/ngraph/op/roi_pooling.hpp +++ b/ngraph/core/include/ngraph/op/roi_pooling.hpp @@ -32,7 +32,7 @@ namespace ngraph ROIPooling() = default; /// \brief Constructs a ROIPooling operation /// - /// \param input Input feature map {N, C, ...} + /// \param input Input feature map {N, C, H, W} /// \param coords Coordinates of bounding boxes /// \param output_size Height/Width of ROI output features /// \param spatial_scale Ratio of input feature map over input image size @@ -41,7 +41,7 @@ namespace ngraph const Output& coords, const Shape& output_size, const float spatial_scale, - const std::string& method); + const std::string& method = "max"); void validate_and_infer_types() override; @@ -58,7 +58,10 @@ namespace ngraph float m_spatial_scale; std::string m_method; }; - } + + } // namespace v0 using v0::ROIPooling; - } -} + + } // namespace op + +} // namespace ngraph diff --git a/ngraph/core/reference/include/ngraph/runtime/reference/roi_pooling.hpp b/ngraph/core/reference/include/ngraph/runtime/reference/roi_pooling.hpp new file mode 100644 index 00000000000..8ea19700e4d --- /dev/null +++ b/ngraph/core/reference/include/ngraph/runtime/reference/roi_pooling.hpp @@ -0,0 +1,231 @@ +//***************************************************************************** +// Copyright 2017-2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#pragma once + +#include "ngraph/shape.hpp" + +namespace ngraph +{ + namespace runtime + { + namespace reference + { + template + void roi_pooling(const T* feature_maps, + const T* rois, + T* output, + const Shape& feature_maps_shape, + const Shape& rois_shape, + const Shape& output_shape, + const float spatial_scale, + const std::string& pooling_method) + { + // Feature maps input shape: {N, C, H, W} + const int batches = feature_maps_shape[0]; + const int channels = feature_maps_shape[1]; + const int height = feature_maps_shape[2]; + const int width = feature_maps_shape[3]; + + // Output shape: {NUM_ROIS, C, pooled_h, pooled_w} + const int pooled_h = output_shape[2]; + const int pooled_w = output_shape[3]; + + // ROIs shape: {NUM_ROIS, 5} + const int num_rois = rois_shape[0]; + + for (unsigned int roi_num = 0; roi_num < num_rois; roi_num++) + { + // ROI tuple: [roi_batch_id, roi_w_start, roi_h_start, roi_w_end, roi_h_end] + // ROI index + int roi_idx = rois_shape[1] * roi_num; + + // ROI batch id + int roi_batch_id = rois[roi_idx + 0]; + + // ROI batch id must be in the range of [0, N-1] + NGRAPH_CHECK(0 <= roi_batch_id && roi_batch_id < batches, + "ROI batch id must be in the range of [0, N-1]"); + + if (pooling_method == "max") + { + // ROI coordinates scaled to input feature maps + int roi_w_start = std::round(rois[roi_idx + 1] * spatial_scale); + int roi_h_start = std::round(rois[roi_idx + 2] * spatial_scale); + int roi_w_end = std::round(rois[roi_idx + 3] * spatial_scale); + int roi_h_end = std::round(rois[roi_idx + 4] * spatial_scale); + + // Force malformed ROIs to be 1x1 + int roi_height = std::max(roi_h_end - roi_h_start + 1, 1); + int roi_width = std::max(roi_w_end - roi_w_start + 1, 1); + + // Divide ROIs into sub-regions for max pooling + T bin_size_h = static_cast(roi_height) / pooled_h; + T bin_size_w = static_cast(roi_width) / pooled_w; + + const T* batch_data = + feature_maps + roi_batch_id * channels * height * width; + + for (unsigned int c = 0; c < channels; c++) + { + for (unsigned int ph = 0; ph < pooled_h; ph++) + { + for (unsigned int pw = 0; pw < pooled_w; pw++) + { + // Compute pooling region for this output unit: + // start (included) = floor(ph * roi_height / pooled_h) + // end (excluded) = ceil((ph + 1) * roi_height / pooled_h) + int h_start = static_cast( + std::floor(static_cast(ph) * bin_size_h)); + int w_start = static_cast( + std::floor(static_cast(pw) * bin_size_w)); + int h_end = static_cast( + std::ceil(static_cast(ph + 1) * bin_size_h)); + int w_end = static_cast( + std::ceil(static_cast(pw + 1) * bin_size_w)); + + // Add ROI offsets and clip to input boundaries + h_start = std::min(std::max(h_start + roi_h_start, 0), height); + w_start = std::min(std::max(w_start + roi_w_start, 0), width); + h_end = std::min(std::max(h_end + roi_h_start, 0), height); + w_end = std::min(std::max(w_end + roi_w_start, 0), width); + + const size_t pool_index = + roi_num * channels * pooled_h * pooled_w + + c * pooled_h * pooled_w + ph * pooled_w + pw; + + // Define an empty pooling region to be zero + bool is_empty = (h_end <= h_start) || (w_end <= w_start); + output[pool_index] = + is_empty ? 0 : std::numeric_limits::lowest(); + + for (unsigned int h = h_start; h < h_end; h++) + { + for (unsigned int w = w_start; w < w_end; w++) + { + const size_t index = h * width + w; + output[pool_index] = + std::max(batch_data[index], output[pool_index]); + } + } + } + } + // Increment batch data pointer by one channel + batch_data += height * width; + } + } + else if (pooling_method == "bilinear") + { + // ROI coordinates, normalized + T roi_w_start = rois[roi_idx + 1]; + T roi_h_start = rois[roi_idx + 2]; + T roi_w_end = rois[roi_idx + 3]; + T roi_h_end = rois[roi_idx + 4]; + + T roi_height = (roi_h_end - roi_h_start) * (height - 1); + T roi_width = (roi_w_end - roi_w_start) * (width - 1); + + T roi_height_scale = (pooled_h > 1) ? roi_height / (pooled_h - 1) : 0; + T roi_width_scale = (pooled_w > 1) ? roi_width / (pooled_w - 1) : 0; + + for (unsigned int c = 0; c < channels; c++) + { + for (unsigned int ph = 0; ph < pooled_h; ph++) + { + for (unsigned int pw = 0; pw < pooled_w; pw++) + { + T in_y = + (pooled_h > 1) + ? (ph * roi_height_scale + roi_h_start * (height - 1)) + : 0.5 * (roi_h_start + roi_h_end) * (height - 1); + T in_x = + (pooled_w > 1) + ? (pw * roi_width_scale + roi_w_start * (width - 1)) + : 0.5 * (roi_w_end + roi_w_start) * (width - 1); + + const size_t pool_index = + roi_num * channels * pooled_h * pooled_w + + c * pooled_h * pooled_w + ph * pooled_w + pw; + // Define invalid pooling region to be zero + if (in_y < 0 || in_y > height - 1 || in_x < 0 || + in_x > width - 1) + { + output[pool_index] = 0; + } + else + { + int top_y_index = static_cast(std::floor(in_y)); + int bottom_y_index = static_cast(std::ceil(in_y)); + int left_x_index = static_cast(std::floor(in_x)); + int right_x_index = static_cast(std::ceil(in_x)); + + // Clip to input width boundaries + if (right_x_index > width - 1) + { + right_x_index = width - 1; + } + + // Clip to input height boundaries + if (bottom_y_index > height - 1) + { + bottom_y_index = height - 1; + } + + size_t top_left_idx = + roi_batch_id * channels * height * width + + c * height * width + top_y_index * width + left_x_index; + + size_t top_right_idx = + roi_batch_id * channels * height * width + + c * height * width + top_y_index * width + + right_x_index; + + size_t bottom_left_idx = + roi_batch_id * channels * height * width + + c * height * width + bottom_y_index * width + + left_x_index; + + size_t bottom_right_idx = + roi_batch_id * channels * height * width + + c * height * width + bottom_y_index * width + + right_x_index; + + const T top_left = feature_maps[top_left_idx]; + const T top_right = feature_maps[top_right_idx]; + const T bottom_left = feature_maps[bottom_left_idx]; + const T bottom_right = feature_maps[bottom_right_idx]; + + const T top = + top_left + + (top_right - top_left) * (in_x - left_x_index); + const T bottom = + bottom_left + + (bottom_right - bottom_left) * (in_x - left_x_index); + + output[pool_index] = + top + (bottom - top) * (in_y - top_y_index); + } + } + } + } + } + } + } + } // namespace reference + + } // namespace runtime + +} // namespace ngraph diff --git a/ngraph/core/src/op/roi_pooling.cpp b/ngraph/core/src/op/roi_pooling.cpp index 92cc49bf1e7..31dc072ea09 100644 --- a/ngraph/core/src/op/roi_pooling.cpp +++ b/ngraph/core/src/op/roi_pooling.cpp @@ -36,32 +36,104 @@ op::ROIPooling::ROIPooling(const Output& input, void op::ROIPooling::validate_and_infer_types() { - auto input_et = get_input_element_type(0); - if (get_input_partial_shape(0).is_static() && get_input_partial_shape(1).is_static()) + auto feat_maps_et = get_input_element_type(0); + auto coords_et = get_input_element_type(1); + NODE_VALIDATION_CHECK( + this, + feat_maps_et.is_real() && coords_et.is_real(), + "The data type for input and ROIs is expected to be a floating point type. Got: ", + feat_maps_et, + " and: ", + coords_et); + + NODE_VALIDATION_CHECK( + this, + feat_maps_et == coords_et, + "Type of feature maps (inputs) and rois is expected to be the same. Got: ", + feat_maps_et, + " and: ", + coords_et); + + NODE_VALIDATION_CHECK(this, + m_output_size.size() == 2, + "The dimension of pooled size is expected to be equal to 2. Got: ", + m_output_size.size()); + + NODE_VALIDATION_CHECK(this, + m_output_size[0] > 0 && m_output_size[1] > 0, + "Pooled size attributes pooled_h and pooled_w should should be " + "non-negative integers. Got: ", + m_output_size[0], + " and: ", + m_output_size[1], + "respectively"); + + NODE_VALIDATION_CHECK( + this, + m_spatial_scale > 0, + "The spatial scale attribute should be a positive floating point number. Got: ", + m_spatial_scale); + + NODE_VALIDATION_CHECK( + this, + m_method == "max" || m_method == "bilinear", + "Pooling method attribute should be either \'max\' or \'bilinear\'. Got: ", + m_method); + + const auto& feat_maps_ps = get_input_partial_shape(0); + NODE_VALIDATION_CHECK(this, + feat_maps_ps.rank().compatible(4), + "Expected a 4D tensor for the feature maps input. Got: ", + feat_maps_ps); + + const auto& coords_ps = get_input_partial_shape(1); + NODE_VALIDATION_CHECK(this, + coords_ps.rank().compatible(2), + "Expected a 2D tensor for the ROIs input with box coordinates. Got: ", + coords_ps); + + if (coords_ps.rank().is_static()) { - Shape input_shape = get_input_partial_shape(0).to_shape(); - Shape coords_shape = get_input_partial_shape(1).to_shape(); - NODE_VALIDATION_CHECK(this, - input_shape.size() >= 3, - "ROIPooling expects 3 or higher dimensions for input. Got ", - input_shape.size()); - NODE_VALIDATION_CHECK(this, - coords_shape.size() == 2, - "ROIPooling expects 2 dimensions for box coordinates. Got ", - coords_shape.size()); - NODE_VALIDATION_CHECK(this, - input_shape.size() - 2 == m_output_size.size(), - "Spatial dimensions on input: ", - input_shape.size() - 2, - " doesn't match dimensions on requested output_size: ", - m_output_size.size()); - Shape output_shape{coords_shape[0], input_shape[1]}; - output_shape.insert(output_shape.end(), m_output_size.begin(), m_output_size.end()); - set_output_type(0, input_et, output_shape); + const auto coords_second_dim = coords_ps[1]; + NODE_VALIDATION_CHECK( + this, + coords_second_dim.compatible(5), + "The second dimension of ROIs input should contain batch id and box coordinates. ", + "This dimension is expected to be equal to 5. Got: ", + coords_second_dim); } - else + + // output shape should be {NUM_ROIS, C, pooled_h, pooled_w} + auto output_shape = PartialShape{{Dimension::dynamic(), + Dimension::dynamic(), + Dimension{static_cast(m_output_size[0])}, + Dimension{static_cast(m_output_size[1])}}}; + + if (coords_ps.rank().is_static() && coords_ps[0].is_static()) { - set_output_type(0, input_et, PartialShape::dynamic()); + output_shape[0] = coords_ps[0]; + } + + if (feat_maps_ps.rank().is_static() && feat_maps_ps[1].is_static()) + { + output_shape[1] = feat_maps_ps[1]; + } + + set_output_size(1); + set_output_type(0, feat_maps_et, output_shape); + + // if channel dimension, C, not known + // feature maps input is used by shape specialization pass + if (feat_maps_ps.rank().is_static() && feat_maps_ps[1].is_dynamic()) + { + set_input_is_relevant_to_shape(0); + } + + // if number of ROIs, NUM_ROIS, not known + // coordinate input is used by shape specialization pass + if (coords_ps.rank().is_static() && coords_ps[0].is_dynamic()) + { + set_input_is_relevant_to_shape(1); } } diff --git a/ngraph/test/CMakeLists.txt b/ngraph/test/CMakeLists.txt index 81db61806dc..38bc076f815 100644 --- a/ngraph/test/CMakeLists.txt +++ b/ngraph/test/CMakeLists.txt @@ -79,6 +79,7 @@ set(SRC op_eval/reduce_l1.cpp op_eval/reduce_l2.cpp op_eval/roi_align.cpp + op_eval/roi_pooling.cpp op_eval/round.cpp op_eval/softplus.cpp op_eval/split.cpp @@ -162,6 +163,7 @@ set(SRC type_prop/reverse.cpp type_prop/reverse_sequence.cpp type_prop/roi_align.cpp + type_prop/roi_pooling.cpp type_prop/round.cpp type_prop/rnn_cell.cpp type_prop/rnn_sequence.cpp @@ -328,6 +330,7 @@ set(MULTI_TEST_SRC backend/reshape.in.cpp backend/reverse_sequence.in.cpp backend/reverse.in.cpp + backend/roi_pooling.in.cpp backend/round.in.cpp backend/select.in.cpp backend/shape_of.in.cpp diff --git a/ngraph/test/attributes.cpp b/ngraph/test/attributes.cpp index 64a5a60451d..88efac34f63 100644 --- a/ngraph/test/attributes.cpp +++ b/ngraph/test/attributes.cpp @@ -1350,10 +1350,10 @@ TEST(attributes, reorg_yolo_op_strides) TEST(attributes, roi_pooling_op) { FactoryRegistry::get().register_factory(); - const auto data = make_shared(element::i32, Shape{2, 3, 4, 5}); - const auto coords = make_shared(element::i32, Shape{2, 3}); + const auto data = make_shared(element::f32, Shape{2, 3, 4, 5}); + const auto coords = make_shared(element::f32, Shape{2, 5}); - const auto op = make_shared(data, coords, Shape{5, 5}, 0.123, "Bilinear"); + const auto op = make_shared(data, coords, Shape{5, 5}, 0.123, "bilinear"); NodeBuilder builder(op); const auto g_op = as_type_ptr(builder.create()); diff --git a/ngraph/test/backend/roi_pooling.in.cpp b/ngraph/test/backend/roi_pooling.in.cpp new file mode 100644 index 00000000000..37004ba1d41 --- /dev/null +++ b/ngraph/test/backend/roi_pooling.in.cpp @@ -0,0 +1,216 @@ +//***************************************************************************** +// Copyright 2017-2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "gtest/gtest.h" +#include "ngraph/ngraph.hpp" +#include "util/engine/test_engines.hpp" +#include "util/test_case.hpp" +#include "util/test_control.hpp" + +NGRAPH_SUPPRESS_DEPRECATED_START + +using namespace std; +using namespace ngraph; + +static string s_manifest = "${MANIFEST}"; +using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME}); + +NGRAPH_TEST(${BACKEND_NAME}, roi_pooling_1x1_max) +{ + const int H = 6; + const int W = 6; + const int image_size = H * W; + const int channels = 3; + const int num_rois = 3; + + const int pooled_h = 1; + const int pooled_w = 1; + const float spatial_scale = 1.f; + + Shape feat_maps_shape{1, channels, H, W}; + Shape rois_shape{num_rois, 5}; + Shape pooled_shape{pooled_h, pooled_w}; + Shape output_shape{num_rois, channels, pooled_h, pooled_w}; + + const auto feat_maps = make_shared(element::f32, feat_maps_shape); + const auto rois = make_shared(element::f32, rois_shape); + const auto roi_pooling = + make_shared(feat_maps, rois, pooled_shape, spatial_scale, "max"); + const auto f = make_shared(roi_pooling, ParameterVector{feat_maps, rois}); + + vector feat_maps_vect; + for (unsigned int i = 0; i < channels * image_size; i++) + { + feat_maps_vect.push_back(1.f * i / 10); + } + + vector rois_vect = {0, 1, 1, 2, 3, 0, 1, 1, 2, 3, 0, 1, 1, 2, 3}; + + const vector expected_vect = {2.0f, 5.6f, 9.2f, 2.0f, 5.6f, 9.2f, 2.0f, 5.6f, 9.2f}; + + auto test_case = test::TestCase(f); + test_case.add_input(feat_maps_shape, feat_maps_vect); + test_case.add_input(rois_shape, rois_vect); + test_case.add_expected_output(output_shape, expected_vect); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, roi_pooling_2x2_max) +{ + const int H = 6; + const int W = 6; + const int image_size = H * W; + const int channels = 1; + const int num_rois = 3; + + const int pooled_h = 2; + const int pooled_w = 2; + const float spatial_scale = 1.f; + + Shape feat_maps_shape{1, channels, H, W}; + Shape rois_shape{num_rois, 5}; + Shape pooled_shape{pooled_h, pooled_w}; + Shape output_shape{num_rois, channels, pooled_h, pooled_w}; + + const auto feat_maps = make_shared(element::f32, feat_maps_shape); + const auto rois = make_shared(element::f32, rois_shape); + const auto roi_pooling = + make_shared(feat_maps, rois, pooled_shape, spatial_scale, "max"); + const auto f = make_shared(roi_pooling, ParameterVector{feat_maps, rois}); + + vector feat_maps_vect; + for (unsigned int i = 0; i < channels * image_size; i++) + { + feat_maps_vect.push_back(1.f * i / 10); + } + + vector rois_vect = {0, 1, 1, 3, 3, 0, 1, 2, 2, 4, 0, 0, 1, 4, 5}; + + const vector expected_vect = { + 1.4f, 1.5f, 2.0f, 2.1f, 1.9f, 2.0f, 2.5f, 2.6f, 2.0f, 2.2f, 3.2f, 3.4f}; + + auto test_case = test::TestCase(f); + test_case.add_input(feat_maps_shape, feat_maps_vect); + test_case.add_input(rois_shape, rois_vect); + test_case.add_expected_output(output_shape, expected_vect); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, roi_pooling_1x1_bilinear) +{ + const int H = 6; + const int W = 6; + const int image_size = H * W; + const int channels = 3; + const int num_rois = 2; + + const int pooled_h = 1; + const int pooled_w = 1; + const float spatial_scale = 1.f; + + Shape feat_maps_shape{1, channels, H, W}; + Shape rois_shape{num_rois, 5}; + Shape pooled_shape{pooled_h, pooled_w}; + Shape output_shape{num_rois, channels, pooled_h, pooled_w}; + + const auto feat_maps = make_shared(element::f32, feat_maps_shape); + const auto rois = make_shared(element::f32, rois_shape); + const auto roi_pooling = + make_shared(feat_maps, rois, pooled_shape, spatial_scale, "bilinear"); + const auto f = make_shared(roi_pooling, ParameterVector{feat_maps, rois}); + + vector feat_maps_vect; + for (unsigned int i = 0; i < channels * image_size; i++) + { + feat_maps_vect.push_back(1.f * i / 10); + } + + vector rois_vect = {0, 0.2, 0.2, 0.4, 0.4, 0, 0.2, 0.2, 0.6, 0.6}; + + const vector expected_vect = {1.05f, 4.65f, 8.25f, 1.4f, 5.0f, 8.6f}; + + auto test_case = test::TestCase(f); + test_case.add_input(feat_maps_shape, feat_maps_vect); + test_case.add_input(rois_shape, rois_vect); + test_case.add_expected_output(output_shape, expected_vect); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, roi_pooling_2x2_bilinear) +{ + const int H = 8; + const int W = 8; + const int image_size = H * W; + const int channels = 1; + const int num_rois = 3; + + const int pooled_h = 2; + const int pooled_w = 2; + const float spatial_scale = 1.f; + + Shape feat_maps_shape{1, channels, H, W}; + Shape rois_shape{num_rois, 5}; + Shape pooled_shape{pooled_h, pooled_w}; + Shape output_shape{num_rois, channels, pooled_h, pooled_w}; + + const auto feat_maps = make_shared(element::f32, feat_maps_shape); + const auto rois = make_shared(element::f32, rois_shape); + const auto roi_pooling = + make_shared(feat_maps, rois, pooled_shape, spatial_scale, "bilinear"); + const auto f = make_shared(roi_pooling, ParameterVector{feat_maps, rois}); + + vector feat_maps_vect; + for (unsigned int i = 0; i < channels * image_size; i++) + { + feat_maps_vect.push_back(1.f * i / 10); + } + + vector rois_vect = {0.f, + 0.15f, + 0.2f, + 0.75f, + 0.8f, + 0.f, + 0.15f, + 0.2f, + 0.75f, + 0.8f, + 0.f, + 0.15f, + 0.2f, + 0.75f, + 0.8f}; + + const auto count = shape_size(output_shape); + const vector expected_vect = {1.225f, + 1.645f, + 4.585f, + 5.005f, + 1.225f, + 1.645f, + 4.585f, + 5.005f, + 1.225f, + 1.645f, + 4.585f, + 5.005f}; + + auto test_case = test::TestCase(f); + test_case.add_input(feat_maps_shape, feat_maps_vect); + test_case.add_input(rois_shape, rois_vect); + test_case.add_expected_output(output_shape, expected_vect); + test_case.run(); +} diff --git a/ngraph/test/op_eval/roi_pooling.cpp b/ngraph/test/op_eval/roi_pooling.cpp new file mode 100644 index 00000000000..c7057e866c7 --- /dev/null +++ b/ngraph/test/op_eval/roi_pooling.cpp @@ -0,0 +1,64 @@ +//***************************************************************************** +// Copyright 2017-2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "gtest/gtest.h" +#include "ngraph/ngraph.hpp" +#include "util/engine/interpreter_engine.hpp" +#include "util/engine/test_engines.hpp" +#include "util/test_case.hpp" +#include "util/test_control.hpp" + +using namespace std; +using namespace ngraph; + +static string s_manifest = "${MANIFEST}"; + +NGRAPH_TEST(op_eval, roi_pooling_invalid_roi_batch_id) +{ + const int H = 6; + const int W = 6; + const int image_size = H * W; + const int channels = 1; + const int num_rois = 1; + + const int pooled_h = 1; + const int pooled_w = 1; + const float spatial_scale = 1.f; + + Shape feat_maps_shape{1, channels, H, W}; + Shape rois_shape{num_rois, 5}; + Shape pooled_shape{pooled_h, pooled_w}; + Shape output_shape{num_rois, channels, pooled_h, pooled_w}; + + const auto feat_maps = make_shared(element::f32, feat_maps_shape); + const auto rois = make_shared(element::f32, rois_shape); + const auto roi_pooling = + make_shared(feat_maps, rois, pooled_shape, spatial_scale, "max"); + const auto f = make_shared(roi_pooling, ParameterVector{feat_maps, rois}); + + vector feat_maps_vect; + for (unsigned int i = 0; i < channels * image_size; i++) + { + feat_maps_vect.push_back(1.f * i / 10); + } + + auto test_case = test::TestCase(f); + test_case.add_input(feat_maps_shape, feat_maps_vect); + // ROI with invalid batch id, should throw exception + test_case.add_input(rois_shape, {-1, 1, 1, 2, 3}); + test_case.add_expected_output(output_shape, {2.0f}); + ASSERT_THROW(test_case.run(), ngraph::CheckFailure); +} diff --git a/ngraph/test/runtime/ie/unit_test.manifest b/ngraph/test/runtime/ie/unit_test.manifest index 3da8e8c21d9..719da7f9ab9 100644 --- a/ngraph/test/runtime/ie/unit_test.manifest +++ b/ngraph/test/runtime/ie/unit_test.manifest @@ -1141,6 +1141,9 @@ IE_CPU.nonmaxsuppression_suppress_by_IOU_and_scores IE_CPU.nonmaxsuppression_two_batches IE_CPU.nonmaxsuppression_two_classes +# Bug in CPU plugin for ROIPooling when pooled size is 1x1 and method is bilinear +IE_CPU.roi_pooling_1x1_bilinear + #------------------------------------------------------------------------------- # # Inference Engine GPU plugin excludes diff --git a/ngraph/test/runtime/interpreter/int_executable.hpp b/ngraph/test/runtime/interpreter/int_executable.hpp index 60732fe17c6..b63c32fbd83 100644 --- a/ngraph/test/runtime/interpreter/int_executable.hpp +++ b/ngraph/test/runtime/interpreter/int_executable.hpp @@ -85,6 +85,7 @@ #include "ngraph/runtime/reference/reverse.hpp" #include "ngraph/runtime/reference/reverse_sequence.hpp" #include "ngraph/runtime/reference/rnn_cell.hpp" +#include "ngraph/runtime/reference/roi_pooling.hpp" #include "ngraph/runtime/reference/round.hpp" #include "ngraph/runtime/reference/scatter_nd_update.hpp" #include "ngraph/runtime/reference/select.hpp" @@ -1195,6 +1196,19 @@ protected: } break; } + case OP_TYPEID::ROIPooling_v0: + { + const op::ROIPooling* roi_pooling = static_cast(&node); + reference::roi_pooling(args[0]->get_data_ptr(), + args[1]->get_data_ptr(), + out[0]->get_data_ptr(), + node.get_input_shape(0), + node.get_input_shape(1), + node.get_output_shape(0), + roi_pooling->get_spatial_scale(), + roi_pooling->get_method()); + break; + } case OP_TYPEID::Select: { size_t element_count = shape_size(node.get_output_shape(0)); diff --git a/ngraph/test/runtime/interpreter/opset_int_tbl.hpp b/ngraph/test/runtime/interpreter/opset_int_tbl.hpp index 8cebe8d3ade..985070bc251 100644 --- a/ngraph/test/runtime/interpreter/opset_int_tbl.hpp +++ b/ngraph/test/runtime/interpreter/opset_int_tbl.hpp @@ -25,6 +25,7 @@ NGRAPH_OP(LSTMCell, op::v0) NGRAPH_OP(RegionYolo, op::v0) NGRAPH_OP(ReorgYolo, op::v0) NGRAPH_OP(RNNCell, op::v0) +NGRAPH_OP(ROIPooling, op::v0) #undef ID_SUFFIX #define ID_SUFFIX(NAME) NAME##_v1 diff --git a/ngraph/test/type_prop/roi_pooling.cpp b/ngraph/test/type_prop/roi_pooling.cpp new file mode 100644 index 00000000000..5ab2b7759fe --- /dev/null +++ b/ngraph/test/type_prop/roi_pooling.cpp @@ -0,0 +1,136 @@ +//***************************************************************************** +// Copyright 2017-2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "gtest/gtest.h" +#include "ngraph/ngraph.hpp" + +using namespace std; +using namespace ngraph; + +TEST(type_prop, roi_pooling_basic_shape_inference) +{ + const auto feat_maps = make_shared(element::f32, Shape{1, 3, 6, 6}); + const auto rois = make_shared(element::f32, Shape{4, 5}); + const auto op = make_shared(feat_maps, rois, Shape{2, 2}, 0.625f); + ASSERT_EQ(op->get_method(), "max"); + ASSERT_EQ(op->get_shape(), (Shape{4, 3, 2, 2})); +} + +TEST(type_prop, roi_pooling_dynamic_channels_dim) +{ + const auto feat_maps = + make_shared(element::f32, PartialShape{1, Dimension(), 6, 6}); + const auto rois = make_shared(element::f32, Shape{4, 5}); + const auto op = make_shared(feat_maps, rois, Shape{2, 2}, 0.625f, "max"); + ASSERT_TRUE(op->get_output_partial_shape(0).same_scheme(PartialShape{4, Dimension(), 2, 2})); +} + +TEST(type_prop, roi_pooling_dynamic_num_rois_dim) +{ + const auto feat_maps = make_shared(element::f32, Shape{1, 3, 6, 6}); + const auto rois = make_shared(element::f32, PartialShape{Dimension(), 5}); + const auto op = make_shared(feat_maps, rois, Shape{2, 2}, 0.625f); + ASSERT_TRUE(op->get_output_partial_shape(0).same_scheme(PartialShape{Dimension(), 3, 2, 2})); +} + +TEST(type_prop, roi_pooling_dynamic_rank_feat_maps) +{ + const auto feat_maps = make_shared(element::f32, PartialShape::dynamic()); + const auto rois = make_shared(element::f32, Shape{4, 5}); + const auto op = make_shared(feat_maps, rois, Shape{2, 2}, 0.625f); + ASSERT_TRUE(op->get_output_partial_shape(0).same_scheme(PartialShape{4, Dimension(), 2, 2})); +} + +TEST(type_prop, roi_pooling_dynamic_rank_rois) +{ + const auto feat_maps = make_shared(element::f32, Shape{1, 3, 6, 6}); + const auto rois = make_shared(element::f32, PartialShape::dynamic()); + const auto op = make_shared(feat_maps, rois, Shape{2, 2}, 0.625f); + ASSERT_TRUE(op->get_output_partial_shape(0).same_scheme(PartialShape{Dimension(), 3, 2, 2})); +} + +TEST(type_prop, roi_pooling_incompatible_input_rank) +{ + const auto feat_maps = make_shared(element::f32, Shape{1, 3, 2, 6, 6}); + const auto rois = make_shared(element::f32, Shape{3, 5}); + // feat_maps must be of rank 4 + ASSERT_THROW(make_shared(feat_maps, rois, Shape{2, 2}, 0.625f, "max"), + ngraph::NodeValidationFailure); +} + +TEST(type_prop, roi_pooling_incompatible_pooling_shape) +{ + Shape pool_shape{2, 2, 2}; + const auto feat_maps = make_shared(element::f32, Shape{3, 2, 6, 6}); + const auto rois = make_shared(element::f32, Shape{3, 5}); + // pool_shape must be of rank 2 {pooled_h, pooled_w} + ASSERT_THROW(make_shared(feat_maps, rois, pool_shape, 0.625f, "max"), + ngraph::NodeValidationFailure); +} + +TEST(type_prop, roi_pooling_incompatible_rois_second_dim) +{ + const auto feat_maps = make_shared(element::f32, Shape{3, 2, 6, 6}); + const auto rois = make_shared(element::f32, Shape{3, 4}); + // the second dim of rois must be 5. [batch_id, x_1, y_1, x_2, y_2] + ASSERT_THROW(make_shared(feat_maps, rois, Shape{2, 2}, 0.625f, "max"), + ngraph::NodeValidationFailure); +} + +TEST(type_prop, roi_pooling_incompatible_feature_maps_element_type) +{ + const auto feat_maps = make_shared(element::i32, Shape{3, 2, 6, 6}); + const auto rois = make_shared(element::f32, Shape{3, 5}); + // feat_maps element type must be floating point type + ASSERT_THROW(make_shared(feat_maps, rois, Shape{2, 2}, 0.625f, "max"), + ngraph::NodeValidationFailure); +} + +TEST(type_prop, roi_pooling_incompatible_rois_element_type) +{ + const auto feat_maps = make_shared(element::f32, Shape{3, 2, 6, 6}); + const auto rois = make_shared(element::f16, Shape{3, 5}); + // rois element type must be equal to feat_maps element type (floating point type) + ASSERT_THROW(make_shared(feat_maps, rois, Shape{2, 2}, 0.625f, "bilinear"), + ngraph::NodeValidationFailure); +} + +TEST(type_prop, roi_pooling_invalid_pooling_method) +{ + const auto feat_maps = make_shared(element::f32, Shape{3, 2, 6, 6}); + const auto rois = make_shared(element::f16, Shape{3, 5}); + // ROIPooling method is invalid: not max nor bilinear + ASSERT_THROW(make_shared(feat_maps, rois, Shape{2, 2}, 0.625f, "invalid"), + ngraph::NodeValidationFailure); +} + +TEST(type_prop, roi_pooling_invalid_spatial_scale) +{ + const auto feat_maps = make_shared(element::f32, Shape{3, 2, 6, 6}); + const auto rois = make_shared(element::f16, Shape{3, 5}); + // ROIPooling spatial scale attribute must be a positive floating point number + ASSERT_THROW(make_shared(feat_maps, rois, Shape{2, 2}, -0.625f, "max"), + ngraph::NodeValidationFailure); +} + +TEST(type_prop, roi_pooling_invalid_pooled_size) +{ + const auto feat_maps = make_shared(element::f32, Shape{3, 2, 6, 6}); + const auto rois = make_shared(element::f16, Shape{3, 5}); + // ROIPooling pooled_h and pooled_w must be non-negative integers + ASSERT_THROW(make_shared(feat_maps, rois, Shape{1, 0}, 0.625f, "max"), + ngraph::NodeValidationFailure); +} diff --git a/ngraph/test/type_prop_layers.cpp b/ngraph/test/type_prop_layers.cpp index 159c84a56bc..1e2c0f01a79 100644 --- a/ngraph/test/type_prop_layers.cpp +++ b/ngraph/test/type_prop_layers.cpp @@ -168,6 +168,6 @@ TEST(type_prop_layers, roi_pooling) { auto inputs = make_shared(element::f32, Shape{2, 3, 4, 5}); auto coords = make_shared(element::f32, Shape{150, 5}); - auto op = make_shared(inputs, coords, Shape{6, 6}, 0.0625, "Max"); + auto op = make_shared(inputs, coords, Shape{6, 6}, 0.0625, "max"); ASSERT_EQ(op->get_shape(), (Shape{150, 3, 6, 6})); }