diff --git a/inference-engine/src/inference_engine/generic_ie.cpp b/inference-engine/src/inference_engine/generic_ie.cpp index b0da220bfeb..840491493f0 100644 --- a/inference-engine/src/inference_engine/generic_ie.cpp +++ b/inference-engine/src/inference_engine/generic_ie.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2021 Intel Corporation +// Copyright (C) 2017-2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -132,11 +132,6 @@ void ngraph::op::GenericIE::validate_and_infer_types() { } } - // WA: shape infer has to know number of outputs - if (type == "ExperimentalDetectronROIFeatureExtractor" && parameters.find("num_outputs") == parameters.end()) { - parameters["num_outputs"] = std::to_string(outputs.size()); - } - ret = impl->inferShapes(inputs, parameters, blobs, outShapes, nullptr); IE_SUPPRESS_DEPRECATED_END @@ -151,12 +146,6 @@ void ngraph::op::GenericIE::validate_and_infer_types() { // Extensions are not loaded when we create nGraph function // First call: create node if (initialized < 1) { - if (type == "ExperimentalDetectronROIFeatureExtractor" && outputs.size() < 2) { - // Add fake port - PortIE port; - port.precision = InferenceEngine::Precision::FP32; - outputs.emplace_back(port); - } if (outputs.size()) set_output_size(outputs.size()); for (size_t output_index = 0; output_index < outputs.size(); output_index++) { diff --git a/inference-engine/src/inference_engine/shape_infer/ie_built_in_holder.cpp b/inference-engine/src/inference_engine/shape_infer/ie_built_in_holder.cpp index 415b42a587b..795823419f5 100644 --- a/inference-engine/src/inference_engine/shape_infer/ie_built_in_holder.cpp +++ b/inference-engine/src/inference_engine/shape_infer/ie_built_in_holder.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2021 Intel Corporation +// Copyright (C) 2017-2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -10,7 +10,6 @@ #include "shape_infer/ie_proposal_onnx_shape_infer.hpp" #include "shape_infer/ie_proposal_shape_infer.hpp" #include "shape_infer/ie_rnn_cell_shape_infer.hpp" -#include "shape_infer/ie_roifeatureextractor_onnx_shape_infer.hpp" #include "shape_infer/ie_simpler_nms_shape_infer.hpp" #include "shape_infer/ie_sparse_to_dense_shape_infer.hpp" #include "shape_infer/ie_topkrois_onnx_shape_infer.hpp" @@ -67,7 +66,6 @@ public: static ImplRegisterBase<__prim> __bi_reg__##__type(#__type) REG_SHAPE_INFER_FOR_TYPE(ExperimentalDetectronGenerateProposalsSingleImageShapeProp, ExperimentalDetectronGenerateProposalsSingleImage); -REG_SHAPE_INFER_FOR_TYPE(ExperimentalDetectronROIFeatureExtractorShapeProp, ExperimentalDetectronROIFeatureExtractor); REG_SHAPE_INFER_FOR_TYPE(ExperimentalDetectronTopKROIsShapeProp, ExperimentalDetectronTopKROIs); REG_SHAPE_INFER_FOR_TYPE(SimplerNMSShapeProp, SimplerNMS); REG_SHAPE_INFER_FOR_TYPE(SparseToDenseShapeProp, SparseToDense); diff --git a/inference-engine/src/inference_engine/shape_infer/ie_roifeatureextractor_onnx_shape_infer.hpp b/inference-engine/src/inference_engine/shape_infer/ie_roifeatureextractor_onnx_shape_infer.hpp deleted file mode 100644 index e335ce224c4..00000000000 --- a/inference-engine/src/inference_engine/shape_infer/ie_roifeatureextractor_onnx_shape_infer.hpp +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (C) 2018-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include -#include - -#include "ie_built_in_impl.hpp" - -namespace InferenceEngine { -namespace ShapeInfer { - -/** - * @brief Implementation of Shape inference for ExperimentalDetectronROIFeatureExtractor layer - */ -class ExperimentalDetectronROIFeatureExtractorShapeProp : public BuiltInShapeInferImpl { -protected: - const int ROIS = 0; - const int FEATMAPS = 1; - -public: - explicit ExperimentalDetectronROIFeatureExtractorShapeProp(const std::string& type): BuiltInShapeInferImpl(type) {} - - void inferShapesImpl(const std::vector& inBlobs, const std::map& params, - const std::map& blobs, std::vector& outShapes) override { - size_t rois_num = inShapes.at(ROIS).at(0); - size_t channels_num = inShapes.at(FEATMAPS).at(1); - size_t output_size = static_cast(GetParamAsInt("output_size", params)); - outShapes.push_back({rois_num, channels_num, output_size, output_size}); - - auto num_outputs = GetParamAsUInt("num_outputs", params); - if (num_outputs > 2) THROW_IE_EXCEPTION << "Incorrect value num_outputs: " << num_outputs; - if (num_outputs == 2) { - outShapes.push_back({rois_num, 4}); - } - } -}; - -} // namespace ShapeInfer -} // namespace InferenceEngine diff --git a/inference-engine/src/readers/ir_reader/ie_ir_parser.cpp b/inference-engine/src/readers/ir_reader/ie_ir_parser.cpp index 50c4d3539cc..08b7b0a8d8a 100644 --- a/inference-engine/src/readers/ir_reader/ie_ir_parser.cpp +++ b/inference-engine/src/readers/ir_reader/ie_ir_parser.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2021 Intel Corporation +// Copyright (C) 2017-2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -680,7 +680,8 @@ std::shared_ptr V10Parser::XmlDeserializer::createNode( // Try to create operation from loaded opsets auto version = params.version; static const std::unordered_set experimental_detectrons = {"ExperimentalDetectronDetectionOutput", - "ExperimentalDetectronPriorGridGenerator"}; + "ExperimentalDetectronPriorGridGenerator", + "ExperimentalDetectronROIFeatureExtractor"}; if (experimental_detectrons.count(params.type)) { version = "opset6"; } diff --git a/inference-engine/tests/functional/inference_engine/cnn_network/cnn_ngraph_impl_tests.cpp b/inference-engine/tests/functional/inference_engine/cnn_network/cnn_ngraph_impl_tests.cpp index fdbdd8dbc13..9d77b90d522 100644 --- a/inference-engine/tests/functional/inference_engine/cnn_network/cnn_ngraph_impl_tests.cpp +++ b/inference-engine/tests/functional/inference_engine/cnn_network/cnn_ngraph_impl_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2020 Intel Corporation +// Copyright (C) 2017-2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -935,72 +935,286 @@ TEST(CNNNGraphImplTests, addOutputForParameter) { } } -TEST(CNNNGraphImplTests, AddOutputToExperimentalOp) { +TEST(CNNNGraphImplTests, AddOutputToExperimentalOpOpset6) { std::string model = R"V0G0N( - - + + + + + 1000 + 4 + + + + + 1 - 3 - 22 - 22 + 256 + 200 + 336 - - - - 1 - 3 - 22 - 22 - - + + - - 1 - 3 - 22 - 22 - - - - - - - 1 - 3 - 22 - 22 - - - - - 1 - 3 - 22 - 22 - - - - - 1 - 3 - 22 - 22 + 256 + 100 + 168 + + + + + + + + 1 + 256 + 50 + 84 + + + + + + + + 1 + 256 + 25 + 42 + + + + + + + + 1000 + 4 + + + 1 + 256 + 200 + 336 + + + 1 + 256 + 100 + 168 + + + 1 + 256 + 50 + 84 + + + 1 + 256 + 25 + 42 + + + + + 1000 + 256 + 7 + 7 + + + + + + + 1000 + 256 + 7 + 7 + + + + + 1000 + 256 + 7 + 7 + + + + + + + 1000 + 256 + 7 + 7 - - - + + + + + + + + + +)V0G0N"; + InferenceEngine::Core core; + CNNNetwork network = core.ReadNetwork(model, InferenceEngine::Blob::CPtr()); + network.addOutput("exp"); + auto outputs = network.getOutputsInfo(); + ASSERT_NE(outputs.find("exp.0"), outputs.end()); +} + +TEST(CNNNGraphImplTests, AddOutputToExperimentalOp) { + std::string model = R"V0G0N( + + + + + + + 1000 + 4 + + + + + + + + 1 + 256 + 200 + 336 + + + + + + + + 1 + 256 + 100 + 168 + + + + + + + + 1 + 256 + 50 + 84 + + + + + + + + 1 + 256 + 25 + 42 + + + + + + + + 1000 + 4 + + + 1 + 256 + 200 + 336 + + + 1 + 256 + 100 + 168 + + + 1 + 256 + 50 + 84 + + + 1 + 256 + 25 + 42 + + + + + 1000 + 256 + 7 + 7 + + + + + + + 1000 + 256 + 7 + 7 + + + + + 1000 + 256 + 7 + 7 + + + + + + + 1000 + 256 + 7 + 7 + + + + + + + + + + + + )V0G0N"; @@ -1012,39 +1226,104 @@ TEST(CNNNGraphImplTests, AddOutputToExperimentalOp) { } TEST(CNNNGraphImplTests, SaveOriginalResultNameForMultiOutputOp) { - std::string model = R"V0G0N( + std::string model = R"V0G0N( - - + + + + + 1000 + 4 + + + + + 1 - 3 - 22 - 22 + 256 + 200 + 336 - - - + + + + 1 - 3 - 22 - 22 + 256 + 100 + 168 + + + + + + + + 1 + 256 + 50 + 84 + + + + + + + + 1 + 256 + 25 + 42 + + + + + + + + 1000 + 4 + + + 1 + 256 + 200 + 336 + + + 1 + 256 + 100 + 168 + + + 1 + 256 + 50 + 84 + + + 1 + 256 + 25 + 42 - - 1 - 3 - 22 - 22 + + 1000 + 256 + 7 + 7 - + @@ -1055,13 +1334,13 @@ TEST(CNNNGraphImplTests, SaveOriginalResultNameForMultiOutputOp) { - + - 1 - 3 - 22 - 22 + 1000 + 256 + 7 + 7 1 @@ -1072,32 +1351,229 @@ TEST(CNNNGraphImplTests, SaveOriginalResultNameForMultiOutputOp) { - 1 - 3 - 22 - 22 + 1000 + 256 + 7 + 7 - + - 1 - 3 - 22 - 22 + 1000 + 256 + 7 + 7 - - - - + + + + + + + + )V0G0N"; + + InferenceEngine::Core core; + Blob::Ptr data = make_shared_blob(TensorDesc(Precision::FP32, {4}, Layout::C)); + data->allocate(); + { + auto lockData = data->buffer(); + float *dataPtr = lockData.as(); + + for (size_t i = 0; i < 4; ++i) { + dataPtr[i] = 0; + } + } + CNNNetwork network = core.ReadNetwork(model, data); + { + auto outputs = network.getOutputsInfo(); + ASSERT_NE(outputs.find("text_features"), outputs.end()); + } + + auto nGraphFunc = network.getFunction(); + // Disable shape inference (WA for generic operations) + ngraph::op::GenericIE::DisableReshape noReshape(nGraphFunc); + + ngraph::pass::Manager manager; + + manager.register_pass(); + + manager.run_passes(nGraphFunc); + + auto clonedNetwork = InferenceEngine::details::convertFunctionToICNNNetwork(nGraphFunc, network); + { + OutputsDataMap outputs; + clonedNetwork->getOutputsInfo(outputs); + ASSERT_NE(outputs.find("text_features"), outputs.end()); + } +} + +TEST(CNNNGraphImplTests, SaveOriginalResultNameForMultiOutputOpOpset6) { + std::string model = R"V0G0N( + + + + + + + 1000 + 4 + + + + + + + + 1 + 256 + 200 + 336 + + + + + + + + 1 + 256 + 100 + 168 + + + + + + + + 1 + 256 + 50 + 84 + + + + + + + + 1 + 256 + 25 + 42 + + + + + + + + 1000 + 4 + + + 1 + 256 + 200 + 336 + + + 1 + 256 + 100 + 168 + + + 1 + 256 + 50 + 84 + + + 1 + 256 + 25 + 42 + + + + + 1000 + 256 + 7 + 7 + + + + + + + + 1 + 1 + 1 + 1 + + + + + + + 1000 + 256 + 7 + 7 + + + 1 + 1 + 1 + 1 + + + + + 1000 + 256 + 7 + 7 + + + + + + + 1000 + 256 + 7 + 7 + + + + + + + + + + + + + + + +)V0G0N"; + InferenceEngine::Core core; Blob::Ptr data = make_shared_blob(TensorDesc(Precision::FP32, {4}, Layout::C)); data->allocate(); diff --git a/inference-engine/tests/functional/inference_engine/ir_serialization/models/experimental_detectron_roi_feature_extractor.xml b/inference-engine/tests/functional/inference_engine/ir_serialization/models/experimental_detectron_roi_feature_extractor.xml index 56eb115b46b..bb111712984 100644 --- a/inference-engine/tests/functional/inference_engine/ir_serialization/models/experimental_detectron_roi_feature_extractor.xml +++ b/inference-engine/tests/functional/inference_engine/ir_serialization/models/experimental_detectron_roi_feature_extractor.xml @@ -1,38 +1,118 @@ - + - - - - - - 1 - - - - - - - - 1 - - - - - 1 - - - - - - - 1 - - - - - - - - + + + + + + 1000 + 4 + + + + + + + + 1 + 256 + 200 + 336 + + + + + + + + 1 + 256 + 100 + 168 + + + + + + + + 1 + 256 + 50 + 84 + + + + + + + + 1 + 256 + 25 + 42 + + + + + + + + 1000 + 4 + + + 1 + 256 + 200 + 336 + + + 1 + 256 + 100 + 168 + + + 1 + 256 + 50 + 84 + + + 1 + 256 + 25 + 42 + + + + + 1000 + 256 + 7 + 7 + + + + + + + 1000 + 256 + 7 + 7 + + + + + + + + + + + + diff --git a/inference-engine/tests/functional/inference_engine/ir_serialization/models/experimental_detectron_roi_feature_extractor_opset6.xml b/inference-engine/tests/functional/inference_engine/ir_serialization/models/experimental_detectron_roi_feature_extractor_opset6.xml new file mode 100644 index 00000000000..dff25984ad2 --- /dev/null +++ b/inference-engine/tests/functional/inference_engine/ir_serialization/models/experimental_detectron_roi_feature_extractor_opset6.xml @@ -0,0 +1,118 @@ + + + + + + + + + 1000 + 4 + + + + + + + + 1 + 256 + 200 + 336 + + + + + + + + 1 + 256 + 100 + 168 + + + + + + + + 1 + 256 + 50 + 84 + + + + + + + + 1 + 256 + 25 + 42 + + + + + + + + 1000 + 4 + + + 1 + 256 + 200 + 336 + + + 1 + 256 + 100 + 168 + + + 1 + 256 + 50 + 84 + + + 1 + 256 + 25 + 42 + + + + + 1000 + 256 + 7 + 7 + + + + + + + 1000 + 256 + 7 + 7 + + + + + + + + + + + + + diff --git a/inference-engine/tests/functional/inference_engine/ir_serialization/serialize.cpp b/inference-engine/tests/functional/inference_engine/ir_serialization/serialize.cpp index dd76caa1b5b..1b3fc8f132c 100644 --- a/inference-engine/tests/functional/inference_engine/ir_serialization/serialize.cpp +++ b/inference-engine/tests/functional/inference_engine/ir_serialization/serialize.cpp @@ -64,6 +64,7 @@ INSTANTIATE_TEST_CASE_P(IRSerialization, SerializationTest, std::make_tuple("addmul_abc.xml", "addmul_abc.bin"), std::make_tuple("add_abc_initializers.xml", "add_abc_initializers.bin"), std::make_tuple("experimental_detectron_roi_feature_extractor.xml", ""), + std::make_tuple("experimental_detectron_roi_feature_extractor_opset6.xml", ""), std::make_tuple("experimental_detectron_detection_output.xml", ""), std::make_tuple("experimental_detectron_detection_output_opset6.xml", ""), std::make_tuple("nms5.xml", "nms5.bin"), @@ -73,4 +74,4 @@ INSTANTIATE_TEST_CASE_P(ONNXSerialization, SerializationTest, testing::Values(std::make_tuple("add_abc.prototxt", ""), std::make_tuple("split_equal_parts_2d.prototxt", ""), std::make_tuple("addmul_abc.prototxt", ""), - std::make_tuple("add_abc_initializers.prototxt", ""))); \ No newline at end of file + std::make_tuple("add_abc_initializers.prototxt", ""))); diff --git a/inference-engine/tests/functional/inference_engine/ngraph_reshape_tests.cpp b/inference-engine/tests/functional/inference_engine/ngraph_reshape_tests.cpp index aaf1d8172e7..ff2067c5662 100644 --- a/inference-engine/tests/functional/inference_engine/ngraph_reshape_tests.cpp +++ b/inference-engine/tests/functional/inference_engine/ngraph_reshape_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2021 Intel Corporation +// Copyright (C) 2017-2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -577,38 +577,38 @@ TEST_F(NGraphReshapeTests, ReshapeEDDetectionOutput) { - - - - 1000 - 4 - - - 1000 - 324 - - - 1000 - 81 - - - 1 - 3 - - - - - 100 - 4 - - - 100 - - - 100 - - - + + + + 1000 + 4 + + + 1000 + 324 + + + 1000 + 81 + + + 1 + 3 + + + + + 100 + 4 + + + 100 + + + 100 + + + @@ -688,37 +688,37 @@ TEST_F(NGraphReshapeTests, ReshapeEDPriorGridGenerator) { - - - - 3 - 4 - - - 1 - 256 - 200 - 336 - - - 1 - 3 - 800 - 1344 - - - - - 201600 - 4 - - - + + + + 3 + 4 + + + 1 + 256 + 200 + 336 + + + 1 + 3 + 800 + 1344 + + + + + 201600 + 4 + + + 201600 - 4 + 4 @@ -757,7 +757,7 @@ TEST_F(NGraphReshapeTests, ReshapeEDGenerateProposalsSingleImage) { 201600 - 4 + 4 @@ -766,8 +766,8 @@ TEST_F(NGraphReshapeTests, ReshapeEDGenerateProposalsSingleImage) { 12 - 200 - 336 + 200 + 336 @@ -776,47 +776,47 @@ TEST_F(NGraphReshapeTests, ReshapeEDGenerateProposalsSingleImage) { 3 - 200 - 336 + 200 + 336 - - - - 3 - - - 201600 - 4 - - - 12 - 200 - 336 - - - 3 - 200 - 336 - - - - - 1000 - 4 - - - 1000 - - - + + + + 3 + + + 201600 + 4 + + + 12 + 200 + 336 + + + 3 + 200 + 336 + + + + + 1000 + 4 + + + 1000 + + + 1000 - 4 + 4 @@ -872,35 +872,35 @@ TEST_F(NGraphReshapeTests, ReshapeEDROIFeatureExtractor) { - - - - 1000 - 4 - - - 1 - 256 - 200 - 336 - - - - - 1000 - 256 - 7 - 7 - - - + + + + 1000 + 4 + + + 1 + 256 + 200 + 336 + + + + + 1000 + 256 + 7 + 7 + + + 1000 - 256 - 7 - 7 + 256 + 7 + 7 @@ -916,8 +916,82 @@ TEST_F(NGraphReshapeTests, ReshapeEDROIFeatureExtractor) { Blob::Ptr weights; auto network = ie.ReadNetwork(model, weights); InferenceEngine::ICNNNetwork::InputShapes newShapes; - newShapes["in0"] = {1, 256, 300, 400}; - newShapes["in1"] = {1000, 256, 7, 7}; + newShapes["in0"] = {1256, 4}; + newShapes["in1"] = {1, 256, 7, 7}; + ASSERT_NO_THROW(network.reshape(newShapes)); +} + +TEST_F(NGraphReshapeTests, ReshapeEDROIFeatureExtractorOpset6) { + std::string model = R"V0G0N( + + + + + + + 1000 + 4 + + + + + + + + 1 + 256 + 200 + 336 + + + + + + + + 1000 + 4 + + + 1 + 256 + 200 + 336 + + + + + 1000 + 256 + 7 + 7 + + + + + + + 1000 + 256 + 7 + 7 + + + + + + + + + + +)V0G0N"; + InferenceEngine::Core ie; + Blob::Ptr weights; + auto network = ie.ReadNetwork(model, weights); + InferenceEngine::ICNNNetwork::InputShapes newShapes; + newShapes["in0"] = {1256, 4}; + newShapes["in1"] = {1, 256, 7, 7}; ASSERT_NO_THROW(network.reshape(newShapes)); } @@ -943,28 +1017,28 @@ TEST_F(NGraphReshapeTests, ReshapeEDTopKROIs) { - - - - 5000 - 4 - - - 5000 - - - - - 1000 - 4 - - - + + + + 5000 + 4 + + + 5000 + + + + + 1000 + 4 + + + 1000 - 4 + 4 diff --git a/inference-engine/tests_deprecated/functional/vpu/common/layers/myriad_layers_roi_feature_extractor_test.hpp b/inference-engine/tests_deprecated/functional/vpu/common/layers/myriad_layers_roi_feature_extractor_test.hpp index 039afc28410..b475cc6246e 100644 --- a/inference-engine/tests_deprecated/functional/vpu/common/layers/myriad_layers_roi_feature_extractor_test.hpp +++ b/inference-engine/tests_deprecated/functional/vpu/common/layers/myriad_layers_roi_feature_extractor_test.hpp @@ -89,9 +89,6 @@ TEST_P(myriadLayersTestsROIFeatureExtractor_smoke, ROIFeatureExtractor) { {"output_size", std::to_string(test_params.output_size)}, {"sampling_ratio", std::to_string(test_params.sampling_ratio)}, {"pyramid_scales", pyramid_scales_str}, - {"distribute_rois_between_levels", "1"}, - {"preserve_rois_order", "1"}, - {"image_id", "0"}, }; makeSingleLayerNetwork(LayerInitParams("ExperimentalDetectronROIFeatureExtractor").params(layer_params)); diff --git a/model-optimizer/extensions/front/onnx/mask_rcnn_conversion.py b/model-optimizer/extensions/front/onnx/mask_rcnn_conversion.py index 7b6b30d7831..12ca0cfbb62 100644 --- a/model-optimizer/extensions/front/onnx/mask_rcnn_conversion.py +++ b/model-optimizer/extensions/front/onnx/mask_rcnn_conversion.py @@ -99,13 +99,11 @@ def insert_ExperimentalDetectronROIFeatureExtractor1(graph: Graph, replacement_d old_output_node = Node(graph, replacement_descriptions['ROIFeatureExtractor1_output']) input_fpn_head_nodes = [Node(graph, node_id) for node_id in input_fpn_heads] fpn_roi_align = ExperimentalDetectronROIFeatureExtractor(graph, {'name': 'ROIFeatureExtractor_1', - 'distribute_rois_between_levels': 1, - 'image_id': 0, 'output_size': 14, - 'preserve_rois_order': 1, 'pyramid_scales': int64_array( [4, 8, 16, 32, 64]), - 'sampling_ratio': 2, }).create_node() + 'sampling_ratio': 2, + 'in_ports_count': 5}).create_node() fpn_roi_align.in_port(0).connect(Node(graph, 'DetectionOutput').out_port(0)) for ind, fpn_node in enumerate(input_fpn_head_nodes): fpn_roi_align.in_port(ind + 1).connect(fpn_node.out_port(0)) @@ -118,13 +116,11 @@ def insert_ExperimentalDetectronROIFeatureExtractor2(graph: Graph, replacement_d old_output_node = Node(graph, replacement_descriptions['ROIFeatureExtractor2_output']) input_fpn_head_nodes = [Node(graph, node_id) for node_id in input_fpn_heads] fpn_roi_align = ExperimentalDetectronROIFeatureExtractor(graph, {'name': 'ROIFeatureExtractor_2', - 'distribute_rois_between_levels': 1, - 'image_id': 0, 'output_size': 7, - 'preserve_rois_order': 1, 'pyramid_scales': int64_array( [4, 8, 16, 32, 64]), - 'sampling_ratio': 2, }).create_node() + 'sampling_ratio': 2, + 'in_ports_count': 5}).create_node() fpn_roi_align.in_port(0).connect(Node(graph, replacement_descriptions['ROIFeatureExtractor2_input']).out_port(0)) for ind, fpn_node in enumerate(input_fpn_head_nodes): fpn_roi_align.in_port(ind + 1).connect(fpn_node.out_port(0)) diff --git a/model-optimizer/extensions/front/onnx/person_detection_crossroad_conversion.py b/model-optimizer/extensions/front/onnx/person_detection_crossroad_conversion.py index 329f3dd21e4..a84117a89d7 100644 --- a/model-optimizer/extensions/front/onnx/person_detection_crossroad_conversion.py +++ b/model-optimizer/extensions/front/onnx/person_detection_crossroad_conversion.py @@ -1,5 +1,5 @@ """ - Copyright (c) 2020 Intel Corporation + Copyright (c) 2018-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -41,10 +41,7 @@ def insert_experimental_layers(graph: Graph, input_fpn_heads: list, inp: str, ou input_fpn_head_nodes = [Node(graph, node_id) for node_id in input_fpn_heads] fpn_roi_align = ExperimentalDetectronROIFeatureExtractor(graph, {'name': output_name, - 'distribute_rois_between_levels': 1, - 'image_id': 0, 'output_size': 7, - 'preserve_rois_order': 1, 'pyramid_scales': int64_array( [4, 8, 16, 32, 64]), 'sampling_ratio': 2, }).create_node() diff --git a/model-optimizer/extensions/front/onnx/roifeatureextractor_ext.py b/model-optimizer/extensions/front/onnx/roifeatureextractor_ext.py index 9c070aa8a58..a6a25fbfab4 100644 --- a/model-optimizer/extensions/front/onnx/roifeatureextractor_ext.py +++ b/model-optimizer/extensions/front/onnx/roifeatureextractor_ext.py @@ -1,5 +1,5 @@ """ - Copyright (C) 2018-2020 Intel Corporation + Copyright (C) 2018-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -29,8 +29,6 @@ class ExperimentalDetectronROIFeatureExtractorFrontExtractor(FrontExtractorOp): def extract(cls, node): attrs = dict(output_size=onnx_attr(node, 'output_size', 'i', 7), sampling_ratio=onnx_attr(node, 'sampling_ratio', 'i', 2), - distribute_rois_between_levels=onnx_attr(node, 'distribute_rois_between_levels', 'i', 1), - preserve_rois_order=onnx_attr(node, 'preserve_rois_order', 'i', 1), aligned=onnx_attr(node, 'aligned', 'i', 0), num_classes=onnx_attr(node, 'num_classes', 'i', 81), post_nms_count=onnx_attr(node, 'post_nms_count', 'i', 2000), diff --git a/model-optimizer/extensions/ops/roifeatureextractor_onnx.py b/model-optimizer/extensions/ops/roifeatureextractor_onnx.py index 26c2e63b497..fc34290a4ea 100644 --- a/model-optimizer/extensions/ops/roifeatureextractor_onnx.py +++ b/model-optimizer/extensions/ops/roifeatureextractor_onnx.py @@ -1,5 +1,5 @@ """ - Copyright (C) 2018-2020 Intel Corporation + Copyright (C) 2018-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -26,9 +26,8 @@ class ExperimentalDetectronROIFeatureExtractor(Op): mandatory_props = dict( type=__class__.op, op=__class__.op, - version='experimental', + version='opset6', infer=__class__.infer, - in_ports_count=5, out_ports_count=2, ) @@ -36,13 +35,10 @@ class ExperimentalDetectronROIFeatureExtractor(Op): def backend_attrs(self): return [ - 'distribute_rois_between_levels', ('pyramid_scales', lambda node: ','.join(map(str, node['pyramid_scales']))), - 'image_id', 'output_size', 'sampling_ratio', - 'preserve_rois_order', - 'aligned'] + ('aligned', lambda node: str(bool(node.soft_get('aligned', False))).lower())] @staticmethod def infer(node): diff --git a/model-optimizer/mo/utils/ir_engine/ir_engine.py b/model-optimizer/mo/utils/ir_engine/ir_engine.py index fcd9c9252f0..5424b7bcaf9 100644 --- a/model-optimizer/mo/utils/ir_engine/ir_engine.py +++ b/model-optimizer/mo/utils/ir_engine/ir_engine.py @@ -331,8 +331,8 @@ class IREngine(object): n_value.append(int(val)) elif IREngine.__isfloat(val): n_value.append(float(val)) - elif val in ['True', 'False']: - n_value.append(val == 'True') + elif val in ['True', 'False', 'true', 'false']: + n_value.append(val in ['True', 'true']) else: n_value.append(val) diff --git a/ngraph/core/include/ngraph/op/experimental_detectron_roi_feature.hpp b/ngraph/core/include/ngraph/op/experimental_detectron_roi_feature.hpp new file mode 100644 index 00000000000..4a74aaf3343 --- /dev/null +++ b/ngraph/core/include/ngraph/op/experimental_detectron_roi_feature.hpp @@ -0,0 +1,76 @@ +//***************************************************************************** +// Copyright 2017-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#pragma once + +#include +#include +#include +#include "ngraph/attribute_adapter.hpp" +#include "ngraph/op/op.hpp" +#include "ngraph/op/util/attr_types.hpp" + +namespace ngraph +{ + namespace op + { + namespace v6 + { + /// \brief An operation ExperimentalDetectronROIFeatureExtractor, according to + /// the repository https://github.com/openvinotoolkit/training_extensions (see the file + /// pytorch_toolkit/instance_segmentation/segmentoly/rcnn/roi_feature_extractor.py). + class NGRAPH_API ExperimentalDetectronROIFeatureExtractor : public Op + { + public: + NGRAPH_RTTI_DECLARATION; + + /// \brief Structure that specifies attributes of the operation + struct Attributes + { + int64_t output_size; + int64_t sampling_ratio; + std::vector pyramid_scales; + bool aligned; + }; + + ExperimentalDetectronROIFeatureExtractor() = default; + /// \brief Constructs a ExperimentalDetectronROIFeatureExtractor operation. + /// + /// \param args Inputs of ExperimentalDetectronROIFeatureExtractor + /// \param attrs Operation attributes + ExperimentalDetectronROIFeatureExtractor(const OutputVector& args, + const Attributes& attrs); + + /// \brief Constructs a ExperimentalDetectronROIFeatureExtractor operation. + /// + /// \param args Inputs of ExperimentalDetectronROIFeatureExtractor + /// \param attrs Operation attributes + ExperimentalDetectronROIFeatureExtractor(const NodeVector& args, + const Attributes& attrs); + bool visit_attributes(AttributeVisitor& visitor) override; + + void validate_and_infer_types() override; + + std::shared_ptr + clone_with_new_inputs(const OutputVector& new_args) const override; + /// \brief Returns attributes of the operation. + const Attributes& get_attrs() const { return m_attrs; } + private: + Attributes m_attrs; + }; + } + } +} diff --git a/ngraph/core/include/ngraph/ops.hpp b/ngraph/core/include/ngraph/ops.hpp index 198d12692dc..de05f068769 100644 --- a/ngraph/core/include/ngraph/ops.hpp +++ b/ngraph/core/include/ngraph/ops.hpp @@ -61,6 +61,7 @@ #include "ngraph/op/exp.hpp" #include "ngraph/op/experimental_detectron_detection_output.hpp" #include "ngraph/op/experimental_detectron_prior_grid_generator.hpp" +#include "ngraph/op/experimental_detectron_roi_feature.hpp" #include "ngraph/op/extractimagepatches.hpp" #include "ngraph/op/fake_quantize.hpp" #include "ngraph/op/floor.hpp" diff --git a/ngraph/core/include/ngraph/opsets/opset6_tbl.hpp b/ngraph/core/include/ngraph/opsets/opset6_tbl.hpp index d103ed48232..907a8ba2441 100644 --- a/ngraph/core/include/ngraph/opsets/opset6_tbl.hpp +++ b/ngraph/core/include/ngraph/opsets/opset6_tbl.hpp @@ -176,5 +176,6 @@ NGRAPH_OP(Round, ngraph::op::v5) NGRAPH_OP(CTCGreedyDecoderSeqLen, ngraph::op::v6) NGRAPH_OP(ExperimentalDetectronDetectionOutput, ngraph::op::v6) NGRAPH_OP(ExperimentalDetectronPriorGridGenerator, ngraph::op::v6) +NGRAPH_OP(ExperimentalDetectronROIFeatureExtractor, ngraph::op::v6) NGRAPH_OP(GatherElements, ngraph::op::v6) NGRAPH_OP(MVN, ngraph::op::v6) diff --git a/ngraph/core/src/op/experimental_detectron_roi_feature.cpp b/ngraph/core/src/op/experimental_detectron_roi_feature.cpp new file mode 100644 index 00000000000..037ae0bffba --- /dev/null +++ b/ngraph/core/src/op/experimental_detectron_roi_feature.cpp @@ -0,0 +1,125 @@ +//***************************************************************************** +// Copyright 2017-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include + +#include "itt.hpp" +#include "ngraph/attribute_visitor.hpp" +#include "ngraph/op/experimental_detectron_roi_feature.hpp" + +using namespace std; +using namespace ngraph; + +NGRAPH_RTTI_DEFINITION(op::v6::ExperimentalDetectronROIFeatureExtractor, + "ExperimentalDetectronROIFeatureExtractor", + 6); + +op::v6::ExperimentalDetectronROIFeatureExtractor::ExperimentalDetectronROIFeatureExtractor( + const OutputVector& args, const Attributes& attrs) + : Op(args) + , m_attrs(attrs) +{ + constructor_validate_and_infer_types(); +} + +op::v6::ExperimentalDetectronROIFeatureExtractor::ExperimentalDetectronROIFeatureExtractor( + const NodeVector& args, const Attributes& attrs) + : ExperimentalDetectronROIFeatureExtractor(as_output_vector(args), attrs) +{ +} + +bool op::v6::ExperimentalDetectronROIFeatureExtractor::visit_attributes(AttributeVisitor& visitor) +{ + NGRAPH_OP_SCOPE(v6_ExperimentalDetectronROIFeatureExtractor_visit_attributes); + visitor.on_attribute("output_size", m_attrs.output_size); + visitor.on_attribute("sampling_ratio", m_attrs.sampling_ratio); + visitor.on_attribute("pyramid_scales", m_attrs.pyramid_scales); + visitor.on_attribute("aligned", m_attrs.aligned); + return true; +} + +void op::v6::ExperimentalDetectronROIFeatureExtractor::validate_and_infer_types() +{ + NGRAPH_OP_SCOPE(v6_ExperimentalDetectronROIFeatureExtractor_validate_and_infer_types); + NODE_VALIDATION_CHECK(this, get_input_size() >= 2, "At least two argument required."); + + auto rois_shape = get_input_partial_shape(0); + auto input_et = get_input_element_type(0); + + PartialShape out_shape = { + Dimension::dynamic(), Dimension::dynamic(), m_attrs.output_size, m_attrs.output_size}; + + PartialShape out_rois_shape = {Dimension::dynamic(), 4}; + if (rois_shape.rank().is_static()) + { + NODE_VALIDATION_CHECK( + this, rois_shape.rank().get_length() == 2, "Input rois rank must be equal to 2."); + + NODE_VALIDATION_CHECK(this, + rois_shape[1].is_static() && rois_shape[1].get_length() == 4u, + "The last dimension of the 'input_rois' input must be equal to 4. " + "Got: ", + rois_shape[1]); + + out_shape[0] = rois_shape[0]; + out_rois_shape[0] = rois_shape[0]; + } + + size_t num_of_inputs = get_input_size(); + Dimension channels_intersection; + + for (size_t i = 1; i < num_of_inputs; i++) + { + auto current_shape = get_input_partial_shape(i); + auto current_rank = current_shape.rank(); + + if (current_rank.is_static()) + { + NODE_VALIDATION_CHECK(this, + current_rank.get_length() == 4, + "Rank of each element of the pyramid must be equal to 4. Got: ", + current_rank); + + NODE_VALIDATION_CHECK(this, + current_shape[0].is_static() && + current_shape[0].get_length() == 1u, + "The first dimension of each pyramid element must be equal to 1. " + "Got: ", + current_shape[0]); + + channels_intersection &= current_shape[1]; + } + } + + NODE_VALIDATION_CHECK(this, + !channels_intersection.get_interval().empty(), + "The number of channels must be the same for all layers of the pyramid."); + + out_shape[1] = channels_intersection; + + set_output_type(0, input_et, out_shape); + set_output_type(1, input_et, out_rois_shape); +} + +shared_ptr op::v6::ExperimentalDetectronROIFeatureExtractor::clone_with_new_inputs( + const OutputVector& new_args) const +{ + NGRAPH_OP_SCOPE(v6_ExperimentalDetectronROIFeatureExtractor_clone_with_new_inputs); + check_new_args_count(this, new_args); + return make_shared(new_args, m_attrs); +} diff --git a/ngraph/test/CMakeLists.txt b/ngraph/test/CMakeLists.txt index 524592e2bf2..65829cd97f7 100644 --- a/ngraph/test/CMakeLists.txt +++ b/ngraph/test/CMakeLists.txt @@ -126,6 +126,7 @@ set(SRC type_prop/detection_output.cpp type_prop/depth_to_space.cpp type_prop/dyn_reshape.cpp + type_prop/experimental_detectron_roi_feature_extractor.cpp type_prop/strided_slice.cpp type_prop/elu.cpp type_prop/embeddingbag_offsetssum.cpp diff --git a/ngraph/test/type_prop/experimental_detectron_roi_feature_extractor.cpp b/ngraph/test/type_prop/experimental_detectron_roi_feature_extractor.cpp new file mode 100644 index 00000000000..4bc3f1a34fc --- /dev/null +++ b/ngraph/test/type_prop/experimental_detectron_roi_feature_extractor.cpp @@ -0,0 +1,171 @@ +//***************************************************************************** +// Copyright 2017-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include + +#include "gtest/gtest.h" +#include "ngraph/ngraph.hpp" +#include "util/type_prop.hpp" + +using namespace ngraph; + +using Attrs = op::v6::ExperimentalDetectronROIFeatureExtractor::Attributes; +using ExperimentalROI = op::v6::ExperimentalDetectronROIFeatureExtractor; + +TEST(type_prop, detectron_roi_feature_extractor) +{ + Attrs attrs; + attrs.aligned = false; + attrs.output_size = 14; + attrs.sampling_ratio = 2; + attrs.pyramid_scales = {4, 8, 16, 32}; + + auto input = std::make_shared(element::f32, Shape{1000, 4}); + auto pyramid_layer0 = std::make_shared(element::f32, Shape{1, 256, 200, 336}); + auto pyramid_layer1 = std::make_shared(element::f32, Shape{1, 256, 100, 168}); + auto pyramid_layer2 = std::make_shared(element::f32, Shape{1, 256, 50, 84}); + auto pyramid_layer3 = std::make_shared(element::f32, Shape{1, 256, 25, 42}); + + auto roi = std::make_shared( + NodeVector{input, pyramid_layer0, pyramid_layer1, pyramid_layer2, pyramid_layer3}, attrs); + + ASSERT_EQ(roi->get_output_element_type(0), element::f32); + EXPECT_EQ(roi->get_output_shape(0), (Shape{1000, 256, 14, 14})); +} + +TEST(type_prop, detectron_roi_feature_extractor_dynamic) +{ + Attrs attrs; + attrs.aligned = false; + attrs.output_size = 14; + attrs.sampling_ratio = 2; + attrs.pyramid_scales = {4, 8, 16, 32}; + + struct Shapes + { + PartialShape input_shape; + Dimension channels; + }; + + const auto dyn_dim = Dimension::dynamic(); + + std::vector shapes = { + {{1000, 4}, dyn_dim}, {{dyn_dim, 4}, 256}, {{dyn_dim, 4}, dyn_dim}}; + for (const auto& s : shapes) + { + auto layer0_shape = PartialShape{1, s.channels, 200, 336}; + auto layer1_shape = PartialShape{1, s.channels, 100, 168}; + auto layer2_shape = PartialShape{1, s.channels, 50, 84}; + auto layer3_shape = PartialShape{1, s.channels, 25, 42}; + + auto ref_out_shape = PartialShape{s.input_shape[0], s.channels, 14, 14}; + + auto input = std::make_shared(element::f32, s.input_shape); + auto pyramid_layer0 = std::make_shared(element::f32, layer0_shape); + auto pyramid_layer1 = std::make_shared(element::f32, layer1_shape); + auto pyramid_layer2 = std::make_shared(element::f32, layer2_shape); + auto pyramid_layer3 = std::make_shared(element::f32, layer3_shape); + + auto roi = std::make_shared( + NodeVector{input, pyramid_layer0, pyramid_layer1, pyramid_layer2, pyramid_layer3}, + attrs); + + ASSERT_EQ(roi->get_output_element_type(0), element::f32); + ASSERT_TRUE(roi->get_output_partial_shape(0).same_scheme(ref_out_shape)); + } +} + +TEST(type_prop, detectron_roi_feature_extractor_intervals) +{ + Attrs attrs; + attrs.aligned = false; + attrs.output_size = 14; + attrs.sampling_ratio = 2; + attrs.pyramid_scales = {4, 8, 16, 32}; + + struct Shapes + { + PartialShape input_shape; + Dimension channels[4]; + }; + + const auto dyn_dim = Dimension::dynamic(); + + std::vector shapes = { + {{1000, 4}, {Dimension(0, 128), Dimension(0, 256), Dimension(0, 64), Dimension(0, 33)}}, + {{1000, 4}, {Dimension(0, 128), Dimension(0, 256), Dimension(0, 64), Dimension(33)}}, + {{1000, 4}, {Dimension(0, 128), Dimension(0, 256), Dimension(64), Dimension(0, 72)}}, + {{1000, 4}, {Dimension(0, 128), Dimension(0, 256), Dimension(64), Dimension(64)}}, + {{1000, 4}, {Dimension(0, 512), Dimension(256), Dimension(0, 640), Dimension(0, 330)}}, + {{1000, 4}, {Dimension(0, 512), Dimension(256), Dimension(0, 640), Dimension(256)}}, + {{1000, 4}, {Dimension(0, 512), Dimension(256), Dimension(256), Dimension(0, 720)}}, + {{1000, 4}, {Dimension(0, 380), Dimension(256), Dimension(256), Dimension(256)}}, + {{1000, 4}, {Dimension(128), Dimension(0, 256), Dimension(0, 640), Dimension(0, 330)}}, + {{1000, 4}, {Dimension(128), Dimension(0, 256), Dimension(0, 640), Dimension(128)}}, + {{1000, 4}, {Dimension(128), Dimension(0, 256), Dimension(128), Dimension(0, 720)}}, + {{1000, 4}, {Dimension(128), Dimension(0, 256), Dimension(128), Dimension(128)}}, + {{1000, 4}, {Dimension(256), Dimension(256), Dimension(0, 640), Dimension(0, 330)}}, + {{1000, 4}, {Dimension(256), Dimension(256), Dimension(0, 640), Dimension(256)}}, + {{1000, 4}, {Dimension(256), Dimension(256), Dimension(256), Dimension(0, 330)}}, + {{1000, 4}, {Dimension(256), Dimension(256), Dimension(256), Dimension(256)}}, + {{dyn_dim, 4}, {Dimension(0, 128), Dimension(0, 256), Dimension(0, 64), Dimension(0, 33)}}, + {{dyn_dim, 4}, {Dimension(0, 128), Dimension(0, 256), Dimension(0, 64), Dimension(33)}}, + {{dyn_dim, 4}, {Dimension(0, 128), Dimension(0, 256), Dimension(64), Dimension(0, 72)}}, + {{dyn_dim, 4}, {Dimension(0, 128), Dimension(0, 256), Dimension(64), Dimension(64)}}, + {{dyn_dim, 4}, {Dimension(0, 512), Dimension(256), Dimension(0, 640), Dimension(0, 330)}}, + {{dyn_dim, 4}, {Dimension(0, 512), Dimension(256), Dimension(0, 640), Dimension(256)}}, + {{dyn_dim, 4}, {Dimension(0, 512), Dimension(256), Dimension(256), Dimension(0, 720)}}, + {{dyn_dim, 4}, {Dimension(0, 380), Dimension(256), Dimension(256), Dimension(256)}}, + {{dyn_dim, 4}, {Dimension(128), Dimension(0, 256), Dimension(0, 640), Dimension(0, 330)}}, + {{dyn_dim, 4}, {Dimension(128), Dimension(0, 256), Dimension(0, 640), Dimension(128)}}, + {{dyn_dim, 4}, {Dimension(128), Dimension(0, 256), Dimension(128), Dimension(0, 720)}}, + {{dyn_dim, 4}, {Dimension(128), Dimension(0, 256), Dimension(128), Dimension(128)}}, + {{dyn_dim, 4}, {Dimension(256), Dimension(256), Dimension(0, 640), Dimension(0, 330)}}, + {{dyn_dim, 4}, {Dimension(256), Dimension(256), Dimension(0, 640), Dimension(256)}}, + {{dyn_dim, 4}, {Dimension(256), Dimension(256), Dimension(256), Dimension(0, 330)}}, + {{dyn_dim, 4}, {Dimension(256), Dimension(256), Dimension(256), Dimension(256)}}}; + + for (const auto& s : shapes) + { + auto layer0_channels = s.channels[0]; + auto layer1_channels = s.channels[1]; + auto layer2_channels = s.channels[2]; + auto layer3_channels = s.channels[3]; + + auto layer0_shape = PartialShape{1, layer0_channels, 200, 336}; + auto layer1_shape = PartialShape{1, layer1_channels, 100, 168}; + auto layer2_shape = PartialShape{1, layer2_channels, 50, 84}; + auto layer3_shape = PartialShape{1, layer3_channels, 25, 42}; + + auto expected_channels = + layer0_channels & layer1_channels & layer2_channels & layer3_channels; + + auto ref_out_shape = PartialShape{s.input_shape[0], expected_channels, 14, 14}; + + auto input = std::make_shared(element::f32, s.input_shape); + auto pyramid_layer0 = std::make_shared(element::f32, layer0_shape); + auto pyramid_layer1 = std::make_shared(element::f32, layer1_shape); + auto pyramid_layer2 = std::make_shared(element::f32, layer2_shape); + auto pyramid_layer3 = std::make_shared(element::f32, layer3_shape); + + auto roi = std::make_shared( + NodeVector{input, pyramid_layer0, pyramid_layer1, pyramid_layer2, pyramid_layer3}, + attrs); + + ASSERT_EQ(roi->get_output_element_type(0), element::f32); + ASSERT_TRUE(roi->get_output_partial_shape(0).same_scheme(ref_out_shape)); + } +}