IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated convert_nms_to_nms_ie transformation to support dynamic shapes (#614)

Browse Source
This commit is contained in:
Gleb Kazantaev
2020-05-27 00:38:25 +03:00
committed by GitHub
parent 851f64946a
commit 6788153ba9
5 changed files with 186 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
inference-engine/src/transformations/include/ngraph_ops/nms_ie.hpp
View File

@@ -24,17 +24,15 @@ public:
const Output<Node>& max_output_boxes_per_class,
const Output<Node>& iou_threshold,
const Output<Node>& score_threshold,
const Shape& output_shape,
int center_point_box,
bool sort_result_descending);
void validate_and_infer_types() override;
std::shared_ptr<Node> copy_with_new_args(const NodeVector& new_args) const override;
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector & new_args) const override;
int m_center_point_box;
bool m_sort_result_descending = true;
Shape m_output_shape;
};
} // namespace op

8
inference-engine/src/transformations/include/transformations/convert_opset1_to_legacy/convert_nms_to_nms_ie.hpp
View File

@@ -20,6 +20,14 @@ class INFERENCE_ENGINE_API_CLASS(ConvertNMSToNMSIE);
} // namespace pass
} // namespace ngraph
/*
* Description:
* This transformation converts opset1::NonMaxSuppression to legacy NonMaxSuppressionIE
* NonMaxSuppressionIE takes max_output_boxes_per_class, iou_threshold and score_threshold
* inputs as 1D tensors when original operation requires scalars. And for this inputs
* we insert Unsqueeze operations.
*/
class ngraph::pass::ConvertNMSToNMSIE : public ngraph::pass::GraphRewrite {
public:
ConvertNMSToNMSIE() : GraphRewrite() {

25
inference-engine/src/transformations/src/ngraph_ops/nms_ie.cpp
View File

@@ -6,6 +6,8 @@
#include <memory>
#include <ngraph/opsets/opset1.hpp>
using namespace std;
using namespace ngraph;
@@ -16,24 +18,27 @@ op::NonMaxSuppressionIE::NonMaxSuppressionIE(const Output<Node> &boxes,
const Output<Node> &max_output_boxes_per_class,
const Output<Node> &iou_threshold,
const Output<Node> &score_threshold,
const Shape &output_shape,
int center_point_box,
bool sort_result_descending)
: Op({boxes, scores, max_output_boxes_per_class, iou_threshold, score_threshold}),
m_center_point_box{center_point_box}, m_sort_result_descending{sort_result_descending}, m_output_shape{output_shape} {
: Op({boxes, scores, max_output_boxes_per_class, iou_threshold, score_threshold})
, m_center_point_box(center_point_box)
, m_sort_result_descending(sort_result_descending) {
constructor_validate_and_infer_types();
}
std::shared_ptr<Node> op::NonMaxSuppressionIE::copy_with_new_args(const NodeVector &new_args) const {
if (new_args.size() != 5) {
throw ngraph_error("Incorrect number of new arguments");
}
std::shared_ptr<Node> op::NonMaxSuppressionIE::clone_with_new_inputs(const ngraph::OutputVector & new_args) const {
check_new_args_count(this, new_args);
return make_shared<NonMaxSuppressionIE>(new_args.at(0), new_args.at(1), new_args.at(2), new_args.at(3),
new_args.at(4), m_output_shape, m_center_point_box, m_sort_result_descending);
new_args.at(4), m_center_point_box, m_sort_result_descending);
}
void op::NonMaxSuppressionIE::validate_and_infer_types() {
set_output_type(0, element::i32, m_output_shape);
// Calculate output shape using opset1::NonMaxSuppression
auto nms = std::make_shared<opset1::NonMaxSuppression>(input_value(0),
input_value(1),
std::make_shared<opset1::Squeeze>(input_value(2), opset1::Constant::create(element::i64, Shape{1}, {0})),
std::make_shared<opset1::Squeeze>(input_value(3), opset1::Constant::create(element::i64, Shape{1}, {0})),
std::make_shared<opset1::Squeeze>(input_value(4), opset1::Constant::create(element::i64, Shape{1}, {0})));
set_output_type(0, nms->output(0).get_element_type(), nms->output(0).get_partial_shape());
}

49
inference-engine/src/transformations/src/transformations/convert_opset1_to_legacy/convert_nms_to_nms_ie.cpp
View File

@@ -13,45 +13,51 @@
#include <ngraph/rt_info.hpp>
void ngraph::pass::ConvertNMSToNMSIE::convert_nms_to_nms_ie() {
auto input_0 = std::make_shared<pattern::op::Label>(element::f32, Shape{1, 1, 4});
auto input_1 = std::make_shared<pattern::op::Label>(element::f32, Shape{1, 1, 1});
auto max_per_class = std::make_shared<pattern::op::Label>(element::i64, Shape{});
auto iou_threshold = std::make_shared<pattern::op::Label>(element::f32, Shape{});
auto score_threshold = std::make_shared<pattern::op::Label>(element::f32, Shape{});
auto nms = std::make_shared<ngraph::opset1::NonMaxSuppression>(input_0, input_1, max_per_class, iou_threshold,
score_threshold);
auto nms = std::make_shared<pattern::op::Label>(element::f32, Shape{}, pattern::has_class<opset1::NonMaxSuppression>());
ngraph::graph_rewrite_callback callback = [](pattern::Matcher &m) {
auto nms = std::dynamic_pointer_cast<ngraph::opset1::NonMaxSuppression>(m.get_match_root());
auto nms = std::dynamic_pointer_cast<opset1::NonMaxSuppression>(m.get_match_root());
if (!nms) {
return false;
}
if (nms->input(2).get_shape().size() == 1 && nms->input(3).get_shape().size() == 1 &&
nms->input(4).get_shape().size() == 1) {
const auto max_output_boxes_per_class_rank = nms->input(2).get_partial_shape().rank();
const auto iou_threshold_rank = nms->input(3).get_partial_shape().rank();
const auto score_threshold_rank = nms->input(4).get_partial_shape().rank();
// Check that required ranks are not dynamic
if (max_output_boxes_per_class_rank.is_dynamic() ||
iou_threshold_rank.is_dynamic() ||
score_threshold_rank.is_dynamic()) {
return false;
}
if (max_output_boxes_per_class_rank.get_length() == 1 &&
iou_threshold_rank.get_length() == 1 &&
score_threshold_rank.get_length() == 1) {
return false;
}
// vector of new nGraph operations
NodeVector new_ops;
auto new_max_per_class = nms->input(2).get_source_output();
if (nms->input(2).get_shape().empty()) {
auto new_max_per_class = nms->input_value(2);
if (max_output_boxes_per_class_rank.get_length() == 0) {
new_max_per_class = std::make_shared<ngraph::op::Unsqueeze>(
nms->input(2).get_source_output().get_node_shared_ptr(),
nms->input_value(2),
opset1::Constant::create(element::i64, Shape{1}, {0}));
new_ops.push_back(new_max_per_class.get_node_shared_ptr());
}
auto new_iou_threshold = nms->input(3).get_source_output();
if (nms->input(3).get_shape().empty()) {
auto new_iou_threshold = nms->input_value(3);
if (iou_threshold_rank.get_length() == 0) {
new_iou_threshold = std::make_shared<ngraph::op::Unsqueeze>(
nms->input(3).get_source_output().get_node_shared_ptr(),
nms->input_value(3),
opset1::Constant::create(element::i64, Shape{1}, {0}));
new_ops.push_back(new_iou_threshold.get_node_shared_ptr());
}
auto new_score_threshold = nms->input(4).get_source_output();
if (nms->input(4).get_shape().empty()) {
auto new_score_threshold = nms->input_value(4);
if (score_threshold_rank.get_length() == 0) {
new_score_threshold = std::make_shared<ngraph::op::Unsqueeze>(
nms->input(4).get_source_output().get_node_shared_ptr(),
nms->input_value(4),
opset1::Constant::create(element::i64, Shape{1}, {0}));
new_ops.push_back(new_score_threshold.get_node_shared_ptr());
}
@@ -67,12 +73,11 @@ void ngraph::pass::ConvertNMSToNMSIE::convert_nms_to_nms_ie() {
throw ngraph_error("NonMaxSuppression layer " + nms->get_friendly_name() +
" has unsupported box encoding");
}
auto new_nms = std::make_shared<ngraph::op::NonMaxSuppressionIE>(nms->input(0).get_source_output(),
nms->input(1).get_source_output(),
auto new_nms = std::make_shared<ngraph::op::NonMaxSuppressionIE>(nms->input_value(0),
nms->input_value(1),
new_max_per_class,
new_iou_threshold,
new_score_threshold,
nms->output(0).get_shape(),
center_point_box,
nms->get_sort_result_descending());
new_ops.push_back(new_nms);

135
inference-engine/tests/functional/inference_engine/transformations/convert_nms_to_nms_ie_test.cpp Normal file
View File

@@ -0,0 +1,135 @@
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <gtest/gtest.h>
#include <string>
#include <memory>
#include <queue>
#include <ngraph/function.hpp>
#include <ngraph/opsets/opset1.hpp>
#include <transformations/convert_opset1_to_legacy/convert_nms_to_nms_ie.hpp>
#include <transformations/init_node_info.hpp>
#include <transformations/utils/utils.hpp>
#include <ngraph_ops/nms_ie.hpp>
#include "ngraph_test_utils.hpp"
using namespace testing;
using namespace ngraph;
TEST(TransformationTests, ConvertNMSToNMSIEStatic) {
std::shared_ptr<Function> f(nullptr), f_ref(nullptr);
{
auto boxes = std::make_shared<opset1::Parameter>(element::f32, Shape{1, 1000, 4});
auto scores = std::make_shared<opset1::Parameter>(element::f32, Shape{1, 1, 1000});
auto max_output_boxes_per_class = opset1::Constant::create(element::i64, Shape{}, {10});
auto iou_threshold = opset1::Constant::create(element::f32, Shape{}, {0.75});
auto score_threshold = opset1::Constant::create(element::f32, Shape{}, {0.7});
auto nms = std::make_shared<opset1::NonMaxSuppression>(boxes, scores, max_output_boxes_per_class,
iou_threshold, score_threshold, opset1::NonMaxSuppression::BoxEncodingType::CORNER, true);
f = std::make_shared<Function>(NodeVector{nms}, ParameterVector{boxes, scores});
pass::InitNodeInfo().run_on_function(f);
pass::ConvertNMSToNMSIE().run_on_function(f);
f->validate_nodes_and_infer_types();
ASSERT_NO_THROW(check_rt_info(f));
}
{
auto boxes = std::make_shared<opset1::Parameter>(element::f32, Shape{1, 1000, 4});
auto scores = std::make_shared<opset1::Parameter>(element::f32, Shape{1, 1, 1000});
auto max_output_boxes_per_class = opset1::Constant::create(element::i64, Shape{}, {10});
auto iou_threshold = opset1::Constant::create(element::f32, Shape{}, {0.75});
auto score_threshold = opset1::Constant::create(element::f32, Shape{}, {0.7});
auto nms = std::make_shared<op::NonMaxSuppressionIE>(boxes, scores,
std::make_shared<opset1::Unsqueeze>(max_output_boxes_per_class, opset1::Constant::create(element::i64, Shape{1}, {0})),
std::make_shared<opset1::Unsqueeze>(iou_threshold, opset1::Constant::create(element::i64, Shape{1}, {0})),
std::make_shared<opset1::Unsqueeze>(score_threshold, opset1::Constant::create(element::i64, Shape{1}, {0})),
0, true);
f_ref = std::make_shared<Function>(NodeVector{nms}, ParameterVector{boxes, scores});
}
auto res = compare_functions(f, f_ref);
ASSERT_TRUE(res.first) << res.second;
}
TEST(TransformationTests, ConvertNMSToNMSIEDynamic1) {
std::shared_ptr<Function> f(nullptr), f_ref(nullptr);
{
auto boxes = std::make_shared<opset1::Parameter>(element::f32, PartialShape::dynamic());
auto scores = std::make_shared<opset1::Parameter>(element::f32, PartialShape::dynamic());
auto max_output_boxes_per_class = opset1::Constant::create(element::i64, Shape{}, {10});
auto iou_threshold = opset1::Constant::create(element::f32, Shape{}, {0.75});
auto score_threshold = opset1::Constant::create(element::f32, Shape{}, {0.7});
auto nms = std::make_shared<opset1::NonMaxSuppression>(boxes, scores, max_output_boxes_per_class,
iou_threshold, score_threshold, opset1::NonMaxSuppression::BoxEncodingType::CORNER, true);
f = std::make_shared<Function>(NodeVector{nms}, ParameterVector{boxes, scores});
pass::InitNodeInfo().run_on_function(f);
pass::ConvertNMSToNMSIE().run_on_function(f);
f->validate_nodes_and_infer_types();
ASSERT_NO_THROW(check_rt_info(f));
}
{
auto boxes = std::make_shared<opset1::Parameter>(element::f32, PartialShape::dynamic());
auto scores = std::make_shared<opset1::Parameter>(element::f32, PartialShape::dynamic());
auto max_output_boxes_per_class = opset1::Constant::create(element::i64, Shape{}, {10});
auto iou_threshold = opset1::Constant::create(element::f32, Shape{}, {0.75});
auto score_threshold = opset1::Constant::create(element::f32, Shape{}, {0.7});
auto nms = std::make_shared<op::NonMaxSuppressionIE>(boxes, scores,
std::make_shared<opset1::Unsqueeze>(max_output_boxes_per_class, opset1::Constant::create(element::i64, Shape{1}, {0})),
std::make_shared<opset1::Unsqueeze>(iou_threshold, opset1::Constant::create(element::i64, Shape{1}, {0})),
std::make_shared<opset1::Unsqueeze>(score_threshold, opset1::Constant::create(element::i64, Shape{1}, {0})),
0, true);
f_ref = std::make_shared<Function>(NodeVector{nms}, ParameterVector{boxes, scores});
}
auto res = compare_functions(f, f_ref);
ASSERT_TRUE(res.first) << res.second;
}
TEST(TransformationTests, ConvertNMSToNMSIEDynamic2) {
std::shared_ptr<Function> f(nullptr), f_ref(nullptr);
{
auto boxes = std::make_shared<opset1::Parameter>(element::f32, PartialShape{DYN, 1000, 4});
auto scores = std::make_shared<opset1::Parameter>(element::f32, PartialShape{DYN, 1, 1000});
auto max_output_boxes_per_class = opset1::Constant::create(element::i64, Shape{}, {10});
auto iou_threshold = opset1::Constant::create(element::f32, Shape{}, {0.75});
auto score_threshold = opset1::Constant::create(element::f32, Shape{}, {0.7});
auto nms = std::make_shared<opset1::NonMaxSuppression>(boxes, scores, max_output_boxes_per_class,
iou_threshold, score_threshold, opset1::NonMaxSuppression::BoxEncodingType::CORNER, true);
f = std::make_shared<Function>(NodeVector{nms}, ParameterVector{boxes, scores});
pass::InitNodeInfo().run_on_function(f);
pass::ConvertNMSToNMSIE().run_on_function(f);
f->validate_nodes_and_infer_types();
ASSERT_NO_THROW(check_rt_info(f));
}
{
auto boxes = std::make_shared<opset1::Parameter>(element::f32, PartialShape{DYN, 1000, 4});
auto scores = std::make_shared<opset1::Parameter>(element::f32, PartialShape{DYN, 1, 1000});
auto max_output_boxes_per_class = opset1::Constant::create(element::i64, Shape{}, {10});
auto iou_threshold = opset1::Constant::create(element::f32, Shape{}, {0.75});
auto score_threshold = opset1::Constant::create(element::f32, Shape{}, {0.7});
auto nms = std::make_shared<op::NonMaxSuppressionIE>(boxes, scores,
std::make_shared<opset1::Unsqueeze>(max_output_boxes_per_class, opset1::Constant::create(element::i64, Shape{1}, {0})),
std::make_shared<opset1::Unsqueeze>(iou_threshold, opset1::Constant::create(element::i64, Shape{1}, {0})),
std::make_shared<opset1::Unsqueeze>(score_threshold, opset1::Constant::create(element::i64, Shape{1}, {0})),
0, true);
f_ref = std::make_shared<Function>(NodeVector{nms}, ParameterVector{boxes, scores});
}
auto res = compare_functions(f, f_ref);
ASSERT_TRUE(res.first) << res.second;
}