IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for nonconstant optional NMS-5 inputs (#3640)

Browse Source
This commit is contained in:
Bartosz Sledz 2021-01-26 14:41:28 +01:00 committed by GitHub
parent d58b4c65c8
commit 247606baf6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 456 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
inference-engine/src/legacy_api/src/transformations/convert_opset1_to_legacy/convert_nms_5_to_legacy.cpp
View File

@ -73,7 +73,7 @@ ngraph::pass::ConvertNMS5ToLegacyMatcher::ConvertNMS5ToLegacyMatcher(bool force_
std::shared_ptr<op::NonMaxSuppressionIE3> nms_legacy{nullptr};
auto output_type = force_i32_output_type ? element::i32 : nms_5->get_output_type();
if (num_of_inputs > 5 && nms_5->soft_nms_sigma_from_input() != 0.0f) {
if (num_of_inputs > 5 && !nms_5->is_soft_nms_sigma_constant_and_default()) {
new_soft_nms_sigma = std::make_shared<opset1::Reshape>(new_args.at(5), new_shape_for_soft_nms_sigma, true);
new_ops.emplace_back(new_soft_nms_sigma.get_node_shared_ptr());
nms_legacy = std::make_shared<op::NonMaxSuppressionIE3>(

2
inference-engine/src/transformations/src/transformations/op_conversions/convert_nms_to_nms_ie_internal.cpp
View File

@ -73,7 +73,7 @@ ngraph::pass::ConvertNMSToNMSIEInternal::ConvertNMSToNMSIEInternal() {
std::shared_ptr<op::internal::NonMaxSuppressionIEInternal> nms_legacy{nullptr};
if (num_of_inputs > 5 && nms_5->soft_nms_sigma_from_input() != 0.0f) {
if (num_of_inputs > 5 && !nms_5->is_soft_nms_sigma_constant_and_default()) {
new_soft_nms_sigma = std::make_shared<opset1::Reshape>(new_args.at(5), new_shape_for_soft_nms_sigma, true);
new_ops.emplace_back(new_soft_nms_sigma.get_node_shared_ptr());
nms_legacy = std::make_shared<op::internal::NonMaxSuppressionIEInternal>(

1
ngraph/core/include/ngraph/op/non_max_suppression.hpp
View File

@ -378,6 +378,7 @@ namespace ngraph
float iou_threshold_from_input() const;
float score_threshold_from_input() const;
float soft_nms_sigma_from_input() const;
bool is_soft_nms_sigma_constant_and_default() const;
protected:
BoxEncodingType m_box_encoding = BoxEncodingType::CORNER;

11
ngraph/core/src/op/non_max_suppression.cpp
View File

@ -892,6 +892,17 @@ float op::v5::NonMaxSuppression::soft_nms_sigma_from_input() const
return soft_nms_sigma;
}
bool op::v5::NonMaxSuppression::is_soft_nms_sigma_constant_and_default() const
{
auto soft_nms_sigma_node = input_value(soft_nms_sigma_port).get_node_shared_ptr();
if (inputs().size() < 6 || !ngraph::op::is_constant(soft_nms_sigma_node))
{
return false;
}
const auto soft_nms_sigma_input = as_type_ptr<op::Constant>(soft_nms_sigma_node);
return soft_nms_sigma_input->cast_vector<float>().at(0) == 0.0f;
}
bool ngraph::op::v5::NonMaxSuppression::visit_attributes(AttributeVisitor& visitor)
{
NGRAPH_OP_SCOPE(v5_NonMaxSuppression_visit_attributes);

4
ngraph/python/tests/__init__.py
View File

@ -201,9 +201,7 @@ xfail_issue_39658 = xfail_test(reason="RuntimeError: Tile operation has a form t
" z should be converted to TileIE operation.")
xfail_issue_39659 = xfail_test(reason="RuntimeError: Broadcast operation has a form that is not supported."
" y should be converted to Tile operation.")
xfail_issue_39661 = xfail_test(reason="RuntimeError: NonMaxSuppression operation has a form that is not "
"supported. selected_indices should be converted to "
"NonMaxSuppressionIE operation.")
xfail_issue_45344 = xfail_test(reason="Unsupported dynamic ops: v3::NonMaxSuppressionIE3")
xfail_issue_39662 = xfail_test(reason="RuntimeError: 'ScatterElementsUpdate' layer with name 'y' have "
"indices value that points to non-existing output tensor element")

4
ngraph/python/tests/test_onnx/test_backend.py
View File

@ -29,7 +29,7 @@ from tests import (BACKEND_NAME,
xfail_issue_39656,
xfail_issue_39658,
xfail_issue_39659,
xfail_issue_39661,
xfail_issue_45344,
xfail_issue_39662,
xfail_issue_33540,
xfail_issue_34314,
@ -165,7 +165,7 @@ tests_expected_to_fail = [
"OnnxBackendNodeModelTest.test_constantofshape_float_ones_cpu",
"OnnxBackendNodeModelTest.test_constantofshape_int_zeros_cpu",
"OnnxBackendNodeModelTest.test_constantofshape_int_shape_zero_cpu"),
(xfail_issue_39661,
(xfail_issue_45344,
"OnnxBackendNodeModelTest.test_nonmaxsuppression_center_point_box_format_cpu",
"OnnxBackendNodeModelTest.test_nonmaxsuppression_flipped_coordinates_cpu",
"OnnxBackendNodeModelTest.test_nonmaxsuppression_identical_boxes_cpu",

81
ngraph/test/backend/non_max_suppression.in.cpp
View File

@ -612,3 +612,84 @@ NGRAPH_TEST(${BACKEND_NAME}, nonmaxsuppression_two_classes)
EXPECT_EQ(expected_selected_scores, selected_scores_value);
EXPECT_EQ(expected_valid_outputs, valid_outputs_value);
}
NGRAPH_TEST(${BACKEND_NAME}, nonmaxsuppression_suppress_by_IOU_and_scores_without_constants)
{
std::vector<float> boxes_data = {0.0f, 0.0f, 1.0f, 1.0f, 0.0f, 0.1f, 1.0f, 1.1f,
0.0f, -0.1f, 1.0f, 0.9f, 0.0f, 10.0f, 1.0f, 11.0f,
0.0f, 10.1f, 1.0f, 11.1f, 0.0f, 100.0f, 1.0f, 101.0f};
std::vector<float> scores_data = {0.9f, 0.75f, 0.6f, 0.95f, 0.5f, 0.3f};
std::vector<int64_t> max_output_boxes_per_class_data = {1};
std::vector<float> iou_threshold_data = {0.4f};
std::vector<float> score_threshold_data = {0.2f};
const auto box_encoding = op::v5::NonMaxSuppression::BoxEncodingType::CORNER;
const auto boxes_shape = Shape{1, 6, 4};
const auto scores_shape = Shape{1, 1, 6};
const auto boxes = make_shared<op::Parameter>(element::f32, boxes_shape);
const auto scores = make_shared<op::Parameter>(element::f32, scores_shape);
const auto max_output_boxes_per_class = make_shared<op::Parameter>(element::i64, Shape{1});
const auto score_treshold = make_shared<op::Parameter>(element::f32, Shape{1});
const auto iou_threshold = make_shared<op::Parameter>(element::f32, Shape{1});
const auto soft_nms_sigma = make_shared<op::Parameter>(element::f32, Shape{1});
auto nms = make_shared<op::v5::NonMaxSuppression>(boxes,
scores,
max_output_boxes_per_class,
iou_threshold,
score_treshold,
soft_nms_sigma,
box_encoding,
false);
auto f = make_shared<Function>(nms,
ParameterVector{boxes,
scores,
max_output_boxes_per_class,
iou_threshold,
score_treshold,
soft_nms_sigma});
auto backend = runtime::Backend::create("${BACKEND_NAME}");
auto selected_indeces = backend->create_tensor(element::i64, Shape{1, 3});
auto selected_scores = backend->create_tensor(element::f32, Shape{1, 3});
auto valid_outputs = backend->create_tensor(element::i64, Shape{1});
auto backend_boxes = backend->create_tensor(element::f32, boxes_shape);
auto backend_scores = backend->create_tensor(element::f32, scores_shape);
auto backend_max_output_boxes_per_class = backend->create_tensor(element::i64, {1});
auto backend_iou_threshold = backend->create_tensor(element::f32, {1});
auto backend_score_threshold = backend->create_tensor(element::f32, {1});
auto backend_soft_nms_sigma = backend->create_tensor(element::f32, {1});
copy_data(backend_boxes, boxes_data);
copy_data(backend_scores, scores_data);
copy_data(backend_max_output_boxes_per_class, max_output_boxes_per_class_data);
copy_data(backend_iou_threshold, iou_threshold_data);
copy_data(backend_score_threshold, score_threshold_data);
copy_data(backend_soft_nms_sigma, std::vector<float>(0.0));
auto handle = backend->compile(f);
handle->call({selected_indeces, selected_scores, valid_outputs},
{backend_boxes,
backend_scores,
backend_max_output_boxes_per_class,
backend_iou_threshold,
backend_score_threshold,
backend_soft_nms_sigma});
auto selected_indeces_value = read_vector<int64_t>(selected_indeces);
auto selected_scores_value = read_vector<float>(selected_scores);
auto valid_outputs_value = read_vector<int64_t>(valid_outputs);
std::vector<int64_t> expected_selected_indices = {0, 0, 3};
std::vector<float> expected_selected_scores = {0.0f, 0.0f, 0.95f};
std::vector<int64_t> expected_valid_outputs = {1};
EXPECT_EQ(expected_selected_indices, selected_indeces_value);
EXPECT_EQ(expected_selected_scores, selected_scores_value);
EXPECT_EQ(expected_valid_outputs, valid_outputs_value);
}

115
ngraph/test/models/onnx/nonmaxsuppression_center_point_box_format.prototxt Normal file
View File

@ -0,0 +1,115 @@
ir_version: 5
producer_name: "backend-test"
graph {
node {
input: "boxes"
input: "scores"
input: "max_output_boxes_per_class"
input: "iou_threshold"
input: "score_threshold"
output: "selected_indices"
op_type: "NonMaxSuppression"
attribute {
name: "center_point_box"
i: 1
type: INT
}
}
name: "test_nonmaxsuppression_center_point_box_format"
input {
name: "boxes"
type {
tensor_type {
elem_type: 1
shape {
dim {
dim_value: 1
}
dim {
dim_value: 6
}
dim {
dim_value: 4
}
}
}
}
}
input {
name: "scores"
type {
tensor_type {
elem_type: 1
shape {
dim {
dim_value: 1
}
dim {
dim_value: 1
}
dim {
dim_value: 6
}
}
}
}
}
input {
name: "max_output_boxes_per_class"
type {
tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
}
}
input {
name: "iou_threshold"
type {
tensor_type {
elem_type: 1
shape {
dim {
dim_value: 1
}
}
}
}
}
input {
name: "score_threshold"
type {
tensor_type {
elem_type: 1
shape {
dim {
dim_value: 1
}
}
}
}
}
output {
name: "selected_indices"
type {
tensor_type {
elem_type: 7
shape {
dim {
dim_value: 3
}
dim {
dim_value: 3
}
}
}
}
}
}
opset_import {
version: 10
}

110
ngraph/test/models/onnx/nonmaxsuppression_single_box.prototxt Normal file
View File

@ -0,0 +1,110 @@
ir_version: 5
producer_name: "backend-test"
graph {
node {
input: "boxes"
input: "scores"
input: "max_output_boxes_per_class"
input: "iou_threshold"
input: "score_threshold"
output: "selected_indices"
op_type: "NonMaxSuppression"
}
name: "test_nonmaxsuppression_single_box"
input {
name: "boxes"
type {
tensor_type {
elem_type: 1
shape {
dim {
dim_value: 1
}
dim {
dim_value: 1
}
dim {
dim_value: 4
}
}
}
}
}
input {
name: "scores"
type {
tensor_type {
elem_type: 1
shape {
dim {
dim_value: 1
}
dim {
dim_value: 1
}
dim {
dim_value: 1
}
}
}
}
}
input {
name: "max_output_boxes_per_class"
type {
tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
}
}
input {
name: "iou_threshold"
type {
tensor_type {
elem_type: 1
shape {
dim {
dim_value: 1
}
}
}
}
}
input {
name: "score_threshold"
type {
tensor_type {
elem_type: 1
shape {
dim {
dim_value: 1
}
}
}
}
}
output {
name: "selected_indices"
type {
tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
dim {
dim_value: 3
}
}
}
}
}
}
opset_import {
version: 10
}

36
ngraph/test/onnx/onnx_import.in.cpp
View File

@ -925,6 +925,42 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_add_bcast)
test_case.run();
}
NGRAPH_TEST(${BACKEND_NAME}, onnx_model_nonmaxsuppression_center_point_box_format)
{
auto function = onnx_import::import_onnx_model(file_util::path_join(
SERIALIZED_ZOO, "onnx/nonmaxsuppression_center_point_box_format.prototxt"));
auto test_case = test::TestCase<TestEngine, test::TestCaseType::DYNAMIC>(function);
test_case.add_input(std::vector<float>(
{0.5f, 0.5f, 1.0f, 1.0f, 0.5f, 0.6f, 1.0f, 1.0f, 0.5f, 0.4f, 1.0f, 1.0f,
0.5f, 10.5f, 1.0f, 1.0f, 0.5f, 10.6f, 1.0f, 1.0f, 0.5f, 100.5f, 1.0f, 1.0f})); // boxes
test_case.add_input(std::vector<float>({0.9f, 0.75f, 0.6f, 0.95f, 0.5f, 0.3f})); // scores
test_case.add_input(std::vector<int64_t>({3})); // max_output_boxes_per_class
test_case.add_input(std::vector<float>({0.5f})); // iou_threshold
test_case.add_input(std::vector<float>({0.0f})); // score_threshold
test_case.add_expected_output<int64_t>(Shape{3, 3}, {0, 0, 3, 0, 0, 0, 0, 0, 5});
test_case.run();
}
NGRAPH_TEST(${BACKEND_NAME}, onnx_model_nonmaxsuppression_single_box)
{
auto function = onnx_import::import_onnx_model(
file_util::path_join(SERIALIZED_ZOO, "onnx/nonmaxsuppression_single_box.prototxt"));
auto test_case = test::TestCase<TestEngine, test::TestCaseType::DYNAMIC>(function);
test_case.add_input(std::vector<float>({0.0f, 0.0f, 1.0f, 1.0f})); // boxes
test_case.add_input(std::vector<float>({0.9f})); // scores
test_case.add_input(std::vector<int64_t>({3})); // max_output_boxes_per_class
test_case.add_input(std::vector<float>({0.5f})); // iou_threshold
test_case.add_input(std::vector<float>({0.0f})); // score_threshold
test_case.add_expected_output<int64_t>(Shape{1, 3}, {0, 0, 0});
test_case.run();
}
NGRAPH_TEST(${BACKEND_NAME}, onnx_model_reduce_log_sum)
{
auto function = onnx_import::import_onnx_model(

5
ngraph/test/runtime/ie/unit_test.manifest
View File

@ -1171,6 +1171,11 @@ IE_CPU.nonmaxsuppression_suppress_by_IOU_and_scores
IE_CPU.nonmaxsuppression_two_batches
IE_CPU.nonmaxsuppression_two_classes
# Unsupported dynamic ops: v3::NonMaxSuppressionIE3
IE_CPU.onnx_model_nonmaxsuppression_center_point_box_format
IE_CPU.onnx_model_nonmaxsuppression_single_box
IE_CPU.nonmaxsuppression_suppress_by_IOU_and_scores_without_constants
# Bug in CPU plugin for ROIPooling when pooled size is 1x1 and method is bilinear
IE_CPU.roi_pooling_1x1_bilinear

96
ngraph/test/runtime/interpreter/evaluates_map.cpp
View File

@ -510,6 +510,93 @@ namespace
scores_ps[0].get_length();
}
}
return result;
}
std::vector<int64_t> get_integers(const std::shared_ptr<HostTensor>& input,
const Shape& shape)
{
size_t input_size = shape_size(shape);
std::vector<int64_t> result(input_size);
switch (input->get_element_type())
{
case element::Type_t::i8:
{
auto p = input->get_data_ptr<int8_t>();
for (size_t i = 0; i < input_size; ++i)
{
result[i] = int64_t(p[i]);
}
}
break;
case element::Type_t::i16:
{
auto p = input->get_data_ptr<int16_t>();
for (size_t i = 0; i < input_size; ++i)
{
result[i] = int64_t(p[i]);
}
}
break;
case element::Type_t::i32:
{
auto p = input->get_data_ptr<int32_t>();
for (size_t i = 0; i < input_size; ++i)
{
result[i] = int64_t(p[i]);
}
}
break;
case element::Type_t::i64:
{
auto p = input->get_data_ptr<int64_t>();
for (size_t i = 0; i < input_size; ++i)
{
result[i] = int64_t(p[i]);
}
}
break;
case element::Type_t::u8:
{
auto p = input->get_data_ptr<uint8_t>();
for (size_t i = 0; i < input_size; ++i)
{
result[i] = int64_t(p[i]);
}
}
break;
case element::Type_t::u16:
{
auto p = input->get_data_ptr<uint16_t>();
for (size_t i = 0; i < input_size; ++i)
{
result[i] = int64_t(p[i]);
}
}
break;
case element::Type_t::u32:
{
auto p = input->get_data_ptr<uint32_t>();
for (size_t i = 0; i < input_size; ++i)
{
result[i] = int64_t(p[i]);
}
}
break;
case element::Type_t::u64:
{
auto p = input->get_data_ptr<uint64_t>();
for (size_t i = 0; i < input_size; ++i)
{
result[i] = int64_t(p[i]);
}
}
break;
default:
throw std::runtime_error("Unsupported data type in op NonMaxSuppression-5");
break;
}
return result;
}
@ -636,10 +723,11 @@ namespace
{
InfoForNMS5 result;
result.max_output_boxes_per_class = nms5->max_boxes_output_from_input();
result.iou_threshold = nms5->iou_threshold_from_input();
result.score_threshold = nms5->score_threshold_from_input();
result.soft_nms_sigma = nms5->soft_nms_sigma_from_input();
result.max_output_boxes_per_class =
inputs.size() > 2 ? get_integers(inputs[2], Shape({}))[0] : 0;
result.iou_threshold = inputs.size() > 3 ? get_floats(inputs[3], Shape({}))[0] : 0.0f;
result.score_threshold = inputs.size() > 4 ? get_floats(inputs[4], Shape({}))[0] : 0.0f;
result.soft_nms_sigma = inputs.size() > 5 ? get_floats(inputs[5], Shape({}))[0] : 0.0f;
auto selected_indices_shape =
infer_selected_indices_shape(inputs, result.max_output_boxes_per_class);