Review multiclass nms class for shape inference aspects (#18908)

* Review NMS shape inference

* Fix test issues

* Review MatrixNms shape inference

* Unregister NMS with dynamic output from CPU plugin

* Review MulticlassNms for shape inference
This commit is contained in:
Pawel Raasz 2023-08-02 10:01:06 +02:00 committed by GitHub
parent a2039e8410
commit 1f08d3520c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 457 additions and 619 deletions

View File

@ -38,23 +38,17 @@ std::shared_ptr<Node> op::internal::MulticlassNmsIEInternal::clone_with_new_inpu
void op::internal::MulticlassNmsIEInternal::validate_and_infer_types() {
INTERNAL_OP_SCOPE(internal_MulticlassNmsIEInternal_validate_and_infer_types);
const auto output_type = get_attrs().output_type;
OPENVINO_SUPPRESS_DEPRECATED_START
const auto input_shapes = get_node_input_partial_shapes(*this);
OPENVINO_SUPPRESS_DEPRECATED_END
const auto output_shapes = shape_infer(this, input_shapes, false, true);
validate();
const auto& boxes_ps = get_input_partial_shape(0);
const auto& scores_ps = get_input_partial_shape(1);
std::vector<PartialShape> input_shapes = {boxes_ps, scores_ps};
if (get_input_size() == 3) {
const auto& roisnum_ps = get_input_partial_shape(2);
input_shapes.push_back(roisnum_ps);
}
std::vector<PartialShape> output_shapes = {{Dimension::dynamic(), 6},
{Dimension::dynamic(), 1},
{Dimension::dynamic()}};
shape_infer(this, input_shapes, output_shapes, true, true);
set_output_type(0, get_input_element_type(0), output_shapes[0]);
set_output_type(1, output_type, output_shapes[1]);
const auto& output_type = get_attrs().output_type;
set_output_type(0, get_input_element_type(0), output_shapes[0].get_max_shape());
set_output_type(1, output_type, output_shapes[1].get_max_shape());
set_output_type(2, output_type, output_shapes[2]);
}

View File

@ -64,6 +64,8 @@ public:
return m_attrs;
}
void set_attrs(Attributes attrs);
void set_output_type(const element::Type& output_type) {
m_attrs.output_type = output_type;
}

View File

@ -22,27 +22,6 @@
namespace ngraph {
namespace runtime {
namespace reference {
namespace multiclass_nms_impl {
struct InfoForNMS {
Shape selected_outputs_shape;
Shape selected_indices_shape;
Shape selected_numrois_shape;
Shape boxes_shape;
Shape scores_shape;
Shape roisnum_shape;
std::vector<float> boxes_data;
std::vector<float> scores_data;
std::vector<int64_t> roisnum_data;
size_t selected_outputs_shape_size;
size_t selected_indices_shape_size;
size_t selected_numrois_shape_size;
};
OPENVINO_SUPPRESS_DEPRECATED_START
InfoForNMS get_info_for_nms_eval(const std::shared_ptr<op::util::MulticlassNmsBase>& nms,
const std::vector<std::shared_ptr<HostTensor>>& inputs);
OPENVINO_SUPPRESS_DEPRECATED_END
} // namespace multiclass_nms_impl
void multiclass_nms(const float* boxes_data,
const Shape& boxes_data_shape,

View File

@ -10,7 +10,6 @@
#include <queue>
#include <vector>
#include "../shape_inference/include/multiclass_nms_shape_inference.hpp"
#include "ngraph/runtime/reference/multiclass_nms.hpp"
#include "ngraph/runtime/reference/utils/nms_common.hpp"
#include "ngraph/shape.hpp"
@ -21,162 +20,6 @@ namespace reference {
namespace multiclass_nms_impl {
OPENVINO_SUPPRESS_DEPRECATED_START
namespace {
std::vector<float> get_floats(const std::shared_ptr<HostTensor>& input, const Shape& shape) {
size_t input_size = shape_size(shape);
std::vector<float> result(input_size);
switch (input->get_element_type()) {
case element::Type_t::bf16: {
bfloat16* p = input->get_data_ptr<bfloat16>();
for (size_t i = 0; i < input_size; ++i) {
result[i] = float(p[i]);
}
} break;
case element::Type_t::f16: {
float16* p = input->get_data_ptr<float16>();
for (size_t i = 0; i < input_size; ++i) {
result[i] = float(p[i]);
}
} break;
case element::Type_t::f32: {
float* p = input->get_data_ptr<float>();
memcpy(result.data(), p, input_size * sizeof(float));
} break;
default:
throw std::runtime_error("Unsupported data type.");
break;
}
return result;
}
std::vector<int64_t> get_integers(const std::shared_ptr<HostTensor>& input, const Shape& shape) {
size_t input_size = shape_size(shape);
std::vector<int64_t> result(input_size);
switch (input->get_element_type()) {
case element::Type_t::i8: {
auto p = input->get_data_ptr<int8_t>();
for (size_t i = 0; i < input_size; ++i) {
result[i] = int64_t(p[i]);
}
} break;
case element::Type_t::i16: {
auto p = input->get_data_ptr<int16_t>();
for (size_t i = 0; i < input_size; ++i) {
result[i] = int64_t(p[i]);
}
} break;
case element::Type_t::i32: {
auto p = input->get_data_ptr<int32_t>();
for (size_t i = 0; i < input_size; ++i) {
result[i] = int64_t(p[i]);
}
} break;
case element::Type_t::i64: {
auto p = input->get_data_ptr<int64_t>();
for (size_t i = 0; i < input_size; ++i) {
result[i] = int64_t(p[i]);
}
} break;
case element::Type_t::u8: {
auto p = input->get_data_ptr<uint8_t>();
for (size_t i = 0; i < input_size; ++i) {
result[i] = int64_t(p[i]);
}
} break;
case element::Type_t::u16: {
auto p = input->get_data_ptr<uint16_t>();
for (size_t i = 0; i < input_size; ++i) {
result[i] = int64_t(p[i]);
}
} break;
case element::Type_t::u32: {
auto p = input->get_data_ptr<uint32_t>();
for (size_t i = 0; i < input_size; ++i) {
result[i] = int64_t(p[i]);
}
} break;
case element::Type_t::u64: {
auto p = input->get_data_ptr<uint64_t>();
for (size_t i = 0; i < input_size; ++i) {
result[i] = int64_t(p[i]);
}
} break;
default:
throw std::runtime_error("Unsupported data type");
break;
}
return result;
}
static std::vector<float> prepare_boxes_data(const std::shared_ptr<HostTensor>& boxes, const Shape& boxes_shape) {
auto result = get_floats(boxes, boxes_shape);
return result;
}
static std::vector<float> prepare_scores_data(const std::shared_ptr<HostTensor>& scores, const Shape& scores_shape) {
auto result = get_floats(scores, scores_shape);
return result;
}
static std::vector<int64_t> prepare_roisnum_data(const std::shared_ptr<HostTensor>& roisnum,
const Shape& roisnum_shape) {
auto result = get_integers(roisnum, roisnum_shape);
return result;
}
} // namespace
constexpr size_t boxes_port = 0;
constexpr size_t scores_port = 1;
constexpr size_t roisnum_port = 2;
InfoForNMS get_info_for_nms_eval(const std::shared_ptr<op::util::MulticlassNmsBase>& nms,
const std::vector<std::shared_ptr<HostTensor>>& inputs) {
InfoForNMS result;
const auto boxes_ps = inputs[boxes_port]->get_partial_shape();
const auto scores_ps = inputs[scores_port]->get_partial_shape();
std::vector<PartialShape> input_shapes = {boxes_ps, scores_ps};
if (nms->get_input_size() == 3) {
const auto roisnum_ps = inputs[roisnum_port]->get_partial_shape();
input_shapes.push_back(roisnum_ps);
}
std::vector<PartialShape> output_shapes = {{Dimension::dynamic(), 6},
{Dimension::dynamic(), 1},
{Dimension::dynamic()}};
ov::op::util::shape_infer(nms.get(),
input_shapes,
output_shapes,
true,
false); // here just for upper boundary estimation.
result.selected_outputs_shape = output_shapes[0].to_shape();
result.selected_indices_shape = output_shapes[1].to_shape();
result.selected_numrois_shape = output_shapes[2].to_shape();
result.boxes_shape = inputs[boxes_port]->get_shape();
result.scores_shape = inputs[scores_port]->get_shape();
result.boxes_data = prepare_boxes_data(inputs[boxes_port], result.boxes_shape);
result.scores_data = prepare_scores_data(inputs[scores_port], result.scores_shape);
if (inputs.size() == 3) {
result.roisnum_shape = inputs[roisnum_port]->get_shape();
result.roisnum_data = prepare_roisnum_data(inputs[roisnum_port], result.roisnum_shape);
}
result.selected_outputs_shape_size = shape_size(result.selected_outputs_shape);
result.selected_indices_shape_size = shape_size(result.selected_indices_shape);
result.selected_numrois_shape_size = shape_size(result.selected_numrois_shape);
return result;
}
using Rectangle = runtime::reference::nms_common::Rectangle;
using BoxInfo = runtime::reference::nms_common::BoxInfo;
static float intersectionOverUnion(const Rectangle& boxI, const Rectangle& boxJ, const bool normalized) {

View File

@ -4,197 +4,119 @@
#pragma once
#include <openvino/op/multiclass_nms.hpp>
#include <openvino/op/util/multiclass_nms_base.hpp>
#include <vector>
#include "nms_shape_inference.hpp"
#include "openvino/op/multiclass_nms.hpp"
#include "openvino/op/util/multiclass_nms_base.hpp"
#include "utils.hpp"
namespace ov {
namespace op {
namespace util {
namespace multiclass_nms {
namespace validate {
template <class TShape>
void scores_shape(const Node* const op, const std::vector<TShape>& input_shapes) {
const auto scores_rank = input_shapes[1].rank();
NODE_SHAPE_INFER_CHECK(op, input_shapes, scores_rank.compatible(2), "Expected a 2D tensor for the 'scores' input");
}
template <class TShape>
void rois_num_shape(const Node* const op, const std::vector<TShape>& input_shapes) {
NODE_SHAPE_INFER_CHECK(op,
input_shapes,
input_shapes[2].rank().compatible(1),
"Expected a 1D tensor for the 'roisnum' input");
}
template <class T>
void shape_infer(const ov::op::util::MulticlassNmsBase* op,
const std::vector<T>& input_shapes,
std::vector<T>& output_shapes,
bool static_output = false,
bool ignore_bg_class = false) {
NODE_VALIDATION_CHECK(op, (input_shapes.size() == 2 || input_shapes.size() == 3) && output_shapes.size() == 3);
template <class TShape>
void num_boxes(const Node* const op, const std::vector<TShape>& input_shapes) {
NODE_SHAPE_INFER_CHECK(op,
input_shapes,
input_shapes[0][1].compatible(input_shapes[1][1]),
"'boxes' and 'scores' input shapes must match at the second dimension respectively");
}
} // namespace validate
} // namespace multiclass_nms
const auto& boxes_ps = input_shapes[0];
const auto& scores_ps = input_shapes[1];
template <class TShape, class TRShape = result_shape_t<TShape>>
std::vector<TRShape> shape_infer(const util::MulticlassNmsBase* op,
const std::vector<TShape>& input_shapes,
const bool static_output = !std::is_same<PartialShape, TShape>::value,
const bool ignore_bg_class = false) {
const auto inputs_size = input_shapes.size();
const auto has_rois_num = inputs_size == 3;
NODE_VALIDATION_CHECK(op, (input_shapes.size() == 2 || has_rois_num));
using TDim = typename TRShape::value_type;
using V = typename TDim::value_type;
using namespace ov::util;
nms::validate::boxes_shape(op, input_shapes);
if (has_rois_num) {
multiclass_nms::validate::scores_shape(op, input_shapes);
multiclass_nms::validate::rois_num_shape(op, input_shapes);
} else {
nms::validate::scores_shape(op, input_shapes);
}
auto output_shapes = std::vector<TRShape>{TRShape{TDim(dim::inf_bound), 6},
TRShape{TDim(dim::inf_bound), 1},
TRShape{TDim(dim::inf_bound)}};
const auto& boxes_shape = input_shapes[0];
const auto& scores_shape = input_shapes[1];
const auto& rois_num_shape = has_rois_num ? input_shapes[2] : PartialShape::dynamic();
if (boxes_shape.rank().is_static()) {
const auto scores_rank = scores_shape.rank();
nms::validate::num_batches(op, input_shapes);
nms::validate::boxes_last_dim(op, input_shapes);
bool can_infer;
if (has_rois_num && scores_rank.is_static()) {
multiclass_nms::validate::num_boxes(op, input_shapes);
can_infer = rois_num_shape.rank().is_static();
} else if (!has_rois_num && scores_rank.is_static()) {
nms::validate::num_boxes(op, input_shapes);
can_infer = true;
} else {
can_infer = false;
}
if (can_infer) {
const auto& nms_attrs = op->get_attrs();
const auto nms_top_k = nms_attrs.nms_top_k;
const auto keep_top_k = nms_attrs.keep_top_k;
const auto background_class = nms_attrs.background_class;
// validate rank of each input
if (boxes_ps.rank().is_dynamic() || scores_ps.rank().is_dynamic()) {
return;
}
const auto& num_classes = has_rois_num ? boxes_shape[0] : scores_shape[1];
const auto& num_images = has_rois_num ? rois_num_shape[0] : scores_shape[0];
if (op->get_input_size() == 3) {
NODE_VALIDATION_CHECK(op, input_shapes.size() == 3);
const auto& roisnum_ps = input_shapes[2];
if (roisnum_ps.rank().is_dynamic()) {
return;
}
}
auto& selected_boxes = output_shapes[0][0];
selected_boxes =
(nms_top_k > -1) ? TDim(std::min<V>(boxes_shape[1].get_max_length(), nms_top_k)) : boxes_shape[1];
// validate shape of each input
NODE_VALIDATION_CHECK(op,
boxes_ps.rank().is_static() && boxes_ps.rank().get_length() == 3,
"Expected a 3D tensor for the 'boxes' input. Got: ",
boxes_ps);
NODE_VALIDATION_CHECK(op,
boxes_ps[2].is_static() && boxes_ps[2].get_length() == 4,
"The third dimension of the 'boxes' must be 4. Got: ",
boxes_ps[2]);
if (ov::is_type<ov::op::v8::MulticlassNms>(op)) {
NODE_VALIDATION_CHECK(op,
scores_ps.rank().is_static() && scores_ps.rank().get_length() == 3,
"Expected a 3D tensor for the 'scores' input. Got: ",
scores_ps);
if (ignore_bg_class && (background_class > -1) && (background_class < num_classes.get_max_length())) {
selected_boxes *= std::max<V>(1, num_classes.get_max_length() - 1);
} else {
NODE_VALIDATION_CHECK(
op,
scores_ps.rank().is_static() && (scores_ps.rank().get_length() == 3 || scores_ps.rank().get_length() == 2),
"Expected a 2D or 3D tensor for the 'scores' input. Got: ",
scores_ps);
selected_boxes *= num_classes;
}
if (op->get_input_size() == 3) {
const auto& roisnum_ps = input_shapes[2];
NODE_VALIDATION_CHECK(op,
roisnum_ps.rank().is_static() && roisnum_ps.rank().get_length() == 1,
"Expected a 1D tensor for the 'roisnum' input. Got: ",
roisnum_ps);
if (keep_top_k > -1 && (keep_top_k < selected_boxes.get_max_length())) {
selected_boxes = TDim(keep_top_k);
}
// validate compatibility of input shapes
if (scores_ps.rank().is_static() && scores_ps.rank().get_length() == 3) { // if scores shape (N, C, M)
const auto num_batches_boxes = boxes_ps[0];
const auto num_batches_scores = scores_ps[0];
NODE_VALIDATION_CHECK(op,
num_batches_boxes.compatible(num_batches_scores),
"The first dimension of both 'boxes' and 'scores' must match. Boxes: ",
num_batches_boxes,
"; Scores: ",
num_batches_scores);
const auto num_boxes_boxes = boxes_ps[1];
const auto num_boxes_scores = scores_ps[2];
NODE_VALIDATION_CHECK(op,
num_boxes_boxes.compatible(num_boxes_scores),
"'boxes' and 'scores' input shapes must match at the second and third "
"dimension respectively. Boxes: ",
num_boxes_boxes,
"; Scores: ",
num_boxes_scores);
selected_boxes *= num_images;
if (std::is_same<PartialShape, TShape>::value) {
selected_boxes =
static_output ? TDim(selected_boxes.get_max_length()) : TDim(0, selected_boxes.get_max_length());
}
if (scores_ps.rank().is_static() && scores_ps.rank().get_length() == 2) { // if scores shape (C, M)
NODE_VALIDATION_CHECK(op,
op->get_input_size() == 3,
"Expected the 'roisnum' input when the input 'scores' is a 2D tensor.");
const auto num_classes_boxes = boxes_ps[0];
const auto num_classes_scores = scores_ps[0];
NODE_VALIDATION_CHECK(op,
num_classes_boxes.compatible(num_classes_scores),
"'boxes' and 'scores' input shapes must match. Boxes: ",
num_classes_boxes,
"; Scores: ",
num_classes_scores);
const auto num_boxes_boxes = boxes_ps[1];
const auto num_boxes_scores = scores_ps[1];
NODE_VALIDATION_CHECK(op,
num_boxes_boxes.compatible(num_boxes_scores),
"'boxes' and 'scores' input shapes must match. Boxes: ",
num_boxes_boxes,
"; Scores: ",
num_boxes_scores);
}
/* rank of inputs have been static since here. */
auto _ready_infer = [&]() {
if (boxes_ps.rank().is_dynamic() || scores_ps.rank().is_dynamic()) {
return false;
}
const bool shared = (scores_ps.rank().get_length() == 3);
if (shared) {
return boxes_ps[1].is_static() && scores_ps[1].is_static() && scores_ps[0].is_static();
} else {
const auto& roisnum_ps = input_shapes[2];
if (roisnum_ps.rank().is_dynamic()) {
return false;
}
return boxes_ps[1].is_static() && boxes_ps[0].is_static() && roisnum_ps[0].is_static();
}
};
// Here output 0 and output 1 is not the real dimension of output.
// It will be rewritten in the computing runtime.
// But we still need it here for static shape only backends.
auto first_dim_shape = Dimension::dynamic();
if (_ready_infer()) {
const bool shared = (scores_ps.rank().get_length() == 3);
ov::PartialShape roisnum_ps;
if (!shared) {
roisnum_ps = input_shapes[2];
}
const auto num_boxes = shared ? boxes_ps[1].get_length() : boxes_ps[1].get_length();
auto num_classes = shared ? scores_ps[1].get_length() : boxes_ps[0].get_length();
auto num_images = shared ? scores_ps[0].get_length() : roisnum_ps[0].get_length();
if (ignore_bg_class) {
if (nms_attrs.background_class >= 0 && nms_attrs.background_class < num_classes) {
num_classes = std::max(int64_t{1}, num_classes - 1);
output_shapes[1][0] = selected_boxes;
output_shapes[2][0] = has_rois_num ? rois_num_shape[0] : boxes_shape[0];
}
}
int64_t max_output_boxes_per_class = 0;
if (nms_top_k >= 0)
max_output_boxes_per_class = std::min(num_boxes, (int64_t)nms_top_k);
else
max_output_boxes_per_class = num_boxes;
auto max_output_boxes_per_batch = max_output_boxes_per_class * num_classes;
if (keep_top_k >= 0)
max_output_boxes_per_batch = std::min(max_output_boxes_per_batch, (int64_t)keep_top_k);
first_dim_shape = static_output ? max_output_boxes_per_batch * num_images
: Dimension(0, max_output_boxes_per_batch * num_images);
return output_shapes;
}
// 'selected_outputs' have the following format:
// [number of selected boxes, [class_id, box_score, xmin, ymin, xmax, ymax]]
output_shapes[0] = {first_dim_shape, 6};
// 'selected_indices' have the following format:
// [number of selected boxes, ]
output_shapes[1] = {first_dim_shape, 1};
// 'selected_num' have the following format:
// [num_batches, ]
if (op->get_input_size() == 3) {
const auto& roisnum_ps = input_shapes[2];
if (roisnum_ps.rank().is_static() && roisnum_ps.rank().get_length() > 0) {
output_shapes[2] = {roisnum_ps[0]};
} else {
output_shapes[2] = {Dimension::dynamic()};
}
} else { // shared
if (boxes_ps.rank().is_static() && boxes_ps.rank().get_length() > 0) {
output_shapes[2] = {boxes_ps[0]};
} else {
output_shapes[2] = {Dimension::dynamic()};
}
}
}
} // namespace util
} // namespace op
} // namespace ov

View File

@ -141,11 +141,8 @@ std::vector<TRShape> shape_infer(const Node* op,
const std::vector<T>& input_shapes,
const ITensorAccessor& ta,
const bool static_output) {
// Note: static_output this input make this function to compatible shape_infer function pattern
// but it has special usage in GPU plugin to force static output in special condition.
// This could be removed as for CPU plugin the using StaticShape force output to be static and GPU
// could use in this case ov::Shape but this shape class is not compatible with interface of PartialShape
// ans StaticShape.
// Note: static_output parameter of this shape_infer is exclusively made for GPU internal needs
// To be removed after GPU supports dynamic NMS
const auto inputs_size = input_shapes.size();
NODE_VALIDATION_CHECK(op, cmp::Between<size_t>(1, 7)(inputs_size));
using TDim = typename TRShape::value_type;

View File

@ -2,22 +2,23 @@
// SPDX-License-Identifier: Apache-2.0
//
#include "ngraph/op/multiclass_nms.hpp"
#include "openvino/op/multiclass_nms.hpp"
#include "itt.hpp"
#include "multiclass_nms_shape_inference.hpp"
#include "openvino/core/validation_util.hpp"
using namespace ngraph;
using namespace op::util;
namespace ov {
namespace op {
// ------------------------------ V8 ------------------------------
namespace v8 {
op::v8::MulticlassNms::MulticlassNms(const Output<Node>& boxes, const Output<Node>& scores, const Attributes& attrs)
MulticlassNms::MulticlassNms(const Output<Node>& boxes, const Output<Node>& scores, const Attributes& attrs)
: MulticlassNmsBase({boxes, scores}, attrs) {
constructor_validate_and_infer_types();
}
std::shared_ptr<Node> op::v8::MulticlassNms::clone_with_new_inputs(const OutputVector& new_args) const {
std::shared_ptr<Node> MulticlassNms::clone_with_new_inputs(const OutputVector& new_args) const {
OV_OP_SCOPE(MulticlassNms_v8_clone_with_new_inputs);
check_new_args_count(this, new_args);
NODE_VALIDATION_CHECK(this, new_args.size() >= 2, "Number of inputs must be 2 at least");
@ -25,32 +26,31 @@ std::shared_ptr<Node> op::v8::MulticlassNms::clone_with_new_inputs(const OutputV
return std::make_shared<MulticlassNms>(new_args.at(0), new_args.at(1), m_attrs);
}
void op::v8::MulticlassNms::validate_and_infer_types() {
void MulticlassNms::validate_and_infer_types() {
OV_OP_SCOPE(MulticlassNms_v9_validate_and_infer_types);
const auto output_type = get_attrs().output_type;
OPENVINO_SUPPRESS_DEPRECATED_START
const auto input_shapes = get_node_input_partial_shapes(*this);
OPENVINO_SUPPRESS_DEPRECATED_END
const auto output_shapes = shape_infer(this, input_shapes, false);
validate();
const auto& boxes_ps = get_input_partial_shape(0);
const auto& scores_ps = get_input_partial_shape(1);
std::vector<PartialShape> input_shapes = {boxes_ps, scores_ps};
std::vector<PartialShape> output_shapes = {{Dimension::dynamic(), 6},
{Dimension::dynamic(), 1},
{Dimension::dynamic()}};
shape_infer(this, input_shapes, output_shapes, false, false);
const auto& output_type = get_attrs().output_type;
set_output_type(0, get_input_element_type(0), output_shapes[0]);
set_output_type(1, output_type, output_shapes[1]);
set_output_type(2, output_type, output_shapes[2]);
}
} // namespace v8
// ------------------------------ V9 ------------------------------
op::v9::MulticlassNms::MulticlassNms(const Output<Node>& boxes, const Output<Node>& scores, const Attributes& attrs)
namespace v9 {
MulticlassNms::MulticlassNms(const Output<Node>& boxes, const Output<Node>& scores, const Attributes& attrs)
: MulticlassNmsBase({boxes, scores}, attrs) {
constructor_validate_and_infer_types();
}
op::v9::MulticlassNms::MulticlassNms(const Output<Node>& boxes,
MulticlassNms::MulticlassNms(const Output<Node>& boxes,
const Output<Node>& scores,
const Output<Node>& roisnum,
const Attributes& attrs)
@ -58,7 +58,7 @@ op::v9::MulticlassNms::MulticlassNms(const Output<Node>& boxes,
constructor_validate_and_infer_types();
}
std::shared_ptr<Node> op::v9::MulticlassNms::clone_with_new_inputs(const OutputVector& new_args) const {
std::shared_ptr<Node> MulticlassNms::clone_with_new_inputs(const OutputVector& new_args) const {
OV_OP_SCOPE(MulticlassNms_v9_clone_with_new_inputs);
check_new_args_count(this, new_args);
NODE_VALIDATION_CHECK(this, new_args.size() == 2 || new_args.size() == 3, "Number of inputs must be 2 or 3");
@ -71,25 +71,22 @@ std::shared_ptr<Node> op::v9::MulticlassNms::clone_with_new_inputs(const OutputV
}
}
void op::v9::MulticlassNms::validate_and_infer_types() {
void MulticlassNms::validate_and_infer_types() {
OV_OP_SCOPE(MulticlassNms_v9_validate_and_infer_types);
const auto output_type = get_attrs().output_type;
OPENVINO_SUPPRESS_DEPRECATED_START
const auto input_shapes = get_node_input_partial_shapes(*this);
OPENVINO_SUPPRESS_DEPRECATED_END
const auto output_shapes = shape_infer(this, input_shapes, false);
validate();
const auto& boxes_ps = get_input_partial_shape(0);
const auto& scores_ps = get_input_partial_shape(1);
std::vector<PartialShape> input_shapes = {boxes_ps, scores_ps};
if (get_input_size() == 3) {
const auto& roisnum_ps = get_input_partial_shape(2);
input_shapes.push_back(roisnum_ps);
}
std::vector<PartialShape> output_shapes = {{Dimension::dynamic(), 6},
{Dimension::dynamic(), 1},
{Dimension::dynamic()}};
shape_infer(this, input_shapes, output_shapes, false, false);
const auto& output_type = get_attrs().output_type;
set_output_type(0, get_input_element_type(0), output_shapes[0]);
set_output_type(1, output_type, output_shapes[1]);
set_output_type(2, output_type, output_shapes[2]);
}
} // namespace v9
} // namespace op
} // namespace ov

View File

@ -6,8 +6,7 @@
#include "itt.hpp"
using namespace ov;
namespace ov {
op::util::MulticlassNmsBase::MulticlassNmsBase(const OutputVector& arguments, const Attributes& attrs)
: Op(arguments),
m_attrs{attrs} {}
@ -81,13 +80,16 @@ bool op::util::MulticlassNmsBase::visit_attributes(AttributeVisitor& visitor) {
return true;
}
std::ostream& ov::operator<<(std::ostream& s, const op::util::MulticlassNmsBase::SortResultType& type) {
void op::util::MulticlassNmsBase::set_attrs(op::util::MulticlassNmsBase::Attributes attrs) {
m_attrs = std::move(attrs);
}
std::ostream& operator<<(std::ostream& s, const op::util::MulticlassNmsBase::SortResultType& type) {
return s << as_string(type);
}
namespace ov {
template <>
NGRAPH_API EnumNames<op::util::MulticlassNmsBase::SortResultType>&
OPENVINO_API EnumNames<op::util::MulticlassNmsBase::SortResultType>&
EnumNames<op::util::MulticlassNmsBase::SortResultType>::get() {
static auto enum_names = EnumNames<op::util::MulticlassNmsBase::SortResultType>(
"op::util::MulticlassNmsBase::SortResultType",

View File

@ -2,249 +2,236 @@
// SPDX-License-Identifier: Apache-2.0
//
#include "openvino/op/multiclass_nms.hpp"
#include "common_test_utils/test_assertions.hpp"
#include "common_test_utils/type_prop.hpp"
#include "gtest/gtest.h"
#include "ngraph/ngraph.hpp"
using namespace std;
using namespace ngraph;
using namespace ov;
using namespace testing;
template <typename T>
class type_prop : public testing::Test {};
class type_prop : public testing::Test {
protected:
using Attributes = op::util::MulticlassNmsBase::Attributes;
typedef testing::Types<op::v8::MulticlassNms, op::v9::MulticlassNms> test_prop_types;
TYPED_TEST_SUITE(type_prop, test_prop_types);
Attributes attrs;
};
using MulticlassNmsTypes = testing::Types<op::v8::MulticlassNms, op::v9::MulticlassNms>;
TYPED_TEST_SUITE(type_prop, MulticlassNmsTypes);
TYPED_TEST(type_prop, multiclass_nms_incorrect_boxes_rank) {
try {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{1, 2, 3, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{1, 2, 3});
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 3, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 3});
const auto unused = make_shared<TypeParam>(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes());
} catch (const NodeValidationFailure& error) {
EXPECT_HAS_SUBSTRING(error.what(), "Expected a 3D tensor for the 'boxes' input");
}
OV_EXPECT_THROW(ignore = make_shared<TypeParam>(boxes, scores, this->attrs),
NodeValidationFailure,
HasSubstr("Expected a 3D tensor for the 'boxes' input"));
}
TEST(type_prop2, multiclass_nms_incorrect_boxes_rank) {
try {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{1, 2, 3, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{1, 2, 3});
const auto roisnum = make_shared<op::Parameter>(element::i32, Shape{1});
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 3, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 3});
const auto roisnum = make_shared<op::v0::Parameter>(element::i32, Shape{1});
const auto unused =
make_shared<op::v9::MulticlassNms>(boxes, scores, roisnum, ov::op::util::MulticlassNmsBase::Attributes());
} catch (const NodeValidationFailure& error) {
EXPECT_HAS_SUBSTRING(error.what(), "Expected a 3D tensor for the 'boxes' input");
}
OV_EXPECT_THROW(
ignore = make_shared<op::v9::MulticlassNms>(boxes, scores, op::util::MulticlassNmsBase::Attributes()),
NodeValidationFailure,
HasSubstr("Expected a 3D tensor for the 'boxes' input"));
}
TYPED_TEST(type_prop, multiclass_nms_incorrect_scores_rank) {
try {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{1});
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{1});
const auto unused = make_shared<TypeParam>(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes());
} catch (const NodeValidationFailure& error) {
EXPECT_HAS_SUBSTRING(error.what(),
TypeParam::get_type_info_static().get_version() == "opset8"
? "Expected a 3D tensor for the 'scores' input"
: "Expected a 2D or 3D tensor for the 'scores' input");
}
OV_EXPECT_THROW(ignore = make_shared<TypeParam>(boxes, scores, this->attrs),
NodeValidationFailure,
HasSubstr("Expected a 3D tensor for the 'scores' input"));
}
TEST(type_prop2, multiclass_nms_incorrect_scores_rank2) {
try {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{1, 2});
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2});
const auto unused =
make_shared<op::v9::MulticlassNms>(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes());
} catch (const NodeValidationFailure& error) {
EXPECT_HAS_SUBSTRING(error.what(), "Expected the 'roisnum' input when the input 'scores' is a 2D tensor.");
}
OV_EXPECT_THROW(
ignore = make_shared<op::v9::MulticlassNms>(boxes, scores, op::util::MulticlassNmsBase::Attributes()),
NodeValidationFailure,
HasSubstr("Expected a 3D tensor for the 'scores' input"));
}
TEST(type_prop2, multiclass_nms_incorrect_roisnum_rank) {
try {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{2, 3, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{2, 3});
const auto roisnum = make_shared<op::Parameter>(element::i32, Shape{1, 2});
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{2, 3, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{2, 3});
const auto roisnum = make_shared<op::v0::Parameter>(element::i32, Shape{1, 2});
const auto unused =
make_shared<op::v9::MulticlassNms>(boxes, scores, roisnum, ov::op::util::MulticlassNmsBase::Attributes());
} catch (const NodeValidationFailure& error) {
EXPECT_HAS_SUBSTRING(error.what(), "Expected a 1D tensor for the 'roisnum' input");
}
OV_EXPECT_THROW(
ignore = make_shared<op::v9::MulticlassNms>(boxes, scores, roisnum, op::util::MulticlassNmsBase::Attributes()),
NodeValidationFailure,
HasSubstr("Expected a 1D tensor for the 'roisnum' input"));
}
TYPED_TEST(type_prop, multiclass_nms_incorrect_scheme_num_batches) {
try {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{2, 2, 3});
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{2, 2, 3});
const auto unused = make_shared<TypeParam>(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes());
} catch (const NodeValidationFailure& error) {
EXPECT_HAS_SUBSTRING(error.what(), "The first dimension of both 'boxes' and 'scores' must match");
}
OV_EXPECT_THROW(ignore = make_shared<TypeParam>(boxes, scores, this->attrs),
NodeValidationFailure,
HasSubstr("The first dimension of both 'boxes' and 'scores' must match"));
}
TEST(type_prop2, multiclass_nms_incorrect_scheme_num_classes) {
try {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{2, 3, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{1, 3});
const auto roisnum = make_shared<op::Parameter>(element::i32, Shape{1});
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{2, 3, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{1, 3});
const auto roisnum = make_shared<op::v0::Parameter>(element::i32, Shape{1});
const auto unused =
make_shared<op::v9::MulticlassNms>(boxes, scores, roisnum, ov::op::util::MulticlassNmsBase::Attributes());
} catch (const NodeValidationFailure& error) {
EXPECT_HAS_SUBSTRING(error.what(), "'boxes' and 'scores' input shapes must match");
}
OV_EXPECT_THROW(
ignore = make_shared<op::v9::MulticlassNms>(boxes, scores, roisnum, op::util::MulticlassNmsBase::Attributes()),
NodeValidationFailure,
HasSubstr("The first dimension of both 'boxes' and 'scores' must match"));
}
TYPED_TEST(type_prop, multiclass_nms_incorrect_scheme_num_boxes) {
try {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{1, 2, 3});
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 3});
const auto unused = make_shared<TypeParam>(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes());
} catch (const NodeValidationFailure& error) {
EXPECT_HAS_SUBSTRING(error.what(),
"'boxes' and 'scores' input shapes must match at the second and third "
"dimension respectively");
}
OV_EXPECT_THROW(ignore = make_shared<TypeParam>(boxes, scores, this->attrs),
NodeValidationFailure,
HasSubstr("'boxes' and 'scores' input shapes must match at the second and third "
"dimension respectively"));
}
TEST(type_prop2, multiclass_nms_incorrect_scheme_num_boxes) {
try {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{1, 3});
const auto roisnum = make_shared<op::Parameter>(element::i32, Shape{1});
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{1, 3});
const auto roisnum = make_shared<op::v0::Parameter>(element::i32, Shape{1});
const auto unused =
make_shared<op::v9::MulticlassNms>(boxes, scores, roisnum, ov::op::util::MulticlassNmsBase::Attributes());
} catch (const NodeValidationFailure& error) {
EXPECT_HAS_SUBSTRING(error.what(), "'boxes' and 'scores' input shapes must match.");
}
OV_EXPECT_THROW(
ignore = make_shared<op::v9::MulticlassNms>(boxes, scores, roisnum, op::util::MulticlassNmsBase::Attributes()),
NodeValidationFailure,
HasSubstr("'boxes' and 'scores' input shapes must match at the second dimension respectively"));
}
TEST(type_prop2, multiclass_nms_incorrect_scheme_num_boxes2) {
try {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{3, 2});
const auto roisnum = make_shared<op::Parameter>(element::i32, Shape{1});
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{3, 2});
const auto roisnum = make_shared<op::v0::Parameter>(element::i32, Shape{1});
const auto unused =
make_shared<op::v9::MulticlassNms>(boxes, scores, roisnum, ov::op::util::MulticlassNmsBase::Attributes());
} catch (const NodeValidationFailure& error) {
EXPECT_HAS_SUBSTRING(error.what(), "'boxes' and 'scores' input shapes must match.");
}
OV_EXPECT_THROW(
ignore = make_shared<op::v9::MulticlassNms>(boxes, scores, roisnum, op::util::MulticlassNmsBase::Attributes()),
NodeValidationFailure,
HasSubstr("The first dimension of both 'boxes' and 'scores' must match"));
}
TYPED_TEST(type_prop, multiclass_nms_incorrect_boxes_rank2) {
try {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{1, 2, 3});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{2, 2, 2});
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{2, 2, 3});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{2, 2, 2});
const auto unused = make_shared<TypeParam>(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes());
} catch (const NodeValidationFailure& error) {
EXPECT_HAS_SUBSTRING(error.what(), "The third dimension of the 'boxes' must be 4");
}
OV_EXPECT_THROW(ignore = make_shared<TypeParam>(boxes, scores, this->attrs),
NodeValidationFailure,
HasSubstr("The last dimension of the 'boxes' input must be equal to 4"));
}
TYPED_TEST(type_prop, multiclass_nms_incorrect_output_type) {
try {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{1, 2, 2});
ov::op::util::MulticlassNmsBase::Attributes attrs;
attrs.output_type = ngraph::element::f32;
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 2});
this->attrs.output_type = element::f32;
const auto unused = make_shared<TypeParam>(boxes, scores, attrs);
} catch (const NodeValidationFailure& error) {
EXPECT_HAS_SUBSTRING(error.what(), "Output type must be i32 or i64");
}
OV_EXPECT_THROW(ignore = make_shared<TypeParam>(boxes, scores, this->attrs),
NodeValidationFailure,
HasSubstr("Output type must be i32 or i64"));
}
TYPED_TEST(type_prop, multiclass_nms_incorrect_nms_topk) {
try {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{1, 2, 2});
ov::op::util::MulticlassNmsBase::Attributes attrs;
attrs.nms_top_k = -2;
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 2});
this->attrs.nms_top_k = -2;
const auto unused = make_shared<TypeParam>(boxes, scores, attrs);
} catch (const NodeValidationFailure& error) {
EXPECT_HAS_SUBSTRING(error.what(), "The 'nms_top_k' must be great or equal -1");
}
OV_EXPECT_THROW(ignore = make_shared<TypeParam>(boxes, scores, this->attrs),
NodeValidationFailure,
HasSubstr("The 'nms_top_k' must be great or equal -1"));
}
TYPED_TEST(type_prop, multiclass_nms_incorrect_keep_topk) {
try {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{1, 2, 2});
ov::op::util::MulticlassNmsBase::Attributes attrs;
attrs.keep_top_k = -2;
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 2});
this->attrs.keep_top_k = -2;
const auto unused = make_shared<TypeParam>(boxes, scores, attrs);
} catch (const NodeValidationFailure& error) {
EXPECT_HAS_SUBSTRING(error.what(), "The 'keep_top_k' must be great or equal -1");
}
OV_EXPECT_THROW(ignore = make_shared<TypeParam>(boxes, scores, this->attrs),
NodeValidationFailure,
HasSubstr("The 'keep_top_k' must be great or equal -1"));
}
TYPED_TEST(type_prop, multiclass_nms_incorrect_background_class) {
try {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{1, 2, 2});
ov::op::util::MulticlassNmsBase::Attributes attrs;
attrs.background_class = -2;
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 2});
this->attrs.background_class = -2;
const auto unused = make_shared<TypeParam>(boxes, scores, attrs);
} catch (const NodeValidationFailure& error) {
EXPECT_HAS_SUBSTRING(error.what(), "The 'background_class' must be great or equal -1");
}
OV_EXPECT_THROW(ignore = make_shared<TypeParam>(boxes, scores, this->attrs),
NodeValidationFailure,
HasSubstr("The 'background_class' must be great or equal -1"));
}
TYPED_TEST(type_prop, multiclass_nms_incorrect_eta) {
try {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{1, 2, 2});
ov::op::util::MulticlassNmsBase::Attributes attrs;
attrs.nms_eta = 2.0f;
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 2});
this->attrs.nms_eta = 2.0f;
const auto unused = make_shared<TypeParam>(boxes, scores, attrs);
} catch (const NodeValidationFailure& error) {
EXPECT_HAS_SUBSTRING(error.what(), "The 'nms_eta' must be in close range [0, 1.0]");
}
OV_EXPECT_THROW(ignore = make_shared<TypeParam>(boxes, scores, this->attrs),
NodeValidationFailure,
HasSubstr("The 'nms_eta' must be in close range [0, 1.0]"));
}
TYPED_TEST(type_prop, multiclass_nms_incorrect_input_type) {
try {
const auto boxes = make_shared<op::Parameter>(element::f16, Shape{1, 2, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{1, 2, 2});
const auto boxes = make_shared<op::v0::Parameter>(element::f16, Shape{1, 2, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{1, 2, 2});
const auto unused = make_shared<TypeParam>(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes());
} catch (const NodeValidationFailure& error) {
EXPECT_HAS_SUBSTRING(error.what(), "Expected 'boxes', 'scores' type is same.");
OV_EXPECT_THROW(ignore = make_shared<TypeParam>(boxes, scores, this->attrs),
NodeValidationFailure,
HasSubstr("Expected 'boxes', 'scores' type is same."));
}
TYPED_TEST(type_prop, multiclass_nms_default_ctor) {
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{2, 7, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{2, 5, 7});
this->attrs.nms_top_k = 3;
const auto nms = make_shared<TypeParam>();
nms->set_arguments(OutputVector{boxes, scores});
nms->set_attrs(this->attrs);
nms->validate_and_infer_types();
EXPECT_EQ(nms->get_output_partial_shape(0), (PartialShape{{0, 30}, 6}));
EXPECT_EQ(nms->get_output_partial_shape(1), (PartialShape{{0, 30}, 1}));
EXPECT_EQ(nms->get_output_partial_shape(2), (PartialShape{2}));
}
TYPED_TEST(type_prop, multiclass_nms_output_shape_1dim_dynamic) {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{5, 2, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{5, 3, 2});
auto boxes_shape = PartialShape{5, 2, 4};
auto scores_shape = PartialShape{5, 3, 2};
set_shape_labels(boxes_shape, 10);
set_shape_labels(scores_shape, 20);
const auto nms = make_shared<TypeParam>(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes());
const auto boxes = make_shared<op::v0::Parameter>(element::f32, boxes_shape);
const auto scores = make_shared<op::v0::Parameter>(element::f32, scores_shape);
ASSERT_TRUE(nms->get_output_partial_shape(0).same_scheme(PartialShape{Dimension::dynamic(), 6}));
ASSERT_TRUE(nms->get_output_partial_shape(1).same_scheme(PartialShape{Dimension::dynamic(), 1}));
const auto nms = make_shared<TypeParam>(boxes, scores, this->attrs);
EXPECT_EQ(nms->get_output_shape(2), (Shape{5}));
EXPECT_EQ(nms->get_output_partial_shape(0), (PartialShape{{0, 30}, 6}));
EXPECT_EQ(nms->get_output_partial_shape(1), (PartialShape{{0, 30}, 1}));
EXPECT_EQ(nms->get_output_partial_shape(2), (PartialShape{5}));
EXPECT_THAT(get_shape_labels(nms->get_output_partial_shape(0)), Each(no_label));
EXPECT_THAT(get_shape_labels(nms->get_output_partial_shape(1)), Each(no_label));
EXPECT_THAT(get_shape_labels(nms->get_output_partial_shape(2)), ElementsAre(10));
}
TYPED_TEST(type_prop, multiclass_nms_output_shape_1dim_max_out) {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{2, 7, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{2, 5, 7});
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{2, 7, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{2, 5, 7});
const auto nms = make_shared<TypeParam>(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes());
const auto nms = make_shared<TypeParam>(boxes, scores, this->attrs);
ASSERT_EQ(nms->get_output_element_type(0), element::f32);
ASSERT_EQ(nms->get_output_element_type(1), element::i64);
@ -257,12 +244,11 @@ TYPED_TEST(type_prop, multiclass_nms_output_shape_1dim_max_out) {
}
TYPED_TEST(type_prop, multiclass_nms_output_shape_1dim_nms_topk) {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{2, 7, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{2, 5, 7});
ov::op::util::MulticlassNmsBase::Attributes attrs;
attrs.nms_top_k = 3;
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{2, 7, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{2, 5, 7});
this->attrs.nms_top_k = 3;
const auto nms = make_shared<TypeParam>(boxes, scores, attrs);
const auto nms = make_shared<TypeParam>(boxes, scores, this->attrs);
ASSERT_EQ(nms->get_output_element_type(0), element::f32);
ASSERT_EQ(nms->get_output_element_type(1), element::i64);
@ -274,13 +260,12 @@ TYPED_TEST(type_prop, multiclass_nms_output_shape_1dim_nms_topk) {
}
TYPED_TEST(type_prop, multiclass_nms_output_shape_1dim_keep_topk) {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{2, 7, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{2, 5, 7});
ov::op::util::MulticlassNmsBase::Attributes attrs;
attrs.nms_top_k = 3;
attrs.keep_top_k = 8;
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{2, 7, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{2, 5, 7});
this->attrs.nms_top_k = 3;
this->attrs.keep_top_k = 8;
const auto nms = make_shared<TypeParam>(boxes, scores, attrs);
const auto nms = make_shared<TypeParam>(boxes, scores, this->attrs);
ASSERT_EQ(nms->get_output_element_type(0), element::f32);
ASSERT_EQ(nms->get_output_element_type(1), element::i64);
@ -292,10 +277,10 @@ TYPED_TEST(type_prop, multiclass_nms_output_shape_1dim_keep_topk) {
}
TYPED_TEST(type_prop, multiclass_nms_input_f16) {
const auto boxes = make_shared<op::Parameter>(element::f16, Shape{2, 7, 4});
const auto scores = make_shared<op::Parameter>(element::f16, Shape{2, 5, 7});
const auto boxes = make_shared<op::v0::Parameter>(element::f16, Shape{2, 7, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f16, Shape{2, 5, 7});
const auto nms = make_shared<TypeParam>(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes());
const auto nms = make_shared<TypeParam>(boxes, scores, this->attrs);
ASSERT_EQ(nms->get_output_element_type(0), element::f16);
ASSERT_EQ(nms->get_output_element_type(1), element::i64);
@ -307,12 +292,11 @@ TYPED_TEST(type_prop, multiclass_nms_input_f16) {
}
TYPED_TEST(type_prop, multiclass_nms_output_shape_i32) {
const auto boxes = make_shared<op::Parameter>(element::f32, Shape{2, 7, 4});
const auto scores = make_shared<op::Parameter>(element::f32, Shape{2, 5, 7});
ov::op::util::MulticlassNmsBase::Attributes attrs;
attrs.output_type = ngraph::element::i32;
const auto boxes = make_shared<op::v0::Parameter>(element::f32, Shape{2, 7, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, Shape{2, 5, 7});
this->attrs.output_type = element::i32;
const auto nms = make_shared<TypeParam>(boxes, scores, attrs);
const auto nms = make_shared<TypeParam>(boxes, scores, this->attrs);
ASSERT_EQ(nms->get_output_element_type(0), element::f32);
ASSERT_EQ(nms->get_output_element_type(1), element::i32);
@ -324,10 +308,10 @@ TYPED_TEST(type_prop, multiclass_nms_output_shape_i32) {
}
TYPED_TEST(type_prop, multiclass_nms_dynamic_boxes_and_scores) {
const auto boxes = make_shared<op::Parameter>(element::f32, PartialShape::dynamic());
const auto scores = make_shared<op::Parameter>(element::f32, PartialShape::dynamic());
const auto boxes = make_shared<op::v0::Parameter>(element::f32, PartialShape::dynamic());
const auto scores = make_shared<op::v0::Parameter>(element::f32, PartialShape::dynamic());
const auto nms = make_shared<TypeParam>(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes());
const auto nms = make_shared<TypeParam>(boxes, scores, this->attrs);
ASSERT_EQ(nms->get_output_element_type(0), element::f32);
ASSERT_EQ(nms->get_output_element_type(1), element::i64);
@ -338,9 +322,9 @@ TYPED_TEST(type_prop, multiclass_nms_dynamic_boxes_and_scores) {
}
TEST(type_prop2, multiclass_nms_dynamic_boxes_and_scores) {
const auto boxes = make_shared<op::Parameter>(element::f32, PartialShape::dynamic());
const auto scores = make_shared<op::Parameter>(element::f32, PartialShape::dynamic());
const auto roisnum = make_shared<op::Parameter>(element::i32, PartialShape::dynamic());
const auto boxes = make_shared<op::v0::Parameter>(element::f32, PartialShape::dynamic());
const auto scores = make_shared<op::v0::Parameter>(element::f32, PartialShape::dynamic());
const auto roisnum = make_shared<op::v0::Parameter>(element::i32, PartialShape::dynamic());
const auto nms =
make_shared<op::v9::MulticlassNms>(boxes, scores, roisnum, ov::op::util::MulticlassNmsBase::Attributes());
@ -353,14 +337,55 @@ TEST(type_prop2, multiclass_nms_dynamic_boxes_and_scores) {
EXPECT_EQ(nms->get_output_partial_shape(2), PartialShape({Dimension::dynamic()}));
}
TEST(type_prop2, multiclass_nms_interval_shapes_and_labels) {
auto boxes_shape = PartialShape{2, 7, 4};
auto scores_shape = PartialShape{2, 7};
auto roisnum_shape = PartialShape{4};
set_shape_labels(boxes_shape, 10);
set_shape_labels(scores_shape, 20);
set_shape_labels(roisnum_shape, 30);
const auto boxes = make_shared<op::v0::Parameter>(element::f32, boxes_shape);
const auto scores = make_shared<op::v0::Parameter>(element::f32, scores_shape);
const auto roisnum = make_shared<op::v0::Parameter>(element::i32, roisnum_shape);
const auto nms =
make_shared<op::v9::MulticlassNms>(boxes, scores, roisnum, ov::op::util::MulticlassNmsBase::Attributes());
ASSERT_EQ(nms->get_output_element_type(0), element::f32);
ASSERT_EQ(nms->get_output_element_type(1), element::i64);
ASSERT_EQ(nms->get_output_element_type(2), element::i64);
EXPECT_EQ(nms->get_output_partial_shape(0), PartialShape({{0, 56}, 6}));
EXPECT_EQ(nms->get_output_partial_shape(1), PartialShape({{0, 56}, 1}));
EXPECT_EQ(nms->get_output_partial_shape(2), PartialShape({4}));
EXPECT_THAT(get_shape_labels(nms->get_output_partial_shape(0)), Each(no_label));
EXPECT_THAT(get_shape_labels(nms->get_output_partial_shape(1)), Each(no_label));
EXPECT_THAT(get_shape_labels(nms->get_output_partial_shape(2)), ElementsAre(30));
}
TEST(type_prop2, multiclass_nms_static_shapes) {
const auto boxes = make_shared<op::v0::Parameter>(element::f32, PartialShape{3, 7, 4});
const auto scores = make_shared<op::v0::Parameter>(element::f32, PartialShape{3, 7});
const auto roisnum = make_shared<op::v0::Parameter>(element::i32, PartialShape{4});
ov::op::util::MulticlassNmsBase::Attributes attrs;
attrs.nms_top_k = 3;
attrs.keep_top_k = 8;
const auto nms = make_shared<op::v9::MulticlassNms>(boxes, scores, roisnum, attrs);
EXPECT_EQ(nms->get_output_partial_shape(0), PartialShape({{0, 32}, Dimension(6)}));
EXPECT_EQ(nms->get_output_partial_shape(1), PartialShape({{0, 32}, 1}));
EXPECT_EQ(nms->get_output_partial_shape(2), PartialShape({4}));
}
TYPED_TEST(type_prop, multiclass_nms_dynamic_boxes_and_scores2) {
const auto boxes =
make_shared<op::Parameter>(element::f32, PartialShape({Dimension::dynamic(), Dimension::dynamic(), 4}));
const auto scores =
make_shared<op::Parameter>(element::f32,
make_shared<op::v0::Parameter>(element::f32, PartialShape({Dimension::dynamic(), Dimension::dynamic(), 4}));
const auto scores = make_shared<op::v0::Parameter>(
element::f32,
PartialShape({Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic()}));
const auto nms = make_shared<TypeParam>(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes());
const auto nms = make_shared<TypeParam>(boxes, scores, this->attrs);
ASSERT_EQ(nms->get_output_element_type(0), element::f32);
ASSERT_EQ(nms->get_output_element_type(1), element::i64);
@ -372,10 +397,10 @@ TYPED_TEST(type_prop, multiclass_nms_dynamic_boxes_and_scores2) {
TEST(type_prop2, multiclass_nms_dynamic_boxes_and_scores2) {
const auto boxes =
make_shared<op::Parameter>(element::f32, PartialShape({Dimension::dynamic(), Dimension::dynamic(), 4}));
make_shared<op::v0::Parameter>(element::f32, PartialShape({Dimension::dynamic(), Dimension::dynamic(), 4}));
const auto scores =
make_shared<op::Parameter>(element::f32, PartialShape({Dimension::dynamic(), Dimension::dynamic()}));
const auto roisnum = make_shared<op::Parameter>(element::i32, PartialShape({Dimension::dynamic()}));
make_shared<op::v0::Parameter>(element::f32, PartialShape({Dimension::dynamic(), Dimension::dynamic()}));
const auto roisnum = make_shared<op::v0::Parameter>(element::i32, PartialShape({Dimension::dynamic()}));
const auto nms =
make_shared<op::v9::MulticlassNms>(boxes, scores, roisnum, ov::op::util::MulticlassNmsBase::Attributes());

View File

@ -5,13 +5,90 @@
#include "ngraph/runtime/reference/multiclass_nms.hpp"
#include "evaluate_node.hpp"
#include "evaluates_map.hpp"
#include "multiclass_nms_shape_inference.hpp"
#include "ngraph/runtime/reference/utils/nms_common.hpp"
namespace multiclass_nms {
using namespace ov;
struct InfoForNMS {
Shape selected_outputs_shape;
Shape selected_indices_shape;
Shape selected_numrois_shape;
Shape boxes_shape;
Shape scores_shape;
Shape roisnum_shape;
std::vector<float> boxes_data;
std::vector<float> scores_data;
std::vector<int64_t> roisnum_data;
size_t selected_outputs_shape_size;
size_t selected_indices_shape_size;
size_t selected_numrois_shape_size;
};
static std::vector<float> prepare_boxes_data(const std::shared_ptr<HostTensor>& boxes, const Shape& boxes_shape) {
auto result = get_floats(boxes, boxes_shape);
return result;
}
static std::vector<float> prepare_scores_data(const std::shared_ptr<HostTensor>& scores, const Shape& scores_shape) {
auto result = get_floats(scores, scores_shape);
return result;
}
static std::vector<int64_t> prepare_roisnum_data(const std::shared_ptr<HostTensor>& roisnum,
const Shape& roisnum_shape) {
auto result = get_integers(roisnum, roisnum_shape);
return result;
}
constexpr size_t boxes_port = 0;
constexpr size_t scores_port = 1;
constexpr size_t roisnum_port = 2;
InfoForNMS get_info_for_nms_eval(const std::shared_ptr<op::util::MulticlassNmsBase>& nms,
const std::vector<std::shared_ptr<HostTensor>>& inputs) {
InfoForNMS result;
const auto boxes_ps = inputs[boxes_port]->get_partial_shape();
const auto scores_ps = inputs[scores_port]->get_partial_shape();
std::vector<PartialShape> input_shapes = {boxes_ps, scores_ps};
if (nms->get_input_size() == 3) {
const auto roisnum_ps = inputs[roisnum_port]->get_partial_shape();
input_shapes.push_back(roisnum_ps);
}
const auto output_shapes = ov::op::shape_infer(nms.get(), input_shapes);
result.selected_outputs_shape = output_shapes[0].get_max_shape();
result.selected_indices_shape = output_shapes[1].get_max_shape();
result.selected_numrois_shape = output_shapes[2].to_shape();
result.boxes_shape = inputs[boxes_port]->get_shape();
result.scores_shape = inputs[scores_port]->get_shape();
result.boxes_data = prepare_boxes_data(inputs[boxes_port], result.boxes_shape);
result.scores_data = prepare_scores_data(inputs[scores_port], result.scores_shape);
if (inputs.size() == 3) {
result.roisnum_shape = inputs[roisnum_port]->get_shape();
result.roisnum_data = prepare_roisnum_data(inputs[roisnum_port], result.roisnum_shape);
}
result.selected_outputs_shape_size = shape_size(result.selected_outputs_shape);
result.selected_indices_shape_size = shape_size(result.selected_indices_shape);
result.selected_numrois_shape_size = shape_size(result.selected_numrois_shape);
return result;
}
} // namespace multiclass_nms
template <ngraph::element::Type_t ET>
bool evaluate(const std::shared_ptr<ngraph::op::v8::MulticlassNms>& op,
const ngraph::HostTensorVector& outputs,
const ngraph::HostTensorVector& inputs) {
auto info = ngraph::runtime::reference::multiclass_nms_impl::get_info_for_nms_eval(op, inputs);
auto info = multiclass_nms::get_info_for_nms_eval(op, inputs);
std::vector<float> selected_outputs(info.selected_outputs_shape_size);
std::vector<int64_t> selected_indices(info.selected_indices_shape_size);
@ -62,7 +139,7 @@ template <ngraph::element::Type_t ET>
bool evaluate(const std::shared_ptr<ngraph::op::v9::MulticlassNms>& op,
const ngraph::HostTensorVector& outputs,
const ngraph::HostTensorVector& inputs) {
auto info = ngraph::runtime::reference::multiclass_nms_impl::get_info_for_nms_eval(op, inputs);
auto info = multiclass_nms::get_info_for_nms_eval(op, inputs);
std::vector<float> selected_outputs(info.selected_outputs_shape_size);
std::vector<int64_t> selected_indices(info.selected_indices_shape_size);