diff --git a/src/common/transformations/src/ov_ops/multiclass_nms_ie_internal.cpp b/src/common/transformations/src/ov_ops/multiclass_nms_ie_internal.cpp index 934b9e191c4..a832c17b4d1 100644 --- a/src/common/transformations/src/ov_ops/multiclass_nms_ie_internal.cpp +++ b/src/common/transformations/src/ov_ops/multiclass_nms_ie_internal.cpp @@ -38,23 +38,17 @@ std::shared_ptr op::internal::MulticlassNmsIEInternal::clone_with_new_inpu void op::internal::MulticlassNmsIEInternal::validate_and_infer_types() { INTERNAL_OP_SCOPE(internal_MulticlassNmsIEInternal_validate_and_infer_types); - const auto output_type = get_attrs().output_type; + + OPENVINO_SUPPRESS_DEPRECATED_START + const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END + + const auto output_shapes = shape_infer(this, input_shapes, false, true); validate(); - const auto& boxes_ps = get_input_partial_shape(0); - const auto& scores_ps = get_input_partial_shape(1); - std::vector input_shapes = {boxes_ps, scores_ps}; - if (get_input_size() == 3) { - const auto& roisnum_ps = get_input_partial_shape(2); - input_shapes.push_back(roisnum_ps); - } - - std::vector output_shapes = {{Dimension::dynamic(), 6}, - {Dimension::dynamic(), 1}, - {Dimension::dynamic()}}; - shape_infer(this, input_shapes, output_shapes, true, true); - set_output_type(0, get_input_element_type(0), output_shapes[0]); - set_output_type(1, output_type, output_shapes[1]); + const auto& output_type = get_attrs().output_type; + set_output_type(0, get_input_element_type(0), output_shapes[0].get_max_shape()); + set_output_type(1, output_type, output_shapes[1].get_max_shape()); set_output_type(2, output_type, output_shapes[2]); } diff --git a/src/core/include/openvino/op/util/multiclass_nms_base.hpp b/src/core/include/openvino/op/util/multiclass_nms_base.hpp index 6e4b2d87228..514bac0636f 100644 --- a/src/core/include/openvino/op/util/multiclass_nms_base.hpp +++ b/src/core/include/openvino/op/util/multiclass_nms_base.hpp @@ -64,6 +64,8 @@ public: return m_attrs; } + void set_attrs(Attributes attrs); + void set_output_type(const element::Type& output_type) { m_attrs.output_type = output_type; } diff --git a/src/core/reference/include/ngraph/runtime/reference/multiclass_nms.hpp b/src/core/reference/include/ngraph/runtime/reference/multiclass_nms.hpp index f6bd1eca4ed..ecdfc56cc6b 100644 --- a/src/core/reference/include/ngraph/runtime/reference/multiclass_nms.hpp +++ b/src/core/reference/include/ngraph/runtime/reference/multiclass_nms.hpp @@ -22,27 +22,6 @@ namespace ngraph { namespace runtime { namespace reference { -namespace multiclass_nms_impl { -struct InfoForNMS { - Shape selected_outputs_shape; - Shape selected_indices_shape; - Shape selected_numrois_shape; - Shape boxes_shape; - Shape scores_shape; - Shape roisnum_shape; - std::vector boxes_data; - std::vector scores_data; - std::vector roisnum_data; - size_t selected_outputs_shape_size; - size_t selected_indices_shape_size; - size_t selected_numrois_shape_size; -}; - -OPENVINO_SUPPRESS_DEPRECATED_START -InfoForNMS get_info_for_nms_eval(const std::shared_ptr& nms, - const std::vector>& inputs); -OPENVINO_SUPPRESS_DEPRECATED_END -} // namespace multiclass_nms_impl void multiclass_nms(const float* boxes_data, const Shape& boxes_data_shape, diff --git a/src/core/reference/src/runtime/reference/multiclass_nms.cpp b/src/core/reference/src/runtime/reference/multiclass_nms.cpp index 77818f49a64..a8ba8369b52 100644 --- a/src/core/reference/src/runtime/reference/multiclass_nms.cpp +++ b/src/core/reference/src/runtime/reference/multiclass_nms.cpp @@ -10,7 +10,6 @@ #include #include -#include "../shape_inference/include/multiclass_nms_shape_inference.hpp" #include "ngraph/runtime/reference/multiclass_nms.hpp" #include "ngraph/runtime/reference/utils/nms_common.hpp" #include "ngraph/shape.hpp" @@ -21,162 +20,6 @@ namespace reference { namespace multiclass_nms_impl { OPENVINO_SUPPRESS_DEPRECATED_START -namespace { -std::vector get_floats(const std::shared_ptr& input, const Shape& shape) { - size_t input_size = shape_size(shape); - std::vector result(input_size); - - switch (input->get_element_type()) { - case element::Type_t::bf16: { - bfloat16* p = input->get_data_ptr(); - for (size_t i = 0; i < input_size; ++i) { - result[i] = float(p[i]); - } - } break; - case element::Type_t::f16: { - float16* p = input->get_data_ptr(); - for (size_t i = 0; i < input_size; ++i) { - result[i] = float(p[i]); - } - } break; - case element::Type_t::f32: { - float* p = input->get_data_ptr(); - memcpy(result.data(), p, input_size * sizeof(float)); - } break; - default: - throw std::runtime_error("Unsupported data type."); - break; - } - - return result; -} - -std::vector get_integers(const std::shared_ptr& input, const Shape& shape) { - size_t input_size = shape_size(shape); - std::vector result(input_size); - - switch (input->get_element_type()) { - case element::Type_t::i8: { - auto p = input->get_data_ptr(); - for (size_t i = 0; i < input_size; ++i) { - result[i] = int64_t(p[i]); - } - } break; - case element::Type_t::i16: { - auto p = input->get_data_ptr(); - for (size_t i = 0; i < input_size; ++i) { - result[i] = int64_t(p[i]); - } - } break; - case element::Type_t::i32: { - auto p = input->get_data_ptr(); - for (size_t i = 0; i < input_size; ++i) { - result[i] = int64_t(p[i]); - } - } break; - case element::Type_t::i64: { - auto p = input->get_data_ptr(); - for (size_t i = 0; i < input_size; ++i) { - result[i] = int64_t(p[i]); - } - } break; - case element::Type_t::u8: { - auto p = input->get_data_ptr(); - for (size_t i = 0; i < input_size; ++i) { - result[i] = int64_t(p[i]); - } - } break; - case element::Type_t::u16: { - auto p = input->get_data_ptr(); - for (size_t i = 0; i < input_size; ++i) { - result[i] = int64_t(p[i]); - } - } break; - case element::Type_t::u32: { - auto p = input->get_data_ptr(); - for (size_t i = 0; i < input_size; ++i) { - result[i] = int64_t(p[i]); - } - } break; - case element::Type_t::u64: { - auto p = input->get_data_ptr(); - for (size_t i = 0; i < input_size; ++i) { - result[i] = int64_t(p[i]); - } - } break; - default: - throw std::runtime_error("Unsupported data type"); - break; - } - - return result; -} - -static std::vector prepare_boxes_data(const std::shared_ptr& boxes, const Shape& boxes_shape) { - auto result = get_floats(boxes, boxes_shape); - return result; -} - -static std::vector prepare_scores_data(const std::shared_ptr& scores, const Shape& scores_shape) { - auto result = get_floats(scores, scores_shape); - return result; -} - -static std::vector prepare_roisnum_data(const std::shared_ptr& roisnum, - const Shape& roisnum_shape) { - auto result = get_integers(roisnum, roisnum_shape); - return result; -} - -} // namespace - -constexpr size_t boxes_port = 0; -constexpr size_t scores_port = 1; -constexpr size_t roisnum_port = 2; - -InfoForNMS get_info_for_nms_eval(const std::shared_ptr& nms, - const std::vector>& inputs) { - InfoForNMS result; - - const auto boxes_ps = inputs[boxes_port]->get_partial_shape(); - const auto scores_ps = inputs[scores_port]->get_partial_shape(); - std::vector input_shapes = {boxes_ps, scores_ps}; - if (nms->get_input_size() == 3) { - const auto roisnum_ps = inputs[roisnum_port]->get_partial_shape(); - input_shapes.push_back(roisnum_ps); - } - - std::vector output_shapes = {{Dimension::dynamic(), 6}, - {Dimension::dynamic(), 1}, - {Dimension::dynamic()}}; - ov::op::util::shape_infer(nms.get(), - input_shapes, - output_shapes, - true, - false); // here just for upper boundary estimation. - - result.selected_outputs_shape = output_shapes[0].to_shape(); - result.selected_indices_shape = output_shapes[1].to_shape(); - result.selected_numrois_shape = output_shapes[2].to_shape(); - - result.boxes_shape = inputs[boxes_port]->get_shape(); - result.scores_shape = inputs[scores_port]->get_shape(); - - result.boxes_data = prepare_boxes_data(inputs[boxes_port], result.boxes_shape); - result.scores_data = prepare_scores_data(inputs[scores_port], result.scores_shape); - - if (inputs.size() == 3) { - result.roisnum_shape = inputs[roisnum_port]->get_shape(); - result.roisnum_data = prepare_roisnum_data(inputs[roisnum_port], result.roisnum_shape); - } - - result.selected_outputs_shape_size = shape_size(result.selected_outputs_shape); - result.selected_indices_shape_size = shape_size(result.selected_indices_shape); - result.selected_numrois_shape_size = shape_size(result.selected_numrois_shape); - - return result; -} - using Rectangle = runtime::reference::nms_common::Rectangle; using BoxInfo = runtime::reference::nms_common::BoxInfo; static float intersectionOverUnion(const Rectangle& boxI, const Rectangle& boxJ, const bool normalized) { diff --git a/src/core/shape_inference/include/multiclass_nms_shape_inference.hpp b/src/core/shape_inference/include/multiclass_nms_shape_inference.hpp index 15c022c3578..f6882e12586 100644 --- a/src/core/shape_inference/include/multiclass_nms_shape_inference.hpp +++ b/src/core/shape_inference/include/multiclass_nms_shape_inference.hpp @@ -4,197 +4,119 @@ #pragma once -#include -#include #include +#include "nms_shape_inference.hpp" +#include "openvino/op/multiclass_nms.hpp" +#include "openvino/op/util/multiclass_nms_base.hpp" +#include "utils.hpp" + namespace ov { namespace op { -namespace util { - -template -void shape_infer(const ov::op::util::MulticlassNmsBase* op, - const std::vector& input_shapes, - std::vector& output_shapes, - bool static_output = false, - bool ignore_bg_class = false) { - NODE_VALIDATION_CHECK(op, (input_shapes.size() == 2 || input_shapes.size() == 3) && output_shapes.size() == 3); - - const auto& boxes_ps = input_shapes[0]; - const auto& scores_ps = input_shapes[1]; - - const auto& nms_attrs = op->get_attrs(); - const auto nms_top_k = nms_attrs.nms_top_k; - const auto keep_top_k = nms_attrs.keep_top_k; - - // validate rank of each input - if (boxes_ps.rank().is_dynamic() || scores_ps.rank().is_dynamic()) { - return; - } - - if (op->get_input_size() == 3) { - NODE_VALIDATION_CHECK(op, input_shapes.size() == 3); - const auto& roisnum_ps = input_shapes[2]; - if (roisnum_ps.rank().is_dynamic()) { - return; - } - } - - // validate shape of each input - NODE_VALIDATION_CHECK(op, - boxes_ps.rank().is_static() && boxes_ps.rank().get_length() == 3, - "Expected a 3D tensor for the 'boxes' input. Got: ", - boxes_ps); - - NODE_VALIDATION_CHECK(op, - boxes_ps[2].is_static() && boxes_ps[2].get_length() == 4, - "The third dimension of the 'boxes' must be 4. Got: ", - boxes_ps[2]); - - if (ov::is_type(op)) { - NODE_VALIDATION_CHECK(op, - scores_ps.rank().is_static() && scores_ps.rank().get_length() == 3, - "Expected a 3D tensor for the 'scores' input. Got: ", - scores_ps); - } else { - NODE_VALIDATION_CHECK( - op, - scores_ps.rank().is_static() && (scores_ps.rank().get_length() == 3 || scores_ps.rank().get_length() == 2), - "Expected a 2D or 3D tensor for the 'scores' input. Got: ", - scores_ps); - } - - if (op->get_input_size() == 3) { - const auto& roisnum_ps = input_shapes[2]; - NODE_VALIDATION_CHECK(op, - roisnum_ps.rank().is_static() && roisnum_ps.rank().get_length() == 1, - "Expected a 1D tensor for the 'roisnum' input. Got: ", - roisnum_ps); - } - - // validate compatibility of input shapes - if (scores_ps.rank().is_static() && scores_ps.rank().get_length() == 3) { // if scores shape (N, C, M) - const auto num_batches_boxes = boxes_ps[0]; - const auto num_batches_scores = scores_ps[0]; - - NODE_VALIDATION_CHECK(op, - num_batches_boxes.compatible(num_batches_scores), - "The first dimension of both 'boxes' and 'scores' must match. Boxes: ", - num_batches_boxes, - "; Scores: ", - num_batches_scores); - - const auto num_boxes_boxes = boxes_ps[1]; - const auto num_boxes_scores = scores_ps[2]; - NODE_VALIDATION_CHECK(op, - num_boxes_boxes.compatible(num_boxes_scores), - "'boxes' and 'scores' input shapes must match at the second and third " - "dimension respectively. Boxes: ", - num_boxes_boxes, - "; Scores: ", - num_boxes_scores); - } - - if (scores_ps.rank().is_static() && scores_ps.rank().get_length() == 2) { // if scores shape (C, M) - NODE_VALIDATION_CHECK(op, - op->get_input_size() == 3, - "Expected the 'roisnum' input when the input 'scores' is a 2D tensor."); - - const auto num_classes_boxes = boxes_ps[0]; - const auto num_classes_scores = scores_ps[0]; - NODE_VALIDATION_CHECK(op, - num_classes_boxes.compatible(num_classes_scores), - "'boxes' and 'scores' input shapes must match. Boxes: ", - num_classes_boxes, - "; Scores: ", - num_classes_scores); - - const auto num_boxes_boxes = boxes_ps[1]; - const auto num_boxes_scores = scores_ps[1]; - NODE_VALIDATION_CHECK(op, - num_boxes_boxes.compatible(num_boxes_scores), - "'boxes' and 'scores' input shapes must match. Boxes: ", - num_boxes_boxes, - "; Scores: ", - num_boxes_scores); - } - - /* rank of inputs have been static since here. */ - auto _ready_infer = [&]() { - if (boxes_ps.rank().is_dynamic() || scores_ps.rank().is_dynamic()) { - return false; - } - const bool shared = (scores_ps.rank().get_length() == 3); - if (shared) { - return boxes_ps[1].is_static() && scores_ps[1].is_static() && scores_ps[0].is_static(); - } else { - const auto& roisnum_ps = input_shapes[2]; - if (roisnum_ps.rank().is_dynamic()) { - return false; - } - return boxes_ps[1].is_static() && boxes_ps[0].is_static() && roisnum_ps[0].is_static(); - } - }; - - // Here output 0 and output 1 is not the real dimension of output. - // It will be rewritten in the computing runtime. - // But we still need it here for static shape only backends. - auto first_dim_shape = Dimension::dynamic(); - if (_ready_infer()) { - const bool shared = (scores_ps.rank().get_length() == 3); - ov::PartialShape roisnum_ps; - if (!shared) { - roisnum_ps = input_shapes[2]; - } - - const auto num_boxes = shared ? boxes_ps[1].get_length() : boxes_ps[1].get_length(); - auto num_classes = shared ? scores_ps[1].get_length() : boxes_ps[0].get_length(); - auto num_images = shared ? scores_ps[0].get_length() : roisnum_ps[0].get_length(); - - if (ignore_bg_class) { - if (nms_attrs.background_class >= 0 && nms_attrs.background_class < num_classes) { - num_classes = std::max(int64_t{1}, num_classes - 1); - } - } - - int64_t max_output_boxes_per_class = 0; - if (nms_top_k >= 0) - max_output_boxes_per_class = std::min(num_boxes, (int64_t)nms_top_k); - else - max_output_boxes_per_class = num_boxes; - - auto max_output_boxes_per_batch = max_output_boxes_per_class * num_classes; - if (keep_top_k >= 0) - max_output_boxes_per_batch = std::min(max_output_boxes_per_batch, (int64_t)keep_top_k); - - first_dim_shape = static_output ? max_output_boxes_per_batch * num_images - : Dimension(0, max_output_boxes_per_batch * num_images); - } - - // 'selected_outputs' have the following format: - // [number of selected boxes, [class_id, box_score, xmin, ymin, xmax, ymax]] - output_shapes[0] = {first_dim_shape, 6}; - // 'selected_indices' have the following format: - // [number of selected boxes, ] - output_shapes[1] = {first_dim_shape, 1}; - // 'selected_num' have the following format: - // [num_batches, ] - if (op->get_input_size() == 3) { - const auto& roisnum_ps = input_shapes[2]; - if (roisnum_ps.rank().is_static() && roisnum_ps.rank().get_length() > 0) { - output_shapes[2] = {roisnum_ps[0]}; - } else { - output_shapes[2] = {Dimension::dynamic()}; - } - } else { // shared - if (boxes_ps.rank().is_static() && boxes_ps.rank().get_length() > 0) { - output_shapes[2] = {boxes_ps[0]}; - } else { - output_shapes[2] = {Dimension::dynamic()}; - } - } +namespace multiclass_nms { +namespace validate { +template +void scores_shape(const Node* const op, const std::vector& input_shapes) { + const auto scores_rank = input_shapes[1].rank(); + NODE_SHAPE_INFER_CHECK(op, input_shapes, scores_rank.compatible(2), "Expected a 2D tensor for the 'scores' input"); +} +template +void rois_num_shape(const Node* const op, const std::vector& input_shapes) { + NODE_SHAPE_INFER_CHECK(op, + input_shapes, + input_shapes[2].rank().compatible(1), + "Expected a 1D tensor for the 'roisnum' input"); } -} // namespace util +template +void num_boxes(const Node* const op, const std::vector& input_shapes) { + NODE_SHAPE_INFER_CHECK(op, + input_shapes, + input_shapes[0][1].compatible(input_shapes[1][1]), + "'boxes' and 'scores' input shapes must match at the second dimension respectively"); +} +} // namespace validate +} // namespace multiclass_nms + +template > +std::vector shape_infer(const util::MulticlassNmsBase* op, + const std::vector& input_shapes, + const bool static_output = !std::is_same::value, + const bool ignore_bg_class = false) { + const auto inputs_size = input_shapes.size(); + const auto has_rois_num = inputs_size == 3; + NODE_VALIDATION_CHECK(op, (input_shapes.size() == 2 || has_rois_num)); + + using TDim = typename TRShape::value_type; + using V = typename TDim::value_type; + using namespace ov::util; + + nms::validate::boxes_shape(op, input_shapes); + if (has_rois_num) { + multiclass_nms::validate::scores_shape(op, input_shapes); + multiclass_nms::validate::rois_num_shape(op, input_shapes); + } else { + nms::validate::scores_shape(op, input_shapes); + } + + auto output_shapes = std::vector{TRShape{TDim(dim::inf_bound), 6}, + TRShape{TDim(dim::inf_bound), 1}, + TRShape{TDim(dim::inf_bound)}}; + + const auto& boxes_shape = input_shapes[0]; + const auto& scores_shape = input_shapes[1]; + const auto& rois_num_shape = has_rois_num ? input_shapes[2] : PartialShape::dynamic(); + + if (boxes_shape.rank().is_static()) { + const auto scores_rank = scores_shape.rank(); + nms::validate::num_batches(op, input_shapes); + nms::validate::boxes_last_dim(op, input_shapes); + + bool can_infer; + if (has_rois_num && scores_rank.is_static()) { + multiclass_nms::validate::num_boxes(op, input_shapes); + can_infer = rois_num_shape.rank().is_static(); + } else if (!has_rois_num && scores_rank.is_static()) { + nms::validate::num_boxes(op, input_shapes); + can_infer = true; + } else { + can_infer = false; + } + + if (can_infer) { + const auto& nms_attrs = op->get_attrs(); + const auto nms_top_k = nms_attrs.nms_top_k; + const auto keep_top_k = nms_attrs.keep_top_k; + const auto background_class = nms_attrs.background_class; + + const auto& num_classes = has_rois_num ? boxes_shape[0] : scores_shape[1]; + const auto& num_images = has_rois_num ? rois_num_shape[0] : scores_shape[0]; + + auto& selected_boxes = output_shapes[0][0]; + selected_boxes = + (nms_top_k > -1) ? TDim(std::min(boxes_shape[1].get_max_length(), nms_top_k)) : boxes_shape[1]; + + if (ignore_bg_class && (background_class > -1) && (background_class < num_classes.get_max_length())) { + selected_boxes *= std::max(1, num_classes.get_max_length() - 1); + } else { + selected_boxes *= num_classes; + } + + if (keep_top_k > -1 && (keep_top_k < selected_boxes.get_max_length())) { + selected_boxes = TDim(keep_top_k); + } + + selected_boxes *= num_images; + if (std::is_same::value) { + selected_boxes = + static_output ? TDim(selected_boxes.get_max_length()) : TDim(0, selected_boxes.get_max_length()); + } + output_shapes[1][0] = selected_boxes; + output_shapes[2][0] = has_rois_num ? rois_num_shape[0] : boxes_shape[0]; + } + } + + return output_shapes; +} } // namespace op } // namespace ov diff --git a/src/core/shape_inference/include/nms_shape_inference.hpp b/src/core/shape_inference/include/nms_shape_inference.hpp index 99935403dc0..a141b8f675e 100644 --- a/src/core/shape_inference/include/nms_shape_inference.hpp +++ b/src/core/shape_inference/include/nms_shape_inference.hpp @@ -141,11 +141,8 @@ std::vector shape_infer(const Node* op, const std::vector& input_shapes, const ITensorAccessor& ta, const bool static_output) { - // Note: static_output this input make this function to compatible shape_infer function pattern - // but it has special usage in GPU plugin to force static output in special condition. - // This could be removed as for CPU plugin the using StaticShape force output to be static and GPU - // could use in this case ov::Shape but this shape class is not compatible with interface of PartialShape - // ans StaticShape. + // Note: static_output parameter of this shape_infer is exclusively made for GPU internal needs + // To be removed after GPU supports dynamic NMS const auto inputs_size = input_shapes.size(); NODE_VALIDATION_CHECK(op, cmp::Between(1, 7)(inputs_size)); using TDim = typename TRShape::value_type; diff --git a/src/core/src/op/multiclass_nms.cpp b/src/core/src/op/multiclass_nms.cpp index 559c129cff5..810e3075dba 100644 --- a/src/core/src/op/multiclass_nms.cpp +++ b/src/core/src/op/multiclass_nms.cpp @@ -2,22 +2,23 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "ngraph/op/multiclass_nms.hpp" +#include "openvino/op/multiclass_nms.hpp" #include "itt.hpp" #include "multiclass_nms_shape_inference.hpp" +#include "openvino/core/validation_util.hpp" -using namespace ngraph; -using namespace op::util; - +namespace ov { +namespace op { // ------------------------------ V8 ------------------------------ +namespace v8 { -op::v8::MulticlassNms::MulticlassNms(const Output& boxes, const Output& scores, const Attributes& attrs) +MulticlassNms::MulticlassNms(const Output& boxes, const Output& scores, const Attributes& attrs) : MulticlassNmsBase({boxes, scores}, attrs) { constructor_validate_and_infer_types(); } -std::shared_ptr op::v8::MulticlassNms::clone_with_new_inputs(const OutputVector& new_args) const { +std::shared_ptr MulticlassNms::clone_with_new_inputs(const OutputVector& new_args) const { OV_OP_SCOPE(MulticlassNms_v8_clone_with_new_inputs); check_new_args_count(this, new_args); NODE_VALIDATION_CHECK(this, new_args.size() >= 2, "Number of inputs must be 2 at least"); @@ -25,40 +26,39 @@ std::shared_ptr op::v8::MulticlassNms::clone_with_new_inputs(const OutputV return std::make_shared(new_args.at(0), new_args.at(1), m_attrs); } -void op::v8::MulticlassNms::validate_and_infer_types() { +void MulticlassNms::validate_and_infer_types() { OV_OP_SCOPE(MulticlassNms_v9_validate_and_infer_types); - const auto output_type = get_attrs().output_type; + OPENVINO_SUPPRESS_DEPRECATED_START + const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END + + const auto output_shapes = shape_infer(this, input_shapes, false); validate(); - const auto& boxes_ps = get_input_partial_shape(0); - const auto& scores_ps = get_input_partial_shape(1); - std::vector input_shapes = {boxes_ps, scores_ps}; - std::vector output_shapes = {{Dimension::dynamic(), 6}, - {Dimension::dynamic(), 1}, - {Dimension::dynamic()}}; - shape_infer(this, input_shapes, output_shapes, false, false); + const auto& output_type = get_attrs().output_type; set_output_type(0, get_input_element_type(0), output_shapes[0]); set_output_type(1, output_type, output_shapes[1]); set_output_type(2, output_type, output_shapes[2]); } +} // namespace v8 // ------------------------------ V9 ------------------------------ - -op::v9::MulticlassNms::MulticlassNms(const Output& boxes, const Output& scores, const Attributes& attrs) +namespace v9 { +MulticlassNms::MulticlassNms(const Output& boxes, const Output& scores, const Attributes& attrs) : MulticlassNmsBase({boxes, scores}, attrs) { constructor_validate_and_infer_types(); } -op::v9::MulticlassNms::MulticlassNms(const Output& boxes, - const Output& scores, - const Output& roisnum, - const Attributes& attrs) +MulticlassNms::MulticlassNms(const Output& boxes, + const Output& scores, + const Output& roisnum, + const Attributes& attrs) : MulticlassNmsBase({boxes, scores, roisnum}, attrs) { constructor_validate_and_infer_types(); } -std::shared_ptr op::v9::MulticlassNms::clone_with_new_inputs(const OutputVector& new_args) const { +std::shared_ptr MulticlassNms::clone_with_new_inputs(const OutputVector& new_args) const { OV_OP_SCOPE(MulticlassNms_v9_clone_with_new_inputs); check_new_args_count(this, new_args); NODE_VALIDATION_CHECK(this, new_args.size() == 2 || new_args.size() == 3, "Number of inputs must be 2 or 3"); @@ -71,25 +71,22 @@ std::shared_ptr op::v9::MulticlassNms::clone_with_new_inputs(const OutputV } } -void op::v9::MulticlassNms::validate_and_infer_types() { +void MulticlassNms::validate_and_infer_types() { OV_OP_SCOPE(MulticlassNms_v9_validate_and_infer_types); - const auto output_type = get_attrs().output_type; + + OPENVINO_SUPPRESS_DEPRECATED_START + const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END + + const auto output_shapes = shape_infer(this, input_shapes, false); validate(); - const auto& boxes_ps = get_input_partial_shape(0); - const auto& scores_ps = get_input_partial_shape(1); - std::vector input_shapes = {boxes_ps, scores_ps}; - if (get_input_size() == 3) { - const auto& roisnum_ps = get_input_partial_shape(2); - input_shapes.push_back(roisnum_ps); - } - - std::vector output_shapes = {{Dimension::dynamic(), 6}, - {Dimension::dynamic(), 1}, - {Dimension::dynamic()}}; - shape_infer(this, input_shapes, output_shapes, false, false); + const auto& output_type = get_attrs().output_type; set_output_type(0, get_input_element_type(0), output_shapes[0]); set_output_type(1, output_type, output_shapes[1]); set_output_type(2, output_type, output_shapes[2]); } +} // namespace v9 +} // namespace op +} // namespace ov diff --git a/src/core/src/op/util/multiclass_nms_base.cpp b/src/core/src/op/util/multiclass_nms_base.cpp index ba96f8ae848..5dcbe639106 100644 --- a/src/core/src/op/util/multiclass_nms_base.cpp +++ b/src/core/src/op/util/multiclass_nms_base.cpp @@ -6,8 +6,7 @@ #include "itt.hpp" -using namespace ov; - +namespace ov { op::util::MulticlassNmsBase::MulticlassNmsBase(const OutputVector& arguments, const Attributes& attrs) : Op(arguments), m_attrs{attrs} {} @@ -81,13 +80,16 @@ bool op::util::MulticlassNmsBase::visit_attributes(AttributeVisitor& visitor) { return true; } -std::ostream& ov::operator<<(std::ostream& s, const op::util::MulticlassNmsBase::SortResultType& type) { +void op::util::MulticlassNmsBase::set_attrs(op::util::MulticlassNmsBase::Attributes attrs) { + m_attrs = std::move(attrs); +} + +std::ostream& operator<<(std::ostream& s, const op::util::MulticlassNmsBase::SortResultType& type) { return s << as_string(type); } -namespace ov { template <> -NGRAPH_API EnumNames& +OPENVINO_API EnumNames& EnumNames::get() { static auto enum_names = EnumNames( "op::util::MulticlassNmsBase::SortResultType", diff --git a/src/core/tests/type_prop/multiclass_nms.cpp b/src/core/tests/type_prop/multiclass_nms.cpp index 71938822cec..443c20a35fc 100644 --- a/src/core/tests/type_prop/multiclass_nms.cpp +++ b/src/core/tests/type_prop/multiclass_nms.cpp @@ -2,249 +2,236 @@ // SPDX-License-Identifier: Apache-2.0 // +#include "openvino/op/multiclass_nms.hpp" + +#include "common_test_utils/test_assertions.hpp" #include "common_test_utils/type_prop.hpp" #include "gtest/gtest.h" -#include "ngraph/ngraph.hpp" using namespace std; -using namespace ngraph; +using namespace ov; +using namespace testing; template -class type_prop : public testing::Test {}; +class type_prop : public testing::Test { +protected: + using Attributes = op::util::MulticlassNmsBase::Attributes; -typedef testing::Types test_prop_types; -TYPED_TEST_SUITE(type_prop, test_prop_types); + Attributes attrs; +}; + +using MulticlassNmsTypes = testing::Types; +TYPED_TEST_SUITE(type_prop, MulticlassNmsTypes); TYPED_TEST(type_prop, multiclass_nms_incorrect_boxes_rank) { - try { - const auto boxes = make_shared(element::f32, Shape{1, 2, 3, 4}); - const auto scores = make_shared(element::f32, Shape{1, 2, 3}); + const auto boxes = make_shared(element::f32, Shape{1, 2, 3, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 3}); - const auto unused = make_shared(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes()); - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Expected a 3D tensor for the 'boxes' input"); - } + OV_EXPECT_THROW(ignore = make_shared(boxes, scores, this->attrs), + NodeValidationFailure, + HasSubstr("Expected a 3D tensor for the 'boxes' input")); } TEST(type_prop2, multiclass_nms_incorrect_boxes_rank) { - try { - const auto boxes = make_shared(element::f32, Shape{1, 2, 3, 4}); - const auto scores = make_shared(element::f32, Shape{1, 2, 3}); - const auto roisnum = make_shared(element::i32, Shape{1}); + const auto boxes = make_shared(element::f32, Shape{1, 2, 3, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 3}); + const auto roisnum = make_shared(element::i32, Shape{1}); - const auto unused = - make_shared(boxes, scores, roisnum, ov::op::util::MulticlassNmsBase::Attributes()); - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Expected a 3D tensor for the 'boxes' input"); - } + OV_EXPECT_THROW( + ignore = make_shared(boxes, scores, op::util::MulticlassNmsBase::Attributes()), + NodeValidationFailure, + HasSubstr("Expected a 3D tensor for the 'boxes' input")); } TYPED_TEST(type_prop, multiclass_nms_incorrect_scores_rank) { - try { - const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); - const auto scores = make_shared(element::f32, Shape{1}); + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1}); - const auto unused = make_shared(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes()); - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), - TypeParam::get_type_info_static().get_version() == "opset8" - ? "Expected a 3D tensor for the 'scores' input" - : "Expected a 2D or 3D tensor for the 'scores' input"); - } + OV_EXPECT_THROW(ignore = make_shared(boxes, scores, this->attrs), + NodeValidationFailure, + HasSubstr("Expected a 3D tensor for the 'scores' input")); } TEST(type_prop2, multiclass_nms_incorrect_scores_rank2) { - try { - const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); - const auto scores = make_shared(element::f32, Shape{1, 2}); + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2}); - const auto unused = - make_shared(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes()); - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Expected the 'roisnum' input when the input 'scores' is a 2D tensor."); - } + OV_EXPECT_THROW( + ignore = make_shared(boxes, scores, op::util::MulticlassNmsBase::Attributes()), + NodeValidationFailure, + HasSubstr("Expected a 3D tensor for the 'scores' input")); } TEST(type_prop2, multiclass_nms_incorrect_roisnum_rank) { - try { - const auto boxes = make_shared(element::f32, Shape{2, 3, 4}); - const auto scores = make_shared(element::f32, Shape{2, 3}); - const auto roisnum = make_shared(element::i32, Shape{1, 2}); + const auto boxes = make_shared(element::f32, Shape{2, 3, 4}); + const auto scores = make_shared(element::f32, Shape{2, 3}); + const auto roisnum = make_shared(element::i32, Shape{1, 2}); - const auto unused = - make_shared(boxes, scores, roisnum, ov::op::util::MulticlassNmsBase::Attributes()); - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Expected a 1D tensor for the 'roisnum' input"); - } + OV_EXPECT_THROW( + ignore = make_shared(boxes, scores, roisnum, op::util::MulticlassNmsBase::Attributes()), + NodeValidationFailure, + HasSubstr("Expected a 1D tensor for the 'roisnum' input")); } TYPED_TEST(type_prop, multiclass_nms_incorrect_scheme_num_batches) { - try { - const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); - const auto scores = make_shared(element::f32, Shape{2, 2, 3}); + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{2, 2, 3}); - const auto unused = make_shared(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes()); - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "The first dimension of both 'boxes' and 'scores' must match"); - } + OV_EXPECT_THROW(ignore = make_shared(boxes, scores, this->attrs), + NodeValidationFailure, + HasSubstr("The first dimension of both 'boxes' and 'scores' must match")); } TEST(type_prop2, multiclass_nms_incorrect_scheme_num_classes) { - try { - const auto boxes = make_shared(element::f32, Shape{2, 3, 4}); - const auto scores = make_shared(element::f32, Shape{1, 3}); - const auto roisnum = make_shared(element::i32, Shape{1}); + const auto boxes = make_shared(element::f32, Shape{2, 3, 4}); + const auto scores = make_shared(element::f32, Shape{1, 3}); + const auto roisnum = make_shared(element::i32, Shape{1}); - const auto unused = - make_shared(boxes, scores, roisnum, ov::op::util::MulticlassNmsBase::Attributes()); - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "'boxes' and 'scores' input shapes must match"); - } + OV_EXPECT_THROW( + ignore = make_shared(boxes, scores, roisnum, op::util::MulticlassNmsBase::Attributes()), + NodeValidationFailure, + HasSubstr("The first dimension of both 'boxes' and 'scores' must match")); } TYPED_TEST(type_prop, multiclass_nms_incorrect_scheme_num_boxes) { - try { - const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); - const auto scores = make_shared(element::f32, Shape{1, 2, 3}); + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 3}); - const auto unused = make_shared(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes()); - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), - "'boxes' and 'scores' input shapes must match at the second and third " - "dimension respectively"); - } + OV_EXPECT_THROW(ignore = make_shared(boxes, scores, this->attrs), + NodeValidationFailure, + HasSubstr("'boxes' and 'scores' input shapes must match at the second and third " + "dimension respectively")); } TEST(type_prop2, multiclass_nms_incorrect_scheme_num_boxes) { - try { - const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); - const auto scores = make_shared(element::f32, Shape{1, 3}); - const auto roisnum = make_shared(element::i32, Shape{1}); + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 3}); + const auto roisnum = make_shared(element::i32, Shape{1}); - const auto unused = - make_shared(boxes, scores, roisnum, ov::op::util::MulticlassNmsBase::Attributes()); - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "'boxes' and 'scores' input shapes must match."); - } + OV_EXPECT_THROW( + ignore = make_shared(boxes, scores, roisnum, op::util::MulticlassNmsBase::Attributes()), + NodeValidationFailure, + HasSubstr("'boxes' and 'scores' input shapes must match at the second dimension respectively")); } TEST(type_prop2, multiclass_nms_incorrect_scheme_num_boxes2) { - try { - const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); - const auto scores = make_shared(element::f32, Shape{3, 2}); - const auto roisnum = make_shared(element::i32, Shape{1}); + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{3, 2}); + const auto roisnum = make_shared(element::i32, Shape{1}); - const auto unused = - make_shared(boxes, scores, roisnum, ov::op::util::MulticlassNmsBase::Attributes()); - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "'boxes' and 'scores' input shapes must match."); - } + OV_EXPECT_THROW( + ignore = make_shared(boxes, scores, roisnum, op::util::MulticlassNmsBase::Attributes()), + NodeValidationFailure, + HasSubstr("The first dimension of both 'boxes' and 'scores' must match")); } TYPED_TEST(type_prop, multiclass_nms_incorrect_boxes_rank2) { - try { - const auto boxes = make_shared(element::f32, Shape{1, 2, 3}); - const auto scores = make_shared(element::f32, Shape{2, 2, 2}); + const auto boxes = make_shared(element::f32, Shape{2, 2, 3}); + const auto scores = make_shared(element::f32, Shape{2, 2, 2}); - const auto unused = make_shared(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes()); - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "The third dimension of the 'boxes' must be 4"); - } + OV_EXPECT_THROW(ignore = make_shared(boxes, scores, this->attrs), + NodeValidationFailure, + HasSubstr("The last dimension of the 'boxes' input must be equal to 4")); } TYPED_TEST(type_prop, multiclass_nms_incorrect_output_type) { - try { - const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); - const auto scores = make_shared(element::f32, Shape{1, 2, 2}); - ov::op::util::MulticlassNmsBase::Attributes attrs; - attrs.output_type = ngraph::element::f32; + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 2}); + this->attrs.output_type = element::f32; - const auto unused = make_shared(boxes, scores, attrs); - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Output type must be i32 or i64"); - } + OV_EXPECT_THROW(ignore = make_shared(boxes, scores, this->attrs), + NodeValidationFailure, + HasSubstr("Output type must be i32 or i64")); } TYPED_TEST(type_prop, multiclass_nms_incorrect_nms_topk) { - try { - const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); - const auto scores = make_shared(element::f32, Shape{1, 2, 2}); - ov::op::util::MulticlassNmsBase::Attributes attrs; - attrs.nms_top_k = -2; + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 2}); + this->attrs.nms_top_k = -2; - const auto unused = make_shared(boxes, scores, attrs); - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "The 'nms_top_k' must be great or equal -1"); - } + OV_EXPECT_THROW(ignore = make_shared(boxes, scores, this->attrs), + NodeValidationFailure, + HasSubstr("The 'nms_top_k' must be great or equal -1")); } TYPED_TEST(type_prop, multiclass_nms_incorrect_keep_topk) { - try { - const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); - const auto scores = make_shared(element::f32, Shape{1, 2, 2}); - ov::op::util::MulticlassNmsBase::Attributes attrs; - attrs.keep_top_k = -2; + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 2}); + this->attrs.keep_top_k = -2; - const auto unused = make_shared(boxes, scores, attrs); - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "The 'keep_top_k' must be great or equal -1"); - } + OV_EXPECT_THROW(ignore = make_shared(boxes, scores, this->attrs), + NodeValidationFailure, + HasSubstr("The 'keep_top_k' must be great or equal -1")); } TYPED_TEST(type_prop, multiclass_nms_incorrect_background_class) { - try { - const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); - const auto scores = make_shared(element::f32, Shape{1, 2, 2}); - ov::op::util::MulticlassNmsBase::Attributes attrs; - attrs.background_class = -2; + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 2}); + this->attrs.background_class = -2; - const auto unused = make_shared(boxes, scores, attrs); - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "The 'background_class' must be great or equal -1"); - } + OV_EXPECT_THROW(ignore = make_shared(boxes, scores, this->attrs), + NodeValidationFailure, + HasSubstr("The 'background_class' must be great or equal -1")); } TYPED_TEST(type_prop, multiclass_nms_incorrect_eta) { - try { - const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); - const auto scores = make_shared(element::f32, Shape{1, 2, 2}); - ov::op::util::MulticlassNmsBase::Attributes attrs; - attrs.nms_eta = 2.0f; + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 2}); + this->attrs.nms_eta = 2.0f; - const auto unused = make_shared(boxes, scores, attrs); - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "The 'nms_eta' must be in close range [0, 1.0]"); - } + OV_EXPECT_THROW(ignore = make_shared(boxes, scores, this->attrs), + NodeValidationFailure, + HasSubstr("The 'nms_eta' must be in close range [0, 1.0]")); } TYPED_TEST(type_prop, multiclass_nms_incorrect_input_type) { - try { - const auto boxes = make_shared(element::f16, Shape{1, 2, 4}); - const auto scores = make_shared(element::f32, Shape{1, 2, 2}); + const auto boxes = make_shared(element::f16, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 2}); - const auto unused = make_shared(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes()); - } catch (const NodeValidationFailure& error) { - EXPECT_HAS_SUBSTRING(error.what(), "Expected 'boxes', 'scores' type is same."); - } + OV_EXPECT_THROW(ignore = make_shared(boxes, scores, this->attrs), + NodeValidationFailure, + HasSubstr("Expected 'boxes', 'scores' type is same.")); +} + +TYPED_TEST(type_prop, multiclass_nms_default_ctor) { + const auto boxes = make_shared(element::f32, Shape{2, 7, 4}); + const auto scores = make_shared(element::f32, Shape{2, 5, 7}); + this->attrs.nms_top_k = 3; + + const auto nms = make_shared(); + nms->set_arguments(OutputVector{boxes, scores}); + nms->set_attrs(this->attrs); + nms->validate_and_infer_types(); + + EXPECT_EQ(nms->get_output_partial_shape(0), (PartialShape{{0, 30}, 6})); + EXPECT_EQ(nms->get_output_partial_shape(1), (PartialShape{{0, 30}, 1})); + EXPECT_EQ(nms->get_output_partial_shape(2), (PartialShape{2})); } TYPED_TEST(type_prop, multiclass_nms_output_shape_1dim_dynamic) { - const auto boxes = make_shared(element::f32, Shape{5, 2, 4}); - const auto scores = make_shared(element::f32, Shape{5, 3, 2}); + auto boxes_shape = PartialShape{5, 2, 4}; + auto scores_shape = PartialShape{5, 3, 2}; + set_shape_labels(boxes_shape, 10); + set_shape_labels(scores_shape, 20); - const auto nms = make_shared(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes()); + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); - ASSERT_TRUE(nms->get_output_partial_shape(0).same_scheme(PartialShape{Dimension::dynamic(), 6})); - ASSERT_TRUE(nms->get_output_partial_shape(1).same_scheme(PartialShape{Dimension::dynamic(), 1})); + const auto nms = make_shared(boxes, scores, this->attrs); - EXPECT_EQ(nms->get_output_shape(2), (Shape{5})); + EXPECT_EQ(nms->get_output_partial_shape(0), (PartialShape{{0, 30}, 6})); + EXPECT_EQ(nms->get_output_partial_shape(1), (PartialShape{{0, 30}, 1})); + EXPECT_EQ(nms->get_output_partial_shape(2), (PartialShape{5})); + EXPECT_THAT(get_shape_labels(nms->get_output_partial_shape(0)), Each(no_label)); + EXPECT_THAT(get_shape_labels(nms->get_output_partial_shape(1)), Each(no_label)); + EXPECT_THAT(get_shape_labels(nms->get_output_partial_shape(2)), ElementsAre(10)); } TYPED_TEST(type_prop, multiclass_nms_output_shape_1dim_max_out) { - const auto boxes = make_shared(element::f32, Shape{2, 7, 4}); - const auto scores = make_shared(element::f32, Shape{2, 5, 7}); + const auto boxes = make_shared(element::f32, Shape{2, 7, 4}); + const auto scores = make_shared(element::f32, Shape{2, 5, 7}); - const auto nms = make_shared(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes()); + const auto nms = make_shared(boxes, scores, this->attrs); ASSERT_EQ(nms->get_output_element_type(0), element::f32); ASSERT_EQ(nms->get_output_element_type(1), element::i64); @@ -257,12 +244,11 @@ TYPED_TEST(type_prop, multiclass_nms_output_shape_1dim_max_out) { } TYPED_TEST(type_prop, multiclass_nms_output_shape_1dim_nms_topk) { - const auto boxes = make_shared(element::f32, Shape{2, 7, 4}); - const auto scores = make_shared(element::f32, Shape{2, 5, 7}); - ov::op::util::MulticlassNmsBase::Attributes attrs; - attrs.nms_top_k = 3; + const auto boxes = make_shared(element::f32, Shape{2, 7, 4}); + const auto scores = make_shared(element::f32, Shape{2, 5, 7}); + this->attrs.nms_top_k = 3; - const auto nms = make_shared(boxes, scores, attrs); + const auto nms = make_shared(boxes, scores, this->attrs); ASSERT_EQ(nms->get_output_element_type(0), element::f32); ASSERT_EQ(nms->get_output_element_type(1), element::i64); @@ -274,13 +260,12 @@ TYPED_TEST(type_prop, multiclass_nms_output_shape_1dim_nms_topk) { } TYPED_TEST(type_prop, multiclass_nms_output_shape_1dim_keep_topk) { - const auto boxes = make_shared(element::f32, Shape{2, 7, 4}); - const auto scores = make_shared(element::f32, Shape{2, 5, 7}); - ov::op::util::MulticlassNmsBase::Attributes attrs; - attrs.nms_top_k = 3; - attrs.keep_top_k = 8; + const auto boxes = make_shared(element::f32, Shape{2, 7, 4}); + const auto scores = make_shared(element::f32, Shape{2, 5, 7}); + this->attrs.nms_top_k = 3; + this->attrs.keep_top_k = 8; - const auto nms = make_shared(boxes, scores, attrs); + const auto nms = make_shared(boxes, scores, this->attrs); ASSERT_EQ(nms->get_output_element_type(0), element::f32); ASSERT_EQ(nms->get_output_element_type(1), element::i64); @@ -292,10 +277,10 @@ TYPED_TEST(type_prop, multiclass_nms_output_shape_1dim_keep_topk) { } TYPED_TEST(type_prop, multiclass_nms_input_f16) { - const auto boxes = make_shared(element::f16, Shape{2, 7, 4}); - const auto scores = make_shared(element::f16, Shape{2, 5, 7}); + const auto boxes = make_shared(element::f16, Shape{2, 7, 4}); + const auto scores = make_shared(element::f16, Shape{2, 5, 7}); - const auto nms = make_shared(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes()); + const auto nms = make_shared(boxes, scores, this->attrs); ASSERT_EQ(nms->get_output_element_type(0), element::f16); ASSERT_EQ(nms->get_output_element_type(1), element::i64); @@ -307,12 +292,11 @@ TYPED_TEST(type_prop, multiclass_nms_input_f16) { } TYPED_TEST(type_prop, multiclass_nms_output_shape_i32) { - const auto boxes = make_shared(element::f32, Shape{2, 7, 4}); - const auto scores = make_shared(element::f32, Shape{2, 5, 7}); - ov::op::util::MulticlassNmsBase::Attributes attrs; - attrs.output_type = ngraph::element::i32; + const auto boxes = make_shared(element::f32, Shape{2, 7, 4}); + const auto scores = make_shared(element::f32, Shape{2, 5, 7}); + this->attrs.output_type = element::i32; - const auto nms = make_shared(boxes, scores, attrs); + const auto nms = make_shared(boxes, scores, this->attrs); ASSERT_EQ(nms->get_output_element_type(0), element::f32); ASSERT_EQ(nms->get_output_element_type(1), element::i32); @@ -324,10 +308,10 @@ TYPED_TEST(type_prop, multiclass_nms_output_shape_i32) { } TYPED_TEST(type_prop, multiclass_nms_dynamic_boxes_and_scores) { - const auto boxes = make_shared(element::f32, PartialShape::dynamic()); - const auto scores = make_shared(element::f32, PartialShape::dynamic()); + const auto boxes = make_shared(element::f32, PartialShape::dynamic()); + const auto scores = make_shared(element::f32, PartialShape::dynamic()); - const auto nms = make_shared(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes()); + const auto nms = make_shared(boxes, scores, this->attrs); ASSERT_EQ(nms->get_output_element_type(0), element::f32); ASSERT_EQ(nms->get_output_element_type(1), element::i64); @@ -338,9 +322,9 @@ TYPED_TEST(type_prop, multiclass_nms_dynamic_boxes_and_scores) { } TEST(type_prop2, multiclass_nms_dynamic_boxes_and_scores) { - const auto boxes = make_shared(element::f32, PartialShape::dynamic()); - const auto scores = make_shared(element::f32, PartialShape::dynamic()); - const auto roisnum = make_shared(element::i32, PartialShape::dynamic()); + const auto boxes = make_shared(element::f32, PartialShape::dynamic()); + const auto scores = make_shared(element::f32, PartialShape::dynamic()); + const auto roisnum = make_shared(element::i32, PartialShape::dynamic()); const auto nms = make_shared(boxes, scores, roisnum, ov::op::util::MulticlassNmsBase::Attributes()); @@ -353,14 +337,55 @@ TEST(type_prop2, multiclass_nms_dynamic_boxes_and_scores) { EXPECT_EQ(nms->get_output_partial_shape(2), PartialShape({Dimension::dynamic()})); } +TEST(type_prop2, multiclass_nms_interval_shapes_and_labels) { + auto boxes_shape = PartialShape{2, 7, 4}; + auto scores_shape = PartialShape{2, 7}; + auto roisnum_shape = PartialShape{4}; + set_shape_labels(boxes_shape, 10); + set_shape_labels(scores_shape, 20); + set_shape_labels(roisnum_shape, 30); + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + const auto roisnum = make_shared(element::i32, roisnum_shape); + + const auto nms = + make_shared(boxes, scores, roisnum, ov::op::util::MulticlassNmsBase::Attributes()); + + ASSERT_EQ(nms->get_output_element_type(0), element::f32); + ASSERT_EQ(nms->get_output_element_type(1), element::i64); + ASSERT_EQ(nms->get_output_element_type(2), element::i64); + EXPECT_EQ(nms->get_output_partial_shape(0), PartialShape({{0, 56}, 6})); + EXPECT_EQ(nms->get_output_partial_shape(1), PartialShape({{0, 56}, 1})); + EXPECT_EQ(nms->get_output_partial_shape(2), PartialShape({4})); + EXPECT_THAT(get_shape_labels(nms->get_output_partial_shape(0)), Each(no_label)); + EXPECT_THAT(get_shape_labels(nms->get_output_partial_shape(1)), Each(no_label)); + EXPECT_THAT(get_shape_labels(nms->get_output_partial_shape(2)), ElementsAre(30)); +} + +TEST(type_prop2, multiclass_nms_static_shapes) { + const auto boxes = make_shared(element::f32, PartialShape{3, 7, 4}); + const auto scores = make_shared(element::f32, PartialShape{3, 7}); + const auto roisnum = make_shared(element::i32, PartialShape{4}); + ov::op::util::MulticlassNmsBase::Attributes attrs; + attrs.nms_top_k = 3; + attrs.keep_top_k = 8; + + const auto nms = make_shared(boxes, scores, roisnum, attrs); + + EXPECT_EQ(nms->get_output_partial_shape(0), PartialShape({{0, 32}, Dimension(6)})); + EXPECT_EQ(nms->get_output_partial_shape(1), PartialShape({{0, 32}, 1})); + EXPECT_EQ(nms->get_output_partial_shape(2), PartialShape({4})); +} + TYPED_TEST(type_prop, multiclass_nms_dynamic_boxes_and_scores2) { const auto boxes = - make_shared(element::f32, PartialShape({Dimension::dynamic(), Dimension::dynamic(), 4})); - const auto scores = - make_shared(element::f32, - PartialShape({Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic()})); + make_shared(element::f32, PartialShape({Dimension::dynamic(), Dimension::dynamic(), 4})); + const auto scores = make_shared( + element::f32, + PartialShape({Dimension::dynamic(), Dimension::dynamic(), Dimension::dynamic()})); - const auto nms = make_shared(boxes, scores, ov::op::util::MulticlassNmsBase::Attributes()); + const auto nms = make_shared(boxes, scores, this->attrs); ASSERT_EQ(nms->get_output_element_type(0), element::f32); ASSERT_EQ(nms->get_output_element_type(1), element::i64); @@ -372,10 +397,10 @@ TYPED_TEST(type_prop, multiclass_nms_dynamic_boxes_and_scores2) { TEST(type_prop2, multiclass_nms_dynamic_boxes_and_scores2) { const auto boxes = - make_shared(element::f32, PartialShape({Dimension::dynamic(), Dimension::dynamic(), 4})); + make_shared(element::f32, PartialShape({Dimension::dynamic(), Dimension::dynamic(), 4})); const auto scores = - make_shared(element::f32, PartialShape({Dimension::dynamic(), Dimension::dynamic()})); - const auto roisnum = make_shared(element::i32, PartialShape({Dimension::dynamic()})); + make_shared(element::f32, PartialShape({Dimension::dynamic(), Dimension::dynamic()})); + const auto roisnum = make_shared(element::i32, PartialShape({Dimension::dynamic()})); const auto nms = make_shared(boxes, scores, roisnum, ov::op::util::MulticlassNmsBase::Attributes()); diff --git a/src/plugins/template/backend/ops/multiclass_nms.cpp b/src/plugins/template/backend/ops/multiclass_nms.cpp index 74ff3a8477f..9367a8da3dc 100644 --- a/src/plugins/template/backend/ops/multiclass_nms.cpp +++ b/src/plugins/template/backend/ops/multiclass_nms.cpp @@ -5,13 +5,90 @@ #include "ngraph/runtime/reference/multiclass_nms.hpp" #include "evaluate_node.hpp" +#include "evaluates_map.hpp" +#include "multiclass_nms_shape_inference.hpp" #include "ngraph/runtime/reference/utils/nms_common.hpp" +namespace multiclass_nms { +using namespace ov; + +struct InfoForNMS { + Shape selected_outputs_shape; + Shape selected_indices_shape; + Shape selected_numrois_shape; + Shape boxes_shape; + Shape scores_shape; + Shape roisnum_shape; + std::vector boxes_data; + std::vector scores_data; + std::vector roisnum_data; + size_t selected_outputs_shape_size; + size_t selected_indices_shape_size; + size_t selected_numrois_shape_size; +}; + +static std::vector prepare_boxes_data(const std::shared_ptr& boxes, const Shape& boxes_shape) { + auto result = get_floats(boxes, boxes_shape); + return result; +} + +static std::vector prepare_scores_data(const std::shared_ptr& scores, const Shape& scores_shape) { + auto result = get_floats(scores, scores_shape); + return result; +} + +static std::vector prepare_roisnum_data(const std::shared_ptr& roisnum, + const Shape& roisnum_shape) { + auto result = get_integers(roisnum, roisnum_shape); + return result; +} + +constexpr size_t boxes_port = 0; +constexpr size_t scores_port = 1; +constexpr size_t roisnum_port = 2; + +InfoForNMS get_info_for_nms_eval(const std::shared_ptr& nms, + const std::vector>& inputs) { + InfoForNMS result; + + const auto boxes_ps = inputs[boxes_port]->get_partial_shape(); + const auto scores_ps = inputs[scores_port]->get_partial_shape(); + std::vector input_shapes = {boxes_ps, scores_ps}; + if (nms->get_input_size() == 3) { + const auto roisnum_ps = inputs[roisnum_port]->get_partial_shape(); + input_shapes.push_back(roisnum_ps); + } + + const auto output_shapes = ov::op::shape_infer(nms.get(), input_shapes); + + result.selected_outputs_shape = output_shapes[0].get_max_shape(); + result.selected_indices_shape = output_shapes[1].get_max_shape(); + result.selected_numrois_shape = output_shapes[2].to_shape(); + + result.boxes_shape = inputs[boxes_port]->get_shape(); + result.scores_shape = inputs[scores_port]->get_shape(); + + result.boxes_data = prepare_boxes_data(inputs[boxes_port], result.boxes_shape); + result.scores_data = prepare_scores_data(inputs[scores_port], result.scores_shape); + + if (inputs.size() == 3) { + result.roisnum_shape = inputs[roisnum_port]->get_shape(); + result.roisnum_data = prepare_roisnum_data(inputs[roisnum_port], result.roisnum_shape); + } + + result.selected_outputs_shape_size = shape_size(result.selected_outputs_shape); + result.selected_indices_shape_size = shape_size(result.selected_indices_shape); + result.selected_numrois_shape_size = shape_size(result.selected_numrois_shape); + + return result; +} +} // namespace multiclass_nms + template bool evaluate(const std::shared_ptr& op, const ngraph::HostTensorVector& outputs, const ngraph::HostTensorVector& inputs) { - auto info = ngraph::runtime::reference::multiclass_nms_impl::get_info_for_nms_eval(op, inputs); + auto info = multiclass_nms::get_info_for_nms_eval(op, inputs); std::vector selected_outputs(info.selected_outputs_shape_size); std::vector selected_indices(info.selected_indices_shape_size); @@ -62,7 +139,7 @@ template bool evaluate(const std::shared_ptr& op, const ngraph::HostTensorVector& outputs, const ngraph::HostTensorVector& inputs) { - auto info = ngraph::runtime::reference::multiclass_nms_impl::get_info_for_nms_eval(op, inputs); + auto info = multiclass_nms::get_info_for_nms_eval(op, inputs); std::vector selected_outputs(info.selected_outputs_shape_size); std::vector selected_indices(info.selected_indices_shape_size);