diff --git a/ngraph/core/include/ngraph/op/matrix_nms.hpp b/ngraph/core/include/ngraph/op/matrix_nms.hpp new file mode 100644 index 00000000000..ca2800a921a --- /dev/null +++ b/ngraph/core/include/ngraph/op/matrix_nms.hpp @@ -0,0 +1,102 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "ngraph/op/util/nms_base.hpp" + +namespace ngraph +{ + namespace op + { + namespace v8 + { + /// \brief MatrixNms operation + /// + class NGRAPH_API MatrixNms : public util::NmsBase + { + public: + NGRAPH_RTTI_DECLARATION; + + enum class DecayFunction + { + GAUSSIAN, + LINEAR + }; + + /// \brief Structure that specifies attributes of the operation + struct Attributes + { + // specifies order of output elements + SortResultType sort_result_type = SortResultType::NONE; + // specifies whenever it is necessary to sort selected boxes across batches or + // not + bool sort_result_across_batch = false; + // specifies the output tensor type + ngraph::element::Type output_type = ngraph::element::i64; + // specifies minimum score to consider box for the processing + float score_threshold = 0.0f; + // specifies maximum number of boxes to be selected per class, -1 meaning to + // keep all boxes + int nms_top_k = -1; + // specifies maximum number of boxes to be selected per batch element, -1 + // meaning to keep all boxes + int keep_top_k = -1; + // specifies the background class id, -1 meaning to keep all classes + int background_class = -1; + // specifies decay function used to decay scores + DecayFunction decay_function = DecayFunction::LINEAR; + // specifies gaussian_sigma parameter for gaussian decay_function + float gaussian_sigma = 2.0f; + // specifies threshold to filter out boxes with low confidence score after + // decaying + float post_threshold = 0.0f; + // specifies whether boxes are normalized or not + bool normalized = true; + }; + + MatrixNms(); + + /// \brief Constructs a MatrixNms operation + /// + /// \param boxes Node producing the box coordinates + /// \param scores Node producing the box scores + /// \param attrs Attributes of the operation + MatrixNms(const Output& boxes, + const Output& scores, + const Attributes& attrs); + + bool visit_attributes(AttributeVisitor& visitor) override; + + std::shared_ptr + clone_with_new_inputs(const OutputVector& new_args) const override; + + /// \brief Returns attributes of the operation MatrixNms + const Attributes& get_attrs() const { return m_attrs; } + + protected: + Attributes m_attrs; + + void validate() override; + }; + } // namespace v8 + } // namespace op + NGRAPH_API + std::ostream& operator<<(std::ostream& s, const op::v8::MatrixNms::DecayFunction& type); + + template <> + class NGRAPH_API AttributeAdapter + : public EnumAttributeAdapterBase + { + public: + AttributeAdapter(op::v8::MatrixNms::DecayFunction& value) + : EnumAttributeAdapterBase(value) + { + } + + static constexpr DiscreteTypeInfo type_info{ + "AttributeAdapter", 1}; + const DiscreteTypeInfo& get_type_info() const override { return type_info; } + }; +} // namespace ngraph diff --git a/ngraph/core/include/ngraph/op/multiclass_nms.hpp b/ngraph/core/include/ngraph/op/multiclass_nms.hpp new file mode 100644 index 00000000000..1b351824ab8 --- /dev/null +++ b/ngraph/core/include/ngraph/op/multiclass_nms.hpp @@ -0,0 +1,75 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "ngraph/op/util/nms_base.hpp" + +namespace ngraph +{ + namespace op + { + namespace v8 + { + /// \brief MulticlassNms operation + /// + class NGRAPH_API MulticlassNms : public util::NmsBase + { + public: + NGRAPH_RTTI_DECLARATION; + + /// \brief Structure that specifies attributes of the operation + struct Attributes + { + // specifies order of output elements + SortResultType sort_result_type = SortResultType::NONE; + // specifies whenever it is necessary to sort selected boxes across batches or + // not + bool sort_result_across_batch = false; + // specifies the output tensor type + ngraph::element::Type output_type = ngraph::element::i64; + // specifies intersection over union threshold + float iou_threshold = 0.0f; + // specifies minimum score to consider box for the processing + float score_threshold = 0.0f; + // specifies maximum number of boxes to be selected per class, -1 meaning to + // keep all boxes + int nms_top_k = -1; + // specifies maximum number of boxes to be selected per batch element, -1 + // meaning to keep all boxes + int keep_top_k = -1; + // specifies the background class id, -1 meaning to keep all classes + int background_class = -1; + // specifies eta parameter for adpative NMS, in close range [0, 1.0] + float nms_eta = 1.0f; + // specifies whether boxes are normalized or not + bool normalized = true; + }; + + MulticlassNms(); + + /// \brief Constructs a MulticlassNms operation + /// + /// \param boxes Node producing the box coordinates + /// \param scores Node producing the box scores + /// \param attrs Attributes of the operation + MulticlassNms(const Output& boxes, + const Output& scores, + const Attributes& attrs); + + bool visit_attributes(AttributeVisitor& visitor) override; + + std::shared_ptr + clone_with_new_inputs(const OutputVector& new_args) const override; + + /// \brief Returns attributes of the operation MulticlassNms + const Attributes& get_attrs() const { return m_attrs; } + + protected: + Attributes m_attrs; + void validate() override; + }; + } // namespace v8 + } // namespace op +} // namespace ngraph diff --git a/ngraph/core/include/ngraph/op/util/nms_base.hpp b/ngraph/core/include/ngraph/op/util/nms_base.hpp new file mode 100644 index 00000000000..8983cbe7804 --- /dev/null +++ b/ngraph/core/include/ngraph/op/util/nms_base.hpp @@ -0,0 +1,93 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "ngraph/op/op.hpp" + +namespace ngraph +{ + namespace op + { + namespace util + { + /// \brief Base class for operations NmsBase and MatrixNms + /// + class NGRAPH_API NmsBase : public Op + { + public: + NGRAPH_RTTI_DECLARATION; + enum class SortResultType + { + CLASSID, // sort selected boxes by class id (ascending) in each batch element + SCORE, // sort selected boxes by score (descending) in each batch element + NONE // do not guarantee the order in each batch element + }; + + NmsBase() = delete; + + /// \brief Constructs a NmsBase operation + /// + /// \param output_type Specifies the output tensor type + /// \param nms_top_k Specifies maximum number of boxes to be selected per + /// class, -1 meaning to keep all boxes + /// \param keep_top_k Specifies maximum number of boxes to be selected per + /// batch element, -1 meaning to keep all boxes + NmsBase(ngraph::element::Type& output_type, int& nms_top_k, int& keep_top_k); + + /// \brief Constructs a NmsBase operation + /// + /// \param boxes Node producing the box coordinates + /// \param scores Node producing the box scores + /// \param output_type Specifies the output tensor type + /// \param nms_top_k Specifies maximum number of boxes to be selected per + /// class, -1 meaning to keep all boxes + /// \param keep_top_k Specifies maximum number of boxes to be selected per + /// batch element, -1 meaning to keep all boxes + NmsBase(const Output& boxes, + const Output& scores, + ngraph::element::Type& output_type, + int& nms_top_k, + int& keep_top_k); + + void validate_and_infer_types() override; + + const element::Type& get_output_type() const { return m_output_type; } + void set_output_type(const element::Type& output_type) + { + m_output_type = output_type; + } + using Node::set_output_type; + + int get_nms_top_k() const { return m_nms_top_k; } + + int get_keep_top_k() const { return m_keep_top_k; } + + protected: + ngraph::element::Type& m_output_type; + int& m_nms_top_k; + int& m_keep_top_k; + virtual void validate(); + }; + } // namespace util + } // namespace op + + NGRAPH_API + std::ostream& operator<<(std::ostream& s, const op::util::NmsBase::SortResultType& type); + + template <> + class NGRAPH_API AttributeAdapter + : public EnumAttributeAdapterBase + { + public: + AttributeAdapter(op::util::NmsBase::SortResultType& value) + : EnumAttributeAdapterBase(value) + { + } + + static constexpr DiscreteTypeInfo type_info{ + "AttributeAdapter", 1}; + const DiscreteTypeInfo& get_type_info() const override { return type_info; } + }; +} // namespace ngraph diff --git a/ngraph/core/include/ngraph/ops.hpp b/ngraph/core/include/ngraph/ops.hpp index 6999dc93c9d..4701a2f733f 100644 --- a/ngraph/core/include/ngraph/ops.hpp +++ b/ngraph/core/include/ngraph/ops.hpp @@ -85,6 +85,7 @@ #include "ngraph/op/lstm_cell.hpp" #include "ngraph/op/lstm_sequence.hpp" #include "ngraph/op/matmul.hpp" +#include "ngraph/op/matrix_nms.hpp" #include "ngraph/op/max.hpp" #include "ngraph/op/max_pool.hpp" #include "ngraph/op/maximum.hpp" @@ -92,6 +93,7 @@ #include "ngraph/op/minimum.hpp" #include "ngraph/op/mish.hpp" #include "ngraph/op/mod.hpp" +#include "ngraph/op/multiclass_nms.hpp" #include "ngraph/op/multiply.hpp" #include "ngraph/op/mvn.hpp" #include "ngraph/op/negative.hpp" diff --git a/ngraph/core/include/ngraph/opsets/opset8_tbl.hpp b/ngraph/core/include/ngraph/opsets/opset8_tbl.hpp index ad4d641027d..0004161dc48 100644 --- a/ngraph/core/include/ngraph/opsets/opset8_tbl.hpp +++ b/ngraph/core/include/ngraph/opsets/opset8_tbl.hpp @@ -179,3 +179,5 @@ NGRAPH_OP(Gather, ngraph::op::v8) NGRAPH_OP(AdaptiveAvgPool, ngraph::op::v8) NGRAPH_OP(AdaptiveMaxPool, ngraph::op::v8) NGRAPH_OP(DeformableConvolution, ngraph::op::v8) +NGRAPH_OP(MatrixNms, ngraph::op::v8) +NGRAPH_OP(MulticlassNms, ngraph::op::v8) \ No newline at end of file diff --git a/ngraph/core/reference/include/ngraph/runtime/reference/matrix_nms.hpp b/ngraph/core/reference/include/ngraph/runtime/reference/matrix_nms.hpp new file mode 100644 index 00000000000..c1c045e4349 --- /dev/null +++ b/ngraph/core/reference/include/ngraph/runtime/reference/matrix_nms.hpp @@ -0,0 +1,41 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ngraph/node.hpp" +#include "ngraph/op/matrix_nms.hpp" +#include "ngraph/op/util/op_types.hpp" +#include "ngraph/ops.hpp" +#include "ngraph/shape_util.hpp" + +namespace ngraph +{ + namespace runtime + { + namespace reference + { + void matrix_nms(const float* boxes_data, + const Shape& boxes_data_shape, + const float* scores_data, + const Shape& scores_data_shape, + const op::v8::MatrixNms::Attributes& attrs, + float* selected_outputs, + const Shape& selected_outputs_shape, + int64_t* selected_indices, + const Shape& selected_indices_shape, + int64_t* valid_outputs); + + } // namespace reference + } // namespace runtime +} // namespace ngraph diff --git a/ngraph/core/reference/include/ngraph/runtime/reference/multiclass_nms.hpp b/ngraph/core/reference/include/ngraph/runtime/reference/multiclass_nms.hpp new file mode 100644 index 00000000000..fe14f29689a --- /dev/null +++ b/ngraph/core/reference/include/ngraph/runtime/reference/multiclass_nms.hpp @@ -0,0 +1,41 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ngraph/node.hpp" +#include "ngraph/op/util/nms_base.hpp" +#include "ngraph/op/util/op_types.hpp" +#include "ngraph/ops.hpp" +#include "ngraph/shape_util.hpp" + +namespace ngraph +{ + namespace runtime + { + namespace reference + { + void multiclass_nms(const float* boxes_data, + const Shape& boxes_data_shape, + const float* scores_data, + const Shape& scores_data_shape, + const op::v8::MulticlassNms::Attributes& attrs, + float* selected_outputs, + const Shape& selected_outputs_shape, + int64_t* selected_indices, + const Shape& selected_indices_shape, + int64_t* valid_outputs); + + } // namespace reference + } // namespace runtime +} // namespace ngraph diff --git a/ngraph/core/reference/include/ngraph/runtime/reference/utils/nms_common.hpp b/ngraph/core/reference/include/ngraph/runtime/reference/utils/nms_common.hpp new file mode 100644 index 00000000000..b44e8dbaa41 --- /dev/null +++ b/ngraph/core/reference/include/ngraph/runtime/reference/utils/nms_common.hpp @@ -0,0 +1,88 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include +#include +#include "ngraph/type/element_type.hpp" + +namespace ngraph +{ + namespace runtime + { + namespace reference + { + namespace nms_common + { + struct Rectangle + { + Rectangle(float x_left, float y_left, float x_right, float y_right) + : x1{x_left} + , y1{y_left} + , x2{x_right} + , y2{y_right} + { + } + + Rectangle() = default; + + float x1 = 0.0f; + float y1 = 0.0f; + float x2 = 0.0f; + float y2 = 0.0f; + }; + + struct BoxInfo + { + BoxInfo(const Rectangle& r, + int64_t idx, + float sc, + int64_t suppress_idx, + int64_t batch_idx, + int64_t class_idx) + : box{r} + , index{idx} + , suppress_begin_index{suppress_idx} + , batch_index{batch_idx} + , class_index{class_idx} + , score{sc} + { + } + + BoxInfo() = default; + + inline bool operator<(const BoxInfo& rhs) const + { + return score < rhs.score || (score == rhs.score && index > rhs.index); + } + + inline bool operator>(const BoxInfo& rhs) const + { + return !(score < rhs.score || (score == rhs.score && index > rhs.index)); + } + + Rectangle box; + int64_t index = 0; + int64_t suppress_begin_index = 0; + int64_t batch_index = 0; + int64_t class_index = 0; + float score = 0.0f; + }; + + void nms_common_postprocessing(void* prois, + void* pscores, + void* pselected_num, + const ngraph::element::Type& output_type, + const std::vector& selected_outputs, + const std::vector& selected_indices, + const std::vector& valid_outputs); + + } // namespace nms_common + } // namespace reference + } // namespace runtime +} // namespace ngraph diff --git a/ngraph/core/reference/src/runtime/reference/matrix_nms.cpp b/ngraph/core/reference/src/runtime/reference/matrix_nms.cpp new file mode 100644 index 00000000000..e82d0ee4de3 --- /dev/null +++ b/ngraph/core/reference/src/runtime/reference/matrix_nms.cpp @@ -0,0 +1,354 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "ngraph/op/matrix_nms.hpp" +#include +#include +#include +#include +#include +#include "ngraph/runtime/reference/matrix_nms.hpp" +#include "ngraph/runtime/reference/utils/nms_common.hpp" +#include "ngraph/shape.hpp" + +using namespace ngraph; +using namespace ngraph::runtime::reference; + +namespace ngraph +{ + namespace runtime + { + namespace reference + { + namespace matrix_nms_v8 + { + template + struct decay_score; + + template + struct decay_score + { + T operator()(T iou, T max_iou, T sigma) + { + return std::exp((max_iou * max_iou - iou * iou) * sigma); + } + }; + + template + struct decay_score + { + T operator()(T iou, T max_iou, T sigma) + { + return (1. - iou) / (1. - max_iou + 1e-10f); + } + }; + + template + static inline T BBoxArea(const T* box, const bool normalized) + { + if (box[2] < box[0] || box[3] < box[1]) + { + // If coordinate values are is invalid + // (e.g. xmax < xmin or ymax < ymin), return 0. + return static_cast(0.); + } + else + { + const T w = box[2] - box[0]; + const T h = box[3] - box[1]; + if (normalized) + { + return w * h; + } + else + { + // If coordinate values are not within range [0, 1]. + return (w + 1) * (h + 1); + } + } + } + + template + static inline T + intersectionOverUnion(const T* box1, const T* box2, const bool normalized) + { + if (box2[0] > box1[2] || box2[2] < box1[0] || box2[1] > box1[3] || + box2[3] < box1[1]) + { + return static_cast(0.); + } + else + { + const T inter_xmin = std::max(box1[0], box2[0]); + const T inter_ymin = std::max(box1[1], box2[1]); + const T inter_xmax = std::min(box1[2], box2[2]); + const T inter_ymax = std::min(box1[3], box2[3]); + T norm = normalized ? static_cast(0.) : static_cast(1.); + T inter_w = inter_xmax - inter_xmin + norm; + T inter_h = inter_ymax - inter_ymin + norm; + const T inter_area = inter_w * inter_h; + const T bbox1_area = BBoxArea(box1, normalized); + const T bbox2_area = BBoxArea(box2, normalized); + return inter_area / (bbox1_area + bbox2_area - inter_area); + } + } + } // namespace matrix_nms_v8 + + template + void nms_matrix(const T* boxes_data, + const Shape& boxes_data_shape, + const T* scores_data, + const Shape& scores_data_shape, + const T score_threshold, + const T post_threshold, + const float sigma, + const int64_t top_k, + const bool normalized, + std::vector* selected_indices, + std::vector* decayed_scores) + { + int64_t boxes_num = static_cast(boxes_data_shape[1]); + int64_t box_size = static_cast(boxes_data_shape[2]); + + std::vector candidate_index(boxes_num); + std::iota(candidate_index.begin(), candidate_index.end(), 0); + auto end = std::remove_if(candidate_index.begin(), + candidate_index.end(), + [&scores_data, score_threshold](int32_t idx) { + return scores_data[idx] <= score_threshold; + }); + + int64_t original_size = std::distance(candidate_index.begin(), end); + if (original_size <= 0) + { + return; + } + if (top_k > -1 && original_size > top_k) + { + original_size = top_k; + } + + std::partial_sort(candidate_index.begin(), + candidate_index.begin() + original_size, + end, + [&scores_data](int32_t a, int32_t b) { + return scores_data[a] > scores_data[b]; + }); + + std::vector iou_matrix((original_size * (original_size - 1)) >> 1); + std::vector iou_max(original_size); + + iou_max[0] = 0.; + for (int64_t i = 1; i < original_size; i++) + { + T max_iou = 0.; + auto idx_a = candidate_index[i]; + for (int64_t j = 0; j < i; j++) + { + auto idx_b = candidate_index[j]; + auto iou = + matrix_nms_v8::intersectionOverUnion(boxes_data + idx_a * box_size, + boxes_data + idx_b * box_size, + normalized); + max_iou = std::max(max_iou, iou); + iou_matrix[i * (i - 1) / 2 + j] = iou; + } + iou_max[i] = max_iou; + } + + if (scores_data[candidate_index[0]] > post_threshold) + { + selected_indices->push_back(candidate_index[0]); + decayed_scores->push_back(scores_data[candidate_index[0]]); + } + + matrix_nms_v8::decay_score decay_fn; + for (int64_t i = 1; i < original_size; i++) + { + T min_decay = 1.; + for (int64_t j = 0; j < i; j++) + { + auto max_iou = iou_max[j]; + auto iou = iou_matrix[i * (i - 1) / 2 + j]; + auto decay = decay_fn(iou, max_iou, sigma); + min_decay = std::min(min_decay, decay); + } + auto ds = min_decay * scores_data[candidate_index[i]]; + if (ds <= post_threshold) + continue; + selected_indices->push_back(candidate_index[i]); + decayed_scores->push_back(ds); + } + } + + void matrix_nms(const float* boxes_data, + const Shape& boxes_data_shape, + const float* scores_data, + const Shape& scores_data_shape, + const op::v8::MatrixNms::Attributes& attrs, + float* selected_outputs, + const Shape& selected_outputs_shape, + int64_t* selected_indices, + const Shape& selected_indices_shape, + int64_t* valid_outputs) + { + using Rectangle = runtime::reference::nms_common::Rectangle; + using BoxInfo = runtime::reference::nms_common::BoxInfo; + + // boxes shape: {num_batches, num_boxes, 4} + // scores shape: {num_batches, num_classes, num_boxes} + int64_t num_batches = static_cast(scores_data_shape[0]); + int64_t num_classes = static_cast(scores_data_shape[1]); + int64_t num_boxes = static_cast(boxes_data_shape[1]); + int64_t box_shape = static_cast(boxes_data_shape[2]); + + std::vector num_per_batch; + std::vector filtered_boxes; + filtered_boxes.reserve(6 * num_batches * num_classes * num_boxes); + + for (int64_t batch = 0; batch < num_batches; batch++) + { + const float* boxesPtr = boxes_data + batch * num_boxes * 4; + std::vector all_indices; + std::vector all_scores; + std::vector all_classes; + size_t num_det = 0; + + for (int64_t class_idx = 0; class_idx < num_classes; class_idx++) + { + if (class_idx == attrs.background_class) + continue; + const float* scoresPtr = + scores_data + batch * (num_classes * num_boxes) + class_idx * num_boxes; + if (attrs.decay_function == op::v8::MatrixNms::DecayFunction::GAUSSIAN) + { + nms_matrix(boxesPtr, + boxes_data_shape, + scoresPtr, + scores_data_shape, + attrs.score_threshold, + attrs.post_threshold, + attrs.gaussian_sigma, + attrs.nms_top_k, + attrs.normalized, + &all_indices, + &all_scores); + } + else + { + nms_matrix(boxesPtr, + boxes_data_shape, + scoresPtr, + scores_data_shape, + attrs.score_threshold, + attrs.post_threshold, + attrs.gaussian_sigma, + attrs.nms_top_k, + attrs.normalized, + &all_indices, + &all_scores); + } + for (size_t i = 0; i < all_indices.size() - num_det; i++) + { + all_classes.push_back(class_idx); + } + num_det = all_indices.size(); + } + + if (num_det <= 0) + { + break; + } + + if (attrs.keep_top_k > -1) + { + auto k = static_cast(attrs.keep_top_k); + if (num_det > k) + num_det = k; + } + + std::vector perm(all_indices.size()); + std::iota(perm.begin(), perm.end(), 0); + + std::partial_sort(perm.begin(), + perm.begin() + num_det, + perm.end(), + [&all_scores, &all_classes, &all_indices](int lhs, int rhs) { + return (all_scores[lhs] > all_scores[rhs]) || + (all_scores[lhs] == all_scores[rhs] && + all_classes[lhs] < all_classes[rhs]) || + (all_scores[lhs] == all_scores[rhs] && + all_classes[lhs] == all_classes[rhs] && + all_indices[lhs] < all_indices[rhs]); + }); + + for (size_t i = 0; i < num_det; i++) + { + auto p = perm[i]; + auto idx = all_indices[p]; + auto cls = all_classes[p]; + auto score = all_scores[p]; + auto bbox = boxesPtr + idx * box_shape; + + filtered_boxes.push_back( + BoxInfo{Rectangle{bbox[0], bbox[1], bbox[2], bbox[3]}, + batch * num_boxes + idx, + score, + 0, + batch, + cls}); + } + num_per_batch.push_back(num_det); + } + + if (attrs.sort_result_across_batch) + { /* sort across batch */ + if (attrs.sort_result_type == op::v8::MatrixNms::SortResultType::SCORE) + { + std::sort( + filtered_boxes.begin(), + filtered_boxes.end(), + [](const BoxInfo& l, const BoxInfo& r) { + return (l.score > r.score) || + (l.score == r.score && l.batch_index < r.batch_index) || + (l.score == r.score && l.batch_index == r.batch_index && + l.class_index < r.class_index) || + (l.score == r.score && l.batch_index == r.batch_index && + l.class_index == r.class_index && l.index < r.index); + }); + } + else if (attrs.sort_result_type == op::v8::MatrixNms::SortResultType::CLASSID) + { + std::sort(filtered_boxes.begin(), + filtered_boxes.end(), + [](const BoxInfo& l, const BoxInfo& r) { + return (l.class_index < r.class_index) || + (l.class_index == r.class_index && + l.batch_index < r.batch_index) || + (l.class_index == r.class_index && + l.batch_index == r.batch_index && + l.score > r.score) || + (l.class_index == r.class_index && + l.batch_index == r.batch_index && + l.score == r.score && l.index < r.index); + }); + } + } + + std::copy(num_per_batch.begin(), num_per_batch.end(), valid_outputs); + for (size_t i = 0; i < filtered_boxes.size(); i++) + { + selected_indices[i] = filtered_boxes[i].index; + auto selected_base = selected_outputs + i * 6; + selected_base[0] = filtered_boxes[i].class_index; + selected_base[1] = filtered_boxes[i].score; + selected_base[2] = filtered_boxes[i].box.x1; + selected_base[3] = filtered_boxes[i].box.y1; + selected_base[4] = filtered_boxes[i].box.x2; + selected_base[5] = filtered_boxes[i].box.y2; + } + } + } // namespace reference + } // namespace runtime +} // namespace ngraph diff --git a/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp b/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp new file mode 100644 index 00000000000..3328de9c3aa --- /dev/null +++ b/ngraph/core/reference/src/runtime/reference/multiclass_nms.cpp @@ -0,0 +1,350 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "ngraph/op/multiclass_nms.hpp" +#include +#include +#include +#include +#include +#include "ngraph/runtime/reference/multiclass_nms.hpp" +#include "ngraph/runtime/reference/utils/nms_common.hpp" +#include "ngraph/shape.hpp" + +using namespace ngraph; +using namespace ngraph::runtime::reference; + +namespace ngraph +{ + namespace runtime + { + namespace reference + { + namespace multiclass_nms_v8 + { + using Rectangle = runtime::reference::nms_common::Rectangle; + using BoxInfo = runtime::reference::nms_common::BoxInfo; + static float intersectionOverUnion(const Rectangle& boxI, + const Rectangle& boxJ, + const bool normalized) + { + const float norm = static_cast(normalized == false); + + float areaI = (boxI.y2 - boxI.y1 + norm) * (boxI.x2 - boxI.x1 + norm); + float areaJ = (boxJ.y2 - boxJ.y1 + norm) * (boxJ.x2 - boxJ.x1 + norm); + + if (areaI <= 0.0f || areaJ <= 0.0f) + { + return 0.0f; + } + + float intersection_ymin = std::max(boxI.y1, boxJ.y1); + float intersection_xmin = std::max(boxI.x1, boxJ.x1); + float intersection_ymax = std::min(boxI.y2, boxJ.y2); + float intersection_xmax = std::min(boxI.x2, boxJ.x2); + + float intersection_area = + std::max(intersection_ymax - intersection_ymin + norm, 0.0f) * + std::max(intersection_xmax - intersection_xmin + norm, 0.0f); + + return intersection_area / (areaI + areaJ - intersection_area); + } + + struct SelectedIndex + { + SelectedIndex(int64_t batch_idx, int64_t box_idx, int64_t num_box) + : flattened_index(batch_idx * num_box + box_idx) + { + } + + SelectedIndex() = default; + + int64_t flattened_index = 0; + }; + + struct SelectedOutput + { + SelectedOutput( + float class_idx, float score, float x1, float y1, float x2, float y2) + : class_index{class_idx} + , box_score{score} + , xmin{x1} + , ymin{y1} + , xmax{x2} + , ymax{y2} + { + } + + SelectedOutput() = default; + + float class_index = 0.0f; + float box_score = 0.0f; + float xmin, ymin, xmax, ymax; + }; + } // namespace multiclass_nms_v8 + + void multiclass_nms(const float* boxes_data, + const Shape& boxes_data_shape, + const float* scores_data, + const Shape& scores_data_shape, + const op::v8::MulticlassNms::Attributes& attrs, + float* selected_outputs, + const Shape& selected_outputs_shape, + int64_t* selected_indices, + const Shape& selected_indices_shape, + int64_t* valid_outputs) + { + using SelectedIndex = multiclass_nms_v8::SelectedIndex; + using SelectedOutput = multiclass_nms_v8::SelectedOutput; + using BoxInfo = multiclass_nms_v8::BoxInfo; + using Rectangle = multiclass_nms_v8::Rectangle; + + auto func = [](float iou, float adaptive_threshold) { + return iou <= adaptive_threshold ? 1.0f : 0.0f; + }; + + // boxes shape: {num_batches, num_boxes, 4} + // scores shape: {num_batches, num_classes, num_boxes} + int64_t num_batches = static_cast(scores_data_shape[0]); + int64_t num_classes = static_cast(scores_data_shape[1]); + int64_t num_boxes = static_cast(boxes_data_shape[1]); + + SelectedIndex* selected_indices_ptr = + reinterpret_cast(selected_indices); + SelectedOutput* selected_scores_ptr = + reinterpret_cast(selected_outputs); + + std::vector filteredBoxes; // container for the whole batch + + for (int64_t batch = 0; batch < num_batches; batch++) + { + const float* boxesPtr = boxes_data + batch * num_boxes * 4; + Rectangle* r = reinterpret_cast(const_cast(boxesPtr)); + + int64_t num_dets = 0; + std::vector selected_boxes; // container for a batch element + + for (int64_t class_idx = 0; class_idx < num_classes; class_idx++) + { + if (class_idx == attrs.background_class) + continue; + + auto adaptive_threshold = attrs.iou_threshold; + + const float* scoresPtr = + scores_data + batch * (num_classes * num_boxes) + class_idx * num_boxes; + + std::vector candidate_boxes; + + for (int64_t box_idx = 0; box_idx < num_boxes; box_idx++) + { + if (scoresPtr[box_idx] >= + attrs.score_threshold) /* NOTE: ">=" instead of ">" used in PDPD */ + { + candidate_boxes.emplace_back( + r[box_idx], box_idx, scoresPtr[box_idx], 0, batch, class_idx); + } + } + + int candiate_size = candidate_boxes.size(); + + // threshold nms_top_k for each class + // NOTE: "nms_top_k" in PDPD not exactly equal to + // "max_output_boxes_per_class" in ONNX. + if (attrs.nms_top_k > -1 && attrs.nms_top_k < candiate_size) + { + candiate_size = attrs.nms_top_k; + } + + if (candiate_size <= 0) // early drop + { + continue; + } + + // sort by score in current class + std::partial_sort(candidate_boxes.begin(), + candidate_boxes.begin() + candiate_size, + candidate_boxes.end(), + std::greater()); + + std::priority_queue sorted_boxes(candidate_boxes.begin(), + candidate_boxes.begin() + + candiate_size, + std::less()); + + std::vector selected; // container for a class + + // Get the next box with top score, filter by iou_threshold + BoxInfo next_candidate; + float original_score; + + while (!sorted_boxes.empty()) + { + next_candidate = sorted_boxes.top(); + original_score = next_candidate.score; + sorted_boxes.pop(); + + bool should_hard_suppress = false; + for (int64_t j = static_cast(selected.size()) - 1; + j >= next_candidate.suppress_begin_index; + --j) + { + float iou = multiclass_nms_v8::intersectionOverUnion( + next_candidate.box, selected[j].box, attrs.normalized); + next_candidate.score *= func(iou, adaptive_threshold); + + if (iou >= adaptive_threshold) + { + should_hard_suppress = true; + break; + } + + if (next_candidate.score <= attrs.score_threshold) + { + break; + } + } + + next_candidate.suppress_begin_index = selected.size(); + + if (!should_hard_suppress) + { + if (attrs.nms_eta < 1 && adaptive_threshold > 0.5) + { + adaptive_threshold *= attrs.nms_eta; + } + if (next_candidate.score == original_score) + { + selected.push_back(next_candidate); + continue; + } + if (next_candidate.score > attrs.score_threshold) + { + sorted_boxes.push(next_candidate); + } + } + } + + for (const auto& box_info : selected) + { + selected_boxes.push_back(box_info); + } + num_dets += selected.size(); + } // for each class + + // sort inside batch element before go through keep_top_k + std::sort(selected_boxes.begin(), + selected_boxes.end(), + [](const BoxInfo& l, const BoxInfo& r) { + return ((l.batch_index == r.batch_index) && + ((l.score > r.score) || + ((std::fabs(l.score - r.score) < 1e-6) && + l.class_index < r.class_index) || + ((std::fabs(l.score - r.score) < 1e-6) && + l.class_index == r.class_index && l.index < r.index))); + }); + + // threshold keep_top_k for each batch element + if (attrs.keep_top_k > -1 && attrs.keep_top_k < num_dets) + { + num_dets = attrs.keep_top_k; + selected_boxes.resize(num_dets); + } + + // sort + if (!attrs.sort_result_across_batch) + { + if (attrs.sort_result_type == + op::v8::MulticlassNms::SortResultType::CLASSID) + { + std::sort( + selected_boxes.begin(), + selected_boxes.end(), + [](const BoxInfo& l, const BoxInfo& r) { + return ( + (l.batch_index == r.batch_index) && + ((l.class_index < r.class_index) || + ((l.class_index == r.class_index) && l.score > r.score) || + ((std::fabs(l.score - r.score) <= 1e-6) && + l.class_index == r.class_index && l.index < r.index))); + }); + } + // in case of "SCORE", pass through, as, + // it has already gurranteed. + } + + *valid_outputs++ = num_dets; + for (auto& v : selected_boxes) + { + filteredBoxes.push_back(v); + } + } // for each batch element + + if (attrs.sort_result_across_batch) + { /* sort across batch */ + if (attrs.sort_result_type == op::v8::MulticlassNms::SortResultType::SCORE) + { + std::sort( + filteredBoxes.begin(), + filteredBoxes.end(), + [](const BoxInfo& l, const BoxInfo& r) { + return (l.score > r.score) || + (l.score == r.score && l.batch_index < r.batch_index) || + (l.score == r.score && l.batch_index == r.batch_index && + l.class_index < r.class_index) || + (l.score == r.score && l.batch_index == r.batch_index && + l.class_index == r.class_index && l.index < r.index); + }); + } + else if (attrs.sort_result_type == + op::v8::MulticlassNms::SortResultType::CLASSID) + { + std::sort(filteredBoxes.begin(), + filteredBoxes.end(), + [](const BoxInfo& l, const BoxInfo& r) { + return (l.class_index < r.class_index) || + (l.class_index == r.class_index && + l.batch_index < r.batch_index) || + (l.class_index == r.class_index && + l.batch_index == r.batch_index && + l.score > r.score) || + (l.class_index == r.class_index && + l.batch_index == r.batch_index && + l.score == r.score && l.index < r.index); + }); + } + } + + /* output */ + + size_t max_num_of_selected_indices = selected_indices_shape[0]; + size_t output_size = std::min(filteredBoxes.size(), max_num_of_selected_indices); + + size_t idx; + for (idx = 0; idx < output_size; idx++) + { + const auto& box_info = filteredBoxes[idx]; + SelectedIndex selected_index{box_info.batch_index, box_info.index, num_boxes}; + SelectedOutput selected_score{static_cast(box_info.class_index), + box_info.score, + box_info.box.x1, + box_info.box.y1, + box_info.box.x2, + box_info.box.y2}; + + selected_indices_ptr[idx] = selected_index; + selected_scores_ptr[idx] = selected_score; + } + + SelectedIndex selected_index_filler{0, 0, 0}; + SelectedOutput selected_score_filler{0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; + for (; idx < max_num_of_selected_indices; idx++) + { + selected_indices_ptr[idx] = selected_index_filler; + selected_scores_ptr[idx] = selected_score_filler; + } + } + } // namespace reference + } // namespace runtime +} // namespace ngraph diff --git a/ngraph/core/reference/src/runtime/reference/utils/nms_common.cpp b/ngraph/core/reference/src/runtime/reference/utils/nms_common.cpp new file mode 100644 index 00000000000..c658b0f0cad --- /dev/null +++ b/ngraph/core/reference/src/runtime/reference/utils/nms_common.cpp @@ -0,0 +1,72 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include +#include + +#include "ngraph/runtime/reference/utils/nms_common.hpp" + +namespace ngraph +{ + namespace runtime + { + namespace reference + { + namespace nms_common + { + void nms_common_postprocessing(void* prois, + void* pscores, + void* pselected_num, + const ngraph::element::Type& output_type, + const std::vector& selected_outputs, + const std::vector& selected_indices, + const std::vector& valid_outputs) + { + int64_t total_num = + std::accumulate(valid_outputs.begin(), valid_outputs.end(), 0); + + float* ptr = static_cast(prois); + memcpy(ptr, selected_outputs.data(), total_num * sizeof(float) * 6); + + if (pscores) + { + if (output_type == ngraph::element::i64) + { + int64_t* indices_ptr = static_cast(pscores); + memcpy( + indices_ptr, selected_indices.data(), total_num * sizeof(int64_t)); + } + else + { + int32_t* indices_ptr = static_cast(pscores); + for (int64_t i = 0; i < total_num; ++i) + { + indices_ptr[i] = static_cast(selected_indices[i]); + } + } + } + + if (pselected_num) + { + if (output_type == ngraph::element::i64) + { + int64_t* valid_outputs_ptr = static_cast(pselected_num); + std::copy( + valid_outputs.begin(), valid_outputs.end(), valid_outputs_ptr); + } + else + { + int32_t* valid_outputs_ptr = static_cast(pselected_num); + for (size_t i = 0; i < valid_outputs.size(); ++i) + { + valid_outputs_ptr[i] = static_cast(valid_outputs[i]); + } + } + } + } + } // namespace nms_common + } // namespace reference + } // namespace runtime +} // namespace ngraph diff --git a/ngraph/core/src/op/matrix_nms.cpp b/ngraph/core/src/op/matrix_nms.cpp new file mode 100644 index 00000000000..7d3731f3b11 --- /dev/null +++ b/ngraph/core/src/op/matrix_nms.cpp @@ -0,0 +1,92 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "ngraph/op/matrix_nms.hpp" +#include +#include +#include "itt.hpp" +#include "ngraph/attribute_visitor.hpp" +#include "ngraph/op/constant.hpp" +#include "ngraph/op/util/op_types.hpp" +#include "ngraph/runtime/reference/matrix_nms.hpp" +#include "ngraph/type/bfloat16.hpp" +#include "ngraph/type/float16.hpp" +#include "ngraph/util.hpp" + +using namespace ngraph; + +NGRAPH_RTTI_DEFINITION(op::v8::MatrixNms, "MatrixNms", 8, op::util::NmsBase); + +op::v8::MatrixNms::MatrixNms() + : NmsBase(m_attrs.output_type, m_attrs.nms_top_k, m_attrs.keep_top_k) +{ +} + +op::v8::MatrixNms::MatrixNms(const Output& boxes, + const Output& scores, + const Attributes& attrs) + : NmsBase(boxes, scores, m_attrs.output_type, m_attrs.nms_top_k, m_attrs.keep_top_k) + , m_attrs{attrs} +{ + constructor_validate_and_infer_types(); +} + +std::shared_ptr op::v8::MatrixNms::clone_with_new_inputs(const OutputVector& new_args) const +{ + NGRAPH_OP_SCOPE(v8_MatrixNms_clone_with_new_inputs); + check_new_args_count(this, new_args); + NODE_VALIDATION_CHECK(this, new_args.size() == 2, "Number of inputs must be 2"); + + return std::make_shared(new_args.at(0), new_args.at(1), m_attrs); +} + +void op::v8::MatrixNms::validate() +{ + NGRAPH_OP_SCOPE(v8_MatrixNms_validate); + NmsBase::validate(); + + NODE_VALIDATION_CHECK(this, + m_attrs.background_class >= -1, + "The 'background_class' must be great or equal -1. Got:", + m_attrs.background_class); +} + +bool ngraph::op::v8::MatrixNms::visit_attributes(AttributeVisitor& visitor) +{ + NGRAPH_OP_SCOPE(v8_MatrixNms_visit_attributes); + + visitor.on_attribute("sort_result_type", m_attrs.sort_result_type); + visitor.on_attribute("output_type", m_attrs.output_type); + visitor.on_attribute("nms_top_k", m_attrs.nms_top_k); + visitor.on_attribute("keep_top_k", m_attrs.keep_top_k); + visitor.on_attribute("sort_result_across_batch", m_attrs.sort_result_across_batch); + visitor.on_attribute("score_threshold", m_attrs.score_threshold); + visitor.on_attribute("background_class", m_attrs.background_class); + visitor.on_attribute("decay_function", m_attrs.decay_function); + visitor.on_attribute("gaussian_sigma", m_attrs.gaussian_sigma); + visitor.on_attribute("post_threshold", m_attrs.post_threshold); + visitor.on_attribute("normalized", m_attrs.normalized); + + return true; +} + +namespace ngraph +{ + template <> + EnumNames& EnumNames::get() + { + static auto enum_names = EnumNames( + "op::v8::MatrixNms::DecayFunction", + {{"gaussian", op::v8::MatrixNms::DecayFunction::GAUSSIAN}, + {"linear", op::v8::MatrixNms::DecayFunction::LINEAR}}); + return enum_names; + } + + constexpr DiscreteTypeInfo AttributeAdapter::type_info; + + std::ostream& operator<<(std::ostream& s, const op::v8::MatrixNms::DecayFunction& type) + { + return s << as_string(type); + } +} // namespace ngraph diff --git a/ngraph/core/src/op/multiclass_nms.cpp b/ngraph/core/src/op/multiclass_nms.cpp new file mode 100644 index 00000000000..8f0e7cd5345 --- /dev/null +++ b/ngraph/core/src/op/multiclass_nms.cpp @@ -0,0 +1,77 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "ngraph/op/multiclass_nms.hpp" +#include +#include +#include "itt.hpp" +#include "ngraph/attribute_visitor.hpp" +#include "ngraph/op/constant.hpp" +#include "ngraph/op/util/op_types.hpp" +#include "ngraph/runtime/reference/multiclass_nms.hpp" +#include "ngraph/type/bfloat16.hpp" +#include "ngraph/type/float16.hpp" +#include "ngraph/util.hpp" + +using namespace ngraph; + +NGRAPH_RTTI_DEFINITION(op::v8::MulticlassNms, "MulticlassNms", 8, op::util::NmsBase); + +op::v8::MulticlassNms::MulticlassNms() + : NmsBase(m_attrs.output_type, m_attrs.nms_top_k, m_attrs.keep_top_k) +{ +} + +op::v8::MulticlassNms::MulticlassNms(const Output& boxes, + const Output& scores, + const Attributes& attrs) + : NmsBase(boxes, scores, m_attrs.output_type, m_attrs.nms_top_k, m_attrs.keep_top_k) + , m_attrs{attrs} +{ + constructor_validate_and_infer_types(); +} + +std::shared_ptr + op::v8::MulticlassNms::clone_with_new_inputs(const OutputVector& new_args) const +{ + NGRAPH_OP_SCOPE(v8_MulticlassNms_clone_with_new_inputs); + check_new_args_count(this, new_args); + NODE_VALIDATION_CHECK(this, new_args.size() == 2, "Number of inputs must be 2"); + + return std::make_shared(new_args.at(0), new_args.at(1), m_attrs); +} + +void op::v8::MulticlassNms::validate() +{ + NGRAPH_OP_SCOPE(v8_MulticlassNms_validate); + NmsBase::validate(); + + NODE_VALIDATION_CHECK(this, + m_attrs.background_class >= -1, + "The 'background_class' must be great or equal -1. Got:", + m_attrs.background_class); + + NODE_VALIDATION_CHECK(this, + m_attrs.nms_eta >= 0.0f && m_attrs.nms_eta <= 1.0f, + "The 'nms_eta' must be in close range [0, 1.0]. Got:", + m_attrs.nms_eta); +} + +bool ngraph::op::v8::MulticlassNms::visit_attributes(AttributeVisitor& visitor) +{ + NGRAPH_OP_SCOPE(v8_MulticlassNms_visit_attributes); + + visitor.on_attribute("sort_result_type", m_attrs.sort_result_type); + visitor.on_attribute("output_type", m_attrs.output_type); + visitor.on_attribute("nms_top_k", m_attrs.nms_top_k); + visitor.on_attribute("keep_top_k", m_attrs.keep_top_k); + visitor.on_attribute("sort_result_across_batch", m_attrs.sort_result_across_batch); + visitor.on_attribute("iou_threshold", m_attrs.iou_threshold); + visitor.on_attribute("score_threshold", m_attrs.score_threshold); + visitor.on_attribute("background_class", m_attrs.background_class); + visitor.on_attribute("nms_eta", m_attrs.nms_eta); + visitor.on_attribute("normalized", m_attrs.normalized); + + return true; +} diff --git a/ngraph/core/src/op/util/nms_base.cpp b/ngraph/core/src/op/util/nms_base.cpp new file mode 100644 index 00000000000..4fce4c46fc4 --- /dev/null +++ b/ngraph/core/src/op/util/nms_base.cpp @@ -0,0 +1,183 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "ngraph/op/util/nms_base.hpp" +#include +#include +#include "itt.hpp" +#include "ngraph/attribute_visitor.hpp" +#include "ngraph/op/constant.hpp" +#include "ngraph/op/util/op_types.hpp" +#include "ngraph/type/bfloat16.hpp" +#include "ngraph/type/float16.hpp" +#include "ngraph/util.hpp" + +using namespace ngraph; + +NGRAPH_RTTI_DEFINITION(op::util::NmsBase, "NmsBase", 0); + +op::util::NmsBase::NmsBase(ngraph::element::Type& output_type, int& nms_top_k, int& keep_top_k) + : m_output_type(output_type) + , m_nms_top_k(nms_top_k) + , m_keep_top_k(keep_top_k) +{ +} + +op::util::NmsBase::NmsBase(const Output& boxes, + const Output& scores, + ngraph::element::Type& output_type, + int& nms_top_k, + int& keep_top_k) + : Op({boxes, scores}) + , m_output_type(output_type) + , m_nms_top_k(nms_top_k) + , m_keep_top_k(keep_top_k) +{ +} + +namespace +{ + inline bool is_float_type_admissible(const element::Type& t) + { + return t == element::f32 || t == element::f16 || t == element::bf16; + } +} // namespace + +void op::util::NmsBase::validate() +{ + NGRAPH_OP_SCOPE(util_NmsBase_validate); + + const auto boxes_ps = get_input_partial_shape(0); + const auto scores_ps = get_input_partial_shape(1); + + NODE_VALIDATION_CHECK(this, + m_output_type == element::i64 || m_output_type == element::i32, + "Output type must be i32 or i64"); + + if (boxes_ps.is_dynamic() || scores_ps.is_dynamic()) + { + return; + } + + NODE_VALIDATION_CHECK(this, + is_float_type_admissible(get_input_element_type(0)), + "Expected bf16, fp16 or fp32 as element type for the 'boxes' input."); + + NODE_VALIDATION_CHECK(this, + is_float_type_admissible(get_input_element_type(1)), + "Expected bf16, fp16 or fp32 as element type for the 'scores' input."); + + NODE_VALIDATION_CHECK(this, + boxes_ps.rank().is_static() && boxes_ps.rank().get_length() == 3, + "Expected a 3D tensor for the 'boxes' input. Got: ", + boxes_ps); + + NODE_VALIDATION_CHECK(this, + boxes_ps[2].is_static() && boxes_ps[2].get_length() == 4, + "The third dimension of the 'boxes' must be 4. Got: ", + boxes_ps[2]); + + NODE_VALIDATION_CHECK(this, + scores_ps.rank().is_static() && scores_ps.rank().get_length() == 3, + "Expected a 3D tensor for the 'scores' input. Got: ", + scores_ps); + + NODE_VALIDATION_CHECK( + this, m_nms_top_k >= -1, "The 'nms_top_k' must be great or equal -1. Got:", m_nms_top_k); + + NODE_VALIDATION_CHECK( + this, m_keep_top_k >= -1, "The 'keep_top_k' must be great or equal -1. Got:", m_keep_top_k); + + const auto num_batches_boxes = boxes_ps[0]; + const auto num_batches_scores = scores_ps[0]; + + NODE_VALIDATION_CHECK(this, + num_batches_boxes.same_scheme(num_batches_scores), + "The first dimension of both 'boxes' and 'scores' must match. Boxes: ", + num_batches_boxes, + "; Scores: ", + num_batches_scores); + + const auto num_boxes_boxes = boxes_ps[1]; + const auto num_boxes_scores = scores_ps[2]; + NODE_VALIDATION_CHECK(this, + num_boxes_boxes.same_scheme(num_boxes_scores), + "'boxes' and 'scores' input shapes must match at the second and third " + "dimension respectively. Boxes: ", + num_boxes_boxes, + "; Scores: ", + num_boxes_scores); +} + +void op::util::NmsBase::validate_and_infer_types() +{ + NGRAPH_OP_SCOPE(util_NmsBase_validate_and_infer_types); + const auto boxes_ps = get_input_partial_shape(0); + const auto scores_ps = get_input_partial_shape(1); + + auto first_dim_shape = Dimension::dynamic(); + + validate(); + + if (boxes_ps.rank().is_static() && scores_ps.rank().is_static()) + { + const auto num_boxes_boxes = boxes_ps[1]; + if (num_boxes_boxes.is_static() && scores_ps[0].is_static() && scores_ps[1].is_static()) + { + const auto num_boxes = num_boxes_boxes.get_length(); + const auto num_classes = scores_ps[1].get_length(); + int64_t max_output_boxes_per_class = 0; + if (m_nms_top_k >= 0) + max_output_boxes_per_class = std::min(num_boxes, (int64_t)m_nms_top_k); + else + max_output_boxes_per_class = num_boxes; + + auto max_output_boxes_per_batch = max_output_boxes_per_class * num_classes; + if (m_keep_top_k >= 0) + max_output_boxes_per_batch = + std::min(max_output_boxes_per_batch, (int64_t)m_keep_top_k); + + first_dim_shape = Dimension(0, max_output_boxes_per_batch * scores_ps[0].get_length()); + } + } + + // 'selected_outputs' have the following format: + // [number of selected boxes, [class_id, box_score, xmin, ymin, xmax, ymax]] + set_output_type(0, element::f32, {first_dim_shape, 6}); + // 'selected_indices' have the following format: + // [number of selected boxes, ] + set_output_type(1, m_output_type, {first_dim_shape, 1}); + // 'selected_num' have the following format: + // [num_batches, ] + if (boxes_ps.rank().is_static() && boxes_ps.rank().get_length() > 0) + { + set_output_type(2, m_output_type, {boxes_ps[0]}); + } + else + { + set_output_type(2, m_output_type, {Dimension::dynamic()}); + } +} + +namespace ngraph +{ + template <> + EnumNames& + EnumNames::get() + { + static auto enum_names = EnumNames( + "op::util::NmsBase::SortResultType", + {{"classid", op::util::NmsBase::SortResultType::CLASSID}, + {"score", op::util::NmsBase::SortResultType::SCORE}, + {"none", op::util::NmsBase::SortResultType::NONE}}); + return enum_names; + } + + constexpr DiscreteTypeInfo AttributeAdapter::type_info; + + std::ostream& operator<<(std::ostream& s, const op::util::NmsBase::SortResultType& type) + { + return s << as_string(type); + } +} // namespace ngraph diff --git a/ngraph/core/src/opsets/opset.cpp b/ngraph/core/src/opsets/opset.cpp index ea09eec98c1..c3b791995e3 100644 --- a/ngraph/core/src/opsets/opset.cpp +++ b/ngraph/core/src/opsets/opset.cpp @@ -115,3 +115,15 @@ const ngraph::OpSet& ngraph::get_opset7() }); return opset; } + +const ngraph::OpSet& ngraph::get_opset8() +{ + static OpSet opset; + static std::once_flag flag; + std::call_once(flag, [&]() { +#define NGRAPH_OP(NAME, NAMESPACE) opset.insert(); +#include "ngraph/opsets/opset8_tbl.hpp" +#undef NGRAPH_OP + }); + return opset; +} diff --git a/ngraph/test/CMakeLists.txt b/ngraph/test/CMakeLists.txt index 1001e8abf10..f20292ed934 100644 --- a/ngraph/test/CMakeLists.txt +++ b/ngraph/test/CMakeLists.txt @@ -157,11 +157,13 @@ set(SRC type_prop/lstm_sequence.cpp type_prop/loop.cpp type_prop/matmul.cpp + type_prop/matrix_nms.cpp type_prop/maximum.cpp type_prop/max_pool.cpp type_prop/minimum.cpp type_prop/mish.cpp type_prop/mod.cpp + type_prop/multiclass_nms.cpp type_prop/mvn.cpp type_prop/negative.cpp type_prop/non_max_suppression.cpp @@ -260,9 +262,11 @@ set(SRC visitors/op/lstm_cell.cpp visitors/op/lstm_sequence.cpp visitors/op/matmul.cpp + visitors/op/matrix_nms.cpp visitors/op/max_pool.cpp visitors/op/mish.cpp visitors/op/mod.cpp + visitors/op/multiclass_nms.cpp visitors/op/mvn.cpp visitors/op/negative.cpp visitors/op/non_max_suppression.cpp @@ -430,11 +434,13 @@ set(MULTI_TEST_SRC backend/logical_xor.in.cpp backend/lrn.in.cpp backend/matmul.in.cpp + backend/matrix_nms.in.cpp backend/maximum.in.cpp backend/max_pool.in.cpp backend/minimum.in.cpp backend/mish.in.cpp backend/mod.in.cpp + backend/multiclass_nms.in.cpp backend/multiple_backends.in.cpp backend/multiple_result.in.cpp backend/multiply.in.cpp diff --git a/ngraph/test/backend/matrix_nms.in.cpp b/ngraph/test/backend/matrix_nms.in.cpp new file mode 100644 index 00000000000..0c691831356 --- /dev/null +++ b/ngraph/test/backend/matrix_nms.in.cpp @@ -0,0 +1,667 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +// clang-format off +#ifdef ${BACKEND_NAME}_FLOAT_TOLERANCE_BITS +#define DEFAULT_FLOAT_TOLERANCE_BITS ${BACKEND_NAME}_FLOAT_TOLERANCE_BITS +#endif + +#ifdef ${BACKEND_NAME}_DOUBLE_TOLERANCE_BITS +#define DEFAULT_DOUBLE_TOLERANCE_BITS ${BACKEND_NAME}_DOUBLE_TOLERANCE_BITS +#endif +// clang-format on + +#include "gtest/gtest.h" +#include "ngraph/ngraph.hpp" +#include "util/engine/test_engines.hpp" +#include "util/test_case.hpp" +#include "util/test_control.hpp" + +using namespace std; +using namespace ngraph; + +static string s_manifest = "${MANIFEST}"; +using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME}); + +NGRAPH_TEST(${BACKEND_NAME}, matrix_nms_output_type_i64) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3}; + + op::v8::MatrixNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MatrixNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = 0; + + const auto boxes_shape = Shape{1, 6, 4}; // N 1, C 2, M 6 + const auto scores_shape = Shape{1, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + attrs.decay_function = op::v8::MatrixNms::DecayFunction::LINEAR; + attrs.gaussian_sigma = 2.0f; + attrs.post_threshold = 0.0f; + + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {0, 3, 1}; + std::vector expected_selected_scores = {1.00, 0.95, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.8, 0.00, 10.00, 1.00, 11.00, + 1.00, 0.13636364, 0.0, 0.1, 1.0, 1.1}; + std::vector expected_valid_outputs = {3}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({3, 6}, expected_selected_scores); + test_case.add_expected_output({3, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, matrix_nms_output_type_i32) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3}; + + op::v8::MatrixNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MatrixNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = 0; + + const auto boxes_shape = Shape{1, 6, 4}; // N 1, C 2, M 6 + const auto scores_shape = Shape{1, 2, 6}; + attrs.decay_function = op::v8::MatrixNms::DecayFunction::LINEAR; + attrs.gaussian_sigma = 2.0f; + attrs.post_threshold = 0.0f; + attrs.output_type = ngraph::element::i32; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {0, 3, 1}; + std::vector expected_selected_scores = {1.00, 0.95, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.8, 0.00, 10.00, 1.00, 11.00, + 1.00, 0.13636364, 0.0, 0.1, 1.0, 1.1}; + std::vector expected_valid_outputs = {3}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({3, 6}, expected_selected_scores); + test_case.add_expected_output({3, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, matrix_nms_gaussian) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3}; + + op::v8::MatrixNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MatrixNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = 0; + + const auto boxes_shape = Shape{1, 6, 4}; // N 1, C 2, M 6 + const auto scores_shape = Shape{1, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + attrs.decay_function = op::v8::MatrixNms::DecayFunction::GAUSSIAN; + attrs.gaussian_sigma = 2.0f; + attrs.post_threshold = 0.0f; + + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {0, 3, 1}; + std::vector expected_selected_scores = {1.00, 0.95, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.8, 0.00, 10.00, 1.00, 11.00, + 1.00, 0.1966116, 0.0, 0.1, 1.0, 1.1}; + std::vector expected_valid_outputs = {3}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({3, 6}, expected_selected_scores); + test_case.add_expected_output({3, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, matrix_nms_two_batches_two_classes) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; // 1 + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3}; // 1 + + op::v8::MatrixNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MatrixNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = 0; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + attrs.decay_function = op::v8::MatrixNms::DecayFunction::LINEAR; + attrs.gaussian_sigma = 2.0f; + attrs.post_threshold = 0.0f; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {0, 3, 1, + 6, 9, 7}; + std::vector expected_selected_scores = {1.00, 0.95, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.8, 0.00, 10.00, 1.00, 11.00, + 1.00, 0.13636364, 0.0, 0.1, 1.0, 1.1, + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.8, 0.00, 10.00, 1.00, 11.00, + 1.00, 0.13636364, 0.0, 0.1, 1.0, 1.1}; + std::vector expected_valid_outputs = {3, 3}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({6, 6}, expected_selected_scores); + test_case.add_expected_output({6, 1}, expected_selected_indices); + test_case.add_expected_output({2}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, matrix_nms_two_batches_two_classes_by_score_cross_batch) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; // 1 + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3}; // 1 + + op::v8::MatrixNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MatrixNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = -1; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + attrs.decay_function = op::v8::MatrixNms::DecayFunction::LINEAR; + attrs.gaussian_sigma = 2.0f; + attrs.post_threshold = 0.5; + attrs.sort_result_across_batch = true; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3, 0, 9, 6, + 0, 6, 3, 9}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, //3 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, //0 + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, //9 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, //6 + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, //0 + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, //6 + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, //3 + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00}; // 9 + std::vector expected_valid_outputs = {4, 4}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({8, 6}, expected_selected_scores); + test_case.add_expected_output({8, 1}, expected_selected_indices); + test_case.add_expected_output({2}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, matrix_nms_two_batches_two_classes_by_classid_cross_batch) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; // 1 + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3}; // 1 + + op::v8::MatrixNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MatrixNms::SortResultType::CLASSID; + attrs.keep_top_k = -1; + attrs.background_class = -1; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + attrs.decay_function = op::v8::MatrixNms::DecayFunction::LINEAR; + attrs.gaussian_sigma = 2.0f; + attrs.post_threshold = 0.5; + attrs.sort_result_across_batch = true; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3, 0, 9, 6, + 0, 3, 6, 9}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, //3 + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, //0 + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, //9 + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, //6 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, //0 + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, // 3 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, //6 + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; // 9 + std::vector expected_valid_outputs = {4, 4}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({8, 6}, expected_selected_scores); + test_case.add_expected_output({8, 1}, expected_selected_indices); + test_case.add_expected_output({2}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, matrix_nms_by_keep_top_k) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; // 1 + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3}; // 1 + + op::v8::MatrixNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MatrixNms::SortResultType::CLASSID; + attrs.keep_top_k = 3; + attrs.background_class = 0; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + attrs.decay_function = op::v8::MatrixNms::DecayFunction::LINEAR; + attrs.gaussian_sigma = 2.0f; + attrs.post_threshold = 0.0f; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {0, 3, 1, + 6, 9, 7}; + std::vector expected_selected_scores = {1.00, 0.95, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.8, 0.00, 10.00, 1.00, 11.00, + 1.00, 0.13636364, 0.0, 0.1, 1.0, 1.1, + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.8, 0.00, 10.00, 1.00, 11.00, + 1.00, 0.13636364, 0.0, 0.1, 1.0, 1.1}; + std::vector expected_valid_outputs = {3, 3}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({6, 6}, expected_selected_scores); + test_case.add_expected_output({6, 1}, expected_selected_indices); + test_case.add_expected_output({2}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, matrix_nms_background) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3}; + + op::v8::MatrixNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MatrixNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = -1; + + const auto boxes_shape = Shape{1, 6, 4}; // N 1, C 2, M 6 + const auto scores_shape = Shape{1, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + attrs.decay_function = op::v8::MatrixNms::DecayFunction::LINEAR; + attrs.gaussian_sigma = 2.0f; + attrs.post_threshold = 0.0f; + + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3, 0, 0, 3, 1, 1}; + std::vector expected_selected_scores = {0.00, 0.95, 0.0, 10.0, 1.0, 11.0, + 1.00, 0.95, 0.0, 0.0, 1.0, 1.0, + 0.00, 0.9, 0.0, 0.0, 1.0, 1.0, + 1.00, 0.8, 0.0, 10.0, 1.0, 11.0, + 0.00, 0.13636364, 0.0, 0.1, 1.0, 1.1, + 1.00, 0.13636364, 0.0, 0.1, 1.0, 1.1}; + std::vector expected_valid_outputs = {6}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({6, 6}, expected_selected_scores); + test_case.add_expected_output({6, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, matrix_nms_flipped_coordinates) +{ + std::vector boxes_data = {1.0, 1.0, 0.0, 0.0, 0.0, 0.1, 1.0, 1.1, + 0.0, 0.9, 1.0, -0.1, 0.0, 10.0, 1.0, 11.0, + 1.0, 10.1, 0.0, 11.1, 1.0, 101.0, 0.0, 100.0}; + + std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; + + op::v8::MatrixNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MatrixNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = -1; + + const auto boxes_shape = Shape{1, 6, 4}; // N 1, C 1, M 6 + const auto scores_shape = Shape{1, 1, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + attrs.decay_function = op::v8::MatrixNms::DecayFunction::LINEAR; + attrs.gaussian_sigma = 2.0f; + attrs.post_threshold = 0.0f; + + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3, 0, 1}; + std::vector expected_selected_scores = {0.00, 0.95, 0.0, 10.0, 1.0, 11.0, + 0.00, 0.9, 1.0, 1.0, 0.0, 0.0, + 0.00, 0.75, 0.0, 0.1, 1.0, 1.1}; + std::vector expected_valid_outputs = {3}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({3, 6}, expected_selected_scores); + test_case.add_expected_output({3, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, matrix_nms_post_threshold) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; + + op::v8::MatrixNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.score_threshold = 0.00; + attrs.sort_result_type = op::v8::MatrixNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = -1; + + const auto boxes_shape = Shape{1, 6, 4}; // N 1, C 2, M 6 + const auto scores_shape = Shape{1, 1, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + attrs.decay_function = op::v8::MatrixNms::DecayFunction::LINEAR; + attrs.gaussian_sigma = 2.0f; + attrs.post_threshold = 0.8; + + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3, 0}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, + 0.00, 0.9, 0.00, 0.00, 1.00, 1.00}; + std::vector expected_valid_outputs = {2}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({2, 6}, expected_selected_scores); + test_case.add_expected_output({2, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, matrix_nms_identical_boxes) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, + 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, + 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, + 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0}; + + std::vector scores_data = {0.4, 0.01, 0.2, 0.09, 0.15, 0.05, 0.02, 0.03, 0.05, 0.0}; + + op::v8::MatrixNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MatrixNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = -1; + + const auto boxes_shape = Shape{1, 10, 4}; // N 1, C 1, M 10 + const auto scores_shape = Shape{1, 1, 10}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + attrs.decay_function = op::v8::MatrixNms::DecayFunction::LINEAR; + attrs.gaussian_sigma = 2.0f; + attrs.post_threshold = 0.3; + + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {0}; + std::vector expected_selected_scores = {0.00, 0.40, 0.00, 0.00, 1.00, 1.00}; + std::vector expected_valid_outputs = {1}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({1, 6}, expected_selected_scores); + test_case.add_expected_output({1, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, matrix_nms_nms_top_k) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; + + op::v8::MatrixNms::Attributes attrs; + attrs.nms_top_k = 2; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MatrixNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = -1; + + const auto boxes_shape = Shape{1, 6, 4}; + const auto scores_shape = Shape{1, 1, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + attrs.decay_function = op::v8::MatrixNms::DecayFunction::LINEAR; + attrs.gaussian_sigma = 2.0f; + attrs.post_threshold = 0.0f; + + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3, 0}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00 }; + std::vector expected_valid_outputs = {2}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({2, 6}, expected_selected_scores); + test_case.add_expected_output({2, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, matrix_nms_single_box) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0}; + + std::vector scores_data = {0.9}; + + op::v8::MatrixNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MatrixNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = -1; + + const auto boxes_shape = Shape{1, 1, 4}; + const auto scores_shape = Shape{1, 1, 1}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + attrs.decay_function = op::v8::MatrixNms::DecayFunction::LINEAR; + attrs.gaussian_sigma = 2.0f; + attrs.post_threshold = 0.0f; + + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {0}; + std::vector expected_selected_scores = {0.00, 0.90, 0.00, 0.00, 1.00, 1.00}; + std::vector expected_valid_outputs = {1}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({1, 6}, expected_selected_scores); + test_case.add_expected_output({1, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, matrix_nms_no_output) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; + + op::v8::MatrixNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.score_threshold = 2.0f; + attrs.sort_result_type = op::v8::MatrixNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = -1; + + const auto boxes_shape = Shape{1, 6, 4}; + const auto scores_shape = Shape{1, 1, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + attrs.decay_function = op::v8::MatrixNms::DecayFunction::LINEAR; + attrs.gaussian_sigma = 2.0f; + attrs.post_threshold = 0.0f; + + auto nms = make_shared(boxes, scores, attrs); + + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {}; + std::vector expected_selected_scores = {}; + std::vector expected_valid_outputs = {0}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({0, 6}, expected_selected_scores); + test_case.add_expected_output({0, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} diff --git a/ngraph/test/backend/multiclass_nms.in.cpp b/ngraph/test/backend/multiclass_nms.in.cpp new file mode 100644 index 00000000000..3c0d3153765 --- /dev/null +++ b/ngraph/test/backend/multiclass_nms.in.cpp @@ -0,0 +1,802 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +// clang-format off +#ifdef ${BACKEND_NAME}_FLOAT_TOLERANCE_BITS +#define DEFAULT_FLOAT_TOLERANCE_BITS ${BACKEND_NAME}_FLOAT_TOLERANCE_BITS +#endif + +#ifdef ${BACKEND_NAME}_DOUBLE_TOLERANCE_BITS +#define DEFAULT_DOUBLE_TOLERANCE_BITS ${BACKEND_NAME}_DOUBLE_TOLERANCE_BITS +#endif +// clang-format on + +#include "gtest/gtest.h" +#include "ngraph/ngraph.hpp" +#include "util/engine/test_engines.hpp" +#include "util/test_case.hpp" +#include "util/test_control.hpp" + +using namespace std; +using namespace ngraph; + +static string s_manifest = "${MANIFEST}"; +using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME}); + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_score) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3}; + + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.iou_threshold = 0.5f; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = -1; + attrs.nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 6, 4}; // N 1, C 2, M 6 + const auto scores_shape = Shape{1, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3, 0, 0, 3}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; + std::vector expected_valid_outputs = {4}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({4, 6}, expected_selected_scores); + test_case.add_expected_output({4, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_class_id) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3}; + + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.iou_threshold = 0.5f; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + attrs.keep_top_k = -1; + attrs.background_class = -1; + attrs.nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 6, 4}; // N 1, C 2, M 6 + const auto scores_shape = Shape{1, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3, 0, 0, 3}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00 , + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00 , + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; + std::vector expected_valid_outputs = {4}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({4, 6}, expected_selected_scores); + test_case.add_expected_output({4, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); + +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_output_type_i32) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3}; + + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.iou_threshold = 0.5f; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + attrs.keep_top_k = -1; + attrs.background_class = -1; + attrs.nms_eta = 1.0f; + attrs.output_type = element::i32; + + const auto boxes_shape = Shape{1, 6, 4}; // N 1, C 2, M 6 + const auto scores_shape = Shape{1, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3, 0, 0, 3}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00 , + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00 , + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; + std::vector expected_valid_outputs = {4}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({4, 6}, expected_selected_scores); + test_case.add_expected_output({4, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches_two_classes_by_score) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; + + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.iou_threshold = 0.5f; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = -1; + attrs.nms_eta = 1.0f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3, 0, 0, 3, + 9, 6, 6, 9}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, // 0 + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; // 1 + std::vector expected_valid_outputs = {4, 4}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({8, 6}, expected_selected_scores); + test_case.add_expected_output({8, 1}, expected_selected_indices); + test_case.add_expected_output({2}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches_two_classes_by_class_id) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; + + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.iou_threshold = 0.5f; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + attrs.keep_top_k = -1; + attrs.background_class = -1; + attrs.nms_eta = 1.0f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3, 0, 0, 3, + 9, 6, 6, 9}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, // 0 + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; // 1 + std::vector expected_valid_outputs = {4, 4}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({8, 6}, expected_selected_scores); + test_case.add_expected_output({8, 1}, expected_selected_indices); + test_case.add_expected_output({2}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches_two_classes_by_score_cross_batch) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; + + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.iou_threshold = 0.5f; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = -1; + attrs.nms_eta = 1.0f; + attrs.sort_result_across_batch = true; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3, 0, 9, 6, + 0, 6, 3, 9}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, //3 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, //0 + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, //9 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, //6 + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, //0 + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, //6 + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, //3 + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; // 9 + std::vector expected_valid_outputs = {4, 4}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({8, 6}, expected_selected_scores); + test_case.add_expected_output({8, 1}, expected_selected_indices); + test_case.add_expected_output({2}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_two_batches_two_classes_by_class_id_cross_batch) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; + + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.iou_threshold = 0.5f; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + attrs.keep_top_k = -1; + attrs.background_class = -1; + attrs.nms_eta = 1.0f; + attrs.sort_result_across_batch = true; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3, 0, 9, 6, + 0, 3, 6, 9}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, //3 + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, //0 + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, //9 + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, //6 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, //0 + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, // 3 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, //6 + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; // 9 + std::vector expected_valid_outputs = {4, 4}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({8, 6}, expected_selected_scores); + test_case.add_expected_output({8, 1}, expected_selected_indices); + test_case.add_expected_output({2}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_flipped_coordinates) +{ + std::vector boxes_data = {1.0, 1.0, 0.0, 0.0, 0.0, 0.1, 1.0, 1.1, + 0.0, 0.9, 1.0, -0.1, 0.0, 10.0, 1.0, 11.0, + 1.0, 10.1, 0.0, 11.1, 1.0, 101.0, 0.0, 100.0}; + + std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; + + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.iou_threshold = 0.5f; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = -1; + attrs.nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 6, 4}; // N 1, C 1, M 6 + const auto scores_shape = Shape{1, 1, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3, 0, 1}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 1.00, 1.00, 0.00, 0.00 , + 0.00, 0.75, 0.00, 0.10, 1.00, 1.10}; + std::vector expected_valid_outputs = {3}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({3, 6}, expected_selected_scores); + test_case.add_expected_output({3, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_identical_boxes) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, + 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, + 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, + 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0}; + + std::vector scores_data = {0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9}; + + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.iou_threshold = 0.5f; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = -1; + attrs.nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 10, 4}; // N 1, C 1, M 10 + const auto scores_shape = Shape{1, 1, 10}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {0}; + std::vector expected_selected_scores = {0.00, 0.90, 0.00, 0.00, 1.00, 1.00}; + std::vector expected_valid_outputs = {1}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({1, 6}, expected_selected_scores); + test_case.add_expected_output({1, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_limit_output_size) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; + + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = 2; + attrs.iou_threshold = 0.5f; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = -1; + attrs.nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 6, 4}; + const auto scores_shape = Shape{1, 1, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3, 0}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00 }; + std::vector expected_valid_outputs = {2}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({2, 6}, expected_selected_scores); + test_case.add_expected_output({2, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_single_box) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0}; + + std::vector scores_data = {0.9}; + + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.iou_threshold = 0.5f; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = -1; + attrs.nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 1, 4}; + const auto scores_shape = Shape{1, 1, 1}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {0}; + std::vector expected_selected_scores = {0.00, 0.90, 0.00, 0.00, 1.00, 1.00}; + std::vector expected_valid_outputs = {1}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({1, 6}, expected_selected_scores); + test_case.add_expected_output({1, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_IOU) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; + + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.iou_threshold = 0.2f; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = -1; + attrs.nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 6, 4}; + const auto scores_shape = Shape{1, 1, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3, 0}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00}; + std::vector expected_valid_outputs = {2}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({2, 6}, expected_selected_scores); + test_case.add_expected_output({2, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_IOU_and_scores) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; + + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.iou_threshold = 0.5f; + attrs.score_threshold = 0.95f; + attrs.sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = -1; + attrs.nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 6, 4}; + const auto scores_shape = Shape{1, 1, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00}; + std::vector expected_valid_outputs = {1}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({1, 6}, expected_selected_scores); + test_case.add_expected_output({1, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_no_output) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0}; + + std::vector scores_data = {0.9, 0.75, 0.6, 0.95, 0.5, 0.3}; + + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.iou_threshold = 0.5f; + attrs.score_threshold = 2.0f; + attrs.sort_result_type = op::v8::MulticlassNms::SortResultType::SCORE; + attrs.keep_top_k = -1; + attrs.background_class = -1; + attrs.nms_eta = 1.0f; + + const auto boxes_shape = Shape{1, 6, 4}; + const auto scores_shape = Shape{1, 1, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {}; + std::vector expected_selected_scores = {}; + std::vector expected_valid_outputs = {0}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({0, 6}, expected_selected_scores); + test_case.add_expected_output({0, 1}, expected_selected_indices); + test_case.add_expected_output({1}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_background) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; + + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.iou_threshold = 0.5f; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + attrs.keep_top_k = -1; + attrs.background_class = 0; + attrs.nms_eta = 1.0f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {0, 3, 6, 9}; + std::vector expected_selected_scores = {1.00, 0.95, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00, // 0 + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 }; // 1 + std::vector expected_valid_outputs = {2, 2}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({4, 6}, expected_selected_scores); + test_case.add_expected_output({4, 1}, expected_selected_indices); + test_case.add_expected_output({2}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_keep_top_k) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; + + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.iou_threshold = 0.5f; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + attrs.keep_top_k = 3; + attrs.background_class = -1; + attrs.nms_eta = 1.0f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3, 0, 0, + 9, 6, 6}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00, // 0 + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00, 0.00, 0.90, 0.00, 0.00, 1.00, 1.00, + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00 }; // 1 + std::vector expected_valid_outputs = {3, 3}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({6, 6}, expected_selected_scores); + test_case.add_expected_output({6, 1}, expected_selected_indices); + test_case.add_expected_output({2}, expected_valid_outputs); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, multiclass_nms_by_nms_eta) +{ + std::vector boxes_data = {0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0, // 0 + 0.0, 0.0, 1.0, 1.0, 0.0, 0.1, 1.0, 1.1, + 0.0, -0.1, 1.0, 0.9, 0.0, 10.0, 1.0, 11.0, + 0.0, 10.1, 1.0, 11.1, 0.0, 100.0, 1.0, 101.0 // 1 + }; + + std::vector scores_data = { + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3, // 0 + 0.9, 0.75, 0.6, 0.95, 0.5, 0.3, + 0.95, 0.75, 0.6, 0.80, 0.5, 0.3 // 1 + }; + + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = -1; + attrs.iou_threshold = 1.0f; + attrs.score_threshold = 0.0f; + attrs.sort_result_type = op::v8::MulticlassNms::SortResultType::CLASSID; + attrs.keep_top_k = -1; + attrs.background_class = -1; + attrs.nms_eta = 0.1f; + + const auto boxes_shape = Shape{2, 6, 4}; // N 2, C 2, M 6 + const auto scores_shape = Shape{2, 2, 6}; + + const auto boxes = make_shared(element::f32, boxes_shape); + const auto scores = make_shared(element::f32, scores_shape); + auto nms = make_shared(boxes, scores, attrs); + + auto f = make_shared(nms, ParameterVector{boxes, scores}); + + std::vector expected_selected_indices = {3, 0, 5, 0, 3, 5, + 9, 6, 11, 6, 9, 11}; + std::vector expected_selected_scores = {0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00 , + 0.00, 0.30, 0.00, 100.00, 1.00, 101.00 , + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00 , + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 , + 1.00, 0.30, 0.00, 100.00, 1.00, 101.00 , + 0.00, 0.95, 0.00, 10.00, 1.00, 11.00 , + 0.00, 0.90, 0.00, 0.00, 1.00, 1.00 , + 0.00, 0.30, 0.00, 100.00, 1.00, 101.00 , + 1.00, 0.95, 0.00, 0.00, 1.00, 1.00 , + 1.00, 0.80, 0.00, 10.00, 1.00, 11.00 , + 1.00, 0.30, 0.00, 100.00, 1.00, 101.00 }; + std::vector expected_valid_outputs = {6, 6}; + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs({boxes_data, scores_data}); + test_case.add_expected_output({12, 6}, expected_selected_scores); + test_case.add_expected_output({12, 1}, expected_selected_indices); + test_case.add_expected_output({2}, expected_valid_outputs); + test_case.run(); +} diff --git a/ngraph/test/runtime/ie/unit_test.manifest b/ngraph/test/runtime/ie/unit_test.manifest index 49e221001cb..3483b054310 100644 --- a/ngraph/test/runtime/ie/unit_test.manifest +++ b/ngraph/test/runtime/ie/unit_test.manifest @@ -1105,6 +1105,41 @@ IE_CPU.onnx_model_nonmaxsuppression_center_point_box_format IE_CPU.onnx_model_nonmaxsuppression_single_box IE_CPU.nonmaxsuppression_suppress_by_IOU_and_scores_without_constants +# Unsupported dynamic op +IE_CPU.multiclass_nms_by_score +IE_CPU.multiclass_nms_by_class_id +IE_CPU.multiclass_nms_output_type_i32 +IE_CPU.multiclass_nms_two_batches_two_classes_by_score +IE_CPU.multiclass_nms_two_batches_two_classes_by_class_id +IE_CPU.multiclass_nms_two_batches_two_classes_by_score_cross_batch +IE_CPU.multiclass_nms_two_batches_two_classes_by_class_id_cross_batch +IE_CPU.multiclass_nms_no_output +IE_CPU.multiclass_nms_by_background +IE_CPU.multiclass_nms_by_keep_top_k +IE_CPU.multiclass_nms_by_nms_eta +IE_CPU.multiclass_nms_flipped_coordinates +IE_CPU.multiclass_nms_identical_boxes +IE_CPU.multiclass_nms_limit_output_size +IE_CPU.multiclass_nms_single_box +IE_CPU.multiclass_nms_by_IOU +IE_CPU.multiclass_nms_by_IOU_and_scores + +# Unsupported dynamic op +IE_CPU.matrix_nms_output_type_i64 +IE_CPU.matrix_nms_output_type_i32 +IE_CPU.matrix_nms_gaussian +IE_CPU.matrix_nms_two_batches_two_classes +IE_CPU.matrix_nms_two_batches_two_classes_by_score_cross_batch +IE_CPU.matrix_nms_two_batches_two_classes_by_classid_cross_batch +IE_CPU.matrix_nms_by_keep_top_k +IE_CPU.matrix_nms_background +IE_CPU.matrix_nms_flipped_coordinates +IE_CPU.matrix_nms_post_threshold +IE_CPU.matrix_nms_identical_boxes +IE_CPU.matrix_nms_nms_top_k +IE_CPU.matrix_nms_single_box +IE_CPU.matrix_nms_no_output + # Unsupported dynamic op IE_CPU.range_v4_trunc_inputs IE_CPU.onnx_model_reduce_sum_13_axes_as_input diff --git a/ngraph/test/runtime/interpreter/evaluates_map.cpp b/ngraph/test/runtime/interpreter/evaluates_map.cpp index b3e854c72de..6bad2c31fa4 100644 --- a/ngraph/test/runtime/interpreter/evaluates_map.cpp +++ b/ngraph/test/runtime/interpreter/evaluates_map.cpp @@ -49,7 +49,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -70,6 +72,7 @@ #include #include #include +#include using namespace ngraph; using namespace std; @@ -922,6 +925,285 @@ namespace return true; } + namespace matrix_nms_v8 + { + using SortResultType = op::v8::MatrixNms::SortResultType; + struct InfoForNMS + { + Shape selected_outputs_shape; + Shape selected_indices_shape; + Shape boxes_shape; + Shape scores_shape; + std::vector boxes_data; + std::vector scores_data; + size_t selected_outputs_shape_size; + size_t selected_indices_shape_size; + }; + + constexpr size_t boxes_port = 0; + constexpr size_t scores_port = 1; + + PartialShape + infer_selected_outputs_shape(const std::vector>& inputs, + int nms_top_k, int keep_top_k) + { + const auto boxes_ps = inputs[boxes_port]->get_partial_shape(); + const auto scores_ps = inputs[scores_port]->get_partial_shape(); + + PartialShape result = {Dimension::dynamic(), 6}; + + if (boxes_ps.rank().is_static() && scores_ps.rank().is_static()) + { + const auto num_boxes_boxes = boxes_ps[1]; + if (num_boxes_boxes.is_static() && scores_ps[0].is_static() && scores_ps[1].is_static()) + { + const auto num_boxes = num_boxes_boxes.get_length(); + const auto num_classes = scores_ps[1].get_length(); + int64_t max_output_boxes_per_class = 0; + if (nms_top_k >= 0) + max_output_boxes_per_class = std::min(num_boxes, (int64_t)nms_top_k); + else + max_output_boxes_per_class = num_boxes; + + auto max_output_boxes_per_batch = max_output_boxes_per_class * num_classes; + if (keep_top_k >= 0) + max_output_boxes_per_batch = + std::min(max_output_boxes_per_batch, (int64_t)keep_top_k); + + result[0] = max_output_boxes_per_batch * scores_ps[0].get_length(); + } + } + + return result; + } + + std::vector prepare_boxes_data(const std::shared_ptr& boxes, + const Shape& boxes_shape) + { + auto result = get_floats(boxes, boxes_shape); + return result; + } + + std::vector prepare_scores_data(const std::shared_ptr& scores, + const Shape& scores_shape) + { + auto result = get_floats(scores, scores_shape); + return result; + } + + InfoForNMS get_info_for_nms_eval(const std::shared_ptr& nms, + const std::vector>& inputs) + { + InfoForNMS result; + + auto selected_outputs_shape = + infer_selected_outputs_shape(inputs, nms->get_nms_top_k(), nms->get_keep_top_k()); + result.selected_outputs_shape = selected_outputs_shape.to_shape(); + result.selected_indices_shape = {result.selected_outputs_shape[0], 1}; + + result.boxes_shape = inputs[boxes_port]->get_shape(); + result.scores_shape = inputs[scores_port]->get_shape(); + + result.boxes_data = prepare_boxes_data(inputs[boxes_port], result.boxes_shape); + result.scores_data = prepare_scores_data(inputs[scores_port], result.scores_shape); + + result.selected_outputs_shape_size = shape_size(result.selected_outputs_shape); + result.selected_indices_shape_size = shape_size(result.selected_indices_shape); + + return result; + } + } // namespace matrix_nms_v8 + + template + bool evaluate(const shared_ptr& op, + const HostTensorVector& outputs, + const HostTensorVector& inputs) + { + auto info = matrix_nms_v8::get_info_for_nms_eval(op, inputs); + + std::vector selected_outputs(info.selected_outputs_shape_size); + std::vector selected_indices(info.selected_indices_shape_size); + std::vector valid_outputs(info.boxes_shape[0]); + + runtime::reference::matrix_nms(info.boxes_data.data(), + info.boxes_shape, + info.scores_data.data(), + info.scores_shape, + op->get_attrs(), + selected_outputs.data(), + info.selected_outputs_shape, + selected_indices.data(), + info.selected_indices_shape, + valid_outputs.data()); + + void* pscores = nullptr; + void* pselected_num = nullptr; + void* prois; + size_t num_selected = static_cast(std::accumulate(valid_outputs.begin(), valid_outputs.end(), 0)); + + outputs[0]->set_shape({num_selected, 6}); + prois = outputs[0]->get_data_ptr(); + + if (outputs.size() >= 2) + { + outputs[1]->set_shape({num_selected, 1}); + pscores = outputs[1]->get_data_ptr(); + } + if (outputs.size() >= 3) + { + pselected_num = outputs[2]->get_data_ptr(); + } + + runtime::reference::nms_common::nms_common_postprocessing(prois, + pscores, + pselected_num, + op->get_output_type(), + selected_outputs, + selected_indices, + valid_outputs); + return true; + } + + namespace multiclass_nms_v8 + { + using SortResultType = op::v8::MulticlassNms::SortResultType; + struct InfoForNMS + { + Shape selected_outputs_shape; + Shape selected_indices_shape; + Shape boxes_shape; + Shape scores_shape; + std::vector boxes_data; + std::vector scores_data; + size_t selected_outputs_shape_size; + size_t selected_indices_shape_size; + }; + + constexpr size_t boxes_port = 0; + constexpr size_t scores_port = 1; + + PartialShape + infer_selected_outputs_shape(const std::vector>& inputs, + int nms_top_k, int keep_top_k) + { + const auto boxes_ps = inputs[boxes_port]->get_partial_shape(); + const auto scores_ps = inputs[scores_port]->get_partial_shape(); + + PartialShape result = {Dimension::dynamic(), 6}; + + if (boxes_ps.rank().is_static() && scores_ps.rank().is_static()) + { + const auto num_boxes_boxes = boxes_ps[1]; + if (num_boxes_boxes.is_static() && scores_ps[0].is_static() && scores_ps[1].is_static()) + { + const auto num_boxes = num_boxes_boxes.get_length(); + const auto num_classes = scores_ps[1].get_length(); + int64_t max_output_boxes_per_class = 0; + if (nms_top_k >= 0) + max_output_boxes_per_class = std::min(num_boxes, (int64_t)nms_top_k); + else + max_output_boxes_per_class = num_boxes; + + auto max_output_boxes_per_batch = max_output_boxes_per_class * num_classes; + if (keep_top_k >= 0) + max_output_boxes_per_batch = + std::min(max_output_boxes_per_batch, (int64_t)keep_top_k); + + result[0] = max_output_boxes_per_batch * scores_ps[0].get_length(); + } + } + + return result; + } + + std::vector prepare_boxes_data(const std::shared_ptr& boxes, + const Shape& boxes_shape) + { + auto result = get_floats(boxes, boxes_shape); + return result; + } + + std::vector prepare_scores_data(const std::shared_ptr& scores, + const Shape& scores_shape) + { + auto result = get_floats(scores, scores_shape); + return result; + } + + InfoForNMS get_info_for_nms_eval(const std::shared_ptr& nms, + const std::vector>& inputs) + { + InfoForNMS result; + + auto selected_outputs_shape = + infer_selected_outputs_shape(inputs, nms->get_nms_top_k(), nms->get_keep_top_k()); + result.selected_outputs_shape = selected_outputs_shape.to_shape(); + result.selected_indices_shape = {result.selected_outputs_shape[0], 1}; + + result.boxes_shape = inputs[boxes_port]->get_shape(); + result.scores_shape = inputs[scores_port]->get_shape(); + + result.boxes_data = prepare_boxes_data(inputs[boxes_port], result.boxes_shape); + result.scores_data = prepare_scores_data(inputs[scores_port], result.scores_shape); + + result.selected_outputs_shape_size = shape_size(result.selected_outputs_shape); + result.selected_indices_shape_size = shape_size(result.selected_indices_shape); + + return result; + } + } // namespace multiclass_nms_v8 + + template + bool evaluate(const shared_ptr& op, + const HostTensorVector& outputs, + const HostTensorVector& inputs) + { + auto info = multiclass_nms_v8::get_info_for_nms_eval(op, inputs); + + std::vector selected_outputs(info.selected_outputs_shape_size); + std::vector selected_indices(info.selected_indices_shape_size); + std::vector valid_outputs(inputs[0]->get_shape()[0]); + + runtime::reference::multiclass_nms(info.boxes_data.data(), + info.boxes_shape, + info.scores_data.data(), + info.scores_shape, + op->get_attrs(), + selected_outputs.data(), + info.selected_outputs_shape, + selected_indices.data(), + info.selected_indices_shape, + valid_outputs.data()); + + void* pscores = nullptr; + void* pselected_num = nullptr; + void* prois; + size_t num_selected = static_cast(std::accumulate(valid_outputs.begin(), valid_outputs.end(), 0)); + + outputs[0]->set_shape({num_selected, 6}); + prois = outputs[0]->get_data_ptr(); + + if (outputs.size() >= 2) + { + outputs[1]->set_shape({num_selected, 1}); + pscores = outputs[1]->get_data_ptr(); + } + if (outputs.size() >= 3) + { + pselected_num = outputs[2]->get_data_ptr(); + } + + runtime::reference::nms_common::nms_common_postprocessing(prois, + pscores, + pselected_num, + op->get_output_type(), + selected_outputs, + selected_indices, + valid_outputs); + + return true; + } + namespace experimental_prior_grid { struct InfoForEDPriorGrid @@ -2585,16 +2867,14 @@ namespace for (size_t i = 1; i < node->outputs().size(); i++) { if ((is_type(node) || + is_type(node) || + is_type(node) || is_type(node) || is_type(node)) && - i == 1) + i == 1) { continue; } - if (element_type != node->get_output_element_type(i)) - { - throw std::logic_error("Output node element types is not equal"); - } } switch (element_type) { diff --git a/ngraph/test/runtime/interpreter/opset_int_tbl.hpp b/ngraph/test/runtime/interpreter/opset_int_tbl.hpp index 11e64dbab6f..2893e716af5 100644 --- a/ngraph/test/runtime/interpreter/opset_int_tbl.hpp +++ b/ngraph/test/runtime/interpreter/opset_int_tbl.hpp @@ -99,3 +99,5 @@ NGRAPH_OP(Roll, ngraph::op::v7) NGRAPH_OP(AdaptiveAvgPool, ngraph::op::v8) NGRAPH_OP(AdaptiveMaxPool, ngraph::op::v8) +NGRAPH_OP(MatrixNms, op::v8) +NGRAPH_OP(MulticlassNms, op::v8) diff --git a/ngraph/test/type_prop/matrix_nms.cpp b/ngraph/test/type_prop/matrix_nms.cpp new file mode 100644 index 00000000000..6817786cd1f --- /dev/null +++ b/ngraph/test/type_prop/matrix_nms.cpp @@ -0,0 +1,263 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "gtest/gtest.h" +#include "ngraph/ngraph.hpp" +#include "util/type_prop.hpp" + +using namespace std; +using namespace ngraph; + +TEST(type_prop, matrix_nms_incorrect_boxes_rank) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 3, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 3}); + + make_shared(boxes, scores, op::v8::MatrixNms::Attributes()); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), "Expected a 3D tensor for the 'boxes' input"); + } +} + +TEST(type_prop, matrix_nms_incorrect_scores_rank) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2}); + + make_shared(boxes, scores, op::v8::MatrixNms::Attributes()); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), "Expected a 3D tensor for the 'scores' input"); + } +} + +TEST(type_prop, matrix_nms_incorrect_scheme_num_batches) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{2, 2, 3}); + + make_shared(boxes, scores, op::v8::MatrixNms::Attributes()); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "The first dimension of both 'boxes' and 'scores' must match"); + } +} + +TEST(type_prop, matrix_nms_incorrect_scheme_num_boxes) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 3}); + + make_shared(boxes, scores, op::v8::MatrixNms::Attributes()); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "'boxes' and 'scores' input shapes must match at the second and third " + "dimension respectively"); + } +} + +TEST(type_prop, matrix_nms_incorrect_boxes_rank2) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 3}); + const auto scores = make_shared(element::f32, Shape{2, 2, 2}); + + make_shared(boxes, scores, op::v8::MatrixNms::Attributes()); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "The third dimension of the 'boxes' must be 4"); + } +} + +TEST(type_prop, matrix_nms_incorrect_output_type) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 2}); + op::v8::MatrixNms::Attributes attrs; + attrs.output_type = ngraph::element::f32; + + make_shared(boxes, scores, attrs); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "Output type must be i32 or i64"); + } +} + +TEST(type_prop, matrix_nms_incorrect_nms_topk) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 2}); + op::v8::MatrixNms::Attributes attrs; + attrs.nms_top_k = -2; + + make_shared(boxes, scores, attrs); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "The 'nms_top_k' must be great or equal -1"); + } +} + +TEST(type_prop, matrix_nms_incorrect_keep_topk) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 2}); + op::v8::MatrixNms::Attributes attrs; + attrs.keep_top_k = -2; + + make_shared(boxes, scores, attrs); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "The 'keep_top_k' must be great or equal -1"); + } +} + +TEST(type_prop, matrix_nms_incorrect_background_class) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 2}); + op::v8::MatrixNms::Attributes attrs; + attrs.background_class = -2; + + make_shared(boxes, scores, attrs); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "The 'background_class' must be great or equal -1"); + } +} + +TEST(type_prop, matrix_nms_output_shape_1dim_dynamic) +{ + const auto boxes = make_shared(element::f32, Shape{5, 2, 4}); + const auto scores = make_shared(element::f32, Shape{5, 3, 2}); + + const auto nms = make_shared(boxes, scores, op::v8::MatrixNms::Attributes()); + + ASSERT_TRUE( + nms->get_output_partial_shape(0).same_scheme(PartialShape{Dimension::dynamic(), 6})); + ASSERT_TRUE( + nms->get_output_partial_shape(1).same_scheme(PartialShape{Dimension::dynamic(), 1})); + + EXPECT_EQ(nms->get_output_shape(2), (Shape{5})); +} + +TEST(type_prop, matrix_nms_output_shape_1dim_max_out) +{ + const auto boxes = make_shared(element::f32, Shape{2, 7, 4}); + const auto scores = make_shared(element::f32, Shape{2, 5, 7}); + + const auto nms = make_shared(boxes, scores, op::v8::MatrixNms::Attributes()); + + ASSERT_EQ(nms->get_output_element_type(0), element::f32); + ASSERT_EQ(nms->get_output_element_type(1), element::i64); + ASSERT_EQ(nms->get_output_element_type(2), element::i64); + + // batch * class * box + EXPECT_EQ(nms->get_output_partial_shape(0), PartialShape({Dimension(0, 2 * 5 * 7), Dimension(6)})); + EXPECT_EQ(nms->get_output_partial_shape(1), PartialShape({Dimension(0, 2 * 5 * 7), 1})); + EXPECT_EQ(nms->get_output_shape(2), (Shape{2})); +} + +TEST(type_prop, matrix_nms_output_shape_1dim_nms_topk) +{ + const auto boxes = make_shared(element::f32, Shape{2, 7, 4}); + const auto scores = make_shared(element::f32, Shape{2, 5, 7}); + op::v8::MatrixNms::Attributes attrs; + attrs.nms_top_k = 3; + + const auto nms = make_shared(boxes, scores, attrs); + + ASSERT_EQ(nms->get_output_element_type(0), element::f32); + ASSERT_EQ(nms->get_output_element_type(1), element::i64); + ASSERT_EQ(nms->get_output_element_type(2), element::i64); + // batch * class * min(nms_topk, box) + EXPECT_EQ(nms->get_output_partial_shape(0), PartialShape({Dimension(0, 2 * 5 * 3), Dimension(6)})); + EXPECT_EQ(nms->get_output_partial_shape(1), PartialShape({Dimension(0, 2 * 5 * 3), 1})); + EXPECT_EQ(nms->get_output_shape(2), (Shape{2})); +} + +TEST(type_prop, matrix_nms_output_shape_1dim_keep_topk) +{ + const auto boxes = make_shared(element::f32, Shape{2, 7, 4}); + const auto scores = make_shared(element::f32, Shape{2, 5, 7}); + op::v8::MatrixNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.keep_top_k = 8; + + const auto nms = make_shared(boxes, scores, attrs); + + ASSERT_EQ(nms->get_output_element_type(0), element::f32); + ASSERT_EQ(nms->get_output_element_type(1), element::i64); + ASSERT_EQ(nms->get_output_element_type(2), element::i64); + // batch * min(keep_topk, class * box)) + EXPECT_EQ(nms->get_output_partial_shape(0), PartialShape({Dimension(0, 2 * 8), Dimension(6)})); + EXPECT_EQ(nms->get_output_partial_shape(1), PartialShape({Dimension(0, 2 * 8), 1})); + EXPECT_EQ(nms->get_output_shape(2), (Shape{2})); +} + +TEST(type_prop, matrix_nms_output_shape_i32) +{ + const auto boxes = make_shared(element::f32, Shape{2, 7, 4}); + const auto scores = make_shared(element::f32, Shape{2, 5, 7}); + op::v8::MatrixNms::Attributes attrs; + attrs.output_type = ngraph::element::i32; + + const auto nms = make_shared(boxes, scores, attrs); + + ASSERT_EQ(nms->get_output_element_type(0), element::f32); + ASSERT_EQ(nms->get_output_element_type(1), element::i32); + ASSERT_EQ(nms->get_output_element_type(2), element::i32); + // batch * class * box + EXPECT_EQ(nms->get_output_partial_shape(0), PartialShape({Dimension(0, 2 * 5 * 7), Dimension(6)})); + EXPECT_EQ(nms->get_output_partial_shape(1), PartialShape({Dimension(0, 2 * 5 * 7), 1})); + EXPECT_EQ(nms->get_output_shape(2), (Shape{2})); +} + +TEST(type_prop, matrix_nms_dynamic_boxes_and_scores) +{ + const auto boxes = make_shared(element::f32, PartialShape::dynamic()); + const auto scores = make_shared(element::f32, PartialShape::dynamic()); + + const auto nms = make_shared(boxes, scores, op::v8::MatrixNms::Attributes()); + + ASSERT_EQ(nms->get_output_element_type(0), element::f32); + ASSERT_EQ(nms->get_output_element_type(1), element::i64); + ASSERT_EQ(nms->get_output_element_type(2), element::i64); + EXPECT_EQ(nms->get_output_partial_shape(0), PartialShape({Dimension::dynamic(), 6})); + EXPECT_EQ(nms->get_output_partial_shape(1), PartialShape({Dimension::dynamic(), 1})); + EXPECT_EQ(nms->get_output_partial_shape(2), PartialShape({Dimension::dynamic()})); +} diff --git a/ngraph/test/type_prop/multiclass_nms.cpp b/ngraph/test/type_prop/multiclass_nms.cpp new file mode 100644 index 00000000000..ad13d8e2dec --- /dev/null +++ b/ngraph/test/type_prop/multiclass_nms.cpp @@ -0,0 +1,281 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "gtest/gtest.h" +#include "ngraph/ngraph.hpp" +#include "util/type_prop.hpp" + +using namespace std; +using namespace ngraph; + +TEST(type_prop, multiclass_nms_incorrect_boxes_rank) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 3, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 3}); + + make_shared(boxes, scores, op::v8::MulticlassNms::Attributes()); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), "Expected a 3D tensor for the 'boxes' input"); + } +} + +TEST(type_prop, multiclass_nms_incorrect_scores_rank) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2}); + + make_shared(boxes, scores, op::v8::MulticlassNms::Attributes()); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), "Expected a 3D tensor for the 'scores' input"); + } +} + +TEST(type_prop, multiclass_nms_incorrect_scheme_num_batches) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{2, 2, 3}); + + make_shared(boxes, scores, op::v8::MulticlassNms::Attributes()); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "The first dimension of both 'boxes' and 'scores' must match"); + } +} + +TEST(type_prop, multiclass_nms_incorrect_scheme_num_boxes) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 3}); + + make_shared(boxes, scores, op::v8::MulticlassNms::Attributes()); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "'boxes' and 'scores' input shapes must match at the second and third " + "dimension respectively"); + } +} + +TEST(type_prop, multiclass_nms_incorrect_boxes_rank2) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 3}); + const auto scores = make_shared(element::f32, Shape{2, 2, 2}); + + make_shared(boxes, scores, op::v8::MulticlassNms::Attributes()); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "The third dimension of the 'boxes' must be 4"); + } +} + +TEST(type_prop, multiclass_nms_incorrect_output_type) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 2}); + op::v8::MulticlassNms::Attributes attrs; + attrs.output_type = ngraph::element::f32; + + make_shared(boxes, scores, attrs); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "Output type must be i32 or i64"); + } +} + +TEST(type_prop, multiclass_nms_incorrect_nms_topk) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 2}); + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = -2; + + make_shared(boxes, scores, attrs); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "The 'nms_top_k' must be great or equal -1"); + } +} + +TEST(type_prop, multiclass_nms_incorrect_keep_topk) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 2}); + op::v8::MulticlassNms::Attributes attrs; + attrs.keep_top_k = -2; + + make_shared(boxes, scores, attrs); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "The 'keep_top_k' must be great or equal -1"); + } +} + +TEST(type_prop, multiclass_nms_incorrect_background_class) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 2}); + op::v8::MulticlassNms::Attributes attrs; + attrs.background_class = -2; + + make_shared(boxes, scores, attrs); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "The 'background_class' must be great or equal -1"); + } +} + +TEST(type_prop, multiclass_nms_incorrect_eta) +{ + try + { + const auto boxes = make_shared(element::f32, Shape{1, 2, 4}); + const auto scores = make_shared(element::f32, Shape{1, 2, 2}); + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_eta = 2.0f; + + make_shared(boxes, scores, attrs); + } + catch (const NodeValidationFailure& error) + { + EXPECT_HAS_SUBSTRING(error.what(), + "The 'nms_eta' must be in close range [0, 1.0]"); + } +} + +TEST(type_prop, multiclass_nms_output_shape_1dim_dynamic) +{ + const auto boxes = make_shared(element::f32, Shape{5, 2, 4}); + const auto scores = make_shared(element::f32, Shape{5, 3, 2}); + + const auto nms = make_shared(boxes, scores, op::v8::MulticlassNms::Attributes()); + + ASSERT_TRUE( + nms->get_output_partial_shape(0).same_scheme(PartialShape{Dimension::dynamic(), 6})); + ASSERT_TRUE( + nms->get_output_partial_shape(1).same_scheme(PartialShape{Dimension::dynamic(), 1})); + + EXPECT_EQ(nms->get_output_shape(2), (Shape{5})); +} + +TEST(type_prop, multiclass_nms_output_shape_1dim_max_out) +{ + const auto boxes = make_shared(element::f32, Shape{2, 7, 4}); + const auto scores = make_shared(element::f32, Shape{2, 5, 7}); + + const auto nms = make_shared(boxes, scores, op::v8::MulticlassNms::Attributes()); + + ASSERT_EQ(nms->get_output_element_type(0), element::f32); + ASSERT_EQ(nms->get_output_element_type(1), element::i64); + ASSERT_EQ(nms->get_output_element_type(2), element::i64); + + // batch * class * box + EXPECT_EQ(nms->get_output_partial_shape(0), PartialShape({Dimension(0, 2 * 5 * 7), Dimension(6)})); + EXPECT_EQ(nms->get_output_partial_shape(1), PartialShape({Dimension(0, 2 * 5 * 7), 1})); + EXPECT_EQ(nms->get_output_shape(2), (Shape{2})); +} + +TEST(type_prop, multiclass_nms_output_shape_1dim_nms_topk) +{ + const auto boxes = make_shared(element::f32, Shape{2, 7, 4}); + const auto scores = make_shared(element::f32, Shape{2, 5, 7}); + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = 3; + + const auto nms = make_shared(boxes, scores, attrs); + + ASSERT_EQ(nms->get_output_element_type(0), element::f32); + ASSERT_EQ(nms->get_output_element_type(1), element::i64); + ASSERT_EQ(nms->get_output_element_type(2), element::i64); + // batch * class * min(nms_topk, box) + EXPECT_EQ(nms->get_output_partial_shape(0), PartialShape({Dimension(0, 2 * 5 * 3), Dimension(6)})); + EXPECT_EQ(nms->get_output_partial_shape(1), PartialShape({Dimension(0, 2 * 5 * 3), 1})); + EXPECT_EQ(nms->get_output_shape(2), (Shape{2})); +} + +TEST(type_prop, multiclass_nms_output_shape_1dim_keep_topk) +{ + const auto boxes = make_shared(element::f32, Shape{2, 7, 4}); + const auto scores = make_shared(element::f32, Shape{2, 5, 7}); + op::v8::MulticlassNms::Attributes attrs; + attrs.nms_top_k = 3; + attrs.keep_top_k = 8; + + const auto nms = make_shared(boxes, scores, attrs); + + ASSERT_EQ(nms->get_output_element_type(0), element::f32); + ASSERT_EQ(nms->get_output_element_type(1), element::i64); + ASSERT_EQ(nms->get_output_element_type(2), element::i64); + // batch * min(keep_topk, class * box)) + EXPECT_EQ(nms->get_output_partial_shape(0), PartialShape({Dimension(0, 2 * 8), Dimension(6)})); + EXPECT_EQ(nms->get_output_partial_shape(1), PartialShape({Dimension(0, 2 * 8), 1})); + EXPECT_EQ(nms->get_output_shape(2), (Shape{2})); +} + +TEST(type_prop, multiclass_nms_output_shape_i32) +{ + const auto boxes = make_shared(element::f32, Shape{2, 7, 4}); + const auto scores = make_shared(element::f32, Shape{2, 5, 7}); + op::v8::MulticlassNms::Attributes attrs; + attrs.output_type = ngraph::element::i32; + + const auto nms = make_shared(boxes, scores, attrs); + + ASSERT_EQ(nms->get_output_element_type(0), element::f32); + ASSERT_EQ(nms->get_output_element_type(1), element::i32); + ASSERT_EQ(nms->get_output_element_type(2), element::i32); + // batch * class * box + EXPECT_EQ(nms->get_output_partial_shape(0), PartialShape({Dimension(0, 2 * 5 * 7), Dimension(6)})); + EXPECT_EQ(nms->get_output_partial_shape(1), PartialShape({Dimension(0, 2 * 5 * 7), 1})); + EXPECT_EQ(nms->get_output_shape(2), (Shape{2})); +} + +TEST(type_prop, multiclass_nms_dynamic_boxes_and_scores) +{ + const auto boxes = make_shared(element::f32, PartialShape::dynamic()); + const auto scores = make_shared(element::f32, PartialShape::dynamic()); + + const auto nms = make_shared(boxes, scores, op::v8::MulticlassNms::Attributes()); + + ASSERT_EQ(nms->get_output_element_type(0), element::f32); + ASSERT_EQ(nms->get_output_element_type(1), element::i64); + ASSERT_EQ(nms->get_output_element_type(2), element::i64); + EXPECT_EQ(nms->get_output_partial_shape(0), PartialShape({Dimension::dynamic(), 6})); + EXPECT_EQ(nms->get_output_partial_shape(1), PartialShape({Dimension::dynamic(), 1})); + EXPECT_EQ(nms->get_output_partial_shape(2), PartialShape({Dimension::dynamic()})); +} diff --git a/ngraph/test/visitors/op/matrix_nms.cpp b/ngraph/test/visitors/op/matrix_nms.cpp new file mode 100644 index 00000000000..7486e7791e2 --- /dev/null +++ b/ngraph/test/visitors/op/matrix_nms.cpp @@ -0,0 +1,101 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "gtest/gtest.h" + +#include "ngraph/ngraph.hpp" +#include "ngraph/op/util/attr_types.hpp" +#include "ngraph/opsets/opset1.hpp" +#include "ngraph/opsets/opset3.hpp" +#include "ngraph/opsets/opset4.hpp" +#include "ngraph/opsets/opset5.hpp" +#include "ngraph/opsets/opset8.hpp" + +#include "util/visitor.hpp" + +using namespace std; +using namespace ngraph; +using ngraph::test::NodeBuilder; +using ngraph::test::ValueMap; + +TEST(attributes, matrix_nms_v8_op_custom_attributes) +{ + NodeBuilder::get_ops().register_factory(); + auto boxes = make_shared(element::f32, Shape{1, 1, 4}); + auto scores = make_shared(element::f32, Shape{1, 1, 1}); + + opset8::MatrixNms::Attributes attrs; + attrs.sort_result_type = opset8::MatrixNms::SortResultType::SCORE; + attrs.output_type = ngraph::element::i32; + attrs.nms_top_k = 100; + attrs.keep_top_k = 10; + attrs.sort_result_across_batch = true; + attrs.score_threshold = 0.1f; + attrs.background_class = 2; + attrs.decay_function = opset8::MatrixNms::DecayFunction::GAUSSIAN; + attrs.gaussian_sigma = 0.2f; + attrs.post_threshold = 0.3f; + attrs.normalized = false; + + auto nms = make_shared(boxes, scores, attrs); + NodeBuilder builder(nms); + auto g_nms = as_type_ptr(builder.create()); + const auto expected_attr_count = 11; + EXPECT_EQ(builder.get_value_map_size(), expected_attr_count); + + auto& g_nms_attrs = g_nms->get_attrs(); + auto& nms_attrs = nms->get_attrs(); + + EXPECT_EQ(g_nms_attrs.sort_result_type, nms_attrs.sort_result_type); + EXPECT_EQ(g_nms_attrs.output_type, nms_attrs.output_type); + EXPECT_EQ(g_nms_attrs.nms_top_k, nms_attrs.nms_top_k); + EXPECT_EQ(g_nms_attrs.keep_top_k, nms_attrs.keep_top_k); + EXPECT_EQ(g_nms_attrs.sort_result_across_batch, nms_attrs.sort_result_across_batch); + EXPECT_EQ(g_nms_attrs.score_threshold, nms_attrs.score_threshold); + EXPECT_EQ(g_nms_attrs.background_class, nms_attrs.background_class); + EXPECT_EQ(g_nms_attrs.decay_function, nms_attrs.decay_function); + EXPECT_EQ(g_nms_attrs.gaussian_sigma, nms_attrs.gaussian_sigma); + EXPECT_EQ(g_nms_attrs.post_threshold, nms_attrs.post_threshold); + EXPECT_EQ(g_nms_attrs.normalized, nms_attrs.normalized); + + EXPECT_EQ(attrs.sort_result_type, nms_attrs.sort_result_type); + EXPECT_EQ(attrs.output_type, nms_attrs.output_type); + EXPECT_EQ(attrs.nms_top_k, nms_attrs.nms_top_k); + EXPECT_EQ(attrs.keep_top_k, nms_attrs.keep_top_k); + EXPECT_EQ(attrs.sort_result_across_batch, nms_attrs.sort_result_across_batch); + EXPECT_EQ(attrs.score_threshold, nms_attrs.score_threshold); + EXPECT_EQ(attrs.background_class, nms_attrs.background_class); + EXPECT_EQ(attrs.decay_function, nms_attrs.decay_function); + EXPECT_EQ(attrs.gaussian_sigma, nms_attrs.gaussian_sigma); + EXPECT_EQ(attrs.post_threshold, nms_attrs.post_threshold); + EXPECT_EQ(attrs.normalized, nms_attrs.normalized); +} + +TEST(attributes, matrix_nms_v8_op_default_attributes) +{ + NodeBuilder::get_ops().register_factory(); + auto boxes = make_shared(element::f32, Shape{1, 1, 4}); + auto scores = make_shared(element::f32, Shape{1, 1, 1}); + + auto nms = make_shared(boxes, scores, opset8::MatrixNms::Attributes()); + NodeBuilder builder(nms); + auto g_nms = as_type_ptr(builder.create()); + const auto expected_attr_count = 11; + EXPECT_EQ(builder.get_value_map_size(), expected_attr_count); + + auto& g_nms_attrs = g_nms->get_attrs(); + auto& nms_attrs = nms->get_attrs(); + + EXPECT_EQ(g_nms_attrs.sort_result_type, nms_attrs.sort_result_type); + EXPECT_EQ(g_nms_attrs.output_type, nms_attrs.output_type); + EXPECT_EQ(g_nms_attrs.nms_top_k, nms_attrs.nms_top_k); + EXPECT_EQ(g_nms_attrs.keep_top_k, nms_attrs.keep_top_k); + EXPECT_EQ(g_nms_attrs.sort_result_across_batch, nms_attrs.sort_result_across_batch); + EXPECT_EQ(g_nms_attrs.score_threshold, nms_attrs.score_threshold); + EXPECT_EQ(g_nms_attrs.background_class, nms_attrs.background_class); + EXPECT_EQ(g_nms_attrs.decay_function, nms_attrs.decay_function); + EXPECT_EQ(g_nms_attrs.gaussian_sigma, nms_attrs.gaussian_sigma); + EXPECT_EQ(g_nms_attrs.post_threshold, nms_attrs.post_threshold); + EXPECT_EQ(g_nms_attrs.normalized, nms_attrs.normalized); +} diff --git a/ngraph/test/visitors/op/multiclass_nms.cpp b/ngraph/test/visitors/op/multiclass_nms.cpp new file mode 100644 index 00000000000..50bb9ec5e23 --- /dev/null +++ b/ngraph/test/visitors/op/multiclass_nms.cpp @@ -0,0 +1,97 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "gtest/gtest.h" + +#include "ngraph/ngraph.hpp" +#include "ngraph/op/util/attr_types.hpp" +#include "ngraph/opsets/opset1.hpp" +#include "ngraph/opsets/opset3.hpp" +#include "ngraph/opsets/opset4.hpp" +#include "ngraph/opsets/opset5.hpp" +#include "ngraph/opsets/opset8.hpp" + +#include "util/visitor.hpp" + +using namespace std; +using namespace ngraph; +using ngraph::test::NodeBuilder; +using ngraph::test::ValueMap; + +TEST(attributes, multiclass_nms_v8_op_custom_attributes) +{ + NodeBuilder::get_ops().register_factory(); + auto boxes = make_shared(element::f32, Shape{1, 1, 4}); + auto scores = make_shared(element::f32, Shape{1, 1, 1}); + + opset8::MulticlassNms::Attributes attrs; + attrs.sort_result_type = opset8::MulticlassNms::SortResultType::SCORE; + attrs.sort_result_across_batch = true; + attrs.output_type = ngraph::element::i32; + attrs.nms_top_k = 100; + attrs.keep_top_k = 10; + attrs.iou_threshold = 0.1f; + attrs.score_threshold = 0.2f; + attrs.background_class = 2; + attrs.nms_eta = 0.3f; + attrs.normalized = false; + + auto nms = make_shared(boxes, scores, attrs); + NodeBuilder builder(nms); + auto g_nms = as_type_ptr(builder.create()); + const auto expected_attr_count = 10; + EXPECT_EQ(builder.get_value_map_size(), expected_attr_count); + + auto& g_nms_attrs = g_nms->get_attrs(); + auto& nms_attrs = nms->get_attrs(); + + EXPECT_EQ(g_nms_attrs.sort_result_type, nms_attrs.sort_result_type); + EXPECT_EQ(g_nms_attrs.sort_result_across_batch, nms_attrs.sort_result_across_batch); + EXPECT_EQ(g_nms_attrs.output_type, nms_attrs.output_type); + EXPECT_EQ(g_nms_attrs.nms_top_k, nms_attrs.nms_top_k); + EXPECT_EQ(g_nms_attrs.keep_top_k, nms_attrs.keep_top_k); + EXPECT_EQ(g_nms_attrs.iou_threshold, nms_attrs.iou_threshold); + EXPECT_EQ(g_nms_attrs.score_threshold, nms_attrs.score_threshold); + EXPECT_EQ(g_nms_attrs.background_class, nms_attrs.background_class); + EXPECT_EQ(g_nms_attrs.nms_eta, nms_attrs.nms_eta); + EXPECT_EQ(g_nms_attrs.normalized, nms_attrs.normalized); + + EXPECT_EQ(attrs.sort_result_type, nms_attrs.sort_result_type); + EXPECT_EQ(attrs.sort_result_across_batch, nms_attrs.sort_result_across_batch); + EXPECT_EQ(attrs.output_type, nms_attrs.output_type); + EXPECT_EQ(attrs.nms_top_k, nms_attrs.nms_top_k); + EXPECT_EQ(attrs.keep_top_k, nms_attrs.keep_top_k); + EXPECT_EQ(attrs.iou_threshold, nms_attrs.iou_threshold); + EXPECT_EQ(attrs.score_threshold, nms_attrs.score_threshold); + EXPECT_EQ(attrs.background_class, nms_attrs.background_class); + EXPECT_EQ(attrs.nms_eta, nms_attrs.nms_eta); + EXPECT_EQ(attrs.normalized, nms_attrs.normalized); +} + +TEST(attributes, multiclass_nms_v8_op_default_attributes) +{ + NodeBuilder::get_ops().register_factory(); + auto boxes = make_shared(element::f32, Shape{1, 1, 4}); + auto scores = make_shared(element::f32, Shape{1, 1, 1}); + + auto nms = make_shared(boxes, scores, opset8::MulticlassNms::Attributes()); + NodeBuilder builder(nms); + auto g_nms = as_type_ptr(builder.create()); + const auto expected_attr_count = 10; + EXPECT_EQ(builder.get_value_map_size(), expected_attr_count); + + auto& g_nms_attrs = g_nms->get_attrs(); + auto& nms_attrs = nms->get_attrs(); + + EXPECT_EQ(g_nms_attrs.sort_result_type, nms_attrs.sort_result_type); + EXPECT_EQ(g_nms_attrs.sort_result_across_batch, nms_attrs.sort_result_across_batch); + EXPECT_EQ(g_nms_attrs.output_type, nms_attrs.output_type); + EXPECT_EQ(g_nms_attrs.nms_top_k, nms_attrs.nms_top_k); + EXPECT_EQ(g_nms_attrs.keep_top_k, nms_attrs.keep_top_k); + EXPECT_EQ(g_nms_attrs.iou_threshold, nms_attrs.iou_threshold); + EXPECT_EQ(g_nms_attrs.score_threshold, nms_attrs.score_threshold); + EXPECT_EQ(g_nms_attrs.background_class, nms_attrs.background_class); + EXPECT_EQ(g_nms_attrs.nms_eta, nms_attrs.nms_eta); + EXPECT_EQ(g_nms_attrs.normalized, nms_attrs.normalized); +}