[MO] Extend MO for NonMaximumSupression-9 (#11576)

src/core/shape_inference/include/nms_shape_inference.hpp

@@ -23,39 +23,41 @@ void shape_infer(const NonMaxSuppression* op,
     const auto& boxes_ps = input_shapes[0];
     const auto& scores_ps = input_shapes[1];
 
-    NODE_VALIDATION_CHECK(op,
-                          boxes_ps.rank().is_static() && boxes_ps.rank().get_length() == 3,
-                          "Expected a 3D tensor for the 'boxes' input. Got: ",
-                          boxes_ps);
+    if (!boxes_ps.is_dynamic() && !scores_ps.is_dynamic()) {
+        NODE_VALIDATION_CHECK(op,
+                            boxes_ps.rank().is_static() && boxes_ps.rank().get_length() == 3,
+                            "Expected a 3D tensor for the 'boxes' input. Got: ",
+                            boxes_ps);
 
-    NODE_VALIDATION_CHECK(op,
-                          scores_ps.rank().is_static() && scores_ps.rank().get_length() == 3,
-                          "Expected a 3D tensor for the 'scores' input. Got: ",
-                          scores_ps);
+        NODE_VALIDATION_CHECK(op,
+                            scores_ps.rank().is_static() && scores_ps.rank().get_length() == 3,
+                            "Expected a 3D tensor for the 'scores' input. Got: ",
+                            scores_ps);
 
-    const auto num_batches_boxes = boxes_ps[0];
-    const auto num_batches_scores = scores_ps[0];
-    NODE_VALIDATION_CHECK(op,
-                          num_batches_boxes.same_scheme(num_batches_scores),
-                          "The first dimension of both 'boxes' and 'scores' must match. Boxes: ",
-                          num_batches_boxes,
-                          "; Scores: ",
-                          num_batches_scores);
+        const auto num_batches_boxes = boxes_ps[0];
+        const auto num_batches_scores = scores_ps[0];
+        NODE_VALIDATION_CHECK(op,
+                            num_batches_boxes.same_scheme(num_batches_scores),
+                            "The first dimension of both 'boxes' and 'scores' must match. Boxes: ",
+                            num_batches_boxes,
+                            "; Scores: ",
+                            num_batches_scores);
 
-    const auto num_boxes_boxes = boxes_ps[1];
-    const auto num_boxes_scores = scores_ps[2];
-    NODE_VALIDATION_CHECK(op,
-                          num_boxes_boxes.same_scheme(num_boxes_scores),
-                          "'boxes' and 'scores' input shapes must match at the second and third "
-                          "dimension respectively. Boxes: ",
-                          num_boxes_boxes,
-                          "; Scores: ",
-                          num_boxes_scores);
+        const auto num_boxes_boxes = boxes_ps[1];
+        const auto num_boxes_scores = scores_ps[2];
+        NODE_VALIDATION_CHECK(op,
+                            num_boxes_boxes.same_scheme(num_boxes_scores),
+                            "'boxes' and 'scores' input shapes must match at the second and third "
+                            "dimension respectively. Boxes: ",
+                            num_boxes_boxes,
+                            "; Scores: ",
+                            num_boxes_scores);
 
-    NODE_VALIDATION_CHECK(op,
-                          boxes_ps[2].is_static() && boxes_ps[2].get_length() == 4u,
-                          "The last dimension of the 'boxes' input must be equal to 4. Got:",
-                          boxes_ps[2]);
+        NODE_VALIDATION_CHECK(op,
+                            boxes_ps[2].is_static() && boxes_ps[2].get_length() == 4u,
+                            "The last dimension of the 'boxes' input must be equal to 4. Got:",
+                            boxes_ps[2]);
+    }
 
     // NonMaxSuppression produces triplets
     // that have the following format: [batch_index, class_index, box_index]

tests/layer_tests/tensorflow_tests/test_tf_NonMaxSupression.py

@@ -0,0 +1,104 @@
+# Copyright (C) 2018-2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import numpy as np
+import pytest
+import tensorflow as tf
+
+from common.tf_layer_test_class import CommonTFLayerTest
+
+
+class TestNonMaxSupression(CommonTFLayerTest):
+
+    # overload inputs generation to suit NMS use case
+    def _prepare_input(self, inputs_dict):
+        channel = ':0' if self.api_2 else ''
+        input_data = {}
+        for input in inputs_dict.keys():
+            input_data[input + channel] = np.random.uniform(low=0, high=1,
+                                                            size=inputs_dict[input]).astype(np.float32)
+        return input_data
+
+    def create_nms_net(self, test_params: dict, with_scores: bool = False):
+
+        tf.compat.v1.reset_default_graph()
+        with tf.compat.v1.Session() as sess:
+
+            # parametrized inputs
+            number_of_boxes = test_params["number_of_boxes"]
+            max_output_size = tf.constant(test_params["max_output_size"])
+            iou_threshold = tf.constant(test_params["iou_threshold"])
+            score_threshold = tf.constant(test_params["score_threshold"])
+
+            # inputs to be generated
+            boxes = tf.compat.v1.placeholder(tf.float32, [number_of_boxes, 4], "Input")
+
+            # randomize boxes' confidence scores
+            np.random.seed(42)
+            scores = np.random.uniform(low=0.2, high=1.0, size=[number_of_boxes])
+
+            if with_scores:
+                soft_nms_sigma = tf.constant(test_params["soft_nms_sigma"])
+                _ = tf.image.non_max_suppression_with_scores(boxes, scores, max_output_size,
+                                             iou_threshold, score_threshold, soft_nms_sigma, name="NMS")
+            else:
+                _ = tf.image.non_max_suppression(boxes, scores, max_output_size,
+                                                iou_threshold, score_threshold, name="NMS")
+            tf_net = sess.graph_def
+
+        ref_net = None
+        return tf_net, ref_net
+
+    test_params = [
+            (
+                {
+                    "number_of_boxes": 50,
+                    "max_output_size": 5,
+                    "iou_threshold": 0.7,
+                    "score_threshold": 0.8,
+                    "soft_nms_sigma": 0.1
+                }
+            ),
+            (
+                {
+                    "number_of_boxes": 50,
+                    "max_output_size": 9,
+                    "iou_threshold": 0.7,
+                    "score_threshold": 0.7,
+                    "soft_nms_sigma": 0.4
+                }
+            ),
+            (
+                {
+                    "number_of_boxes": 50,
+                    "max_output_size": 3,
+                    "iou_threshold": 0.3,
+                    "score_threshold": 0.8,
+                    "soft_nms_sigma": 0.7
+                }
+            )
+        ]
+
+    @pytest.mark.parametrize("test_params", test_params)
+    @pytest.mark.nightly
+    @pytest.mark.precommit
+    def test_NonMaxSupression(self, test_params, ie_device, precision, ir_version, temp_dir,
+                              use_new_frontend, api_2):
+        if ie_device == 'GPU':
+            pytest.skip("Skip TF NonMaxSuppresion test on GPU")
+        self.api_2 = api_2
+        self._test(*self.create_nms_net(test_params), ie_device, precision,
+                   ir_version, temp_dir=temp_dir, use_new_frontend=use_new_frontend,
+                   api_2=api_2)
+
+    @pytest.mark.parametrize("test_params", test_params)
+    @pytest.mark.nightly
+    @pytest.mark.precommit
+    def test_NonMaxSupressionWithScores(self, test_params, ie_device, precision, ir_version, temp_dir,
+                                        use_new_frontend, api_2):
+        if ie_device == 'GPU':
+            pytest.skip("Skip TF NonMaxSuppresionWithScores test on GPU")
+        self.api_2 = api_2
+        self._test(*self.create_nms_net(test_params, with_scores=True), ie_device, precision,
+                   ir_version, temp_dir=temp_dir, use_new_frontend=use_new_frontend,
+                   api_2=api_2)

tools/mo/openvino/tools/mo/ops/non_max_suppression.py

@@ -21,7 +21,7 @@ class NonMaxSuppression(Op):
         mandatory_props = {
             'type': self.op,
             'op': self.op,
-            'version': 'opset5',
+            'version': 'opset9',
             'infer': self.infer,
             'reverse_infer': self.reverse_infer,
             'output_type': np.int64,
@@ -36,14 +36,14 @@ class NonMaxSuppression(Op):
         version = self.get_opset()
         if version in ['opset1', 'opset3', 'opset4']:
             self.attrs['out_ports_count'] = 1
-        elif version == 'opset5':
+        elif version in ['opset5', 'opset9']:
             self.attrs['out_ports_count'] = 3
         else:
             raise Error('Unsupported operation opset version "{}"'.format(version))
 
     def backend_attrs(self):
         version = self.get_opset()
-        if version in ['opset3', 'opset4', 'opset5']:
+        if version in ['opset3', 'opset4', 'opset5', 'opset9']:
             return [('sort_result_descending', lambda node: bool_to_str(node, 'sort_result_descending')),
                     'box_encoding',
                     ('output_type', lambda node: np_data_type_to_destination_type(node.output_type))]
@@ -57,7 +57,7 @@ class NonMaxSuppression(Op):
     def infer(node: Node):
         num_of_inputs = len(node.in_ports())
         opset = node.get_opset()
-        max_num_of_inputs = 6 if opset == 'opset5' else 5
+        max_num_of_inputs = 6 if opset in ['opset5', 'opset9'] else 5
         input_msg_fmt = 'NonMaxSuppression node {} from {} must have from 2 to {} inputs'
         node_name = node.soft_get('name', node.id)
         inputs_msg = input_msg_fmt.format(node_name, opset, max_num_of_inputs)
@@ -91,13 +91,13 @@ class NonMaxSuppression(Op):
         assert scores_shape[2] is dynamic_dimension or scores_shape[2] == num_input_boxes or scores_shape[2] is None \
                or num_input_boxes is None, 'Number of boxes mismatch for operation {}'.format(node_name)
 
-        if node.get_opset() in ['opset4', 'opset5']:
+        if node.get_opset() in ['opset4', 'opset5', 'opset9']:
             max_number_of_boxes = min(num_input_boxes, max_output_boxes_per_class) * boxes_shape[0] * num_classes
         else:
             max_number_of_boxes = min(num_input_boxes, boxes_shape[0] * max_output_boxes_per_class * num_classes)
         node.out_port(0).data.set_shape(shape_array([max_number_of_boxes, 3]))
 
-        if opset == 'opset5':
+        if opset in ['opset5', 'opset9']:
             node.out_port(0).data.set_shape(shape_array([dynamic_dimension_value, 3]))
             num_of_outputs = len([port for port in node.out_ports().values() if not port.disconnected()])
             if num_of_outputs >= 2 and node.has_port('out', 1):
@@ -108,7 +108,7 @@ class NonMaxSuppression(Op):
     @staticmethod
     def type_infer(node):
         opset = node.get_opset()
-        if opset == 'opset5':
+        if opset in ['opset5', 'opset9']:
             node.out_port(0).set_data_type(node.output_type)
             if node.has_port('out', 1):
                 node.out_port(1).set_data_type(np.float32)

tools/mo/openvino/tools/mo/utils/check_ie_bindings.py

@@ -85,7 +85,7 @@ def import_core_modules(silent: bool, path_to_module: str):
             extracted_mo_release_version = v.extract_release_version(mo_version)
             mo_is_custom = extracted_mo_release_version == (None, None)
 
-            print("[ WARNING ] Model Optimizer and OpenVINO runtime versions do no match.")
+            print("[ WARNING ] Model Optimizer and OpenVINO runtime versions do not match.")
             print("[ WARNING ] Consider building the OpenVINO Python API from sources or reinstall OpenVINO "
                   "(TM) toolkit using", end=" ")
             if mo_is_custom:

tools/mo/unit_tests/mo/ops/non_max_suppression_test.py

@@ -188,3 +188,85 @@ class TestNonMaxSuppressionInfer(unittest.TestCase):
         self.assertTrue(nms_node.out_port(0).get_data_type() == np.int64)
         self.assertTrue(nms_node.out_port(1).get_data_type() == np.float32)
         self.assertTrue(nms_node.out_port(2).get_data_type() == np.int64)
+
+    def test_nms_infer_i32_opset9_1_out(self):
+        nms_node = Node(self.graph, 'nms')
+        nms_node['version'] = 'opset9'
+        nms_node['output_type'] = np.int32
+        NonMaxSuppression.infer(nms_node)
+        NonMaxSuppression.type_infer(nms_node)
+
+        self.assertTrue(np.array_equal(nms_node.out_port(0).data.get_shape(),
+                                       shape_array([dynamic_dimension_value, 3])))
+        self.assertTrue(nms_node.out_port(0).get_data_type() == np.int32)
+
+    def test_nms_infer_i64_opset9_1_out(self):
+        nms_node = Node(self.graph, 'nms')
+        nms_node['version'] = 'opset9'
+        nms_node['output_type'] = np.int64
+        NonMaxSuppression.infer(nms_node)
+        NonMaxSuppression.type_infer(nms_node)
+
+        self.assertTrue(np.array_equal(nms_node.out_port(0).data.get_shape(),
+                                       shape_array([dynamic_dimension_value, 3])))
+        self.assertTrue(nms_node.out_port(0).get_data_type() == np.int64)
+
+    def test_nms_infer_i32_opset9_2_outs(self):
+        nms_node = Node(self.graph_nms_5_2_outs, 'nms')
+        nms_node['version'] = 'opset9'
+        nms_node['output_type'] = np.int32
+        NonMaxSuppression.infer(nms_node)
+        NonMaxSuppression.type_infer(nms_node)
+
+        self.assertTrue(np.array_equal(nms_node.out_port(0).data.get_shape(),
+                                       shape_array([dynamic_dimension_value, 3])))
+        self.assertTrue(np.array_equal(nms_node.out_port(1).data.get_shape(),
+                                       shape_array([dynamic_dimension_value, 3])))
+        self.assertTrue(nms_node.out_port(0).get_data_type() == np.int32)
+        self.assertTrue(nms_node.out_port(1).get_data_type() == np.float32)
+
+    def test_nms_infer_i64_opset9_2_outs(self):
+        nms_node = Node(self.graph_nms_5_2_outs, 'nms')
+        nms_node['version'] = 'opset9'
+        nms_node['output_type'] = np.int64
+        NonMaxSuppression.infer(nms_node)
+        NonMaxSuppression.type_infer(nms_node)
+
+        self.assertTrue(np.array_equal(nms_node.out_port(0).data.get_shape(),
+                                       shape_array([dynamic_dimension_value, 3])))
+        self.assertTrue(np.array_equal(nms_node.out_port(1).data.get_shape(),
+                                       shape_array([dynamic_dimension_value, 3])))
+        self.assertTrue(nms_node.out_port(0).get_data_type() == np.int64)
+        self.assertTrue(nms_node.out_port(1).get_data_type() == np.float32)
+
+    def test_nms_infer_i32_opset9_3_outs(self):
+        nms_node = Node(self.graph_nms_5_3_outs, 'nms')
+        nms_node['version'] = 'opset9'
+        nms_node['output_type'] = np.int32
+        NonMaxSuppression.infer(nms_node)
+        NonMaxSuppression.type_infer(nms_node)
+
+        self.assertTrue(np.array_equal(nms_node.out_port(0).data.get_shape(),
+                                       shape_array([dynamic_dimension_value, 3])))
+        self.assertTrue(np.array_equal(nms_node.out_port(1).data.get_shape(),
+                                       shape_array([dynamic_dimension_value, 3])))
+        self.assertTrue(np.array_equal(nms_node.out_port(2).data.get_shape(), [1]))
+        self.assertTrue(nms_node.out_port(0).get_data_type() == np.int32)
+        self.assertTrue(nms_node.out_port(1).get_data_type() == np.float32)
+        self.assertTrue(nms_node.out_port(2).get_data_type() == np.int64)
+
+    def test_nms_infer_i64_opset9_3_outs(self):
+        nms_node = Node(self.graph_nms_5_3_outs, 'nms')
+        nms_node['version'] = 'opset9'
+        nms_node['output_type'] = np.int64
+        NonMaxSuppression.infer(nms_node)
+        NonMaxSuppression.type_infer(nms_node)
+
+        self.assertTrue(np.array_equal(nms_node.out_port(0).data.get_shape(),
+                                       shape_array([dynamic_dimension_value, 3])))
+        self.assertTrue(np.array_equal(nms_node.out_port(1).data.get_shape(),
+                                       shape_array([dynamic_dimension_value, 3])))
+        self.assertTrue(np.array_equal(nms_node.out_port(2).data.get_shape(), [1]))
+        self.assertTrue(nms_node.out_port(0).get_data_type() == np.int64)
+        self.assertTrue(nms_node.out_port(1).get_data_type() == np.float32)
+        self.assertTrue(nms_node.out_port(2).get_data_type() == np.int64)

tools/mo/unit_tests/utils/graph.py

@@ -157,10 +157,8 @@ def build_graph(nodes_attrs: dict, edges: list, update_attributes: dict = None,
         # filter nodes to keep only ones with edges connected
         filtered_nodes = {}
         for item in edges:
-            if len(item) == 2:  # TODO: is there any better way in python to do that?
-                node1, node2 = item
-            else:
-                node1, node2, _ = item
+            node1, node2, *_ = item
+
             filtered_nodes[node1] = nodes_attrs[node1]
             filtered_nodes[node2] = nodes_attrs[node2]
         nodes_attrs = filtered_nodes
@@ -172,11 +170,8 @@ def build_graph(nodes_attrs: dict, edges: list, update_attributes: dict = None,
 
     # connect nodes with edges
     for item in edges:
-        if len(item) == 2:  # TODO: is there any better way in python to do that?
-            node_1, node_2 = item
-            edge_attrs = {}
-        else:
-            node_1, node_2, edge_attrs = item
+        node_1, node_2, *edge_attrs_list = item
+        edge_attrs = dict(edge_attrs_list[0]) if edge_attrs_list else {}
 
         common_attrs = {'in': len(graph.in_edges(node_2)),
                         'out': len(graph.out_edges(node_1)),