From 1db4446e2a6ead55d066e0b4e718fa37f509353a Mon Sep 17 00:00:00 2001 From: Przemyslaw Wysocki Date: Thu, 2 Jun 2022 18:18:14 +0200 Subject: [PATCH] [MO] Extend MO for NonMaximumSupression-9 (#11576) --- .../include/nms_shape_inference.hpp | 60 +++++----- .../test_tf_NonMaxSupression.py | 104 ++++++++++++++++++ .../tools/mo/ops/non_max_suppression.py | 14 +-- .../tools/mo/utils/check_ie_bindings.py | 2 +- .../mo/ops/non_max_suppression_test.py | 82 ++++++++++++++ tools/mo/unit_tests/utils/graph.py | 13 +-- 6 files changed, 229 insertions(+), 46 deletions(-) create mode 100644 tests/layer_tests/tensorflow_tests/test_tf_NonMaxSupression.py diff --git a/src/core/shape_inference/include/nms_shape_inference.hpp b/src/core/shape_inference/include/nms_shape_inference.hpp index efc1d10162b..9e9e2cffae0 100644 --- a/src/core/shape_inference/include/nms_shape_inference.hpp +++ b/src/core/shape_inference/include/nms_shape_inference.hpp @@ -23,39 +23,41 @@ void shape_infer(const NonMaxSuppression* op, const auto& boxes_ps = input_shapes[0]; const auto& scores_ps = input_shapes[1]; - NODE_VALIDATION_CHECK(op, - boxes_ps.rank().is_static() && boxes_ps.rank().get_length() == 3, - "Expected a 3D tensor for the 'boxes' input. Got: ", - boxes_ps); + if (!boxes_ps.is_dynamic() && !scores_ps.is_dynamic()) { + NODE_VALIDATION_CHECK(op, + boxes_ps.rank().is_static() && boxes_ps.rank().get_length() == 3, + "Expected a 3D tensor for the 'boxes' input. Got: ", + boxes_ps); - NODE_VALIDATION_CHECK(op, - scores_ps.rank().is_static() && scores_ps.rank().get_length() == 3, - "Expected a 3D tensor for the 'scores' input. Got: ", - scores_ps); + NODE_VALIDATION_CHECK(op, + scores_ps.rank().is_static() && scores_ps.rank().get_length() == 3, + "Expected a 3D tensor for the 'scores' input. Got: ", + scores_ps); - const auto num_batches_boxes = boxes_ps[0]; - const auto num_batches_scores = scores_ps[0]; - NODE_VALIDATION_CHECK(op, - num_batches_boxes.same_scheme(num_batches_scores), - "The first dimension of both 'boxes' and 'scores' must match. Boxes: ", - num_batches_boxes, - "; Scores: ", - num_batches_scores); + const auto num_batches_boxes = boxes_ps[0]; + const auto num_batches_scores = scores_ps[0]; + NODE_VALIDATION_CHECK(op, + num_batches_boxes.same_scheme(num_batches_scores), + "The first dimension of both 'boxes' and 'scores' must match. Boxes: ", + num_batches_boxes, + "; Scores: ", + num_batches_scores); - const auto num_boxes_boxes = boxes_ps[1]; - const auto num_boxes_scores = scores_ps[2]; - NODE_VALIDATION_CHECK(op, - num_boxes_boxes.same_scheme(num_boxes_scores), - "'boxes' and 'scores' input shapes must match at the second and third " - "dimension respectively. Boxes: ", - num_boxes_boxes, - "; Scores: ", - num_boxes_scores); + const auto num_boxes_boxes = boxes_ps[1]; + const auto num_boxes_scores = scores_ps[2]; + NODE_VALIDATION_CHECK(op, + num_boxes_boxes.same_scheme(num_boxes_scores), + "'boxes' and 'scores' input shapes must match at the second and third " + "dimension respectively. Boxes: ", + num_boxes_boxes, + "; Scores: ", + num_boxes_scores); - NODE_VALIDATION_CHECK(op, - boxes_ps[2].is_static() && boxes_ps[2].get_length() == 4u, - "The last dimension of the 'boxes' input must be equal to 4. Got:", - boxes_ps[2]); + NODE_VALIDATION_CHECK(op, + boxes_ps[2].is_static() && boxes_ps[2].get_length() == 4u, + "The last dimension of the 'boxes' input must be equal to 4. Got:", + boxes_ps[2]); + } // NonMaxSuppression produces triplets // that have the following format: [batch_index, class_index, box_index] diff --git a/tests/layer_tests/tensorflow_tests/test_tf_NonMaxSupression.py b/tests/layer_tests/tensorflow_tests/test_tf_NonMaxSupression.py new file mode 100644 index 00000000000..61ac7b99012 --- /dev/null +++ b/tests/layer_tests/tensorflow_tests/test_tf_NonMaxSupression.py @@ -0,0 +1,104 @@ +# Copyright (C) 2018-2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import numpy as np +import pytest +import tensorflow as tf + +from common.tf_layer_test_class import CommonTFLayerTest + + +class TestNonMaxSupression(CommonTFLayerTest): + + # overload inputs generation to suit NMS use case + def _prepare_input(self, inputs_dict): + channel = ':0' if self.api_2 else '' + input_data = {} + for input in inputs_dict.keys(): + input_data[input + channel] = np.random.uniform(low=0, high=1, + size=inputs_dict[input]).astype(np.float32) + return input_data + + def create_nms_net(self, test_params: dict, with_scores: bool = False): + + tf.compat.v1.reset_default_graph() + with tf.compat.v1.Session() as sess: + + # parametrized inputs + number_of_boxes = test_params["number_of_boxes"] + max_output_size = tf.constant(test_params["max_output_size"]) + iou_threshold = tf.constant(test_params["iou_threshold"]) + score_threshold = tf.constant(test_params["score_threshold"]) + + # inputs to be generated + boxes = tf.compat.v1.placeholder(tf.float32, [number_of_boxes, 4], "Input") + + # randomize boxes' confidence scores + np.random.seed(42) + scores = np.random.uniform(low=0.2, high=1.0, size=[number_of_boxes]) + + if with_scores: + soft_nms_sigma = tf.constant(test_params["soft_nms_sigma"]) + _ = tf.image.non_max_suppression_with_scores(boxes, scores, max_output_size, + iou_threshold, score_threshold, soft_nms_sigma, name="NMS") + else: + _ = tf.image.non_max_suppression(boxes, scores, max_output_size, + iou_threshold, score_threshold, name="NMS") + tf_net = sess.graph_def + + ref_net = None + return tf_net, ref_net + + test_params = [ + ( + { + "number_of_boxes": 50, + "max_output_size": 5, + "iou_threshold": 0.7, + "score_threshold": 0.8, + "soft_nms_sigma": 0.1 + } + ), + ( + { + "number_of_boxes": 50, + "max_output_size": 9, + "iou_threshold": 0.7, + "score_threshold": 0.7, + "soft_nms_sigma": 0.4 + } + ), + ( + { + "number_of_boxes": 50, + "max_output_size": 3, + "iou_threshold": 0.3, + "score_threshold": 0.8, + "soft_nms_sigma": 0.7 + } + ) + ] + + @pytest.mark.parametrize("test_params", test_params) + @pytest.mark.nightly + @pytest.mark.precommit + def test_NonMaxSupression(self, test_params, ie_device, precision, ir_version, temp_dir, + use_new_frontend, api_2): + if ie_device == 'GPU': + pytest.skip("Skip TF NonMaxSuppresion test on GPU") + self.api_2 = api_2 + self._test(*self.create_nms_net(test_params), ie_device, precision, + ir_version, temp_dir=temp_dir, use_new_frontend=use_new_frontend, + api_2=api_2) + + @pytest.mark.parametrize("test_params", test_params) + @pytest.mark.nightly + @pytest.mark.precommit + def test_NonMaxSupressionWithScores(self, test_params, ie_device, precision, ir_version, temp_dir, + use_new_frontend, api_2): + if ie_device == 'GPU': + pytest.skip("Skip TF NonMaxSuppresionWithScores test on GPU") + self.api_2 = api_2 + self._test(*self.create_nms_net(test_params, with_scores=True), ie_device, precision, + ir_version, temp_dir=temp_dir, use_new_frontend=use_new_frontend, + api_2=api_2) diff --git a/tools/mo/openvino/tools/mo/ops/non_max_suppression.py b/tools/mo/openvino/tools/mo/ops/non_max_suppression.py index 17ce1193a4e..95efb33bd8c 100644 --- a/tools/mo/openvino/tools/mo/ops/non_max_suppression.py +++ b/tools/mo/openvino/tools/mo/ops/non_max_suppression.py @@ -21,7 +21,7 @@ class NonMaxSuppression(Op): mandatory_props = { 'type': self.op, 'op': self.op, - 'version': 'opset5', + 'version': 'opset9', 'infer': self.infer, 'reverse_infer': self.reverse_infer, 'output_type': np.int64, @@ -36,14 +36,14 @@ class NonMaxSuppression(Op): version = self.get_opset() if version in ['opset1', 'opset3', 'opset4']: self.attrs['out_ports_count'] = 1 - elif version == 'opset5': + elif version in ['opset5', 'opset9']: self.attrs['out_ports_count'] = 3 else: raise Error('Unsupported operation opset version "{}"'.format(version)) def backend_attrs(self): version = self.get_opset() - if version in ['opset3', 'opset4', 'opset5']: + if version in ['opset3', 'opset4', 'opset5', 'opset9']: return [('sort_result_descending', lambda node: bool_to_str(node, 'sort_result_descending')), 'box_encoding', ('output_type', lambda node: np_data_type_to_destination_type(node.output_type))] @@ -57,7 +57,7 @@ class NonMaxSuppression(Op): def infer(node: Node): num_of_inputs = len(node.in_ports()) opset = node.get_opset() - max_num_of_inputs = 6 if opset == 'opset5' else 5 + max_num_of_inputs = 6 if opset in ['opset5', 'opset9'] else 5 input_msg_fmt = 'NonMaxSuppression node {} from {} must have from 2 to {} inputs' node_name = node.soft_get('name', node.id) inputs_msg = input_msg_fmt.format(node_name, opset, max_num_of_inputs) @@ -91,13 +91,13 @@ class NonMaxSuppression(Op): assert scores_shape[2] is dynamic_dimension or scores_shape[2] == num_input_boxes or scores_shape[2] is None \ or num_input_boxes is None, 'Number of boxes mismatch for operation {}'.format(node_name) - if node.get_opset() in ['opset4', 'opset5']: + if node.get_opset() in ['opset4', 'opset5', 'opset9']: max_number_of_boxes = min(num_input_boxes, max_output_boxes_per_class) * boxes_shape[0] * num_classes else: max_number_of_boxes = min(num_input_boxes, boxes_shape[0] * max_output_boxes_per_class * num_classes) node.out_port(0).data.set_shape(shape_array([max_number_of_boxes, 3])) - if opset == 'opset5': + if opset in ['opset5', 'opset9']: node.out_port(0).data.set_shape(shape_array([dynamic_dimension_value, 3])) num_of_outputs = len([port for port in node.out_ports().values() if not port.disconnected()]) if num_of_outputs >= 2 and node.has_port('out', 1): @@ -108,7 +108,7 @@ class NonMaxSuppression(Op): @staticmethod def type_infer(node): opset = node.get_opset() - if opset == 'opset5': + if opset in ['opset5', 'opset9']: node.out_port(0).set_data_type(node.output_type) if node.has_port('out', 1): node.out_port(1).set_data_type(np.float32) diff --git a/tools/mo/openvino/tools/mo/utils/check_ie_bindings.py b/tools/mo/openvino/tools/mo/utils/check_ie_bindings.py index 8701bc88f34..4e1ed61ead2 100644 --- a/tools/mo/openvino/tools/mo/utils/check_ie_bindings.py +++ b/tools/mo/openvino/tools/mo/utils/check_ie_bindings.py @@ -85,7 +85,7 @@ def import_core_modules(silent: bool, path_to_module: str): extracted_mo_release_version = v.extract_release_version(mo_version) mo_is_custom = extracted_mo_release_version == (None, None) - print("[ WARNING ] Model Optimizer and OpenVINO runtime versions do no match.") + print("[ WARNING ] Model Optimizer and OpenVINO runtime versions do not match.") print("[ WARNING ] Consider building the OpenVINO Python API from sources or reinstall OpenVINO " "(TM) toolkit using", end=" ") if mo_is_custom: diff --git a/tools/mo/unit_tests/mo/ops/non_max_suppression_test.py b/tools/mo/unit_tests/mo/ops/non_max_suppression_test.py index 162558f7794..fb46b48d2e4 100644 --- a/tools/mo/unit_tests/mo/ops/non_max_suppression_test.py +++ b/tools/mo/unit_tests/mo/ops/non_max_suppression_test.py @@ -188,3 +188,85 @@ class TestNonMaxSuppressionInfer(unittest.TestCase): self.assertTrue(nms_node.out_port(0).get_data_type() == np.int64) self.assertTrue(nms_node.out_port(1).get_data_type() == np.float32) self.assertTrue(nms_node.out_port(2).get_data_type() == np.int64) + + def test_nms_infer_i32_opset9_1_out(self): + nms_node = Node(self.graph, 'nms') + nms_node['version'] = 'opset9' + nms_node['output_type'] = np.int32 + NonMaxSuppression.infer(nms_node) + NonMaxSuppression.type_infer(nms_node) + + self.assertTrue(np.array_equal(nms_node.out_port(0).data.get_shape(), + shape_array([dynamic_dimension_value, 3]))) + self.assertTrue(nms_node.out_port(0).get_data_type() == np.int32) + + def test_nms_infer_i64_opset9_1_out(self): + nms_node = Node(self.graph, 'nms') + nms_node['version'] = 'opset9' + nms_node['output_type'] = np.int64 + NonMaxSuppression.infer(nms_node) + NonMaxSuppression.type_infer(nms_node) + + self.assertTrue(np.array_equal(nms_node.out_port(0).data.get_shape(), + shape_array([dynamic_dimension_value, 3]))) + self.assertTrue(nms_node.out_port(0).get_data_type() == np.int64) + + def test_nms_infer_i32_opset9_2_outs(self): + nms_node = Node(self.graph_nms_5_2_outs, 'nms') + nms_node['version'] = 'opset9' + nms_node['output_type'] = np.int32 + NonMaxSuppression.infer(nms_node) + NonMaxSuppression.type_infer(nms_node) + + self.assertTrue(np.array_equal(nms_node.out_port(0).data.get_shape(), + shape_array([dynamic_dimension_value, 3]))) + self.assertTrue(np.array_equal(nms_node.out_port(1).data.get_shape(), + shape_array([dynamic_dimension_value, 3]))) + self.assertTrue(nms_node.out_port(0).get_data_type() == np.int32) + self.assertTrue(nms_node.out_port(1).get_data_type() == np.float32) + + def test_nms_infer_i64_opset9_2_outs(self): + nms_node = Node(self.graph_nms_5_2_outs, 'nms') + nms_node['version'] = 'opset9' + nms_node['output_type'] = np.int64 + NonMaxSuppression.infer(nms_node) + NonMaxSuppression.type_infer(nms_node) + + self.assertTrue(np.array_equal(nms_node.out_port(0).data.get_shape(), + shape_array([dynamic_dimension_value, 3]))) + self.assertTrue(np.array_equal(nms_node.out_port(1).data.get_shape(), + shape_array([dynamic_dimension_value, 3]))) + self.assertTrue(nms_node.out_port(0).get_data_type() == np.int64) + self.assertTrue(nms_node.out_port(1).get_data_type() == np.float32) + + def test_nms_infer_i32_opset9_3_outs(self): + nms_node = Node(self.graph_nms_5_3_outs, 'nms') + nms_node['version'] = 'opset9' + nms_node['output_type'] = np.int32 + NonMaxSuppression.infer(nms_node) + NonMaxSuppression.type_infer(nms_node) + + self.assertTrue(np.array_equal(nms_node.out_port(0).data.get_shape(), + shape_array([dynamic_dimension_value, 3]))) + self.assertTrue(np.array_equal(nms_node.out_port(1).data.get_shape(), + shape_array([dynamic_dimension_value, 3]))) + self.assertTrue(np.array_equal(nms_node.out_port(2).data.get_shape(), [1])) + self.assertTrue(nms_node.out_port(0).get_data_type() == np.int32) + self.assertTrue(nms_node.out_port(1).get_data_type() == np.float32) + self.assertTrue(nms_node.out_port(2).get_data_type() == np.int64) + + def test_nms_infer_i64_opset9_3_outs(self): + nms_node = Node(self.graph_nms_5_3_outs, 'nms') + nms_node['version'] = 'opset9' + nms_node['output_type'] = np.int64 + NonMaxSuppression.infer(nms_node) + NonMaxSuppression.type_infer(nms_node) + + self.assertTrue(np.array_equal(nms_node.out_port(0).data.get_shape(), + shape_array([dynamic_dimension_value, 3]))) + self.assertTrue(np.array_equal(nms_node.out_port(1).data.get_shape(), + shape_array([dynamic_dimension_value, 3]))) + self.assertTrue(np.array_equal(nms_node.out_port(2).data.get_shape(), [1])) + self.assertTrue(nms_node.out_port(0).get_data_type() == np.int64) + self.assertTrue(nms_node.out_port(1).get_data_type() == np.float32) + self.assertTrue(nms_node.out_port(2).get_data_type() == np.int64) diff --git a/tools/mo/unit_tests/utils/graph.py b/tools/mo/unit_tests/utils/graph.py index aab769c724e..1ed57b01dd6 100644 --- a/tools/mo/unit_tests/utils/graph.py +++ b/tools/mo/unit_tests/utils/graph.py @@ -157,10 +157,8 @@ def build_graph(nodes_attrs: dict, edges: list, update_attributes: dict = None, # filter nodes to keep only ones with edges connected filtered_nodes = {} for item in edges: - if len(item) == 2: # TODO: is there any better way in python to do that? - node1, node2 = item - else: - node1, node2, _ = item + node1, node2, *_ = item + filtered_nodes[node1] = nodes_attrs[node1] filtered_nodes[node2] = nodes_attrs[node2] nodes_attrs = filtered_nodes @@ -172,11 +170,8 @@ def build_graph(nodes_attrs: dict, edges: list, update_attributes: dict = None, # connect nodes with edges for item in edges: - if len(item) == 2: # TODO: is there any better way in python to do that? - node_1, node_2 = item - edge_attrs = {} - else: - node_1, node_2, edge_attrs = item + node_1, node_2, *edge_attrs_list = item + edge_attrs = dict(edge_attrs_list[0]) if edge_attrs_list else {} common_attrs = {'in': len(graph.in_edges(node_2)), 'out': len(graph.out_edges(node_1)),