Move TF OD API docs to code + several fixes for TF OD API models conversion (#7377)

* Refactored code, updated comments and documentation related to TF OD API models pre-processing.

* Improved MO messages related to pre-processor block removal during conversion of the TD OD API models. Remove mean/scale if padding is used and mean/scale is applied before resize

* Updated TF OD API transformation and documentation for SSD models

* Updated comments and documentation for the ObjectDetectionAPIMaskRCNNSigmoidReplacement transformation

* Updated comments and documentation for the ObjectDetectionAPIMaskRCNNROIPoolingSecondReplacement transformation

* Updated comments and documentation for the ObjectDetectionAPIPSROIPoolingReplacement transformation

* Updated comments and documentation for the ObjectDetectionAPIProposalReplacement transformation

* Updated comments and documentation for the ObjectDetectionAPIDetectionOutputReplacement transformation

* Minor code style fixes

* Fixed unit tests for ObjectDetectionAPIPreprocessor2Replacement transformation

* Improved unit test for pipeline.config parser. Fixed very long bug with incorrect test data for the PipelineConfig parser class

* Code style fixes

* Get rid of "coordinates_swap_method" parameter in the JSON configuration file for TF OD API models

* Code style fixes and minor refactoring

* Simplied code related to swapping Proposal coordinates

* Removed incorrectly removed code

* Fixed code review comments about the code comments
This commit is contained in:
Evgeny Lazarev
2021-09-08 10:03:01 +03:00
committed by GitHub
parent 4d377901bf
commit 4547818fb1
25 changed files with 708 additions and 1489 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -60,8 +60,7 @@
{
"custom_attributes": {
"clip_before_nms": true,
"clip_after_nms": false,
"coordinates_swap_method": "swap_weights"
"clip_after_nms": false
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"inputs": [

View File

@@ -61,8 +61,7 @@
{
"custom_attributes": {
"clip_before_nms": false,
"clip_after_nms": true,
"coordinates_swap_method": "swap_weights"
"clip_after_nms": true
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"inputs": [

View File

@@ -61,8 +61,7 @@
{
"custom_attributes": {
"clip_before_nms": false,
"clip_after_nms": true,
"coordinates_swap_method": "swap_weights"
"clip_after_nms": true
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"inputs": [

View File

@@ -61,8 +61,7 @@
{
"custom_attributes": {
"clip_before_nms": false,
"clip_after_nms": true,
"coordinates_swap_method": "swap_weights"
"clip_after_nms": true
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"inputs": [

View File

@@ -61,8 +61,7 @@
{
"custom_attributes": {
"clip_before_nms": false,
"clip_after_nms": true,
"coordinates_swap_method": "swap_weights"
"clip_after_nms": true
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"inputs": [

View File

@@ -61,8 +61,7 @@
{
"custom_attributes": {
"clip_before_nms": true,
"clip_after_nms": false,
"coordinates_swap_method": "swap_weights"
"clip_after_nms": false
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"inputs": [

View File

@@ -38,8 +38,7 @@
"custom_attributes": {
"clip_before_nms": false,
"clip_after_nms": true,
"background_label_id": 0,
"coordinates_swap_method": "swap_weights"
"background_label_id": 0
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"inputs": [

View File

@@ -38,8 +38,7 @@
"custom_attributes": {
"clip_before_nms": false,
"clip_after_nms": true,
"background_label_id": 0,
"coordinates_swap_method": "swap_weights"
"background_label_id": 0
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"inputs": [

View File

@@ -61,8 +61,7 @@
{
"custom_attributes": {
"clip_before_nms": true,
"clip_after_nms": false,
"coordinates_swap_method": "swap_weights"
"clip_after_nms": false
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"include_inputs_to_sub_graph": true,

View File

@@ -61,8 +61,7 @@
{
"custom_attributes": {
"clip_before_nms": false,
"clip_after_nms": true,
"coordinates_swap_method": "swap_weights"
"clip_after_nms": true
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"include_inputs_to_sub_graph": true,

View File

@@ -61,8 +61,7 @@
{
"custom_attributes": {
"clip_before_nms": false,
"clip_after_nms": true,
"coordinates_swap_method": "swap_weights"
"clip_after_nms": true
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"include_inputs_to_sub_graph": true,

View File

@@ -61,8 +61,7 @@
{
"custom_attributes": {
"clip_before_nms": false,
"clip_after_nms": true,
"coordinates_swap_method": "swap_weights"
"clip_after_nms": true
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"include_inputs_to_sub_graph": true,

View File

@@ -61,8 +61,7 @@
{
"custom_attributes": {
"clip_before_nms": false,
"clip_after_nms": true,
"coordinates_swap_method": "swap_weights"
"clip_after_nms": true
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"include_inputs_to_sub_graph": true,

View File

@@ -61,8 +61,7 @@
{
"custom_attributes": {
"clip_before_nms": true,
"clip_after_nms": false,
"coordinates_swap_method": "swap_weights"
"clip_after_nms": false
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"include_inputs_to_sub_graph": true,

View File

@@ -38,8 +38,7 @@
"custom_attributes": {
"clip_before_nms": false,
"clip_after_nms": true,
"background_label_id": 0,
"coordinates_swap_method": "swap_weights"
"background_label_id": 0
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"include_inputs_to_sub_graph": true,

View File

@@ -38,8 +38,7 @@
"custom_attributes": {
"clip_before_nms": false,
"clip_after_nms": true,
"background_label_id": 0,
"coordinates_swap_method": "swap_weights"
"background_label_id": 0
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"include_inputs_to_sub_graph": true,

View File

@@ -62,7 +62,6 @@
"custom_attributes": {
"clip_before_nms": true,
"clip_after_nms": false,
"coordinates_swap_method": "add_convolution",
"swap_proposals": true
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",

View File

@@ -60,8 +60,7 @@
{
"custom_attributes": {
"clip_before_nms": false,
"clip_after_nms": true,
"coordinates_swap_method": "add_convolution"
"clip_after_nms": true
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"inputs": [

View File

@@ -60,8 +60,7 @@
{
"custom_attributes": {
"clip_before_nms": false,
"clip_after_nms": true,
"coordinates_swap_method": "add_convolution"
"clip_after_nms": true
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"inputs": [

View File

@@ -60,8 +60,7 @@
{
"custom_attributes": {
"clip_before_nms": false,
"clip_after_nms": true,
"coordinates_swap_method": "add_convolution"
"clip_after_nms": true
},
"id": "ObjectDetectionAPIDetectionOutputReplacement",
"inputs": [

View File

@@ -1,8 +1,6 @@
# Copyright (C) 2018-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import collections
import logging as log
from typing import Dict
import numpy as np
@@ -52,43 +50,6 @@ def create_op_with_const_inputs(graph: Graph, op: callable, port_value_dict: Dic
return node
def mark_squeeze_reshape_concat_before_detection_output(start_nodes: list):
"""
The function looks for Reshape, Concat and Squeeze ops after the 'start_nodes' with 4D output and marks them with
proper attributes to infer them in original NHWC layout. This is a case of the TensorFlow Object Detection API
models for the SSD heads output which produces 4D tensor with bounding box deltas.
:param start_nodes: list of nodes to start search from.
:return: None
"""
q = collections.deque()
q.extend(start_nodes)
while len(q) != 0:
cur_node = q.popleft()
if cur_node.has_valid('type'):
if cur_node.soft_get('type') == 'DetectionOutput': # do not go beyond the DetectionOutput node
continue
# the input to Reshape comes from Convolution so it will be converted from NCHW to NHWC layout in the
# InsertLayoutPropagationTransposes transformation. But the output should be kept in the original layout
if cur_node.soft_get('type') == 'Reshape' and len(cur_node.out_port(0).data.get_shape()) == 4:
mark_output_as_in_correct_layout(cur_node, 0)
# Concat should be inferred in the original layout so the input with concatenation axis should not be
# updated from NHWC to NCHW layout
if cur_node.soft_get('type') == 'Concat' and len(cur_node.out_port(0).data.get_shape()) == 4:
cur_node.in_port(1).__setattr__('input_permutation', None)
cur_node['nchw_layout'] = True
cur_node.out_node(0)['nchw_layout'] = True
# Squeeze should be inferred in the original layout so the input with squeeze axis should not be updated
# from NHWC to NCHW layout. The input is marked as in correct layout to prevent from inserting Transpose
# from NHWC to NCHW.
if cur_node.soft_get('type') == 'Squeeze' and len(cur_node.in_port(0).data.get_shape()) == 4:
cur_node.in_port(1).__setattr__('input_permutation', None)
mark_input_as_in_correct_layout(cur_node, 0)
[q.append(port.node) for port in cur_node.out_port(0).get_destinations()]
def add_convolution_to_swap_xy_coordinates(graph: Graph, input_node: Node, coordinates_size: int):
"""
The function add convolution node after the node 'input_node' to swap xy coordinates of the boxes produced

View File

@@ -2,9 +2,10 @@
# SPDX-License-Identifier: Apache-2.0
import unittest
from argparse import Namespace
from unittest.mock import patch
from generator import generator, generate
from unittest.mock import patch
from extensions.front.tf.ObjectDetectionAPI import calculate_shape_keeping_aspect_ratio, \
calculate_placeholder_spatial_shape, ObjectDetectionAPIPreprocessor2Replacement
@@ -14,6 +15,7 @@ from mo.graph.graph import Graph
from mo.utils.custom_replacement_config import CustomReplacementDescriptor
from mo.utils.error import Error
from mo.utils.ir_engine.compare_graphs import compare_graphs
from unit_tests.mo.utils.pipeline_config_test import file_content
from unit_tests.utils.graph import const, regular_op, result, build_graph, connect_front
@@ -32,28 +34,24 @@ class TestCalculateShape(unittest.TestCase):
min_size = 600
max_size = 1024
@generate(*[(100, 300, 341, 1024, False),
(100, 600, 171, 1024, False),
(100, 3000, 34, 1024, False),
(300, 300, 600, 600, False),
(300, 400, 600, 800, False),
(300, 600, 512, 1024, False),
(1000, 2500, 410, 1024, False),
(1800, 2000, 600, 667, False),
(300, 100, 1024, 341, False),
(600, 100, 1024, 171, False),
(3000, 100, 1024, 34, False),
(400, 300, 800, 600, False),
(600, 300, 1024, 512, False),
(2500, 1000, 1024, 410, False),
(2000, 1800, 667, 600, False),
(300, 300, 1024, 1024, True),
(900, 300, 1024, 1024, True),
(1300, 900, 1024, 1024, True),
(1025, 1025, 1024, 1024, True),
@generate(*[(100, 300, 341, 1024),
(100, 600, 171, 1024),
(100, 3000, 34, 1024),
(300, 300, 600, 600),
(300, 400, 600, 800),
(300, 600, 512, 1024),
(1000, 2500, 410, 1024),
(1800, 2000, 600, 667),
(300, 100, 1024, 341),
(600, 100, 1024, 171),
(3000, 100, 1024, 34),
(400, 300, 800, 600),
(600, 300, 1024, 512),
(2500, 1000, 1024, 410),
(2000, 1800, 667, 600),
])
def test_calculate_shape(self, h, w, th, tw, pad):
self.assertTupleEqual(calculate_shape_keeping_aspect_ratio(h, w, self.min_size, self.max_size, pad), (th, tw))
def test_calculate_shape(self, h, w, th, tw):
self.assertTupleEqual(calculate_shape_keeping_aspect_ratio(h, w, self.min_size, self.max_size), (th, tw))
class TestCalculatePlaceholderSpatialShape(unittest.TestCase):
@@ -147,9 +145,10 @@ class TestObjectDetectionAPIPreprocessor2Replacement(unittest.TestCase):
ref_graph.stage = 'front'
return ref_graph
def test_case_1(self, update_parameter_shape_mock):
def test_case_1_pad_to_max_dim(self, update_parameter_shape_mock):
# test for case #1 described in the ObjectDetectionAPIPreprocessor2Replacement
update_parameter_shape_mock.return_value = None
# sub/mul should be removed because they are applied before prep-processing and pad_to_max_dimension is True
update_parameter_shape_mock.return_value = (None, None)
edges = [*connect_front('input', '0:mul'),
*connect_front('mul_const', '1:mul'),
*connect_front('sub_const', '0:sub'),
@@ -161,15 +160,42 @@ class TestObjectDetectionAPIPreprocessor2Replacement(unittest.TestCase):
]
graph = build_graph(self.nodes, edges)
graph.stage = 'front'
graph.graph['cmd_params'] = Namespace(tensorflow_object_detection_api_pipeline_config=__file__)
ObjectDetectionAPIPreprocessor2Replacement().transform_graph(graph, self.replacement_desc)
with unittest.mock.patch('builtins.open', unittest.mock.mock_open(read_data=file_content)):
ObjectDetectionAPIPreprocessor2Replacement().transform_graph(graph, self.replacement_desc)
(flag, resp) = compare_graphs(graph, self.build_ref_graph(False), 'result', check_op_attrs=True)
self.assertTrue(flag, resp)
def test_case_1_no_pad_to_max_dim(self, update_parameter_shape_mock):
# test for case #1 described in the ObjectDetectionAPIPreprocessor2Replacement
# sub/mul should be kept even though they are applied before prep-processing and pad_to_max_dimension is False
update_parameter_shape_mock.return_value = (None, None)
edges = [*connect_front('input', '0:mul'),
*connect_front('mul_const', '1:mul'),
*connect_front('sub_const', '0:sub'),
*connect_front('mul', '1:sub'),
*connect_front('sub', self.start_node_name),
*connect_front(self.start_node_name, 'resize'),
*connect_front('resize', self.end_node_name),
*connect_front(self.end_node_name, 'result'),
]
graph = build_graph(self.nodes, edges)
graph.stage = 'front'
graph.graph['cmd_params'] = Namespace(tensorflow_object_detection_api_pipeline_config=__file__)
updated_pipeline_config_content = file_content.replace('pad_to_max_dimension: true',
'pad_to_max_dimension: false')
with unittest.mock.patch('builtins.open', unittest.mock.mock_open(read_data=updated_pipeline_config_content)):
ObjectDetectionAPIPreprocessor2Replacement().transform_graph(graph, self.replacement_desc)
(flag, resp) = compare_graphs(graph, self.build_ref_graph(True), 'result', check_op_attrs=True)
self.assertTrue(flag, resp)
def test_case_2(self, update_parameter_shape_mock):
# test for case #2 described in the ObjectDetectionAPIPreprocessor2Replacement
update_parameter_shape_mock.return_value = None
update_parameter_shape_mock.return_value = (None, None)
edges = [*connect_front('input', self.start_node_name),
*connect_front(self.start_node_name, 'resize'),
@@ -182,15 +208,17 @@ class TestObjectDetectionAPIPreprocessor2Replacement(unittest.TestCase):
]
graph = build_graph(self.nodes, edges)
graph.stage = 'front'
graph.graph['cmd_params'] = Namespace(tensorflow_object_detection_api_pipeline_config=__file__)
ObjectDetectionAPIPreprocessor2Replacement().transform_graph(graph, self.replacement_desc)
with unittest.mock.patch('builtins.open', unittest.mock.mock_open(read_data=file_content)):
ObjectDetectionAPIPreprocessor2Replacement().transform_graph(graph, self.replacement_desc)
(flag, resp) = compare_graphs(graph, self.build_ref_graph(True), 'result', check_op_attrs=True)
self.assertTrue(flag, resp)
def test_case_3(self, update_parameter_shape_mock):
# test for case #3 described in the ObjectDetectionAPIPreprocessor2Replacement
update_parameter_shape_mock.return_value = None
update_parameter_shape_mock.return_value = (None, None)
edges = [*connect_front('input', self.start_node_name),
*connect_front(self.start_node_name, 'resize'),
@@ -199,8 +227,10 @@ class TestObjectDetectionAPIPreprocessor2Replacement(unittest.TestCase):
]
graph = build_graph(self.nodes, edges)
graph.stage = 'front'
graph.graph['cmd_params'] = Namespace(tensorflow_object_detection_api_pipeline_config=__file__)
ObjectDetectionAPIPreprocessor2Replacement().transform_graph(graph, self.replacement_desc)
with unittest.mock.patch('builtins.open', unittest.mock.mock_open(read_data=file_content)):
ObjectDetectionAPIPreprocessor2Replacement().transform_graph(graph, self.replacement_desc)
(flag, resp) = compare_graphs(graph, self.build_ref_graph(False), 'result', check_op_attrs=True)
self.assertTrue(flag, resp)
@@ -221,7 +251,6 @@ class TestObjectDetectionAPIPreprocessor2Replacement(unittest.TestCase):
**regular_op('resize', {'type': 'Interpolate'}),
**result('result'),
}
edges = None
if pre_processing == 'no':
edges = [*connect_front('input', self.loop_start_node_name),
*connect_front(self.loop_start_node_name, 'resize'),
@@ -265,33 +294,39 @@ class TestObjectDetectionAPIPreprocessor2Replacement(unittest.TestCase):
def test_case_4(self, update_parameter_shape_mock):
# test for case #4 described in the ObjectDetectionAPIPreprocessor2Replacement
update_parameter_shape_mock.return_value = None
update_parameter_shape_mock.return_value = (None, None)
graph = self.build_main_graph('leading')
graph.graph['cmd_params'] = Namespace(tensorflow_object_detection_api_pipeline_config=__file__)
ObjectDetectionAPIPreprocessor2Replacement().transform_graph(graph, self.replacement_desc)
with unittest.mock.patch('builtins.open', unittest.mock.mock_open(read_data=file_content)):
ObjectDetectionAPIPreprocessor2Replacement().transform_graph(graph, self.replacement_desc)
(flag, resp) = compare_graphs(graph, self.build_ref_graph(True), 'result', check_op_attrs=True)
self.assertTrue(flag, resp)
def test_case_5(self, update_parameter_shape_mock):
# test for case #5 described in the ObjectDetectionAPIPreprocessor2Replacement
update_parameter_shape_mock.return_value = None
update_parameter_shape_mock.return_value = (None, None)
graph = self.build_main_graph('trailing')
graph.graph['cmd_params'] = Namespace(tensorflow_object_detection_api_pipeline_config=__file__)
ObjectDetectionAPIPreprocessor2Replacement().transform_graph(graph, self.replacement_desc)
with unittest.mock.patch('builtins.open', unittest.mock.mock_open(read_data=file_content)):
ObjectDetectionAPIPreprocessor2Replacement().transform_graph(graph, self.replacement_desc)
(flag, resp) = compare_graphs(graph, self.build_ref_graph(True), 'result', check_op_attrs=True)
self.assertTrue(flag, resp)
def test_case_6(self, update_parameter_shape_mock):
# test for case #6 described in the ObjectDetectionAPIPreprocessor2Replacement
update_parameter_shape_mock.return_value = None
update_parameter_shape_mock.return_value = (None, None)
graph = self.build_main_graph('no')
graph.graph['cmd_params'] = Namespace(tensorflow_object_detection_api_pipeline_config=__file__)
ObjectDetectionAPIPreprocessor2Replacement().transform_graph(graph, self.replacement_desc)
with unittest.mock.patch('builtins.open', unittest.mock.mock_open(read_data=file_content)):
ObjectDetectionAPIPreprocessor2Replacement().transform_graph(graph, self.replacement_desc)
(flag, resp) = compare_graphs(graph, self.build_ref_graph(False), 'result', check_op_attrs=True)
self.assertTrue(flag, resp)

View File

@@ -1,9 +1,7 @@
# Copyright (C) 2018-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import os
import tempfile
import unittest
import unittest.mock
from mo.utils.error import Error
from mo.utils.pipeline_config import PipelineConfig
@@ -15,6 +13,7 @@ file_content = """model {
keep_aspect_ratio_resizer {
min_dimension: 600
max_dimension: 1024
pad_to_max_dimension: true
}
}
feature_extractor {
@@ -71,8 +70,6 @@ file_content = """model {
mode: FAN_AVG
}
}
}
}
}
use_dropout: false
dropout_keep_probability: 1.0
@@ -83,7 +80,7 @@ file_content = """model {
score_threshold: 0.300000011921
iou_threshold: 0.600000023842
max_detections_per_class: 100
max_total_detections: 100
max_total_detections: 200
}
score_converter: SOFTMAX
}
@@ -99,20 +96,12 @@ class TestingSimpleProtoParser(unittest.TestCase):
self.assertRaises(Error, PipelineConfig, "/abc/def")
def test_pipeline_config_non_model_file(self):
file = tempfile.NamedTemporaryFile('wt', delete=False)
file.write("non_model {}")
file_name = file.name
file.close()
self.assertRaises(Error, PipelineConfig, file_name)
with unittest.mock.patch('builtins.open', unittest.mock.mock_open(read_data="non_model {}")):
self.assertRaises(Error, PipelineConfig, __file__)
def test_pipeline_config_existing_file(self):
file = tempfile.NamedTemporaryFile('wt', delete=False)
file.write(file_content)
file_name = file.name
file.close()
pipeline_config = PipelineConfig(file_name)
with unittest.mock.patch('builtins.open', unittest.mock.mock_open(read_data=file_content)):
pipeline_config = PipelineConfig(__file__)
expected_result = {'resizer_min_dimension': 600,
'first_stage_nms_score_threshold': 0.0,
'anchor_generator_aspect_ratios': [0.5, 1.0, 2.0],
@@ -138,7 +127,11 @@ class TestingSimpleProtoParser(unittest.TestCase):
'use_matmul_crop_and_resize': False,
'add_background_class': True,
'share_box_across_classes': False,
'pad_to_max_dimension': False,
'pad_to_max_dimension': True,
'postprocessing_score_threshold': 0.300000011921,
'postprocessing_score_converter': 'SOFTMAX',
'postprocessing_iou_threshold': 0.600000023842,
'postprocessing_max_detections_per_class': 100,
'postprocessing_max_total_detections': 200,
}
os.unlink(file_name)
self.assertDictEqual(pipeline_config._model_params, expected_result)