IntenseWebs/openvino
3
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed bug in the MO transformation UnsqueezeTileReshapeBlockToInterpolate (#4710)

Browse Source 
* Some additional checks in the MO transformation UnsqueezeTileReshapeBlockToInterpolate.

* Refactored transformation UnsqueezeTileReshapeBlockToInterpolate: checks of applicabilty were moved into the separate function.

* Rewritten the MO transformation UnsqueezeTileReshapeBlockToInterpolate.

* Now we replace all block Unsqueeze+Tile+Reshape by Interpolate-4.

* Fixed comment.

* Added comments about applicability criteria.

* Some fixes.

* Small fix.

* Deleted redundant type casts.

* Added an example into comment.

* Now an interpolated axis length is calculated using the function node_to_get_shape_value_of_indices.

* Fixed tests.

* Optimized imports.
This commit is contained in:
Vladimir Gavrilov authored and GitHub committed 2021-03-17 21:56:25 +03:00
1 parent 3d838067d8
commit 24c31fba6a
2 files changed
+167 -130
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

model-optimizer/extensions/middle/UnsqueezeTileReshapeBlockToInterpolate.py
+118 -77
View File
@@ -15,41 +15,68 @@
"""
import logging as log
import numpy as np
from extensions.ops.activation_ops import Floor
from extensions.ops.Cast import Cast
from extensions.ops.elementwise import Mul
from extensions.ops.interpolate import Interpolate
from mo.front.common.partial_infer.utils import int64_array, float32_array
from mo.front.tf.graph_utils import create_op_with_const_inputs
from mo.graph.graph import Graph, rename_nodes
from mo.middle.replacement import MiddleReplacementPattern
from mo.ops.const import Const
from mo.ops.shape import Shape
from mo.ops.strided_slice import StridedSlice
from mo.utils.shape import node_to_get_shape_value_of_indices
class UnsqueezeTileReshapeBlockToInterpolate(MiddleReplacementPattern):
"""
This transformation looks for Interpolation layer implemented using simple operations, i.e. Unsqueeze,
Tile, Reshape, and replaces found pattern with a sequence of Shape, StridedSlice, Const, Mul, Interpolate.
The transformation looks for a sub-graph performing unsqueeze-ing input tensor by some "axis" and then tiling over
it fixed number of "times". This pattern can be represented with the Interpolate operation of mode "nearest"
performing interpolation over specific "axis" with fixed output dimension size equal to "times".
Here we assume that the input of 'unsqueeze' is in NDHWC layout and is a 5D-tensor.
Note, that the transformation expects that the output from Tile is reshaped back to the tensor with rank equal to
the input tensor rank. This constraints occurs because the pattern appears in the models where these patterns appear
one after another, performing unsqueeze-ing over different dimensions, effectively performing interpolation over
several dimensions.
Found pattern will be replaced with
nodes=[
('shape', dict(kind='op', op='Shape')),
('strided_slice', dict(kind='op', op='StridedSlice')),
('scales', dict(kind='op', op='Const')),
('scaled_shape', dict(kind='op', op='Mul')),
('interp', dict(kind='op', op='Interpolate'))
],
edges=[
('shape', 'strided_slice', {'in': 0}),
('strided_slice', 'scaled_shape', {'in': 0}),
('scales', 'scaled_shape', {'in': 1}),
('scaled_shape', 'interp', {'in': 1}),
]
These sequences are merged in the 'optimizer/extensions/middle/InterpolateSequenceToInterpolate.py' transformation
into a single Interpolate operation.
The transformation is applicable only when all following conditions are fulfilled:
1. 'Unsqueeze' must be performed with respect to only one axis.
2. The length of the value of the second input of 'Tile' must be equal to the input rank of 'Unsqueeze' plus 1.
3. All elements of the value of the second input of 'Tile' must be equal to 1,
except the value corresponding the interpolated axis.
4. The input rank of 'Unsqueeze' and the output rank of 'Reshape' must be equal.
Finally, because plugins support only Interpolate-4 with 4D or 5D tensor with interpolated data,
we need to check that the input rank of 'Unsqueeze' is equal to 4 or 5.
Example.
Let data = np.arange(0, 1 * 2 * 3 * 4).reshape((1, 2, 3, 4)).astype(np.float32), that is
data = np.array([[[[ 0, 1, 2, 3],
[ 4, 5, 6, 7],
[ 8, 9, 10, 11]],
[[12, 13, 14, 15],
[16, 17, 18, 19],
[20, 21, 22, 23]]]], dtype=np.float32)
After np.tile(np.expand_dims(data, 3), [1, 1, 1, 2, 1]).reshape((1, 2, 3 * 2, 4)) we get
array([[[[ 0, 1, 2, 3],
[ 0, 1, 2, 3],
[ 4, 5, 6, 7],
[ 4, 5, 6, 7],
[ 8, 9, 10, 11],
[ 8, 9, 10, 11]],
[[12, 13, 14, 15],
[12, 13, 14, 15],
[16, 17, 18, 19],
[16, 17, 18, 19],
[20, 21, 22, 23],
[20, 21, 22, 23]]]], dtype=np.float32)
This result is equal to nearest interpolation along with axis = 2 (the second argument of 'expand_dims')
and scale = 2 (the element from the second argument of 'tile' that is not equal to 1).
"""
enabled = True
force_shape_inference = True
@@ -76,79 +103,93 @@ class UnsqueezeTileReshapeBlockToInterpolate(MiddleReplacementPattern):
]
)
def replace_pattern(self, graph: Graph, match: dict):
@staticmethod
def is_applicable(match: dict) -> bool:
"""
This function checks whether this transformation is applicable.
:param match: dictionary with nodes from the found pattern
:return: True, if the transformation is applicable
False, otherwise
"""
unsqueeze_node = match['unsqueeze']
unsqueeze_name = unsqueeze_node.name
second_input_of_unsqueeze = unsqueeze_node.in_port(1).get_connection().get_source().node
if not second_input_of_unsqueeze.has_valid('value'):
return
if not second_input_of_unsqueeze.has_valid('value') or len(second_input_of_unsqueeze.value) != 1:
return False
d_idx = int(second_input_of_unsqueeze.value)
if d_idx == 0:
return False
second_input_of_tile = match['tile'].in_port(1).get_connection().get_source().node
if not second_input_of_tile.has_valid('value'):
return
return False
input_shape_of_unsqueeze = unsqueeze_node.in_port(0).data.get_shape()
if len(input_shape_of_unsqueeze) not in {4, 5}:
return
input_rank_of_unsqueeze = len(input_shape_of_unsqueeze)
if input_rank_of_unsqueeze not in {4, 5}:
return False
scale = float32_array([second_input_of_tile.value[d_idx]])
axis = d_idx - 1
axis_node = Const(graph, {'name': unsqueeze_name + '/axis', 'value': int64_array([axis])}).create_node()
if input_rank_of_unsqueeze + 1 != len(second_input_of_tile.value):
return False
shape_node = Shape(graph, dict(name=unsqueeze_name + '/Shape')).create_node()
scales_node = Const(graph, dict(name=unsqueeze_name + '/scales', value=scale)).create_node()
mul_node = Mul(graph, dict(name=unsqueeze_name + '/Mul')).create_node()
scales_node.out_port(0).connect(mul_node.in_port(1))
expected_tile_constant = np.ones(input_rank_of_unsqueeze + 1)
expected_tile_constant[d_idx] = float(second_input_of_tile.value[d_idx])
slice_begin = Const(graph, dict(name=unsqueeze_name + '/slice_begin', value=int64_array([axis]))).create_node()
slice_end = Const(graph, dict(name=unsqueeze_name + '/slice_end', value=int64_array([axis + 1]))).create_node()
strided_slice_node = StridedSlice(graph,
{'name': unsqueeze_name + '/StridedSlice',
'begin_mask': int64_array([1]),
'end_mask': int64_array([1]),
'new_axis_mask': int64_array([0]),
'shrink_axis_mask': int64_array([0]),
'ellipsis_mask': int64_array([0]),
}).create_node()
shape_node.out_port(0).connect(strided_slice_node.in_port(0))
slice_begin.out_port(0).connect(strided_slice_node.in_port(1))
slice_end.out_port(0).connect(strided_slice_node.in_port(2))
cast_shape_to_float = Cast(graph, {'dst_type': np.float32}).create_node()
strided_slice_node.out_port(0).connect(cast_shape_to_float.in_port(0))
cast_shape_to_float.out_port(0).connect(mul_node.in_port(0))
interp_node = Interpolate(graph,
dict(mode='nearest',
antialias=0, pads_begin=int64_array([0]),
pads_end=int64_array([0]), coordinate_transformation_mode='half_pixel',
nearest_mode='round_prefer_floor', cube_coeff=-0.75,
version='opset4', shape_calculation_mode='scales',
in_ports_count=4,
maybe_part_of_sequence=True)).create_node()
floor_node = Floor(graph, {'name': unsqueeze_name + '/Floor'}).create_node()
cast_mul_result_to_int = Cast(graph, {'dst_type': np.int64}).create_node()
mul_node.out_port(0).connect(floor_node.in_port(0))
floor_node.out_port(0).connect(cast_mul_result_to_int.in_port(0))
cast_mul_result_to_int.out_port(0).connect(interp_node.in_port(1))
scales_node.out_port(0).connect(interp_node.in_port(2))
axis_node.out_port(0).connect(interp_node.in_port(3))
if not np.array_equal(expected_tile_constant, float32_array(second_input_of_tile.value)):
return False
reshape_node = match['reshape']
new_shape = reshape_node.in_port(1).data.get_value()
if new_shape is None or input_rank_of_unsqueeze != len(new_shape):
return False
return True
def replace_pattern(self, graph: Graph, match: dict):
if not self.is_applicable(match):
return
unsqueeze_node = match['unsqueeze']
unsqueeze_name = unsqueeze_node.soft_get('name', unsqueeze_node.id)
second_input_of_unsqueeze = unsqueeze_node.in_port(1).get_connection().get_source().node
d_idx = int(second_input_of_unsqueeze.value)
axis = d_idx - 1
shape_node = Shape(graph, dict(name=unsqueeze_name + '/Shape')).create_node()
axis_len_node = node_to_get_shape_value_of_indices(shape_node, [axis])
second_input_of_tile = match['tile'].in_port(1).get_connection().get_source().node
scale = int64_array([second_input_of_tile.value[d_idx]])
float_scale = float32_array([second_input_of_tile.value[d_idx]])
mul_node = create_op_with_const_inputs(graph, Mul, {1: scale}, {'name': unsqueeze_name + '/Mul'})
axis_len_node.out_port(0).connect(mul_node.in_port(0))
interp_node = create_op_with_const_inputs(graph,
Interpolate,
{
2: float_scale,
3: int64_array([axis])},
{
'mode': 'nearest',
'antialias': 0,
'pads_begin': int64_array([0]),
'pads_end': int64_array([0]),
'coordinate_transformation_mode': 'half_pixel',
'nearest_mode': 'round_prefer_floor',
'cube_coeff': -0.75,
'version': 'opset4',
'shape_calculation_mode': 'scales',
'in_ports_count': 4,
'maybe_part_of_sequence': True
})
mul_node.out_port(0).connect(interp_node.in_port(1))
reshape_node = match['reshape']
reshape_node.out_port(0).get_connection().set_source(interp_node.out_port(0))
reshape_name = reshape_node.soft_get('name', reshape_node.id)
rename_nodes([(reshape_node, reshape_name + '/delete'), (interp_node, reshape_name)])
unsqueeze_connection = match['unsqueeze'].in_port(0).get_connection()
before_unsqueeze = unsqueeze_connection.get_source().node
unsqueeze_connection = unsqueeze_node.in_port(0).get_connection()
unsqueeze_connection.set_destination(interp_node.in_port(0))
before_unsqueeze.out_port(0).connect(shape_node.in_port(0))
unsqueeze_connection.get_source().connect(shape_node.in_port(0))
model-optimizer/extensions/middle/UnsqueezeTileReshapeBlockToInterpolate_test.py
+49 -53
View File
@@ -127,41 +127,54 @@ ref_graph_node_attrs_with_4_inputs_interpolate = {
'shape': None,
'value': None,
},
'strided_slice': {
'type': 'StridedSlice',
'gather': {
'type': 'Gather',
'kind': 'op',
'op': 'StridedSlice',
'begin_mask': int64_array([1]),
'end_mask': int64_array([1]),
'new_axis_mask': int64_array([0]),
'shrink_axis_mask': int64_array([0]),
'ellipsis_mask': int64_array([0]),
'op': 'Gather'
},
'strided_slice_data': {
'gather_data': {
'kind': 'data',
'shape': None,
'value': None,
},
'begin': {
'indices': {
'kind': 'op',
'op': 'Const',
'type': 'Const',
'value': int64_array([1]),
'shape': int64_array([1]),
},
'begin_data': {
'indices_data': {
'kind': 'data',
'value': None,
'shape': None,
},
'end': {
'gather_axis': {
'kind': 'op',
'op': 'Const',
'type': 'Const',
'value': int64_array([2]),
'value': np.array(0, dtype=np.int64),
'shape': np.array(0, dtype=np.int64).shape,
},
'gather_axis_data': {
'kind': 'data',
'value': None,
'shape': None,
},
'scales_m': {
'kind': 'op',
'op': 'Const',
'type': 'Const',
'value': np.array([2], dtype=np.int64),
'shape': int64_array([1]),
},
'end_data': {
'scales_m_data': {
'kind': 'data',
'value': np.array([2], dtype=np.float32),
'shape': int64_array([1]),
},
'mul': {'type': 'Mul', 'kind': 'op', 'op': 'Mul'},
'mul_data': {
'kind': 'data',
'value': None,
'shape': None,
@@ -175,26 +188,8 @@ ref_graph_node_attrs_with_4_inputs_interpolate = {
},
'scales_data': {
'kind': 'data',
'value': None,
'shape': None,
},
'cast_shape_to_float': {'kind': 'op', 'op': 'Cast', 'type': 'Convert', 'dst_type': np.float32},
'cast_shape_to_float_data': {'kind': 'data', 'shape': None},
'mul': {'type': 'Mul', 'kind': 'op', 'op': 'Mul'},
'mul_data': {
'kind': 'data',
'value': None,
'shape': None,
},
'floor': {'kind': 'op', 'op': 'Floor', 'type': 'Floor'},
'floor_data': {'kind': 'data', 'shape': None},
'cast_mul_to_float': {'kind': 'op', 'op': 'Cast', 'type': 'Convert', 'dst_type': np.int64},
'cast_mul_to_float_data': {'kind': 'data', 'shape': None},
'interpolate': {'type': 'Interpolate', 'kind': 'op', 'op': 'Interpolate'},
'interpolate_data': {
'kind': 'data',
'value': None,
'shape': int64_array([1, 16, 32, 32, 64]),
'value': np.array([2], dtype=np.float32),
'shape': int64_array([1]),
},
'axes': {
'kind': 'op',
@@ -204,9 +199,15 @@ ref_graph_node_attrs_with_4_inputs_interpolate = {
'shape': int64_array([1]),
},
'axes_data': {
'kind': 'data',
'value': int64_array([1]),
'shape': int64_array([1]),
},
'interpolate': {'type': 'Interpolate', 'kind': 'op', 'op': 'Interpolate'},
'interpolate_data': {
'kind': 'data',
'value': None,
'shape': None,
'shape': int64_array([1, 16, 32, 32, 64]),
},
'abs': {'type': 'Abs', 'kind': 'op', 'op': 'Abs'},
'abs_data': {
@@ -222,26 +223,21 @@ ref_graph_edges_attrs_with_4_inputs_interpolate = [
('placeholder', 'placeholder_data'),
('placeholder_data', 'shapeof'),
('shapeof', 'shapeof_data'),
('shapeof_data', 'strided_slice', {'in': 0}),
('strided_slice', 'strided_slice_data'),
('begin', 'begin_data'),
('begin_data', 'strided_slice', {'in': 1}),
('end', 'end_data'),
('end_data', 'strided_slice', {'in': 2}),
('scales', 'scales_data'),
('strided_slice_data', 'cast_shape_to_float'),
('cast_shape_to_float', 'cast_shape_to_float_data'),
('cast_shape_to_float_data', 'mul', {'in': 0}),
('scales_data', 'mul', {'out': 0, 'in': 1}),
('scales_data', 'interpolate', {'out': 0, 'in': 2}),
('shapeof_data', 'gather', {'in': 0}),
('gather', 'gather_data'),
('indices', 'indices_data'),
('indices_data', 'gather', {'in': 1}),
('gather_axis', 'gather_axis_data'),
('gather_axis_data', 'gather', {'in': 2}),
('scales_m', 'scales_m_data'),
('gather_data', 'mul', {'in': 0}),
('scales_m_data', 'mul', {'in': 1}),
('mul', 'mul_data'),
('mul_data', 'floor'),
('floor', 'floor_data'),
('floor_data', 'cast_mul_to_float'),
('cast_mul_to_float', 'cast_mul_to_float_data'),
('cast_mul_to_float_data', 'interpolate', {'in': 1}),
('placeholder_data', 'interpolate', {'in': 0}),
('scales', 'scales_data'),
('axes', 'axes_data'),
('scales_data', 'interpolate', {'out': 0, 'in': 2}),
('mul_data', 'interpolate', {'in': 1}),
('placeholder_data', 'interpolate', {'in': 0}),
('axes_data', 'interpolate', {'in': 3}),
('interpolate', 'interpolate_data'),
('interpolate_data', 'abs'),