Leaky relu transformation refactor (#2640)

* Refactored LeakyRelu transformation

* Added unit test for LeakyRelu transformation + removed duplicate test function valued_const

model-optimizer/automation/package_BOM.txt

@@ -879,7 +879,6 @@ mo/middle/passes/fusing/helpers.py
 mo/middle/passes/fusing/mark_unfused_nodes.py
 mo/middle/passes/fusing/resnet_optimization.py
 mo/middle/passes/infer.py
-mo/middle/passes/leaky_relu.py
 mo/middle/passes/tensor_names.py
 mo/middle/pattern_match.py
 mo/middle/replacement.py

model-optimizer/extensions/back/ConvolutionNormalizer_test.py

@@ -24,7 +24,7 @@ from mo.front.common.partial_infer.utils import int64_array
 from mo.ops.reshape import Reshape
 from mo.utils.ir_engine.compare_graphs import compare_graphs
 from mo.utils.unittest.graph import build_graph, result, regular_op_with_shaped_data, regular_op_with_empty_data, \
-    valued_const_with_data, const_with_data, connect
+    valued_const_with_data, connect
 
 
 def graph_template(weights_initial_shape, new_reshape_shape, limits_initial_shape, limits_new_shape=None):
@@ -40,7 +40,7 @@ def graph_template(weights_initial_shape, new_reshape_shape, limits_initial_shap
 
         **valued_const_with_data('weights', np.ones(weights_shape)),
 
-        **const_with_data('dim', int64_array(reshape_shape)),
+        **valued_const_with_data('dim', int64_array(reshape_shape)),
         **regular_op_with_shaped_data('reshape', reshape_shape, {'type': 'Reshape', 'infer': Reshape.infer, 'op': 'Reshape'}),
 
         **valued_const_with_data('il', np.ones(limit_shape)),
@@ -118,7 +118,7 @@ class TestV7ConvolutionWithGroupsResolver(unittest.TestCase):
 
             **valued_const_with_data('weights', np.ones([3, 8, 7, 7])),
 
-            **const_with_data('dim', int64_array([24, -1, 7, 7])),
+            **valued_const_with_data('dim', int64_array([24, -1, 7, 7])),
             **regular_op_with_empty_data('reshape', {'type': 'Reshape'}),
 
             **regular_op_with_shaped_data('convolution', None, {'type': 'Convolution', 'group': 3, 'output': 24}),
@@ -169,7 +169,7 @@ class TestV10ConvolutionWithGroupsResolver(unittest.TestCase):
 
             **valued_const_with_data('weights', np.ones([3, 8, 7, 7])),
 
-            **const_with_data('dim', int64_array([3, 8, 1, 7, 7])),
+            **valued_const_with_data('dim', int64_array([3, 8, 1, 7, 7])),
             **regular_op_with_empty_data('reshape', {'type': 'Reshape'}),
 
             **regular_op_with_shaped_data('convolution', None, {'type': 'Convolution', 'group': 3, 'output': 24}),

model-optimizer/extensions/back/FakeOutputResolver_test.py

@@ -19,8 +19,8 @@ import unittest
 from extensions.back.FakeOutputResolver import FakeOutputResolver
 from mo.front.common.partial_infer.utils import int64_array
 from mo.utils.ir_engine.compare_graphs import compare_graphs
-from mo.utils.unittest.graph import build_graph, result, regular_op_with_empty_data, const_with_data, connect, \
-    empty_data
+from mo.utils.unittest.graph import build_graph, result, regular_op_with_empty_data, connect, empty_data, \
+    valued_const_with_data
 
 
 class FakeOutputResolverTest(unittest.TestCase):
@@ -59,8 +59,8 @@ class FakeOutputResolverTest(unittest.TestCase):
             **regular_op_with_empty_data('fake_output2',
                                          {'type': None, 'kind': 'op', 'op': 'FakeOutput', 'name': 'my_output_name2'}),
 
-            **const_with_data('const1', int64_array(0)),
-            **const_with_data('const2', int64_array(0)),
+            **valued_const_with_data('const1', int64_array(0)),
+            **valued_const_with_data('const2', int64_array(0)),
             **regular_op_with_empty_data('add1', {'type': None, 'kind': 'op', 'op': 'Add', 'name': 'my_output_name1'}),
             **regular_op_with_empty_data('add2', {'type': None, 'kind': 'op', 'op': 'Add', 'name': 'my_output_name2'}),
             **result('result1'),

model-optimizer/extensions/back/MatMulNormalizer_test.py

@@ -24,7 +24,7 @@ from extensions.ops.MatMul import MatMul
 from mo.front.common.partial_infer.utils import int64_array
 from mo.ops.reshape import Reshape
 from mo.utils.ir_engine.compare_graphs import compare_graphs
-from mo.utils.unittest.graph import build_graph, regular_op_with_shaped_data, const_with_data, \
+from mo.utils.unittest.graph import build_graph, regular_op_with_shaped_data, valued_const_with_data, \
     result, connect
 from mo.utils.unittest.graph import regular_op_with_empty_data as op_with_empty_data
 
@@ -44,7 +44,7 @@ class SmartReshape_HC_Reshape_MatMulTest(unittest.TestCase):
         nodes = {
             **regular_op_with_shaped_data('in_1', in1_shape, dict(type='Parameter', op='Parameter')),
             **regular_op_with_shaped_data('in_2', in2_shape, dict(type='Parameter', op='Parameter')),
-            **const_with_data('dim', int64_array(reshape_pattern)),
+            **valued_const_with_data('dim', int64_array(reshape_pattern)),
             **op_with_empty_data('reshape',
                                  dict(type='Reshape', op='Reshape', infer=Reshape.infer, need_shape_inference=True)),
             **op_with_empty_data('matmul',
@@ -82,7 +82,7 @@ class SmartReshape_HC_Reshape_MatMulTest(unittest.TestCase):
         nodes = {
             **regular_op_with_shaped_data('in_1', in1_shape, dict(type='Parameter', op='Parameter')),
             **regular_op_with_shaped_data('in_2', in2_shape, dict(type='Parameter', op='Parameter')),
-            **const_with_data('dim', int64_array(reshape_pattern)),
+            **valued_const_with_data('dim', int64_array(reshape_pattern)),
             **op_with_empty_data('reshape',
                                  dict(type='Reshape', op='Reshape', infer=Reshape.infer, need_shape_inference=True)),
             **op_with_empty_data('matmul',

model-optimizer/extensions/middle/LeakyReluPattern.py

@@ -13,15 +13,25 @@
  See the License for the specific language governing permissions and
  limitations under the License.
 """
+import logging as log
 from extensions.middle.fusings import Fusing
 from extensions.middle.pass_separator import PostMiddleStart
-from mo.graph.graph import Graph
-from mo.middle.passes.leaky_relu import convert_mul_eltwise_to_leaky_relu
+from extensions.ops.activation_ops import LeakyReLU
+from mo.graph.graph import Graph, rename_nodes
 from mo.middle.replacement import MiddleReplacementPattern
 
 
-class LeakyReLU(MiddleReplacementPattern):
+class LeakyReLUFusion(MiddleReplacementPattern):
+    """
+    The transformation  finds next subgraph:
+
+    -->Data-------->Maximum-->Data
+          `-->Mul---`
+
+    and replaces with ReLU with negative slope (LeakyRelu)
+    """
     enabled = True
+    force_clean_up = True
 
     def run_after(self):
         return [Fusing]
@@ -29,5 +39,47 @@ class LeakyReLU(MiddleReplacementPattern):
     def run_before(self):
         return [PostMiddleStart]
 
-    def find_and_replace_pattern(self, graph: Graph):
-        convert_mul_eltwise_to_leaky_relu(graph)
+    @staticmethod
+    def pattern():
+        return dict(
+            nodes=[
+                ('data', dict(kind='data')),
+                ('mul_data', dict(kind='data')),
+                ('max_op', dict(kind='op', type='Maximum')),
+                ('const_op', dict(kind='op', type='Const')),
+                ('const_data', dict(kind='data')),
+                ('mul_op', dict(kind='op', type='Multiply')),
+            ],
+            edges=[
+                ('data', 'mul_op'),
+                ('mul_op', 'mul_data'),
+                ('data', 'max_op'),
+                ('mul_data', 'max_op'),
+                ('const_op', 'const_data'),
+                ('const_data', 'mul_op')
+            ],
+        )
+
+    def replace_pattern(self, graph: Graph, match: dict):
+        mul_node = match['mul_op']
+        const_node = match['const_op']
+        max_node = match['max_op']
+        max_name = max_node.soft_get('name', max_node.id)
+
+        const_value = const_node.out_port(0).data.get_value()
+        if const_value is None or const_value.size != 1:
+            log.debug('Mul layer "{}" can not participate in conversion to the LeakyReLU because constant "{}" '
+                      'contains more than one element: {}'.format(mul_node.id, const_node.id, const_value.size))
+            return
+
+        # Create new LeakyReLU operation
+        leaky_relu_node = LeakyReLU(graph, dict(negative_slope=const_value.item(0))).create_node()
+
+        data_in_port = int(mul_node.in_port(0).get_source().node.type == 'Const')
+        mul_node.in_port(data_in_port).get_source().connect(leaky_relu_node.in_port(0))
+        max_node.out_port(0).get_connection().set_source(leaky_relu_node.out_port(0))
+
+        rename_nodes([(max_node, max_name + '/TBR'), (leaky_relu_node, max_name)])
+
+        log.debug('Successful conversion from {} {} to ReLU with negative slope (leaky ReLU)'
+                  ''.format(max_node.id, mul_node.id))

model-optimizer/extensions/middle/LeakyReluPattern_test.py

@@ -0,0 +1,108 @@
+"""
+ Copyright (C) 2018-2020 Intel Corporation
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+"""
+
+import unittest
+
+from extensions.middle.LeakyReluPattern import LeakyReLUFusion
+from mo.front.common.partial_infer.utils import float_array, int64_array
+from mo.graph.graph import Node
+from mo.ops.result import Result
+from mo.utils.ir_engine.compare_graphs import compare_graphs
+from mo.utils.unittest.graph import build_graph, result, build_graph_with_edge_attrs, connect, \
+    regular_op_with_shaped_data, valued_const_with_data, connect_data
+
+shape = int64_array([1, 3, 5, 2])
+nodes = {**regular_op_with_shaped_data('input', shape, {'type': 'Parameter', 'op': 'Parameter'}),
+         **regular_op_with_shaped_data('mul', shape, {'type': 'Multiply', 'name': 'mul'}),
+         **regular_op_with_shaped_data('max', shape, {'type': 'Maximum', 'name': 'final_max'}),
+         **valued_const_with_data('const', float_array([0.5])),
+         **result('result')
+         }
+
+edges = [*connect('input:0', '0:mul'),
+         *connect('const', '1:mul'),
+         *connect_data('input', '0:max'),
+         *connect('mul:0', '1:max'),
+         *connect('max:0', 'result'),
+         ]
+
+ref_nodes = {**regular_op_with_shaped_data('input', shape, {'type': 'Parameter', 'op': 'Parameter'}),
+             **regular_op_with_shaped_data('leaky_relu', shape, {'type': 'LeakyReLU', 'name': 'max_final',
+                                                                 'negative_slope': None}),
+             **result('result')
+         }
+ref_edges = [*connect('input:0', 'leaky_relu'), *connect('leaky_relu', 'result')]
+
+
+class LeakyReluFusionTest(unittest.TestCase):
+    def test_leaky_relu_data_port_0(self):
+        graph = build_graph_with_edge_attrs(nodes, edges, {})
+        graph_ref = build_graph(ref_nodes, ref_edges)
+        Node(graph_ref, 'leaky_relu')['negative_slope'] = 0.5
+
+        LeakyReLUFusion().find_and_replace_pattern(graph)
+        graph.clean_up()
+
+        (flag, resp) = compare_graphs(graph, graph_ref, 'result')
+        self.assertTrue(flag, resp)
+        self.assertTrue(len(graph.get_op_nodes(name='final_max')) == 1 and
+                        graph.get_op_nodes(name='final_max')[0].op == 'LeakyReLU')
+
+    def test_leaky_relu_not_applicable_non_scalar_const(self):
+        # const value is not a scalar or 1D tensor with 1 element so the transformation is not applicable
+        graph = build_graph_with_edge_attrs(nodes, edges, {})
+        Node(graph, 'const')['value'] = float_array([0.5, 0.7])
+        Node(graph, 'const_d')['value'] = float_array([0.5, 0.7])
+        graph_ref = graph.copy()
+
+        LeakyReLUFusion().find_and_replace_pattern(graph)
+        graph.clean_up()
+
+        (flag, resp) = compare_graphs(graph, graph_ref, 'result')
+        self.assertTrue(flag, resp)
+
+    def test_leaky_relu_mul_multiple_consumers(self):
+        # multiple consumers of Mul operation
+        graph = build_graph_with_edge_attrs(nodes, edges, {})
+        additional_result = Result(graph, {'name': 'result_2'}).create_node()
+        Node(graph, 'mul').out_port(0).connect(additional_result.in_port(0))
+
+        ref_nodes = {**regular_op_with_shaped_data('input', shape, {'type': 'Parameter', 'op': 'Parameter'}),
+                     **regular_op_with_shaped_data('mul', shape, {'type': 'Multiply', 'name': 'mul'}),
+                     **regular_op_with_shaped_data('max', shape, {'type': 'Maximum', 'name': 'final_max'}),
+                     **valued_const_with_data('const', float_array([0.5])),
+                     **regular_op_with_shaped_data('leaky_relu', shape, {'type': 'LeakyReLU', 'name': 'max_final',
+                                                                         'negative_slope': None}),
+                     **result('result'),
+                     **result('result_2')
+                     }
+        ref_edges = [*connect('input:0', '0:mul'),
+                     *connect('const', '1:mul'),
+                     *connect('max:0', 'result'),
+                     *connect('mul:0', 'result_2'),
+                     *connect_data('input', 'leaky_relu'),
+                     *connect('leaky_relu', 'result')
+                     ]
+        graph_ref = build_graph_with_edge_attrs(ref_nodes, ref_edges)
+
+        LeakyReLUFusion().find_and_replace_pattern(graph)
+        graph.clean_up()
+
+        (flag, resp) = compare_graphs(graph, graph_ref, 'result')
+        self.assertTrue(flag, resp)
+
+        (flag, resp) = compare_graphs(graph, graph_ref, 'result_2')
+        self.assertTrue(flag, resp)

model-optimizer/extensions/middle/preprocessing.py

@@ -13,7 +13,7 @@
  See the License for the specific language governing permissions and
  limitations under the License.
 """
-from extensions.middle.LeakyReluPattern import LeakyReLU
+from extensions.middle.LeakyReluPattern import LeakyReLUFusion
 from extensions.middle.pass_separator import PostMiddleStart
 from mo.graph.graph import Graph
 from mo.middle.replacement import MiddleReplacementPattern
@@ -28,7 +28,7 @@ class CaffeMeanFileProcessing(MiddleReplacementPattern):
     graph_condition = [lambda graph: graph.graph['fw'] == 'caffe']
 
     def run_after(self):
-        return [LeakyReLU]
+        return [LeakyReLUFusion]
 
     def run_before(self):
         return [PostMiddleStart]

model-optimizer/extensions/ops/one_hot_test.py

@@ -22,16 +22,16 @@ from generator import generator, generate
 from extensions.ops.one_hot import OneHot
 from mo.front.common.partial_infer.utils import int64_array, float_array
 from mo.graph.graph import Node
-from mo.utils.unittest.graph import build_graph, regular_op_with_shaped_data, const_with_data, connect
+from mo.utils.unittest.graph import build_graph, regular_op_with_shaped_data, valued_const_with_data, connect
 
 
 def generate_nodes(data, axis=-1, depth=4, on_value=1., off_value=0.):
     return {
         'indices': {'Op': 'Parameter', 'value': data, 'shape': int64_array(data.shape)},
         'indices_d': {'kind': 'data', 'value': data, 'shape': int64_array(data.shape)},
-        **const_with_data('depth', int64_array(depth)),
-        **const_with_data('on_value', float_array(on_value)),
-        **const_with_data('off_value', float_array(off_value)),
+        **valued_const_with_data('depth', int64_array(depth)),
+        **valued_const_with_data('on_value', float_array(on_value)),
+        **valued_const_with_data('off_value', float_array(off_value)),
         **regular_op_with_shaped_data('one_hot', None, {'type': 'OneHot', 'axis': axis, 'Op': 'OneHot'})
     }
 

model-optimizer/mo/middle/passes/eliminate.py

@@ -86,7 +86,7 @@ def mark_undead_nodes(graph, undead_types: list):
     undead_types_with_result = undead_types + ['Result']
     undead_nodes = []
     for node in graph.get_op_nodes():
-        node_type = node.soft_get('type', node.op)
+        node_type = node.soft_get('type', node.soft_get('op'))
         if node_type in undead_types_with_result:
             undead_nodes.append(node.id)
 

model-optimizer/mo/middle/passes/fusing/fuse_linear_ops_test.py

@@ -22,7 +22,7 @@ from mo.front.common.partial_infer.eltwise import eltwise_infer
 from mo.graph.graph import Node
 from mo.middle.passes.fusing.fuse_linear_ops import _fuse_mul, fuse_linear_ops
 from mo.utils.ir_engine.compare_graphs import compare_graphs
-from mo.utils.unittest.graph import build_graph, regular_op_with_empty_data, const_with_data, connect
+from mo.utils.unittest.graph import build_graph
 
 nodes_attributes = {
     'placeholder_1': {'shape': None, 'type': 'Parameter', 'kind': 'op', 'op': 'Parameter'},

model-optimizer/mo/middle/passes/leaky_relu.py

@@ -1,93 +0,0 @@
-"""
- Copyright (C) 2018-2020 Intel Corporation
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-"""
-
-import logging as log
-
-from extensions.ops.activation_ops import LeakyReLU
-from mo.graph.graph import Graph
-from mo.middle.pattern_match import apply_pattern
-
-
-def _convert_to_leaky_relu_action(graph: Graph, matches: dict):
-    """
-    This function checks given pattern and if pattern satisfies all requirements, converts to ReLU with negative slope
-    """
-    mul_op = matches['mul_op']
-    mul_value_data = matches['const_data']
-    mul_data = matches['mul_data']
-    input_data = matches['data']
-    max_op = matches['max_op']
-    max_data = max_op.out_node()
-
-    # Check that all nodes satisfies conversion requirements
-    if len(max_op.in_nodes()) > 2:
-        log.debug('Maximum layer ({}) can not participate in conversion to leaky ReLU due to it has more than two '
-                  'inputs ({})'.format(max_op.id, len(max_op.in_nodes())))
-        return
-
-    if mul_value_data.has_valid('value') and mul_value_data.value.size != 1:
-        log.debug('Mul layer ({}) can not participate in conversion to leaky ReLU due to value {}'
-                  ''.format(mul_op.id, mul_value_data.soft_get('value')))
-        return
-
-    value = mul_value_data.value.item(0)
-
-    if len(mul_data.out_nodes()) > 1:
-        log.debug('Mul layer({}) can not participate in conversion to leaky ReLU due to it has more than one consumer'
-                  ''.format(mul_op.id))
-        return
-
-    # Disconnect data nodes from ops
-    graph.remove_edge(max_op.id, max_data.id)
-    graph.remove_edge(input_data.id, mul_op.id)
-    graph.remove_edge(input_data.id, max_op.id)
-
-    # Create new ReLU operation
-    relu_op = LeakyReLU(graph, dict(name="LeakyReLU_", negative_slope=value))
-    relu_op.create_node_with_data(inputs=[input_data], data_nodes=max_data)
-
-    log.debug('Successful conversion from {} {} to ReLU with negative slope (leaky ReLU)'
-              ''.format(max_op.id, mul_op.id))
-
-
-def convert_mul_eltwise_to_leaky_relu(graph: Graph):
-    """
-    This function finds next subgraph:
-    -->Data-------->Maximum-->Data
-          `-->Mul---`
-    and replace with ReLU with negative slope
-    """
-    apply_pattern(
-        graph,
-        nodes=[
-            ('data', dict(kind='data')),
-            ('mul_data', dict(kind='data')),
-            ('max_op', dict(kind='op', type='Maximum')),
-            ('const_op', dict(kind='op', type='Const')),
-            ('const_data', dict(kind='data')),
-            ('mul_op', dict(kind='op', type='Multiply')),
-        ],
-        edges=[
-            ('data', 'mul_op'),
-            ('mul_op', 'mul_data'),
-            ('data', 'max_op'),
-            ('mul_data', 'max_op'),
-            ('const_op', 'const_data'),
-            ('const_data', 'mul_op')
-        ],
-        action=_convert_to_leaky_relu_action
-    )
-    return graph

model-optimizer/mo/utils/unittest/graph.py

@@ -302,8 +302,6 @@ shaped_const_with_data = lambda name, shape: {**fake_const(name, shape), **shape
 
 valued_const_with_data = lambda name, value: {**const(name, value), **valued_data(name + '_d', value)}
 
-const_with_data = lambda name, value: {**const(name, value), **valued_data(name + '_d', value)}
-
 
 def extract_port_from_string(node_name: str):
     """