Adding v7::Gelu operation (#4497)
* Added support for Gelu-6 to the MO * Adding Gelu-6 to ngraph and python API + some tests * Fixed typo in the Gelu approximation mode * Fixed Gelu-6 reference implementation for Tanh mode * Added transformation to downgrade v6::Gelu to v2::Gelu * Added specification for the Gelu-6 * Code style fixes * The Gelu-6 operation specification update * Fixed compilation issue in reference implementation for Gelu * Fix compilation issues for some OSs * Code style fix * One more cpplint issue fix * Fixed Gelu6 reference implementation compilation on Windows. * Code style fix * Fixed various ngraph unit tests * Code style check * Reverted Gelu-2 to be fused op * Fixed Gelu6 downgrade transformation * Added unit test for Gelu6Downgrade transformation * Update copyright year * Updated copyright year * Replaced tab characters with 4 spaces in IR reader tests * Code style fixes * Added default value for GeluApproximation mode for Gelu-6 op * Fixed code style for Gelu-6 * Changed order of parameters for the Gelu evaluate to potentially avoid backward compatibility issues with ARM plugin * Fixed code style * Introduced opset7. Moved Gelu6 to opset7 * Fixed non-updated transformation * Fixed opset version in ngraph Python API for Gelu operation * Fixed typo in the opset number in the documentation * Reverted some changes related to Gelu6 * Updated MO to produce Gelu7 * Updated unit tests for Gelu * Updated Gelu7 specification * Changed gelu reference implementation. Added opset7 to Python packages * Updated Python API tests for Gelu operation * Code style fix * Marked get_approximation_mode function as const * Added missing "const" qualifier * Fixed code style issues in tests * Added extractor for MxNet operation Gelu * Spelling issues fix * Updated MxNet supported symbols * Added NGRAPH_OP_SCOPE for Gelu7 validate_and_infer_types * Fixed a typo in the comment
This commit is contained in:
Evgeny Lazarev authored and 
GitHub committed
1 parent
a5e2497788
commit
99f94ca09c
40 files changed
+1501
-197
No files matched your search
@@ -126,7 +126,7 @@ class GeLUMergerErf(FrontReplacementPattern):
|
||||
# check that the values match the approximation
|
||||
if fabs(div_param - sqrt2) < 1e-06 and mul_param == 0.5 and add_param == 1.0:
|
||||
log.debug('Confirmed Erf-based GELU pattern after {} with name {}'.format(inp_node.op, inp_name))
|
||||
gelu = GeLUOP(graph, dict(name=inp_name + '/GELU_')).create_node()
|
||||
gelu = GeLUOP(graph, dict(name=inp_name + '/GELU_', approximation='erf')).create_node()
|
||||
div.in_port(0).get_connection().set_destination(gelu.in_port(0))
|
||||
out_node.out_port(0).get_connection().set_source(gelu.out_port(0))
|
||||
rename_nodes([(out_node, node_name + '/TBD'), (gelu, node_name)])
|
||||
@@ -23,7 +23,7 @@ from mo.utils.ir_engine.compare_graphs import compare_graphs
|
||||
from mo.utils.unittest.graph import build_graph, const, regular_op, result, build_graph
|
||||
|
||||
ref_nodes = {**regular_op('input', {'type': 'Parameter'}),
|
||||
**regular_op('gelu', {'type': 'Gelu', 'name': 'final_mul'}),
|
||||
**regular_op('gelu', {'type': 'Gelu', 'approximation': 'erf', 'name': 'final_mul'}),
|
||||
**result('result')
|
||||
}
|
||||
ref_edges = [('input', 'gelu'), ('gelu', 'result')]
|
||||
@@ -65,6 +65,7 @@ class GeLUMergerErfTest(unittest.TestCase):
|
||||
|
||||
(flag, resp) = compare_graphs(graph, graph_ref, 'result')
|
||||
self.assertTrue(flag, resp)
|
||||
self.assertTrue(graph.get_op_nodes(op='Gelu')[0].approximation == 'erf')
|
||||
self.assertTrue(len(graph.get_op_nodes(name='final_mul')) == 1 and
|
||||
graph.get_op_nodes(name='final_mul')[0].op == 'Gelu')
|
||||
|
||||
@@ -89,6 +90,7 @@ class GeLUMergerErfTest(unittest.TestCase):
|
||||
|
||||
(flag, resp) = compare_graphs(graph, graph_ref, 'result')
|
||||
self.assertTrue(flag, resp)
|
||||
self.assertTrue(graph.get_op_nodes(op='Gelu')[0].approximation == 'erf')
|
||||
self.assertTrue(len(graph.get_op_nodes(name='final_mul')) == 1 and
|
||||
graph.get_op_nodes(name='final_mul')[0].op == 'Gelu')
|
||||
|
||||
@@ -113,5 +115,6 @@ class GeLUMergerErfTest(unittest.TestCase):
|
||||
|
||||
(flag, resp) = compare_graphs(graph, graph_ref, 'result')
|
||||
self.assertTrue(flag, resp)
|
||||
self.assertTrue(graph.get_op_nodes(op='Gelu')[0].approximation == 'erf')
|
||||
self.assertTrue(len(graph.get_op_nodes(name='final_mul')) == 1 and
|
||||
graph.get_op_nodes(name='final_mul')[0].op == 'Gelu')
|
||||
@@ -74,6 +74,6 @@ class GeLUMergerTanh(FrontReplacementSubgraph):
|
||||
# check that the values match the approximation
|
||||
if fabs(mul0_param - sqrt2pi) < 1e-06 and fabs(mul_param - 0.044715) < 1e-06 and mul1_param == 0.5:
|
||||
log.debug('Confirmed TanH-based GELU pattern after {} with name {}'.format(inp.op, inp.name))
|
||||
gelu = GeLUOP(graph, dict(name=inp.name + '/GELU_')).create_node()
|
||||
gelu = GeLUOP(graph, dict(name=inp.name + '/GELU_', approximation='tanh')).create_node()
|
||||
inp_port.connect(gelu.in_port(0))
|
||||
match['mul2'].out_port(0).get_connection().set_source(gelu.out_port(0))
|
||||
+6
-36
@@ -15,11 +15,10 @@
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
from math import sqrt
|
||||
|
||||
from extensions.front.GeLUMerger_Erf import GeLUMergerErf
|
||||
import numpy as np
|
||||
|
||||
from extensions.front.GeLUMerger_Tanh import GeLUMergerTanh
|
||||
from mo.utils.ir_engine.compare_graphs import compare_graphs
|
||||
from mo.utils.unittest.graph import build_graph
|
||||
@@ -55,42 +54,13 @@ nodes_attributes_tanh = {
|
||||
|
||||
nodes_attributes_ref = {
|
||||
'inp': {'kind': 'op', 'op': 'AnyOp'},
|
||||
'gelu': {'kind': 'op', 'op': 'Gelu'},
|
||||
'gelu': {'kind': 'op', 'op': 'Gelu', 'approximation': 'tanh'},
|
||||
'out': {'kind': 'op', 'op': 'AnyOp'},
|
||||
}
|
||||
|
||||
|
||||
class TestGeLUMergerReplacement(unittest.TestCase):
|
||||
def test_GeLUMergerErf_test_1(self):
|
||||
graph = build_graph(nodes_attributes_erf,
|
||||
[('inp', 'mul0', {'out': 0}),
|
||||
('inp', 'div', {'out': 0}),
|
||||
('mul', 'mul0'),
|
||||
('div', 'erf'),
|
||||
('erf', 'add'),
|
||||
('add', 'mul0'),
|
||||
('mul_param', 'mul'),
|
||||
('div_param', 'div'),
|
||||
('add_param', 'add'),
|
||||
('mul0', 'out'),
|
||||
],
|
||||
{'mul_param': {'shape': np.array([1]), 'value': np.array(0.5)},
|
||||
'add_param': {'shape': np.array([1]), 'value': np.array(1.0)},
|
||||
'div_param': {'shape': np.array([1]), 'value': np.array(sqrt(2.0))}
|
||||
},
|
||||
nodes_with_edges_only=True)
|
||||
graph_ref = build_graph(nodes_attributes_ref,
|
||||
[('inp', 'gelu'),
|
||||
('gelu', 'out')],
|
||||
{}, nodes_with_edges_only=True)
|
||||
graph.stage = 'front'
|
||||
|
||||
replacer = GeLUMergerErf()
|
||||
replacer.find_and_replace_pattern(graph)
|
||||
|
||||
(flag, resp) = compare_graphs(graph, graph_ref, 'out', check_op_attrs=True)
|
||||
self.assertTrue(flag, resp)
|
||||
|
||||
def test_GeLUMergerTanh_test_2(self):
|
||||
def test_GeLUMergerTanh(self):
|
||||
graph = build_graph(nodes_attributes_tanh,
|
||||
[('inp', 'mul2', {'out': 0}),
|
||||
('inp', 'add', {'out': 0}),
|
||||
@@ -122,4 +92,4 @@ class TestGeLUMergerReplacement(unittest.TestCase):
|
||||
replacer.find_and_replace_pattern(graph)
|
||||
|
||||
(flag, resp) = compare_graphs(graph, graph_ref, 'out', check_op_attrs=True)
|
||||
self.assertTrue(flag, resp)
|
||||
self.assertTrue(flag, resp)
|
||||
@@ -15,6 +15,7 @@
|
||||
"""
|
||||
|
||||
from extensions.ops.activation_ops import Elu, LeakyReLU, ReLU
|
||||
from extensions.ops.gelu import GeLUOP
|
||||
from extensions.ops.prelu import PReLU
|
||||
from mo.front.extractor import FrontExtractorOp
|
||||
from mo.front.mxnet.extractors.utils import get_mxnet_layer_attrs
|
||||
@@ -50,6 +51,8 @@ class LeakyReLUFrontExtractor(FrontExtractorOp):
|
||||
ReLU.update_node_stat(node)
|
||||
else:
|
||||
LeakyReLU.update_node_stat(node, {'negative_slope': negative_slope})
|
||||
elif act_type == 'gelu':
|
||||
GeLUOP.update_node_stat(node, {'approximation': 'erf'})
|
||||
else:
|
||||
raise Error(
|
||||
"Operation '{}' not supported. Please register it as custom op. " +
|
||||
|
||||
@@ -24,11 +24,17 @@ class GeLUOP(Op):
|
||||
|
||||
def __init__(self, graph: Graph, attrs: dict):
|
||||
mandatory_props = {
|
||||
'type': __class__.op,
|
||||
'op': __class__.op,
|
||||
'type': self.op,
|
||||
'op': self.op,
|
||||
'in_ports_count': 1,
|
||||
'out_ports_count': 1,
|
||||
'version': 'opset2',
|
||||
'version': 'opset7',
|
||||
'infer': copy_shape_infer
|
||||
}
|
||||
super().__init__(graph, mandatory_props, attrs)
|
||||
super().__init__(graph, mandatory_props, attrs)
|
||||
|
||||
def backend_attrs(self):
|
||||
if self.get_opset() == 'opset7':
|
||||
return ['approximation']
|
||||
else:
|
||||
return []
|
||||
Reference in new issue
Block a user