[MO] Range output_type correction for FP16 (#6590)
* added ChangeRangeOutputType.py * applied review comments * corrected error message - warn user to use FP32 * renamed ChangeCastOutputType.py et ell. * merged ChangeRangeOutputType.py, ChangeCastOutputType.py into a singe file * corrections * typo fix * applied comments: faster find_and_replace loop, wording correction
This commit is contained in:
Pavel Esir authored and 
GitHub committed
1 parent
a177eb2f2e
commit
bfca47ad5e
6 files changed
+223
-47
No files matched your search
@@ -11,7 +11,7 @@ extensions/back/__init__.py
|
||||
extensions/back/AvgPool.py
|
||||
extensions/back/blob_normalizer.py
|
||||
extensions/back/CellNormalizer.py
|
||||
extensions/back/ChangeCastOutputType.py
|
||||
extensions/back/ChangeOutputTypeAttributes.py
|
||||
extensions/back/ClampNormalizer.py
|
||||
extensions/back/compress_quantized_weights.py
|
||||
extensions/back/ConvolutionNormalizer.py
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
# Copyright (C) 2018-2021 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging as log
|
||||
|
||||
import numpy as np
|
||||
|
||||
from mo.back.replacement import BackReplacementPattern
|
||||
from mo.graph.graph import Graph
|
||||
from mo.middle.passes.convert_data_type import data_type_str_to_np
|
||||
|
||||
|
||||
class ChangeCastOutputType(BackReplacementPattern):
|
||||
"""
|
||||
Change the Cast dst_type from fp64 to fp32 since not all plugins support fp64 data type.
|
||||
Change the Cast dst_type from fp32 to fp16 when generating IR for fp16.
|
||||
But leave fp32 if node returns shape value even if --data_type=FP16 (look extensions/back/MarkNodesWithShapeValues.py).
|
||||
"""
|
||||
enabled = True
|
||||
force_shape_inference = True
|
||||
|
||||
def run_after(self):
|
||||
from extensions.back.MarkNodesWithShapeValues import MarkNodesWithShapeValues
|
||||
return [MarkNodesWithShapeValues]
|
||||
|
||||
def run_before(self):
|
||||
return []
|
||||
|
||||
def find_and_replace_pattern(self, graph: Graph):
|
||||
for node in graph.get_op_nodes(op='Cast'):
|
||||
if node.dst_type == np.float64:
|
||||
log.warning('Change data type from {} to {} for node {}'.format(node.dst_type, np.float32, node.name))
|
||||
node.dst_type = np.float32
|
||||
|
||||
ir_data_type = data_type_str_to_np(node.graph.graph['cmd_params'].data_type)
|
||||
if node.dst_type == np.float32 and ir_data_type == np.float16 and not node.has_and_set('returns_shape_value'):
|
||||
log.warning('Change data type from {} to {} for node {}'.format(node.dst_type, ir_data_type, node.name))
|
||||
node.dst_type = ir_data_type
|
||||
elif node.has_and_set('returns_shape_value') and node.dst_type == np.float16:
|
||||
# return back FP32 for all Convert nodes with shape values
|
||||
log.warning('Change data type from {} to {} for node {} in ShapeOf subgraph'.
|
||||
format(node.dst_type, np.float32, node.name))
|
||||
node.dst_type = np.float32
|
||||
@@ -0,0 +1,100 @@
|
||||
# Copyright (C) 2018-2021 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging as log
|
||||
|
||||
import numpy as np
|
||||
|
||||
from mo.back.replacement import BackReplacementPattern
|
||||
from mo.graph.graph import Graph
|
||||
from mo.graph.graph import Node
|
||||
from mo.middle.passes.convert_data_type import data_type_str_to_np
|
||||
from mo.utils.error import Error
|
||||
|
||||
operations_with_data_type_attributes = {
|
||||
'Cast': {'attr_name': 'dst_type', 'in_ports_to_check': (0,)},
|
||||
'Range': {'attr_name': 'output_type', 'in_ports_to_check': (0, 1, 2)},
|
||||
}
|
||||
|
||||
|
||||
class ChangeOutputTypeAttributes(BackReplacementPattern):
|
||||
"""
|
||||
The transformation changes output type for the specific operations defined in the
|
||||
operations_with_data_type_attributes dictionary if one of the following conditions is met:
|
||||
- The operation output type is fp64. Since not all plugins support fp64 data type it is converted to fp32.
|
||||
- Changes output type from fp32 to fp16 (and ensure that this is possible) when generating fp16 IR.
|
||||
- Keep operation output type equal to fp32 for operations located in the shape calculation sub-graphs to
|
||||
avoid floating point overflow.
|
||||
"""
|
||||
enabled = True
|
||||
force_shape_inference = True
|
||||
|
||||
def run_after(self):
|
||||
from extensions.back.MarkNodesWithShapeValues import MarkNodesWithShapeValues
|
||||
return [MarkNodesWithShapeValues]
|
||||
|
||||
def run_before(self):
|
||||
return []
|
||||
|
||||
def find_and_replace_pattern(self, graph: Graph):
|
||||
ir_data_type = data_type_str_to_np(graph.graph['cmd_params'].data_type)
|
||||
|
||||
for node in graph.get_op_nodes():
|
||||
if node.op in operations_with_data_type_attributes:
|
||||
dst_type = operations_with_data_type_attributes[node.op]['attr_name']
|
||||
node_name = node.soft_get('name', node.id)
|
||||
assert node.has_valid(dst_type), '{} attribute is missing for node {}'.format(dst_type, node_name)
|
||||
|
||||
final_type = None
|
||||
if node[dst_type] == np.float64:
|
||||
final_type = np.float32
|
||||
|
||||
if node[dst_type] in [np.float32, np.float64] and ir_data_type == np.float16 and \
|
||||
not node.has_and_set('returns_shape_value'):
|
||||
final_type = np.float16
|
||||
elif node.has_and_set('returns_shape_value') and node.dst_type == np.float16:
|
||||
# return back FP32 for all nodes with shape values
|
||||
final_type = np.float32
|
||||
|
||||
if final_type is not None:
|
||||
log.warning('Change data type from {} to {} for node {}'.format(node[dst_type], final_type,
|
||||
node_name))
|
||||
node[dst_type] = final_type
|
||||
|
||||
if final_type == np.float16:
|
||||
assert_that_is_castable_to_fp16(node)
|
||||
|
||||
|
||||
def assert_that_is_castable_to_fp16(node: Node):
|
||||
op_name = node.soft_get('op')
|
||||
node_name = node.soft_get('name', node.id)
|
||||
|
||||
for i in operations_with_data_type_attributes[op_name]['in_ports_to_check']:
|
||||
val = node.in_port(i).data.get_value()
|
||||
if val is None:
|
||||
return
|
||||
|
||||
if np.any(val > np.finfo(np.float16).max) or np.any(val < np.finfo(np.float16).min):
|
||||
raise Error("Try to convert with --data_type=FP32 argument. "
|
||||
"This model can not be converted to FP16 precision, since "
|
||||
"'{}' node value {} exceeds FP16 allowed limits: [{}, {}]"
|
||||
.format(node_name, val, np.finfo(np.float16).min, np.finfo(np.float16).max))
|
||||
# further this input values will be rewritten since force_shape_inference=True
|
||||
node.in_port(i).data.set_value(val.astype(np.float16))
|
||||
|
||||
original_output = node.out_port(0).data.get_value()
|
||||
node.infer(node)
|
||||
casted_output = node.out_port(0).data.get_value()
|
||||
original_output_len = len(original_output) if hasattr(original_output, '__len__') else None
|
||||
casted_output_len = len(casted_output) if hasattr(casted_output, '__len__') else None
|
||||
|
||||
if original_output_len != casted_output_len:
|
||||
raise Error("Try to convert with --data_type=FP32 argument. "
|
||||
"This model can not be converted to FP16 precision, since "
|
||||
"after conversion of '{}' node to FP16 output shape {} differs from the original {}."
|
||||
.format(node_name, casted_output_len, original_output_len))
|
||||
|
||||
diff_count = np.count_nonzero(np.subtract(original_output, casted_output) > 1.e-4)
|
||||
if diff_count > 0:
|
||||
log.warning("{} elements of {} of Range node '{}' output differ from the original values while "
|
||||
"converting network to FP16 precision".format(diff_count, len(original_output), node_name))
|
||||
@@ -23,7 +23,6 @@ shape_accepting_ops = {
|
||||
'Tile': [1], # repeats input
|
||||
'TopK': [1], # K input
|
||||
'Pad': [1, 2], # pads_begin, pads_end
|
||||
'Range': [0, 1, 2], # start, stop, step inputs
|
||||
'OneHot': [1], # depth input
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# Copyright (C) 2018-2021 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging as log
|
||||
|
||||
import numpy as np
|
||||
|
||||
from mo.graph.graph import Node, Graph
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
# Copyright (C) 2018-2021 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import unittest
|
||||
from copy import deepcopy
|
||||
|
||||
import numpy as np
|
||||
|
||||
from extensions.back.ChangeOutputTypeAttributes import ChangeOutputTypeAttributes
|
||||
from extensions.ops.Cast import Cast
|
||||
from extensions.ops.range import Range
|
||||
from mo.front.common.partial_infer.utils import float32_array
|
||||
from mo.middle.passes.convert_data_type import convert_blobs, data_type_str_to_np
|
||||
from mo.middle.passes.infer import partial_infer
|
||||
from mo.utils.error import Error
|
||||
from mo.utils.ir_engine.compare_graphs import compare_graphs
|
||||
from unit_tests.utils.graph import build_graph, result, regular_op_with_empty_data, connect
|
||||
from unit_tests.utils.graph import valued_const_with_data
|
||||
|
||||
|
||||
class ChangeOutputTypeAttributesTests(unittest.TestCase):
|
||||
|
||||
def test_range_correct_case(self):
|
||||
graph, graph_ref = build_range_test_graphs(start=0, limit=10, delta=1, dst_type_str='FP16')
|
||||
ChangeOutputTypeAttributes().find_and_replace_pattern(graph)
|
||||
(flag, resp) = compare_graphs(graph, graph_ref, 'res', check_op_attrs=True)
|
||||
self.assertTrue(flag, resp)
|
||||
|
||||
# starting from ~1000 FP16 absolute difference between neighbor values is more than 1
|
||||
# fails because of shape inconsistency
|
||||
def test_range_different_values(self):
|
||||
graph, graph_ref = build_range_test_graphs(start=0, limit=50000, delta=1, dst_type_str='FP16')
|
||||
self.assertRaises(Error, ChangeOutputTypeAttributes().find_and_replace_pattern, graph)
|
||||
|
||||
def test_range_out_of_fp16_max(self):
|
||||
graph, graph_ref = build_range_test_graphs(start=0, limit=100000, delta=1, dst_type_str='FP16')
|
||||
self.assertRaises(Error, ChangeOutputTypeAttributes().find_and_replace_pattern, graph)
|
||||
|
||||
def test_range_out_of_fp16_min(self):
|
||||
graph, graph_ref = build_range_test_graphs(start=0, limit=-100000, delta=-1, dst_type_str='FP16')
|
||||
self.assertRaises(Error, ChangeOutputTypeAttributes().find_and_replace_pattern, graph)
|
||||
|
||||
def test_cast_correct_case(self):
|
||||
input_data = np.array([0, 1000, 4, 9, 0])
|
||||
graph, graph_ref = build_cast_test_graphs(input_data, dst_type_str='FP16')
|
||||
ChangeOutputTypeAttributes().find_and_replace_pattern(graph)
|
||||
(flag, resp) = compare_graphs(graph, graph_ref, 'res', check_op_attrs=True)
|
||||
self.assertTrue(flag, resp)
|
||||
|
||||
def test_cast_out_of_fp16_max(self):
|
||||
input_data = np.array([0, 100000, 4, 9, 0])
|
||||
graph, graph_ref = build_cast_test_graphs(input_data, dst_type_str='FP16')
|
||||
self.assertRaises(Error, ChangeOutputTypeAttributes().find_and_replace_pattern, graph)
|
||||
|
||||
def test_cast_out_of_fp16_min(self):
|
||||
input_data = np.array([0, -100000, 4, 9, 0])
|
||||
graph, graph_ref = build_cast_test_graphs(input_data, dst_type_str='FP16')
|
||||
self.assertRaises(Error, ChangeOutputTypeAttributes().find_and_replace_pattern, graph)
|
||||
|
||||
|
||||
def build_range_test_graphs(start=0, limit=10, delta=1, dst_type_str='FP16'):
|
||||
nodes = {
|
||||
**valued_const_with_data('start', float32_array(start)),
|
||||
**valued_const_with_data('limit', float32_array(limit)),
|
||||
**valued_const_with_data('delta', float32_array(delta)),
|
||||
**regular_op_with_empty_data('range', {'type': 'Range', 'op': 'Range',
|
||||
'output_type': np.float32,
|
||||
'infer': Range.infer}),
|
||||
**result('res'),
|
||||
}
|
||||
|
||||
nodes_ref = deepcopy(nodes)
|
||||
nodes_ref.update({
|
||||
**regular_op_with_empty_data('range', {'type': 'Range', 'op': 'Range',
|
||||
'output_type': data_type_str_to_np(dst_type_str),
|
||||
'infer': Range.infer}),
|
||||
})
|
||||
|
||||
edges = [
|
||||
*connect('start', '0:range'),
|
||||
*connect('limit', '1:range'),
|
||||
*connect('delta', '2:range'),
|
||||
*connect('range', 'res'),
|
||||
]
|
||||
graph = build_graph(nodes, edges)
|
||||
graph_ref = build_graph(nodes_ref, edges)
|
||||
|
||||
graph = partial_infer(graph)
|
||||
|
||||
graph.graph['cmd_params'].data_type = dst_type_str
|
||||
convert_blobs(graph, dst_type_str)
|
||||
return graph, graph_ref
|
||||
|
||||
|
||||
def build_cast_test_graphs(input_data, dst_type_str='FP16'):
|
||||
nodes = {
|
||||
**valued_const_with_data('input', float32_array(input_data)),
|
||||
**regular_op_with_empty_data('cast', {'type': 'Convert', 'op': 'Cast',
|
||||
'dst_type': np.float32,
|
||||
'infer': Cast.infer}),
|
||||
**result('res'),
|
||||
}
|
||||
|
||||
nodes_ref = deepcopy(nodes)
|
||||
nodes_ref.update({
|
||||
**regular_op_with_empty_data('cast', {'type': 'Convert', 'op': 'Cast',
|
||||
'dst_type': data_type_str_to_np(dst_type_str),
|
||||
'infer': Cast.infer}),
|
||||
})
|
||||
|
||||
edges = [
|
||||
*connect('input', 'cast'),
|
||||
*connect('cast', 'res'),
|
||||
]
|
||||
graph = build_graph(nodes, edges)
|
||||
graph_ref = build_graph(nodes_ref, edges)
|
||||
|
||||
graph = partial_infer(graph)
|
||||
|
||||
graph.graph['cmd_params'].data_type = dst_type_str
|
||||
convert_blobs(graph, dst_type_str)
|
||||
return graph, graph_ref
|
||||
Reference in new issue
Block a user