IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[MO] compress_to_fp16=True by default (2nd attempt) (#18652)

Browse Source 
* [MO] compress_to_fp16=True by default (2dn attempt)

* fix unit-tests

* second round of fixin unit-tests

* set compress_to_fp16 default to True in ovc/cli_parser.py

* use save_model in mo_python_api_tests

* enforce compress_to_fp16=False in test_zero_copy

* selectively compress depending on the path user has chosen to generate IR

* corrected doc

* allow compress_to_fp16=False/True for ovc

* doc and unit-tests failing fix

* user save_model in ovc cli tool

* revert back serialize and compress_model but into main instead of moc_emit_ir

* cover more argument combinations for cli tool and convert_model
This commit is contained in:
Pavel Esir
2023-07-27 12:32:50 +04:00
committed by GitHub
parent a0a1102499
commit 98df87099a
12 changed files with 267 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/MO_DG/prepare_model/FP16_Compression.md
+6 -5
View File
@@ -2,12 +2,13 @@
@sphinxdirective
Optionally, all relevant floating-point weights can be compressed to ``FP16`` data type during model conversion.
By default, when IR is saved all relevant floating-point weights are compressed to ``FP16`` data type during model conversion.
It results in creating a "compressed ``FP16`` model", which occupies about half of
the original space in the file system. The compression may introduce a minor drop in accuracy,
but it is negligible for most models.
In case if accuracy drop is significant user can disable compression explicitly.
To compress the model, use the ``compress_to_fp16=True`` option:
To disable compression, use the ``compress_to_fp16=False`` option:
.. tab-set::
@@ -17,15 +18,15 @@ To compress the model, use the ``compress_to_fp16=True`` option:
.. code-block:: py
:force:
from openvino.tools.mo import convert_model
ov_model = convert_model(INPUT_MODEL, compress_to_fp16=True)
from openvino.runtime import save_model
ov_model = save_model(INPUT_MODEL, compress_to_fp16=False)
.. tab-item:: CLI
:sync: cli
.. code-block:: sh
mo --input_model INPUT_MODEL --compress_to_fp16=True
mo --input_model INPUT_MODEL --compress_to_fp16=False
For details on how plugins handle compressed ``FP16`` models, see
tests/layer_tests/common/layer_test_class.py
+1 -1
View File
@@ -148,7 +148,7 @@ class CommonLayerTest:
# It is possible to redefine this function and generate your own input
def _prepare_input(self, inputs_dict):
for input in inputs_dict.keys():
inputs_dict[input] = np.random.randint(-255, 255, inputs_dict[input]).astype(np.float32)
inputs_dict[input] = np.random.randint(-10, 10, inputs_dict[input]).astype(np.float32)
return inputs_dict
def compare_ie_results_with_framework(self, infer_res, framework_res, framework_eps):
tests/layer_tests/common/mo_convert_test_class.py
+9 -3
View File
@@ -3,7 +3,7 @@
from pathlib import Path
from openvino.runtime import serialize
from openvino.runtime import serialize, save_model
from openvino.tools.ovc import convert_model
from openvino.tools.mo import convert_model as legacy_convert_model
from openvino.test_utils import compare_functions
@@ -22,9 +22,15 @@ class CommonMOConvertTest:
if 'use_convert_model_from_mo' in kwargs:
del kwargs['use_convert_model_from_mo']
model = legacy_convert_model(**kwargs)
else:
model = convert_model(**kwargs)
serialize(model, str(Path(output_dir, model_name + '.xml')))
else:
# ovc.convert_model does not have 'compress_to_fp16' arg, it's moved into save model
compress_to_fp16 = True
if 'compress_to_fp16' in kwargs:
compress_to_fp16 = kwargs['compress_to_fp16']
del kwargs['compress_to_fp16']
model = convert_model(**kwargs)
save_model(model, str(Path(output_dir, model_name + '.xml')), compress_to_fp16)
def _test(self, temp_dir, test_params, ref_params):
"""
tests/layer_tests/mo_python_api_tests/test_mo_convert_complex_params.py
+55 -20
View File
@@ -134,11 +134,12 @@ class TestComplexParams(CommonMOConvertTest):
{'params_test': {'input_shape': [PartialShape([2, 3, 4]),
[2, 3, 4],
[Dimension(2), Dimension(3), Dimension(4)]],
'input':['Input1', 'Input2', 'Relu3'], 'use_convert_model_from_mo': True},
'input':['Input1', 'Input2', 'Relu3'], 'use_convert_model_from_mo': True, 'compress_to_fp16': True},
'params_ref': {'input_shape': "[2,3,4],[2,3,4],[2,3,4]", 'input': 'Input1,Input2,Relu3'}},
{'params_test': {'input_shape': [PartialShape([Dimension(), Dimension(1, 3), Dimension(4, -1), Dimension(-1, 5)]),
[Dimension(), Dimension(1, 3), 4, Dimension(-1, 5)],
[Dimension(), 3, Dimension(4, -1), Dimension(-1, 5)]], 'use_convert_model_from_mo': True,
'compress_to_fp16': True,
'input':['Input1', 'Input2', 'Relu3']},
'params_ref': {'input_shape': "[?,1..3,4..,..5],[?,1..3,4,..5],[?,3,4..,..5]", 'input': 'Input1,Input2,Relu3'}},
{'params_test': {'input': [InputCutInfo("Relu1", Shape([3, 2]), Type(np.int32)),
@@ -152,27 +153,32 @@ class TestComplexParams(CommonMOConvertTest):
{'params_test': {'output': ["Sigmoid_0", "Sigmoid_2"]},
'params_ref': {'output': "Sigmoid_0,Sigmoid_2"}},
{'params_test': {'mean_values': {'Input1': [0.5,1.3,0.67], 'Input2':[4.2, 6.7, 3.15], 'Input3':[0.757, 4.6, 7.3]},
'use_convert_model_from_mo': True},
'use_convert_model_from_mo': True, 'compress_to_fp16': True},
'params_ref': {'mean_values': "Input1[0.5,1.3,0.67],Input2[4.2,6.7,3.15],Input3[0.757,4.6,7.3]"}},
{'params_test': {
'mean_values': [[0.5, 1.3, 0.67], [4.2, 6.7, 3.15], [0.757, 4.6, 7.3]], 'use_convert_model_from_mo': True},
'mean_values': [[0.5, 1.3, 0.67], [4.2, 6.7, 3.15], [0.757, 4.6, 7.3]], 'use_convert_model_from_mo': True,
'compress_to_fp16': True},
'params_ref': {'mean_values': "[0.5,1.3,0.67],[4.2,6.7,3.15],[0.757,4.6,7.3]"}},
{'params_test': {'scale_values': {'Input1': [0.5,1.3,0.67], 'Input2':[4.2, 6.7, 3.15], 'Input3':[0.757, 4.6, 7.3]},
'use_convert_model_from_mo': True},
'use_convert_model_from_mo': True, 'compress_to_fp16': True},
'params_ref': {'scale_values': "Input1[0.5,1.3,0.67],Input2[4.2,6.7,3.15],Input3[0.757,4.6,7.3]"}},
{'params_test': {
'scale_values': [[0.5, 1.3, 0.67], [4.2, 6.7, 3.15], [0.757, 4.6, 7.3]], 'use_convert_model_from_mo': True},
'scale_values': [[0.5, 1.3, 0.67], [4.2, 6.7, 3.15], [0.757, 4.6, 7.3]], 'use_convert_model_from_mo': True,
'compress_to_fp16': True},
'params_ref': {'scale_values': "[0.5,1.3,0.67],[4.2,6.7,3.15],[0.757,4.6,7.3]"}},
{'params_test': {
'source_layout': {'Input1': Layout("nchw"), 'Input2': "nchw", 'Input3': "nc??"}, 'use_convert_model_from_mo': True},
'source_layout': {'Input1': Layout("nchw"), 'Input2': "nchw", 'Input3': "nc??"}, 'use_convert_model_from_mo': True,
'compress_to_fp16': True},
'params_ref': {'source_layout': "Input1(nchw),Input2(nchw),Input3(nc??)"}},
{'params_test': {
'target_layout': {'Input1': Layout("nhwc"), 'Input2': "nhwc", 'Input3': "n??c"}, 'use_convert_model_from_mo': True},
'target_layout': {'Input1': Layout("nhwc"), 'Input2': "nhwc", 'Input3': "n??c"}, 'use_convert_model_from_mo': True,
'compress_to_fp16': True},
'params_ref': {'target_layout': "Input1(nhwc),Input2(nhwc),Input3(n??c)"}},
{'params_test': {
'layout': {'Input1': LayoutMap(source_layout=Layout("nchw"), target_layout="nhwc"),
'Input2': LayoutMap(source_layout="nc??", target_layout=Layout("n??c")),
'Input3': LayoutMap(source_layout="abcd", target_layout="acdb")}, 'use_convert_model_from_mo': True},
'Input3': LayoutMap(source_layout="abcd", target_layout="acdb")}, 'use_convert_model_from_mo': True,
'compress_to_fp16': True},
'params_ref': {'layout': "Input1(nchw->nhwc),Input2(nc??->n??c),Input3(abcd->acdb)"}},
{'params_test': {'input': [PartialShape([2, 3, 4]), [2, 3, 4], [Dimension(2), Dimension(3), Dimension(4)]]},
'params_ref': {'input_shape': "[2,3,4],[2,3,4],[2,3,4]", 'input': 'Input1,Input2,Input3'}},
@@ -226,14 +232,42 @@ class TestComplexParams(CommonMOConvertTest):
test_params = params['params_test']
ref_params = params['params_ref']
test_params.update({'input_model': tf_net_path})
test_params.update({'use_convert_model_from_mo': True})
test_params.update({'use_convert_model_from_mo': True, 'compress_to_fp16': True})
ref_params.update({'input_model': tf_net_path})
self._test(temp_dir, test_params, ref_params)
test_data = [
{'params_test': {'input_shape': PartialShape([2, 3, 4]), 'use_convert_model_from_mo': True},
# When use_convert_model_from_mo=True legacy openvino.tools.mo.convert_model is used
# By default compress_to_fp16 in Python API is False but for mo cli tool (used for params_ref) it's True.
# compress_to_fp16 should be specified explicitly either in 'param_test' or 'params_ref' (or in both)
# Check all args combinations.
{'params_test': {'input_shape': PartialShape([2, 3, 4]), 'use_convert_model_from_mo': True,
'compress_to_fp16': True},
'params_ref': {'input_shape': "[2,3,4]"}},
{'params_test': {'input_shape': [Dimension(), Dimension(1, 3), 4, Dimension(-1, 5)], 'use_convert_model_from_mo': True},
{'params_test': {'input_shape': PartialShape([2, 3, 4]), 'use_convert_model_from_mo': True},
'params_ref': {'input_shape': "[2,3,4]", 'compress_to_fp16': False}},
{'params_test': {'input_shape': PartialShape([2, 3, 4]), 'use_convert_model_from_mo': True,
'compress_to_fp16': True},
'params_ref': {'input_shape': "[2,3,4]", 'compress_to_fp16': True}},
{'params_test': {'input_shape': PartialShape([2, 3, 4]), 'use_convert_model_from_mo': True,
'compress_to_fp16': False},
'params_ref': {'input_shape': "[2,3,4]", 'compress_to_fp16': False}},
# ovc.convert_model with save_model are used, by default save_model compresses to fp16 same as cli tool.
# Check all args combinations.
{'params_test': {'input': InputCutInfo("Relu", [3, 2], Type(np.int32), [1, 2, 3, 4, 5, 6])},
'params_ref': {'input': "Relu[3 2]{i32}->[1 2 3 4 5 6]"}},
{'params_test': {'input': InputCutInfo("Relu", [3, 2], Type(np.int32), [1, 2, 3, 4, 5, 6]), 'compress_to_fp16': True},
'params_ref': {'input': "Relu[3 2]{i32}->[1 2 3 4 5 6]"}},
{'params_test': {'input': InputCutInfo("Relu", [3, 2], Type(np.int32), [1, 2, 3, 4, 5, 6])},
'params_ref': {'input': "Relu[3 2]{i32}->[1 2 3 4 5 6]", 'compress_to_fp16': True}},
{'params_test': {'input': InputCutInfo("Relu", [3, 2], Type(np.int32), [1, 2, 3, 4, 5, 6]), 'compress_to_fp16': True},
'params_ref': {'input': "Relu[3 2]{i32}->[1 2 3 4 5 6]", 'compress_to_fp16': True}},
{'params_test': {'input': InputCutInfo("Relu", [3, 2], Type(np.int32), [1, 2, 3, 4, 5, 6]), 'compress_to_fp16': False},
'params_ref': {'input': "Relu[3 2]{i32}->[1 2 3 4 5 6]", 'compress_to_fp16': False}},
{'params_test': {'input_shape': [Dimension(), Dimension(1, 3), 4, Dimension(-1, 5)], 'use_convert_model_from_mo': True,
'compress_to_fp16': True},
'params_ref': {'input_shape': "[?,1..3,4,..5]"}},
{'params_test': {'input': InputCutInfo("Relu", [3, 2], Type(np.int32), [1, 2, 3, 4, 5, 6])},
'params_ref': {'input': "Relu[3 2]{i32}->[1 2 3 4 5 6]"}},
@@ -245,17 +279,18 @@ class TestComplexParams(CommonMOConvertTest):
'params_ref': {'input': "Relu[3 2]"}},
{'params_test': {'input': ("Relu")},
'params_ref': {'input': "Relu"}},
{'params_test': {'mean_values': [0.5, 1.3, 0.67], 'use_convert_model_from_mo': True},
{'params_test': {'mean_values': [0.5, 1.3, 0.67], 'use_convert_model_from_mo': True, 'compress_to_fp16': True},
'params_ref': {'mean_values': "[0.5,1.3,0.67]"}},
{'params_test': {'scale_values': [0.5, 1.3, 0.67], 'use_convert_model_from_mo': True},
{'params_test': {'scale_values': [0.5, 1.3, 0.67], 'use_convert_model_from_mo': True, 'compress_to_fp16': True},
'params_ref': {'scale_values': "[0.5,1.3,0.67]"}},
{'params_test': {'source_layout': Layout("nchw"), 'use_convert_model_from_mo': True},
{'params_test': {'source_layout': Layout("nchw"), 'use_convert_model_from_mo': True, 'compress_to_fp16': True},
'params_ref': {'source_layout': "nchw"}},
{'params_test': {'target_layout': Layout("nchw"), 'use_convert_model_from_mo': True},
{'params_test': {'target_layout': Layout("nchw"), 'use_convert_model_from_mo': True, 'compress_to_fp16': True},
'params_ref': {'target_layout': "nchw"}},
{'params_test': {'layout': LayoutMap(source_layout=Layout("nchw"), target_layout="nhwc"), 'use_convert_model_from_mo': True},
{'params_test': {'layout': LayoutMap(source_layout=Layout("nchw"), target_layout="nhwc"),
'use_convert_model_from_mo': True, 'compress_to_fp16': True},
'params_ref': {'layout': "nchw->nhwc"}},
{'params_test': {'layout': Layout("nchw"), 'use_convert_model_from_mo': True},
{'params_test': {'layout': Layout("nchw"), 'use_convert_model_from_mo': True, 'compress_to_fp16': True},
'params_ref': {'layout': "nchw"}},
{'params_test': {'input': [3, 2]},
'params_ref': {'input': "Input[3 2]"}},
@@ -271,13 +306,13 @@ class TestComplexParams(CommonMOConvertTest):
'params_ref': {'input': "Input[1]{i32}->[10]"}},
{'params_test': {'input': (np.int32, [1, 2, 3])},
'params_ref': {'input': "Input[1,2,3]{i32}"}},
{'params_test': {'input_shape': [Dimension(3, 10), 10, -1], 'use_convert_model_from_mo': True},
{'params_test': {'input_shape': [Dimension(3, 10), 10, -1], 'use_convert_model_from_mo': True, 'compress_to_fp16': True},
'params_ref': {'input_shape': '[3..10,10,?]'}},
{'params_test': {'input': [Dimension(3, 10), 10, -1]},
'params_ref': {'input': 'Input[3..10,10,?]'}},
{'params_test': {'input': PartialShape([1, 100, 100, 3]), 'mean_values': [0.5, 1.3, 0.67], 'use_convert_model_from_mo': True},
{'params_test': {'input': PartialShape([1, 100, 100, 3]), 'mean_values': [0.5, 1.3, 0.67], 'use_convert_model_from_mo': True, 'compress_to_fp16': True},
'params_ref': {'input': "Input[1,100,100,3]", 'mean_values': "[0.5,1.3,0.67]"}},
{'params_test': {'input': [1, 100, 100, 3], 'scale_values': [0.5, 1.3, 0.67], 'use_convert_model_from_mo': True},
{'params_test': {'input': [1, 100, 100, 3], 'scale_values': [0.5, 1.3, 0.67], 'use_convert_model_from_mo': True, 'compress_to_fp16': True},
'params_ref': {'input': "Input[1,100,100,3]", 'scale_values': "[0.5,1.3,0.67]"}},
]
tests/layer_tests/mo_python_api_tests/test_mo_convert_onnx.py
+3 -2
View File
@@ -49,8 +49,9 @@ def make_graph_proto_model():
def create_ref_model(shape):
param1 = ov.opset8.parameter(shape, dtype=np.float32)
slope_const = ov.opset8.constant([0.1], dtype=np.float32)
prelu = ov.opset8.prelu(param1, slope=slope_const)
slope_const = ov.opset8.constant([0.1], dtype=np.float16)
decompress_slope = ov.opset8.convert(slope_const, np.float32)
prelu = ov.opset8.prelu(param1, slope=decompress_slope)
relu = ov.opset8.elu(prelu, alpha=np.float32(0.1))
parameter_list = [param1]
return Model([relu], parameter_list, "test")
tests/layer_tests/mo_python_api_tests/test_mo_convert_pytorch.py
+149 -21
View File
@@ -333,8 +333,7 @@ def create_pytorch_nn_module_mean_list(tmp_dir):
'use_convert_model_from_mo': True}
def create_pytorch_nn_module_mean_list_default_no_compression(tmp_dir):
# by default compression is disabled (same as setting 'compress_to_fp16': False)
def create_pytorch_nn_module_mean_list_compression_disabled(tmp_dir):
pt_model = make_pt_model_two_inputs()
shape = [1, 10, 10, 3]
@@ -352,7 +351,32 @@ def create_pytorch_nn_module_mean_list_default_no_compression(tmp_dir):
parameter_list = [param1, param2]
ref_model = Model([sigm], parameter_list, "test")
return pt_model, ref_model, {'input_shape': [shape, shape], 'mean_values': [[0, 0, 0], [0, 0, 0]], 'use_convert_model_from_mo': True}
return pt_model, ref_model, {'input_shape': [shape, shape], 'mean_values': [[0, 0, 0], [0, 0, 0]],
'compress_to_fp16': False, 'use_convert_model_from_mo': True}
def create_pytorch_nn_module_mean_list_compression_default(tmp_dir):
# when 'use_convert_model_from_mo': True by default compression in convert_model is disabled
# therefore decompression Converts will not be present
pt_model = make_pt_model_two_inputs()
shape = [1, 10, 10, 3]
shape = PartialShape(shape)
param1 = ov.opset8.parameter(shape)
param2 = ov.opset8.parameter(shape)
const1 = ov.opset8.constant([[[[-0.0, -0.0, -0.0]]]], dtype=np.float32)
const2 = ov.opset8.constant([[[[-0.0, -0.0, -0.0]]]], dtype=np.float32)
add1 = ov.opset8.add(param1, const1)
add2 = ov.opset8.add(param2, const2)
mul = ov.opset8.multiply(add1, add2)
relu = ov.opset8.relu(mul)
sigm = ov.opset8.sigmoid(relu)
parameter_list = [param1, param2]
ref_model = Model([sigm], parameter_list, "test")
return pt_model, ref_model, {'input_shape': [shape, shape], 'mean_values': [[0, 0, 0], [0, 0, 0]],
'use_convert_model_from_mo': True}
def create_pytorch_nn_module_mean_list_compression_enabled(tmp_dir):
@@ -362,10 +386,15 @@ def create_pytorch_nn_module_mean_list_compression_enabled(tmp_dir):
shape = PartialShape(shape)
param1 = ov.opset8.parameter(shape)
param2 = ov.opset8.parameter(shape)
const1 = ov.opset8.constant([[[[-0.0, -0.0, -0.0]]]], dtype=np.float32)
const2 = ov.opset8.constant([[[[-0.0, -0.0, -0.0]]]], dtype=np.float32)
add1 = ov.opset8.add(param1, const1)
add2 = ov.opset8.add(param2, const2)
const1 = ov.opset8.constant([[[[-0.0, -0.0, -0.0]]]], dtype=np.float16)
const2 = ov.opset8.constant([[[[-0.0, -0.0, -0.0]]]], dtype=np.float16)
const1_decompressed = ov.opset8.convert(
const1, destination_type=np.float32)
const2_decompressed = ov.opset8.convert(
const2, destination_type=np.float32)
add1 = ov.opset8.add(param1, const1_decompressed)
add2 = ov.opset8.add(param2, const2_decompressed)
mul = ov.opset8.multiply(add1, add2)
relu = ov.opset8.relu(mul)
sigm = ov.opset8.sigmoid(relu)
@@ -375,7 +404,7 @@ def create_pytorch_nn_module_mean_list_compression_enabled(tmp_dir):
return pt_model, ref_model, {
'input_shape': [shape, shape], 'mean_values': [[0, 0, 0], [0, 0, 0]],
'compress_to_fp16': False, 'use_convert_model_from_mo': True}
'compress_to_fp16': True, 'use_convert_model_from_mo': True}
def create_pytorch_nn_module_scale_list(tmp_dir):
@@ -400,8 +429,7 @@ def create_pytorch_nn_module_scale_list(tmp_dir):
'use_convert_model_from_mo': True}
def create_pytorch_nn_module_scale_list_default_no_compression(tmp_dir):
# by default compression is disabled (same as setting 'compress_to_fp16': False)
def create_pytorch_nn_module_scale_list_compression_disabled(tmp_dir):
pt_model = make_pt_model_two_inputs()
shape = [1, 10, 10, 3]
@@ -419,7 +447,32 @@ def create_pytorch_nn_module_scale_list_default_no_compression(tmp_dir):
parameter_list = [param1, param2]
ref_model = Model([sigm], parameter_list, "test")
return pt_model, ref_model, {'input_shape': [shape, shape], 'scale_values': [[1, 1, 1], [1, 1, 1]], 'use_convert_model_from_mo': True}
return pt_model, ref_model, {'input_shape': [shape, shape], 'scale_values': [[1, 1, 1], [1, 1, 1]],
'compress_to_fp16': False, 'use_convert_model_from_mo': True}
def create_pytorch_nn_module_scale_list_compression_default(tmp_dir):
# when 'use_convert_model_from_mo': True by default compression in convert_model is disabled
# therefore decompression Converts will not be present
pt_model = make_pt_model_two_inputs()
shape = [1, 10, 10, 3]
shape = PartialShape(shape)
param1 = ov.opset8.parameter(shape)
param2 = ov.opset8.parameter(shape)
const1 = ov.opset8.constant([[[[1, 1, 1]]]], dtype=np.float32)
const2 = ov.opset8.constant([[[[1, 1, 1]]]], dtype=np.float32)
sub1 = ov.opset8.multiply(param1, const1)
sub2 = ov.opset8.multiply(param2, const2)
mul = ov.opset8.multiply(sub1, sub2)
relu = ov.opset8.relu(mul)
sigm = ov.opset8.sigmoid(relu)
parameter_list = [param1, param2]
ref_model = Model([sigm], parameter_list, "test")
return pt_model, ref_model, {'input_shape': [shape, shape], 'scale_values': [[1, 1, 1], [1, 1, 1]],
'use_convert_model_from_mo': True}
def create_pytorch_nn_module_scale_list_compression_enabled(tmp_dir):
@@ -646,8 +699,9 @@ def create_pytorch_module_convert_pytorch_frontend_oob(tmp_dir):
net = ConvModel()
shape = PartialShape([-1, 3, -1, -1])
param1 = ov.opset10.parameter(shape, dtype=np.float32)
weights = ov.opset10.constant(net.weights.numpy(force=True))
conv = ov.opset10.convolution(param1, weights, strides=[1, 1],
weights = ov.opset10.constant(net.weights.numpy(force=True), dtype=np.float16)
decompress_weights = ov.opset10.convert(weights, np.float32)
conv = ov.opset10.convolution(param1, decompress_weights, strides=[1, 1],
pads_begin=[0, 0], pads_end=[0, 0],
dilations=[1, 1])
parameter_list = [param1]
@@ -695,6 +749,43 @@ def create_pytorch_module_with_optional_inputs_case5(tmp_dir):
return net, ref_model, {"input": [("x",[1, 3, -1, -1]), ("z", [1, 3, -1, -1])]}
def create_pytorch_module_with_compressed_int8_constant_compress_to_fp16_default(tmp_dir):
import torch
import torch.nn.functional as F
class Int8Model(torch.nn.Module):
def __init__(self):
super(Int8Model, self).__init__()
self.weights = torch.randint(-127, 128,
[1, 3, 3, 3], dtype=torch.int8)
def forward(self, x):
cast = self.weights.to(torch.float32)
sub = cast - 0.5
mul = sub * 0.02
return F.conv2d(x, mul)
net = Int8Model()
example_input = (torch.rand((1, 3, 10, 10)),)
traced_model = torch.jit.trace(net, example_input)
shape = [-1, -1, -1, -1]
shape = PartialShape(shape)
param1 = ov.opset10.parameter(shape, dtype=np.float32)
weights = ov.opset10.constant(net.weights.numpy(force=True))
cast1 = ov.opset10.convert(weights, np.float32)
sub1_const = np.float16(0.5).reshape(1, 1, 1, 1)
mul1_const = np.float16(0.02).reshape(1, 1, 1, 1)
sub1_const_decompress = ov.opset10.convert(sub1_const, np.float32)
mul1_const_decompress = ov.opset10.convert(mul1_const, np.float32)
sub1 = ov.opset10.subtract(cast1, sub1_const_decompress)
mul1 = ov.opset10.multiply(sub1, mul1_const_decompress)
conv = ov.opset10.convolution(param1, mul1, strides=[1, 1],
pads_begin=[0, 0], pads_end=[0, 0],
dilations=[1, 1])
ref_model = Model([conv], [param1], "test")
return traced_model, ref_model, {"example_input": example_input}
def create_pytorch_module_with_compressed_int8_constant(tmp_dir):
import torch
import torch.nn.functional as F
@@ -725,7 +816,8 @@ def create_pytorch_module_with_compressed_int8_constant(tmp_dir):
pads_begin=[0, 0], pads_end=[0, 0],
dilations=[1, 1])
ref_model = Model([conv], [param1], "test")
return traced_model, ref_model, {"example_input": example_input}
return traced_model, ref_model, {"example_input": example_input, "compress_to_fp16": False}
def create_pytorch_module_with_nested_inputs(tmp_dir):
class PTModel(torch.nn.Module):
@@ -746,6 +838,31 @@ def create_pytorch_module_with_nested_inputs(tmp_dir):
concat1 = ov.opset10.concat([param1, constant_zeros1], 1)
concat2 = ov.opset10.concat([param2, constant_zeros2], 2)
ref_model = Model([concat2, concat1], [param1, param2], "test")
return net, ref_model, {"example_input": {"z": (torch.zeros((1, 10)), torch.ones((1, 5, 2)))},
"compress_to_fp16": False}
def create_pytorch_module_with_nested_inputs_compress_to_fp16_default(tmp_dir):
class PTModel(torch.nn.Module):
def forward(self, z:Tuple[torch.Tensor, torch.Tensor]):
z1, z2 = z
zeros1 = torch.zeros((1, 1))
zeros2 = torch.zeros((1, 5, 1))
return torch.cat([z1, zeros1], 1), torch.cat([z2, zeros2], 2)
net = PTModel()
constant_zeros1 = ov.opset10.constant(np.zeros((1, 1), dtype=np.float32), dtype=np.float16)
constant_zeros2 = ov.opset10.constant(np.zeros((1, 5, 1), dtype=np.float32), dtype=np.float16)
const1_decompress = ov.opset10.convert(constant_zeros1, np.float32)
const2_decompress = ov.opset10.convert(constant_zeros2, np.float32)
shape1 = PartialShape([1, -1])
shape2 = PartialShape([1, 5, -1])
param1 = ov.opset10.parameter(shape1, dtype=np.float32)
param2 = ov.opset10.parameter(shape2, dtype=np.float32)
concat1 = ov.opset10.concat([param1, const1_decompress], 1)
concat2 = ov.opset10.concat([param2, const2_decompress], 2)
ref_model = Model([concat2, concat1], [param1, param2], "test")
return net, ref_model, {"example_input": {"z": (torch.zeros((1, 10)), torch.ones((1, 5, 2)))}}
@@ -770,7 +887,9 @@ def create_pytorch_module_with_nested_inputs2(tmp_dir):
concat2 = ov.opset10.concat([param2, constant_zeros2], 2)
add = ov.opset10.add(concat1, param0)
ref_model = Model([concat2, add], [param0, param1, param2], "test")
return net, ref_model, {"example_input": {"x": torch.ones((1, 10)), "z": (torch.zeros((1, 10)), torch.ones((1, 5, 5)))}}
return net, ref_model, {"example_input": {"x": torch.ones((1, 10)), "z": (torch.zeros((1, 10)), torch.ones((1, 5, 5)))},
"compress_to_fp16": False}
def create_pytorch_module_with_nested_inputs3(tmp_dir):
class PTModel(torch.nn.Module):
@@ -793,7 +912,8 @@ def create_pytorch_module_with_nested_inputs3(tmp_dir):
concat2 = ov.opset10.concat([param2, constant_zeros2], 2)
add = ov.opset10.add(concat1, param3)
ref_model = Model([concat2, add], [param1, param2, param3], "test")
return net, ref_model, {"example_input": {"x": torch.ones((1, 10)), "z": (torch.zeros((1, 10)), torch.ones((1, 5, 3)))}}
return net, ref_model, {"example_input": {"x": torch.ones((1, 10)), "z": (torch.zeros((1, 10)), torch.ones((1, 5, 3)))},
"compress_to_fp16": False}
def create_pytorch_module_with_nested_inputs4(tmp_dir):
@@ -819,7 +939,9 @@ def create_pytorch_module_with_nested_inputs4(tmp_dir):
add = ov.opset10.add(concat1, param3)
mul = ov.opset10.multiply(concat2, param4)
ref_model = Model([mul, add], [param3, param1, param2, param4], "test")
return net, ref_model, {"example_input": {"x": torch.ones((1, 10)), "z": (torch.zeros((1, 10)), torch.ones((1, 5, 10))), "y": torch.ones((1,))}}
return net, ref_model, {"example_input": {"x": torch.ones((1, 10)), "z": (torch.zeros((1, 10)), torch.ones((1, 5, 10))), "y": torch.ones((1,))},
"compress_to_fp16": False}
def create_pytorch_module_with_nested_inputs5(tmp_dir):
class PTModel(torch.nn.Module):
@@ -844,7 +966,9 @@ def create_pytorch_module_with_nested_inputs5(tmp_dir):
add = ov.opset10.add(concat1, param0)
mul = ov.opset10.multiply(concat2, param4)
ref_model = Model([mul, add], [param0, param1, param2, param4], "test")
return net, ref_model, {"example_input": [torch.ones((1, 10)), (torch.zeros((1, 10)), torch.ones((1, 5, 10))), torch.ones((1,))]}
return net, ref_model, {"example_input": [torch.ones((1, 10)), (torch.zeros((1, 10)), torch.ones((1, 5, 10))), torch.ones((1,))],
"compress_to_fp16": False}
def create_pytorch_module_with_nested_inputs6(tmp_dir):
class PTModel(torch.nn.Module):
@@ -869,7 +993,8 @@ def create_pytorch_module_with_nested_inputs6(tmp_dir):
concat2 = ov.opset10.concat([param2, constant_zeros2], 2)
add1 = ov.opset10.add(concat1, param0)
ref_model = Model([concat2, add1], [param0, param1, param2], "test")
return net, ref_model, {"example_input": {"x": torch.ones((1, 11)), "z": (torch.zeros((1, 10)), torch.ones((1, 5, 10)))}}
return net, ref_model, {"example_input": {"x": torch.ones((1, 11)), "z": (torch.zeros((1, 10)), torch.ones((1, 5, 10)))},
"compress_to_fp16": False}
class TestMoConvertPyTorch(CommonMOConvertTest):
@@ -889,10 +1014,12 @@ class TestMoConvertPyTorch(CommonMOConvertTest):
create_pytorch_nn_module_layout_list,
create_pytorch_nn_module_layout_list_case2,
create_pytorch_nn_module_mean_list,
create_pytorch_nn_module_mean_list_default_no_compression,
create_pytorch_nn_module_mean_list_compression_default,
create_pytorch_nn_module_mean_list_compression_disabled,
create_pytorch_nn_module_mean_list_compression_enabled,
create_pytorch_nn_module_scale_list,
create_pytorch_nn_module_scale_list_default_no_compression,
create_pytorch_nn_module_scale_list_compression_default,
create_pytorch_nn_module_scale_list_compression_disabled,
create_pytorch_nn_module_scale_list_compression_enabled,
create_pytorch_nn_module_shapes_list_static,
create_pytorch_nn_module_shapes_list_static_via_input,
@@ -916,6 +1043,7 @@ class TestMoConvertPyTorch(CommonMOConvertTest):
create_pytorch_module_with_optional_inputs_case5,
create_pytorch_nn_module_with_scalar_input,
create_pytorch_module_with_compressed_int8_constant,
create_pytorch_module_with_compressed_int8_constant_compress_to_fp16_default,
create_pytorch_module_with_nested_inputs,
create_pytorch_module_with_nested_inputs2,
create_pytorch_module_with_nested_inputs3,
tests/layer_tests/mo_python_api_tests/test_mo_convert_tf.py
+31 -3
View File
@@ -505,6 +505,18 @@ def two_params_function_reference(shapes, const_value):
return Model([mul], parameter_list, "test")
def two_params_function_reference_fp16_compressed(shapes, const_value):
param1 = ov.opset8.parameter(shapes[0], dtype=np.float32)
param2 = ov.opset8.parameter(shapes[1], dtype=np.float32)
const_value = ov.opset8.constant(const_value, dtype=np.float16)
const_decompress = ov.opset8.convert(const_value, np.float32)
sigm = ov.opset8.sigmoid(param1)
add = ov.opset8.add(sigm, param2)
mul = ov.opset8.multiply(add, const_decompress)
parameter_list = [param1, param2]
return Model([mul], parameter_list, "test")
def create_keras_layer_with_example_input_1(tmp_dir):
model, model_ref = create_keras_layer_input_list()
example_input = (np.random.rand(1,2,3).astype(np.float32), np.random.rand(1,2,3).astype(np.float32))
@@ -550,6 +562,22 @@ def create_keras_layer_with_tf_function_call(tmp_dir):
return sigm * self.var1
model = LayerModel()
model_ref = two_params_function_reference([[1, 2], [1, 2]], [[5.0]])
return model, model_ref, {'compress_to_fp16': False}
def create_keras_layer_with_tf_function_call_default_compressed_to_fp16(tmp_dir):
import tensorflow as tf
class LayerModel(tf.Module):
def __init__(self):
super(LayerModel, self).__init__()
self.var1 = tf.Variable(5.0)
@tf.function(input_signature=[tf.TensorSpec([1, 2], tf.float32), tf.TensorSpec([1, 2], tf.float32)])
def __call__(self, input1, input2):
sigm = tf.nn.sigmoid(input1) + input2
return sigm * self.var1
model = LayerModel()
model_ref = two_params_function_reference_fp16_compressed([[1, 2], [1, 2]], [[5.0]])
return model, model_ref, {}
@@ -568,7 +596,7 @@ def create_keras_layer_with_tf_function_call_no_signature(tmp_dir):
example_input = [np.random.rand(2, 3).astype(np.float32), np.random.rand(2, 3).astype(np.float32)]
model_ref = two_params_function_reference([[2, 3], [2, 3]], [[5.0]])
return model, model_ref, {'example_input': example_input}
return model, model_ref, {'example_input': example_input, 'compress_to_fp16': False}
def create_keras_layer_with_tf_function_call_no_signature_single_input(tmp_dir):
@@ -586,7 +614,7 @@ def create_keras_layer_with_tf_function_call_no_signature_single_input(tmp_dir):
example_input = np.random.rand(2, 3).astype(np.float32)
model_ref = single_param_function_reference([2, 3], [[5.0]])
return model, model_ref, {'example_input': example_input}
return model, model_ref, {'example_input': example_input, 'compress_to_fp16': False}
def create_keras_layer_with_string_tensor(tmp_dir):
@@ -631,6 +659,7 @@ class TestMoConvertTF(CommonMOConvertTest):
create_keras_layer_with_input_shapes_case3,
create_keras_layer_with_input_shapes_case4,
create_keras_layer_with_tf_function_call,
create_keras_layer_with_tf_function_call_default_compressed_to_fp16,
create_keras_layer_with_tf_function_call_no_signature,
create_keras_layer_with_tf_function_call_no_signature_single_input,
create_keras_layer_with_string_tensor,
@@ -641,7 +670,6 @@ class TestMoConvertTF(CommonMOConvertTest):
create_tf1_wrap_function,
create_tf_session,
]
test_data_legacy = [
# TF2
create_keras_model,
tools/mo/openvino/tools/mo/utils/cli_parser.py
+6 -1
View File
@@ -805,13 +805,18 @@ def add_args_by_description(args_group, params_description):
# Bool params common setting
if signature.parameters[param_name].annotation == bool and param_name != 'version':
default_flag = signature.parameters[param_name].default
# tools.mo.convert_model by default does not compress,
# but if we convert from cli we need to compress_to_fp16 if user did not specify otherwise
if param_name == 'compress_to_fp16':
default_flag = True
args_group.add_argument(
cli_param_name, *param_alias,
type=check_bool if param_type is None else param_type,
nargs="?",
const=True,
help=help_text,
default=signature.parameters[param_name].default)
default=default_flag)
# File paths common setting
elif param_name in filepath_args:
action = action if action is not None else CanonicalizePathCheckExistenceAction
tools/mo/openvino/tools/mo/utils/get_ov_update_message.py
+2 -2
View File
@@ -38,8 +38,8 @@ def get_tf_fe_message():
def get_compression_message():
link = "https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_FP16_Compression.html"
message = '[ INFO ] Generated IR will be compressed to FP16. ' \
'If you get lower accuracy, please consider disabling compression ' \
'by removing argument "compress_to_fp16" or set it to false "compress_to_fp16=False".\n' \
'If you get lower accuracy, please consider disabling compression explicitly ' \
'by adding argument --compress_to_fp16=False.\n' \
'Find more information about compression to FP16 at {}'.format(link)
return message
tools/ovc/openvino/tools/ovc/cli_parser.py
+1 -1
View File
@@ -646,7 +646,7 @@ def get_common_cli_parser(parser: argparse.ArgumentParser = None):
# Command line tool specific params
common_group.add_argument('--output_model',
help='This parameter is used to name output .xml/.bin files with converted model.')
common_group.add_argument('--compress_to_fp16', action='store_true',
common_group.add_argument('--compress_to_fp16', type=check_bool, default=True,
help='Compress weights in output IR .xml/bin files to FP16.')
common_group.add_argument('--version', action='version',
help='Print ovc version and exit.',
tools/ovc/openvino/tools/ovc/main.py
+5
View File
@@ -26,6 +26,11 @@ def main():
model_path_no_ext = os.path.normpath(os.path.join(output_dir, argv.output_model))
model_path = model_path_no_ext + '.xml'
# TODO: replace compress_model + serialize with save_model
if argv.compress_to_fp16:
from openvino.tools.ovc.moc_frontend.offline_transformations import compress_model
compress_model(ngraph_function)
serialize(ngraph_function, model_path.encode('utf-8'), model_path.replace('.xml', '.bin').encode('utf-8'))
print('[ SUCCESS ] XML file: {}'.format(model_path))
tools/ovc/openvino/tools/ovc/moc_frontend/moc_emit_ir.py
-5
View File
@@ -26,11 +26,6 @@ def moc_emit_ir(ngraph_function: Model, argv: argparse.Namespace):
else list(argv.placeholder_data_types.keys())
apply_moc_legacy_transformations(ngraph_function, params_with_custom_types)
# TODO: Move compression to save_model at the level of main function where serialize is called
if not argv.is_python_api_used and argv.compress_to_fp16:
from openvino.tools.ovc.moc_frontend.offline_transformations import compress_model
compress_model(ngraph_function)
apply_fused_names_cleanup(ngraph_function)
del argv.feManager