diff --git a/src/common/offline_transformations/src/compress_quantize_weigths.cpp b/src/common/offline_transformations/src/compress_quantize_weigths.cpp index a1d89520cbf..294178c979c 100644 --- a/src/common/offline_transformations/src/compress_quantize_weigths.cpp +++ b/src/common/offline_transformations/src/compress_quantize_weigths.cpp @@ -95,13 +95,13 @@ ngraph::pass::CompressQuantizeWeights::CompressQuantizeWeights() { output_high = levels - 1 + output_low The FakeQuantize result is converted to low precision type and then constant folded */ - std::shared_ptr new_input_low; - auto new_output_low = op::Constant::create(input_type, Shape{}, {-static_cast(levels / 2)}); - auto new_output_high = + std::shared_ptr new_output_low = + op::Constant::create(input_type, Shape{}, {-static_cast(levels / 2)}); + std::shared_ptr new_output_high = std::make_shared(new_output_low, op::Constant::create(input_type, Shape{}, {levels - 1})); const auto& weights_const = pattern_value_map.at(weights_const_pattern); - const auto& input_low = pattern_value_map.at(input_low_pattern); - const auto& input_high = pattern_value_map.at(input_high_pattern); + Output input_low = pattern_value_map.at(input_low_pattern); + Output input_high = pattern_value_map.at(input_high_pattern); const auto& fq_data_input = pattern_value_map.count(weigths_convert_pattern) ? pattern_value_map.at(weigths_convert_pattern) : weights_const; @@ -143,8 +143,18 @@ ngraph::pass::CompressQuantizeWeights::CompressQuantizeWeights() { scale = (output_high - output_low) / (new_output_high - new_output_low) zero_point = new_output_low - output_low / scale */ - const auto& output_low = pattern_value_map.at(output_low_pattern); - const auto& output_high = pattern_value_map.at(output_high_pattern); + Output output_low = pattern_value_map.at(output_low_pattern); + Output output_high = pattern_value_map.at(output_high_pattern); + const auto& fq_type = fq->get_output_element_type(0); + const bool should_convert = fq_type.is_real() && fq_type.size() < element::f32.size(); + if (should_convert) { + input_low = std::make_shared(input_low, element::f32); + input_high = std::make_shared(input_high, element::f32); + output_low = std::make_shared(output_low, element::f32); + output_high = std::make_shared(output_high, element::f32); + new_output_low = std::make_shared(new_output_low, element::f32); + new_output_high = std::make_shared(new_output_high, element::f32); + } auto output_range = std::make_shared(output_high, output_low); auto input_range = std::make_shared(new_output_high, new_output_low); std::shared_ptr scale = std::make_shared(output_range, input_range); @@ -155,11 +165,17 @@ ngraph::pass::CompressQuantizeWeights::CompressQuantizeWeights() { OPENVINO_SUPPRESS_DEPRECATED_END scale = constant; } - auto zero = op::Constant::create(input_type, Shape{}, {0}); + auto zero = op::Constant::create(scale->get_output_element_type(0), Shape{}, {0}); auto scale_eq_zero = std::make_shared(scale, zero); // shift equals to input_low - output_low / scale // for positions where scale == 0, we put zero as shift std::shared_ptr zero_point = std::make_shared(scale_eq_zero, zero, shift); + + if (should_convert) { + scale = std::make_shared(scale, fq_type); + zero_point = std::make_shared(zero_point, fq_type); + } + OPENVINO_SUPPRESS_DEPRECATED_START if (auto constant = ov::get_constant_from_source(zero_point)) { OPENVINO_SUPPRESS_DEPRECATED_END diff --git a/src/common/transformations/tests/utils/compress_quantize_weights.cpp b/src/common/transformations/tests/utils/compress_quantize_weights.cpp index c71a9754a2f..df5f60ece77 100644 --- a/src/common/transformations/tests/utils/compress_quantize_weights.cpp +++ b/src/common/transformations/tests/utils/compress_quantize_weights.cpp @@ -184,6 +184,36 @@ TEST_F(TransformationTestsF, CompressQuantizeWeightsWithZeroPointOptimizer) { comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } +TEST_F(TransformationTestsF, CompressQuantizeWeightsWithZeroPointOptimizerFP16) { + { + auto data = opset8::Constant::create(element::f16, Shape{3, 1, 1, 1}, {0.2, 1.2, 1.2}); + auto input_low = + opset8::Constant::create(element::f16, Shape{3, 1, 1, 1}, {0.59033203125, 1.4833984375, 1.2900390625}); + auto input_high = + opset8::Constant::create(element::f16, Shape{3, 1, 1, 1}, {-0.59033203125, -1.4833984375, -1.2900390625}); + auto output_low = + opset8::Constant::create(element::f16, Shape{3, 1, 1, 1}, {0.295166015625, 0.74169921875, 0.64501953125}); + auto output_high = opset8::Constant::create(element::f16, + Shape{3, 1, 1, 1}, + {-0.295166015625, -0.74169921875, -0.64501953125}); + auto fq = std::make_shared(data, input_low, input_high, output_low, output_high, 255); + function = std::make_shared(NodeVector{fq}, ParameterVector{}); + + manager.register_pass(); + manager.register_pass(); + } + + { + auto data = opset8::Constant::create(element::i8, Shape{3, 1, 1, 1}, {-43, -103, -118}); + auto convert = std::make_shared(data, element::f16); + auto scale = opset8::Constant::create(element::f16, Shape{3, 1, 1, 1}, {-0.002325, -0.00584, -0.005077}); + auto mul = std::make_shared(convert, scale); + function_ref = std::make_shared(NodeVector{mul}, ParameterVector{}); + } + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ACCURACY); +} + TEST_F(TransformationTestsF, NegativeCompressQuantizeWeightsWithZeroPointOptimizer) { { auto data = opset8::Constant::create(element::f32, Shape{2, 4, 1, 1}, {-1, 0, 1, 2, 3, 4, 5, 11}); diff --git a/tools/mo/openvino/tools/mo/back/compress_quantized_weights.py b/tools/mo/openvino/tools/mo/back/compress_quantized_weights.py index 8a65be057a4..8152ec0a1a6 100644 --- a/tools/mo/openvino/tools/mo/back/compress_quantized_weights.py +++ b/tools/mo/openvino/tools/mo/back/compress_quantized_weights.py @@ -7,6 +7,7 @@ import numpy as np from openvino.tools.mo.ops.Cast import Cast from openvino.tools.mo.ops.elementwise import Sub, Div, Mul, Equal +from openvino.tools.mo.ops.Cast import Cast from openvino.tools.mo.ops.select import Select from openvino.tools.mo.back.replacement import BackReplacementPattern from openvino.tools.mo.front.common.partial_infer.utils import mo_array @@ -183,6 +184,24 @@ class CompressQuantizeWeights(BackReplacementPattern): out_low = fake_quantize.in_port(3).get_source() out_high = fake_quantize.in_port(4).get_source() + need_cast_to_f32 = fake_quantize.out_port(0).is_data_type_defined() and fake_quantize.out_port(0).get_data_type() < np.float32 + if need_cast_to_f32: + in_low_cast = Cast(graph, {'name': name + '/in_low/convert_to_f32', 'dst_type': np.float32}).create_node() + in_low_cast.in_port(0).connect(in_low) + in_low = in_low_cast.out_port(0) + + in_high_cast = Cast(graph, {'name': name + '/in_high/convert_to_f32', 'dst_type': np.float32}).create_node() + in_high_cast.in_port(0).connect(in_high) + in_high = in_high_cast.out_port(0) + + out_low_cast = Cast(graph, {'name': name + '/out_low/convert_to_f32', 'dst_type': np.float32}).create_node() + out_low_cast.in_port(0).connect(out_low) + out_low = out_low_cast.out_port(0) + + out_high_cast = Cast(graph, {'name': name + '/out_high/convert_to_f32', 'dst_type': np.float32}).create_node() + out_high_cast.in_port(0).connect(out_high) + out_high = out_high_cast.out_port(0) + # scale calculation output_range = Sub(graph, {'name': name + '/output_range'}).create_node() output_range.in_port(0).connect(out_high) @@ -215,6 +234,15 @@ class CompressQuantizeWeights(BackReplacementPattern): zero_point.in_port(1).connect(zero.out_port(0)) zero_point.in_port(2).connect(shift.out_port(0)) + if need_cast_to_f32: + fq_dtype = fake_quantize.out_port(0).get_data_type() + scale_cast = Cast(graph, {'name': name + '/scale/convert_back', 'dst_type': fq_dtype}).create_node() + scale_cast.in_port(0).connect(scale.out_port(0)) + scale = scale_cast + zero_point_cast = Cast(graph, {'name': name + '/zero_point/convert_back', 'dst_type': fq_dtype}).create_node() + zero_point_cast.in_port(0).connect(zero_point.out_port(0)) + zero_point = zero_point_cast + # DeQuantize(x) == Mul(Sub(x, zero_point), scale) sub_zp = Sub(graph, {'name': name + '/minus_zp'}).create_node() sub_zp.in_port(0).connect(dequantizing_cast.out_port(0)) diff --git a/tools/mo/unit_tests/mo/back/compress_quantized_weights_test.py b/tools/mo/unit_tests/mo/back/compress_quantized_weights_test.py index 1d3ca9307fa..3f2afc477b8 100644 --- a/tools/mo/unit_tests/mo/back/compress_quantized_weights_test.py +++ b/tools/mo/unit_tests/mo/back/compress_quantized_weights_test.py @@ -13,14 +13,18 @@ from openvino.tools.mo.ops.elementwise import Sub, Mul from openvino.tools.mo.ops.fakequantize import FakeQuantize from openvino.tools.mo.front.common.partial_infer.eltwise import eltwise_infer from openvino.tools.mo.utils.ir_engine.compare_graphs import compare_graphs +from openvino.tools.mo.middle.passes.infer import type_infer from unit_tests.utils.graph import build_graph, regular_op_with_shaped_data, valued_const_with_data, result, connect, \ shaped_const_with_data -def nodes_dict(original, transformed=None, levels=255, data=None, il=[-127], ih=[127], ol=[-127], oh=[127]): +def nodes_dict(original, transformed=None, levels=255, data=None, + il=[-127], ih=[127], ol=[-127], oh=[127], + scale=np.array([1]), zp=np.array([0]), int_data=None): shape = [1, 2, 3, 4] if data is None else np.array(data).shape data = np.ones(shape, dtype=original) if data is None else np.array(data, dtype=original) - int_data = data.astype(dtype=np.int8) + if int_data is None: + int_data = data.astype(dtype=np.int8) transformed = transformed if transformed is not None else original return { @@ -42,8 +46,8 @@ def nodes_dict(original, transformed=None, levels=255, data=None, il=[-127], ih= 'FQ', shape, {'type': 'FakeQuantize', 'infer': FakeQuantize.infer, 'stop_value_propagation': True, 'levels': levels, 'op': 'FakeQuantize'}), - **valued_const_with_data('zp', np.array([0])), - **valued_const_with_data('scale', np.array([1])), + **valued_const_with_data('zp', zp), + **valued_const_with_data('scale', scale), **regular_op_with_shaped_data( 'sub', shape, {'type': 'Subtract', 'op': 'Sub', 'infer': lambda node: eltwise_infer(node, Sub.operation)}), @@ -233,6 +237,51 @@ class CompressionDataTypeTest(unittest.TestCase): self.assertTrue(flag, resp) + def test_fp16_fake_quantize(self): + original_type = np.float16 + input_low = np.array([-0.59033203125, -1.4833984375, -1.2900390625], dtype=np.float16) + input_high = np.array([0.59033203125, 1.4833984375, 1.2900390625], dtype=np.float16) + output_low = np.array([0.295166015625, 0.74169921875, 0.64501953125], dtype=np.float16) + output_high = np.array([-0.295166015625, -0.74169921875, -0.64501953125], dtype=np.float16) + scale = np.array([-0.002325, -0.00584, -0.005077], dtype=np.float16) + int_data = np.array([43, 103, 118], dtype=np.int8) + nodes = nodes_dict(original_type, transformed=np.int8, + levels=255, data=np.array([0.2, 1.2, 1.2], dtype=np.float16), + il=input_low, ih=input_high, ol=output_low, oh=output_high, scale=scale, int_data=int_data) + + graph = build_graph(nodes, [ + *connect('weights:0', '0:FQ'), + *connect('il:0', '1:FQ'), + *connect('ih:0', '2:FQ'), + *connect('ol:0', '3:FQ'), + *connect('oh:0', '4:FQ'), + *connect('FQ:0', 'output'), + ], nodes_with_edges_only=True) + type_infer(graph) + + error_message = 'Unexpected number of {} nodes {} CompressQuantizeWeights.dequantize_data call `{}`' + fq_nodes = graph.get_op_nodes(type='FakeQuantize') + self.assertEqual(len(fq_nodes), 1, error_message.format('FakeQuantize', 'before', len(fq_nodes))) + + CompressQuantizeWeights().find_and_replace_pattern(graph) + graph.clean_up() + ZeroPointOptimizer().find_and_replace_pattern(graph) + graph.clean_up() + + fq_nodes = graph.get_op_nodes(type='FakeQuantize') + self.assertEqual(len(fq_nodes), 0, error_message.format('FakeQuantize', 'after', len(fq_nodes))) + + graph_ref = build_graph(nodes, [ + *connect('int_weights:0', '0:cast'), + *connect('cast:0', '0:mul'), + *connect('scale:0', '1:mul'), + *connect('mul:0', 'output'), + ], {'cast': {'dst_type': original_type}}, nodes_with_edges_only=True) + + (flag, resp) = compare_graphs(graph, graph_ref, 'output', check_op_attrs=True) + self.assertTrue(flag, resp) + + @generator class AccuracyCheckFP32Test(unittest.TestCase): eps = np.finfo(np.float32).eps