IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

CompressQuantizeWeights - use f32 precision when computing scale and zero point (#16794)

Browse Source 
Ticket: 101825
This commit is contained in:
Mateusz Tabaka
2023-04-12 12:42:39 +02:00
committed by GitHub
parent 496fe7a7db
commit bdd79fe931
4 changed files with 135 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/offline_transformations/src/compress_quantize_weigths.cpp
+24 -8
View File
@@ -95,13 +95,13 @@ ngraph::pass::CompressQuantizeWeights::CompressQuantizeWeights() {
output_high = levels - 1 + output_low
The FakeQuantize result is converted to low precision type and then constant folded
*/
std::shared_ptr<Node> new_input_low;
auto new_output_low = op::Constant::create(input_type, Shape{}, {-static_cast<float>(levels / 2)});
auto new_output_high =
std::shared_ptr<Node> new_output_low =
op::Constant::create(input_type, Shape{}, {-static_cast<float>(levels / 2)});
std::shared_ptr<Node> new_output_high =
std::make_shared<opset8::Add>(new_output_low, op::Constant::create(input_type, Shape{}, {levels - 1}));
const auto& weights_const = pattern_value_map.at(weights_const_pattern);
const auto& input_low = pattern_value_map.at(input_low_pattern);
const auto& input_high = pattern_value_map.at(input_high_pattern);
Output<Node> input_low = pattern_value_map.at(input_low_pattern);
Output<Node> input_high = pattern_value_map.at(input_high_pattern);
const auto& fq_data_input = pattern_value_map.count(weigths_convert_pattern)
? pattern_value_map.at(weigths_convert_pattern)
: weights_const;
@@ -143,8 +143,18 @@ ngraph::pass::CompressQuantizeWeights::CompressQuantizeWeights() {
scale = (output_high - output_low) / (new_output_high - new_output_low)
zero_point = new_output_low - output_low / scale
*/
const auto& output_low = pattern_value_map.at(output_low_pattern);
const auto& output_high = pattern_value_map.at(output_high_pattern);
Output<Node> output_low = pattern_value_map.at(output_low_pattern);
Output<Node> output_high = pattern_value_map.at(output_high_pattern);
const auto& fq_type = fq->get_output_element_type(0);
const bool should_convert = fq_type.is_real() && fq_type.size() < element::f32.size();
if (should_convert) {
input_low = std::make_shared<opset8::Convert>(input_low, element::f32);
input_high = std::make_shared<opset8::Convert>(input_high, element::f32);
output_low = std::make_shared<opset8::Convert>(output_low, element::f32);
output_high = std::make_shared<opset8::Convert>(output_high, element::f32);
new_output_low = std::make_shared<opset8::Convert>(new_output_low, element::f32);
new_output_high = std::make_shared<opset8::Convert>(new_output_high, element::f32);
}
auto output_range = std::make_shared<opset8::Subtract>(output_high, output_low);
auto input_range = std::make_shared<opset8::Subtract>(new_output_high, new_output_low);
std::shared_ptr<Node> scale = std::make_shared<opset8::Divide>(output_range, input_range);
@@ -155,11 +165,17 @@ ngraph::pass::CompressQuantizeWeights::CompressQuantizeWeights() {
OPENVINO_SUPPRESS_DEPRECATED_END
scale = constant;
}
auto zero = op::Constant::create(input_type, Shape{}, {0});
auto zero = op::Constant::create(scale->get_output_element_type(0), Shape{}, {0});
auto scale_eq_zero = std::make_shared<opset8::Equal>(scale, zero);
// shift equals to input_low - output_low / scale
// for positions where scale == 0, we put zero as shift
std::shared_ptr<Node> zero_point = std::make_shared<opset8::Select>(scale_eq_zero, zero, shift);
if (should_convert) {
scale = std::make_shared<opset8::Convert>(scale, fq_type);
zero_point = std::make_shared<opset8::Convert>(zero_point, fq_type);
}
OPENVINO_SUPPRESS_DEPRECATED_START
if (auto constant = ov::get_constant_from_source(zero_point)) {
OPENVINO_SUPPRESS_DEPRECATED_END
src/common/transformations/tests/utils/compress_quantize_weights.cpp
+30
View File
@@ -184,6 +184,36 @@ TEST_F(TransformationTestsF, CompressQuantizeWeightsWithZeroPointOptimizer) {
comparator.enable(FunctionsComparator::CmpValues::ACCURACY);
}
TEST_F(TransformationTestsF, CompressQuantizeWeightsWithZeroPointOptimizerFP16) {
{
auto data = opset8::Constant::create(element::f16, Shape{3, 1, 1, 1}, {0.2, 1.2, 1.2});
auto input_low =
opset8::Constant::create(element::f16, Shape{3, 1, 1, 1}, {0.59033203125, 1.4833984375, 1.2900390625});
auto input_high =
opset8::Constant::create(element::f16, Shape{3, 1, 1, 1}, {-0.59033203125, -1.4833984375, -1.2900390625});
auto output_low =
opset8::Constant::create(element::f16, Shape{3, 1, 1, 1}, {0.295166015625, 0.74169921875, 0.64501953125});
auto output_high = opset8::Constant::create(element::f16,
Shape{3, 1, 1, 1},
{-0.295166015625, -0.74169921875, -0.64501953125});
auto fq = std::make_shared<opset8::FakeQuantize>(data, input_low, input_high, output_low, output_high, 255);
function = std::make_shared<Function>(NodeVector{fq}, ParameterVector{});
manager.register_pass<pass::CompressQuantizeWeights>();
manager.register_pass<pass::ZeroPointOptimizer>();
}
{
auto data = opset8::Constant::create(element::i8, Shape{3, 1, 1, 1}, {-43, -103, -118});
auto convert = std::make_shared<opset8::Convert>(data, element::f16);
auto scale = opset8::Constant::create(element::f16, Shape{3, 1, 1, 1}, {-0.002325, -0.00584, -0.005077});
auto mul = std::make_shared<opset8::Multiply>(convert, scale);
function_ref = std::make_shared<Function>(NodeVector{mul}, ParameterVector{});
}
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
comparator.enable(FunctionsComparator::CmpValues::ACCURACY);
}
TEST_F(TransformationTestsF, NegativeCompressQuantizeWeightsWithZeroPointOptimizer) {
{
auto data = opset8::Constant::create(element::f32, Shape{2, 4, 1, 1}, {-1, 0, 1, 2, 3, 4, 5, 11});
tools/mo/openvino/tools/mo/back/compress_quantized_weights.py
+28
View File
@@ -7,6 +7,7 @@ import numpy as np
from openvino.tools.mo.ops.Cast import Cast
from openvino.tools.mo.ops.elementwise import Sub, Div, Mul, Equal
from openvino.tools.mo.ops.Cast import Cast
from openvino.tools.mo.ops.select import Select
from openvino.tools.mo.back.replacement import BackReplacementPattern
from openvino.tools.mo.front.common.partial_infer.utils import mo_array
@@ -183,6 +184,24 @@ class CompressQuantizeWeights(BackReplacementPattern):
out_low = fake_quantize.in_port(3).get_source()
out_high = fake_quantize.in_port(4).get_source()
need_cast_to_f32 = fake_quantize.out_port(0).is_data_type_defined() and fake_quantize.out_port(0).get_data_type() < np.float32
if need_cast_to_f32:
in_low_cast = Cast(graph, {'name': name + '/in_low/convert_to_f32', 'dst_type': np.float32}).create_node()
in_low_cast.in_port(0).connect(in_low)
in_low = in_low_cast.out_port(0)
in_high_cast = Cast(graph, {'name': name + '/in_high/convert_to_f32', 'dst_type': np.float32}).create_node()
in_high_cast.in_port(0).connect(in_high)
in_high = in_high_cast.out_port(0)
out_low_cast = Cast(graph, {'name': name + '/out_low/convert_to_f32', 'dst_type': np.float32}).create_node()
out_low_cast.in_port(0).connect(out_low)
out_low = out_low_cast.out_port(0)
out_high_cast = Cast(graph, {'name': name + '/out_high/convert_to_f32', 'dst_type': np.float32}).create_node()
out_high_cast.in_port(0).connect(out_high)
out_high = out_high_cast.out_port(0)
# scale calculation
output_range = Sub(graph, {'name': name + '/output_range'}).create_node()
output_range.in_port(0).connect(out_high)
@@ -215,6 +234,15 @@ class CompressQuantizeWeights(BackReplacementPattern):
zero_point.in_port(1).connect(zero.out_port(0))
zero_point.in_port(2).connect(shift.out_port(0))
if need_cast_to_f32:
fq_dtype = fake_quantize.out_port(0).get_data_type()
scale_cast = Cast(graph, {'name': name + '/scale/convert_back', 'dst_type': fq_dtype}).create_node()
scale_cast.in_port(0).connect(scale.out_port(0))
scale = scale_cast
zero_point_cast = Cast(graph, {'name': name + '/zero_point/convert_back', 'dst_type': fq_dtype}).create_node()
zero_point_cast.in_port(0).connect(zero_point.out_port(0))
zero_point = zero_point_cast
# DeQuantize(x) == Mul(Sub(x, zero_point), scale)
sub_zp = Sub(graph, {'name': name + '/minus_zp'}).create_node()
sub_zp.in_port(0).connect(dequantizing_cast.out_port(0))
tools/mo/unit_tests/mo/back/compress_quantized_weights_test.py
+53 -4
View File
@@ -13,14 +13,18 @@ from openvino.tools.mo.ops.elementwise import Sub, Mul
from openvino.tools.mo.ops.fakequantize import FakeQuantize
from openvino.tools.mo.front.common.partial_infer.eltwise import eltwise_infer
from openvino.tools.mo.utils.ir_engine.compare_graphs import compare_graphs
from openvino.tools.mo.middle.passes.infer import type_infer
from unit_tests.utils.graph import build_graph, regular_op_with_shaped_data, valued_const_with_data, result, connect, \
shaped_const_with_data
def nodes_dict(original, transformed=None, levels=255, data=None, il=[-127], ih=[127], ol=[-127], oh=[127]):
def nodes_dict(original, transformed=None, levels=255, data=None,
il=[-127], ih=[127], ol=[-127], oh=[127],
scale=np.array([1]), zp=np.array([0]), int_data=None):
shape = [1, 2, 3, 4] if data is None else np.array(data).shape
data = np.ones(shape, dtype=original) if data is None else np.array(data, dtype=original)
int_data = data.astype(dtype=np.int8)
if int_data is None:
int_data = data.astype(dtype=np.int8)
transformed = transformed if transformed is not None else original
return {
@@ -42,8 +46,8 @@ def nodes_dict(original, transformed=None, levels=255, data=None, il=[-127], ih=
'FQ', shape, {'type': 'FakeQuantize', 'infer': FakeQuantize.infer, 'stop_value_propagation': True,
'levels': levels, 'op': 'FakeQuantize'}),
**valued_const_with_data('zp', np.array([0])),
**valued_const_with_data('scale', np.array([1])),
**valued_const_with_data('zp', zp),
**valued_const_with_data('scale', scale),
**regular_op_with_shaped_data(
'sub', shape, {'type': 'Subtract', 'op': 'Sub', 'infer': lambda node: eltwise_infer(node, Sub.operation)}),
@@ -233,6 +237,51 @@ class CompressionDataTypeTest(unittest.TestCase):
self.assertTrue(flag, resp)
def test_fp16_fake_quantize(self):
original_type = np.float16
input_low = np.array([-0.59033203125, -1.4833984375, -1.2900390625], dtype=np.float16)
input_high = np.array([0.59033203125, 1.4833984375, 1.2900390625], dtype=np.float16)
output_low = np.array([0.295166015625, 0.74169921875, 0.64501953125], dtype=np.float16)
output_high = np.array([-0.295166015625, -0.74169921875, -0.64501953125], dtype=np.float16)
scale = np.array([-0.002325, -0.00584, -0.005077], dtype=np.float16)
int_data = np.array([43, 103, 118], dtype=np.int8)
nodes = nodes_dict(original_type, transformed=np.int8,
levels=255, data=np.array([0.2, 1.2, 1.2], dtype=np.float16),
il=input_low, ih=input_high, ol=output_low, oh=output_high, scale=scale, int_data=int_data)
graph = build_graph(nodes, [
*connect('weights:0', '0:FQ'),
*connect('il:0', '1:FQ'),
*connect('ih:0', '2:FQ'),
*connect('ol:0', '3:FQ'),
*connect('oh:0', '4:FQ'),
*connect('FQ:0', 'output'),
], nodes_with_edges_only=True)
type_infer(graph)
error_message = 'Unexpected number of {} nodes {} CompressQuantizeWeights.dequantize_data call `{}`'
fq_nodes = graph.get_op_nodes(type='FakeQuantize')
self.assertEqual(len(fq_nodes), 1, error_message.format('FakeQuantize', 'before', len(fq_nodes)))
CompressQuantizeWeights().find_and_replace_pattern(graph)
graph.clean_up()
ZeroPointOptimizer().find_and_replace_pattern(graph)
graph.clean_up()
fq_nodes = graph.get_op_nodes(type='FakeQuantize')
self.assertEqual(len(fq_nodes), 0, error_message.format('FakeQuantize', 'after', len(fq_nodes)))
graph_ref = build_graph(nodes, [
*connect('int_weights:0', '0:cast'),
*connect('cast:0', '0:mul'),
*connect('scale:0', '1:mul'),
*connect('mul:0', 'output'),
], {'cast': {'dst_type': original_type}}, nodes_with_edges_only=True)
(flag, resp) = compare_graphs(graph, graph_ref, 'output', check_op_attrs=True)
self.assertTrue(flag, resp)
@generator
class AccuracyCheckFP32Test(unittest.TestCase):
eps = np.finfo(np.float32).eps