[PT FE] Add quantized::linear (#18570)
* Support GetAttr with packed params * Apply suggestions from code review * [PT FE] Add quantized types as normal types to decoder * [PT FE] Add decoder dequantize, add dtypes to quantize * [PT FE] Add dequantize example * [PT FE] Implement replacer for quantized nodes * [PT FE] Register replacer for quantize/dequantize * [PT FE] Remove unwanted junk from previous version * [PT FE] Fix building mistakes for frontend * [PT FE] Clang fix * [PT FE] Ease of use upgrade to quantize funcs * [PT FE] Clang format * [PT FE] Introduce new version of quantize/dequantize * [PT FE] Remove unwanted files from new version * [PT FE] Fix style * [PT FE] Add QuantizedPtNode replacer, fix accuracy error * [PT FE] Add improved version of quantize/dequantize with shared_ptrs * [PT FE] Fix utils shared ptr reference error * [PT FE] Quantize now takes correct input for operations * [PT FE] Upgrade quantize method * [PT FE] Add BFS for dequantize, add quantize_per_channel * [PT FE] Add missing replacer to frontend, improve tests * [PT FE] Rename replacer -> remover, remove unwanted header files * [PT FE] Change function declarations to return ov::Output instead of shared ptr * [PT FE] Add missing context mark node * [PT FE] Remove unknown modifications to ie_c_api * [PT FE] Remove fp16 support, turn off int32 tests * [PT FE] Clang format * [PT FE] Fix quantize_per_tensor * [PT FE] Minor fixes from review * [PT FE] Remove dequantize, remove helpers, replacer now removes nodes instead * [PT FE] Rename Replacer to Remover for dequantize nodes * [PT FE] Clang format * [PT FE] Move comments to header files, minor import fixes * [PT FE] Fix clang format * [PT FE] Fix dtype issue * [PT FE] Fix quantize_per_channel tests * Add quantized::linear * Update quantized_linear.cpp * add contructors * Improve tests * Add requested checks --------- Co-authored-by: Maxim Vafin <maxim.vafin@intel.com> Co-authored-by: Roboreptile <piotr.krzeminski@intel.com>
This commit is contained in:
Mateusz Mikolajczyk
co-authored by
Maxim Vafin
Roboreptile
Maxim Vafin
Roboreptile
parent
6d8dcb059d
commit
b315880cc3
@@ -7,6 +7,7 @@
|
||||
from openvino.frontend.pytorch.py_pytorch_frontend import _FrontEndPytorchDecoder as Decoder
|
||||
from openvino.frontend.pytorch.py_pytorch_frontend import _Type as DecoderType
|
||||
from openvino.runtime import op, PartialShape, Type as OVType, OVAny, Shape, Tensor
|
||||
from openvino.runtime import opset11 as ops
|
||||
|
||||
import typing
|
||||
from packaging.version import parse
|
||||
@@ -404,10 +405,59 @@ class TorchScriptPythonDecoder (Decoder):
|
||||
node.set_friendly_name(name)
|
||||
return node
|
||||
|
||||
@staticmethod
|
||||
def convert_quantized_tensor(qtensor: torch.Tensor):
|
||||
# need to represent as Constant(u8) -> Convert(f32) -> Subtract(zero_point) -> Multiply (scale)
|
||||
qscheme = qtensor.qscheme() # torch.per_channel_affine (per_tensor)
|
||||
if qscheme == torch.per_channel_affine:
|
||||
int8_tensor = qtensor.int_repr()
|
||||
scale = qtensor.q_per_channel_scales().numpy().astype(np.float32) # (weight.q_scale() for per_tensor)
|
||||
zero_point = qtensor.q_per_channel_zero_points().numpy().astype(np.float32) # (weight.q_zero_point() for per_tensor)
|
||||
axis = np.int32(qtensor.q_per_channel_axis())
|
||||
|
||||
new_shape = np.ones(len(int8_tensor.shape), dtype=np.int32)
|
||||
new_shape[axis] = -1
|
||||
zero_point_bc = np.reshape(zero_point, new_shape)
|
||||
scale_bc = np.reshape(scale, new_shape)
|
||||
|
||||
int8_const = op.Constant(int8_tensor.numpy())
|
||||
convert = ops.convert(int8_const, np.float32)
|
||||
sub = ops.subtract(convert, zero_point_bc)
|
||||
return ops.multiply(sub, scale_bc).outputs()
|
||||
elif qscheme == torch.per_tensor_affine:
|
||||
int8_tensor = qtensor.int_repr()
|
||||
scale = np.float32(qtensor.q_scale())
|
||||
zero_point = np.float32(qtensor.q_zero_point())
|
||||
|
||||
int8_const = op.Constant(int8_tensor.numpy())
|
||||
convert = ops.convert(int8_const, np.float32)
|
||||
sub = ops.subtract(convert, zero_point)
|
||||
return ops.multiply(sub, scale).outputs()
|
||||
assert False, "Unsupported qscheme"
|
||||
|
||||
def try_decode_get_attr(self):
|
||||
pt_value = get_value_from_getattr(self.graph_element, self.pt_module)
|
||||
assert pt_value is not None, "Couldn't retrieve value from prim::GetAttr"
|
||||
if not isinstance(pt_value, (torch.jit.ScriptModule, torch.jit.TracedModule)):
|
||||
if isinstance(pt_value, torch.ScriptObject):
|
||||
# We assume this is __torch__.torch.classes.quantized.Conv2dPackedParamsBase or __torch__.torch.classes.quantized.LinearPackedParamsBase
|
||||
# TODO: but can be anything. Figure a better way to distinguish
|
||||
weight, bias = pt_value.unpack()
|
||||
res = self.convert_quantized_tensor(weight)
|
||||
if isinstance(bias, torch.Tensor):
|
||||
res += ivalue_to_constant(bias)
|
||||
else:
|
||||
res += ops.convert_like(ivalue_to_constant(torch.zeros(1))[0], res[0]).outputs()
|
||||
try:
|
||||
# these params exist only for conv params
|
||||
stride = pt_value.stride()
|
||||
padding = pt_value.padding()
|
||||
dilation = pt_value.dilation()
|
||||
groups = pt_value.groups()
|
||||
res += ivalue_to_constant(stride) + ivalue_to_constant(padding) + ivalue_to_constant(dilation) + ivalue_to_constant(groups)
|
||||
except:
|
||||
pass
|
||||
return res
|
||||
elif not isinstance(pt_value, (torch.jit.ScriptModule, torch.jit.TracedModule)):
|
||||
return ivalue_to_constant(pt_value)
|
||||
else:
|
||||
return []
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
//
|
||||
|
||||
#include "openvino/frontend/pytorch/node_context.hpp"
|
||||
#include "openvino/opsets/opset10.hpp"
|
||||
#include "pt_framework_node.hpp"
|
||||
#include "utils.hpp"
|
||||
|
||||
@@ -14,11 +13,23 @@ namespace op {
|
||||
|
||||
OutputVector translate_get_attr(const NodeContext& context) {
|
||||
auto res = context.get_decoder()->try_decode_get_attr();
|
||||
FRONT_END_OP_CONVERSION_CHECK(res.size() > 0, "GetAttr must have at least one output.");
|
||||
return res;
|
||||
FRONT_END_OP_CONVERSION_CHECK(res.size() > 0,
|
||||
"Failed to obtain data from GetAttr with output tensor name: ",
|
||||
context.get_decoder()->get_output_debug_name(0));
|
||||
if (res.size() == 1) {
|
||||
return res;
|
||||
} else {
|
||||
// Packed params case
|
||||
std::shared_ptr<Node> fw_node = std::make_shared<PtFrameworkNode>(context.get_decoder(), res, 1);
|
||||
add_exception_to_fw_node(
|
||||
fw_node,
|
||||
"PackedParams represented as FrameworkNode, all contained params represented as inputs to this "
|
||||
"node.");
|
||||
return {context.mark_node(fw_node)};
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace op
|
||||
} // namespace pytorch
|
||||
} // namespace frontend
|
||||
} // namespace ov
|
||||
} // namespace ov
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
// Copyright (C) 2018-2023 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "openvino/frontend/pytorch/node_context.hpp"
|
||||
#include "openvino/op/add.hpp"
|
||||
#include "openvino/op/matmul.hpp"
|
||||
#include "utils.hpp"
|
||||
#include "utils_quantize.hpp"
|
||||
|
||||
namespace ov {
|
||||
namespace frontend {
|
||||
namespace pytorch {
|
||||
namespace op {
|
||||
|
||||
OutputVector translate_quantized_linear(const NodeContext& context) {
|
||||
// "quantized::linear(Tensor X, __torch__.torch.classes.quantized.LinearPackedParamsBase W_prepack, float Y_scale_i,
|
||||
// int Y_zero_point_i) -> Tensor Y"
|
||||
num_inputs_check(context, 4, 4);
|
||||
auto x = context.get_input(0);
|
||||
auto packed_params_node =
|
||||
std::dynamic_pointer_cast<ov::op::util::FrameworkNode>(context.get_input(1).get_node_shared_ptr());
|
||||
FRONT_END_OP_CONVERSION_CHECK(packed_params_node, "Packed params input node type is required to be FrameworkNode.");
|
||||
const auto& attrs = packed_params_node->get_attrs();
|
||||
FRONT_END_OP_CONVERSION_CHECK((attrs.find(PtFrameworkNode::op_type_key) != attrs.end()),
|
||||
"Packed params input node does not contain information about op type.");
|
||||
FRONT_END_OP_CONVERSION_CHECK((attrs.at(PtFrameworkNode::op_type_key) == "prim::GetAttr"),
|
||||
"Incorrect packed params input node operator type, expected prim::GetAttr.");
|
||||
|
||||
auto packed_params = packed_params_node->inputs();
|
||||
FRONT_END_OP_CONVERSION_CHECK(packed_params.size() == 2,
|
||||
"Packed parameters for quantized linear should contain 2 items.");
|
||||
auto weights = packed_params[0].get_source_output();
|
||||
auto bias = packed_params[1].get_source_output();
|
||||
|
||||
auto linear = context.mark_node(std::make_shared<ov::op::v0::MatMul>(x, weights, false, true));
|
||||
linear = context.mark_node(std::make_shared<ov::op::v1::Add>(linear, bias));
|
||||
auto scale = context.get_input(2);
|
||||
auto zero_point = context.get_input(3);
|
||||
return {context.mark_output(quantize(context, linear, scale, zero_point, x))};
|
||||
};
|
||||
|
||||
} // namespace op
|
||||
} // namespace pytorch
|
||||
} // namespace frontend
|
||||
} // namespace ov
|
||||
@@ -158,6 +158,7 @@ OP_CONVERTER(translate_var_mean);
|
||||
OP_CONVERTER(translate_where);
|
||||
OP_CONVERTER(translate_zeros);
|
||||
OP_CONVERTER(translate_zeros_like);
|
||||
OP_CONVERTER(translate_quantized_linear);
|
||||
|
||||
} // namespace op
|
||||
|
||||
@@ -418,6 +419,7 @@ const std::map<std::string, CreatorFunction> get_supported_ops() {
|
||||
{"prim::requires_grad", op::return_false_scalar},
|
||||
{"prim::PythonOp", op::translate_pythonop},
|
||||
{"prim::type", op::skip_node}, // Used with prim::device, pass PtFrameworkNode.
|
||||
{"quantized::linear", op::translate_quantized_linear},
|
||||
{"torchvision::deform_conv2d", op::translate_deform_conv},
|
||||
{"torchvision::nms", op::translate_nms},
|
||||
{"torchvision::roi_align", op::translate_roi_align},
|
||||
|
||||
@@ -152,6 +152,40 @@ ov::Output<ov::Node> quantize(const NodeContext& context,
|
||||
quantization_type);
|
||||
}
|
||||
|
||||
ov::Output<ov::Node> quantize(const NodeContext& context,
|
||||
ov::Output<ov::Node> input,
|
||||
ov::Output<ov::Node> quantized_node) {
|
||||
std::shared_ptr<QuantizedPtNode> quantized_pt_node;
|
||||
if ((quantized_pt_node = cast_quantized_fw_node(quantized_node.get_node_shared_ptr()))) {
|
||||
return quantize(context,
|
||||
input.get_node_shared_ptr(),
|
||||
quantized_pt_node->get_scale(),
|
||||
quantized_pt_node->get_zero_point(),
|
||||
quantized_pt_node->get_axis(),
|
||||
quantized_pt_node->get_dtype(),
|
||||
quantized_pt_node->get_type());
|
||||
}
|
||||
FRONT_END_OP_CONVERSION_CHECK(false, "Failed to convert a node to QuantizedPtNode");
|
||||
}
|
||||
|
||||
ov::Output<ov::Node> quantize(const NodeContext& context,
|
||||
ov::Output<ov::Node> input,
|
||||
ov::Output<ov::Node> scale,
|
||||
ov::Output<ov::Node> zero_point,
|
||||
ov::Output<ov::Node> quantized_node) {
|
||||
std::shared_ptr<QuantizedPtNode> quantized_pt_node;
|
||||
if ((quantized_pt_node = cast_quantized_fw_node(quantized_node.get_node_shared_ptr()))) {
|
||||
return quantize(context,
|
||||
input.get_node_shared_ptr(),
|
||||
scale.get_node_shared_ptr(),
|
||||
zero_point.get_node_shared_ptr(),
|
||||
quantized_pt_node->get_axis(),
|
||||
quantized_pt_node->get_dtype(),
|
||||
quantized_pt_node->get_type());
|
||||
}
|
||||
FRONT_END_OP_CONVERSION_CHECK(false, "Failed to convert a node to QuantizedPtNode");
|
||||
}
|
||||
|
||||
std::shared_ptr<QuantizedPtNode> cast_quantized_fw_node(ov::Output<Node> node) {
|
||||
auto quant_node = std::dynamic_pointer_cast<QuantizedPtNode>(node.get_node_shared_ptr());
|
||||
if (!quant_node) {
|
||||
|
||||
@@ -108,6 +108,23 @@ ov::Output<ov::Node> quantize(const NodeContext& context,
|
||||
ov::element::Type dtype,
|
||||
QuantizedPtNodeType quantization_type);
|
||||
|
||||
/**
|
||||
* Quantizes input node like the quantized node. Returns a shared pointer to the new QuantizedPtNode.
|
||||
*/
|
||||
ov::Output<ov::Node> quantize(const NodeContext& context,
|
||||
ov::Output<ov::Node> input,
|
||||
ov::Output<ov::Node> quantized_node);
|
||||
|
||||
/**
|
||||
* Quantizes input node like the quantized node, with new scale and zero_point parameters. Returns a shared pointer to
|
||||
* the new QuantizedPtNode.
|
||||
*/
|
||||
ov::Output<ov::Node> quantize(const NodeContext& context,
|
||||
ov::Output<ov::Node> input,
|
||||
ov::Output<ov::Node> scale,
|
||||
ov::Output<ov::Node> zero_point,
|
||||
ov::Output<ov::Node> quantized_node);
|
||||
|
||||
std::shared_ptr<QuantizedPtNode> cast_quantized_fw_node(ov::Output<ov::Node> node);
|
||||
std::shared_ptr<QuantizedPtNode> cast_quantized_fw_node(ov::Output<ov::Node> node, const std::string& type);
|
||||
} // namespace pytorch
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
# Copyright (C) 2018-2023 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import numpy as np
|
||||
from pytorch_layer_test_class import PytorchLayerTest
|
||||
|
||||
class TestQuantizedLinear(PytorchLayerTest):
|
||||
def _prepare_input(self, input_shape=(2, 2)):
|
||||
return (np.random.randn(*input_shape).astype(np.float32),)
|
||||
|
||||
def create_model(self, weight_shape, is_bias, scale, zero_point):
|
||||
|
||||
class aten_quantized_linear(torch.nn.Module):
|
||||
def __init__(self, weight_shape, is_bias, scale, zero_point):
|
||||
super(aten_quantized_linear, self).__init__()
|
||||
if is_bias:
|
||||
self.linear = torch.ao.nn.quantized.Linear(weight_shape[-1], weight_shape[0], True)
|
||||
torch.nn.init.normal_(self.linear.bias())
|
||||
else:
|
||||
self.linear = torch.ao.nn.quantized.Linear(weight_shape[-1], weight_shape[0], False)
|
||||
self.linear.scale = float(scale)
|
||||
self.linear.zero_point = int(zero_point)
|
||||
|
||||
def forward(self, inp):
|
||||
inp_q = torch.quantize_per_tensor(inp, 1., 0, torch.quint8)
|
||||
return torch.dequantize(self.linear(inp_q))
|
||||
|
||||
ref_net = None
|
||||
|
||||
return aten_quantized_linear(weight_shape, is_bias, scale, zero_point), ref_net, "quantized::linear"
|
||||
|
||||
@pytest.mark.parametrize("params", [
|
||||
{'input_shape': [3, 9], 'weight_shape': [10, 9]},
|
||||
{'input_shape': [3, 9], 'weight_shape': [9]},
|
||||
{'input_shape': [2, 3, 9], 'weight_shape': [10, 9]},
|
||||
{'input_shape': [2, 3, 9], 'weight_shape': [9]},
|
||||
{'input_shape': [3, 9], 'weight_shape': [9], "bias": True},
|
||||
{'input_shape': [3, 9], 'weight_shape': [10, 9], "bias": True},
|
||||
{'input_shape': [2, 3, 9], 'weight_shape': [10, 9], "bias": True},
|
||||
])
|
||||
@pytest.mark.parametrize("scale", [1., 0.3, 1.3])
|
||||
@pytest.mark.parametrize("zero_point", [0, 1])
|
||||
@pytest.mark.parametrize("trace", [True, False])
|
||||
@pytest.mark.nightly
|
||||
@pytest.mark.precommit
|
||||
def test_quantized_linear(self, params, scale, zero_point, trace, ie_device, precision, ir_version):
|
||||
input_shape = params.get("input_shape")
|
||||
weight_shape = params.get("weight_shape")
|
||||
bias = params.get("bias", False)
|
||||
self._test(*self.create_model(weight_shape, bias, scale, zero_point), ie_device, precision, ir_version,
|
||||
kwargs_to_prepare_input={"input_shape": input_shape}, trace_model=trace, freeze_model=False)
|
||||
Reference in New Issue
Block a user