From 296b6215770ca02c52e5a722007649842b61a927 Mon Sep 17 00:00:00 2001 From: Szymon Irzabek Date: Thu, 13 Oct 2022 11:21:59 +0200 Subject: [PATCH] [GNA] Replace fixed quantization code with a more flexible approach (#13237) --- .../intel_gna/src/backend/gna_limitations.cpp | 2 +- .../intel_gna/src/backend/gna_limitations.hpp | 4 +- .../src/frontend/layer_quantizer.hpp | 892 +++++++----------- .../src/frontend/model_quantizer.hpp | 186 ++-- .../intel_gna/src/frontend/quantization.cpp | 550 ++++------- .../intel_gna/src/frontend/quantization.h | 54 -- .../intel_gna/src/frontend/quantization.hpp | 38 + .../src/frontend/quantized_layer_params.hpp | 15 +- .../src/frontend/scale_factor_calc.hpp | 148 ++- .../intel_gna/src/gna_graph_compiler.cpp | 52 +- src/plugins/intel_gna/src/gna_plugin.cpp | 22 +- .../src/optimizer/gna_pass_manager.cpp | 3 +- .../tests/unit/gna_get_scale_factor.cpp | 8 +- .../unit/engines/gna/I8_quantisation_test.cpp | 17 +- .../gna/graph_tools/graph_copy_tests.cpp | 5 +- .../engines/gna/i16_quantisation_test.cpp | 16 +- .../engines/gna/matchers/conv_matcher.hpp | 2 +- .../engines/gna/matchers/diag_matcher.hpp | 2 +- .../engines/gna/matchers/pool_matcher.hpp | 2 +- .../engines/gna/matchers/weights_matcher.hpp | 2 +- 20 files changed, 744 insertions(+), 1276 deletions(-) delete mode 100644 src/plugins/intel_gna/src/frontend/quantization.h create mode 100644 src/plugins/intel_gna/src/frontend/quantization.hpp diff --git a/src/plugins/intel_gna/src/backend/gna_limitations.cpp b/src/plugins/intel_gna/src/backend/gna_limitations.cpp index 92d2ea45cc4..80bc9f98094 100644 --- a/src/plugins/intel_gna/src/backend/gna_limitations.cpp +++ b/src/plugins/intel_gna/src/backend/gna_limitations.cpp @@ -107,7 +107,7 @@ std::string RectLimitByChannels::GetErrorOrEmpty(const uint32_t h, const uint32_ } RectLimitByChannels RectLimitByChannelsAndPrecision::GetByPrecision(const OvGnaType precision) const { - return precision == OvGnaTypeInt8 ? lowPrecision : defaultPrecision; + return precision == OvGnaTypeInt8 ? limit_for_int8 : limit_for_int16; } bool RectLimitByChannelsAndPrecision::isValid(const uint32_t h, const uint32_t w, const OvGnaType precision, const uint32_t channels) const { diff --git a/src/plugins/intel_gna/src/backend/gna_limitations.hpp b/src/plugins/intel_gna/src/backend/gna_limitations.hpp index d1b63f8973b..48efbb2e1fc 100644 --- a/src/plugins/intel_gna/src/backend/gna_limitations.hpp +++ b/src/plugins/intel_gna/src/backend/gna_limitations.hpp @@ -112,8 +112,8 @@ struct RectLimitByChannels { }; struct RectLimitByChannelsAndPrecision { - RectLimitByChannels lowPrecision; - RectLimitByChannels defaultPrecision; + RectLimitByChannels limit_for_int8; + RectLimitByChannels limit_for_int16; RectLimitByChannels GetByPrecision(const OvGnaType precision) const; bool isValid(const uint32_t h, const uint32_t w, const OvGnaType precision, const uint32_t channels) const; std::string GetErrorOrEmpty(const uint32_t h, const uint32_t w, diff --git a/src/plugins/intel_gna/src/frontend/layer_quantizer.hpp b/src/plugins/intel_gna/src/frontend/layer_quantizer.hpp index 479704b9619..ee7c5fefc8a 100644 --- a/src/plugins/intel_gna/src/frontend/layer_quantizer.hpp +++ b/src/plugins/intel_gna/src/frontend/layer_quantizer.hpp @@ -4,99 +4,91 @@ #pragma once -#include -#include -#include - -#include "backend/gna_types.h" -#include "gna_plugin_log.hpp" -#include "quantized_layer_params.hpp" -#include "quantization.h" -#include "gna_graph_tools.hpp" -#include "blob_factory.hpp" -#include "precision_ex.hpp" +#include "ie_precision.hpp" #include "layers/gna_layer_info.hpp" +#include "quantized_layer_params.hpp" +#include "backend/gna_types.h" +#include "quantization.hpp" #include "weights_converter.hpp" -#include + namespace GNAPluginNS { namespace frontend { -/** - * @brief description of quantisation precision - * @tparam Ip - input precision - * @tparam Op - output precision - * @tparam Wp - weights precision - * @tparam Bp - biases precision - * @tparam Np - network precision - can be auto generated in future - */ -template -struct QuantDescTmpl { - using WeightsPrecision = Wp; - using BiasesPrecision = Bp; +// TODO: POT AAQ support can be added by modifying these functions +static InferenceEngine::Precision GetInputPrecision() { + return InferenceEngine::Precision::I16; +} - InferenceEngine::TPrecision _Ip; - InferenceEngine::TPrecision _Op; - InferenceEngine::TPrecision _Wp; - InferenceEngine::TPrecision _Bp; +static InferenceEngine::Precision GetOutputPrecision() { + return InferenceEngine::Precision::I32; +} - QuantDescTmpl() = default; - QuantDescTmpl(InferenceEngine::TPrecision _Ip, - InferenceEngine::TPrecision _Op, - InferenceEngine::TPrecision _Wp, - InferenceEngine::TPrecision _Bp) : _Op(_Op), _Ip(_Ip), _Wp(_Wp), _Bp(_Bp) { +// TODO: These should take into account the HW platform limitations +static InferenceEngine::Precision GetWeightsPrecision(const LayerInfo& layer_info, + const QuantizedLayerParams& quant_layer_params) { + if (layer_info.isConvolution() || layer_info.isConvolutionFilter() || layer_info.isScaleShift()) { + return InferenceEngine::Precision::I16; } - InferenceEngine::Precision getInputPrecision() const { - return _Ip; + if (quant_layer_params._weights_quant.IsStatsSet()) { + // For networks with FakeQuantize layers + if (quant_layer_params._weights_quant.GetLevels() <= std::numeric_limits::max()) { + return InferenceEngine::Precision::I8; + } else { + return InferenceEngine::Precision::I16; + } + } else { + return quant_layer_params._weights_int8_precision ? InferenceEngine::Precision::I8 + : InferenceEngine::Precision::I16; } - InferenceEngine::Precision getWeightsPrecision() const { - return _Wp; +} + +// TODO: In future these should take into account the HW platform limitations +static InferenceEngine::Precision GetBiasesPrecision(const LayerInfo& layer_info, + const QuantizedLayerParams& quant_layer_params) { + if (layer_info.isConvolution() || layer_info.isConvolutionFilter() || layer_info.isScaleShift()) { + return InferenceEngine::Precision::I32; } - InferenceEngine::Precision getBiasesPrecision() const { - return _Bp; + + if (quant_layer_params._weights_quant.IsStatsSet()) { + // For networks with FakeQuantize layers + const auto fq_levels = quant_layer_params._weights_quant.GetLevels(); + + if (fq_levels <= std::numeric_limits::max()) { + return InferenceEngine::Precision::fromType(); + // TODO: enable when all available precisions should be supported; it will require tests modifications + //} else if (fqLevels <= std::numeric_limits::max()) { + // return InferenceEngine::Precision::I16; + //} else if (GetInputPrecision() != InferenceEngine::Precision::I16 || + // GetWeightsPrecision(layer_info, quant_data) != InferenceEngine::Precision::I8) { + } else { + return InferenceEngine::Precision::I32; + } + } else { + if (quant_layer_params._inputs_int8_precision) { + return InferenceEngine::Precision::I8; + } else { + return (GetWeightsPrecision(layer_info, quant_layer_params) == InferenceEngine::Precision::I8) + ? InferenceEngine::Precision::fromType() + : InferenceEngine::Precision{InferenceEngine::Precision::I32}; + } } - InferenceEngine::Precision getOutputPrecision() const { - return _Op; - } -}; +} -#define P_TYPE(X)\ -typename InferenceEngine::PrecisionTrait::value_type - -#define PRECISION_TYPE(A, B, C, D, E)\ - P_TYPE(A), P_TYPE(B), P_TYPE(C), P_TYPE(D), P_TYPE(E) - - -struct QuantI16 : public QuantDescTmpl { -}; -struct QuantI8 : public QuantDescTmpl { -}; -// Low precision path quantizer (I8 inputs, weights, biases) -struct QuantI8_I8 : public QuantDescTmpl { -}; - -// for support proper trait instantiation for quantization function callback -struct FakeQuant : public QuantDescTmpl { -}; -struct FakeQuantI16 : public QuantI16 {}; -struct FakeQuantI8 : public QuantI8 {}; - -template -struct QuantPair { - using MandatoryType = A; - using OptionalType = B; - static A mandatory () { return A();} - static B optional () { return B();} -}; +static bool IsBiasCompound(const LayerInfo& layer_info, const QuantizedLayerParams* quant_layer_params) { + auto biases_precision = GetBiasesPrecision(layer_info, *quant_layer_params); + auto compound_bias_precision = InferenceEngine::Precision::fromType(); + return (biases_precision == compound_bias_precision ? true : false); +} struct FakeQuantizeParams { - bool paramsSet = false; - uint32_t levelsNum = 1; - float inputMinValue = 1.0f; - float inputMaxValue = 1.0f; - float outputMinValue = 1.0f; - float outputMaxValue = 1.0f; + bool params_set = false; + uint32_t levels_num = 1; + float input_min_value = 1.0f; + float input_max_value = 1.0f; + float output_min_value = 1.0f; + float output_max_value = 1.0f; }; /** @@ -105,90 +97,22 @@ struct FakeQuantizeParams { * @return */ template -inline bool shouldAlwaysAllocate() { +inline bool ShouldAlwaysAllocate() { return false; } template <> -inline bool shouldAlwaysAllocate() { +inline bool ShouldAlwaysAllocate() { return true; } - -#undef P_TYPE -#undef PRECISION_TYPE - -/** - * @brief designate actual data quantisation functions trait - */ -template -class Quant { -public: - template - void operator()(Args && ... args) const { } -}; - -template<> -class Quant { - public: - template - void operator()(Args && ... args) const { - QuantizationCallback { - std::forward(args)... - }.runQuantize(); - } -}; - -template<> -class Quant { - public: - template - void operator()(Args && ... args) const { - QuantizationCallback { - std::forward(args)... - }.runQuantize(); - } -}; - -template<> -class Quant { -public: - template - void operator()(Args && ... args) const { - QuantizationCallback { - std::forward(args)... - }.runQuantize(); - } -}; - -template<> -class Quant { - public: - template - void operator()(Args && ... args) const { - QuantizationCallback { - std::forward(args)... - }.runFakeQuantize(); - } -}; - -template<> -class Quant { - public: - template - void operator()(Args && ... args) const { - QuantizationCallback{ - std::forward(args)... - }.runFakeQuantize(); - } -}; - - template -inline InferenceEngine::Blob::Ptr fp32_to_precision_blob(InferenceEngine::Blob::Ptr fp32_blob, InferenceEngine::Precision precision, - float scale_factor, const FakeQuantizeParams& fqParams) { - auto prec_blob = InferenceEngine::make_shared_blob({ precision, - fp32_blob->getTensorDesc().getDims(), fp32_blob->getTensorDesc().getLayout() }); +inline InferenceEngine::Blob::Ptr FP32ToPrecisionBlob(InferenceEngine::Blob::Ptr fp32_blob, + InferenceEngine::Precision precision, + float scale_factor, + const FakeQuantizeParams& fq_params) { + auto prec_blob = InferenceEngine::make_shared_blob( + {precision, fp32_blob->getTensorDesc().getDims(), fp32_blob->getTensorDesc().getLayout()}); prec_blob->allocate(); auto input_low = 0.0f; @@ -196,57 +120,66 @@ inline InferenceEngine::Blob::Ptr fp32_to_precision_blob(InferenceEngine::Blob:: auto output_low = 0.0f; auto output_high = 0.0f; auto levels = 1; - if (fqParams.paramsSet) { - input_low = fqParams.inputMinValue; - input_high = fqParams.inputMaxValue; - output_low = fqParams.outputMinValue; - output_high = fqParams.outputMaxValue; - levels = fqParams.levelsNum; + if (fq_params.params_set) { + input_low = fq_params.input_min_value; + input_high = fq_params.input_max_value; + output_low = fq_params.output_min_value; + output_high = fq_params.output_max_value; + levels = fq_params.levels_num; } int i = 0; - for (auto& precValue : *prec_blob) { - auto f32Value = fp32_blob->buffer().template as::value_type*>()[i++]; - if (fqParams.paramsSet) { - auto x = f32Value; + + auto f32_value_array = + fp32_blob->buffer() + .template as::value_type*>(); + + for (auto& prec_value : *prec_blob) { + auto f32_value = f32_value_array[i++]; + + if (fq_params.params_set) { + auto x = f32_value; if (x <= std::min(input_low, input_high)) { - f32Value = output_low; + f32_value = output_low; } else if (x > std::max(input_low, input_high)) { - f32Value = output_high; + f32_value = output_high; } else { - f32Value = nearbyint((x - input_low) / (input_high - input_low) * (levels - 1)) / - (levels - 1) * (output_high - output_low) + output_low; + f32_value = nearbyint((x - input_low) / (input_high - input_low) * (levels - 1)) / (levels - 1) * + (output_high - output_low) + + output_low; } } - f32Value = f32Value * scale_factor; - if (f32Value > static_cast(std::numeric_limits::max())) { - precValue = std::numeric_limits::max(); - } else if (f32Value < std::numeric_limits::min()) { - precValue = std::numeric_limits::min(); + f32_value = f32_value * scale_factor; + if (f32_value > static_cast(std::numeric_limits::max())) { + prec_value = std::numeric_limits::max(); + } else if (f32_value < std::numeric_limits::min()) { + prec_value = std::numeric_limits::min(); } else { - precValue = static_cast(f32Value); + prec_value = static_cast(f32_value); } } - return static_cast(prec_blob); + return static_cast(prec_blob); } -inline InferenceEngine::Blob::Ptr fp32_to_precision_blob(InferenceEngine::Blob::Ptr fp32_blob, InferenceEngine::Precision precision, - float scale_factor, const FakeQuantizeParams &fqParams) { +inline InferenceEngine::Blob::Ptr FP32ToPrecisionBlob(InferenceEngine::Blob::Ptr fp32_blob, + InferenceEngine::Precision precision, + float scale_factor, + const FakeQuantizeParams& fq_params) { InferenceEngine::Blob::Ptr result_ptr = nullptr; switch (precision) { case InferenceEngine::Precision::FP32: - result_ptr = fp32_to_precision_blob(fp32_blob, precision, scale_factor, fqParams); + result_ptr = FP32ToPrecisionBlob(fp32_blob, precision, scale_factor, fq_params); break; case InferenceEngine::Precision::I32: - result_ptr = fp32_to_precision_blob(fp32_blob, precision, scale_factor, fqParams); + result_ptr = FP32ToPrecisionBlob(fp32_blob, precision, scale_factor, fq_params); break; case InferenceEngine::Precision::I16: - result_ptr = fp32_to_precision_blob(fp32_blob, precision, scale_factor, fqParams); + result_ptr = FP32ToPrecisionBlob(fp32_blob, precision, scale_factor, fq_params); break; case InferenceEngine::Precision::I8: - result_ptr = fp32_to_precision_blob(fp32_blob, precision, scale_factor, fqParams); + result_ptr = FP32ToPrecisionBlob(fp32_blob, precision, scale_factor, fq_params); break; default: THROW_GNA_EXCEPTION << "FP32 to " << precision << " not supported"; @@ -254,323 +187,257 @@ inline InferenceEngine::Blob::Ptr fp32_to_precision_blob(InferenceEngine::Blob:: return result_ptr; } -template -InferenceEngine::Blob::Ptr make_custom_blob(Args&&... args) { - return InferenceEngine::make_shared_blob(InferenceEngine::Precision::fromType(), std::forward(args)...); -} - -template -InferenceEngine::Blob::Ptr make_custom_blob(InferenceEngine::Layout layout, InferenceEngine::SizeVector size) { - return InferenceEngine::make_shared_blob( - InferenceEngine::TensorDesc(InferenceEngine::Precision::fromType(), size, layout)); -} - -template -inline void quantizeWeightsBiases(const QuantDesc & quantDesc, - InferenceEngine::WeightableLayer *wl, - const QuantFunc &fnc, - bool isDiagonal = false) { // for diagonal layer number of weights and biases significatly smaller - // for quantized weights - auto intWeights = - make_custom_blob(InferenceEngine::C, InferenceEngine::SizeVector({wl->_weights->size()})); - intWeights->allocate(); - if (intWeights->buffer() == nullptr) { - IE_THROW(NotAllocated) - << "[GNAPlugin] in function " << __PRETTY_FUNCTION__<< ": " - << "cannot copy weights for layer :"<< wl->name << " of size" << intWeights->byteSize(); - } - - int oIdx = wl->outData[0]->getDims().size() - 1; - int iIdx = wl->insData[0].lock().get()->getDims().size() - 1; - - auto getBiasSizeForLayer = [&oIdx](InferenceEngine::WeightableLayer *wl) { - if (wl->_biases) { - return wl->_biases->size(); - } - // calculating biases len using outdata dims - auto & dims = wl->outData.front()->getDims(); - return dims[oIdx]; - }; - - using BiasesPrecision = typename QuantDesc::BiasesPrecision; - auto biasMaker = [&] () { - InferenceEngine::Blob::Ptr zero; - if (!wl->_biases && !shouldAlwaysAllocate()) { - return zero; - } - auto bias = make_custom_blob(InferenceEngine::C, InferenceEngine::SizeVector({ - getBiasSizeForLayer(wl) - })); - bias->allocate(); - if (bias->buffer() == nullptr) { - IE_THROW(NotAllocated) - << "[GNAPlugin] in function " << __PRETTY_FUNCTION__<< ": " - << "cannot copy bias for layer :"<< wl->name <<"of size" << bias->byteSize(); - } - - memset(bias->buffer(), 0, bias->byteSize()); - - return bias; - }; - auto intBiases = biasMaker(); - - float input_scale_factor = 1.f; - if (InferenceEngine::CNNNetHasPrevLayer(wl)) { - auto quantDataForInputLayer = - InferenceEngine::getInjectedData(*InferenceEngine::CNNNetPrevLayer(wl).get()); - input_scale_factor = quantDataForInputLayer->_dst_quant.GetScale(); - if (std::isnan(input_scale_factor) || - std::isinf(input_scale_factor)) { - IE_THROW() << "Unsupported input scale factor value " << input_scale_factor; - } - } - - uint32_t num_rows = isDiagonal ? 1 : wl->outData[0]->getDims()[oIdx]; - uint32_t num_columns = isDiagonal ? wl->_weights->size() : wl->insData[0].lock().get()->getDims()[iIdx]; - - if (LayerInfo(wl).isAffineFilter() || LayerInfo(wl).isConcatAlignFilter()) { - // for affine filter layer insdata size not equal to actual coded in input layer - num_columns = wl->_weights->size() / num_rows; - } - - if (isDiagonal) { - std::swap(num_rows, num_columns); - } - - uint32_t num_rows_padded = num_rows; - uint32_t num_columns_padded = num_columns; - - // TODO: replace this into fixed scale quantizer then - - auto quantData = InferenceEngine::getInjectedData(*wl); - { - auto weightsStats = !quantData->_weights_quant.GetMinValues().empty(); - auto weightsScale = quantData->_weights_quant.GetScale(); - auto dstScale = quantData->_dst_quant.GetScale(); - auto blob_precision = wl->_weights->getTensorDesc().getPrecision(); - auto quantizedWeights = blob_precision != InferenceEngine::Precision::FP32 && blob_precision != InferenceEngine::Precision::FP16; - fnc(wl->_weights->buffer().as(), - wl->_biases ? wl->_biases->buffer().as() : nullptr, - intWeights->buffer(), - intBiases ? intBiases->buffer() : static_cast(nullptr), - input_scale_factor, - &weightsScale, - &dstScale, - num_rows, - num_columns, - num_rows_padded, - num_columns_padded, - quantizedWeights, - quantData->_weights_quant.GetLevels(), - quantData->_weights_quant.GetMinValues().size(), - weightsStats ? &quantData->_weights_quant.GetMinValues(true).front() : nullptr, - weightsStats ? &quantData->_weights_quant.GetMaxValues(true).front() : nullptr, - weightsStats ? &quantData->_weights_quant.GetMinValues(false).front() : nullptr, - weightsStats ? &quantData->_weights_quant.GetMaxValues(false).front() : nullptr); - } - wl->_weights = intWeights; - wl->_biases = intBiases; - - /** - * correcting precision for outdata - */ - wl->precision = quantDesc.getWeightsPrecision(); - for (auto &&outData : wl->outData) { - outData->setPrecision(quantDesc.getOutputPrecision()); - } -} - - -template -inline void quantizeWeightsBiasesConv(const QuantDesc & quantDesc, - InferenceEngine::WeightableLayer *conv, - const QuantFunc &fnc) { - // for quantized weights - auto intWeights = make_custom_blob(InferenceEngine::C, InferenceEngine::SizeVector({conv->_weights->size()})); - intWeights->allocate(); - if (intWeights->buffer() == nullptr) { - IE_THROW(NotAllocated) - << "[GNAPlugin] in function " << __PRETTY_FUNCTION__<< ": " - << "cannot copy weights for layer :"<< conv->name << " of size" << intWeights->byteSize(); - } - - auto getBiasSizeForLayer = [](InferenceEngine::WeightableLayer *wl) -> size_t { - if (wl->_biases) { - return wl->_biases->size(); - } +static size_t GetBiasSizeForLayer(InferenceEngine::WeightableLayer& wl) { + if (wl._biases) { + return wl._biases->size(); + } else if (LayerInfo(wl).isConvolution()) { // calculating biases len using outdata dims: biases number should be equal to output channels number - return InferenceEngine::GetDataDimSize(wl->outData.front(), InferenceEngine::DataDimName::C); - }; + size_t size = InferenceEngine::GetDataDimSize(wl.outData.front(), InferenceEngine::DataDimName::C); + return size; + } else { + // calculating biases size using outData dimensions + const auto& dims = wl.outData.front()->getDims(); + const auto& o_idx = wl.outData[0]->getDims().size() - 1; + return dims[o_idx]; + } +} - using BiasesPrecision = typename QuantDesc::BiasesPrecision; - auto biasMaker = [&] () { +static std::pair GetNumRowsColumns(InferenceEngine::WeightableLayer& wl) { + size_t num_rows; + size_t num_columns; + const auto& o_idx = wl.outData[0]->getDims().size() - 1; + const auto& i_idx = wl.insData[0].lock().get()->getDims().size() - 1; + + if (LayerInfo(wl).isScaleShift()) { + num_columns = 1; + num_rows = wl._weights->size(); + } else if (LayerInfo(wl).isConvolution() || LayerInfo(wl).isConvolutionFilter()) { + if (wl.outData[0]->getDims().size() < 2) { + IE_THROW() << "Unsupported output dims size for " << wl.name << ". Should be > 1, but is " + << wl.outData[0]->getDims().size(); + } + if (wl.insData[0].lock().get()->getDims().size() < 2) { + IE_THROW() << "Unsupported input dims size for " << wl.name << ". Should be > 1, but is " + << wl.insData[0].lock().get()->getDims().size(); + } + + num_rows = GetBiasSizeForLayer(wl); + + if (num_rows == 0) { + THROW_GNA_EXCEPTION << "Invalid nummber of rows"; + } + + num_columns = wl._weights->size() / num_rows; + } else if (LayerInfo(wl).isAffineFilter() || LayerInfo(wl).isConcatAlignFilter()) { + // For affine filter layer insdata size is not equal to the one stored in input layer + num_rows = wl.outData[0]->getDims()[o_idx]; + num_columns = wl._weights->size() / num_rows; + } else { + num_rows = wl.outData[0]->getDims()[o_idx]; + num_columns = wl.insData[0].lock().get()->getDims()[i_idx]; + } + + return {num_rows, num_columns}; +} + +template +void QuantizeWeightsPrep(InferenceEngine::WeightableLayer& wl, QuantizationData& common_data) { + const auto weights_precision = InferenceEngine::Precision::fromType(); + const auto int_weights = InferenceEngine::make_shared_blob( + InferenceEngine::TensorDesc(weights_precision, + InferenceEngine::SizeVector({wl._weights->size()}), + InferenceEngine::C)); + + int_weights->allocate(); + + if (int_weights->buffer() == nullptr) { + IE_THROW(NotAllocated) << "[GNAPlugin] in function " << __PRETTY_FUNCTION__ << ": " + << "cannot copy weights for layer :" << wl.name << " of size" + << int_weights->byteSize(); + } + + common_data.scale_factor = InferenceEngine::getInjectedData(wl)->_weights_quant.GetScale(); + const auto& blob_precision = wl._weights->getTensorDesc().getPrecision(); + const auto& quantized_weights = + blob_precision != InferenceEngine::Precision::FP32 && blob_precision != InferenceEngine::Precision::FP16; + const bool& compound_bias = + IsBiasCompound(LayerInfo(wl), InferenceEngine::getInjectedData(wl)); + const auto& compound_bias_ptr = + (compound_bias && wl._biases) ? wl._biases->buffer().as() : nullptr; + + QuantizeWeights(common_data, + wl._weights->buffer().as(), + int_weights->buffer(), + compound_bias_ptr, + quantized_weights); + + wl._weights = int_weights; + + /** + * correcting precision for outdata + */ + wl.precision = weights_precision; +} + +template +void QuantizeBiasesPrep(InferenceEngine::WeightableLayer& wl, QuantizationData& common_data) { + const auto bias_maker = [&]() { InferenceEngine::Blob::Ptr zero; - if (!conv->_biases && !shouldAlwaysAllocate()) { + if (!wl._biases && !ShouldAlwaysAllocate()) { return zero; } - auto bias = make_custom_blob(InferenceEngine::C, InferenceEngine::SizeVector({ - getBiasSizeForLayer(conv) - })); + + InferenceEngine::Blob::Ptr bias = InferenceEngine::make_shared_blob( + InferenceEngine::TensorDesc(InferenceEngine::Precision::fromType(), + InferenceEngine::SizeVector({GetBiasSizeForLayer(wl)}), + InferenceEngine::C)); + bias->allocate(); + if (bias->buffer() == nullptr) { - IE_THROW(NotAllocated) - << "[GNAPlugin] in function " << __PRETTY_FUNCTION__<< ": " - << "cannot copy bias for layer :"<< conv->name <<"of size" << bias->byteSize(); + IE_THROW(NotAllocated) << "[GNAPlugin] in function " << __PRETTY_FUNCTION__ << ": " + << "cannot copy bias for layer :" << wl.name << "of size" << bias->byteSize(); } + memset(bias->buffer(), 0, bias->byteSize()); return bias; }; - auto intBiases = biasMaker(); + const auto int_biases = bias_maker(); + common_data.scale_factor = InferenceEngine::getInjectedData(wl)->_dst_quant.GetScale(); + + QuantizeBiases(common_data, + wl._biases ? wl._biases->buffer().as() : nullptr, + int_biases ? int_biases->buffer() : static_cast(nullptr)); + + wl._biases = int_biases; +} + +static const std::map GetWeightsPreicisionsMap() { + return {{InferenceEngine::Precision::I8, QuantizeWeightsPrep}, + {InferenceEngine::Precision::I16, QuantizeWeightsPrep}}; +} + +static const std::map GetBiasPreicisionsMap() { + return {{InferenceEngine::Precision::I8, QuantizeBiasesPrep}, + {InferenceEngine::Precision::I16, QuantizeBiasesPrep}, + {InferenceEngine::Precision::I32, QuantizeBiasesPrep}, + {InferenceEngine::Precision::fromType(), QuantizeBiasesPrep}}; +} + +static void QuantizeWeightsBiases(InferenceEngine::WeightableLayer* wl) { float input_scale_factor = 1.f; - if (InferenceEngine::CNNNetHasPrevLayer(conv)) { - auto quantDataForInputLayer = - InferenceEngine::getInjectedData(*InferenceEngine::CNNNetPrevLayer(conv).get()); - input_scale_factor = quantDataForInputLayer->_dst_quant.GetScale(); + + if (InferenceEngine::CNNNetHasPrevLayer(wl)) { + auto quant_data_for_input_layer = + InferenceEngine::getInjectedData(InferenceEngine::CNNNetPrevLayer(wl)); + input_scale_factor = quant_data_for_input_layer->_dst_quant.GetScale(); if (std::isnan(input_scale_factor) || std::isinf(input_scale_factor)) { IE_THROW() << "Unsupported input scale factor value " << input_scale_factor; } } - if (conv->outData[0]->getDims().size() < 2) { - IE_THROW() << "Unsupported output dims size for " << conv->name <<", should be > 1, but " << conv->outData[0]->getDims().size(); - } - if (conv->insData[0].lock().get()->getDims().size() < 2) { - IE_THROW() << "Unsupported input dims size for " << conv->name << ", should be > 1, but " << conv->insData[0].lock().get()->getDims().size(); - } - auto inputData = conv->insData[0].lock(); - uint32_t num_rows = getBiasSizeForLayer(conv); - if (num_rows == 0) { - THROW_GNA_EXCEPTION << "Invalid num rows"; - } - uint32_t num_columns = conv->_weights->size() / num_rows; + size_t num_rows; + size_t num_columns; - uint32_t num_rows_padded = num_rows; - uint32_t num_columns_padded = num_columns; + std::tie(num_rows, num_columns) = GetNumRowsColumns(*wl); - // TODO: replace this into fixed scale quantizer then + auto quant_layer_params = InferenceEngine::getInjectedData(*wl); - auto quantData = InferenceEngine::getInjectedData(*conv); - { - auto weightsStats = !quantData->_weights_quant.GetMinValues().empty(); - auto weightsScale = quantData->_weights_quant.GetScale(); - auto dstScale = quantData->_dst_quant.GetScale(); - auto blob_precision = conv->_weights->getTensorDesc().getPrecision(); - auto quantizedWeights = blob_precision != InferenceEngine::Precision::FP32 && blob_precision != InferenceEngine::Precision::FP16; - fnc(conv->_weights->buffer().as(), - conv->_biases ? conv->_biases->buffer().as() : nullptr, - intWeights->buffer(), - intBiases ? intBiases->buffer() : static_cast(nullptr), - input_scale_factor, - &weightsScale, - &dstScale, - num_rows, - num_columns, - num_rows_padded, - num_columns_padded, - quantizedWeights, - quantData->_weights_quant.GetLevels(), - quantData->_weights_quant.GetMinValues().size(), - weightsStats ? &quantData->_weights_quant.GetMinValues(true).front() : nullptr, - weightsStats ? &quantData->_weights_quant.GetMaxValues(true).front() : nullptr, - weightsStats ? &quantData->_weights_quant.GetMinValues(false).front() : nullptr, - weightsStats ? &quantData->_weights_quant.GetMaxValues(false).front() : nullptr); - } - conv->_weights = intWeights; - conv->_biases = intBiases; + QuantizationData common_data{ + num_rows, + num_columns, + 1.0f, + quant_layer_params->_weights_quant + }; - /** - * correcting precision for outdata - */ - conv->precision = quantDesc.getWeightsPrecision(); - for (auto &&outData : conv->outData) { - outData->setPrecision(quantDesc.getOutputPrecision()); + GetBiasPreicisionsMap().at(GetBiasesPrecision(LayerInfo(wl), *quant_layer_params))(*wl, common_data); + GetWeightsPreicisionsMap().at(GetWeightsPrecision(LayerInfo(wl), *quant_layer_params))(*wl, common_data); + + // Correct precision for outdata + for (auto &&outData : wl->outData) { + outData->setPrecision(GetOutputPrecision()); } } - -class DataQuantizerBase { - public: - explicit DataQuantizerBase(float scaleFactor) : scaleFactor(scaleFactor) { - } - protected: - float scaleFactor = 1.0; -}; /** * Helper class to use partial specialisation of Layer type - * @tparam Desc * @tparam Layer */ -template -class DataQuantizer : public DataQuantizerBase { +template +class DataQuantizer { public: - explicit DataQuantizer(float scaleFactor) : DataQuantizerBase(scaleFactor) {} - bool operator()(Layer cnnLayer) const { + bool operator()(Layer cnn_layer) const { return false; } }; -template -class DataQuantizer : public DataQuantizerBase { - public: - explicit DataQuantizer(float scaleFactor) : DataQuantizerBase(scaleFactor) {} - - bool operator()(InferenceEngine::CNNLayer *cnnLayer) const { - for (auto &&outData : cnnLayer->outData) { - outData->setPrecision(Desc::mandatory().getOutputPrecision()); +template<> +class DataQuantizer { +public: + bool operator()(InferenceEngine::CNNLayer* cnn_layer) const { + for (auto&& out_data : cnn_layer->outData) { + out_data->setPrecision(GetOutputPrecision()); } - // set scale factor for input layers - if (cnnLayer->insData.empty()) { - for (auto &&outData : cnnLayer->outData) { - outData->setPrecision(Desc::mandatory().getInputPrecision()); + + // Set scale factor for input layers + if (cnn_layer->insData.empty() || LayerInfo(*cnn_layer).isCrop() || LayerInfo(*cnn_layer).isConcat() || + LayerInfo(*cnn_layer).isSplit()) { + for (auto&& out_data : cnn_layer->outData) { + out_data->setPrecision(GetInputPrecision()); } } else { - if (LayerInfo(*cnnLayer).isActivation() || - LayerInfo(*cnnLayer).isCopy() || - LayerInfo(*cnnLayer).isNonFunctional() || - LayerInfo(*cnnLayer).isPermute() || - LayerInfo(*cnnLayer).isConst()) { - // precision of activation layers is always equal input precision - for (auto &&outData : cnnLayer->outData) { - outData->setPrecision(Desc::mandatory().getInputPrecision()); + if (LayerInfo(*cnn_layer).isActivation() || LayerInfo(*cnn_layer).isCopy() || + LayerInfo(*cnn_layer).isNonFunctional() || LayerInfo(*cnn_layer).isPermute() || + LayerInfo(*cnn_layer).isConst()) { + // Precision of activation layers is always equal input precision + for (auto&& out_data : cnn_layer->outData) { + out_data->setPrecision(GetInputPrecision()); } } - // for pooling layer output precision is the same as input precision - if (LayerInfo(*cnnLayer).isMaxPooling()) { - const auto inputPrecision = cnnLayer->insData.front().lock()->getPrecision(); - for (auto&& outData : cnnLayer->outData) { - outData->setPrecision(inputPrecision); + // For pooling layer output precision is the same as input precision + if (LayerInfo(*cnn_layer).isMaxPooling()) { + const auto inputPrecision = cnn_layer->insData.front().lock()->getPrecision(); + for (auto&& out_data : cnn_layer->outData) { + out_data->setPrecision(inputPrecision); } } } - cnnLayer->precision = Desc::mandatory().getInputPrecision(); - if (LayerInfo(*cnnLayer).isConst()) { - auto initial_precision = cnnLayer->blobs["custom"]->getTensorDesc().getPrecision(); - // TODO I32 must be handled separately when it'll be supported + cnn_layer->precision = GetInputPrecision(); + + if (LayerInfo(*cnn_layer).isConst()) { + auto initial_precision = cnn_layer->blobs["custom"]->getTensorDesc().getPrecision(); + // TODO: I32 must be handled separately when it's supported IE_ASSERT(initial_precision != InferenceEngine::Precision::I32); if (initial_precision == InferenceEngine::Precision::FP16) { - cnnLayer->blobs["custom"] = make_fp32_blob(cnnLayer->blobs["custom"]); + cnn_layer->blobs["custom"] = make_fp32_blob(cnn_layer->blobs["custom"]); } - auto quantParams = InferenceEngine::getInjectedData(*cnnLayer); - auto new_const_blob = InferenceEngine::Blob::CreateFromData(cnnLayer->outData[0]); - auto const_blob = cnnLayer->blobs["custom"]; + auto quant_params = InferenceEngine::getInjectedData(*cnn_layer); + auto new_const_blob = InferenceEngine::Blob::CreateFromData(cnn_layer->outData[0]); + auto const_blob = cnn_layer->blobs["custom"]; if (const_blob->getTensorDesc().getPrecision() == InferenceEngine::Precision::FP32) { - auto fqParams = FakeQuantizeParams{}; - if (quantParams->_dst_quant.IsStatsSet()) { - fqParams.paramsSet = true; - fqParams.levelsNum = quantParams->_dst_quant.GetLevels(); - fqParams.inputMinValue = quantParams->_dst_quant.GetMinValues(true).front(); - fqParams.inputMaxValue = quantParams->_dst_quant.GetMaxValues(true).front(); - fqParams.outputMinValue = quantParams->_dst_quant.GetMinValues(false).front(); - fqParams.outputMaxValue = quantParams->_dst_quant.GetMaxValues(false).front(); + auto fq_params = FakeQuantizeParams{}; + if (quant_params->_dst_quant.IsStatsSet()) { + fq_params.params_set = true; + fq_params.levels_num = quant_params->_dst_quant.GetLevels(); + fq_params.input_min_value = quant_params->_dst_quant.GetMinValues(true).front(); + fq_params.input_max_value = quant_params->_dst_quant.GetMaxValues(true).front(); + fq_params.output_min_value = quant_params->_dst_quant.GetMinValues(false).front(); + fq_params.output_max_value = quant_params->_dst_quant.GetMaxValues(false).front(); } - cnnLayer->blobs["custom"] = fp32_to_precision_blob(const_blob, cnnLayer->outData[0]->getPrecision(), - quantParams->_dst_quant.GetScale(), fqParams); + cnn_layer->blobs["custom"] = FP32ToPrecisionBlob(const_blob, + cnn_layer->outData[0]->getPrecision(), + quant_params->_dst_quant.GetScale(), + fq_params); + } else if (LayerInfo(*cnn_layer).isReshape()) { + for (auto&& out_data : cnn_layer->outData) { + out_data->setPrecision(cnn_layer->insData.front().lock()->getPrecision()); + } } } @@ -578,142 +445,25 @@ class DataQuantizer : public DataQuantizerBas } }; - -template -class DataQuantizer : public DataQuantizer { - using base = DataQuantizer; +template<> +class DataQuantizer { public: - explicit DataQuantizer(float scaleFactor) : base(scaleFactor) {} - bool operator()(InferenceEngine::SplitLayer *splitLayer) const { - base::operator()(splitLayer); - // split layer doesnt change it's data at all - for (auto &&outData : splitLayer->outData) { - outData->setPrecision(Desc::mandatory().getInputPrecision()); - } - return true; - } -}; - -template -class DataQuantizer : public DataQuantizer { - using base = DataQuantizer; - public: - explicit DataQuantizer(float scaleFactor) : base(scaleFactor) {} - bool operator()(InferenceEngine::ConcatLayer *concatLayer) const { - base::operator()(concatLayer); - for (auto &&outData : concatLayer->outData) { - outData->setPrecision(Desc::mandatory().getInputPrecision()); - } - return true; - } -}; - -template -class DataQuantizer : public DataQuantizer { - using base = DataQuantizer; - public: - explicit DataQuantizer(float scaleFactor) : base(scaleFactor) {} - bool operator()(InferenceEngine::CropLayer *cropLayer) const { - base::operator()(cropLayer); - for (auto &&outData : cropLayer->outData) { - outData->setPrecision(Desc::mandatory().getInputPrecision()); - } - return true; - } -}; - -template -class DataQuantizer : public DataQuantizer { - using base = DataQuantizer; - public: - explicit DataQuantizer(float scaleFactor) : base(scaleFactor) {} - bool operator()(InferenceEngine::ReshapeLayer *reshapeLayer) const { - base::operator()(reshapeLayer); - // reshape layer doesnt change it's data at all - for (auto &&outData : reshapeLayer->outData) { - outData->setPrecision(reshapeLayer->insData.front().lock()->getPrecision()); - } - return true; - } -}; - -template -class DataQuantizer : public DataQuantizerBase { - public: - explicit DataQuantizer(float scaleFactor) : DataQuantizerBase(scaleFactor) {} - bool operator()(InferenceEngine::WeightableLayer *wl) const { - (*this)(wl, typename Desc::MandatoryType()); - return true; - } - - template - void operator()(InferenceEngine::WeightableLayer *wl, const T&) const { - quantizeWeightsBiases(T(), wl, Quant()); - } - - void operator()(InferenceEngine::WeightableLayer *wl, const FakeQuant&) const { - auto quantData = InferenceEngine::getInjectedData(*wl); - if (quantData->_weights_quant.IsStatsSet()) { - if (quantData->_weights_quant.GetLevels() <= std::numeric_limits::max()) { - quantizeWeightsBiases(FakeQuantI8(), wl, Quant()); - } else { - quantizeWeightsBiases(FakeQuantI16(), wl, Quant()); - } - } else { - if (std::is_same()) { - THROW_GNA_EXCEPTION << "Infinite recursion. The type Desc::OptionalType is equal FakeQuant."; - } - - (*this)(wl, typename Desc::OptionalType()); - } - } -}; - -template -class DataQuantizer : public DataQuantizerBase { - public: - explicit DataQuantizer(float scaleFactor) : DataQuantizerBase(scaleFactor) {} - bool operator()(InferenceEngine::ConvolutionLayer *cl) const { - quantizeWeightsBiasesConv(Desc::optional(), cl, Quant()); - return true; - } -}; - -template -class DataQuantizer : public DataQuantizerBase { - public: - explicit DataQuantizer(float scaleFactor) : DataQuantizerBase(scaleFactor) {} - bool operator()(InferenceEngine::ScaleShiftLayer *ssl) const { - quantizeWeightsBiases(Desc::optional(), ssl, Quant(), true); + bool operator()(InferenceEngine::WeightableLayer* wl) const { + QuantizeWeightsBiases(wl); return true; } }; } // namespace frontend -template -class LayersQuantizer : public frontend::DataQuantizerBase { - public: - explicit LayersQuantizer(float scaleFactor) : DataQuantizerBase(scaleFactor) {} +struct LayersQuantizer { + explicit LayersQuantizer(float scaleFactor) {} template bool operator()(T input) const { - return frontend::DataQuantizer(scaleFactor)(input); + return frontend::DataQuantizer()(input); } }; -/* - * Major of layers will be executed in I16 mode - * most of auto generated primitives like one for aligning support - * GNA 1.0, 2.0 doesn’t support I8 for convolution layer. - * Some layers will be switched into I16 mode to not lose accuracy while memory and - * runtime performance of layers like scaleshifts still OK since it is O(N). - */ -using QuantI16 = frontend::QuantPair; -using QuantI8 = frontend::QuantPair; -using QuantI8_I8 = frontend::QuantPair; - -using FakeQuant = frontend::QuantPair; - enum class QuantizedDataType { input, output, @@ -727,7 +477,7 @@ enum class QuantizedDataType { * @param data_type Type of data to be quantized * @return scale factor */ -inline float getScaleFactor(InferenceEngine::CNNLayerPtr layer, QuantizedDataType data_type) { +inline float GetScaleFactor(InferenceEngine::CNNLayerPtr layer, QuantizedDataType data_type) { IE_ASSERT(layer != nullptr); auto quantized = InferenceEngine::getInjectedData(layer); float scale_factor; @@ -739,7 +489,7 @@ inline float getScaleFactor(InferenceEngine::CNNLayerPtr layer, QuantizedDataTyp scale_factor = quantized->_src_quant.GetScale(); break; case QuantizedDataType::output: - scale_factor = quantized->_dst_quant.GetScale(); + scale_factor = quantized->_dst_quant.GetScale(); break; case QuantizedDataType::weights: scale_factor = quantized->_weights_quant.GetScale(); diff --git a/src/plugins/intel_gna/src/frontend/model_quantizer.hpp b/src/plugins/intel_gna/src/frontend/model_quantizer.hpp index 86d989da837..84dc1f661af 100644 --- a/src/plugins/intel_gna/src/frontend/model_quantizer.hpp +++ b/src/plugins/intel_gna/src/frontend/model_quantizer.hpp @@ -22,125 +22,125 @@ namespace GNAPluginNS { /** - * Quantize entire cnn - network - * @tparam T - type trait for weights and biases + * Quantize entire network */ -template class ModelQuantizer { public: - InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork &model, float scaleFactor) const { - return quantize(model, [](const InferenceEngine::CNNNetwork &, bool runBeforeCopy, bool lowPrecision){}, scaleFactor); + // Below four functions are used only in unit tests + InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork& model, float scale_factor) const { + return quantize(model, [](const InferenceEngine::CNNNetwork&, bool run_before_copy, bool low_precision){}, scale_factor); } template - InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork &model, const PreQuantisationCb &cb, float scaleFactor) const { - return quantize(model, cb, std::vector({scaleFactor})); + InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork& model, const PreQuantisationCb& cb, float scale_factor) const { + return quantize(model, cb, std::vector({scale_factor})); } - InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork &model, std::vector scaleFactors) const { - return quantize(model, [](InferenceEngine::CNNNetwork &, bool runBeforeCopy, bool lowPrecision){}, scaleFactors); + InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork& model, std::vector scale_factors) const { + return quantize(model, [](InferenceEngine::CNNNetwork&, bool run_before_copy, bool low_precision){}, scale_factors); } template - InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork &model, const PreQuantisationCb &cb, std::vector scaleFactors) const { + InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork& model, const PreQuantisationCb& cb, std::vector scale_factors) const { GNAPluginNS::GnaInputs inputs; InferenceEngine::InputsDataMap inputs_map = model.getInputsInfo(); int sf_id = 0; for (auto &&input_data : inputs_map) { auto input_layer = getCreatorLayer(input_data.second->getInputData()).lock(); - if (scaleFactors.size() <= sf_id) { + if (scale_factors.size() <= sf_id) { THROW_GNA_EXCEPTION << "Scale factors are not set for some of the inputs"; } - inputs[input_layer->name].scale_factor = scaleFactors[sf_id++]; + inputs[input_layer->name].scale_factor = scale_factors[sf_id++]; } - return quantize(model, cb, inputs); + return quantize(model, cb, inputs, InferenceEngine::Precision::I16, false); } + // This function is acutally used by the plugin template - InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork &model, - const PreQuantisationCb &cb, - const GNAPluginNS::GnaInputs &inputs) const { + InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork& model, + const PreQuantisationCb& cb, + const GNAPluginNS::GnaInputs& inputs, + const InferenceEngine::Precision& gna_precision, + const bool& inputs_int8_precision) const { OV_ITT_SCOPED_TASK(itt::domains::GNA_LT, "ModelQuantizer::quantize"); auto visitor = [&](InferenceEngine::CNNLayerPtr lp) { - auto newLayer = InferenceEngine::injectData(lp); - transformLayer(newLayer, WeightsConverter()); - return newLayer; + auto new_layer = InferenceEngine::injectData(lp); + transformLayer(new_layer, WeightsConverter()); + return new_layer; }; - bool lowPrecision = (T::mandatory().getInputPrecision().size() == sizeof(uint8_t)); - InferenceEngine::CNNNetwork copiedNet = InferenceEngine::CNNNetCopy(model); - cb(copiedNet, true, lowPrecision); - copiedNet = InferenceEngine::CNNNetCopy(copiedNet, visitor); + // User's hints for GNA weights precision + const bool weights_int8_precision = (gna_precision == InferenceEngine::Precision::I8) ? true : false; - // allow client code to access copied topology, to avoid copies if user would like to chain quantisation with + InferenceEngine::CNNNetwork copied_net = InferenceEngine::CNNNetCopy(model); + cb(copied_net, true, inputs_int8_precision); + + copied_net = InferenceEngine::CNNNetCopy(copied_net, visitor); + + // Allow client code to access copied topology, to avoid copies if user would like to chain quantisation with // another preprocessing - cb(copiedNet, false, lowPrecision); + cb(copied_net, false, inputs_int8_precision); - float lc_sf = GNAPluginNS::kScaleFactorDefault; - if (inputs.empty()) { - gnawarn() << "Inputs structure is empty, will be used default scale factor: " << lc_sf << std::endl; - } else { - lc_sf = inputs.Get().begin()->scale_factor; - } - - LayersQuantizer lc(lc_sf); - auto sortedNewNet = InferenceEngine::details::CNNNetSortTopologically(copiedNet); + LayersQuantizer lc(GNAPluginNS::kScaleFactorDefault); + auto sorted_new_net = InferenceEngine::details::CNNNetSortTopologically(copied_net); gnalog() << "Sorted layers: " << std::endl; - for (auto &&layer : sortedNewNet) { - auto quantData = InferenceEngine::getInjectedData(layer); - quantData->lowPrecision = lowPrecision; + for (auto &&layer : sorted_new_net) { + auto quant_data = InferenceEngine::getInjectedData(layer); + quant_data->_inputs_int8_precision = inputs_int8_precision; + quant_data->_weights_int8_precision = weights_int8_precision; gnalog() << layer->name << std::endl; } - /// filling scale factors for input layers, memory layers will have scaleFactor of 1.0 by default - InferenceEngine::InputsDataMap dm = copiedNet.getInputsInfo(); + + // Filling scale factors for input layers, memory layers will have scale factor of 1.0 by default + InferenceEngine::InputsDataMap dm = copied_net.getInputsInfo(); for (auto &&inputData : dm) { - auto inputLayer = getCreatorLayer(inputData.second->getInputData()).lock(); - auto quantData = InferenceEngine::getInjectedData(inputLayer); - IE_ASSERT(quantData != nullptr); - quantData->_src_quant.SetScale(inputs.at(inputLayer->name).scale_factor); + auto input_layer = getCreatorLayer(inputData.second->getInputData()).lock(); + auto quant_data = InferenceEngine::getInjectedData(input_layer); + IE_ASSERT(quant_data != nullptr); + quant_data->_src_quant.SetScale(inputs.at(input_layer->name).scale_factor); } - propagateScaleFactor(sortedNewNet); + propagateScaleFactor(sorted_new_net); - // sorted order gives possibility for propagate quantisation along depended layers - for (auto &&layer : sortedNewNet) { + // Sorted order gives possibility for propagate quantisation along depended layers + for (auto &&layer : sorted_new_net) { transformLayer(layer, lc); } - return copiedNet; + return copied_net; } private : - void propagateScaleFactor(std::vector & net) const { - ScaleFactorCalculator sf(net); + void propagateScaleFactor(std::vector& net) const { + ScaleFactorCalculator sf(net); - int infiniteLoopCount = 0; - std::vector infiniteLoopPattern; - std::vector infiniteLoopHistory; - while (!sf.allLayersProcessed() && infiniteLoopCount <= 2) { + uint32_t inf_loop_count = 0; + std::vector inf_loop_pattern; + std::vector inf_loop_history; + while (!sf.allLayersProcessed() && inf_loop_count <= 2) { auto layers = sf.getStartLayers(); - infiniteLoopHistory.emplace_back(layers.front()->name); + inf_loop_history.emplace_back(layers.front()->name); for (auto &&layer : layers) { transformLayer(layer, sf); // transforming until we reached cases where output scale updated due to situation in downstream layer if (sf.needToRestart()) { - infiniteLoopHistory.back() += "#" + layer->name; + inf_loop_history.back() += "#" + layer->name; break; } } // We are looking for infinite loop by using algorithm of compute prefix function, complexity O(N) // (a part of the Knuth–Morris–Pratt algorithm). - std::map prefixFunction; - int k = infiniteLoopHistory.size(); - for (int i = infiniteLoopHistory.size() - 2; i >= 0; i--) { - while (k < infiniteLoopHistory.size() && infiniteLoopHistory[k - 1] != infiniteLoopHistory[i]) { - auto iter = prefixFunction.find(k); - k = iter == prefixFunction.end() ? infiniteLoopHistory.size() : iter->second; + std::map prefix_function; + int32_t k = inf_loop_history.size(); + for (int32_t i = inf_loop_history.size() - 2; i >= 0; i--) { + while (k < inf_loop_history.size() && inf_loop_history[k - 1] != inf_loop_history[i]) { + auto iter = prefix_function.find(k); + k = iter == prefix_function.end() ? inf_loop_history.size() : iter->second; } - if (infiniteLoopHistory[k - 1] == infiniteLoopHistory[i]) { + if (inf_loop_history[k - 1] == inf_loop_history[i]) { k--; } @@ -154,56 +154,56 @@ class ModelQuantizer { // // In the case of pattern length is 1, an infinite loop can be found on 2 consecutive strings. // To avoid this, we will expect the appearance of 4 equal strings for the case pattern length is 1. - if ((infiniteLoopHistory.size() - i) % 2 == 0 && - (infiniteLoopHistory.size() - i) / 2 == infiniteLoopHistory.size() - k && - ((infiniteLoopHistory.size() - i) / 2 > 1 || - std::distance(infiniteLoopHistory.rbegin(), - std::find_if_not(infiniteLoopHistory.rbegin(), infiniteLoopHistory.rend(), - [&infiniteLoopHistory](const std::string& str) { return str == infiniteLoopHistory.back(); })) > 3)) { - gnalog() << "infiniteLoopPattern:\n"; - for (const auto& s : infiniteLoopPattern) { + if ((inf_loop_history.size() - i) % 2 == 0 && + (inf_loop_history.size() - i) / 2 == inf_loop_history.size() - k && + ((inf_loop_history.size() - i) / 2 > 1 || + std::distance(inf_loop_history.rbegin(), + std::find_if_not(inf_loop_history.rbegin(), inf_loop_history.rend(), + [&inf_loop_history](const std::string& str) { return str == inf_loop_history.back(); })) > 3)) { + gnalog() << "inf_loop_pattern:\n"; + for (const auto& s : inf_loop_pattern) { gnalog() << "\t " << s << '\n'; } - infiniteLoopPattern.clear(); - int patternLength = (infiniteLoopHistory.size() - i) / 2; - gnalog() << "patternLength: " << patternLength << '\n'; - for (int j = 0; j < patternLength; j++) { - infiniteLoopPattern.emplace_back(infiniteLoopHistory[infiniteLoopHistory.size() - patternLength + j]); + inf_loop_pattern.clear(); + int pattern_len = (inf_loop_history.size() - i) / 2; + gnalog() << "pattern_len: " << pattern_len << '\n'; + for (int j = 0; j < pattern_len; j++) { + inf_loop_pattern.emplace_back(inf_loop_history[inf_loop_history.size() - pattern_len + j]); } - gnalog() << "infiniteLoopHistory:\n"; - for (const auto& s : infiniteLoopHistory) { + gnalog() << "inf_loop_history:\n"; + for (const auto& s : inf_loop_history) { gnalog() << "\t " << s << '\n'; } - infiniteLoopHistory.clear(); + inf_loop_history.clear(); gnalog() << "infinite loop detected\n"; break; } - prefixFunction.emplace(i, k); + prefix_function.emplace(i, k); } - if (infiniteLoopHistory.empty()) { - infiniteLoopCount++; + if (inf_loop_history.empty()) { + inf_loop_count++; } else { - if (infiniteLoopCount > 0 && - (infiniteLoopHistory.size()%infiniteLoopPattern.size() == 0 || sf.allLayersProcessed()) && - !std::equal(infiniteLoopHistory.begin() + (infiniteLoopHistory.size() - infiniteLoopPattern.size()), - infiniteLoopHistory.end(), infiniteLoopPattern.begin())) { - infiniteLoopCount = 0; - infiniteLoopPattern.clear(); + if (inf_loop_count > 0 && + (inf_loop_history.size()%inf_loop_pattern.size() == 0 || sf.allLayersProcessed()) && + !std::equal(inf_loop_history.begin() + (inf_loop_history.size() - inf_loop_pattern.size()), + inf_loop_history.end(), inf_loop_pattern.begin())) { + inf_loop_count = 0; + inf_loop_pattern.clear(); gnalog() << "infinite loop fixed\n"; } } - sf.SetInfiniteLoopCount(infiniteLoopCount); + sf.SetInfiniteLoopCount(inf_loop_count); } - if (infiniteLoopCount > 0) { - std::string additionalInformation; - for (const auto& p : infiniteLoopPattern) { - additionalInformation += '\n' + p; + if (inf_loop_count > 0) { + std::string additional_info; + for (const auto& p : inf_loop_pattern) { + additional_info += '\n' + p; } - THROW_GNA_EXCEPTION << "infinite loop: " + additionalInformation; + THROW_GNA_EXCEPTION << "infinite loop: " + additional_info; } } }; diff --git a/src/plugins/intel_gna/src/frontend/quantization.cpp b/src/plugins/intel_gna/src/frontend/quantization.cpp index db36f398392..5deb0977c35 100644 --- a/src/plugins/intel_gna/src/frontend/quantization.cpp +++ b/src/plugins/intel_gna/src/frontend/quantization.cpp @@ -6,161 +6,9 @@ #include #include #include "backend/gna_types.h" -#include "quantization.h" +#include "quantization.hpp" #include -#ifdef DEBUG -#define QUANTWARNING(...) (fprintf(stderr, __VA_ARGS__)) -#else -#define QUANTWARNING(...) -#endif - - -template<> -void QuantizationCallback::runFakeQuantize() const { - if (quantizedWeights) { - THROW_GNA_EXCEPTION << "Quantized weights are not yet supported in int16 quantization mode"; - } - - uint32_t num_saturate = 0; - auto input_low = 0.0f; - auto input_high = 0.0f; - auto output_low = 0.0f; - auto output_high = 0.0f; - size_t levels = 1; - if (fq_num_stats > 0) { - input_low = *fq_ptr_input_low; - input_high = *fq_ptr_input_high; - output_low = *fq_ptr_output_low; - output_high = *fq_ptr_output_high; - levels = fq_levels; - } - - for (uint32_t row = 0; row < num_rows; row++) { - for (uint32_t col = 0; col < num_columns; col++) { - float rounding_value = (ptr_float_weights[row * num_columns + col] > 0) ? 0.5f : -0.5f; - float value = ptr_float_weights[row * num_columns + col]; - if (fq_num_stats > 0) { - auto x = value; - if (x <= std::min(input_low, input_high)) { - value = output_low; - } else if (x > std::max(input_low, input_high)) { - value = output_high; - } else { - value = nearbyint((x - input_low) / (input_high - input_low) * (levels - 1)) / - (levels - 1) * (output_high - output_low) + output_low; - } - } - - value = value * *ptr_weight_scale_factor + rounding_value; - - int16_t* ptr_weight_16 = ptr_int_weights + (row * num_columns_padded + col); - - if (value > std::numeric_limits::max()) { - *ptr_weight_16 = std::numeric_limits::max(); - num_saturate++; - } else if (value < std::numeric_limits::min()) { - *ptr_weight_16 = std::numeric_limits::min(); - num_saturate++; - } else { - *ptr_weight_16 = (int16_t)value; - } - } - for (uint32_t col = num_columns; col < num_columns_padded; col++) { - int16_t* ptr_weight_16 = ptr_int_weights + (row * num_columns_padded + col); - *ptr_weight_16 = 0; - } - } - for (uint32_t row = num_rows; row < num_rows_padded; row++) { - for (uint32_t col = 0; col < num_columns_padded; col++) { - int16_t* ptr_weight_16 = ptr_int_weights + (row * num_columns_padded + col); - *ptr_weight_16 = 0; - } - } - - // case for element wise layer - if (ptr_float_biases != nullptr && ptr_int_biases != nullptr) { - for (uint32_t j = 0; j < num_rows; j++) { - float rounding_value = (ptr_float_biases[j] > 0) ? 0.5f : -0.5f; - float value = ptr_float_biases[j] * *ptr_output_scale_factor + rounding_value; - if (value > 2147483647.0) { - ptr_int_biases[j] = 2147483647L; - num_saturate++; - } else if (value < -2147483648.0) { - ptr_int_biases[j] = -2147483648LL; - num_saturate++; - } else { - ptr_int_biases[j] = (int32_t)value; - } - } - for (uint32_t j = num_rows; j < num_rows_padded; j++) { - ptr_int_biases[j] = 0; - } - } - - if (num_saturate > 0) { - QUANTWARNING("Warning: %d / %d saturations in QuantizeAffine16()\n", - num_saturate, - num_rows * num_columns + num_rows); - } -} - -template<> -void QuantizationCallback::runQuantize() const { - uint32_t num_saturate = 0; - for (uint32_t row = 0; row < num_rows; row++) { - for (uint32_t col = 0; col < num_columns; col++) { - float rounding_value = (ptr_float_weights[row * num_columns + col] > 0) ? 0.5f : -0.5f; - float value = ptr_float_weights[row * num_columns + col] * *ptr_weight_scale_factor + rounding_value; - int16_t *ptr_weight_16 = ptr_int_weights + (row * num_columns_padded + col); - if (value > 32767.0) { - *ptr_weight_16 = 32767; - num_saturate++; - } else if (value < -32768.0) { - *ptr_weight_16 = -32768; - num_saturate++; - } else { - *ptr_weight_16 = (int16_t) value; - } - } - for (uint32_t col = num_columns; col < num_columns_padded; col++) { - int16_t *ptr_weight_16 = ptr_int_weights + (row * num_columns_padded + col); - *ptr_weight_16 = 0; - } - } - for (uint32_t row = num_rows; row < num_rows_padded; row++) { - for (uint32_t col = 0; col < num_columns_padded; col++) { - int16_t *ptr_weight_16 = ptr_int_weights + (row * num_columns_padded + col); - *ptr_weight_16 = 0; - } - } - - // case for element wise layer - if (ptr_float_biases != nullptr && ptr_int_biases != nullptr) { - for (uint32_t j = 0; j < num_rows; j++) { - float rounding_value = (ptr_float_biases[j] > 0) ? 0.5f : -0.5f; - float value = ptr_float_biases[j] * *ptr_output_scale_factor + rounding_value; - if (value > 2147483647.0) { - ptr_int_biases[j] = 2147483647L; - num_saturate++; - } else if (value < -2147483648.0) { - ptr_int_biases[j] = -2147483648LL; - num_saturate++; - } else { - ptr_int_biases[j] = (int32_t) value; - } - } - for (uint32_t j = num_rows; j < num_rows_padded; j++) { - ptr_int_biases[j] = 0; - } - } - - if (num_saturate > 0) { - QUANTWARNING("Warning: %d / %d saturations in QuantizeAffine16()\n", - num_saturate, - num_rows * num_columns + num_rows); - } -} std::pair FindMinMaxValues(void* ptr_float_memory, size_t num_elements) { float* ptr_float_feat = reinterpret_cast(ptr_float_memory); @@ -180,9 +28,9 @@ std::pair FindMinMaxValues(void* ptr_float_memory, size_t num_elem return { min, max }; } -float ScaleFactorForQuantization(void *ptr_float_memory, float target_max, size_t num_elements) { +float ScaleFactorForQuantization(void* ptr_float_memory, float target_max, size_t num_elements) { float *ptr_float_feat = reinterpret_cast(ptr_float_memory); - float max = 0.0; + float max = 0.0f; float scale_factor; for (size_t i = 0; i < num_elements; i++) { @@ -200,262 +48,236 @@ float ScaleFactorForQuantization(void *ptr_float_memory, float target_max, size_ return (scale_factor); } -void QuantizeVector16(float *ptr_float_memory, int16_t *ptr_int_memory, uint32_t num_elements, float scale_factor) { - float *ptr_float_feat = reinterpret_cast(ptr_float_memory); - uint32_t num_saturate = 0; +template +void QuantizeWeights(const QuantizationData& data, float* ptr_float_weights, T* ptr_int_weights, + gna_compound_bias_t* ptr_int_biases, const bool quantized_weights) {} - int16_t *ptr_int_feat = reinterpret_cast(ptr_int_memory); - for (uint32_t i = 0; i < num_elements; i++) { - float rounding_value = (ptr_float_feat[i] > 0) ? 0.5f : -0.5f; - float value = ptr_float_feat[i] * scale_factor + rounding_value; - if (value > 32767.0) { - ptr_int_feat[i] = 32767; - num_saturate++; - } else if (value < -32768.0) { - ptr_int_feat[i] = -32768; - num_saturate++; - } else { - ptr_int_feat[i] = (int16_t) value; - } +template +void QuantizeBiases(const QuantizationData& data, float* ptr_float_biases, T* ptr_int_biases) {} + +template <> +void QuantizeWeights(const QuantizationData& data, float* ptr_float_weights, int8_t* ptr_int_weights, + gna_compound_bias_t* ptr_int_biases, const bool quantized_weights) { + if (!ptr_int_biases && quantized_weights) { + THROW_GNA_EXCEPTION << "Quantized weights are not yet supported in int8 quantization mode"; } - if (num_saturate > 0) { - QUANTWARNING("Warning: %d / %d saturations during QuantizeVector16()\n", num_saturate, num_elements); - } -} - -template<> -void QuantizationCallback::runFakeQuantize() const { uint32_t num_saturate = 0; - auto input_low = 0.0f; auto input_high = 0.0f; auto output_low = 0.0f; auto output_high = 0.0f; size_t levels = 1; float valueAcc = 0.0f; - for (uint32_t i = 0; i < num_rows; i++) { + const auto min_values_size = data.weights_quant_params.GetMinValues().size(); + + if (min_values_size > 0) { + input_low = data.weights_quant_params.GetMinValues(true).front(); + input_high = data.weights_quant_params.GetMaxValues(true).front(); + output_low = data.weights_quant_params.GetMinValues(false).front(); + output_high = data.weights_quant_params.GetMaxValues(false).front(); + levels = data.weights_quant_params.GetLevels(); + } + + for (size_t row = 0; row < data.num_rows; row++) { uint32_t channel_multiplier = 1; - if (fq_num_stats > 0) { - auto idx = fq_num_stats == 1 ? 0 : i; - input_low = fq_ptr_input_low[idx]; - input_high = fq_ptr_input_high[idx]; - output_low = fq_ptr_output_low[idx]; - output_high = fq_ptr_output_high[idx]; - levels = fq_levels; - channel_multiplier = static_cast(((input_high - input_low) * *ptr_weight_scale_factor) / (levels - 1)); - } else { - float scaled_row_max = 0; - for (uint32_t col = 0; col < num_columns; col++) { - float value = ptr_float_weights[i * num_columns + col] * *ptr_weight_scale_factor; - valueAcc += value; - if (fabs(value) > scaled_row_max) { - scaled_row_max = fabs(value); - } - } - - channel_multiplier = static_cast(scaled_row_max / static_cast(MAX_VAL_1B_WEIGHT)); - } - - // channel multiplier shouldn't be 0 - channel_multiplier = channel_multiplier == 0 ? 1 : channel_multiplier; - - ptr_int_biases[i].multiplier = static_cast (channel_multiplier + 0.5f); - if (channel_multiplier > MAX_OUT_MULTIPLIER) { - THROW_GNA_EXCEPTION << "invalid channel multiplier: " << channel_multiplier; - } - - for (uint32_t j = 0; j < num_columns; j++) { - auto offset = i * num_columns + j; - auto rounding_value = (ptr_float_weights[i * num_columns + j] > 0) ? 0.5f : -0.5f; - float value = ptr_float_weights[offset]; - if (!quantizedWeights) { - if (fq_num_stats > 0) { - auto x = value; - if (x <= std::min(input_low, input_high)) { - value = output_low; - } else if (x > std::max(input_low, input_high)) { - value = output_high; - } else { - value = nearbyint((x - input_low) / (input_high - input_low) * (levels - 1)) / - (levels - 1) * (output_high - output_low) + output_low; + if (ptr_int_biases) { + if (min_values_size > 0) { + auto idx = min_values_size == 1 ? 0 : row; + input_low = data.weights_quant_params.GetMinValues(true).at(idx); + input_high = data.weights_quant_params.GetMaxValues(true).at(idx); + output_low = data.weights_quant_params.GetMinValues(false).at(idx); + output_high = data.weights_quant_params.GetMaxValues(false).at(idx); + levels = data.weights_quant_params.GetLevels(); + channel_multiplier = ((input_high - input_low) * data.scale_factor) / (levels - 1); + } else { + float scaled_row_max = 0; + for (size_t col = 0; col < data.num_columns; col++) { + float value = ptr_float_weights[row * data.num_columns + col] * data.scale_factor; + valueAcc += value; + if (fabs(value) > scaled_row_max) { + scaled_row_max = fabs(value); } } - value = value * (*ptr_weight_scale_factor / ptr_int_biases[i].multiplier) + rounding_value; - } else { - value -= MAX_VAL_1B_WEIGHT; + channel_multiplier = (scaled_row_max / static_cast(MAX_VAL_1B_WEIGHT) + 0.5f); } - auto normalizedWeight = static_cast(value); + + if (channel_multiplier > MAX_OUT_MULTIPLIER) { + THROW_GNA_EXCEPTION << "invalid channel multiplier: " << channel_multiplier; + } + + // channel multiplier shouldn't be 0 + ptr_int_biases[row].multiplier = (channel_multiplier == 0) ? 1 : channel_multiplier; + } + + for (uint32_t col = 0; col < data.num_columns; col++) { + auto offset = row * data.num_columns + col; + float rounding_value = (ptr_float_weights[offset] > 0) ? 0.5f : -0.5f; + float value = ptr_float_weights[offset]; + + if (min_values_size > 0) { + auto x = value; + if (x <= std::min(input_low, input_high)) { + value = output_low; + } else if (x > std::max(input_low, input_high)) { + value = output_high; + } else { + value = nearbyint((x - input_low) / (input_high - input_low) * (levels - 1)) / (levels - 1) * + (output_high - output_low) + output_low; + } + } + + if (ptr_int_biases) { + if (quantized_weights) { + value -= MAX_VAL_1B_WEIGHT; + } else { + value = value * (data.scale_factor / channel_multiplier) + rounding_value; + } + } else { + value = value * data.scale_factor + rounding_value; + } + + int8_t* ptr_weight_8 = ptr_int_weights + offset; if (value > std::numeric_limits::max()) { - normalizedWeight = std::numeric_limits::max(); + *ptr_weight_8 = std::numeric_limits::max(); num_saturate++; } else if (value < std::numeric_limits::min()) { - normalizedWeight = std::numeric_limits::min(); + *ptr_weight_8 = std::numeric_limits::min(); num_saturate++; } else { - normalizedWeight = (int8_t)value; - } - - // range checking - ptr_int_weights[offset] = static_cast(normalizedWeight); - } - - for (uint32_t j = num_columns; j < num_columns_padded; j++) { - ptr_int_weights[i * num_columns + j] = 0; - } - } - - for (uint32_t i = num_rows; i < num_rows_padded; i++) { - for (uint32_t j = 0; j < num_columns_padded; j++) { - ptr_int_weights[i * num_columns + j] = 0; - } - ptr_int_biases[i].multiplier = 0; - } - - if (ptr_float_biases != nullptr) { - for (uint32_t j = 0; j < num_rows; j++) { - float rounding_value = (ptr_float_biases[j] > 0) ? 0.5f : -0.5f; - float value = ptr_float_biases[j] * *ptr_output_scale_factor + rounding_value; - if (value > 2147483647.0) { - ptr_int_biases[j].bias = 2147483647L; - num_saturate++; - } else if (value < -2147483648.0) { - ptr_int_biases[j].bias = -2147483648LL; - num_saturate++; - } else { - ptr_int_biases[j].bias = (int32_t) value; - } - } - } - if (num_saturate > 0) { - QUANTWARNING("Warning: %d / %d saturations in QuantizeAffine8()\n", num_saturate, num_rows * num_columns + num_rows); - } -} - -template<> -void QuantizationCallback::runQuantize() const { - if (ptr_int_biases == nullptr) { - IE_THROW() << "Int biases are empty"; - } - uint32_t num_saturate = 0; - - float valueAcc = 0.0; - for (uint32_t row = 0; row < num_rows; row++) { - float scaled_row_max = 0; - float rounding_value, value; - for (uint32_t col = 0; col < num_columns; col++) { - value = ptr_float_weights[row*num_columns + col] * *ptr_weight_scale_factor; - valueAcc += value; - if (fabs(value) > scaled_row_max) { - scaled_row_max = fabs(value); - } - } - - value = scaled_row_max / static_cast(MAX_VAL_1B_WEIGHT); - ptr_int_biases[row].multiplier = (uint8_t) (value + 0.5); - for (uint32_t col = 0; col < num_columns; col++) { - int8_t *ptr_weight_8 = ptr_int_weights + (row * num_columns_padded + col); - rounding_value = (ptr_float_weights[row * num_columns + col] > 0) ? 0.5f : -0.5f; - - value = ptr_float_weights[row * num_columns + col] * (*ptr_weight_scale_factor / ptr_int_biases[row].multiplier) + rounding_value; - if (value > 127.0) { - *ptr_weight_8 = 127; - num_saturate++; - } else if (value < -128.0) { - *ptr_weight_8 = -128; - num_saturate++; - } else { - *ptr_weight_8 = (int8_t) value; - } - } - for (uint32_t col = num_columns; col < num_columns_padded; col++) { - int8_t *ptr_weight_8 = ptr_int_weights + (row * num_columns_padded + col); - *ptr_weight_8 = 0; - } - } - for (uint32_t row = num_rows; row < num_rows_padded; row++) { - for (uint32_t col = 0; col < num_columns_padded; col++) { - int8_t *ptr_weight_8 = ptr_int_weights + (row*num_columns_padded + col); - *ptr_weight_8 = 0; - } - ptr_int_biases[row].multiplier = 0; - } - - // bias value of the bas will be only used when input bias provided - if (ptr_float_biases != nullptr) { - for (uint32_t j = 0; j < num_rows; j++) { - float rounding_value = (ptr_float_biases[j] > 0) ? 0.5f : -0.5f; - float value = ptr_float_biases[j] * *ptr_output_scale_factor + rounding_value; - if (value > 2147483647.0) { - ptr_int_biases[j].bias = 2147483647L; - num_saturate++; - } else if (value < -2147483648.0) { - ptr_int_biases[j].bias = -2147483648LL; - num_saturate++; - } else { - ptr_int_biases[j].bias = (int32_t) value; + *ptr_weight_8 = static_cast(value); } } } if (num_saturate > 0) { - QUANTWARNING("Warning: %d / %d saturations in QuantizeAffine8()\n", num_saturate, num_rows * num_columns + num_rows); + gnalog() << "Warning: " << num_saturate << " / " << (data.num_rows * data.num_columns) + << " saturations in int8 weights quantization." << std::endl; } } template<> -void QuantizationCallback::runQuantize() const { +void QuantizeWeights(const QuantizationData& data, float* ptr_float_weights, int16_t* ptr_int_weights, + gna_compound_bias_t* ptr_int_biases, const bool quantized_weights) { + if (quantized_weights) { + THROW_GNA_EXCEPTION << "Quantized weights are not yet supported in int16 quantization mode"; + } + uint32_t num_saturate = 0; - for (uint32_t row = 0; row < num_rows; row++) { - for (uint32_t col = 0; col < num_columns; col++) { - float rounding_value = (ptr_float_weights[row * num_columns + col] > 0) ? 0.5f : -0.5f; - float value = ptr_float_weights[row * num_columns + col] * *ptr_weight_scale_factor + rounding_value; - int8_t* ptr_weight_8 = ptr_int_weights + (row * num_columns_padded + col); - if (value > 127.0) { - *ptr_weight_8 = 127; + auto input_low = 0.0f; + auto input_high = 0.0f; + auto output_low = 0.0f; + auto output_high = 0.0f; + size_t levels = 1; + const auto min_values_size = data.weights_quant_params.GetMinValues().size(); + + if (min_values_size > 0) { + input_low = data.weights_quant_params.GetMinValues(true).front(); + input_high = data.weights_quant_params.GetMaxValues(true).front(); + output_low = data.weights_quant_params.GetMinValues(false).front(); + output_high = data.weights_quant_params.GetMaxValues(false).front(); + levels = data.weights_quant_params.GetLevels(); + } + + for (size_t row = 0; row < data.num_rows; row++) { + for (size_t col = 0; col < data.num_columns; col++) { + float rounding_value = (ptr_float_weights[row * data.num_columns + col] > 0) ? 0.5f : -0.5f; + float value = ptr_float_weights[row * data.num_columns + col]; + if (min_values_size > 0) { + auto x = value; + if (x <= std::min(input_low, input_high)) { + value = output_low; + } else if (x > std::max(input_low, input_high)) { + value = output_high; + } else { + value = nearbyint((x - input_low) / (input_high - input_low) * (levels - 1)) / + (levels - 1) * (output_high - output_low) + output_low; + } + } + + value = value * data.scale_factor + rounding_value; + + int16_t* ptr_weight_16 = ptr_int_weights + (row * data.num_columns + col); + + if (value > std::numeric_limits::max()) { + *ptr_weight_16 = std::numeric_limits::max(); num_saturate++; - } else if (value < -128.0) { - *ptr_weight_8 = -128; + } else if (value < std::numeric_limits::min()) { + *ptr_weight_16 = std::numeric_limits::min(); num_saturate++; } else { - *ptr_weight_8 = (int8_t)value; + *ptr_weight_16 = (int16_t)value; } } - for (uint32_t col = num_columns; col < num_columns_padded; col++) { - int8_t* ptr_weight_8 = ptr_int_weights + (row * num_columns_padded + col); - *ptr_weight_8 = 0; - } - } - for (uint32_t row = num_rows; row < num_rows_padded; row++) { - for (uint32_t col = 0; col < num_columns_padded; col++) { - int8_t* ptr_weight_8 = ptr_int_weights + (row * num_columns_padded + col); - *ptr_weight_8 = 0; - } } + if (num_saturate > 0) { + gnalog() << "Warning: " << num_saturate << " / " << (data.num_rows * data.num_columns) + << " saturations in int16 weights quantization." << std::endl; + } +} + +template<> +void QuantizeBiases(const QuantizationData& data, float* ptr_float_biases, int8_t* ptr_int_biases) { + // Stub +} + +template<> +void QuantizeBiases(const QuantizationData& data, float* ptr_float_biases, int16_t* ptr_int_biases) { + // Stub +} + +template<> +void QuantizeBiases(const QuantizationData& data, float* ptr_float_biases, int32_t* ptr_int_biases) { + uint32_t num_saturate = 0; + + // case for element wise layer if (ptr_float_biases != nullptr && ptr_int_biases != nullptr) { - for (uint32_t j = 0; j < num_rows; j++) { - float rounding_value = (ptr_float_biases[j] > 0) ? 0.5f : -0.5f; - float value = ptr_float_biases[j] * *ptr_output_scale_factor + rounding_value; - if (value > 127.0) { - ptr_int_biases[j] = 127; + for (size_t row = 0; row < data.num_rows; row++) { + float rounding_value = (ptr_float_biases[row] > 0) ? 0.5f : -0.5f; + float value = ptr_float_biases[row] * data.scale_factor + rounding_value; + if (value > std::numeric_limits::max()) { + ptr_int_biases[row] = std::numeric_limits::max(); num_saturate++; - } else if (value < -128.0) { - ptr_int_biases[j] = -128; + } else if (value < std::numeric_limits::min()) { + ptr_int_biases[row] = std::numeric_limits::min(); num_saturate++; } else { - ptr_int_biases[j] = (int8_t)value; + ptr_int_biases[row] = static_cast(value); } } - for (uint32_t j = num_rows; j < num_rows_padded; j++) { - ptr_int_biases[j] = 0; - } } if (num_saturate > 0) { - QUANTWARNING("Warning: %d / %d saturations in QuantizeAffine8_8()\n", num_saturate, num_rows * num_columns + num_rows); + gnalog() << "Warning: " << num_saturate << " / " << data.num_rows + << " saturations in int32 biases quantization." << std::endl; + } +} + +template<> +void QuantizeBiases(const QuantizationData& data, float* ptr_float_biases, gna_compound_bias_t* ptr_int_biases) { + uint32_t num_saturate = 0; + + if (ptr_float_biases != nullptr) { + for (size_t row = 0; row < data.num_rows; row++) { + float rounding_value = (ptr_float_biases[row] > 0) ? 0.5f : -0.5f; + float value = ptr_float_biases[row] * data.scale_factor + rounding_value; + if (value > std::numeric_limits::max()) { + ptr_int_biases[row].bias = std::numeric_limits::max(); + num_saturate++; + } else if (value < std::numeric_limits::min()) { + ptr_int_biases[row].bias = std::numeric_limits::min(); + num_saturate++; + } else { + ptr_int_biases[row].bias = static_cast(value); + } + } + } + if (num_saturate > 0) { + gnalog() << "Warning: " << num_saturate << " / " << data.num_rows + << " saturations in compound biases quantization." << std::endl; } } diff --git a/src/plugins/intel_gna/src/frontend/quantization.h b/src/plugins/intel_gna/src/frontend/quantization.h deleted file mode 100644 index d01f2e89495..00000000000 --- a/src/plugins/intel_gna/src/frontend/quantization.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (C) 2018-2022 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include -#include -#include -#include "backend/gna_types.h" - -#define MAX_OUT_MULTIPLIER 230 -#define MAX_VAL_1B_WEIGHT 127 -#define MAX_VAL_1B_FEAT 64 -#define MAX_VAL_1B_BIAS 127 -#define MAX_VAL_2B_WEIGHT 16384 -#define MAX_VAL_2B_FEAT 16384 -#define MAX_VAL_4B_BIAS 1073741824 - -template -struct QuantizationCallback { - float *ptr_float_weights; - float *ptr_float_biases; - WeightsType* ptr_int_weights; - BiasType* ptr_int_biases; - float input_scale_factor; - float *ptr_weight_scale_factor; - float *ptr_output_scale_factor; - uint32_t num_rows; - uint32_t num_columns; - uint32_t num_rows_padded; - uint32_t num_columns_padded; - - bool quantizedWeights; - size_t fq_levels; - const size_t fq_num_stats; - const float *fq_ptr_input_low; - const float *fq_ptr_input_high; - const float* fq_ptr_output_low; - const float* fq_ptr_output_high; - - void runQuantize() const; - void runFakeQuantize() const; -}; - -template struct QuantizationCallback; -template struct QuantizationCallback; -template struct QuantizationCallback; - -std::pair FindMinMaxValues(void* ptr_float_memory, size_t num_elements); -float ScaleFactorForQuantization(void *ptr_float_memory, float target_max, size_t num_elements); -void QuantizeVector16(float *ptr_float_memory, int16_t *ptr_int_memory, uint32_t num_elements, float scale_factor); diff --git a/src/plugins/intel_gna/src/frontend/quantization.hpp b/src/plugins/intel_gna/src/frontend/quantization.hpp new file mode 100644 index 00000000000..7dc1a5be6c4 --- /dev/null +++ b/src/plugins/intel_gna/src/frontend/quantization.hpp @@ -0,0 +1,38 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include +#include +#include "quantized_layer_params.hpp" +#include "backend/gna_types.h" + +#define MAX_OUT_MULTIPLIER 230 +#define MAX_VAL_1B_WEIGHT 127 +#define MAX_VAL_1B_FEAT 64 +#define MAX_VAL_1B_BIAS 127 +#define MAX_VAL_2B_WEIGHT 16384 +#define MAX_VAL_2B_FEAT 16384 +#define MAX_VAL_4B_BIAS 1073741824 + +// Common data required for quantization of weights and biases +struct QuantizationData { + const size_t num_rows; + const size_t num_columns; + float scale_factor; + // This field is currently used for weights as well as for biases + GNAPluginNS::QuantizationParams& weights_quant_params; +}; + +std::pair FindMinMaxValues(void* ptr_float_memory, size_t num_elements); +float ScaleFactorForQuantization(void *ptr_float_memory, float target_max, size_t num_elements); +template +extern void QuantizeWeights(const QuantizationData& data, float* ptr_float_weights, T* ptr_int_weights, + gna_compound_bias_t* ptr_int_biases, const bool quantized_weights); +template +extern void QuantizeBiases(const QuantizationData& data, float* ptr_float_biases, T* ptr_int_biases); diff --git a/src/plugins/intel_gna/src/frontend/quantized_layer_params.hpp b/src/plugins/intel_gna/src/frontend/quantized_layer_params.hpp index 9bbb625b20d..e0c66cf0344 100644 --- a/src/plugins/intel_gna/src/frontend/quantized_layer_params.hpp +++ b/src/plugins/intel_gna/src/frontend/quantized_layer_params.hpp @@ -6,7 +6,7 @@ namespace GNAPluginNS { -class Quantization { +class QuantizationParams { public: void SetScale(float s) { scale = s; @@ -59,7 +59,7 @@ public: return output_max_values; } - void CopyStats(Quantization &src) { + void CopyStats(QuantizationParams &src) { levels = src.GetLevels(); SetMinValues(src.GetMinValues(true), true); SetMaxValues(src.GetMaxValues(true), true); @@ -78,14 +78,15 @@ private: }; struct QuantizedLayerParams { - Quantization _src_quant; - Quantization _dst_quant; + QuantizationParams _src_quant; + QuantizationParams _dst_quant; // deprecate this - Quantization _weights_quant; - Quantization _bias_quant; + QuantizationParams _weights_quant; + QuantizationParams _bias_quant; - bool lowPrecision = false; + bool _weights_int8_precision = false; + bool _inputs_int8_precision = false; }; } // namespace GNAPluginNS diff --git a/src/plugins/intel_gna/src/frontend/scale_factor_calc.hpp b/src/plugins/intel_gna/src/frontend/scale_factor_calc.hpp index 3fb2c9768d1..b1e5a89c801 100644 --- a/src/plugins/intel_gna/src/frontend/scale_factor_calc.hpp +++ b/src/plugins/intel_gna/src/frontend/scale_factor_calc.hpp @@ -23,9 +23,6 @@ namespace GNAPluginNS { -template -class ScaleFactorCalculator; - namespace frontend { static const float MIN_SEARCH_WEIGHTS_VAL = 1.0f; static const float MAX_SEARCH_WEIGHTS_VAL = 1024.0f; @@ -197,7 +194,7 @@ static std::vector generateScaleFactors(float startRange, float endRange, * @param dst_quant destination quantization statistics * @return Weights reducer */ -static double calculateWeightsReducerFromDstStats(Quantization dst_quant) { +static double calculateWeightsReducerFromDstStats(QuantizationParams dst_quant) { auto maxAbsVal = std::max(std::abs(dst_quant.GetMinValues().front()), std::abs(dst_quant.GetMaxValues().front())); @@ -301,11 +298,17 @@ static bool requantizeInput(InferenceEngine::CNNLayerPtr input, float newOutputS return requantizeInput(prevLayer, newOutputScale, result, infiniteLoopCount); } +//TODO: verify if this is correct way to check FQ presence for all layer types +static bool isFakeQuantize(const InferenceEngine::CNNLayer& cnnLayer) { + auto quantParams = InferenceEngine::getInjectedData(cnnLayer); + return (quantParams && quantParams->_src_quant.IsStatsSet()); +} + /** * @brief calculates output scale factor per layer * @tparam T */ -template +template class ScaleFactorPerLayer { public: /** @@ -320,8 +323,8 @@ class ScaleFactorPerLayer { } }; -template -class ScaleFactorPerLayer { +template<> +class ScaleFactorPerLayer { private : const float activation_scale_factor = 2048.f; const float low_prec_activation_scale_factor = 4.f; @@ -604,13 +607,13 @@ class ScaleFactorPerLayer { } public : - bool operator()(InferenceEngine::CNNLayer *cnnLayer, ScaleFactorUpdateResult &result, int infiniteLoopCount) { + bool operator()(InferenceEngine::CNNLayer* cnnLayer, ScaleFactorUpdateResult& result, int infiniteLoopCount) { if ( !cnnLayer ) { IE_THROW() << "Incorrect Layer pointer \n"; } - int inputsSize = ScaleFactorCalculator::GetInputsBytesSize(); - bool fakeQuantize = ScaleFactorCalculator::IsFakeQuantize(); + int inputsSize = frontend::GetInputPrecision().size(); + bool fakeQuantize = isFakeQuantize(*cnnLayer); LayerInfo layerInfo(*cnnLayer); // TODO: current approach set input scale factor for true input layer(s) equals to provided factor, @@ -813,15 +816,15 @@ class ScaleFactorPerLayer { } }; -template -class ScaleFactorPerLayer { +template<> +class ScaleFactorPerLayer { public: - bool operator()(InferenceEngine::EltwiseLayer* eltwiseLayer, ScaleFactorUpdateResult &result, int infiniteLoopCount) { + bool operator()(InferenceEngine::EltwiseLayer* eltwiseLayer, ScaleFactorUpdateResult& result, int infiniteLoopCount) { if ( !eltwiseLayer ) { THROW_GNA_EXCEPTION << "Incorrect Eltwise Layer pointer \n"; } - int inputsSize = ScaleFactorCalculator::GetInputsBytesSize(); - bool fakeQuantize = ScaleFactorCalculator::IsFakeQuantize(); + int inputsSize = frontend::GetInputPrecision().size(); + bool fakeQuantize = isFakeQuantize(*eltwiseLayer); bool lowPrecision = (inputsSize == sizeof(int8_t)); auto in0 = InferenceEngine::CNNNetPrevLayer(eltwiseLayer, 0); @@ -878,7 +881,8 @@ class ScaleFactorPerLayer { for (size_t i = MAX_VAL_2B_FEAT; i > 0; --i) { auto scaleIn1Dst = i * scaleIn1Src; auto eltwiseWeightsScale = scaleIn1Dst / scaleIn0Dst; - if (eltwiseWeightsScale < 1.0 || eltwiseWeightsScale > std::numeric_limits::max() - 1) { + if (eltwiseWeightsScale < 1.0 || + eltwiseWeightsScale > std::numeric_limits::max() - 1) { continue; } @@ -893,14 +897,17 @@ class ScaleFactorPerLayer { } } - if (bestWeightsScale > 0.0f && !fp32eq(bestWeightsScale, quantParams1->_weights_quant.GetScale())) { + if (bestWeightsScale > 0.0f && + !fp32eq(bestWeightsScale, quantParams1->_weights_quant.GetScale())) { quantParams1->_weights_quant.SetScale(bestWeightsScale); - quantParams1->_dst_quant.SetScale(quantParams1->_weights_quant.GetScale() * quantParams1->_src_quant.GetScale()); + quantParams1->_dst_quant.SetScale(quantParams1->_weights_quant.GetScale() * + quantParams1->_src_quant.GetScale()); result = ScaleFactorUpdateResult(in1.get()); return true; } } } + quantData->_weights_quant.SetScale(weightsScale); quantData->_dst_quant.SetScale(quantParams1->_dst_quant.GetScale()); @@ -937,15 +944,15 @@ class ScaleFactorPerLayer { } }; -template -class ScaleFactorPerLayer { +template<> +class ScaleFactorPerLayer { public: - bool operator()(InferenceEngine::ConcatLayer* concatLayer, ScaleFactorUpdateResult &result, int infiniteLoopCount) { + bool operator()(InferenceEngine::ConcatLayer* concatLayer, ScaleFactorUpdateResult& result, int infiniteLoopCount) { if ( !concatLayer ) { THROW_GNA_EXCEPTION << "Incorrect Concat Layer pointer \n"; } - bool fakeQuantize = ScaleFactorCalculator::IsFakeQuantize(); + bool fakeQuantize = isFakeQuantize(*concatLayer); if (concatLayer->insData.size() < 2) { THROW_GNA_EXCEPTION << "Concat layer has unsupported number of incoming layers."; @@ -1012,7 +1019,7 @@ class ScaleFactorPerLayer { for (auto it = inputLayers.begin(); it != inputLayers.end(); ++it) { auto quantParams = InferenceEngine::getInjectedData(*it); if ((quantParams->_dst_quant.GetScale() < minScaleFactor && - !fp32eq(quantParams->_dst_quant.GetScale(), 1.0f)) || + !fp32eq(quantParams->_dst_quant.GetScale(), 1.0f)) || fp32eq(minScaleFactor, 1.0f)) { minScaleFactor = quantParams->_dst_quant.GetScale(); sourceLayerIt = it; @@ -1168,8 +1175,8 @@ class ScaleFactorPerLayer { } }; -template -class ScaleFactorPerLayer { +template<> +class ScaleFactorPerLayer { private: std::vector> thresholds = { // tuple values: scale factor threshold, scale factor reduction factor for I16 precision, for I8 precision @@ -1188,8 +1195,8 @@ class ScaleFactorPerLayer { THROW_GNA_EXCEPTION << "Incorrect weight value for " << wl->name << ":" << wl->type << "\n"; } - int inputsSize = ScaleFactorCalculator::GetInputsBytesSize(); - bool fakeQuantize = ScaleFactorCalculator::IsFakeQuantize(); + int inputsSize = frontend::GetInputPrecision().size(); + bool fakeQuantize = InferenceEngine::getInjectedData(*wl)->_weights_quant.IsStatsSet(); auto prevLayer = CNNNetPrevLayer(wl); auto quantDataForInputLayer = InferenceEngine::getInjectedData(*InferenceEngine::CNNNetPrevLayer(wl).get()); @@ -1219,7 +1226,8 @@ class ScaleFactorPerLayer { } // TODO: pass 8 bits somehow - int weightsSize = ScaleFactorCalculator::GetMandatoryWeightsBytesSize(wl); + int weightsSize = frontend::GetWeightsPrecision(LayerInfo(wl), *quant).size(); + if (!quant->_weights_quant.IsScaleSet()) { size_t scaleRange = 0; if (weightsSize == 2) { @@ -1236,23 +1244,13 @@ class ScaleFactorPerLayer { } if (wl->_biases) { - // for now the only case of INT8 bias we support comes with INT8 inputs and weights as well - if (inputsSize == 1 && weightsSize == 1) { - quant->_bias_quant.SetScale(ScaleFactorForQuantization(wl->_biases->buffer().as(), - MAX_VAL_1B_BIAS, - wl->_biases->size())); - } else { - quant->_bias_quant.SetScale(ScaleFactorForQuantization(wl->_biases->buffer().as(), - MAX_VAL_4B_BIAS, - wl->_biases->size())); - } + // for now the only case of bias we support with INT8 inputs and weights is INT32 + quant->_bias_quant.SetScale(ScaleFactorForQuantization(wl->_biases->buffer().as(), + MAX_VAL_4B_BIAS, + wl->_biases->size())); if (quant->_bias_quant.GetScale() != -1.0f) { - // for low precision we don't change bias scale factor based on source and weights scale factors - // in order not to loose too much precision - if (inputsSize != 1 || weightsSize != 1) { - quant->_bias_quant.SetScale( - std::min(quant->_weights_quant.GetScale() * quant->_src_quant.GetScale(), quant->_bias_quant.GetScale())); - } + quant->_bias_quant.SetScale( + std::min(quant->_weights_quant.GetScale() * quant->_src_quant.GetScale(), quant->_bias_quant.GetScale())); quant->_weights_quant.SetScale(quant->_bias_quant.GetScale() / quant->_src_quant.GetScale()); } } @@ -1326,18 +1324,18 @@ class ScaleFactorPerLayer { } }; -template -class ScaleFactorPerLayer : - public ScaleFactorPerLayer { +template<> +class ScaleFactorPerLayer : + public ScaleFactorPerLayer { }; -template -class ScaleFactorPerLayer : - public ScaleFactorPerLayer { +template<> +class ScaleFactorPerLayer : + public ScaleFactorPerLayer { }; -template -class ScaleFactorPerLayer { +template<> +class ScaleFactorPerLayer { public: bool operator() (InferenceEngine::GemmLayer* gemmLayer, ScaleFactorUpdateResult &result, int infiniteLoopCount) { if ( !gemmLayer ) { @@ -1392,12 +1390,11 @@ public: /** * @brief scale factor calculator will calculate only output scale factors for the layer - * if scale factor propagation not possible, it will fall indicate a restart condition + * if scale factor propagation is not possible, it will fall indicate a restart condition */ -template class ScaleFactorCalculator { using Cnt = std::vector; - Cnt net; + Cnt net; mutable Cnt::const_iterator idx; mutable bool needRestart = false; int infiniteLoopCount = 0; @@ -1422,7 +1419,7 @@ class ScaleFactorCalculator { bool operator()(T ptr) const { needRestart = false; frontend::ScaleFactorUpdateResult result; - if (!frontend::ScaleFactorPerLayer()(ptr, result, infiniteLoopCount)) { + if (!frontend::ScaleFactorPerLayer()(ptr, result, infiniteLoopCount)) { return false; } if (result) { @@ -1442,45 +1439,6 @@ class ScaleFactorCalculator { needRestart = true; return true; } - - template - static int GetMandatoryWeightsBytesSize(T ptr) { - auto info = LayerInfo(ptr); - if (info.isConvolution() || info.isScaleShift()) { - return GetOptionalWeightsBytesSize(); - } - - if (IsFakeQuantize()) { - auto quantData = InferenceEngine::getInjectedData(*ptr); - if (quantData->_weights_quant.IsStatsSet()) { - if (quantData->_weights_quant.GetLevels() <= std::numeric_limits::max()) { - return frontend::FakeQuantI8().getWeightsPrecision().size(); - } else { - return frontend::FakeQuantI16().getWeightsPrecision().size(); - } - } else { - if (!info.isSynthetic()) { - gnawarn() << "The layer (" << ptr->name << ") has not quantization statistics\n"; - } - - return GetOptionalWeightsBytesSize(); - } - } - - return QUANT_DESC::mandatory().getWeightsPrecision().size(); - } - - static int GetOptionalWeightsBytesSize() { - return QUANT_DESC::optional().getWeightsPrecision().size(); - } - - static int GetInputsBytesSize() { - return QUANT_DESC::mandatory().getInputPrecision().size(); - } - - static bool IsFakeQuantize() { - return std::is_same(); - } }; // class ScaleFactorCalculator } // namespace GNAPluginNS diff --git a/src/plugins/intel_gna/src/gna_graph_compiler.cpp b/src/plugins/intel_gna/src/gna_graph_compiler.cpp index a518bb12858..d8714f0369b 100644 --- a/src/plugins/intel_gna/src/gna_graph_compiler.cpp +++ b/src/plugins/intel_gna/src/gna_graph_compiler.cpp @@ -488,8 +488,8 @@ void GNAGraphCompiler::finalizeConvolution1DPrimitive(InferenceEngine::CNNLayerP uint32_t num_bytes_per_weight = convolution._weights->getTensorDesc().getPrecision().size(); uint32_t num_bytes_per_bias = biasPrecision.size(); - float weight_scale_factor = getScaleFactor(layer, QuantizedDataType::weights); - float output_scale_factor = getScaleFactor(layer, QuantizedDataType::output); + float weight_scale_factor = GetScaleFactor(layer, QuantizedDataType::weights); + float output_scale_factor = GetScaleFactor(layer, QuantizedDataType::output); auto& currentComponent = dnnComponents.addComponent(convolution.name, "convolution"); dnn->InitConvolutional1DComponent(currentComponent, @@ -680,8 +680,8 @@ void GNAGraphCompiler::finalizeConvolution2DPrimitive(InferenceEngine::CNNLayerP convolution._kernel_y, convolution._kernel_x, filter_n, convolution._stride_y, convolution._stride_x, convolution._dilation_y, convolution._dilation_x, inputPrec); - float weight_scale_factor = getScaleFactor(layer, QuantizedDataType::weights); - float output_scale_factor = getScaleFactor(layer, QuantizedDataType::output); + float weight_scale_factor = GetScaleFactor(layer, QuantizedDataType::weights); + float output_scale_factor = GetScaleFactor(layer, QuantizedDataType::output); auto& currentComponent = dnnComponents.addComponent(convolution.name, "convolution"); dnn->InitConvolutional2DComponent(currentComponent, @@ -849,8 +849,8 @@ void GNAGraphCompiler::PowerPrimitive(InferenceEngine::CNNLayerPtr layer) { gna_pwl_segment_t* ptr_pwl_segments_target = nullptr; - float output_pwl_scale_factor = getScaleFactor(layer, QuantizedDataType::output); - float input_pwl_scale_factor = getScaleFactor(layer, QuantizedDataType::input); + float output_pwl_scale_factor = GetScaleFactor(layer, QuantizedDataType::output); + float input_pwl_scale_factor = GetScaleFactor(layer, QuantizedDataType::input); if (!gnaFlags->sw_fp32 && gnaFlags->uniformPwlDesign) { uint32_t num_segments = POW_NUM_SEGMENTS; @@ -957,7 +957,7 @@ void GNAGraphCompiler::PoolingPrimitive(InferenceEngine::CNNLayerPtr layer) { outputs->getPrecision().size(), { pooling._kernel[X_AXIS], pooling._kernel[Y_AXIS] }, { pooling._stride[X_AXIS], pooling._stride[Y_AXIS] }, - getScaleFactor(layer, QuantizedDataType::output), + GetScaleFactor(layer, QuantizedDataType::output), ptr_inputs, ptr_outputs); size_t num_data_bytes_out = InferenceEngine::details::product(begin(outputs->getDims()), end(outputs->getDims())); @@ -1028,7 +1028,7 @@ void GNAGraphCompiler::CopyPrimitive(InferenceEngine::CNNLayerPtr layer) { num_columns_out, inputs->getPrecision().size(), outputs->getPrecision().size(), - getScaleFactor(layer, QuantizedDataType::output), + GetScaleFactor(layer, QuantizedDataType::output), num_rows_out + num_padding_out, num_columns_out, ptr_inputs, @@ -1205,8 +1205,8 @@ void GNAGraphCompiler::CropPrimitive(InferenceEngine::CNNLayerPtr layer) { outputs->getPrecision().size(), quantized == nullptr ? inputs->getPrecision().size() : (gnaFlags->input_low_precision ? 1 : 2), gnaFlags->input_low_precision ? 1 : 4, - getScaleFactor(layer, QuantizedDataType::weights), - getScaleFactor(layer, QuantizedDataType::output), + GetScaleFactor(layer, QuantizedDataType::weights), + GetScaleFactor(layer, QuantizedDataType::output), ptr_inputs, ptr_outputs, ptr_weights, @@ -1341,8 +1341,8 @@ void GNAGraphCompiler::EltwisePrimitive(InferenceEngine::CNNLayerPtr layer) { // TODO: only fp32 and Int16 tested quantized == nullptr ? inputs2Bytes->getPrecision().size() : (gnaFlags->input_low_precision ? 1 : 2), quantized == nullptr ? inputs4Bytes->getPrecision().size() : (gnaFlags->input_low_precision ? 1 : 4), - getScaleFactor(layer, QuantizedDataType::weights), - getScaleFactor(layer, QuantizedDataType::output), + GetScaleFactor(layer, QuantizedDataType::weights), + GetScaleFactor(layer, QuantizedDataType::output), ptr_inputs, ptr_outputs, ptr_weights, @@ -1448,8 +1448,8 @@ void GNAGraphCompiler::GemmPrimitive(InferenceEngine::CNNLayerPtr layer) { outputs->getPrecision().size(), quantized == nullptr ? input_2->getPrecision().size() : 2, quantized == nullptr ? input_2->getPrecision().size() : 4, - getScaleFactor(layer, QuantizedDataType::weights), - getScaleFactor(layer, QuantizedDataType::output), + GetScaleFactor(layer, QuantizedDataType::weights), + GetScaleFactor(layer, QuantizedDataType::output), ptr_input_1, ptr_outputs, ptr_input_2, @@ -1537,8 +1537,8 @@ void GNAGraphCompiler::AffinePrimitive(InferenceEngine::CNNLayerPtr layer, bool outputs->getPrecision().size(), weightable._weights->getTensorDesc().getPrecision().size(), biasPrecisionSize, - getScaleFactor(layer, QuantizedDataType::weights), - getScaleFactor(layer, QuantizedDataType::output), + GetScaleFactor(layer, QuantizedDataType::weights), + GetScaleFactor(layer, QuantizedDataType::output), ptr_inputs, ptr_outputs, ptr_weights, @@ -1723,7 +1723,7 @@ void GNAGraphCompiler::ConcatAlignFilterPrimitive(InferenceEngine::CNNLayerPtr l num_columns_in, inputs->getPrecision().size(), inputs->getPrecision().size(), - getScaleFactor(layer, QuantizedDataType::output), + GetScaleFactor(layer, QuantizedDataType::output), num_rows_copied, num_columns_in, ptr_inputs, @@ -1759,8 +1759,8 @@ void GNAGraphCompiler::ConcatAlignFilterPrimitive(InferenceEngine::CNNLayerPtr l outputs->getPrecision().size(), filterLayer->_weights->getTensorDesc().getPrecision().size(), biasPrecisionSize, - getScaleFactor(layer, QuantizedDataType::weights), - getScaleFactor(layer, QuantizedDataType::output), + GetScaleFactor(layer, QuantizedDataType::weights), + GetScaleFactor(layer, QuantizedDataType::output), ptr_inputs, ptr_outputs, ptr_weights, @@ -1864,8 +1864,8 @@ void GNAGraphCompiler::ConvolutionFilterPrimitive(InferenceEngine::CNNLayerPtr l numberOfFilters, filterWidth, convolutionStride, - getScaleFactor(layer, QuantizedDataType::weights), - getScaleFactor(layer, QuantizedDataType::output), + GetScaleFactor(layer, QuantizedDataType::weights), + GetScaleFactor(layer, QuantizedDataType::output), ptr_inputs, ptr_outputs, ptr_weights, @@ -1924,8 +1924,8 @@ void GNAGraphCompiler::PWLPrimitive(InferenceEngine::CNNLayerPtr layer) { auto inputs = layer->insData.begin()->lock(); auto outputs = *layer->outData.begin(); - float output_pwl_scale_factor = getScaleFactor(layer, QuantizedDataType::output); - float input_pwl_scale_factor = getScaleFactor(layer, QuantizedDataType::input); + float output_pwl_scale_factor = GetScaleFactor(layer, QuantizedDataType::output); + float input_pwl_scale_factor = GetScaleFactor(layer, QuantizedDataType::input); auto orientation = kDnnInterleavedOrientation; @@ -2173,7 +2173,7 @@ void GNAGraphCompiler::PermutePrimitive(InferenceEngine::CNNLayerPtr layer) { squeezedInputOrder[1], inputs->getPrecision().size(), outputs->getPrecision().size(), - getScaleFactor(layer, QuantizedDataType::output), + GetScaleFactor(layer, QuantizedDataType::output), ptr_inputs, ptr_outputs); } @@ -2188,7 +2188,7 @@ void GNAGraphCompiler::PermutePrimitive(InferenceEngine::CNNLayerPtr layer) { squeezedInputOrder[1], inputs->getPrecision().size(), outputs->getPrecision().size(), - getScaleFactor(layer, QuantizedDataType::output), + GetScaleFactor(layer, QuantizedDataType::output), ptr_inputs, ptr_outputs); } @@ -2465,7 +2465,7 @@ GNAPluginNS::ConnectionDetails GNAGraphCompiler::connectInput(CNNLayerPtr layer, if (LayerInfo(prevLayer).isInput()) { auto quantized = getInjectedData(prevLayer); if (quantized) { - if (quantized->lowPrecision) { + if (quantized->_inputs_int8_precision) { inputs_ptr_->at(prevLayer->name).set_precision(InferenceEngine::Precision::I8); } else { inputs_ptr_->at(prevLayer->name).set_precision(InferenceEngine::Precision::I16); diff --git a/src/plugins/intel_gna/src/gna_plugin.cpp b/src/plugins/intel_gna/src/gna_plugin.cpp index 1eff112f12e..6f5b562d178 100644 --- a/src/plugins/intel_gna/src/gna_plugin.cpp +++ b/src/plugins/intel_gna/src/gna_plugin.cpp @@ -863,27 +863,9 @@ void GNAPlugin::LoadNetwork(const CNNNetwork& _network) { // to run all passes need to have two calls to pass manager run_passes(newNet, true, gnaFlags->input_low_precision); run_passes(newNet, false, gnaFlags->input_low_precision); - } else if (fake_quantized) { - ModelQuantizer modelQuantizer; - newNet = modelQuantizer.quantize(network, run_passes, *inputs_ptr_); } else { - switch (config.gnaPrecision) { - case Precision::I16: - ModelQuantizer q16; - newNet = q16.quantize(network, run_passes, *inputs_ptr_); - break; - case Precision::I8: - if (gnaFlags->input_low_precision == false) { - ModelQuantizer q8; - newNet = q8.quantize(network, run_passes, *inputs_ptr_); - } else { - ModelQuantizer q8_8; - newNet = q8_8.quantize(network, run_passes, *inputs_ptr_); - } - break; - default: - THROW_GNA_EXCEPTION << "unsupported GNA precision for quantisation: " << config.gnaPrecision; - } + ModelQuantizer modelQuantizer; + newNet = modelQuantizer.quantize(network, run_passes, *inputs_ptr_, config.gnaPrecision, gnaFlags->input_low_precision); } auto inputLayers = CNNNetGetAllInputLayers(newNet); diff --git a/src/plugins/intel_gna/src/optimizer/gna_pass_manager.cpp b/src/plugins/intel_gna/src/optimizer/gna_pass_manager.cpp index 07e94a17808..e22cb073606 100644 --- a/src/plugins/intel_gna/src/optimizer/gna_pass_manager.cpp +++ b/src/plugins/intel_gna/src/optimizer/gna_pass_manager.cpp @@ -26,7 +26,7 @@ #include "backend/dnn_types.h" #include "gna_plugin_log.hpp" -#include "frontend/quantization.h" +#include "frontend/quantization.hpp" #include "frontend/quantized_layer_params.hpp" #include #include @@ -35,7 +35,6 @@ #include "gna_pass_manager.hpp" #include "layers/gna_layer_info.hpp" #include "gna_upstream_iterator.hpp" -#include "frontend/quantization.h" #include "gna_groups.hpp" #include "gna_graph_patterns.hpp" #include "gna_data_types.hpp" diff --git a/src/plugins/intel_gna/tests/unit/gna_get_scale_factor.cpp b/src/plugins/intel_gna/tests/unit/gna_get_scale_factor.cpp index c60d71fb45d..2e419cfe5eb 100644 --- a/src/plugins/intel_gna/tests/unit/gna_get_scale_factor.cpp +++ b/src/plugins/intel_gna/tests/unit/gna_get_scale_factor.cpp @@ -24,10 +24,10 @@ class GnaGetScaleFactorTest : public ::testing::Test { quant->_dst_quant.SetScale(dst_scale); quant->_weights_quant.SetScale(weights_scale); quant->_bias_quant.SetScale(bias_scale); - ASSERT_EQ(GNAPluginNS::getScaleFactor(layer, GNAPluginNS::QuantizedDataType::input), src_scale); - ASSERT_EQ(GNAPluginNS::getScaleFactor(layer, GNAPluginNS::QuantizedDataType::output), dst_scale); - ASSERT_EQ(GNAPluginNS::getScaleFactor(layer, GNAPluginNS::QuantizedDataType::weights), weights_scale); - ASSERT_EQ(GNAPluginNS::getScaleFactor(layer, GNAPluginNS::QuantizedDataType::bias), bias_scale); + ASSERT_EQ(GNAPluginNS::GetScaleFactor(layer, GNAPluginNS::QuantizedDataType::input), src_scale); + ASSERT_EQ(GNAPluginNS::GetScaleFactor(layer, GNAPluginNS::QuantizedDataType::output), dst_scale); + ASSERT_EQ(GNAPluginNS::GetScaleFactor(layer, GNAPluginNS::QuantizedDataType::weights), weights_scale); + ASSERT_EQ(GNAPluginNS::GetScaleFactor(layer, GNAPluginNS::QuantizedDataType::bias), bias_scale); } }; diff --git a/src/tests_deprecated/unit/engines/gna/I8_quantisation_test.cpp b/src/tests_deprecated/unit/engines/gna/I8_quantisation_test.cpp index 9ed4db6cf0f..b3402573079 100644 --- a/src/tests_deprecated/unit/engines/gna/I8_quantisation_test.cpp +++ b/src/tests_deprecated/unit/engines/gna/I8_quantisation_test.cpp @@ -16,7 +16,8 @@ using namespace GNATestIRs; class I8QuantisationTest : public GNATest<> { protected: - LayersQuantizer lc = LayersQuantizer (1.0f); + ModelQuantizer q; + LayersQuantizer lc = LayersQuantizer (1.0f); InferenceEngine::CNNLayerPtr quantize (InferenceEngine::CNNLayerPtr lp) { auto newLayer = InferenceEngine::injectData(lp); @@ -63,8 +64,6 @@ TEST_F(I8QuantisationTest, canQuantizeActivation){ } TEST_F(I8QuantisationTest, inputPrecisionIs16Bits){ - ModelQuantizer q; - auto weights = make_shared_blob({ Precision::U8, {440}, C }); weights->allocate(); fillWeights(weights); @@ -80,8 +79,6 @@ TEST_F(I8QuantisationTest, inputPrecisionIs16Bits){ } TEST_F(I8QuantisationTest, FCDimensionIs1){ - ModelQuantizer q; - auto weights = make_shared_blob({ Precision::U8, {440}, C }); weights->allocate(); fillWeights(weights); @@ -93,8 +90,6 @@ TEST_F(I8QuantisationTest, FCDimensionIs1){ } TEST_F(I8QuantisationTest, outputAffinePrecisionIs32Bits){ - ModelQuantizer q; - auto weights = make_shared_blob({ Precision::U8, {440}, C }); weights->allocate(); fillWeights(weights); @@ -110,8 +105,6 @@ TEST_F(I8QuantisationTest, outputAffinePrecisionIs32Bits){ } TEST_F(I8QuantisationTest, fp16tofp32_on_fullyConnected_model) { - ModelQuantizer q; - auto weights = make_shared_blob({ Precision::U8, {220}, Layout::C }); weights->allocate(); fillWeights(weights); @@ -123,8 +116,6 @@ TEST_F(I8QuantisationTest, fp16tofp32_on_fullyConnected_model) { } TEST_F(I8QuantisationTest, LSTMCell_quantize) { - ModelQuantizer q; - auto weights = make_shared_blob({ Precision::U8, {33664}, C }); weights->allocate(); fillWeights(weights); @@ -136,8 +127,6 @@ TEST_F(I8QuantisationTest, LSTMCell_quantize) { } TEST_F(I8QuantisationTest, LSTMCell_unaligned_quantize) { - ModelQuantizer q; - auto weights = make_shared_blob({ Precision::U8, {3480}, C }); weights->allocate(); fillWeights(weights); @@ -149,8 +138,6 @@ TEST_F(I8QuantisationTest, LSTMCell_unaligned_quantize) { } TEST_F(I8QuantisationTest, TI_quantize) { - ModelQuantizer q; - auto weights = make_shared_blob({ Precision::U8, {249748}, C }); weights->allocate(); fillWeights(weights); diff --git a/src/tests_deprecated/unit/engines/gna/graph_tools/graph_copy_tests.cpp b/src/tests_deprecated/unit/engines/gna/graph_tools/graph_copy_tests.cpp index bf92bd0603a..aa035816e93 100644 --- a/src/tests_deprecated/unit/engines/gna/graph_tools/graph_copy_tests.cpp +++ b/src/tests_deprecated/unit/engines/gna/graph_tools/graph_copy_tests.cpp @@ -97,12 +97,9 @@ TEST_F(GraphCopyTests, canPreserveAttributes) { #ifdef ENABLE_INTEL_GNA using namespace GNAPluginNS; -struct _FP32_2_FP32 : public GNAPluginNS::frontend::QuantDescTmpl { -}; -using FP32_2_FP32 = GNAPluginNS::frontend::QuantPair<_FP32_2_FP32 , _FP32_2_FP32 >; TEST_F(GraphCopyTests, canQuantizeTopology) { - auto clone = ModelQuantizer().quantize(CNNNetwork(mockNet), std::vector({1.0f, 1.0f})); + auto clone = ModelQuantizer().quantize(CNNNetwork(mockNet), std::vector({1.0f, 1.0f})); CNNNetBFS(CommonTestUtils::getLayerByName(clone, "1"), [&](CNNLayerPtr layer) { auto params = getInjectedData(layer); diff --git a/src/tests_deprecated/unit/engines/gna/i16_quantisation_test.cpp b/src/tests_deprecated/unit/engines/gna/i16_quantisation_test.cpp index 55e46da6c8c..89e38497dfc 100644 --- a/src/tests_deprecated/unit/engines/gna/i16_quantisation_test.cpp +++ b/src/tests_deprecated/unit/engines/gna/i16_quantisation_test.cpp @@ -18,7 +18,8 @@ using namespace GNATestIRs; class I16QuantisationTest : public GNATest<> { protected: - LayersQuantizer lc = LayersQuantizer(1.0f); + ModelQuantizer q; + LayersQuantizer lc = LayersQuantizer(1.0f); InferenceEngine::CNNLayerPtr quantize (InferenceEngine::CNNLayerPtr lp) { auto newLayer = InferenceEngine::injectData(lp); @@ -84,8 +85,6 @@ TEST_F(I16QuantisationTest, canQuantizeActivation){ } TEST_F(I16QuantisationTest, outputAffinePrecisionIs32Bits){ - ModelQuantizer q; - auto weights = make_shared_blob({ Precision::U8, {440}, C }); weights->allocate(); fillWeights(weights); @@ -102,8 +101,6 @@ TEST_F(I16QuantisationTest, outputAffinePrecisionIs32Bits){ TEST_F(I16QuantisationTest, canQuantizeLstmLikeTopology) { - ModelQuantizer q; - auto weights = setWeights(make_shared_blob({ Precision::U8, {440}, C })); //std::fill_n(weights->buffer().as(), weights->byteSize()/sizeof(float), 0); @@ -114,7 +111,6 @@ TEST_F(I16QuantisationTest, canQuantizeLstmLikeTopology) { } TEST_F(I16QuantisationTest, DISABLED_outputScaleFactorForAffineIsCorrect){ - ModelQuantizer q; const float inputScaleFactorTest = 1000; const float weightValueTest = 100; @@ -350,8 +346,6 @@ TEST_F(I16QuantisationTest, DISABLED_noPermutationOfWeightsBetweenConvAndAffineI } TEST_F(I16QuantisationTest, fp16tofp32_on_fullyConnected_model) { - ModelQuantizer q; - auto weights = make_shared_blob({ Precision::U8, {220}, Layout::C }); weights->allocate(); fillWeights(weights); @@ -405,8 +399,6 @@ TEST_F(I16QuantisationTest, ConcatWithConstInputPropagatedForward) { } TEST_F(I16QuantisationTest, LSTMCell_quantize) { - ModelQuantizer q; - auto weights = make_shared_blob({ Precision::U8, {33664}, C }); weights->allocate(); fillWeights(weights); @@ -418,8 +410,6 @@ TEST_F(I16QuantisationTest, LSTMCell_quantize) { } TEST_F(I16QuantisationTest, LSTMCell_unaligned_quantize) { - ModelQuantizer q; - auto weights = make_shared_blob({ Precision::U8, {3480}, C }); weights->allocate(); fillWeights(weights); @@ -457,8 +447,6 @@ TEST_F(I16QuantisationTest, ConcatWithDifferentInputScaleFactorsPropagateForward } TEST_F(I16QuantisationTest, TI_quantize) { - ModelQuantizer q; - auto weights = make_shared_blob({ Precision::U8, {249748}, C }); weights->allocate(); fillWeights(weights); diff --git a/src/tests_deprecated/unit/engines/gna/matchers/conv_matcher.hpp b/src/tests_deprecated/unit/engines/gna/matchers/conv_matcher.hpp index 1dc60e4e656..a6b72e31e68 100644 --- a/src/tests_deprecated/unit/engines/gna/matchers/conv_matcher.hpp +++ b/src/tests_deprecated/unit/engines/gna/matchers/conv_matcher.hpp @@ -6,7 +6,7 @@ #include "backend/gna_types.h" #include "nnet_base_matcher.hpp" -#include "frontend/quantization.h" +#include "frontend/quantization.hpp" class ConvoluionLayerMatcher : public ::testing::MatcherInterface { bool matchInserted; diff --git a/src/tests_deprecated/unit/engines/gna/matchers/diag_matcher.hpp b/src/tests_deprecated/unit/engines/gna/matchers/diag_matcher.hpp index e30cb5e12f7..3443ae5f420 100644 --- a/src/tests_deprecated/unit/engines/gna/matchers/diag_matcher.hpp +++ b/src/tests_deprecated/unit/engines/gna/matchers/diag_matcher.hpp @@ -6,7 +6,7 @@ #include "backend/gna_types.h" #include "nnet_base_matcher.hpp" -#include "frontend/quantization.h" +#include "frontend/quantization.hpp" class DiagLayerMatcher : public ::testing::MatcherInterface { bool matchInserted; diff --git a/src/tests_deprecated/unit/engines/gna/matchers/pool_matcher.hpp b/src/tests_deprecated/unit/engines/gna/matchers/pool_matcher.hpp index 5276d8f1898..d2650da0160 100644 --- a/src/tests_deprecated/unit/engines/gna/matchers/pool_matcher.hpp +++ b/src/tests_deprecated/unit/engines/gna/matchers/pool_matcher.hpp @@ -6,7 +6,7 @@ #include "backend/gna_types.h" #include "nnet_base_matcher.hpp" -#include "frontend/quantization.h" +#include "frontend/quantization.hpp" class PoolingLayerMatcher : public ::testing::MatcherInterface { bool matchInserted; diff --git a/src/tests_deprecated/unit/engines/gna/matchers/weights_matcher.hpp b/src/tests_deprecated/unit/engines/gna/matchers/weights_matcher.hpp index 040decfdd3a..2a998f58371 100644 --- a/src/tests_deprecated/unit/engines/gna/matchers/weights_matcher.hpp +++ b/src/tests_deprecated/unit/engines/gna/matchers/weights_matcher.hpp @@ -5,7 +5,7 @@ #pragma once #include "backend/gna_types.h" #include "nnet_base_matcher.hpp" -#include "frontend/quantization.h" +#include "frontend/quantization.hpp" using TranspozedData = std::tuple*, int, int>;