IntenseWebs/openvino
3
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[GNA] Replace fixed quantization code with a more flexible approach (#13237)

Browse Source
This commit is contained in:
Szymon Irzabek authored and GitHub committed 2022-10-13 10:21:59 +01:00
1 parent 3f14ec7fcb
commit 296b621577
20 files changed
+744 -1276
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/plugins/intel_gna/src/backend/gna_limitations.cpp
+1 -1
View File
@@ -107,7 +107,7 @@ std::string RectLimitByChannels::GetErrorOrEmpty(const uint32_t h, const uint32_
}
RectLimitByChannels RectLimitByChannelsAndPrecision::GetByPrecision(const OvGnaType precision) const {
return precision == OvGnaTypeInt8 ? lowPrecision : defaultPrecision;
return precision == OvGnaTypeInt8 ? limit_for_int8 : limit_for_int16;
}
bool RectLimitByChannelsAndPrecision::isValid(const uint32_t h, const uint32_t w, const OvGnaType precision, const uint32_t channels) const {
src/plugins/intel_gna/src/backend/gna_limitations.hpp
+2 -2
View File
@@ -112,8 +112,8 @@ struct RectLimitByChannels {
};
struct RectLimitByChannelsAndPrecision {
RectLimitByChannels lowPrecision;
RectLimitByChannels defaultPrecision;
RectLimitByChannels limit_for_int8;
RectLimitByChannels limit_for_int16;
RectLimitByChannels GetByPrecision(const OvGnaType precision) const;
bool isValid(const uint32_t h, const uint32_t w, const OvGnaType precision, const uint32_t channels) const;
std::string GetErrorOrEmpty(const uint32_t h, const uint32_t w,
src/plugins/intel_gna/src/frontend/layer_quantizer.hpp
+321 -571
View File
File diff suppressed because it is too large. Load diff
src/plugins/intel_gna/src/frontend/model_quantizer.hpp
+93 -93
View File
@@ -22,125 +22,125 @@
namespace GNAPluginNS {
/**
* Quantize entire cnn - network
* @tparam T - type trait for weights and biases
* Quantize entire network
*/
template<class T>
class ModelQuantizer {
public:
InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork &model, float scaleFactor) const {
return quantize(model, [](const InferenceEngine::CNNNetwork &, bool runBeforeCopy, bool lowPrecision){}, scaleFactor);
// Below four functions are used only in unit tests
InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork& model, float scale_factor) const {
return quantize(model, [](const InferenceEngine::CNNNetwork&, bool run_before_copy, bool low_precision){}, scale_factor);
}
template <class PreQuantisationCb>
InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork &model, const PreQuantisationCb &cb, float scaleFactor) const {
return quantize(model, cb, std::vector<float>({scaleFactor}));
InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork& model, const PreQuantisationCb& cb, float scale_factor) const {
return quantize(model, cb, std::vector<float>({scale_factor}));
}
InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork &model, std::vector<float> scaleFactors) const {
return quantize(model, [](InferenceEngine::CNNNetwork &, bool runBeforeCopy, bool lowPrecision){}, scaleFactors);
InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork& model, std::vector<float> scale_factors) const {
return quantize(model, [](InferenceEngine::CNNNetwork&, bool run_before_copy, bool low_precision){}, scale_factors);
}
template <class PreQuantisationCb>
InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork &model, const PreQuantisationCb &cb, std::vector<float> scaleFactors) const {
InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork& model, const PreQuantisationCb& cb, std::vector<float> scale_factors) const {
GNAPluginNS::GnaInputs inputs;
InferenceEngine::InputsDataMap inputs_map = model.getInputsInfo();
int sf_id = 0;
for (auto &&input_data : inputs_map) {
auto input_layer = getCreatorLayer(input_data.second->getInputData()).lock();
if (scaleFactors.size() <= sf_id) {
if (scale_factors.size() <= sf_id) {
THROW_GNA_EXCEPTION << "Scale factors are not set for some of the inputs";
}
inputs[input_layer->name].scale_factor = scaleFactors[sf_id++];
inputs[input_layer->name].scale_factor = scale_factors[sf_id++];
}
return quantize(model, cb, inputs);
return quantize(model, cb, inputs, InferenceEngine::Precision::I16, false);
}
// This function is acutally used by the plugin
template <class PreQuantisationCb>
InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork &model,
const PreQuantisationCb &cb,
const GNAPluginNS::GnaInputs &inputs) const {
InferenceEngine::CNNNetwork quantize(const InferenceEngine::CNNNetwork& model,
const PreQuantisationCb& cb,
const GNAPluginNS::GnaInputs& inputs,
const InferenceEngine::Precision& gna_precision,
const bool& inputs_int8_precision) const {
OV_ITT_SCOPED_TASK(itt::domains::GNA_LT, "ModelQuantizer::quantize");
auto visitor = [&](InferenceEngine::CNNLayerPtr lp) {
auto newLayer = InferenceEngine::injectData<QuantizedLayerParams>(lp);
transformLayer(newLayer, WeightsConverter());
return newLayer;
auto new_layer = InferenceEngine::injectData<QuantizedLayerParams>(lp);
transformLayer(new_layer, WeightsConverter());
return new_layer;
};
bool lowPrecision = (T::mandatory().getInputPrecision().size() == sizeof(uint8_t));
InferenceEngine::CNNNetwork copiedNet = InferenceEngine::CNNNetCopy(model);
cb(copiedNet, true, lowPrecision);
copiedNet = InferenceEngine::CNNNetCopy(copiedNet, visitor);
// User's hints for GNA weights precision
const bool weights_int8_precision = (gna_precision == InferenceEngine::Precision::I8) ? true : false;
// allow client code to access copied topology, to avoid copies if user would like to chain quantisation with
InferenceEngine::CNNNetwork copied_net = InferenceEngine::CNNNetCopy(model);
cb(copied_net, true, inputs_int8_precision);
copied_net = InferenceEngine::CNNNetCopy(copied_net, visitor);
// Allow client code to access copied topology, to avoid copies if user would like to chain quantisation with
// another preprocessing
cb(copiedNet, false, lowPrecision);
cb(copied_net, false, inputs_int8_precision);
float lc_sf = GNAPluginNS::kScaleFactorDefault;
if (inputs.empty()) {
gnawarn() << "Inputs structure is empty, will be used default scale factor: " << lc_sf << std::endl;
} else {
lc_sf = inputs.Get().begin()->scale_factor;
}
LayersQuantizer<T> lc(lc_sf);
auto sortedNewNet = InferenceEngine::details::CNNNetSortTopologically(copiedNet);
LayersQuantizer lc(GNAPluginNS::kScaleFactorDefault);
auto sorted_new_net = InferenceEngine::details::CNNNetSortTopologically(copied_net);
gnalog() << "Sorted layers: " << std::endl;
for (auto &&layer : sortedNewNet) {
auto quantData = InferenceEngine::getInjectedData<QuantizedLayerParams>(layer);
quantData->lowPrecision = lowPrecision;
for (auto &&layer : sorted_new_net) {
auto quant_data = InferenceEngine::getInjectedData<QuantizedLayerParams>(layer);
quant_data->_inputs_int8_precision = inputs_int8_precision;
quant_data->_weights_int8_precision = weights_int8_precision;
gnalog() << layer->name << std::endl;
}
/// filling scale factors for input layers, memory layers will have scaleFactor of 1.0 by default
InferenceEngine::InputsDataMap dm = copiedNet.getInputsInfo();
// Filling scale factors for input layers, memory layers will have scale factor of 1.0 by default
InferenceEngine::InputsDataMap dm = copied_net.getInputsInfo();
for (auto &&inputData : dm) {
auto inputLayer = getCreatorLayer(inputData.second->getInputData()).lock();
auto quantData = InferenceEngine::getInjectedData<QuantizedLayerParams>(inputLayer);
IE_ASSERT(quantData != nullptr);
quantData->_src_quant.SetScale(inputs.at(inputLayer->name).scale_factor);
auto input_layer = getCreatorLayer(inputData.second->getInputData()).lock();
auto quant_data = InferenceEngine::getInjectedData<QuantizedLayerParams>(input_layer);
IE_ASSERT(quant_data != nullptr);
quant_data->_src_quant.SetScale(inputs.at(input_layer->name).scale_factor);
}
propagateScaleFactor(sortedNewNet);
propagateScaleFactor(sorted_new_net);
// sorted order gives possibility for propagate quantisation along depended layers
for (auto &&layer : sortedNewNet) {
// Sorted order gives possibility for propagate quantisation along depended layers
for (auto &&layer : sorted_new_net) {
transformLayer(layer, lc);
}
return copiedNet;
return copied_net;
}
private :
void propagateScaleFactor(std::vector<InferenceEngine::CNNLayerPtr> & net) const {
ScaleFactorCalculator<T> sf(net);
void propagateScaleFactor(std::vector<InferenceEngine::CNNLayerPtr>& net) const {
ScaleFactorCalculator sf(net);
int infiniteLoopCount = 0;
std::vector<std::string> infiniteLoopPattern;
std::vector<std::string> infiniteLoopHistory;
while (!sf.allLayersProcessed() && infiniteLoopCount <= 2) {
uint32_t inf_loop_count = 0;
std::vector<std::string> inf_loop_pattern;
std::vector<std::string> inf_loop_history;
while (!sf.allLayersProcessed() && inf_loop_count <= 2) {
auto layers = sf.getStartLayers();
infiniteLoopHistory.emplace_back(layers.front()->name);
inf_loop_history.emplace_back(layers.front()->name);
for (auto &&layer : layers) {
transformLayer(layer, sf);
// transforming until we reached cases where output scale updated due to situation in downstream layer
if (sf.needToRestart()) {
infiniteLoopHistory.back() += "#" + layer->name;
inf_loop_history.back() += "#" + layer->name;
break;
}
}
// We are looking for infinite loop by using algorithm of compute prefix function, complexity O(N)
// (a part of the Knuth–Morris–Pratt algorithm).
std::map<int, int> prefixFunction;
int k = infiniteLoopHistory.size();
for (int i = infiniteLoopHistory.size() - 2; i >= 0; i--) {
while (k < infiniteLoopHistory.size() && infiniteLoopHistory[k - 1] != infiniteLoopHistory[i]) {
auto iter = prefixFunction.find(k);
k = iter == prefixFunction.end() ? infiniteLoopHistory.size() : iter->second;
std::map<int, int> prefix_function;
int32_t k = inf_loop_history.size();
for (int32_t i = inf_loop_history.size() - 2; i >= 0; i--) {
while (k < inf_loop_history.size() && inf_loop_history[k - 1] != inf_loop_history[i]) {
auto iter = prefix_function.find(k);
k = iter == prefix_function.end() ? inf_loop_history.size() : iter->second;
}
if (infiniteLoopHistory[k - 1] == infiniteLoopHistory[i]) {
if (inf_loop_history[k - 1] == inf_loop_history[i]) {
k--;
}
@@ -154,56 +154,56 @@ class ModelQuantizer {
//
// In the case of pattern length is 1, an infinite loop can be found on 2 consecutive strings.
// To avoid this, we will expect the appearance of 4 equal strings for the case pattern length is 1.
if ((infiniteLoopHistory.size() - i) % 2 == 0 &&
(infiniteLoopHistory.size() - i) / 2 == infiniteLoopHistory.size() - k &&
((infiniteLoopHistory.size() - i) / 2 > 1 ||
std::distance(infiniteLoopHistory.rbegin(),
std::find_if_not(infiniteLoopHistory.rbegin(), infiniteLoopHistory.rend(),
[&infiniteLoopHistory](const std::string& str) { return str == infiniteLoopHistory.back(); })) > 3)) {
gnalog() << "infiniteLoopPattern:\n";
for (const auto& s : infiniteLoopPattern) {
if ((inf_loop_history.size() - i) % 2 == 0 &&
(inf_loop_history.size() - i) / 2 == inf_loop_history.size() - k &&
((inf_loop_history.size() - i) / 2 > 1 ||
std::distance(inf_loop_history.rbegin(),
std::find_if_not(inf_loop_history.rbegin(), inf_loop_history.rend(),
[&inf_loop_history](const std::string& str) { return str == inf_loop_history.back(); })) > 3)) {
gnalog() << "inf_loop_pattern:\n";
for (const auto& s : inf_loop_pattern) {
gnalog() << "\t " << s << '\n';
}
infiniteLoopPattern.clear();
int patternLength = (infiniteLoopHistory.size() - i) / 2;
gnalog() << "patternLength: " << patternLength << '\n';
for (int j = 0; j < patternLength; j++) {
infiniteLoopPattern.emplace_back(infiniteLoopHistory[infiniteLoopHistory.size() - patternLength + j]);
inf_loop_pattern.clear();
int pattern_len = (inf_loop_history.size() - i) / 2;
gnalog() << "pattern_len: " << pattern_len << '\n';
for (int j = 0; j < pattern_len; j++) {
inf_loop_pattern.emplace_back(inf_loop_history[inf_loop_history.size() - pattern_len + j]);
}
gnalog() << "infiniteLoopHistory:\n";
for (const auto& s : infiniteLoopHistory) {
gnalog() << "inf_loop_history:\n";
for (const auto& s : inf_loop_history) {
gnalog() << "\t " << s << '\n';
}
infiniteLoopHistory.clear();
inf_loop_history.clear();
gnalog() << "infinite loop detected\n";
break;
}
prefixFunction.emplace(i, k);
prefix_function.emplace(i, k);
}
if (infiniteLoopHistory.empty()) {
infiniteLoopCount++;
if (inf_loop_history.empty()) {
inf_loop_count++;
} else {
if (infiniteLoopCount > 0 &&
(infiniteLoopHistory.size()%infiniteLoopPattern.size() == 0 || sf.allLayersProcessed()) &&
!std::equal(infiniteLoopHistory.begin() + (infiniteLoopHistory.size() - infiniteLoopPattern.size()),
infiniteLoopHistory.end(), infiniteLoopPattern.begin())) {
infiniteLoopCount = 0;
infiniteLoopPattern.clear();
if (inf_loop_count > 0 &&
(inf_loop_history.size()%inf_loop_pattern.size() == 0 || sf.allLayersProcessed()) &&
!std::equal(inf_loop_history.begin() + (inf_loop_history.size() - inf_loop_pattern.size()),
inf_loop_history.end(), inf_loop_pattern.begin())) {
inf_loop_count = 0;
inf_loop_pattern.clear();
gnalog() << "infinite loop fixed\n";
}
}
sf.SetInfiniteLoopCount(infiniteLoopCount);
sf.SetInfiniteLoopCount(inf_loop_count);
}
if (infiniteLoopCount > 0) {
std::string additionalInformation;
for (const auto& p : infiniteLoopPattern) {
additionalInformation += '\n' + p;
if (inf_loop_count > 0) {
std::string additional_info;
for (const auto& p : inf_loop_pattern) {
additional_info += '\n' + p;
}
THROW_GNA_EXCEPTION << "infinite loop: " + additionalInformation;
THROW_GNA_EXCEPTION << "infinite loop: " + additional_info;
}
}
};
src/plugins/intel_gna/src/frontend/quantization.cpp
+186 -364
View File
@@ -6,161 +6,9 @@
#include <gna_plugin_log.hpp>
#include <limits>
#include "backend/gna_types.h"
#include "quantization.h"
#include "quantization.hpp"
#include <algorithm>
#ifdef DEBUG
#define QUANTWARNING(...) (fprintf(stderr, __VA_ARGS__))
#else
#define QUANTWARNING(...)
#endif
template<>
void QuantizationCallback<int16_t, int32_t>::runFakeQuantize() const {
if (quantizedWeights) {
THROW_GNA_EXCEPTION << "Quantized weights are not yet supported in int16 quantization mode";
}
uint32_t num_saturate = 0;
auto input_low = 0.0f;
auto input_high = 0.0f;
auto output_low = 0.0f;
auto output_high = 0.0f;
size_t levels = 1;
if (fq_num_stats > 0) {
input_low = *fq_ptr_input_low;
input_high = *fq_ptr_input_high;
output_low = *fq_ptr_output_low;
output_high = *fq_ptr_output_high;
levels = fq_levels;
}
for (uint32_t row = 0; row < num_rows; row++) {
for (uint32_t col = 0; col < num_columns; col++) {
float rounding_value = (ptr_float_weights[row * num_columns + col] > 0) ? 0.5f : -0.5f;
float value = ptr_float_weights[row * num_columns + col];
if (fq_num_stats > 0) {
auto x = value;
if (x <= std::min(input_low, input_high)) {
value = output_low;
} else if (x > std::max(input_low, input_high)) {
value = output_high;
} else {
value = nearbyint((x - input_low) / (input_high - input_low) * (levels - 1)) /
(levels - 1) * (output_high - output_low) + output_low;
}
}
value = value * *ptr_weight_scale_factor + rounding_value;
int16_t* ptr_weight_16 = ptr_int_weights + (row * num_columns_padded + col);
if (value > std::numeric_limits<int16_t>::max()) {
*ptr_weight_16 = std::numeric_limits<int16_t>::max();
num_saturate++;
} else if (value < std::numeric_limits<int16_t>::min()) {
*ptr_weight_16 = std::numeric_limits<int16_t>::min();
num_saturate++;
} else {
*ptr_weight_16 = (int16_t)value;
}
}
for (uint32_t col = num_columns; col < num_columns_padded; col++) {
int16_t* ptr_weight_16 = ptr_int_weights + (row * num_columns_padded + col);
*ptr_weight_16 = 0;
}
}
for (uint32_t row = num_rows; row < num_rows_padded; row++) {
for (uint32_t col = 0; col < num_columns_padded; col++) {
int16_t* ptr_weight_16 = ptr_int_weights + (row * num_columns_padded + col);
*ptr_weight_16 = 0;
}
}
// case for element wise layer
if (ptr_float_biases != nullptr && ptr_int_biases != nullptr) {
for (uint32_t j = 0; j < num_rows; j++) {
float rounding_value = (ptr_float_biases[j] > 0) ? 0.5f : -0.5f;
float value = ptr_float_biases[j] * *ptr_output_scale_factor + rounding_value;
if (value > 2147483647.0) {
ptr_int_biases[j] = 2147483647L;
num_saturate++;
} else if (value < -2147483648.0) {
ptr_int_biases[j] = -2147483648LL;
num_saturate++;
} else {
ptr_int_biases[j] = (int32_t)value;
}
}
for (uint32_t j = num_rows; j < num_rows_padded; j++) {
ptr_int_biases[j] = 0;
}
}
if (num_saturate > 0) {
QUANTWARNING("Warning: %d / %d saturations in QuantizeAffine16()\n",
num_saturate,
num_rows * num_columns + num_rows);
}
}
template<>
void QuantizationCallback<int16_t, int32_t>::runQuantize() const {
uint32_t num_saturate = 0;
for (uint32_t row = 0; row < num_rows; row++) {
for (uint32_t col = 0; col < num_columns; col++) {
float rounding_value = (ptr_float_weights[row * num_columns + col] > 0) ? 0.5f : -0.5f;
float value = ptr_float_weights[row * num_columns + col] * *ptr_weight_scale_factor + rounding_value;
int16_t *ptr_weight_16 = ptr_int_weights + (row * num_columns_padded + col);
if (value > 32767.0) {
*ptr_weight_16 = 32767;
num_saturate++;
} else if (value < -32768.0) {
*ptr_weight_16 = -32768;
num_saturate++;
} else {
*ptr_weight_16 = (int16_t) value;
}
}
for (uint32_t col = num_columns; col < num_columns_padded; col++) {
int16_t *ptr_weight_16 = ptr_int_weights + (row * num_columns_padded + col);
*ptr_weight_16 = 0;
}
}
for (uint32_t row = num_rows; row < num_rows_padded; row++) {
for (uint32_t col = 0; col < num_columns_padded; col++) {
int16_t *ptr_weight_16 = ptr_int_weights + (row * num_columns_padded + col);
*ptr_weight_16 = 0;
}
}
// case for element wise layer
if (ptr_float_biases != nullptr && ptr_int_biases != nullptr) {
for (uint32_t j = 0; j < num_rows; j++) {
float rounding_value = (ptr_float_biases[j] > 0) ? 0.5f : -0.5f;
float value = ptr_float_biases[j] * *ptr_output_scale_factor + rounding_value;
if (value > 2147483647.0) {
ptr_int_biases[j] = 2147483647L;
num_saturate++;
} else if (value < -2147483648.0) {
ptr_int_biases[j] = -2147483648LL;
num_saturate++;
} else {
ptr_int_biases[j] = (int32_t) value;
}
}
for (uint32_t j = num_rows; j < num_rows_padded; j++) {
ptr_int_biases[j] = 0;
}
}
if (num_saturate > 0) {
QUANTWARNING("Warning: %d / %d saturations in QuantizeAffine16()\n",
num_saturate,
num_rows * num_columns + num_rows);
}
}
std::pair<float, float> FindMinMaxValues(void* ptr_float_memory, size_t num_elements) {
float* ptr_float_feat = reinterpret_cast<float*>(ptr_float_memory);
@@ -180,9 +28,9 @@ std::pair<float, float> FindMinMaxValues(void* ptr_float_memory, size_t num_elem
return { min, max };
}
float ScaleFactorForQuantization(void *ptr_float_memory, float target_max, size_t num_elements) {
float ScaleFactorForQuantization(void* ptr_float_memory, float target_max, size_t num_elements) {
float *ptr_float_feat = reinterpret_cast<float *>(ptr_float_memory);
float max = 0.0;
float max = 0.0f;
float scale_factor;
for (size_t i = 0; i < num_elements; i++) {
@@ -200,262 +48,236 @@ float ScaleFactorForQuantization(void *ptr_float_memory, float target_max, size_
return (scale_factor);
}
void QuantizeVector16(float *ptr_float_memory, int16_t *ptr_int_memory, uint32_t num_elements, float scale_factor) {
float *ptr_float_feat = reinterpret_cast<float *>(ptr_float_memory);
uint32_t num_saturate = 0;
template <typename T>
void QuantizeWeights(const QuantizationData& data, float* ptr_float_weights, T* ptr_int_weights,
gna_compound_bias_t* ptr_int_biases, const bool quantized_weights) {}
int16_t *ptr_int_feat = reinterpret_cast<int16_t *>(ptr_int_memory);
for (uint32_t i = 0; i < num_elements; i++) {
float rounding_value = (ptr_float_feat[i] > 0) ? 0.5f : -0.5f;
float value = ptr_float_feat[i] * scale_factor + rounding_value;
if (value > 32767.0) {
ptr_int_feat[i] = 32767;
num_saturate++;
} else if (value < -32768.0) {
ptr_int_feat[i] = -32768;
num_saturate++;
} else {
ptr_int_feat[i] = (int16_t) value;
}
template <typename T>
void QuantizeBiases(const QuantizationData& data, float* ptr_float_biases, T* ptr_int_biases) {}
template <>
void QuantizeWeights<int8_t>(const QuantizationData& data, float* ptr_float_weights, int8_t* ptr_int_weights,
gna_compound_bias_t* ptr_int_biases, const bool quantized_weights) {
if (!ptr_int_biases && quantized_weights) {
THROW_GNA_EXCEPTION << "Quantized weights are not yet supported in int8 quantization mode";
}
if (num_saturate > 0) {
QUANTWARNING("Warning: %d / %d saturations during QuantizeVector16()\n", num_saturate, num_elements);
}
}
template<>
void QuantizationCallback<int8_t, gna_compound_bias_t>::runFakeQuantize() const {
uint32_t num_saturate = 0;
auto input_low = 0.0f;
auto input_high = 0.0f;
auto output_low = 0.0f;
auto output_high = 0.0f;
size_t levels = 1;
float valueAcc = 0.0f;
for (uint32_t i = 0; i < num_rows; i++) {
const auto min_values_size = data.weights_quant_params.GetMinValues().size();
if (min_values_size > 0) {
input_low = data.weights_quant_params.GetMinValues(true).front();
input_high = data.weights_quant_params.GetMaxValues(true).front();
output_low = data.weights_quant_params.GetMinValues(false).front();
output_high = data.weights_quant_params.GetMaxValues(false).front();
levels = data.weights_quant_params.GetLevels();
}
for (size_t row = 0; row < data.num_rows; row++) {
uint32_t channel_multiplier = 1;
if (fq_num_stats > 0) {
auto idx = fq_num_stats == 1 ? 0 : i;
input_low = fq_ptr_input_low[idx];
input_high = fq_ptr_input_high[idx];
output_low = fq_ptr_output_low[idx];
output_high = fq_ptr_output_high[idx];
levels = fq_levels;
channel_multiplier = static_cast<uint32_t>(((input_high - input_low) * *ptr_weight_scale_factor) / (levels - 1));
} else {
float scaled_row_max = 0;
for (uint32_t col = 0; col < num_columns; col++) {
float value = ptr_float_weights[i * num_columns + col] * *ptr_weight_scale_factor;
valueAcc += value;
if (fabs(value) > scaled_row_max) {
scaled_row_max = fabs(value);
}
}
channel_multiplier = static_cast<uint32_t>(scaled_row_max / static_cast<float>(MAX_VAL_1B_WEIGHT));
}
// channel multiplier shouldn't be 0
channel_multiplier = channel_multiplier == 0 ? 1 : channel_multiplier;
ptr_int_biases[i].multiplier = static_cast<uint8_t> (channel_multiplier + 0.5f);
if (channel_multiplier > MAX_OUT_MULTIPLIER) {
THROW_GNA_EXCEPTION << "invalid channel multiplier: " << channel_multiplier;
}
for (uint32_t j = 0; j < num_columns; j++) {
auto offset = i * num_columns + j;
auto rounding_value = (ptr_float_weights[i * num_columns + j] > 0) ? 0.5f : -0.5f;
float value = ptr_float_weights[offset];
if (!quantizedWeights) {
if (fq_num_stats > 0) {
auto x = value;
if (x <= std::min(input_low, input_high)) {
value = output_low;
} else if (x > std::max(input_low, input_high)) {
value = output_high;
} else {
value = nearbyint((x - input_low) / (input_high - input_low) * (levels - 1)) /
(levels - 1) * (output_high - output_low) + output_low;
if (ptr_int_biases) {
if (min_values_size > 0) {
auto idx = min_values_size == 1 ? 0 : row;
input_low = data.weights_quant_params.GetMinValues(true).at(idx);
input_high = data.weights_quant_params.GetMaxValues(true).at(idx);
output_low = data.weights_quant_params.GetMinValues(false).at(idx);
output_high = data.weights_quant_params.GetMaxValues(false).at(idx);
levels = data.weights_quant_params.GetLevels();
channel_multiplier = ((input_high - input_low) * data.scale_factor) / (levels - 1);
} else {
float scaled_row_max = 0;
for (size_t col = 0; col < data.num_columns; col++) {
float value = ptr_float_weights[row * data.num_columns + col] * data.scale_factor;
valueAcc += value;
if (fabs(value) > scaled_row_max) {
scaled_row_max = fabs(value);
}
}
value = value * (*ptr_weight_scale_factor / ptr_int_biases[i].multiplier) + rounding_value;
} else {
value -= MAX_VAL_1B_WEIGHT;
channel_multiplier = (scaled_row_max / static_cast<float>(MAX_VAL_1B_WEIGHT) + 0.5f);
}
auto normalizedWeight = static_cast<int32_t>(value);
if (channel_multiplier > MAX_OUT_MULTIPLIER) {
THROW_GNA_EXCEPTION << "invalid channel multiplier: " << channel_multiplier;
}
// channel multiplier shouldn't be 0
ptr_int_biases[row].multiplier = (channel_multiplier == 0) ? 1 : channel_multiplier;
}
for (uint32_t col = 0; col < data.num_columns; col++) {
auto offset = row * data.num_columns + col;
float rounding_value = (ptr_float_weights[offset] > 0) ? 0.5f : -0.5f;
float value = ptr_float_weights[offset];
if (min_values_size > 0) {
auto x = value;
if (x <= std::min(input_low, input_high)) {
value = output_low;
} else if (x > std::max(input_low, input_high)) {
value = output_high;
} else {
value = nearbyint((x - input_low) / (input_high - input_low) * (levels - 1)) / (levels - 1) *
(output_high - output_low) + output_low;
}
}
if (ptr_int_biases) {
if (quantized_weights) {
value -= MAX_VAL_1B_WEIGHT;
} else {
value = value * (data.scale_factor / channel_multiplier) + rounding_value;
}
} else {
value = value * data.scale_factor + rounding_value;
}
int8_t* ptr_weight_8 = ptr_int_weights + offset;
if (value > std::numeric_limits<int8_t>::max()) {
normalizedWeight = std::numeric_limits<int8_t>::max();
*ptr_weight_8 = std::numeric_limits<int8_t>::max();
num_saturate++;
} else if (value < std::numeric_limits<int8_t>::min()) {
normalizedWeight = std::numeric_limits<int8_t>::min();
*ptr_weight_8 = std::numeric_limits<int8_t>::min();
num_saturate++;
} else {
normalizedWeight = (int8_t)value;
}
// range checking
ptr_int_weights[offset] = static_cast<int8_t>(normalizedWeight);
}
for (uint32_t j = num_columns; j < num_columns_padded; j++) {
ptr_int_weights[i * num_columns + j] = 0;
}
}
for (uint32_t i = num_rows; i < num_rows_padded; i++) {
for (uint32_t j = 0; j < num_columns_padded; j++) {
ptr_int_weights[i * num_columns + j] = 0;
}
ptr_int_biases[i].multiplier = 0;
}
if (ptr_float_biases != nullptr) {
for (uint32_t j = 0; j < num_rows; j++) {
float rounding_value = (ptr_float_biases[j] > 0) ? 0.5f : -0.5f;
float value = ptr_float_biases[j] * *ptr_output_scale_factor + rounding_value;
if (value > 2147483647.0) {
ptr_int_biases[j].bias = 2147483647L;
num_saturate++;
} else if (value < -2147483648.0) {
ptr_int_biases[j].bias = -2147483648LL;
num_saturate++;
} else {
ptr_int_biases[j].bias = (int32_t) value;
}
}
}
if (num_saturate > 0) {
QUANTWARNING("Warning: %d / %d saturations in QuantizeAffine8()\n", num_saturate, num_rows * num_columns + num_rows);
}
}
template<>
void QuantizationCallback<int8_t, gna_compound_bias_t>::runQuantize() const {
if (ptr_int_biases == nullptr) {
IE_THROW() << "Int biases are empty";
}
uint32_t num_saturate = 0;
float valueAcc = 0.0;
for (uint32_t row = 0; row < num_rows; row++) {
float scaled_row_max = 0;
float rounding_value, value;
for (uint32_t col = 0; col < num_columns; col++) {
value = ptr_float_weights[row*num_columns + col] * *ptr_weight_scale_factor;
valueAcc += value;
if (fabs(value) > scaled_row_max) {
scaled_row_max = fabs(value);
}
}
value = scaled_row_max / static_cast<float>(MAX_VAL_1B_WEIGHT);
ptr_int_biases[row].multiplier = (uint8_t) (value + 0.5);
for (uint32_t col = 0; col < num_columns; col++) {
int8_t *ptr_weight_8 = ptr_int_weights + (row * num_columns_padded + col);
rounding_value = (ptr_float_weights[row * num_columns + col] > 0) ? 0.5f : -0.5f;
value = ptr_float_weights[row * num_columns + col] * (*ptr_weight_scale_factor / ptr_int_biases[row].multiplier) + rounding_value;
if (value > 127.0) {
*ptr_weight_8 = 127;
num_saturate++;
} else if (value < -128.0) {
*ptr_weight_8 = -128;
num_saturate++;
} else {
*ptr_weight_8 = (int8_t) value;
}
}
for (uint32_t col = num_columns; col < num_columns_padded; col++) {
int8_t *ptr_weight_8 = ptr_int_weights + (row * num_columns_padded + col);
*ptr_weight_8 = 0;
}
}
for (uint32_t row = num_rows; row < num_rows_padded; row++) {
for (uint32_t col = 0; col < num_columns_padded; col++) {
int8_t *ptr_weight_8 = ptr_int_weights + (row*num_columns_padded + col);
*ptr_weight_8 = 0;
}
ptr_int_biases[row].multiplier = 0;
}
// bias value of the bas will be only used when input bias provided
if (ptr_float_biases != nullptr) {
for (uint32_t j = 0; j < num_rows; j++) {
float rounding_value = (ptr_float_biases[j] > 0) ? 0.5f : -0.5f;
float value = ptr_float_biases[j] * *ptr_output_scale_factor + rounding_value;
if (value > 2147483647.0) {
ptr_int_biases[j].bias = 2147483647L;
num_saturate++;
} else if (value < -2147483648.0) {
ptr_int_biases[j].bias = -2147483648LL;
num_saturate++;
} else {
ptr_int_biases[j].bias = (int32_t) value;
*ptr_weight_8 = static_cast<int8_t>(value);
}
}
}
if (num_saturate > 0) {
QUANTWARNING("Warning: %d / %d saturations in QuantizeAffine8()\n", num_saturate, num_rows * num_columns + num_rows);
gnalog() << "Warning: " << num_saturate << " / " << (data.num_rows * data.num_columns)
<< " saturations in int8 weights quantization." << std::endl;
}
}
template<>
void QuantizationCallback<int8_t, int8_t>::runQuantize() const {
void QuantizeWeights<int16_t>(const QuantizationData& data, float* ptr_float_weights, int16_t* ptr_int_weights,
gna_compound_bias_t* ptr_int_biases, const bool quantized_weights) {
if (quantized_weights) {
THROW_GNA_EXCEPTION << "Quantized weights are not yet supported in int16 quantization mode";
}
uint32_t num_saturate = 0;
for (uint32_t row = 0; row < num_rows; row++) {
for (uint32_t col = 0; col < num_columns; col++) {
float rounding_value = (ptr_float_weights[row * num_columns + col] > 0) ? 0.5f : -0.5f;
float value = ptr_float_weights[row * num_columns + col] * *ptr_weight_scale_factor + rounding_value;
int8_t* ptr_weight_8 = ptr_int_weights + (row * num_columns_padded + col);
if (value > 127.0) {
*ptr_weight_8 = 127;
auto input_low = 0.0f;
auto input_high = 0.0f;
auto output_low = 0.0f;
auto output_high = 0.0f;
size_t levels = 1;
const auto min_values_size = data.weights_quant_params.GetMinValues().size();
if (min_values_size > 0) {
input_low = data.weights_quant_params.GetMinValues(true).front();
input_high = data.weights_quant_params.GetMaxValues(true).front();
output_low = data.weights_quant_params.GetMinValues(false).front();
output_high = data.weights_quant_params.GetMaxValues(false).front();
levels = data.weights_quant_params.GetLevels();
}
for (size_t row = 0; row < data.num_rows; row++) {
for (size_t col = 0; col < data.num_columns; col++) {
float rounding_value = (ptr_float_weights[row * data.num_columns + col] > 0) ? 0.5f : -0.5f;
float value = ptr_float_weights[row * data.num_columns + col];
if (min_values_size > 0) {
auto x = value;
if (x <= std::min(input_low, input_high)) {
value = output_low;
} else if (x > std::max(input_low, input_high)) {
value = output_high;
} else {
value = nearbyint((x - input_low) / (input_high - input_low) * (levels - 1)) /
(levels - 1) * (output_high - output_low) + output_low;
}
}
value = value * data.scale_factor + rounding_value;
int16_t* ptr_weight_16 = ptr_int_weights + (row * data.num_columns + col);
if (value > std::numeric_limits<int16_t>::max()) {
*ptr_weight_16 = std::numeric_limits<int16_t>::max();
num_saturate++;
} else if (value < -128.0) {
*ptr_weight_8 = -128;
} else if (value < std::numeric_limits<int16_t>::min()) {
*ptr_weight_16 = std::numeric_limits<int16_t>::min();
num_saturate++;
} else {
*ptr_weight_8 = (int8_t)value;
*ptr_weight_16 = (int16_t)value;
}
}
for (uint32_t col = num_columns; col < num_columns_padded; col++) {
int8_t* ptr_weight_8 = ptr_int_weights + (row * num_columns_padded + col);
*ptr_weight_8 = 0;
}
}
for (uint32_t row = num_rows; row < num_rows_padded; row++) {
for (uint32_t col = 0; col < num_columns_padded; col++) {
int8_t* ptr_weight_8 = ptr_int_weights + (row * num_columns_padded + col);
*ptr_weight_8 = 0;
}
}
if (num_saturate > 0) {
gnalog() << "Warning: " << num_saturate << " / " << (data.num_rows * data.num_columns)
<< " saturations in int16 weights quantization." << std::endl;
}
}
template<>
void QuantizeBiases<int8_t>(const QuantizationData& data, float* ptr_float_biases, int8_t* ptr_int_biases) {
// Stub
}
template<>
void QuantizeBiases<int16_t>(const QuantizationData& data, float* ptr_float_biases, int16_t* ptr_int_biases) {
// Stub
}
template<>
void QuantizeBiases<int32_t>(const QuantizationData& data, float* ptr_float_biases, int32_t* ptr_int_biases) {
uint32_t num_saturate = 0;
// case for element wise layer
if (ptr_float_biases != nullptr && ptr_int_biases != nullptr) {
for (uint32_t j = 0; j < num_rows; j++) {
float rounding_value = (ptr_float_biases[j] > 0) ? 0.5f : -0.5f;
float value = ptr_float_biases[j] * *ptr_output_scale_factor + rounding_value;
if (value > 127.0) {
ptr_int_biases[j] = 127;
for (size_t row = 0; row < data.num_rows; row++) {
float rounding_value = (ptr_float_biases[row] > 0) ? 0.5f : -0.5f;
float value = ptr_float_biases[row] * data.scale_factor + rounding_value;
if (value > std::numeric_limits<int32_t>::max()) {
ptr_int_biases[row] = std::numeric_limits<int32_t>::max();
num_saturate++;
} else if (value < -128.0) {
ptr_int_biases[j] = -128;
} else if (value < std::numeric_limits<int32_t>::min()) {
ptr_int_biases[row] = std::numeric_limits<int32_t>::min();
num_saturate++;
} else {
ptr_int_biases[j] = (int8_t)value;
ptr_int_biases[row] = static_cast<int32_t>(value);
}
}
for (uint32_t j = num_rows; j < num_rows_padded; j++) {
ptr_int_biases[j] = 0;
}
}
if (num_saturate > 0) {
QUANTWARNING("Warning: %d / %d saturations in QuantizeAffine8_8()\n", num_saturate, num_rows * num_columns + num_rows);
gnalog() << "Warning: " << num_saturate << " / " << data.num_rows
<< " saturations in int32 biases quantization." << std::endl;
}
}
template<>
void QuantizeBiases<gna_compound_bias_t>(const QuantizationData& data, float* ptr_float_biases, gna_compound_bias_t* ptr_int_biases) {
uint32_t num_saturate = 0;
if (ptr_float_biases != nullptr) {
for (size_t row = 0; row < data.num_rows; row++) {
float rounding_value = (ptr_float_biases[row] > 0) ? 0.5f : -0.5f;
float value = ptr_float_biases[row] * data.scale_factor + rounding_value;
if (value > std::numeric_limits<int32_t>::max()) {
ptr_int_biases[row].bias = std::numeric_limits<int32_t>::max();
num_saturate++;
} else if (value < std::numeric_limits<int32_t>::min()) {
ptr_int_biases[row].bias = std::numeric_limits<int32_t>::min();
num_saturate++;
} else {
ptr_int_biases[row].bias = static_cast<int32_t>(value);
}
}
}
if (num_saturate > 0) {
gnalog() << "Warning: " << num_saturate << " / " << data.num_rows
<< " saturations in compound biases quantization." << std::endl;
}
}
src/plugins/intel_gna/src/frontend/quantization.h
-54
View File
@@ -1,54 +0,0 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <vector>
#include <cstdint>
#include "backend/gna_types.h"
#define MAX_OUT_MULTIPLIER 230
#define MAX_VAL_1B_WEIGHT 127
#define MAX_VAL_1B_FEAT 64
#define MAX_VAL_1B_BIAS 127
#define MAX_VAL_2B_WEIGHT 16384
#define MAX_VAL_2B_FEAT 16384
#define MAX_VAL_4B_BIAS 1073741824
template <class WeightsType, class BiasType>
struct QuantizationCallback {
float *ptr_float_weights;
float *ptr_float_biases;
WeightsType* ptr_int_weights;
BiasType* ptr_int_biases;
float input_scale_factor;
float *ptr_weight_scale_factor;
float *ptr_output_scale_factor;
uint32_t num_rows;
uint32_t num_columns;
uint32_t num_rows_padded;
uint32_t num_columns_padded;
bool quantizedWeights;
size_t fq_levels;
const size_t fq_num_stats;
const float *fq_ptr_input_low;
const float *fq_ptr_input_high;
const float* fq_ptr_output_low;
const float* fq_ptr_output_high;
void runQuantize() const;
void runFakeQuantize() const;
};
template struct QuantizationCallback<int16_t, int32_t>;
template struct QuantizationCallback<int8_t, gna_compound_bias_t>;
template struct QuantizationCallback<int8_t, int8_t>;
std::pair<float, float> FindMinMaxValues(void* ptr_float_memory, size_t num_elements);
float ScaleFactorForQuantization(void *ptr_float_memory, float target_max, size_t num_elements);
void QuantizeVector16(float *ptr_float_memory, int16_t *ptr_int_memory, uint32_t num_elements, float scale_factor);
src/plugins/intel_gna/src/frontend/quantization.hpp
+38
View File
@@ -0,0 +1,38 @@
// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <vector>
#include <cstdint>
#include "quantized_layer_params.hpp"
#include "backend/gna_types.h"
#define MAX_OUT_MULTIPLIER 230
#define MAX_VAL_1B_WEIGHT 127
#define MAX_VAL_1B_FEAT 64
#define MAX_VAL_1B_BIAS 127
#define MAX_VAL_2B_WEIGHT 16384
#define MAX_VAL_2B_FEAT 16384
#define MAX_VAL_4B_BIAS 1073741824
// Common data required for quantization of weights and biases
struct QuantizationData {
const size_t num_rows;
const size_t num_columns;
float scale_factor;
// This field is currently used for weights as well as for biases
GNAPluginNS::QuantizationParams& weights_quant_params;
};
std::pair<float, float> FindMinMaxValues(void* ptr_float_memory, size_t num_elements);
float ScaleFactorForQuantization(void *ptr_float_memory, float target_max, size_t num_elements);
template <typename T>
extern void QuantizeWeights(const QuantizationData& data, float* ptr_float_weights, T* ptr_int_weights,
gna_compound_bias_t* ptr_int_biases, const bool quantized_weights);
template <typename T>
extern void QuantizeBiases(const QuantizationData& data, float* ptr_float_biases, T* ptr_int_biases);
src/plugins/intel_gna/src/frontend/quantized_layer_params.hpp
+8 -7
View File
@@ -6,7 +6,7 @@
namespace GNAPluginNS {
class Quantization {
class QuantizationParams {
public:
void SetScale(float s) {
scale = s;
@@ -59,7 +59,7 @@ public:
return output_max_values;
}
void CopyStats(Quantization &src) {
void CopyStats(QuantizationParams &src) {
levels = src.GetLevels();
SetMinValues(src.GetMinValues(true), true);
SetMaxValues(src.GetMaxValues(true), true);
@@ -78,14 +78,15 @@ private:
};
struct QuantizedLayerParams {
Quantization _src_quant;
Quantization _dst_quant;
QuantizationParams _src_quant;
QuantizationParams _dst_quant;
// deprecate this
Quantization _weights_quant;
Quantization _bias_quant;
QuantizationParams _weights_quant;
QuantizationParams _bias_quant;
bool lowPrecision = false;
bool _weights_int8_precision = false;
bool _inputs_int8_precision = false;
};
} // namespace GNAPluginNS
src/plugins/intel_gna/src/frontend/scale_factor_calc.hpp
+53 -95
View File
@@ -23,9 +23,6 @@
namespace GNAPluginNS {
template<typename QUANT_DESC>
class ScaleFactorCalculator;
namespace frontend {
static const float MIN_SEARCH_WEIGHTS_VAL = 1.0f;
static const float MAX_SEARCH_WEIGHTS_VAL = 1024.0f;
@@ -197,7 +194,7 @@ static std::vector<float> generateScaleFactors(float startRange, float endRange,
* @param dst_quant destination quantization statistics
* @return Weights reducer
*/
static double calculateWeightsReducerFromDstStats(Quantization dst_quant) {
static double calculateWeightsReducerFromDstStats(QuantizationParams dst_quant) {
auto maxAbsVal = std::max(std::abs(dst_quant.GetMinValues().front()),
std::abs(dst_quant.GetMaxValues().front()));
@@ -301,11 +298,17 @@ static bool requantizeInput(InferenceEngine::CNNLayerPtr input, float newOutputS
return requantizeInput(prevLayer, newOutputScale, result, infiniteLoopCount);
}
//TODO: verify if this is correct way to check FQ presence for all layer types
static bool isFakeQuantize(const InferenceEngine::CNNLayer& cnnLayer) {
auto quantParams = InferenceEngine::getInjectedData<QuantizedLayerParams>(cnnLayer);
return (quantParams && quantParams->_src_quant.IsStatsSet());
}
/**
* @brief calculates output scale factor per layer
* @tparam T
*/
template<typename T, typename QUANT_DESC>
template<typename T>
class ScaleFactorPerLayer {
public:
/**
@@ -320,8 +323,8 @@ class ScaleFactorPerLayer {
}
};
template<typename QUANT_DESC>
class ScaleFactorPerLayer<InferenceEngine::CNNLayer*, QUANT_DESC> {
template<>
class ScaleFactorPerLayer<InferenceEngine::CNNLayer*> {
private :
const float activation_scale_factor = 2048.f;
const float low_prec_activation_scale_factor = 4.f;
@@ -604,13 +607,13 @@ class ScaleFactorPerLayer<InferenceEngine::CNNLayer*, QUANT_DESC> {
}
public :
bool operator()(InferenceEngine::CNNLayer *cnnLayer, ScaleFactorUpdateResult &result, int infiniteLoopCount) {
bool operator()(InferenceEngine::CNNLayer* cnnLayer, ScaleFactorUpdateResult& result, int infiniteLoopCount) {
if ( !cnnLayer ) {
IE_THROW() << "Incorrect Layer pointer \n";
}
int inputsSize = ScaleFactorCalculator<QUANT_DESC>::GetInputsBytesSize();
bool fakeQuantize = ScaleFactorCalculator<QUANT_DESC>::IsFakeQuantize();
int inputsSize = frontend::GetInputPrecision().size();
bool fakeQuantize = isFakeQuantize(*cnnLayer);
LayerInfo layerInfo(*cnnLayer);
// TODO: current approach set input scale factor for true input layer(s) equals to provided factor,
@@ -813,15 +816,15 @@ class ScaleFactorPerLayer<InferenceEngine::CNNLayer*, QUANT_DESC> {
}
};
template<typename QUANT_DESC>
class ScaleFactorPerLayer<InferenceEngine::EltwiseLayer*, QUANT_DESC> {
template<>
class ScaleFactorPerLayer<InferenceEngine::EltwiseLayer*> {
public:
bool operator()(InferenceEngine::EltwiseLayer* eltwiseLayer, ScaleFactorUpdateResult &result, int infiniteLoopCount) {
bool operator()(InferenceEngine::EltwiseLayer* eltwiseLayer, ScaleFactorUpdateResult& result, int infiniteLoopCount) {
if ( !eltwiseLayer ) {
THROW_GNA_EXCEPTION << "Incorrect Eltwise Layer pointer \n";
}
int inputsSize = ScaleFactorCalculator<QUANT_DESC>::GetInputsBytesSize();
bool fakeQuantize = ScaleFactorCalculator<QUANT_DESC>::IsFakeQuantize();
int inputsSize = frontend::GetInputPrecision().size();
bool fakeQuantize = isFakeQuantize(*eltwiseLayer);
bool lowPrecision = (inputsSize == sizeof(int8_t));
auto in0 = InferenceEngine::CNNNetPrevLayer(eltwiseLayer, 0);
@@ -878,7 +881,8 @@ class ScaleFactorPerLayer<InferenceEngine::EltwiseLayer*, QUANT_DESC> {
for (size_t i = MAX_VAL_2B_FEAT; i > 0; --i) {
auto scaleIn1Dst = i * scaleIn1Src;
auto eltwiseWeightsScale = scaleIn1Dst / scaleIn0Dst;
if (eltwiseWeightsScale < 1.0 || eltwiseWeightsScale > std::numeric_limits<int16_t>::max() - 1) {
if (eltwiseWeightsScale < 1.0 ||
eltwiseWeightsScale > std::numeric_limits<int16_t>::max() - 1) {
continue;
}
@@ -893,14 +897,17 @@ class ScaleFactorPerLayer<InferenceEngine::EltwiseLayer*, QUANT_DESC> {
}
}
if (bestWeightsScale > 0.0f && !fp32eq(bestWeightsScale, quantParams1->_weights_quant.GetScale())) {
if (bestWeightsScale > 0.0f &&
!fp32eq(bestWeightsScale, quantParams1->_weights_quant.GetScale())) {
quantParams1->_weights_quant.SetScale(bestWeightsScale);
quantParams1->_dst_quant.SetScale(quantParams1->_weights_quant.GetScale() * quantParams1->_src_quant.GetScale());
quantParams1->_dst_quant.SetScale(quantParams1->_weights_quant.GetScale() *
quantParams1->_src_quant.GetScale());
result = ScaleFactorUpdateResult(in1.get());
return true;
}
}
}
quantData->_weights_quant.SetScale(weightsScale);
quantData->_dst_quant.SetScale(quantParams1->_dst_quant.GetScale());
@@ -937,15 +944,15 @@ class ScaleFactorPerLayer<InferenceEngine::EltwiseLayer*, QUANT_DESC> {
}
};
template<typename QUANT_DESC>
class ScaleFactorPerLayer<InferenceEngine::ConcatLayer*, QUANT_DESC> {
template<>
class ScaleFactorPerLayer<InferenceEngine::ConcatLayer*> {
public:
bool operator()(InferenceEngine::ConcatLayer* concatLayer, ScaleFactorUpdateResult &result, int infiniteLoopCount) {
bool operator()(InferenceEngine::ConcatLayer* concatLayer, ScaleFactorUpdateResult& result, int infiniteLoopCount) {
if ( !concatLayer ) {
THROW_GNA_EXCEPTION << "Incorrect Concat Layer pointer \n";
}
bool fakeQuantize = ScaleFactorCalculator<QUANT_DESC>::IsFakeQuantize();
bool fakeQuantize = isFakeQuantize(*concatLayer);
if (concatLayer->insData.size() < 2) {
THROW_GNA_EXCEPTION << "Concat layer has unsupported number of incoming layers.";
@@ -1012,7 +1019,7 @@ class ScaleFactorPerLayer<InferenceEngine::ConcatLayer*, QUANT_DESC> {
for (auto it = inputLayers.begin(); it != inputLayers.end(); ++it) {
auto quantParams = InferenceEngine::getInjectedData<QuantizedLayerParams>(*it);
if ((quantParams->_dst_quant.GetScale() < minScaleFactor &&
!fp32eq(quantParams->_dst_quant.GetScale(), 1.0f)) ||
!fp32eq(quantParams->_dst_quant.GetScale(), 1.0f)) ||
fp32eq(minScaleFactor, 1.0f)) {
minScaleFactor = quantParams->_dst_quant.GetScale();
sourceLayerIt = it;
@@ -1168,8 +1175,8 @@ class ScaleFactorPerLayer<InferenceEngine::ConcatLayer*, QUANT_DESC> {
}
};
template<typename QUANT_DESC>
class ScaleFactorPerLayer<InferenceEngine::WeightableLayer*, QUANT_DESC> {
template<>
class ScaleFactorPerLayer<InferenceEngine::WeightableLayer*> {
private:
std::vector<std::tuple<uint16_t const, float const, float const>> thresholds = {
// tuple values: scale factor threshold, scale factor reduction factor for I16 precision, for I8 precision
@@ -1188,8 +1195,8 @@ class ScaleFactorPerLayer<InferenceEngine::WeightableLayer*, QUANT_DESC> {
THROW_GNA_EXCEPTION << "Incorrect weight value for " << wl->name << ":" << wl->type << "\n";
}
int inputsSize = ScaleFactorCalculator<QUANT_DESC>::GetInputsBytesSize();
bool fakeQuantize = ScaleFactorCalculator<QUANT_DESC>::IsFakeQuantize();
int inputsSize = frontend::GetInputPrecision().size();
bool fakeQuantize = InferenceEngine::getInjectedData<QuantizedLayerParams>(*wl)->_weights_quant.IsStatsSet();
auto prevLayer = CNNNetPrevLayer(wl);
auto quantDataForInputLayer =
InferenceEngine::getInjectedData<QuantizedLayerParams>(*InferenceEngine::CNNNetPrevLayer(wl).get());
@@ -1219,7 +1226,8 @@ class ScaleFactorPerLayer<InferenceEngine::WeightableLayer*, QUANT_DESC> {
}
// TODO: pass 8 bits somehow
int weightsSize = ScaleFactorCalculator<QUANT_DESC>::GetMandatoryWeightsBytesSize(wl);
int weightsSize = frontend::GetWeightsPrecision(LayerInfo(wl), *quant).size();
if (!quant->_weights_quant.IsScaleSet()) {
size_t scaleRange = 0;
if (weightsSize == 2) {
@@ -1236,23 +1244,13 @@ class ScaleFactorPerLayer<InferenceEngine::WeightableLayer*, QUANT_DESC> {
}
if (wl->_biases) {
// for now the only case of INT8 bias we support comes with INT8 inputs and weights as well
if (inputsSize == 1 && weightsSize == 1) {
quant->_bias_quant.SetScale(ScaleFactorForQuantization(wl->_biases->buffer().as<float*>(),
MAX_VAL_1B_BIAS,
wl->_biases->size()));
} else {
quant->_bias_quant.SetScale(ScaleFactorForQuantization(wl->_biases->buffer().as<float*>(),
MAX_VAL_4B_BIAS,
wl->_biases->size()));
}
// for now the only case of bias we support with INT8 inputs and weights is INT32
quant->_bias_quant.SetScale(ScaleFactorForQuantization(wl->_biases->buffer().as<float*>(),
MAX_VAL_4B_BIAS,
wl->_biases->size()));
if (quant->_bias_quant.GetScale() != -1.0f) {
// for low precision we don't change bias scale factor based on source and weights scale factors
// in order not to loose too much precision
if (inputsSize != 1 || weightsSize != 1) {
quant->_bias_quant.SetScale(
std::min(quant->_weights_quant.GetScale() * quant->_src_quant.GetScale(), quant->_bias_quant.GetScale()));
}
quant->_bias_quant.SetScale(
std::min(quant->_weights_quant.GetScale() * quant->_src_quant.GetScale(), quant->_bias_quant.GetScale()));
quant->_weights_quant.SetScale(quant->_bias_quant.GetScale() / quant->_src_quant.GetScale());
}
}
@@ -1326,18 +1324,18 @@ class ScaleFactorPerLayer<InferenceEngine::WeightableLayer*, QUANT_DESC> {
}
};
template<typename QUANT_DESC>
class ScaleFactorPerLayer<InferenceEngine::ScaleShiftLayer*, QUANT_DESC> :
public ScaleFactorPerLayer<InferenceEngine::WeightableLayer*, QUANT_DESC> {
template<>
class ScaleFactorPerLayer<InferenceEngine::ScaleShiftLayer*> :
public ScaleFactorPerLayer<InferenceEngine::WeightableLayer*> {
};
template<typename QUANT_DESC>
class ScaleFactorPerLayer<InferenceEngine::ConvolutionLayer*, QUANT_DESC> :
public ScaleFactorPerLayer<InferenceEngine::WeightableLayer*, QUANT_DESC> {
template<>
class ScaleFactorPerLayer<InferenceEngine::ConvolutionLayer*> :
public ScaleFactorPerLayer<InferenceEngine::WeightableLayer*> {
};
template<typename QUANT_DESC>
class ScaleFactorPerLayer<InferenceEngine::GemmLayer*, QUANT_DESC> {
template<>
class ScaleFactorPerLayer<InferenceEngine::GemmLayer*> {
public:
bool operator() (InferenceEngine::GemmLayer* gemmLayer, ScaleFactorUpdateResult &result, int infiniteLoopCount) {
if ( !gemmLayer ) {
@@ -1392,12 +1390,11 @@ public:
/**
* @brief scale factor calculator will calculate only output scale factors for the layer
* if scale factor propagation not possible, it will fall indicate a restart condition
* if scale factor propagation is not possible, it will fall indicate a restart condition
*/
template<typename QUANT_DESC>
class ScaleFactorCalculator {
using Cnt = std::vector<InferenceEngine::CNNLayerPtr>;
Cnt net;
Cnt net;
mutable Cnt::const_iterator idx;
mutable bool needRestart = false;
int infiniteLoopCount = 0;
@@ -1422,7 +1419,7 @@ class ScaleFactorCalculator {
bool operator()(T ptr) const {
needRestart = false;
frontend::ScaleFactorUpdateResult result;
if (!frontend::ScaleFactorPerLayer<T, QUANT_DESC>()(ptr, result, infiniteLoopCount)) {
if (!frontend::ScaleFactorPerLayer<T>()(ptr, result, infiniteLoopCount)) {
return false;
}
if (result) {
@@ -1442,45 +1439,6 @@ class ScaleFactorCalculator {
needRestart = true;
return true;
}
template<class T>
static int GetMandatoryWeightsBytesSize(T ptr) {
auto info = LayerInfo(ptr);
if (info.isConvolution() || info.isScaleShift()) {
return GetOptionalWeightsBytesSize();
}
if (IsFakeQuantize()) {
auto quantData = InferenceEngine::getInjectedData<QuantizedLayerParams>(*ptr);
if (quantData->_weights_quant.IsStatsSet()) {
if (quantData->_weights_quant.GetLevels() <= std::numeric_limits<uint8_t>::max()) {
return frontend::FakeQuantI8().getWeightsPrecision().size();
} else {
return frontend::FakeQuantI16().getWeightsPrecision().size();
}
} else {
if (!info.isSynthetic()) {
gnawarn() << "The layer (" << ptr->name << ") has not quantization statistics\n";
}
return GetOptionalWeightsBytesSize();
}
}
return QUANT_DESC::mandatory().getWeightsPrecision().size();
}
static int GetOptionalWeightsBytesSize() {
return QUANT_DESC::optional().getWeightsPrecision().size();
}
static int GetInputsBytesSize() {
return QUANT_DESC::mandatory().getInputPrecision().size();
}
static bool IsFakeQuantize() {
return std::is_same<QUANT_DESC, FakeQuant>();
}
}; // class ScaleFactorCalculator
} // namespace GNAPluginNS
src/plugins/intel_gna/src/gna_graph_compiler.cpp
+26 -26
View File
@@ -488,8 +488,8 @@ void GNAGraphCompiler::finalizeConvolution1DPrimitive(InferenceEngine::CNNLayerP
uint32_t num_bytes_per_weight = convolution._weights->getTensorDesc().getPrecision().size();
uint32_t num_bytes_per_bias = biasPrecision.size();
float weight_scale_factor = getScaleFactor(layer, QuantizedDataType::weights);
float output_scale_factor = getScaleFactor(layer, QuantizedDataType::output);
float weight_scale_factor = GetScaleFactor(layer, QuantizedDataType::weights);
float output_scale_factor = GetScaleFactor(layer, QuantizedDataType::output);
auto& currentComponent = dnnComponents.addComponent(convolution.name, "convolution");
dnn->InitConvolutional1DComponent(currentComponent,
@@ -680,8 +680,8 @@ void GNAGraphCompiler::finalizeConvolution2DPrimitive(InferenceEngine::CNNLayerP
convolution._kernel_y, convolution._kernel_x, filter_n, convolution._stride_y, convolution._stride_x,
convolution._dilation_y, convolution._dilation_x, inputPrec);
float weight_scale_factor = getScaleFactor(layer, QuantizedDataType::weights);
float output_scale_factor = getScaleFactor(layer, QuantizedDataType::output);
float weight_scale_factor = GetScaleFactor(layer, QuantizedDataType::weights);
float output_scale_factor = GetScaleFactor(layer, QuantizedDataType::output);
auto& currentComponent = dnnComponents.addComponent(convolution.name, "convolution");
dnn->InitConvolutional2DComponent(currentComponent,
@@ -849,8 +849,8 @@ void GNAGraphCompiler::PowerPrimitive(InferenceEngine::CNNLayerPtr layer) {
gna_pwl_segment_t* ptr_pwl_segments_target = nullptr;
float output_pwl_scale_factor = getScaleFactor(layer, QuantizedDataType::output);
float input_pwl_scale_factor = getScaleFactor(layer, QuantizedDataType::input);
float output_pwl_scale_factor = GetScaleFactor(layer, QuantizedDataType::output);
float input_pwl_scale_factor = GetScaleFactor(layer, QuantizedDataType::input);
if (!gnaFlags->sw_fp32 && gnaFlags->uniformPwlDesign) {
uint32_t num_segments = POW_NUM_SEGMENTS;
@@ -957,7 +957,7 @@ void GNAGraphCompiler::PoolingPrimitive(InferenceEngine::CNNLayerPtr layer) {
outputs->getPrecision().size(),
{ pooling._kernel[X_AXIS], pooling._kernel[Y_AXIS] },
{ pooling._stride[X_AXIS], pooling._stride[Y_AXIS] },
getScaleFactor(layer, QuantizedDataType::output),
GetScaleFactor(layer, QuantizedDataType::output),
ptr_inputs,
ptr_outputs);
size_t num_data_bytes_out = InferenceEngine::details::product(begin(outputs->getDims()), end(outputs->getDims()));
@@ -1028,7 +1028,7 @@ void GNAGraphCompiler::CopyPrimitive(InferenceEngine::CNNLayerPtr layer) {
num_columns_out,
inputs->getPrecision().size(),
outputs->getPrecision().size(),
getScaleFactor(layer, QuantizedDataType::output),
GetScaleFactor(layer, QuantizedDataType::output),
num_rows_out + num_padding_out,
num_columns_out,
ptr_inputs,
@@ -1205,8 +1205,8 @@ void GNAGraphCompiler::CropPrimitive(InferenceEngine::CNNLayerPtr layer) {
outputs->getPrecision().size(),
quantized == nullptr ? inputs->getPrecision().size() : (gnaFlags->input_low_precision ? 1 : 2),
gnaFlags->input_low_precision ? 1 : 4,
getScaleFactor(layer, QuantizedDataType::weights),
getScaleFactor(layer, QuantizedDataType::output),
GetScaleFactor(layer, QuantizedDataType::weights),
GetScaleFactor(layer, QuantizedDataType::output),
ptr_inputs,
ptr_outputs,
ptr_weights,
@@ -1341,8 +1341,8 @@ void GNAGraphCompiler::EltwisePrimitive(InferenceEngine::CNNLayerPtr layer) {
// TODO: only fp32 and Int16 tested
quantized == nullptr ? inputs2Bytes->getPrecision().size() : (gnaFlags->input_low_precision ? 1 : 2),
quantized == nullptr ? inputs4Bytes->getPrecision().size() : (gnaFlags->input_low_precision ? 1 : 4),
getScaleFactor(layer, QuantizedDataType::weights),
getScaleFactor(layer, QuantizedDataType::output),
GetScaleFactor(layer, QuantizedDataType::weights),
GetScaleFactor(layer, QuantizedDataType::output),
ptr_inputs,
ptr_outputs,
ptr_weights,
@@ -1448,8 +1448,8 @@ void GNAGraphCompiler::GemmPrimitive(InferenceEngine::CNNLayerPtr layer) {
outputs->getPrecision().size(),
quantized == nullptr ? input_2->getPrecision().size() : 2,
quantized == nullptr ? input_2->getPrecision().size() : 4,
getScaleFactor(layer, QuantizedDataType::weights),
getScaleFactor(layer, QuantizedDataType::output),
GetScaleFactor(layer, QuantizedDataType::weights),
GetScaleFactor(layer, QuantizedDataType::output),
ptr_input_1,
ptr_outputs,
ptr_input_2,
@@ -1537,8 +1537,8 @@ void GNAGraphCompiler::AffinePrimitive(InferenceEngine::CNNLayerPtr layer, bool
outputs->getPrecision().size(),
weightable._weights->getTensorDesc().getPrecision().size(),
biasPrecisionSize,
getScaleFactor(layer, QuantizedDataType::weights),
getScaleFactor(layer, QuantizedDataType::output),
GetScaleFactor(layer, QuantizedDataType::weights),
GetScaleFactor(layer, QuantizedDataType::output),
ptr_inputs,
ptr_outputs,
ptr_weights,
@@ -1723,7 +1723,7 @@ void GNAGraphCompiler::ConcatAlignFilterPrimitive(InferenceEngine::CNNLayerPtr l
num_columns_in,
inputs->getPrecision().size(),
inputs->getPrecision().size(),
getScaleFactor(layer, QuantizedDataType::output),
GetScaleFactor(layer, QuantizedDataType::output),
num_rows_copied,
num_columns_in,
ptr_inputs,
@@ -1759,8 +1759,8 @@ void GNAGraphCompiler::ConcatAlignFilterPrimitive(InferenceEngine::CNNLayerPtr l
outputs->getPrecision().size(),
filterLayer->_weights->getTensorDesc().getPrecision().size(),
biasPrecisionSize,
getScaleFactor(layer, QuantizedDataType::weights),
getScaleFactor(layer, QuantizedDataType::output),
GetScaleFactor(layer, QuantizedDataType::weights),
GetScaleFactor(layer, QuantizedDataType::output),
ptr_inputs,
ptr_outputs,
ptr_weights,
@@ -1864,8 +1864,8 @@ void GNAGraphCompiler::ConvolutionFilterPrimitive(InferenceEngine::CNNLayerPtr l
numberOfFilters,
filterWidth,
convolutionStride,
getScaleFactor(layer, QuantizedDataType::weights),
getScaleFactor(layer, QuantizedDataType::output),
GetScaleFactor(layer, QuantizedDataType::weights),
GetScaleFactor(layer, QuantizedDataType::output),
ptr_inputs,
ptr_outputs,
ptr_weights,
@@ -1924,8 +1924,8 @@ void GNAGraphCompiler::PWLPrimitive(InferenceEngine::CNNLayerPtr layer) {
auto inputs = layer->insData.begin()->lock();
auto outputs = *layer->outData.begin();
float output_pwl_scale_factor = getScaleFactor(layer, QuantizedDataType::output);
float input_pwl_scale_factor = getScaleFactor(layer, QuantizedDataType::input);
float output_pwl_scale_factor = GetScaleFactor(layer, QuantizedDataType::output);
float input_pwl_scale_factor = GetScaleFactor(layer, QuantizedDataType::input);
auto orientation = kDnnInterleavedOrientation;
@@ -2173,7 +2173,7 @@ void GNAGraphCompiler::PermutePrimitive(InferenceEngine::CNNLayerPtr layer) {
squeezedInputOrder[1],
inputs->getPrecision().size(),
outputs->getPrecision().size(),
getScaleFactor(layer, QuantizedDataType::output),
GetScaleFactor(layer, QuantizedDataType::output),
ptr_inputs,
ptr_outputs);
}
@@ -2188,7 +2188,7 @@ void GNAGraphCompiler::PermutePrimitive(InferenceEngine::CNNLayerPtr layer) {
squeezedInputOrder[1],
inputs->getPrecision().size(),
outputs->getPrecision().size(),
getScaleFactor(layer, QuantizedDataType::output),
GetScaleFactor(layer, QuantizedDataType::output),
ptr_inputs,
ptr_outputs);
}
@@ -2465,7 +2465,7 @@ GNAPluginNS::ConnectionDetails GNAGraphCompiler::connectInput(CNNLayerPtr layer,
if (LayerInfo(prevLayer).isInput()) {
auto quantized = getInjectedData<QuantizedLayerParams>(prevLayer);
if (quantized) {
if (quantized->lowPrecision) {
if (quantized->_inputs_int8_precision) {
inputs_ptr_->at(prevLayer->name).set_precision(InferenceEngine::Precision::I8);
} else {
inputs_ptr_->at(prevLayer->name).set_precision(InferenceEngine::Precision::I16);
src/plugins/intel_gna/src/gna_plugin.cpp
+2 -20
View File
@@ -863,27 +863,9 @@ void GNAPlugin::LoadNetwork(const CNNNetwork& _network) {
// to run all passes need to have two calls to pass manager
run_passes(newNet, true, gnaFlags->input_low_precision);
run_passes(newNet, false, gnaFlags->input_low_precision);
} else if (fake_quantized) {
ModelQuantizer<FakeQuant> modelQuantizer;
newNet = modelQuantizer.quantize(network, run_passes, *inputs_ptr_);
} else {
switch (config.gnaPrecision) {
case Precision::I16:
ModelQuantizer<QuantI16> q16;
newNet = q16.quantize(network, run_passes, *inputs_ptr_);
break;
case Precision::I8:
if (gnaFlags->input_low_precision == false) {
ModelQuantizer<QuantI8> q8;
newNet = q8.quantize(network, run_passes, *inputs_ptr_);
} else {
ModelQuantizer<QuantI8_I8> q8_8;
newNet = q8_8.quantize(network, run_passes, *inputs_ptr_);
}
break;
default:
THROW_GNA_EXCEPTION << "unsupported GNA precision for quantisation: " << config.gnaPrecision;
}
ModelQuantizer modelQuantizer;
newNet = modelQuantizer.quantize(network, run_passes, *inputs_ptr_, config.gnaPrecision, gnaFlags->input_low_precision);
}
auto inputLayers = CNNNetGetAllInputLayers(newNet);
src/plugins/intel_gna/src/optimizer/gna_pass_manager.cpp
+1 -2
View File
@@ -26,7 +26,7 @@
#include "backend/dnn_types.h"
#include "gna_plugin_log.hpp"
#include "frontend/quantization.h"
#include "frontend/quantization.hpp"
#include "frontend/quantized_layer_params.hpp"
#include <layers/gna_copy_layer.hpp>
#include <layers/gna_fake_quantize_layer.hpp>
@@ -35,7 +35,6 @@
#include "gna_pass_manager.hpp"
#include "layers/gna_layer_info.hpp"
#include "gna_upstream_iterator.hpp"
#include "frontend/quantization.h"
#include "gna_groups.hpp"
#include "gna_graph_patterns.hpp"
#include "gna_data_types.hpp"
src/plugins/intel_gna/tests/unit/gna_get_scale_factor.cpp
+4 -4
View File
@@ -24,10 +24,10 @@ class GnaGetScaleFactorTest : public ::testing::Test {
quant->_dst_quant.SetScale(dst_scale);
quant->_weights_quant.SetScale(weights_scale);
quant->_bias_quant.SetScale(bias_scale);
ASSERT_EQ(GNAPluginNS::getScaleFactor(layer, GNAPluginNS::QuantizedDataType::input), src_scale);
ASSERT_EQ(GNAPluginNS::getScaleFactor(layer, GNAPluginNS::QuantizedDataType::output), dst_scale);
ASSERT_EQ(GNAPluginNS::getScaleFactor(layer, GNAPluginNS::QuantizedDataType::weights), weights_scale);
ASSERT_EQ(GNAPluginNS::getScaleFactor(layer, GNAPluginNS::QuantizedDataType::bias), bias_scale);
ASSERT_EQ(GNAPluginNS::GetScaleFactor(layer, GNAPluginNS::QuantizedDataType::input), src_scale);
ASSERT_EQ(GNAPluginNS::GetScaleFactor(layer, GNAPluginNS::QuantizedDataType::output), dst_scale);
ASSERT_EQ(GNAPluginNS::GetScaleFactor(layer, GNAPluginNS::QuantizedDataType::weights), weights_scale);
ASSERT_EQ(GNAPluginNS::GetScaleFactor(layer, GNAPluginNS::QuantizedDataType::bias), bias_scale);
}
};
src/tests_deprecated/unit/engines/gna/I8_quantisation_test.cpp
+2 -15
View File
@@ -16,7 +16,8 @@ using namespace GNATestIRs;
class I8QuantisationTest : public GNATest<> {
protected:
LayersQuantizer<QuantI8> lc = LayersQuantizer<QuantI8> (1.0f);
ModelQuantizer q;
LayersQuantizer lc = LayersQuantizer (1.0f);
InferenceEngine::CNNLayerPtr quantize (InferenceEngine::CNNLayerPtr lp) {
auto newLayer = InferenceEngine::injectData<QuantizedLayerParams>(lp);
@@ -63,8 +64,6 @@ TEST_F(I8QuantisationTest, canQuantizeActivation){
}
TEST_F(I8QuantisationTest, inputPrecisionIs16Bits){
ModelQuantizer<QuantI8> q;
auto weights = make_shared_blob<uint8_t >({ Precision::U8, {440}, C });
weights->allocate();
fillWeights(weights);
@@ -80,8 +79,6 @@ TEST_F(I8QuantisationTest, inputPrecisionIs16Bits){
}
TEST_F(I8QuantisationTest, FCDimensionIs1){
ModelQuantizer<QuantI8> q;
auto weights = make_shared_blob<uint8_t >({ Precision::U8, {440}, C });
weights->allocate();
fillWeights(weights);
@@ -93,8 +90,6 @@ TEST_F(I8QuantisationTest, FCDimensionIs1){
}
TEST_F(I8QuantisationTest, outputAffinePrecisionIs32Bits){
ModelQuantizer<QuantI8> q;
auto weights = make_shared_blob<uint8_t >({ Precision::U8, {440}, C });
weights->allocate();
fillWeights(weights);
@@ -110,8 +105,6 @@ TEST_F(I8QuantisationTest, outputAffinePrecisionIs32Bits){
}
TEST_F(I8QuantisationTest, fp16tofp32_on_fullyConnected_model) {
ModelQuantizer<QuantI8> q;
auto weights = make_shared_blob<uint8_t>({ Precision::U8, {220}, Layout::C });
weights->allocate();
fillWeights(weights);
@@ -123,8 +116,6 @@ TEST_F(I8QuantisationTest, fp16tofp32_on_fullyConnected_model) {
}
TEST_F(I8QuantisationTest, LSTMCell_quantize) {
ModelQuantizer<QuantI8> q;
auto weights = make_shared_blob<uint8_t>({ Precision::U8, {33664}, C });
weights->allocate();
fillWeights(weights);
@@ -136,8 +127,6 @@ TEST_F(I8QuantisationTest, LSTMCell_quantize) {
}
TEST_F(I8QuantisationTest, LSTMCell_unaligned_quantize) {
ModelQuantizer<QuantI8> q;
auto weights = make_shared_blob<uint8_t>({ Precision::U8, {3480}, C });
weights->allocate();
fillWeights(weights);
@@ -149,8 +138,6 @@ TEST_F(I8QuantisationTest, LSTMCell_unaligned_quantize) {
}
TEST_F(I8QuantisationTest, TI_quantize) {
ModelQuantizer<QuantI8> q;
auto weights = make_shared_blob<uint8_t>({ Precision::U8, {249748}, C });
weights->allocate();
fillWeights(weights);
src/tests_deprecated/unit/engines/gna/graph_tools/graph_copy_tests.cpp
+1 -4
View File
@@ -97,12 +97,9 @@ TEST_F(GraphCopyTests, canPreserveAttributes) {
#ifdef ENABLE_INTEL_GNA
using namespace GNAPluginNS;
struct _FP32_2_FP32 : public GNAPluginNS::frontend::QuantDescTmpl<float, float, float, float, float> {
};
using FP32_2_FP32 = GNAPluginNS::frontend::QuantPair<_FP32_2_FP32 , _FP32_2_FP32 >;
TEST_F(GraphCopyTests, canQuantizeTopology) {
auto clone = ModelQuantizer<FP32_2_FP32>().quantize(CNNNetwork(mockNet), std::vector<float >({1.0f, 1.0f}));
auto clone = ModelQuantizer().quantize(CNNNetwork(mockNet), std::vector<float >({1.0f, 1.0f}));
CNNNetBFS(CommonTestUtils::getLayerByName(clone, "1"), [&](CNNLayerPtr layer) {
auto params = getInjectedData<QuantizedLayerParams>(layer);
src/tests_deprecated/unit/engines/gna/i16_quantisation_test.cpp
+2 -14
View File
@@ -18,7 +18,8 @@ using namespace GNATestIRs;
class I16QuantisationTest : public GNATest<> {
protected:
LayersQuantizer<QuantI16> lc = LayersQuantizer<QuantI16>(1.0f);
ModelQuantizer q;
LayersQuantizer lc = LayersQuantizer(1.0f);
InferenceEngine::CNNLayerPtr quantize (InferenceEngine::CNNLayerPtr lp) {
auto newLayer = InferenceEngine::injectData<QuantizedLayerParams>(lp);
@@ -84,8 +85,6 @@ TEST_F(I16QuantisationTest, canQuantizeActivation){
}
TEST_F(I16QuantisationTest, outputAffinePrecisionIs32Bits){
ModelQuantizer<QuantI16> q;
auto weights = make_shared_blob<uint8_t>({ Precision::U8, {440}, C });
weights->allocate();
fillWeights(weights);
@@ -102,8 +101,6 @@ TEST_F(I16QuantisationTest, outputAffinePrecisionIs32Bits){
TEST_F(I16QuantisationTest, canQuantizeLstmLikeTopology) {
ModelQuantizer<QuantI16> q;
auto weights = setWeights(make_shared_blob<uint8_t >({ Precision::U8, {440}, C }));
//std::fill_n(weights->buffer().as<float*>(), weights->byteSize()/sizeof(float), 0);
@@ -114,7 +111,6 @@ TEST_F(I16QuantisationTest, canQuantizeLstmLikeTopology) {
}
TEST_F(I16QuantisationTest, DISABLED_outputScaleFactorForAffineIsCorrect){
ModelQuantizer<QuantI16> q;
const float inputScaleFactorTest = 1000;
const float weightValueTest = 100;
@@ -350,8 +346,6 @@ TEST_F(I16QuantisationTest, DISABLED_noPermutationOfWeightsBetweenConvAndAffineI
}
TEST_F(I16QuantisationTest, fp16tofp32_on_fullyConnected_model) {
ModelQuantizer<QuantI16> q;
auto weights = make_shared_blob<uint8_t>({ Precision::U8, {220}, Layout::C });
weights->allocate();
fillWeights(weights);
@@ -405,8 +399,6 @@ TEST_F(I16QuantisationTest, ConcatWithConstInputPropagatedForward) {
}
TEST_F(I16QuantisationTest, LSTMCell_quantize) {
ModelQuantizer<QuantI16> q;
auto weights = make_shared_blob<uint8_t>({ Precision::U8, {33664}, C });
weights->allocate();
fillWeights(weights);
@@ -418,8 +410,6 @@ TEST_F(I16QuantisationTest, LSTMCell_quantize) {
}
TEST_F(I16QuantisationTest, LSTMCell_unaligned_quantize) {
ModelQuantizer<QuantI16> q;
auto weights = make_shared_blob<uint8_t>({ Precision::U8, {3480}, C });
weights->allocate();
fillWeights(weights);
@@ -457,8 +447,6 @@ TEST_F(I16QuantisationTest, ConcatWithDifferentInputScaleFactorsPropagateForward
}
TEST_F(I16QuantisationTest, TI_quantize) {
ModelQuantizer<QuantI16> q;
auto weights = make_shared_blob<uint8_t>({ Precision::U8, {249748}, C });
weights->allocate();
fillWeights(weights);
src/tests_deprecated/unit/engines/gna/matchers/conv_matcher.hpp
+1 -1
View File
@@ -6,7 +6,7 @@
#include "backend/gna_types.h"
#include "nnet_base_matcher.hpp"
#include "frontend/quantization.h"
#include "frontend/quantization.hpp"
class ConvoluionLayerMatcher : public ::testing::MatcherInterface<const gna_nnet_type_t*> {
bool matchInserted;
src/tests_deprecated/unit/engines/gna/matchers/diag_matcher.hpp
+1 -1
View File
@@ -6,7 +6,7 @@
#include "backend/gna_types.h"
#include "nnet_base_matcher.hpp"
#include "frontend/quantization.h"
#include "frontend/quantization.hpp"
class DiagLayerMatcher : public ::testing::MatcherInterface<const gna_nnet_type_t*> {
bool matchInserted;
src/tests_deprecated/unit/engines/gna/matchers/pool_matcher.hpp
+1 -1
View File
@@ -6,7 +6,7 @@
#include "backend/gna_types.h"
#include "nnet_base_matcher.hpp"
#include "frontend/quantization.h"
#include "frontend/quantization.hpp"
class PoolingLayerMatcher : public ::testing::MatcherInterface<const gna_nnet_type_t*> {
bool matchInserted;
src/tests_deprecated/unit/engines/gna/matchers/weights_matcher.hpp
+1 -1
View File
@@ -5,7 +5,7 @@
#pragma once
#include "backend/gna_types.h"
#include "nnet_base_matcher.hpp"
#include "frontend/quantization.h"
#include "frontend/quantization.hpp"
using TranspozedData = std::tuple<std::vector<uint16_t>*, int, int>;