IntenseWebs/openvino
3
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[nG] [IE] add support of mixed FP16/FP32 precision ov::Model for mixed inference (#13249)

Browse Source 
* add pattern mathing for MVN, Exp->ReduceSum, L2Normalize, Div with eps for mixed precision inference

* added necessary includes

* clang_format_fix_all

* fix warning_as_error for unused variable

* fix warning_as_error for specifying float literals

* enable marking for fp32 IRs as well

* cosmetic improvements in unit-tests

* fix warnings as error

* added unit-tests for compress_float_constants.cpp for out of range values

* Update align_mixed_fp32_fp16_types.cpp

* Apply suggestions from code review

Co-authored-by: Maxim Vafin <maxim.vafin@intel.com>

* some grooming: mainly in imports

* build fix: replaced ngraph:: -> ov::

* collected all markings in a single file

* shortened pass names

* style fix

* made MarkNormalizationOps as a separate pass

* removed redundant comment, fixed description of MarkSugraphsToKeepInMixedPrecision pass

* comments on Up and Down marking in MarkSugraphsToKeepInMixedPrecision

* cleared info messages in compress_float_constants.cpp, removed threshold adjusting from ENV

* moved declarations of MarkNormalizationOps, MarkExpInReduceOpPath, MarkDivWithEps to hide them from outside users

* simplified pattern matching for max_or_add

* moved `reduceop_path` rt_info inside mark_subgraphs_to_keep_in_mixed_precision.cpp

* fix potential bug with Convert

* removed redundant check for Converts in `insert_converts_after_if_needed` as well

* set Convert types more safely

* corrections in opset10 namespaces; some minor corrections

---------

Co-authored-by: Maxim Vafin <maxim.vafin@intel.com>
This commit is contained in:
Pavel EsirandMaxim Vafin authored and GitHub committed 2023-01-30 10:04:12 +04:00
1 parent 635bfd848e
commit 7b6eefbbf8
12 files changed
+2444 -334
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/common/transformations/include/transformations/common_optimizations/align_mixed_fp32_fp16_types.hpp
+27
View File
@@ -0,0 +1,27 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "openvino/pass/graph_rewrite.hpp"
#include "openvino/pass/pass.hpp"
#include "transformations_visibility.hpp"
namespace ov {
namespace pass {
class TRANSFORMATIONS_API AlignMixedFP32FP16Types;
} // namespace pass
} // namespace ov
/**
* @ingroup ie_transformation_common_api
* @brief AlignMixedFP32FP16Types adds Converts to keep mixed FP16/FP32 graph type consistent
*/
class ov::pass::AlignMixedFP32FP16Types : public ov::pass::ModelPass {
public:
OPENVINO_RTTI("AlignMixedFP32FP16Types", "0");
bool run_on_model(const std::shared_ptr<ov::Model>& m) override;
};
src/common/transformations/include/transformations/common_optimizations/division_by_zero_fp16_resolver.hpp
-34
View File
@@ -1,34 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <memory>
#include <openvino/pass/graph_rewrite.hpp>
#include <openvino/pass/pattern/matcher.hpp>
#include <transformations_visibility.hpp>
#include <utility>
namespace ov {
namespace pass {
class TRANSFORMATIONS_API DivisionByZeroFP16Resolver;
} // namespace pass
} // namespace ov
/**
* @ingroup ie_transformation_common_api
* @brief: clamps eps into fp16 minimal normalized value in input_1/Maximum(input_2, eps); input_1/Add(input_2, eps);
* and input_1*Pow(Maximum[Add](input_2, eps), -z) patterns to prevent division by zero.
*
* eps must be always nonzero to prevent from NaNs in such expressions if input_1 and input_2 simultaneously happened to
* be zero. We should keep in such patterns eps >= fp16 minimal normalized value so that CompressFloatConstants should
* not cast them into zero during compression into f16.
*/
class ov::pass::DivisionByZeroFP16Resolver : public ov::pass::MatcherPass {
public:
OPENVINO_RTTI("DivisionByZeroFP16Resolver", "0");
DivisionByZeroFP16Resolver();
};
src/common/transformations/include/transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.hpp
+31
View File
@@ -0,0 +1,31 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "openvino/pass/graph_rewrite.hpp"
#include "openvino/pass/pass.hpp"
#include "transformations_visibility.hpp"
namespace ov {
namespace pass {
class TRANSFORMATIONS_API MarkSugraphsToKeepInMixedPrecision;
constexpr auto float16_min_normalized = float16::from_bits(0x0400);
} // namespace pass
} // namespace ov
/*
* @ingroup ie_transformation_common_api
* @brief: MarkSugraphsToKeepInMixedPrecision container for marking passes which marks subgraphs
* to be kept in f32 for mixed precision inference. Includes passes for the following patterns:
* L2Normalize, MVN, ShapeOf subgraphs, Exp in ReduceOp paths and Division with small eps values.
*/
class ov::pass::MarkSugraphsToKeepInMixedPrecision : public ov::pass::ModelPass {
public:
OPENVINO_RTTI("MarkSugraphsToKeepInMixedPrecision", "0");
bool run_on_model(const std::shared_ptr<ov::Model>& m) override;
};
src/common/transformations/src/transformations/common_optimizations/align_mixed_fp32_fp16_types.cpp
+80
View File
@@ -0,0 +1,80 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "transformations/common_optimizations/align_mixed_fp32_fp16_types.hpp"
#include "itt.hpp"
#include "openvino/core/rt_info.hpp"
#include "openvino/op/util/precision_sensitive_attribute.hpp"
#include "openvino/opsets/opset10.hpp"
#include "transformations/convert_precision.hpp"
#include "transformations/rt_info/decompression.hpp"
#include "transformations/rt_info/disable_fp16_compression.hpp"
using namespace ov;
bool ov::pass::AlignMixedFP32FP16Types::run_on_model(const std::shared_ptr<ov::Model>& model) {
RUN_ON_MODEL_SCOPE(AlignMixedFP32FP16Types);
std::function<bool(const std::shared_ptr<Node>&)> insert_converts_before_if_needed =
[&](const std::shared_ptr<Node>& node) {
bool is_changed = false;
for (const auto& input : node->inputs()) {
const auto& incoming_output = input.get_source_output();
const auto& incoming_node = incoming_output.get_node_shared_ptr();
if (fp16_compression_is_disabled(incoming_node))
continue; // we are in the middle
if (!incoming_output.get_element_type().is_real())
continue;
auto convert = std::make_shared<opset10::Convert>(incoming_output, incoming_output.get_element_type());
convert->set_friendly_name(incoming_node->get_friendly_name() + "_decompressed_to_f32");
copy_runtime_info(incoming_node, convert);
input.replace_source_output(convert);
disable_fp16_compression(convert);
is_changed = true;
}
return is_changed;
};
std::function<bool(const std::shared_ptr<Node>&)> insert_converts_after_if_needed =
[&](const std::shared_ptr<Node>& node) {
bool is_changed = false;
for (const auto& output : node->outputs()) {
for (const auto& out_inputs : output.get_target_inputs()) {
auto out_node = out_inputs.get_node()->shared_from_this();
if (fp16_compression_is_disabled(out_node) || is_precision_sensitive(out_inputs))
continue;
if (!out_inputs.get_element_type().is_real())
continue;
// todo xxx-101766: if we don't skip Results there is an error on GPU
if (std::dynamic_pointer_cast<opset10::Result>(out_node))
continue;
// element_type of this convert will be changed automatically to f16 after
// ConvertPrecision(f32 -> f16). It's kept here f32 to keep ov::Model validatable
auto convert = std::make_shared<opset10::Convert>(output, out_inputs.get_element_type());
copy_runtime_info(node, convert);
convert->set_friendly_name(node->get_friendly_name() + "_compressed_to_f16");
out_inputs.replace_source_output(convert);
is_changed = true;
}
}
return is_changed;
};
bool is_changed = false;
for (auto& node : model->get_ordered_ops()) {
if (!fp16_compression_is_disabled(node))
continue;
is_changed |= insert_converts_before_if_needed(node);
is_changed |= insert_converts_after_if_needed(node);
}
return true;
}
src/common/transformations/src/transformations/common_optimizations/compress_float_constants.cpp
+21 -10
View File
@@ -5,7 +5,7 @@
#include "transformations/common_optimizations/compress_float_constants.hpp"
#include "itt.hpp"
#include "ngraph/rt_info.hpp"
#include "openvino/core/rt_info.hpp"
#include "openvino/opsets/opset8.hpp"
#include "openvino/pass/pattern/op/wrap_type.hpp"
#include "transformations/rt_info/decompression.hpp"
@@ -25,23 +25,30 @@ std::shared_ptr<ov::Node> change_constant_precision_to_fp16(std::shared_ptr<ov::
if (dst_data == nullptr)
return nullptr;
bool is_overflow = false;
int num_out_of_range = 0;
for (size_t i = 0; i < size; ++i) {
if (src_data[i] > std::numeric_limits<ov::float16>::max()) {
// if it is smaller than the smallest positive normal fp16 but not zero
if (std::abs(src_data[i]) <= ov::float16::from_bits(0x0400) && src_data[i] != 0.0f) {
num_out_of_range++;
} else if (src_data[i] > std::numeric_limits<ov::float16>::max()) {
dst_data[i] = std::numeric_limits<ov::float16>::max();
is_overflow = true;
num_out_of_range++;
} else if (src_data[i] < std::numeric_limits<ov::float16>::lowest()) {
dst_data[i] = std::numeric_limits<ov::float16>::lowest();
is_overflow = true;
num_out_of_range++;
} else {
dst_data[i] = static_cast<ov::float16>(src_data[i]);
}
}
if (is_overflow) {
std::cerr << "Warning: One or more of the values of the Constant can't fit in the float16 data type."
" Those values were casted to the nearest limit value, the model can produce incorrect results."
<< std::endl;
// if more than 75% of a FP32 constant do not fit into FP16 keep in FP32
float keep_threshold = 0.75f;
float out_of_range_proportion = static_cast<float>(num_out_of_range) / static_cast<float>(size);
if (out_of_range_proportion >= keep_threshold) {
return nullptr;
}
return new_constant;
}
} // namespace
@@ -70,11 +77,15 @@ ov::pass::CompressFloatConstantsImpl::CompressFloatConstantsImpl() {
} else {
return false;
}
if (!new_const) {
return false;
}
auto convert = std::make_shared<ov::opset8::Convert>(new_const, const_node->get_element_type());
new_const->set_friendly_name(const_node->get_friendly_name() + "_compressed");
convert->set_friendly_name(const_node->get_friendly_name());
ngraph::copy_runtime_info(const_node, convert);
ov::copy_runtime_info(const_node, convert);
ov::mark_as_decompression(convert);
ov::replace_node(const_node, convert);
src/common/transformations/src/transformations/common_optimizations/convert_compressed_to_mixed_precision.cpp
+5 -4
View File
@@ -7,8 +7,9 @@
#include "itt.hpp"
#include "openvino/opsets/opset8.hpp"
#include "openvino/pass/manager.hpp"
#include "transformations/common_optimizations/align_mixed_fp32_fp16_types.hpp"
#include "transformations/common_optimizations/convert_compression_only_to_legacy.hpp"
#include "transformations/common_optimizations/mark_precision_sensitive_shapeof_subgraphs.hpp"
#include "transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.hpp"
#include "transformations/convert_precision.hpp"
#include "transformations/utils/utils.hpp"
@@ -20,13 +21,13 @@ bool ov::pass::ConvertCompressedToMixedPrecision::run_on_model(const std::shared
return false;
Manager manager(get_pass_config());
// Mark nodes in ShapeOf subgraphs with disable_fp16_compression rt_info to keep them in FP32 precision
REGISTER_PASS(manager, MarkPrecisionSensitiveShapeOfSubgraphs)
REGISTER_PASS(manager, MarkSugraphsToKeepInMixedPrecision)
REGISTER_PASS(manager, AlignMixedFP32FP16Types)
const precisions_array convert_precision_list{{ov::element::f32, ov::element::f16}};
type_to_fuse_map additional_fuse_map = {};
// call ConvertPrecision with keep_precision_sensitive_in_fp32 = true
REGISTER_PASS(manager, ConvertPrecision, convert_precision_list, additional_fuse_map, true);
REGISTER_PASS(manager, ConvertPrecision, convert_precision_list, additional_fuse_map, true)
REGISTER_PASS(manager, EnableDecompressionConvertConstantFolding)
REGISTER_PASS(manager, ConstantFolding)
src/common/transformations/src/transformations/common_optimizations/division_by_zero_fp16_resolver.cpp
-79
View File
@@ -1,79 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "transformations/common_optimizations/division_by_zero_fp16_resolver.hpp"
#include <memory>
#include <openvino/opsets/opset8.hpp>
#include <openvino/pass/pattern/op/or.hpp>
#include <openvino/pass/pattern/op/wrap_type.hpp>
#include <vector>
#include "itt.hpp"
#include "ngraph/rt_info.hpp"
#include "transformations/utils/utils.hpp"
constexpr float normalized_fp16_min = 6.103515625e-05f; // fp16 minimal normalized value
using namespace ov;
ov::pass::DivisionByZeroFP16Resolver::DivisionByZeroFP16Resolver() {
MATCHER_SCOPE(DivisionByZeroFP16Resolver);
// to detect the following patterns where eps is used to prevent division by zero:
// input_1/Maximum(input_2, eps)
// input_1/Add(input_2, eps)
// input_1/Sqrt(Maximum(input_2, eps))
// input_1/Sqrt(Add(input_2, eps))
// input_1*Pow(Maximum(input_2, eps), -z)
// input_1*Pow(Add(input_2, eps), -z)
auto input_1 = pattern::any_input();
auto input_2 = pattern::any_input();
auto eps_const_pattern = pattern::wrap_type<opset8::Constant>();
auto max = std::make_shared<opset8::Maximum>(input_2, eps_const_pattern);
auto add = std::make_shared<opset8::Add>(input_2, eps_const_pattern);
auto max_or_add = std::make_shared<pattern::op::Or>(OutputVector{max, add});
auto sqrt = std::make_shared<opset8::Sqrt>(max_or_add);
auto sqrt_or_max_add = std::make_shared<pattern::op::Or>(OutputVector{max_or_add, sqrt});
// whether is divided directly or after sqrt (e.g. in L2Norm after sqrt, in MVN is divided directly)
auto divide = std::make_shared<opset8::Divide>(input_1, sqrt_or_max_add);
auto pow_exp = pattern::wrap_type<opset8::Constant>();
auto pow_pattern = std::make_shared<opset8::Power>(max_or_add, pow_exp);
auto mul_pattern = std::make_shared<opset8::Multiply>(input_1, pow_pattern);
auto div_or_mul_to_negative_pow = std::make_shared<pattern::op::Or>(OutputVector{divide, mul_pattern});
matcher_pass_callback callback = [=](pattern::Matcher& m) {
const auto& pattern_to_output = m.get_pattern_map();
const auto mul = std::dynamic_pointer_cast<opset8::Multiply>(m.get_match_root());
if (mul) {
// pattern input_1*Pow(Maximum(input_2, eps), z) or input_1*Pow(Add(input_2, eps), z) is matched
const auto pow_const = std::dynamic_pointer_cast<opset8::Constant>(pattern_to_output.at(pow_exp));
for (float val : pow_const->get_vector<float>())
if (val >= 0) // continue only if exponent is negative (z < 0)
return false;
}
const auto eps_const = std::dynamic_pointer_cast<opset8::Constant>(pattern_to_output.at(eps_const_pattern));
if (!eps_const || eps_const->get_element_type() != ov::element::f32)
return false;
for (float val : eps_const->get_vector<float>())
if (val >= normalized_fp16_min)
return false;
auto new_constant = std::make_shared<opset8::Constant>(eps_const->get_element_type(),
eps_const->get_shape(),
normalized_fp16_min);
copy_runtime_info(eps_const, new_constant);
replace_node(eps_const, new_constant);
return true;
};
auto m = std::make_shared<pattern::Matcher>(div_or_mul_to_negative_pow, matcher_name);
register_matcher(m, callback);
}
src/common/transformations/src/transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.cpp
+362
View File
@@ -0,0 +1,362 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.hpp"
#include "itt.hpp"
#include "openvino/op/util/broadcast_base.hpp"
#include "openvino/op/util/gather_base.hpp"
#include "openvino/opsets/opset10.hpp"
#include "openvino/opsets/opset2.hpp"
#include "openvino/pass/manager.hpp"
#include "openvino/pass/pattern/op/or.hpp"
#include "openvino/pass/pattern/op/wrap_type.hpp"
#include "transformations/common_optimizations/mark_precision_sensitive_shapeof_subgraphs.hpp"
#include "transformations/convert_precision.hpp"
#include "transformations/rt_info/disable_fp16_compression.hpp"
#include "transformations/utils/utils.hpp"
using namespace std;
using namespace ov::opset10;
namespace ov {
namespace pass {
/*
* MarkNormalizationOps marks MVN and NormalizeL2 to be kept in f32 precision.
*/
class MarkNormalizationOps : public MatcherPass {
public:
OPENVINO_RTTI("MarkNormalizationOps", "0");
MarkNormalizationOps() {
MATCHER_SCOPE(MarkNormalizationOps);
auto ops_to_be_kept_fp32 = pattern::wrap_type<opset2::MVN, MVN, NormalizeL2>();
matcher_pass_callback callback = [=](pattern::Matcher& m) {
const auto& node = m.get_match_root();
if (!node)
return false;
disable_fp16_compression(node);
return true;
};
auto m = make_shared<pattern::Matcher>(ops_to_be_kept_fp32, matcher_name);
register_matcher(m, callback);
}
};
// Marking continues to propagate through these ops.
std::shared_ptr<Node> propagate_through_ops = pattern::wrap_type<Squeeze,
Unsqueeze,
Reshape,
op::util::BroadcastBase,
op::util::BinaryElementwiseArithmetic,
op::util::UnaryElementwiseArithmetic,
MVN,
opset2::MVN,
NormalizeL2,
Sqrt,
StridedSlice,
ReduceSum,
ReduceMean,
Slice,
VariadicSplit,
Split,
op::util::GatherBase,
Concat,
Convert, // through Convert can go only to Constants
Constant,
Tile>();
/* After PropagateDownMark we need to go also once up to include side branches of ops with several args:
* Elementwise, Concat and so. E.g. if one of the argument of Concat was marked
* during PropagateDownMark we need also mark its other inputs and this is done in this PropagateUpMark pass.
* Propagation stops when we face ops not listed in propagate_through_ops: e.g. if we face Conv or MatMul.
*/
class PropagateUpMarkToKeepInMixedPrecision : public pass::MatcherPass {
public:
OPENVINO_RTTI("PropagateUpMarkToKeepInMixedPrecision", "0");
PropagateUpMarkToKeepInMixedPrecision() {
MATCHER_SCOPE(PropagateUpMarkToKeepInMixedPrecision);
matcher_pass_callback callback = [=](pattern::Matcher& m) {
const auto& node = m.get_match_root();
bool has_marked_output = false;
for (const auto& output : node->outputs()) {
for (const auto& out_inputs : output.get_target_inputs()) {
if (out_inputs.get_element_type().is_real() &&
fp16_compression_is_disabled(out_inputs.get_node()->shared_from_this())) {
has_marked_output = true;
}
}
}
if (!has_marked_output)
return false;
auto convert_node = dynamic_pointer_cast<Convert>(node);
if (convert_node) {
// if during propagating up there is a Convert it must go to Const,
// otherwise interrupt propagation
auto const_node = dynamic_pointer_cast<Constant>(node->input_value(0).get_node_shared_ptr());
if (!const_node)
return false;
}
disable_fp16_compression(node);
return true;
};
auto m = make_shared<pattern::Matcher>(propagate_through_ops, matcher_name);
register_matcher(m, callback);
}
};
/* Starting from the marked precision sensitive nodes we need to propagate down to include neighboring
* ops like Slice, ReduceSum, Reshape, Elementwise, et al. to be kept in f32 as well.
* Propagation stops when ops not listed in propagate_through_ops are faced: e.g. if we face Conv or MatMul.
*/
class PropagateDownMarkToKeepInMixedPrecision : public pass::MatcherPass {
public:
OPENVINO_RTTI("PropagateDownMarkToKeepInMixedPrecision", "0");
PropagateDownMarkToKeepInMixedPrecision() {
MATCHER_SCOPE(PropagateDownMarkToKeepInMixedPrecision);
matcher_pass_callback callback = [=](pattern::Matcher& m) {
const auto& node = m.get_match_root();
if (!node)
return false;
// on convert down propagation should be interrupted
auto convert_node = dynamic_pointer_cast<Convert>(node);
if (convert_node)
return false;
bool is_changed = false;
for (const auto& in_node : node->input_values()) {
if (!in_node.get_element_type().is_real())
continue;
if (fp16_compression_is_disabled(in_node.get_node_shared_ptr())) {
disable_fp16_compression(node);
is_changed = true;
break;
}
}
return is_changed;
};
auto m = make_shared<pattern::Matcher>(propagate_through_ops, matcher_name);
register_matcher(m, callback);
}
};
void mark_reduceop_path(const std::shared_ptr<Node>& node) {
node->get_rt_info().emplace("reduceop_path", true);
}
bool is_reduceop_path(const std::shared_ptr<const Node>& node) {
return node->get_rt_info().count("reduceop_path");
}
void erase_reduceop_path(const std::shared_ptr<Node>& node) {
auto& rt_info = node->get_rt_info();
rt_info.erase("reduceop_path");
}
class InitMarkReduceOpPath : public pass::MatcherPass {
public:
OPENVINO_RTTI("InitMarkReduceOpPath", "0");
InitMarkReduceOpPath() {
MATCHER_SCOPE(InitMarkReduceOpPath);
auto reduce_ops = pattern::wrap_type<ReduceSum, ReduceMean>();
matcher_pass_callback callback = [=](pattern::Matcher& m) {
const auto& node = m.get_match_root();
if (!node)
return false;
mark_reduceop_path(node);
return true;
};
auto m = make_shared<pattern::Matcher>(reduce_ops, matcher_name);
register_matcher(m, callback);
}
};
class PropagateMarkUpReduceOpPath : public pass::MatcherPass {
public:
OPENVINO_RTTI("PropagateMarkUpReduceOpPath", "0");
PropagateMarkUpReduceOpPath() {
MATCHER_SCOPE(PropagateMarkUpReduceOpPath);
matcher_pass_callback callback = [=](pattern::Matcher& m) {
const auto& node = m.get_match_root();
if (!node)
return false;
for (const auto& output : node->outputs()) {
for (const auto& out_inputs : output.get_target_inputs()) {
if (out_inputs.get_element_type().is_real() &&
is_reduceop_path(out_inputs.get_node()->shared_from_this())) {
mark_reduceop_path(node);
return false;
}
}
}
return false;
};
auto m = make_shared<pattern::Matcher>(propagate_through_ops, matcher_name);
register_matcher(m, callback);
}
};
class MarkExp : public pass::MatcherPass {
public:
// only exponent that go into ReduceOp should be marked as precision sensitive and kept in f32
OPENVINO_RTTI("MarkExp", "0");
MarkExp() {
MATCHER_SCOPE(MarkExp);
auto exp_pattern = pattern::wrap_type<Exp>();
matcher_pass_callback callback = [=](pattern::Matcher& m) {
const auto& node = m.get_match_root();
if (!node)
return false;
if (!is_reduceop_path(node))
return false;
disable_fp16_compression(node);
return true;
};
auto m = make_shared<pattern::Matcher>(exp_pattern, matcher_name);
register_matcher(m, callback);
}
};
/* MarkExpInReduceOpPath marks path that goes into ReduceSum and ReduceMean.
* Values that go from Exp to ReduceSum/ReduceMean are precision
* sensitive and should be kept in f32 precision for mixed inference.
*/
class MarkExpInReduceOpPath : public BackwardGraphRewrite {
public:
OPENVINO_RTTI("MarkExpInReduceOpPath", "0");
MarkExpInReduceOpPath() {
// marking of ReduceOp path is needed to mark only Exponents that go into ReduceSum/ReduceMean
ADD_MATCHER_FOR_THIS(InitMarkReduceOpPath);
ADD_MATCHER_FOR_THIS(PropagateMarkUpReduceOpPath);
ADD_MATCHER_FOR_THIS(MarkExp);
}
};
/* MarkDivWithEps martk pattern that matches the patterns input_1/Maximum(input_2, eps); input_1/Add(input_2, eps);
* and input_1*Pow(Maximum[Add](input_2, eps), -z) and marks subgraph root to be kept in fp32.
*
* If both input_1 and input_2 simultaneously happen to be zero to prevent from NaNs and not to loose accuracy,
* we should calculate such patterns always in fp32 precision even if ov::Model is compressed to fp16.
*/
class MarkDivWithEps : public MatcherPass {
public:
OPENVINO_RTTI("MarkDivWithEps", "0");
MarkDivWithEps() {
MATCHER_SCOPE(MarkDivWithEps);
// to detect the following patterns where eps is used to prevent division by zero:
// input_1/Maximum(input_2, eps)
// input_1/Add(input_2, eps)
// input_1/Sqrt(Maximum(input_2, eps))
// input_1/Sqrt(Add(input_2, eps))
// input_1*Pow(Maximum(input_2, eps), -z)
// input_1*Pow(Add(input_2, eps), -z)
auto input_1 = pattern::any_input();
auto input_2 = pattern::any_input();
auto eps_const_pattern = pattern::wrap_type<Constant>();
auto max_or_add = pattern::wrap_type<Maximum, Add>(OutputVector{input_2, eps_const_pattern});
auto sqrt = std::make_shared<Sqrt>(max_or_add);
auto sqrt_or_max_add = std::make_shared<pattern::op::Or>(OutputVector{max_or_add, sqrt});
// whether is divided directly or after sqrt (e.g. in L2Norm after sqrt, in MVN is divided directly)
auto divide = std::make_shared<Divide>(input_1, sqrt_or_max_add);
auto pow_exp = pattern::wrap_type<Constant>();
auto convert_pattern = pattern::wrap_type<Convert>({pow_exp});
auto pow_exp_or_convert = std::make_shared<pattern::op::Or>(OutputVector{pow_exp, convert_pattern});
auto pow_pattern = std::make_shared<Power>(max_or_add, pow_exp_or_convert);
auto mul_pattern = std::make_shared<Multiply>(input_1, pow_pattern);
auto div_or_mul_to_negative_pow = std::make_shared<pattern::op::Or>(OutputVector{divide, mul_pattern});
matcher_pass_callback callback = [=](pattern::Matcher& m) {
const auto& pattern_to_output = m.get_pattern_map();
if (!m.get_match_root())
return false;
const auto mul = std::dynamic_pointer_cast<Multiply>(m.get_match_root());
// if pattern input_1*Pow(Maximum(input_2, eps), z) or input_1*Pow(Add(input_2, eps), z) is matched
// need to check that power is negative
if (mul) {
const auto pow_const = std::dynamic_pointer_cast<Constant>(pattern_to_output.at(pow_exp));
if (pow_const) {
// continue only if exponent is negative (z < 0)
if (pow_const->get_element_type() == element::f16) {
for (auto val : pow_const->get_vector<float16>())
if (val >= 0.0f)
return false;
} else if (pow_const->get_element_type() == element::f32) {
for (auto val : pow_const->get_vector<float>())
if (val >= 0.0f)
return false;
}
}
}
const auto eps_const = std::dynamic_pointer_cast<Constant>(pattern_to_output.at(eps_const_pattern));
if (!eps_const)
return false;
if (eps_const->get_element_type() == element::f32) {
for (const auto& val : eps_const->get_vector<float>())
if (val > static_cast<float>(float16_min_normalized))
return false;
} else if (eps_const->get_element_type() == element::f16) {
for (const auto& val : eps_const->get_vector<float16>())
if (val > float16_min_normalized)
return false;
}
disable_fp16_compression(m.get_match_root());
return true;
};
auto m = make_shared<pattern::Matcher>(div_or_mul_to_negative_pow, matcher_name);
register_matcher(m, callback);
}
};
bool MarkSugraphsToKeepInMixedPrecision::run_on_model(const shared_ptr<ov::Model>& m) {
RUN_ON_MODEL_SCOPE(MarkSugraphsToKeepInMixedPrecision);
Manager manager(get_pass_config());
// Mark root of Division with eps pattern to keep in FP32
REGISTER_PASS(manager, MarkDivWithEps)
REGISTER_PASS(manager, MarkExpInReduceOpPath)
REGISTER_PASS(manager, MarkNormalizationOps)
// both Up and Down propagations are needed.
// Why both of them are needed is explained in comments in passes declarations.
REGISTER_PASS(manager, PropagateDownMarkToKeepInMixedPrecision)
auto propagate_up = manager.register_pass<BackwardGraphRewrite>();
ADD_MATCHER(propagate_up, PropagateUpMarkToKeepInMixedPrecision)
// Mark nodes in ShapeOf subgraphs to keep in FP32
REGISTER_PASS(manager, MarkPrecisionSensitiveShapeOfSubgraphs)
manager.run_passes(m);
for (auto& node : m->get_ops()) {
erase_reduceop_path(node);
}
return false; // no need to revalidate
}
} // namespace pass
} // namespace ov
src/common/transformations/tests/common_optimizations/align_mixed_fp32_fp16_types_test.cpp
+304
View File
@@ -0,0 +1,304 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <gtest/gtest.h>
#include <openvino/core/model.hpp>
#include <openvino/opsets/opset10.hpp>
#include <openvino/pass/manager.hpp>
#include <transformations/common_optimizations/align_mixed_fp32_fp16_types.hpp>
#include <transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.hpp>
#include <transformations/convert_precision.hpp>
#include "common_test_utils/ngraph_test_utils.hpp"
using namespace testing;
using namespace ov;
using namespace std;
using namespace ov::opset10;
TEST_F(TransformationTestsF, align_mixed_fp16_fp32_1) {
{
auto input_1 = make_shared<Parameter>(element::f32, Shape{1, 3, 224, 224});
auto exp_1 = make_shared<Exp>(input_1);
auto input_2 = make_shared<Parameter>(element::f32, Shape{1, 3, 224, 224});
auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1});
auto reduce_sum_1 = make_shared<ReduceSum>(exp_1, reduction_axes);
auto factor_const = Constant::create(element::f16, Shape{1}, {-1});
auto factor_const_decompressed = make_shared<Convert>(factor_const, element::f32);
auto mul_1 = make_shared<Multiply>(reduce_sum_1, factor_const_decompressed);
auto matmul_1 = make_shared<MatMul>(mul_1, input_2);
model = make_shared<Model>(NodeVector{matmul_1}, ParameterVector{input_1, input_2});
pass::Manager manager;
manager.register_pass<pass::MarkSugraphsToKeepInMixedPrecision>();
manager.register_pass<pass::AlignMixedFP32FP16Types>();
const precisions_array convert_precision_list{{ov::element::f32, ov::element::f16}};
type_to_fuse_map empty_type_to_fuse_map = {};
manager.register_pass<pass::ConvertPrecision>(convert_precision_list, empty_type_to_fuse_map, true);
manager.run_passes(model);
}
{
auto input_1 = make_shared<Parameter>(element::f16, Shape{1, 3, 224, 224});
auto convert_to_f32_1 = make_shared<Convert>(input_1, element::f32);
auto exp_1 = make_shared<Exp>(convert_to_f32_1);
auto input_2 = make_shared<Parameter>(element::f16, Shape{1, 3, 224, 224});
auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1});
auto reduce_sum_1 = make_shared<ReduceSum>(exp_1, reduction_axes);
auto factor_const = Constant::create(element::f16, Shape{1}, {-1});
auto factor_const_decompressed = make_shared<Convert>(factor_const, element::f32);
auto mul_1 = make_shared<Multiply>(reduce_sum_1, factor_const_decompressed);
auto convert_to_f16_1 = make_shared<Convert>(mul_1, element::f16);
auto matmul_1 = make_shared<MatMul>(convert_to_f16_1, input_2);
model_ref = make_shared<Model>(NodeVector{matmul_1}, ParameterVector{input_1, input_2});
}
}
TEST_F(TransformationTestsF, align_mixed_fp16_fp32_2) {
{
auto input_1 = make_shared<Parameter>(element::f32, Shape{1, 3, 224, 224});
auto exp_1 = make_shared<Exp>(input_1);
auto input_2 = make_shared<Parameter>(element::f32, Shape{1, 3, 224, 224});
auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1});
auto unsqueeze_axes = Constant::create(element::i64, Shape{1}, {1});
auto unsqueeze_1 = make_shared<Unsqueeze>(exp_1, unsqueeze_axes);
auto reduce_sum_1 = make_shared<ReduceSum>(unsqueeze_1, reduction_axes);
auto factor_const = Constant::create(element::f16, Shape{1}, {-1});
auto factor_const_decompressed = make_shared<Convert>(factor_const, element::f32);
auto mul_1 = make_shared<Multiply>(reduce_sum_1, factor_const_decompressed);
auto matmul_1 = make_shared<MatMul>(mul_1, input_2);
model = make_shared<Model>(NodeVector{matmul_1}, ParameterVector{input_1, input_2});
pass::Manager manager;
manager.register_pass<pass::MarkSugraphsToKeepInMixedPrecision>();
manager.register_pass<pass::AlignMixedFP32FP16Types>();
const precisions_array convert_precision_list{{ov::element::f32, ov::element::f16}};
type_to_fuse_map empty_type_to_fuse_map = {};
manager.register_pass<pass::ConvertPrecision>(convert_precision_list, empty_type_to_fuse_map, true);
manager.run_passes(model);
}
{
auto input_1 = make_shared<Parameter>(element::f16, Shape{1, 3, 224, 224});
auto convert_to_f32_1 = make_shared<Convert>(input_1, element::f32);
auto exp_1 = make_shared<Exp>(convert_to_f32_1);
auto input_2 = make_shared<Parameter>(element::f16, Shape{1, 3, 224, 224});
auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1});
auto unsqueeze_axes = Constant::create(element::i64, Shape{1}, {1});
auto unsqueeze_1 = make_shared<Unsqueeze>(exp_1, unsqueeze_axes);
auto reduce_sum_1 = make_shared<ReduceSum>(unsqueeze_1, reduction_axes);
auto factor_const = Constant::create(element::f16, Shape{1}, {-1});
auto factor_const_decompressed = make_shared<Convert>(factor_const, element::f32);
auto mul_1 = make_shared<Multiply>(reduce_sum_1, factor_const_decompressed);
auto convert_to_f16_1 = make_shared<Convert>(mul_1, element::f16);
auto matmul_1 = make_shared<MatMul>(convert_to_f16_1, input_2);
model_ref = make_shared<Model>(NodeVector{matmul_1}, ParameterVector{input_1, input_2});
}
}
TEST_F(TransformationTestsF, align_mixed_fp16_fp32_3) {
{
auto input_1 = make_shared<Parameter>(element::f32, Shape{1, 3, 224, 224});
auto exp_1 = make_shared<Exp>(input_1);
auto input_2 = make_shared<Parameter>(element::f32, Shape{1, 3, 224, 224});
auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1});
auto reduce_sum_1 = make_shared<ReduceSum>(exp_1, reduction_axes);
auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f});
auto add_1 = make_shared<Add>(reduce_sum_1, addition_const);
auto factor_const = Constant::create(element::f16, Shape{1}, {-1});
auto factor_const_decompressed = make_shared<Convert>(factor_const, element::f32);
auto mul_1 = make_shared<Multiply>(add_1, factor_const_decompressed);
auto matmul_1 = make_shared<MatMul>(mul_1, input_2);
model = make_shared<Model>(NodeVector{matmul_1}, ParameterVector{input_1, input_2});
pass::Manager manager;
manager.register_pass<pass::MarkSugraphsToKeepInMixedPrecision>();
manager.register_pass<pass::AlignMixedFP32FP16Types>();
const precisions_array convert_precision_list{{ov::element::f32, ov::element::f16}};
type_to_fuse_map empty_type_to_fuse_map = {};
manager.register_pass<pass::ConvertPrecision>(convert_precision_list, empty_type_to_fuse_map, true);
manager.run_passes(model);
}
{
auto input_1 = make_shared<Parameter>(element::f16, Shape{1, 3, 224, 224});
auto convert_to_f32_1 = make_shared<Convert>(input_1, element::f32);
auto exp_1 = make_shared<Exp>(convert_to_f32_1);
auto input_2 = make_shared<Parameter>(element::f16, Shape{1, 3, 224, 224});
auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1});
auto reduce_sum_1 = make_shared<ReduceSum>(exp_1, reduction_axes);
auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f});
auto add_1 = make_shared<Add>(reduce_sum_1, addition_const);
auto factor_const = Constant::create(element::f16, Shape{1}, {-1});
auto factor_const_decompressed = make_shared<Convert>(factor_const, element::f32);
auto mul_1 = make_shared<Multiply>(add_1, factor_const_decompressed);
auto convert_to_f16_1 = make_shared<Convert>(mul_1, element::f16);
auto matmul_1 = make_shared<MatMul>(convert_to_f16_1, input_2);
model_ref = make_shared<Model>(NodeVector{matmul_1}, ParameterVector{input_1, input_2});
}
}
TEST_F(TransformationTestsF, align_mixed_fp16_fp32_4) {
{
auto input_1 = make_shared<Parameter>(element::f32, Shape{1, 3, 224, 224});
auto input_2 = make_shared<Parameter>(element::f32, Shape{1, 3, 224, 224});
auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1});
auto mvn_1 = make_shared<MVN>(input_1, reduction_axes, true, 1.0e-8f, op::MVNEpsMode::INSIDE_SQRT);
auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f});
auto add_1 = make_shared<Add>(mvn_1, addition_const);
auto matmul_1 = make_shared<MatMul>(add_1, input_2);
model = make_shared<Model>(NodeVector{matmul_1}, ParameterVector{input_1, input_2});
pass::Manager manager;
manager.register_pass<pass::MarkSugraphsToKeepInMixedPrecision>();
manager.register_pass<pass::AlignMixedFP32FP16Types>();
const precisions_array convert_precision_list{{ov::element::f32, ov::element::f16}};
type_to_fuse_map empty_type_to_fuse_map = {};
manager.register_pass<pass::ConvertPrecision>(convert_precision_list, empty_type_to_fuse_map, true);
manager.run_passes(model);
}
{
auto input_1 = make_shared<Parameter>(element::f16, Shape{1, 3, 224, 224});
auto input_2 = make_shared<Parameter>(element::f16, Shape{1, 3, 224, 224});
auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1});
auto convert_to_f32_1 = make_shared<Convert>(input_1, element::f32);
auto mvn_1 = make_shared<MVN>(convert_to_f32_1, reduction_axes, true, 1.0e-8f, op::MVNEpsMode::INSIDE_SQRT);
auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f});
auto add_1 = make_shared<Add>(mvn_1, addition_const);
auto convert_to_f16_1 = make_shared<Convert>(add_1, element::f16);
auto matmul_1 = make_shared<MatMul>(convert_to_f16_1, input_2);
model_ref = make_shared<Model>(NodeVector{matmul_1}, ParameterVector{input_1, input_2});
}
}
TEST_F(TransformationTestsF, align_mixed_fp16_fp32_mnv_with_split) {
{
auto input_1 = make_shared<Parameter>(element::f32, Shape{1, 3, 224, 224});
auto input_2 = make_shared<Parameter>(element::f32, Shape{1, 3, 56, 224});
auto split_axis = Constant::create(element::i64, Shape{}, {3});
auto split = make_shared<Split>(input_1, split_axis, 4);
auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1});
auto mvn_1 = make_shared<MVN>(split->output(0), reduction_axes, true, 1.0e-8f, op::MVNEpsMode::INSIDE_SQRT);
auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f});
auto add_1 = make_shared<Add>(mvn_1, addition_const);
auto matmul_1 = make_shared<MatMul>(add_1, input_2);
auto result_1 = make_shared<Result>(matmul_1);
auto result_2 = make_shared<Result>(split->output(1));
auto result_3 = make_shared<Result>(split->output(2));
model = make_shared<Model>(OutputVector{result_1, result_2, result_3}, ParameterVector{input_1, input_2});
pass::Manager manager;
manager.register_pass<pass::MarkSugraphsToKeepInMixedPrecision>();
manager.register_pass<pass::AlignMixedFP32FP16Types>();
const precisions_array convert_precision_list{{ov::element::f32, ov::element::f16}};
type_to_fuse_map empty_type_to_fuse_map = {};
manager.register_pass<pass::ConvertPrecision>(convert_precision_list, empty_type_to_fuse_map, true);
manager.run_passes(model);
}
{
auto input_1 = make_shared<Parameter>(element::f16, Shape{1, 3, 224, 224});
auto input_2 = make_shared<Parameter>(element::f16, Shape{1, 3, 56, 224});
auto convert_to_f32_1 = make_shared<Convert>(input_1, element::f32);
auto split_axis = Constant::create(element::i64, Shape{}, {3});
auto split = make_shared<Split>(convert_to_f32_1, split_axis, 4);
auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1});
auto mvn_1 = make_shared<MVN>(split->output(0), reduction_axes, true, 1.0e-8f, op::MVNEpsMode::INSIDE_SQRT);
auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f});
auto add_1 = make_shared<Add>(mvn_1, addition_const);
auto convert_to_f16_1 = make_shared<Convert>(add_1, element::f16);
auto matmul_1 = make_shared<MatMul>(convert_to_f16_1, input_2);
// todo: without Converts to fp16 because of GPU
auto result_1 = make_shared<Result>(matmul_1);
auto result_2 = make_shared<Result>(split->output(1));
auto result_3 = make_shared<Result>(split->output(2));
model_ref = make_shared<Model>(OutputVector{result_1, result_2, result_3}, ParameterVector{input_1, input_2});
}
}
TEST_F(TransformationTestsF, align_mixed_fp16_fp32_with_rand_uniform) {
{
auto input_1 = make_shared<Parameter>(element::f32, Shape{1, 3, 224, 224});
auto exp_1 = make_shared<Exp>(input_1);
auto input_2 = make_shared<Parameter>(element::f32, Shape{1, 3, 224, 224});
auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1});
auto reduce_sum_1 = make_shared<ReduceSum>(exp_1, reduction_axes);
auto factor_const = Constant::create(element::f16, Shape{1}, {-1});
auto factor_const_decompressed = make_shared<Convert>(factor_const, element::f32);
auto out_shape = Constant::create(element::i64, Shape{3}, {1, 3, 224});
auto minval = Constant::create(element::f32, Shape{}, {1});
auto maxval = Constant::create(element::f32, Shape{}, {10});
auto rand_uniform = make_shared<RandomUniform>(out_shape, minval, maxval, element::f32);
auto rand_uniform_add_factor = make_shared<Add>(rand_uniform, factor_const_decompressed);
auto mul_1 = make_shared<Multiply>(reduce_sum_1, rand_uniform_add_factor);
auto matmul_1 = make_shared<MatMul>(mul_1, input_2);
model = make_shared<Model>(NodeVector{matmul_1}, ParameterVector{input_1, input_2});
pass::Manager manager;
manager.register_pass<pass::MarkSugraphsToKeepInMixedPrecision>();
manager.register_pass<pass::AlignMixedFP32FP16Types>();
const precisions_array convert_precision_list{{ov::element::f32, ov::element::f16}};
type_to_fuse_map empty_type_to_fuse_map = {};
manager.register_pass<pass::ConvertPrecision>(convert_precision_list, empty_type_to_fuse_map, true);
manager.run_passes(model);
}
{
auto input_1 = make_shared<Parameter>(element::f16, Shape{1, 3, 224, 224});
auto convert_to_f32_1 = make_shared<Convert>(input_1, element::f32);
auto exp_1 = make_shared<Exp>(convert_to_f32_1);
auto input_2 = make_shared<Parameter>(element::f16, Shape{1, 3, 224, 224});
auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1});
auto reduce_sum_1 = make_shared<ReduceSum>(exp_1, reduction_axes);
auto factor_const = Constant::create(element::f16, Shape{1}, {-1});
auto factor_const_decompressed = make_shared<Convert>(factor_const, element::f32);
auto out_shape = Constant::create(element::i64, Shape{3}, {1, 3, 224});
auto minval = Constant::create(element::f16, Shape{}, {1});
auto maxval = Constant::create(element::f16, Shape{}, {10});
auto rand_uniform = make_shared<RandomUniform>(out_shape, minval, maxval, element::f16);
auto rand_uniform_decompressed = make_shared<Convert>(rand_uniform, element::f32);
auto rand_uniform_add_factor = make_shared<Add>(rand_uniform_decompressed, factor_const_decompressed);
auto mul_1 = make_shared<Multiply>(reduce_sum_1, rand_uniform_add_factor);
auto convert_to_f16_1 = make_shared<Convert>(mul_1, element::f16);
auto matmul_1 = make_shared<MatMul>(convert_to_f16_1, input_2);
model_ref = make_shared<Model>(NodeVector{matmul_1}, ParameterVector{input_1, input_2});
}
}
src/common/transformations/tests/common_optimizations/compress_float_constants_test.cpp
+196
View File
@@ -271,3 +271,199 @@ TEST_F(TransformationTestsF, CompressConstants_f64) {
function_ref = std::make_shared<ov::Model>(ov::NodeVector{conv}, ov::ParameterVector{input});
}
}
TEST_F(TransformationTestsF, CompressConstants_keep_in_f32_small_eps_out_of_range) {
float fp16_eps = static_cast<float>(ov::float16::from_bits(0x0400));
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f32, ov::Shape{1, 3, 12, 12});
auto const_weights = ov::opset8::Constant::create(ov::element::f32,
ov::Shape{1, 3, 4, 1},
{0.0f,
1.0f,
2.0f,
fp16_eps,
fp16_eps,
fp16_eps,
fp16_eps,
fp16_eps,
fp16_eps,
fp16_eps,
fp16_eps,
fp16_eps});
auto conv = std::make_shared<ov::opset8::Convolution>(input,
const_weights,
ov::Strides{1, 1},
ov::CoordinateDiff{0, 0},
ov::CoordinateDiff{0, 0},
ov::Strides{1, 1});
function = std::make_shared<ov::Model>(ov::NodeVector{conv}, ov::ParameterVector{input});
manager.register_pass<ov::pass::MarkPrecisionSensitiveConstants>();
manager.register_pass<ov::pass::CompressFloatConstants>();
}
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f32, ov::Shape{1, 3, 12, 12});
auto const_weights = ov::opset8::Constant::create(ov::element::f32,
ov::Shape{1, 3, 4, 1},
{0.0f,
1.0f,
2.0f,
fp16_eps,
fp16_eps,
fp16_eps,
fp16_eps,
fp16_eps,
fp16_eps,
fp16_eps,
fp16_eps,
fp16_eps});
auto conv = std::make_shared<ov::opset8::Convolution>(input,
const_weights,
ov::Strides{1, 1},
ov::CoordinateDiff{0, 0},
ov::CoordinateDiff{0, 0},
ov::Strides{1, 1});
function_ref = std::make_shared<ov::Model>(ov::NodeVector{conv}, ov::ParameterVector{input});
}
}
TEST_F(TransformationTestsF, CompressConstants_keep_in_f32_max_out_of_range_val) {
// if fp16 out of range values fraction is greater than threshold (75%) then keep them in fp32
// no decompression converts should be inserted
float fp16_oor = static_cast<float>(std::numeric_limits<ov::float16>::max()) + 100.0f;
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f32, ov::Shape{1, 3, 12, 12});
auto const_weights = ov::opset8::Constant::create(ov::element::f32,
ov::Shape{1, 3, 4, 1},
{0.0f,
1.0f,
2.0f,
fp16_oor,
fp16_oor,
fp16_oor,
fp16_oor,
fp16_oor,
fp16_oor,
fp16_oor,
fp16_oor,
fp16_oor});
auto conv = std::make_shared<ov::opset8::Convolution>(input,
const_weights,
ov::Strides{1, 1},
ov::CoordinateDiff{0, 0},
ov::CoordinateDiff{0, 0},
ov::Strides{1, 1});
function = std::make_shared<ov::Model>(ov::NodeVector{conv}, ov::ParameterVector{input});
manager.register_pass<ov::pass::MarkPrecisionSensitiveConstants>();
manager.register_pass<ov::pass::CompressFloatConstants>();
}
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f32, ov::Shape{1, 3, 12, 12});
auto const_weights = ov::opset8::Constant::create(ov::element::f32,
ov::Shape{1, 3, 4, 1},
{0.0f,
1.0f,
2.0f,
fp16_oor,
fp16_oor,
fp16_oor,
fp16_oor,
fp16_oor,
fp16_oor,
fp16_oor,
fp16_oor,
fp16_oor});
auto conv = std::make_shared<ov::opset8::Convolution>(input,
const_weights,
ov::Strides{1, 1},
ov::CoordinateDiff{0, 0},
ov::CoordinateDiff{0, 0},
ov::Strides{1, 1});
function_ref = std::make_shared<ov::Model>(ov::NodeVector{conv}, ov::ParameterVector{input});
}
}
TEST_F(TransformationTestsF, CompressConstants_compress_to_f16_max_out_of_range_val) {
// fp16 out of range should be clipped to fp16_max_val if fraction of out of range values is less than threshold
float fp16_oor = static_cast<float>(std::numeric_limits<ov::float16>::max()) + 100.0f;
float fp16_max = static_cast<float>(std::numeric_limits<ov::float16>::max());
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f32, ov::Shape{1, 3, 12, 12});
// only half of values are out of range, therefore they will be compressed to fp16
auto const_weights = ov::opset8::Constant::create(
ov::element::f32,
ov::Shape{1, 3, 4, 1},
{0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, fp16_oor, fp16_oor, fp16_oor, fp16_oor, fp16_oor, fp16_oor});
auto conv = std::make_shared<ov::opset8::Convolution>(input,
const_weights,
ov::Strides{1, 1},
ov::CoordinateDiff{0, 0},
ov::CoordinateDiff{0, 0},
ov::Strides{1, 1});
function = std::make_shared<ov::Model>(ov::NodeVector{conv}, ov::ParameterVector{input});
manager.register_pass<ov::pass::MarkPrecisionSensitiveConstants>();
manager.register_pass<ov::pass::CompressFloatConstants>();
}
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f32, ov::Shape{1, 3, 12, 12});
auto const_weights = ov::opset8::Constant::create(
ov::element::f16,
ov::Shape{1, 3, 4, 1},
{0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, fp16_max, fp16_max, fp16_max, fp16_max, fp16_max, fp16_max});
auto convert_ins1 = std::make_shared<ov::opset8::Convert>(const_weights, ov::element::f32);
auto conv = std::make_shared<ov::opset8::Convolution>(input,
convert_ins1,
ov::Strides{1, 1},
ov::CoordinateDiff{0, 0},
ov::CoordinateDiff{0, 0},
ov::Strides{1, 1});
function_ref = std::make_shared<ov::Model>(ov::NodeVector{conv}, ov::ParameterVector{input});
}
}
TEST_F(TransformationTestsF, CompressConstants_not_keep_in_f32_when_zeros) {
// zero values are less than fp16_eps, but they are exactly expressed in fp16
// not need to keep them in fp32
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f32, ov::Shape{1, 3, 12, 12});
auto const_weights =
ov::opset8::Constant::create(ov::element::f32,
ov::Shape{1, 3, 4, 1},
{0.0f, 1.0f, 2.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f});
auto conv = std::make_shared<ov::opset8::Convolution>(input,
const_weights,
ov::Strides{1, 1},
ov::CoordinateDiff{0, 0},
ov::CoordinateDiff{0, 0},
ov::Strides{1, 1});
function = std::make_shared<ov::Model>(ov::NodeVector{conv}, ov::ParameterVector{input});
manager.register_pass<ov::pass::MarkPrecisionSensitiveConstants>();
manager.register_pass<ov::pass::CompressFloatConstants>();
}
{
auto input = std::make_shared<ov::opset8::Parameter>(ov::element::f32, ov::Shape{1, 3, 12, 12});
auto const_weights =
ov::opset8::Constant::create(ov::element::f16,
ov::Shape{1, 3, 4, 1},
{0.0f, 1.0f, 2.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f});
auto convert_ins1 = std::make_shared<ov::opset8::Convert>(const_weights, ov::element::f32);
auto conv = std::make_shared<ov::opset8::Convolution>(input,
convert_ins1,
ov::Strides{1, 1},
ov::CoordinateDiff{0, 0},
ov::CoordinateDiff{0, 0},
ov::Strides{1, 1});
function_ref = std::make_shared<ov::Model>(ov::NodeVector{conv}, ov::ParameterVector{input});
}
}
src/common/transformations/tests/common_optimizations/division_by_zero_fp16_resolver_test.cpp
-207
View File
@@ -1,207 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <gtest/gtest.h>
#include <memory>
#include <ngraph/function.hpp>
#include <openvino/opsets/opset4.hpp>
#include <openvino/pass/manager.hpp>
#include <string>
#include <transformations/common_optimizations/division_by_zero_fp16_resolver.hpp>
#include <transformations/init_node_info.hpp>
#include "common_test_utils/ngraph_test_utils.hpp"
using namespace testing;
using namespace ov;
constexpr float normalized_fp16_min = 6.103515625e-05f; // fp16 minimal normalized value
TEST_F(TransformationTestsF, DivisionByZeroMinimalPattern) {
const float eps_value = 1.e-12f;
{
auto input_1 = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto input_2 = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {eps_value});
auto add = std::make_shared<opset4::Add>(input_2, eps_const);
auto divide = std::make_shared<opset4::Divide>(input_1, add);
function = std::make_shared<Model>(NodeVector{divide}, ParameterVector{input_1, input_2});
manager.register_pass<ov::pass::DivisionByZeroFP16Resolver>();
}
{
auto input_1 = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto input_2 = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {normalized_fp16_min});
auto add = std::make_shared<opset4::Add>(input_2, eps_const);
auto divide = std::make_shared<opset4::Divide>(input_1, add);
function_ref = std::make_shared<Model>(NodeVector{divide}, ParameterVector{input_1, input_2});
}
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES);
}
TEST_F(TransformationTestsF, PowWithNegativeExponent) {
const float eps_value = 1.e-12f;
{
auto input_1 = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto input_2 = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {eps_value});
auto add = std::make_shared<opset4::Add>(input_2, eps_const);
auto pow_exp_const = opset4::Constant::create(element::f32, Shape{1}, {-1.77});
auto pow = std::make_shared<opset4::Power>(add, pow_exp_const);
auto mul = std::make_shared<opset4::Multiply>(input_1, pow);
function = std::make_shared<Model>(NodeVector{mul}, ParameterVector{input_1, input_2});
manager.register_pass<ov::pass::DivisionByZeroFP16Resolver>();
}
{
auto input_1 = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto input_2 = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {normalized_fp16_min});
auto add = std::make_shared<opset4::Add>(input_2, eps_const);
auto pow_exp_const = opset4::Constant::create(element::f32, Shape{1}, {-1.77});
auto pow = std::make_shared<opset4::Power>(add, pow_exp_const);
auto mul = std::make_shared<opset4::Multiply>(input_1, pow);
function_ref = std::make_shared<Model>(NodeVector{mul}, ParameterVector{input_1, input_2});
}
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES);
}
TEST_F(TransformationTestsF, PowWithPositiveExponent) {
// graph should be left unchanged
const float eps_value = 1.e-12f;
{
auto input_1 = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto input_2 = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {eps_value});
auto add = std::make_shared<opset4::Add>(input_2, eps_const);
auto pow_exp_const = opset4::Constant::create(element::f32, Shape{1}, {1.77});
auto pow = std::make_shared<opset4::Power>(add, pow_exp_const);
auto mul = std::make_shared<opset4::Multiply>(input_1, pow);
function = std::make_shared<Model>(NodeVector{mul}, ParameterVector{input_1, input_2});
manager.register_pass<ov::pass::DivisionByZeroFP16Resolver>();
}
{
auto input_1 = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto input_2 = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {eps_value});
auto add = std::make_shared<opset4::Add>(input_2, eps_const);
auto pow_exp_const = opset4::Constant::create(element::f32, Shape{1}, {1.77});
auto pow = std::make_shared<opset4::Power>(add, pow_exp_const);
auto mul = std::make_shared<opset4::Multiply>(input_1, pow);
function_ref = std::make_shared<Model>(NodeVector{mul}, ParameterVector{input_1, input_2});
}
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES);
}
TEST_F(TransformationTestsF, DivisionByZeroMinimalPatternUnchanged) {
// if eps_value is greater than normalized_fp16_min then leave graph unchanged
const float eps_value = 0.0001f;
{
auto input_1 = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto input_2 = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {eps_value});
auto add = std::make_shared<opset4::Add>(input_2, eps_const);
auto divide = std::make_shared<opset4::Divide>(input_1, add);
function = std::make_shared<Model>(NodeVector{divide}, ParameterVector{input_1, input_2});
manager.register_pass<ov::pass::DivisionByZeroFP16Resolver>();
}
{
auto input_1 = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto input_2 = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {eps_value});
auto add = std::make_shared<opset4::Add>(input_2, eps_const);
auto divide = std::make_shared<opset4::Divide>(input_1, add);
function_ref = std::make_shared<Model>(NodeVector{divide}, ParameterVector{input_1, input_2});
}
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES);
}
TEST_F(TransformationTestsF, DivisionByZeroInL2NormWithSqrtAndWithMax) {
const float eps_value = 1.e-12f;
{
auto input = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto exp = opset4::Constant::create(element::f32, Shape{}, {2.f});
auto pow = std::make_shared<opset4::Power>(input, exp);
auto axes_const = opset4::Constant::create(element::i64, Shape{2}, {0, 1});
auto reduce_sum = std::make_shared<opset4::ReduceSum>(pow, axes_const);
auto eps_const = opset4::Constant::create(element::f32, Shape{}, {eps_value});
auto max = std::make_shared<opset4::Maximum>(reduce_sum, eps_const);
auto sqrt = std::make_shared<opset4::Sqrt>(max);
auto divide = std::make_shared<opset4::Divide>(input, sqrt);
function = std::make_shared<Model>(NodeVector{divide}, ParameterVector{input});
manager.register_pass<ov::pass::DivisionByZeroFP16Resolver>();
}
{
auto input = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto exp = opset4::Constant::create(element::f32, Shape{}, {2.f});
auto pow = std::make_shared<opset4::Power>(input, exp);
auto axes_const = opset4::Constant::create(element::i64, Shape{2}, {0, 1});
auto reduce_sum = std::make_shared<opset4::ReduceSum>(pow, axes_const);
auto eps_const = opset4::Constant::create(element::f32, Shape{}, {normalized_fp16_min});
auto max = std::make_shared<opset4::Maximum>(reduce_sum, eps_const);
auto sqrt = std::make_shared<opset4::Sqrt>(max);
auto divide = std::make_shared<opset4::Divide>(input, sqrt);
function_ref = std::make_shared<Model>(NodeVector{divide}, ParameterVector{input});
}
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES);
}
TEST_F(TransformationTestsF, DivisionByZeroInL2NormWithSqrtAndWithAdd) {
const float eps_value = 1.e-12f;
{
auto input = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto exp = opset4::Constant::create(element::f32, Shape{}, {2.f});
auto pow = std::make_shared<opset4::Power>(input, exp);
auto axes_const = opset4::Constant::create(element::i64, Shape{2}, {0, 1});
auto reduce_sum = std::make_shared<opset4::ReduceSum>(pow, axes_const);
auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {eps_value});
auto add = std::make_shared<opset4::Add>(reduce_sum, eps_const);
auto sqrt = std::make_shared<opset4::Sqrt>(add);
auto divide = std::make_shared<opset4::Divide>(input, sqrt);
function = std::make_shared<Model>(NodeVector{divide}, ParameterVector{input});
manager.register_pass<ov::pass::DivisionByZeroFP16Resolver>();
}
{
auto input = std::make_shared<opset4::Parameter>(element::f32, PartialShape::dynamic(3));
auto exp = opset4::Constant::create(element::f32, Shape{}, {2.f});
auto pow = std::make_shared<opset4::Power>(input, exp);
auto axes_const = opset4::Constant::create(element::i64, Shape{2}, {0, 1});
auto reduce_sum = std::make_shared<opset4::ReduceSum>(pow, axes_const);
auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {normalized_fp16_min});
auto add = std::make_shared<opset4::Add>(reduce_sum, eps_const);
auto sqrt = std::make_shared<opset4::Sqrt>(add);
auto divide = std::make_shared<opset4::Divide>(input, sqrt);
function_ref = std::make_shared<Model>(NodeVector{divide}, ParameterVector{input});
}
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES);
}
src/common/transformations/tests/common_optimizations/mark_subgraph_to_keep_in_mixed_precision_test.cpp
+1418
View File
File diff suppressed because it is too large. Load diff