From 7b6eefbbf8a431439a3f30866cef59e7d4974983 Mon Sep 17 00:00:00 2001 From: Pavel Esir Date: Mon, 30 Jan 2023 07:04:12 +0100 Subject: [PATCH] [nG] [IE] add support of mixed FP16/FP32 precision ov::Model for mixed inference (#13249) * add pattern mathing for MVN, Exp->ReduceSum, L2Normalize, Div with eps for mixed precision inference * added necessary includes * clang_format_fix_all * fix warning_as_error for unused variable * fix warning_as_error for specifying float literals * enable marking for fp32 IRs as well * cosmetic improvements in unit-tests * fix warnings as error * added unit-tests for compress_float_constants.cpp for out of range values * Update align_mixed_fp32_fp16_types.cpp * Apply suggestions from code review Co-authored-by: Maxim Vafin * some grooming: mainly in imports * build fix: replaced ngraph:: -> ov:: * collected all markings in a single file * shortened pass names * style fix * made MarkNormalizationOps as a separate pass * removed redundant comment, fixed description of MarkSugraphsToKeepInMixedPrecision pass * comments on Up and Down marking in MarkSugraphsToKeepInMixedPrecision * cleared info messages in compress_float_constants.cpp, removed threshold adjusting from ENV * moved declarations of MarkNormalizationOps, MarkExpInReduceOpPath, MarkDivWithEps to hide them from outside users * simplified pattern matching for max_or_add * moved `reduceop_path` rt_info inside mark_subgraphs_to_keep_in_mixed_precision.cpp * fix potential bug with Convert * removed redundant check for Converts in `insert_converts_after_if_needed` as well * set Convert types more safely * corrections in opset10 namespaces; some minor corrections --------- Co-authored-by: Maxim Vafin --- .../align_mixed_fp32_fp16_types.hpp | 27 + .../division_by_zero_fp16_resolver.hpp | 34 - ...k_subgraphs_to_keep_in_mixed_precision.hpp | 31 + .../align_mixed_fp32_fp16_types.cpp | 80 + .../compress_float_constants.cpp | 31 +- .../convert_compressed_to_mixed_precision.cpp | 9 +- .../division_by_zero_fp16_resolver.cpp | 79 - ...k_subgraphs_to_keep_in_mixed_precision.cpp | 362 +++++ .../align_mixed_fp32_fp16_types_test.cpp | 304 ++++ .../compress_float_constants_test.cpp | 196 +++ .../division_by_zero_fp16_resolver_test.cpp | 207 --- ...bgraph_to_keep_in_mixed_precision_test.cpp | 1418 +++++++++++++++++ 12 files changed, 2444 insertions(+), 334 deletions(-) create mode 100644 src/common/transformations/include/transformations/common_optimizations/align_mixed_fp32_fp16_types.hpp create mode 100644 src/common/transformations/include/transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.hpp create mode 100644 src/common/transformations/src/transformations/common_optimizations/align_mixed_fp32_fp16_types.cpp delete mode 100644 src/common/transformations/src/transformations/common_optimizations/division_by_zero_fp16_resolver.cpp create mode 100644 src/common/transformations/src/transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.cpp create mode 100644 src/common/transformations/tests/common_optimizations/align_mixed_fp32_fp16_types_test.cpp create mode 100644 src/common/transformations/tests/common_optimizations/mark_subgraph_to_keep_in_mixed_precision_test.cpp diff --git a/src/common/transformations/include/transformations/common_optimizations/align_mixed_fp32_fp16_types.hpp b/src/common/transformations/include/transformations/common_optimizations/align_mixed_fp32_fp16_types.hpp new file mode 100644 index 00000000000..016bb852066 --- /dev/null +++ b/src/common/transformations/include/transformations/common_optimizations/align_mixed_fp32_fp16_types.hpp @@ -0,0 +1,27 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/pass/graph_rewrite.hpp" +#include "openvino/pass/pass.hpp" +#include "transformations_visibility.hpp" + +namespace ov { +namespace pass { + +class TRANSFORMATIONS_API AlignMixedFP32FP16Types; + +} // namespace pass +} // namespace ov + +/** + * @ingroup ie_transformation_common_api + * @brief AlignMixedFP32FP16Types adds Converts to keep mixed FP16/FP32 graph type consistent + */ +class ov::pass::AlignMixedFP32FP16Types : public ov::pass::ModelPass { +public: + OPENVINO_RTTI("AlignMixedFP32FP16Types", "0"); + bool run_on_model(const std::shared_ptr& m) override; +}; diff --git a/src/common/transformations/include/transformations/common_optimizations/division_by_zero_fp16_resolver.hpp b/src/common/transformations/include/transformations/common_optimizations/division_by_zero_fp16_resolver.hpp index 07ac88e7f40..e69de29bb2d 100644 --- a/src/common/transformations/include/transformations/common_optimizations/division_by_zero_fp16_resolver.hpp +++ b/src/common/transformations/include/transformations/common_optimizations/division_by_zero_fp16_resolver.hpp @@ -1,34 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include -#include -#include - -namespace ov { -namespace pass { - -class TRANSFORMATIONS_API DivisionByZeroFP16Resolver; - -} // namespace pass -} // namespace ov - -/** - * @ingroup ie_transformation_common_api - * @brief: clamps eps into fp16 minimal normalized value in input_1/Maximum(input_2, eps); input_1/Add(input_2, eps); - * and input_1*Pow(Maximum[Add](input_2, eps), -z) patterns to prevent division by zero. - * - * eps must be always nonzero to prevent from NaNs in such expressions if input_1 and input_2 simultaneously happened to - * be zero. We should keep in such patterns eps >= fp16 minimal normalized value so that CompressFloatConstants should - * not cast them into zero during compression into f16. - */ -class ov::pass::DivisionByZeroFP16Resolver : public ov::pass::MatcherPass { -public: - OPENVINO_RTTI("DivisionByZeroFP16Resolver", "0"); - DivisionByZeroFP16Resolver(); -}; diff --git a/src/common/transformations/include/transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.hpp b/src/common/transformations/include/transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.hpp new file mode 100644 index 00000000000..47dad40bdf9 --- /dev/null +++ b/src/common/transformations/include/transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.hpp @@ -0,0 +1,31 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/pass/graph_rewrite.hpp" +#include "openvino/pass/pass.hpp" +#include "transformations_visibility.hpp" + +namespace ov { +namespace pass { + +class TRANSFORMATIONS_API MarkSugraphsToKeepInMixedPrecision; + +constexpr auto float16_min_normalized = float16::from_bits(0x0400); + +} // namespace pass +} // namespace ov + +/* + * @ingroup ie_transformation_common_api + * @brief: MarkSugraphsToKeepInMixedPrecision container for marking passes which marks subgraphs + * to be kept in f32 for mixed precision inference. Includes passes for the following patterns: + * L2Normalize, MVN, ShapeOf subgraphs, Exp in ReduceOp paths and Division with small eps values. + */ +class ov::pass::MarkSugraphsToKeepInMixedPrecision : public ov::pass::ModelPass { +public: + OPENVINO_RTTI("MarkSugraphsToKeepInMixedPrecision", "0"); + bool run_on_model(const std::shared_ptr& m) override; +}; diff --git a/src/common/transformations/src/transformations/common_optimizations/align_mixed_fp32_fp16_types.cpp b/src/common/transformations/src/transformations/common_optimizations/align_mixed_fp32_fp16_types.cpp new file mode 100644 index 00000000000..11d039047a2 --- /dev/null +++ b/src/common/transformations/src/transformations/common_optimizations/align_mixed_fp32_fp16_types.cpp @@ -0,0 +1,80 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/common_optimizations/align_mixed_fp32_fp16_types.hpp" + +#include "itt.hpp" +#include "openvino/core/rt_info.hpp" +#include "openvino/op/util/precision_sensitive_attribute.hpp" +#include "openvino/opsets/opset10.hpp" +#include "transformations/convert_precision.hpp" +#include "transformations/rt_info/decompression.hpp" +#include "transformations/rt_info/disable_fp16_compression.hpp" + +using namespace ov; + +bool ov::pass::AlignMixedFP32FP16Types::run_on_model(const std::shared_ptr& model) { + RUN_ON_MODEL_SCOPE(AlignMixedFP32FP16Types); + + std::function&)> insert_converts_before_if_needed = + [&](const std::shared_ptr& node) { + bool is_changed = false; + for (const auto& input : node->inputs()) { + const auto& incoming_output = input.get_source_output(); + const auto& incoming_node = incoming_output.get_node_shared_ptr(); + + if (fp16_compression_is_disabled(incoming_node)) + continue; // we are in the middle + + if (!incoming_output.get_element_type().is_real()) + continue; + + auto convert = std::make_shared(incoming_output, incoming_output.get_element_type()); + convert->set_friendly_name(incoming_node->get_friendly_name() + "_decompressed_to_f32"); + copy_runtime_info(incoming_node, convert); + input.replace_source_output(convert); + disable_fp16_compression(convert); + is_changed = true; + } + return is_changed; + }; + + std::function&)> insert_converts_after_if_needed = + [&](const std::shared_ptr& node) { + bool is_changed = false; + for (const auto& output : node->outputs()) { + for (const auto& out_inputs : output.get_target_inputs()) { + auto out_node = out_inputs.get_node()->shared_from_this(); + if (fp16_compression_is_disabled(out_node) || is_precision_sensitive(out_inputs)) + continue; + if (!out_inputs.get_element_type().is_real()) + continue; + + // todo xxx-101766: if we don't skip Results there is an error on GPU + if (std::dynamic_pointer_cast(out_node)) + continue; + + // element_type of this convert will be changed automatically to f16 after + // ConvertPrecision(f32 -> f16). It's kept here f32 to keep ov::Model validatable + auto convert = std::make_shared(output, out_inputs.get_element_type()); + copy_runtime_info(node, convert); + convert->set_friendly_name(node->get_friendly_name() + "_compressed_to_f16"); + out_inputs.replace_source_output(convert); + is_changed = true; + } + } + return is_changed; + }; + + bool is_changed = false; + for (auto& node : model->get_ordered_ops()) { + if (!fp16_compression_is_disabled(node)) + continue; + + is_changed |= insert_converts_before_if_needed(node); + is_changed |= insert_converts_after_if_needed(node); + } + + return true; +} diff --git a/src/common/transformations/src/transformations/common_optimizations/compress_float_constants.cpp b/src/common/transformations/src/transformations/common_optimizations/compress_float_constants.cpp index 3efe23ec550..3c54a15347c 100644 --- a/src/common/transformations/src/transformations/common_optimizations/compress_float_constants.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/compress_float_constants.cpp @@ -5,7 +5,7 @@ #include "transformations/common_optimizations/compress_float_constants.hpp" #include "itt.hpp" -#include "ngraph/rt_info.hpp" +#include "openvino/core/rt_info.hpp" #include "openvino/opsets/opset8.hpp" #include "openvino/pass/pattern/op/wrap_type.hpp" #include "transformations/rt_info/decompression.hpp" @@ -25,23 +25,30 @@ std::shared_ptr change_constant_precision_to_fp16(std::shared_ptr std::numeric_limits::max()) { + // if it is smaller than the smallest positive normal fp16 but not zero + if (std::abs(src_data[i]) <= ov::float16::from_bits(0x0400) && src_data[i] != 0.0f) { + num_out_of_range++; + } else if (src_data[i] > std::numeric_limits::max()) { dst_data[i] = std::numeric_limits::max(); - is_overflow = true; + num_out_of_range++; } else if (src_data[i] < std::numeric_limits::lowest()) { dst_data[i] = std::numeric_limits::lowest(); - is_overflow = true; + num_out_of_range++; } else { dst_data[i] = static_cast(src_data[i]); } } - if (is_overflow) { - std::cerr << "Warning: One or more of the values of the Constant can't fit in the float16 data type." - " Those values were casted to the nearest limit value, the model can produce incorrect results." - << std::endl; + + // if more than 75% of a FP32 constant do not fit into FP16 keep in FP32 + float keep_threshold = 0.75f; + float out_of_range_proportion = static_cast(num_out_of_range) / static_cast(size); + + if (out_of_range_proportion >= keep_threshold) { + return nullptr; } + return new_constant; } } // namespace @@ -70,11 +77,15 @@ ov::pass::CompressFloatConstantsImpl::CompressFloatConstantsImpl() { } else { return false; } + + if (!new_const) { + return false; + } auto convert = std::make_shared(new_const, const_node->get_element_type()); new_const->set_friendly_name(const_node->get_friendly_name() + "_compressed"); convert->set_friendly_name(const_node->get_friendly_name()); - ngraph::copy_runtime_info(const_node, convert); + ov::copy_runtime_info(const_node, convert); ov::mark_as_decompression(convert); ov::replace_node(const_node, convert); diff --git a/src/common/transformations/src/transformations/common_optimizations/convert_compressed_to_mixed_precision.cpp b/src/common/transformations/src/transformations/common_optimizations/convert_compressed_to_mixed_precision.cpp index 4ce68ad6277..aefe31a3ece 100644 --- a/src/common/transformations/src/transformations/common_optimizations/convert_compressed_to_mixed_precision.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/convert_compressed_to_mixed_precision.cpp @@ -7,8 +7,9 @@ #include "itt.hpp" #include "openvino/opsets/opset8.hpp" #include "openvino/pass/manager.hpp" +#include "transformations/common_optimizations/align_mixed_fp32_fp16_types.hpp" #include "transformations/common_optimizations/convert_compression_only_to_legacy.hpp" -#include "transformations/common_optimizations/mark_precision_sensitive_shapeof_subgraphs.hpp" +#include "transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.hpp" #include "transformations/convert_precision.hpp" #include "transformations/utils/utils.hpp" @@ -20,13 +21,13 @@ bool ov::pass::ConvertCompressedToMixedPrecision::run_on_model(const std::shared return false; Manager manager(get_pass_config()); - // Mark nodes in ShapeOf subgraphs with disable_fp16_compression rt_info to keep them in FP32 precision - REGISTER_PASS(manager, MarkPrecisionSensitiveShapeOfSubgraphs) + REGISTER_PASS(manager, MarkSugraphsToKeepInMixedPrecision) + REGISTER_PASS(manager, AlignMixedFP32FP16Types) const precisions_array convert_precision_list{{ov::element::f32, ov::element::f16}}; type_to_fuse_map additional_fuse_map = {}; // call ConvertPrecision with keep_precision_sensitive_in_fp32 = true - REGISTER_PASS(manager, ConvertPrecision, convert_precision_list, additional_fuse_map, true); + REGISTER_PASS(manager, ConvertPrecision, convert_precision_list, additional_fuse_map, true) REGISTER_PASS(manager, EnableDecompressionConvertConstantFolding) REGISTER_PASS(manager, ConstantFolding) diff --git a/src/common/transformations/src/transformations/common_optimizations/division_by_zero_fp16_resolver.cpp b/src/common/transformations/src/transformations/common_optimizations/division_by_zero_fp16_resolver.cpp deleted file mode 100644 index 469476319fa..00000000000 --- a/src/common/transformations/src/transformations/common_optimizations/division_by_zero_fp16_resolver.cpp +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "transformations/common_optimizations/division_by_zero_fp16_resolver.hpp" - -#include -#include -#include -#include -#include - -#include "itt.hpp" -#include "ngraph/rt_info.hpp" -#include "transformations/utils/utils.hpp" - -constexpr float normalized_fp16_min = 6.103515625e-05f; // fp16 minimal normalized value - -using namespace ov; - -ov::pass::DivisionByZeroFP16Resolver::DivisionByZeroFP16Resolver() { - MATCHER_SCOPE(DivisionByZeroFP16Resolver); - - // to detect the following patterns where eps is used to prevent division by zero: - // input_1/Maximum(input_2, eps) - // input_1/Add(input_2, eps) - // input_1/Sqrt(Maximum(input_2, eps)) - // input_1/Sqrt(Add(input_2, eps)) - // input_1*Pow(Maximum(input_2, eps), -z) - // input_1*Pow(Add(input_2, eps), -z) - auto input_1 = pattern::any_input(); - auto input_2 = pattern::any_input(); - - auto eps_const_pattern = pattern::wrap_type(); - auto max = std::make_shared(input_2, eps_const_pattern); - auto add = std::make_shared(input_2, eps_const_pattern); - auto max_or_add = std::make_shared(OutputVector{max, add}); - - auto sqrt = std::make_shared(max_or_add); - auto sqrt_or_max_add = std::make_shared(OutputVector{max_or_add, sqrt}); - // whether is divided directly or after sqrt (e.g. in L2Norm after sqrt, in MVN is divided directly) - auto divide = std::make_shared(input_1, sqrt_or_max_add); - - auto pow_exp = pattern::wrap_type(); - auto pow_pattern = std::make_shared(max_or_add, pow_exp); - auto mul_pattern = std::make_shared(input_1, pow_pattern); - auto div_or_mul_to_negative_pow = std::make_shared(OutputVector{divide, mul_pattern}); - - matcher_pass_callback callback = [=](pattern::Matcher& m) { - const auto& pattern_to_output = m.get_pattern_map(); - - const auto mul = std::dynamic_pointer_cast(m.get_match_root()); - if (mul) { - // pattern input_1*Pow(Maximum(input_2, eps), z) or input_1*Pow(Add(input_2, eps), z) is matched - const auto pow_const = std::dynamic_pointer_cast(pattern_to_output.at(pow_exp)); - for (float val : pow_const->get_vector()) - if (val >= 0) // continue only if exponent is negative (z < 0) - return false; - } - - const auto eps_const = std::dynamic_pointer_cast(pattern_to_output.at(eps_const_pattern)); - if (!eps_const || eps_const->get_element_type() != ov::element::f32) - return false; - - for (float val : eps_const->get_vector()) - if (val >= normalized_fp16_min) - return false; - - auto new_constant = std::make_shared(eps_const->get_element_type(), - eps_const->get_shape(), - normalized_fp16_min); - copy_runtime_info(eps_const, new_constant); - replace_node(eps_const, new_constant); - return true; - }; - - auto m = std::make_shared(div_or_mul_to_negative_pow, matcher_name); - register_matcher(m, callback); -} diff --git a/src/common/transformations/src/transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.cpp b/src/common/transformations/src/transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.cpp new file mode 100644 index 00000000000..1cf7f2fdee8 --- /dev/null +++ b/src/common/transformations/src/transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.cpp @@ -0,0 +1,362 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.hpp" + +#include "itt.hpp" +#include "openvino/op/util/broadcast_base.hpp" +#include "openvino/op/util/gather_base.hpp" +#include "openvino/opsets/opset10.hpp" +#include "openvino/opsets/opset2.hpp" +#include "openvino/pass/manager.hpp" +#include "openvino/pass/pattern/op/or.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "transformations/common_optimizations/mark_precision_sensitive_shapeof_subgraphs.hpp" +#include "transformations/convert_precision.hpp" +#include "transformations/rt_info/disable_fp16_compression.hpp" +#include "transformations/utils/utils.hpp" + +using namespace std; +using namespace ov::opset10; + +namespace ov { +namespace pass { + +/* + * MarkNormalizationOps marks MVN and NormalizeL2 to be kept in f32 precision. + */ +class MarkNormalizationOps : public MatcherPass { +public: + OPENVINO_RTTI("MarkNormalizationOps", "0"); + + MarkNormalizationOps() { + MATCHER_SCOPE(MarkNormalizationOps); + auto ops_to_be_kept_fp32 = pattern::wrap_type(); + + matcher_pass_callback callback = [=](pattern::Matcher& m) { + const auto& node = m.get_match_root(); + if (!node) + return false; + + disable_fp16_compression(node); + return true; + }; + auto m = make_shared(ops_to_be_kept_fp32, matcher_name); + register_matcher(m, callback); + } +}; + +// Marking continues to propagate through these ops. +std::shared_ptr propagate_through_ops = pattern::wrap_type(); + +/* After PropagateDownMark we need to go also once up to include side branches of ops with several args: + * Elementwise, Concat and so. E.g. if one of the argument of Concat was marked + * during PropagateDownMark we need also mark its other inputs and this is done in this PropagateUpMark pass. + * Propagation stops when we face ops not listed in propagate_through_ops: e.g. if we face Conv or MatMul. + */ +class PropagateUpMarkToKeepInMixedPrecision : public pass::MatcherPass { +public: + OPENVINO_RTTI("PropagateUpMarkToKeepInMixedPrecision", "0"); + PropagateUpMarkToKeepInMixedPrecision() { + MATCHER_SCOPE(PropagateUpMarkToKeepInMixedPrecision); + + matcher_pass_callback callback = [=](pattern::Matcher& m) { + const auto& node = m.get_match_root(); + bool has_marked_output = false; + for (const auto& output : node->outputs()) { + for (const auto& out_inputs : output.get_target_inputs()) { + if (out_inputs.get_element_type().is_real() && + fp16_compression_is_disabled(out_inputs.get_node()->shared_from_this())) { + has_marked_output = true; + } + } + } + + if (!has_marked_output) + return false; + + auto convert_node = dynamic_pointer_cast(node); + if (convert_node) { + // if during propagating up there is a Convert it must go to Const, + // otherwise interrupt propagation + auto const_node = dynamic_pointer_cast(node->input_value(0).get_node_shared_ptr()); + if (!const_node) + return false; + } + + disable_fp16_compression(node); + return true; + }; + + auto m = make_shared(propagate_through_ops, matcher_name); + register_matcher(m, callback); + } +}; + +/* Starting from the marked precision sensitive nodes we need to propagate down to include neighboring + * ops like Slice, ReduceSum, Reshape, Elementwise, et al. to be kept in f32 as well. + * Propagation stops when ops not listed in propagate_through_ops are faced: e.g. if we face Conv or MatMul. + */ +class PropagateDownMarkToKeepInMixedPrecision : public pass::MatcherPass { +public: + OPENVINO_RTTI("PropagateDownMarkToKeepInMixedPrecision", "0"); + PropagateDownMarkToKeepInMixedPrecision() { + MATCHER_SCOPE(PropagateDownMarkToKeepInMixedPrecision); + + matcher_pass_callback callback = [=](pattern::Matcher& m) { + const auto& node = m.get_match_root(); + if (!node) + return false; + + // on convert down propagation should be interrupted + auto convert_node = dynamic_pointer_cast(node); + if (convert_node) + return false; + + bool is_changed = false; + for (const auto& in_node : node->input_values()) { + if (!in_node.get_element_type().is_real()) + continue; + if (fp16_compression_is_disabled(in_node.get_node_shared_ptr())) { + disable_fp16_compression(node); + is_changed = true; + break; + } + } + return is_changed; + }; + auto m = make_shared(propagate_through_ops, matcher_name); + register_matcher(m, callback); + } +}; + +void mark_reduceop_path(const std::shared_ptr& node) { + node->get_rt_info().emplace("reduceop_path", true); +} +bool is_reduceop_path(const std::shared_ptr& node) { + return node->get_rt_info().count("reduceop_path"); +} + +void erase_reduceop_path(const std::shared_ptr& node) { + auto& rt_info = node->get_rt_info(); + rt_info.erase("reduceop_path"); +} + +class InitMarkReduceOpPath : public pass::MatcherPass { +public: + OPENVINO_RTTI("InitMarkReduceOpPath", "0"); + InitMarkReduceOpPath() { + MATCHER_SCOPE(InitMarkReduceOpPath); + + auto reduce_ops = pattern::wrap_type(); + + matcher_pass_callback callback = [=](pattern::Matcher& m) { + const auto& node = m.get_match_root(); + if (!node) + return false; + mark_reduceop_path(node); + return true; + }; + auto m = make_shared(reduce_ops, matcher_name); + register_matcher(m, callback); + } +}; + +class PropagateMarkUpReduceOpPath : public pass::MatcherPass { +public: + OPENVINO_RTTI("PropagateMarkUpReduceOpPath", "0"); + PropagateMarkUpReduceOpPath() { + MATCHER_SCOPE(PropagateMarkUpReduceOpPath); + + matcher_pass_callback callback = [=](pattern::Matcher& m) { + const auto& node = m.get_match_root(); + if (!node) + return false; + + for (const auto& output : node->outputs()) { + for (const auto& out_inputs : output.get_target_inputs()) { + if (out_inputs.get_element_type().is_real() && + is_reduceop_path(out_inputs.get_node()->shared_from_this())) { + mark_reduceop_path(node); + return false; + } + } + } + return false; + }; + auto m = make_shared(propagate_through_ops, matcher_name); + register_matcher(m, callback); + } +}; + +class MarkExp : public pass::MatcherPass { +public: + // only exponent that go into ReduceOp should be marked as precision sensitive and kept in f32 + OPENVINO_RTTI("MarkExp", "0"); + MarkExp() { + MATCHER_SCOPE(MarkExp); + auto exp_pattern = pattern::wrap_type(); + + matcher_pass_callback callback = [=](pattern::Matcher& m) { + const auto& node = m.get_match_root(); + if (!node) + return false; + + if (!is_reduceop_path(node)) + return false; + + disable_fp16_compression(node); + return true; + }; + auto m = make_shared(exp_pattern, matcher_name); + register_matcher(m, callback); + } +}; +/* MarkExpInReduceOpPath marks path that goes into ReduceSum and ReduceMean. + * Values that go from Exp to ReduceSum/ReduceMean are precision + * sensitive and should be kept in f32 precision for mixed inference. + */ +class MarkExpInReduceOpPath : public BackwardGraphRewrite { +public: + OPENVINO_RTTI("MarkExpInReduceOpPath", "0"); + MarkExpInReduceOpPath() { + // marking of ReduceOp path is needed to mark only Exponents that go into ReduceSum/ReduceMean + ADD_MATCHER_FOR_THIS(InitMarkReduceOpPath); + ADD_MATCHER_FOR_THIS(PropagateMarkUpReduceOpPath); + ADD_MATCHER_FOR_THIS(MarkExp); + } +}; + +/* MarkDivWithEps martk pattern that matches the patterns input_1/Maximum(input_2, eps); input_1/Add(input_2, eps); + * and input_1*Pow(Maximum[Add](input_2, eps), -z) and marks subgraph root to be kept in fp32. + * + * If both input_1 and input_2 simultaneously happen to be zero to prevent from NaNs and not to loose accuracy, + * we should calculate such patterns always in fp32 precision even if ov::Model is compressed to fp16. + */ +class MarkDivWithEps : public MatcherPass { +public: + OPENVINO_RTTI("MarkDivWithEps", "0"); + MarkDivWithEps() { + MATCHER_SCOPE(MarkDivWithEps); + + // to detect the following patterns where eps is used to prevent division by zero: + // input_1/Maximum(input_2, eps) + // input_1/Add(input_2, eps) + // input_1/Sqrt(Maximum(input_2, eps)) + // input_1/Sqrt(Add(input_2, eps)) + // input_1*Pow(Maximum(input_2, eps), -z) + // input_1*Pow(Add(input_2, eps), -z) + auto input_1 = pattern::any_input(); + auto input_2 = pattern::any_input(); + + auto eps_const_pattern = pattern::wrap_type(); + auto max_or_add = pattern::wrap_type(OutputVector{input_2, eps_const_pattern}); + + auto sqrt = std::make_shared(max_or_add); + auto sqrt_or_max_add = std::make_shared(OutputVector{max_or_add, sqrt}); + // whether is divided directly or after sqrt (e.g. in L2Norm after sqrt, in MVN is divided directly) + auto divide = std::make_shared(input_1, sqrt_or_max_add); + + auto pow_exp = pattern::wrap_type(); + auto convert_pattern = pattern::wrap_type({pow_exp}); + auto pow_exp_or_convert = std::make_shared(OutputVector{pow_exp, convert_pattern}); + + auto pow_pattern = std::make_shared(max_or_add, pow_exp_or_convert); + auto mul_pattern = std::make_shared(input_1, pow_pattern); + auto div_or_mul_to_negative_pow = std::make_shared(OutputVector{divide, mul_pattern}); + + matcher_pass_callback callback = [=](pattern::Matcher& m) { + const auto& pattern_to_output = m.get_pattern_map(); + if (!m.get_match_root()) + return false; + + const auto mul = std::dynamic_pointer_cast(m.get_match_root()); + // if pattern input_1*Pow(Maximum(input_2, eps), z) or input_1*Pow(Add(input_2, eps), z) is matched + // need to check that power is negative + if (mul) { + const auto pow_const = std::dynamic_pointer_cast(pattern_to_output.at(pow_exp)); + if (pow_const) { + // continue only if exponent is negative (z < 0) + if (pow_const->get_element_type() == element::f16) { + for (auto val : pow_const->get_vector()) + if (val >= 0.0f) + return false; + } else if (pow_const->get_element_type() == element::f32) { + for (auto val : pow_const->get_vector()) + if (val >= 0.0f) + return false; + } + } + } + + const auto eps_const = std::dynamic_pointer_cast(pattern_to_output.at(eps_const_pattern)); + if (!eps_const) + return false; + if (eps_const->get_element_type() == element::f32) { + for (const auto& val : eps_const->get_vector()) + if (val > static_cast(float16_min_normalized)) + return false; + } else if (eps_const->get_element_type() == element::f16) { + for (const auto& val : eps_const->get_vector()) + if (val > float16_min_normalized) + return false; + } + disable_fp16_compression(m.get_match_root()); + return true; + }; + + auto m = make_shared(div_or_mul_to_negative_pow, matcher_name); + register_matcher(m, callback); + } +}; + +bool MarkSugraphsToKeepInMixedPrecision::run_on_model(const shared_ptr& m) { + RUN_ON_MODEL_SCOPE(MarkSugraphsToKeepInMixedPrecision); + + Manager manager(get_pass_config()); + // Mark root of Division with eps pattern to keep in FP32 + REGISTER_PASS(manager, MarkDivWithEps) + + REGISTER_PASS(manager, MarkExpInReduceOpPath) + REGISTER_PASS(manager, MarkNormalizationOps) + + // both Up and Down propagations are needed. + // Why both of them are needed is explained in comments in passes declarations. + REGISTER_PASS(manager, PropagateDownMarkToKeepInMixedPrecision) + auto propagate_up = manager.register_pass(); + ADD_MATCHER(propagate_up, PropagateUpMarkToKeepInMixedPrecision) + + // Mark nodes in ShapeOf subgraphs to keep in FP32 + REGISTER_PASS(manager, MarkPrecisionSensitiveShapeOfSubgraphs) + manager.run_passes(m); + + for (auto& node : m->get_ops()) { + erase_reduceop_path(node); + } + + return false; // no need to revalidate +} + +} // namespace pass +} // namespace ov diff --git a/src/common/transformations/tests/common_optimizations/align_mixed_fp32_fp16_types_test.cpp b/src/common/transformations/tests/common_optimizations/align_mixed_fp32_fp16_types_test.cpp new file mode 100644 index 00000000000..cfe06a0800d --- /dev/null +++ b/src/common/transformations/tests/common_optimizations/align_mixed_fp32_fp16_types_test.cpp @@ -0,0 +1,304 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include +#include +#include +#include +#include +#include + +#include "common_test_utils/ngraph_test_utils.hpp" + +using namespace testing; +using namespace ov; +using namespace std; +using namespace ov::opset10; + +TEST_F(TransformationTestsF, align_mixed_fp16_fp32_1) { + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto exp_1 = make_shared(input_1); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto reduce_sum_1 = make_shared(exp_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_sum_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + pass::Manager manager; + manager.register_pass(); + manager.register_pass(); + const precisions_array convert_precision_list{{ov::element::f32, ov::element::f16}}; + type_to_fuse_map empty_type_to_fuse_map = {}; + manager.register_pass(convert_precision_list, empty_type_to_fuse_map, true); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f16, Shape{1, 3, 224, 224}); + auto convert_to_f32_1 = make_shared(input_1, element::f32); + auto exp_1 = make_shared(convert_to_f32_1); + auto input_2 = make_shared(element::f16, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto reduce_sum_1 = make_shared(exp_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_sum_1, factor_const_decompressed); + auto convert_to_f16_1 = make_shared(mul_1, element::f16); + auto matmul_1 = make_shared(convert_to_f16_1, input_2); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + } +} + +TEST_F(TransformationTestsF, align_mixed_fp16_fp32_2) { + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto exp_1 = make_shared(input_1); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + + auto unsqueeze_axes = Constant::create(element::i64, Shape{1}, {1}); + auto unsqueeze_1 = make_shared(exp_1, unsqueeze_axes); + auto reduce_sum_1 = make_shared(unsqueeze_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_sum_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + pass::Manager manager; + manager.register_pass(); + manager.register_pass(); + const precisions_array convert_precision_list{{ov::element::f32, ov::element::f16}}; + type_to_fuse_map empty_type_to_fuse_map = {}; + manager.register_pass(convert_precision_list, empty_type_to_fuse_map, true); + + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f16, Shape{1, 3, 224, 224}); + auto convert_to_f32_1 = make_shared(input_1, element::f32); + auto exp_1 = make_shared(convert_to_f32_1); + auto input_2 = make_shared(element::f16, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + + auto unsqueeze_axes = Constant::create(element::i64, Shape{1}, {1}); + auto unsqueeze_1 = make_shared(exp_1, unsqueeze_axes); + auto reduce_sum_1 = make_shared(unsqueeze_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_sum_1, factor_const_decompressed); + auto convert_to_f16_1 = make_shared(mul_1, element::f16); + auto matmul_1 = make_shared(convert_to_f16_1, input_2); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + } +} + +TEST_F(TransformationTestsF, align_mixed_fp16_fp32_3) { + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto exp_1 = make_shared(input_1); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + + auto reduce_sum_1 = make_shared(exp_1, reduction_axes); + auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); + auto add_1 = make_shared(reduce_sum_1, addition_const); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(add_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + pass::Manager manager; + manager.register_pass(); + manager.register_pass(); + const precisions_array convert_precision_list{{ov::element::f32, ov::element::f16}}; + type_to_fuse_map empty_type_to_fuse_map = {}; + manager.register_pass(convert_precision_list, empty_type_to_fuse_map, true); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f16, Shape{1, 3, 224, 224}); + auto convert_to_f32_1 = make_shared(input_1, element::f32); + auto exp_1 = make_shared(convert_to_f32_1); + auto input_2 = make_shared(element::f16, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + + auto reduce_sum_1 = make_shared(exp_1, reduction_axes); + auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); + auto add_1 = make_shared(reduce_sum_1, addition_const); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(add_1, factor_const_decompressed); + auto convert_to_f16_1 = make_shared(mul_1, element::f16); + auto matmul_1 = make_shared(convert_to_f16_1, input_2); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + } +} + +TEST_F(TransformationTestsF, align_mixed_fp16_fp32_4) { + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto mvn_1 = make_shared(input_1, reduction_axes, true, 1.0e-8f, op::MVNEpsMode::INSIDE_SQRT); + auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); + auto add_1 = make_shared(mvn_1, addition_const); + auto matmul_1 = make_shared(add_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + pass::Manager manager; + manager.register_pass(); + manager.register_pass(); + const precisions_array convert_precision_list{{ov::element::f32, ov::element::f16}}; + type_to_fuse_map empty_type_to_fuse_map = {}; + manager.register_pass(convert_precision_list, empty_type_to_fuse_map, true); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f16, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f16, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto convert_to_f32_1 = make_shared(input_1, element::f32); + auto mvn_1 = make_shared(convert_to_f32_1, reduction_axes, true, 1.0e-8f, op::MVNEpsMode::INSIDE_SQRT); + auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); + auto add_1 = make_shared(mvn_1, addition_const); + auto convert_to_f16_1 = make_shared(add_1, element::f16); + auto matmul_1 = make_shared(convert_to_f16_1, input_2); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + } +} + +TEST_F(TransformationTestsF, align_mixed_fp16_fp32_mnv_with_split) { + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f32, Shape{1, 3, 56, 224}); + + auto split_axis = Constant::create(element::i64, Shape{}, {3}); + auto split = make_shared(input_1, split_axis, 4); + + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto mvn_1 = make_shared(split->output(0), reduction_axes, true, 1.0e-8f, op::MVNEpsMode::INSIDE_SQRT); + auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); + auto add_1 = make_shared(mvn_1, addition_const); + auto matmul_1 = make_shared(add_1, input_2); + + auto result_1 = make_shared(matmul_1); + auto result_2 = make_shared(split->output(1)); + auto result_3 = make_shared(split->output(2)); + model = make_shared(OutputVector{result_1, result_2, result_3}, ParameterVector{input_1, input_2}); + + pass::Manager manager; + manager.register_pass(); + manager.register_pass(); + const precisions_array convert_precision_list{{ov::element::f32, ov::element::f16}}; + type_to_fuse_map empty_type_to_fuse_map = {}; + manager.register_pass(convert_precision_list, empty_type_to_fuse_map, true); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f16, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f16, Shape{1, 3, 56, 224}); + + auto convert_to_f32_1 = make_shared(input_1, element::f32); + + auto split_axis = Constant::create(element::i64, Shape{}, {3}); + auto split = make_shared(convert_to_f32_1, split_axis, 4); + + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto mvn_1 = make_shared(split->output(0), reduction_axes, true, 1.0e-8f, op::MVNEpsMode::INSIDE_SQRT); + auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); + auto add_1 = make_shared(mvn_1, addition_const); + auto convert_to_f16_1 = make_shared(add_1, element::f16); + auto matmul_1 = make_shared(convert_to_f16_1, input_2); + + // todo: without Converts to fp16 because of GPU + auto result_1 = make_shared(matmul_1); + auto result_2 = make_shared(split->output(1)); + auto result_3 = make_shared(split->output(2)); + + model_ref = make_shared(OutputVector{result_1, result_2, result_3}, ParameterVector{input_1, input_2}); + } +} + +TEST_F(TransformationTestsF, align_mixed_fp16_fp32_with_rand_uniform) { + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto exp_1 = make_shared(input_1); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto reduce_sum_1 = make_shared(exp_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + + auto out_shape = Constant::create(element::i64, Shape{3}, {1, 3, 224}); + auto minval = Constant::create(element::f32, Shape{}, {1}); + auto maxval = Constant::create(element::f32, Shape{}, {10}); + auto rand_uniform = make_shared(out_shape, minval, maxval, element::f32); + auto rand_uniform_add_factor = make_shared(rand_uniform, factor_const_decompressed); + + auto mul_1 = make_shared(reduce_sum_1, rand_uniform_add_factor); + auto matmul_1 = make_shared(mul_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + pass::Manager manager; + manager.register_pass(); + manager.register_pass(); + const precisions_array convert_precision_list{{ov::element::f32, ov::element::f16}}; + type_to_fuse_map empty_type_to_fuse_map = {}; + manager.register_pass(convert_precision_list, empty_type_to_fuse_map, true); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f16, Shape{1, 3, 224, 224}); + auto convert_to_f32_1 = make_shared(input_1, element::f32); + auto exp_1 = make_shared(convert_to_f32_1); + auto input_2 = make_shared(element::f16, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto reduce_sum_1 = make_shared(exp_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + + auto out_shape = Constant::create(element::i64, Shape{3}, {1, 3, 224}); + auto minval = Constant::create(element::f16, Shape{}, {1}); + auto maxval = Constant::create(element::f16, Shape{}, {10}); + auto rand_uniform = make_shared(out_shape, minval, maxval, element::f16); + auto rand_uniform_decompressed = make_shared(rand_uniform, element::f32); + auto rand_uniform_add_factor = make_shared(rand_uniform_decompressed, factor_const_decompressed); + + auto mul_1 = make_shared(reduce_sum_1, rand_uniform_add_factor); + auto convert_to_f16_1 = make_shared(mul_1, element::f16); + auto matmul_1 = make_shared(convert_to_f16_1, input_2); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + } +} diff --git a/src/common/transformations/tests/common_optimizations/compress_float_constants_test.cpp b/src/common/transformations/tests/common_optimizations/compress_float_constants_test.cpp index 239b5e6d555..ab7bbb11a3d 100644 --- a/src/common/transformations/tests/common_optimizations/compress_float_constants_test.cpp +++ b/src/common/transformations/tests/common_optimizations/compress_float_constants_test.cpp @@ -271,3 +271,199 @@ TEST_F(TransformationTestsF, CompressConstants_f64) { function_ref = std::make_shared(ov::NodeVector{conv}, ov::ParameterVector{input}); } } + +TEST_F(TransformationTestsF, CompressConstants_keep_in_f32_small_eps_out_of_range) { + float fp16_eps = static_cast(ov::float16::from_bits(0x0400)); + { + auto input = std::make_shared(ov::element::f32, ov::Shape{1, 3, 12, 12}); + + auto const_weights = ov::opset8::Constant::create(ov::element::f32, + ov::Shape{1, 3, 4, 1}, + {0.0f, + 1.0f, + 2.0f, + fp16_eps, + fp16_eps, + fp16_eps, + fp16_eps, + fp16_eps, + fp16_eps, + fp16_eps, + fp16_eps, + fp16_eps}); + auto conv = std::make_shared(input, + const_weights, + ov::Strides{1, 1}, + ov::CoordinateDiff{0, 0}, + ov::CoordinateDiff{0, 0}, + ov::Strides{1, 1}); + function = std::make_shared(ov::NodeVector{conv}, ov::ParameterVector{input}); + + manager.register_pass(); + manager.register_pass(); + } + + { + auto input = std::make_shared(ov::element::f32, ov::Shape{1, 3, 12, 12}); + auto const_weights = ov::opset8::Constant::create(ov::element::f32, + ov::Shape{1, 3, 4, 1}, + {0.0f, + 1.0f, + 2.0f, + fp16_eps, + fp16_eps, + fp16_eps, + fp16_eps, + fp16_eps, + fp16_eps, + fp16_eps, + fp16_eps, + fp16_eps}); + auto conv = std::make_shared(input, + const_weights, + ov::Strides{1, 1}, + ov::CoordinateDiff{0, 0}, + ov::CoordinateDiff{0, 0}, + ov::Strides{1, 1}); + function_ref = std::make_shared(ov::NodeVector{conv}, ov::ParameterVector{input}); + } +} + +TEST_F(TransformationTestsF, CompressConstants_keep_in_f32_max_out_of_range_val) { + // if fp16 out of range values fraction is greater than threshold (75%) then keep them in fp32 + // no decompression converts should be inserted + float fp16_oor = static_cast(std::numeric_limits::max()) + 100.0f; + { + auto input = std::make_shared(ov::element::f32, ov::Shape{1, 3, 12, 12}); + + auto const_weights = ov::opset8::Constant::create(ov::element::f32, + ov::Shape{1, 3, 4, 1}, + {0.0f, + 1.0f, + 2.0f, + fp16_oor, + fp16_oor, + fp16_oor, + fp16_oor, + fp16_oor, + fp16_oor, + fp16_oor, + fp16_oor, + fp16_oor}); + auto conv = std::make_shared(input, + const_weights, + ov::Strides{1, 1}, + ov::CoordinateDiff{0, 0}, + ov::CoordinateDiff{0, 0}, + ov::Strides{1, 1}); + function = std::make_shared(ov::NodeVector{conv}, ov::ParameterVector{input}); + + manager.register_pass(); + manager.register_pass(); + } + + { + auto input = std::make_shared(ov::element::f32, ov::Shape{1, 3, 12, 12}); + auto const_weights = ov::opset8::Constant::create(ov::element::f32, + ov::Shape{1, 3, 4, 1}, + {0.0f, + 1.0f, + 2.0f, + fp16_oor, + fp16_oor, + fp16_oor, + fp16_oor, + fp16_oor, + fp16_oor, + fp16_oor, + fp16_oor, + fp16_oor}); + auto conv = std::make_shared(input, + const_weights, + ov::Strides{1, 1}, + ov::CoordinateDiff{0, 0}, + ov::CoordinateDiff{0, 0}, + ov::Strides{1, 1}); + function_ref = std::make_shared(ov::NodeVector{conv}, ov::ParameterVector{input}); + } +} + +TEST_F(TransformationTestsF, CompressConstants_compress_to_f16_max_out_of_range_val) { + // fp16 out of range should be clipped to fp16_max_val if fraction of out of range values is less than threshold + float fp16_oor = static_cast(std::numeric_limits::max()) + 100.0f; + float fp16_max = static_cast(std::numeric_limits::max()); + { + auto input = std::make_shared(ov::element::f32, ov::Shape{1, 3, 12, 12}); + + // only half of values are out of range, therefore they will be compressed to fp16 + auto const_weights = ov::opset8::Constant::create( + ov::element::f32, + ov::Shape{1, 3, 4, 1}, + {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, fp16_oor, fp16_oor, fp16_oor, fp16_oor, fp16_oor, fp16_oor}); + auto conv = std::make_shared(input, + const_weights, + ov::Strides{1, 1}, + ov::CoordinateDiff{0, 0}, + ov::CoordinateDiff{0, 0}, + ov::Strides{1, 1}); + function = std::make_shared(ov::NodeVector{conv}, ov::ParameterVector{input}); + + manager.register_pass(); + manager.register_pass(); + } + + { + auto input = std::make_shared(ov::element::f32, ov::Shape{1, 3, 12, 12}); + auto const_weights = ov::opset8::Constant::create( + ov::element::f16, + ov::Shape{1, 3, 4, 1}, + {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, fp16_max, fp16_max, fp16_max, fp16_max, fp16_max, fp16_max}); + auto convert_ins1 = std::make_shared(const_weights, ov::element::f32); + auto conv = std::make_shared(input, + convert_ins1, + ov::Strides{1, 1}, + ov::CoordinateDiff{0, 0}, + ov::CoordinateDiff{0, 0}, + ov::Strides{1, 1}); + function_ref = std::make_shared(ov::NodeVector{conv}, ov::ParameterVector{input}); + } +} + +TEST_F(TransformationTestsF, CompressConstants_not_keep_in_f32_when_zeros) { + // zero values are less than fp16_eps, but they are exactly expressed in fp16 + // not need to keep them in fp32 + { + auto input = std::make_shared(ov::element::f32, ov::Shape{1, 3, 12, 12}); + + auto const_weights = + ov::opset8::Constant::create(ov::element::f32, + ov::Shape{1, 3, 4, 1}, + {0.0f, 1.0f, 2.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}); + auto conv = std::make_shared(input, + const_weights, + ov::Strides{1, 1}, + ov::CoordinateDiff{0, 0}, + ov::CoordinateDiff{0, 0}, + ov::Strides{1, 1}); + function = std::make_shared(ov::NodeVector{conv}, ov::ParameterVector{input}); + + manager.register_pass(); + manager.register_pass(); + } + + { + auto input = std::make_shared(ov::element::f32, ov::Shape{1, 3, 12, 12}); + auto const_weights = + ov::opset8::Constant::create(ov::element::f16, + ov::Shape{1, 3, 4, 1}, + {0.0f, 1.0f, 2.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}); + auto convert_ins1 = std::make_shared(const_weights, ov::element::f32); + auto conv = std::make_shared(input, + convert_ins1, + ov::Strides{1, 1}, + ov::CoordinateDiff{0, 0}, + ov::CoordinateDiff{0, 0}, + ov::Strides{1, 1}); + function_ref = std::make_shared(ov::NodeVector{conv}, ov::ParameterVector{input}); + } +} diff --git a/src/common/transformations/tests/common_optimizations/division_by_zero_fp16_resolver_test.cpp b/src/common/transformations/tests/common_optimizations/division_by_zero_fp16_resolver_test.cpp index a0a54b70bc3..e69de29bb2d 100644 --- a/src/common/transformations/tests/common_optimizations/division_by_zero_fp16_resolver_test.cpp +++ b/src/common/transformations/tests/common_optimizations/division_by_zero_fp16_resolver_test.cpp @@ -1,207 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "common_test_utils/ngraph_test_utils.hpp" - -using namespace testing; -using namespace ov; -constexpr float normalized_fp16_min = 6.103515625e-05f; // fp16 minimal normalized value - -TEST_F(TransformationTestsF, DivisionByZeroMinimalPattern) { - const float eps_value = 1.e-12f; - { - auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {eps_value}); - auto add = std::make_shared(input_2, eps_const); - auto divide = std::make_shared(input_1, add); - - function = std::make_shared(NodeVector{divide}, ParameterVector{input_1, input_2}); - - manager.register_pass(); - } - - { - auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {normalized_fp16_min}); - auto add = std::make_shared(input_2, eps_const); - auto divide = std::make_shared(input_1, add); - - function_ref = std::make_shared(NodeVector{divide}, ParameterVector{input_1, input_2}); - } - comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); - comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); -} - -TEST_F(TransformationTestsF, PowWithNegativeExponent) { - const float eps_value = 1.e-12f; - { - auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {eps_value}); - auto add = std::make_shared(input_2, eps_const); - auto pow_exp_const = opset4::Constant::create(element::f32, Shape{1}, {-1.77}); - auto pow = std::make_shared(add, pow_exp_const); - auto mul = std::make_shared(input_1, pow); - - function = std::make_shared(NodeVector{mul}, ParameterVector{input_1, input_2}); - - manager.register_pass(); - } - - { - auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {normalized_fp16_min}); - auto add = std::make_shared(input_2, eps_const); - auto pow_exp_const = opset4::Constant::create(element::f32, Shape{1}, {-1.77}); - auto pow = std::make_shared(add, pow_exp_const); - auto mul = std::make_shared(input_1, pow); - - function_ref = std::make_shared(NodeVector{mul}, ParameterVector{input_1, input_2}); - } - comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); - comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); -} - -TEST_F(TransformationTestsF, PowWithPositiveExponent) { - // graph should be left unchanged - const float eps_value = 1.e-12f; - { - auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {eps_value}); - auto add = std::make_shared(input_2, eps_const); - auto pow_exp_const = opset4::Constant::create(element::f32, Shape{1}, {1.77}); - auto pow = std::make_shared(add, pow_exp_const); - auto mul = std::make_shared(input_1, pow); - - function = std::make_shared(NodeVector{mul}, ParameterVector{input_1, input_2}); - - manager.register_pass(); - } - - { - auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {eps_value}); - auto add = std::make_shared(input_2, eps_const); - auto pow_exp_const = opset4::Constant::create(element::f32, Shape{1}, {1.77}); - auto pow = std::make_shared(add, pow_exp_const); - auto mul = std::make_shared(input_1, pow); - - function_ref = std::make_shared(NodeVector{mul}, ParameterVector{input_1, input_2}); - } - comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); - comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); -} - -TEST_F(TransformationTestsF, DivisionByZeroMinimalPatternUnchanged) { - // if eps_value is greater than normalized_fp16_min then leave graph unchanged - const float eps_value = 0.0001f; - { - auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {eps_value}); - auto add = std::make_shared(input_2, eps_const); - auto divide = std::make_shared(input_1, add); - - function = std::make_shared(NodeVector{divide}, ParameterVector{input_1, input_2}); - - manager.register_pass(); - } - - { - auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {eps_value}); - auto add = std::make_shared(input_2, eps_const); - auto divide = std::make_shared(input_1, add); - - function_ref = std::make_shared(NodeVector{divide}, ParameterVector{input_1, input_2}); - } - comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); - comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); -} - -TEST_F(TransformationTestsF, DivisionByZeroInL2NormWithSqrtAndWithMax) { - const float eps_value = 1.e-12f; - { - auto input = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto exp = opset4::Constant::create(element::f32, Shape{}, {2.f}); - auto pow = std::make_shared(input, exp); - auto axes_const = opset4::Constant::create(element::i64, Shape{2}, {0, 1}); - auto reduce_sum = std::make_shared(pow, axes_const); - auto eps_const = opset4::Constant::create(element::f32, Shape{}, {eps_value}); - auto max = std::make_shared(reduce_sum, eps_const); - auto sqrt = std::make_shared(max); - auto divide = std::make_shared(input, sqrt); - - function = std::make_shared(NodeVector{divide}, ParameterVector{input}); - - manager.register_pass(); - } - - { - auto input = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto exp = opset4::Constant::create(element::f32, Shape{}, {2.f}); - auto pow = std::make_shared(input, exp); - auto axes_const = opset4::Constant::create(element::i64, Shape{2}, {0, 1}); - auto reduce_sum = std::make_shared(pow, axes_const); - auto eps_const = opset4::Constant::create(element::f32, Shape{}, {normalized_fp16_min}); - auto max = std::make_shared(reduce_sum, eps_const); - auto sqrt = std::make_shared(max); - auto divide = std::make_shared(input, sqrt); - - function_ref = std::make_shared(NodeVector{divide}, ParameterVector{input}); - } - comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); - comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); -} - -TEST_F(TransformationTestsF, DivisionByZeroInL2NormWithSqrtAndWithAdd) { - const float eps_value = 1.e-12f; - { - auto input = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto exp = opset4::Constant::create(element::f32, Shape{}, {2.f}); - auto pow = std::make_shared(input, exp); - auto axes_const = opset4::Constant::create(element::i64, Shape{2}, {0, 1}); - auto reduce_sum = std::make_shared(pow, axes_const); - auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {eps_value}); - auto add = std::make_shared(reduce_sum, eps_const); - auto sqrt = std::make_shared(add); - auto divide = std::make_shared(input, sqrt); - - function = std::make_shared(NodeVector{divide}, ParameterVector{input}); - - manager.register_pass(); - } - - { - auto input = std::make_shared(element::f32, PartialShape::dynamic(3)); - auto exp = opset4::Constant::create(element::f32, Shape{}, {2.f}); - auto pow = std::make_shared(input, exp); - auto axes_const = opset4::Constant::create(element::i64, Shape{2}, {0, 1}); - auto reduce_sum = std::make_shared(pow, axes_const); - auto eps_const = opset4::Constant::create(element::f32, Shape{1}, {normalized_fp16_min}); - auto add = std::make_shared(reduce_sum, eps_const); - auto sqrt = std::make_shared(add); - auto divide = std::make_shared(input, sqrt); - - function_ref = std::make_shared(NodeVector{divide}, ParameterVector{input}); - } - comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); - comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); -} diff --git a/src/common/transformations/tests/common_optimizations/mark_subgraph_to_keep_in_mixed_precision_test.cpp b/src/common/transformations/tests/common_optimizations/mark_subgraph_to_keep_in_mixed_precision_test.cpp new file mode 100644 index 00000000000..3a8bca535b9 --- /dev/null +++ b/src/common/transformations/tests/common_optimizations/mark_subgraph_to_keep_in_mixed_precision_test.cpp @@ -0,0 +1,1418 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include "common_test_utils/ngraph_test_utils.hpp" +#include "openvino/opsets/opset10.hpp" +#include "openvino/opsets/opset2.hpp" +#include "openvino/pass/manager.hpp" +#include "transformations/common_optimizations/mark_subgraphs_to_keep_in_mixed_precision.hpp" +#include "transformations/rt_info/disable_fp16_compression.hpp" + +using namespace testing; +using namespace ov; +using namespace std; +using namespace ov::opset10; + +TEST(TransformationTests, keep_precission_sensitive_fp32_1) { + shared_ptr model, model_ref; + pass::Manager manager; + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto exp_1 = make_shared(input_1); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto reduce_sum_1 = make_shared(exp_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_sum_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto exp_1 = make_shared(input_1); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto reduce_sum_1 = make_shared(exp_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_sum_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + // marking nodes to be kept in fp32 for mixed precision + disable_fp16_compression(exp_1); + disable_fp16_compression(reduce_sum_1); + disable_fp16_compression(mul_1); + disable_fp16_compression(factor_const); + disable_fp16_compression(factor_const_decompressed); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + } + + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, keep_precission_sensitive_fp32_with_reducemean) { + shared_ptr model, model_ref; + pass::Manager manager; + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto exp_1 = make_shared(input_1); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto reduce_sum_1 = make_shared(exp_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_sum_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto exp_1 = make_shared(input_1); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto reduce_mean_1 = make_shared(exp_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_mean_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + // marking nodes to be kept in fp32 for mixed precision + disable_fp16_compression(exp_1); + disable_fp16_compression(reduce_mean_1); + disable_fp16_compression(mul_1); + disable_fp16_compression(factor_const); + disable_fp16_compression(factor_const_decompressed); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + } + + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, MarkSugraphsToKeepInMixedPrecision_reducesum_without_exp) { + // ReduceSum without Exp is not a precision sensitive case, + // no nodes should be marked with disable_fp16_compression + shared_ptr model, model_ref; + pass::Manager manager; + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto reduce_sum_1 = make_shared(input_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_sum_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto reduce_sum_1 = make_shared(input_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_sum_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + } + + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, MarkNormalizationOps_1) { + shared_ptr model, model_ref; + pass::Manager manager; + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto mvn_1 = make_shared(input_1, reduction_axes, true, 1.0e-8f, op::MVNEpsMode::INSIDE_SQRT); + auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); + auto matmul_1 = make_shared(mvn_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto mvn_1 = make_shared(input_1, reduction_axes, true, 1.0e-8f, op::MVNEpsMode::INSIDE_SQRT); + auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); + auto matmul_1 = make_shared(mvn_1, input_2); + + // marking nodes to be kept in fp32 for mixed precision + disable_fp16_compression(addition_const); + disable_fp16_compression(mvn_1); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + } + + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, MarkNormalizationOps_2) { + shared_ptr model, model_ref; + pass::Manager manager; + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto normalizel2_1 = make_shared(input_1, reduction_axes, 1.0e-8f, ov::op::EpsMode::MAX); + auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); + auto matmul_1 = make_shared(normalizel2_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto normalizel2_1 = make_shared(input_1, reduction_axes, 1.0e-8f, ov::op::EpsMode::MAX); + auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); + auto matmul_1 = make_shared(normalizel2_1, input_2); + + // marking nodes to be kept in fp32 for mixed precision + disable_fp16_compression(addition_const); + disable_fp16_compression(normalizel2_1); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + } + + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, MarkNormalizationOps_3) { + shared_ptr model, model_ref; + pass::Manager manager; + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto mvn_1 = make_shared(input_1, AxisSet{3}, true, 1.0e-8); + auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); + auto matmul_1 = make_shared(mvn_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto mvn_1 = make_shared(input_1, AxisSet{3}, true, 1.0e-8); + auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); + auto matmul_1 = make_shared(mvn_1, input_2); + + // marking nodes to be kept in fp32 for mixed precision + disable_fp16_compression(addition_const); + disable_fp16_compression(mvn_1); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + } + + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, keep_precission_sensitive_fp32_2) { + shared_ptr model, model_ref; + pass::Manager manager; + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto exp_1 = make_shared(input_1); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + + auto unsqueeze_axes = Constant::create(element::i64, Shape{1}, {1}); + auto unsqueeze_1 = make_shared(exp_1, unsqueeze_axes); + auto reduce_sum_1 = make_shared(unsqueeze_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_sum_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto exp_1 = make_shared(input_1); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + + auto unsqueeze_axes = Constant::create(element::i64, Shape{1}, {1}); + auto unsqueeze_1 = make_shared(exp_1, unsqueeze_axes); + auto reduce_sum_1 = make_shared(unsqueeze_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_sum_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + // marking nodes to be kept in fp32 for mixed precision + disable_fp16_compression(exp_1); + disable_fp16_compression(reduce_sum_1); + disable_fp16_compression(mul_1); + disable_fp16_compression(unsqueeze_1); + disable_fp16_compression(factor_const); + disable_fp16_compression(factor_const_decompressed); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + } + + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, keep_precission_sensitive_fp32_3) { + shared_ptr model, model_ref; + pass::Manager manager; + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto exp_1 = make_shared(input_1); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + + auto reduce_sum_1 = make_shared(exp_1, reduction_axes); + auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); + auto add_1 = make_shared(reduce_sum_1, addition_const); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(add_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto exp_1 = make_shared(input_1); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + + auto reduce_sum_1 = make_shared(exp_1, reduction_axes); + auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); + auto add_1 = make_shared(reduce_sum_1, addition_const); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(add_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + // marking nodes to be kept in fp32 for mixed precision + disable_fp16_compression(exp_1); + disable_fp16_compression(reduce_sum_1); + disable_fp16_compression(mul_1); + disable_fp16_compression(add_1); + disable_fp16_compression(addition_const); + disable_fp16_compression(factor_const); + disable_fp16_compression(factor_const_decompressed); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + } + + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + // ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, keep_precission_sensitive_fp32_4) { + shared_ptr model, model_ref; + pass::Manager manager; + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto mvn_1 = make_shared(input_1, reduction_axes, true, 1.0e-8f, op::MVNEpsMode::INSIDE_SQRT); + auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); + auto unsqueeze_1 = make_shared(mvn_1, addition_const); + auto matmul_1 = make_shared(unsqueeze_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto mvn_1 = make_shared(input_1, reduction_axes, true, 1.0e-8f, op::MVNEpsMode::INSIDE_SQRT); + auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); + auto unsqueeze_1 = make_shared(mvn_1, addition_const); + auto matmul_1 = make_shared(unsqueeze_1, input_2); + + // marking nodes to be kept in fp32 for mixed precision + disable_fp16_compression(addition_const); + disable_fp16_compression(unsqueeze_1); + disable_fp16_compression(mvn_1); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + } + + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, keep_precission_sensitive_fp32_5) { + shared_ptr model, model_ref; + pass::Manager manager; + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto normalizel2_1 = make_shared(input_1, reduction_axes, 1.0e-8f, ov::op::EpsMode::MAX); + auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); + auto unsqueeze_1 = make_shared(normalizel2_1, addition_const); + auto matmul_1 = make_shared(unsqueeze_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto normalizel2_1 = make_shared(input_1, reduction_axes, 1.0e-8f, ov::op::EpsMode::MAX); + auto addition_const = Constant::create(element::f32, Shape{1}, {0.1f}); + auto unsqueeze_1 = make_shared(normalizel2_1, addition_const); + auto matmul_1 = make_shared(unsqueeze_1, input_2); + + // marking nodes to be kept in fp32 for mixed precision + disable_fp16_compression(addition_const); + disable_fp16_compression(unsqueeze_1); + disable_fp16_compression(normalizel2_1); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + } + + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, keep_precission_sensitive_fp32_6) { + shared_ptr model, model_ref; + pass::Manager manager; + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto normalizel2_1 = make_shared(input_1, reduction_axes, 1.0e-8f, ov::op::EpsMode::MAX); + auto matmul_1 = make_shared(normalizel2_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto normalizel2_1 = make_shared(input_1, reduction_axes, 1.0e-8f, ov::op::EpsMode::MAX); + auto matmul_1 = make_shared(normalizel2_1, input_2); + + disable_fp16_compression(normalizel2_1); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + } + + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, keep_precission_sensitive_fp32_7) { + shared_ptr model, model_ref; + pass::Manager manager; + // subgraph from t2t-vit-7 + { + auto input_1 = make_shared(element::f32, Shape{1, 3136, 32}); + auto input_2 = make_shared(element::f32, Shape{1, 3136, 32}); + auto input_3 = make_shared(element::f32, Shape{1, 3136, 64, 1}); + auto input_4 = make_shared(element::f32, Shape{128, 64}); + auto exp_1 = make_shared(input_1); + auto exp_2 = make_shared(input_2); + + auto factor_1 = Constant::create(element::f32, Shape{1}, {0.5}); // add decompression + auto mul_1 = make_shared(exp_1, factor_1); + auto factor_2 = Constant::create(element::f32, Shape{1}, {0.5}); + auto mul_2 = make_shared(exp_2, factor_2); + + auto const_unsqueeze_1 = Constant::create(element::i64, Shape{4}, {1, 3136, 1, 32}); + auto unsqueeze_1 = make_shared(mul_1, const_unsqueeze_1, false); + + auto const_unsqueeze_2 = Constant::create(element::i64, Shape{4}, {1, 3136, 1, 32}); + auto unsqueeze_2 = make_shared(mul_2, const_unsqueeze_1, false); + auto reduction_axes_1 = Constant::create(element::i64, Shape{1}, {1}); + auto reduce_sum_1 = make_shared(mul_2, reduction_axes_1, true); + auto mul_3 = make_shared(reduce_sum_1, mul_1); + auto mul_4 = make_shared(input_3, unsqueeze_2); + + auto reduction_axes_2 = Constant::create(element::i64, Shape{1}, {1}); + auto reduce_sum_2 = make_shared(mul_4, reduction_axes_2); + auto reduction_axes_3 = Constant::create(element::i64, Shape{1}, {2}); + auto reduce_sum_3 = make_shared(mul_3, reduction_axes_3, true); + + auto broadcast_to_shape = Constant::create(element::i64, Shape{3}, {1, 1, 1}); + auto broadcast = make_shared(reduce_sum_3, broadcast_to_shape, ov::op::BroadcastType::BIDIRECTIONAL); + auto tile_shape = Constant::create(element::i64, Shape{3}, {1, 1, 64}); + auto tile = make_shared(broadcast, tile_shape); + auto eps_const = Constant::create(element::f32, Shape{1}, {1.e-10}); + auto add_1 = make_shared(tile, eps_const); + + auto const_unsqueeze_3 = Constant::create(element::i64, Shape{4}, {1, 1, 64, 32}); + auto unsqueeze_3 = make_shared(reduce_sum_2, const_unsqueeze_3, false); + auto mul_5 = make_shared(unsqueeze_1, unsqueeze_3); + + auto reduction_axes_4 = Constant::create(element::i64, Shape{1}, {-1}); + auto reduce_sum_4 = make_shared(mul_5, reduction_axes_4); + + auto div_1 = make_shared(reduce_sum_4, add_1); + auto matmul_1 = make_shared(div_1, input_4, false, true); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2, input_3, input_4}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f32, Shape{1, 3136, 32}); + auto input_2 = make_shared(element::f32, Shape{1, 3136, 32}); + auto input_3 = make_shared(element::f32, Shape{1, 3136, 64, 1}); + auto input_4 = make_shared(element::f32, Shape{128, 64}); + auto exp_1 = make_shared(input_1); + auto exp_2 = make_shared(input_2); + + auto factor_1 = Constant::create(element::f32, Shape{1}, {0.5}); // add decompression + auto mul_1 = make_shared(exp_1, factor_1); + auto factor_2 = Constant::create(element::f32, Shape{1}, {0.5}); + auto mul_2 = make_shared(exp_2, factor_2); + + auto const_unsqueeze_1 = Constant::create(element::i64, Shape{4}, {1, 3136, 1, 32}); + auto unsqueeze_1 = make_shared(mul_1, const_unsqueeze_1, false); + + auto const_unsqueeze_2 = Constant::create(element::i64, Shape{4}, {1, 3136, 1, 32}); + auto unsqueeze_2 = make_shared(mul_2, const_unsqueeze_2, false); + auto reduction_axes_1 = Constant::create(element::i64, Shape{1}, {1}); + auto reduce_sum_1 = make_shared(mul_2, reduction_axes_1, true); + auto mul_3 = make_shared(reduce_sum_1, mul_1); + auto mul_4 = make_shared(input_3, unsqueeze_2); + + auto reduction_axes_2 = Constant::create(element::i64, Shape{1}, {1}); + auto reduce_sum_2 = make_shared(mul_4, reduction_axes_2); + auto reduction_axes_3 = Constant::create(element::i64, Shape{1}, {2}); + auto reduce_sum_3 = make_shared(mul_3, reduction_axes_3, true); + + auto broadcast_to_shape = Constant::create(element::i64, Shape{3}, {1, 1, 1}); + auto broadcast = make_shared(reduce_sum_3, broadcast_to_shape, ov::op::BroadcastType::BIDIRECTIONAL); + auto tile_shape = Constant::create(element::i64, Shape{3}, {1, 1, 64}); + auto tile = make_shared(broadcast, tile_shape); + auto eps_const = Constant::create(element::f32, Shape{1}, {1.e-10}); + auto add_1 = make_shared(tile, eps_const); + + auto const_unsqueeze_3 = Constant::create(element::i64, Shape{4}, {1, 1, 64, 32}); + auto unsqueeze_3 = make_shared(reduce_sum_2, const_unsqueeze_3, false); + auto mul_5 = make_shared(unsqueeze_1, unsqueeze_3); + + auto reduction_axes_4 = Constant::create(element::i64, Shape{1}, {-1}); + auto reduce_sum_4 = make_shared(mul_5, reduction_axes_4); + + auto div_1 = make_shared(reduce_sum_4, add_1); + auto matmul_1 = make_shared(div_1, input_4, false, true); + + // marking nodes to be kept in fp32 for mixed precision + disable_fp16_compression(mul_1); + disable_fp16_compression(mul_2); + disable_fp16_compression(mul_3); + disable_fp16_compression(mul_4); + disable_fp16_compression(mul_5); + disable_fp16_compression(unsqueeze_1); + disable_fp16_compression(unsqueeze_2); + disable_fp16_compression(unsqueeze_3); + disable_fp16_compression(reduce_sum_1); + disable_fp16_compression(reduce_sum_2); + disable_fp16_compression(reduce_sum_3); + disable_fp16_compression(reduce_sum_4); + disable_fp16_compression(exp_1); + disable_fp16_compression(exp_2); + disable_fp16_compression(tile); + disable_fp16_compression(eps_const); + disable_fp16_compression(add_1); + disable_fp16_compression(broadcast); + disable_fp16_compression(div_1); + + disable_fp16_compression(factor_1); + disable_fp16_compression(factor_2); + + disable_fp16_compression(broadcast_to_shape); + disable_fp16_compression(tile_shape); + disable_fp16_compression(const_unsqueeze_1); + disable_fp16_compression(const_unsqueeze_2); + disable_fp16_compression(const_unsqueeze_3); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2, input_3, input_4}); + } + + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, DivisionByZeroMinimalPattern) { + shared_ptr model, model_ref; + pass::Manager manager; + + const float eps_value = 1.0e-12f; + { + auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(input_2, eps_const); + auto divide = std::make_shared(input_1, add); + model = std::make_shared(NodeVector{divide}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(input_2, eps_const); + auto divide = std::make_shared(input_1, add); + disable_fp16_compression(divide); + disable_fp16_compression(eps_const); + disable_fp16_compression(add); + + model_ref = std::make_shared(NodeVector{divide}, ParameterVector{input_1, input_2}); + } + + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, PowWithNegativeExponent) { + shared_ptr model, model_ref; + pass::Manager manager; + const float eps_value = 1.0e-12f; + { + auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(input_2, eps_const); + auto pow_exp_const = Constant::create(element::f32, Shape{1}, {-1.77}); + auto pow = std::make_shared(add, pow_exp_const); + auto mul = std::make_shared(input_1, pow); + + model = std::make_shared(NodeVector{mul}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(input_2, eps_const); + auto pow_exp_const = Constant::create(element::f32, Shape{1}, {-1.77}); + auto pow = std::make_shared(add, pow_exp_const); + auto mul = std::make_shared(input_1, pow); + + // marking nodes to be kept in fp32 for mixed precision + disable_fp16_compression(eps_const); + disable_fp16_compression(add); + disable_fp16_compression(pow_exp_const); + disable_fp16_compression(pow); + disable_fp16_compression(mul); + + model_ref = std::make_shared(NodeVector{mul}, ParameterVector{input_1, input_2}); + } + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, PowWithPositiveExponent) { + shared_ptr model, model_ref; + pass::Manager manager; + // graph should be left unchanged + const float eps_value = 1.0e-12f; + { + auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(input_2, eps_const); + auto pow_exp_const = Constant::create(element::f32, Shape{1}, {1.77}); + auto pow = std::make_shared(add, pow_exp_const); + auto mul = std::make_shared(input_1, pow); + + model = std::make_shared(NodeVector{mul}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(input_2, eps_const); + auto pow_exp_const = Constant::create(element::f32, Shape{1}, {1.77}); + auto pow = std::make_shared(add, pow_exp_const); + auto mul = std::make_shared(input_1, pow); + + model_ref = std::make_shared(NodeVector{mul}, ParameterVector{input_1, input_2}); + } + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, DivisionByZeroMinimalPatternUnchanged) { + shared_ptr model, model_ref; + pass::Manager manager; + // if eps_value is greater than normalized_fp16_min then leave graph unchanged + const float eps_value = 0.0001f; + { + auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(input_2, eps_const); + auto divide = std::make_shared(input_1, add); + + model = std::make_shared(NodeVector{divide}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(input_2, eps_const); + auto divide = std::make_shared(input_1, add); + + model_ref = std::make_shared(NodeVector{divide}, ParameterVector{input_1, input_2}); + } + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, DivisionByZeroInL2NormWithSqrtAndWithMax) { + shared_ptr model, model_ref; + pass::Manager manager; + const float eps_value = 1.0e-12f; + { + auto input = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto exp = Constant::create(element::f32, Shape{}, {2.f}); + auto pow = std::make_shared(input, exp); + auto axes_const = Constant::create(element::i64, Shape{2}, {0, 1}); + auto reduce_sum = std::make_shared(pow, axes_const); + auto eps_const = Constant::create(element::f32, Shape{}, {eps_value}); + auto max = std::make_shared(reduce_sum, eps_const); + auto sqrt = std::make_shared(max); + auto divide = std::make_shared(input, sqrt); + + model = std::make_shared(NodeVector{divide}, ParameterVector{input}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto exp = Constant::create(element::f32, Shape{}, {2.f}); + auto pow = std::make_shared(input, exp); + auto axes_const = Constant::create(element::i64, Shape{2}, {0, 1}); + auto reduce_sum = std::make_shared(pow, axes_const); + auto eps_const = Constant::create(element::f32, Shape{}, {eps_value}); + auto max = std::make_shared(reduce_sum, eps_const); + auto sqrt = std::make_shared(max); + auto divide = std::make_shared(input, sqrt); + + // marking nodes to be kept in fp32 for mixed precision + disable_fp16_compression(exp); + disable_fp16_compression(pow); + disable_fp16_compression(reduce_sum); + disable_fp16_compression(max); + disable_fp16_compression(eps_const); + disable_fp16_compression(sqrt); + disable_fp16_compression(divide); + + model_ref = std::make_shared(NodeVector{divide}, ParameterVector{input}); + } + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, DivisionByZeroInL2NormWithSqrtAndWithAdd) { + shared_ptr model, model_ref; + pass::Manager manager; + const float eps_value = 1.e-12f; + { + auto input = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto exp = Constant::create(element::f32, Shape{}, {2.f}); + auto pow = std::make_shared(input, exp); + auto axes_const = Constant::create(element::i64, Shape{2}, {0, 1}); + auto reduce_sum = std::make_shared(pow, axes_const); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(reduce_sum, eps_const); + auto sqrt = std::make_shared(add); + auto divide = std::make_shared(input, sqrt); + + model = std::make_shared(NodeVector{divide}, ParameterVector{input}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto exp = Constant::create(element::f32, Shape{}, {2.f}); + auto pow = std::make_shared(input, exp); + auto axes_const = Constant::create(element::i64, Shape{2}, {0, 1}); + auto reduce_sum = std::make_shared(pow, axes_const); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(reduce_sum, eps_const); + auto sqrt = std::make_shared(add); + auto divide = std::make_shared(input, sqrt); + + // marking nodes to be kept in fp32 for mixed precision + disable_fp16_compression(exp); + disable_fp16_compression(pow); + disable_fp16_compression(sqrt); + disable_fp16_compression(reduce_sum); + disable_fp16_compression(eps_const); + disable_fp16_compression(add); + disable_fp16_compression(divide); + + model_ref = std::make_shared(NodeVector{divide}, ParameterVector{input}); + } + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, MarkReduceOpExpToKeepInMixedPrecision_with_reducesum) { + shared_ptr model, model_ref; + pass::Manager manager; + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto exp_1 = make_shared(input_1); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto reduce_sum_1 = make_shared(exp_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_sum_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto exp_1 = make_shared(input_1); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto reduce_sum_1 = make_shared(exp_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_sum_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + disable_fp16_compression(exp_1); + disable_fp16_compression(mul_1); + disable_fp16_compression(reduce_sum_1); + disable_fp16_compression(factor_const_decompressed); + disable_fp16_compression(factor_const); + } + + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, MarkReduceOpExpToKeepInMixedPrecision_with_reducemean) { + shared_ptr model, model_ref; + pass::Manager manager; + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto exp_1 = make_shared(input_1); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto reduce_mean_1 = make_shared(exp_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_mean_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto exp_1 = make_shared(input_1); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto reduce_mean_1 = make_shared(exp_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_mean_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + disable_fp16_compression(exp_1); + disable_fp16_compression(mul_1); + disable_fp16_compression(reduce_mean_1); + disable_fp16_compression(factor_const_decompressed); + disable_fp16_compression(factor_const); + } + + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, MarkReduceOpExpToKeepInMixedPrecision_reducesum_without_exp) { + // ReduceSum without Exp is not a precision sensitive case + shared_ptr model, model_ref; + pass::Manager manager; + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto reduce_sum_1 = make_shared(input_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_sum_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + auto reduce_sum_1 = make_shared(input_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_sum_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + } + + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, MarkReduceOpExpToKeepInMixedPrecision_reducesum_exp_through_unsqueeze) { + shared_ptr model, model_ref; + pass::Manager manager; + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto exp_1 = make_shared(input_1); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + + auto unsqueeze_axes = Constant::create(element::i64, Shape{1}, {1}); + auto unsqueeze_1 = make_shared(exp_1, unsqueeze_axes); + auto reduce_sum_1 = make_shared(unsqueeze_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_sum_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + model = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto exp_1 = make_shared(input_1); + auto input_2 = make_shared(element::f32, Shape{1, 3, 224, 224}); + auto reduction_axes = Constant::create(element::i64, Shape{1}, {-1}); + + auto unsqueeze_axes = Constant::create(element::i64, Shape{1}, {1}); + auto unsqueeze_1 = make_shared(exp_1, unsqueeze_axes); + auto reduce_sum_1 = make_shared(unsqueeze_1, reduction_axes); + + auto factor_const = Constant::create(element::f16, Shape{1}, {-1}); + auto factor_const_decompressed = make_shared(factor_const, element::f32); + auto mul_1 = make_shared(reduce_sum_1, factor_const_decompressed); + auto matmul_1 = make_shared(mul_1, input_2); + + model_ref = make_shared(NodeVector{matmul_1}, ParameterVector{input_1, input_2}); + disable_fp16_compression(exp_1); + disable_fp16_compression(mul_1); + disable_fp16_compression(reduce_sum_1); + disable_fp16_compression(factor_const_decompressed); + disable_fp16_compression(factor_const); + disable_fp16_compression(unsqueeze_1); + } + + const FunctionsComparator func_comparator = + FunctionsComparator::with_default().enable(FunctionsComparator::RUNTIME_KEYS); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = func_comparator(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = func_comparator(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +// check marking of Division with eps +TEST(TransformationTests, MarkDivWithEps) { + const float eps_value = 1.e-12f; + shared_ptr model, model_ref; + pass::Manager manager; + { + auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(input_2, eps_const); + auto divide = std::make_shared(input_1, add); + + model = std::make_shared(NodeVector{divide}, ParameterVector{input_1, input_2}); + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(input_2, eps_const); + auto divide = std::make_shared(input_1, add); + disable_fp16_compression(divide); + disable_fp16_compression(add); + disable_fp16_compression(eps_const); + + model_ref = std::make_shared(NodeVector{divide}, ParameterVector{input_1, input_2}); + } + const auto fc = FunctionsComparator::with_default() + .enable(FunctionsComparator::PRECISIONS) + .enable(FunctionsComparator::RUNTIME_KEYS) + .enable(FunctionsComparator::CONST_VALUES); + + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = fc(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = fc(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, MarkDivWithEpsToKeepInMixedPrecision_PowWithNegativeExponent) { + const float eps_value = 1.e-12f; + shared_ptr model, model_ref; + pass::Manager manager; + { + auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(input_2, eps_const); + auto pow_exp_const = Constant::create(element::f32, Shape{1}, {-1.77}); + auto pow = std::make_shared(add, pow_exp_const); + auto mul = std::make_shared(input_1, pow); + + model = std::make_shared(NodeVector{mul}, ParameterVector{input_1, input_2}); + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(input_2, eps_const); + auto pow_exp_const = Constant::create(element::f32, Shape{1}, {-1.77}); + auto pow = std::make_shared(add, pow_exp_const); + auto mul = std::make_shared(input_1, pow); + disable_fp16_compression(mul); + disable_fp16_compression(eps_const); + disable_fp16_compression(add); + disable_fp16_compression(pow_exp_const); + disable_fp16_compression(pow); + + model_ref = std::make_shared(NodeVector{mul}, ParameterVector{input_1, input_2}); + } + const auto fc = FunctionsComparator::with_default() + .enable(FunctionsComparator::PRECISIONS) + .enable(FunctionsComparator::RUNTIME_KEYS) + .enable(FunctionsComparator::CONST_VALUES); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = fc(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = fc(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, MarkDivWithEpsToKeepInMixedPrecision_PowWithPositiveExponent) { + // graph should be left unchanged because of the positive z in pow + const float eps_value = 1.e-12f; + shared_ptr model, model_ref; + pass::Manager manager; + { + auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(input_2, eps_const); + auto pow_exp_const = Constant::create(element::f32, Shape{1}, {1.77}); + auto pow = std::make_shared(add, pow_exp_const); + auto mul = std::make_shared(input_1, pow); + + model = std::make_shared(NodeVector{mul}, ParameterVector{input_1, input_2}); + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(input_2, eps_const); + auto pow_exp_const = Constant::create(element::f32, Shape{1}, {1.77}); + auto pow = std::make_shared(add, pow_exp_const); + auto mul = std::make_shared(input_1, pow); + + model_ref = std::make_shared(NodeVector{mul}, ParameterVector{input_1, input_2}); + } + const auto fc = FunctionsComparator::with_default() + .enable(FunctionsComparator::PRECISIONS) + .enable(FunctionsComparator::RUNTIME_KEYS) + .enable(FunctionsComparator::CONST_VALUES); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = fc(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = fc(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, MarkDivWithEpsToKeepInMixedPrecision_MinimalPatternUnchanged) { + // if eps_value is greater than normalized_fp16_min then leave graph unchanged + const float eps_value = 0.0001f; + shared_ptr model, model_ref; + pass::Manager manager; + { + auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(input_2, eps_const); + auto divide = std::make_shared(input_1, add); + + model = std::make_shared(NodeVector{divide}, ParameterVector{input_1, input_2}); + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input_1 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto input_2 = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(input_2, eps_const); + auto divide = std::make_shared(input_1, add); + + model_ref = std::make_shared(NodeVector{divide}, ParameterVector{input_1, input_2}); + } + const auto fc = FunctionsComparator::with_default() + .enable(FunctionsComparator::PRECISIONS) + .enable(FunctionsComparator::RUNTIME_KEYS) + .enable(FunctionsComparator::CONST_VALUES); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = fc(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = fc(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, MarkDivWithEpsToKeepInMixedPrecision_InL2NormWithSqrtAndWithMax) { + const float eps_value = 1.e-12f; + shared_ptr model, model_ref; + pass::Manager manager; + { + auto input = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto exp = Constant::create(element::f32, Shape{}, {2.f}); + auto pow = std::make_shared(input, exp); + auto axes_const = Constant::create(element::i64, Shape{2}, {0, 1}); + auto reduce_sum = std::make_shared(pow, axes_const); + auto eps_const = Constant::create(element::f32, Shape{}, {eps_value}); + auto max = std::make_shared(reduce_sum, eps_const); + auto sqrt = std::make_shared(max); + auto divide = std::make_shared(input, sqrt); + + model = std::make_shared(NodeVector{divide}, ParameterVector{input}); + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto exp = Constant::create(element::f32, Shape{}, {2.f}); + auto pow = std::make_shared(input, exp); + auto axes_const = Constant::create(element::i64, Shape{2}, {0, 1}); + auto reduce_sum = std::make_shared(pow, axes_const); + auto eps_const = Constant::create(element::f32, Shape{}, {eps_value}); + auto max = std::make_shared(reduce_sum, eps_const); + auto sqrt = std::make_shared(max); + auto divide = std::make_shared(input, sqrt); + disable_fp16_compression(divide); + + disable_fp16_compression(exp); + disable_fp16_compression(pow); + disable_fp16_compression(reduce_sum); + disable_fp16_compression(eps_const); + disable_fp16_compression(max); + disable_fp16_compression(sqrt); + + model_ref = std::make_shared(NodeVector{divide}, ParameterVector{input}); + } + const auto fc = FunctionsComparator::with_default() + .enable(FunctionsComparator::PRECISIONS) + .enable(FunctionsComparator::RUNTIME_KEYS) + .enable(FunctionsComparator::CONST_VALUES); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = fc(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = fc(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +} + +TEST(TransformationTests, MarkDivWithEpsToKeepInMixedPrecision_InL2NormWithSqrtAndWithAdd) { + const float eps_value = 1.e-12f; + shared_ptr model, model_ref; + pass::Manager manager; + { + auto input = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto exp = Constant::create(element::f32, Shape{}, {2.f}); + auto pow = std::make_shared(input, exp); + auto axes_const = Constant::create(element::i64, Shape{2}, {0, 1}); + auto reduce_sum = std::make_shared(pow, axes_const); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(reduce_sum, eps_const); + auto sqrt = std::make_shared(add); + auto divide = std::make_shared(input, sqrt); + + model = std::make_shared(NodeVector{divide}, ParameterVector{input}); + manager.register_pass(); + manager.run_passes(model); + } + + { + auto input = std::make_shared(element::f32, PartialShape::dynamic(3)); + auto exp = Constant::create(element::f32, Shape{}, {2.f}); + auto pow = std::make_shared(input, exp); + auto axes_const = Constant::create(element::i64, Shape{2}, {0, 1}); + auto reduce_sum = std::make_shared(pow, axes_const); + auto eps_const = Constant::create(element::f32, Shape{1}, {eps_value}); + auto add = std::make_shared(reduce_sum, eps_const); + auto sqrt = std::make_shared(add); + auto divide = std::make_shared(input, sqrt); + disable_fp16_compression(divide); + + disable_fp16_compression(exp); + disable_fp16_compression(pow); + disable_fp16_compression(reduce_sum); + disable_fp16_compression(eps_const); + disable_fp16_compression(add); + disable_fp16_compression(sqrt); + + model_ref = std::make_shared(NodeVector{divide}, ParameterVector{input}); + } + const auto fc = FunctionsComparator::with_default() + .enable(FunctionsComparator::PRECISIONS) + .enable(FunctionsComparator::RUNTIME_KEYS) + .enable(FunctionsComparator::CONST_VALUES); + // need to compare twice to ensure that no extra nodes are marked + FunctionsComparator::Result result = fc(model_ref, model); + ASSERT_TRUE(result.valid) << result.message; + result = fc(model, model_ref); + ASSERT_TRUE(result.valid) << result.message; +}