From 456347e1b6a2aba352e13b6221752963acf3391d Mon Sep 17 00:00:00 2001 From: Vladislav Golubev Date: Thu, 13 Jan 2022 10:32:44 +0300 Subject: [PATCH] [Transformations] ConvertDivide disabled transformation for divide ops from precision sensitive subgraphs (#9561) * [Transformations] Skip ConvertDivide if Divide is on fp16 shapeOf subgraph * lambda definition moved out of loop * review fixes --- .../mark_precision_sensitive_divides.hpp | 28 +++++++++++ .../rt_info/nonconvertible_divide.hpp | 34 ++++++++++++++ .../include/transformations/utils/utils.hpp | 4 ++ .../common_optimizations.cpp | 3 ++ .../mark_precision_sensitive_divides.cpp | 47 +++++++++++++++++++ .../mark_precision_sensitive_subgraphs.cpp | 35 +++----------- .../op_conversions/convert_divide.cpp | 6 ++- .../rt_info/nonconvertible_divide.cpp | 20 ++++++++ .../src/transformations/utils/utils.cpp | 26 ++++++++++ .../transformations/convert_divide.cpp | 26 ++++++++++ 10 files changed, 199 insertions(+), 30 deletions(-) create mode 100644 src/common/transformations/include/transformations/common_optimizations/mark_precision_sensitive_divides.hpp create mode 100644 src/common/transformations/include/transformations/rt_info/nonconvertible_divide.hpp create mode 100644 src/common/transformations/src/transformations/common_optimizations/mark_precision_sensitive_divides.cpp create mode 100644 src/common/transformations/src/transformations/rt_info/nonconvertible_divide.cpp diff --git a/src/common/transformations/include/transformations/common_optimizations/mark_precision_sensitive_divides.hpp b/src/common/transformations/include/transformations/common_optimizations/mark_precision_sensitive_divides.hpp new file mode 100644 index 00000000000..73826217d9a --- /dev/null +++ b/src/common/transformations/include/transformations/common_optimizations/mark_precision_sensitive_divides.hpp @@ -0,0 +1,28 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/pass/pass.hpp" +#include "transformations_visibility.hpp" + +namespace ov { +namespace pass { + +class TRANSFORMATIONS_API MarkPrecisionSensitiveDivides; + +} // namespace pass +} // namespace ov + +/** + * @ingroup ie_transformation_common_api + * @brief MarkPrecisionSensitiveDivides transformation marks the Divide fp16 layers + * inside the subgraph starting from precision-sensitive input and ending at + * the ShapeOf node as disabled for ConvertDivide transformation. + */ +class ov::pass::MarkPrecisionSensitiveDivides : public ModelPass { +public: + OPENVINO_RTTI("MarkPrecisionSensitiveDivides", "0"); + bool run_on_model(const std::shared_ptr& m) override; +}; diff --git a/src/common/transformations/include/transformations/rt_info/nonconvertible_divide.hpp b/src/common/transformations/include/transformations/rt_info/nonconvertible_divide.hpp new file mode 100644 index 00000000000..4c29ee2ccc4 --- /dev/null +++ b/src/common/transformations/include/transformations/rt_info/nonconvertible_divide.hpp @@ -0,0 +1,34 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/core/node.hpp" +#include "openvino/core/runtime_attribute.hpp" +#include "transformations_visibility.hpp" + + +namespace ov { + +TRANSFORMATIONS_API void disable_divide_conversion(const std::shared_ptr& node); + +TRANSFORMATIONS_API void enable_divide_conversion(const std::shared_ptr& node); + +TRANSFORMATIONS_API bool divide_is_nonconvertible(const std::shared_ptr& node); + +/** + * @ingroup ie_runtime_attr_api + * @brief NonconvertibleDivide class represents runtime info attribute that marks + * a Divide as prohibitted to transform it to power. + */ +class TRANSFORMATIONS_API NonconvertibleDivide : public RuntimeAttribute { +public: + OPENVINO_RTTI("nonconvertable_divide", "0"); + + NonconvertibleDivide() = default; + + bool is_copyable() const override { return false; } +}; + +} // namespace ov diff --git a/src/common/transformations/include/transformations/utils/utils.hpp b/src/common/transformations/include/transformations/utils/utils.hpp index 7d6da94f75c..576f7d127dd 100644 --- a/src/common/transformations/include/transformations/utils/utils.hpp +++ b/src/common/transformations/include/transformations/utils/utils.hpp @@ -180,6 +180,10 @@ TRANSFORMATIONS_API std::shared_ptr clone_try_fold(const std::shared_ptr& node, + std::unordered_set>& visited, + std::function)> func); + template std::shared_ptr make_try_fold(Args&&... args) { auto unary_output_node = std::make_shared(std::forward(args)...); diff --git a/src/common/transformations/src/transformations/common_optimizations/common_optimizations.cpp b/src/common/transformations/src/transformations/common_optimizations/common_optimizations.cpp index d548824fe06..8f58f3446d7 100644 --- a/src/common/transformations/src/transformations/common_optimizations/common_optimizations.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/common_optimizations.cpp @@ -53,6 +53,7 @@ #include "transformations/common_optimizations/convert_compression_only_to_legacy.hpp" #include #include "transformations/common_optimizations/matmul_multiply_fusion.hpp" +#include "transformations/common_optimizations/mark_precision_sensitive_divides.hpp" #include "transformations/op_conversions/bidirectional_sequences_decomposition.hpp" #include "transformations/op_conversions/convert_pad_to_group_conv.hpp" #include "transformations/op_conversions/convert_divide.hpp" @@ -117,6 +118,8 @@ bool ngraph::pass::CommonOptimizations::run_on_model(const std::shared_ptr(); + manager.register_pass(); + // TODO: move to KMB manager.register_pass(); diff --git a/src/common/transformations/src/transformations/common_optimizations/mark_precision_sensitive_divides.cpp b/src/common/transformations/src/transformations/common_optimizations/mark_precision_sensitive_divides.cpp new file mode 100644 index 00000000000..ed982bbba6f --- /dev/null +++ b/src/common/transformations/src/transformations/common_optimizations/mark_precision_sensitive_divides.cpp @@ -0,0 +1,47 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/common_optimizations/mark_precision_sensitive_divides.hpp" + +#include +#include + +#include "openvino/op/util/precision_sensitive_attribute.hpp" +#include "openvino/opsets/opset8.hpp" +#include "transformations/rt_info/nonconvertible_divide.hpp" +#include "transformations/utils/utils.hpp" + +bool ov::pass::MarkPrecisionSensitiveDivides::run_on_model(const std::shared_ptr& m) { + std::deque> nodes; + std::unordered_set> visited; + for (auto& r : m->get_results()) + nodes.push_back(r); + for (auto& r : m->get_sinks()) + nodes.emplace_back(r); + + auto markup_func = [](std::shared_ptr node) { + if (ov::is_type(node) && node->get_output_element_type(0) == ngraph::element::f16) { + ov::disable_divide_conversion(node); + } + }; + + while (!nodes.empty()) { + auto curr_node = nodes.front(); + nodes.pop_front(); + if (visited.count(curr_node)) + continue; + for (auto& input : curr_node->inputs()) { + if (ov::is_precision_sensitive(input)) + ngraph::op::util::visit_shape_path(input.get_source_output().get_node_shared_ptr(), visited, markup_func); + } + visited.insert(curr_node); + + for (auto& input_value : curr_node->input_values()) { + // continue searching + const auto& input_node = input_value.get_node_shared_ptr(); + nodes.push_front(input_node); + } + } + return true; +} diff --git a/src/common/transformations/src/transformations/common_optimizations/mark_precision_sensitive_subgraphs.cpp b/src/common/transformations/src/transformations/common_optimizations/mark_precision_sensitive_subgraphs.cpp index 88511d8b201..ea9ba40d890 100644 --- a/src/common/transformations/src/transformations/common_optimizations/mark_precision_sensitive_subgraphs.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/mark_precision_sensitive_subgraphs.cpp @@ -17,33 +17,6 @@ using namespace std; -namespace { -void visit_shape_path(const shared_ptr& node, unordered_set>& visited) { - if (!node) - return; - visited.insert(node); - deque> nodes{node}; - while (!nodes.empty()) { - auto curr_node = nodes.front(); - nodes.pop_front(); - // Do not check if already visited - if (ov::is_type(curr_node) || ov::is_type(curr_node)) { - continue; - } - visited.insert(curr_node); - if (ov::is_type(curr_node)) { - ov::disable_fp16_compression(curr_node); - } else { - for (auto& input_value : curr_node->input_values()) { - // continue searching - const auto& input_node = input_value.get_node_shared_ptr(); - nodes.push_front(input_node); - } - } - } -} -} // namespace - bool ov::pass::MarkPrecisionSensitiveSubgraphs::run_on_model(const std::shared_ptr& f) { deque> nodes; unordered_set> visited; @@ -52,6 +25,12 @@ bool ov::pass::MarkPrecisionSensitiveSubgraphs::run_on_model(const std::shared_p for (auto& r : f->get_sinks()) nodes.emplace_back(r); + auto markup_func = [](shared_ptr node) { + if (ov::is_type(node)) { + ov::disable_fp16_compression(node); + } + }; + while (!nodes.empty()) { auto curr_node = nodes.front(); nodes.pop_front(); @@ -59,7 +38,7 @@ bool ov::pass::MarkPrecisionSensitiveSubgraphs::run_on_model(const std::shared_p continue; for (auto& input : curr_node->inputs()) { if (ov::is_precision_sensitive(input)) - visit_shape_path(input.get_source_output().get_node_shared_ptr(), visited); + ngraph::op::util::visit_shape_path(input.get_source_output().get_node_shared_ptr(), visited, markup_func); } visited.insert(curr_node); diff --git a/src/common/transformations/src/transformations/op_conversions/convert_divide.cpp b/src/common/transformations/src/transformations/op_conversions/convert_divide.cpp index 07c3624839b..79862b3186e 100644 --- a/src/common/transformations/src/transformations/op_conversions/convert_divide.cpp +++ b/src/common/transformations/src/transformations/op_conversions/convert_divide.cpp @@ -14,6 +14,8 @@ #include #include +#include "transformations/rt_info/nonconvertible_divide.hpp" + NGRAPH_RTTI_DEFINITION(ngraph::pass::ConvertDivide, "ConvertDivide", 0); NGRAPH_RTTI_DEFINITION(ngraph::pass::ConvertDivideWithConstant, "ConvertDivideWithConstant", 0); @@ -21,8 +23,8 @@ namespace { bool convert_divide(std::shared_ptr node) { auto div = std::dynamic_pointer_cast(node); // We can not apply this transformation in case with integer input data type - if (!div || div->get_input_element_type(0).is_integral() - || div->get_input_element_type(1).is_integral()) { + if (!div || ov::divide_is_nonconvertible(div) + || div->get_input_element_type(0).is_integral()) { return false; } diff --git a/src/common/transformations/src/transformations/rt_info/nonconvertible_divide.cpp b/src/common/transformations/src/transformations/rt_info/nonconvertible_divide.cpp new file mode 100644 index 00000000000..833bce59e89 --- /dev/null +++ b/src/common/transformations/src/transformations/rt_info/nonconvertible_divide.cpp @@ -0,0 +1,20 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/rt_info/nonconvertible_divide.hpp" + +void ov::disable_divide_conversion(const std::shared_ptr& node) { + auto& rt_info = node->get_rt_info(); + rt_info[NonconvertibleDivide::get_type_info_static()] = NonconvertibleDivide{}; +} + +void ov::enable_divide_conversion(const std::shared_ptr& node) { + auto& rt_info = node->get_rt_info(); + rt_info.erase(NonconvertibleDivide::get_type_info_static()); +} + +bool ov::divide_is_nonconvertible(const std::shared_ptr& node) { + const auto& rt_info = node->get_rt_info(); + return rt_info.count(NonconvertibleDivide::get_type_info_static()); +} diff --git a/src/common/transformations/src/transformations/utils/utils.cpp b/src/common/transformations/src/transformations/utils/utils.cpp index 602cf06c0d2..5f98f8f2992 100644 --- a/src/common/transformations/src/transformations/utils/utils.cpp +++ b/src/common/transformations/src/transformations/utils/utils.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -177,6 +178,31 @@ bool shapes_equal_except_dynamic_expected_batch(const ngraph::PartialShape& expe } } +void visit_shape_path(const std::shared_ptr& node, + std::unordered_set>& visited, + std::function)> func) { + if (!node) + return; + visited.insert(node); + std::deque> nodes{node}; + while (!nodes.empty()) { + auto curr_node = nodes.front(); + nodes.pop_front(); + // Do not check if already visited + if (ngraph::is_type(curr_node) || ngraph::is_type(curr_node)) { + continue; + } + + visited.insert(curr_node); + func(curr_node); + for (auto& input_value : curr_node->input_values()) { + // continue searching + const auto& input_node = input_value.get_node_shared_ptr(); + nodes.push_front(input_node); + } + } +} + } // namespace util } // namespace op } // namespace ngraph diff --git a/src/tests/functional/inference_engine/transformations/convert_divide.cpp b/src/tests/functional/inference_engine/transformations/convert_divide.cpp index b4df08a778b..0b9d96a8ac2 100644 --- a/src/tests/functional/inference_engine/transformations/convert_divide.cpp +++ b/src/tests/functional/inference_engine/transformations/convert_divide.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -122,3 +123,28 @@ TEST_F(TransformationTestsF, ConvertDivideWithConstantNegative) { function_ref = std::make_shared(ngraph::NodeVector{divide}, ngraph::ParameterVector{data1, data2}); } } + +TEST_F(TransformationTestsF, ConvertDivideFP16ShapeOfSubgraphNegative) { + { + auto data = std::make_shared(ngraph::element::f16, ngraph::Shape{1, 3, 22, 22}); + auto gather = ngraph::op::util::node_to_get_shape_value_of_indices_from_shape_source(data, {2, 3}); + auto convert = std::make_shared(gather, ngraph::element::f16); + auto divide_constant = ngraph::opset1::Constant::create(ngraph::element::f16, ngraph::Shape{1}, {0.5}); + auto divide = std::make_shared(convert, divide_constant); + auto convert_after = std::make_shared(divide, ngraph::element::i32); + + ngraph::opset1::Interpolate::Attributes interp_attr; + interp_attr.antialias = false; + interp_attr.axes = {2, 3}; + interp_attr.mode = "nearest"; + interp_attr.pads_begin = {0, 0, 0, 0}; + interp_attr.pads_end = {0, 0, 0, 0}; + + auto interpolate = std::make_shared(data, convert_after, interp_attr); + + function = std::make_shared(ngraph::NodeVector{interpolate}, ngraph::ParameterVector{data}); + + ov::pass::MarkPrecisionSensitiveDivides().run_on_model(function); + manager.register_pass(); + } +}