Efficient FP32 -> FP16 conversion for convert_precision, save_model, ovc and mo (#18988)

* WIP Postpone fp16 in CompressFloatConstantsImpl

* Apply suggestions from code review

Co-authored-by: Ilya Lavrenov <ilya.lavrenov@intel.com>

* WIP: Compression to FP16 in Serialize

* Prepared for efficient fp32 to fp16 conversion

* Update src/core/reference/src/runtime/reference/convert.cpp

* Called real slow reference implementations in the place where the optimized versions are supposed to be implemented

* Code style

* Fixed 0 values in the fast f64 to f16 compression

* Optimized convert_from_f32_to_f16_with_clamp

* Added optimized f32->f16 instance of change_constant_precision

* compression transformation Python test

* use tmp dir, minor corrections

* Update src/bindings/python/tests/test_transformations/test_compression.py

* Update src/bindings/python/tests/test_transformations/test_compression.py

* style fix

* define rt_info for postponed_fp16_compression

* remove redundant class

* fix temp dir for Win in test_compression.py

* update definitions in convert.hpp

* Update implementation in convert.cpp

* Update serialize.cpp

* Update compress_float_constants.cpp

* added macros for ARM/non_x86 in convert.cpp

* fix macros in convert.cpp

* change fixme placement in serialize.cpp

* style_fix

* Update src/core/reference/src/runtime/reference/convert.cpp

* style_fix

* Optimized count_out_of_f16_range

* Code style

* Revert unused

* Update src/core/src/pass/serialize.cpp

Co-authored-by: Ilya Lavrenov <ilya.lavrenov@intel.com>

* Update src/core/reference/src/runtime/reference/convert.cpp

Co-authored-by: Ilya Lavrenov <ilya.lavrenov@intel.com>

* use optimized convert_from_f32_to_f16_with_clamp for non postponed

* minor corrections

* Update src/common/transformations/src/transformations/common_optimizations/compress_float_constants.cpp

* Update compress_float_constants.cpp

* Switched mo and ovc to save_model instead of serialize to leverage performance improvements in fp32->fp16

* Applied minor code imporvements to address review feedback

* Minor changes in code

* Update tools/ovc/openvino/tools/ovc/main.py

* Apply suggestions from code review

* Fixed failed test in case when both usual xml compression and fp16 compression are applied simultaneously (disabled for now)

* Added description for CompressFloatConstantImpl postponed parameter

* Description of postponed parameter for CompressFloatConstants

* Reverted switching to save_model in mo as the compression can be applied not only via CLI and old code should be kept for Python path (not applicable for ovc)

* Removed remining committed test artefacts and reverted remaining changes in mo

---------

Co-authored-by: Ilya Lavrenov <ilya.lavrenov@intel.com>
Co-authored-by: dmitrygo <dmitry.gorokhov@intel.com>
Co-authored-by: Vladimir Paramuzov <vladimir.paramuzov@intel.com>
Co-authored-by: Pavel Esir <pavel.esir@intel.com>
Co-authored-by: Pavel Esir <pavel.esir@gmail.com>

src/common/transformations/include/transformations/common_optimizations/compress_float_constants.hpp

@@ -24,7 +24,13 @@ class TRANSFORMATIONS_API CompressFloatConstants;
 class ov::pass::CompressFloatConstantsImpl : public ov::pass::MatcherPass {
 public:
     OPENVINO_RTTI("CompressFloatConstantsImpl", "0");
-    CompressFloatConstantsImpl();
+    /// @brief Transformation constructor
+    /// @param postponed If true then the transformation won't compress the constants
+    ///                  keeping them in the original type but still will insert Converts. This is
+    ///                  a special mode of operation that requires another transformation to
+    ///                  apply a real compression on constants. Constants eligible for
+    ///                  postponed compression are marked with a special rt_info tag.
+    CompressFloatConstantsImpl(bool postponed = false);
 };
 
 /**
@@ -44,8 +50,10 @@ public:
 class ov::pass::CompressFloatConstants : public ov::pass::GraphRewrite {
 public:
     OPENVINO_RTTI("CompressFloatConstants", "0");
-    CompressFloatConstants() {
-        add_matcher<ov::pass::CompressFloatConstantsImpl>();
+    /// @brief Transformation constructor
+    /// @param postponed Postponed compression, see ov::pass::CompressFloatConstantsImpl for details.
+    CompressFloatConstants(bool postponed = false) {
+        add_matcher<ov::pass::CompressFloatConstantsImpl>(postponed);
         add_matcher<ov::pass::AddOldApiMapToParameters>();
     }
 };

src/common/transformations/include/transformations/rt_info/disable_fp16_compression.hpp

@@ -16,6 +16,12 @@ TRANSFORMATIONS_API void enable_fp16_compression(const std::shared_ptr<Node>& no
 
 TRANSFORMATIONS_API bool fp16_compression_is_disabled(const std::shared_ptr<const Node>& node);
 
+TRANSFORMATIONS_API void postpone_fp16_compression(RTMap& rt_info);
+
+TRANSFORMATIONS_API bool is_fp16_compression_postponed(const RTMap& rt_info);
+
+TRANSFORMATIONS_API void do_not_postpone_fp16_compression(RTMap& rt_info);
+
 /**
  * @ingroup ie_runtime_attr_api
  * @brief DisableFP16Compression class represents runtime info attribute that marks operation

src/common/transformations/src/transformations/common_optimizations/compress_float_constants.cpp

@@ -5,6 +5,7 @@
 #include "transformations/common_optimizations/compress_float_constants.hpp"
 
 #include "itt.hpp"
+#include "ngraph/runtime/reference/convert.hpp"
 #include "openvino/core/rt_info.hpp"
 #include "openvino/op/constant.hpp"
 #include "openvino/op/convert.hpp"
@@ -16,7 +17,8 @@
 
 namespace {
 template <ov::element::Type_t PREC_FROM>
-std::shared_ptr<ov::Node> change_constant_precision_to_fp16(std::shared_ptr<ov::op::v0::Constant>& constant) {
+std::shared_ptr<ov::Node> change_constant_precision_to_fp16(std::shared_ptr<ov::op::v0::Constant>& constant,
+                                                            bool postponed = false) {
     using src_type = typename ov::element_type_traits<PREC_FROM>::value_type;
 
     const auto* src_data = constant->get_data_ptr<src_type>();
@@ -24,9 +26,10 @@ std::shared_ptr<ov::Node> change_constant_precision_to_fp16(std::shared_ptr<ov::
 
     auto new_constant = std::make_shared<ov::op::v0::Constant>(ov::element::f16, constant->get_shape());
     auto* dst_data = const_cast<ov::float16*>(reinterpret_cast<const ov::float16*>(new_constant->get_data_ptr()));
-    if (dst_data == nullptr)
+    if (!dst_data || !size)
         return nullptr;
 
+    // slow implementation: is used when optimized ones are not available: f64 or for ARM (both for f64 and f32)
     int num_out_of_range = 0;
     for (size_t i = 0; i < size; ++i) {
         // if abs value is smaller than the smallest positive fp16, but not zero
@@ -44,18 +47,24 @@ std::shared_ptr<ov::Node> change_constant_precision_to_fp16(std::shared_ptr<ov::
     }
 
     // if more than 75% of a FP32 constant do not fit into FP16 keep in FP32
-    float keep_threshold = 0.75f;
-    float out_of_range_proportion = static_cast<float>(num_out_of_range) / static_cast<float>(size);
+    const float keep_threshold = 0.75f;
+    const float out_of_range_proportion = static_cast<float>(num_out_of_range) / static_cast<float>(size);
 
     if (out_of_range_proportion >= keep_threshold) {
         return nullptr;
     }
 
-    return new_constant;
+    if (postponed) {
+        // dispose just converted constant to avoid allocation too much memory
+        // it will be converted again while serialization
+        return constant;
+    } else {
+        return new_constant;
+    }
 }
 }  // namespace
 
-ov::pass::CompressFloatConstantsImpl::CompressFloatConstantsImpl() {
+ov::pass::CompressFloatConstantsImpl::CompressFloatConstantsImpl(bool postponed) {
     MATCHER_SCOPE(CompressFloatConstantsImpl);
     auto const_pattern = pattern::wrap_type<ov::op::v0::Constant>();
 
@@ -72,26 +81,68 @@ ov::pass::CompressFloatConstantsImpl::CompressFloatConstantsImpl() {
 
         auto c_type = const_node->get_element_type();
         std::shared_ptr<ov::Node> new_const;
+
+#if !defined(OPENVINO_ARCH_X86) && !defined(OPENVINO_ARCH_X86_64)
         if (c_type == ov::element::f32) {
-            new_const = change_constant_precision_to_fp16<ov::element::Type_t::f32>(const_node);
+            new_const = change_constant_precision_to_fp16<ov::element::Type_t::f32>(const_node, postponed);
         } else if (c_type == ov::element::f64) {
-            new_const = change_constant_precision_to_fp16<ov::element::Type_t::f64>(const_node);
+            new_const = change_constant_precision_to_fp16<ov::element::Type_t::f64>(const_node, postponed);
+        }
+        if (!new_const)  // if out of range > threshold -> then new_const == nullptr
+            return false;
+#else
+        if (c_type == ov::element::f32) {
+            auto size = shape_size(const_node->get_output_shape(0));
+            if (size == 0)
+                return false;
+            auto num_out_of_range =
+                ngraph::runtime::reference::count_out_of_f16_range(const_node->get_data_ptr<ov::element::f32>(), size);
+
+            // if more than 75% of a FP32 constant do not fit into FP16 keep in FP32
+            const float keep_threshold = 0.75f;
+            const float out_of_range_proportion = static_cast<float>(num_out_of_range) / static_cast<float>(size);
+            if (out_of_range_proportion >= keep_threshold)
+                return false;
+
+            if (postponed) {
+                new_const = const_node;
+            } else {
+                const auto* src_data = const_node->get_data_ptr<float>();
+                auto compressed_const =
+                    std::make_shared<ov::op::v0::Constant>(ov::element::f16, const_node->get_shape());
+                auto* dst_data =
+                    const_cast<ov::float16*>(reinterpret_cast<const ov::float16*>(compressed_const->get_data_ptr()));
+                OPENVINO_ASSERT(dst_data);
+                ngraph::runtime::reference::convert_from_f32_to_f16_with_clamp(src_data, dst_data, size);
+                new_const = compressed_const;
+            }
+        } else if (c_type == ov::element::f64) {
+            new_const = change_constant_precision_to_fp16<ov::element::Type_t::f64>(const_node, postponed);
         } else {
             return false;
         }
+#endif  // !defined(OPENVINO_ARCH_X86) && !defined(OPENVINO_ARCH_X86_64)
 
         if (!new_const) {
             return false;
         }
+        auto constant_target_inputs = const_node->get_output_target_inputs(0);
         auto convert = std::make_shared<ov::op::v0::Convert>(new_const, const_node->get_element_type());
 
-        new_const->set_friendly_name(const_node->get_friendly_name() + "_compressed");
         convert->set_friendly_name(const_node->get_friendly_name());
+        new_const->set_friendly_name(const_node->get_friendly_name() + "_compressed");
         ov::copy_runtime_info(const_node, convert);
         ov::mark_as_decompression(convert);
+        if (postponed) {
+            postpone_fp16_compression(new_const->get_rt_info());
+            postpone_fp16_compression(new_const->get_output_tensor(0).get_rt_info());
 
-        ov::replace_node(const_node, convert);
-
+            for (const auto& target_input : constant_target_inputs) {
+                target_input.replace_source_output(convert);
+            }
+        } else {
+            ov::replace_node(const_node, convert);
+        }
         return true;
     };
 

src/common/transformations/src/transformations/convert_precision.cpp

@@ -895,6 +895,26 @@ std::shared_ptr<ngraph::Node> change_constant_precision(std::shared_ptr<opset4::
     return new_constant;
 }
 
+template <>
+std::shared_ptr<Node> change_constant_precision<ov::element::Type_t::f32, ov::element::Type_t::f16>(
+    std::shared_ptr<opset4::Constant>& constant) {
+    using src_type = typename element_type_traits<ov::element::Type_t::f32>::value_type;
+    using dst_type = typename element_type_traits<ov::element::Type_t::f16>::value_type;
+
+    const auto* src_data = constant->get_data_ptr<src_type>();
+    const auto size = shape_size(constant->get_shape());
+
+    auto new_constant = std::make_shared<opset4::Constant>(ov::element::Type_t::f16, constant->get_shape());
+    new_constant->output(0).set_names(constant->output(0).get_names());
+    auto* dst_data = const_cast<dst_type*>(reinterpret_cast<const dst_type*>(new_constant->get_data_ptr()));
+    if (dst_data == nullptr)
+        OPENVINO_THROW("Can't get destination data pointer");
+
+    ngraph::runtime::reference::convert_from_f32_to_f16_with_clamp(src_data, dst_data, size);
+
+    return new_constant;
+}
+
 template <>
 std::shared_ptr<Node> change_constant_precision<ov::element::Type_t::f16, ov::element::Type_t::f32>(
     std::shared_ptr<opset4::Constant>& constant) {

src/common/transformations/src/transformations/rt_info/disable_fp16_compression.cpp

@@ -4,6 +4,10 @@
 
 #include "transformations/rt_info/disable_fp16_compression.hpp"
 
+namespace {
+const std::string& postponed_fp16_compression_tag = "postponed_fp16_compression";
+}
+
 void ov::disable_fp16_compression(const std::shared_ptr<Node>& node) {
     auto& rt_info = node->get_rt_info();
     rt_info[DisableFP16Compression::get_type_info_static()] = DisableFP16Compression{};
@@ -18,3 +22,15 @@ bool ov::fp16_compression_is_disabled(const std::shared_ptr<const Node>& node) {
     const auto& rt_info = node->get_rt_info();
     return rt_info.count(DisableFP16Compression::get_type_info_static());
 }
+
+void ov::postpone_fp16_compression(ov::RTMap& rt_info) {
+    rt_info[postponed_fp16_compression_tag] = true;
+}
+
+bool ov::is_fp16_compression_postponed(const ov::RTMap& rt_info) {
+    return rt_info.count(postponed_fp16_compression_tag);
+}
+
+void ov::do_not_postpone_fp16_compression(ov::RTMap& rt_info) {
+    rt_info.erase(postponed_fp16_compression_tag);
+}