IntenseWebs/openvino
3
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fuse GeLU (#20428)

Browse Source 
* add transformation + test

* move transformation to GeluFusion

* add comments

* code review fixes

* fix

* code review fixes

* code style fix

* fix windows build warning
This commit is contained in:
Evgeny Kotov authored and GitHub committed 2023-10-24 18:23:14 +04:00
1 parent 5ffde7d8d6
commit 251602636c
3 files changed
+448 -293
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/common/transformations/include/transformations/common_optimizations/gelu_fusion.hpp
+13
View File
@@ -19,6 +19,7 @@ class TRANSFORMATIONS_API GeluFusionWithErfTwo;
class TRANSFORMATIONS_API GeluFusionWithErfThree;
class TRANSFORMATIONS_API GeluFusionWithErfFour;
class TRANSFORMATIONS_API GeluFusionWithTanh;
class TRANSFORMATIONS_API GeluFusionWithTanhNoPower;
} // namespace pass
} // namespace ov
@@ -78,6 +79,17 @@ public:
GeluFusionWithTanh();
};
/**
* @ingroup ie_transformation_common_api
* @brief GeluFusion transformation replaces a sub-graph
* x * 0.5 * (1 + tanh((x * 0.044715 * x + 1) * x * sqrt(2 / pi))) with a Gelu (Tanh) op.
*/
class ov::pass::GeluFusionWithTanhNoPower : public ov::pass::MatcherPass {
public:
OPENVINO_RTTI("GeluFusionWithTanhNoPower", "0");
GeluFusionWithTanhNoPower();
};
/**
* @ingroup ie_transformation_common_api
* @brief GeluFusion transformation replaces various sub-graphs with a Gelu op.
@@ -91,5 +103,6 @@ public:
add_matcher<ov::pass::GeluFusionWithErfThree>();
add_matcher<ov::pass::GeluFusionWithErfFour>();
add_matcher<ov::pass::GeluFusionWithTanh>();
add_matcher<ov::pass::GeluFusionWithTanhNoPower>();
}
};
src/common/transformations/src/transformations/common_optimizations/gelu_fusion.cpp
+76 -3
View File
@@ -8,6 +8,7 @@
#include <math.h>
#include <cmath>
#include <memory>
#include "itt.hpp"
@@ -16,8 +17,8 @@
#include "openvino/op/constant.hpp"
#include "openvino/op/divide.hpp"
#include "openvino/op/erf.hpp"
#include "openvino/op/gelu.hpp"
#include "openvino/op/multiply.hpp"
#include "openvino/op/parameter.hpp"
#include "openvino/op/power.hpp"
#include "openvino/op/tanh.hpp"
#include "openvino/pass/pattern/op/wrap_type.hpp"
@@ -302,11 +303,10 @@ ov::pass::GeluFusionWithTanh::GeluFusionWithTanh() {
return false;
}
constexpr float pi = 3.141592653589793238462643383279502884f;
bool valid_constant_values =
op::util::has_constant_value<float>(pow_constant_value, 3.0f) &&
op::util::has_constant_value<float>(mul_0_constant_value, 0.044715f, 0.001f) &&
op::util::has_constant_value<float>(mul_1_constant_value, std::sqrt(2.0f / pi), 0.01f) &&
op::util::has_constant_value<double>(mul_1_constant_value, std::sqrt(2.0 / M_PI), 0.01) &&
op::util::has_constant_value<float>(mul_2_constant_value, 0.5f) &&
op::util::has_constant_value<float>(add_1_constant_value, 1.0f);
@@ -336,3 +336,76 @@ ov::pass::GeluFusionWithTanh::GeluFusionWithTanh() {
auto m = std::make_shared<ov::pass::pattern::Matcher>(mul_3, matcher_name);
register_matcher(m, callback);
}
ov::pass::GeluFusionWithTanhNoPower::GeluFusionWithTanhNoPower() {
// Replaces a sub-graph with a Gelu (ov::op::v0::Tanh) op
// x * 0.5 * (1 + tanh((x * 0.044715 * x + 1) * x * sqrt(2 / pi)))
MATCHER_SCOPE(GeluFusionWithTanhNoPower);
auto input = pattern::any_input();
auto const1 = pattern::wrap_type<ov::op::v0::Constant>();
auto mul1 = pattern::wrap_type<ov::op::v1::Multiply>({input, const1});
auto mul2 = pattern::wrap_type<ov::op::v1::Multiply>({mul1, input});
auto const2 = pattern::wrap_type<ov::op::v0::Constant>();
auto add1 = pattern::wrap_type<ov::op::v1::Add>({const2, mul2});
auto const3 = pattern::wrap_type<ov::op::v0::Constant>();
auto mul3 = pattern::wrap_type<ov::op::v1::Multiply>({input, const3});
auto mul4 = pattern::wrap_type<ov::op::v1::Multiply>({add1, mul3});
auto tanh = pattern::wrap_type<ov::op::v0::Tanh>({mul4});
auto const4 = pattern::wrap_type<ov::op::v0::Constant>();
auto add2 = pattern::wrap_type<ov::op::v1::Add>({tanh, const4});
auto const5 = pattern::wrap_type<ov::op::v0::Constant>();
auto mul5 = pattern::wrap_type<ov::op::v1::Multiply>({input, const5});
auto mul6 = pattern::wrap_type<ov::op::v1::Multiply>({add2, mul5});
matcher_pass_callback callback = [=](pattern::Matcher& m) {
auto& pattern_to_output = m.get_pattern_value_map();
auto x_output = pattern_to_output.at(input);
auto const1_value = pattern_to_output.at(const1).get_node_shared_ptr();
auto const2_value = pattern_to_output.at(const2).get_node_shared_ptr();
auto const3_value = pattern_to_output.at(const3).get_node_shared_ptr();
auto const4_value = pattern_to_output.at(const4).get_node_shared_ptr();
auto const5_value = pattern_to_output.at(const5).get_node_shared_ptr();
bool valid_constant_values = op::util::has_constant_value<float>(const1_value, 0.044715f, 0.001f) &&
op::util::has_constant_value<float>(const2_value, 1.0f) &&
op::util::has_constant_value<double>(const3_value, std::sqrt(2.0 / M_PI), 0.01) &&
op::util::has_constant_value<float>(const4_value, 1.0f) &&
op::util::has_constant_value<float>(const5_value, 0.5f);
if (!valid_constant_values) {
return false;
}
auto gelu = std::make_shared<ov::op::v7::Gelu>(x_output, op::GeluApproximationMode::TANH);
gelu->set_friendly_name(m.get_match_root()->get_friendly_name());
ov::copy_runtime_info(
{
pattern_to_output.at(mul1).get_node_shared_ptr(),
pattern_to_output.at(mul2).get_node_shared_ptr(),
pattern_to_output.at(add1).get_node_shared_ptr(),
pattern_to_output.at(mul3).get_node_shared_ptr(),
pattern_to_output.at(mul4).get_node_shared_ptr(),
pattern_to_output.at(tanh).get_node_shared_ptr(),
pattern_to_output.at(add2).get_node_shared_ptr(),
pattern_to_output.at(mul5).get_node_shared_ptr(),
pattern_to_output.at(mul6).get_node_shared_ptr(),
},
gelu);
ov::replace_node(m.get_match_root(), gelu);
return true;
};
auto m = std::make_shared<pattern::Matcher>(mul6, matcher_name);
this->register_matcher(m, callback);
}
src/common/transformations/tests/common_optimizations/gelu_fusion.cpp
+359 -290
View File
@@ -9,14 +9,21 @@
#include <gtest/gtest.h>
#include <math.h>
#include <cmath>
#include <memory>
#include <queue>
#include <string>
#include "common_test_utils/ov_test_utils.hpp"
#include "openvino/core/model.hpp"
#include "openvino/opsets/opset7.hpp"
#include "openvino/opsets/opset9.hpp"
#include "openvino/op/add.hpp"
#include "openvino/op/constant.hpp"
#include "openvino/op/divide.hpp"
#include "openvino/op/erf.hpp"
#include "openvino/op/multiply.hpp"
#include "openvino/op/parameter.hpp"
#include "openvino/op/power.hpp"
#include "openvino/op/tanh.hpp"
#include "openvino/pass/constant_folding.hpp"
#include "openvino/pass/manager.hpp"
#include "transformations/convert_precision.hpp"
@@ -28,17 +35,17 @@ using namespace ov;
TEST_F(TransformationTestsF, GeluFusionPatternOne) {
{
auto data = std::make_shared<opset7::Parameter>(element::f32, Shape{2, 2});
auto data = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto div_const = opset7::Constant::create(element::f32, Shape{1}, {M_SQRT2});
auto add_const = opset7::Constant::create(element::f32, Shape{1}, {1.0});
auto mul_const = opset7::Constant::create(element::f32, Shape{1}, {0.5});
auto div_const = ov::op::v0::Constant::create(element::f32, Shape{1}, {M_SQRT2});
auto add_const = ov::op::v0::Constant::create(element::f32, Shape{1}, {1.0});
auto mul_const = ov::op::v0::Constant::create(element::f32, Shape{1}, {0.5});
auto div = std::make_shared<opset7::Divide>(data, div_const);
auto erf = std::make_shared<opset7::Erf>(div);
auto add = std::make_shared<opset7::Add>(erf, add_const);
auto mul_first = std::make_shared<opset7::Multiply>(data, mul_const);
auto mul = std::make_shared<opset7::Multiply>(mul_first, add);
auto div = std::make_shared<ov::op::v1::Divide>(data, div_const);
auto erf = std::make_shared<ov::op::v0::Erf>(div);
auto add = std::make_shared<ov::op::v1::Add>(erf, add_const);
auto mul_first = std::make_shared<ov::op::v1::Multiply>(data, mul_const);
auto mul = std::make_shared<ov::op::v1::Multiply>(mul_first, add);
model = std::make_shared<Model>(NodeVector{mul}, ParameterVector{data});
@@ -47,24 +54,24 @@ TEST_F(TransformationTestsF, GeluFusionPatternOne) {
{
auto data = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto gelu = std::make_shared<opset7::Gelu>(data);
auto gelu = std::make_shared<ov::op::v7::Gelu>(data);
model_ref = std::make_shared<Model>(NodeVector{gelu}, ParameterVector{data});
}
}
TEST_F(TransformationTestsF, GeluFusionPatternOneF16) {
{
auto data = std::make_shared<opset7::Parameter>(element::f16, Shape{2, 2});
auto data = std::make_shared<ov::op::v0::Parameter>(element::f16, Shape{2, 2});
auto div_const = opset7::Constant::create(element::f16, Shape{1}, {M_SQRT2});
auto add_const = opset7::Constant::create(element::f16, Shape{1}, {1.0});
auto mul_const = opset7::Constant::create(element::f16, Shape{1}, {0.5});
auto div_const = ov::op::v0::Constant::create(element::f16, Shape{1}, {M_SQRT2});
auto add_const = ov::op::v0::Constant::create(element::f16, Shape{1}, {1.0});
auto mul_const = ov::op::v0::Constant::create(element::f16, Shape{1}, {0.5});
auto div = std::make_shared<opset7::Divide>(data, div_const);
auto erf = std::make_shared<opset7::Erf>(div);
auto add = std::make_shared<opset7::Add>(erf, add_const);
auto mul_first = std::make_shared<opset7::Multiply>(data, mul_const);
auto mul = std::make_shared<opset7::Multiply>(mul_first, add);
auto div = std::make_shared<ov::op::v1::Divide>(data, div_const);
auto erf = std::make_shared<ov::op::v0::Erf>(div);
auto add = std::make_shared<ov::op::v1::Add>(erf, add_const);
auto mul_first = std::make_shared<ov::op::v1::Multiply>(data, mul_const);
auto mul = std::make_shared<ov::op::v1::Multiply>(mul_first, add);
model = std::make_shared<Model>(NodeVector{mul}, ParameterVector{data});
@@ -73,24 +80,24 @@ TEST_F(TransformationTestsF, GeluFusionPatternOneF16) {
{
auto data = std::make_shared<ov::op::v0::Parameter>(element::f16, Shape{2, 2});
auto gelu = std::make_shared<opset7::Gelu>(data);
auto gelu = std::make_shared<ov::op::v7::Gelu>(data);
model_ref = std::make_shared<Model>(NodeVector{gelu}, ParameterVector{data});
}
}
TEST_F(TransformationTestsF, GeluFusionPatternTwo) {
{
auto data = std::make_shared<opset7::Parameter>(element::f32, Shape{2, 2});
auto data = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto div_const = opset7::Constant::create(element::f32, Shape{1}, {M_SQRT2});
auto add_const = opset7::Constant::create(element::f32, Shape{1}, {1.0});
auto mul_const = opset7::Constant::create(element::f32, Shape{1}, {0.5});
auto div_const = ov::op::v0::Constant::create(element::f32, Shape{1}, {M_SQRT2});
auto add_const = ov::op::v0::Constant::create(element::f32, Shape{1}, {1.0});
auto mul_const = ov::op::v0::Constant::create(element::f32, Shape{1}, {0.5});
auto div = std::make_shared<opset7::Divide>(data, div_const);
auto erf = std::make_shared<opset7::Erf>(div);
auto add = std::make_shared<opset7::Add>(erf, add_const);
auto mul_first = std::make_shared<opset7::Multiply>(data, add);
auto mul = std::make_shared<opset7::Multiply>(mul_first, mul_const);
auto div = std::make_shared<ov::op::v1::Divide>(data, div_const);
auto erf = std::make_shared<ov::op::v0::Erf>(div);
auto add = std::make_shared<ov::op::v1::Add>(erf, add_const);
auto mul_first = std::make_shared<ov::op::v1::Multiply>(data, add);
auto mul = std::make_shared<ov::op::v1::Multiply>(mul_first, mul_const);
model = std::make_shared<Model>(NodeVector{mul}, ParameterVector{data});
@@ -99,24 +106,24 @@ TEST_F(TransformationTestsF, GeluFusionPatternTwo) {
{
auto data = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto gelu = std::make_shared<opset7::Gelu>(data);
auto gelu = std::make_shared<ov::op::v7::Gelu>(data);
model_ref = std::make_shared<Model>(NodeVector{gelu}, ParameterVector{data});
}
}
TEST_F(TransformationTestsF, GeluFusionPatternTwoF16) {
{
auto data = std::make_shared<opset7::Parameter>(element::f16, Shape{2, 2});
auto data = std::make_shared<ov::op::v0::Parameter>(element::f16, Shape{2, 2});
auto div_const = opset7::Constant::create(element::f16, Shape{1}, {M_SQRT2});
auto add_const = opset7::Constant::create(element::f16, Shape{1}, {1.0});
auto mul_const = opset7::Constant::create(element::f16, Shape{1}, {0.5});
auto div_const = ov::op::v0::Constant::create(element::f16, Shape{1}, {M_SQRT2});
auto add_const = ov::op::v0::Constant::create(element::f16, Shape{1}, {1.0});
auto mul_const = ov::op::v0::Constant::create(element::f16, Shape{1}, {0.5});
auto div = std::make_shared<opset7::Divide>(data, div_const);
auto erf = std::make_shared<opset7::Erf>(div);
auto add = std::make_shared<opset7::Add>(erf, add_const);
auto mul_first = std::make_shared<opset7::Multiply>(data, add);
auto mul = std::make_shared<opset7::Multiply>(mul_first, mul_const);
auto div = std::make_shared<ov::op::v1::Divide>(data, div_const);
auto erf = std::make_shared<ov::op::v0::Erf>(div);
auto add = std::make_shared<ov::op::v1::Add>(erf, add_const);
auto mul_first = std::make_shared<ov::op::v1::Multiply>(data, add);
auto mul = std::make_shared<ov::op::v1::Multiply>(mul_first, mul_const);
model = std::make_shared<Model>(NodeVector{mul}, ParameterVector{data});
@@ -125,24 +132,24 @@ TEST_F(TransformationTestsF, GeluFusionPatternTwoF16) {
{
auto data = std::make_shared<ov::op::v0::Parameter>(element::f16, Shape{2, 2});
auto gelu = std::make_shared<opset7::Gelu>(data);
auto gelu = std::make_shared<ov::op::v7::Gelu>(data);
model_ref = std::make_shared<Model>(NodeVector{gelu}, ParameterVector{data});
}
}
TEST_F(TransformationTestsF, GeluFusionPatternThree) {
{
auto data = std::make_shared<opset7::Parameter>(element::f32, Shape{2, 2});
auto data = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto div_const = opset7::Constant::create(element::f32, Shape{1}, {M_SQRT2});
auto add_const = opset7::Constant::create(element::f32, Shape{1}, {1.0});
auto mul_const = opset7::Constant::create(element::f32, Shape{1}, {0.5});
auto div_const = ov::op::v0::Constant::create(element::f32, Shape{1}, {M_SQRT2});
auto add_const = ov::op::v0::Constant::create(element::f32, Shape{1}, {1.0});
auto mul_const = ov::op::v0::Constant::create(element::f32, Shape{1}, {0.5});
auto div = std::make_shared<opset7::Divide>(data, div_const);
auto erf = std::make_shared<opset7::Erf>(div);
auto add = std::make_shared<opset7::Add>(erf, add_const);
auto mul_first = std::make_shared<opset7::Multiply>(add, mul_const);
auto mul = std::make_shared<opset7::Multiply>(data, mul_first);
auto div = std::make_shared<ov::op::v1::Divide>(data, div_const);
auto erf = std::make_shared<ov::op::v0::Erf>(div);
auto add = std::make_shared<ov::op::v1::Add>(erf, add_const);
auto mul_first = std::make_shared<ov::op::v1::Multiply>(add, mul_const);
auto mul = std::make_shared<ov::op::v1::Multiply>(data, mul_first);
model = std::make_shared<Model>(NodeVector{mul}, ParameterVector{data});
@@ -151,24 +158,24 @@ TEST_F(TransformationTestsF, GeluFusionPatternThree) {
{
auto data = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto gelu = std::make_shared<opset7::Gelu>(data);
auto gelu = std::make_shared<ov::op::v7::Gelu>(data);
model_ref = std::make_shared<Model>(NodeVector{gelu}, ParameterVector{data});
}
}
TEST_F(TransformationTestsF, GeluFusionPatternThreeF16) {
{
auto data = std::make_shared<opset7::Parameter>(element::f16, Shape{2, 2});
auto data = std::make_shared<ov::op::v0::Parameter>(element::f16, Shape{2, 2});
auto div_const = opset7::Constant::create(element::f16, Shape{1}, {M_SQRT2});
auto add_const = opset7::Constant::create(element::f16, Shape{1}, {1.0});
auto mul_const = opset7::Constant::create(element::f16, Shape{1}, {0.5});
auto div_const = ov::op::v0::Constant::create(element::f16, Shape{1}, {M_SQRT2});
auto add_const = ov::op::v0::Constant::create(element::f16, Shape{1}, {1.0});
auto mul_const = ov::op::v0::Constant::create(element::f16, Shape{1}, {0.5});
auto div = std::make_shared<opset7::Divide>(data, div_const);
auto erf = std::make_shared<opset7::Erf>(div);
auto add = std::make_shared<opset7::Add>(erf, add_const);
auto mul_first = std::make_shared<opset7::Multiply>(add, mul_const);
auto mul = std::make_shared<opset7::Multiply>(data, mul_first);
auto div = std::make_shared<ov::op::v1::Divide>(data, div_const);
auto erf = std::make_shared<ov::op::v0::Erf>(div);
auto add = std::make_shared<ov::op::v1::Add>(erf, add_const);
auto mul_first = std::make_shared<ov::op::v1::Multiply>(add, mul_const);
auto mul = std::make_shared<ov::op::v1::Multiply>(data, mul_first);
model = std::make_shared<Model>(NodeVector{mul}, ParameterVector{data});
@@ -177,24 +184,24 @@ TEST_F(TransformationTestsF, GeluFusionPatternThreeF16) {
{
auto data = std::make_shared<ov::op::v0::Parameter>(element::f16, Shape{2, 2});
auto gelu = std::make_shared<opset7::Gelu>(data);
auto gelu = std::make_shared<ov::op::v7::Gelu>(data);
model_ref = std::make_shared<Model>(NodeVector{gelu}, ParameterVector{data});
}
}
TEST_F(TransformationTestsF, GeluFusionPatternFour) {
{
auto data = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto data = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto mul1_const = opset9::Constant::create(element::f32, Shape{1}, {1.0f / M_SQRT2});
auto add_const = opset9::Constant::create(element::f32, Shape{1}, {0.5f});
auto mul2_const = opset9::Constant::create(element::f32, Shape{1}, {0.5f});
auto mul1_const = ov::op::v0::Constant::create(element::f32, Shape{1}, {1.0f / M_SQRT2});
auto add_const = ov::op::v0::Constant::create(element::f32, Shape{1}, {0.5f});
auto mul2_const = ov::op::v0::Constant::create(element::f32, Shape{1}, {0.5f});
auto mul1 = std::make_shared<opset9::Multiply>(data, mul1_const);
auto erf = std::make_shared<opset9::Erf>(mul1);
auto mul2 = std::make_shared<opset9::Multiply>(erf, mul2_const);
auto add = std::make_shared<opset9::Add>(mul2, add_const);
auto mul3 = std::make_shared<opset9::Multiply>(data, add);
auto mul1 = std::make_shared<ov::op::v1::Multiply>(data, mul1_const);
auto erf = std::make_shared<ov::op::v0::Erf>(mul1);
auto mul2 = std::make_shared<ov::op::v1::Multiply>(erf, mul2_const);
auto add = std::make_shared<ov::op::v1::Add>(mul2, add_const);
auto mul3 = std::make_shared<ov::op::v1::Multiply>(data, add);
model = std::make_shared<Model>(NodeVector{mul3}, ParameterVector{data});
@@ -203,24 +210,24 @@ TEST_F(TransformationTestsF, GeluFusionPatternFour) {
{
auto data = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto gelu = std::make_shared<opset9::Gelu>(data);
auto gelu = std::make_shared<ov::op::v7::Gelu>(data);
model_ref = std::make_shared<Model>(NodeVector{gelu}, ParameterVector{data});
}
}
TEST_F(TransformationTestsF, GeluFusionPatternFourF16) {
{
auto data = std::make_shared<opset9::Parameter>(element::f16, Shape{2, 2});
auto data = std::make_shared<ov::op::v0::Parameter>(element::f16, Shape{2, 2});
auto mul1_const = opset9::Constant::create(element::f16, Shape{1}, {1.0f / M_SQRT2});
auto add_const = opset9::Constant::create(element::f16, Shape{1}, {0.5f});
auto mul2_const = opset9::Constant::create(element::f16, Shape{1}, {0.5f});
auto mul1_const = ov::op::v0::Constant::create(element::f16, Shape{1}, {1.0f / M_SQRT2});
auto add_const = ov::op::v0::Constant::create(element::f16, Shape{1}, {0.5f});
auto mul2_const = ov::op::v0::Constant::create(element::f16, Shape{1}, {0.5f});
auto mul1 = std::make_shared<opset9::Multiply>(data, mul1_const);
auto erf = std::make_shared<opset9::Erf>(mul1);
auto mul2 = std::make_shared<opset9::Multiply>(erf, mul2_const);
auto add = std::make_shared<opset9::Add>(mul2, add_const);
auto mul3 = std::make_shared<opset9::Multiply>(data, add);
auto mul1 = std::make_shared<ov::op::v1::Multiply>(data, mul1_const);
auto erf = std::make_shared<ov::op::v0::Erf>(mul1);
auto mul2 = std::make_shared<ov::op::v1::Multiply>(erf, mul2_const);
auto add = std::make_shared<ov::op::v1::Add>(mul2, add_const);
auto mul3 = std::make_shared<ov::op::v1::Multiply>(data, add);
model = std::make_shared<Model>(NodeVector{mul3}, ParameterVector{data});
@@ -229,24 +236,24 @@ TEST_F(TransformationTestsF, GeluFusionPatternFourF16) {
{
auto data = std::make_shared<ov::op::v0::Parameter>(element::f16, Shape{2, 2});
auto gelu = std::make_shared<opset9::Gelu>(data);
auto gelu = std::make_shared<ov::op::v7::Gelu>(data);
model_ref = std::make_shared<Model>(NodeVector{gelu}, ParameterVector{data});
}
}
TEST_F(TransformationTestsF, GeluFusionPatternIncorrectDivConstValue) {
{
auto data = std::make_shared<opset7::Parameter>(element::f32, Shape{2, 2});
auto data = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto div_const = opset7::Constant::create(element::f32, Shape{1}, {1.4149});
auto add_const = opset7::Constant::create(element::f32, Shape{1}, {1.0});
auto mul_const = opset7::Constant::create(element::f32, Shape{1}, {0.5});
auto div_const = ov::op::v0::Constant::create(element::f32, Shape{1}, {1.4149});
auto add_const = ov::op::v0::Constant::create(element::f32, Shape{1}, {1.0});
auto mul_const = ov::op::v0::Constant::create(element::f32, Shape{1}, {0.5});
auto div = std::make_shared<opset7::Divide>(data, div_const);
auto erf = std::make_shared<opset7::Erf>(div);
auto add = std::make_shared<opset7::Add>(erf, add_const);
auto mul_first = std::make_shared<opset7::Multiply>(data, add);
auto mul = std::make_shared<opset7::Multiply>(mul_first, mul_const);
auto div = std::make_shared<ov::op::v1::Divide>(data, div_const);
auto erf = std::make_shared<ov::op::v0::Erf>(div);
auto add = std::make_shared<ov::op::v1::Add>(erf, add_const);
auto mul_first = std::make_shared<ov::op::v1::Multiply>(data, add);
auto mul = std::make_shared<ov::op::v1::Multiply>(mul_first, mul_const);
model = std::make_shared<Model>(NodeVector{mul}, ParameterVector{data});
model_ref = std::make_shared<Model>(NodeVector{mul}, ParameterVector{data});
@@ -257,17 +264,17 @@ TEST_F(TransformationTestsF, GeluFusionPatternIncorrectDivConstValue) {
TEST_F(TransformationTestsF, GeluFusionPatternTooShortDivConstValue) {
{
auto data = std::make_shared<opset7::Parameter>(element::f32, Shape{2, 2});
auto data = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto div_const = opset7::Constant::create(element::f32, Shape{1}, {1.4142});
auto add_const = opset7::Constant::create(element::f32, Shape{1}, {1.0});
auto mul_const = opset7::Constant::create(element::f32, Shape{1}, {0.5});
auto div_const = ov::op::v0::Constant::create(element::f32, Shape{1}, {1.4142});
auto add_const = ov::op::v0::Constant::create(element::f32, Shape{1}, {1.0});
auto mul_const = ov::op::v0::Constant::create(element::f32, Shape{1}, {0.5});
auto div = std::make_shared<opset7::Divide>(data, div_const);
auto erf = std::make_shared<opset7::Erf>(div);
auto add = std::make_shared<opset7::Add>(erf, add_const);
auto mul_first = std::make_shared<opset7::Multiply>(data, add);
auto mul = std::make_shared<opset7::Multiply>(mul_first, mul_const);
auto div = std::make_shared<ov::op::v1::Divide>(data, div_const);
auto erf = std::make_shared<ov::op::v0::Erf>(div);
auto add = std::make_shared<ov::op::v1::Add>(erf, add_const);
auto mul_first = std::make_shared<ov::op::v1::Multiply>(data, add);
auto mul = std::make_shared<ov::op::v1::Multiply>(mul_first, mul_const);
model = std::make_shared<Model>(NodeVector{mul}, ParameterVector{data});
model_ref = std::make_shared<Model>(NodeVector{mul}, ParameterVector{data});
@@ -278,60 +285,62 @@ TEST_F(TransformationTestsF, GeluFusionPatternTooShortDivConstValue) {
TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_equal_const_values) {
{
auto input = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f});
auto pow = std::make_shared<opset9::Power>(input, pow_constant);
auto mul_0_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.044715f});
auto mul_0 = std::make_shared<opset9::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<opset9::Add>(input, mul_0);
auto input = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f});
auto pow = std::make_shared<ov::op::v1::Power>(input, pow_constant);
auto mul_0_constant =
std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.044715f});
auto mul_0 = std::make_shared<ov::op::v1::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<ov::op::v1::Add>(input, mul_0);
constexpr float pi = 3.141592653589793238462643383279502884f;
auto mul_1_constant =
std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{std::sqrt(2.0f / pi)});
auto mul_1 = std::make_shared<opset9::Multiply>(add_0, mul_1_constant);
std::make_shared<ov::op::v0::Constant>(element::f32,
Shape{1},
std::vector<float>{static_cast<float>(std::sqrt(2.0 / M_PI))});
auto mul_1 = std::make_shared<ov::op::v1::Multiply>(add_0, mul_1_constant);
auto tanh = std::make_shared<opset9::Tanh>(mul_1);
auto tanh = std::make_shared<ov::op::v0::Tanh>(mul_1);
auto add_1_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f});
auto add_1 = std::make_shared<opset9::Add>(tanh, add_1_constant);
auto add_1_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f});
auto add_1 = std::make_shared<ov::op::v1::Add>(tanh, add_1_constant);
auto mul_2_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f});
auto mul_2 = std::make_shared<opset9::Multiply>(add_1, mul_2_constant);
auto mul_2_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f});
auto mul_2 = std::make_shared<ov::op::v1::Multiply>(add_1, mul_2_constant);
auto mul_3 = std::make_shared<opset9::Multiply>(input, mul_2);
auto mul_3 = std::make_shared<ov::op::v1::Multiply>(input, mul_2);
model = std::make_shared<Model>(NodeVector{mul_3}, ParameterVector{input});
manager.register_pass<ov::pass::GeluFusionWithTanh>();
}
{
auto data = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto gelu = std::make_shared<opset9::Gelu>(data, op::GeluApproximationMode::TANH);
auto data = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto gelu = std::make_shared<ov::op::v7::Gelu>(data, op::GeluApproximationMode::TANH);
model_ref = std::make_shared<Model>(NodeVector{gelu}, ParameterVector{data});
}
}
TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_params_no_conversion) {
{
auto input = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto pow_param = std::make_shared<opset9::Parameter>(element::f32, Shape{1});
auto pow = std::make_shared<opset9::Power>(input, pow_param);
auto mul_0_param = std::make_shared<opset9::Parameter>(element::f32, Shape{1});
auto mul_0 = std::make_shared<opset9::Multiply>(pow, mul_0_param);
auto add_0 = std::make_shared<opset9::Add>(input, mul_0);
auto input = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto pow_param = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{1});
auto pow = std::make_shared<ov::op::v1::Power>(input, pow_param);
auto mul_0_param = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{1});
auto mul_0 = std::make_shared<ov::op::v1::Multiply>(pow, mul_0_param);
auto add_0 = std::make_shared<ov::op::v1::Add>(input, mul_0);
auto mul_1_param = std::make_shared<opset9::Parameter>(element::f32, Shape{1});
auto mul_1 = std::make_shared<opset9::Multiply>(add_0, mul_1_param);
auto mul_1_param = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{1});
auto mul_1 = std::make_shared<ov::op::v1::Multiply>(add_0, mul_1_param);
auto tanh = std::make_shared<opset9::Tanh>(mul_1);
auto tanh = std::make_shared<ov::op::v0::Tanh>(mul_1);
auto add_1_param = std::make_shared<opset9::Parameter>(element::f32, Shape{1});
auto add_1 = std::make_shared<opset9::Add>(tanh, add_1_param);
auto add_1_param = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{1});
auto add_1 = std::make_shared<ov::op::v1::Add>(tanh, add_1_param);
auto mul_2_param = std::make_shared<opset9::Parameter>(element::f32, Shape{1});
auto mul_2 = std::make_shared<opset9::Multiply>(add_1, mul_2_param);
auto mul_2_param = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{1});
auto mul_2 = std::make_shared<ov::op::v1::Multiply>(add_1, mul_2_param);
auto mul_3 = std::make_shared<opset9::Multiply>(input, mul_2);
auto mul_3 = std::make_shared<ov::op::v1::Multiply>(input, mul_2);
model = std::make_shared<Model>(
NodeVector{mul_3},
@@ -342,63 +351,67 @@ TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_params_no_conversion) {
TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_epsilon_pow_value) {
{
auto input = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto input = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto pow_constant =
std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f + 1.0e-8f});
auto pow = std::make_shared<opset9::Power>(input, pow_constant);
auto mul_0_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.044715f});
auto mul_0 = std::make_shared<opset9::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<opset9::Add>(input, mul_0);
std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f + 1.0e-8f});
auto pow = std::make_shared<ov::op::v1::Power>(input, pow_constant);
auto mul_0_constant =
std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.044715f});
auto mul_0 = std::make_shared<ov::op::v1::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<ov::op::v1::Add>(input, mul_0);
constexpr float pi = 3.141592653589793238462643383279502884f;
auto mul_1_constant =
std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{std::sqrt(2.0f / pi)});
auto mul_1 = std::make_shared<opset9::Multiply>(add_0, mul_1_constant);
std::make_shared<ov::op::v0::Constant>(element::f32,
Shape{1},
std::vector<float>{static_cast<float>(std::sqrt(2.0 / M_PI))});
auto mul_1 = std::make_shared<ov::op::v1::Multiply>(add_0, mul_1_constant);
auto tanh = std::make_shared<opset9::Tanh>(mul_1);
auto tanh = std::make_shared<ov::op::v0::Tanh>(mul_1);
auto add_1_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f});
auto add_1 = std::make_shared<opset9::Add>(tanh, add_1_constant);
auto add_1_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f});
auto add_1 = std::make_shared<ov::op::v1::Add>(tanh, add_1_constant);
auto mul_2_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f});
auto mul_2 = std::make_shared<opset9::Multiply>(add_1, mul_2_constant);
auto mul_2_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f});
auto mul_2 = std::make_shared<ov::op::v1::Multiply>(add_1, mul_2_constant);
auto mul_3 = std::make_shared<opset9::Multiply>(input, mul_2);
auto mul_3 = std::make_shared<ov::op::v1::Multiply>(input, mul_2);
model = std::make_shared<Model>(NodeVector{mul_3}, ParameterVector{input});
manager.register_pass<ov::pass::GeluFusionWithTanh>();
}
{
auto data = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto gelu = std::make_shared<opset9::Gelu>(data, op::GeluApproximationMode::TANH);
auto data = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto gelu = std::make_shared<ov::op::v7::Gelu>(data, op::GeluApproximationMode::TANH);
model_ref = std::make_shared<Model>(NodeVector{gelu}, ParameterVector{data});
}
}
TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_wrong_pow_value) {
{
auto input = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{2.0f});
auto pow = std::make_shared<opset9::Power>(input, pow_constant);
auto mul_0_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.044715f});
auto mul_0 = std::make_shared<opset9::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<opset9::Add>(input, mul_0);
auto input = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{2.0f});
auto pow = std::make_shared<ov::op::v1::Power>(input, pow_constant);
auto mul_0_constant =
std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.044715f});
auto mul_0 = std::make_shared<ov::op::v1::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<ov::op::v1::Add>(input, mul_0);
constexpr float pi = 3.141592653589793238462643383279502884f;
auto mul_1_constant =
std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{std::sqrt(2.0f / pi)});
auto mul_1 = std::make_shared<opset9::Multiply>(add_0, mul_1_constant);
std::make_shared<ov::op::v0::Constant>(element::f32,
Shape{1},
std::vector<float>{static_cast<float>(std::sqrt(2.0 / M_PI))});
auto mul_1 = std::make_shared<ov::op::v1::Multiply>(add_0, mul_1_constant);
auto tanh = std::make_shared<opset9::Tanh>(mul_1);
auto tanh = std::make_shared<ov::op::v0::Tanh>(mul_1);
auto add_1_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f});
auto add_1 = std::make_shared<opset9::Add>(tanh, add_1_constant);
auto add_1_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f});
auto add_1 = std::make_shared<ov::op::v1::Add>(tanh, add_1_constant);
auto mul_2_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f});
auto mul_2 = std::make_shared<opset9::Multiply>(add_1, mul_2_constant);
auto mul_2_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f});
auto mul_2 = std::make_shared<ov::op::v1::Multiply>(add_1, mul_2_constant);
auto mul_3 = std::make_shared<opset9::Multiply>(input, mul_2);
auto mul_3 = std::make_shared<ov::op::v1::Multiply>(input, mul_2);
model = std::make_shared<Model>(NodeVector{mul_3}, ParameterVector{input});
manager.register_pass<ov::pass::GeluFusionWithTanh>();
@@ -407,62 +420,66 @@ TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_wrong_pow_value) {
TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_epsilon_mul_0_value) {
{
auto input = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f});
auto pow = std::make_shared<opset9::Power>(input, pow_constant);
auto mul_0_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.04515f});
auto mul_0 = std::make_shared<opset9::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<opset9::Add>(input, mul_0);
auto input = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f});
auto pow = std::make_shared<ov::op::v1::Power>(input, pow_constant);
auto mul_0_constant =
std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.04515f});
auto mul_0 = std::make_shared<ov::op::v1::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<ov::op::v1::Add>(input, mul_0);
constexpr float pi = 3.141592653589793238462643383279502884f;
auto mul_1_constant =
std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{std::sqrt(2.0f / pi)});
auto mul_1 = std::make_shared<opset9::Multiply>(add_0, mul_1_constant);
std::make_shared<ov::op::v0::Constant>(element::f32,
Shape{1},
std::vector<float>{static_cast<float>(std::sqrt(2.0 / M_PI))});
auto mul_1 = std::make_shared<ov::op::v1::Multiply>(add_0, mul_1_constant);
auto tanh = std::make_shared<opset9::Tanh>(mul_1);
auto tanh = std::make_shared<ov::op::v0::Tanh>(mul_1);
auto add_1_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f});
auto add_1 = std::make_shared<opset9::Add>(tanh, add_1_constant);
auto add_1_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f});
auto add_1 = std::make_shared<ov::op::v1::Add>(tanh, add_1_constant);
auto mul_2_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f});
auto mul_2 = std::make_shared<opset9::Multiply>(add_1, mul_2_constant);
auto mul_2_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f});
auto mul_2 = std::make_shared<ov::op::v1::Multiply>(add_1, mul_2_constant);
auto mul_3 = std::make_shared<opset9::Multiply>(input, mul_2);
auto mul_3 = std::make_shared<ov::op::v1::Multiply>(input, mul_2);
model = std::make_shared<Model>(NodeVector{mul_3}, ParameterVector{input});
manager.register_pass<ov::pass::GeluFusionWithTanh>();
}
{
auto data = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto gelu = std::make_shared<opset9::Gelu>(data, op::GeluApproximationMode::TANH);
auto data = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto gelu = std::make_shared<ov::op::v7::Gelu>(data, op::GeluApproximationMode::TANH);
model_ref = std::make_shared<Model>(NodeVector{gelu}, ParameterVector{data});
}
}
TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_wrong_mul_0_value) {
{
auto input = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f});
auto pow = std::make_shared<opset9::Power>(input, pow_constant);
auto mul_0_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{1.4715f});
auto mul_0 = std::make_shared<opset9::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<opset9::Add>(input, mul_0);
auto input = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f});
auto pow = std::make_shared<ov::op::v1::Power>(input, pow_constant);
auto mul_0_constant =
std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{1.4715f});
auto mul_0 = std::make_shared<ov::op::v1::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<ov::op::v1::Add>(input, mul_0);
constexpr float pi = 3.141592653589793238462643383279502884f;
auto mul_1_constant =
std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{std::sqrt(2.0f / pi)});
auto mul_1 = std::make_shared<opset9::Multiply>(add_0, mul_1_constant);
std::make_shared<ov::op::v0::Constant>(element::f32,
Shape{1},
std::vector<float>{static_cast<float>(std::sqrt(2.0 / M_PI))});
auto mul_1 = std::make_shared<ov::op::v1::Multiply>(add_0, mul_1_constant);
auto tanh = std::make_shared<opset9::Tanh>(mul_1);
auto tanh = std::make_shared<ov::op::v0::Tanh>(mul_1);
auto add_1_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f});
auto add_1 = std::make_shared<opset9::Add>(tanh, add_1_constant);
auto add_1_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f});
auto add_1 = std::make_shared<ov::op::v1::Add>(tanh, add_1_constant);
auto mul_2_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f});
auto mul_2 = std::make_shared<opset9::Multiply>(add_1, mul_2_constant);
auto mul_2_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f});
auto mul_2 = std::make_shared<ov::op::v1::Multiply>(add_1, mul_2_constant);
auto mul_3 = std::make_shared<opset9::Multiply>(input, mul_2);
auto mul_3 = std::make_shared<ov::op::v1::Multiply>(input, mul_2);
model = std::make_shared<Model>(NodeVector{mul_3}, ParameterVector{input});
manager.register_pass<ov::pass::GeluFusionWithTanh>();
@@ -471,61 +488,64 @@ TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_wrong_mul_0_value) {
TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_epsilon_mul_1_value) {
{
auto input = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f});
auto pow = std::make_shared<opset9::Power>(input, pow_constant);
auto mul_0_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.044715f});
auto mul_0 = std::make_shared<opset9::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<opset9::Add>(input, mul_0);
auto input = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f});
auto pow = std::make_shared<ov::op::v1::Power>(input, pow_constant);
auto mul_0_constant =
std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.044715f});
auto mul_0 = std::make_shared<ov::op::v1::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<ov::op::v1::Add>(input, mul_0);
auto mul_1_constant =
std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.7980868f});
auto mul_1 = std::make_shared<opset9::Multiply>(add_0, mul_1_constant);
std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.7980868f});
auto mul_1 = std::make_shared<ov::op::v1::Multiply>(add_0, mul_1_constant);
auto tanh = std::make_shared<opset9::Tanh>(mul_1);
auto tanh = std::make_shared<ov::op::v0::Tanh>(mul_1);
auto add_1_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f});
auto add_1 = std::make_shared<opset9::Add>(tanh, add_1_constant);
auto add_1_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f});
auto add_1 = std::make_shared<ov::op::v1::Add>(tanh, add_1_constant);
auto mul_2_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f});
auto mul_2 = std::make_shared<opset9::Multiply>(add_1, mul_2_constant);
auto mul_2_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f});
auto mul_2 = std::make_shared<ov::op::v1::Multiply>(add_1, mul_2_constant);
auto mul_3 = std::make_shared<opset9::Multiply>(input, mul_2);
auto mul_3 = std::make_shared<ov::op::v1::Multiply>(input, mul_2);
model = std::make_shared<Model>(NodeVector{mul_3}, ParameterVector{input});
manager.register_pass<ov::pass::GeluFusionWithTanh>();
}
{
auto data = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto gelu = std::make_shared<opset9::Gelu>(data, op::GeluApproximationMode::TANH);
auto data = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto gelu = std::make_shared<ov::op::v7::Gelu>(data, op::GeluApproximationMode::TANH);
model_ref = std::make_shared<Model>(NodeVector{gelu}, ParameterVector{data});
}
}
TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_wrong_mul_1_value) {
{
auto input = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f});
auto pow = std::make_shared<opset9::Power>(input, pow_constant);
auto mul_0_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.044715f});
auto mul_0 = std::make_shared<opset9::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<opset9::Add>(input, mul_0);
auto input = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f});
auto pow = std::make_shared<ov::op::v1::Power>(input, pow_constant);
auto mul_0_constant =
std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.044715f});
auto mul_0 = std::make_shared<ov::op::v1::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<ov::op::v1::Add>(input, mul_0);
constexpr float pi = 3.141592653589793238462643383279502884f;
auto mul_1_constant =
std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{std::sqrt(10.0f / pi)});
auto mul_1 = std::make_shared<opset9::Multiply>(add_0, mul_1_constant);
std::make_shared<ov::op::v0::Constant>(element::f32,
Shape{1},
std::vector<float>{static_cast<float>(std::sqrt(10.0 / M_PI))});
auto mul_1 = std::make_shared<ov::op::v1::Multiply>(add_0, mul_1_constant);
auto tanh = std::make_shared<opset9::Tanh>(mul_1);
auto tanh = std::make_shared<ov::op::v0::Tanh>(mul_1);
auto add_1_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f});
auto add_1 = std::make_shared<opset9::Add>(tanh, add_1_constant);
auto add_1_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f});
auto add_1 = std::make_shared<ov::op::v1::Add>(tanh, add_1_constant);
auto mul_2_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f});
auto mul_2 = std::make_shared<opset9::Multiply>(add_1, mul_2_constant);
auto mul_2_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f});
auto mul_2 = std::make_shared<ov::op::v1::Multiply>(add_1, mul_2_constant);
auto mul_3 = std::make_shared<opset9::Multiply>(input, mul_2);
auto mul_3 = std::make_shared<ov::op::v1::Multiply>(input, mul_2);
model = std::make_shared<Model>(NodeVector{mul_3}, ParameterVector{input});
manager.register_pass<ov::pass::GeluFusionWithTanh>();
@@ -534,63 +554,67 @@ TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_wrong_mul_1_value) {
TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_epsilon_add_1_value) {
{
auto input = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f});
auto pow = std::make_shared<opset9::Power>(input, pow_constant);
auto mul_0_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.044715f});
auto mul_0 = std::make_shared<opset9::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<opset9::Add>(input, mul_0);
auto input = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f});
auto pow = std::make_shared<ov::op::v1::Power>(input, pow_constant);
auto mul_0_constant =
std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.044715f});
auto mul_0 = std::make_shared<ov::op::v1::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<ov::op::v1::Add>(input, mul_0);
constexpr float pi = 3.141592653589793238462643383279502884f;
auto mul_1_constant =
std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{std::sqrt(2.0f / pi)});
auto mul_1 = std::make_shared<opset9::Multiply>(add_0, mul_1_constant);
std::make_shared<ov::op::v0::Constant>(element::f32,
Shape{1},
std::vector<float>{static_cast<float>(std::sqrt(2.0 / M_PI))});
auto mul_1 = std::make_shared<ov::op::v1::Multiply>(add_0, mul_1_constant);
auto tanh = std::make_shared<opset9::Tanh>(mul_1);
auto tanh = std::make_shared<ov::op::v0::Tanh>(mul_1);
auto add_1_constant =
std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f + 1.0e-8f});
auto add_1 = std::make_shared<opset9::Add>(tanh, add_1_constant);
std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f + 1.0e-8f});
auto add_1 = std::make_shared<ov::op::v1::Add>(tanh, add_1_constant);
auto mul_2_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f});
auto mul_2 = std::make_shared<opset9::Multiply>(add_1, mul_2_constant);
auto mul_2_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f});
auto mul_2 = std::make_shared<ov::op::v1::Multiply>(add_1, mul_2_constant);
auto mul_3 = std::make_shared<opset9::Multiply>(input, mul_2);
auto mul_3 = std::make_shared<ov::op::v1::Multiply>(input, mul_2);
model = std::make_shared<Model>(NodeVector{mul_3}, ParameterVector{input});
manager.register_pass<ov::pass::GeluFusionWithTanh>();
}
{
auto data = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto gelu = std::make_shared<opset9::Gelu>(data, op::GeluApproximationMode::TANH);
auto data = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto gelu = std::make_shared<ov::op::v7::Gelu>(data, op::GeluApproximationMode::TANH);
model_ref = std::make_shared<Model>(NodeVector{gelu}, ParameterVector{data});
}
}
TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_wrong_add_1_value) {
{
auto input = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f});
auto pow = std::make_shared<opset9::Power>(input, pow_constant);
auto mul_0_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.044715f});
auto mul_0 = std::make_shared<opset9::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<opset9::Add>(input, mul_0);
auto input = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f});
auto pow = std::make_shared<ov::op::v1::Power>(input, pow_constant);
auto mul_0_constant =
std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.044715f});
auto mul_0 = std::make_shared<ov::op::v1::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<ov::op::v1::Add>(input, mul_0);
constexpr float pi = 3.141592653589793238462643383279502884f;
auto mul_1_constant =
std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{std::sqrt(2.0f / pi)});
auto mul_1 = std::make_shared<opset9::Multiply>(add_0, mul_1_constant);
std::make_shared<ov::op::v0::Constant>(element::f32,
Shape{1},
std::vector<float>{static_cast<float>(std::sqrt(2.0 / M_PI))});
auto mul_1 = std::make_shared<ov::op::v1::Multiply>(add_0, mul_1_constant);
auto tanh = std::make_shared<opset9::Tanh>(mul_1);
auto tanh = std::make_shared<ov::op::v0::Tanh>(mul_1);
auto add_1_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{2.0f});
auto add_1 = std::make_shared<opset9::Add>(tanh, add_1_constant);
auto add_1_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{2.0f});
auto add_1 = std::make_shared<ov::op::v1::Add>(tanh, add_1_constant);
auto mul_2_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f});
auto mul_2 = std::make_shared<opset9::Multiply>(add_1, mul_2_constant);
auto mul_2_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f});
auto mul_2 = std::make_shared<ov::op::v1::Multiply>(add_1, mul_2_constant);
auto mul_3 = std::make_shared<opset9::Multiply>(input, mul_2);
auto mul_3 = std::make_shared<ov::op::v1::Multiply>(input, mul_2);
model = std::make_shared<Model>(NodeVector{mul_3}, ParameterVector{input});
manager.register_pass<ov::pass::GeluFusionWithTanh>();
@@ -599,65 +623,110 @@ TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_wrong_add_1_value) {
TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_epsilon_mul_2_value) {
{
auto input = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f});
auto pow = std::make_shared<opset9::Power>(input, pow_constant);
auto mul_0_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.044715f});
auto mul_0 = std::make_shared<opset9::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<opset9::Add>(input, mul_0);
auto input = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f});
auto pow = std::make_shared<ov::op::v1::Power>(input, pow_constant);
auto mul_0_constant =
std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.044715f});
auto mul_0 = std::make_shared<ov::op::v1::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<ov::op::v1::Add>(input, mul_0);
constexpr float pi = 3.141592653589793238462643383279502884f;
auto mul_1_constant =
std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{std::sqrt(2.0f / pi)});
auto mul_1 = std::make_shared<opset9::Multiply>(add_0, mul_1_constant);
std::make_shared<ov::op::v0::Constant>(element::f32,
Shape{1},
std::vector<float>{static_cast<float>(std::sqrt(2.0 / M_PI))});
auto mul_1 = std::make_shared<ov::op::v1::Multiply>(add_0, mul_1_constant);
auto tanh = std::make_shared<opset9::Tanh>(mul_1);
auto tanh = std::make_shared<ov::op::v0::Tanh>(mul_1);
auto add_1_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f});
auto add_1 = std::make_shared<opset9::Add>(tanh, add_1_constant);
auto add_1_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f});
auto add_1 = std::make_shared<ov::op::v1::Add>(tanh, add_1_constant);
auto mul_2_constant =
std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f + 1.0e-8f});
auto mul_2 = std::make_shared<opset9::Multiply>(add_1, mul_2_constant);
std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.5f + 1.0e-8f});
auto mul_2 = std::make_shared<ov::op::v1::Multiply>(add_1, mul_2_constant);
auto mul_3 = std::make_shared<opset9::Multiply>(input, mul_2);
auto mul_3 = std::make_shared<ov::op::v1::Multiply>(input, mul_2);
model = std::make_shared<Model>(NodeVector{mul_3}, ParameterVector{input});
manager.register_pass<ov::pass::GeluFusionWithTanh>();
}
{
auto data = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto gelu = std::make_shared<opset9::Gelu>(data, op::GeluApproximationMode::TANH);
auto data = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto gelu = std::make_shared<ov::op::v7::Gelu>(data, op::GeluApproximationMode::TANH);
model_ref = std::make_shared<Model>(NodeVector{gelu}, ParameterVector{data});
}
}
TEST_F(TransformationTestsF, GeluFusionTanhWithTanh_wrong_mul_2_value) {
{
auto input = std::make_shared<opset9::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f});
auto pow = std::make_shared<opset9::Power>(input, pow_constant);
auto mul_0_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{0.044715f});
auto mul_0 = std::make_shared<opset9::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<opset9::Add>(input, mul_0);
auto input = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{2, 2});
auto pow_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{3.0f});
auto pow = std::make_shared<ov::op::v1::Power>(input, pow_constant);
auto mul_0_constant =
std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{0.044715f});
auto mul_0 = std::make_shared<ov::op::v1::Multiply>(pow, mul_0_constant);
auto add_0 = std::make_shared<ov::op::v1::Add>(input, mul_0);
constexpr float pi = 3.141592653589793238462643383279502884f;
auto mul_1_constant =
std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{std::sqrt(2.0f / pi)});
auto mul_1 = std::make_shared<opset9::Multiply>(add_0, mul_1_constant);
std::make_shared<ov::op::v0::Constant>(element::f32,
Shape{1},
std::vector<float>{static_cast<float>(std::sqrt(2.0 / M_PI))});
auto mul_1 = std::make_shared<ov::op::v1::Multiply>(add_0, mul_1_constant);
auto tanh = std::make_shared<opset9::Tanh>(mul_1);
auto tanh = std::make_shared<ov::op::v0::Tanh>(mul_1);
auto add_1_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f});
auto add_1 = std::make_shared<opset9::Add>(tanh, add_1_constant);
auto add_1_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{1.0f});
auto add_1 = std::make_shared<ov::op::v1::Add>(tanh, add_1_constant);
auto mul_2_constant = std::make_shared<opset9::Constant>(element::f32, Shape{1}, std::vector<float>{5.0f});
auto mul_2 = std::make_shared<opset9::Multiply>(add_1, mul_2_constant);
auto mul_2_constant = std::make_shared<ov::op::v0::Constant>(element::f32, Shape{1}, std::vector<float>{5.0f});
auto mul_2 = std::make_shared<ov::op::v1::Multiply>(add_1, mul_2_constant);
auto mul_3 = std::make_shared<opset9::Multiply>(input, mul_2);
auto mul_3 = std::make_shared<ov::op::v1::Multiply>(input, mul_2);
model = std::make_shared<Model>(NodeVector{mul_3}, ParameterVector{input});
manager.register_pass<ov::pass::GeluFusionWithTanh>();
}
}
TEST_F(TransformationTestsF, FoldGeluOperation) {
{
auto param = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{1006, 2, 100, 3, 4096});
auto const1 = ov::op::v0::Constant::create(element::f32, Shape{1, 1, 1}, std::vector<float>{0.044715f});
auto mul1 = std::make_shared<ov::op::v1::Multiply>(param, const1);
auto mul2 = std::make_shared<ov::op::v1::Multiply>(mul1, param);
auto const2 = ov::op::v0::Constant::create(element::f32, Shape{1, 1, 1}, std::vector<float>{1.0});
auto add1 = std::make_shared<ov::op::v1::Add>(const2, mul2);
auto const3 = ov::op::v0::Constant::create(element::f32,
Shape{1, 1, 1},
std::vector<float>{static_cast<float>(std::sqrt(2.0 / M_PI))});
auto mul3 = std::make_shared<ov::op::v1::Multiply>(param, const3);
auto mul4 = std::make_shared<ov::op::v1::Multiply>(add1, mul3);
auto tan = std::make_shared<ov::op::v0::Tanh>(mul4);
auto const4 = ov::op::v0::Constant::create(element::f32, Shape{1, 1, 1}, std::vector<float>{1.0});
auto add2 = std::make_shared<ov::op::v1::Add>(tan, const4);
auto const5 = ov::op::v0::Constant::create(element::f32, Shape{1, 1, 1}, std::vector<float>{0.5});
auto mul5 = std::make_shared<ov::op::v1::Multiply>(param, const5);
auto mul6 = std::make_shared<ov::op::v1::Multiply>(add2, mul5);
auto result = std::make_shared<ov::op::v0::Result>(mul6);
model = std::make_shared<Model>(NodeVector{result}, ParameterVector{param});
manager.register_pass<ov::pass::GeluFusionWithTanhNoPower>();
}
{
auto param = std::make_shared<ov::op::v0::Parameter>(element::f32, Shape{1006, 2, 100, 3, 4096});
auto gelu = std::make_shared<ov::op::v7::Gelu>(param, ov::op::GeluApproximationMode::TANH);
auto result = std::make_shared<ov::op::v0::Result>(gelu);
model_ref = std::make_shared<Model>(NodeVector{result}, ParameterVector{param});
}
}