diff --git a/inference-engine/src/cldnn_engine/cldnn_program.cpp b/inference-engine/src/cldnn_engine/cldnn_program.cpp index 5897b94aaab..33b5670350d 100644 --- a/inference-engine/src/cldnn_engine/cldnn_program.cpp +++ b/inference-engine/src/cldnn_engine/cldnn_program.cpp @@ -575,6 +575,7 @@ Program::LayerType Program::LayerTypeFromStr(const std::string &str) { { "Sinh" , Sinh }, { "Cosh" , Cosh }, { "Swish" , Swish }, + { "HSwish", HSwish }, { "Mish" , Mish }, { "Gelu" , Gelu }, { "Atanh" , Atanh }, @@ -1162,6 +1163,7 @@ void Program::CreateSingleLayerPrimitive(cldnn::topology& topology, InferenceEng case SoftPlus: case SoftSign: case Swish: + case HSwish: case Mish: case Gelu: CreateActivationPrimitive(topology, layer, LayerTypeFromStr(layer->type)); @@ -2780,6 +2782,8 @@ void Program::CreateActivationPrimitive(cldnn::topology& topology, InferenceEngi activationType = ELU; } else if (activation_type == "swish") { activationType = Swish; + } else if (activation_type == "hswish") { + activationType = HSwish; } else if (activation_type == "mish") { activationType = Mish; } else if (activation_type == "gelu") { @@ -2977,6 +2981,11 @@ void Program::CreateActivationPrimitive(cldnn::topology& topology, InferenceEngi func = cldnn::activation_func::swish; break; } + case HSwish: + { + func = cldnn::activation_func::hswish; + break; + } case Mish: { func = cldnn::activation_func::mish; diff --git a/inference-engine/src/cldnn_engine/cldnn_program.h b/inference-engine/src/cldnn_engine/cldnn_program.h index b7c1cff4187..d736b354222 100644 --- a/inference-engine/src/cldnn_engine/cldnn_program.h +++ b/inference-engine/src/cldnn_engine/cldnn_program.h @@ -202,6 +202,7 @@ public: SoftPlus, SoftSign, Swish, + HSwish, Mish, Gelu, Sin, diff --git a/inference-engine/thirdparty/clDNN/api/activation.hpp b/inference-engine/thirdparty/clDNN/api/activation.hpp index e19deacb3ea..9b892d6030a 100644 --- a/inference-engine/thirdparty/clDNN/api/activation.hpp +++ b/inference-engine/thirdparty/clDNN/api/activation.hpp @@ -68,6 +68,7 @@ enum class activation_func { softplus, // ln(exp(val) + 1) softsign, // (val/(1+|val|)) swish, // (val*sigmoid(val)) + hswish, // val * min(max(0, val + 3), 6) / 6 mish, // val*tanh(ln(1 + exp(val))) gelu // (0.5*val*(1 + erf(val / sqrt(2))) }; diff --git a/inference-engine/thirdparty/clDNN/kernel_selector/common/common_types.h b/inference-engine/thirdparty/clDNN/kernel_selector/common/common_types.h index 7f1f977aea0..6797532c49e 100644 --- a/inference-engine/thirdparty/clDNN/kernel_selector/common/common_types.h +++ b/inference-engine/thirdparty/clDNN/kernel_selector/common/common_types.h @@ -150,6 +150,7 @@ enum class ActivationFunction { SOFTPLUS, SOFTSIGN, SWISH, + HSWISH, MISH, GELU }; diff --git a/inference-engine/thirdparty/clDNN/kernel_selector/core/common/jitter.cpp b/inference-engine/thirdparty/clDNN/kernel_selector/core/common/jitter.cpp index a9910c12aa4..0c6ba7a02d1 100644 --- a/inference-engine/thirdparty/clDNN/kernel_selector/core/common/jitter.cpp +++ b/inference-engine/thirdparty/clDNN/kernel_selector/core/common/jitter.cpp @@ -717,6 +717,15 @@ JitConstants MakeActivationJitConstants(ActivationFunction activation_function, (input / (one + exp(neg(input)))).str())); break; } + case ActivationFunction::HSWISH: { + std::string type_suffix = out_dt == Datatype::F32 ? "f" : "h"; + const JitTerm three("3." + type_suffix); + const JitTerm six("6." + type_suffix); + jitConstants.AddConstant(MakeJitConstant( + macro_def, + (input * min_func(max_func(zero, input + three), six) / six).str())); + break; + } case ActivationFunction::MISH: { std::string type_suffix = out_dt == Datatype::F32 ? "f" : "h"; auto bound = out_dt == Datatype::F32 ? "9.9f"_jit : "4.75h"_jit; diff --git a/inference-engine/thirdparty/clDNN/kernel_selector/core/kernel_selector_common.cpp b/inference-engine/thirdparty/clDNN/kernel_selector/core/kernel_selector_common.cpp index 593838e2dfe..03c51b5f737 100644 --- a/inference-engine/thirdparty/clDNN/kernel_selector/core/kernel_selector_common.cpp +++ b/inference-engine/thirdparty/clDNN/kernel_selector/core/kernel_selector_common.cpp @@ -82,6 +82,7 @@ std::string toString(ActivationFunction activation) { case ActivationFunction::SOFTPLUS: method = "SOFTPLUS"; break; case ActivationFunction::SOFTSIGN: method = "SOFTSIGN"; break; case ActivationFunction::SWISH: method = "SWISH"; break; + case ActivationFunction::HSWISH: method = "HSWISH"; break; case ActivationFunction::MISH: method = "MISH"; break; case ActivationFunction::GELU: method = "GELU"; break; default: break; diff --git a/inference-engine/thirdparty/clDNN/src/kernel_selector_helper.cpp b/inference-engine/thirdparty/clDNN/src/kernel_selector_helper.cpp index d45c90327fd..79d9bebf6f7 100644 --- a/inference-engine/thirdparty/clDNN/src/kernel_selector_helper.cpp +++ b/inference-engine/thirdparty/clDNN/src/kernel_selector_helper.cpp @@ -666,6 +666,8 @@ kernel_selector::activation_function get_kernel_selector_activation_param(activa return kernel_selector::activation_function::HARD_SIGMOID; case cldnn::activation_func::swish: return kernel_selector::activation_function::SWISH; + case cldnn::activation_func::hswish: + return kernel_selector::activation_function::HSWISH; case cldnn::activation_func::mish: return kernel_selector::activation_function::MISH; case cldnn::activation_func::gelu: diff --git a/inference-engine/thirdparty/clDNN/tests/test_cases/activation_simple_gpu_test.cpp b/inference-engine/thirdparty/clDNN/tests/test_cases/activation_simple_gpu_test.cpp index da9f4612d95..8c801b5305a 100644 --- a/inference-engine/thirdparty/clDNN/tests/test_cases/activation_simple_gpu_test.cpp +++ b/inference-engine/thirdparty/clDNN/tests/test_cases/activation_simple_gpu_test.cpp @@ -695,6 +695,46 @@ TEST(activation_f16_fw_gpu, basic_yxfb_mish) { } } +TEST(activation_f16_fw_gpu, basic_yxfb_hswish) { + const auto& engine = get_test_engine(); + + auto input = memory::allocate(engine, { data_types::f16, format::yxfb, { 1, 2, 5, 2 } }); + set_values(input, + { FLOAT16(0.0f), FLOAT16(-2.0f), FLOAT16(-3.0f), FLOAT16(4.0f), FLOAT16(5.0f), + FLOAT16(2.0f), FLOAT16(2.0f), FLOAT16(3.0f), FLOAT16(4.0f), FLOAT16(-6.0f), + FLOAT16(3.0f), FLOAT16(-3.0f), FLOAT16(3.0f), FLOAT16(5.0f), FLOAT16(1.0f), + FLOAT16(1.0f), FLOAT16(1.0f), FLOAT16(1.0f), FLOAT16(-1.0f), FLOAT16(1.0f) }); + + topology topology( + input_layout("input", input.get_layout()), + activation("hswish", "input", activation_func::hswish)); + network network(engine, topology); + network.set_input_data("input", input); + auto outputs = network.execute(); + EXPECT_EQ(outputs.size(), size_t(1)); + EXPECT_EQ(outputs.begin()->first, "hswish"); + + auto output_memory = outputs.at("hswish").get_memory(); + auto output_layout = output_memory.get_layout(); + auto output_ptr = output_memory.pointer(); + auto input_ptr = input.pointer(); + + int y_size = output_layout.size.spatial[1]; + int x_size = output_layout.size.spatial[0]; + int f_size = output_layout.size.feature[0]; + int b_size = output_layout.size.batch[0]; + EXPECT_EQ(output_layout.format, format::yxfb); + EXPECT_EQ(y_size, 2); + EXPECT_EQ(x_size, 5); + EXPECT_EQ(f_size, 2); + EXPECT_EQ(b_size, 1); + + for (size_t i = 0; i < output_layout.get_linear_size(); ++i) { + EXPECT_NEAR((FLOAT16)((float)input_ptr[i] * std::fmin(std::fmax(0.f, (float)input_ptr[i] + 3.f), 6.f) / 6.f), + output_ptr[i], 1e-3f); + } +} + TEST(activation_f32_fw_gpu, basic_yxfb_all_functions) { // Input: @@ -740,6 +780,7 @@ TEST(activation_f32_fw_gpu, basic_yxfb_all_functions) activation_func::negative, activation_func::abs, activation_func::swish, + activation_func::hswish, activation_func::mish, activation_func::gelu }; @@ -858,6 +899,9 @@ TEST(activation_f32_fw_gpu, basic_yxfb_all_functions) case activation_func::swish: EXPECT_FLOAT_EQ((float)input_ptr[i] / (1.f + std::exp((float)(-input_ptr[i]))), output_ptr[i]); break; + case activation_func::hswish: + EXPECT_FLOAT_EQ((float)input_ptr[i] * std::fmin(std::fmax(0.f, (float)input_ptr[i] + 3.f), 6.f) / 6.f, output_ptr[i]); + break; case activation_func::mish: EXPECT_NEAR((float)input_ptr[i] * std::tanh(std::log(1.f + std::exp((float)input_ptr[i]))), output_ptr[i], 1e-5f);