From 77365bcb4c337394c10f0cb79a5f28ec506b00f6 Mon Sep 17 00:00:00 2001 From: Roman Lyamin Date: Tue, 27 Oct 2020 10:56:15 +0300 Subject: [PATCH] [IE CLDNN] Added Round-5 operation (#2838) --- .../src/cldnn_engine/cldnn_program.cpp | 25 ++ .../src/cldnn_engine/cldnn_program.h | 2 + .../single_layer_tests/activation.cpp | 60 ++--- .../include/single_layer_tests/activation.hpp | 84 +++--- .../ngraph_functions/utils/ngraph_helpers.hpp | 4 +- .../tests/ngraph_functions/src/activation.cpp | 4 + .../thirdparty/clDNN/api/activation.hpp | 88 ++++--- .../kernel_selector/common/common_types.h | 4 +- .../kernel_selector/core/common/jitter.cpp | 6 + .../core/kernel_selector_common.cpp | 88 ++++--- .../clDNN/src/kernel_selector_helper.cpp | 4 + .../test_cases/activation_simple_gpu_test.cpp | 244 +++++++----------- 12 files changed, 299 insertions(+), 314 deletions(-) diff --git a/inference-engine/src/cldnn_engine/cldnn_program.cpp b/inference-engine/src/cldnn_engine/cldnn_program.cpp index 3c88dc3b7da..a532f4f6c0c 100644 --- a/inference-engine/src/cldnn_engine/cldnn_program.cpp +++ b/inference-engine/src/cldnn_engine/cldnn_program.cpp @@ -869,6 +869,7 @@ Program::LayerType Program::LayerTypeFromStr(const std::string &str) { { "CTCGreedyDecoder", CTCGreedyDecoder }, { "PriorBoxClustered", PriorBoxClustered }, { "CumSum", CumSum }, + { "Round", Round }, { "EmbeddingBagPackedSum", EmbeddingBagPackedSum }, { "EmbeddingBagOffsetsSum", EmbeddingBagOffsetsSum }, { "EmbeddingSegmentsSum", EmbeddingSegmentsSum }, @@ -1563,6 +1564,8 @@ void Program::CreateSingleLayerPrimitive(cldnn::topology& topology, InferenceEng break; case CumSum: CreateCumSumPrimitive(topology, layer); break; + case Round: CreateRoundPrimitive(topology, layer); + break; case EmbeddingBagPackedSum: CreateEmbeddingBagPackedSumPrimitive(topology, layer); break; case EmbeddingBagOffsetsSum: CreateEmbeddingBagOffsetsSumPrimitive(topology, layer); @@ -5273,6 +5276,28 @@ void Program::CreateCumSumPrimitive(cldnn::topology& topology, InferenceEngine:: AddPrimitiveToProfiler(layerName, layer); } +void Program::CreateRoundPrimitive(cldnn::topology& topology, InferenceEngine::CNNLayerPtr& layer) { + ValidateLayer(layer, 1); + auto inputPrimitives = GetPrevLayersPrimitives(layer); + auto layerName = layer_type_name_ID(layer); + + std::string mode = layer->GetParamAsString("mode", "half_to_even"); + + if ((mode != "half_to_even") && (mode != "half_away_from_zero")) { + THROW_CLDNN_EXCEPTION("Unsupported mode (" + mode + ") in layer " + layerName); + } + + auto func = mode == "half_to_even" ? cldnn::activation_func::round_half_to_even : cldnn::activation_func::round_half_away_from_zero; + + auto primitive = cldnn::activation( + layerName, + inputPrimitives[0], + func); + + topology.add(primitive); + AddPrimitiveToProfiler(layerName, layer); +} + void Program::CreatePriorBoxClusteredPrimitive(cldnn::topology& topology, InferenceEngine::CNNLayerPtr& layer) { ValidateLayer(layer, 2); auto pbcLayer = as(layer); diff --git a/inference-engine/src/cldnn_engine/cldnn_program.h b/inference-engine/src/cldnn_engine/cldnn_program.h index ed677552ac1..6802fa319d7 100644 --- a/inference-engine/src/cldnn_engine/cldnn_program.h +++ b/inference-engine/src/cldnn_engine/cldnn_program.h @@ -224,6 +224,7 @@ public: CTCGreedyDecoder, PriorBoxClustered, CumSum, + Round, EmbeddingBagPackedSum, EmbeddingBagOffsetsSum, EmbeddingSegmentsSum, @@ -387,6 +388,7 @@ private: void CreateCTCGreedyDecoderPrimitive(cldnn::topology& topology, InferenceEngine::CNNLayerPtr& layer); void CreatePriorBoxClusteredPrimitive(cldnn::topology& topology, InferenceEngine::CNNLayerPtr& layer); void CreateCumSumPrimitive(cldnn::topology& topology, InferenceEngine::CNNLayerPtr& layer); + void CreateRoundPrimitive(cldnn::topology& topology, InferenceEngine::CNNLayerPtr& layer); void CreateEmbeddingBagPackedSumPrimitive(cldnn::topology& topology, InferenceEngine::CNNLayerPtr& layer); void CreateEmbeddingBagOffsetsSumPrimitive(cldnn::topology& topology, InferenceEngine::CNNLayerPtr& layer); void CreateEmbeddingSegmentsSumPrimitive(cldnn::topology& topology, InferenceEngine::CNNLayerPtr& layer); diff --git a/inference-engine/tests/functional/plugin/gpu/shared_tests_instances/single_layer_tests/activation.cpp b/inference-engine/tests/functional/plugin/gpu/shared_tests_instances/single_layer_tests/activation.cpp index 154a9986a60..e5ea6780423 100644 --- a/inference-engine/tests/functional/plugin/gpu/shared_tests_instances/single_layer_tests/activation.cpp +++ b/inference-engine/tests/functional/plugin/gpu/shared_tests_instances/single_layer_tests/activation.cpp @@ -17,35 +17,37 @@ const std::vector netPrecisions = { }; const std::map>> activationTypes = { - {Sigmoid, {}}, - {Tanh, {}}, - {Relu, {}}, - {Exp, {}}, - {Log, {}}, - {Sign, {}}, - {Abs, {}}, - {Gelu, {}}, - {Clamp, {{-2.0f, 2.0f}}}, - {Negative, {}}, - {Acos, {}}, - {Asin, {}}, - {Atan, {}}, - {Cos, {}}, - {Cosh, {}}, - {Floor, {}}, - {Sin, {}}, - {Sinh, {}}, - {Sqrt, {}}, - {Tan, {}}, - {Elu, {{0.1f}}}, - {Erf, {}}, - {HardSigmoid, {{0.2f, 0.5f}}}, - {Selu, {{1.6732f, 1.0507f}}}, - {Ceiling, {}}, - {Mish, {}}, - {HSwish, {}}, - {SoftPlus, {}}, - {HSigmoid, {}} + {Sigmoid, {}}, + {Tanh, {}}, + {Relu, {}}, + {Exp, {}}, + {Log, {}}, + {Sign, {}}, + {Abs, {}}, + {Gelu, {}}, + {Clamp, {{-2.0f, 2.0f}}}, + {Negative, {}}, + {Acos, {}}, + {Asin, {}}, + {Atan, {}}, + {Cos, {}}, + {Cosh, {}}, + {Floor, {}}, + {Sin, {}}, + {Sinh, {}}, + {Sqrt, {}}, + {Tan, {}}, + {Elu, {{0.1f}}}, + {Erf, {}}, + {HardSigmoid, {{0.2f, 0.5f}}}, + {Selu, {{1.6732f, 1.0507f}}}, + {Ceiling, {}}, + {Mish, {}}, + {HSwish, {}}, + {SoftPlus, {}}, + {HSigmoid, {}}, + {RoundHalfToEven, {}}, + {RoundHalfAwayFromZero, {}} }; std::map, std::vector>> basic = { diff --git a/inference-engine/tests/functional/plugin/shared/include/single_layer_tests/activation.hpp b/inference-engine/tests/functional/plugin/shared/include/single_layer_tests/activation.hpp index bbf6e9f32b6..97d3c7d346e 100644 --- a/inference-engine/tests/functional/plugin/shared/include/single_layer_tests/activation.hpp +++ b/inference-engine/tests/functional/plugin/shared/include/single_layer_tests/activation.hpp @@ -31,47 +31,49 @@ namespace LayerTestsDefinitions { static std::map activationNames = { - {ngraph::helpers::ActivationTypes::Sigmoid, "Sigmoid"}, - {ngraph::helpers::ActivationTypes::Tanh, "Tanh"}, - {ngraph::helpers::ActivationTypes::Relu, "Relu"}, - {ngraph::helpers::ActivationTypes::LeakyRelu, "LeakyRelu"}, - {ngraph::helpers::ActivationTypes::Exp, "Exp"}, - {ngraph::helpers::ActivationTypes::Log, "Log"}, - {ngraph::helpers::ActivationTypes::Sign, "Sign"}, - {ngraph::helpers::ActivationTypes::Abs, "Abs"}, - {ngraph::helpers::ActivationTypes::Gelu, "Gelu"}, - {ngraph::helpers::ActivationTypes::Clamp, "Clamp"}, - {ngraph::helpers::ActivationTypes::Negative, "Negative"}, - {ngraph::helpers::ActivationTypes::Acos, "Acos"}, - {ngraph::helpers::ActivationTypes::Asin, "Asin"}, - {ngraph::helpers::ActivationTypes::Atan, "Atan"}, - {ngraph::helpers::ActivationTypes::Cos, "Cos"}, - {ngraph::helpers::ActivationTypes::Cosh, "Cosh"}, - {ngraph::helpers::ActivationTypes::Floor, "Floor"}, - {ngraph::helpers::ActivationTypes::Sin, "Sin"}, - {ngraph::helpers::ActivationTypes::Sinh, "Sinh"}, - {ngraph::helpers::ActivationTypes::Sqrt, "Sqrt"}, - {ngraph::helpers::ActivationTypes::Tan, "Tan"}, - {ngraph::helpers::ActivationTypes::Elu, "Elu"}, - {ngraph::helpers::ActivationTypes::Erf, "Erf"}, - {ngraph::helpers::ActivationTypes::HardSigmoid, "HardSigmoid"}, - {ngraph::helpers::ActivationTypes::Selu, "Selu"}, - {ngraph::helpers::ActivationTypes::Sigmoid, "Sigmoid"}, - {ngraph::helpers::ActivationTypes::Tanh, "Tanh"}, - {ngraph::helpers::ActivationTypes::Relu, "Relu"}, - {ngraph::helpers::ActivationTypes::LeakyRelu, "LeakyRelu"}, - {ngraph::helpers::ActivationTypes::Exp, "Exp"}, - {ngraph::helpers::ActivationTypes::Log, "Log"}, - {ngraph::helpers::ActivationTypes::Sign, "Sign"}, - {ngraph::helpers::ActivationTypes::Abs, "Abs"}, - {ngraph::helpers::ActivationTypes::Gelu, "Gelu"}, - {ngraph::helpers::ActivationTypes::Ceiling, "Ceiling"}, - {ngraph::helpers::ActivationTypes::PReLu, "PReLu"}, - {ngraph::helpers::ActivationTypes::Mish, "Mish"}, - {ngraph::helpers::ActivationTypes::HSwish, "HSwish"}, - {ngraph::helpers::ActivationTypes::SoftPlus, "SoftPlus"}, - {ngraph::helpers::ActivationTypes::Swish, "Swish"}, - {ngraph::helpers::ActivationTypes::HSigmoid, "HSigmoid"}, + {ngraph::helpers::ActivationTypes::Sigmoid, "Sigmoid"}, + {ngraph::helpers::ActivationTypes::Tanh, "Tanh"}, + {ngraph::helpers::ActivationTypes::Relu, "Relu"}, + {ngraph::helpers::ActivationTypes::LeakyRelu, "LeakyRelu"}, + {ngraph::helpers::ActivationTypes::Exp, "Exp"}, + {ngraph::helpers::ActivationTypes::Log, "Log"}, + {ngraph::helpers::ActivationTypes::Sign, "Sign"}, + {ngraph::helpers::ActivationTypes::Abs, "Abs"}, + {ngraph::helpers::ActivationTypes::Gelu, "Gelu"}, + {ngraph::helpers::ActivationTypes::Clamp, "Clamp"}, + {ngraph::helpers::ActivationTypes::Negative, "Negative"}, + {ngraph::helpers::ActivationTypes::Acos, "Acos"}, + {ngraph::helpers::ActivationTypes::Asin, "Asin"}, + {ngraph::helpers::ActivationTypes::Atan, "Atan"}, + {ngraph::helpers::ActivationTypes::Cos, "Cos"}, + {ngraph::helpers::ActivationTypes::Cosh, "Cosh"}, + {ngraph::helpers::ActivationTypes::Floor, "Floor"}, + {ngraph::helpers::ActivationTypes::Sin, "Sin"}, + {ngraph::helpers::ActivationTypes::Sinh, "Sinh"}, + {ngraph::helpers::ActivationTypes::Sqrt, "Sqrt"}, + {ngraph::helpers::ActivationTypes::Tan, "Tan"}, + {ngraph::helpers::ActivationTypes::Elu, "Elu"}, + {ngraph::helpers::ActivationTypes::Erf, "Erf"}, + {ngraph::helpers::ActivationTypes::HardSigmoid, "HardSigmoid"}, + {ngraph::helpers::ActivationTypes::Selu, "Selu"}, + {ngraph::helpers::ActivationTypes::Sigmoid, "Sigmoid"}, + {ngraph::helpers::ActivationTypes::Tanh, "Tanh"}, + {ngraph::helpers::ActivationTypes::Relu, "Relu"}, + {ngraph::helpers::ActivationTypes::LeakyRelu, "LeakyRelu"}, + {ngraph::helpers::ActivationTypes::Exp, "Exp"}, + {ngraph::helpers::ActivationTypes::Log, "Log"}, + {ngraph::helpers::ActivationTypes::Sign, "Sign"}, + {ngraph::helpers::ActivationTypes::Abs, "Abs"}, + {ngraph::helpers::ActivationTypes::Gelu, "Gelu"}, + {ngraph::helpers::ActivationTypes::Ceiling, "Ceiling"}, + {ngraph::helpers::ActivationTypes::PReLu, "PReLu"}, + {ngraph::helpers::ActivationTypes::Mish, "Mish"}, + {ngraph::helpers::ActivationTypes::HSwish, "HSwish"}, + {ngraph::helpers::ActivationTypes::SoftPlus, "SoftPlus"}, + {ngraph::helpers::ActivationTypes::Swish, "Swish"}, + {ngraph::helpers::ActivationTypes::HSigmoid, "HSigmoid"}, + {ngraph::helpers::ActivationTypes::RoundHalfToEven, "RoundHalfToEven"}, + {ngraph::helpers::ActivationTypes::RoundHalfAwayFromZero, "RoundHalfAwayFromZero"} }; typedef std::tuple< diff --git a/inference-engine/tests/ngraph_functions/include/ngraph_functions/utils/ngraph_helpers.hpp b/inference-engine/tests/ngraph_functions/include/ngraph_functions/utils/ngraph_helpers.hpp index 2114f3fd4b1..9daf61a4167 100644 --- a/inference-engine/tests/ngraph_functions/include/ngraph_functions/utils/ngraph_helpers.hpp +++ b/inference-engine/tests/ngraph_functions/include/ngraph_functions/utils/ngraph_helpers.hpp @@ -112,7 +112,9 @@ enum ActivationTypes { HSwish, SoftPlus, Swish, - HSigmoid + HSigmoid, + RoundHalfToEven, + RoundHalfAwayFromZero }; enum EltwiseTypes { diff --git a/inference-engine/tests/ngraph_functions/src/activation.cpp b/inference-engine/tests/ngraph_functions/src/activation.cpp index cbccfb670de..ee0add014b9 100644 --- a/inference-engine/tests/ngraph_functions/src/activation.cpp +++ b/inference-engine/tests/ngraph_functions/src/activation.cpp @@ -104,6 +104,10 @@ std::shared_ptr makeActivation(const ngraph::Output &in, } case ngraph::helpers::ActivationTypes::HSigmoid: return std::make_shared(in); + case ngraph::helpers::ActivationTypes::RoundHalfToEven: + return std::make_shared(in, ngraph::op::v5::Round::RoundMode::HALF_TO_EVEN); + case ngraph::helpers::ActivationTypes::RoundHalfAwayFromZero: + return std::make_shared(in, ngraph::op::v5::Round::RoundMode::HALF_AWAY_FROM_ZERO); default: throw std::runtime_error("Can't create layer for this activation type"); } diff --git a/inference-engine/thirdparty/clDNN/api/activation.hpp b/inference-engine/thirdparty/clDNN/api/activation.hpp index ce5d8e8db7e..e571eb595f9 100644 --- a/inference-engine/thirdparty/clDNN/api/activation.hpp +++ b/inference-engine/thirdparty/clDNN/api/activation.hpp @@ -29,49 +29,51 @@ namespace cldnn { /// @brief activation functions enum class activation_func { - none, // val - logistic, // 1/(1 + exp(-val)) - hyperbolic_tan, // tanh(val) - relu, // max(0, val) - relu_negative_slope, // max(0, val) + a * min(0, val) (a is additional param) - clamp, // max(a, min(b, val) (a,b are additional param) - softrelu, // log(1 + exp(val)) - abs, // abs(val) - linear, // a*val + b (a,b are additional params) - square, // val*val - sqrt, // sqrt(val) - elu, // max(0, val) + a * (exp(min(0, val) - 1) (a is additional param) - sin, // sin(val) - asin, // asin(val) - sinh, // sinh(val) - asinh, // asinh(val) - cos, // cos(val) - acos, // acos(val) - cosh, // cosh(val) - acosh, // acosh(val) - log, // log(val) - log2, // log2(val) - exp, // exp(val) - tan, // tan(val) - atan, // atan(val) - atanh, // atanh(val) - floor, // floor(val) - ceil, // ceil(val) - negative, // -val - negation, // !val - pow, // pow(val, a) - reciprocal, // (1/val) - erf, // Gauss error function - hard_sigmoid, // max(0, min(1, a * val + b)) (a,b are additional params) - hsigmoid, // min(max(val + 3, 0), 6) / 6 - selu, // for val <= 0: b * (a * e^val - a); for val > 0: b * val (a,b are additional params) - sign, // val > 0: 1; val < 0: -1; val == 0: 0 - softplus, // ln(exp(val) + 1) - softsign, // (val/(1+|val|)) - swish, // (val*sigmoid(val)) - hswish, // val * min(max(0, val + 3), 6) / 6 - mish, // val*tanh(ln(1 + exp(val))) - gelu // (0.5*val*(1 + erf(val / sqrt(2))) + none, // val + logistic, // 1/(1 + exp(-val)) + hyperbolic_tan, // tanh(val) + relu, // max(0, val) + relu_negative_slope, // max(0, val) + a * min(0, val) (a is additional param) + clamp, // max(a, min(b, val) (a,b are additional param) + softrelu, // log(1 + exp(val)) + abs, // abs(val) + linear, // a*val + b (a,b are additional params) + square, // val*val + sqrt, // sqrt(val) + elu, // max(0, val) + a * (exp(min(0, val) - 1) (a is additional param) + sin, // sin(val) + asin, // asin(val) + sinh, // sinh(val) + asinh, // asinh(val) + cos, // cos(val) + acos, // acos(val) + cosh, // cosh(val) + acosh, // acosh(val) + log, // log(val) + log2, // log2(val) + exp, // exp(val) + tan, // tan(val) + atan, // atan(val) + atanh, // atanh(val) + floor, // floor(val) + ceil, // ceil(val) + negative, // -val + negation, // !val + pow, // pow(val, a) + reciprocal, // (1/val) + erf, // Gauss error function + hard_sigmoid, // max(0, min(1, a * val + b)) (a,b are additional params) + hsigmoid, // min(max(val + 3, 0), 6) / 6 + selu, // for val <= 0: b * (a * e^val - a); for val > 0: b * val (a,b are additional params) + sign, // val > 0: 1; val < 0: -1; val == 0: 0 + softplus, // ln(exp(val) + 1) + softsign, // (val/(1+|val|)) + swish, // (val*sigmoid(val)) + hswish, // val * min(max(0, val + 3), 6) / 6 + mish, // val*tanh(ln(1 + exp(val))) + gelu, // (0.5*val*(1 + erf(val / sqrt(2))) + round_half_to_even, // round halfs to the nearest even integer + round_half_away_from_zero // round the number so it's further away from zero }; /// @brief activation additional params diff --git a/inference-engine/thirdparty/clDNN/kernel_selector/common/common_types.h b/inference-engine/thirdparty/clDNN/kernel_selector/common/common_types.h index fc06bd1115f..3978673e8e1 100644 --- a/inference-engine/thirdparty/clDNN/kernel_selector/common/common_types.h +++ b/inference-engine/thirdparty/clDNN/kernel_selector/common/common_types.h @@ -154,7 +154,9 @@ enum class ActivationFunction { SWISH, HSWISH, MISH, - GELU + GELU, + ROUND_HALF_TO_EVEN, + ROUND_HALF_AWAY_FROM_ZERO }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/inference-engine/thirdparty/clDNN/kernel_selector/core/common/jitter.cpp b/inference-engine/thirdparty/clDNN/kernel_selector/core/common/jitter.cpp index 7df057de511..9ba94c9a90c 100644 --- a/inference-engine/thirdparty/clDNN/kernel_selector/core/common/jitter.cpp +++ b/inference-engine/thirdparty/clDNN/kernel_selector/core/common/jitter.cpp @@ -1095,6 +1095,12 @@ JitConstants MakeActivationJitConstants(ActivationFunction activation_function, ternary(input.eq(zero), one, zero) .str())); // the workaround for OpenCL's vector type result (!input) break; + case ActivationFunction::ROUND_HALF_TO_EVEN: + jitConstants.AddConstant(MakeJitConstant( macro_def, "rint(input)")); + break; + case ActivationFunction::ROUND_HALF_AWAY_FROM_ZERO: + jitConstants.AddConstant(MakeJitConstant(macro_def, "(round(input))")); + break; case ActivationFunction::NONE: default: jitConstants.AddConstant(MakeJitConstant(macro_def, "input")); diff --git a/inference-engine/thirdparty/clDNN/kernel_selector/core/kernel_selector_common.cpp b/inference-engine/thirdparty/clDNN/kernel_selector/core/kernel_selector_common.cpp index ea1e3a9618b..41e7f506143 100644 --- a/inference-engine/thirdparty/clDNN/kernel_selector/core/kernel_selector_common.cpp +++ b/inference-engine/thirdparty/clDNN/kernel_selector/core/kernel_selector_common.cpp @@ -43,49 +43,51 @@ std::string GetStringEnv(const char* varName) { std::string toString(ActivationFunction activation) { std::string method("LINEAR"); switch (activation) { - case ActivationFunction::LOGISTIC: method = "LOGISTIC"; break; - case ActivationFunction::HYPERBOLIC_TAN: method = "HYPERBOLIC_TAN"; break; - case ActivationFunction::RELU: method = "RELU"; break; - case ActivationFunction::RELU_NEGATIVE_SLOPE: method = "RELU_NEGATIVE_SLOPE"; break; - case ActivationFunction::CLAMP: method = "CLAMP"; break; - case ActivationFunction::SOFTRELU: method = "SOFTRELU"; break; - case ActivationFunction::ABS: method = "ABS"; break; - case ActivationFunction::SQUARE: method = "SQUARE"; break; - case ActivationFunction::SQRT: method = "SQRT"; break; - case ActivationFunction::LINEAR: method = "LINEAR"; break; - case ActivationFunction::ELU: method = "ELU"; break; - case ActivationFunction::SIN: method = "SIN"; break; - case ActivationFunction::ASIN: method = "ASIN"; break; - case ActivationFunction::SINH: method = "SINH"; break; - case ActivationFunction::ASINH: method = "ASINH"; break; - case ActivationFunction::COS: method = "COS"; break; - case ActivationFunction::ACOS: method = "ACOS"; break; - case ActivationFunction::COSH: method = "COSH"; break; - case ActivationFunction::ACOSH: method = "ACOSH"; break; - case ActivationFunction::LOG: method = "LOG"; break; - case ActivationFunction::LOG2: method = "LOG2"; break; - case ActivationFunction::EXP: method = "EXP"; break; - case ActivationFunction::NOT: method = "NOT"; break; - case ActivationFunction::POW: method = "POW"; break; - case ActivationFunction::NONE: method = "NONE"; break; - case ActivationFunction::TAN: method = "TAN"; break; - case ActivationFunction::ATAN: method = "ATAN"; break; - case ActivationFunction::ATANH: method = "ATANH"; break; - case ActivationFunction::FLOOR: method = "FLOOR"; break; - case ActivationFunction::CEIL: method = "CEIL"; break; - case ActivationFunction::NEGATIVE: method = "NEGATIVE"; break; - case ActivationFunction::ERF: method = "ERF"; break; - case ActivationFunction::HARD_SIGMOID: method = "HARD_SIGMOID"; break; - case ActivationFunction::HSIGMOID: method = "HSIGMOID"; break; - case ActivationFunction::RECIPROCAL: method = "RECIPROCAL"; break; - case ActivationFunction::SELU: method = "SELU"; break; - case ActivationFunction::SIGN: method = "SIGN"; break; - case ActivationFunction::SOFTPLUS: method = "SOFTPLUS"; break; - case ActivationFunction::SOFTSIGN: method = "SOFTSIGN"; break; - case ActivationFunction::SWISH: method = "SWISH"; break; - case ActivationFunction::HSWISH: method = "HSWISH"; break; - case ActivationFunction::MISH: method = "MISH"; break; - case ActivationFunction::GELU: method = "GELU"; break; + case ActivationFunction::LOGISTIC: method = "LOGISTIC"; break; + case ActivationFunction::HYPERBOLIC_TAN: method = "HYPERBOLIC_TAN"; break; + case ActivationFunction::RELU: method = "RELU"; break; + case ActivationFunction::RELU_NEGATIVE_SLOPE: method = "RELU_NEGATIVE_SLOPE"; break; + case ActivationFunction::CLAMP: method = "CLAMP"; break; + case ActivationFunction::SOFTRELU: method = "SOFTRELU"; break; + case ActivationFunction::ABS: method = "ABS"; break; + case ActivationFunction::SQUARE: method = "SQUARE"; break; + case ActivationFunction::SQRT: method = "SQRT"; break; + case ActivationFunction::LINEAR: method = "LINEAR"; break; + case ActivationFunction::ELU: method = "ELU"; break; + case ActivationFunction::SIN: method = "SIN"; break; + case ActivationFunction::ASIN: method = "ASIN"; break; + case ActivationFunction::SINH: method = "SINH"; break; + case ActivationFunction::ASINH: method = "ASINH"; break; + case ActivationFunction::COS: method = "COS"; break; + case ActivationFunction::ACOS: method = "ACOS"; break; + case ActivationFunction::COSH: method = "COSH"; break; + case ActivationFunction::ACOSH: method = "ACOSH"; break; + case ActivationFunction::LOG: method = "LOG"; break; + case ActivationFunction::LOG2: method = "LOG2"; break; + case ActivationFunction::EXP: method = "EXP"; break; + case ActivationFunction::NOT: method = "NOT"; break; + case ActivationFunction::POW: method = "POW"; break; + case ActivationFunction::NONE: method = "NONE"; break; + case ActivationFunction::TAN: method = "TAN"; break; + case ActivationFunction::ATAN: method = "ATAN"; break; + case ActivationFunction::ATANH: method = "ATANH"; break; + case ActivationFunction::FLOOR: method = "FLOOR"; break; + case ActivationFunction::CEIL: method = "CEIL"; break; + case ActivationFunction::NEGATIVE: method = "NEGATIVE"; break; + case ActivationFunction::ERF: method = "ERF"; break; + case ActivationFunction::HARD_SIGMOID: method = "HARD_SIGMOID"; break; + case ActivationFunction::HSIGMOID: method = "HSIGMOID"; break; + case ActivationFunction::RECIPROCAL: method = "RECIPROCAL"; break; + case ActivationFunction::SELU: method = "SELU"; break; + case ActivationFunction::SIGN: method = "SIGN"; break; + case ActivationFunction::SOFTPLUS: method = "SOFTPLUS"; break; + case ActivationFunction::SOFTSIGN: method = "SOFTSIGN"; break; + case ActivationFunction::SWISH: method = "SWISH"; break; + case ActivationFunction::HSWISH: method = "HSWISH"; break; + case ActivationFunction::MISH: method = "MISH"; break; + case ActivationFunction::GELU: method = "GELU"; break; + case ActivationFunction::ROUND_HALF_TO_EVEN: method = "ROUND_HALF_TO_EVEN"; break; + case ActivationFunction::ROUND_HALF_AWAY_FROM_ZERO: method = "ROUND_HALF_AWAY_FROM_ZERO"; break; default: break; } return method; diff --git a/inference-engine/thirdparty/clDNN/src/kernel_selector_helper.cpp b/inference-engine/thirdparty/clDNN/src/kernel_selector_helper.cpp index 0d6f168eaaa..0997dfc6976 100644 --- a/inference-engine/thirdparty/clDNN/src/kernel_selector_helper.cpp +++ b/inference-engine/thirdparty/clDNN/src/kernel_selector_helper.cpp @@ -703,6 +703,10 @@ kernel_selector::activation_function get_kernel_selector_activation_param(activa return kernel_selector::activation_function::MISH; case cldnn::activation_func::gelu: return kernel_selector::activation_function::GELU; + case cldnn::activation_func::round_half_to_even: + return kernel_selector::activation_function::ROUND_HALF_TO_EVEN; + case cldnn::activation_func::round_half_away_from_zero: + return kernel_selector::activation_function::ROUND_HALF_AWAY_FROM_ZERO; default: throw std::runtime_error("Unknown activation function"); break; diff --git a/inference-engine/thirdparty/clDNN/tests/test_cases/activation_simple_gpu_test.cpp b/inference-engine/thirdparty/clDNN/tests/test_cases/activation_simple_gpu_test.cpp index 5f99ac86474..369984777a0 100644 --- a/inference-engine/thirdparty/clDNN/tests/test_cases/activation_simple_gpu_test.cpp +++ b/inference-engine/thirdparty/clDNN/tests/test_cases/activation_simple_gpu_test.cpp @@ -484,42 +484,6 @@ TEST(activation_f16_fw_gpu, pow_basic_yxfb) { } } -TEST(activation_f16_fw_gpu, linear_basic_yxfb) { - const auto& engine = get_test_engine(); - - auto input = memory::allocate(engine, {data_types::f16, format::yxfb, {1, 1, 2, 2}}); - set_values(input, - {FLOAT16(1.0f), FLOAT16(2.0f), FLOAT16(3.0f), FLOAT16(4.5f)}); - VF output_vec = {FLOAT16(5.0f), FLOAT16(8.0f), FLOAT16(11.0f), FLOAT16(15.5f)}; - - topology topology( - input_layout("input", input.get_layout()), - activation("linear", "input", activation_func::linear, {FLOAT16(3.0f), FLOAT16(2.0f)})); - network network(engine, topology); - network.set_input_data("input", input); - auto outputs = network.execute(); - EXPECT_EQ(outputs.size(), size_t(1)); - EXPECT_EQ(outputs.begin()->first, "linear"); - - auto output_memory = outputs.at("linear").get_memory(); - auto output_layout = output_memory.get_layout(); - auto output_ptr = output_memory.pointer(); - - int y_size = output_layout.size.spatial[1]; - int x_size = output_layout.size.spatial[0]; - int f_size = output_layout.size.feature[0]; - int b_size = output_layout.size.batch[0]; - EXPECT_EQ(output_layout.format, format::yxfb); - EXPECT_EQ(y_size, 2); - EXPECT_EQ(x_size, 2); - EXPECT_EQ(f_size, 1); - EXPECT_EQ(b_size, 1); - - for (size_t i = 0; i < output_vec.size(); ++i) { - EXPECT_FLOAT_EQ(output_vec[i], output_ptr[i]); - } -} - TEST(activation_f32_fw_gpu, relu_basic_yxfb) { // Input: // 1 -2 -3 4 5 @@ -655,126 +619,6 @@ TEST(activation_f32_fw_gpu, relu_basic_bfzyx) { } } -TEST(activation_f16_fw_gpu, basic_yxfb_mish) { - const auto& engine = get_test_engine(); - - auto input = memory::allocate(engine, { data_types::f16, format::yxfb, { 1, 1, 5, 4 } }); - set_values(input, - { FLOAT16(0.0f), FLOAT16(-2.0f), FLOAT16(-3.0f), FLOAT16(4.0f), FLOAT16(5.0f), - FLOAT16(2.0f), FLOAT16(2.0f), FLOAT16(3.0f), FLOAT16(4.0f), FLOAT16(-6.0f), - FLOAT16(3.0f), FLOAT16(-3.0f), FLOAT16(3.0f), FLOAT16(5.0f), FLOAT16(1.0f), - FLOAT16(1.0f), FLOAT16(1.0f), FLOAT16(1.0f), FLOAT16(-1.0f), FLOAT16(1.0f) }); - - topology topology( - input_layout("input", input.get_layout()), - activation("mish", "input", activation_func::mish)); - network network(engine, topology); - network.set_input_data("input", input); - auto outputs = network.execute(); - EXPECT_EQ(outputs.size(), size_t(1)); - EXPECT_EQ(outputs.begin()->first, "mish"); - - auto output_memory = outputs.at("mish").get_memory(); - auto output_layout = output_memory.get_layout(); - auto output_ptr = output_memory.pointer(); - auto input_ptr = input.pointer(); - - int y_size = output_layout.size.spatial[1]; - int x_size = output_layout.size.spatial[0]; - int f_size = output_layout.size.feature[0]; - int b_size = output_layout.size.batch[0]; - EXPECT_EQ(output_layout.format, format::yxfb); - EXPECT_EQ(y_size, 4); - EXPECT_EQ(x_size, 5); - EXPECT_EQ(f_size, 1); - EXPECT_EQ(b_size, 1); - - for (size_t i = 0; i < output_layout.get_linear_size(); ++i) { - EXPECT_NEAR((FLOAT16)((float)input_ptr[i] * std::tanh(std::log(1.f + std::exp((float)input_ptr[i])))), - output_ptr[i], 1e-2f); - } -} - -TEST(activation_f16_fw_gpu, basic_yxfb_hswish) { - const auto& engine = get_test_engine(); - - auto input = memory::allocate(engine, { data_types::f16, format::yxfb, { 1, 2, 5, 2 } }); - set_values(input, - { FLOAT16(0.0f), FLOAT16(-2.0f), FLOAT16(-3.0f), FLOAT16(4.0f), FLOAT16(5.0f), - FLOAT16(2.0f), FLOAT16(2.0f), FLOAT16(3.0f), FLOAT16(4.0f), FLOAT16(-6.0f), - FLOAT16(3.0f), FLOAT16(-3.0f), FLOAT16(3.0f), FLOAT16(5.0f), FLOAT16(1.0f), - FLOAT16(1.0f), FLOAT16(1.0f), FLOAT16(1.0f), FLOAT16(-1.0f), FLOAT16(1.0f) }); - - topology topology( - input_layout("input", input.get_layout()), - activation("hswish", "input", activation_func::hswish)); - network network(engine, topology); - network.set_input_data("input", input); - auto outputs = network.execute(); - EXPECT_EQ(outputs.size(), size_t(1)); - EXPECT_EQ(outputs.begin()->first, "hswish"); - - auto output_memory = outputs.at("hswish").get_memory(); - auto output_layout = output_memory.get_layout(); - auto output_ptr = output_memory.pointer(); - auto input_ptr = input.pointer(); - - int y_size = output_layout.size.spatial[1]; - int x_size = output_layout.size.spatial[0]; - int f_size = output_layout.size.feature[0]; - int b_size = output_layout.size.batch[0]; - EXPECT_EQ(output_layout.format, format::yxfb); - EXPECT_EQ(y_size, 2); - EXPECT_EQ(x_size, 5); - EXPECT_EQ(f_size, 2); - EXPECT_EQ(b_size, 1); - - for (size_t i = 0; i < output_layout.get_linear_size(); ++i) { - EXPECT_NEAR((FLOAT16)((float)input_ptr[i] * std::fmin(std::fmax(0.f, (float)input_ptr[i] + 3.f), 6.f) / 6.f), - output_ptr[i], 1e-3f); - } -} - -TEST(activation_f16_fw_gpu, basic_yxfb_hsigmoid) { - const auto& engine = get_test_engine(); - - auto input = memory::allocate(engine, { data_types::f16, format::yxfb, { 1, 2, 5, 2 } }); - set_values(input, - { FLOAT16(0.0f), FLOAT16(-2.0f), FLOAT16(-3.0f), FLOAT16(4.0f), FLOAT16(5.0f), - FLOAT16(2.0f), FLOAT16(2.0f), FLOAT16(3.0f), FLOAT16(4.0f), FLOAT16(-6.0f), - FLOAT16(3.0f), FLOAT16(-3.0f), FLOAT16(3.0f), FLOAT16(5.0f), FLOAT16(1.0f), - FLOAT16(1.0f), FLOAT16(1.0f), FLOAT16(1.0f), FLOAT16(-1.0f), FLOAT16(1.0f) }); - - topology topology( - input_layout("input", input.get_layout()), - activation("hsigmoid", "input", activation_func::hsigmoid)); - network network(engine, topology); - network.set_input_data("input", input); - auto outputs = network.execute(); - EXPECT_EQ(outputs.size(), size_t(1)); - EXPECT_EQ(outputs.begin()->first, "hsigmoid"); - - auto output_memory = outputs.at("hsigmoid").get_memory(); - auto output_layout = output_memory.get_layout(); - auto output_ptr = output_memory.pointer(); - auto input_ptr = input.pointer(); - - int y_size = output_layout.size.spatial[1]; - int x_size = output_layout.size.spatial[0]; - int f_size = output_layout.size.feature[0]; - int b_size = output_layout.size.batch[0]; - EXPECT_EQ(output_layout.format, format::yxfb); - EXPECT_EQ(y_size, 2); - EXPECT_EQ(x_size, 5); - EXPECT_EQ(f_size, 2); - EXPECT_EQ(b_size, 1); - - for (size_t i = 0; i < output_layout.get_linear_size(); ++i) { - EXPECT_NEAR((FLOAT16)(std::fmin(std::fmax(0.f, (float)input_ptr[i] + 3.f), 6.f) / 6.f), - output_ptr[i], 1e-3f); - } -} - TEST(activation_f32_fw_gpu, basic_yxfb_all_functions) { // Input: @@ -962,6 +806,94 @@ TEST(activation_f32_fw_gpu, basic_yxfb_all_functions) } } +TEST(activation_f16_fw_gpu, basic_bfyx_all_functions) +{ + const auto& engine = get_test_engine(); + + auto input = memory::allocate(engine, { data_types::f16, format::bfyx, { 1, 1, 2, 4 } }); + auto input_params = memory::allocate(engine, { data_types::f16, format::bfyx, { 1, 2, 1, 1 } }); + + set_values(input, { FLOAT16(-4.5f), FLOAT16(-2.5f), FLOAT16(-1.5f), FLOAT16(0.5f), + FLOAT16(0.9f), FLOAT16(1.5f), FLOAT16(2.0f), FLOAT16(2.5f) }); + + std::vector funcs = { + activation_func::linear, + activation_func::mish, + activation_func::hswish, + activation_func::hsigmoid, + activation_func::round_half_to_even, + activation_func::round_half_away_from_zero + }; + + activation_additional_params params = { 3.f, 2.f }; + set_values(input_params, { FLOAT16(params.a), FLOAT16(params.b) }); + + for (uint8_t i = 0 ; i < 2 ; i++) { + for (auto func : funcs) { + topology topology(input_layout("input", input.get_layout())); + + if (i == 0) { + topology.add(activation("activation", "input", func, params)); + } else { + topology.add(data("input_params", input_params)); + topology.add(activation("activation", "input", "input_params", func)); + } + + network network(engine, topology); + network.set_input_data("input", input); + auto outputs = network.execute(); + EXPECT_EQ(outputs.size(), size_t(1)); + EXPECT_EQ(outputs.begin()->first, "activation"); + + auto output_memory = outputs.at("activation").get_memory(); + auto output_layout = output_memory.get_layout(); + auto output_ptr = output_memory.pointer(); + auto input_ptr = input.pointer(); + + int y_size = output_layout.size.spatial[1]; + int x_size = output_layout.size.spatial[0]; + int f_size = output_layout.size.feature[0]; + int b_size = output_layout.size.batch[0]; + EXPECT_EQ(output_layout.format, format::bfyx); + EXPECT_EQ(y_size, 4); + EXPECT_EQ(x_size, 2); + EXPECT_EQ(f_size, 1); + EXPECT_EQ(b_size, 1); + + for (size_t i = 0; i < output_layout.get_linear_size(); ++i) { + switch (func) { + case activation_func::linear: { + VF output_vec = {FLOAT16(-11.5f), FLOAT16(-5.5f), FLOAT16(-2.5f), FLOAT16(3.5f), + FLOAT16(4.7f), FLOAT16(6.5f), FLOAT16(8.0f), FLOAT16(9.5f)}; + EXPECT_FLOAT_EQ(output_vec[i], output_ptr[i]); + break; + } + case activation_func::mish: + EXPECT_NEAR((FLOAT16)((float)input_ptr[i] * std::tanh(std::log(1.f + std::exp((float)input_ptr[i])))), + output_ptr[i], 1e-2f); + break; + case activation_func::hswish: + EXPECT_NEAR((FLOAT16)((float)input_ptr[i] * std::fmin(std::fmax(0.f, (float)input_ptr[i] + 3.f), 6.f) / 6.f), + output_ptr[i], 1e-3f); + break; + case activation_func::hard_sigmoid: + EXPECT_NEAR((FLOAT16)(std::fmin(std::fmax(0.f, (float)input_ptr[i] + 3.f), 6.f) / 6.f), + output_ptr[i], 1e-3f); + break; + case activation_func::round_half_to_even: + EXPECT_FLOAT_EQ((FLOAT16)std::rint((float)input_ptr[i]), output_ptr[i]); + break; + case activation_func::round_half_away_from_zero: + EXPECT_FLOAT_EQ((FLOAT16)std::round((float)input_ptr[i]), output_ptr[i]); + break; + default: + break; + } + } + } + } +} + TEST(activation_f32_fw_gpu, basic_yxfb_asin_acos_log_atan) { const auto& engine = get_test_engine();