IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CPU] Fixed SoftPlus for large positive values (#4932)

Browse Source
This commit is contained in:
Alexandra Sidorova 2021-04-30 13:34:33 +03:00 committed by GitHub
parent 8b1b900591
commit 03ca3d1ef7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 50 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
docs/ops/activation/SoftPlus_4.md
View File

@ -13,9 +13,23 @@
*SoftPlus* performs element-wise activation function on a given input tensor, based on the following mathematical formula:
\f[
SoftPlus(x) = \ln(1+e^{x})
SoftPlus(x) = \left\{\begin{array}{r}
x \qquad \mbox{if } x \geq threshold \\
log(e^{x} + 1.0) \qquad \mbox{if } x < threshold
\end{array}\right.
\f]
**Note**: For numerical stability the operation reverts to the linear function when `x > threshold` where `threshold` depends on *T* and
is chosen in such a way that the difference between the linear function and exact calculation is no more than `1e-6`.
The `threshold` can be calculated with the following formula where `alpha` is the number of digits after the decimal point,
`beta` is maximum value of *T* data type:
\f[
-log(e^{10^{-\alpha}} - 1.0) < threshold < log(\beta)
\f]
For example, if *T* is `fp32`, `threshold` should be `20` or if *T* is `fp16`, `threshold` should be `12`.
**Attributes**: *SoftPlus* operation has no attributes.

10
inference-engine/src/mkldnn_plugin/mkldnn_graph_optimizer.cpp
View File

@ -615,7 +615,7 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndActivation(MKLDNNGraph &graph) {
(eltwiseNode->getOpType() == Relu ||
(conv->getCnnLayer()->precision == Precision::FP32 &&
IsOneOf(eltwiseNode->getOpType(), {Elu, Logistic, BoundedRelu, Clamp, Swish, Hswish, Mish, Hsigmoid,
Round})));
Round, SoftRelu})));
};
for (int i = 0; i < graphNodes.size(); i++) {
@ -694,7 +694,7 @@ void MKLDNNGraphOptimizer::FuseFullyConnectedAndSimpleOperation(MKLDNNGraph &gra
IE_THROW() << "Cannot get Eltwise node " << childNode->getName();
if (IsOneOf(eltwiseNode->getOpType(), {Relu, Gelu, Elu, Logistic, BoundedRelu, Clamp, Swish, Hswish, Mish,
Hsigmoid, Round})) {
Hsigmoid, Round, SoftRelu})) {
return true;
} else if (IsOneOf(eltwiseNode->getOpType(), {MulAdd, Prelu})) {
if (eltwiseNode->getOpType() == MulAdd && eltwiseNode->getCnnLayer()->blobs.size() != 2)
@ -1053,7 +1053,7 @@ void MKLDNNGraphOptimizer::FuseConvolutionAndSimpleOperation(MKLDNNGraph &graph)
return ((eltwiseNode->getOpType() == MulAdd && node->getCnnLayer()->blobs.size() == 2) ||
(eltwiseNode->getOpType() == Prelu) ||
IsOneOf(eltwiseNode->getOpType(), {Relu, Elu, Logistic, BoundedRelu, Clamp, Swish, Hswish, Mish,
Hsigmoid, Round}));
Hsigmoid, Round, SoftRelu}));
}
return false;
@ -1269,7 +1269,7 @@ void MKLDNNGraphOptimizer::FuseConvolutionSumAndConvolutionSumActivation(MKLDNNG
(eltwiseNode->getOpType() == Relu ||
(conv->getCnnLayer()->precision == Precision::FP32 &&
IsOneOf(eltwiseNode->getOpType(), {Elu, Logistic, BoundedRelu, Clamp, Swish, Hswish, Mish, Hsigmoid,
Round})));
Round, SoftRelu})));
};
for (auto &graphNode : graphNodes) {
@ -1568,7 +1568,7 @@ void MKLDNNGraphOptimizer::FuseNormalizeAndSimpleOperation(MKLDNNGraph &graph) {
if (eltwiseNode == nullptr)
IE_THROW() << "Cannot get Eltwise node " << node->getName();
return IsOneOf(eltwiseNode->getOpType(), {Relu, Gelu, Elu, Logistic, BoundedRelu, Clamp, Tanh, Swish,
Hswish, Mish, Hsigmoid, Round, Linear, Abs, Square, Sqrt}) ||
Hswish, Mish, Hsigmoid, Round, Linear, Abs, Square, Sqrt, SoftRelu}) ||
((eltwiseNode->getOpType() == MulAdd && eltwiseNode->getCnnLayer()->blobs.size() == 2) ||
(eltwiseNode->getOpType() == Prelu));
}

1
inference-engine/src/mkldnn_plugin/mkldnn_node.cpp
View File

@ -80,6 +80,7 @@ static const InferenceEngine::details::caseless_unordered_map<std::string, Type>
{ "Round", Eltwise },
{ "ScaleShift", Eltwise },
{ "PReLU", Eltwise },
{ "SoftPlus", Eltwise },
{ "Norm", Lrn },
{ "LRN", Lrn },
{ "Pooling", Pooling },

1
inference-engine/src/mkldnn_plugin/nodes/list_tbl.hpp
View File

@ -32,7 +32,6 @@ MKLDNN_EXTENSION_NODE(MathImpl, Selu);
MKLDNN_EXTENSION_NODE(MathImpl, Sign);
MKLDNN_EXTENSION_NODE(MathImpl, Sin);
MKLDNN_EXTENSION_NODE(MathImpl, Sinh);
MKLDNN_EXTENSION_NODE(MathImpl, SoftPlus);
MKLDNN_EXTENSION_NODE(MathImpl, Softsign);
MKLDNN_EXTENSION_NODE(MathImpl, Tan);
MKLDNN_EXTENSION_NODE(ExperimentalDetectronTopKROIsImpl, ExperimentalDetectronTopKROIs);

2
inference-engine/src/mkldnn_plugin/nodes/mkldnn_bin_conv_node.cpp
View File

@ -1114,7 +1114,7 @@ bool MKLDNNBinaryConvolutionNode::canFuse(const MKLDNNNodePtr& node) const {
}
return eltwiseNode->isSum() ||
isOneOf(eltwiseNode->getOpType(), {MulAdd, Prelu, Relu, Gelu, Elu, Logistic, BoundedRelu, Clamp,
isOneOf(eltwiseNode->getOpType(), {MulAdd, Prelu, Relu, Gelu, Elu, Logistic, BoundedRelu, Clamp, SoftRelu,
Tanh, Swish, Hswish, Mish, Hsigmoid, Round, Linear, Abs, Square, Sqrt});
}

5
inference-engine/src/mkldnn_plugin/nodes/mkldnn_eltwise_node.cpp
View File

@ -843,7 +843,7 @@ MKLDNNEltwiseNode::initializers = {
opType = BoundedRelu;
algorithm = mkldnn::algorithm::eltwise_bounded_relu;
}},
{"soft_relu", [](GenericLayer* activationLayer, EltwiseOpType& opType, mkldnn::algorithm& algorithm, float& alpha, float& beta) {
{"softplus", [](GenericLayer* activationLayer, EltwiseOpType& opType, mkldnn::algorithm& algorithm, float& alpha, float& beta) {
alpha = 0.0f;
beta = 0.0f;
opType = SoftRelu;
@ -983,7 +983,8 @@ void MKLDNNEltwiseNode::init() {
comparator(layerType, "hswish") ||
comparator(layerType, "mish") ||
comparator(layerType, "hsigmoid") ||
comparator(layerType, "round")) {
comparator(layerType, "round") ||
comparator(layerType, "softplus")) {
initializers[layerType](getCnnLayer().get(), eltwiseOp, eltwiseAlgorithm, alpha, beta);
} else if (comparator(layerType, "erf")) {
eltwiseOp = Erf;

2
inference-engine/src/mkldnn_plugin/nodes/mkldnn_interpolate_node.cpp
View File

@ -3197,7 +3197,7 @@ bool MKLDNNInterpolateNode::canFuse(const MKLDNNNodePtr& node) const {
auto* eltwiseNode = dynamic_cast<MKLDNNEltwiseNode*>(node.get());
if (eltwiseNode == nullptr)
IE_THROW() << "Cannot get eltwise node " << node->getName();
return isOneOf(eltwiseNode->getOpType(), {Prelu, Relu, Gelu, Elu, Logistic, BoundedRelu, Clamp,
return isOneOf(eltwiseNode->getOpType(), {Prelu, Relu, Gelu, Elu, Logistic, BoundedRelu, Clamp, SoftRelu,
Tanh, Swish, Hswish, Mish, Hsigmoid, Round, Linear, Abs, Square, Sqrt}) ||
(eltwiseNode->getOpType() == MulAdd && eltwiseNode->getCnnLayer()->blobs.size() == 2);
}

1
inference-engine/tests/functional/plugin/cpu/single_layer_tests/convolution.cpp
View File

@ -112,6 +112,7 @@ const std::vector<fusingSpecificParams> fusingParamsSet{
fusingSwish,
fusingHSwish,
fusingMish,
fusingSoftPlus,
// other patterns
fusingReluScaleShift,
fusingFakeQuantizePerTensorRelu,

1
inference-engine/tests/functional/plugin/cpu/single_layer_tests/group_convolution.cpp
View File

@ -123,6 +123,7 @@ std::vector<fusingSpecificParams> fusingParamsSet {
fusingSwish,
fusingHSwish,
fusingMish,
fusingSoftPlus,
// other patterns
fusingReluScaleShift,
fusingFakeQuantizePerTensorRelu,

4
inference-engine/tests/functional/plugin/cpu/test_utils/fusing_test_utils.hpp
View File

@ -112,6 +112,10 @@ const auto fusingMish = fusingSpecificParams{std::make_shared<postNodesMgr>(std:
{[](std::shared_ptr<ngraph::Node> inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){
return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Mish, {}, {});
}, "Mish"}}), {"Mish"}};
const auto fusingSoftPlus = fusingSpecificParams{std::make_shared<postNodesMgr>(std::vector<postNodeBuilder>{
{[](std::shared_ptr<ngraph::Node> inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){
return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::SoftPlus, {}, {});
}, "SoftPlus"}}), {"SoftPlus"}};
const auto fusingTanh = fusingSpecificParams{std::make_shared<postNodesMgr>(std::vector<postNodeBuilder>{
{[](std::shared_ptr<ngraph::Node> inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){
return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Tanh, {}, {});

2
inference-engine/tests/functional/plugin/gpu/shared_tests_instances/skip_tests_config.cpp
View File

@ -59,5 +59,7 @@ std::vector<std::string> disabledTestPatterns() {
R"(.*ConstantResultSubgraphTest.*inPrc=I16.*)",
// TODO: Issue: 54436
R"(.*LSTMSequence.*CompareWithRefs.*mode=PURE_SEQ_RAND_SEQ_LEN_PARAM.*direction=bidirectional_clip=0.7_netPRC=FP32.*)",
// TODO: Issue: 54194
R"(.*ActivationLayerTest.*SoftPlus.*)",
};
}

2
inference-engine/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp
View File

@ -37,5 +37,7 @@ std::vector<std::string> disabledTestPatterns() {
R"(.*CTCGreedyDecoderSeqLen.*?\(1.1.1\).*)",
// TODO: Issue 51804
".*PreprocessConversionTest.*oPRC=U8.*",
// TODO: Issue 54163
R"(.*ActivationLayerTest.*SoftPlus.*)",
};
}

6
inference-engine/tests/functional/shared_test_classes/src/single_layer/activation.cpp
View File

@ -101,6 +101,12 @@ InferenceEngine::Blob::Ptr ActivationLayerTest::GenerateInput(const InferenceEng
resolution = 32768;
break;
}
case ngraph::helpers::ActivationTypes::SoftPlus: {
data_start_from = -100;
data_range = 200;
resolution = 32768;
break;
}
default: {
data_start_from = -10;
data_range = 20;

2
inference-engine/thirdparty/mkl-dnn vendored

@ -1 +1 @@
Subproject commit 0292c2a2a2525ff86590de3b499ceb61a5e2355f
Subproject commit 2dd787262134c20f91f222bfa776225d2dddbc9a

5
ngraph/core/reference/include/ngraph/runtime/reference/softplus.hpp
View File

@ -16,9 +16,12 @@ namespace ngraph
template <typename T>
void softplus(const T* arg, T* out, size_t count)
{
const T threshold = static_cast<T>(-std::log(std::exp(std::pow(10, -6)) - 1));
for (size_t i = 0; i < count; i++)
{
out[i] = std::log(std::exp(arg[i]) + 1.0);
out[i] = (arg[i] < threshold) ? static_cast<T>(std::log(std::exp(arg[i]) + 1))
: arg[i];
}
}
} // namespace reference

2
ngraph/python/tests/test_onnx/test_ops_unary.py
View File

@ -266,7 +266,7 @@ def test_logsoftmax():
def test_softplus():
def softplus(x):
return np.log(np.exp(x) + 1)
return np.where(x < 20, np.log(np.exp(x) + 1), x)
np.random.seed(133391)
data = np.random.randn(3, 4, 5).astype(np.float32)

4
ngraph/test/onnx/onnx_import.in.cpp
View File

@ -2402,9 +2402,9 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_softplus)
0.6931471824645996094,
1.313261628150939941,
10.0000457763671875,
inf,
100.0,
0.0,
inf,
1000.0,
0.0,
0.6931471824645996094,
0.6931471824645996094,