diff --git a/ngraph/frontend/onnx/frontend/src/op/com.microsoft/skip_layer_normalization.cpp b/ngraph/frontend/onnx/frontend/src/op/com.microsoft/skip_layer_normalization.cpp new file mode 100644 index 00000000000..2b7641d242a --- /dev/null +++ b/ngraph/frontend/onnx/frontend/src/op/com.microsoft/skip_layer_normalization.cpp @@ -0,0 +1,45 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "op/com.microsoft/skip_layer_normalization.hpp" + +#include "default_opset.hpp" + +namespace ngraph { +namespace onnx_import { +namespace op { +namespace set_1 { +OutputVector skip_layer_normalization(const Node& node) { + auto nodes = node.get_ng_inputs(); + auto num_nodes = nodes.size(); + NGRAPH_CHECK(num_nodes >= 3 && num_nodes <= 5, + "SkipLayerNormalization takes 3, 4 or 5 inputs. Provided " + std::to_string(num_nodes)); + + // input + skip + std::shared_ptr input = std::make_shared(nodes[0], nodes[1]); + // add bias if available + if (num_nodes == 5) { + input = std::make_shared(input, nodes[4]); + } + float eps = node.get_attribute_value("epsilon"); + // reduce over hidden_size + int hidden_size_dim = 2; + const auto reduction_axes = default_opset::Constant::create(element::i32, Shape{1}, {hidden_size_dim}); + std::shared_ptr result = + std::make_shared(input, reduction_axes, true, eps, ngraph::op::MVNEpsMode::INSIDE_SQRT); + // multiply by gamma + result = std::make_shared(result, nodes[2]); + // add beta if available + if (num_nodes > 3) { + result = std::make_shared(result, nodes[3]); + } + // spec mentions three outputs (output, mean, inv_std_var) while we support only first one, but: + // - onnxruntime also doesn't support the last two + // - we'd have to unroll MVN to have them + return result->outputs(); +} +} // namespace set_1 +} // namespace op +} // namespace onnx_import +} // namespace ngraph diff --git a/ngraph/frontend/onnx/frontend/src/op/com.microsoft/skip_layer_normalization.hpp b/ngraph/frontend/onnx/frontend/src/op/com.microsoft/skip_layer_normalization.hpp new file mode 100644 index 00000000000..6c134ad7684 --- /dev/null +++ b/ngraph/frontend/onnx/frontend/src/op/com.microsoft/skip_layer_normalization.hpp @@ -0,0 +1,17 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "onnx_import/core/node.hpp" + +namespace ngraph { +namespace onnx_import { +namespace op { +namespace set_1 { +OutputVector skip_layer_normalization(const Node& node); +} // namespace set_1 +} // namespace op +} // namespace onnx_import +} // namespace ngraph diff --git a/ngraph/frontend/onnx/frontend/src/ops_bridge.cpp b/ngraph/frontend/onnx/frontend/src/ops_bridge.cpp index 75a0c78e03d..20484c78953 100644 --- a/ngraph/frontend/onnx/frontend/src/ops_bridge.cpp +++ b/ngraph/frontend/onnx/frontend/src/ops_bridge.cpp @@ -30,6 +30,7 @@ #include "op/ceil.hpp" #include "op/clip.hpp" #include "op/com.microsoft/bias_gelu.hpp" +#include "op/com.microsoft/skip_layer_normalization.hpp" #include "op/compress.hpp" #include "op/concat.hpp" #include "op/constant.hpp" @@ -479,6 +480,7 @@ OperatorsBridge::OperatorsBridge() { REGISTER_OPERATOR_WITH_DOMAIN(OPENVINO_ONNX_DOMAIN, "Swish", 1, swish); REGISTER_OPERATOR_WITH_DOMAIN(MICROSOFT_DOMAIN, "BiasGelu", 1, bias_gelu); + REGISTER_OPERATOR_WITH_DOMAIN(MICROSOFT_DOMAIN, "SkipLayerNormalization", 1, skip_layer_normalization); } #undef REGISTER_OPERATOR diff --git a/ngraph/test/CMakeLists.txt b/ngraph/test/CMakeLists.txt index 4b0a08076e3..a50ee6c8571 100644 --- a/ngraph/test/CMakeLists.txt +++ b/ngraph/test/CMakeLists.txt @@ -546,6 +546,7 @@ set(MULTI_TEST_SRC if (NGRAPH_ONNX_FRONTEND_ENABLE) list(APPEND MULTI_TEST_SRC onnx/onnx_import.in.cpp + onnx/onnx_import_com_microsoft.in.cpp onnx/onnx_import_controlflow.in.cpp onnx/onnx_import_const_folding.in.cpp onnx/onnx_import_convpool.in.cpp diff --git a/ngraph/test/models/onnx/dynamic_shapes/skip_layer_normalization.prototxt b/ngraph/test/models/onnx/dynamic_shapes/skip_layer_normalization.prototxt new file mode 100644 index 00000000000..dea7e08d9a6 --- /dev/null +++ b/ngraph/test/models/onnx/dynamic_shapes/skip_layer_normalization.prototxt @@ -0,0 +1,76 @@ +ir_version: 6 +producer_name: "nGraph" +graph { + node { + input: "input" + input: "skip" + input: "gamma" + input: "beta" + input: "bias" + output: "out" + name: "SkipLayerNorm" + op_type: "SkipLayerNormalization" + attribute { + name: "epsilon" + f: 9.999999960041972e-13 + type: FLOAT + } + domain: "com.microsoft" + } + name: "torch-jit-export" + input { + name: "input" + type { + tensor_type { + elem_type: 1 + } + } + } + input { + name: "skip" + type { + tensor_type { + elem_type: 1 + } + } + } + input { + name: "gamma" + type { + tensor_type { + elem_type: 1 + } + } + } + input { + name: "beta" + type { + tensor_type { + elem_type: 1 + } + } + } + input { + name: "bias" + type { + tensor_type { + elem_type: 1 + } + } + } + output { + name: "out" + type { + tensor_type { + elem_type: 1 + } + } + } +} +opset_import { + version: 11 +} +opset_import { + domain: "com.microsoft" + version: 1 +} diff --git a/ngraph/test/models/onnx/skip_layer_normalization_with_gamma.prototxt b/ngraph/test/models/onnx/skip_layer_normalization_with_gamma.prototxt new file mode 100644 index 00000000000..6d7dab3c676 --- /dev/null +++ b/ngraph/test/models/onnx/skip_layer_normalization_with_gamma.prototxt @@ -0,0 +1,92 @@ +ir_version: 6 +producer_name: "nGraph" +graph { + node { + input: "input" + input: "skip" + input: "gamma" + output: "out" + name: "SkipLayerNorm" + op_type: "SkipLayerNormalization" + attribute { + name: "epsilon" + f: 9.999999960041972e-13 + type: FLOAT + } + domain: "com.microsoft" + } + name: "torch-jit-export" + initializer { + dims: 4 + data_type: 1 + name: "gamma" + float_data: 0.1 + float_data: 0.2 + float_data: 0.3 + float_data: 0.4 + } + input { + name: "input" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 3 + } + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + } + } + } + } + input { + name: "skip" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 3 + } + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + } + } + } + } + output { + name: "out" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 3 + } + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + } + } + } + } +} +opset_import { + version: 11 +} +opset_import { + domain: "com.microsoft" + version: 1 +} diff --git a/ngraph/test/models/onnx/skip_layer_normalization_with_gamma_beta.prototxt b/ngraph/test/models/onnx/skip_layer_normalization_with_gamma_beta.prototxt new file mode 100644 index 00000000000..90e92553245 --- /dev/null +++ b/ngraph/test/models/onnx/skip_layer_normalization_with_gamma_beta.prototxt @@ -0,0 +1,102 @@ +ir_version: 6 +producer_name: "nGraph" +graph { + node { + input: "input" + input: "skip" + input: "gamma" + input: "beta" + output: "out" + name: "SkipLayerNorm" + op_type: "SkipLayerNormalization" + attribute { + name: "epsilon" + f: 9.999999960041972e-13 + type: FLOAT + } + domain: "com.microsoft" + } + name: "torch-jit-export" + initializer { + dims: 4 + data_type: 1 + name: "gamma" + float_data: 0.1 + float_data: 0.2 + float_data: 0.3 + float_data: 0.4 + } + initializer { + dims: 4 + data_type: 1 + name: "beta" + float_data: -0.07 + float_data: -0.4 + float_data: 0.22 + float_data: 0 + } + input { + name: "input" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 3 + } + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + } + } + } + } + input { + name: "skip" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 3 + } + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + } + } + } + } + output { + name: "out" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 3 + } + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + } + } + } + } +} +opset_import { + version: 11 +} +opset_import { + domain: "com.microsoft" + version: 1 +} diff --git a/ngraph/test/models/onnx/skip_layer_normalization_with_gamma_beta_bias.prototxt b/ngraph/test/models/onnx/skip_layer_normalization_with_gamma_beta_bias.prototxt new file mode 100644 index 00000000000..8d61aca7a57 --- /dev/null +++ b/ngraph/test/models/onnx/skip_layer_normalization_with_gamma_beta_bias.prototxt @@ -0,0 +1,112 @@ +ir_version: 6 +producer_name: "nGraph" +graph { + node { + input: "input" + input: "skip" + input: "gamma" + input: "beta" + input: "bias" + output: "out" + name: "SkipLayerNorm" + op_type: "SkipLayerNormalization" + attribute { + name: "epsilon" + f: 9.999999960041972e-13 + type: FLOAT + } + domain: "com.microsoft" + } + name: "torch-jit-export" + initializer { + dims: 4 + data_type: 1 + name: "gamma" + float_data: 0.1 + float_data: 0.2 + float_data: 0.3 + float_data: 0.4 + } + initializer { + dims: 4 + data_type: 1 + name: "beta" + float_data: -0.07 + float_data: -0.4 + float_data: 0.22 + float_data: 0 + } + initializer { + dims: 4 + data_type: 1 + name: "bias" + float_data: -0.07 + float_data: -0.4 + float_data: 0.22 + float_data: 0 + } + input { + name: "input" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 3 + } + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + } + } + } + } + input { + name: "skip" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 3 + } + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + } + } + } + } + output { + name: "out" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 3 + } + dim { + dim_value: 2 + } + dim { + dim_value: 4 + } + } + } + } + } +} +opset_import { + version: 11 +} +opset_import { + domain: "com.microsoft" + version: 1 +} diff --git a/ngraph/test/onnx/onnx_import.in.cpp b/ngraph/test/onnx/onnx_import.in.cpp index e3be4a31d70..d31f4a3b0a8 100644 --- a/ngraph/test/onnx/onnx_import.in.cpp +++ b/ngraph/test/onnx/onnx_import.in.cpp @@ -4150,23 +4150,3 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_random_uniform_like) { test_case.add_input(Shape{2, 2}, {41, 42, 43, 44}); test_case.run(); } - -NGRAPH_TEST(${BACKEND_NAME}, onnx_model_bias_gelu) { - const auto function = onnx_import::import_onnx_model(file_util::path_join(SERIALIZED_ZOO, "onnx/bias_gelu.onnx")); - - auto test_case = test::TestCase(function); - test_case.add_input({0.5488135, - 0.71518934, - 0.60276335, - 0.5448832, - 0.4236548, - 0.6458941, - 0.4375872, - 0.891773, - 0.96366274, - 0.3834415}); - test_case.add_input({0.79172504, 0.5288949, 0.56804454, 0.92559665, 0.07103606}); - test_case.add_expected_output( - {1.2198428, 1.1112978, 1.0293297, 1.366493, 0.3411342, 1.329408, 0.8051748, 1.354462, 1.8336612, 0.3068893}); - test_case.run(); -} diff --git a/ngraph/test/onnx/onnx_import_com_microsoft.in.cpp b/ngraph/test/onnx/onnx_import_com_microsoft.in.cpp new file mode 100644 index 00000000000..43f74e64b06 --- /dev/null +++ b/ngraph/test/onnx/onnx_import_com_microsoft.in.cpp @@ -0,0 +1,177 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +// clang-format off +#ifdef ${BACKEND_NAME}_FLOAT_TOLERANCE_BITS +#define DEFAULT_FLOAT_TOLERANCE_BITS ${BACKEND_NAME}_FLOAT_TOLERANCE_BITS +#endif +#ifdef ${BACKEND_NAME}_DOUBLE_TOLERANCE_BITS +#define DEFAULT_DOUBLE_TOLERANCE_BITS ${BACKEND_NAME}_DOUBLE_TOLERANCE_BITS +#endif +// clang-format on + +#include "onnx_import/onnx.hpp" +#include "default_opset.hpp" +#include "engines_util/test_case.hpp" +#include "engines_util/test_engines.hpp" +#include "util/test_control.hpp" + +NGRAPH_SUPPRESS_DEPRECATED_START + +using namespace ngraph; + +static std::string s_manifest = "${MANIFEST}"; + +using Inputs = std::vector>; +using Outputs = std::vector>; + +using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME}); + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_bias_gelu) { + const auto function = onnx_import::import_onnx_model(file_util::path_join(SERIALIZED_ZOO, "onnx/bias_gelu.onnx")); + + auto test_case = test::TestCase(function); + test_case.add_input({0.5488135, + 0.71518934, + 0.60276335, + 0.5448832, + 0.4236548, + 0.6458941, + 0.4375872, + 0.891773, + 0.96366274, + 0.3834415}); + test_case.add_input({0.79172504, 0.5288949, 0.56804454, 0.92559665, 0.07103606}); + test_case.add_expected_output( + {1.2198428, 1.1112978, 1.0293297, 1.366493, 0.3411342, 1.329408, 0.8051748, 1.354462, 1.8336612, 0.3068893}); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_skip_layer_normalization_with_gamma_beta_bias) { + const auto function = onnx_import::import_onnx_model( + file_util::path_join(SERIALIZED_ZOO, "onnx/skip_layer_normalization_with_gamma_beta_bias.onnx")); + + std::vector input = { + 0.54881352, 0.71518934, 0.60276335, 0.54488319, 0.42365479, 0.64589411, 0.43758720, 0.89177299, + 0.96366274, 0.38344151, 0.79172504, 0.52889490, 0.56804454, 0.92559665, 0.07103606, 0.08712930, + 0.02021840, 0.83261985, 0.77815676, 0.87001216, 0.97861832, 0.79915857, 0.46147937, 0.78052920, + }; + std::vector skip = { + 0.11827443, 0.63992101, 0.14335328, 0.94466889, 0.52184832, 0.41466194, 0.26455560, 0.77423370, + 0.45615032, 0.56843394, 0.01878980, 0.61763549, 0.61209571, 0.61693400, 0.94374806, 0.68182027, + 0.35950789, 0.43703195, 0.69763118, 0.06022547, 0.66676670, 0.67063785, 0.21038257, 0.12892629, + }; + std::vector expected = { + -0.19721794, -0.42944565, 0.18620640, 0.61282152, -0.11097327, -0.59518522, 0.13393641, 0.66901535, + 0.04256713, -0.71902490, 0.23107991, 0.17300847, -0.04390603, -0.31109563, 0.51021838, -0.66914201, + -0.20009395, -0.43313017, 0.67281967, -0.01712347, 0.09767530, -0.43024653, -0.01836969, -0.29238200, + }; + auto test_case = test::TestCase(function); + test_case.add_input(input); + test_case.add_input(skip); + test_case.add_expected_output(expected); + test_case.run(5); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_skip_layer_normalization_with_gamma_beta) { + const auto function = onnx_import::import_onnx_model( + file_util::path_join(SERIALIZED_ZOO, "onnx/skip_layer_normalization_with_gamma_beta.onnx")); + + std::vector input = { + 0.54881352, 0.71518934, 0.60276335, 0.54488319, 0.42365479, 0.64589411, 0.43758720, 0.89177299, + 0.96366274, 0.38344151, 0.79172504, 0.52889490, 0.56804454, 0.92559665, 0.07103606, 0.08712930, + 0.02021840, 0.83261985, 0.77815676, 0.87001216, 0.97861832, 0.79915857, 0.46147937, 0.78052920, + }; + std::vector skip = { + 0.11827443, 0.63992101, 0.14335328, 0.94466889, 0.52184832, 0.41466194, 0.26455560, 0.77423370, + 0.45615032, 0.56843394, 0.01878980, 0.61763549, 0.61209571, 0.61693400, 0.94374806, 0.68182027, + 0.35950789, 0.43703195, 0.69763118, 0.06022547, 0.66676670, 0.67063785, 0.21038257, 0.12892629, + }; + std::vector expected = { + -0.17974678, -0.23946194, -0.04376268, 0.46959469, -0.11171167, -0.41859278, -0.11082965, 0.64513868, + 0.07773457, -0.51403606, -0.13661698, 0.11262375, -0.05096011, -0.10416907, 0.10070466, -0.50876135, + -0.22290939, -0.27663514, 0.55416691, -0.08064821, 0.04857478, -0.25121087, -0.15912610, -0.26637587, + }; + auto test_case = test::TestCase(function); + test_case.add_input(input); + test_case.add_input(skip); + test_case.add_expected_output(expected); + test_case.run(7); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_skip_layer_normalization_with_gamma) { + const auto function = onnx_import::import_onnx_model( + file_util::path_join(SERIALIZED_ZOO, "onnx/skip_layer_normalization_with_gamma.onnx")); + + std::vector input = { + 0.54881352, 0.71518934, 0.60276335, 0.54488319, 0.42365479, 0.64589411, 0.43758720, 0.89177299, + 0.96366274, 0.38344151, 0.79172504, 0.52889490, 0.56804454, 0.92559665, 0.07103606, 0.08712930, + 0.02021840, 0.83261985, 0.77815676, 0.87001216, 0.97861832, 0.79915857, 0.46147937, 0.78052920, + }; + std::vector skip = { + 0.11827443, 0.63992101, 0.14335328, 0.94466889, 0.52184832, 0.41466194, 0.26455560, 0.77423370, + 0.45615032, 0.56843394, 0.01878980, 0.61763549, 0.61209571, 0.61693400, 0.94374806, 0.68182027, + 0.35950789, 0.43703195, 0.69763118, 0.06022547, 0.66676670, 0.67063785, 0.21038257, 0.12892629, + }; + std::vector expected = { + -0.10974677, 0.16053806, -0.26376268, 0.46959469, -0.04171166, -0.01859277, -0.33082965, 0.64513868, + 0.14773457, -0.11403608, -0.35661697, 0.11262375, 0.01903989, 0.29583094, -0.11929534, -0.50876135, + -0.15290938, 0.12336487, 0.33416691, -0.08064821, 0.11857478, 0.14878914, -0.37912610, -0.26637587, + }; + auto test_case = test::TestCase(function); + test_case.add_input(input); + test_case.add_input(skip); + test_case.add_expected_output(expected); + test_case.run(6); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_skip_layer_normalization_dynamic_shapes) { + const auto function = onnx_import::import_onnx_model( + file_util::path_join(SERIALIZED_ZOO, "onnx/dynamic_shapes/skip_layer_normalization.onnx")); + + std::vector input = { + 0.54881352, 0.71518934, 0.60276335, 0.54488319, 0.42365479, 0.64589411, 0.43758720, 0.89177299, + 0.96366274, 0.38344151, 0.79172504, 0.52889490, 0.56804454, 0.92559665, 0.07103606, 0.08712930, + 0.02021840, 0.83261985, 0.77815676, 0.87001216, 0.97861832, 0.79915857, 0.46147937, 0.78052920, + }; + std::vector skip = { + 0.11827443, 0.63992101, 0.14335328, 0.94466889, 0.52184832, 0.41466194, 0.26455560, 0.77423370, + 0.45615032, 0.56843394, 0.01878980, 0.61763549, 0.61209571, 0.61693400, 0.94374806, 0.68182027, + 0.35950789, 0.43703195, 0.69763118, 0.06022547, 0.66676670, 0.67063785, 0.21038257, 0.12892629, + }; + std::vector gamma = { + 0.31542835, + 0.36371076, + 0.57019675, + 0.43860152, + }; + std::vector beta = { + 0.98837382, + 0.10204481, + 0.20887676, + 0.16130951, + }; + std::vector bias = { + 0.65310830, + 0.25329161, + 0.46631077, + 0.24442559, + }; + std::vector expected = { + 0.76600611, 0.34308332, -0.48470584, 0.71335256, 1.10028172, -0.13354334, -0.45232186, 0.79840088, + 1.52454257, -0.19450217, -0.13759643, 0.03988872, 1.27861762, 0.39529073, 0.12247884, -0.52944231, + 0.64228040, 0.21059875, 1.05966032, -0.14278713, 1.46366918, 0.21215858, -0.31640187, -0.22832340, + }; + + auto test_case = test::TestCase(function); + test_case.add_input(Shape{3, 2, 4}, input); + test_case.add_input(Shape{3, 2, 4}, skip); + test_case.add_input(Shape{4}, gamma); + test_case.add_input(Shape{4}, beta); + test_case.add_input(Shape{4}, bias); + test_case.add_expected_output(Shape{3, 2, 4}, expected); + test_case.run(7); +} diff --git a/ngraph/test/runtime/ie/unit_test.manifest b/ngraph/test/runtime/ie/unit_test.manifest index 59ddde899f6..fb73dc50382 100644 --- a/ngraph/test/runtime/ie/unit_test.manifest +++ b/ngraph/test/runtime/ie/unit_test.manifest @@ -1572,3 +1572,6 @@ IE_GPU.deformable_convolution_opset8_2D_integral_offsets_groups_and_deforgroups # Not yet implemented - refer PR#6601 IE_CPU.onnx_model_gather_float_2D_neg_indices + +# CPU plug-in doesn't support operation with dynamic rank +onnx_model_skip_layer_normalization_dynamic_shapes