diff --git a/src/frontends/onnx/frontend/src/op/com.microsoft/embed_layer_normalization.cpp b/src/frontends/onnx/frontend/src/op/com.microsoft/embed_layer_normalization.cpp index 6313032c716..f98f6ab7c8e 100644 --- a/src/frontends/onnx/frontend/src/op/com.microsoft/embed_layer_normalization.cpp +++ b/src/frontends/onnx/frontend/src/op/com.microsoft/embed_layer_normalization.cpp @@ -15,8 +15,8 @@ OutputVector embed_layer_normalization(const Node& node) { auto nodes = node.get_ng_inputs(); auto num_nodes = nodes.size(); - NGRAPH_CHECK(num_nodes >= 7 && num_nodes <= 8, - "EmbedLayerNormalization takes 7 or 8 inputs. Provided " + std::to_string(num_nodes)); + NGRAPH_CHECK(num_nodes >= 7 && num_nodes <= 9, + "EmbedLayerNormalization takes 7 or 9 inputs. Provided " + std::to_string(num_nodes)); NGRAPH_CHECK(nodes[0].get_element_type() == element::i32, "input_ids must have int32 type"); const auto& input_ids = nodes[0]; @@ -27,10 +27,29 @@ OutputVector embed_layer_normalization(const Node& node) { const auto& gamma = nodes[5]; const auto& beta = nodes[6]; - auto zero = default_opset::Constant::create(element::i32, Shape{1}, {0}); + const auto zero = default_opset::Constant::create(element::i32, Shape{1}, {0}); std::shared_ptr input = std::make_shared(word_embeddings, input_ids, zero, 0); - input = std::make_shared(input, position_embeddings); - + // add position embeddings + if (num_nodes > 8 && !ngraph::op::is_null(nodes[8])) { + // if we have position_ids + const auto& position_ids = nodes[8]; + const auto gathered_position_embeddings = + std::make_shared(position_embeddings, position_ids, zero, 0); + input = std::make_shared(input, gathered_position_embeddings); + } else { + // input_ids' shape is [batchsize, sequence_length] + // input's shape is [batchsize, sequence_length, hidden_size] + // position_embeddings's shape is [max_sequence_length, hidden_size] + // therefore input and position_embeddings cannot be added together + // so we need slice the position_embeddings to [sequence_length, hidden_size] first + // then add it with input. + const auto one = default_opset::Constant::create(element::i32, Shape{1}, {1}); + const auto input_ids_shape = std::make_shared(input_ids, element::i32); + const auto seqlen = std::make_shared(input_ids_shape, one, zero, 0); + const auto gathered_position_embeddings = + std::make_shared(position_embeddings, zero, seqlen, one, zero); + input = std::make_shared(input, gathered_position_embeddings); + } // add segment embeddings if available if (!ngraph::op::is_null(segment_ids)) { NGRAPH_CHECK(!ngraph::op::is_null(segment_embeddings), diff --git a/src/frontends/onnx/tests/models/com.microsoft/embed_layer_normalization_dynamic_shapes_with_position_ids.prototxt b/src/frontends/onnx/tests/models/com.microsoft/embed_layer_normalization_dynamic_shapes_with_position_ids.prototxt new file mode 100644 index 00000000000..0cc8e8a3bd3 --- /dev/null +++ b/src/frontends/onnx/tests/models/com.microsoft/embed_layer_normalization_dynamic_shapes_with_position_ids.prototxt @@ -0,0 +1,203 @@ +ir_version: 6 +producer_name: "nGraph" +graph { + node { + input: "input_ids" + input: "segment_ids" + input: "word_embeddings" + input: "position_embeddings" + input: "segment_embeddings" + input: "gamma" + input: "beta" + input: "mask" + input: "position_ids" + output: "output" + output: "mask_index" + name: "EmbedLayerNormalization_1" + op_type: "EmbedLayerNormalization" + attribute { + name: "epsilon" + f: 9.999999960041972e-13 + type: FLOAT + } + domain: "com.microsoft" + } + name: "graph" + input { + name: "input_ids" + type { + tensor_type { + elem_type: 6 + shape { + dim { + dim_param: "batch_size" + } + dim { + dim_param: "seq_len" + } + } + } + } + } + input { + name: "segment_ids" + type { + tensor_type { + elem_type: 6 + shape { + dim { + dim_param: "batch_size" + } + dim { + dim_param: "seq_len" + } + } + } + } + } + input { + name: "word_embeddings" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_param: "word_embed_len" + } + dim { + dim_value: 5 + } + } + } + } + } + input { + name: "position_embeddings" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_param: "pos_embed_len" + } + dim { + dim_value: 5 + } + } + } + } + } + input { + name: "segment_embeddings" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_param: "segment_embed_len" + } + dim { + dim_value: 5 + } + } + } + } + } + input { + name: "gamma" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 5 + } + } + } + } + } + input { + name: "beta" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_value: 5 + } + } + } + } + } + input { + name: "mask" + type { + tensor_type { + elem_type: 6 + shape { + dim { + dim_param: "batch_size" + } + dim { + dim_param: "seq_len" + } + } + } + } + } + input { + name: "position_ids" + type { + tensor_type { + elem_type: 6 + shape { + dim { + dim_param: "batch_size" + } + dim { + dim_param: "seq_len" + } + } + } + } + } + output { + name: "output" + type { + tensor_type { + elem_type: 1 + shape { + dim { + dim_param: "batch_size" + } + dim { + dim_param: "seq_len" + } + dim { + dim_value: 5 + } + } + } + } + } + output { + name: "mask_index" + type { + tensor_type { + elem_type: 6 + shape { + dim { + dim_param: "batch_size" + } + } + } + } + } +} +opset_import { + version: 11 +} +opset_import { + domain: "com.microsoft" + version: 1 +} diff --git a/src/frontends/onnx/tests/onnx_import_com_microsoft.in.cpp b/src/frontends/onnx/tests/onnx_import_com_microsoft.in.cpp index c08acd45622..895499031e4 100644 --- a/src/frontends/onnx/tests/onnx_import_com_microsoft.in.cpp +++ b/src/frontends/onnx/tests/onnx_import_com_microsoft.in.cpp @@ -405,6 +405,206 @@ NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization_dynamic_shapes test_case.run_with_tolerance_as_fp(1e-6); } +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization_diff_seq_len_pos_embed_len) { + const auto function = onnx_import::import_onnx_model( + file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/com.microsoft/embed_layer_normalization_dynamic_shapes.onnx")); + + std::vector input_ids = { + 8, 1, 5, 9, 8, 9, 4, 3, 0, 3, 5, 0, 2, 3, 8, 1, 3, 3, 3, 7, 0, 1, 9, 9, + }; + std::vector segment_ids = { + 0, 2, 0, 2, 2, 0, 2, 0, 0, 0, 1, 1, 2, 0, 0, 1, 0, 1, 2, 2, 0, 1, 1, 1, + }; + std::vector word_embeddings = { + 0.96980906, 0.65314001, 0.17090958, 0.35815218, 0.75068617, 0.60783064, 0.32504722, 0.03842543, 0.63427407, + 0.95894927, 0.65279031, 0.63505888, 0.99529958, 0.58185035, 0.41436860, 0.47469750, 0.62351012, 0.33800763, + 0.67475230, 0.31720173, 0.77834547, 0.94957107, 0.66252685, 0.01357164, 0.62284607, 0.67365962, 0.97194499, + 0.87819350, 0.50962436, 0.05571469, 0.45115921, 0.01998767, 0.44171092, 0.97958672, 0.35944447, 0.48089352, + 0.68866116, 0.88047588, 0.91823548, 0.21682213, 0.56518888, 0.86510259, 0.50896895, 0.91672295, 0.92115760, + 0.08311249, 0.27771857, 0.00935670, 0.84234208, 0.64717412, + }; + std::vector position_embeddings = { + 0.84138614, 0.26473016, 0.39782074, 0.55282146, 0.16494046, 0.36980811, 0.14644176, 0.56961840, 0.70373726, + 0.28847644, 0.43328807, 0.75610667, 0.39609829, 0.89603841, 0.63892108, 0.89155442, 0.68005556, 0.44919774, + 0.97857094, 0.11620191, 0.76702368, 0.41182014, 0.67543906, 0.24979627, 0.31321833, 0.96541619, 0.58846509, + 0.65966839, 0.53320622, 0.23053302, 0.39486930, 0.61880857, 0.47486752, 0.47013220, 0.71607453, 0.28799102, + 0.38346222, 0.74916983, 0.87845218, 0.10286336, 0.84138614, 0.26473016, 0.39782074, 0.55282146, 0.16494046, + }; + std::vector segment_embeddings = { + 0.09237389, + 0.35404667, + 0.55181628, + 0.03362509, + 0.96896178, + 0.32099724, + 0.22126268, + 0.14126390, + 0.09725992, + 0.98404223, + 0.26034093, + 0.53702253, + 0.44792616, + 0.09956909, + 0.35231167, + }; + std::vector gamma = { + 0.46924916, + 0.84114015, + 0.90464777, + 0.03755938, + 0.50831544, + }; + std::vector beta = { + 0.16684751, + 0.77905101, + 0.86493331, + 0.41139671, + 0.13997258, + }; + std::vector mask = { + 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, + }; + std::vector expected_output = { + -0.04089922, 0.35108989, 0.30442458, 0.39546335, 1.15422225, 0.10419128, -0.19301927, 0.01070970, + 0.43977541, 0.89119899, -0.51436460, 1.99256825, 1.41077507, 0.38642293, 0.17583044, 0.03320138, + 1.16508031, -0.24356931, 0.47440714, -0.17844005, 0.20463173, 1.90038323, 1.14138567, 0.34504607, + 0.16403235, -0.24976699, 0.29362509, 0.34502214, 0.41751838, 1.09390712, 0.12354189, 1.83025289, + 1.05569196, 0.34413773, 0.35469764, -0.69760042, 0.76338542, 1.75443077, 0.44126555, 0.18181801, + 0.73277575, 0.45443264, 0.17068321, 0.36591727, 0.72869974, -0.56090516, 0.14415455, 1.47314119, + 0.42908576, 0.73084539, -0.22373237, 2.26550221, 0.05606699, 0.39417523, 0.35234636, 0.78569502, + 0.77521765, -0.65131050, 0.40168875, 0.45527256, 0.38715565, 0.98521245, 2.21446753, 0.36345237, + -0.33269632, 0.36558092, 1.36846578, 1.37523413, 0.33698002, 0.28889543, -0.40639281, 1.01643157, + 0.59668219, 0.39197800, 1.03101778, 0.02551098, -0.03612846, -0.01371557, 0.43444607, 0.96746695, + 0.60583955, -0.10362893, 0.40574494, 0.38046724, 0.87445319, -0.00880148, -0.15437943, 0.08118075, + 0.44650543, 0.85956848, -0.27865338, 2.10837507, 0.04798460, 0.43948367, -0.10185169, 0.19978794, + 1.32323360, 1.20525467, 0.44288942, -0.84200430, 0.52563053, 0.69949460, 0.73987913, 0.34668452, + 0.74545687, 0.57696682, 0.22452033, -0.27099937, 0.39649010, 0.87083614, -0.18965788, 0.58206403, + -0.08108193, 0.42067638, 1.05117214, -0.34287399, 0.20424896, 0.27994895, 0.46011117, 0.70890665, + }; + std::vector expected_mask_index = { + 6, + 5, + 5, + }; + + auto test_case = test::TestCase(function, s_device); + test_case.add_input(Shape{3, 8}, input_ids); + test_case.add_input(Shape{3, 8}, segment_ids); + test_case.add_input(Shape{10, 5}, word_embeddings); + test_case.add_input(Shape{9, 5}, position_embeddings); + test_case.add_input(Shape{3, 5}, segment_embeddings); + test_case.add_input(Shape{5}, gamma); + test_case.add_input(Shape{5}, beta); + test_case.add_input(Shape{3, 8}, mask); + test_case.add_expected_output(Shape{3, 8, 5}, expected_output); + test_case.add_expected_output(Shape{3}, expected_mask_index); + test_case.run_with_tolerance_as_fp(1e-6); +} + +NGRAPH_TEST(${BACKEND_NAME}, onnx_model_embed_layer_normalization_with_position_ids) { + const auto function = onnx_import::import_onnx_model( + file_util::path_join(CommonTestUtils::getExecutableDirectory(), + SERIALIZED_ZOO, + "onnx/com.microsoft/embed_layer_normalization_dynamic_shapes_with_position_ids.onnx")); + + std::vector input_ids = { + 8, 1, 5, 9, 8, 9, 4, 3, 0, 3, 5, 0, 2, 3, 8, 1, 3, 3, 3, 7, 0, 1, 9, 9, + }; + std::vector segment_ids = { + 0, 2, 0, 2, 2, 0, 2, 0, 0, 0, 1, 1, 2, 0, 0, 1, 0, 1, 2, 2, 0, 1, 1, 1, + }; + std::vector word_embeddings = { + 0.96980906, 0.65314001, 0.17090958, 0.35815218, 0.75068617, 0.60783064, 0.32504722, 0.03842543, 0.63427407, + 0.95894927, 0.65279031, 0.63505888, 0.99529958, 0.58185035, 0.41436860, 0.47469750, 0.62351012, 0.33800763, + 0.67475230, 0.31720173, 0.77834547, 0.94957107, 0.66252685, 0.01357164, 0.62284607, 0.67365962, 0.97194499, + 0.87819350, 0.50962436, 0.05571469, 0.45115921, 0.01998767, 0.44171092, 0.97958672, 0.35944447, 0.48089352, + 0.68866116, 0.88047588, 0.91823548, 0.21682213, 0.56518888, 0.86510259, 0.50896895, 0.91672295, 0.92115760, + 0.08311249, 0.27771857, 0.00935670, 0.84234208, 0.64717412, + }; + std::vector position_embeddings = { + 0.84138614, 0.26473016, 0.39782074, 0.55282146, 0.16494046, 0.43328807, 0.75610667, 0.39609829, 0.89603841, + 0.63892108, 0.36980811, 0.14644176, 0.56961840, 0.70373726, 0.28847644, 0.89155442, 0.68005556, 0.44919774, + 0.97857094, 0.11620191, 0.76702368, 0.41182014, 0.67543906, 0.24979627, 0.31321833, 0.28799102, 0.38346222, + 0.74916983, 0.87845218, 0.10286336, 0.96541619, 0.58846509, 0.65966839, 0.53320622, 0.23053302, 0.39486930, + 0.61880857, 0.47486752, 0.47013220, 0.71607453, 0.84138614, 0.26473016, 0.39782074, 0.55282146, 0.16494046, + }; + std::vector segment_embeddings = { + 0.09237389, + 0.35404667, + 0.55181628, + 0.03362509, + 0.96896178, + 0.32099724, + 0.22126268, + 0.14126390, + 0.09725992, + 0.98404223, + 0.26034093, + 0.53702253, + 0.44792616, + 0.09956909, + 0.35231167, + }; + std::vector gamma = { + 0.46924916, + 0.84114015, + 0.90464777, + 0.03755938, + 0.50831544, + }; + std::vector beta = { + 0.16684751, + 0.77905101, + 0.86493331, + 0.41139671, + 0.13997258, + }; + std::vector mask = { + 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, + }; + std::vector position_ids = { + 0, 2, 1, 3, 4, 6, 7, 5, 8, 2, 1, 3, 4, 6, 7, 5, 0, 2, 1, 3, 4, 6, 7, 5, + }; + std::vector expected_output = { + -0.04089922, 0.35108989, 0.30442458, 0.39546335, 1.15422225, 0.10419128, -0.19301927, 0.01070970, + 0.43977541, 0.89119899, -0.51436460, 1.99256825, 1.41077507, 0.38642293, 0.17583044, 0.03320138, + 1.16508031, -0.24356931, 0.47440714, -0.17844005, 0.20463173, 1.90038323, 1.14138567, 0.34504607, + 0.16403235, -0.24976699, 0.29362509, 0.34502214, 0.41751838, 1.09390712, 0.12354189, 1.83025289, + 1.05569196, 0.34413773, 0.35469764, -0.69760042, 0.76338542, 1.75443077, 0.44126555, 0.18181801, + 0.73277575, 0.45443264, 0.17068321, 0.36591727, 0.72869974, -0.56090516, 0.14415455, 1.47314119, + 0.42908576, 0.73084539, -0.22373237, 2.26550221, 0.05606699, 0.39417523, 0.35234636, 0.78569502, + 0.77521765, -0.65131050, 0.40168875, 0.45527256, 0.38715565, 0.98521245, 2.21446753, 0.36345237, + -0.33269632, 0.36558092, 1.36846578, 1.37523413, 0.33698002, 0.28889543, -0.40639281, 1.01643157, + 0.59668219, 0.39197800, 1.03101778, 0.02551098, -0.03612846, -0.01371557, 0.43444607, 0.96746695, + 0.60583955, -0.10362893, 0.40574494, 0.38046724, 0.87445319, -0.00880148, -0.15437943, 0.08118075, + 0.44650543, 0.85956848, -0.27865338, 2.10837507, 0.04798460, 0.43948367, -0.10185169, 0.19978794, + 1.32323360, 1.20525467, 0.44288942, -0.84200430, 0.52563053, 0.69949460, 0.73987913, 0.34668452, + 0.74545687, 0.57696682, 0.22452033, -0.27099937, 0.39649010, 0.87083614, -0.18965788, 0.58206403, + -0.08108193, 0.42067638, 1.05117214, -0.34287399, 0.20424896, 0.27994895, 0.46011117, 0.70890665, + }; + std::vector expected_mask_index = { + 6, + 5, + 5, + }; + + auto test_case = test::TestCase(function, s_device); + test_case.add_input(Shape{3, 8}, input_ids); + test_case.add_input(Shape{3, 8}, segment_ids); + test_case.add_input(Shape{10, 5}, word_embeddings); + test_case.add_input(Shape{9, 5}, position_embeddings); + test_case.add_input(Shape{3, 5}, segment_embeddings); + test_case.add_input(Shape{5}, gamma); + test_case.add_input(Shape{5}, beta); + test_case.add_input(Shape{3, 8}, mask); + test_case.add_input(Shape{3, 8}, position_ids); + test_case.add_expected_output(Shape{3, 8, 5}, expected_output); + test_case.add_expected_output(Shape{3}, expected_mask_index); + test_case.run_with_tolerance_as_fp(1e-6); +} + NGRAPH_TEST(${BACKEND_NAME}, onnx_model_attention) { const auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), SERIALIZED_ZOO,