From ff14899a961cddf0f2f6cfd459d871f74aae5ea6 Mon Sep 17 00:00:00 2001 From: "song, bell" Date: Thu, 21 Oct 2021 16:26:53 +0800 Subject: [PATCH] revise GRU cell/sequence (#7901) * add typepro/vistor tests Signed-off-by: fishbell * remove redundant test file Signed-off-by: fishbell * fix clang format Signed-off-by: fishbell * typo Signed-off-by: fishbell * update doc Signed-off-by: fishbell --- docs/ops/sequence/GRUCell_3.md | 19 +- ngraph/test/CMakeLists.txt | 2 + ngraph/test/type_prop/gru_sequence.cpp | 214 +++++++++++++++++++++++ ngraph/test/visitors/op/gru_cell.cpp | 51 ++++++ ngraph/test/visitors/op/gru_sequence.cpp | 66 +++++++ 5 files changed, 350 insertions(+), 2 deletions(-) create mode 100644 ngraph/test/visitors/op/gru_cell.cpp create mode 100644 ngraph/test/visitors/op/gru_sequence.cpp diff --git a/docs/ops/sequence/GRUCell_3.md b/docs/ops/sequence/GRUCell_3.md index df5b14046d9..4453c6c1028 100644 --- a/docs/ops/sequence/GRUCell_3.md +++ b/docs/ops/sequence/GRUCell_3.md @@ -6,6 +6,21 @@ **Short description**: *GRUCell* represents a single GRU Cell that computes the output using the formula described in the [paper](https://arxiv.org/abs/1406.1078). +**Detailed description**: *GRUCell* computes the output *Ht* for the current time step based on the followint formula: + +``` +Formula: + * - matrix multiplication + (.) - Hadamard product(element-wise) + [,] - concatenation + f, g - are activation functions. + zt = f(Xt*(Wz^T) + Ht-1*(Rz^T) + Wbz + Rbz) + rt = f(Xt*(Wr^T) + Ht-1*(Rr^T) + Wbr + Rbr) + ht = g(Xt*(Wh^T) + (rt (.) Ht-1)*(Rh^T) + Rbh + Wbh) # default, when linear_before_reset = 0 + ht = g(Xt*(Wh^T) + (rt (.) (Ht-1*(Rh^T) + Rbh)) + Wbh) # when linear_before_reset != 0 + Ht = (1 - zt) (.) ht + zt (.) Ht-1 +``` + **Attributes** * *hidden_size* @@ -20,7 +35,7 @@ * **Description**: activation functions for gates * **Range of values**: any combination of *relu*, *sigmoid*, *tanh* * **Type**: a list of strings - * **Default value**: *sigmoid,tanh* + * **Default value**: *sigmoid* for f, *tanh* for g * **Required**: *no* * *activations_alpha, activations_beta* @@ -57,7 +72,7 @@ * **4**: `R` - 2D tensor of type *T* `[3 * hidden_size, hidden_size]`, the recurrence weights for matrix multiplication, gate order: zrh. **Required.** -* **5**: `B` - 1D tensor of type *T*. If *linear_before_reset* is set to 1, then the shape is `[4 * hidden_size]` - the sum of biases for z and r gates (weights and recurrence weights), the biases for h gate are placed separately. Otherwise the shape is `[3 * hidden_size]`, the sum of biases (weights and recurrence weights). **Required.** +* **5**: `B` - 1D tensor of type *T*. If *linear_before_reset* is set to 1, then the shape is `[4 * hidden_size]` - the sum of biases for z and r gates (weights and recurrence weights), the biases for h gate are placed separately. Otherwise the shape is `[3 * hidden_size]`, the sum of biases (weights and recurrence weights). **Optional.** **Outputs** diff --git a/ngraph/test/CMakeLists.txt b/ngraph/test/CMakeLists.txt index c6e98e9eb59..67b88ebac74 100644 --- a/ngraph/test/CMakeLists.txt +++ b/ngraph/test/CMakeLists.txt @@ -302,6 +302,8 @@ set(SRC visitors/op/greater_equal.cpp visitors/op/greater.cpp visitors/op/grn.cpp + visitors/op/gru_cell.cpp + visitors/op/gru_sequence.cpp visitors/op/group_conv.cpp visitors/op/hard_sigmoid.cpp visitors/op/hsigmoid.cpp diff --git a/ngraph/test/type_prop/gru_sequence.cpp b/ngraph/test/type_prop/gru_sequence.cpp index 0625eb3ef5c..2788aefb885 100644 --- a/ngraph/test/type_prop/gru_sequence.cpp +++ b/ngraph/test/type_prop/gru_sequence.cpp @@ -10,6 +10,43 @@ using namespace std; using namespace ngraph; +struct gru_sequence_parameters { + Dimension batch_size = 8; + Dimension num_directions = 1; + Dimension seq_length = 6; + Dimension input_size = 4; + Dimension hidden_size = 128; + ngraph::element::Type et = element::f32; +}; + +shared_ptr gru_seq_tensor_initialization(const gru_sequence_parameters& param) { + auto batch_size = param.batch_size; + auto seq_length = param.seq_length; + auto input_size = param.input_size; + auto num_directions = param.num_directions; + auto hidden_size = param.hidden_size; + auto et = param.et; + + const auto X = make_shared(et, PartialShape{batch_size, seq_length, input_size}); + const auto initial_hidden_state = + make_shared(et, PartialShape{batch_size, num_directions, hidden_size}); + const auto sequence_lengths = make_shared(et, PartialShape{batch_size}); + const auto W = make_shared(et, PartialShape{num_directions, hidden_size * 3, input_size}); + const auto R = make_shared(et, PartialShape{num_directions, hidden_size * 3, hidden_size}); + const auto B = make_shared(et, PartialShape{num_directions, hidden_size * 3}); + + const auto gru_sequence = make_shared(); + + gru_sequence->set_argument(0, X); + gru_sequence->set_argument(1, initial_hidden_state); + gru_sequence->set_argument(2, sequence_lengths); + gru_sequence->set_argument(3, W); + gru_sequence->set_argument(4, R); + gru_sequence->set_argument(5, B); + + return gru_sequence; +} + TEST(type_prop, gru_sequence_forward) { const size_t batch_size = 8; const size_t num_directions = 1; @@ -44,3 +81,180 @@ TEST(type_prop, gru_sequence_forward) { EXPECT_EQ(sequence->get_output_element_type(1), element::f32); EXPECT_EQ(sequence->get_output_shape(1), (Shape{batch_size, num_directions, hidden_size})); } + +TEST(type_prop, gru_sequence_bidirectional) { + const size_t batch_size = 8; + const size_t num_directions = 1; + const size_t seq_length = 6; + const size_t input_size = 4; + const size_t hidden_size = 128; + + const auto X = make_shared(element::f32, Shape{batch_size, seq_length, input_size}); + const auto initial_hidden_state = + make_shared(element::f32, Shape{batch_size, num_directions, hidden_size}); + const auto sequence_lengths = make_shared(element::i32, Shape{batch_size}); + const auto W = make_shared(element::f32, Shape{num_directions, 3 * hidden_size, input_size}); + const auto R = make_shared(element::f32, Shape{num_directions, 3 * hidden_size, hidden_size}); + const auto B = make_shared(element::f32, Shape{num_directions, 3 * hidden_size}); + + const auto direction = op::RecurrentSequenceDirection::BIDIRECTIONAL; + const std::vector activations_alpha = {2.7, 7.0, 32.367}; + const std::vector activations_beta = {0.0, 5.49, 6.0}; + const std::vector activations = {"tanh", "sigmoid"}; + + const auto sequence = make_shared(X, + initial_hidden_state, + sequence_lengths, + W, + R, + B, + hidden_size, + direction, + activations, + activations_alpha, + activations_beta); + + EXPECT_EQ(sequence->get_hidden_size(), hidden_size); + EXPECT_EQ(sequence->get_direction(), op::RecurrentSequenceDirection::BIDIRECTIONAL); + EXPECT_EQ(sequence->get_activations_alpha(), activations_alpha); + EXPECT_EQ(sequence->get_activations_beta(), activations_beta); + EXPECT_EQ(sequence->get_activations()[0], "tanh"); + EXPECT_EQ(sequence->get_activations()[1], "sigmoid"); + EXPECT_EQ(sequence->get_clip(), 0.f); + EXPECT_EQ(sequence->get_linear_before_reset(), false); + EXPECT_EQ(sequence->get_output_element_type(0), element::f32); + EXPECT_EQ(sequence->outputs().size(), 2); + EXPECT_EQ(sequence->get_output_shape(0), (Shape{batch_size, num_directions, seq_length, hidden_size})); + EXPECT_EQ(sequence->get_output_element_type(1), element::f32); + EXPECT_EQ(sequence->get_output_shape(1), (Shape{batch_size, num_directions, hidden_size})); +} + +TEST(type_prop, gru_sequence_dynamic_batch_size) { + gru_sequence_parameters param; + param.batch_size = Dimension::dynamic(); + param.num_directions = 2; + param.seq_length = 6; + param.input_size = 4; + param.hidden_size = 128; + param.et = element::f32; + + auto gru_sequence = gru_seq_tensor_initialization(param); + gru_sequence->validate_and_infer_types(); + + EXPECT_EQ(gru_sequence->get_output_partial_shape(0), + (PartialShape{param.batch_size, param.num_directions, param.seq_length, param.hidden_size})); + EXPECT_EQ(gru_sequence->get_output_partial_shape(1), + (PartialShape{param.batch_size, param.num_directions, param.hidden_size})); + EXPECT_EQ(gru_sequence->get_output_element_type(0), param.et); + EXPECT_EQ(gru_sequence->get_output_element_type(1), param.et); +} + +TEST(type_prop, gru_sequence_dynamic_num_directions) { + gru_sequence_parameters param; + param.batch_size = 8; + param.num_directions = Dimension::dynamic(); + param.seq_length = 6; + param.input_size = 4; + param.hidden_size = 128; + param.et = element::f32; + + auto gru_sequence = gru_seq_tensor_initialization(param); + gru_sequence->validate_and_infer_types(); + + EXPECT_EQ(gru_sequence->get_output_partial_shape(0), + (PartialShape{param.batch_size, param.num_directions, param.seq_length, param.hidden_size})); + EXPECT_EQ(gru_sequence->get_output_partial_shape(1), + (PartialShape{param.batch_size, param.num_directions, param.hidden_size})); + EXPECT_EQ(gru_sequence->get_output_element_type(0), param.et); + EXPECT_EQ(gru_sequence->get_output_element_type(1), param.et); +} + +TEST(type_prop, gru_sequence_dynamic_seq_length) { + gru_sequence_parameters param; + param.batch_size = 8; + param.num_directions = 1; + param.seq_length = Dimension::dynamic(); + param.input_size = 4; + param.hidden_size = 128; + param.et = element::f32; + + auto gru_sequence = gru_seq_tensor_initialization(param); + gru_sequence->validate_and_infer_types(); + + EXPECT_EQ(gru_sequence->get_output_partial_shape(0), + (PartialShape{param.batch_size, param.num_directions, param.seq_length, param.hidden_size})); + EXPECT_EQ(gru_sequence->get_output_partial_shape(1), + (PartialShape{param.batch_size, param.num_directions, param.hidden_size})); + EXPECT_EQ(gru_sequence->get_output_element_type(0), param.et); + EXPECT_EQ(gru_sequence->get_output_element_type(1), param.et); +} + +TEST(type_prop, gru_sequence_dynamic_hidden_size) { + gru_sequence_parameters param; + param.batch_size = 8; + param.num_directions = 1; + param.seq_length = 6; + param.input_size = 4; + param.hidden_size = Dimension::dynamic(); + param.et = element::f32; + + auto gru_sequence = gru_seq_tensor_initialization(param); + gru_sequence->validate_and_infer_types(); + + EXPECT_EQ(gru_sequence->get_output_partial_shape(0), + (PartialShape{param.batch_size, param.num_directions, param.seq_length, param.hidden_size})); + EXPECT_EQ(gru_sequence->get_output_partial_shape(1), + (PartialShape{param.batch_size, param.num_directions, param.hidden_size})); + EXPECT_EQ(gru_sequence->get_output_element_type(0), param.et); + EXPECT_EQ(gru_sequence->get_output_element_type(1), param.et); +} + +TEST(type_prop, gru_sequence_invalid_input_dimension) { + gru_sequence_parameters param; + + param.batch_size = 8; + param.num_directions = 1; + param.seq_length = 6; + param.input_size = 4; + param.hidden_size = 128; + param.et = element::f32; + + auto gru_sequence = gru_seq_tensor_initialization(param); + auto invalid_rank0_tensor = make_shared(param.et, PartialShape{}); + + // Validate invalid rank0 tensor for all inputs: X, initial_hidden_state, W, R, B + for (size_t i = 0; i < gru_sequence->get_input_size(); i++) { + gru_sequence = gru_seq_tensor_initialization(param); + gru_sequence->set_argument(i, invalid_rank0_tensor); + ASSERT_THROW(gru_sequence->validate_and_infer_types(), ngraph::CheckFailure) + << "GRUSequence node was created with invalid data."; + } +} + +TEST(type_prop, gru_sequence_invalid_input_dynamic_rank) { + gru_sequence_parameters param; + + param.batch_size = 8; + param.num_directions = 2; + param.seq_length = 6; + param.input_size = 4; + param.hidden_size = 128; + param.et = element::f32; + + auto check_dynamic_gru = [](const shared_ptr& gru) -> bool { + return gru->output(0).get_partial_shape() == PartialShape::dynamic() && + gru->output(1).get_partial_shape() == PartialShape::dynamic() && + gru->output(0).get_element_type() == gru->input(0).get_element_type(); + }; + + auto gru_sequence = gru_seq_tensor_initialization(param); + auto invalid_dynamic_tensor = make_shared(param.et, PartialShape::dynamic(Rank::dynamic())); + + // Validate invalid dynamic tensor for all inputs: X, initial_hidden_state, W, R, B + for (size_t i = 0; i < gru_sequence->get_input_size(); i++) { + gru_sequence = gru_seq_tensor_initialization(param); + gru_sequence->set_argument(i, invalid_dynamic_tensor); + gru_sequence->validate_and_infer_types(); + EXPECT_EQ(check_dynamic_gru(gru_sequence), true); + } +} diff --git a/ngraph/test/visitors/op/gru_cell.cpp b/ngraph/test/visitors/op/gru_cell.cpp new file mode 100644 index 00000000000..794e41507c3 --- /dev/null +++ b/ngraph/test/visitors/op/gru_cell.cpp @@ -0,0 +1,51 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "gtest/gtest.h" +#include "ngraph/ngraph.hpp" +#include "ngraph/op/util/attr_types.hpp" +#include "ngraph/opsets/opset1.hpp" +#include "ngraph/opsets/opset3.hpp" +#include "ngraph/opsets/opset4.hpp" +#include "ngraph/opsets/opset5.hpp" +#include "util/visitor.hpp" + +using namespace std; +using namespace ngraph; +using ngraph::test::NodeBuilder; +using ngraph::test::ValueMap; + +TEST(attributes, gru_cell_op) { + NodeBuilder::get_ops().register_factory(); + auto X = make_shared(element::f32, Shape{2, 3}); + auto H = make_shared(element::f32, Shape{2, 3}); + auto W = make_shared(element::f32, Shape{9, 3}); + auto R = make_shared(element::f32, Shape{9, 3}); + const auto initial_hidden_state = make_shared(element::f32, Shape{2, 3}); + + const auto hidden_size = 3; + const std::vector activations = {"tanh", "sigmoid"}; + auto activations_alpha = std::vector{1.0, 1.5}; + auto activations_beta = std::vector{2.0, 1.0}; + const float clip = 0.5f; + const auto gru_cell = make_shared(X, + initial_hidden_state, + W, + R, + hidden_size, + activations, + activations_alpha, + activations_beta, + clip, + false); + NodeBuilder builder(gru_cell); + auto g_gru_cell = ov::as_type_ptr(builder.create()); + + EXPECT_EQ(g_gru_cell->get_hidden_size(), gru_cell->get_hidden_size()); + EXPECT_EQ(g_gru_cell->get_activations(), gru_cell->get_activations()); + EXPECT_EQ(g_gru_cell->get_activations_alpha(), gru_cell->get_activations_alpha()); + EXPECT_EQ(g_gru_cell->get_activations_beta(), gru_cell->get_activations_beta()); + EXPECT_EQ(g_gru_cell->get_clip(), gru_cell->get_clip()); + EXPECT_EQ(g_gru_cell->get_linear_before_reset(), gru_cell->get_linear_before_reset()); +} diff --git a/ngraph/test/visitors/op/gru_sequence.cpp b/ngraph/test/visitors/op/gru_sequence.cpp new file mode 100644 index 00000000000..8e87a8ea4c1 --- /dev/null +++ b/ngraph/test/visitors/op/gru_sequence.cpp @@ -0,0 +1,66 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "gtest/gtest.h" +#include "ngraph/ngraph.hpp" +#include "ngraph/op/util/attr_types.hpp" +#include "ngraph/opsets/opset1.hpp" +#include "ngraph/opsets/opset3.hpp" +#include "ngraph/opsets/opset4.hpp" +#include "ngraph/opsets/opset5.hpp" +#include "util/visitor.hpp" + +using namespace std; +using namespace ngraph; +using ngraph::test::NodeBuilder; +using ngraph::test::ValueMap; + +TEST(attributes, gru_sequence_op) { + NodeBuilder::get_ops().register_factory(); + + const size_t batch_size = 4; + const size_t num_directions = 2; + const size_t seq_length = 8; + const size_t input_size = 16; + const size_t hidden_size = 64; + + const auto X = make_shared(element::f32, Shape{batch_size, seq_length, input_size}); + const auto initial_hidden_state = + make_shared(element::f32, Shape{batch_size, num_directions, hidden_size}); + const auto initial_cell_state = + make_shared(element::f32, Shape{batch_size, num_directions, hidden_size}); + const auto sequence_lengths = make_shared(element::i32, Shape{batch_size}); + const auto W = make_shared(element::f32, Shape{num_directions, 3 * hidden_size, input_size}); + const auto R = make_shared(element::f32, Shape{num_directions, 3 * hidden_size, hidden_size}); + const auto B = make_shared(element::f32, Shape{num_directions, 3 * hidden_size}); + + const auto gru_direction = op::RecurrentSequenceDirection::BIDIRECTIONAL; + const std::vector activations_alpha = {1, 2}; + const std::vector activations_beta = {4, 5}; + const std::vector activations = {"tanh", "sigmoid"}; + const float clip_threshold = 0.5f; + + const auto gru_sequence = make_shared(X, + initial_hidden_state, + sequence_lengths, + W, + R, + B, + hidden_size, + gru_direction, + activations, + activations_alpha, + activations_beta, + clip_threshold); + NodeBuilder builder(gru_sequence); + auto g_gru_sequence = ov::as_type_ptr(builder.create()); + + EXPECT_EQ(g_gru_sequence->get_hidden_size(), gru_sequence->get_hidden_size()); + EXPECT_EQ(g_gru_sequence->get_activations(), gru_sequence->get_activations()); + EXPECT_EQ(g_gru_sequence->get_activations_alpha(), gru_sequence->get_activations_alpha()); + EXPECT_EQ(g_gru_sequence->get_activations_beta(), gru_sequence->get_activations_beta()); + EXPECT_EQ(g_gru_sequence->get_clip(), gru_sequence->get_clip()); + EXPECT_EQ(g_gru_sequence->get_direction(), gru_sequence->get_direction()); + EXPECT_EQ(g_gru_sequence->get_linear_before_reset(), gru_sequence->get_linear_before_reset()); +}