IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

revise GRU cell/sequence (#7901)

Browse Source 
* add typepro/vistor tests

Signed-off-by: fishbell <bell.song@intel.com>

* remove redundant test file

Signed-off-by: fishbell <bell.song@intel.com>

* fix clang format

Signed-off-by: fishbell <bell.song@intel.com>

* typo

Signed-off-by: fishbell <bell.song@intel.com>

* update doc

Signed-off-by: fishbell <bell.song@intel.com>
This commit is contained in:
song, bell
2021-10-21 11:26:53 +03:00
committed by GitHub
parent e0062fc274
commit ff14899a96
5 changed files with 350 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/ops/sequence/GRUCell_3.md
+17 -2
View File
@@ -6,6 +6,21 @@
**Short description**: *GRUCell* represents a single GRU Cell that computes the output using the formula described in the [paper](https://arxiv.org/abs/1406.1078).
**Detailed description**: *GRUCell* computes the output *Ht* for the current time step based on the followint formula:
```
Formula:
* - matrix multiplication
(.) - Hadamard product(element-wise)
[,] - concatenation
f, g - are activation functions.
zt = f(Xt*(Wz^T) + Ht-1*(Rz^T) + Wbz + Rbz)
rt = f(Xt*(Wr^T) + Ht-1*(Rr^T) + Wbr + Rbr)
ht = g(Xt*(Wh^T) + (rt (.) Ht-1)*(Rh^T) + Rbh + Wbh) # default, when linear_before_reset = 0
ht = g(Xt*(Wh^T) + (rt (.) (Ht-1*(Rh^T) + Rbh)) + Wbh) # when linear_before_reset != 0
Ht = (1 - zt) (.) ht + zt (.) Ht-1
```
**Attributes**
* *hidden_size*
@@ -20,7 +35,7 @@
* **Description**: activation functions for gates
* **Range of values**: any combination of *relu*, *sigmoid*, *tanh*
* **Type**: a list of strings
* **Default value**: *sigmoid,tanh*
* **Default value**: *sigmoid* for f, *tanh* for g
* **Required**: *no*
* *activations_alpha, activations_beta*
@@ -57,7 +72,7 @@
* **4**: `R` - 2D tensor of type *T* `[3 * hidden_size, hidden_size]`, the recurrence weights for matrix multiplication, gate order: zrh. **Required.**
* **5**: `B` - 1D tensor of type *T*. If *linear_before_reset* is set to 1, then the shape is `[4 * hidden_size]` - the sum of biases for z and r gates (weights and recurrence weights), the biases for h gate are placed separately. Otherwise the shape is `[3 * hidden_size]`, the sum of biases (weights and recurrence weights). **Required.**
* **5**: `B` - 1D tensor of type *T*. If *linear_before_reset* is set to 1, then the shape is `[4 * hidden_size]` - the sum of biases for z and r gates (weights and recurrence weights), the biases for h gate are placed separately. Otherwise the shape is `[3 * hidden_size]`, the sum of biases (weights and recurrence weights). **Optional.**
**Outputs**
ngraph/test/CMakeLists.txt
+2
View File
@@ -302,6 +302,8 @@ set(SRC
visitors/op/greater_equal.cpp
visitors/op/greater.cpp
visitors/op/grn.cpp
visitors/op/gru_cell.cpp
visitors/op/gru_sequence.cpp
visitors/op/group_conv.cpp
visitors/op/hard_sigmoid.cpp
visitors/op/hsigmoid.cpp
ngraph/test/type_prop/gru_sequence.cpp
+214
View File
@@ -10,6 +10,43 @@
using namespace std;
using namespace ngraph;
struct gru_sequence_parameters {
Dimension batch_size = 8;
Dimension num_directions = 1;
Dimension seq_length = 6;
Dimension input_size = 4;
Dimension hidden_size = 128;
ngraph::element::Type et = element::f32;
};
shared_ptr<opset5::GRUSequence> gru_seq_tensor_initialization(const gru_sequence_parameters& param) {
auto batch_size = param.batch_size;
auto seq_length = param.seq_length;
auto input_size = param.input_size;
auto num_directions = param.num_directions;
auto hidden_size = param.hidden_size;
auto et = param.et;
const auto X = make_shared<opset5::Parameter>(et, PartialShape{batch_size, seq_length, input_size});
const auto initial_hidden_state =
make_shared<opset5::Parameter>(et, PartialShape{batch_size, num_directions, hidden_size});
const auto sequence_lengths = make_shared<opset5::Parameter>(et, PartialShape{batch_size});
const auto W = make_shared<opset5::Parameter>(et, PartialShape{num_directions, hidden_size * 3, input_size});
const auto R = make_shared<opset5::Parameter>(et, PartialShape{num_directions, hidden_size * 3, hidden_size});
const auto B = make_shared<opset5::Parameter>(et, PartialShape{num_directions, hidden_size * 3});
const auto gru_sequence = make_shared<opset5::GRUSequence>();
gru_sequence->set_argument(0, X);
gru_sequence->set_argument(1, initial_hidden_state);
gru_sequence->set_argument(2, sequence_lengths);
gru_sequence->set_argument(3, W);
gru_sequence->set_argument(4, R);
gru_sequence->set_argument(5, B);
return gru_sequence;
}
TEST(type_prop, gru_sequence_forward) {
const size_t batch_size = 8;
const size_t num_directions = 1;
@@ -44,3 +81,180 @@ TEST(type_prop, gru_sequence_forward) {
EXPECT_EQ(sequence->get_output_element_type(1), element::f32);
EXPECT_EQ(sequence->get_output_shape(1), (Shape{batch_size, num_directions, hidden_size}));
}
TEST(type_prop, gru_sequence_bidirectional) {
const size_t batch_size = 8;
const size_t num_directions = 1;
const size_t seq_length = 6;
const size_t input_size = 4;
const size_t hidden_size = 128;
const auto X = make_shared<opset5::Parameter>(element::f32, Shape{batch_size, seq_length, input_size});
const auto initial_hidden_state =
make_shared<opset5::Parameter>(element::f32, Shape{batch_size, num_directions, hidden_size});
const auto sequence_lengths = make_shared<op::Parameter>(element::i32, Shape{batch_size});
const auto W = make_shared<opset5::Parameter>(element::f32, Shape{num_directions, 3 * hidden_size, input_size});
const auto R = make_shared<opset5::Parameter>(element::f32, Shape{num_directions, 3 * hidden_size, hidden_size});
const auto B = make_shared<opset5::Parameter>(element::f32, Shape{num_directions, 3 * hidden_size});
const auto direction = op::RecurrentSequenceDirection::BIDIRECTIONAL;
const std::vector<float> activations_alpha = {2.7, 7.0, 32.367};
const std::vector<float> activations_beta = {0.0, 5.49, 6.0};
const std::vector<std::string> activations = {"tanh", "sigmoid"};
const auto sequence = make_shared<opset5::GRUSequence>(X,
initial_hidden_state,
sequence_lengths,
W,
R,
B,
hidden_size,
direction,
activations,
activations_alpha,
activations_beta);
EXPECT_EQ(sequence->get_hidden_size(), hidden_size);
EXPECT_EQ(sequence->get_direction(), op::RecurrentSequenceDirection::BIDIRECTIONAL);
EXPECT_EQ(sequence->get_activations_alpha(), activations_alpha);
EXPECT_EQ(sequence->get_activations_beta(), activations_beta);
EXPECT_EQ(sequence->get_activations()[0], "tanh");
EXPECT_EQ(sequence->get_activations()[1], "sigmoid");
EXPECT_EQ(sequence->get_clip(), 0.f);
EXPECT_EQ(sequence->get_linear_before_reset(), false);
EXPECT_EQ(sequence->get_output_element_type(0), element::f32);
EXPECT_EQ(sequence->outputs().size(), 2);
EXPECT_EQ(sequence->get_output_shape(0), (Shape{batch_size, num_directions, seq_length, hidden_size}));
EXPECT_EQ(sequence->get_output_element_type(1), element::f32);
EXPECT_EQ(sequence->get_output_shape(1), (Shape{batch_size, num_directions, hidden_size}));
}
TEST(type_prop, gru_sequence_dynamic_batch_size) {
gru_sequence_parameters param;
param.batch_size = Dimension::dynamic();
param.num_directions = 2;
param.seq_length = 6;
param.input_size = 4;
param.hidden_size = 128;
param.et = element::f32;
auto gru_sequence = gru_seq_tensor_initialization(param);
gru_sequence->validate_and_infer_types();
EXPECT_EQ(gru_sequence->get_output_partial_shape(0),
(PartialShape{param.batch_size, param.num_directions, param.seq_length, param.hidden_size}));
EXPECT_EQ(gru_sequence->get_output_partial_shape(1),
(PartialShape{param.batch_size, param.num_directions, param.hidden_size}));
EXPECT_EQ(gru_sequence->get_output_element_type(0), param.et);
EXPECT_EQ(gru_sequence->get_output_element_type(1), param.et);
}
TEST(type_prop, gru_sequence_dynamic_num_directions) {
gru_sequence_parameters param;
param.batch_size = 8;
param.num_directions = Dimension::dynamic();
param.seq_length = 6;
param.input_size = 4;
param.hidden_size = 128;
param.et = element::f32;
auto gru_sequence = gru_seq_tensor_initialization(param);
gru_sequence->validate_and_infer_types();
EXPECT_EQ(gru_sequence->get_output_partial_shape(0),
(PartialShape{param.batch_size, param.num_directions, param.seq_length, param.hidden_size}));
EXPECT_EQ(gru_sequence->get_output_partial_shape(1),
(PartialShape{param.batch_size, param.num_directions, param.hidden_size}));
EXPECT_EQ(gru_sequence->get_output_element_type(0), param.et);
EXPECT_EQ(gru_sequence->get_output_element_type(1), param.et);
}
TEST(type_prop, gru_sequence_dynamic_seq_length) {
gru_sequence_parameters param;
param.batch_size = 8;
param.num_directions = 1;
param.seq_length = Dimension::dynamic();
param.input_size = 4;
param.hidden_size = 128;
param.et = element::f32;
auto gru_sequence = gru_seq_tensor_initialization(param);
gru_sequence->validate_and_infer_types();
EXPECT_EQ(gru_sequence->get_output_partial_shape(0),
(PartialShape{param.batch_size, param.num_directions, param.seq_length, param.hidden_size}));
EXPECT_EQ(gru_sequence->get_output_partial_shape(1),
(PartialShape{param.batch_size, param.num_directions, param.hidden_size}));
EXPECT_EQ(gru_sequence->get_output_element_type(0), param.et);
EXPECT_EQ(gru_sequence->get_output_element_type(1), param.et);
}
TEST(type_prop, gru_sequence_dynamic_hidden_size) {
gru_sequence_parameters param;
param.batch_size = 8;
param.num_directions = 1;
param.seq_length = 6;
param.input_size = 4;
param.hidden_size = Dimension::dynamic();
param.et = element::f32;
auto gru_sequence = gru_seq_tensor_initialization(param);
gru_sequence->validate_and_infer_types();
EXPECT_EQ(gru_sequence->get_output_partial_shape(0),
(PartialShape{param.batch_size, param.num_directions, param.seq_length, param.hidden_size}));
EXPECT_EQ(gru_sequence->get_output_partial_shape(1),
(PartialShape{param.batch_size, param.num_directions, param.hidden_size}));
EXPECT_EQ(gru_sequence->get_output_element_type(0), param.et);
EXPECT_EQ(gru_sequence->get_output_element_type(1), param.et);
}
TEST(type_prop, gru_sequence_invalid_input_dimension) {
gru_sequence_parameters param;
param.batch_size = 8;
param.num_directions = 1;
param.seq_length = 6;
param.input_size = 4;
param.hidden_size = 128;
param.et = element::f32;
auto gru_sequence = gru_seq_tensor_initialization(param);
auto invalid_rank0_tensor = make_shared<opset5::Parameter>(param.et, PartialShape{});
// Validate invalid rank0 tensor for all inputs: X, initial_hidden_state, W, R, B
for (size_t i = 0; i < gru_sequence->get_input_size(); i++) {
gru_sequence = gru_seq_tensor_initialization(param);
gru_sequence->set_argument(i, invalid_rank0_tensor);
ASSERT_THROW(gru_sequence->validate_and_infer_types(), ngraph::CheckFailure)
<< "GRUSequence node was created with invalid data.";
}
}
TEST(type_prop, gru_sequence_invalid_input_dynamic_rank) {
gru_sequence_parameters param;
param.batch_size = 8;
param.num_directions = 2;
param.seq_length = 6;
param.input_size = 4;
param.hidden_size = 128;
param.et = element::f32;
auto check_dynamic_gru = [](const shared_ptr<opset5::GRUSequence>& gru) -> bool {
return gru->output(0).get_partial_shape() == PartialShape::dynamic() &&
gru->output(1).get_partial_shape() == PartialShape::dynamic() &&
gru->output(0).get_element_type() == gru->input(0).get_element_type();
};
auto gru_sequence = gru_seq_tensor_initialization(param);
auto invalid_dynamic_tensor = make_shared<opset5::Parameter>(param.et, PartialShape::dynamic(Rank::dynamic()));
// Validate invalid dynamic tensor for all inputs: X, initial_hidden_state, W, R, B
for (size_t i = 0; i < gru_sequence->get_input_size(); i++) {
gru_sequence = gru_seq_tensor_initialization(param);
gru_sequence->set_argument(i, invalid_dynamic_tensor);
gru_sequence->validate_and_infer_types();
EXPECT_EQ(check_dynamic_gru(gru_sequence), true);
}
}
ngraph/test/visitors/op/gru_cell.cpp
+51
View File
@@ -0,0 +1,51 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "gtest/gtest.h"
#include "ngraph/ngraph.hpp"
#include "ngraph/op/util/attr_types.hpp"
#include "ngraph/opsets/opset1.hpp"
#include "ngraph/opsets/opset3.hpp"
#include "ngraph/opsets/opset4.hpp"
#include "ngraph/opsets/opset5.hpp"
#include "util/visitor.hpp"
using namespace std;
using namespace ngraph;
using ngraph::test::NodeBuilder;
using ngraph::test::ValueMap;
TEST(attributes, gru_cell_op) {
NodeBuilder::get_ops().register_factory<opset5::GRUCell>();
auto X = make_shared<op::Parameter>(element::f32, Shape{2, 3});
auto H = make_shared<op::Parameter>(element::f32, Shape{2, 3});
auto W = make_shared<op::Parameter>(element::f32, Shape{9, 3});
auto R = make_shared<op::Parameter>(element::f32, Shape{9, 3});
const auto initial_hidden_state = make_shared<op::Parameter>(element::f32, Shape{2, 3});
const auto hidden_size = 3;
const std::vector<std::string> activations = {"tanh", "sigmoid"};
auto activations_alpha = std::vector<float>{1.0, 1.5};
auto activations_beta = std::vector<float>{2.0, 1.0};
const float clip = 0.5f;
const auto gru_cell = make_shared<opset5::GRUCell>(X,
initial_hidden_state,
W,
R,
hidden_size,
activations,
activations_alpha,
activations_beta,
clip,
false);
NodeBuilder builder(gru_cell);
auto g_gru_cell = ov::as_type_ptr<opset5::GRUCell>(builder.create());
EXPECT_EQ(g_gru_cell->get_hidden_size(), gru_cell->get_hidden_size());
EXPECT_EQ(g_gru_cell->get_activations(), gru_cell->get_activations());
EXPECT_EQ(g_gru_cell->get_activations_alpha(), gru_cell->get_activations_alpha());
EXPECT_EQ(g_gru_cell->get_activations_beta(), gru_cell->get_activations_beta());
EXPECT_EQ(g_gru_cell->get_clip(), gru_cell->get_clip());
EXPECT_EQ(g_gru_cell->get_linear_before_reset(), gru_cell->get_linear_before_reset());
}
ngraph/test/visitors/op/gru_sequence.cpp
+66
View File
@@ -0,0 +1,66 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "gtest/gtest.h"
#include "ngraph/ngraph.hpp"
#include "ngraph/op/util/attr_types.hpp"
#include "ngraph/opsets/opset1.hpp"
#include "ngraph/opsets/opset3.hpp"
#include "ngraph/opsets/opset4.hpp"
#include "ngraph/opsets/opset5.hpp"
#include "util/visitor.hpp"
using namespace std;
using namespace ngraph;
using ngraph::test::NodeBuilder;
using ngraph::test::ValueMap;
TEST(attributes, gru_sequence_op) {
NodeBuilder::get_ops().register_factory<opset5::GRUSequence>();
const size_t batch_size = 4;
const size_t num_directions = 2;
const size_t seq_length = 8;
const size_t input_size = 16;
const size_t hidden_size = 64;
const auto X = make_shared<op::Parameter>(element::f32, Shape{batch_size, seq_length, input_size});
const auto initial_hidden_state =
make_shared<op::Parameter>(element::f32, Shape{batch_size, num_directions, hidden_size});
const auto initial_cell_state =
make_shared<op::Parameter>(element::f32, Shape{batch_size, num_directions, hidden_size});
const auto sequence_lengths = make_shared<op::Parameter>(element::i32, Shape{batch_size});
const auto W = make_shared<op::Parameter>(element::f32, Shape{num_directions, 3 * hidden_size, input_size});
const auto R = make_shared<op::Parameter>(element::f32, Shape{num_directions, 3 * hidden_size, hidden_size});
const auto B = make_shared<op::Parameter>(element::f32, Shape{num_directions, 3 * hidden_size});
const auto gru_direction = op::RecurrentSequenceDirection::BIDIRECTIONAL;
const std::vector<float> activations_alpha = {1, 2};
const std::vector<float> activations_beta = {4, 5};
const std::vector<std::string> activations = {"tanh", "sigmoid"};
const float clip_threshold = 0.5f;
const auto gru_sequence = make_shared<opset5::GRUSequence>(X,
initial_hidden_state,
sequence_lengths,
W,
R,
B,
hidden_size,
gru_direction,
activations,
activations_alpha,
activations_beta,
clip_threshold);
NodeBuilder builder(gru_sequence);
auto g_gru_sequence = ov::as_type_ptr<opset5::GRUSequence>(builder.create());
EXPECT_EQ(g_gru_sequence->get_hidden_size(), gru_sequence->get_hidden_size());
EXPECT_EQ(g_gru_sequence->get_activations(), gru_sequence->get_activations());
EXPECT_EQ(g_gru_sequence->get_activations_alpha(), gru_sequence->get_activations_alpha());
EXPECT_EQ(g_gru_sequence->get_activations_beta(), gru_sequence->get_activations_beta());
EXPECT_EQ(g_gru_sequence->get_clip(), gru_sequence->get_clip());
EXPECT_EQ(g_gru_sequence->get_direction(), gru_sequence->get_direction());
EXPECT_EQ(g_gru_sequence->get_linear_before_reset(), gru_sequence->get_linear_before_reset());
}