From 68e7f062c55e8f236c3de81b777bcc2cdb4717f0 Mon Sep 17 00:00:00 2001 From: Ivan Tikhonov Date: Sat, 19 Nov 2022 11:57:44 +0400 Subject: [PATCH] LowLatency v2 tests + resolving the review comments from PR 93758 (#14084) * Enable SequenceFusion, TensorIteratorToSequence, GRUCellFusion transformation in MOC, the transformations have been deleted from plugin pipelines * clean-up: delete augru transformation header * Temporary disabled TensorIterator IR reader tests * temporary disable tests, it's not TI issue, usqueeze doesn't have reference impl for fp64 and bf16 * Update ConvertTensorItearatorToSequence transformations: add Unsqueeze to pattern * Codestyle * Specify negative values for LSTM/GRU/RNN Sequence ops * fix conflicts with master branch * Insert ShapeOf subgraph in ConvertSequencesToTensorIterator transformation in case of dynamic input shape * codestyle * fix conflict with master branch * fix onednn version * Update low latency v2 to support GRU/RNN/LSTM Sequence ops, fix accuracy issue * fix tensor names * fix tensor name issue on gna * fix transformation tests, fix copying rt_info in the transformation * fix warning, enable bf16, f64 ti tests * codestyle * fix functional tests * EliminateDuplicateTIInputs transformation, fix copy_with_new_inputs method of TI op * move EliminateDuplicateTIInputs transformation * codestyle * add LLTv2 tests, resolve review comments * resolve conflicts with master Co-authored-by: Andrei Kochin --- .../eliminate_duplicate_ti_inputs.cpp | 2 +- src/core/src/op/unsqueeze.cpp | 2 + src/core/src/pass/low_latency.cpp | 22 ++- .../src/plugin/transformations_pipeline.cpp | 14 +- .../eliminate_duplicate_ti_inputs.cpp | 1 + .../low_latency_v2_test.cpp | 164 ++++++++++++++++++ 6 files changed, 192 insertions(+), 13 deletions(-) diff --git a/src/common/transformations/src/transformations/common_optimizations/eliminate_duplicate_ti_inputs.cpp b/src/common/transformations/src/transformations/common_optimizations/eliminate_duplicate_ti_inputs.cpp index 8931b458efc..367874fcb7f 100644 --- a/src/common/transformations/src/transformations/common_optimizations/eliminate_duplicate_ti_inputs.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/eliminate_duplicate_ti_inputs.cpp @@ -69,9 +69,9 @@ ov::pass::EliminateDuplicateTIInputs::EliminateDuplicateTIInputs() { const auto& body = ti->get_function(); // re-connect outputs of duplicate Parameters + auto parameters = body->get_parameters(); for (const auto& it : need_to_eliminate) { for (const auto& redundant : it.second) { - auto parameters = body->get_parameters(); parameters[redundant->m_body_parameter_index]->output(0).replace( parameters[it.first->m_body_parameter_index]); } diff --git a/src/core/src/op/unsqueeze.cpp b/src/core/src/op/unsqueeze.cpp index 14e8b2f05fa..e5bf025d394 100644 --- a/src/core/src/op/unsqueeze.cpp +++ b/src/core/src/op/unsqueeze.cpp @@ -119,6 +119,8 @@ bool op::v0::Unsqueeze::has_evaluate() const { case ngraph::element::u64: case ngraph::element::f16: case ngraph::element::f32: + case ngraph::element::f64: + case ngraph::element::bf16: return true; default: break; diff --git a/src/core/src/pass/low_latency.cpp b/src/core/src/pass/low_latency.cpp index 26df26f0a73..fb923f35fa9 100644 --- a/src/core/src/pass/low_latency.cpp +++ b/src/core/src/pass/low_latency.cpp @@ -177,7 +177,6 @@ shared_ptr replace_with_memory(const ov::Input& in } auto read_value = to.make(read_value_in, variable); input.replace_source_output(read_value->output(0)); - read_value->set_friendly_name(variable_name); auto assign = to.make(output, variable); // control dependency so that ReadValue is processed before Assign assign->add_control_dependency(read_value); @@ -189,10 +188,15 @@ std::vector> replace_with_memory(const shared_ptr bool use_const_initializer, ov::pass::NodeRegistry& to) { std::vector> new_assigns; + size_t var_idx = 0; for (const auto& idx : indexes) { auto in = node->input(idx); auto out = node->output(idx); - new_assigns.push_back(replace_with_memory(in, out, node->get_friendly_name(), use_const_initializer, to)); + new_assigns.push_back(replace_with_memory(in, + out, + node->get_friendly_name() + "/variable_" + to_string(var_idx++), + use_const_initializer, + to)); } return new_assigns; } @@ -211,7 +215,7 @@ ov::OutputVector prepare_inputs(const shared_ptr& op, size_t seq_len_i auto axis_0 = to.make(ov::element::i32, ov::Shape{1}, 0); auto axis_1 = to.make(ov::element::i32, ov::Shape{1}, 1); size_t num_lstm_inputs_without_peepholes = 7; - for (size_t i = 0; i < max(op->get_input_size(), num_lstm_inputs_without_peepholes); ++i) { + for (size_t i = 0; i < min(op->get_input_size(), num_lstm_inputs_without_peepholes); ++i) { if (i < seq_len_idx) { inputs.push_back(to.make(op->get_input_source_output(i), axis_1)); } else if (i > seq_len_idx) { @@ -305,13 +309,19 @@ std::vector> process_sequence(const shared_ptr(ov::element::i32, ov::Shape{1}, 1); ov::OutputVector outputs; - outputs.push_back(to.make(cell->output(0), axis_1_2)); + auto unsqueeze_Y = to.make(cell->output(0), axis_1_2); + unsqueeze_Y->set_friendly_name(op->get_friendly_name() + ":0"); + outputs.push_back(unsqueeze_Y); + + size_t idx = 1; for (const auto& out : cell->outputs()) { - outputs.push_back(to.make(out, axis_1)); + auto unsqueeze_state = to.make(out, axis_1); + unsqueeze_state->set_friendly_name(op->get_friendly_name() + ":" + to_string(idx++)); + outputs.push_back(unsqueeze_state); } replace_node(op, outputs); - copy_runtime_info(op, to.get()); } + copy_runtime_info(op, to.get()); return new_assigns; } } // namespace diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index 1d14af77252..3d07b1e417c 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -323,12 +323,14 @@ void TransformationsPipeline::apply(std::shared_ptr func) { return isCellPrimitiveSupported(node); }); - pass_config->set_callback( - [isSequencePrimitiveSupported](const_node_ptr &node) -> bool { - return isSequencePrimitiveSupported(node); - }); + if (config.enable_loop_unrolling) { + pass_config->set_callback( + [isSequencePrimitiveSupported](const_node_ptr &node) -> bool { + return isSequencePrimitiveSupported(node); + }); + } pass_config->set_callback( [](const_node_ptr &node) -> bool { diff --git a/src/tests/functional/inference_engine/transformations/common_optimizations/eliminate_duplicate_ti_inputs.cpp b/src/tests/functional/inference_engine/transformations/common_optimizations/eliminate_duplicate_ti_inputs.cpp index 7f0e4223fc7..68aa3beef12 100644 --- a/src/tests/functional/inference_engine/transformations/common_optimizations/eliminate_duplicate_ti_inputs.cpp +++ b/src/tests/functional/inference_engine/transformations/common_optimizations/eliminate_duplicate_ti_inputs.cpp @@ -76,4 +76,5 @@ TEST(TransformationTests, EliminateDuplicateTIInputs) { EXPECT_NE(ti_after_transformation, nullptr); EXPECT_EQ(ti_after_transformation->get_body()->get_parameters().size(), 3); EXPECT_EQ(ti_after_transformation->inputs().size(), 3); + EXPECT_EQ(ti_after_transformation->get_input_descriptions().size(), 3); } diff --git a/src/tests/functional/inference_engine/transformations/common_optimizations/low_latency_v2_test.cpp b/src/tests/functional/inference_engine/transformations/common_optimizations/low_latency_v2_test.cpp index ddd5dc6a244..fd53103032e 100644 --- a/src/tests/functional/inference_engine/transformations/common_optimizations/low_latency_v2_test.cpp +++ b/src/tests/functional/inference_engine/transformations/common_optimizations/low_latency_v2_test.cpp @@ -841,3 +841,167 @@ TEST(TransformationTests, LowLatencyLSTM_LLTv1_LLTv2) { EXPECT_NO_THROW(manager.run_passes(f)); } } + +namespace { +using OutPtr = Output; +enum class RNNType : size_t { + RNN = 1, + GRU = 3, + LSTM = 4, +}; + +struct LLT2Params { + RNNType rnn_type; + size_t seq_len; +}; + +struct RNNAttributes { + size_t batch; + size_t num_dir; + size_t hidden_size; + size_t input_size; + size_t seq_len; +}; + +OutputVector create_sequence(RNNType rnn_type, RNNAttributes attrs, const OutPtr& X, const OutPtr& H, const OutPtr& C) { + auto gate = static_cast(rnn_type); + auto shape_of = std::make_shared(X); + auto batch_dimension = std::make_shared( + shape_of, + Constant::create(ngraph::element::i64, {1}, {0}), + Constant::create(ngraph::element::i64, {}, {0})); + auto seq_len_dim = std::make_shared( + shape_of, + Constant::create(element::i64, {1}, {1}), + Constant::create(element::i64, {}, {0})); + auto seq_lengths = std::make_shared(seq_len_dim, batch_dimension); + auto W = make_shared(element::f32, Shape{attrs.num_dir, gate * attrs.hidden_size, attrs.input_size}, 0); + auto R = make_shared(element::f32, Shape{attrs.num_dir, gate * attrs.hidden_size, attrs.hidden_size}, 0); + auto B = make_shared(element::f32, Shape{attrs.num_dir, gate * attrs.hidden_size}, 0); + + shared_ptr sequence; + if (rnn_type == RNNType::LSTM) { + sequence = make_shared(X, H, C, seq_lengths, W, R, B, attrs.hidden_size, LSTMSequence::direction::FORWARD); + } else if (rnn_type == RNNType::RNN) { + sequence = make_shared(X, H, seq_lengths, W, R, B, attrs.hidden_size, op::RecurrentSequenceDirection::FORWARD); + } else if (rnn_type == RNNType::GRU) { + sequence = make_shared(X, H, seq_lengths, W, R, B, attrs.hidden_size, op::RecurrentSequenceDirection::FORWARD); + } + return sequence->outputs(); +} + +shared_ptr create_read_value(const shared_ptr& param, const VariablePtr& variable) { + auto const_zero = make_shared(param->get_element_type(), ov::Shape{1}, 0); + auto shape_of = make_shared(param); + auto broadcast = make_shared(const_zero, shape_of); + auto read_value = make_shared(broadcast, variable); + return read_value; +} + +OutputVector create_cell_reference(RNNType rnn_type, RNNAttributes attrs, const OutPtr& X, const OutPtr& H, const OutPtr& C) { + auto gate = static_cast(rnn_type); + auto axis_0 = Constant::create(element::i32, Shape{1}, {0}); + auto axis_1 = Constant::create(element::i32, Shape{1}, {1}); + + shared_ptr squeeze_C; + if (rnn_type == RNNType::LSTM) { + squeeze_C = make_shared(C, axis_1); + } + + auto squeeze_X = make_shared(X, axis_1); + auto squeeze_H = make_shared(H, axis_1); + + auto W = make_shared(element::f32, Shape{attrs.num_dir, gate * attrs.hidden_size, attrs.input_size}, 0); + auto R = make_shared(element::f32, Shape{attrs.num_dir, gate * attrs.hidden_size, attrs.hidden_size}, 0); + auto B = make_shared(element::f32, Shape{attrs.num_dir, gate * attrs.hidden_size}, 0); + + auto squeeze_W = std::make_shared(W, axis_0); + auto squeeze_R = std::make_shared(R, axis_0); + auto squeeze_B = std::make_shared(B, axis_0); + + shared_ptr cell; + if (rnn_type == RNNType::LSTM) { + cell = make_shared(squeeze_X, squeeze_H, squeeze_C, squeeze_W, squeeze_R, squeeze_B, attrs.hidden_size); + } else if (rnn_type == RNNType::RNN) { + cell = make_shared(squeeze_X, squeeze_H, squeeze_W, squeeze_R, squeeze_B, attrs.hidden_size); + } else if (rnn_type == RNNType::GRU) { + cell = make_shared(squeeze_X, squeeze_H, squeeze_W, squeeze_R, squeeze_B, attrs.hidden_size); + } + + auto axis_12 = Constant::create(element::i32, Shape{2}, {1, 2}); + auto unsqueeze_Y = make_shared(cell->output(0), axis_12); + auto unsqueeze_H = make_shared(cell->output(0), axis_1); + OutputVector outputs = {unsqueeze_Y, unsqueeze_H}; + + if (rnn_type == RNNType::LSTM) { + auto unsqueeze_C = make_shared(cell->output(1), axis_1); + outputs.push_back(unsqueeze_C); + } + return outputs; +} +} // namespace + +class LLT2Sequence + : public WithParamInterface, + public TransformationTestsF { +}; + +TEST_P(LLT2Sequence, RNNLowLatency_v2) { + const auto& p = GetParam(); + RNNAttributes attrs = {1, 1, 10, 4, p.seq_len}; + { + auto X = make_shared(element::f32, Shape{attrs.batch, attrs.seq_len, attrs.input_size}); + auto H = make_shared(element::f32, Shape{attrs.batch, attrs.num_dir, attrs.hidden_size}); + auto C = make_shared(element::f32, Shape{attrs.batch, attrs.num_dir, attrs.hidden_size}); + auto outputs = create_sequence(p.rnn_type, attrs, X, H, C); + ParameterVector params{X, H}; + if (p.rnn_type == RNNType::LSTM) { + params.push_back(C); + } + model = make_shared(outputs, params); + manager.register_pass(); + } + + { + auto X = make_shared(element::f32, Shape{attrs.batch, attrs.seq_len, attrs.input_size}); + auto H = make_shared(element::f32, Shape{attrs.batch, attrs.num_dir, attrs.hidden_size}); + auto C = make_shared(element::f32, Shape{attrs.batch, attrs.num_dir, attrs.hidden_size}); + auto variable_h = make_shared(VariableInfo{PartialShape::dynamic(), element::dynamic, "node_28/variable_0"}); + auto variable_c = make_shared(VariableInfo{PartialShape::dynamic(), element::dynamic, "node_28/variable_1"}); + auto read_val_H = create_read_value(H, variable_h); + auto read_val_C = create_read_value(C, variable_c); + + OutputVector outputs; + if (p.seq_len == 1) { + outputs = create_cell_reference(p.rnn_type, attrs, X, read_val_H, read_val_C); + } else { + outputs = create_sequence(p.rnn_type, attrs, X, read_val_H, read_val_C); + } + + SinkVector sinks; + auto assign_h = make_shared(outputs[1], variable_h); + sinks.push_back(assign_h); + assign_h->add_control_dependency(read_val_H); + + ParameterVector params = {X, H}; + if (p.rnn_type == RNNType::LSTM) { + params.push_back(C); + auto assign_c = make_shared(outputs[2], variable_c); + assign_c->add_control_dependency(read_val_C); + sinks.push_back(assign_c); + } + model_ref = make_shared(outputs, sinks, params); + } + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); +} + +static const std::vector llt_v2_params = { + LLT2Params{RNNType::RNN, 1}, + LLT2Params{RNNType::GRU, 1}, + LLT2Params{RNNType::LSTM, 1}, + LLT2Params{RNNType::RNN, 10}, + LLT2Params{RNNType::GRU, 10}, + LLT2Params{RNNType::LSTM, 10}, +}; + +INSTANTIATE_TEST_SUITE_P(LLT2Sequence, LLT2Sequence, ValuesIn(llt_v2_params));