From 5fd6b521df29e9302315a1f77ebdb9616cbb0a81 Mon Sep 17 00:00:00 2001 From: p-wysocki Date: Thu, 16 Nov 2023 13:07:49 +0100 Subject: [PATCH] Initial commit --- .../src/openvino/runtime/opset13/__init__.py | 3 +- .../openvino/op/multi_lstm_sequence.hpp | 176 ++++++++++++++ src/core/include/openvino/op/ops.hpp | 1 + .../include/openvino/opsets/opset13_tbl.hpp | 1 + src/core/src/op/multi_lstm_sequence.cpp | 220 ++++++++++++++++++ src/core/tests/opset.cpp | 2 +- 6 files changed, 401 insertions(+), 2 deletions(-) create mode 100644 src/core/include/openvino/op/multi_lstm_sequence.hpp create mode 100644 src/core/src/op/multi_lstm_sequence.cpp diff --git a/src/bindings/python/src/openvino/runtime/opset13/__init__.py b/src/bindings/python/src/openvino/runtime/opset13/__init__.py index 80741ecd4af..fdf181b58bb 100644 --- a/src/bindings/python/src/openvino/runtime/opset13/__init__.py +++ b/src/bindings/python/src/openvino/runtime/opset13/__init__.py @@ -105,7 +105,8 @@ from openvino.runtime.opset1.ops import maximum from openvino.runtime.opset1.ops import minimum from openvino.runtime.opset4.ops import mish from openvino.runtime.opset1.ops import mod -from openvino.runtime.opset9.ops import multiclass_nms +from openvino.runtime.opset13.ops import multi_lstm_sequence +from openvino.runtime.opset9.ops import from openvino.runtime.opset13.ops import multinomial from openvino.runtime.opset1.ops import multiply from openvino.runtime.opset6.ops import mvn diff --git a/src/core/include/openvino/op/multi_lstm_sequence.hpp b/src/core/include/openvino/op/multi_lstm_sequence.hpp new file mode 100644 index 00000000000..f29b9db58ae --- /dev/null +++ b/src/core/include/openvino/op/multi_lstm_sequence.hpp @@ -0,0 +1,176 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include +#include + +#include "openvino/op/constant.hpp" +#include "openvino/op/lstm_cell.hpp" +#include "openvino/op/util/attr_types.hpp" +#include "openvino/op/util/rnn_cell_base.hpp" + +namespace ov { +namespace op { +namespace v0 { + +/// +/// \brief Class for lstm sequence node. +/// +/// \note It follows notation and equations defined as in ONNX standard: +/// https://github.com/onnx/onnx/blob/master/docs/Operators.md#LSTM +/// +/// \sa LSTMCell, RNNCell, GRUCell +/// +/// +/// \ingroup ov_ops_cpp_api +class OPENVINO_API LSTMSequence : public util::RNNCellBase { +public: + OPENVINO_OP("LSTMSequence", "opset1", util::RNNCellBase); + LSTMSequence() = default; + + using direction = RecurrentSequenceDirection; + + size_t get_default_output_index() const override { + return no_default_index(); + } + explicit LSTMSequence(const Output& X, + const Output& initial_hidden_state, + const Output& initial_cell_state, + const Output& sequence_lengths, + const Output& W, + const Output& R, + const Output& B, + const Output& P, + const std::int64_t hidden_size, + const direction lstm_direction, + LSTMWeightsFormat weights_format = LSTMWeightsFormat::IFCO, + const std::vector activations_alpha = {}, + const std::vector activations_beta = {}, + const std::vector activations = {"sigmoid", "tanh", "tanh"}, + const float clip_threshold = 0, + const bool input_forget = false); + + explicit LSTMSequence(const Output& X, + const Output& initial_hidden_state, + const Output& initial_cell_state, + const Output& sequence_lengths, + const Output& W, + const Output& R, + const Output& B, + const std::int64_t hidden_size, + const direction lstm_direction, + LSTMWeightsFormat weights_format = LSTMWeightsFormat::IFCO, + const std::vector& activations_alpha = {}, + const std::vector& activations_beta = {}, + const std::vector& activations = {"sigmoid", "tanh", "tanh"}, + const float clip_threshold = 0, + const bool input_forget = false); + + void validate_and_infer_types() override; + bool visit_attributes(AttributeVisitor& visitor) override; + + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + + std::vector get_activations_alpha() const { + return m_activations_alpha; + } + std::vector get_activations_beta() const { + return m_activations_beta; + } + std::vector get_activations() const { + return m_activations; + } + float get_clip_threshold() const { + return m_clip; + } + direction get_direction() const { + return m_direction; + } + void set_direction(const direction& dir) { + m_direction = dir; + } + std::int64_t get_hidden_size() const { + return m_hidden_size; + } + bool get_input_forget() const { + return m_input_forget; + } + LSTMWeightsFormat get_weights_format() const { + return m_weights_format; + } + +private: + direction m_direction; + bool m_input_forget; + LSTMWeightsFormat m_weights_format; +}; +} // namespace v0 + +namespace v5 { +/// +/// \brief Class for lstm sequence node. +/// +/// \note It follows notation and equations defined as in ONNX standard: +/// https://github.com/onnx/onnx/blob/master/docs/Operators.md#LSTM +/// +/// \sa LSTMCell, RNNCell, GRUCell +/// +/// +/// \ingroup ov_ops_cpp_api +class OPENVINO_API LSTMSequence : public util::RNNCellBase { +public: + OPENVINO_OP("LSTMSequence", "opset5", util::RNNCellBase); + LSTMSequence() = default; + + using direction = RecurrentSequenceDirection; + + size_t get_default_output_index() const override { + return no_default_index(); + } + explicit LSTMSequence(const Output& X, + const Output& initial_hidden_state, + const Output& initial_cell_state, + const Output& sequence_lengths, + const Output& W, + const Output& R, + const Output& B, + const std::int64_t hidden_size, + const direction lstm_direction, + const std::vector& activations_alpha = {}, + const std::vector& activations_beta = {}, + const std::vector& activations = {"sigmoid", "tanh", "tanh"}, + const float clip = 0.f) + : RNNCellBase({X, initial_hidden_state, initial_cell_state, sequence_lengths, W, R, B}, + hidden_size, + clip, + activations, + activations_alpha, + activations_beta), + m_direction(lstm_direction) { + constructor_validate_and_infer_types(); + } + + void validate_and_infer_types() override; + bool visit_attributes(AttributeVisitor& visitor) override; + + std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + + direction get_direction() const { + return m_direction; + } + void set_direction(const direction& dir) { + m_direction = dir; + } + +private: + direction m_direction{direction::FORWARD}; +}; +} // namespace v5 +} // namespace op +} // namespace ov diff --git a/src/core/include/openvino/op/ops.hpp b/src/core/include/openvino/op/ops.hpp index 5b28762933a..9e3b2527029 100644 --- a/src/core/include/openvino/op/ops.hpp +++ b/src/core/include/openvino/op/ops.hpp @@ -109,6 +109,7 @@ #include "openvino/op/minimum.hpp" #include "openvino/op/mish.hpp" #include "openvino/op/mod.hpp" +#include "openvino/op/multi_lstm_sequence.hpp" #include "openvino/op/multiclass_nms.hpp" #include "openvino/op/multinomial.hpp" #include "openvino/op/multiply.hpp" diff --git a/src/core/include/openvino/opsets/opset13_tbl.hpp b/src/core/include/openvino/opsets/opset13_tbl.hpp index 13b598eaca1..28451789df4 100644 --- a/src/core/include/openvino/opsets/opset13_tbl.hpp +++ b/src/core/include/openvino/opsets/opset13_tbl.hpp @@ -61,6 +61,7 @@ _OPENVINO_OP_REG(MatMul, ov::op::v0) _OPENVINO_OP_REG(Maximum, ov::op::v1) _OPENVINO_OP_REG(Minimum, ov::op::v1) _OPENVINO_OP_REG(Mod, ov::op::v1) +_OPENVINO_OP_REG(yMultiLSTMSequence, ov::op::v13) _OPENVINO_OP_REG(Multiply, ov::op::v1) _OPENVINO_OP_REG(Negative, ov::op::v0) _OPENVINO_OP_REG(NormalizeL2, ov::op::v0) diff --git a/src/core/src/op/multi_lstm_sequence.cpp b/src/core/src/op/multi_lstm_sequence.cpp new file mode 100644 index 00000000000..492f42d051d --- /dev/null +++ b/src/core/src/op/multi_lstm_sequence.cpp @@ -0,0 +1,220 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/op/lstm_sequence.hpp" + +#include "itt.hpp" +#include "lstm_sequence_shape_inference.hpp" +#include "openvino/core/attribute_visitor.hpp" +#include "openvino/op/util/recurrent_sequence.hpp" + +namespace ov { +op::v0::LSTMSequence::LSTMSequence(const Output& X, + const Output& initial_hidden_state, + const Output& initial_cell_state, + const Output& sequence_lengths, + const Output& W, + const Output& R, + const Output& B, + const Output& P, + const std::int64_t hidden_size, + const LSTMSequence::direction lstm_direction, + LSTMWeightsFormat weights_format, + const std::vector activations_alpha, + const std::vector activations_beta, + const std::vector activations, + const float clip_threshold, + const bool input_forget) + : RNNCellBase({X, initial_hidden_state, initial_cell_state, sequence_lengths, W, R, B, P}, + hidden_size, + clip_threshold, + activations, + activations_alpha, + activations_beta), + m_direction(lstm_direction), + m_input_forget(input_forget), + m_weights_format(weights_format) { + constructor_validate_and_infer_types(); +} + +op::v0::LSTMSequence::LSTMSequence(const Output& X, + const Output& initial_hidden_state, + const Output& initial_cell_state, + const Output& sequence_lengths, + const Output& W, + const Output& R, + const Output& B, + const std::int64_t hidden_size, + const LSTMSequence::direction lstm_direction, + LSTMWeightsFormat weights_format, + const std::vector& activations_alpha, + const std::vector& activations_beta, + const std::vector& activations, + const float clip_threshold, + const bool input_forget) + : op::v0::LSTMSequence( + X, + initial_hidden_state, + initial_cell_state, + sequence_lengths, + W, + R, + B, + Constant::create(element::f32, + Shape{(lstm_direction == LSTMSequence::direction::BIDIRECTIONAL ? 2UL : 1UL), + 3UL * static_cast(hidden_size)}, + std::vector{0.f}), + hidden_size, + lstm_direction, + weights_format, + activations_alpha, + activations_beta, + activations, + clip_threshold, + input_forget) {} + +bool op::v0::LSTMSequence::visit_attributes(AttributeVisitor& visitor) { + OV_OP_SCOPE(v0_LSTMSequence_visit_attributes); + visitor.on_attribute("hidden_size", m_hidden_size); + visitor.on_attribute("activations", m_activations); + visitor.on_attribute("activations_alpha", m_activations_alpha); + visitor.on_attribute("activations_beta", m_activations_beta); + visitor.on_attribute("clip", m_clip); + visitor.on_attribute("direction", m_direction); + + visitor.on_attribute("input_forget", m_input_forget); + visitor.on_attribute("weights_format", m_weights_format); + return true; +} + +std::shared_ptr op::v0::LSTMSequence::clone_with_new_inputs(const OutputVector& new_args) const { + OV_OP_SCOPE(v0_LSTMSequence_clone_with_new_inputs); + check_new_args_count(this, new_args); + if (new_args.size() == 8) { + return std::make_shared(new_args.at(0), // X + new_args.at(1), // initial_hidden_state + new_args.at(2), // initial_cell_state + new_args.at(3), // sequence_lengths + new_args.at(4), // W + new_args.at(5), // R + new_args.at(6), // B + new_args.at(7), // P + m_hidden_size, + m_direction, + m_weights_format, + m_activations_alpha, + m_activations_beta, + m_activations, + m_clip, + m_input_forget); + } else if (new_args.size() == 7) { + return std::make_shared(new_args.at(0), // X + new_args.at(1), // initial_hidden_state + new_args.at(2), // initial_cell_state + new_args.at(3), // sequence_lengths + new_args.at(4), // W + new_args.at(5), // R + new_args.at(6), // B + m_hidden_size, + m_direction, + m_weights_format, + m_activations_alpha, + m_activations_beta, + m_activations, + m_clip, + m_input_forget); + } else { + OPENVINO_THROW("Incorrect number of new arguments"); + } +} + +void op::v0::LSTMSequence::validate_and_infer_types() { + OV_OP_SCOPE(v0_LSTMSequence_validate_and_infer_types); + auto result_et = element::dynamic; + + // Validate input types and save result for output type + NODE_VALIDATION_CHECK(this, + element::Type::merge(result_et, result_et, get_input_element_type(0)) && + element::Type::merge(result_et, result_et, get_input_element_type(1)) && + element::Type::merge(result_et, result_et, get_input_element_type(2)) && + element::Type::merge(result_et, result_et, get_input_element_type(4)) && + element::Type::merge(result_et, result_et, get_input_element_type(5)) && + element::Type::merge(result_et, result_et, get_input_element_type(6)), + "Element types for X, initial_hidden_state, initial_cell_state, W, R and B inputs do " + "not match."); + + // Mark inputs which are relevant to output parameters + for (size_t i = 0; i <= 6; ++i) + set_input_is_relevant_to_shape(i); + + OPENVINO_SUPPRESS_DEPRECATED_START + const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END + auto output_shapes = shape_infer(this, input_shapes); + + // Set output size, type and shape + set_output_type(0, result_et, output_shapes[0]); + set_output_type(1, result_et, output_shapes[1]); + set_output_type(2, result_et, output_shapes[2]); +} + +bool op::v5::LSTMSequence::visit_attributes(AttributeVisitor& visitor) { + OV_OP_SCOPE(v5_LSTMSequence_visit_attributes); + visitor.on_attribute("direction", m_direction); + return op::util::RNNCellBase::visit_attributes(visitor); +} + +std::shared_ptr op::v5::LSTMSequence::clone_with_new_inputs(const OutputVector& new_args) const { + OV_OP_SCOPE(v5_LSTMSequence_clone_with_new_inputs); + check_new_args_count(this, new_args); + if (new_args.size() == 7) { + return std::make_shared(new_args.at(0), // X + new_args.at(1), // initial_hidden_state + new_args.at(2), // initial_cell_state + new_args.at(3), // sequence_lengths + new_args.at(4), // W + new_args.at(5), // R + new_args.at(6), // B + m_hidden_size, + m_direction, + m_activations_alpha, + m_activations_beta, + m_activations, + m_clip); + } else { + OPENVINO_THROW("Incorrect number of new arguments"); + } +} + +void op::v5::LSTMSequence::validate_and_infer_types() { + OV_OP_SCOPE(v5_LSTMSequence_validate_and_infer_types); + + auto result_et = element::dynamic; + + // Validate input types and save result for output type + NODE_VALIDATION_CHECK(this, + element::Type::merge(result_et, result_et, get_input_element_type(0)) && + element::Type::merge(result_et, result_et, get_input_element_type(1)) && + element::Type::merge(result_et, result_et, get_input_element_type(2)) && + element::Type::merge(result_et, result_et, get_input_element_type(4)) && + element::Type::merge(result_et, result_et, get_input_element_type(5)) && + element::Type::merge(result_et, result_et, get_input_element_type(6)), + "Element types for X, initial_hidden_state, initial_cell_state, W, R and B inputs do " + "not match."); + + // Mark inputs which are relevant to output parameters + for (size_t i = 0; i <= 6; ++i) + set_input_is_relevant_to_shape(i); + + OPENVINO_SUPPRESS_DEPRECATED_START + const auto input_shapes = get_node_input_partial_shapes(*this); + OPENVINO_SUPPRESS_DEPRECATED_END + auto output_shapes = shape_infer(this, input_shapes); + + // Set output size, type and shape + set_output_type(0, result_et, output_shapes[0]); + set_output_type(1, result_et, output_shapes[1]); + set_output_type(2, result_et, output_shapes[2]); +} +} // namespace ov diff --git a/src/core/tests/opset.cpp b/src/core/tests/opset.cpp index 29e9df2b4da..37eb02e3842 100644 --- a/src/core/tests/opset.cpp +++ b/src/core/tests/opset.cpp @@ -71,7 +71,7 @@ INSTANTIATE_TEST_SUITE_P(opset, OpsetTestParams{ov::get_opset10, 177}, OpsetTestParams{ov::get_opset11, 177}, OpsetTestParams{ov::get_opset12, 178}, - OpsetTestParams{ov::get_opset13, 185}), + OpsetTestParams{ov::get_opset13, 186}), OpsetTestNameGenerator{}); class MyOpOld : public ov::op::Op {