From 93e30468b1346f739e5e95d22109d7aa45648961 Mon Sep 17 00:00:00 2001 From: Ilya Churaev Date: Fri, 14 Jul 2023 10:47:53 +0400 Subject: [PATCH] Change plugin api (#18441) * Remove vector of so objects from Tensor, Any, VariableState and RemoteContext * Change IRemoteContext * Fixed build without Proxy * Moved API to ITensor * Remove friends from Tensor class * Changed query_state API * Remote SoPtr for some returned values * Fixed auto plugin * Add so to converted objects * Fixed build all on macOS * Check that tensor pointers != nullptr * Add SO to converter * Added new constructors for SO ptrs * Changed IVariableState API * Remove proxy friend * Remove friends and nullptrs from auto * Fixed build * Fixed HETERO plugin * Fixed code style --- src/core/dev_api/openvino/runtime/itensor.hpp | 7 + src/core/include/openvino/runtime/tensor.hpp | 41 ++--- src/core/src/runtime/itensor.cpp | 22 ++- src/core/src/runtime/ov_tensor.cpp | 23 +-- .../ie_iexecutable_network_internal.hpp | 2 + .../openvino/runtime/iasync_infer_request.hpp | 11 +- .../openvino/runtime/icompiled_model.hpp | 8 +- .../dev_api/openvino/runtime/icore.hpp | 8 +- .../openvino/runtime/iinfer_request.hpp | 12 +- .../dev_api/openvino/runtime/iplugin.hpp | 8 +- .../openvino/runtime/iremote_context.hpp | 9 +- .../openvino/runtime/isync_infer_request.hpp | 22 +-- .../openvino/runtime/ivariable_state.hpp | 7 +- .../openvino/runtime}/make_tensor.hpp | 42 +++-- .../dev_api/openvino/runtime/so_ptr.hpp | 14 ++ .../openvino/runtime/remote_context.hpp | 28 --- .../openvino/runtime/remote_tensor.hpp | 6 - src/inference/src/compiled_model.cpp | 4 +- src/inference/src/core.cpp | 9 +- src/inference/src/cpp/ie_remote_context.cpp | 6 +- src/inference/src/cpp/ie_variable_state.cpp | 8 +- src/inference/src/dev/converter_utils.cpp | 170 ++++++++++-------- src/inference/src/dev/converter_utils.hpp | 16 +- src/inference/src/dev/core_impl.cpp | 64 ++++--- src/inference/src/dev/core_impl.hpp | 14 +- src/inference/src/dev/core_impl_ie.cpp | 24 +-- .../src/dev/iasync_infer_request.cpp | 10 +- src/inference/src/dev/icompiled_model.cpp | 6 +- .../src/dev/icompiled_model_wrapper.cpp | 4 +- src/inference/src/dev/iplugin_wrapper.cpp | 37 ++-- src/inference/src/dev/iplugin_wrapper.hpp | 11 +- src/inference/src/dev/iremote_context.cpp | 7 +- src/inference/src/dev/iremote_tensor.cpp | 1 - src/inference/src/dev/isync_infer_request.cpp | 90 +++++----- src/inference/src/dev/ivariable_state.cpp | 4 +- src/inference/src/dev/make_tensor.cpp | 73 +++++--- src/inference/src/dev/plugin.cpp | 16 +- src/inference/src/dev/plugin.hpp | 9 +- .../src/dev/remote_context_wrapper.hpp | 9 +- src/inference/src/ie_blob_common.cpp | 2 +- src/inference/src/ie_core.cpp | 10 +- src/inference/src/ie_remote_context.cpp | 4 +- src/inference/src/infer_request.cpp | 17 +- src/inference/src/remote_tensor.cpp | 3 +- .../get_supported_property_test.cpp | 10 +- src/inference/tests/unit/ie_blob_test.cpp | 6 +- src/plugins/auto/src/infer_request.cpp | 47 +++-- src/plugins/auto/src/infer_request.hpp | 6 +- src/plugins/auto/src/plugin.cpp | 9 +- src/plugins/auto/src/plugin.hpp | 8 +- src/plugins/auto/src/schedule.cpp | 6 +- .../auto/tests/unit/include/mock_common.hpp | 6 +- src/plugins/auto/tests/unit/mock_common.cpp | 15 +- src/plugins/hetero/src/plugin.cpp | 15 +- src/plugins/hetero/src/plugin.hpp | 10 +- src/plugins/hetero/src/sync_infer_request.cpp | 19 +- src/plugins/hetero/src/sync_infer_request.hpp | 13 +- src/plugins/hetero/src/variable_state.hpp | 52 ------ .../dev_api/openvino/proxy/infer_request.hpp | 11 +- .../proxy/dev_api/openvino/proxy/plugin.hpp | 4 +- src/plugins/proxy/src/compiled_model.hpp | 2 +- src/plugins/proxy/src/infer_request.cpp | 34 ++-- src/plugins/proxy/src/plugin.cpp | 23 +-- src/plugins/proxy/src/plugin.hpp | 8 +- src/plugins/proxy/src/remote_context.cpp | 42 ++--- src/plugins/proxy/src/remote_context.hpp | 19 +- src/plugins/proxy/src/remote_tensor.cpp | 40 +++-- src/plugins/proxy/src/remote_tensor.hpp | 9 +- src/plugins/proxy/src/variable_state.hpp | 52 ------ src/plugins/proxy/tests/proxy_tests.cpp | 53 +++--- src/plugins/proxy/tests/proxy_tests.hpp | 1 + src/plugins/template/src/compiled_model.cpp | 2 +- src/plugins/template/src/compiled_model.hpp | 2 +- src/plugins/template/src/plugin.cpp | 14 +- src/plugins/template/src/plugin.hpp | 8 +- src/plugins/template/src/remote_context.cpp | 8 +- src/plugins/template/src/remote_context.hpp | 6 +- .../template/src/sync_infer_request.cpp | 35 ++-- .../template/src/sync_infer_request.hpp | 7 +- src/plugins/template/src/variable_state.hpp | 14 +- .../cpp_interfaces/interface/mock_icore.hpp | 11 +- .../mocks/mock_engine/mock_plugin.cpp | 16 +- .../mocks/mock_engine/mock_plugin.hpp | 8 +- 83 files changed, 793 insertions(+), 756 deletions(-) rename src/inference/{src/dev => dev_api/openvino/runtime}/make_tensor.hpp (50%) delete mode 100644 src/plugins/hetero/src/variable_state.hpp delete mode 100644 src/plugins/proxy/src/variable_state.hpp diff --git a/src/core/dev_api/openvino/runtime/itensor.hpp b/src/core/dev_api/openvino/runtime/itensor.hpp index c66c2a15197..46cb96cb9ac 100644 --- a/src/core/dev_api/openvino/runtime/itensor.hpp +++ b/src/core/dev_api/openvino/runtime/itensor.hpp @@ -69,6 +69,13 @@ public: return static_cast(data(element::from())); } + /** + * @brief Reports whether the tensor is continuous or not + * + * @return true if tensor is continuous + */ + bool is_continuous() const; + protected: virtual ~ITensor(); }; diff --git a/src/core/include/openvino/runtime/tensor.hpp b/src/core/include/openvino/runtime/tensor.hpp index 3b8ba269282..9c20cd41f43 100644 --- a/src/core/include/openvino/runtime/tensor.hpp +++ b/src/core/include/openvino/runtime/tensor.hpp @@ -17,32 +17,20 @@ #include "openvino/core/type/element_type.hpp" #include "openvino/runtime/allocator.hpp" -namespace InferenceEngine { -class IAsyncInferRequestWrapper; -class IVariableStateWrapper; -} // namespace InferenceEngine - namespace ov { -class Core; -class CoreImpl; -class InferRequest; -class RemoteContext; -class VariableState; -class ISyncInferRequest; -class IInferRequestInternalWrapper; -class IVariableStateInternalWrapper; +class Tensor; class ITensor; -class RemoteTensor; -namespace proxy { -class RemoteContext; -} +namespace util { +ov::Tensor make_tensor(const std::shared_ptr& tensor, const std::shared_ptr& so); +void get_tensor_impl(const ov::Tensor& tensor, std::shared_ptr& tensor_impl, std::shared_ptr& so); +} // namespace util namespace op { namespace util { class VariableValue; -} +} // namespace util } // namespace op /** @@ -63,19 +51,12 @@ protected: */ Tensor(const std::shared_ptr& impl, const std::shared_ptr& so); - friend class ov::Core; - friend class ov::CoreImpl; - friend class ov::InferRequest; - friend class ov::RemoteTensor; - friend class ov::RemoteContext; - friend class ov::VariableState; - friend class ov::ISyncInferRequest; - friend class ov::IInferRequestInternalWrapper; - friend class ov::IVariableStateInternalWrapper; - friend class ov::proxy::RemoteContext; - friend class InferenceEngine::IAsyncInferRequestWrapper; - friend class InferenceEngine::IVariableStateWrapper; friend class ov::op::util::VariableValue; + friend ov::Tensor ov::util::make_tensor(const std::shared_ptr& tensor, + const std::shared_ptr& so); + friend void ov::util::get_tensor_impl(const ov::Tensor& tensor, + std::shared_ptr& tensor_impl, + std::shared_ptr& so); public: /// @brief Default constructor diff --git a/src/core/src/runtime/itensor.cpp b/src/core/src/runtime/itensor.cpp index 4800948c1a0..81fed73e889 100644 --- a/src/core/src/runtime/itensor.cpp +++ b/src/core/src/runtime/itensor.cpp @@ -4,7 +4,6 @@ #include "openvino/runtime/itensor.hpp" -#include "dev/make_tensor.hpp" #include "openvino/core/except.hpp" #include "openvino/runtime/allocator.hpp" #include "openvino/runtime/properties.hpp" @@ -21,4 +20,25 @@ size_t ITensor::get_byte_size() const { return (get_size() * get_element_type().bitwidth() + 8 - 1) / 8; } +bool ITensor::is_continuous() const { + if (get_element_type().bitwidth() < 8) + // OpenVINO doesn't support strides for lp types + return true; + const auto& shape = get_shape(); + const auto& type = get_element_type(); + std::vector strides(shape.size()); + if (!shape.empty()) { + strides[shape.size() - 1] = 1; + } + auto size = shape.size(); + for (size_t i = 1; i < size; i++) { + strides[size - i - 1] = strides[size - i] * shape[size - i]; + } + + ov::Strides byte_strides(strides.size()); + for (size_t i = 0; i < strides.size(); ++i) + byte_strides[i] = strides[i] * type.size(); + return byte_strides == get_strides(); +} + } // namespace ov diff --git a/src/core/src/runtime/ov_tensor.cpp b/src/core/src/runtime/ov_tensor.cpp index e686338ca2f..c963c6239c6 100644 --- a/src/core/src/runtime/ov_tensor.cpp +++ b/src/core/src/runtime/ov_tensor.cpp @@ -4,12 +4,12 @@ #include -#include "dev/make_tensor.hpp" #include "openvino/core/except.hpp" #include "openvino/core/node_output.hpp" #include "openvino/core/shape.hpp" #include "openvino/core/strides.hpp" #include "openvino/runtime/itensor.hpp" +#include "openvino/runtime/make_tensor.hpp" #include "openvino/runtime/remote_tensor.hpp" #include "openvino/runtime/tensor.hpp" #include "shape_util.hpp" @@ -230,26 +230,7 @@ Tensor::operator bool() const noexcept { } bool Tensor::is_continuous() const { - OV_TENSOR_STATEMENT({ - if (get_element_type().bitwidth() < 8) - // OpenVINO doesn't support strides for lp types - return true; - const auto& shape = get_shape(); - const auto& type = get_element_type(); - std::vector strides(shape.size()); - if (!shape.empty()) { - strides[shape.size() - 1] = 1; - } - auto size = shape.size(); - for (size_t i = 1; i < size; i++) { - strides[size - i - 1] = strides[size - i] * shape[size - i]; - } - - ov::Strides byte_strides(strides.size()); - for (size_t i = 0; i < strides.size(); ++i) - byte_strides[i] = strides[i] * type.size(); - return byte_strides == get_strides(); - }); + OV_TENSOR_STATEMENT(return _impl->is_continuous()); } } // namespace ov diff --git a/src/inference/dev_api/cpp_interfaces/interface/ie_iexecutable_network_internal.hpp b/src/inference/dev_api/cpp_interfaces/interface/ie_iexecutable_network_internal.hpp index 11822600815..e488cd406c8 100644 --- a/src/inference/dev_api/cpp_interfaces/interface/ie_iexecutable_network_internal.hpp +++ b/src/inference/dev_api/cpp_interfaces/interface/ie_iexecutable_network_internal.hpp @@ -27,6 +27,7 @@ class Result; namespace InferenceEngine { class IInferencePlugin; +class IPluginWrapper; class IInferRequestInternal; class RemoteContext; class IVariableStateInternal; @@ -216,6 +217,7 @@ protected: bool _loadedFromCache = false; friend InferenceEngine::ICompiledModelWrapper; + friend InferenceEngine::IPluginWrapper; }; /** diff --git a/src/inference/dev_api/openvino/runtime/iasync_infer_request.hpp b/src/inference/dev_api/openvino/runtime/iasync_infer_request.hpp index 84705d082ac..fa94e376a48 100644 --- a/src/inference/dev_api/openvino/runtime/iasync_infer_request.hpp +++ b/src/inference/dev_api/openvino/runtime/iasync_infer_request.hpp @@ -90,7 +90,7 @@ public: * @param port Port of the tensor to get. * @return Tensor for the port @p port. */ - ov::Tensor get_tensor(const ov::Output& port) const override; + ov::SoPtr get_tensor(const ov::Output& port) const override; /** * @brief Sets an input/output tensor to infer. @@ -98,7 +98,7 @@ public: * @param tensor Reference to a tensor. The element_type and shape of a tensor must match * the model's input/output element_type and size. */ - void set_tensor(const ov::Output& port, const ov::Tensor& tensor) override; + void set_tensor(const ov::Output& port, const ov::SoPtr& tensor) override; /** * @brief Gets a batch of tensors for input data to infer by input port. @@ -111,7 +111,7 @@ public: * input element type and shape (except batch dimension). Total size of tensors must match the input size. * @return vector of tensors */ - std::vector get_tensors(const ov::Output& port) const override; + std::vector> get_tensors(const ov::Output& port) const override; /** * @brief Sets a batch of tensors for input data to infer by input port. * Model input must have batch dimension, and the number of @p tensors must match the batch size. @@ -122,7 +122,8 @@ public: * @param tensors Input tensors for batched infer request. The type of each tensor must match the model * input element type and shape (except batch dimension). Total size of tensors must match the input size. */ - void set_tensors(const ov::Output& port, const std::vector& tensors) override; + void set_tensors(const ov::Output& port, + const std::vector>& tensors) override; /** * @brief Gets state control interface for the given infer request. @@ -130,7 +131,7 @@ public: * State control essential for recurrent models. * @return Vector of Variable State objects. */ - std::vector> query_state() const override; + std::vector> query_state() const override; /** * @brief Gets pointer to compiled model (usually synchronous request holds the compiled model) diff --git a/src/inference/dev_api/openvino/runtime/icompiled_model.hpp b/src/inference/dev_api/openvino/runtime/icompiled_model.hpp index fa8e6eec7c4..9cbe08b7d54 100644 --- a/src/inference/dev_api/openvino/runtime/icompiled_model.hpp +++ b/src/inference/dev_api/openvino/runtime/icompiled_model.hpp @@ -15,8 +15,10 @@ #include "openvino/core/node_output.hpp" #include "openvino/runtime/common.hpp" +#include "openvino/runtime/iremote_context.hpp" #include "openvino/runtime/isync_infer_request.hpp" #include "openvino/runtime/remote_context.hpp" +#include "openvino/runtime/so_ptr.hpp" #include "openvino/runtime/threading/cpu_streams_executor.hpp" #include "openvino/runtime/threading/itask_executor.hpp" @@ -72,7 +74,7 @@ public: ICompiledModel( const std::shared_ptr& model, const std::shared_ptr& plugin, - const ov::RemoteContext& context, + const ov::SoPtr& context, const std::shared_ptr& task_executor = std::make_shared(ov::threading::IStreamsExecutor::Config{"Default"}), const std::shared_ptr& callback_executor = @@ -134,7 +136,7 @@ public: * * @return OpenVINO RemoteContext */ - std::shared_ptr get_context() const; + ov::SoPtr get_context() const; virtual ~ICompiledModel() = default; @@ -142,7 +144,7 @@ private: std::shared_ptr m_plugin; std::vector> m_inputs; std::vector> m_outputs; - ov::RemoteContext m_context; + ov::SoPtr m_context; std::shared_ptr m_task_executor = nullptr; //!< Holds a task executor std::shared_ptr m_callback_executor = nullptr; //!< Holds a callback executor diff --git a/src/inference/dev_api/openvino/runtime/icore.hpp b/src/inference/dev_api/openvino/runtime/icore.hpp index e7fd6b2bfe3..e4d0a98f5be 100644 --- a/src/inference/dev_api/openvino/runtime/icore.hpp +++ b/src/inference/dev_api/openvino/runtime/icore.hpp @@ -80,7 +80,7 @@ public: * @return A pointer to compiled model */ virtual ov::SoPtr compile_model(const std::shared_ptr& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& config = {}) const = 0; /** @@ -138,7 +138,7 @@ public: * @return A pointer to compiled model */ virtual ov::SoPtr import_model(std::istream& modelStream, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& config = {}) const = 0; /** @@ -168,7 +168,7 @@ public: * @param params Map of device-specific shared context parameters. * @return A shared pointer to a created remote context. */ - virtual ov::RemoteContext create_context(const std::string& device_name, const AnyMap& args) const = 0; + virtual ov::SoPtr create_context(const std::string& device_name, const AnyMap& args) const = 0; virtual bool is_new_api() const = 0; @@ -177,7 +177,7 @@ public: * @param device_name - A name of a device to get create shared context from. * @return A shared pointer to a default remote context. */ - virtual ov::RemoteContext get_default_context(const std::string& device_name) const = 0; + virtual ov::SoPtr get_default_context(const std::string& device_name) const = 0; /** * @brief Gets properties related to device behaviour. diff --git a/src/inference/dev_api/openvino/runtime/iinfer_request.hpp b/src/inference/dev_api/openvino/runtime/iinfer_request.hpp index c0ec9ee81f5..7ce1270d9c8 100644 --- a/src/inference/dev_api/openvino/runtime/iinfer_request.hpp +++ b/src/inference/dev_api/openvino/runtime/iinfer_request.hpp @@ -17,6 +17,7 @@ #include "openvino/runtime/common.hpp" #include "openvino/runtime/ivariable_state.hpp" #include "openvino/runtime/profiling_info.hpp" +#include "openvino/runtime/so_ptr.hpp" #include "openvino/runtime/tensor.hpp" namespace ov { @@ -51,7 +52,7 @@ public: * @param port Port of the tensor to get. * @return Tensor for the port @p port. */ - virtual ov::Tensor get_tensor(const ov::Output& port) const = 0; + virtual ov::SoPtr get_tensor(const ov::Output& port) const = 0; /** * @brief Sets an input/output tensor to infer. @@ -59,7 +60,7 @@ public: * @param tensor Reference to a tensor. The element_type and shape of a tensor must match * the model's input/output element_type and size. */ - virtual void set_tensor(const ov::Output& port, const ov::Tensor& tensor) = 0; + virtual void set_tensor(const ov::Output& port, const ov::SoPtr& tensor) = 0; /** * @brief Gets a batch of tensors for input data to infer by input port. @@ -72,7 +73,7 @@ public: * input element type and shape (except batch dimension). Total size of tensors must match the input size. * @return vector of tensors */ - virtual std::vector get_tensors(const ov::Output& port) const = 0; + virtual std::vector> get_tensors(const ov::Output& port) const = 0; /** * @brief Sets a batch of tensors for input data to infer by input port. @@ -84,7 +85,8 @@ public: * @param tensors Input tensors for batched infer request. The type of each tensor must match the model * input element type and shape (except batch dimension). Total size of tensors must match the input size. */ - virtual void set_tensors(const ov::Output& port, const std::vector& tensors) = 0; + virtual void set_tensors(const ov::Output& port, + const std::vector>& tensors) = 0; /** * @brief Gets state control interface for the given infer request. @@ -92,7 +94,7 @@ public: * State control essential for recurrent models. * @return Vector of Variable State objects. */ - virtual std::vector> query_state() const = 0; + virtual std::vector> query_state() const = 0; /** * @brief Gets pointer to compiled model (usually synchronous request holds the compiled model) diff --git a/src/inference/dev_api/openvino/runtime/iplugin.hpp b/src/inference/dev_api/openvino/runtime/iplugin.hpp index 4b2ee3bfa90..e994bd7cab0 100644 --- a/src/inference/dev_api/openvino/runtime/iplugin.hpp +++ b/src/inference/dev_api/openvino/runtime/iplugin.hpp @@ -133,7 +133,7 @@ public: */ virtual std::shared_ptr compile_model(const std::shared_ptr& model, const ov::AnyMap& properties, - const ov::RemoteContext& context) const = 0; + const ov::SoPtr& context) const = 0; /** * @brief Sets properties for plugin, acceptable keys can be found in openvino/runtime/properties.hpp @@ -157,7 +157,7 @@ public: * * @return A remote context object */ - virtual std::shared_ptr create_context(const ov::AnyMap& remote_properties) const = 0; + virtual ov::SoPtr create_context(const ov::AnyMap& remote_properties) const = 0; /** * @brief Provides a default remote context instance if supported by a plugin @@ -165,7 +165,7 @@ public: * * @return The default context. */ - virtual std::shared_ptr get_default_context(const ov::AnyMap& remote_properties) const = 0; + virtual ov::SoPtr get_default_context(const ov::AnyMap& remote_properties) const = 0; /** * @brief Creates an compiled model from an previously exported model using plugin implementation @@ -187,7 +187,7 @@ public: * @return An Compiled model */ virtual std::shared_ptr import_model(std::istream& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& properties) const = 0; /** diff --git a/src/inference/dev_api/openvino/runtime/iremote_context.hpp b/src/inference/dev_api/openvino/runtime/iremote_context.hpp index 9fddfae1e1d..27e7256dbf6 100644 --- a/src/inference/dev_api/openvino/runtime/iremote_context.hpp +++ b/src/inference/dev_api/openvino/runtime/iremote_context.hpp @@ -16,6 +16,7 @@ #include "openvino/core/type/element_type.hpp" #include "openvino/runtime/common.hpp" #include "openvino/runtime/iremote_tensor.hpp" +#include "openvino/runtime/so_ptr.hpp" namespace ov { @@ -50,9 +51,9 @@ public: * @param params Map of the low-level tensor object parameters. * @return Pointer to a plugin object that implements the RemoteTensor interface. */ - virtual std::shared_ptr create_tensor(const ov::element::Type& type, - const ov::Shape& shape, - const ov::AnyMap& params = {}) = 0; + virtual ov::SoPtr create_tensor(const ov::element::Type& type, + const ov::Shape& shape, + const ov::AnyMap& params = {}) = 0; /** * @brief This method is used to create a host tensor object friendly for the device in current context. @@ -62,7 +63,7 @@ public: * @param shape Tensor shape. * @return A tensor instance with device friendly memory. */ - virtual std::shared_ptr create_host_tensor(const ov::element::Type type, const ov::Shape& shape); + virtual ov::SoPtr create_host_tensor(const ov::element::Type type, const ov::Shape& shape); }; } // namespace ov diff --git a/src/inference/dev_api/openvino/runtime/isync_infer_request.hpp b/src/inference/dev_api/openvino/runtime/isync_infer_request.hpp index 989205c404a..2d32c1f036f 100644 --- a/src/inference/dev_api/openvino/runtime/isync_infer_request.hpp +++ b/src/inference/dev_api/openvino/runtime/isync_infer_request.hpp @@ -41,7 +41,7 @@ public: * @param port Port of the tensor to get. * @return Tensor for the port @p port. */ - ov::Tensor get_tensor(const ov::Output& port) const override; + ov::SoPtr get_tensor(const ov::Output& port) const override; /** * @brief Sets an input/output tensor to infer. @@ -49,7 +49,7 @@ public: * @param tensor Reference to a tensor. The element_type and shape of a tensor must match * the model's input/output element_type and size. */ - void set_tensor(const ov::Output& port, const ov::Tensor& tensor) override; + void set_tensor(const ov::Output& port, const ov::SoPtr& tensor) override; /** * @brief Gets a batch of tensors for input data to infer by input port. @@ -62,7 +62,7 @@ public: * input element type and shape (except batch dimension). Total size of tensors must match the input size. * @return vector of tensors */ - std::vector get_tensors(const ov::Output& port) const override; + std::vector> get_tensors(const ov::Output& port) const override; /** * @brief Sets a batch of tensors for input data to infer by input port. @@ -74,7 +74,8 @@ public: * @param tensors Input tensors for batched infer request. The type of each tensor must match the model * input element type and shape (except batch dimension). Total size of tensors must match the input size. */ - void set_tensors(const ov::Output& port, const std::vector& tensors) override; + void set_tensors(const ov::Output& port, + const std::vector>& tensors) override; /** * @brief Plugin implementation for set tensors @@ -83,7 +84,8 @@ public: * @param tensors Input tensors for batched infer request. The type of each tensor must match the model * input element type and shape (except batch dimension). Total size of tensors must match the input size. */ - virtual void set_tensors_impl(const ov::Output port, const std::vector& tensors); + virtual void set_tensors_impl(const ov::Output port, + const std::vector>& tensors); /** * @brief Gets inputs for infer request @@ -132,7 +134,7 @@ protected: * @param port Input/Output port * @param tensor Input/Output tensor */ - void check_tensor(const ov::Output& port, const ov::Tensor& tensor) const; + void check_tensor(const ov::Output& port, const ov::SoPtr& tensor) const; /** * @brief Check that all tensors are valid. Throws an exception if it's not. @@ -146,15 +148,15 @@ protected: * @param allocate_callback function which allocates the tensor */ void allocate_tensor(const ov::Output& port, - const std::function& allocate_callback); + const std::function& tensor)>& allocate_callback); - std::unordered_map, std::vector> m_batched_tensors; + std::unordered_map, std::vector>> m_batched_tensors; private: std::shared_ptr m_compiled_model; // Mutable to return reference to ov::Tensor - mutable std::unordered_map, ov::Tensor> m_tensors; - ov::Tensor& get_ref_tensor(const ov::Output& port) const; + mutable std::unordered_map, ov::SoPtr> m_tensors; + ov::SoPtr& get_tensor_ptr(const ov::Output& port) const; /** * @brief Finds input or output port diff --git a/src/inference/dev_api/openvino/runtime/ivariable_state.hpp b/src/inference/dev_api/openvino/runtime/ivariable_state.hpp index b2dea052060..c25219ec4a8 100644 --- a/src/inference/dev_api/openvino/runtime/ivariable_state.hpp +++ b/src/inference/dev_api/openvino/runtime/ivariable_state.hpp @@ -13,6 +13,7 @@ #include #include "openvino/runtime/common.hpp" +#include "openvino/runtime/so_ptr.hpp" #include "openvino/runtime/tensor.hpp" namespace ov { @@ -42,13 +43,13 @@ public: * @brief Sets the new state for the next inference * @param newState A new state */ - virtual void set_state(const ov::Tensor& state); + virtual void set_state(const ov::SoPtr& state); /** * @brief Returns the value of the variable state. * @return The value of the variable state */ - virtual const ov::Tensor& get_state() const; + virtual const ov::SoPtr& get_state() const; protected: /** @@ -57,7 +58,7 @@ protected: virtual ~IVariableState(); std::string m_name; - ov::Tensor m_state; + ov::SoPtr m_state; }; } // namespace ov diff --git a/src/inference/src/dev/make_tensor.hpp b/src/inference/dev_api/openvino/runtime/make_tensor.hpp similarity index 50% rename from src/inference/src/dev/make_tensor.hpp rename to src/inference/dev_api/openvino/runtime/make_tensor.hpp index 10f35ea66c8..64c7ccee965 100644 --- a/src/inference/src/dev/make_tensor.hpp +++ b/src/inference/dev_api/openvino/runtime/make_tensor.hpp @@ -5,7 +5,9 @@ #pragma once #include "ie_blob.h" +#include "openvino/runtime/common.hpp" #include "openvino/runtime/itensor.hpp" +#include "openvino/runtime/so_ptr.hpp" namespace ov { @@ -15,7 +17,9 @@ namespace ov { * @param shape Tensor shape * @param allocator allocates memory for internal tensor storage */ -std::shared_ptr make_tensor(const element::Type type, const Shape& shape, const Allocator& allocator = {}); +OPENVINO_RUNTIME_API std::shared_ptr make_tensor(const element::Type type, + const Shape& shape, + const Allocator& allocator = {}); /** * @brief Constructs Tensor using element type and shape. Wraps allocated host memory. @@ -26,10 +30,10 @@ std::shared_ptr make_tensor(const element::Type type, const Shape& shap * @param strides Optional strides parameters in bytes. Strides are supposed to be computed automatically based * on shape and element size */ -std::shared_ptr make_tensor(const element::Type type, - const Shape& shape, - void* host_ptr, - const Strides& strides = {}); +OPENVINO_RUNTIME_API std::shared_ptr make_tensor(const element::Type type, + const Shape& shape, + void* host_ptr, + const Strides& strides = {}); /** * @brief Constructs region of interest (ROI) tensor form another tensor. @@ -39,17 +43,35 @@ std::shared_ptr make_tensor(const element::Type type, * @param end end coordinate of ROI object inside of the original object. * @note A Number of dimensions in `begin` and `end` must match number of dimensions in `other.get_shape()` */ -std::shared_ptr make_tensor(const std::shared_ptr& other, - const Coordinate& begin, - const Coordinate& end); +OPENVINO_RUNTIME_API std::shared_ptr make_tensor(const std::shared_ptr& other, + const Coordinate& begin, + const Coordinate& end); + +/** + * @brief Constructs public ov::Tensor class + * + * @param tensor Tensor implementation + * + * @return OpenVINO Tensor + */ +OPENVINO_RUNTIME_API ov::Tensor make_tensor(const ov::SoPtr& tensor); + +/** + * @brief Returns tensor implementation + * + * @param tensor OpenVINO Tensor + * + * @return SoPtr to ITensor + */ +OPENVINO_RUNTIME_API ov::SoPtr get_tensor_impl(const ov::Tensor& tensor); IE_SUPPRESS_DEPRECATED_START /** @cond INTERNAL */ -std::shared_ptr make_tensor(const std::shared_ptr& tensor); +ov::SoPtr make_tensor(const std::shared_ptr& tensor); const InferenceEngine::Blob* get_hardware_blob(const InferenceEngine::Blob* blob); InferenceEngine::Blob* get_hardware_blob(InferenceEngine::Blob* blob); -std::shared_ptr tensor_to_blob(const std::shared_ptr& tensor, bool unwrap = true); +std::shared_ptr tensor_to_blob(const ov::SoPtr& tensor, bool unwrap = true); /** @endcond */ IE_SUPPRESS_DEPRECATED_END diff --git a/src/inference/dev_api/openvino/runtime/so_ptr.hpp b/src/inference/dev_api/openvino/runtime/so_ptr.hpp index d7d6adba197..ce3c6792422 100644 --- a/src/inference/dev_api/openvino/runtime/so_ptr.hpp +++ b/src/inference/dev_api/openvino/runtime/so_ptr.hpp @@ -43,6 +43,20 @@ struct SoPtr { */ SoPtr(const std::shared_ptr& ptr, const std::shared_ptr& so) : _ptr{ptr}, _so{so} {} + /** + * @brief Constructs an object with existing shared object reference + * @param ptr pointer to the loaded object + */ + SoPtr(const std::shared_ptr& ptr) : _ptr{ptr}, _so{nullptr} {} + + /** + * @brief Constructs an object with existing shared object reference + * @param ptr pointer to the loaded object + */ + template ::value, bool>::type = true> + SoPtr(const std::shared_ptr& ptr) : _ptr{std::dynamic_pointer_cast(ptr)}, + _so{nullptr} {} + /** * @brief The copy-like constructor, can create So Pointer that dereferenced into child type if T is derived of U * @param that copied SoPtr object diff --git a/src/inference/include/openvino/runtime/remote_context.hpp b/src/inference/include/openvino/runtime/remote_context.hpp index 7dc5cc5e7ce..9c56c8d9e46 100644 --- a/src/inference/include/openvino/runtime/remote_context.hpp +++ b/src/inference/include/openvino/runtime/remote_context.hpp @@ -18,29 +18,12 @@ #include "openvino/runtime/properties.hpp" #include "openvino/runtime/remote_tensor.hpp" -namespace InferenceEngine { -class IPluginWrapper; -class ICompiledModelWrapper; -class Core; -} // namespace InferenceEngine - namespace ov { class Core; -class CoreImpl; -class Plugin; -class IPlugin; class IRemoteContext; -class ISyncInferRequest; -class IInferencePluginWrapper; -class IExecutableNetworkWrapper; -class ICompiledModel; class CompiledModel; -namespace proxy { -class RemoteContext; -} - /** * @brief This class represents an abstraction * @ingroup ov_runtime_cpp_api @@ -60,19 +43,8 @@ protected: * object is destroyed. */ RemoteContext(const std::shared_ptr& impl, const std::shared_ptr& so); - friend class InferenceEngine::Core; - friend class InferenceEngine::IPluginWrapper; - friend class InferenceEngine::ICompiledModelWrapper; friend class ov::Core; - friend class ov::CoreImpl; - friend class ov::Plugin; - friend class ov::IPlugin; - friend class ov::ISyncInferRequest; - friend class ov::IInferencePluginWrapper; - friend class ov::IExecutableNetworkWrapper; - friend class ov::ICompiledModel; friend class ov::CompiledModel; - friend class ov::proxy::RemoteContext; public: /** diff --git a/src/inference/include/openvino/runtime/remote_tensor.hpp b/src/inference/include/openvino/runtime/remote_tensor.hpp index 78a07b7c066..7323ad3ea4a 100644 --- a/src/inference/include/openvino/runtime/remote_tensor.hpp +++ b/src/inference/include/openvino/runtime/remote_tensor.hpp @@ -14,10 +14,6 @@ namespace ov { -namespace proxy { -class RemoteTensor; -} - class RemoteContext; /** @@ -26,8 +22,6 @@ class RemoteContext; */ class OPENVINO_RUNTIME_API RemoteTensor : public Tensor { using Tensor::Tensor; - friend class ov::RemoteContext; - friend class ov::proxy::RemoteTensor; public: /** diff --git a/src/inference/src/compiled_model.cpp b/src/inference/src/compiled_model.cpp index aa7f9d2d707..c51f5402ef4 100644 --- a/src/inference/src/compiled_model.cpp +++ b/src/inference/src/compiled_model.cpp @@ -121,7 +121,9 @@ Any CompiledModel::get_property(const std::string& name) const { RemoteContext CompiledModel::get_context() const { OV_COMPILED_MODEL_CALL_STATEMENT({ auto ctx = _impl->get_context(); - return {ctx, {_so}}; + if (!ctx._so) + ctx._so = _so; + return {ctx._ptr, ctx._so}; }); } diff --git a/src/inference/src/core.cpp b/src/inference/src/core.cpp index 2b2ed684682..42a7c7c121a 100644 --- a/src/inference/src/core.cpp +++ b/src/inference/src/core.cpp @@ -11,6 +11,7 @@ #include "ie_itt.hpp" #include "openvino/core/so_extension.hpp" #include "openvino/runtime/device_id_parser.hpp" +#include "openvino/runtime/iremote_context.hpp" namespace { std::string resolve_extension_path(const std::string& path) { @@ -156,7 +157,7 @@ CompiledModel Core::compile_model(const std::shared_ptr& model, const RemoteContext& context, const AnyMap& config) { OV_CORE_CALL_STATEMENT({ - auto exec = _impl->compile_model(model, context, config); + auto exec = _impl->compile_model(model, ov::SoPtr{context._impl, context._so}, config); return {exec._ptr, exec._so}; }); } @@ -222,7 +223,7 @@ CompiledModel Core::import_model(std::istream& modelStream, const RemoteContext& OV_ITT_SCOPED_TASK(ov::itt::domains::IE, "Core::import_model"); OV_CORE_CALL_STATEMENT({ - auto exec = _impl->import_model(modelStream, context, config); + auto exec = _impl->import_model(modelStream, ov::SoPtr{context._impl, context._so}, config); return {exec._ptr, exec._so}; }); } @@ -279,7 +280,7 @@ RemoteContext Core::create_context(const std::string& device_name, const AnyMap& OV_CORE_CALL_STATEMENT({ auto parsed = parseDeviceNameIntoConfig(device_name, params); auto remoteContext = _impl->get_plugin(parsed._deviceName).create_context(parsed._config); - return {remoteContext._impl, {remoteContext._so}}; + return {remoteContext._ptr, remoteContext._so}; }); } @@ -292,7 +293,7 @@ RemoteContext Core::get_default_context(const std::string& device_name) { OV_CORE_CALL_STATEMENT({ auto parsed = parseDeviceNameIntoConfig(device_name, AnyMap{}); auto remoteContext = _impl->get_plugin(parsed._deviceName).get_default_context(parsed._config); - return {remoteContext._impl, {remoteContext._so}}; + return {remoteContext._ptr, remoteContext._so}; }); } diff --git a/src/inference/src/cpp/ie_remote_context.cpp b/src/inference/src/cpp/ie_remote_context.cpp index 2c97a66bb14..b23d6735f26 100644 --- a/src/inference/src/cpp/ie_remote_context.cpp +++ b/src/inference/src/cpp/ie_remote_context.cpp @@ -7,12 +7,12 @@ #include #include "any_copy.hpp" -#include "dev/make_tensor.hpp" #include "ie_ngraph_utils.hpp" #include "ie_remote_blob.hpp" #include "openvino/core/except.hpp" #include "openvino/runtime/iremote_context.hpp" #include "openvino/runtime/itensor.hpp" +#include "openvino/runtime/make_tensor.hpp" #include "openvino/runtime/remote_context.hpp" #define OV_REMOTE_CONTEXT_STATEMENT(...) \ @@ -71,14 +71,14 @@ std::string RemoteContext::get_device_name() const { RemoteTensor RemoteContext::create_tensor(const element::Type& type, const Shape& shape, const AnyMap& params) { OV_REMOTE_CONTEXT_STATEMENT({ auto tensor = _impl->create_tensor(type, shape, params); - return {tensor, {_so}}; + return make_tensor(tensor).as(); }); } Tensor RemoteContext::create_host_tensor(const element::Type element_type, const Shape& shape) { OV_REMOTE_CONTEXT_STATEMENT({ auto tensor = _impl->create_host_tensor(element_type, shape); - return {tensor, {_so}}; + return make_tensor(tensor); }); } diff --git a/src/inference/src/cpp/ie_variable_state.cpp b/src/inference/src/cpp/ie_variable_state.cpp index 4801fb04352..e89b7e3413a 100644 --- a/src/inference/src/cpp/ie_variable_state.cpp +++ b/src/inference/src/cpp/ie_variable_state.cpp @@ -6,6 +6,7 @@ #include "cpp_interfaces/interface/ie_ivariable_state_internal.hpp" #include "openvino/core/except.hpp" #include "openvino/runtime/ivariable_state.hpp" +#include "openvino/runtime/make_tensor.hpp" #include "openvino/runtime/variable_state.hpp" IE_SUPPRESS_DEPRECATED_START @@ -81,11 +82,14 @@ std::string VariableState::get_name() const { } Tensor VariableState::get_state() const { - OV_VARIABLE_CALL_STATEMENT(return _impl->get_state()); + OV_VARIABLE_CALL_STATEMENT({ + auto tensor = _impl->get_state(); + return make_tensor(tensor); + }); } void VariableState::set_state(const Tensor& state) { - OV_VARIABLE_CALL_STATEMENT(_impl->set_state(state)); + OV_VARIABLE_CALL_STATEMENT(_impl->set_state(get_tensor_impl(state))); } } // namespace ov diff --git a/src/inference/src/dev/converter_utils.cpp b/src/inference/src/dev/converter_utils.cpp index 3eedfffd08f..405225e4630 100644 --- a/src/inference/src/dev/converter_utils.cpp +++ b/src/inference/src/dev/converter_utils.cpp @@ -13,7 +13,6 @@ #include "cpp_interfaces/interface/ie_iexecutable_network_internal.hpp" #include "cpp_interfaces/interface/ie_iplugin_internal.hpp" #include "cpp_interfaces/interface/ie_ivariable_state_internal.hpp" -#include "dev/make_tensor.hpp" #include "icompiled_model_wrapper.hpp" #include "ie_blob.h" #include "ie_common.h" @@ -31,8 +30,10 @@ #include "openvino/runtime/icompiled_model.hpp" #include "openvino/runtime/iinfer_request.hpp" #include "openvino/runtime/iplugin.hpp" +#include "openvino/runtime/iremote_context.hpp" #include "openvino/runtime/itensor.hpp" #include "openvino/runtime/ivariable_state.hpp" +#include "openvino/runtime/make_tensor.hpp" #include "openvino/runtime/profiling_info.hpp" #include "openvino/runtime/remote_context.hpp" #include "openvino/runtime/so_ptr.hpp" @@ -197,10 +198,10 @@ std::shared_ptr ov::legacy_convert::convert_model(const Inferen namespace ov { class IVariableStateInternalWrapper : public InferenceEngine::IVariableStateInternal { - std::shared_ptr m_state; + ov::SoPtr m_state; public: - IVariableStateInternalWrapper(const std::shared_ptr& state) + IVariableStateInternalWrapper(const ov::SoPtr& state) : InferenceEngine::IVariableStateInternal(state->get_name()), m_state(state) {} @@ -213,17 +214,17 @@ public: } void SetState(const InferenceEngine::Blob::Ptr& newState) override { - m_state->set_state(ov::Tensor(ov::make_tensor(newState), {})); + m_state->set_state(ov::make_tensor(newState)); } InferenceEngine::Blob::CPtr GetState() const override { - return tensor_to_blob(m_state->get_state()._impl); + return tensor_to_blob(m_state->get_state()); } }; class IInferencePluginWrapper : public InferenceEngine::IInferencePlugin { public: - IInferencePluginWrapper(const std::shared_ptr& plugin) : m_plugin(plugin) { + IInferencePluginWrapper(const ov::SoPtr& plugin) : m_plugin(plugin) { auto& ver = plugin->get_version(); InferenceEngine::Version version; version.buildNumber = ver.buildNumber; @@ -247,8 +248,9 @@ public: const InferenceEngine::CNNNetwork& network, const std::map& config) override { return ov::legacy_convert::convert_compiled_model( - m_plugin->compile_model(ov::legacy_convert::convert_model(network, m_plugin->is_new_api()), - ov::any_copy(config))); + {m_plugin->compile_model(ov::legacy_convert::convert_model(network, m_plugin->is_new_api()), + ov::any_copy(config)), + m_plugin._so}); } std::shared_ptr LoadNetwork( @@ -256,17 +258,19 @@ public: const std::map& config, const std::shared_ptr& context) override { return ov::legacy_convert::convert_compiled_model( - m_plugin->compile_model(ov::legacy_convert::convert_model(network, m_plugin->is_new_api()), - ov::any_copy(config), - ov::RemoteContext{ov::legacy_convert::convert_remote_context(context), {}})); + {m_plugin->compile_model(ov::legacy_convert::convert_model(network, m_plugin->is_new_api()), + ov::any_copy(config), + ov::legacy_convert::convert_remote_context(context)), + m_plugin._so}); } ov::SoPtr LoadNetwork( const std::string& modelPath, const std::map& config) override { return ov::SoPtr( - ov::legacy_convert::convert_compiled_model(m_plugin->compile_model(modelPath, ov::any_copy(config))), - {}); + ov::legacy_convert::convert_compiled_model( + {m_plugin->compile_model(modelPath, ov::any_copy(config)), m_plugin._so}), + m_plugin._so); } void AddExtension(const std::shared_ptr& extension) override { @@ -306,13 +310,15 @@ public: const std::string& modelFileName, const std::map& config) override { std::ifstream model(modelFileName, std::ios::binary); - return ov::legacy_convert::convert_compiled_model(m_plugin->import_model(model, ov::any_copy(config))); + return ov::legacy_convert::convert_compiled_model( + {m_plugin->import_model(model, ov::any_copy(config)), m_plugin._so}); } std::shared_ptr ImportNetwork( std::istream& networkModel, const std::map& config) override { - return ov::legacy_convert::convert_compiled_model(m_plugin->import_model(networkModel, ov::any_copy(config))); + return ov::legacy_convert::convert_compiled_model( + {m_plugin->import_model(networkModel, ov::any_copy(config)), m_plugin._so}); } std::shared_ptr ImportNetwork( @@ -320,9 +326,10 @@ public: const std::shared_ptr& context, const std::map& config) override { return ov::legacy_convert::convert_compiled_model( - m_plugin->import_model(networkModel, - ov::RemoteContext{ov::legacy_convert::convert_remote_context(context), {}}, - ov::any_copy(config))); + {m_plugin->import_model(networkModel, + ov::legacy_convert::convert_remote_context(context), + ov::any_copy(config)), + m_plugin._so}); } void SetCore(std::weak_ptr core) override { @@ -348,19 +355,19 @@ public: return ret; } - std::shared_ptr get_plugin() { + ov::SoPtr get_plugin() { return m_plugin; } private: - std::shared_ptr m_plugin; + ov::SoPtr m_plugin; }; } // namespace ov std::shared_ptr<::InferenceEngine::IInferencePlugin> ov::legacy_convert::convert_plugin( - const std::shared_ptr<::ov::IPlugin>& plugin) { - if (auto wrapper = std::dynamic_pointer_cast(plugin)) + const ov::SoPtr<::ov::IPlugin>& plugin) { + if (auto wrapper = std::dynamic_pointer_cast(plugin._ptr)) return wrapper->get_plugin(); return std::make_shared(plugin); } @@ -375,7 +382,7 @@ namespace ov { class IExecutableNetworkWrapper : public InferenceEngine::IExecutableNetworkInternal { public: - explicit IExecutableNetworkWrapper(const std::shared_ptr& model) : m_model(model) { + explicit IExecutableNetworkWrapper(const ov::SoPtr& model) : m_model(model) { for (const auto& input : m_model->inputs()) { InferenceEngine::InputInfo::Ptr input_info; ov::legacy_convert::fill_input_info(input, input_info); @@ -390,11 +397,13 @@ public: _networkOutputs[output_info->getName()] = output_info; _results.emplace_back(output.get_node_shared_ptr()); } - _plugin = ov::legacy_convert::convert_plugin(std::const_pointer_cast(m_model->m_plugin)); + _plugin = + ov::legacy_convert::convert_plugin({std::const_pointer_cast(m_model->m_plugin), m_model._so}); + _so = model._so; } std::shared_ptr CreateInferRequest() override { - auto infer_request = legacy_convert::convert_infer_request(m_model->create_infer_request()); + auto infer_request = legacy_convert::convert_infer_request({m_model->create_infer_request(), m_model._so}); infer_request->setPointerToExecutableNetworkInternal(shared_from_this()); return infer_request; } @@ -428,29 +437,29 @@ public: return ov::legacy_convert::convert_remote_context(m_model->get_context()); } - std::shared_ptr get_compiled_model() { + ov::SoPtr get_compiled_model() { return m_model; } private: - std::shared_ptr m_model; + ov::SoPtr m_model; }; } // namespace ov std::shared_ptr ov::legacy_convert::convert_compiled_model( - const std::shared_ptr& model) { - if (auto comp_model = std::dynamic_pointer_cast(model)) { + const ov::SoPtr& model) { + if (auto comp_model = std::dynamic_pointer_cast(model._ptr)) { return comp_model->get_executable_network(); } return std::make_shared(model); } -std::shared_ptr ov::legacy_convert::convert_compiled_model( +ov::SoPtr ov::legacy_convert::convert_compiled_model( const std::shared_ptr& model) { if (auto comp_model = std::dynamic_pointer_cast(model)) { return comp_model->get_compiled_model(); } - return std::make_shared(model); + return {std::make_shared(model), model->GetPointerToSo()}; } namespace ov { @@ -469,8 +478,9 @@ class IInferRequestInternalWrapper : public InferenceEngine::IInferRequestIntern } public: - explicit IInferRequestInternalWrapper(const std::shared_ptr& request) - : m_request(request) {} + explicit IInferRequestInternalWrapper(const ov::SoPtr& request) : m_request(request) { + _so = request._so; + } void Infer() override { m_request->infer(); @@ -511,7 +521,7 @@ public: void SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr& data) override { try { - m_request->set_tensor(find_port(name), ov::Tensor{ov::make_tensor(data), {}}); + m_request->set_tensor(find_port(name), ov::make_tensor(data)); } catch (const ov::Exception& ex) { const std::string what = ex.what(); if (what.find("Failed to set tensor") != std::string::npos) { @@ -523,9 +533,9 @@ public: void SetBlobs(const std::string& name, const std::vector& blobs) override { try { - std::vector tensors; + std::vector> tensors; for (const auto& blob : blobs) { - tensors.emplace_back(ov::Tensor{ov::make_tensor(blob), {}}); + tensors.emplace_back(ov::make_tensor(blob)); } m_request->set_tensors(find_port(name), tensors); } catch (const ov::Exception& ex) { @@ -534,22 +544,22 @@ public: } InferenceEngine::Blob::Ptr GetBlob(const std::string& name) override { - return tensor_to_blob(m_request->get_tensor(find_port(name))._impl); + return tensor_to_blob(m_request->get_tensor(find_port(name))); } InferenceEngine::BatchedBlob::Ptr GetBlobs(const std::string& name) override { auto tensors = m_request->get_tensors(find_port(name)); std::vector blobs; for (const auto& tensor : tensors) { - blobs.emplace_back(tensor_to_blob(tensor._impl)); + blobs.emplace_back(tensor_to_blob(tensor)); } return std::make_shared(blobs); } const InferenceEngine::PreProcessInfo& GetPreProcess(const std::string& name) const override { #ifdef PROXY_PLUGIN_ENABLED - if (auto proxy_request = std::dynamic_pointer_cast(m_request)) { - return ov::legacy_convert::convert_infer_request(proxy_request->get_hardware_request()._ptr) + if (auto proxy_request = std::dynamic_pointer_cast(m_request._ptr)) { + return ov::legacy_convert::convert_infer_request(proxy_request->get_hardware_request()) ->GetPreProcess(name); } #endif @@ -591,12 +601,12 @@ public: m_request->set_callback(std::move(callback)); } - std::shared_ptr get_infer_request() { + ov::SoPtr get_infer_request() { return m_request; } private: - std::shared_ptr m_request; + ov::SoPtr m_request; }; } // namespace ov @@ -606,7 +616,7 @@ namespace InferenceEngine { class IVariableStateWrapper : public ov::IVariableState { private: std::shared_ptr m_state; - mutable ov::Tensor m_converted_state; + mutable ov::SoPtr m_converted_state; public: explicit IVariableStateWrapper(const std::shared_ptr& state) @@ -617,13 +627,13 @@ public: m_state->Reset(); } - void set_state(const ov::Tensor& state) override { - m_state->SetState(ov::tensor_to_blob(state._impl)); + void set_state(const ov::SoPtr& state) override { + m_state->SetState(ov::tensor_to_blob(state)); } - const ov::Tensor& get_state() const override { - m_converted_state = - ov::Tensor(ov::make_tensor(std::const_pointer_cast(m_state->GetState())), {}); + const ov::SoPtr& get_state() const override { + m_converted_state = ov::make_tensor(std::const_pointer_cast(m_state->GetState())); + return m_converted_state; } }; @@ -716,7 +726,7 @@ public: return infos; } - ov::Tensor get_tensor(const ov::Output& port) const override { + ov::SoPtr get_tensor(const ov::Output& port) const override { const auto& name = get_legacy_name_from_port(port); OPENVINO_ASSERT(!m_request->GetBlobs(name), "get_tensor shall not be used together with batched " @@ -724,35 +734,42 @@ public: name, "'"); auto blob = m_request->GetBlob(name); - ov::Tensor tensor = {ov::make_tensor(blob), {m_request->getPointerToSo()}}; + ov::SoPtr tensor = ov::make_tensor(blob); + if (!tensor._so) + tensor._so = m_request->getPointerToSo(); return tensor; } - void set_tensor(const ov::Output& port, const ov::Tensor& tensor) override { - m_request->SetBlob(get_legacy_name_from_port(port), ov::tensor_to_blob(tensor._impl, m_unwrap_tensor)); + void set_tensor(const ov::Output& port, const ov::SoPtr& tensor) override { + m_request->SetBlob(get_legacy_name_from_port(port), ov::tensor_to_blob(tensor, m_unwrap_tensor)); } - std::vector get_tensors(const ov::Output& port) const override { + std::vector> get_tensors(const ov::Output& port) const override { auto blobs = m_request->GetBlobs(get_legacy_name_from_port(port)); - std::vector ret; + std::vector> ret; if (!blobs) return ret; for (size_t i = 0; i < blobs->size(); i++) { - ret.emplace_back(ov::Tensor{ov::make_tensor(blobs->getBlob(i)), {m_request->getPointerToSo()}}); + ov::SoPtr tensor = ov::make_tensor(blobs->getBlob(i)); + if (!tensor._so) + tensor._so = m_request->getPointerToSo(); + ret.emplace_back(tensor); } return ret; } - void set_tensors(const ov::Output& port, const std::vector& tensors) override { + void set_tensors(const ov::Output& port, + const std::vector>& tensors) override { std::vector blobs; for (const auto& tensor : tensors) { - blobs.emplace_back(ov::tensor_to_blob(tensor._impl, m_unwrap_tensor)); + blobs.emplace_back(ov::tensor_to_blob(tensor, m_unwrap_tensor)); } m_request->SetBlobs(get_legacy_name_from_port(port), blobs); } - std::vector> query_state() const override { - std::vector> variable_states; + std::vector> query_state() const override { + std::vector> variable_states; for (auto&& state : m_request->QueryState()) { - variable_states.emplace_back(std::make_shared(state)); + variable_states.push_back( + {std::make_shared(state), m_request->getPointerToSo()}); } return variable_states; } @@ -771,7 +788,7 @@ public: } } OPENVINO_ASSERT(m_compiled_model); - return m_compiled_model; + return m_compiled_model._ptr; } const std::vector>& get_inputs() const override { @@ -783,7 +800,7 @@ public: private: std::shared_ptr m_request; - mutable std::shared_ptr m_compiled_model; + mutable ov::SoPtr m_compiled_model; mutable std::mutex m_mutex; const bool m_unwrap_tensor; }; @@ -791,19 +808,20 @@ private: } // namespace InferenceEngine std::shared_ptr<::InferenceEngine::IInferRequestInternal> ov::legacy_convert::convert_infer_request( - const std::shared_ptr<::ov::IAsyncInferRequest>& request) { - if (auto comp_model = std::dynamic_pointer_cast(request)) { + const ov::SoPtr<::ov::IAsyncInferRequest>& request) { + if (auto comp_model = std::dynamic_pointer_cast(request._ptr)) { return comp_model->get_infer_request(); } return std::make_shared(request); } -std::shared_ptr<::ov::IAsyncInferRequest> ov::legacy_convert::convert_infer_request( +ov::SoPtr<::ov::IAsyncInferRequest> ov::legacy_convert::convert_infer_request( const std::shared_ptr<::InferenceEngine::IInferRequestInternal>& request, const std::string& plugin_name) { if (auto comp_model = std::dynamic_pointer_cast(request)) { return comp_model->get_infer_request(); } - return std::make_shared(request, plugin_name); + return {std::make_shared(request, plugin_name), + request->getPointerToSo()}; } namespace InferenceEngine { @@ -830,18 +848,19 @@ public: return m_params; } - std::shared_ptr create_tensor(const ov::element::Type& type, - const ov::Shape& shape, - const ov::AnyMap& params = {}) override { + ov::SoPtr create_tensor(const ov::element::Type& type, + const ov::Shape& shape, + const ov::AnyMap& params = {}) override { InferenceEngine::TensorDesc desc(InferenceEngine::details::convertPrecision(type), shape, InferenceEngine::TensorDesc::getLayoutByDims(shape)); auto blob = m_context->CreateBlob(desc, params); blob->allocate(); - return std::dynamic_pointer_cast(ov::make_tensor(blob)); + auto tensor = ov::make_tensor(blob); + return {std::dynamic_pointer_cast(tensor._ptr), tensor._so}; } - std::shared_ptr create_host_tensor(const ov::element::Type type, const ov::Shape& shape) override { + ov::SoPtr create_host_tensor(const ov::element::Type type, const ov::Shape& shape) override { InferenceEngine::TensorDesc desc(InferenceEngine::details::convertPrecision(type), shape, InferenceEngine::TensorDesc::getLayoutByDims(shape)); @@ -854,16 +873,17 @@ public: } // namespace InferenceEngine std::shared_ptr ov::legacy_convert::convert_remote_context( - const std::shared_ptr& context) { - if (auto ctx = std::dynamic_pointer_cast(context)) { + const ov::SoPtr& context) { + if (auto ctx = std::dynamic_pointer_cast(context._ptr)) { return ctx->get_context(); } return std::make_shared(context); } -std::shared_ptr ov::legacy_convert::convert_remote_context( + +ov::SoPtr ov::legacy_convert::convert_remote_context( const std::shared_ptr& context) { if (auto ctx = std::dynamic_pointer_cast(context)) { return ctx->get_context(); } - return std::make_shared(context); + return {std::make_shared(context)}; } diff --git a/src/inference/src/dev/converter_utils.hpp b/src/inference/src/dev/converter_utils.hpp index bd0dc77604b..d7ad03c67e3 100644 --- a/src/inference/src/dev/converter_utils.hpp +++ b/src/inference/src/dev/converter_utils.hpp @@ -23,24 +23,22 @@ void fill_output_info(const ov::Output& output, InferenceEngine: InferenceEngine::CNNNetwork convert_model(const std::shared_ptr& model, bool is_new_api); std::shared_ptr convert_model(const InferenceEngine::CNNNetwork& model, bool is_new_api); -std::shared_ptr<::InferenceEngine::IInferencePlugin> convert_plugin(const std::shared_ptr<::ov::IPlugin>& plugin); +std::shared_ptr<::InferenceEngine::IInferencePlugin> convert_plugin(const ov::SoPtr<::ov::IPlugin>& plugin); std::shared_ptr<::ov::IPlugin> convert_plugin(const std::shared_ptr<::InferenceEngine::IInferencePlugin>& plugin); std::shared_ptr<::InferenceEngine::IExecutableNetworkInternal> convert_compiled_model( - const std::shared_ptr<::ov::ICompiledModel>& model); -std::shared_ptr<::ov::ICompiledModel> convert_compiled_model( + const ov::SoPtr<::ov::ICompiledModel>& model); +ov::SoPtr<::ov::ICompiledModel> convert_compiled_model( const std::shared_ptr<::InferenceEngine::IExecutableNetworkInternal>& model); std::shared_ptr<::InferenceEngine::IInferRequestInternal> convert_infer_request( - const std::shared_ptr<::ov::IAsyncInferRequest>& request); -std::shared_ptr<::ov::IAsyncInferRequest> convert_infer_request( + const ov::SoPtr<::ov::IAsyncInferRequest>& request); +ov::SoPtr<::ov::IAsyncInferRequest> convert_infer_request( const std::shared_ptr<::InferenceEngine::IInferRequestInternal>& request, const std::string& plugin_name = ""); -std::shared_ptr convert_remote_context( - const std::shared_ptr& context); -std::shared_ptr convert_remote_context( - const std::shared_ptr& context); +std::shared_ptr convert_remote_context(const ov::SoPtr& context); +ov::SoPtr convert_remote_context(const std::shared_ptr& context); } // namespace legacy_convert } // namespace ov diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp index dab0b6dc033..15c840a6f7e 100644 --- a/src/inference/src/dev/core_impl.cpp +++ b/src/inference/src/dev/core_impl.cpp @@ -14,7 +14,6 @@ #include "cpp_interfaces/interface/ie_iplugin_internal.hpp" #include "dev/converter_utils.hpp" #include "dev/icompiled_model_wrapper.hpp" -#include "dev/make_tensor.hpp" #include "file_utils.h" #include "ie_itt.hpp" #include "ie_network_reader.hpp" @@ -32,6 +31,7 @@ #include "openvino/runtime/device_id_parser.hpp" #include "openvino/runtime/icompiled_model.hpp" #include "openvino/runtime/itensor.hpp" +#include "openvino/runtime/make_tensor.hpp" #include "openvino/runtime/remote_context.hpp" #include "openvino/runtime/threading/executor_manager.hpp" #include "openvino/util/common_util.hpp" @@ -573,6 +573,8 @@ ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const { so = ov::util::load_shared_object(desc.libraryLocation.c_str()); std::shared_ptr plugin_impl; reinterpret_cast(ov::util::get_symbol(so, ov::create_plugin_function))(plugin_impl); + if (auto wrapper = std::dynamic_pointer_cast(plugin_impl)) + wrapper->set_shared_object(so); plugin = Plugin{plugin_impl, so}; } @@ -706,23 +708,27 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< // Proxy plugin fallback to lowlevel device if (!is_proxy_device(plugin)) lock = cacheGuard.get_hash_lock(cacheContent.blobId); - res = load_model_from_cache(cacheContent, plugin, parsed._config, ov::RemoteContext{}, [&]() { - return compile_model_and_cache(model, plugin, parsed._config, ov::RemoteContext{}, cacheContent); + res = load_model_from_cache(cacheContent, plugin, parsed._config, ov::SoPtr{}, [&]() { + return compile_model_and_cache(model, + plugin, + parsed._config, + ov::SoPtr{}, + cacheContent); }); } else { - res = compile_model_with_preprocess(plugin, model, ov::RemoteContext{}, parsed._config); + res = compile_model_with_preprocess(plugin, model, ov::SoPtr{}, parsed._config); } return res; } ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr& model_, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& config) const { OV_ITT_SCOPE(FIRST_INFERENCE, ie::itt::domains::IE_LT, "Core::compile_model::RemoteContext"); if (!context) { IE_THROW() << "Remote context is null"; } - std::string deviceName = context.get_device_name(); + std::string deviceName = context->get_device_name(); ov::AnyMap config_with_batch = config; // if auto-batching is applicable, the below function will patch the device name and config accordingly: auto model = apply_auto_batching(model_, deviceName, config_with_batch); @@ -749,7 +755,7 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< ov::SoPtr ov::CoreImpl::compile_model_with_preprocess(ov::Plugin& plugin, const std::shared_ptr& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& config) const { std::shared_ptr preprocessed_model = model; @@ -785,18 +791,21 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod // Proxy plugin fallback to lowlevel device if (!is_proxy_device(plugin)) lock = cacheGuard.get_hash_lock(cacheContent.blobId); - compiled_model = load_model_from_cache(cacheContent, plugin, parsed._config, ov::RemoteContext{}, [&]() { - auto cnnNetwork = ReadNetwork(model_path, std::string()); - return compile_model_and_cache(cnnNetwork.getFunction(), plugin, parsed._config, {}, cacheContent); - }); + compiled_model = + load_model_from_cache(cacheContent, plugin, parsed._config, ov::SoPtr{}, [&]() { + auto cnnNetwork = ReadNetwork(model_path, std::string()); + return compile_model_and_cache(cnnNetwork.getFunction(), plugin, parsed._config, {}, cacheContent); + }); } else if (cacheManager) { // this code path is enabled for AUTO / MULTI / BATCH devices which don't support // import / export explicitly, but can redirect this functionality to actual HW plugin compiled_model = plugin.compile_model(model_path, parsed._config); } else { auto cnnNetwork = ReadNetwork(model_path, std::string()); - compiled_model = - compile_model_with_preprocess(plugin, cnnNetwork.getFunction(), ov::RemoteContext{}, parsed._config); + compiled_model = compile_model_with_preprocess(plugin, + cnnNetwork.getFunction(), + ov::SoPtr{}, + parsed._config); } return compiled_model; } @@ -820,13 +829,18 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod // Proxy plugin fallback to lowlevel device if (!is_proxy_device(plugin)) lock = cacheGuard.get_hash_lock(cacheContent.blobId); - compiled_model = load_model_from_cache(cacheContent, plugin, parsed._config, ov::RemoteContext{}, [&]() { - auto cnnNetwork = read_model(model_str, weights); - return compile_model_and_cache(cnnNetwork, plugin, parsed._config, ov::RemoteContext{}, cacheContent); - }); + compiled_model = + load_model_from_cache(cacheContent, plugin, parsed._config, ov::SoPtr{}, [&]() { + auto cnnNetwork = read_model(model_str, weights); + return compile_model_and_cache(cnnNetwork, + plugin, + parsed._config, + ov::SoPtr{}, + cacheContent); + }); } else { auto model = read_model(model_str, weights); - compiled_model = compile_model_with_preprocess(plugin, model, ov::RemoteContext{}, parsed._config); + compiled_model = compile_model_with_preprocess(plugin, model, ov::SoPtr{}, parsed._config); } return compiled_model; } @@ -845,10 +859,10 @@ ov::SoPtr ov::CoreImpl::import_model(std::istream& model, } ov::SoPtr ov::CoreImpl::import_model(std::istream& modelStream, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& config) const { OV_ITT_SCOPED_TASK(ov::itt::domains::IE, "Core::import_model"); - auto parsed = parseDeviceNameIntoConfig(context.get_device_name(), config); + auto parsed = parseDeviceNameIntoConfig(context->get_device_name(), config); auto compiled_model = get_plugin(parsed._deviceName).import_model(modelStream, context, parsed._config); if (auto wrapper = std::dynamic_pointer_cast(compiled_model._ptr)) { wrapper->get_executable_network()->loadedFromCache(); @@ -924,7 +938,7 @@ std::vector ov::CoreImpl::get_available_devices() const { return devices; } -ov::RemoteContext ov::CoreImpl::create_context(const std::string& device_name, const AnyMap& params) const { +ov::SoPtr ov::CoreImpl::create_context(const std::string& device_name, const AnyMap& params) const { auto parsed = ov::parseDeviceNameIntoConfig(device_name, params); return get_plugin(parsed._deviceName).create_context(parsed._config); } @@ -1002,7 +1016,7 @@ bool ov::CoreImpl::is_new_api() const { return m_new_api; } -ov::RemoteContext ov::CoreImpl::get_default_context(const std::string& device_name) const { +ov::SoPtr ov::CoreImpl::get_default_context(const std::string& device_name) const { auto parsed = ov::parseDeviceNameIntoConfig(device_name); return get_plugin(parsed._deviceName).get_default_context(parsed._config); } @@ -1357,7 +1371,7 @@ bool ov::CoreImpl::device_supports_cache_dir(const ov::Plugin& plugin) const { ov::SoPtr ov::CoreImpl::compile_model_and_cache(const std::shared_ptr& model, ov::Plugin& plugin, const ov::AnyMap& parsedConfig, - const ov::RemoteContext& context, + const ov::SoPtr& context, const CacheContent& cacheContent) const { OV_ITT_SCOPED_TASK(ov::itt::domains::IE, "CoreImpl::compile_model_and_cache"); ov::SoPtr execNetwork; @@ -1383,7 +1397,7 @@ ov::SoPtr ov::CoreImpl::load_model_from_cache( const CacheContent& cacheContent, ov::Plugin& plugin, const ov::AnyMap& config, - const ov::RemoteContext& context, + const ov::SoPtr& context, std::function()> compile_model_lambda) { ov::SoPtr compiled_model; struct HeaderException {}; @@ -1589,7 +1603,7 @@ std::shared_ptr ov::CoreImpl::read_model(const std::string& model, bool frontendMode) const { InferenceEngine::Blob::Ptr blob; if (weights) { - blob = tensor_to_blob(weights._impl); + blob = tensor_to_blob(get_tensor_impl(weights)); } OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::IE_RT, "CoreImpl::read_model from memory"); return ReadNetwork(model, blob, frontendMode).getFunction(); diff --git a/src/inference/src/dev/core_impl.hpp b/src/inference/src/dev/core_impl.hpp index 4a1157d3dfc..90d6fd81423 100644 --- a/src/inference/src/dev/core_impl.hpp +++ b/src/inference/src/dev/core_impl.hpp @@ -158,14 +158,14 @@ private: ov::SoPtr compile_model_and_cache(const std::shared_ptr& model, ov::Plugin& plugin, const ov::AnyMap& parsedConfig, - const ov::RemoteContext& context, + const ov::SoPtr& context, const CacheContent& cacheContent) const; static ov::SoPtr load_model_from_cache( const CacheContent& cacheContent, ov::Plugin& plugin, const ov::AnyMap& config, - const ov::RemoteContext& context, + const ov::SoPtr& context, std::function()> compile_model_lambda); bool device_supports_model_caching(const ov::Plugin& plugin) const; @@ -178,7 +178,7 @@ private: ov::SoPtr compile_model_with_preprocess(ov::Plugin& plugin, const std::shared_ptr& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& config) const; ov::AnyMap create_compile_config(const ov::Plugin& plugin, const ov::AnyMap& origConfig) const; @@ -377,7 +377,7 @@ public: const ov::AnyMap& config = {}) const override; ov::SoPtr compile_model(const std::shared_ptr& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& config = {}) const override; ov::SoPtr compile_model(const std::string& model_path, @@ -394,7 +394,7 @@ public: const ov::AnyMap& config = {}) const override; ov::SoPtr import_model(std::istream& modelStream, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& config) const override; ov::SupportedOpsMap query_model(const std::shared_ptr& model, @@ -403,13 +403,13 @@ public: std::vector get_available_devices() const override; - ov::RemoteContext create_context(const std::string& device_name, const AnyMap& args) const override; + ov::SoPtr create_context(const std::string& device_name, const AnyMap& args) const override; ov::AnyMap get_supported_property(const std::string& device_name, const ov::AnyMap& config) const override; bool is_new_api() const override; - ov::RemoteContext get_default_context(const std::string& device_name) const override; + ov::SoPtr get_default_context(const std::string& device_name) const override; /** * @brief Sets properties for a device, acceptable keys can be found in openvino/runtime/properties.hpp. diff --git a/src/inference/src/dev/core_impl_ie.cpp b/src/inference/src/dev/core_impl_ie.cpp index 1103fb274d0..c7750926294 100644 --- a/src/inference/src/dev/core_impl_ie.cpp +++ b/src/inference/src/dev/core_impl_ie.cpp @@ -11,7 +11,6 @@ #include "cpp_interfaces/interface/ie_iplugin_internal.hpp" #include "dev/converter_utils.hpp" #include "dev/icompiled_model_wrapper.hpp" -#include "dev/make_tensor.hpp" #include "ie_itt.hpp" #include "ie_network_reader.hpp" #include "iplugin_wrapper.hpp" @@ -20,7 +19,9 @@ #include "openvino/itt.hpp" #include "openvino/runtime/device_id_parser.hpp" #include "openvino/runtime/icompiled_model.hpp" +#include "openvino/runtime/iplugin.hpp" #include "openvino/runtime/itensor.hpp" +#include "openvino/runtime/make_tensor.hpp" #include "openvino/util/common_util.hpp" bool ov::CoreImpl::isNewAPI() const { @@ -44,7 +45,7 @@ ov::SoPtr ov::CoreImpl::LoadNetwork } InferenceEngine::RemoteContext::Ptr ov::CoreImpl::GetDefaultContext(const std::string& deviceName) { - return ov::legacy_convert::convert_remote_context(get_default_context(deviceName)._impl); + return ov::legacy_convert::convert_remote_context(get_default_context(deviceName)); } InferenceEngine::CNNNetwork ov::CoreImpl::ReadNetwork(const std::string& modelPath, const std::string& binPath) const { @@ -70,10 +71,10 @@ ov::SoPtr ov::CoreImpl::LoadNetwork const std::map& config) { OV_ITT_SCOPE(FIRST_INFERENCE, InferenceEngine::itt::domains::IE_LT, "Core::LoadNetwork::RemoteContext"); if (network.getFunction()) { - ov::RemoteContext ctx{ov::legacy_convert::convert_remote_context(context), {nullptr}}; + auto ctx = ov::legacy_convert::convert_remote_context(context); auto compiled_model = compile_model(ov::legacy_convert::convert_model(network, isNewAPI()), ctx, any_copy(config)); - return {ov::legacy_convert::convert_compiled_model(compiled_model._ptr), compiled_model._so}; + return {ov::legacy_convert::convert_compiled_model(compiled_model), compiled_model._so}; } if (context == nullptr) { IE_THROW() << "Remote context is null"; @@ -93,7 +94,7 @@ InferenceEngine::SoExecutableNetworkInternal ov::CoreImpl::LoadNetwork( if (network.getFunction()) { auto compiled_model = compile_model(ov::legacy_convert::convert_model(network, isNewAPI()), deviceName, any_copy(config)); - return {ov::legacy_convert::convert_compiled_model(compiled_model._ptr), compiled_model._so}; + return {ov::legacy_convert::convert_compiled_model(compiled_model), compiled_model._so}; } auto parsed = parseDeviceNameIntoConfig(deviceName, any_copy(config)); auto plugin = get_plugin(parsed._deviceName); @@ -109,7 +110,7 @@ InferenceEngine::SoExecutableNetworkInternal ov::CoreImpl::LoadNetwork( OV_ITT_SCOPE(FIRST_INFERENCE, ie::itt::domains::IE_LT, "Core::LoadNetwork::Path"); auto compiled_model = compile_model(modelPath, deviceName, any_copy(config)); - return {ov::legacy_convert::convert_compiled_model(compiled_model._ptr), compiled_model._so}; + return {ov::legacy_convert::convert_compiled_model(compiled_model), compiled_model._so}; } InferenceEngine::SoExecutableNetworkInternal ov::CoreImpl::LoadNetwork( @@ -122,10 +123,10 @@ InferenceEngine::SoExecutableNetworkInternal ov::CoreImpl::LoadNetwork( auto compiled_model = compile_model(modelStr, - ov::Tensor{ov::make_tensor(std::const_pointer_cast(weights)), {}}, + ov::make_tensor(ov::make_tensor(std::const_pointer_cast(weights))), deviceName, ov::any_copy(config)); - return {ov::legacy_convert::convert_compiled_model(compiled_model._ptr), compiled_model._so}; + return {ov::legacy_convert::convert_compiled_model(compiled_model), compiled_model._so}; } InferenceEngine::SoExecutableNetworkInternal ov::CoreImpl::ImportNetwork( @@ -136,7 +137,7 @@ InferenceEngine::SoExecutableNetworkInternal ov::CoreImpl::ImportNetwork( if (auto wrapper = std::dynamic_pointer_cast(compiled_model._ptr)) { wrapper->get_executable_network()->loadedFromCache(); } - return {ov::legacy_convert::convert_compiled_model(compiled_model._ptr), compiled_model._so}; + return {ov::legacy_convert::convert_compiled_model(compiled_model), compiled_model._so}; } InferenceEngine::QueryNetworkResult ov::CoreImpl::QueryNetwork(const InferenceEngine::CNNNetwork& network, @@ -207,7 +208,7 @@ std::vector ov::CoreImpl::GetAvailableDevices() const { InferenceEngine::RemoteContext::Ptr ov::CoreImpl::CreateContext(const std::string& deviceName, const InferenceEngine::ParamMap& params) { - return ov::legacy_convert::convert_remote_context(create_context(deviceName, params)._impl); + return ov::legacy_convert::convert_remote_context(create_context(deviceName, params)); } /** @@ -269,7 +270,8 @@ std::map ov::CoreImpl::GetVersions(const ov::Plugin cppPlugin = get_plugin(deviceNameLocal); - versions[deviceNameLocal] = ov::legacy_convert::convert_plugin(cppPlugin.m_ptr)->GetVersion(); + versions[deviceNameLocal] = + ov::legacy_convert::convert_plugin(ov::SoPtr{cppPlugin.m_ptr, cppPlugin.m_so})->GetVersion(); } return versions; diff --git a/src/inference/src/dev/iasync_infer_request.cpp b/src/inference/src/dev/iasync_infer_request.cpp index 578b6fdb8c8..a780ffd8441 100644 --- a/src/inference/src/dev/iasync_infer_request.cpp +++ b/src/inference/src/dev/iasync_infer_request.cpp @@ -102,7 +102,7 @@ void ov::IAsyncInferRequest::set_callback(std::function> ov::IAsyncInferRequest::query_state() const { +std::vector> ov::IAsyncInferRequest::query_state() const { check_state(); return m_sync_request->query_state(); } @@ -206,23 +206,23 @@ std::vector ov::IAsyncInferRequest::get_profiling_info() cons return m_sync_request->get_profiling_info(); } -ov::Tensor ov::IAsyncInferRequest::get_tensor(const ov::Output& port) const { +ov::SoPtr ov::IAsyncInferRequest::get_tensor(const ov::Output& port) const { check_state(); return m_sync_request->get_tensor(port); } -void ov::IAsyncInferRequest::set_tensor(const ov::Output& port, const ov::Tensor& tensor) { +void ov::IAsyncInferRequest::set_tensor(const ov::Output& port, const ov::SoPtr& tensor) { check_state(); return m_sync_request->set_tensor(port, tensor); } -std::vector ov::IAsyncInferRequest::get_tensors(const ov::Output& port) const { +std::vector> ov::IAsyncInferRequest::get_tensors(const ov::Output& port) const { check_state(); return m_sync_request->get_tensors(port); } void ov::IAsyncInferRequest::set_tensors(const ov::Output& port, - const std::vector& tensors) { + const std::vector>& tensors) { check_state(); return m_sync_request->set_tensors(port, tensors); } diff --git a/src/inference/src/dev/icompiled_model.cpp b/src/inference/src/dev/icompiled_model.cpp index 01db907cc6d..6e81c091719 100644 --- a/src/inference/src/dev/icompiled_model.cpp +++ b/src/inference/src/dev/icompiled_model.cpp @@ -18,7 +18,7 @@ ov::ICompiledModel::ICompiledModel(const std::shared_ptr& model ov::ICompiledModel::ICompiledModel(const std::shared_ptr& model, const std::shared_ptr& plugin, - const ov::RemoteContext& context, + const ov::SoPtr& context, const std::shared_ptr& task_executor, const std::shared_ptr& callback_executor) : m_plugin(plugin), @@ -136,11 +136,11 @@ void ov::ICompiledModel::set_callback_executor(const std::shared_ptr ov::ICompiledModel::get_context() const { +ov::SoPtr ov::ICompiledModel::get_context() const { if (auto wrapper = dynamic_cast(this)) { return ov::legacy_convert::convert_remote_context(wrapper->get_executable_network()->GetContext()); } if (m_context) - return m_context._impl; + return m_context; return m_plugin->get_default_context({}); } diff --git a/src/inference/src/dev/icompiled_model_wrapper.cpp b/src/inference/src/dev/icompiled_model_wrapper.cpp index 354ca304fd2..fd6a994d6c8 100644 --- a/src/inference/src/dev/icompiled_model_wrapper.cpp +++ b/src/inference/src/dev/icompiled_model_wrapper.cpp @@ -23,7 +23,9 @@ InferenceEngine::ICompiledModelWrapper::ICompiledModelWrapper( } std::shared_ptr InferenceEngine::ICompiledModelWrapper::create_infer_request() const { - return ov::legacy_convert::convert_infer_request(m_model->CreateInferRequest(), m_model->_plugin->GetName()); + auto infer_request = m_model->CreateInferRequest(); + infer_request->setPointerToSo(m_model->GetPointerToSo()); + return ov::legacy_convert::convert_infer_request(infer_request, m_model->_plugin->GetName())._ptr; } void InferenceEngine::ICompiledModelWrapper::export_model(std::ostream& model) const { diff --git a/src/inference/src/dev/iplugin_wrapper.cpp b/src/inference/src/dev/iplugin_wrapper.cpp index c7955d49444..43a72fb38b3 100644 --- a/src/inference/src/dev/iplugin_wrapper.cpp +++ b/src/inference/src/dev/iplugin_wrapper.cpp @@ -9,6 +9,7 @@ #include "any_copy.hpp" #include "dev/converter_utils.hpp" #include "ie_icore.hpp" +#include "openvino/runtime/iremote_context.hpp" #include "threading/ie_executor_manager.hpp" namespace InferenceEngine { @@ -27,6 +28,9 @@ IPluginWrapper::IPluginWrapper(const std::shared_ptr& IPluginWrapper::update_exec_network( const std::shared_ptr& network) const { network->SetPointerToPlugin(m_old_plugin); + if (!network->GetPointerToSo()) + network->_so = m_so; + return network; } @@ -34,22 +38,23 @@ std::shared_ptr IPluginWrapper::compile_model(const std::sha const ov::AnyMap& properties) const { auto exec_network = m_old_plugin->LoadNetwork(ov::legacy_convert::convert_model(model, is_new_api()), ov::any_copy(properties)); - return ov::legacy_convert::convert_compiled_model(update_exec_network(exec_network)); + return ov::legacy_convert::convert_compiled_model(update_exec_network(exec_network))._ptr; } std::shared_ptr IPluginWrapper::compile_model(const std::string& model_path, const ov::AnyMap& properties) const { auto exec_network = m_old_plugin->LoadNetwork(model_path, any_copy(properties)); - return ov::legacy_convert::convert_compiled_model(update_exec_network(exec_network._ptr)); + return ov::legacy_convert::convert_compiled_model(update_exec_network(exec_network._ptr))._ptr; } std::shared_ptr IPluginWrapper::compile_model(const std::shared_ptr& model, const ov::AnyMap& properties, - const ov::RemoteContext& context) const { + const ov::SoPtr& context) const { return ov::legacy_convert::convert_compiled_model( - update_exec_network(m_old_plugin->LoadNetwork(ov::legacy_convert::convert_model(model, is_new_api()), - any_copy(properties), - ov::legacy_convert::convert_remote_context(context._impl)))); + update_exec_network(m_old_plugin->LoadNetwork(ov::legacy_convert::convert_model(model, is_new_api()), + any_copy(properties), + ov::legacy_convert::convert_remote_context(context)))) + ._ptr; } void IPluginWrapper::set_property(const ov::AnyMap& properties) { @@ -64,27 +69,29 @@ ov::Any IPluginWrapper::get_property(const std::string& name, const ov::AnyMap& } } -std::shared_ptr IPluginWrapper::create_context(const ov::AnyMap& remote_properties) const { +ov::SoPtr IPluginWrapper::create_context(const ov::AnyMap& remote_properties) const { return ov::legacy_convert::convert_remote_context(m_old_plugin->CreateContext(remote_properties)); } -std::shared_ptr IPluginWrapper::get_default_context(const ov::AnyMap& remote_properties) const { +ov::SoPtr IPluginWrapper::get_default_context(const ov::AnyMap& remote_properties) const { return ov::legacy_convert::convert_remote_context(m_old_plugin->GetDefaultContext(remote_properties)); } std::shared_ptr IPluginWrapper::import_model(std::istream& model, const ov::AnyMap& properties) const { return ov::legacy_convert::convert_compiled_model( - update_exec_network(m_old_plugin->ImportNetwork(model, any_copy(properties)))); + update_exec_network(m_old_plugin->ImportNetwork(model, any_copy(properties)))) + ._ptr; } std::shared_ptr IPluginWrapper::import_model(std::istream& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& properties) const { return ov::legacy_convert::convert_compiled_model( - update_exec_network(m_old_plugin->ImportNetwork(model, - ov::legacy_convert::convert_remote_context(context._impl), - any_copy(properties)))); + update_exec_network(m_old_plugin->ImportNetwork(model, + ov::legacy_convert::convert_remote_context(context), + any_copy(properties)))) + ._ptr; } ov::SupportedOpsMap IPluginWrapper::query_model(const std::shared_ptr& model, @@ -115,4 +122,8 @@ void IPluginWrapper::set_device_name(const std::string& device_name) { m_old_plugin->SetName(device_name); } +void IPluginWrapper::set_shared_object(const std::shared_ptr& so) { + m_so = so; +} + } // namespace InferenceEngine diff --git a/src/inference/src/dev/iplugin_wrapper.hpp b/src/inference/src/dev/iplugin_wrapper.hpp index 0d40b75da55..d6c2f085598 100644 --- a/src/inference/src/dev/iplugin_wrapper.hpp +++ b/src/inference/src/dev/iplugin_wrapper.hpp @@ -59,7 +59,7 @@ public: */ std::shared_ptr compile_model(const std::shared_ptr& model, const ov::AnyMap& properties, - const ov::RemoteContext& context) const override; + const ov::SoPtr& context) const override; /** * @brief Specifies some plugin properties @@ -85,7 +85,7 @@ public: * * @return Remote context */ - std::shared_ptr create_context(const ov::AnyMap& remote_properties) const override; + ov::SoPtr create_context(const ov::AnyMap& remote_properties) const override; /** * @brief Create default remote context @@ -94,7 +94,7 @@ public: * * @return Remote context */ - std::shared_ptr get_default_context(const ov::AnyMap& remote_properties) const override; + ov::SoPtr get_default_context(const ov::AnyMap& remote_properties) const override; /** * @brief Import model to the plugin @@ -116,7 +116,7 @@ public: * @return shared pointer to compiled model interface */ std::shared_ptr import_model(std::istream& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& properties) const override; /** @@ -160,8 +160,11 @@ public: */ void set_device_name(const std::string& device_name); + void set_shared_object(const std::shared_ptr& so); + private: std::shared_ptr m_old_plugin; + std::shared_ptr m_so; const std::shared_ptr& update_exec_network( const std::shared_ptr& network) const; diff --git a/src/inference/src/dev/iremote_context.cpp b/src/inference/src/dev/iremote_context.cpp index 4e4a5074565..c5d373361d9 100644 --- a/src/inference/src/dev/iremote_context.cpp +++ b/src/inference/src/dev/iremote_context.cpp @@ -4,9 +4,8 @@ #include "openvino/runtime/iremote_context.hpp" -#include "dev/make_tensor.hpp" +#include "openvino/runtime/make_tensor.hpp" -std::shared_ptr ov::IRemoteContext::create_host_tensor(const ov::element::Type type, - const ov::Shape& shape) { - return ov::make_tensor(type, shape); +ov::SoPtr ov::IRemoteContext::create_host_tensor(const ov::element::Type type, const ov::Shape& shape) { + return ov::SoPtr(ov::make_tensor(type, shape), nullptr); } diff --git a/src/inference/src/dev/iremote_tensor.cpp b/src/inference/src/dev/iremote_tensor.cpp index fb74f24f0a8..a1730ef4b1e 100644 --- a/src/inference/src/dev/iremote_tensor.cpp +++ b/src/inference/src/dev/iremote_tensor.cpp @@ -6,7 +6,6 @@ #include -#include "dev/make_tensor.hpp" #include "ie_blob.h" #include "ie_ngraph_utils.hpp" #include "ie_remote_blob.hpp" diff --git a/src/inference/src/dev/isync_infer_request.cpp b/src/inference/src/dev/isync_infer_request.cpp index d9626797ccd..5382d8626fc 100644 --- a/src/inference/src/dev/isync_infer_request.cpp +++ b/src/inference/src/dev/isync_infer_request.cpp @@ -4,6 +4,7 @@ #include "openvino/runtime/isync_infer_request.hpp" +#include #include #include "cpp_interfaces/plugin_itt.hpp" @@ -14,10 +15,12 @@ #include "openvino/runtime/icompiled_model.hpp" #include "openvino/runtime/iinfer_request.hpp" #include "openvino/runtime/iremote_context.hpp" +#include "openvino/runtime/make_tensor.hpp" #include "openvino/runtime/tensor.hpp" namespace { -void check_batched_tensors(const ov::Output& input, const std::vector& tensors) { +void check_batched_tensors(const ov::Output& input, + const std::vector>& tensors) { OPENVINO_ASSERT(!tensors.empty(), "set_input_tensors/set_tensors can't be called with empty tensors"); OPENVINO_ASSERT( tensors.size() != 1, @@ -31,16 +34,17 @@ void check_batched_tensors(const ov::Output& input, const std::v auto batch_idx = ov::layout::batch_idx(layout); if (batch_idx < 0) { // TODO: Do we need this logic? - batch_idx += static_cast(tensors[0].get_shape().size()); + batch_idx += static_cast(tensors[0]->get_shape().size()); } OPENVINO_ASSERT(batch_idx == 0, "set_input_tensors/set_tensors is not currently supported for batch dimension index ", batch_idx, " != 0"); - std::for_each(tensors.begin(), tensors.end(), [&batch_idx](const ov::Tensor& item) { - OPENVINO_ASSERT(item.get_shape()[batch_idx] == 1, + std::for_each(tensors.begin(), tensors.end(), [&batch_idx](const ov::SoPtr& item) { + OPENVINO_ASSERT(item, "Unintialized tensor is provided!"); + OPENVINO_ASSERT(item->get_shape()[batch_idx] == 1, "set_input_tensors/set_tensors. Tensors shall represent one item in a batch, ", - item.get_shape()[batch_idx], + item->get_shape()[batch_idx], " provided"); }); auto tensors_size = static_cast(tensors.size()); @@ -62,22 +66,23 @@ void check_batched_tensors(const ov::Output& input, const std::v } // In future consider checking if blobs point to contiguous range of memory and use single 'SetBlob' instead - auto batched_shape = tensors[0].get_shape(); - auto element_type = tensors[0].get_element_type(); + auto batched_shape = tensors[0]->get_shape(); + auto element_type = tensors[0]->get_element_type(); batched_shape[batch_idx] = tensors_size; for (const auto& item : tensors) { - auto item_shape = item.get_shape(); + OPENVINO_ASSERT(item, "Unintialized tensor is provided!"); + auto item_shape = item->get_shape(); item_shape[batch_idx] = batched_shape[batch_idx]; - OPENVINO_ASSERT(item_shape == batched_shape && item.get_element_type() == element_type && + OPENVINO_ASSERT(item_shape == batched_shape && item->get_element_type() == element_type && "set_input_tensors/set_tensors error. Tensor with element type ", - item.get_element_type(), + item->get_element_type(), " and shape ", item_shape, " is not compatible with batched tensor with element type ", element_type, " and shape ", batched_shape); - OPENVINO_ASSERT(item.is_continuous(), "Strides for batched tensors should be default."); + OPENVINO_ASSERT(item->is_continuous(), "Strides for batched tensors should be default."); } } @@ -91,11 +96,11 @@ ov::ISyncInferRequest::ISyncInferRequest(const std::shared_ptr(); } for (const auto& output : get_outputs()) { if (m_tensors.find(output.get_tensor_ptr()) == m_tensors.end()) - m_tensors[output.get_tensor_ptr()] = ov::Tensor(); + m_tensors[output.get_tensor_ptr()] = ov::SoPtr(); } } @@ -132,13 +137,14 @@ ov::ISyncInferRequest::FoundPort ov::ISyncInferRequest::find_port(const ov::Outp } void ov::ISyncInferRequest::convert_batched_tensors() { - std::unordered_map, ov::Tensor> prepared_tensors; + std::unordered_map, ov::SoPtr> prepared_tensors; for (const auto& item : m_batched_tensors) { - auto tmp_shape = item.second.at(0).get_shape(); - auto tmp_et = item.second.at(0).get_element_type(); + OPENVINO_ASSERT(item.second.at(0), "Unintialized tensor is provided!"); + auto tmp_shape = item.second.at(0)->get_shape(); + auto tmp_et = item.second.at(0)->get_element_type(); tmp_shape[0] = item.second.size(); - std::shared_ptr remote_context; - ov::Tensor input_tensor; + ov::SoPtr remote_context; + ov::SoPtr input_tensor; try { auto net = get_compiled_model(); if (net) { @@ -147,16 +153,16 @@ void ov::ISyncInferRequest::convert_batched_tensors() { } catch (const ov::NotImplemented&) { } if (remote_context) { - input_tensor = ov::Tensor(remote_context->create_host_tensor(tmp_et, tmp_shape), {}); + input_tensor = remote_context->create_host_tensor(tmp_et, tmp_shape); } else { - input_tensor = ov::Tensor(tmp_et, tmp_shape); + input_tensor = {ov::make_tensor(tmp_et, tmp_shape), nullptr}; } - auto ptr = static_cast(input_tensor.data()); + auto ptr = static_cast(input_tensor->data()); // Perform memory copy ov::parallel_for(item.second.size(), [&](size_t i) { const auto& tensor = item.second.at(i); - memcpy(ptr + i * tensor.get_byte_size(), static_cast(tensor.data()), tensor.get_byte_size()); + memcpy(ptr + i * tensor->get_byte_size(), static_cast(tensor->data()), tensor->get_byte_size()); }); prepared_tensors[item.first] = input_tensor; } @@ -167,7 +173,7 @@ void ov::ISyncInferRequest::convert_batched_tensors() { } } -ov::Tensor& ov::ISyncInferRequest::get_ref_tensor(const ov::Output& port) const { +ov::SoPtr& ov::ISyncInferRequest::get_tensor_ptr(const ov::Output& port) const { auto found_port = find_port(port); OPENVINO_ASSERT(found_port.found(), "Cannot find tensor for port ", port); auto ports = found_port.is_input() ? get_inputs() : get_outputs(); @@ -177,12 +183,12 @@ ov::Tensor& ov::ISyncInferRequest::get_ref_tensor(const ov::Outputsecond; } -ov::Tensor ov::ISyncInferRequest::get_tensor(const ov::Output& port) const { +ov::SoPtr ov::ISyncInferRequest::get_tensor(const ov::Output& port) const { OV_ITT_SCOPED_TASK(InferenceEngine::itt::domains::Plugin, "get_tensor"); - return get_ref_tensor(port); + return get_tensor_ptr(port); } -void ov::ISyncInferRequest::set_tensor(const ov::Output& port, const ov::Tensor& tensor) { +void ov::ISyncInferRequest::set_tensor(const ov::Output& port, const ov::SoPtr& tensor) { OV_ITT_SCOPED_TASK(InferenceEngine::itt::domains::Plugin, "set_tensor"); auto found_port = find_port(port); OPENVINO_ASSERT(found_port.found(), "Cannot find tensor for port ", port); @@ -199,7 +205,7 @@ void ov::ISyncInferRequest::set_tensor(const ov::Output& port, c } } -std::vector ov::ISyncInferRequest::get_tensors(const ov::Output& port) const { +std::vector> ov::ISyncInferRequest::get_tensors(const ov::Output& port) const { OV_ITT_SCOPED_TASK(InferenceEngine::itt::domains::Plugin, "get_tensors"); auto found_port = find_port(port); OPENVINO_ASSERT(found_port.found(), "Cannot find input tensors for port ", port); @@ -209,7 +215,7 @@ std::vector ov::ISyncInferRequest::get_tensors(const ov::Output& port, - const std::vector& tensors) { + const std::vector>& tensors) { OV_ITT_SCOPED_TASK(InferenceEngine::itt::domains::Plugin, "set_tensors"); auto found_port = find_port(port); OPENVINO_ASSERT(found_port.found() && found_port.is_input(), "Cannot find input tensors for port ", port); @@ -223,7 +229,7 @@ void ov::ISyncInferRequest::set_tensors(const ov::Output& port, } void ov::ISyncInferRequest::set_tensors_impl(const ov::Output port, - const std::vector& tensors) { + const std::vector>& tensors) { OPENVINO_ASSERT_HELPER(::ov::NotImplemented, "", false, @@ -231,42 +237,46 @@ void ov::ISyncInferRequest::set_tensors_impl(const ov::Output po "set_input_tensors/set_tensors are not supported by this plugin"); } -void ov::ISyncInferRequest::check_tensor(const ov::Output& port, const ov::Tensor& tensor) const { +void ov::ISyncInferRequest::check_tensor(const ov::Output& port, + const ov::SoPtr& tensor) const { + OPENVINO_ASSERT(tensor); bool is_input = ov::op::util::is_parameter(port.get_node()); std::string tensor_type = is_input ? "input" : "output"; - OPENVINO_ASSERT(port.get_element_type() == tensor.get_element_type(), + OPENVINO_ASSERT(port.get_element_type() == tensor->get_element_type(), "The tensor element type is not corresponding with output element type (", - tensor.get_element_type(), + tensor->get_element_type(), " != ", port.get_element_type()); bool is_dynamic = port.get_partial_shape().is_dynamic(); - OPENVINO_ASSERT(is_dynamic || port.get_shape() == tensor.get_shape(), + OPENVINO_ASSERT(is_dynamic || port.get_shape() == tensor->get_shape(), "The ", tensor_type, " tensor size is not equal to the model ", tensor_type, " type: got ", - tensor.get_shape(), + tensor->get_shape(), " expecting ", port.get_shape(), "."); - OPENVINO_ASSERT(tensor.is() || tensor.data() != nullptr, "Tensor data equal nullptr!"); + OPENVINO_ASSERT(std::dynamic_pointer_cast(tensor._ptr) || tensor->data() != nullptr, + "Tensor data equal nullptr!"); } -void ov::ISyncInferRequest::allocate_tensor(const ov::Output& port, - const std::function& allocate_callback) { - auto& tensor = get_ref_tensor(port); +void ov::ISyncInferRequest::allocate_tensor( + const ov::Output& port, + const std::function& tensor)>& allocate_callback) { + auto& tensor = get_tensor_ptr(port); allocate_callback(tensor); } void ov::ISyncInferRequest::check_tensors() const { const auto& inputs = m_compiled_model->inputs(); for (size_t i = 0; i < inputs.size(); i++) { - check_tensor(inputs[i], get_ref_tensor(inputs[i])); + check_tensor(inputs[i], get_tensor_ptr(inputs[i])); } const auto& outputs = m_compiled_model->outputs(); for (size_t i = 0; i < outputs.size(); i++) { - check_tensor(outputs[i], get_ref_tensor(outputs[i])); + check_tensor(outputs[i], get_tensor_ptr(outputs[i])); } } diff --git a/src/inference/src/dev/ivariable_state.cpp b/src/inference/src/dev/ivariable_state.cpp index d65409024a9..f46f420b702 100644 --- a/src/inference/src/dev/ivariable_state.cpp +++ b/src/inference/src/dev/ivariable_state.cpp @@ -18,10 +18,10 @@ void ov::IVariableState::reset() { OPENVINO_NOT_IMPLEMENTED; } -void ov::IVariableState::set_state(const ov::Tensor& state) { +void ov::IVariableState::set_state(const ov::SoPtr& state) { m_state = state; } -const ov::Tensor& ov::IVariableState::get_state() const { +const ov::SoPtr& ov::IVariableState::get_state() const { return m_state; } diff --git a/src/inference/src/dev/make_tensor.cpp b/src/inference/src/dev/make_tensor.cpp index 05101d82782..137f1412e08 100644 --- a/src/inference/src/dev/make_tensor.cpp +++ b/src/inference/src/dev/make_tensor.cpp @@ -2,11 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "dev/make_tensor.hpp" +#include "openvino/runtime/make_tensor.hpp" #include -#include "dev/make_tensor.hpp" #include "ie_blob.h" #include "ie_ngraph_utils.hpp" #include "ie_remote_blob.hpp" @@ -442,19 +441,24 @@ public: */ class TensorRemoteBlob : public ie::RemoteBlob { public: - TensorRemoteBlob(const std::shared_ptr& tensor) + TensorRemoteBlob(const ov::SoPtr& tensor) : ie::RemoteBlob{ie::TensorDesc{ie::details::convertPrecision(tensor->get_element_type()), tensor->get_shape(), ie::TensorDesc::getLayoutByRank(tensor->get_shape().size())}}, - tensor{std::dynamic_pointer_cast(tensor)} { + tensor{tensor} { OPENVINO_ASSERT(this->tensor); } + std::shared_ptr cast_tensor() const { + auto remote = std::dynamic_pointer_cast(tensor._ptr); + OPENVINO_ASSERT(remote); + return remote; + } AnyMap getParams() const override { - return tensor->get_properties(); + return cast_tensor()->get_properties(); } std::string getDeviceName() const noexcept override { try { - return tensor->get_device_name(); + return cast_tensor()->get_device_name(); } catch (...) { return {}; } @@ -489,7 +493,7 @@ public: return nullptr; } - std::shared_ptr tensor; + ov::SoPtr tensor; private: std::shared_ptr m_allocator; @@ -504,7 +508,7 @@ template class TensorMemoryBlob : public ie::TBlob { public: ~TensorMemoryBlob() override = default; - explicit TensorMemoryBlob(const std::shared_ptr& tensor_) try : ie + explicit TensorMemoryBlob(const ov::SoPtr& tensor_) try : ie ::TBlob{[&] { auto element_type = tensor_->get_element_type(); auto shape = tensor_->get_shape(); @@ -536,7 +540,7 @@ public: static_cast(tensor_->data()), tensor_->get_byte_size()}, tensor{tensor_} { - OPENVINO_ASSERT(!std::dynamic_pointer_cast(tensor)); + OPENVINO_ASSERT(!std::dynamic_pointer_cast(tensor._ptr)); } catch (const std::exception& ex) { OPENVINO_THROW(ex.what()); @@ -552,10 +556,10 @@ public: } } - std::shared_ptr tensor; + ov::SoPtr tensor; }; -std::shared_ptr make_tensor(const std::shared_ptr& blob) { +ov::SoPtr make_tensor(const std::shared_ptr& blob) { #define ELSE_IF(type) \ else if (auto tblob = dynamic_cast*>(blob.get())) { \ return tblob->tensor; \ @@ -565,7 +569,7 @@ std::shared_ptr make_tensor(const std::shared_ptr& blob) { } else if (auto remote_blob = std::dynamic_pointer_cast(blob)) { return remote_blob->tensor; } else if (auto remote_blob = std::dynamic_pointer_cast(blob)) { - return std::make_shared(remote_blob); + return {std::make_shared(remote_blob), nullptr}; } ELSE_IF(float) ELSE_IF(double) @@ -581,7 +585,7 @@ std::shared_ptr make_tensor(const std::shared_ptr& blob) { ELSE_IF(uint64_t) ELSE_IF(int8_t) ELSE_IF(bool) else { - return std::make_shared(blob); + return {std::make_shared(blob), nullptr}; } #undef IF } @@ -590,9 +594,9 @@ ie::Blob* get_hardware_blob(ie::Blob* blob) { #ifdef PROXY_PLUGIN_ENABLED if (auto remote_blob = dynamic_cast(blob)) { const auto& tensor = ov::proxy::get_hardware_tensor(remote_blob->tensor); - if (auto blob_tensor = std::dynamic_pointer_cast(tensor)) { + if (auto blob_tensor = std::dynamic_pointer_cast(tensor._ptr)) { return blob_tensor->blob.get(); - } else if (auto blob_tensor = std::dynamic_pointer_cast(tensor)) { + } else if (auto blob_tensor = std::dynamic_pointer_cast(tensor._ptr)) { return blob_tensor->blob.get(); } OPENVINO_NOT_IMPLEMENTED; @@ -605,9 +609,9 @@ const ie::Blob* get_hardware_blob(const ie::Blob* blob) { #ifdef PROXY_PLUGIN_ENABLED if (auto remote_blob = dynamic_cast(blob)) { const auto& tensor = ov::proxy::get_hardware_tensor(remote_blob->tensor); - if (auto blob_tensor = std::dynamic_pointer_cast(tensor)) { + if (auto blob_tensor = std::dynamic_pointer_cast(tensor._ptr)) { return blob_tensor->blob.get(); - } else if (auto blob_tensor = std::dynamic_pointer_cast(tensor)) { + } else if (auto blob_tensor = std::dynamic_pointer_cast(tensor._ptr)) { return blob_tensor->blob.get(); } OPENVINO_NOT_IMPLEMENTED; @@ -616,7 +620,7 @@ const ie::Blob* get_hardware_blob(const ie::Blob* blob) { return blob; } -ie::Blob::Ptr tensor_to_blob(const std::shared_ptr& orig_tensor, bool unwrap) { +ie::Blob::Ptr tensor_to_blob(const ov::SoPtr& orig_tensor, bool unwrap) { #ifdef PROXY_PLUGIN_ENABLED const auto& tensor = unwrap ? ov::proxy::get_hardware_tensor(orig_tensor) : orig_tensor; #else @@ -624,13 +628,11 @@ ie::Blob::Ptr tensor_to_blob(const std::shared_ptr& orig_tensor, bool u #endif if (tensor == nullptr) { return {}; - } else if (auto blob_tensor = std::dynamic_pointer_cast(tensor)) { + } else if (auto blob_tensor = std::dynamic_pointer_cast(tensor._ptr)) { return blob_tensor->blob; - } else if (auto blob_tensor = std::dynamic_pointer_cast(tensor)) { + } else if (auto blob_tensor = std::dynamic_pointer_cast(tensor._ptr)) { return blob_tensor->blob; - } else if (auto blob_tensor = dynamic_cast(tensor.get())) { - return blob_tensor->blob; - } else if (std::dynamic_pointer_cast(tensor)) { + } else if (std::dynamic_pointer_cast(tensor._ptr)) { return std::make_shared(tensor); } else { #define CASE(precision, T) \ @@ -662,4 +664,29 @@ ie::Blob::Ptr tensor_to_blob(const std::shared_ptr& orig_tensor, bool u } OPENVINO_THROW("Cannot convert tensor to blob!"); } + +namespace util { + +ov::Tensor make_tensor(const std::shared_ptr& tensor, const std::shared_ptr& so) { + return ov::Tensor(tensor, so); +} + +void get_tensor_impl(const ov::Tensor& tensor, std::shared_ptr& tensor_impl, std::shared_ptr& so) { + tensor_impl = tensor._impl; + so = tensor._so; +} + +} // namespace util + +ov::Tensor make_tensor(const ov::SoPtr& tensor) { + return util::make_tensor(tensor._ptr, tensor._so); +} + +ov::SoPtr get_tensor_impl(const ov::Tensor& tensor) { + std::shared_ptr tensor_impl; + std::shared_ptr so; + util::get_tensor_impl(tensor, tensor_impl, so); + return ov::SoPtr(tensor_impl, so); +} + } // namespace ov diff --git a/src/inference/src/dev/plugin.cpp b/src/inference/src/dev/plugin.cpp index 19dafff4069..88ac29c7535 100644 --- a/src/inference/src/dev/plugin.cpp +++ b/src/inference/src/dev/plugin.cpp @@ -70,7 +70,7 @@ ov::SoPtr ov::Plugin::compile_model(const std::string& model } ov::SoPtr ov::Plugin::compile_model(const std::shared_ptr& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& properties) const { OV_PLUGIN_CALL_STATEMENT(return {m_ptr->compile_model(model, properties, context), m_so}); } @@ -85,22 +85,26 @@ ov::SoPtr ov::Plugin::import_model(std::istream& model, cons } ov::SoPtr ov::Plugin::import_model(std::istream& networkModel, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& config) const { OV_PLUGIN_CALL_STATEMENT(return {m_ptr->import_model(networkModel, context, config), m_so}); } -ov::RemoteContext ov::Plugin::create_context(const AnyMap& params) const { +ov::SoPtr ov::Plugin::create_context(const AnyMap& params) const { OV_PLUGIN_CALL_STATEMENT({ auto remote = m_ptr->create_context(params); - return {remote, {m_so}}; + if (!remote._so) + remote._so = m_so; + return remote; }); } -ov::RemoteContext ov::Plugin::get_default_context(const AnyMap& params) const { +ov::SoPtr ov::Plugin::get_default_context(const AnyMap& params) const { OV_PLUGIN_CALL_STATEMENT({ auto remote = m_ptr->get_default_context(params); - return {remote, {m_so}}; + if (!remote._so) + remote._so = m_so; + return remote; }); } diff --git a/src/inference/src/dev/plugin.hpp b/src/inference/src/dev/plugin.hpp index be029c8b9a7..b392fad14d6 100644 --- a/src/inference/src/dev/plugin.hpp +++ b/src/inference/src/dev/plugin.hpp @@ -12,6 +12,7 @@ #include "ie_iextension.h" #include "openvino/runtime/icompiled_model.hpp" #include "openvino/runtime/iplugin.hpp" +#include "openvino/runtime/so_ptr.hpp" namespace ov { @@ -50,7 +51,7 @@ public: SoPtr compile_model(const std::string& model_path, const ov::AnyMap& properties) const; SoPtr compile_model(const std::shared_ptr& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& properties) const; ov::SupportedOpsMap query_model(const std::shared_ptr& model, const ov::AnyMap& properties) const; @@ -58,12 +59,12 @@ public: SoPtr import_model(std::istream& model, const ov::AnyMap& properties) const; SoPtr import_model(std::istream& networkModel, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& config) const; - ov::RemoteContext create_context(const AnyMap& params) const; + ov::SoPtr create_context(const AnyMap& params) const; - ov::RemoteContext get_default_context(const AnyMap& params) const; + ov::SoPtr get_default_context(const AnyMap& params) const; Any get_property(const std::string& name, const AnyMap& arguments) const; diff --git a/src/inference/src/dev/remote_context_wrapper.hpp b/src/inference/src/dev/remote_context_wrapper.hpp index 78be59eeed7..c061e4d3883 100644 --- a/src/inference/src/dev/remote_context_wrapper.hpp +++ b/src/inference/src/dev/remote_context_wrapper.hpp @@ -4,21 +4,22 @@ #include -#include "dev/make_tensor.hpp" #include "ie_ngraph_utils.hpp" #include "ie_remote_context.hpp" #include "openvino/runtime/iremote_context.hpp" +#include "openvino/runtime/make_tensor.hpp" +#include "openvino/runtime/so_ptr.hpp" namespace ov { class RemoteContextWrapper : public InferenceEngine::RemoteContext { private: - std::shared_ptr m_context; + ov::SoPtr m_context; public: - RemoteContextWrapper(const std::shared_ptr& context) : m_context(context) {} + RemoteContextWrapper(const ov::SoPtr& context) : m_context(context) {} - const std::shared_ptr& get_context() const { + const ov::SoPtr& get_context() const { return m_context; } diff --git a/src/inference/src/ie_blob_common.cpp b/src/inference/src/ie_blob_common.cpp index c5d496bbf57..40347650838 100644 --- a/src/inference/src/ie_blob_common.cpp +++ b/src/inference/src/ie_blob_common.cpp @@ -6,8 +6,8 @@ #include #include -#include "dev/make_tensor.hpp" #include "ie_blob.h" +#include "openvino/runtime/make_tensor.hpp" #include "system_allocator.hpp" namespace InferenceEngine { diff --git a/src/inference/src/ie_core.cpp b/src/inference/src/ie_core.cpp index 479b2a37caa..b81eb3d9e99 100644 --- a/src/inference/src/ie_core.cpp +++ b/src/inference/src/ie_core.cpp @@ -203,7 +203,7 @@ ExecutableNetwork Core::ImportNetwork(const std::string& modelFileName, if (!modelStream.is_open()) IE_THROW(NetworkNotRead) << "Model file " << modelFileName << " cannot be opened!"; auto exec = _impl->get_plugin(parsed._deviceName).import_model(modelStream, parsed._config); - return {ov::legacy_convert::convert_compiled_model(exec._ptr), exec._so}; + return {ov::legacy_convert::convert_compiled_model(exec), exec._so}; } ExecutableNetwork Core::ImportNetwork(std::istream& networkModel, @@ -233,7 +233,7 @@ ExecutableNetwork Core::ImportNetwork(std::istream& networkModel) { networkModel.seekg(currentPos, networkModel.beg); auto exec = _impl->get_plugin(deviceName).import_model(networkModel, {}); - return {ov::legacy_convert::convert_compiled_model(exec._ptr), exec._so}; + return {ov::legacy_convert::convert_compiled_model(exec), exec._so}; } ExecutableNetwork Core::ImportNetwork(std::istream& networkModel, @@ -251,10 +251,8 @@ ExecutableNetwork Core::ImportNetwork(std::istream& networkModel, auto parsed = ov::parseDeviceNameIntoConfig(deviceName, ov::any_copy(config)); auto exec = _impl->get_plugin(deviceName) - .import_model(networkModel, - ov::RemoteContext{ov::legacy_convert::convert_remote_context(context), {}}, - parsed._config); - return {ov::legacy_convert::convert_compiled_model(exec._ptr), exec._so}; + .import_model(networkModel, ov::legacy_convert::convert_remote_context(context), parsed._config); + return {ov::legacy_convert::convert_compiled_model(exec), exec._so}; } QueryNetworkResult Core::QueryNetwork(const CNNNetwork& network, diff --git a/src/inference/src/ie_remote_context.cpp b/src/inference/src/ie_remote_context.cpp index 9430a05e71b..2bab079fbaf 100644 --- a/src/inference/src/ie_remote_context.cpp +++ b/src/inference/src/ie_remote_context.cpp @@ -31,7 +31,7 @@ const std::shared_ptr RemoteContext::GetHardware if (auto wrapper = dynamic_cast(this)) { auto ov_context = wrapper->get_context(); auto hw_context = ov::proxy::get_hardware_context(ov_context); - return ov::legacy_convert::convert_remote_context(hw_context); + return ov::legacy_convert::convert_remote_context(hw_context._ptr); } #endif return shared_from_this(); @@ -42,7 +42,7 @@ const std::shared_ptr RemoteContext::GetHa if (auto wrapper = dynamic_cast(this)) { auto ov_context = wrapper->get_context(); auto hw_context = ov::proxy::get_hardware_context(ov_context); - return ov::legacy_convert::convert_remote_context(hw_context); + return ov::legacy_convert::convert_remote_context(hw_context._ptr); } #endif return shared_from_this(); diff --git a/src/inference/src/infer_request.cpp b/src/inference/src/infer_request.cpp index e025f2b70ee..02027935f2b 100644 --- a/src/inference/src/infer_request.cpp +++ b/src/inference/src/infer_request.cpp @@ -14,6 +14,8 @@ #include "openvino/runtime/compiled_model.hpp" #include "openvino/runtime/exception.hpp" #include "openvino/runtime/iasync_infer_request.hpp" +#include "openvino/runtime/make_tensor.hpp" +#include "openvino/runtime/so_ptr.hpp" #include "transformations/utils/utils.hpp" #define OV_INFER_REQ_CALL_STATEMENT(...) \ @@ -62,7 +64,7 @@ InferRequest::InferRequest(const std::shared_ptr& impl, } void InferRequest::set_tensor(const ov::Output& port, const Tensor& tensor) { - OV_INFER_REQ_CALL_STATEMENT({ _impl->set_tensor(port, tensor); }); + OV_INFER_REQ_CALL_STATEMENT({ _impl->set_tensor(port, get_tensor_impl(tensor)); }); } void InferRequest::set_tensor(const ov::Output& port, const Tensor& tensor) { @@ -90,7 +92,12 @@ void InferRequest::set_tensors(const std::string& name, const std::vector& port, const std::vector& tensors) { - OV_INFER_REQ_CALL_STATEMENT({ _impl->set_tensors(port, tensors); }) + std::vector> tensor_ptrs; + tensor_ptrs.reserve(tensors.size()); + for (const auto& tensor : tensors) { + tensor_ptrs.emplace_back(get_tensor_impl(tensor)); + } + OV_INFER_REQ_CALL_STATEMENT({ _impl->set_tensors(port, tensor_ptrs); }) } void InferRequest::set_input_tensor(size_t idx, const Tensor& tensor) { @@ -170,7 +177,7 @@ Tensor InferRequest::get_tensor(const ov::Output& port) { if (!tensor._so) tensor._so = _so; - return tensor; + return make_tensor(tensor); }); } @@ -269,7 +276,9 @@ std::vector InferRequest::query_state() { std::vector variable_states; OV_INFER_REQ_CALL_STATEMENT({ for (auto&& state : _impl->query_state()) { - variable_states.emplace_back(ov::VariableState{state, {_so}}); + if (!state._so) + state._so = _so; + variable_states.emplace_back(ov::VariableState{state._ptr, state._so}); } }) return variable_states; diff --git a/src/inference/src/remote_tensor.cpp b/src/inference/src/remote_tensor.cpp index 909cec1c866..93ef71d3da2 100644 --- a/src/inference/src/remote_tensor.cpp +++ b/src/inference/src/remote_tensor.cpp @@ -8,13 +8,14 @@ #include "openvino/runtime/iremote_tensor.hpp" #include "openvino/runtime/itensor.hpp" +#include "openvino/runtime/make_tensor.hpp" #include "openvino/runtime/properties.hpp" namespace ov { void RemoteTensor::type_check(const Tensor& tensor, const std::map>& type_info) { OPENVINO_ASSERT(tensor, "Could not check empty tensor type"); - auto remote_tensor = std::dynamic_pointer_cast(tensor._impl); + auto remote_tensor = std::dynamic_pointer_cast(get_tensor_impl(tensor)._ptr); OPENVINO_ASSERT(remote_tensor, "Tensor is not remote."); if (!type_info.empty()) { auto remote_properties = remote_tensor->get_properties(); diff --git a/src/inference/tests/functional/get_supported_property_test.cpp b/src/inference/tests/functional/get_supported_property_test.cpp index 7e517f623da..10618f0256e 100644 --- a/src/inference/tests/functional/get_supported_property_test.cpp +++ b/src/inference/tests/functional/get_supported_property_test.cpp @@ -36,7 +36,7 @@ class MockPlugin : public ov::IPlugin { std::shared_ptr compile_model(const std::shared_ptr& model, const ov::AnyMap& properties, - const ov::RemoteContext& context) const override { + const ov::SoPtr& context) const override { OPENVINO_NOT_IMPLEMENTED; } @@ -62,11 +62,11 @@ class MockPlugin : public ov::IPlugin { return ""; } - std::shared_ptr create_context(const ov::AnyMap& remote_properties) const override { + ov::SoPtr create_context(const ov::AnyMap& remote_properties) const override { OPENVINO_NOT_IMPLEMENTED; } - std::shared_ptr get_default_context(const ov::AnyMap& remote_properties) const override { + ov::SoPtr get_default_context(const ov::AnyMap& remote_properties) const override { OPENVINO_NOT_IMPLEMENTED; } @@ -75,7 +75,7 @@ class MockPlugin : public ov::IPlugin { } std::shared_ptr import_model(std::istream& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& properties) const override { OPENVINO_NOT_IMPLEMENTED; } @@ -175,4 +175,4 @@ static const std::vector test_variants = { INSTANTIATE_TEST_SUITE_P(GetSupportedPropertyTest, GetPropertyTest, ::testing::ValuesIn(test_variants), - getTestCaseName); \ No newline at end of file + getTestCaseName); diff --git a/src/inference/tests/unit/ie_blob_test.cpp b/src/inference/tests/unit/ie_blob_test.cpp index 68b4fef8a47..ecc8792d7b5 100644 --- a/src/inference/tests/unit/ie_blob_test.cpp +++ b/src/inference/tests/unit/ie_blob_test.cpp @@ -6,7 +6,7 @@ #include #include -#include "dev/make_tensor.hpp" +#include "openvino/runtime/make_tensor.hpp" #include "unit_test_utils/mocks/mock_allocator.hpp" IE_SUPPRESS_DEPRECATED_START @@ -600,7 +600,7 @@ TEST_F(BlobTests, readRangeRoiBlob) { TEST_F(BlobTests, setBiggerShapeOnPreAllocatedMemory) { const auto t = ov::make_tensor(ov::element::i64, ov::Shape{2, 6}); - const auto b = ov::tensor_to_blob(t); + const auto b = ov::tensor_to_blob({t, nullptr}); const auto origin_ptr = t->data(); b->setShape({2, 8}); @@ -612,7 +612,7 @@ TEST_F(BlobTests, setBiggerShapeOnPreAllocatedMemory) { TEST_F(BlobTests, setSmallerShapeOnPreAllocatedMemory) { const auto t = ov::make_tensor(ov::element::i64, ov::Shape{2, 6}); - const auto b = ov::tensor_to_blob(t); + const auto b = ov::tensor_to_blob({t, nullptr}); const auto origin_ptr = t->data(); b->setShape({2, 4}); diff --git a/src/plugins/auto/src/infer_request.cpp b/src/plugins/auto/src/infer_request.cpp index 9766187cb6e..62aa40ffba3 100644 --- a/src/plugins/auto/src/infer_request.cpp +++ b/src/plugins/auto/src/infer_request.cpp @@ -12,6 +12,8 @@ #include "itt.hpp" #include "openvino/core/except.hpp" +#include "openvino/runtime/iremote_tensor.hpp" +#include "openvino/runtime/make_tensor.hpp" #include "openvino/runtime/profiling_info.hpp" #include "openvino/runtime/tensor.hpp" #include "plugin.hpp" @@ -20,11 +22,11 @@ using Time = std::chrono::high_resolution_clock; namespace { -void allocate_tensor_impl(ov::Tensor& tensor, const ov::element::Type& element_type, const ov::Shape& shape) { - if (!tensor || tensor.get_element_type() != element_type) { - tensor = ov::Tensor(element_type, shape); +void allocate_tensor_impl(ov::SoPtr& tensor, const ov::element::Type& element_type, const ov::Shape& shape) { + if (!tensor || tensor->get_element_type() != element_type) { + tensor = ov::make_tensor(element_type, shape); } else { - tensor.set_shape(shape); + tensor->set_shape(shape); } } @@ -37,7 +39,7 @@ ov::auto_plugin::InferRequest::InferRequest(const std::shared_ptr& tensor) { // Can add a check to avoid double work in case of shared tensors allocate_tensor_impl(tensor, input.get_element_type(), @@ -45,7 +47,7 @@ ov::auto_plugin::InferRequest::InferRequest(const std::shared_ptr& tensor) { // Can add a check to avoid double work in case of shared tensors allocate_tensor_impl(tensor, output.get_element_type(), @@ -54,10 +56,10 @@ ov::auto_plugin::InferRequest::InferRequest(const std::shared_ptrget_tensor(input), m_shared_request._so)); + ov::ISyncInferRequest::set_tensor(input, m_shared_request->get_tensor(input)); } for (const auto& output : get_outputs()) { - ov::ISyncInferRequest::set_tensor(output, ov::Tensor(m_shared_request->get_tensor(output), m_shared_request._so)); + ov::ISyncInferRequest::set_tensor(output, m_shared_request->get_tensor(output)); } } } @@ -75,22 +77,24 @@ void ov::auto_plugin::InferRequest::set_tensors_to_another_request(const SoAsync for (const auto &it : get_inputs()) { // this request is already in BUSY state, so using the internal functions safely auto tensor = get_tensor(it); - auto type = tensor.get_element_type(); - bool is_remote = tensor.is() || req->get_tensor(it).is(); - if (is_remote || req->get_tensor(it).data(type) != tensor.data(type)) + auto type = tensor->get_element_type(); + bool is_remote = std::dynamic_pointer_cast(tensor._ptr) || + std::dynamic_pointer_cast(req->get_tensor(it)._ptr); + if (is_remote || req->get_tensor(it)->data(type) != tensor->data(type)) req->set_tensor(it, tensor); } for (const auto &it : get_outputs()) { // this request is already in BUSY state, so using the internal functions safely auto tensor = get_tensor(it); - auto type = tensor.get_element_type(); - bool is_remote = tensor.is() || req->get_tensor(it).is(); - if (is_remote || req->get_tensor(it).data(type) != tensor.data(type)) + auto type = tensor->get_element_type(); + bool is_remote = std::dynamic_pointer_cast(tensor._ptr) || + std::dynamic_pointer_cast(req->get_tensor(it)._ptr); + if (is_remote || req->get_tensor(it)->data(type) != tensor->data(type)) req->set_tensor(it, tensor); } } -void ov::auto_plugin::InferRequest::set_tensor(const ov::Output& port, const ov::Tensor& tensor) { +void ov::auto_plugin::InferRequest::set_tensor(const ov::Output& port, const ov::SoPtr& tensor) { if (m_shared_request) m_shared_request->set_tensor(port, tensor); ov::ISyncInferRequest::set_tensor(port, tensor); @@ -111,8 +115,15 @@ std::vector ov::auto_plugin::InferRequest::get_profiling_info ov::auto_plugin::InferRequest::~InferRequest() = default; -std::vector> ov::auto_plugin::InferRequest::query_state() const { - if (m_shared_request) - return m_shared_request->query_state(); +std::vector> ov::auto_plugin::InferRequest::query_state() const { + if (m_shared_request) { + auto states = m_shared_request->query_state(); + for (auto&& state : states) { + if (!state._so) { + state._so = m_shared_request._so; + } + } + return states; + } OPENVINO_NOT_IMPLEMENTED; } diff --git a/src/plugins/auto/src/infer_request.hpp b/src/plugins/auto/src/infer_request.hpp index e4afa4ee802..732b168fe80 100644 --- a/src/plugins/auto/src/infer_request.hpp +++ b/src/plugins/auto/src/infer_request.hpp @@ -26,18 +26,18 @@ public: ~InferRequest(); void infer() override; - std::vector> query_state() const override; + std::vector> query_state() const override; std::vector get_profiling_info() const override; const SoAsyncInferRequest& get_shared_request(); void set_scheduled_request(SoAsyncInferRequest request); // Auto-Device impl specific: sets the data (tensors from the device-less requests to the specific device request) void set_tensors_to_another_request(const SoAsyncInferRequest& req); - void set_tensor(const ov::Output& port, const ov::Tensor& tensor) override; + void set_tensor(const ov::Output& port, const ov::SoPtr& tensor) override; private: SoAsyncInferRequest m_shared_request; SoAsyncInferRequest m_scheduled_request; }; } // namespace auto_plugin -} // namespace ov \ No newline at end of file +} // namespace ov diff --git a/src/plugins/auto/src/plugin.cpp b/src/plugins/auto/src/plugin.cpp index ae8f5eed569..9e3f0eeb2cc 100644 --- a/src/plugins/auto/src/plugin.cpp +++ b/src/plugins/auto/src/plugin.cpp @@ -17,6 +17,7 @@ #include "openvino/runtime/auto/properties.hpp" #include "openvino/runtime/device_id_parser.hpp" #include "openvino/runtime/internal_properties.hpp" +#include "openvino/runtime/iremote_context.hpp" #include "plugin.hpp" #include "auto_schedule.hpp" #include "auto_compiled_model.hpp" @@ -78,11 +79,11 @@ namespace auto_plugin { std::mutex Plugin::m_mtx; std::map> Plugin::m_priority_map; -std::shared_ptr Plugin::create_context(const ov::AnyMap& remote_properties) const { +ov::SoPtr Plugin::create_context(const ov::AnyMap& remote_properties) const { OPENVINO_NOT_IMPLEMENTED; } -std::shared_ptr Plugin::get_default_context(const ov::AnyMap& remote_properties) const { +ov::SoPtr Plugin::get_default_context(const ov::AnyMap& remote_properties) const { OPENVINO_NOT_IMPLEMENTED; } @@ -92,7 +93,7 @@ std::shared_ptr Plugin::import_model(std::istream& model, } std::shared_ptr Plugin::import_model(std::istream& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& properties) const { OPENVINO_NOT_IMPLEMENTED; } @@ -350,7 +351,7 @@ Plugin::Plugin() { std::shared_ptr Plugin::compile_model(const std::shared_ptr& model, const ov::AnyMap& properties, - const ov::RemoteContext& context) const { + const ov::SoPtr& context) const { OPENVINO_NOT_IMPLEMENTED; } diff --git a/src/plugins/auto/src/plugin.hpp b/src/plugins/auto/src/plugin.hpp index 11186beff2b..a41b1113084 100644 --- a/src/plugins/auto/src/plugin.hpp +++ b/src/plugins/auto/src/plugin.hpp @@ -36,7 +36,7 @@ public: std::shared_ptr compile_model(const std::shared_ptr& model, const ov::AnyMap& properties, - const ov::RemoteContext& context) const override; + const ov::SoPtr& context) const override; std::shared_ptr compile_model(const std::string& model_path, const ov::AnyMap& properties) const override; @@ -55,16 +55,16 @@ public: void unregister_priority(const unsigned int& priority, const std::string& device_name); void register_priority(const unsigned int& priority, const std::string& device_name); - std::shared_ptr create_context(const ov::AnyMap& remote_properties) const override; + ov::SoPtr create_context(const ov::AnyMap& remote_properties) const override; - std::shared_ptr get_default_context(const ov::AnyMap& remote_properties) const override; + ov::SoPtr get_default_context(const ov::AnyMap& remote_properties) const override; std::shared_ptr import_model(std::istream& model, const ov::AnyMap& properties) const override; std::shared_ptr import_model(std::istream& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& properties) const override; protected: diff --git a/src/plugins/auto/src/schedule.cpp b/src/plugins/auto/src/schedule.cpp index 1370d2d7e4f..3fc1aec6fb1 100644 --- a/src/plugins/auto/src/schedule.cpp +++ b/src/plugins/auto/src/schedule.cpp @@ -206,8 +206,8 @@ Pipeline Schedule::get_async_pipeline(const ISyncInferPtr& infer_request, Worker // if any input is remote (e.g. was set with set_tensor), let' use the corresponding device for (const auto& it : compiled_model->inputs()) { auto tensor = infer_request->get_tensor(it); - if (tensor.is()) { - const auto name = tensor.as().get_device_name(); + if (auto remote_tensor = std::dynamic_pointer_cast(tensor._ptr)) { + const auto name = remote_tensor->get_device_name(); const auto res = std::find_if( m_context->m_device_priorities_initial.cbegin(), m_context->m_device_priorities_initial.cend(), @@ -303,4 +303,4 @@ Schedule::~Schedule() { LOG_INFO_TAG("scheduler ending"); } } // namespace auto_plugin -} // namespace ov \ No newline at end of file +} // namespace ov diff --git a/src/plugins/auto/tests/unit/include/mock_common.hpp b/src/plugins/auto/tests/unit/include/mock_common.hpp index c9a9e8b44ea..a2f11b7de57 100644 --- a/src/plugins/auto/tests/unit/include/mock_common.hpp +++ b/src/plugins/auto/tests/unit/include/mock_common.hpp @@ -62,7 +62,7 @@ public: //MOCK_METHOD(std::vector, get_tensors, (const Output&), (const, override)); //MOCK_METHOD(void, set_tensors, (const Output&, const std::vector&), (override)); MOCK_METHOD(void, infer, (), (override)); - MOCK_METHOD(std::vector>, query_state, (), (const, override)); + MOCK_METHOD(std::vector>, query_state, (), (const, override)); //MOCK_METHOD(const std::shared_ptr&, get_compiled_model, (), (const, override)); //MOCK_METHOD(const std::vector>&, get_inputs, (), (const, override)); //MOCK_METHOD(const std::vector>&, get_outputs, (), (const, override)); @@ -70,6 +70,6 @@ public: ~MockSyncInferRequest() = default; private: - void allocate_tensor_impl(ov::Tensor& tensor, const ov::element::Type& element_type, const ov::Shape& shape); + void allocate_tensor_impl(ov::SoPtr& tensor, const ov::element::Type& element_type, const ov::Shape& shape); }; -} // namespace ov \ No newline at end of file +} // namespace ov diff --git a/src/plugins/auto/tests/unit/mock_common.cpp b/src/plugins/auto/tests/unit/mock_common.cpp index 681a6d35662..122fe8b9ecc 100644 --- a/src/plugins/auto/tests/unit/mock_common.cpp +++ b/src/plugins/auto/tests/unit/mock_common.cpp @@ -3,6 +3,7 @@ // #include "include/mock_common.hpp" +#include "openvino/runtime/make_tensor.hpp" // getMetric will return a fake ov::Any, gmock will call ostreamer << ov::Any // it will cause core dump, so add this special implemented @@ -29,11 +30,11 @@ MockAsyncInferRequest::MockAsyncInferRequest(const std::shared_ptr& tensor, const element::Type& element_type, const Shape& shape) { + if (!tensor || tensor->get_element_type() != element_type) { + tensor = ov::make_tensor(element_type, shape); } else { - tensor.set_shape(shape); + tensor->set_shape(shape); } } @@ -42,7 +43,7 @@ MockSyncInferRequest::MockSyncInferRequest(const std::shared_ptr& tensor) { // Can add a check to avoid double work in case of shared tensors allocate_tensor_impl(tensor, input.get_element_type(), @@ -50,7 +51,7 @@ MockSyncInferRequest::MockSyncInferRequest(const std::shared_ptr& tensor) { // Can add a check to avoid double work in case of shared tensors allocate_tensor_impl(tensor, output.get_element_type(), @@ -58,4 +59,4 @@ MockSyncInferRequest::MockSyncInferRequest(const std::shared_ptr ov::hetero::Plugin::compile_model(const std: return compiled_model; } -std::shared_ptr ov::hetero::Plugin::compile_model(const std::shared_ptr& model, - const ov::AnyMap& properties, - const ov::RemoteContext& context) const { +std::shared_ptr ov::hetero::Plugin::compile_model( + const std::shared_ptr& model, + const ov::AnyMap& properties, + const ov::SoPtr& context) const { OPENVINO_NOT_IMPLEMENTED; } std::shared_ptr ov::hetero::Plugin::import_model(std::istream& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& properties) const { OPENVINO_NOT_IMPLEMENTED; } @@ -191,13 +192,13 @@ ov::Any ov::hetero::Plugin::caching_device_properties(const std::string& device_ return ov::Any(result); } -std::shared_ptr ov::hetero::Plugin::create_context(const ov::AnyMap& remote_properties) const { +ov::SoPtr ov::hetero::Plugin::create_context(const ov::AnyMap& remote_properties) const { OPENVINO_NOT_IMPLEMENTED; } -std::shared_ptr ov::hetero::Plugin::get_default_context(const ov::AnyMap& remote_properties) const { +ov::SoPtr ov::hetero::Plugin::get_default_context(const ov::AnyMap& remote_properties) const { OPENVINO_NOT_IMPLEMENTED; } static const ov::Version version = {CI_BUILD_NUMBER, "openvino_hetero_plugin"}; -OV_DEFINE_PLUGIN_CREATE_FUNCTION(ov::hetero::Plugin, version) \ No newline at end of file +OV_DEFINE_PLUGIN_CREATE_FUNCTION(ov::hetero::Plugin, version) diff --git a/src/plugins/hetero/src/plugin.hpp b/src/plugins/hetero/src/plugin.hpp index 5a6de6a6e80..43ed4ddb407 100644 --- a/src/plugins/hetero/src/plugin.hpp +++ b/src/plugins/hetero/src/plugin.hpp @@ -32,20 +32,20 @@ public: std::shared_ptr compile_model(const std::shared_ptr& model, const ov::AnyMap& properties, - const ov::RemoteContext& context) const override; + const ov::SoPtr& context) const override; void set_property(const ov::AnyMap& properties) override; ov::Any get_property(const std::string& name, const ov::AnyMap& properties) const override; - std::shared_ptr create_context(const ov::AnyMap& remote_properties) const override; + ov::SoPtr create_context(const ov::AnyMap& remote_properties) const override; - std::shared_ptr get_default_context(const ov::AnyMap& remote_properties) const override; + ov::SoPtr get_default_context(const ov::AnyMap& remote_properties) const override; std::shared_ptr import_model(std::istream& model, const ov::AnyMap& properties) const override; std::shared_ptr import_model(std::istream& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& properties) const override; ov::SupportedOpsMap query_model(const std::shared_ptr& model, @@ -63,4 +63,4 @@ private: }; } // namespace hetero -} // namespace ov \ No newline at end of file +} // namespace ov diff --git a/src/plugins/hetero/src/sync_infer_request.cpp b/src/plugins/hetero/src/sync_infer_request.cpp index 694a93cce15..b1d621167fe 100644 --- a/src/plugins/hetero/src/sync_infer_request.cpp +++ b/src/plugins/hetero/src/sync_infer_request.cpp @@ -14,7 +14,6 @@ #include "itt.hpp" #include "openvino/core/except.hpp" #include "plugin.hpp" -#include "variable_state.hpp" ov::hetero::InferRequest::InferRequest(const std::shared_ptr& compiled_model) : ov::ISyncInferRequest(compiled_model) { @@ -66,20 +65,22 @@ ov::SoPtr ov::hetero::InferRequest::get_request(const ov OPENVINO_THROW("Cannot find infer request for port ", port); } -ov::Tensor ov::hetero::InferRequest::get_tensor(const ov::Output& port) const { +ov::SoPtr ov::hetero::InferRequest::get_tensor(const ov::Output& port) const { return get_request(port)->get_tensor(port); } -void ov::hetero::InferRequest::set_tensor(const ov::Output& port, const ov::Tensor& tensor) { +void ov::hetero::InferRequest::set_tensor(const ov::Output& port, + const ov::SoPtr& tensor) { get_request(port)->set_tensor(port, tensor); } -std::vector ov::hetero::InferRequest::get_tensors(const ov::Output& port) const { +std::vector> ov::hetero::InferRequest::get_tensors( + const ov::Output& port) const { return get_request(port)->get_tensors(port); } void ov::hetero::InferRequest::set_tensors(const ov::Output& port, - const std::vector& tensors) { + const std::vector>& tensors) { return get_request(port)->set_tensors(port, tensors); } @@ -89,12 +90,14 @@ void ov::hetero::InferRequest::check_tensors() const { return; } -std::vector> ov::hetero::InferRequest::query_state() const { - std::vector> variable_states = {}; +std::vector> ov::hetero::InferRequest::query_state() const { + std::vector> variable_states = {}; for (const auto& request : m_subrequests) { OPENVINO_ASSERT(request); for (auto&& state : request->query_state()) { - variable_states.emplace_back(std::make_shared(state, request._so)); + if (!state._so) + state._so = request._so; + variable_states.emplace_back(state); } } return variable_states; diff --git a/src/plugins/hetero/src/sync_infer_request.hpp b/src/plugins/hetero/src/sync_infer_request.hpp index 156ae088fe2..7ccc35c6154 100644 --- a/src/plugins/hetero/src/sync_infer_request.hpp +++ b/src/plugins/hetero/src/sync_infer_request.hpp @@ -30,17 +30,18 @@ public: void infer() override; - std::vector> query_state() const override; + std::vector> query_state() const override; std::vector get_profiling_info() const override; - ov::Tensor get_tensor(const ov::Output& port) const override; + ov::SoPtr get_tensor(const ov::Output& port) const override; - void set_tensor(const ov::Output& port, const ov::Tensor& tensor) override; + void set_tensor(const ov::Output& port, const ov::SoPtr& tensor) override; - std::vector get_tensors(const ov::Output& port) const override; + std::vector> get_tensors(const ov::Output& port) const override; - void set_tensors(const ov::Output& port, const std::vector& tensors) override; + void set_tensors(const ov::Output& port, + const std::vector>& tensors) override; void check_tensors() const override; @@ -54,4 +55,4 @@ private: }; } // namespace hetero -} // namespace ov \ No newline at end of file +} // namespace ov diff --git a/src/plugins/hetero/src/variable_state.hpp b/src/plugins/hetero/src/variable_state.hpp deleted file mode 100644 index 85299f665c2..00000000000 --- a/src/plugins/hetero/src/variable_state.hpp +++ /dev/null @@ -1,52 +0,0 @@ - -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include - -#include "openvino/runtime/ivariable_state.hpp" - -namespace ov { -namespace hetero { -/** - * @brief Simple wrapper for hardware variable states which holds plugin shared object - */ -class VariableState : public ov::IVariableState { - std::shared_ptr m_state; - std::shared_ptr m_so; - -public: - /** - * @brief Constructor of hetero VariableState - * - * @param state hardware state - * @param so shared object - */ - VariableState(const std::shared_ptr& state, const std::shared_ptr& so) - : IVariableState(""), - m_state(state), - m_so(so) { - OPENVINO_ASSERT(m_state); - } - const std::string& get_name() const override { - return m_state->get_name(); - } - - void reset() override { - m_state->reset(); - } - - void set_state(const ov::Tensor& state) override { - m_state->set_state(state); - } - - const ov::Tensor& get_state() const override { - return m_state->get_state(); - } -}; - -} // namespace hetero -} // namespace ov diff --git a/src/plugins/proxy/dev_api/openvino/proxy/infer_request.hpp b/src/plugins/proxy/dev_api/openvino/proxy/infer_request.hpp index 88aec1dd0f8..34887ebaa03 100644 --- a/src/plugins/proxy/dev_api/openvino/proxy/infer_request.hpp +++ b/src/plugins/proxy/dev_api/openvino/proxy/infer_request.hpp @@ -34,15 +34,16 @@ public: std::vector get_profiling_info() const override; - ov::Tensor get_tensor(const ov::Output& port) const override; + ov::SoPtr get_tensor(const ov::Output& port) const override; - void set_tensor(const ov::Output& port, const ov::Tensor& tensor) override; + void set_tensor(const ov::Output& port, const ov::SoPtr& tensor) override; - std::vector get_tensors(const ov::Output& port) const override; + std::vector> get_tensors(const ov::Output& port) const override; - void set_tensors(const ov::Output& port, const std::vector& tensors) override; + void set_tensors(const ov::Output& port, + const std::vector>& tensors) override; - std::vector> query_state() const override; + std::vector> query_state() const override; const std::shared_ptr& get_compiled_model() const override; diff --git a/src/plugins/proxy/dev_api/openvino/proxy/plugin.hpp b/src/plugins/proxy/dev_api/openvino/proxy/plugin.hpp index e69d320ca21..c0bc2164ee4 100644 --- a/src/plugins/proxy/dev_api/openvino/proxy/plugin.hpp +++ b/src/plugins/proxy/dev_api/openvino/proxy/plugin.hpp @@ -26,7 +26,7 @@ void create_plugin(std::shared_ptr& plugin); * * @return Original remote context */ -const std::shared_ptr& get_hardware_context(const std::shared_ptr& context); +const ov::SoPtr& get_hardware_context(const ov::SoPtr& context); /** * @brief Get wrapped remote tensor @@ -35,7 +35,7 @@ const std::shared_ptr& get_hardware_context(const std::share * * @return Original remote tensor */ -const std::shared_ptr& get_hardware_tensor(const std::shared_ptr& tensor); +const ov::SoPtr& get_hardware_tensor(const ov::SoPtr& tensor); } // namespace proxy } // namespace ov diff --git a/src/plugins/proxy/src/compiled_model.hpp b/src/plugins/proxy/src/compiled_model.hpp index 2489624f3c8..2d2f7caf3fe 100644 --- a/src/plugins/proxy/src/compiled_model.hpp +++ b/src/plugins/proxy/src/compiled_model.hpp @@ -18,7 +18,7 @@ public: m_compiled_model(model) {} CompiledModel(const ov::SoPtr& model, const std::shared_ptr& plugin, - const ov::RemoteContext& context) + const ov::SoPtr& context) : ov::ICompiledModel(nullptr, plugin, context), m_compiled_model(model) {} std::shared_ptr create_infer_request() const override { diff --git a/src/plugins/proxy/src/infer_request.cpp b/src/plugins/proxy/src/infer_request.cpp index a1ded2504d6..7bc17b5af9a 100644 --- a/src/plugins/proxy/src/infer_request.cpp +++ b/src/plugins/proxy/src/infer_request.cpp @@ -4,11 +4,14 @@ #include "openvino/proxy/infer_request.hpp" +#include +#include + #include "openvino/runtime/itensor.hpp" +#include "openvino/runtime/make_tensor.hpp" #include "openvino/runtime/remote_context.hpp" #include "openvino/runtime/so_ptr.hpp" #include "remote_context.hpp" -#include "variable_state.hpp" ov::proxy::InferRequest::InferRequest(ov::SoPtr&& request, const std::shared_ptr& compiled_model) @@ -43,42 +46,43 @@ std::vector ov::proxy::InferRequest::get_profiling_info() con return m_infer_request->get_profiling_info(); } -ov::Tensor ov::proxy::InferRequest::get_tensor(const ov::Output& port) const { +ov::SoPtr ov::proxy::InferRequest::get_tensor(const ov::Output& port) const { auto tensor = m_infer_request->get_tensor(port); - if (tensor.is()) { - auto remote_context = std::dynamic_pointer_cast(m_compiled_model->get_context()); + if (std::dynamic_pointer_cast(tensor._ptr)) { + auto remote_context = std::dynamic_pointer_cast(m_compiled_model->get_context()._ptr); OPENVINO_ASSERT(remote_context); - tensor = remote_context->wrap_tensor(tensor.as()); + tensor = remote_context->wrap_tensor(tensor); } - return ov::Tensor(tensor, m_infer_request._so); + return tensor; } -void ov::proxy::InferRequest::set_tensor(const ov::Output& port, const ov::Tensor& tensor) { +void ov::proxy::InferRequest::set_tensor(const ov::Output& port, const ov::SoPtr& tensor) { m_infer_request->set_tensor(port, tensor); } -std::vector ov::proxy::InferRequest::get_tensors(const ov::Output& port) const { +std::vector> ov::proxy::InferRequest::get_tensors(const ov::Output& port) const { auto tensors = m_infer_request->get_tensors(port); for (auto&& tensor : tensors) { - if (tensor.is()) { - auto remote_context = std::dynamic_pointer_cast(m_compiled_model->get_context()); + if (std::dynamic_pointer_cast(tensor._ptr)) { + auto remote_context = + std::dynamic_pointer_cast(m_compiled_model->get_context()._ptr); OPENVINO_ASSERT(remote_context); - tensor = remote_context->wrap_tensor(tensor.as()); + tensor = remote_context->wrap_tensor(tensor); } - tensor = ov::Tensor(tensor, m_infer_request._so); } return tensors; } void ov::proxy::InferRequest::set_tensors(const ov::Output& port, - const std::vector& tensors) { + const std::vector>& tensors) { return m_infer_request->set_tensors(port, tensors); } -std::vector> ov::proxy::InferRequest::query_state() const { +std::vector> ov::proxy::InferRequest::query_state() const { auto states = m_infer_request->query_state(); for (auto&& state : states) { - state = std::make_shared(state, m_infer_request._so); + if (!state._so) + state._so = m_infer_request._so; } return states; } diff --git a/src/plugins/proxy/src/plugin.cpp b/src/plugins/proxy/src/plugin.cpp index 180b085978e..93c2973f607 100644 --- a/src/plugins/proxy/src/plugin.cpp +++ b/src/plugins/proxy/src/plugin.cpp @@ -310,11 +310,12 @@ std::shared_ptr ov::proxy::Plugin::compile_model(const std:: return std::dynamic_pointer_cast(compiled_model); } -std::shared_ptr ov::proxy::Plugin::compile_model(const std::shared_ptr& model, - const ov::AnyMap& properties, - const ov::RemoteContext& context) const { +std::shared_ptr ov::proxy::Plugin::compile_model( + const std::shared_ptr& model, + const ov::AnyMap& properties, + const ov::SoPtr& context) const { auto ctx = ov::proxy::RemoteContext::get_hardware_context(context); - auto dev_name = ctx.get_device_name(); + auto dev_name = ctx->get_device_name(); auto device_config = construct_device_config(dev_name, m_configs, properties); std::shared_ptr plugin = shared_from_this(); @@ -325,7 +326,7 @@ std::shared_ptr ov::proxy::Plugin::compile_model(const std:: return std::dynamic_pointer_cast(compiled_model); } -std::shared_ptr ov::proxy::Plugin::create_context(const ov::AnyMap& remote_properties) const { +ov::SoPtr ov::proxy::Plugin::create_context(const ov::AnyMap& remote_properties) const { // TODO: if no device id, try to create context for each plugin auto dev_name = get_device_name(); auto dev_idx = get_device_from_config(remote_properties); @@ -342,7 +343,7 @@ std::shared_ptr ov::proxy::Plugin::create_context(const ov:: dev_idx, has_dev_idx, is_new_api); - return std::dynamic_pointer_cast(remote_context); + return remote_context; } // Properties doesn't have device id, so try to create context for all devices const auto hidden_devices = get_hidden_devices(); @@ -355,7 +356,7 @@ std::shared_ptr ov::proxy::Plugin::create_context(const ov:: i, has_dev_idx, is_new_api); - return std::dynamic_pointer_cast(remote_context); + return remote_context; } catch (const ov::Exception&) { } } @@ -363,7 +364,7 @@ std::shared_ptr ov::proxy::Plugin::create_context(const ov:: ov::Any(remote_properties).as()); } -std::shared_ptr ov::proxy::Plugin::get_default_context(const ov::AnyMap& remote_properties) const { +ov::SoPtr ov::proxy::Plugin::get_default_context(const ov::AnyMap& remote_properties) const { auto dev_name = get_device_name(); auto dev_idx = get_device_from_config(remote_properties); auto has_dev_idx = is_device_in_config(remote_properties); @@ -378,7 +379,7 @@ std::shared_ptr ov::proxy::Plugin::get_default_context(const dev_idx, has_dev_idx, is_new_api); - return std::dynamic_pointer_cast(remote_context); + return remote_context; } std::shared_ptr ov::proxy::Plugin::import_model(std::istream& model, @@ -391,10 +392,10 @@ std::shared_ptr ov::proxy::Plugin::import_model(std::istream } std::shared_ptr ov::proxy::Plugin::import_model(std::istream& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& properties) const { auto ctx = ov::proxy::RemoteContext::get_hardware_context(context); - auto dev_name = ctx.get_device_name(); + auto dev_name = ctx->get_device_name(); auto device_config = construct_device_config(dev_name, m_configs, properties); return std::make_shared(get_core()->import_model(model, ctx, device_config), diff --git a/src/plugins/proxy/src/plugin.hpp b/src/plugins/proxy/src/plugin.hpp index 34b36bd19b4..2fedf6cc37e 100644 --- a/src/plugins/proxy/src/plugin.hpp +++ b/src/plugins/proxy/src/plugin.hpp @@ -27,16 +27,16 @@ public: std::shared_ptr compile_model(const std::shared_ptr& model, const ov::AnyMap& properties, - const ov::RemoteContext& context) const override; + const ov::SoPtr& context) const override; - std::shared_ptr create_context(const ov::AnyMap& remote_properties) const override; + ov::SoPtr create_context(const ov::AnyMap& remote_properties) const override; - std::shared_ptr get_default_context(const ov::AnyMap& remote_properties) const override; + ov::SoPtr get_default_context(const ov::AnyMap& remote_properties) const override; std::shared_ptr import_model(std::istream& model, const ov::AnyMap& properties) const override; std::shared_ptr import_model(std::istream& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& properties) const override; private: diff --git a/src/plugins/proxy/src/remote_context.cpp b/src/plugins/proxy/src/remote_context.cpp index b543ea97888..5777630171e 100644 --- a/src/plugins/proxy/src/remote_context.cpp +++ b/src/plugins/proxy/src/remote_context.cpp @@ -8,9 +8,10 @@ #include "openvino/proxy/plugin.hpp" #include "openvino/runtime/iremote_context.hpp" +#include "openvino/runtime/so_ptr.hpp" #include "remote_tensor.hpp" -ov::proxy::RemoteContext::RemoteContext(ov::RemoteContext&& ctx, +ov::proxy::RemoteContext::RemoteContext(ov::SoPtr&& ctx, const std::string& dev_name, size_t dev_index, bool has_index, @@ -29,40 +30,35 @@ const std::string& ov::proxy::RemoteContext::get_device_name() const { } const ov::AnyMap& ov::proxy::RemoteContext::get_property() const { - return m_context._impl->get_property(); + return m_context->get_property(); } -ov::Tensor ov::proxy::RemoteContext::wrap_tensor(const ov::RemoteTensor& tensor) { - return ov::Tensor(std::make_shared(tensor, m_tensor_name), {}); +ov::SoPtr ov::proxy::RemoteContext::wrap_tensor(const ov::SoPtr& tensor) { + auto proxy_tensor = std::make_shared(tensor, m_tensor_name); + return ov::SoPtr(std::dynamic_pointer_cast(proxy_tensor), nullptr); } -std::shared_ptr ov::proxy::RemoteContext::create_tensor(const ov::element::Type& type, - const ov::Shape& shape, - const ov::AnyMap& params) { - return std::make_shared(m_context.create_tensor(type, shape, params), m_tensor_name); +ov::SoPtr ov::proxy::RemoteContext::create_tensor(const ov::element::Type& type, + const ov::Shape& shape, + const ov::AnyMap& params) { + auto proxy_tensor = + std::make_shared(m_context->create_tensor(type, shape, params), m_tensor_name); + return ov::SoPtr(std::dynamic_pointer_cast(proxy_tensor), nullptr); } -std::shared_ptr ov::proxy::RemoteContext::create_host_tensor(const ov::element::Type type, - const ov::Shape& shape) { - return m_context._impl->create_host_tensor(type, shape); +ov::SoPtr ov::proxy::RemoteContext::create_host_tensor(const ov::element::Type type, + const ov::Shape& shape) { + return m_context->create_host_tensor(type, shape); } -const ov::RemoteContext& ov::proxy::RemoteContext::get_hardware_context(const ov::RemoteContext& context) { - if (auto proxy_context = std::dynamic_pointer_cast(context._impl)) { +const ov::SoPtr& ov::proxy::RemoteContext::get_hardware_context( + const ov::SoPtr& context) { + if (auto proxy_context = std::dynamic_pointer_cast(context._ptr)) { return proxy_context->m_context; } return context; } -const std::shared_ptr& ov::proxy::RemoteContext::get_hardware_context( - const std::shared_ptr& context) { - if (auto proxy_context = std::dynamic_pointer_cast(context)) { - return proxy_context->m_context._impl; - } - return context; -} - -const std::shared_ptr& ov::proxy::get_hardware_context( - const std::shared_ptr& context) { +const ov::SoPtr& ov::proxy::get_hardware_context(const ov::SoPtr& context) { return ov::proxy::RemoteContext::get_hardware_context(context); } diff --git a/src/plugins/proxy/src/remote_context.hpp b/src/plugins/proxy/src/remote_context.hpp index 8e0ddff2a0b..bf67688a08d 100644 --- a/src/plugins/proxy/src/remote_context.hpp +++ b/src/plugins/proxy/src/remote_context.hpp @@ -8,6 +8,7 @@ #include "openvino/runtime/iremote_context.hpp" #include "openvino/runtime/iremote_tensor.hpp" #include "openvino/runtime/remote_context.hpp" +#include "openvino/runtime/so_ptr.hpp" namespace ov { namespace proxy { @@ -31,7 +32,7 @@ public: * In legacy API remote context doesn't contain the index in the name but Blob contains. * In 2.0 API Tensor and Context always contain device index */ - RemoteContext(ov::RemoteContext&& ctx, + RemoteContext(ov::SoPtr&& ctx, const std::string& dev_name, size_t dev_index, bool has_index, @@ -40,20 +41,18 @@ public: const ov::AnyMap& get_property() const override; - std::shared_ptr create_tensor(const ov::element::Type& type, - const ov::Shape& shape, - const ov::AnyMap& params = {}) override; + ov::SoPtr create_tensor(const ov::element::Type& type, + const ov::Shape& shape, + const ov::AnyMap& params = {}) override; - std::shared_ptr create_host_tensor(const ov::element::Type type, const ov::Shape& shape) override; + ov::SoPtr create_host_tensor(const ov::element::Type type, const ov::Shape& shape) override; - ov::Tensor wrap_tensor(const ov::RemoteTensor& tensor); + ov::SoPtr wrap_tensor(const ov::SoPtr& tensor); - static const ov::RemoteContext& get_hardware_context(const ov::RemoteContext& context); - static const std::shared_ptr& get_hardware_context( - const std::shared_ptr& context); + static const ov::SoPtr& get_hardware_context(const ov::SoPtr& context); private: - ov::RemoteContext m_context; + ov::SoPtr m_context; std::string m_name; std::string m_tensor_name; diff --git a/src/plugins/proxy/src/remote_tensor.cpp b/src/plugins/proxy/src/remote_tensor.cpp index fe6ca6b23c9..4ef5f5b3f1b 100644 --- a/src/plugins/proxy/src/remote_tensor.cpp +++ b/src/plugins/proxy/src/remote_tensor.cpp @@ -9,23 +9,26 @@ #include "openvino/proxy/plugin.hpp" namespace { -std::shared_ptr cast_tensor(const std::shared_ptr& tensor) { - auto rem_tensor = std::dynamic_pointer_cast(tensor); +std::shared_ptr cast_tensor(const ov::SoPtr& tensor) { + auto rem_tensor = std::dynamic_pointer_cast(tensor._ptr); OPENVINO_ASSERT(rem_tensor); return rem_tensor; } } // namespace -ov::proxy::RemoteTensor::RemoteTensor(ov::RemoteTensor&& tensor, const std::string& dev_name) +ov::proxy::RemoteTensor::RemoteTensor(const ov::SoPtr& tensor, const std::string& dev_name) : m_name(dev_name), - m_tensor(std::move(tensor)) {} - -ov::proxy::RemoteTensor::RemoteTensor(const ov::RemoteTensor& tensor, const std::string& dev_name) + m_tensor(tensor) { + OPENVINO_ASSERT(std::dynamic_pointer_cast(m_tensor._ptr)); +} +ov::proxy::RemoteTensor::RemoteTensor(ov::SoPtr&& tensor, const std::string& dev_name) : m_name(dev_name), - m_tensor(tensor) {} + m_tensor(std::move(tensor)) { + OPENVINO_ASSERT(m_tensor); +} const ov::AnyMap& ov::proxy::RemoteTensor::get_properties() const { - return cast_tensor(m_tensor._impl)->get_properties(); + return cast_tensor(m_tensor)->get_properties(); } const std::string& ov::proxy::RemoteTensor::get_device_name() const { @@ -33,36 +36,35 @@ const std::string& ov::proxy::RemoteTensor::get_device_name() const { } void ov::proxy::RemoteTensor::set_shape(ov::Shape shape) { - m_tensor.set_shape(shape); + m_tensor->set_shape(shape); } const ov::element::Type& ov::proxy::RemoteTensor::get_element_type() const { - return m_tensor.get_element_type(); + return m_tensor->get_element_type(); } const ov::Shape& ov::proxy::RemoteTensor::get_shape() const { - return m_tensor.get_shape(); + return m_tensor->get_shape(); } size_t ov::proxy::RemoteTensor::get_size() const { - return m_tensor.get_size(); + return m_tensor->get_size(); } size_t ov::proxy::RemoteTensor::get_byte_size() const { - return m_tensor.get_byte_size(); + return m_tensor->get_byte_size(); } const ov::Strides& ov::proxy::RemoteTensor::get_strides() const { - return cast_tensor(m_tensor._impl)->get_strides(); + return m_tensor->get_strides(); } -const std::shared_ptr& ov::proxy::RemoteTensor::get_hardware_tensor( - const std::shared_ptr& tensor) { - if (auto remote_tensor = std::dynamic_pointer_cast(tensor)) - return remote_tensor->m_tensor._impl; +const ov::SoPtr& ov::proxy::RemoteTensor::get_hardware_tensor(const ov::SoPtr& tensor) { + if (auto remote_tensor = std::dynamic_pointer_cast(tensor._ptr)) + return remote_tensor->m_tensor; return tensor; } -const std::shared_ptr& ov::proxy::get_hardware_tensor(const std::shared_ptr& tensor) { +const ov::SoPtr& ov::proxy::get_hardware_tensor(const ov::SoPtr& tensor) { return ov::proxy::RemoteTensor::get_hardware_tensor(tensor); } diff --git a/src/plugins/proxy/src/remote_tensor.hpp b/src/plugins/proxy/src/remote_tensor.hpp index a1cbde7c2d8..ca32974494e 100644 --- a/src/plugins/proxy/src/remote_tensor.hpp +++ b/src/plugins/proxy/src/remote_tensor.hpp @@ -4,6 +4,7 @@ #pragma once #include +#include #include "openvino/runtime/iremote_tensor.hpp" #include "openvino/runtime/remote_tensor.hpp" @@ -17,8 +18,8 @@ namespace proxy { */ class RemoteTensor : public ov::IRemoteTensor { public: - RemoteTensor(ov::RemoteTensor&& ctx, const std::string& dev_name); - RemoteTensor(const ov::RemoteTensor& ctx, const std::string& dev_name); + RemoteTensor(const ov::SoPtr& ctx, const std::string& dev_name); + RemoteTensor(ov::SoPtr&& ctx, const std::string& dev_name); const AnyMap& get_properties() const override; const std::string& get_device_name() const override; @@ -35,11 +36,11 @@ public: const ov::Strides& get_strides() const override; - static const std::shared_ptr& get_hardware_tensor(const std::shared_ptr& tensor); + static const ov::SoPtr& get_hardware_tensor(const ov::SoPtr& tensor); private: mutable std::string m_name; - ov::RemoteTensor m_tensor; + ov::SoPtr m_tensor; }; } // namespace proxy diff --git a/src/plugins/proxy/src/variable_state.hpp b/src/plugins/proxy/src/variable_state.hpp deleted file mode 100644 index 5ccdadb7293..00000000000 --- a/src/plugins/proxy/src/variable_state.hpp +++ /dev/null @@ -1,52 +0,0 @@ - -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include - -#include "openvino/runtime/ivariable_state.hpp" - -namespace ov { -namespace proxy { -/** - * @brief Simple wrapper for hardware variable states which holds plugin shared object - */ -class VariableState : public ov::IVariableState { - std::shared_ptr m_state; - std::shared_ptr m_so; - -public: - /** - * @brief Constructor of proxy VariableState - * - * @param state hardware state - * @param so shared object - */ - VariableState(const std::shared_ptr& state, const std::shared_ptr& so) - : IVariableState(""), - m_state(state), - m_so(so) { - OPENVINO_ASSERT(m_state); - } - const std::string& get_name() const override { - return m_state->get_name(); - } - - void reset() override { - m_state->reset(); - } - - void set_state(const ov::Tensor& state) override { - m_state->set_state(state); - } - - const ov::Tensor& get_state() const override { - return m_state->get_state(); - } -}; - -} // namespace proxy -} // namespace ov diff --git a/src/plugins/proxy/tests/proxy_tests.cpp b/src/plugins/proxy/tests/proxy_tests.cpp index 5be0eaebcba..4a7db474882 100644 --- a/src/plugins/proxy/tests/proxy_tests.cpp +++ b/src/plugins/proxy/tests/proxy_tests.cpp @@ -18,6 +18,7 @@ #include "openvino/runtime/iplugin.hpp" #include "openvino/runtime/iremote_context.hpp" #include "openvino/runtime/iremote_tensor.hpp" +#include "openvino/runtime/make_tensor.hpp" #include "openvino/runtime/properties.hpp" #include "openvino/util/file_util.hpp" #include "openvino/util/shared_object.hpp" @@ -158,7 +159,7 @@ public: MockCompiledModel(const std::shared_ptr& model, const std::shared_ptr& plugin, const ov::AnyMap& config, - const ov::RemoteContext& context) + const ov::SoPtr& context) : ov::ICompiledModel(model, plugin), m_config(config), m_model(model), @@ -189,7 +190,7 @@ public: return m_model; } - ov::RemoteContext get_context() const { + ov::SoPtr get_context() const { return m_context; } @@ -201,7 +202,7 @@ private: ov::AnyMap m_config; std::shared_ptr m_model; bool m_has_context; - ov::RemoteContext m_context; + ov::SoPtr m_context; }; class MockInferRequest : public ov::ISyncInferRequest { @@ -212,7 +213,7 @@ public: m_model = compiled_model->get_model(); // Allocate input/output tensors for (const auto& input : get_inputs()) { - allocate_tensor(input, [this, input, compiled_model](ov::Tensor& tensor) { + allocate_tensor(input, [this, input, compiled_model](ov::SoPtr& tensor) { // Can add a check to avoid double work in case of shared tensors allocate_tensor_impl(tensor, input.get_element_type(), @@ -222,7 +223,7 @@ public: }); } for (const auto& output : get_outputs()) { - allocate_tensor(output, [this, output, compiled_model](ov::Tensor& tensor) { + allocate_tensor(output, [this, output, compiled_model](ov::SoPtr& tensor) { // Can add a check to avoid double work in case of shared tensors allocate_tensor_impl(tensor, output.get_element_type(), @@ -237,15 +238,15 @@ public: void infer() override { ov::TensorVector input_tensors; for (const auto& input : get_inputs()) { - input_tensors.emplace_back(get_tensor(input)); + input_tensors.emplace_back(ov::make_tensor(get_tensor(input))); } ov::TensorVector output_tensors; for (const auto& output : get_outputs()) { - output_tensors.emplace_back(get_tensor(output)); + output_tensors.emplace_back(ov::make_tensor(get_tensor(output))); } m_model->evaluate(output_tensors, input_tensors); } - std::vector> query_state() const override { + std::vector> query_state() const override { OPENVINO_NOT_IMPLEMENTED; } std::vector get_profiling_info() const override { @@ -253,19 +254,19 @@ public: } private: - void allocate_tensor_impl(ov::Tensor& tensor, + void allocate_tensor_impl(ov::SoPtr& tensor, const ov::element::Type& element_type, const ov::Shape& shape, bool has_context, - ov::RemoteContext context) { - if (!tensor || tensor.get_element_type() != element_type) { + ov::SoPtr context) { + if (!tensor || tensor->get_element_type() != element_type) { if (has_context) { - tensor = context.create_tensor(element_type, shape, {}); + tensor = context->create_tensor(element_type, shape, {}); } else { - tensor = ov::Tensor(element_type, shape); + tensor = ov::SoPtr(ov::make_tensor(element_type, shape), nullptr); } } else { - tensor.set_shape(shape); + tensor->set_shape(shape); } } std::shared_ptr m_model; @@ -318,11 +319,11 @@ public: return m_property; } - std::shared_ptr create_tensor(const ov::element::Type& type, - const ov::Shape& shape, - const ov::AnyMap& params = {}) override { + ov::SoPtr create_tensor(const ov::element::Type& type, + const ov::Shape& shape, + const ov::AnyMap& params = {}) override { auto remote_tensor = std::make_shared(m_dev_name, m_property); - return remote_tensor; + return {remote_tensor, nullptr}; } }; @@ -340,11 +341,11 @@ public: return m_property; } - std::shared_ptr create_tensor(const ov::element::Type& type, - const ov::Shape& shape, - const ov::AnyMap& params = {}) override { + ov::SoPtr create_tensor(const ov::element::Type& type, + const ov::Shape& shape, + const ov::AnyMap& params = {}) override { auto remote_tensor = std::make_shared(m_dev_name, m_property); - return remote_tensor; + return {remote_tensor, nullptr}; } }; @@ -368,7 +369,7 @@ public: std::shared_ptr compile_model(const std::shared_ptr& model, const ov::AnyMap& properties, - const ov::RemoteContext& context) const override { + const ov::SoPtr& context) const override { if (!support_model(model, query_model(model, properties))) OPENVINO_THROW("Unsupported model"); @@ -383,13 +384,13 @@ public: OPENVINO_NOT_IMPLEMENTED; } - std::shared_ptr create_context(const ov::AnyMap& remote_properties) const override { + ov::SoPtr create_context(const ov::AnyMap& remote_properties) const override { if (remote_properties.find("CUSTOM_CTX") == remote_properties.end()) return std::make_shared(get_device_name()); return std::make_shared(get_device_name()); } - std::shared_ptr get_default_context(const ov::AnyMap& remote_properties) const override { + ov::SoPtr get_default_context(const ov::AnyMap& remote_properties) const override { return std::make_shared(get_device_name()); } @@ -424,7 +425,7 @@ public: } std::shared_ptr import_model(std::istream& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& properties) const override { std::string xmlString, xmlInOutString; ov::Tensor weights; diff --git a/src/plugins/proxy/tests/proxy_tests.hpp b/src/plugins/proxy/tests/proxy_tests.hpp index 9bebfe0a36f..c2f94dae0c6 100644 --- a/src/plugins/proxy/tests/proxy_tests.hpp +++ b/src/plugins/proxy/tests/proxy_tests.hpp @@ -8,6 +8,7 @@ #include #include "openvino/runtime/core.hpp" +#include "openvino/runtime/iplugin.hpp" namespace ov { namespace proxy { diff --git a/src/plugins/template/src/compiled_model.cpp b/src/plugins/template/src/compiled_model.cpp index a929d4fbc36..49a61d5c005 100644 --- a/src/plugins/template/src/compiled_model.cpp +++ b/src/plugins/template/src/compiled_model.cpp @@ -20,7 +20,7 @@ // ! [compiled_model:ctor] ov::template_plugin::CompiledModel::CompiledModel(const std::shared_ptr& model, const std::shared_ptr& plugin, - const ov::RemoteContext& context, + const ov::SoPtr& context, const std::shared_ptr& task_executor, const Configuration& cfg, bool loaded_from_cache) diff --git a/src/plugins/template/src/compiled_model.hpp b/src/plugins/template/src/compiled_model.hpp index 911edf72abb..47c8a14a234 100644 --- a/src/plugins/template/src/compiled_model.hpp +++ b/src/plugins/template/src/compiled_model.hpp @@ -25,7 +25,7 @@ class CompiledModel : public ov::ICompiledModel { public: CompiledModel(const std::shared_ptr& model, const std::shared_ptr& plugin, - const ov::RemoteContext& context, + const ov::SoPtr& context, const std::shared_ptr& task_executor, const Configuration& cfg, bool loaded_from_cache = false); diff --git a/src/plugins/template/src/plugin.cpp b/src/plugins/template/src/plugin.cpp index 56bcce2c932..6ae41bac45f 100644 --- a/src/plugins/template/src/plugin.cpp +++ b/src/plugins/template/src/plugin.cpp @@ -47,14 +47,13 @@ ov::template_plugin::Plugin::~Plugin() { // ! [plugin:dtor] // ! [plugin:create_context] -std::shared_ptr ov::template_plugin::Plugin::create_context( - const ov::AnyMap& remote_properties) const { +ov::SoPtr ov::template_plugin::Plugin::create_context(const ov::AnyMap& remote_properties) const { return std::make_shared(); } // ! [plugin:create_context] // ! [plugin:get_default_context] -std::shared_ptr ov::template_plugin::Plugin::get_default_context( +ov::SoPtr ov::template_plugin::Plugin::get_default_context( const ov::AnyMap& remote_properties) const { return std::make_shared(); } @@ -97,7 +96,7 @@ std::shared_ptr ov::template_plugin::Plugin::compile_model( std::shared_ptr ov::template_plugin::Plugin::compile_model( const std::shared_ptr& model, const ov::AnyMap& properties, - const ov::RemoteContext& context) const { + const ov::SoPtr& context) const { OV_ITT_SCOPED_TASK(itt::domains::TemplatePlugin, "Plugin::compile_model"); auto fullConfig = Configuration{properties, m_cfg}; @@ -124,9 +123,10 @@ std::shared_ptr ov::template_plugin::Plugin::import_model(st // ! [plugin:import_model] // ! [plugin:import_model_with_remote] -std::shared_ptr ov::template_plugin::Plugin::import_model(std::istream& model, - const ov::RemoteContext& context, - const ov::AnyMap& properties) const { +std::shared_ptr ov::template_plugin::Plugin::import_model( + std::istream& model, + const ov::SoPtr& context, + const ov::AnyMap& properties) const { OV_ITT_SCOPED_TASK(itt::domains::TemplatePlugin, "Plugin::import_model"); auto fullConfig = Configuration{properties, m_cfg}; diff --git a/src/plugins/template/src/plugin.hpp b/src/plugins/template/src/plugin.hpp index 3382a202d1c..aa2d6d1f42a 100644 --- a/src/plugins/template/src/plugin.hpp +++ b/src/plugins/template/src/plugin.hpp @@ -25,20 +25,20 @@ public: std::shared_ptr compile_model(const std::shared_ptr& model, const ov::AnyMap& properties, - const ov::RemoteContext& context) const override; + const ov::SoPtr& context) const override; void set_property(const ov::AnyMap& properties) override; ov::Any get_property(const std::string& name, const ov::AnyMap& arguments) const override; - std::shared_ptr create_context(const ov::AnyMap& remote_properties) const override; + ov::SoPtr create_context(const ov::AnyMap& remote_properties) const override; - std::shared_ptr get_default_context(const ov::AnyMap& remote_properties) const override; + ov::SoPtr get_default_context(const ov::AnyMap& remote_properties) const override; std::shared_ptr import_model(std::istream& model, const ov::AnyMap& properties) const override; std::shared_ptr import_model(std::istream& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& properties) const override; ov::SupportedOpsMap query_model(const std::shared_ptr& model, diff --git a/src/plugins/template/src/remote_context.cpp b/src/plugins/template/src/remote_context.cpp index 1052add9a93..712144d169c 100644 --- a/src/plugins/template/src/remote_context.cpp +++ b/src/plugins/template/src/remote_context.cpp @@ -149,9 +149,9 @@ const ov::AnyMap& ov::template_plugin::RemoteContext::get_property() const { // ! [remote_context:get_property] // ! [remote_context:create_tensor] -std::shared_ptr ov::template_plugin::RemoteContext::create_tensor(const ov::element::Type& type, - const ov::Shape& shape, - const ov::AnyMap& params) { +ov::SoPtr ov::template_plugin::RemoteContext::create_tensor(const ov::element::Type& type, + const ov::Shape& shape, + const ov::AnyMap& params) { std::shared_ptr tensor; switch (type) { @@ -199,6 +199,6 @@ std::shared_ptr ov::template_plugin::RemoteContext::create_te default: OPENVINO_THROW("Cannot create remote tensor for unsupported type: ", type); } - return std::make_shared(tensor); + return {std::make_shared(tensor), nullptr}; } // ! [remote_context:create_tensor] diff --git a/src/plugins/template/src/remote_context.hpp b/src/plugins/template/src/remote_context.hpp index ff00157406e..02006d671dc 100644 --- a/src/plugins/template/src/remote_context.hpp +++ b/src/plugins/template/src/remote_context.hpp @@ -13,9 +13,9 @@ public: RemoteContext(); const std::string& get_device_name() const override; const ov::AnyMap& get_property() const override; - std::shared_ptr create_tensor(const ov::element::Type& type, - const ov::Shape& shape, - const ov::AnyMap& params = {}) override; + ov::SoPtr create_tensor(const ov::element::Type& type, + const ov::Shape& shape, + const ov::AnyMap& params = {}) override; private: std::string m_name; diff --git a/src/plugins/template/src/sync_infer_request.cpp b/src/plugins/template/src/sync_infer_request.cpp index 96f0bba85c0..41fadbd1588 100644 --- a/src/plugins/template/src/sync_infer_request.cpp +++ b/src/plugins/template/src/sync_infer_request.cpp @@ -14,6 +14,7 @@ #include "openvino/core/except.hpp" #include "openvino/op/util/variable_context.hpp" #include "openvino/runtime/ivariable_state.hpp" +#include "openvino/runtime/make_tensor.hpp" #include "openvino/runtime/profiling_info.hpp" #include "openvino/runtime/tensor.hpp" #include "plugin.hpp" @@ -24,11 +25,13 @@ using Time = std::chrono::high_resolution_clock; namespace { -void allocate_tensor_impl(ov::Tensor& tensor, const ov::element::Type& element_type, const ov::Shape& shape) { - if (!tensor || tensor.get_element_type() != element_type) { - tensor = ov::Tensor(element_type, shape); +void allocate_tensor_impl(ov::SoPtr& tensor, + const ov::element::Type& element_type, + const ov::Shape& shape) { + if (!tensor || tensor->get_element_type() != element_type) { + tensor = ov::make_tensor(element_type, shape); } else { - tensor.set_shape(shape); + tensor->set_shape(shape); } } @@ -61,7 +64,7 @@ ov::template_plugin::InferRequest::InferRequest(const std::shared_ptr& tensor) { // Can add a check to avoid double work in case of shared tensors allocate_tensor_impl(tensor, input.get_element_type(), @@ -69,7 +72,7 @@ ov::template_plugin::InferRequest::InferRequest(const std::shared_ptr& tensor) { // Can add a check to avoid double work in case of shared tensors allocate_tensor_impl(tensor, output.get_element_type(), @@ -86,8 +89,9 @@ ov::template_plugin::InferRequest::InferRequest(const std::shared_ptrget_info().data_type, shape); variable_context.set_variable_value(variable, std::make_shared(tensor)); } - auto state = std::make_shared(variable->get_info().variable_id, - variable_context.get_variable_value(variable)->get_state()); + auto state = std::make_shared( + variable->get_info().variable_id, + get_tensor_impl(variable_context.get_variable_value(variable)->get_state())); m_variable_states.emplace_back(state); } m_eval_context.emplace("VariableContext", variable_context); @@ -100,7 +104,7 @@ ov::template_plugin::InferRequest::~InferRequest() = default; // ! [infer_request:set_tensors_impl] void ov::template_plugin::InferRequest::set_tensors_impl(const ov::Output port, - const std::vector& tensors) { + const std::vector>& tensors) { for (const auto& input : get_inputs()) { if (input == port) { m_batched_tensors[input.get_tensor_ptr()] = tensors; @@ -112,7 +116,7 @@ void ov::template_plugin::InferRequest::set_tensors_impl(const ov::Output> ov::template_plugin::InferRequest::query_state() const { +std::vector> ov::template_plugin::InferRequest::query_state() const { return m_variable_states; } // ! [infer_request:query_state] @@ -145,7 +149,7 @@ void ov::template_plugin::InferRequest::infer_preprocess() { // Allocate backend tensors OPENVINO_ASSERT(get_inputs().size() == m_backend_input_tensors.size()); for (size_t i = 0; i < get_inputs().size(); i++) { - auto tensor = get_tensor(get_inputs()[i]); + auto tensor = make_tensor(get_tensor(get_inputs()[i])); if (tensor.is()) { OPENVINO_ASSERT(tensor.is(), "Template plugin supports only VectorTensor with remote context."); @@ -187,7 +191,7 @@ void ov::template_plugin::InferRequest::infer_preprocess() { m_backend_output_tensors[i] = get_template_model()->get_template_plugin()->m_backend->create_tensor(); continue; } - auto tensor = get_tensor(get_outputs()[i]); + auto tensor = make_tensor(get_tensor(get_outputs()[i])); if (tensor.is_continuous() && !tensor.is()) m_backend_output_tensors[i] = get_template_model()->get_template_plugin()->m_backend->create_tensor(tensor.get_element_type(), @@ -232,12 +236,13 @@ void ov::template_plugin::InferRequest::infer_postprocess() { for (size_t i = 0; i < get_outputs().size(); i++) { const auto& result = get_template_model()->m_model->get_results()[i]; auto host_tensor = m_backend_output_tensors[i]; - auto tensor = get_tensor(get_outputs()[i]); + auto tensor = make_tensor(get_tensor(get_outputs()[i])); if (result->get_output_partial_shape(0).is_dynamic()) { ov::Output output{result->output(0).get_node(), result->output(0).get_index()}; - allocate_tensor(output, [host_tensor](ov::Tensor& tensor) { + allocate_tensor(output, [host_tensor](ov::SoPtr& tensor) { allocate_tensor_impl(tensor, host_tensor.get_element_type(), host_tensor.get_shape()); - host_tensor.copy_to(tensor); + auto ov_tensor = make_tensor(tensor); + host_tensor.copy_to(ov_tensor); }); } else if (!tensor.is_continuous()) { host_tensor.copy_to(tensor); diff --git a/src/plugins/template/src/sync_infer_request.hpp b/src/plugins/template/src/sync_infer_request.hpp index fa75024e3b7..14bc6972cca 100644 --- a/src/plugins/template/src/sync_infer_request.hpp +++ b/src/plugins/template/src/sync_infer_request.hpp @@ -31,7 +31,7 @@ public: ~InferRequest(); void infer() override; - std::vector> query_state() const override; + std::vector> query_state() const override; std::vector get_profiling_info() const override; // pipeline methods-stages which are used in async infer request implementation and assigned to particular executor @@ -41,7 +41,8 @@ public: void infer_postprocess(); void cancel(); - void set_tensors_impl(const ov::Output port, const std::vector& tensors) override; + void set_tensors_impl(const ov::Output port, + const std::vector>& tensors) override; private: std::shared_ptr get_template_model() const; @@ -56,7 +57,7 @@ private: std::vector m_backend_output_tensors; std::shared_ptr m_executable; ov::EvaluationContext m_eval_context; - std::vector> m_variable_states; + std::vector> m_variable_states; }; // ! [infer_request:header] diff --git a/src/plugins/template/src/variable_state.hpp b/src/plugins/template/src/variable_state.hpp index 2dfe4111010..e635f9b3285 100644 --- a/src/plugins/template/src/variable_state.hpp +++ b/src/plugins/template/src/variable_state.hpp @@ -11,18 +11,18 @@ namespace template_plugin { class VariableState : public ov::IVariableState { public: - VariableState(const std::string& name, const ov::Tensor& tensor) : ov::IVariableState(name) { + VariableState(const std::string& name, const ov::SoPtr& tensor) : ov::IVariableState(name) { m_state = tensor; } - void set_state(const ov::Tensor& state) override { - OPENVINO_ASSERT(state.get_shape() == m_state.get_shape(), "Wrong tensor shape."); - OPENVINO_ASSERT(state.get_element_type() == state.get_element_type(), "Wrong tensor type."); - OPENVINO_ASSERT(state.get_byte_size() == state.get_byte_size(), "Blob size of tensors are not equal."); - std::memcpy(m_state.data(), state.data(), state.get_byte_size()); + void set_state(const ov::SoPtr& state) override { + OPENVINO_ASSERT(state->get_shape() == m_state->get_shape(), "Wrong tensor shape."); + OPENVINO_ASSERT(state->get_element_type() == state->get_element_type(), "Wrong tensor type."); + OPENVINO_ASSERT(state->get_byte_size() == state->get_byte_size(), "Blob size of tensors are not equal."); + std::memcpy(m_state->data(), state->data(), state->get_byte_size()); } void reset() override { - std::memset(m_state.data(), 0, m_state.get_byte_size()); + std::memset(m_state->data(), 0, m_state->get_byte_size()); } ~VariableState() override = default; diff --git a/src/tests/ie_test_utils/unit_test_utils/mocks/cpp_interfaces/interface/mock_icore.hpp b/src/tests/ie_test_utils/unit_test_utils/mocks/cpp_interfaces/interface/mock_icore.hpp index 97586d91490..804daf35a27 100644 --- a/src/tests/ie_test_utils/unit_test_utils/mocks/cpp_interfaces/interface/mock_icore.hpp +++ b/src/tests/ie_test_utils/unit_test_utils/mocks/cpp_interfaces/interface/mock_icore.hpp @@ -67,7 +67,8 @@ public: MOCK_METHOD1(GetDefaultContext, InferenceEngine::RemoteContext::Ptr(const std::string&)); MOCK_CONST_METHOD0(is_new_api, bool()); - MOCK_CONST_METHOD2(create_context, ov::RemoteContext(const std::string& deviceName, const ov::AnyMap& params)); + MOCK_CONST_METHOD2(create_context, + ov::SoPtr(const std::string& deviceName, const ov::AnyMap& params)); MOCK_CONST_METHOD0(get_available_devices, std::vector()); MOCK_CONST_METHOD3(query_model, ov::SupportedOpsMap(const std::shared_ptr&, @@ -81,7 +82,7 @@ public: const ov::AnyMap&)); MOCK_CONST_METHOD3(compile_model, ov::SoPtr(const std::shared_ptr&, - const ov::RemoteContext&, + const ov::SoPtr&, const ov::AnyMap&)); MOCK_CONST_METHOD3(compile_model, ov::SoPtr(const std::string&, const std::string&, const ov::AnyMap&)); @@ -90,9 +91,11 @@ public: ov::SoPtr(const std::string&, const ov::Tensor&, const std::string&, const ov::AnyMap&)); MOCK_CONST_METHOD3(read_model, std::shared_ptr(const std::string&, const ov::Tensor&, bool)); MOCK_CONST_METHOD2(read_model, std::shared_ptr(const std::string&, const std::string&)); - MOCK_CONST_METHOD1(get_default_context, ov::RemoteContext(const std::string&)); + MOCK_CONST_METHOD1(get_default_context, ov::SoPtr(const std::string&)); MOCK_CONST_METHOD3(import_model, - ov::SoPtr(std::istream&, const ov::RemoteContext&, const ov::AnyMap&)); + ov::SoPtr(std::istream&, + const ov::SoPtr&, + const ov::AnyMap&)); MOCK_CONST_METHOD1(device_supports_model_caching, bool(const std::string&)); MOCK_METHOD2(set_property, void(const std::string& device_name, const ov::AnyMap& properties)); diff --git a/src/tests/ie_test_utils/unit_test_utils/mocks/mock_engine/mock_plugin.cpp b/src/tests/ie_test_utils/unit_test_utils/mocks/mock_engine/mock_plugin.cpp index 3022cc95e57..bac255154d2 100644 --- a/src/tests/ie_test_utils/unit_test_utils/mocks/mock_engine/mock_plugin.cpp +++ b/src/tests/ie_test_utils/unit_test_utils/mocks/mock_engine/mock_plugin.cpp @@ -52,7 +52,7 @@ public: std::shared_ptr compile_model(const std::shared_ptr& model, const ov::AnyMap& properties, - const ov::RemoteContext& context) const override { + const ov::SoPtr& context) const override { if (m_plugin) return m_plugin->compile_model(model, properties, context); OPENVINO_NOT_IMPLEMENTED; @@ -71,13 +71,13 @@ public: OPENVINO_NOT_IMPLEMENTED; } - std::shared_ptr create_context(const ov::AnyMap& remote_properties) const override { + ov::SoPtr create_context(const ov::AnyMap& remote_properties) const override { if (m_plugin) return m_plugin->create_context(remote_properties); OPENVINO_NOT_IMPLEMENTED; } - std::shared_ptr get_default_context(const ov::AnyMap& remote_properties) const override { + ov::SoPtr get_default_context(const ov::AnyMap& remote_properties) const override { if (m_plugin) return m_plugin->get_default_context(remote_properties); OPENVINO_NOT_IMPLEMENTED; @@ -90,7 +90,7 @@ public: } std::shared_ptr import_model(std::istream& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& properties) const override { if (m_plugin) return m_plugin->import_model(model, context, properties); @@ -160,17 +160,17 @@ std::shared_ptr MockPlugin::compile_model(const std::string& std::shared_ptr MockPlugin::compile_model(const std::shared_ptr& model, const ov::AnyMap& properties, - const ov::RemoteContext& context) const { + const ov::SoPtr& context) const { set_parameters_if_need(); return m_plugin->compile_model(model, properties, context); } -std::shared_ptr MockPlugin::create_context(const ov::AnyMap& remote_properties) const { +ov::SoPtr MockPlugin::create_context(const ov::AnyMap& remote_properties) const { set_parameters_if_need(); return m_plugin->create_context(remote_properties); } -std::shared_ptr MockPlugin::get_default_context(const ov::AnyMap& remote_properties) const { +ov::SoPtr MockPlugin::get_default_context(const ov::AnyMap& remote_properties) const { set_parameters_if_need(); return m_plugin->get_default_context(remote_properties); } @@ -180,7 +180,7 @@ std::shared_ptr MockPlugin::import_model(std::istream& model return m_plugin->import_model(model, properties); } std::shared_ptr MockPlugin::import_model(std::istream& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& properties) const { set_parameters_if_need(); return m_plugin->import_model(model, context, properties); diff --git a/src/tests/ie_test_utils/unit_test_utils/mocks/mock_engine/mock_plugin.hpp b/src/tests/ie_test_utils/unit_test_utils/mocks/mock_engine/mock_plugin.hpp index 95cac305216..505eeca3a32 100644 --- a/src/tests/ie_test_utils/unit_test_utils/mocks/mock_engine/mock_plugin.hpp +++ b/src/tests/ie_test_utils/unit_test_utils/mocks/mock_engine/mock_plugin.hpp @@ -28,19 +28,19 @@ public: std::shared_ptr compile_model(const std::shared_ptr& model, const ov::AnyMap& properties, - const ov::RemoteContext& context) const override; + const ov::SoPtr& context) const override; void set_property(const ov::AnyMap& properties) override; ov::Any get_property(const std::string& name, const ov::AnyMap& arguments) const override; - std::shared_ptr create_context(const ov::AnyMap& remote_properties) const override; + ov::SoPtr create_context(const ov::AnyMap& remote_properties) const override; - std::shared_ptr get_default_context(const ov::AnyMap& remote_properties) const override; + ov::SoPtr get_default_context(const ov::AnyMap& remote_properties) const override; std::shared_ptr import_model(std::istream& model, const ov::AnyMap& properties) const override; std::shared_ptr import_model(std::istream& model, - const ov::RemoteContext& context, + const ov::SoPtr& context, const ov::AnyMap& properties) const override; ov::SupportedOpsMap query_model(const std::shared_ptr& model, const ov::AnyMap& properties) const override;