diff --git a/docs/template_plugin/src/template_executable_network.cpp b/docs/template_plugin/src/template_executable_network.cpp index 237cbeaa712..8479a027550 100644 --- a/docs/template_plugin/src/template_executable_network.cpp +++ b/docs/template_plugin/src/template_executable_network.cpp @@ -151,11 +151,23 @@ InferenceEngine::IInferRequestInternal::Ptr TemplatePlugin::ExecutableNetwork::C networkOutputs, std::static_pointer_cast(shared_from_this())); } + +InferenceEngine::IInferRequestInternal::Ptr TemplatePlugin::ExecutableNetwork::CreateInferRequestImpl( + const std::vector>& inputs, + const std::vector>& outputs) { + return std::make_shared(inputs, + outputs, + std::static_pointer_cast(shared_from_this())); +} // ! [executable_network:create_infer_request_impl] // ! [executable_network:create_infer_request] InferenceEngine::IInferRequestInternal::Ptr TemplatePlugin::ExecutableNetwork::CreateInferRequest() { - auto internalRequest = CreateInferRequestImpl(_networkInputs, _networkOutputs); + InferenceEngine::IInferRequestInternal::Ptr internalRequest; + if (this->_plugin && this->_plugin->GetCore() && this->_plugin->GetCore()->isNewAPI()) + internalRequest = CreateInferRequestImpl(_parameters, _results); + if (!internalRequest) + internalRequest = CreateInferRequestImpl(_networkInputs, _networkOutputs); return std::make_shared(std::static_pointer_cast(internalRequest), _taskExecutor, _plugin->_waitExecutor, diff --git a/docs/template_plugin/src/template_executable_network.hpp b/docs/template_plugin/src/template_executable_network.hpp index bd6e2da53af..bea96e428c3 100644 --- a/docs/template_plugin/src/template_executable_network.hpp +++ b/docs/template_plugin/src/template_executable_network.hpp @@ -37,6 +37,9 @@ public: InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl( InferenceEngine::InputsDataMap networkInputs, InferenceEngine::OutputsDataMap networkOutputs) override; + InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl( + const std::vector>& inputs, + const std::vector>& outputs) override; InferenceEngine::IInferRequestInternal::Ptr CreateInferRequest() override; InferenceEngine::Parameter GetMetric(const std::string& name) const override; InferenceEngine::Parameter GetConfig(const std::string& name) const override; diff --git a/docs/template_plugin/src/template_infer_request.cpp b/docs/template_plugin/src/template_infer_request.cpp index 8030becb3e0..256f4b503a9 100644 --- a/docs/template_plugin/src/template_infer_request.cpp +++ b/docs/template_plugin/src/template_infer_request.cpp @@ -35,6 +35,18 @@ TemplateInferRequest::TemplateInferRequest(const InferenceEngine::InputsDataMap& const std::shared_ptr& executableNetwork) : IInferRequestInternal(networkInputs, networkOutputs), _executableNetwork(executableNetwork) { + createInferRequest(); +} + +TemplateInferRequest::TemplateInferRequest(const std::vector>& inputs, + const std::vector>& outputs, + const std::shared_ptr& executableNetwork) + : IInferRequestInternal(inputs, outputs), + _executableNetwork(executableNetwork) { + createInferRequest(); +} + +void TemplateInferRequest::createInferRequest() { // TODO: allocate infer request device and host buffers if needed, fill actual list of profiling tasks auto requestID = std::to_string(_executableNetwork->_requestId.fetch_add(1)); diff --git a/docs/template_plugin/src/template_infer_request.hpp b/docs/template_plugin/src/template_infer_request.hpp index f7e9a4f9952..12fa18ceea6 100644 --- a/docs/template_plugin/src/template_infer_request.hpp +++ b/docs/template_plugin/src/template_infer_request.hpp @@ -29,6 +29,9 @@ public: TemplateInferRequest(const InferenceEngine::InputsDataMap& networkInputs, const InferenceEngine::OutputsDataMap& networkOutputs, const std::shared_ptr& executableNetwork); + TemplateInferRequest(const std::vector>& inputs, + const std::vector>& outputs, + const std::shared_ptr& executableNetwork); ~TemplateInferRequest(); void InferImpl() override; @@ -44,6 +47,7 @@ public: void SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr& userBlob) override; private: + void createInferRequest(); void allocateDeviceBuffers(); void allocateBlobs(); diff --git a/docs/template_plugin/tests/functional/op_reference/base_reference_test.cpp b/docs/template_plugin/tests/functional/op_reference/base_reference_test.cpp index d2760cd3b8e..b498fae1097 100644 --- a/docs/template_plugin/tests/functional/op_reference/base_reference_test.cpp +++ b/docs/template_plugin/tests/functional/op_reference/base_reference_test.cpp @@ -57,17 +57,15 @@ void CommonReferenceTest::Infer() { const auto& functionParams = function->get_parameters(); for (size_t i = 0; i < functionParams.size(); ++i) { - const auto& param = functionParams[i]; - inferRequest.set_tensor(param->get_friendly_name(), inputData[i]); + inferRequest.set_tensor(executableNetwork.input(i), inputData[i]); } inferRequest.infer(); } void CommonReferenceTest::Validate() { ASSERT_EQ(executableNetwork.outputs().size(), refOutData.size()); - for (const auto& result : function->get_results()) { - auto name = ngraph::op::util::create_ie_output_name(result->input_value(0)); - actualOutData.emplace_back(inferRequest.get_tensor(name)); + for (const auto& output : executableNetwork.outputs()) { + actualOutData.emplace_back(inferRequest.get_tensor(output)); } ASSERT_EQ(refOutData.size(), actualOutData.size()); diff --git a/inference-engine/src/cldnn_engine/cldnn_executable_network.cpp b/inference-engine/src/cldnn_engine/cldnn_executable_network.cpp index 31c31ac2e6f..79f951dd2e1 100644 --- a/inference-engine/src/cldnn_engine/cldnn_executable_network.cpp +++ b/inference-engine/src/cldnn_engine/cldnn_executable_network.cpp @@ -26,6 +26,7 @@ #include "cldnn_executable_network.h" #include "threading/ie_cpu_streams_executor.hpp" #include "cpp_interfaces/interface/ie_iinfer_request_internal.hpp" +#include "ie_icore.hpp" using namespace InferenceEngine; using namespace InferenceEngine::details; @@ -66,6 +67,36 @@ CLDNNExecNetwork::CLDNNExecNetwork(InferenceEngine::CNNNetwork &network, std::sh IInferRequestInternal::Ptr CLDNNExecNetwork::CreateInferRequestImpl(InputsDataMap networkInputs, OutputsDataMap networkOutputs) { OV_ITT_SCOPED_TASK(itt::domains::CLDNNPlugin, "CLDNNExecNetwork::CreateInferRequestImpl"); + auto ptr = std::make_shared(networkInputs, networkOutputs, + std::static_pointer_cast(shared_from_this())); + if (m_config.throughput_streams > 1) { + ptr->EnableStreams(); + } + if (m_config.useProfiling) + ptr->EnableProfiling(); + ptr->SetGraph(m_graphs.front()); + + return ptr; +} + +IInferRequestInternal::Ptr CLDNNExecNetwork::CreateInferRequestImpl(const std::vector>& inputs, + const std::vector>& outputs) { + OV_ITT_SCOPED_TASK(itt::domains::CLDNNPlugin, "CLDNNExecNetwork::CreateInferRequestImpl"); + auto ptr = std::make_shared(inputs, outputs, + std::static_pointer_cast(shared_from_this())); + if (m_config.throughput_streams > 1) { + ptr->EnableStreams(); + } + if (m_config.useProfiling) + ptr->EnableProfiling(); + ptr->SetGraph(m_graphs.front()); + + return ptr; +} + +IInferRequestInternal::Ptr CLDNNExecNetwork::CreateInferRequest() { + OV_ITT_SCOPED_TASK(itt::domains::CLDNNPlugin, "CLDNNExecNetwork::CreateInferRequest"); + InferenceEngine::IInferRequestInternal::Ptr internalRequest; if (m_graphs.empty()) { IE_THROW(NetworkNotLoaded); } @@ -80,21 +111,10 @@ IInferRequestInternal::Ptr CLDNNExecNetwork::CreateInferRequestImpl(InputsDataMa } } - auto ptr = std::make_shared(networkInputs, networkOutputs, - std::static_pointer_cast(shared_from_this())); - if (m_config.throughput_streams > 1) { - ptr->EnableStreams(); - } - if (m_config.useProfiling) - ptr->EnableProfiling(); - ptr->SetGraph(m_graphs.front()); - - return ptr; -} - -IInferRequestInternal::Ptr CLDNNExecNetwork::CreateInferRequest() { - OV_ITT_SCOPED_TASK(itt::domains::CLDNNPlugin, "CLDNNExecNetwork::CreateInferRequest"); - auto internalRequest = CreateInferRequestImpl(_networkInputs, _networkOutputs); + if (this->_plugin && this->_plugin->GetCore() && this->_plugin->GetCore()->isNewAPI()) + internalRequest = CreateInferRequestImpl(_parameters, _results); + if (!internalRequest) + internalRequest = CreateInferRequestImpl(_networkInputs, _networkOutputs); internalRequest->setPointerToExecutableNetworkInternal(shared_from_this()); return std::make_shared(std::static_pointer_cast(internalRequest), m_taskExecutor, diff --git a/inference-engine/src/cldnn_engine/cldnn_executable_network.h b/inference-engine/src/cldnn_engine/cldnn_executable_network.h index 63aba11c72c..7ecb72a738f 100644 --- a/inference-engine/src/cldnn_engine/cldnn_executable_network.h +++ b/inference-engine/src/cldnn_engine/cldnn_executable_network.h @@ -29,6 +29,8 @@ public: InferenceEngine::IInferRequestInternal::Ptr CreateInferRequest() override; InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs, InferenceEngine::OutputsDataMap networkOutputs) override; + InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl(const std::vector>& inputs, + const std::vector>& outputs) override; InferenceEngine::Parameter GetMetric(const std::string &name) const override; InferenceEngine::Parameter GetConfig(const std::string &name) const override; diff --git a/inference-engine/src/cldnn_engine/cldnn_infer_request.cpp b/inference-engine/src/cldnn_engine/cldnn_infer_request.cpp index a8eebaf891a..6577fce8f01 100644 --- a/inference-engine/src/cldnn_engine/cldnn_infer_request.cpp +++ b/inference-engine/src/cldnn_engine/cldnn_infer_request.cpp @@ -457,6 +457,16 @@ CLDNNInferRequest::CLDNNInferRequest(InputsDataMap networkInputs, OutputsDataMap streamExecutor = dynamic_cast(execNetwork->m_taskExecutor.get()); } +CLDNNInferRequest::CLDNNInferRequest(const std::vector>& inputs, + const std::vector>& outputs, + const CLDNNExecNetwork::Ptr& execNetwork) + : IInferRequestInternal(inputs, outputs) + , m_useProfiling(false) + , m_useStreams(false) { + IE_ASSERT(nullptr != execNetwork); + streamExecutor = dynamic_cast(execNetwork->m_taskExecutor.get()); +} + // ----------------------------------------------------------------------------------------- // // ---------------------------- internal pipeline stages ----------------------------------- // // ----------------------------------------------------------------------------------------- // diff --git a/inference-engine/src/cldnn_engine/cldnn_infer_request.h b/inference-engine/src/cldnn_engine/cldnn_infer_request.h index a2095067821..c90836f65b0 100644 --- a/inference-engine/src/cldnn_engine/cldnn_infer_request.h +++ b/inference-engine/src/cldnn_engine/cldnn_infer_request.h @@ -33,6 +33,9 @@ public: CLDNNInferRequest(InferenceEngine::InputsDataMap networkInputs, InferenceEngine::OutputsDataMap networkOutputs, const std::shared_ptr& execNetwork); + CLDNNInferRequest(const std::vector>& inputs, + const std::vector>& outputs, + const std::shared_ptr& execNetwork); CLDNNInferRequest(const CLDNNInferRequest &) = delete; diff --git a/inference-engine/src/gna_plugin/gna_executable_network.hpp b/inference-engine/src/gna_plugin/gna_executable_network.hpp index 40b81ebb84f..d7a9cd1bd71 100644 --- a/inference-engine/src/gna_plugin/gna_executable_network.hpp +++ b/inference-engine/src/gna_plugin/gna_executable_network.hpp @@ -13,6 +13,7 @@ #include #include #include +#include namespace GNAPluginNS { @@ -58,6 +59,14 @@ class GNAExecutableNetwork : public InferenceEngine::IExecutableNetworkInternal return std::make_shared(plg, networkInputs, networkOutputs); } + InferenceEngine::IInferRequestInternal::Ptr + CreateInferRequestImpl(const std::vector>& inputs, + const std::vector>& outputs) override { + if (!this->_plugin || !this->_plugin->GetCore() || !this->_plugin->GetCore()->isNewAPI()) + return nullptr; + return std::make_shared(plg, inputs, outputs); + } + INFERENCE_ENGINE_DEPRECATED("Use InferRequest::QueryState instead") std::vector QueryState() override { IE_SUPPRESS_DEPRECATED_START diff --git a/inference-engine/src/gna_plugin/gna_infer_request.hpp b/inference-engine/src/gna_plugin/gna_infer_request.hpp index 2aaf21341c3..1d2e1312357 100644 --- a/inference-engine/src/gna_plugin/gna_infer_request.hpp +++ b/inference-engine/src/gna_plugin/gna_infer_request.hpp @@ -15,17 +15,10 @@ namespace GNAPluginNS { class GNAInferRequest : public InferenceEngine::IInferRequestInternal { - protected: - std::shared_ptr plg; - uint32_t inferRequestIdx = -1; - - public: - GNAInferRequest(const std::shared_ptr& plg, - InferenceEngine::InputsDataMap networkInputs, - InferenceEngine::OutputsDataMap networkOutputs) - : InferenceEngine::IInferRequestInternal(networkInputs, networkOutputs), plg(plg) { + private: + void CreateInferRequest() { // TODO: internal connection API - better to generalize - if (networkOutputs.empty()) { + if (_networkOutputs.empty()) { THROW_GNA_EXCEPTION << "GNAInferRequest :: network has zero outputs"; } @@ -40,6 +33,24 @@ class GNAInferRequest : public InferenceEngine::IInferRequestInternal { plg->GetInputBlob(input.first, input.second->getTensorDesc().getPrecision()); } } + + protected: + std::shared_ptr plg; + uint32_t inferRequestIdx = -1; + + public: + GNAInferRequest(const std::shared_ptr& plg, + const std::vector>& inputs, + const std::vector>& outputs) + : InferenceEngine::IInferRequestInternal(inputs, outputs), plg(plg) { + CreateInferRequest(); + } + GNAInferRequest(const std::shared_ptr& plg, + InferenceEngine::InputsDataMap networkInputs, + InferenceEngine::OutputsDataMap networkOutputs) + : InferenceEngine::IInferRequestInternal(networkInputs, networkOutputs), plg(plg) { + CreateInferRequest(); + } /** * @brief Infers specified input(s) in synchronous mode * @note blocks all method of InferRequest while request is ongoing (running or waiting in queue) diff --git a/inference-engine/src/hetero_plugin/hetero_executable_network.cpp b/inference-engine/src/hetero_plugin/hetero_executable_network.cpp index 17fddc90d3e..652fa267f57 100644 --- a/inference-engine/src/hetero_plugin/hetero_executable_network.cpp +++ b/inference-engine/src/hetero_plugin/hetero_executable_network.cpp @@ -732,6 +732,25 @@ void HeteroExecutableNetwork::Export(std::ostream& heteroModel) { } } +IInferRequestInternal::Ptr HeteroExecutableNetwork::CreateInferRequestImpl( + const std::vector>& inputs, + const std::vector>& outputs) { + if (!this->_plugin || !this->_plugin->GetCore() || !this->_plugin->GetCore()->isNewAPI()) + return nullptr; + HeteroInferRequest::SubRequestsList inferRequests; + int index = 0; + for (auto&& subnetwork : _networks) { + HeteroInferRequest::SubRequestDesc desc; + desc._network = subnetwork._network; + desc._profilingTask = openvino::itt::handle("Infer" + std::to_string(index++)); + inferRequests.push_back(desc); + } + return std::make_shared(inputs, + outputs, + inferRequests, + _blobNameMap); +} + IInferRequestInternal::Ptr HeteroExecutableNetwork::CreateInferRequestImpl( InputsDataMap networkInputs, OutputsDataMap networkOutputs) { diff --git a/inference-engine/src/hetero_plugin/hetero_executable_network.hpp b/inference-engine/src/hetero_plugin/hetero_executable_network.hpp index 59574ca2ce7..f7ecbe5ea46 100644 --- a/inference-engine/src/hetero_plugin/hetero_executable_network.hpp +++ b/inference-engine/src/hetero_plugin/hetero_executable_network.hpp @@ -49,6 +49,9 @@ public: InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs, InferenceEngine::OutputsDataMap networkOutputs) override; + InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl(const std::vector>& inputs, + const std::vector>& outputs) override; + InferenceEngine::IInferRequestInternal::Ptr CreateInferRequest() override; diff --git a/inference-engine/src/hetero_plugin/hetero_infer_request.cpp b/inference-engine/src/hetero_plugin/hetero_infer_request.cpp index 7ecf4a3a8e9..5094e007419 100644 --- a/inference-engine/src/hetero_plugin/hetero_infer_request.cpp +++ b/inference-engine/src/hetero_plugin/hetero_infer_request.cpp @@ -16,12 +16,26 @@ using namespace HeteroPlugin; using namespace InferenceEngine; using namespace InferenceEngine::details; +HeteroInferRequest::HeteroInferRequest(const std::vector>& inputs, + const std::vector>& outputs, + const SubRequestsList& inferRequests, + const std::unordered_map& subgraphInputToOutputBlobNames) : + IInferRequestInternal(inputs, outputs), + _inferRequests(inferRequests) { + CreateInferRequest(subgraphInputToOutputBlobNames); +} + + HeteroInferRequest::HeteroInferRequest(InferenceEngine::InputsDataMap networkInputs, InferenceEngine::OutputsDataMap networkOutputs, const SubRequestsList& inferRequests, const std::unordered_map& subgraphInputToOutputBlobNames) : IInferRequestInternal(networkInputs, networkOutputs), _inferRequests(inferRequests) { + CreateInferRequest(subgraphInputToOutputBlobNames); +} + +void HeteroInferRequest::CreateInferRequest(const std::unordered_map& subgraphInputToOutputBlobNames) { if (_networkOutputs.empty() || _networkInputs.empty()) { IE_THROW() << "Internal error: no information about network's output/input"; } diff --git a/inference-engine/src/hetero_plugin/hetero_infer_request.hpp b/inference-engine/src/hetero_plugin/hetero_infer_request.hpp index 0b4e55b6606..0d141231f33 100644 --- a/inference-engine/src/hetero_plugin/hetero_infer_request.hpp +++ b/inference-engine/src/hetero_plugin/hetero_infer_request.hpp @@ -27,10 +27,15 @@ public: }; using SubRequestsList = std::vector; - explicit HeteroInferRequest(InferenceEngine::InputsDataMap networkInputs, - InferenceEngine::OutputsDataMap networkOutputs, - const SubRequestsList &inferRequests, - const std::unordered_map& blobNameMap); + HeteroInferRequest(InferenceEngine::InputsDataMap networkInputs, + InferenceEngine::OutputsDataMap networkOutputs, + const SubRequestsList &inferRequests, + const std::unordered_map& blobNameMap); + + HeteroInferRequest(const std::vector>& networkInputs, + const std::vector>& networkOutputs, + const SubRequestsList &inferRequests, + const std::unordered_map& blobNameMap); void InferImpl() override; @@ -49,6 +54,9 @@ public: SubRequestsList _inferRequests; std::map _blobs; std::map _subRequestFromBlobName; + +private: + void CreateInferRequest(const std::unordered_map& subgraphInputToOutputBlobNames); }; } // namespace HeteroPlugin diff --git a/inference-engine/src/inference_engine/include/openvino/runtime/infer_request.hpp b/inference-engine/src/inference_engine/include/openvino/runtime/infer_request.hpp index ec971d53ecd..665508b290f 100644 --- a/inference-engine/src/inference_engine/include/openvino/runtime/infer_request.hpp +++ b/inference-engine/src/inference_engine/include/openvino/runtime/infer_request.hpp @@ -13,6 +13,7 @@ #include #include +#include "openvino/core/node_output.hpp" #include "openvino/runtime/common.hpp" #include "openvino/runtime/profiling_info.hpp" #include "openvino/runtime/tensor.hpp" @@ -54,21 +55,103 @@ public: /** * @brief Sets input/output data to infer * - * @note Memory allocation does not happen * @param name Name of input or output tensor. - * @param tensor Reference to input or output tensor. The type of a tensor must match the network input precision - * and size. + * @param tensor Reference to input or output tensor. The type of a tensor must match the network input/output + * precision and size. */ void set_tensor(const std::string& name, const Tensor& tensor); + /** + * @brief Sets input/output data to infer + * + * @param port Port of input or output tensor. + * @param tensor Reference to input or output tensor. The type of a tensor must match the network input/output + * precision and size. + */ + void set_tensor(const ov::Output& port, const Tensor& tensor); + /** + * @brief Sets input/output data to infer + * + * @param port Port of input or output tensor. + * @param tensor Reference to input or output tensor. The type of a tensor must match the network input/output + * precision and size. + */ + void set_tensor(const ov::Output& port, const Tensor& tensor); + /** + * @brief Sets input tensor to infer + * + * @param idx Index of input tensor. + * @param tensor Reference to input tensor. The type of a tensor must match the network input precision and size. + */ + void set_input_tensor(size_t idx, const Tensor& tensor); + /** + * @brief Sets input tensor to infer models with single input + * + * @param tensor Reference to input tensor. If model has several inputs, an exception is thrown. + */ + void set_input_tensor(const Tensor& tensor); + /** + * @brief Sets output tensor to infer + * + * @param idx Index of output tensor. + * @param tensor Reference to output tensor. The type of a tensor must match the network output precision and size. + */ + void set_output_tensor(size_t idx, const Tensor& tensor); + /** + * @brief Sets output tensor to infer models with single output + * + * @param tensor Reference to output tensor. If model has several outputs, an exception is thrown. + */ + void set_output_tensor(const Tensor& tensor); /** - * @brief Gets input/output data for inference + * @brief Gets input/output tensor for inference * - * @note Memory allocation does not happen * @param name A name of tensor to get * @return A Tensor with a name @p name. If a tensor is not found, an exception is thrown. */ Tensor get_tensor(const std::string& name); + /** + * @brief Gets input/output tensor for inference + * + * @param port Port of tensor to get + * @return A Tensor for the port @p port. If a tensor with specified @p port is not found, an exception is thrown. + */ + Tensor get_tensor(const ov::Output& port); + /** + * @brief Gets input/output tensor for inference + * + * @param port Port of tensor to get + * @return A Tensor for the port @p port. If a tensor with specified @p port is not found, an exception is thrown. + */ + Tensor get_tensor(const ov::Output& port); + /** + * @brief Gets input tensor for inference + * + * @param idx An index of tensor to get + * @return A Tensor with an input index @p idx. If a tensor with specified @p idx is not found, an exception is + * thrown. + */ + Tensor get_input_tensor(size_t idx); + /** + * @brief Gets input tensor for inference + * + * @return An input Tensor for the model. If model has several inputs, an exception is thrown. + */ + Tensor get_input_tensor(); + /** + * @brief Gets output tensor for inference + * + * @param idx An index of tensor to get + * @return A Tensor with an output index @p idx. If a tensor with specified @p idx is not found, an exception is + * thrown. + */ + Tensor get_output_tensor(size_t idx); + /** + * @brief Gets output tensor for inference + * + * @return An output Tensor for the model. If model has several outputs, an exception is thrown. + */ + Tensor get_output_tensor(); /** * @brief Infers specified input(s) in synchronous mode diff --git a/inference-engine/src/inference_engine/src/cpp/ie_executable_network.cpp b/inference-engine/src/inference_engine/src/cpp/ie_executable_network.cpp index 04e42761023..d811b8378d4 100644 --- a/inference-engine/src/inference_engine/src/cpp/ie_executable_network.cpp +++ b/inference-engine/src/inference_engine/src/cpp/ie_executable_network.cpp @@ -145,11 +145,11 @@ std::vector> ExecutableNetwork::inputs() const { ov::Output ExecutableNetwork::input() const { OV_EXEC_NET_CALL_STATEMENT({ - const auto params = _impl->getInputs(); - if (params.size() != 1) { + const auto inputs = _impl->getInputs(); + if (inputs.size() != 1) { throw ov::Exception("input() must be called on a function with exactly one parameter."); } - return params.at(0); + return inputs.at(0); }); } @@ -179,11 +179,11 @@ std::vector> ExecutableNetwork::outputs() const { } ov::Output ExecutableNetwork::output() const { OV_EXEC_NET_CALL_STATEMENT({ - const auto result = _impl->getOutputs(); - if (result.size() != 1) { + const auto outputs = _impl->getOutputs(); + if (outputs.size() != 1) { throw ov::Exception("output() must be called on a function with exactly one parameter."); } - return result.at(0); + return outputs.at(0); }); } ov::Output ExecutableNetwork::output(size_t i) const { diff --git a/inference-engine/src/inference_engine/src/cpp/ie_infer_request.cpp b/inference-engine/src/inference_engine/src/cpp/ie_infer_request.cpp index 271cd710b5d..0c55184e299 100644 --- a/inference-engine/src/inference_engine/src/cpp/ie_infer_request.cpp +++ b/inference-engine/src/inference_engine/src/cpp/ie_infer_request.cpp @@ -8,6 +8,7 @@ #include #include +#include "cpp_interfaces/interface/ie_iexecutable_network_internal.hpp" #include "cpp_interfaces/interface/ie_iinfer_request_internal.hpp" #include "ie_infer_async_request_base.hpp" #include "ie_ngraph_utils.hpp" @@ -16,6 +17,24 @@ #include "openvino/runtime/infer_request.hpp" #include "transformations/utils/utils.hpp" +namespace { + +inline bool getPort(ov::Output& port, + const std::string& name, + const std::vector>>& ports) { + for (const auto& nodes : ports) { + for (const auto& node : nodes) { + const auto& names = node->get_output_tensor(0).get_names(); + if (names.find(name) != names.end()) { + port = node->output(0); + return true; + } + } + } + return false; +} + +} // namespace namespace InferenceEngine { #define INFER_REQ_CALL_STATEMENT(...) \ @@ -204,6 +223,18 @@ bool InferRequest::operator==(const InferRequest& r) const noexcept { } // namespace InferenceEngine +namespace { + +std::string get_legacy_name_from_port(const ov::Output& port) { + ov::Output p(std::const_pointer_cast(port.get_node_shared_ptr()), port.get_index()); + if (auto node = std::dynamic_pointer_cast(p.get_node_shared_ptr())) { + p = node->input_value(0); + } + return ngraph::op::util::create_ie_output_name(p); +} + +} // namespace + namespace ov { namespace runtime { @@ -213,23 +244,114 @@ InferRequest::InferRequest(const std::shared_ptr& so, const ie::IInferRequ OPENVINO_ASSERT(_impl != nullptr, "InferRequest was not initialized."); } -void InferRequest::set_tensor(const std::string& name, const Tensor& tensor){ - OV_INFER_REQ_CALL_STATEMENT({ _impl->SetBlob(name, tensor._impl); })} +void InferRequest::set_tensor(const ov::Output& port, const Tensor& tensor) { + OV_INFER_REQ_CALL_STATEMENT({ _impl->SetBlob(get_legacy_name_from_port(port), tensor._impl); }); +} + +void InferRequest::set_tensor(const ov::Output& port, const Tensor& tensor) { + set_tensor(ov::Output(port.get_node(), port.get_index()), tensor); +} + +void InferRequest::set_tensor(const std::string& name, const Tensor& tensor) { + OV_INFER_REQ_CALL_STATEMENT({ + ov::Output port; + OPENVINO_ASSERT(::getPort(port, name, {_impl->GetInputs(), _impl->GetOutputs()}), + "Port for tensor name " + name + " was not found."); + set_tensor(port, tensor); + }); +} + +void InferRequest::set_input_tensor(size_t idx, const Tensor& tensor) { + OV_INFER_REQ_CALL_STATEMENT({ + const auto& inputs = _impl->GetInputs(); + OPENVINO_ASSERT(inputs.size() > idx, + "Input port for index ", + idx, + " was not found! The model has only ", + inputs.size(), + " inputs."); + set_tensor(inputs.at(idx)->output(0), tensor); + }); +} + +void InferRequest::set_input_tensor(const Tensor& tensor) { + OV_INFER_REQ_CALL_STATEMENT({ + const auto inputs = _impl->GetInputs(); + OPENVINO_ASSERT(inputs.size() == 1, + "set_input_tensor() must be called on a function with exactly one parameter."); + set_tensor(inputs.at(0)->output(0), tensor); + }); +} + +void InferRequest::set_output_tensor(size_t idx, const Tensor& tensor) { + OV_INFER_REQ_CALL_STATEMENT({ + const auto& outputs = _impl->GetOutputs(); + OPENVINO_ASSERT(outputs.size() > idx, + "Output port for index ", + idx, + " was not found! The model has only ", + outputs.size(), + " outputs."); + set_tensor(outputs.at(idx)->output(0), tensor); + }); +} + +void InferRequest::set_output_tensor(const Tensor& tensor) { + OV_INFER_REQ_CALL_STATEMENT({ + const auto outputs = _impl->GetOutputs(); + OPENVINO_ASSERT(outputs.size() == 1, + "set_output_tensor() must be called on a function with exactly one parameter."); + set_tensor(outputs.at(0)->output(0), tensor); + }); +} + +Tensor InferRequest::get_tensor(const ov::Output& port) { + OV_INFER_REQ_CALL_STATEMENT({ + const auto& name = get_legacy_name_from_port(port); + auto blob = _impl->GetBlob(name); + return {_so, blob}; + }); +} + +Tensor InferRequest::get_tensor(const ov::Output& port) { + return get_tensor(ov::Output(port.get_node(), port.get_index())); +} Tensor InferRequest::get_tensor(const std::string& name) { OV_INFER_REQ_CALL_STATEMENT({ - auto blob = _impl->GetBlob(name); - const bool remoteBlobPassed = blob->is(); - if (blob == nullptr) { - IE_THROW(NotAllocated) << "Internal tensor implementation with name `" << name << "` is not allocated!"; + ov::Output port; + OPENVINO_ASSERT(::getPort(port, name, {_impl->GetInputs(), _impl->GetOutputs()}), + "Port for tensor name " + name + " was not found."); + return get_tensor(port); + }); +} + +Tensor InferRequest::get_input_tensor(size_t idx) { + OV_INFER_REQ_CALL_STATEMENT({ return get_tensor(_impl->GetInputs().at(idx)->output(0)); }); +} + +Tensor InferRequest::get_output_tensor(size_t idx) { + OV_INFER_REQ_CALL_STATEMENT({ return get_tensor(_impl->GetOutputs().at(idx)->output(0)); }); +} + +Tensor InferRequest::get_input_tensor() { + OV_INFER_REQ_CALL_STATEMENT({ + const auto inputs = _impl->GetInputs(); + if (inputs.size() != 1) { + throw ov::Exception("get_input_tensor() must be called on a function with exactly one parameter."); } - if (!remoteBlobPassed && blob->buffer() == nullptr) { - IE_THROW(NotAllocated) << "Internal tensor implementation with name `" << name << "` is not allocated!"; + return get_tensor(inputs.at(0)->output(0)); + }); +} + +Tensor InferRequest::get_output_tensor() { + OV_INFER_REQ_CALL_STATEMENT({ + const auto outputs = _impl->GetOutputs(); + if (outputs.size() != 1) { + throw ov::Exception("get_output_tensor() must be called on a function with exactly one parameter."); } - auto tensorDesc = blob->getTensorDesc(); - auto dims = tensorDesc.getDims(); - return {_so, blob}; - }) + return get_tensor(outputs.at(0)->output(0)); + }); } void InferRequest::infer() { @@ -321,4 +443,4 @@ bool InferRequest::operator==(const InferRequest& r) const noexcept { } } // namespace runtime -} // namespace ov \ No newline at end of file +} // namespace ov diff --git a/inference-engine/src/inference_engine/src/cpp_interfaces/interface/ie_iexecutable_network_internal.cpp b/inference-engine/src/inference_engine/src/cpp_interfaces/interface/ie_iexecutable_network_internal.cpp index 40182e8e6de..9a8abf1eec1 100644 --- a/inference-engine/src/inference_engine/src/cpp_interfaces/interface/ie_iexecutable_network_internal.cpp +++ b/inference-engine/src/inference_engine/src/cpp_interfaces/interface/ie_iexecutable_network_internal.cpp @@ -14,7 +14,9 @@ #include "cpp_interfaces/interface/ie_iinfer_request_internal.hpp" #include "cpp_interfaces/interface/ie_iplugin_internal.hpp" #include "ie_icore.hpp" +#include "ie_ngraph_utils.hpp" #include "ie_parameter.hpp" +#include "openvino/core/node.hpp" namespace InferenceEngine { @@ -56,7 +58,13 @@ ConstInputsDataMap IExecutableNetworkInternal::GetInputsInfo() const { } std::shared_ptr IExecutableNetworkInternal::CreateInferRequest() { - auto asyncRequestImpl = CreateInferRequestImpl(_networkInputs, _networkOutputs); + std::shared_ptr asyncRequestImpl; + try { + asyncRequestImpl = CreateInferRequestImpl(_parameters, _results); + } catch (const NotImplemented&) { + } + if (!asyncRequestImpl) + asyncRequestImpl = CreateInferRequestImpl(_networkInputs, _networkOutputs); asyncRequestImpl->setPointerToExecutableNetworkInternal(shared_from_this()); return asyncRequestImpl; } @@ -109,4 +117,10 @@ std::shared_ptr IExecutableNetworkInternal::CreateInferRe IE_THROW(NotImplemented); } +std::shared_ptr IExecutableNetworkInternal::CreateInferRequestImpl( + const std::vector>& inputs, + const std::vector>& outputs) { + IE_THROW(NotImplemented); +} + } // namespace InferenceEngine diff --git a/inference-engine/src/inference_engine/src/cpp_interfaces/interface/ie_iinfer_request_internal.cpp b/inference-engine/src/inference_engine/src/cpp_interfaces/interface/ie_iinfer_request_internal.cpp index 1da9e789a7b..dbd74f4fd6e 100644 --- a/inference-engine/src/inference_engine/src/cpp_interfaces/interface/ie_iinfer_request_internal.cpp +++ b/inference-engine/src/inference_engine/src/cpp_interfaces/interface/ie_iinfer_request_internal.cpp @@ -16,8 +16,10 @@ #include "ie_blob.h" #include "ie_common.h" #include "ie_compound_blob.h" +#include "ie_ngraph_utils.hpp" #include "ie_preprocess.hpp" #include "ie_remote_context.hpp" +#include "transformations/utils/utils.hpp" namespace InferenceEngine { @@ -28,6 +30,43 @@ IInferRequestInternal::IInferRequestInternal(const InputsDataMap& networkInputs, _networkInputs{copyInfo(networkInputs)}, _networkOutputs{copyInfo(networkOutputs)} {} +IInferRequestInternal::IInferRequestInternal(const std::vector>& inputs, + const std::vector>& outputs) + : _parameters(inputs), + _results(outputs) { + const auto& create_old_data = [](const ov::Output& output) -> InferenceEngine::DataPtr { + IE_SUPPRESS_DEPRECATED_START + auto name = ngraph::op::util::get_ie_output_name(output); + auto shape = output.get_partial_shape(); + auto rank = shape.rank().is_static() ? shape.rank().get_length() : -1; + for (const auto& dim : shape) { + if (dim.is_static() && dim.get_length() == 0) + IE_THROW() << name << " has zero dimension which is not allowed"; + } + const Layout rankLayout = rank < 0 ? Layout::BLOCKED : TensorDesc::getLayoutByRank(rank); + const auto precision = InferenceEngine::details::convertPrecision(output.get_element_type()); + return std::make_shared(name, precision, shape, rankLayout); + IE_SUPPRESS_DEPRECATED_END + }; + const auto& create_old_input_data = + [create_old_data](const ov::Output& output) -> InferenceEngine::InputInfo::Ptr { + auto info = std::make_shared(); + info->setInputData(create_old_data(output)); + return info; + }; + + for (const auto& param : _parameters) { + const auto& input = create_old_input_data(param->output(0)); + _networkInputs[input->name()] = input; + } + + for (const auto& result : _results) { + auto input = result->input_value(0); + const auto& output = create_old_data(ov::Output(input.get_node(), input.get_index())); + _networkOutputs[output->getName()] = output; + } +} + void IInferRequestInternal::Infer() { checkBlobs(); InferImpl(); @@ -373,4 +412,12 @@ void* IInferRequestInternal::GetUserData() noexcept { void IInferRequestInternal::SetUserData(void* userData) noexcept { _userData = userData; } + +const std::vector>& IInferRequestInternal::GetInputs() const { + return _parameters; +} + +const std::vector>& IInferRequestInternal::GetOutputs() const { + return _results; +} } // namespace InferenceEngine diff --git a/inference-engine/src/inference_engine/src/ie_core.cpp b/inference-engine/src/inference_engine/src/ie_core.cpp index db9f0154978..650f05ef281 100644 --- a/inference-engine/src/inference_engine/src/ie_core.cpp +++ b/inference-engine/src/inference_engine/src/ie_core.cpp @@ -542,6 +542,54 @@ public: const std::map& config) override { auto parsed = parseDeviceNameIntoConfig(deviceName, config); auto exec = GetCPPPluginByName(parsed._deviceName).import_model(networkModel, parsed._config); + + if (isNewAPI()) { + // create getInputs() based on GetInputsInfo() + using namespace InferenceEngine::details; + + if (exec->getInputs().empty()) { + const auto& inputsInfo = exec->GetInputsInfo(); + OPENVINO_ASSERT(!inputsInfo.empty(), "inputsInfo is empty after network import"); + + std::vector> params; + params.reserve(inputsInfo.size()); + for (auto&& input : inputsInfo) { + auto param = std::make_shared( + convertPrecision(input.second->getPrecision()), + ov::PartialShape(input.second->getTensorDesc().getDims())); + param->set_friendly_name(input.first); + param->get_output_tensor(0).add_names({input.first}); + params.emplace_back(std::move(param)); + } + + exec->setInputs(params); + } + + if (exec->getOutputs().empty()) { + const auto& outputsInfo = exec->GetOutputsInfo(); + OPENVINO_ASSERT(!outputsInfo.empty(), "outputsInfo is empty after network import"); + + std::vector> results; + results.reserve(outputsInfo.size()); + for (auto&& output : outputsInfo) { + auto fake_param = std::make_shared( + convertPrecision(output.second->getPrecision()), + ov::PartialShape(output.second->getTensorDesc().getDims())); + fake_param->set_friendly_name(output.first); + auto result = std::make_shared(fake_param); + result->get_output_tensor(0).add_names({output.first}); + results.emplace_back(std::move(result)); + } + exec->setOutputs(results); + } + + // but for true support plugins need: + // 1. ensure order or parameters and results as in ov::Function + // 2. provide tensor names for inputs and outputs + // 3. precisions for getInputs and getOutputs should be taken from GetInputsInfo / GetOutputsInfo + // not from ngraph. Plugins should use SetExeNetworkInfo + } + return {{exec._so}, exec._ptr}; } @@ -1381,50 +1429,6 @@ ExecutableNetwork Core::import_model(std::istream& modelStream, OV_ITT_SCOPED_TASK(ov::itt::domains::IE, "Core::import_model"); OV_CORE_CALL_STATEMENT({ auto exec = _impl->ImportNetwork(modelStream, deviceName, config); - - // create getInputs() based on GetInputsInfo() - using namespace InferenceEngine::details; - - if (exec->getInputs().empty()) { - const auto& inputsInfo = exec->GetInputsInfo(); - OPENVINO_ASSERT(!inputsInfo.empty(), "inputsInfo is empty after network import"); - - std::vector> params; - params.reserve(inputsInfo.size()); - for (auto&& input : inputsInfo) { - auto param = - std::make_shared(convertPrecision(input.second->getPrecision()), - ov::PartialShape(input.second->getTensorDesc().getDims())); - param->get_output_tensor(0).add_names({input.first}); - params.emplace_back(std::move(param)); - } - - exec->setInputs(params); - } - - if (exec->getOutputs().empty()) { - const auto& outputsInfo = exec->GetOutputsInfo(); - OPENVINO_ASSERT(!outputsInfo.empty(), "outputsInfo is empty after network import"); - - std::vector> results; - results.reserve(outputsInfo.size()); - for (auto&& output : outputsInfo) { - auto fake_param = - std::make_shared(convertPrecision(output.second->getPrecision()), - ov::PartialShape(output.second->getTensorDesc().getDims())); - auto result = std::make_shared(fake_param); - result->get_output_tensor(0).add_names({output.first}); - results.emplace_back(std::move(result)); - } - exec->setOutputs(results); - } - - // but for true support plugins need: - // 1. ensure order or parameters and results as in ov::Function - // 2. provide tensor names for inputs and outputs - // 3. precisions for getInputs and getOutputs should be taken from GetInputsInfo / GetOutputsInfo - // not from ngraph. Plugins should use SetExeNetworkInfo - return {exec._so, exec._ptr}; }); } diff --git a/inference-engine/src/legacy_api/src/ngraph_ops/nms_ie.cpp b/inference-engine/src/legacy_api/src/ngraph_ops/nms_ie.cpp index 785fc811825..c246a2e1857 100644 --- a/inference-engine/src/legacy_api/src/ngraph_ops/nms_ie.cpp +++ b/inference-engine/src/legacy_api/src/ngraph_ops/nms_ie.cpp @@ -162,7 +162,7 @@ int64_t op::NonMaxSuppressionIE3::max_boxes_output_from_input() const { } const auto max_output_boxes_input = - ov::as_type_ptr(input_value(max_output_boxes_per_class_port).get_node_shared_ptr()); + ov::as_type_ptr(input_value(max_output_boxes_per_class_port).get_node_shared_ptr()); max_output_boxes = max_output_boxes_input->cast_vector().at(0); return max_output_boxes; diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_exec_network.cpp b/inference-engine/src/mkldnn_plugin/mkldnn_exec_network.cpp index ff154f4dee4..fdcdf37996f 100644 --- a/inference-engine/src/mkldnn_plugin/mkldnn_exec_network.cpp +++ b/inference-engine/src/mkldnn_plugin/mkldnn_exec_network.cpp @@ -25,11 +25,21 @@ #include #include #include +#include "cpp_interfaces/interface/ie_iplugin_internal.hpp" +#include "ie_icore.hpp" using namespace MKLDNNPlugin; using namespace InferenceEngine; using namespace InferenceEngine::details; +InferenceEngine::IInferRequestInternal::Ptr +MKLDNNExecNetwork::CreateInferRequestImpl(const std::vector>& inputs, + const std::vector>& outputs) { + if (!this->_plugin || !this->_plugin->GetCore() || !this->_plugin->GetCore()->isNewAPI()) + return nullptr; + return std::make_shared(inputs, outputs, std::static_pointer_cast(shared_from_this())); +} + InferenceEngine::IInferRequestInternal::Ptr MKLDNNExecNetwork::CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs, InferenceEngine::OutputsDataMap networkOutputs) { diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_exec_network.h b/inference-engine/src/mkldnn_plugin/mkldnn_exec_network.h index e9373efd4a5..1fab2d16ebb 100644 --- a/inference-engine/src/mkldnn_plugin/mkldnn_exec_network.h +++ b/inference-engine/src/mkldnn_plugin/mkldnn_exec_network.h @@ -23,6 +23,10 @@ class MKLDNNExecNetwork: public InferenceEngine::ExecutableNetworkThreadSafeDefa public: typedef std::shared_ptr Ptr; + std::shared_ptr + CreateInferRequestImpl(const std::vector>& inputs, + const std::vector>& outputs) override; + std::shared_ptr CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs, InferenceEngine::OutputsDataMap networkOutputs) override; diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.cpp b/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.cpp index d606c464ce4..11423ac0f8f 100644 --- a/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.cpp +++ b/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.cpp @@ -29,6 +29,18 @@ MKLDNNPlugin::MKLDNNInferRequest::MKLDNNInferRequest(InferenceEngine::InputsData MKLDNNExecNetwork::Ptr execNetwork_) : IInferRequestInternal(networkInputs, networkOutputs) , execNetwork(execNetwork_) { + CreateInferRequest(); +} + +MKLDNNPlugin::MKLDNNInferRequest::MKLDNNInferRequest(const std::vector>& inputs, + const std::vector>& outputs, + MKLDNNExecNetwork::Ptr execNetwork_) +: IInferRequestInternal(inputs, outputs) +, execNetwork(execNetwork_) { + CreateInferRequest(); +} + +void MKLDNNPlugin::MKLDNNInferRequest::CreateInferRequest() { auto id = (execNetwork->_numRequests)++; profilingTask = openvino::itt::handle("MKLDNN_INFER_" + execNetwork->_name + "_" + std::to_string(id)); diff --git a/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.h b/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.h index 3141ed18ddc..8314165cdf9 100644 --- a/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.h +++ b/inference-engine/src/mkldnn_plugin/mkldnn_infer_request.h @@ -22,6 +22,10 @@ public: InferenceEngine::OutputsDataMap networkOutputs, std::shared_ptr execNetwork); + MKLDNNInferRequest(const std::vector>& inputs, + const std::vector>& outputs, + std::shared_ptr execNetwork); + ~MKLDNNInferRequest(); void InferImpl() override; @@ -48,6 +52,7 @@ public: void ThrowIfCanceled() const; private: + void CreateInferRequest(); void PushInputData(); void PushStates(); void PullStates(); diff --git a/inference-engine/src/multi_device/multi_device_exec_network.cpp b/inference-engine/src/multi_device/multi_device_exec_network.cpp index a1b3fa872f5..7752fd50358 100644 --- a/inference-engine/src/multi_device/multi_device_exec_network.cpp +++ b/inference-engine/src/multi_device/multi_device_exec_network.cpp @@ -372,6 +372,30 @@ std::shared_ptr MultiDeviceExecutableNetwork::GetCore() return _plugin->GetCore(); } +InferenceEngine::IInferRequestInternal::Ptr MultiDeviceExecutableNetwork::CreateInferRequestImpl( + const std::vector>& inputs, + const std::vector>& outputs) { + auto num = _numRequestsCreated++; + size_t sum = 0; + InferenceEngine::SoIInferRequestInternal request_to_share_blobs_with; + + if (_workModeIsAUTO) { + return std::make_shared(inputs, outputs, request_to_share_blobs_with); + } + + // borrowing device-specific blobs from the underlying requests for the device-agnostic, user-facing requests + // this allows to potentially save on the data-copy later (if the requests are scheduled in the same order) + for (const auto& device : _devicePrioritiesInitial) { + auto& dev_requests = _workerRequests[device.deviceName]; + if ((num - sum) < dev_requests.size()) { + request_to_share_blobs_with = dev_requests.at(num - sum)._inferRequest; + break; + } + sum += dev_requests.size(); + } + return std::make_shared(inputs, outputs, request_to_share_blobs_with); +} + InferenceEngine::IInferRequestInternal::Ptr MultiDeviceExecutableNetwork::CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs, InferenceEngine::OutputsDataMap networkOutputs) { auto num = _numRequestsCreated++; @@ -396,7 +420,12 @@ InferenceEngine::IInferRequestInternal::Ptr MultiDeviceExecutableNetwork::Create } IInferRequestInternal::Ptr MultiDeviceExecutableNetwork::CreateInferRequest() { - auto syncRequestImpl = CreateInferRequestImpl(_networkInputs, _networkOutputs); + IInferRequestInternal::Ptr syncRequestImpl; + if (this->_plugin && this->_plugin->GetCore() && GetCore()->isNewAPI()) + syncRequestImpl = CreateInferRequestImpl(_parameters, _results); + + if (!syncRequestImpl) + syncRequestImpl = CreateInferRequestImpl(_networkInputs, _networkOutputs); syncRequestImpl->setPointerToExecutableNetworkInternal(shared_from_this()); return std::make_shared(std::static_pointer_cast(syncRequestImpl), _needPerfCounters, diff --git a/inference-engine/src/multi_device/multi_device_exec_network.hpp b/inference-engine/src/multi_device/multi_device_exec_network.hpp index 4a687377b34..054ee0aac8b 100644 --- a/inference-engine/src/multi_device/multi_device_exec_network.hpp +++ b/inference-engine/src/multi_device/multi_device_exec_network.hpp @@ -131,6 +131,8 @@ public: InferenceEngine::IInferRequestInternal::Ptr CreateInferRequest() override; InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs, InferenceEngine::OutputsDataMap networkOutputs) override; + InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl(const std::vector>& inputs, + const std::vector>& outputs) override; std::shared_ptr GetContext() const override; std::shared_ptr GetCore() const; ~MultiDeviceExecutableNetwork() override; diff --git a/inference-engine/src/multi_device/multi_device_infer_request.cpp b/inference-engine/src/multi_device/multi_device_infer_request.cpp index de6540b88f0..d96754ecdb6 100644 --- a/inference-engine/src/multi_device/multi_device_infer_request.cpp +++ b/inference-engine/src/multi_device/multi_device_infer_request.cpp @@ -14,10 +14,21 @@ namespace MultiDevicePlugin { using namespace InferenceEngine; // ------------------------------MultiDeviceInferRequest---------------------------- +MultiDeviceInferRequest::MultiDeviceInferRequest(const std::vector>& inputs, + const std::vector>& outputs, + const InferenceEngine::SoIInferRequestInternal & request_to_share_blobs_with) + : IInferRequestInternal(inputs, outputs) { + CreateInferRequest(request_to_share_blobs_with); +} + MultiDeviceInferRequest::MultiDeviceInferRequest(const InputsDataMap& networkInputs, const OutputsDataMap& networkOutputs, const SoIInferRequestInternal & request_to_share_blobs_with) : IInferRequestInternal(networkInputs, networkOutputs) { + CreateInferRequest(request_to_share_blobs_with); +} + +void MultiDeviceInferRequest::CreateInferRequest(const InferenceEngine::SoIInferRequestInternal& request_to_share_blobs_with) { if (request_to_share_blobs_with) { // borrow device-friendly blobs from the request for (const auto &it : _networkInputs) @@ -27,7 +38,7 @@ MultiDeviceInferRequest::MultiDeviceInferRequest(const InputsDataMap& networkI return; } // Allocate all input blobs - for (const auto &it : networkInputs) { + for (const auto &it : _networkInputs) { Layout l = it.second->getLayout(); Precision p = it.second->getPrecision(); SizeVector dims = it.second->getTensorDesc().getDims(); @@ -37,7 +48,7 @@ MultiDeviceInferRequest::MultiDeviceInferRequest(const InputsDataMap& networkI _inputs[it.first]->allocate(); } // Allocate all output blobs - for (const auto &it : networkOutputs) { + for (const auto &it : _networkOutputs) { Layout l = it.second->getLayout(); Precision p = it.second->getPrecision(); SizeVector dims = it.second->getTensorDesc().getDims(); @@ -47,7 +58,6 @@ MultiDeviceInferRequest::MultiDeviceInferRequest(const InputsDataMap& networkI _outputs[it.first]->allocate(); } } - void MultiDeviceInferRequest::SetBlobsToAnotherRequest(const SoIInferRequestInternal& req) { for (const auto &it : _networkInputs) { auto &name = it.first; diff --git a/inference-engine/src/multi_device/multi_device_infer_request.hpp b/inference-engine/src/multi_device/multi_device_infer_request.hpp index 9ec23eda918..73f488d892c 100644 --- a/inference-engine/src/multi_device/multi_device_infer_request.hpp +++ b/inference-engine/src/multi_device/multi_device_infer_request.hpp @@ -24,10 +24,16 @@ public: explicit MultiDeviceInferRequest(const InferenceEngine::InputsDataMap& networkInputs, const InferenceEngine::OutputsDataMap& networkOutputs, const InferenceEngine::SoIInferRequestInternal & request_to_share_blobs_with); + explicit MultiDeviceInferRequest(const std::vector>& inputs, + const std::vector>& outputs, + const InferenceEngine::SoIInferRequestInternal & request_to_share_blobs_with); std::map GetPerformanceCounts() const override; void InferImpl() override; // Multi-Device impl specific: sets the data (blobs from the device-less requests to the specific device request) void SetBlobsToAnotherRequest(const InferenceEngine::SoIInferRequestInternal& req); + +private: + void CreateInferRequest(const InferenceEngine::SoIInferRequestInternal& request_to_share_blobs_with); }; } // namespace MultiDevicePlugin diff --git a/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp b/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp index 534599f752d..e42e03369b8 100644 --- a/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp +++ b/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp @@ -58,7 +58,13 @@ protected: */ template IInferRequestInternal::Ptr CreateAsyncInferRequestFromSync() { - auto syncRequestImpl = this->CreateInferRequestImpl(_networkInputs, _networkOutputs); + InferenceEngine::IInferRequestInternal::Ptr syncRequestImpl; + try { + syncRequestImpl = this->CreateInferRequestImpl(_parameters, _results); + } catch (const NotImplemented&) { + } + if (!syncRequestImpl) + syncRequestImpl = this->CreateInferRequestImpl(_networkInputs, _networkOutputs); syncRequestImpl->setPointerToExecutableNetworkInternal(shared_from_this()); return std::make_shared(syncRequestImpl, _taskExecutor, _callbackExecutor); } diff --git a/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_infer_async_request_thread_safe_default.hpp b/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_infer_async_request_thread_safe_default.hpp index c82a9ebb303..f6e63fcbcd7 100644 --- a/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_infer_async_request_thread_safe_default.hpp +++ b/inference-engine/src/plugin_api/cpp_interfaces/impl/ie_infer_async_request_thread_safe_default.hpp @@ -145,7 +145,8 @@ public: AsyncInferRequestThreadSafeDefault(const IInferRequestInternal::Ptr& request, const ITaskExecutor::Ptr& taskExecutor, const ITaskExecutor::Ptr& callbackExecutor) - : _syncRequest{request}, + : IInferRequestInternal(*request), + _syncRequest{request}, _requestExecutor{taskExecutor}, _callbackExecutor{callbackExecutor}, _pipeline{{taskExecutor, diff --git a/inference-engine/src/plugin_api/cpp_interfaces/interface/ie_iexecutable_network_internal.hpp b/inference-engine/src/plugin_api/cpp_interfaces/interface/ie_iexecutable_network_internal.hpp index 4f54e49e234..b454d6a326f 100644 --- a/inference-engine/src/plugin_api/cpp_interfaces/interface/ie_iexecutable_network_internal.hpp +++ b/inference-engine/src/plugin_api/cpp_interfaces/interface/ie_iexecutable_network_internal.hpp @@ -168,6 +168,17 @@ protected: */ virtual std::shared_ptr CreateInferRequestImpl(InputsDataMap networkInputs, OutputsDataMap networkOutputs); + /** + * @brief Creates an inference request internal implementation. + * @note The method is called by IExecutableNetworkInternal::CreateInferRequest as + * plugin-specific implementation. + * @param[in] inputs The function inputs + * @param[in] outputs The function outputs + * @return A shared pointer to inference request object. + */ + virtual std::shared_ptr CreateInferRequestImpl( + const std::vector>& inputs, + const std::vector>& outputs); InferenceEngine::InputsDataMap _networkInputs; //!< Holds information about network inputs info InferenceEngine::OutputsDataMap _networkOutputs; //!< Holds information about network outputs data diff --git a/inference-engine/src/plugin_api/cpp_interfaces/interface/ie_iinfer_request_internal.hpp b/inference-engine/src/plugin_api/cpp_interfaces/interface/ie_iinfer_request_internal.hpp index 0b12bd775e6..7cfa757ee72 100644 --- a/inference-engine/src/plugin_api/cpp_interfaces/interface/ie_iinfer_request_internal.hpp +++ b/inference-engine/src/plugin_api/cpp_interfaces/interface/ie_iinfer_request_internal.hpp @@ -13,6 +13,7 @@ #include "ie_common.h" #include "ie_input_info.hpp" #include "ie_preprocess_data.hpp" +#include "openvino/core/node_output.hpp" #include "so_ptr.hpp" namespace InferenceEngine { @@ -42,6 +43,14 @@ public: */ IInferRequestInternal(const InputsDataMap& networkInputs, const OutputsDataMap& networkOutputs); + /** + * @brief Constructs a new instance. + * @param[in] inputs The network inputs + * @param[in] outputs The network outputs + */ + IInferRequestInternal(const std::vector>& networkInputs, + const std::vector>& networkOutputs); + /** * @brief Infers specified input(s) in synchronous mode * @note blocks all method of InferRequest while request is ongoing (running or waiting in queue) @@ -190,6 +199,9 @@ public: INFERENCE_ENGINE_DEPRECATED("The method will be removed") void SetUserData(void* userData) noexcept; + const std::vector>& GetInputs() const; + const std::vector>& GetOutputs() const; + protected: /** * @brief Destroys the object. @@ -232,8 +244,10 @@ protected: InferenceEngine::BlobMap _inputs; //!< A map of user passed blobs for network inputs InferenceEngine::BlobMap _deviceInputs; //!< A map of actual network inputs, in plugin specific format InferenceEngine::BlobMap _outputs; //!< A map of user passed blobs for network outputs - std::map _preProcData; //!< A map of pre-process data per input - int m_curBatch = -1; //!< Current batch value used in dynamic batching + std::vector> _parameters; //!< A vector of function inputs + std::vector> _results; //!< A vector of function outputs + std::map _preProcData; //!< A map of pre-process data per input + int m_curBatch = -1; //!< Current batch value used in dynamic batching /** * @brief A shared pointer to IInferRequestInternal diff --git a/inference-engine/src/transformations/include/transformations/utils/utils.hpp b/inference-engine/src/transformations/include/transformations/utils/utils.hpp index c3672fb9961..b4e4a8e6142 100644 --- a/inference-engine/src/transformations/include/transformations/utils/utils.hpp +++ b/inference-engine/src/transformations/include/transformations/utils/utils.hpp @@ -49,14 +49,17 @@ bool has_op_with_type(const std::shared_ptr &function) { } return false; } -inline std::string create_ie_output_name(const ngraph::Output& output) { +inline std::string create_ie_output_name(const ngraph::Output& output) { const auto& prev_layer = output.get_node_shared_ptr(); std::string out_name = prev_layer->get_friendly_name(); if (prev_layer->get_output_size() != 1) out_name += "." + std::to_string(output.get_index()); return out_name; } -inline std::string get_ie_output_name(const ngraph::Output& output) { +inline std::string create_ie_output_name(const ngraph::Output& output) { + return create_ie_output_name(ov::Output(output.get_node(), output.get_index())); +} +inline std::string get_ie_output_name(const ngraph::Output& output) { NGRAPH_SUPPRESS_DEPRECATED_START auto name = output.get_tensor().get_name(); NGRAPH_SUPPRESS_DEPRECATED_END @@ -65,6 +68,9 @@ inline std::string get_ie_output_name(const ngraph::Output& output } return name; } +inline std::string get_ie_output_name(const ngraph::Output& output) { + return get_ie_output_name(ov::Output(output.get_node(), output.get_index())); +} template bool has_constant_value(const std::shared_ptr& constant, diff --git a/inference-engine/src/transformations/src/ngraph_ops/nms_ie_internal.cpp b/inference-engine/src/transformations/src/ngraph_ops/nms_ie_internal.cpp index 550be77caab..9c4b8c4db27 100644 --- a/inference-engine/src/transformations/src/ngraph_ops/nms_ie_internal.cpp +++ b/inference-engine/src/transformations/src/ngraph_ops/nms_ie_internal.cpp @@ -80,7 +80,7 @@ int64_t op::internal::NonMaxSuppressionIEInternal::max_boxes_output_from_input() } const auto max_output_boxes_input = - ov::as_type_ptr(input_value(max_output_boxes_per_class_port).get_node_shared_ptr()); + ov::as_type_ptr(input_value(max_output_boxes_per_class_port).get_node_shared_ptr()); max_output_boxes = max_output_boxes_input->cast_vector().at(0); return max_output_boxes; diff --git a/inference-engine/src/vpu/myriad_plugin/myriad_executable_network.h b/inference-engine/src/vpu/myriad_plugin/myriad_executable_network.h index 22e0f3781c9..ceb54714f28 100644 --- a/inference-engine/src/vpu/myriad_plugin/myriad_executable_network.h +++ b/inference-engine/src/vpu/myriad_plugin/myriad_executable_network.h @@ -22,6 +22,8 @@ #include "myriad_executor.h" #include "myriad_infer_request.h" #include "myriad_async_infer_request.h" +#include "cpp_interfaces/interface/ie_iplugin_internal.hpp" +#include "ie_icore.hpp" namespace vpu { namespace MyriadPlugin { @@ -52,7 +54,7 @@ public: } ie::IInferRequestInternal::Ptr CreateInferRequestImpl(ie::InputsDataMap networkInputs, - ie::OutputsDataMap networkOutputs) override { + ie::OutputsDataMap networkOutputs) override { if (!_isNetworkConstant && (_device == nullptr || !_device->isBooted())) { IE_THROW() << "Can not create infer request: there is no available devices with platform "; } @@ -67,11 +69,17 @@ public: if (!_isNetworkConstant && (_device == nullptr || !_device->isBooted())) { IE_THROW() << "Can not create infer request: there is no available devices with platform "; } - - auto syncRequestImpl = std::make_shared(_graphDesc, _networkInputs, _networkOutputs, - _inputInfo, _outputInfo, - _graphMetaData.stagesMeta, _config, _log, - _executor, _constDatas, _isNetworkConstant); + std::shared_ptr syncRequestImpl; + if (this->_plugin && this->_plugin->GetCore() && this->_plugin->GetCore()->isNewAPI()) + syncRequestImpl = std::make_shared(_graphDesc, _parameters, _results, + _inputInfo, _outputInfo, + _graphMetaData.stagesMeta, _config, _log, + _executor, _constDatas, _isNetworkConstant); + if (!syncRequestImpl) + syncRequestImpl = std::make_shared(_graphDesc, _networkInputs, _networkOutputs, + _inputInfo, _outputInfo, + _graphMetaData.stagesMeta, _config, _log, + _executor, _constDatas, _isNetworkConstant); syncRequestImpl->setPointerToExecutableNetworkInternal(shared_from_this()); auto taskExecutorGetResult = getNextTaskExecutor(); return std::make_shared( diff --git a/inference-engine/src/vpu/myriad_plugin/myriad_infer_request.cpp b/inference-engine/src/vpu/myriad_plugin/myriad_infer_request.cpp index 5192f1d2289..73d13b699e4 100644 --- a/inference-engine/src/vpu/myriad_plugin/myriad_infer_request.cpp +++ b/inference-engine/src/vpu/myriad_plugin/myriad_infer_request.cpp @@ -28,6 +28,24 @@ using namespace InferenceEngine; #define MEMCPY(dst, src, bytes) std::copy_n((src), (bytes), (dst)) +MyriadInferRequest::MyriadInferRequest(GraphDesc &graphDesc, + const std::vector>& inputs, + const std::vector>& outputs, + DataInfo& compilerInputsInfo, + DataInfo& compilerOutputsInfo, + const std::vector &blobMetaData, + const PluginConfiguration& myriadConfig, + const Logger::Ptr &log, + const MyriadExecutorPtr &executor, + std::map constDatas, + bool isNetworkConstant = true) : + IInferRequestInternal(inputs, outputs), _executor(executor), + _log(log), _stagesMetaData(blobMetaData), _config(myriadConfig), + _inputInfo(compilerInputsInfo), _outputInfo(compilerOutputsInfo), + _graphDesc(graphDesc), _constDatas(constDatas), _isNetworkConstant(isNetworkConstant) { + CreateInferRequest(); +} + MyriadInferRequest::MyriadInferRequest(GraphDesc &graphDesc, InferenceEngine::InputsDataMap networkInputs, InferenceEngine::OutputsDataMap networkOutputs, @@ -43,6 +61,10 @@ MyriadInferRequest::MyriadInferRequest(GraphDesc &graphDesc, _log(log), _stagesMetaData(blobMetaData), _config(myriadConfig), _inputInfo(compilerInputsInfo), _outputInfo(compilerOutputsInfo), _graphDesc(graphDesc), _constDatas(constDatas), _isNetworkConstant(isNetworkConstant) { + CreateInferRequest(); +} + +void MyriadInferRequest::CreateInferRequest() { VPU_PROFILE(MyriadInferRequest); const auto& ioStrides = _config.get(); @@ -84,8 +106,8 @@ MyriadInferRequest::MyriadInferRequest(GraphDesc &graphDesc, _outputs[networkOutput.first] = outputBlob; } - inputBuffer .resize(compilerInputsInfo.totalSize); - resultBuffer.resize(compilerOutputsInfo.totalSize); + inputBuffer .resize(_inputInfo.totalSize); + resultBuffer.resize(_outputInfo.totalSize); VPU_THROW_UNLESS( !_networkOutputs.empty() && !(_networkInputs.empty() && !_isNetworkConstant), diff --git a/inference-engine/src/vpu/myriad_plugin/myriad_infer_request.h b/inference-engine/src/vpu/myriad_plugin/myriad_infer_request.h index ccfe1ddc6eb..e1a042d35e5 100644 --- a/inference-engine/src/vpu/myriad_plugin/myriad_infer_request.h +++ b/inference-engine/src/vpu/myriad_plugin/myriad_infer_request.h @@ -35,21 +35,34 @@ class MyriadInferRequest : public InferenceEngine::IInferRequestInternal { std::vector inputBuffer; std::map _constDatas; bool _isNetworkConstant; + void CreateInferRequest(); public: typedef std::shared_ptr Ptr; - explicit MyriadInferRequest(GraphDesc &_graphDesc, - InferenceEngine::InputsDataMap networkInputs, - InferenceEngine::OutputsDataMap networkOutputs, - DataInfo& compilerInputsInfo, - DataInfo& compilerOutputsInfo, - const std::vector &blobMetaData, - const PluginConfiguration &myriadConfig, - const Logger::Ptr &log, - const MyriadExecutorPtr &executor, - std::map constDatas, - bool isNetworkConstant); + MyriadInferRequest(GraphDesc &_graphDesc, + InferenceEngine::InputsDataMap networkInputs, + InferenceEngine::OutputsDataMap networkOutputs, + DataInfo& compilerInputsInfo, + DataInfo& compilerOutputsInfo, + const std::vector &blobMetaData, + const PluginConfiguration &myriadConfig, + const Logger::Ptr &log, + const MyriadExecutorPtr &executor, + std::map constDatas, + bool isNetworkConstant); + + MyriadInferRequest(GraphDesc &_graphDesc, + const std::vector>& inputs, + const std::vector>& outputs, + DataInfo& compilerInputsInfo, + DataInfo& compilerOutputsInfo, + const std::vector &blobMetaData, + const PluginConfiguration &myriadConfig, + const Logger::Ptr &log, + const MyriadExecutorPtr &executor, + std::map constDatas, + bool isNetworkConstant); void InferImpl() override; void InferAsync(); diff --git a/inference-engine/tests/functional/inference_engine/ov_infer_request_test.cpp b/inference-engine/tests/functional/inference_engine/ov_infer_request_test.cpp index 65429ddc57c..e757bfaed01 100644 --- a/inference-engine/tests/functional/inference_engine/ov_infer_request_test.cpp +++ b/inference-engine/tests/functional/inference_engine/ov_infer_request_test.cpp @@ -16,12 +16,12 @@ using namespace InferenceEngine::details; TEST(InferRequestOVTests, throwsOnUninitializedSetTensor) { ov::runtime::InferRequest req; - ASSERT_THROW(req.set_tensor({}, {}), ov::Exception); + ASSERT_THROW(req.set_tensor("", {}), ov::Exception); } TEST(InferRequestOVTests, throwsOnUninitializedGetTensor) { ov::runtime::InferRequest req; - ASSERT_THROW(req.get_tensor({}), ov::Exception); + ASSERT_THROW(req.get_tensor(""), ov::Exception); } TEST(InferRequestOVTests, throwsOnUninitializedInfer) { @@ -60,9 +60,26 @@ TEST(InferRequestOVTests, throwsOnUninitializedQueryState) { ASSERT_THROW(req.query_state(), ov::Exception); } +TEST(InferRequestOVTests, throwsOnUninitializedSetRemoteTensorWithName) { + ov::runtime::InferRequest req; + ov::runtime::RemoteTensor remote_tensor; + ASSERT_THROW(req.set_tensor("", remote_tensor), ov::Exception); +} + +TEST(InferRequestOVTests, throwsOnUninitializedSetInputRemoteTensor) { + ov::runtime::InferRequest req; + ov::runtime::RemoteTensor remote_tensor; + ASSERT_THROW(req.set_input_tensor(0, remote_tensor), ov::Exception); +} + +TEST(InferRequestOVTests, throwsOnUninitializedSetOutputRemoteTensor) { + ov::runtime::InferRequest req; + ov::runtime::RemoteTensor remote_tensor; + ASSERT_THROW(req.set_output_tensor(0, remote_tensor), ov::Exception); +} TEST(InferRequestOVTests, throwsOnUninitializedSetRemoteTensor) { ov::runtime::InferRequest req; ov::runtime::RemoteTensor remote_tensor; - ASSERT_THROW(req.set_tensor({}, remote_tensor), ov::Exception); -} \ No newline at end of file + ASSERT_THROW(req.set_tensor(ov::Output(), remote_tensor), ov::Exception); +} diff --git a/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/behavior/infer_request/infer_requset_dynamic.cpp b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/behavior/infer_request/infer_requset_dynamic.cpp index 5100e8828fb..ff940261d53 100644 --- a/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/behavior/infer_request/infer_requset_dynamic.cpp +++ b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/behavior/infer_request/infer_requset_dynamic.cpp @@ -28,12 +28,14 @@ std::shared_ptr getFunction1() { auto params = ngraph::builder::makeParams(ngPrc, {inputShape}); params.front()->set_friendly_name("Param_1"); - params.front()->get_output_tensor(0).set_names({"Tensor_1"}); + params.front()->get_output_tensor(0).set_names({"input_tensor"}); auto in2add = ngraph::builder::makeConstant(ngPrc, {1, 4, 1, 1}, std::vector{}, true); auto add = ngraph::builder::makeEltwise(params[0], in2add, ngraph::helpers::EltwiseTypes::ADD); auto relu1 = std::make_shared(add->output(0)); + relu1->get_output_tensor(0).set_names({"relu1"}); auto relu2 = std::make_shared(add->output(0)); + relu2->get_output_tensor(0).set_names({"relu2"}); ngraph::NodeVector results{relu1, relu2}; return std::make_shared(results, params, "AddTwoOutputEdges"); @@ -45,7 +47,7 @@ std::shared_ptr getFunction2() { auto params = ngraph::builder::makeParams(ngPrc, {inputShape}); params.front()->set_friendly_name("Param_1"); - params.front()->get_output_tensor(0).set_names({"Tensor_1"}); + params.front()->get_output_tensor(0).set_names({"input_tensor"}); auto split = ngraph::builder::makeSplit(params[0], ngPrc, 2, 1); auto in2add = ngraph::builder::makeConstant(ngPrc, {1, 2, 1, 1}, std::vector{}, true); @@ -57,6 +59,7 @@ std::shared_ptr getFunction2() { auto relu2 = std::make_shared(mult); auto concat = std::make_shared(ngraph::OutputVector{relu1->output(0), relu2->output(0)}, 3); + concat->get_output_tensor(0).set_names({"concat"}); return std::make_shared(concat, params, "SplitAddConcat"); } diff --git a/inference-engine/tests/functional/plugin/gna/shared_tests_instances/skip_tests_config.cpp b/inference-engine/tests/functional/plugin/gna/shared_tests_instances/skip_tests_config.cpp index 977cff05e26..5b306e6a99c 100644 --- a/inference-engine/tests/functional/plugin/gna/shared_tests_instances/skip_tests_config.cpp +++ b/inference-engine/tests/functional/plugin/gna/shared_tests_instances/skip_tests_config.cpp @@ -62,6 +62,7 @@ std::vector disabledTestPatterns() { R"(.*OVExecutableNetworkBaseTest.*canLoadCorrectNetworkToGetExecutableWithIncorrectConfig.*)", R"(.*OVExecutableNetworkBaseTest.*CanSetConfigToExecNet.*)", R"(.*OVExecutableNetworkBaseTest.*CanGetInputsInfoAndCheck.*)", + R"(.*OVExecutableNetworkBaseTest.*getOutputsFromSplitFunctionWithSeveralOutputs.*)", R"(.*OVClassHeteroExecutableNetworkGetMetricTest_TARGET_FALLBACK.*GetMetricNoThrow.*)", R"(.*Behavior.*OVExecutableNetworkBaseTest.*get(Inputs|Outputs)FromFunctionWithSeveral(Inputs|Outputs).*)", // TODO: Issue: 29577 diff --git a/inference-engine/tests/functional/plugin/shared/CMakeLists.txt b/inference-engine/tests/functional/plugin/shared/CMakeLists.txt index c3d1ae1af05..89785f98bac 100644 --- a/inference-engine/tests/functional/plugin/shared/CMakeLists.txt +++ b/inference-engine/tests/functional/plugin/shared/CMakeLists.txt @@ -32,6 +32,7 @@ addIeTarget( PUBLIC pugixml::static funcTestUtils + ngraph_test_util ngraphFunctions lptNgraphFunctions sharedTestClasses diff --git a/inference-engine/tests/functional/plugin/shared/include/behavior/infer_request/infer_request_dynamic.hpp b/inference-engine/tests/functional/plugin/shared/include/behavior/infer_request/infer_request_dynamic.hpp index 6cfaa1620df..31c5b29e040 100644 --- a/inference-engine/tests/functional/plugin/shared/include/behavior/infer_request/infer_request_dynamic.hpp +++ b/inference-engine/tests/functional/plugin/shared/include/behavior/infer_request/infer_request_dynamic.hpp @@ -85,7 +85,7 @@ protected: }; TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithoutSetShape) { - const std::string tensor_name = "Tensor_1"; + const std::string tensor_name = "input_tensor"; std::map shapes; shapes[tensor_name] = {ov::Dimension::dynamic(), 4, 20, 20}; ASSERT_NO_THROW(function->reshape(shapes)); @@ -95,11 +95,11 @@ TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithoutSetShape) { ov::runtime::InferRequest req; ov::runtime::Tensor tensor; ASSERT_NO_THROW(req = execNet.create_infer_request()); - ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name())); + ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name())); } TEST_P(InferRequestDynamicTests, InferDynamicNetworkBoundWithoutSetShape) { - const std::string tensor_name = "Tensor_1"; + const std::string tensor_name = "input_tensor"; std::map shapes; shapes[tensor_name] = {ov::Dimension(0, 5), 4, 20, 20}; ASSERT_NO_THROW(function->reshape(shapes)); @@ -109,12 +109,12 @@ TEST_P(InferRequestDynamicTests, InferDynamicNetworkBoundWithoutSetShape) { ov::runtime::InferRequest req; ov::runtime::Tensor tensor; ASSERT_NO_THROW(req = execNet.create_infer_request()); - ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name())); + ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name())); } TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithGetTensor) { - const std::string tensor_name = "Tensor_1"; + const std::string tensor_name = "input_tensor"; const ov::Shape refShape = inOutShapes[0].first; const ov::Shape refOutShape = inOutShapes[0].second; std::map shapes; @@ -125,11 +125,10 @@ TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithGetTensor) { // Create InferRequest ov::runtime::InferRequest req; ov::runtime::Tensor tensor, otensor; - const std::string outputname = ngraph::op::util::create_ie_output_name( - function->get_results().front()->input_value(0)); + const std::string outputname = function->outputs().back().get_any_name(); ASSERT_NO_THROW(req = execNet.create_infer_request()); //ASSERT_NO_THROW(req.SetShape(tensor_name, {1, 4, 20, 20})); - ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name())); + ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name())); ASSERT_NO_THROW(tensor.set_shape({1, 4, 20, 20})); ASSERT_EQ(tensor.get_shape(), refShape); ASSERT_NO_THROW(otensor = req.get_tensor(outputname)); @@ -144,7 +143,7 @@ TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithGetTensor) { } TEST_P(InferRequestDynamicTests, InferUpperBoundNetworkWithGetTensor) { - const std::string tensor_name = "Tensor_1"; + const std::string tensor_name = "input_tensor"; const ov::Shape refShape = inOutShapes[0].first; const ov::Shape refOutShape = inOutShapes[0].second; std::map shapes; @@ -155,14 +154,13 @@ TEST_P(InferRequestDynamicTests, InferUpperBoundNetworkWithGetTensor) { // Create InferRequest ov::runtime::InferRequest req; ov::runtime::Tensor tensor, otensor; - const std::string outputname = ngraph::op::util::create_ie_output_name( - function->get_results().front()->input_value(0)); + const std::string outputname = function->outputs().back().get_any_name(); ASSERT_NO_THROW(req = execNet.create_infer_request()); //ASSERT_NO_THROW(req.SetShape(tensor_name, {1, 4, 20, 20})); ASSERT_NO_THROW(otensor = req.get_tensor(outputname)); ASSERT_EQ(0, otensor.get_size()); // output tensor is not allocated ASSERT_EQ(function->output().get_element_type(), otensor.get_element_type()); // by it has type - ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name())); + ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name())); ASSERT_NO_THROW(tensor.set_shape({1, 4, 20, 20})); ASSERT_EQ(tensor.get_shape(), refShape); ASSERT_NO_THROW(req.infer()); @@ -172,7 +170,7 @@ TEST_P(InferRequestDynamicTests, InferUpperBoundNetworkWithGetTensor) { } TEST_P(InferRequestDynamicTests, InferFullyDynamicNetworkWithGetTensor) { - const std::string tensor_name = "Tensor_1"; + const std::string tensor_name = "input_tensor"; const ov::Shape refShape = inOutShapes[0].first; const ov::Shape refOutShape = inOutShapes[0].second; std::map shapes; @@ -183,25 +181,24 @@ TEST_P(InferRequestDynamicTests, InferFullyDynamicNetworkWithGetTensor) { // Create InferRequest ov::runtime::InferRequest req; ov::runtime::Tensor tensor, otensor; - const std::string outputname = ngraph::op::util::create_ie_output_name( - function->get_results().front()->input_value(0)); + const std::string outputName = function->outputs().back().get_any_name(); ASSERT_NO_THROW(req = execNet.create_infer_request()); //ASSERT_NO_THROW(req.SetShape(tensor_name, {1, 4, 20, 20})); - ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name())); + ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name())); ASSERT_NO_THROW(tensor.set_shape({1, 4, 20, 20})); ASSERT_EQ(tensor.get_shape(), refShape); - ASSERT_NO_THROW(otensor = req.get_tensor(outputname)); + ASSERT_NO_THROW(otensor = req.get_tensor(outputName)); ASSERT_EQ(0, otensor.get_size()); // output tensor is not allocated ASSERT_EQ(function->output().get_element_type(), otensor.get_element_type()); // by it has type ASSERT_NO_THROW(req.infer()); ASSERT_NO_THROW(req.start_async()); ASSERT_NO_THROW(req.wait()); - ASSERT_NO_THROW(otensor = req.get_tensor(ngraph::op::util::create_ie_output_name(function->get_results().front()->input_value(0)))); + ASSERT_NO_THROW(otensor = req.get_tensor(outputName)); ASSERT_EQ(otensor.get_shape(), refOutShape); } TEST_P(InferRequestDynamicTests, InferOutOfRangeShapeNetworkWithGetTensorLower) { - const std::string tensor_name = "Tensor_1"; + const std::string tensor_name = "input_tensor"; const ov::Shape refShape = inOutShapes[0].first; const ov::Shape refOutShape = inOutShapes[0].second; std::map shapes; @@ -213,14 +210,14 @@ TEST_P(InferRequestDynamicTests, InferOutOfRangeShapeNetworkWithGetTensorLower) ov::runtime::InferRequest req; ov::runtime::Tensor tensor; ASSERT_NO_THROW(req = execNet.create_infer_request()); - ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name())); + ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name())); ASSERT_NO_THROW(tensor.set_shape({1, 4, 20, 20})); // Plugin may or may not throw in case if input tensor has dimensions that are out of bounds //ASSERT_THROW(req.infer(), ov::Exception); } TEST_P(InferRequestDynamicTests, InferOutOfRangeShapeNetworkWithGetTensorUpper) { - const std::string tensor_name = "Tensor_1"; + const std::string tensor_name = "input_tensor"; const ov::Shape refShape = inOutShapes[0].first; const ov::Shape refOutShape = inOutShapes[0].second; std::map shapes; @@ -232,14 +229,14 @@ TEST_P(InferRequestDynamicTests, InferOutOfRangeShapeNetworkWithGetTensorUpper) ov::runtime::InferRequest req; ov::runtime::Tensor tensor; ASSERT_NO_THROW(req = execNet.create_infer_request()); - ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name())); + ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name())); ASSERT_NO_THROW(tensor.set_shape({3, 4, 20, 20})); // Plugin may or may not throw in case if input tensor has dimensions that are out of bounds // ASSERT_THROW(req.infer(), ov::Exception); } TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithGetTensor2times) { - const std::string tensor_name = "Tensor_1"; + const std::string tensor_name = "input_tensor"; const ov::Shape refShape = inOutShapes[0].first; const ov::Shape refShape2 = inOutShapes[1].first; const ov::Shape refOutShape = inOutShapes[0].second; @@ -253,28 +250,29 @@ TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithGetTensor2times) { ov::runtime::InferRequest req; ov::runtime::Tensor tensor; ASSERT_NO_THROW(req = execNet.create_infer_request()); - ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name())); + ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name())); ASSERT_NO_THROW(tensor.set_shape(refShape)); ASSERT_EQ(tensor.get_shape(), refShape); ASSERT_NO_THROW(req.infer()); ASSERT_NO_THROW(req.start_async()); req.wait(); - ASSERT_NO_THROW(tensor = req.get_tensor(ngraph::op::util::create_ie_output_name(function->get_results().front()->input_value(0)))); + const std::string outputName = function->outputs().back().get_any_name(); + ASSERT_NO_THROW(tensor = req.get_tensor(outputName)); ASSERT_EQ(tensor.get_shape(), refOutShape); - ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name())); + ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name())); ASSERT_NO_THROW(tensor.set_shape(refShape2)); ASSERT_EQ(tensor.get_shape(), refShape2); ASSERT_NO_THROW(req.infer()); ASSERT_NO_THROW(req.start_async()); req.wait(); - ASSERT_NO_THROW(tensor = req.get_tensor(ngraph::op::util::create_ie_output_name(function->get_results().front()->input_value(0)))); + ASSERT_NO_THROW(tensor = req.get_tensor(outputName)); ASSERT_EQ(tensor.get_shape(), refOutShape2); } TEST_P(InferRequestDynamicTests, GetSameTensor2times) { - const std::string tensor_name = "Tensor_1"; + const std::string tensor_name = "input_tensor"; const ov::Shape refShape = inOutShapes[0].first; std::map shapes; shapes[tensor_name] = {ov::Dimension::dynamic(), 4, 20, 20}; @@ -285,15 +283,15 @@ TEST_P(InferRequestDynamicTests, GetSameTensor2times) { ov::runtime::InferRequest req; ov::runtime::Tensor tensor; ASSERT_NO_THROW(req = execNet.create_infer_request()); - ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name())); + ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name())); ASSERT_NO_THROW(tensor.set_shape(refShape)); ASSERT_EQ(tensor.get_shape(), refShape); - ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name())); + ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name())); ASSERT_EQ(tensor.get_shape(), refShape); } TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithSetTensor) { - const std::string tensor_name = "Tensor_1"; + const std::string tensor_name = "input_tensor"; const ov::Shape refShape = inOutShapes[0].first; const ov::Shape refOutShape = inOutShapes[0].second; std::map shapes; @@ -305,17 +303,18 @@ TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithSetTensor) { ov::runtime::InferRequest req; ov::runtime::Tensor tensor(ov::element::f32, refShape); ASSERT_NO_THROW(req = execNet.create_infer_request()); - ASSERT_NO_THROW(req.set_tensor(function->get_parameters().back()->get_friendly_name(), tensor)); + ASSERT_NO_THROW(req.set_tensor(function->inputs().back().get_any_name(), tensor)); ASSERT_EQ(tensor.get_shape(), refShape); ASSERT_NO_THROW(req.infer()); ASSERT_NO_THROW(req.start_async()); ASSERT_NO_THROW(req.wait()); - ASSERT_NO_THROW(tensor = req.get_tensor(ngraph::op::util::create_ie_output_name(function->get_results().front()->input_value(0)))); + const std::string outputName = function->outputs().back().get_any_name(); + ASSERT_NO_THROW(tensor = req.get_tensor(outputName)); ASSERT_EQ(tensor.get_shape(), refOutShape); } TEST_P(InferRequestDynamicTests, InferFullyDynamicNetworkWithSetTensor) { - const std::string tensor_name = "Tensor_1"; + const std::string tensor_name = "input_tensor"; const ov::Shape refShape = inOutShapes[0].first; const ov::Shape refOutShape = inOutShapes[0].second; std::map shapes; @@ -326,12 +325,11 @@ TEST_P(InferRequestDynamicTests, InferFullyDynamicNetworkWithSetTensor) { // Create InferRequest ov::runtime::InferRequest req; ov::runtime::Tensor tensor(ov::element::f32, refShape), otensor; - const std::string outputname = ngraph::op::util::create_ie_output_name( - function->get_results().front()->input_value(0)); + const std::string outputName = function->outputs().back().get_any_name(); ASSERT_NO_THROW(req = execNet.create_infer_request()); - ASSERT_NO_THROW(req.set_tensor(function->get_parameters().back()->get_friendly_name(), tensor)); + ASSERT_NO_THROW(req.set_tensor(function->inputs().back().get_any_name(), tensor)); ASSERT_EQ(tensor.get_shape(), refShape); - ASSERT_NO_THROW(otensor = req.get_tensor(outputname)); + ASSERT_NO_THROW(otensor = req.get_tensor(outputName)); ASSERT_EQ(0, otensor.get_size()); // output tensor is not allocated ASSERT_EQ(function->output().get_element_type(), otensor.get_element_type()); // by it has type ASSERT_NO_THROW(req.infer()); @@ -339,13 +337,13 @@ TEST_P(InferRequestDynamicTests, InferFullyDynamicNetworkWithSetTensor) { ASSERT_NO_THROW(req.start_async()); ASSERT_NO_THROW(req.wait()); ASSERT_EQ(otensor.get_shape(), refOutShape); - ASSERT_NO_THROW(tensor = req.get_tensor(ngraph::op::util::create_ie_output_name(function->get_results().front()->input_value(0)))); + ASSERT_NO_THROW(tensor = req.get_tensor(outputName)); ASSERT_EQ(tensor.get_shape(), refOutShape); ASSERT_EQ(otensor.get_shape(), refOutShape); } TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithSetTensor2times) { - const std::string tensor_name = "Tensor_1"; + const std::string tensor_name = "input_tensor"; const ov::Shape refShape = inOutShapes[0].first; const ov::Shape refShape2 = inOutShapes[1].first; const ov::Shape refOutShape = inOutShapes[0].second; @@ -353,6 +351,7 @@ TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithSetTensor2times) { std::map shapes; shapes[tensor_name] = {ov::Dimension::dynamic(), 4, 20, 20}; ASSERT_NO_THROW(function->reshape(shapes)); + const std::string outputName = function->outputs().back().get_any_name(); // Load ov::Function to target plugins auto execNet = ie->compile_model(function, targetDevice, configuration); // Create InferRequest @@ -360,22 +359,22 @@ TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithSetTensor2times) { ov::runtime::Tensor tensor(ov::element::f32, refShape); ASSERT_NO_THROW(req = execNet.create_infer_request()); - ASSERT_NO_THROW(req.set_tensor(function->get_parameters().back()->get_friendly_name(), tensor)); + ASSERT_NO_THROW(req.set_tensor(function->inputs().back().get_any_name(), tensor)); ASSERT_EQ(tensor.get_shape(), refShape); ASSERT_NO_THROW(req.infer()); ASSERT_NO_THROW(req.start_async()); ASSERT_NO_THROW(req.wait()); - ASSERT_NO_THROW(tensor = req.get_tensor(ngraph::op::util::create_ie_output_name(function->get_results().front()->input_value(0)))); + ASSERT_NO_THROW(tensor = req.get_tensor(outputName)); ASSERT_EQ(tensor.get_shape(), refOutShape); tensor = ov::runtime::Tensor(ov::element::f32, refShape2); - ASSERT_NO_THROW(req.set_tensor(function->get_parameters().back()->get_friendly_name(), tensor)); + ASSERT_NO_THROW(req.set_tensor(function->inputs().back().get_any_name(), tensor)); ASSERT_EQ(tensor.get_shape(), refShape2); ASSERT_NO_THROW(req.infer()); ASSERT_NO_THROW(req.start_async()); ASSERT_NO_THROW(req.wait()); - ASSERT_NO_THROW(tensor = req.get_tensor(ngraph::op::util::create_ie_output_name(function->get_results().front()->input_value(0)))); + ASSERT_NO_THROW(tensor = req.get_tensor(outputName)); ASSERT_EQ(tensor.get_shape(), refOutShape2); } -} // namespace BehaviorTestsDefinitions \ No newline at end of file +} // namespace BehaviorTestsDefinitions diff --git a/inference-engine/tests/functional/plugin/shared/include/behavior/infer_request/inference_chaining.hpp b/inference-engine/tests/functional/plugin/shared/include/behavior/infer_request/inference_chaining.hpp index 6f8b1ce9195..4ed968e984b 100644 --- a/inference-engine/tests/functional/plugin/shared/include/behavior/infer_request/inference_chaining.hpp +++ b/inference-engine/tests/functional/plugin/shared/include/behavior/infer_request/inference_chaining.hpp @@ -2,8 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 // -#include #include + +#include #include #include #include @@ -12,6 +13,7 @@ #include "base/behavior_test_utils.hpp" #include "openvino/core/attribute_visitor.hpp" +#include "openvino/core/function.hpp" #include "openvino/core/node.hpp" #include "openvino/core/partial_shape.hpp" #include "openvino/core/rank.hpp" @@ -19,8 +21,6 @@ #include "openvino/core/type/element_type.hpp" #include "openvino/core/type/element_type_traits.hpp" #include "openvino/op/parameter.hpp" -#include "openvino/core/function.hpp" -#include "ngraph_functions/builders.hpp" #include "openvino/runtime/infer_request.hpp" #include "openvino/runtime/tensor.hpp" @@ -107,9 +107,9 @@ public: // perform inference chaining if (outputToInput) { - ASSERT_NO_THROW(r1.set_tensor("param_0", r0.get_tensor("result_0"))); + ASSERT_NO_THROW(r1.set_tensor("input_tensor_0", r0.get_tensor("result_tensor_0"))); } else { - ASSERT_NO_THROW(r0.set_tensor("result_0", r1.get_tensor("param_0"))); + ASSERT_NO_THROW(r0.set_tensor("result_tensor_0", r1.get_tensor("input_tensor_0"))); } // create input tensors @@ -118,15 +118,15 @@ public: ov::runtime::Tensor t2 = tensor(std::vector{7.0f, 8.0f, 9.0f}); ov::runtime::Tensor t3 = tensor(std::vector{2.0f, 3.0f, 2.0f}); - ASSERT_NO_THROW(r0.set_tensor("param_0", t0)); - ASSERT_NO_THROW(r0.set_tensor("param_1", t1)); - ASSERT_NO_THROW(r0.set_tensor("param_2", t2)); - ASSERT_NO_THROW(r1.set_tensor("param_1", t3)); + ASSERT_NO_THROW(r0.set_tensor("input_tensor_0", t0)); + ASSERT_NO_THROW(r0.set_tensor("input_tensor_1", t1)); + ASSERT_NO_THROW(r0.set_tensor("input_tensor_2", t2)); + ASSERT_NO_THROW(r1.set_tensor("input_tensor_1", t3)); - ASSERT_NO_THROW(r2.set_tensor("param_0", t0)); - ASSERT_NO_THROW(r2.set_tensor("param_1", t1)); - ASSERT_NO_THROW(r2.set_tensor("param_2", t2)); - ASSERT_NO_THROW(r2.set_tensor("param_3", t3)); + ASSERT_NO_THROW(r2.set_tensor("input_tensor_0", t0)); + ASSERT_NO_THROW(r2.set_tensor("input_tensor_1", t1)); + ASSERT_NO_THROW(r2.set_tensor("input_tensor_2", t2)); + ASSERT_NO_THROW(r2.set_tensor("input_tensor_3", t3)); ASSERT_NO_THROW(r0.infer()); ASSERT_NO_THROW(r1.infer()); @@ -136,9 +136,9 @@ public: std::vector reference1 = {12.0f, 15.0f, 18.0f}; std::vector reference2 = {24.0f, 45.0f, 36.0f}; - auto rti = r0.get_tensor("result_0"); - auto rt0 = r1.get_tensor("result_0"); - auto rt1 = r2.get_tensor("result_0"); + auto rti = r0.get_tensor("result_tensor_0"); + auto rt0 = r1.get_tensor("result_tensor_0"); + auto rt1 = r2.get_tensor("result_tensor_0"); for (size_t i = 0; i < reference1.size(); ++i) { EXPECT_EQ(reference1[i], rti.data()[i]); @@ -148,7 +148,6 @@ public: } }; - TEST_P(OVInferenceChaining, StaticOutputToStaticInput) { // Skip test according to plugin specific disabledTestPatterns() (if any) SKIP_IF_CURRENT_TEST_IS_DISABLED() diff --git a/inference-engine/tests/functional/plugin/shared/include/behavior/ov_executable_network/exec_graph_info.hpp b/inference-engine/tests/functional/plugin/shared/include/behavior/ov_executable_network/exec_graph_info.hpp index 36c2cce21df..cf53b11e39f 100644 --- a/inference-engine/tests/functional/plugin/shared/include/behavior/ov_executable_network/exec_graph_info.hpp +++ b/inference-engine/tests/functional/plugin/shared/include/behavior/ov_executable_network/exec_graph_info.hpp @@ -100,7 +100,7 @@ TEST_P(OVExecGraphImportExportTest, importExportedFunction) { execNet.export_model(strm); ov::runtime::ExecutableNetwork importedExecNet = core->import_model(strm, targetDevice, configuration); - ASSERT_EQ(function->inputs().size(), 2); + EXPECT_EQ(function->inputs().size(), 2); EXPECT_EQ(function->inputs().size(), importedExecNet.inputs().size()); EXPECT_THROW(importedExecNet.input(), ov::Exception); EXPECT_EQ(function->input(0).get_tensor().get_names(), importedExecNet.input(0).get_tensor().get_names()); @@ -194,8 +194,8 @@ TEST_P(OVExecGraphImportExportTest, readFromV10IR) { function = core->read_model(model, ov::runtime::Tensor()); EXPECT_EQ(function->inputs().size(), 1); EXPECT_EQ(function->outputs().size(), 1); - EXPECT_NO_THROW(function->input("in1")); // remove if read_model does not change function names - EXPECT_NO_THROW(function->output("round")); // remove if read_model does not change function names + EXPECT_NO_THROW(function->input("in1")); // remove if read_model does not change function names + EXPECT_NO_THROW(function->output("round")); // remove if read_model does not change function names ov::runtime::ExecutableNetwork execNet = core->compile_model(function, targetDevice, configuration); EXPECT_EQ(execNet.inputs().size(), 1); @@ -256,14 +256,14 @@ TEST_P(OVExecGraphImportExportTest, importExportedIENetwork) { execNet.Export(strm); ov::runtime::ExecutableNetwork importedExecNet = core->import_model(strm, targetDevice, configuration); - ASSERT_EQ(function->inputs().size(), 2); + EXPECT_EQ(function->inputs().size(), 2); EXPECT_EQ(function->inputs().size(), importedExecNet.inputs().size()); EXPECT_THROW(importedExecNet.input(), ov::Exception); EXPECT_NO_THROW(importedExecNet.input("data1").get_node()); EXPECT_NO_THROW(importedExecNet.input("data2").get_node()); EXPECT_NO_THROW(importedExecNet.input("param1").get_node()); EXPECT_NO_THROW(importedExecNet.input("param2").get_node()); - ASSERT_EQ(function->outputs().size(), 2); + EXPECT_EQ(function->outputs().size(), 2); EXPECT_EQ(function->outputs().size(), importedExecNet.outputs().size()); EXPECT_THROW(importedExecNet.output(), ov::Exception); EXPECT_NE(function->output(0).get_tensor().get_names(), @@ -319,11 +319,11 @@ TEST_P(OVExecGraphImportExportTest, ieImportExportedFunction) { execNet.export_model(strm); InferenceEngine::ExecutableNetwork importedExecNet = ie->ImportNetwork(strm, targetDevice, configuration); - ASSERT_EQ(function->inputs().size(), 2); + EXPECT_EQ(function->inputs().size(), 2); EXPECT_EQ(function->inputs().size(), importedExecNet.GetInputsInfo().size()); EXPECT_NO_THROW(importedExecNet.GetInputsInfo()["param1"]); EXPECT_NO_THROW(importedExecNet.GetInputsInfo()["param2"]); - ASSERT_EQ(function->outputs().size(), 2); + EXPECT_EQ(function->outputs().size(), 2); EXPECT_EQ(function->outputs().size(), importedExecNet.GetOutputsInfo().size()); EXPECT_NO_THROW(importedExecNet.GetOutputsInfo()["relu_op"]); EXPECT_NO_THROW(importedExecNet.GetOutputsInfo()["concat_op"]); @@ -338,4 +338,4 @@ TEST_P(OVExecGraphImportExportTest, ieImportExportedFunction) { } // namespace behavior } // namespace test -} // namespace ov \ No newline at end of file +} // namespace ov diff --git a/inference-engine/tests/functional/plugin/shared/include/behavior/ov_executable_network/exec_network_base.hpp b/inference-engine/tests/functional/plugin/shared/include/behavior/ov_executable_network/exec_network_base.hpp index 70eaa9337fe..c38a67fcdac 100644 --- a/inference-engine/tests/functional/plugin/shared/include/behavior/ov_executable_network/exec_network_base.hpp +++ b/inference-engine/tests/functional/plugin/shared/include/behavior/ov_executable_network/exec_network_base.hpp @@ -2,15 +2,15 @@ // SPDX-License-Identifcorer: Apache-2.0 // -#include - #include +#include #include -#include "base/ov_behavior_test_utils.hpp" -#include "common_test_utils/ngraph_test_utils.hpp" -#include "common_test_utils/file_utils.hpp" +#include "base/ov_behavior_test_utils.hpp" +#include "common_test_utils/file_utils.hpp" +#include "common_test_utils/ngraph_test_utils.hpp" #include "functional_test_utils/plugin_cache.hpp" +#include "openvino/runtime/tensor.hpp" namespace ov { namespace test { @@ -26,7 +26,7 @@ public: std::ostringstream result; result << "targetDevice=" << targetDevice << "_"; if (!configuration.empty()) { - for (auto &configItem : configuration) { + for (auto& configItem : configuration) { result << "configItem=" << configItem.first << "_" << configItem.second << "_"; } } @@ -46,6 +46,29 @@ public: } } + bool compareTensors(const ov::runtime::Tensor& t1, const ov::runtime::Tensor& t2) { + void* data1; + void* data2; + try { + data1 = t1.data(); + } catch (const ov::Exception&) { + // Remote tensor + data1 = nullptr; + } + try { + data2 = t2.data(); + } catch (const ov::Exception&) { + // Remote tensor + data2 = nullptr; + } + return t1.get_element_type() == t2.get_element_type() && + t1.get_shape() == t2.get_shape() && + t1.get_byte_size() == t2.get_byte_size() && + t1.get_size() == t2.get_size() && + t1.get_strides() == t2.get_strides() && + data1 == data2; + } + protected: std::shared_ptr core = utils::PluginCache::get().core(); std::string targetDevice; @@ -54,99 +77,99 @@ protected: }; TEST_P(OVExecutableNetworkBaseTest, canLoadCorrectNetworkToGetExecutable) { - ASSERT_NO_THROW(auto execNet = core->compile_model(function, targetDevice, configuration)); + EXPECT_NO_THROW(auto execNet = core->compile_model(function, targetDevice, configuration)); } TEST_P(OVExecutableNetworkBaseTest, canLoadCorrectNetworkToGetExecutableWithIncorrectConfig) { std::map incorrectConfig = {{"abc", "def"}}; - ASSERT_ANY_THROW(auto execNet = core->compile_model(function, targetDevice, configuration)); + EXPECT_ANY_THROW(auto execNet = core->compile_model(function, targetDevice, configuration)); } TEST_P(OVExecutableNetworkBaseTest, canLoadCorrectNetworkToGetExecutableAndCreateInferRequest) { auto execNet = core->compile_model(function, targetDevice, configuration); - ASSERT_NO_THROW(auto req = execNet.create_infer_request()); + EXPECT_NO_THROW(auto req = execNet.create_infer_request()); } TEST_P(OVExecutableNetworkBaseTest, checkGetExecGraphInfoIsNotNullptr) { auto execNet = core->compile_model(function, targetDevice, configuration); auto execGraph = execNet.get_runtime_function(); - ASSERT_NE(execGraph, nullptr); + EXPECT_NE(execGraph, nullptr); } TEST_P(OVExecutableNetworkBaseTest, checkGetMetric) { auto execNet = core->compile_model(function, targetDevice, configuration); - ASSERT_NO_THROW(execNet.get_metric(METRIC_KEY(SUPPORTED_CONFIG_KEYS))); + EXPECT_NO_THROW(execNet.get_metric(METRIC_KEY(SUPPORTED_CONFIG_KEYS))); } TEST_P(OVExecutableNetworkBaseTest, canLoadCorrectNetworkToGetExecutableAndCheckConfig) { auto execNet = core->compile_model(function, targetDevice, configuration); - for (const auto &configItem : configuration) { + for (const auto& configItem : configuration) { InferenceEngine::Parameter param; - ASSERT_NO_THROW(param = execNet.get_config(configItem.first)); - ASSERT_FALSE(param.empty()); - ASSERT_EQ(param, InferenceEngine::Parameter(configItem.second)); + EXPECT_NO_THROW(param = execNet.get_config(configItem.first)); + EXPECT_FALSE(param.empty()); + EXPECT_EQ(param, InferenceEngine::Parameter(configItem.second)); } } TEST_P(OVExecutableNetworkBaseTest, CanSetConfigToExecNet) { auto execNet = core->compile_model(function, targetDevice); std::map config; - for (const auto &confItem : configuration) { + for (const auto& confItem : configuration) { config.insert({confItem.first, InferenceEngine::Parameter(confItem.second)}); } - ASSERT_NO_THROW(execNet.set_config(config)); + EXPECT_NO_THROW(execNet.set_config(config)); } TEST_P(OVExecutableNetworkBaseTest, CanSetConfigToExecNetWithIncorrectConfig) { auto execNet = core->compile_model(function, targetDevice); std::map incorrectConfig = {{"abc", "def"}}; std::map config; - for (const auto &confItem : incorrectConfig) { + for (const auto& confItem : incorrectConfig) { config.insert({confItem.first, InferenceEngine::Parameter(confItem.second)}); } - ASSERT_ANY_THROW(execNet.set_config(config)); + EXPECT_ANY_THROW(execNet.set_config(config)); } TEST_P(OVExecutableNetworkBaseTest, CanSetConfigToExecNetAndCheckConfigAndCheck) { auto execNet = core->compile_model(function, targetDevice); std::map config; - for (const auto &confItem : configuration) { + for (const auto& confItem : configuration) { config.insert({confItem.first, InferenceEngine::Parameter(confItem.second)}); } execNet.set_config(config); - for (const auto &configItem : configuration) { + for (const auto& configItem : configuration) { InferenceEngine::Parameter param; - ASSERT_NO_THROW(param = execNet.get_config(configItem.first)); - ASSERT_FALSE(param.empty()); - ASSERT_EQ(param, InferenceEngine::Parameter(configItem.second)); + EXPECT_NO_THROW(param = execNet.get_config(configItem.first)); + EXPECT_FALSE(param.empty()); + EXPECT_EQ(param, InferenceEngine::Parameter(configItem.second)); } } TEST_P(OVExecutableNetworkBaseTest, CanCreateTwoExeNetworks) { std::vector vec; for (auto i = 0; i < 2; i++) { - ASSERT_NO_THROW(vec.push_back(core->compile_model(function, targetDevice, configuration))); - ASSERT_NE(nullptr, function); + EXPECT_NO_THROW(vec.push_back(core->compile_model(function, targetDevice, configuration))); + EXPECT_NE(nullptr, function); } } TEST_P(OVExecutableNetworkBaseTest, CanCreateTwoExeNetworksAndCheckFunction) { std::vector vec; for (auto i = 0; i < 2; i++) { - ASSERT_NO_THROW(vec.push_back(core->compile_model(function, targetDevice, configuration))); - ASSERT_NE(nullptr, vec[i].get_runtime_function()); - ASSERT_NE(vec.begin()->get_runtime_function(), vec[i].get_runtime_function()); + EXPECT_NO_THROW(vec.push_back(core->compile_model(function, targetDevice, configuration))); + EXPECT_NE(nullptr, vec[i].get_runtime_function()); + EXPECT_NE(vec.begin()->get_runtime_function(), vec[i].get_runtime_function()); } } TEST_P(OVExecutableNetworkBaseTest, CanGetInputsInfo) { auto execNet = core->compile_model(function, targetDevice, configuration); - ASSERT_NO_THROW(auto inInfo = execNet.inputs()); + EXPECT_NO_THROW(auto inInfo = execNet.inputs()); } TEST_P(OVExecutableNetworkBaseTest, CanGetOutputsInfo) { auto execNet = core->compile_model(function, targetDevice, configuration); - ASSERT_NO_THROW(auto outInfo = execNet.outputs()); + EXPECT_NO_THROW(auto outInfo = execNet.outputs()); } TEST_P(OVExecutableNetworkBaseTest, CanGetInputsInfoAndCheck) { @@ -157,8 +180,9 @@ TEST_P(OVExecutableNetworkBaseTest, CanGetInputsInfoAndCheck) { paramVec.push_back(*input.get_tensor().get_names().begin()); } auto params = function->get_parameters(); - for (const auto ¶m : params) { - ASSERT_NE(std::find(paramVec.begin(), paramVec.end(), *param->get_output_tensor(0).get_names().begin()), paramVec.end()); + for (const auto& param : params) { + EXPECT_NE(std::find(paramVec.begin(), paramVec.end(), *param->get_output_tensor(0).get_names().begin()), + paramVec.end()); } } @@ -170,8 +194,9 @@ TEST_P(OVExecutableNetworkBaseTest, CanGetOutputsInfoAndCheck) { resVec.push_back(*out.get_tensor().get_names().begin()); } auto results = function->get_results(); - for (const auto ¶m : results) { - ASSERT_NE(std::find(resVec.begin(), resVec.end(), *param->get_output_tensor(0).get_names().begin()), resVec.end()); + for (const auto& param : results) { + EXPECT_NE(std::find(resVec.begin(), resVec.end(), *param->get_output_tensor(0).get_names().begin()), + resVec.end()); } } @@ -179,20 +204,20 @@ TEST_P(OVExecutableNetworkBaseTest, CheckExecGraphInfoBeforeExecution) { std::shared_ptr execGraph; // Load CNNNetwork to target plugins auto execNet = core->compile_model(function, targetDevice, configuration); - ASSERT_NO_THROW(execGraph = execNet.get_runtime_function()); + EXPECT_NO_THROW(execGraph = execNet.get_runtime_function()); std::map originalLayersMap; - for (const auto &layer : function->get_ops()) { + for (const auto& layer : function->get_ops()) { originalLayersMap[layer->get_friendly_name()] = 0; } int constCnt = 0; std::shared_ptr getFunction = std::dynamic_pointer_cast(execGraph); - ASSERT_NE(getFunction, nullptr); + EXPECT_NE(getFunction, nullptr); - for (const auto &op : getFunction->get_ops()) { - const ov::RTMap &rtInfo = op->get_rt_info(); + for (const auto& op : getFunction->get_ops()) { + const ov::RTMap& rtInfo = op->get_rt_info(); - auto getExecValue = [&rtInfo](const std::string ¶mName) -> std::string { + auto getExecValue = [&rtInfo](const std::string& paramName) -> std::string { auto it = rtInfo.find(paramName); IE_ASSERT(rtInfo.end() != it); auto value = std::dynamic_pointer_cast>(it->second); @@ -202,7 +227,7 @@ TEST_P(OVExecutableNetworkBaseTest, CheckExecGraphInfoBeforeExecution) { }; // Each layer from the execGraphInfo network must have PM data option set - ASSERT_EQ("not_executed", getExecValue(ExecGraphInfoSerialization::PERF_COUNTER)); + EXPECT_EQ("not_executed", getExecValue(ExecGraphInfoSerialization::PERF_COUNTER)); // Parse origin layer names (fused/merged layers) from the executable graph // and compare with layers from the original model auto origFromExecLayer = getExecValue(ExecGraphInfoSerialization::ORIGINAL_NAMES); @@ -210,20 +235,20 @@ TEST_P(OVExecutableNetworkBaseTest, CheckExecGraphInfoBeforeExecution) { constCnt++; } else { auto origFromExecLayerSep = CommonTestUtils::splitStringByDelimiter(origFromExecLayer); - std::for_each(origFromExecLayerSep.begin(), origFromExecLayerSep.end(), [&](const std::string &op) { + std::for_each(origFromExecLayerSep.begin(), origFromExecLayerSep.end(), [&](const std::string& op) { auto origLayer = originalLayersMap.find(op); - ASSERT_NE(originalLayersMap.end(), origLayer) << op; + EXPECT_NE(originalLayersMap.end(), origLayer) << op; origLayer->second++; }); } } // All layers from the original IR must be present with in ExecGraphInfo - for (auto &layer : originalLayersMap) { + for (auto& layer : originalLayersMap) { if ((layer.second == 0) && (constCnt > 0)) { constCnt--; } else { - ASSERT_GE(layer.second, 0); + EXPECT_GE(layer.second, 0); } } } @@ -232,21 +257,21 @@ TEST_P(OVExecutableNetworkBaseTest, CheckExecGraphInfoAfterExecution) { std::shared_ptr execGraph; // Load CNNNetwork to target plugins auto execNet = core->compile_model(function, targetDevice, configuration); - ASSERT_NO_THROW(execGraph = execNet.get_runtime_function()); + EXPECT_NO_THROW(execGraph = execNet.get_runtime_function()); std::map originalLayersMap; - for (const auto &layer : function->get_ops()) { + for (const auto& layer : function->get_ops()) { originalLayersMap[layer->get_friendly_name()] = 0; } int constCnt = 0; // Store all the layers from the executable graph information represented as CNNNetwork bool hasOpWithValidTime = false; auto getFunction = std::dynamic_pointer_cast(execGraph); - ASSERT_NE(nullptr, getFunction); + EXPECT_NE(nullptr, getFunction); - for (const auto &op : getFunction->get_ops()) { - const auto &rtInfo = op->get_rt_info(); + for (const auto& op : getFunction->get_ops()) { + const auto& rtInfo = op->get_rt_info(); - auto getExecValue = [&rtInfo](const std::string ¶mName) -> std::string { + auto getExecValue = [&rtInfo](const std::string& paramName) -> std::string { auto it = rtInfo.find(paramName); IE_ASSERT(rtInfo.end() != it); auto value = std::dynamic_pointer_cast>(it->second); @@ -259,9 +284,10 @@ TEST_P(OVExecutableNetworkBaseTest, CheckExecGraphInfoAfterExecution) { try { float x = static_cast(std::atof(getExecValue(ExecGraphInfoSerialization::PERF_COUNTER).c_str())); std::cout << "TIME: " << x << std::endl; - ASSERT_GE(x, 0.0f); + EXPECT_GE(x, 0.0f); hasOpWithValidTime = true; - } catch (std::exception &) {} + } catch (std::exception&) { + } // Parse origin layer names (fused/merged layers) from the executable graph // and compare with layers from the original model @@ -270,22 +296,22 @@ TEST_P(OVExecutableNetworkBaseTest, CheckExecGraphInfoAfterExecution) { if (origFromExecLayer.empty()) { constCnt++; } else { - std::for_each(origFromExecLayerSep.begin(), origFromExecLayerSep.end(), [&](const std::string &layer) { + std::for_each(origFromExecLayerSep.begin(), origFromExecLayerSep.end(), [&](const std::string& layer) { auto origLayer = originalLayersMap.find(layer); - ASSERT_NE(originalLayersMap.end(), origLayer) << layer; + EXPECT_NE(originalLayersMap.end(), origLayer) << layer; origLayer->second++; }); } } - ASSERT_TRUE(hasOpWithValidTime); + EXPECT_TRUE(hasOpWithValidTime); // All layers from the original IR must be present within ExecGraphInfo - for (auto &layer : originalLayersMap) { + for (auto& layer : originalLayersMap) { if ((layer.second == 0) && (constCnt > 0)) { constCnt--; } else { - ASSERT_GE(layer.second, 0); + EXPECT_GE(layer.second, 0); } } } @@ -295,10 +321,10 @@ TEST_P(OVExecutableNetworkBaseTest, canExport) { std::string modelName = GetTestName().substr(0, CommonTestUtils::maxFileNameLength) + "_" + ts; auto execNet = core->compile_model(function, targetDevice, configuration); std::ofstream out(modelName, std::ios::out); - ASSERT_NO_THROW(execNet.export_model(out)); + EXPECT_NO_THROW(execNet.export_model(out)); out.close(); - ASSERT_TRUE(CommonTestUtils::fileExists(modelName + ".xml")); - ASSERT_TRUE(CommonTestUtils::fileExists(modelName + ".bin")); + EXPECT_TRUE(CommonTestUtils::fileExists(modelName + ".xml")); + EXPECT_TRUE(CommonTestUtils::fileExists(modelName + ".bin")); CommonTestUtils::removeIRFiles(modelName + ".xml", modelName + ".bin"); } @@ -314,12 +340,25 @@ TEST_P(OVExecutableNetworkBaseTest, getInputFromFunctionWithSingleInput) { ov::runtime::ExecutableNetwork execNet; execNet = core->compile_model(function, targetDevice, configuration); - ASSERT_EQ(function->inputs().size(), 1); + EXPECT_EQ(function->inputs().size(), 1); EXPECT_EQ(function->inputs().size(), execNet.inputs().size()); EXPECT_NO_THROW(execNet.input()); EXPECT_EQ(function->input().get_tensor().get_names(), execNet.input().get_tensor().get_names()); EXPECT_EQ(function->input().get_tensor().get_partial_shape(), execNet.input().get_tensor().get_partial_shape()); EXPECT_EQ(function->input().get_tensor().get_element_type(), execNet.input().get_tensor().get_element_type()); + + ov::runtime::InferRequest request = execNet.create_infer_request(); + + ov::runtime::Tensor tensor1, tensor2; + EXPECT_NO_THROW(tensor1 = request.get_tensor(execNet.input())); + EXPECT_NO_THROW(tensor2 = request.get_tensor(function->input())); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_tensor(execNet.input().get_any_name())); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_tensor(function->input().get_any_name())); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_input_tensor(0)); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); } TEST_P(OVExecutableNetworkBaseTest, getOutputFromFunctionWithSingleInput) { @@ -334,6 +373,18 @@ TEST_P(OVExecutableNetworkBaseTest, getOutputFromFunctionWithSingleInput) { EXPECT_EQ(function->output().get_tensor().get_names(), execNet.output().get_tensor().get_names()); EXPECT_EQ(function->output().get_tensor().get_partial_shape(), execNet.output().get_tensor().get_partial_shape()); EXPECT_EQ(function->output().get_tensor().get_element_type(), execNet.output().get_tensor().get_element_type()); + + ov::runtime::InferRequest request = execNet.create_infer_request(); + ov::runtime::Tensor tensor1, tensor2; + EXPECT_NO_THROW(tensor1 = request.get_tensor(execNet.output())); + EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output())); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_tensor(execNet.output().get_any_name())); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output().get_any_name())); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_output_tensor(0)); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); } TEST_P(OVExecutableNetworkBaseTest, getInputsFromFunctionWithSeveralInputs) { @@ -361,10 +412,10 @@ TEST_P(OVExecutableNetworkBaseTest, getInputsFromFunctionWithSeveralInputs) { result2->set_friendly_name("result2"); function = std::make_shared(ngraph::ResultVector{result1, result2}, ngraph::ParameterVector{param1, param2}); - function->set_friendly_name("SingleRuLU"); + function->set_friendly_name("SimpleReLU"); } execNet = core->compile_model(function, targetDevice, configuration); - ASSERT_EQ(function->inputs().size(), 2); + EXPECT_EQ(function->inputs().size(), 2); EXPECT_EQ(function->inputs().size(), execNet.inputs().size()); EXPECT_THROW(execNet.input(), ov::Exception); EXPECT_EQ(function->input(0).get_tensor().get_names(), execNet.input(0).get_tensor().get_names()); @@ -377,6 +428,34 @@ TEST_P(OVExecutableNetworkBaseTest, getInputsFromFunctionWithSeveralInputs) { EXPECT_NE(function->input(1).get_node(), function->input("data1").get_node()); EXPECT_EQ(function->input(1).get_node(), function->input("data2").get_node()); EXPECT_NE(function->input(0).get_node(), function->input("data2").get_node()); + + ov::runtime::InferRequest request = execNet.create_infer_request(); + + ov::runtime::Tensor tensor1, tensor2; + EXPECT_NO_THROW(tensor1 = request.get_tensor(execNet.input(0))); + EXPECT_NO_THROW(tensor2 = request.get_tensor(function->input(0))); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_tensor(execNet.input(0).get_any_name())); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_tensor(function->input(0).get_any_name())); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_input_tensor(0)); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor1 = request.get_tensor(execNet.input(1))); + try { + // To avoid case with remote tensors + tensor1.data(); + EXPECT_FALSE(compareTensors(tensor1, tensor2)); + } catch (const ov::Exception&) { + } + EXPECT_NO_THROW(tensor2 = request.get_tensor(function->input(1))); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_tensor(execNet.input(1).get_any_name())); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_tensor(function->input(1).get_any_name())); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_input_tensor(1)); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); } TEST_P(OVExecutableNetworkBaseTest, getOutputsFromFunctionWithSeveralOutputs) { @@ -404,10 +483,10 @@ TEST_P(OVExecutableNetworkBaseTest, getOutputsFromFunctionWithSeveralOutputs) { result2->set_friendly_name("result2"); function = std::make_shared(ngraph::ResultVector{result1, result2}, ngraph::ParameterVector{param1, param2}); - function->set_friendly_name("SingleRuLU"); + function->set_friendly_name("SimpleReLU"); } execNet = core->compile_model(function, targetDevice, configuration); - ASSERT_EQ(function->outputs().size(), 2); + EXPECT_EQ(function->outputs().size(), 2); EXPECT_EQ(function->outputs().size(), execNet.outputs().size()); EXPECT_THROW(execNet.output(), ov::Exception); EXPECT_EQ(function->output(0).get_tensor().get_names(), execNet.output(0).get_tensor().get_names()); @@ -420,12 +499,107 @@ TEST_P(OVExecutableNetworkBaseTest, getOutputsFromFunctionWithSeveralOutputs) { EXPECT_NE(function->output(1).get_node(), function->output("relu").get_node()); EXPECT_EQ(function->output(1).get_node(), function->output("concat").get_node()); EXPECT_NE(function->output(0).get_node(), function->output("concat").get_node()); + + ov::runtime::InferRequest request = execNet.create_infer_request(); + + ov::runtime::Tensor tensor1, tensor2; + EXPECT_NO_THROW(tensor1 = request.get_tensor(execNet.output(0))); + EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output(0))); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_tensor(execNet.output(0).get_any_name())); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output(0).get_any_name())); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_output_tensor(0)); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor1 = request.get_tensor(execNet.output(1))); + try { + // To avoid case with remote tensors + tensor1.data(); + EXPECT_FALSE(compareTensors(tensor1, tensor2)); + } catch (const ov::Exception&) { + } + EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output(1))); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_tensor(execNet.output(1).get_any_name())); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output(1).get_any_name())); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_output_tensor(1)); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); +} + +TEST_P(OVExecutableNetworkBaseTest, getOutputsFromSplitFunctionWithSeveralOutputs) { + // Skip test according to plugin specific disabledTestPatterns() (if any) + SKIP_IF_CURRENT_TEST_IS_DISABLED() + ov::runtime::ExecutableNetwork execNet; + + // Create simple function + { + auto param1 = std::make_shared(ov::element::Type_t::f32, ngraph::Shape({1, 4, 24, 24})); + param1->set_friendly_name("param1"); + param1->output(0).get_tensor().set_names({"data1"}); + auto axis_node = ov::opset8::Constant::create(element::i64, Shape{}, {1}); + auto split = std::make_shared(param1, axis_node, 2); + split->set_friendly_name("split"); + split->output(0).get_tensor().set_names({"tensor_split_1"}); + split->output(1).get_tensor().set_names({"tensor_split_2"}); + auto result1 = std::make_shared(split->output(0)); + result1->set_friendly_name("result1"); + auto result2 = std::make_shared(split->output(1)); + result2->set_friendly_name("result2"); + function = std::make_shared(ngraph::ResultVector{result1, result2}, + ngraph::ParameterVector{param1}); + function->set_friendly_name("SingleSplit"); + } + execNet = core->compile_model(function, targetDevice, configuration); + EXPECT_EQ(function->outputs().size(), 2); + EXPECT_EQ(function->outputs().size(), execNet.outputs().size()); + EXPECT_THROW(execNet.output(), ov::Exception); + EXPECT_EQ(function->output(0).get_tensor().get_names(), execNet.output(0).get_tensor().get_names()); + EXPECT_EQ(function->output(0).get_tensor().get_partial_shape(), execNet.output(0).get_tensor().get_partial_shape()); + EXPECT_EQ(function->output(0).get_tensor().get_element_type(), execNet.output(0).get_tensor().get_element_type()); + EXPECT_EQ(function->output(1).get_tensor().get_names(), execNet.output(1).get_tensor().get_names()); + EXPECT_EQ(function->output(1).get_tensor().get_partial_shape(), execNet.output(1).get_tensor().get_partial_shape()); + EXPECT_EQ(function->output(1).get_tensor().get_element_type(), execNet.output(1).get_tensor().get_element_type()); + EXPECT_EQ(function->output(0).get_node(), function->output("tensor_split_1").get_node()); + EXPECT_NE(function->output(1).get_node(), function->output("tensor_split_1").get_node()); + EXPECT_EQ(function->output(1).get_node(), function->output("tensor_split_2").get_node()); + EXPECT_NE(function->output(0).get_node(), function->output("tensor_split_2").get_node()); + + ov::runtime::InferRequest request = execNet.create_infer_request(); + + ov::runtime::Tensor tensor1, tensor2; + EXPECT_NO_THROW(tensor1 = request.get_tensor(execNet.output(0))); + EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output(0))); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_tensor(execNet.output(0).get_any_name())); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output(0).get_any_name())); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_output_tensor(0)); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor1 = request.get_tensor(execNet.output(1))); + try { + // To avoid case with remote tensors + tensor1.data(); + EXPECT_FALSE(compareTensors(tensor1, tensor2)); + } catch (const ov::Exception&) { + } + EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output(1))); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_tensor(execNet.output(1).get_any_name())); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output(1).get_any_name())); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); + EXPECT_NO_THROW(tensor2 = request.get_output_tensor(1)); + EXPECT_TRUE(compareTensors(tensor1, tensor2)); } // Load correct network to Plugin to get executable network TEST_P(OVExecutableNetworkBaseTest, precisionsAsInOriginalFunction) { ov::runtime::ExecutableNetwork execNet; - ASSERT_NO_THROW(execNet = core->compile_model(function, targetDevice, configuration)); + EXPECT_NO_THROW(execNet = core->compile_model(function, targetDevice, configuration)); EXPECT_EQ(function->get_parameters().size(), execNet.inputs().size()); auto ref_parameter = function->get_parameters().back(); @@ -449,7 +623,7 @@ TEST_P(OVExecutableNetworkBaseTest, precisionsAsInOriginalIR) { ov::pass::Serialize(m_out_xml_path_1, m_out_bin_path_1).run_on_function(function); ov::runtime::ExecutableNetwork execNet; - ASSERT_NO_THROW(execNet = core->compile_model(m_out_xml_path_1, targetDevice, configuration)); + EXPECT_NO_THROW(execNet = core->compile_model(m_out_xml_path_1, targetDevice, configuration)); EXPECT_EQ(function->get_parameters().size(), execNet.inputs().size()); auto ref_parameter = function->get_parameters().back(); @@ -467,4 +641,4 @@ TEST_P(OVExecutableNetworkBaseTest, precisionsAsInOriginalIR) { } } // namespace behavior } // namespace test -} // namespace ov \ No newline at end of file +} // namespace ov diff --git a/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/subgraph_builders.hpp b/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/subgraph_builders.hpp index 5a14a96d47c..fbf84626652 100644 --- a/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/subgraph_builders.hpp +++ b/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/subgraph_builders.hpp @@ -51,7 +51,7 @@ inline std::shared_ptr makeSplitConvConcat(std::vector ngraph::element::Type_t ngPrc = ngraph::element::Type_t::f32) { auto params = ngraph::builder::makeParams(ngPrc, {inputShape}); params.front()->set_friendly_name("Param_1"); - params.front()->get_output_tensor(0).set_names({"Tensor_1"}); + params.front()->get_output_tensor(0).set_names({"input_tensor"}); auto split = ngraph::builder::makeSplit(params[0], ngPrc, 2, 1); auto conv1 = ngraph::builder::makeConvolution(split->output(0), ngPrc, {3, 3}, {1, 1}, {0, 0}, {0, 0}, {1, 1}, @@ -63,6 +63,7 @@ inline std::shared_ptr makeSplitConvConcat(std::vector auto relu2 = std::make_shared(conv2); auto concat = std::make_shared(ngraph::OutputVector{relu1->output(0), relu2->output(0)}, 1); + concat->get_output_tensor(0).set_names({"concat_tensor"}); ngraph::ResultVector results{std::make_shared(concat)}; std::shared_ptr fnPtr = std::make_shared(results, params); fnPtr->set_friendly_name("SplitConvConcat"); diff --git a/ngraph/core/src/node_output.cpp b/ngraph/core/src/node_output.cpp index 60b301b6493..af21964406b 100644 --- a/ngraph/core/src/node_output.cpp +++ b/ngraph/core/src/node_output.cpp @@ -84,7 +84,7 @@ const std::unordered_set& Output::get_names() const { } std::string Output::get_any_name() const { - return m_node->m_outputs.at(m_index).get_tensor_ptr()->get_any_name(); + return get_tensor().get_any_name(); } void Output::set_names(const std::unordered_set& names) { @@ -100,7 +100,7 @@ const std::unordered_set& Output::get_names() const { } std::string Output::get_any_name() const { - return m_node->m_outputs.at(m_index).get_tensor_ptr()->get_any_name(); + return get_tensor().get_any_name(); } bool Output::operator==(const Output& other) const { diff --git a/ngraph/test/util/all_close.cpp b/ngraph/test/util/all_close.cpp new file mode 100644 index 00000000000..1daa0793011 --- /dev/null +++ b/ngraph/test/util/all_close.cpp @@ -0,0 +1,91 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "all_close.hpp" + +::testing::AssertionResult ov::test::all_close(const ov::runtime::Tensor& a, + const ov::runtime::Tensor& b, + float rtol, + float atol) { + if (a.get_element_type() != b.get_element_type()) { + return ::testing::AssertionFailure() << "Cannot compare tensors with different element types"; + } + switch (a.get_element_type()) { + case ov::element::u8: + return all_close::value_type>( + a, + b, + static_cast::value_type>(rtol), + static_cast::value_type>(atol)); + case ov::element::u16: + return all_close::value_type>( + a, + b, + static_cast::value_type>(rtol), + static_cast::value_type>(atol)); + case ov::element::u32: + return all_close::value_type>( + a, + b, + static_cast::value_type>(rtol), + static_cast::value_type>(atol)); + case ov::element::u64: + return all_close::value_type>( + a, + b, + static_cast::value_type>(rtol), + static_cast::value_type>(atol)); + case ov::element::i8: + return all_close::value_type>( + a, + b, + static_cast::value_type>(rtol), + static_cast::value_type>(atol)); + case ov::element::i16: + return all_close::value_type>( + a, + b, + static_cast::value_type>(rtol), + static_cast::value_type>(atol)); + case ov::element::i32: + return all_close::value_type>( + a, + b, + static_cast::value_type>(rtol), + static_cast::value_type>(atol)); + case ov::element::i64: + return all_close::value_type>( + a, + b, + static_cast::value_type>(rtol), + static_cast::value_type>(atol)); + // case ov::element::bf16: + // return all_close::value_type>( + // a, + // b, + // static_cast::value_type>(rtol), + // static_cast::value_type>(atol)); + // case ov::element::f16: + // return all_close::value_type>( + // a, + // b, + // static_cast::value_type>(rtol), + // static_cast::value_type>(atol)); + case ov::element::f32: + return all_close::value_type>( + a, + b, + static_cast::value_type>(rtol), + static_cast::value_type>(atol)); + case ov::element::f64: + return all_close::value_type>( + a, + b, + static_cast::value_type>(rtol), + static_cast::value_type>(atol)); + default: + return ::testing::AssertionFailure() + << "Cannot compare tensors with unsupported element type: " << a.get_element_type(); + } +} diff --git a/ngraph/test/util/all_close.hpp b/ngraph/test/util/all_close.hpp index 5d8bc01ba97..fdb8fccdfe4 100644 --- a/ngraph/test/util/all_close.hpp +++ b/ngraph/test/util/all_close.hpp @@ -11,6 +11,7 @@ #include "gtest/gtest.h" #include "ngraph/pass/manager.hpp" #include "ngraph/type/element_type.hpp" +#include "openvino/runtime/tensor.hpp" namespace ngraph { namespace test { @@ -116,3 +117,35 @@ template } } // namespace test } // namespace ngraph + +namespace ov { +namespace test { +/// \brief Same as numpy.allclose +/// \param a First tensor to compare +/// \param b Second tensor to compare +/// \param rtol Relative tolerance +/// \param atol Absolute tolerance +/// Returns true if shapes match and for all elements, |a_i-b_i| <= atol + rtol*|b_i|. +template +::testing::AssertionResult all_close(const ov::runtime::Tensor& a, + const ov::runtime::Tensor& b, + T rtol = 1e-5f, + T atol = 1e-8f) { + auto a_t = std::make_shared(a.get_element_type(), a.get_shape(), a.data()); + auto b_t = std::make_shared(b.get_element_type(), b.get_shape(), b.data()); + + return ngraph::test::all_close(a_t, b_t, rtol, atol); +} + +/// \brief Same as numpy.allclose +/// \param a First tensor to compare +/// \param b Second tensor to compare +/// \param rtol Relative tolerance +/// \param atol Absolute tolerance +/// Returns true if shapes match and for all elements, |a_i-b_i| <= atol + rtol*|b_i|. +::testing::AssertionResult all_close(const ov::runtime::Tensor& a, + const ov::runtime::Tensor& b, + float rtol = 1e-5f, + float atol = 1e-8f); +} // namespace test +} // namespace ov