IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Infer request api (#7962)

Browse Source 
* Introduce new API

* Fixed comments

* Fixed code style

* Introduce some new API

* Fixed code style

* Reverted names

* Exend OVExecNetwork tests

* Changed executable network API

* Fixed tests

* Fixed comments

* Fixed style

* More tests

* Applied changes

* Fixed windows build

* Fixed typo

* Fixed some tests

* Fixed code style

* Fixed pre-proc tests

* Fixed documentation comments

* Added more comments

* Fixed code style

* Remove friend

* Removed find tensor API from internal classes

* Avoid old API in new constructors

* Fixed CPU tests

* Fixed comments

* Fixed template tests

* Extended test cases

* Revert data creation

* Fixed GNA and GPU plugins

* Fixed comments

* Fixed GNA tests

* Try to fix tests for GPU

* Fixed GNA tests

* Fixed GNA

* Try to fix Myriad tests

* Removed check to see an exception message

* Fixed HETERO import

* Reverted mkl-dnn submodule

* Fixed clang-format

Co-authored-by: y <ilya.lavrenov@intel.com>
This commit is contained in:
Ilya Churaev 2021-10-20 22:08:55 +03:00 committed by GitHub
parent a9b773e4b2
commit 0f3c10c810
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
51 changed files with 1160 additions and 284 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
docs/template_plugin/src/template_executable_network.cpp
View File

@ -151,11 +151,23 @@ InferenceEngine::IInferRequestInternal::Ptr TemplatePlugin::ExecutableNetwork::C
networkOutputs,
std::static_pointer_cast<ExecutableNetwork>(shared_from_this()));
}
InferenceEngine::IInferRequestInternal::Ptr TemplatePlugin::ExecutableNetwork::CreateInferRequestImpl(
const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) {
return std::make_shared<TemplateInferRequest>(inputs,
outputs,
std::static_pointer_cast<ExecutableNetwork>(shared_from_this()));
}
// ! [executable_network:create_infer_request_impl]
// ! [executable_network:create_infer_request]
InferenceEngine::IInferRequestInternal::Ptr TemplatePlugin::ExecutableNetwork::CreateInferRequest() {
auto internalRequest = CreateInferRequestImpl(_networkInputs, _networkOutputs);
InferenceEngine::IInferRequestInternal::Ptr internalRequest;
if (this->_plugin && this->_plugin->GetCore() && this->_plugin->GetCore()->isNewAPI())
internalRequest = CreateInferRequestImpl(_parameters, _results);
if (!internalRequest)
internalRequest = CreateInferRequestImpl(_networkInputs, _networkOutputs);
return std::make_shared<TemplateAsyncInferRequest>(std::static_pointer_cast<TemplateInferRequest>(internalRequest),
_taskExecutor,
_plugin->_waitExecutor,

3
docs/template_plugin/src/template_executable_network.hpp
View File

@ -37,6 +37,9 @@ public:
InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl(
InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs) override;
InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl(
const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) override;
InferenceEngine::IInferRequestInternal::Ptr CreateInferRequest() override;
InferenceEngine::Parameter GetMetric(const std::string& name) const override;
InferenceEngine::Parameter GetConfig(const std::string& name) const override;

12
docs/template_plugin/src/template_infer_request.cpp
View File

@ -35,6 +35,18 @@ TemplateInferRequest::TemplateInferRequest(const InferenceEngine::InputsDataMap&
const std::shared_ptr<TemplatePlugin::ExecutableNetwork>& executableNetwork)
: IInferRequestInternal(networkInputs, networkOutputs),
_executableNetwork(executableNetwork) {
createInferRequest();
}
TemplateInferRequest::TemplateInferRequest(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs,
const std::shared_ptr<TemplatePlugin::ExecutableNetwork>& executableNetwork)
: IInferRequestInternal(inputs, outputs),
_executableNetwork(executableNetwork) {
createInferRequest();
}
void TemplateInferRequest::createInferRequest() {
// TODO: allocate infer request device and host buffers if needed, fill actual list of profiling tasks
auto requestID = std::to_string(_executableNetwork->_requestId.fetch_add(1));

4
docs/template_plugin/src/template_infer_request.hpp
View File

@ -29,6 +29,9 @@ public:
TemplateInferRequest(const InferenceEngine::InputsDataMap& networkInputs,
const InferenceEngine::OutputsDataMap& networkOutputs,
const std::shared_ptr<ExecutableNetwork>& executableNetwork);
TemplateInferRequest(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs,
const std::shared_ptr<ExecutableNetwork>& executableNetwork);
~TemplateInferRequest();
void InferImpl() override;
@ -44,6 +47,7 @@ public:
void SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr& userBlob) override;
private:
void createInferRequest();
void allocateDeviceBuffers();
void allocateBlobs();

8
docs/template_plugin/tests/functional/op_reference/base_reference_test.cpp
View File

@ -57,17 +57,15 @@ void CommonReferenceTest::Infer() {
const auto& functionParams = function->get_parameters();
for (size_t i = 0; i < functionParams.size(); ++i) {
const auto& param = functionParams[i];
inferRequest.set_tensor(param->get_friendly_name(), inputData[i]);
inferRequest.set_tensor(executableNetwork.input(i), inputData[i]);
}
inferRequest.infer();
}
void CommonReferenceTest::Validate() {
ASSERT_EQ(executableNetwork.outputs().size(), refOutData.size());
for (const auto& result : function->get_results()) {
auto name = ngraph::op::util::create_ie_output_name(result->input_value(0));
actualOutData.emplace_back(inferRequest.get_tensor(name));
for (const auto& output : executableNetwork.outputs()) {
actualOutData.emplace_back(inferRequest.get_tensor(output));
}
ASSERT_EQ(refOutData.size(), actualOutData.size());

50
inference-engine/src/cldnn_engine/cldnn_executable_network.cpp
View File

@ -26,6 +26,7 @@
#include "cldnn_executable_network.h"
#include "threading/ie_cpu_streams_executor.hpp"
#include "cpp_interfaces/interface/ie_iinfer_request_internal.hpp"
#include "ie_icore.hpp"
using namespace InferenceEngine;
using namespace InferenceEngine::details;
@ -66,6 +67,36 @@ CLDNNExecNetwork::CLDNNExecNetwork(InferenceEngine::CNNNetwork &network, std::sh
IInferRequestInternal::Ptr CLDNNExecNetwork::CreateInferRequestImpl(InputsDataMap networkInputs,
OutputsDataMap networkOutputs) {
OV_ITT_SCOPED_TASK(itt::domains::CLDNNPlugin, "CLDNNExecNetwork::CreateInferRequestImpl");
auto ptr = std::make_shared<CLDNNInferRequest>(networkInputs, networkOutputs,
std::static_pointer_cast<CLDNNExecNetwork>(shared_from_this()));
if (m_config.throughput_streams > 1) {
ptr->EnableStreams();
}
if (m_config.useProfiling)
ptr->EnableProfiling();
ptr->SetGraph(m_graphs.front());
return ptr;
}
IInferRequestInternal::Ptr CLDNNExecNetwork::CreateInferRequestImpl(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) {
OV_ITT_SCOPED_TASK(itt::domains::CLDNNPlugin, "CLDNNExecNetwork::CreateInferRequestImpl");
auto ptr = std::make_shared<CLDNNInferRequest>(inputs, outputs,
std::static_pointer_cast<CLDNNExecNetwork>(shared_from_this()));
if (m_config.throughput_streams > 1) {
ptr->EnableStreams();
}
if (m_config.useProfiling)
ptr->EnableProfiling();
ptr->SetGraph(m_graphs.front());
return ptr;
}
IInferRequestInternal::Ptr CLDNNExecNetwork::CreateInferRequest() {
OV_ITT_SCOPED_TASK(itt::domains::CLDNNPlugin, "CLDNNExecNetwork::CreateInferRequest");
InferenceEngine::IInferRequestInternal::Ptr internalRequest;
if (m_graphs.empty()) {
IE_THROW(NetworkNotLoaded);
}
@ -80,21 +111,10 @@ IInferRequestInternal::Ptr CLDNNExecNetwork::CreateInferRequestImpl(InputsDataMa
}
}
auto ptr = std::make_shared<CLDNNInferRequest>(networkInputs, networkOutputs,
std::static_pointer_cast<CLDNNExecNetwork>(shared_from_this()));
if (m_config.throughput_streams > 1) {
ptr->EnableStreams();
}
if (m_config.useProfiling)
ptr->EnableProfiling();
ptr->SetGraph(m_graphs.front());
return ptr;
}
IInferRequestInternal::Ptr CLDNNExecNetwork::CreateInferRequest() {
OV_ITT_SCOPED_TASK(itt::domains::CLDNNPlugin, "CLDNNExecNetwork::CreateInferRequest");
auto internalRequest = CreateInferRequestImpl(_networkInputs, _networkOutputs);
if (this->_plugin && this->_plugin->GetCore() && this->_plugin->GetCore()->isNewAPI())
internalRequest = CreateInferRequestImpl(_parameters, _results);
if (!internalRequest)
internalRequest = CreateInferRequestImpl(_networkInputs, _networkOutputs);
internalRequest->setPointerToExecutableNetworkInternal(shared_from_this());
return std::make_shared<CLDNNAsyncInferRequest>(std::static_pointer_cast<CLDNNInferRequest>(internalRequest),
m_taskExecutor,

2
inference-engine/src/cldnn_engine/cldnn_executable_network.h
View File

@ -29,6 +29,8 @@ public:
InferenceEngine::IInferRequestInternal::Ptr CreateInferRequest() override;
InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs) override;
InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) override;
InferenceEngine::Parameter GetMetric(const std::string &name) const override;
InferenceEngine::Parameter GetConfig(const std::string &name) const override;

10
inference-engine/src/cldnn_engine/cldnn_infer_request.cpp
View File

@ -457,6 +457,16 @@ CLDNNInferRequest::CLDNNInferRequest(InputsDataMap networkInputs, OutputsDataMap
streamExecutor = dynamic_cast<InferenceEngine::IStreamsExecutor*>(execNetwork->m_taskExecutor.get());
}
CLDNNInferRequest::CLDNNInferRequest(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs,
const CLDNNExecNetwork::Ptr& execNetwork)
: IInferRequestInternal(inputs, outputs)
, m_useProfiling(false)
, m_useStreams(false) {
IE_ASSERT(nullptr != execNetwork);
streamExecutor = dynamic_cast<InferenceEngine::IStreamsExecutor*>(execNetwork->m_taskExecutor.get());
}
// ----------------------------------------------------------------------------------------- //
// ---------------------------- internal pipeline stages ----------------------------------- //
// ----------------------------------------------------------------------------------------- //

3
inference-engine/src/cldnn_engine/cldnn_infer_request.h
View File

@ -33,6 +33,9 @@ public:
CLDNNInferRequest(InferenceEngine::InputsDataMap networkInputs, InferenceEngine::OutputsDataMap networkOutputs,
const std::shared_ptr<CLDNNExecNetwork>& execNetwork);
CLDNNInferRequest(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs,
const std::shared_ptr<CLDNNExecNetwork>& execNetwork);
CLDNNInferRequest(const CLDNNInferRequest &) = delete;

9
inference-engine/src/gna_plugin/gna_executable_network.hpp
View File

@ -13,6 +13,7 @@
#include <gna/gna_config.hpp>
#include <threading/ie_executor_manager.hpp>
#include <cpp_interfaces/interface/ie_iexecutable_network_internal.hpp>
#include <ie_icore.hpp>
namespace GNAPluginNS {
@ -58,6 +59,14 @@ class GNAExecutableNetwork : public InferenceEngine::IExecutableNetworkInternal
return std::make_shared<GNAInferRequest>(plg, networkInputs, networkOutputs);
}
InferenceEngine::IInferRequestInternal::Ptr
CreateInferRequestImpl(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) override {
if (!this->_plugin || !this->_plugin->GetCore() || !this->_plugin->GetCore()->isNewAPI())
return nullptr;
return std::make_shared<GNAInferRequest>(plg, inputs, outputs);
}
INFERENCE_ENGINE_DEPRECATED("Use InferRequest::QueryState instead")
std::vector<InferenceEngine::IVariableStateInternal::Ptr> QueryState() override {
IE_SUPPRESS_DEPRECATED_START

31
inference-engine/src/gna_plugin/gna_infer_request.hpp
View File

@ -15,17 +15,10 @@
namespace GNAPluginNS {
class GNAInferRequest : public InferenceEngine::IInferRequestInternal {
protected:
std::shared_ptr<GNAPlugin> plg;
uint32_t inferRequestIdx = -1;
public:
GNAInferRequest(const std::shared_ptr<GNAPlugin>& plg,
InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs)
: InferenceEngine::IInferRequestInternal(networkInputs, networkOutputs), plg(plg) {
private:
void CreateInferRequest() {
// TODO: internal connection API - better to generalize
if (networkOutputs.empty()) {
if (_networkOutputs.empty()) {
THROW_GNA_EXCEPTION << "GNAInferRequest :: network has zero outputs";
}
@ -40,6 +33,24 @@ class GNAInferRequest : public InferenceEngine::IInferRequestInternal {
plg->GetInputBlob(input.first, input.second->getTensorDesc().getPrecision());
}
}
protected:
std::shared_ptr<GNAPlugin> plg;
uint32_t inferRequestIdx = -1;
public:
GNAInferRequest(const std::shared_ptr<GNAPlugin>& plg,
const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs)
: InferenceEngine::IInferRequestInternal(inputs, outputs), plg(plg) {
CreateInferRequest();
}
GNAInferRequest(const std::shared_ptr<GNAPlugin>& plg,
InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs)
: InferenceEngine::IInferRequestInternal(networkInputs, networkOutputs), plg(plg) {
CreateInferRequest();
}
/**
* @brief Infers specified input(s) in synchronous mode
* @note blocks all method of InferRequest while request is ongoing (running or waiting in queue)

19
inference-engine/src/hetero_plugin/hetero_executable_network.cpp
View File

@ -732,6 +732,25 @@ void HeteroExecutableNetwork::Export(std::ostream& heteroModel) {
}
}
IInferRequestInternal::Ptr HeteroExecutableNetwork::CreateInferRequestImpl(
const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) {
if (!this->_plugin || !this->_plugin->GetCore() || !this->_plugin->GetCore()->isNewAPI())
return nullptr;
HeteroInferRequest::SubRequestsList inferRequests;
int index = 0;
for (auto&& subnetwork : _networks) {
HeteroInferRequest::SubRequestDesc desc;
desc._network = subnetwork._network;
desc._profilingTask = openvino::itt::handle("Infer" + std::to_string(index++));
inferRequests.push_back(desc);
}
return std::make_shared<HeteroInferRequest>(inputs,
outputs,
inferRequests,
_blobNameMap);
}
IInferRequestInternal::Ptr HeteroExecutableNetwork::CreateInferRequestImpl(
InputsDataMap networkInputs,
OutputsDataMap networkOutputs) {

3
inference-engine/src/hetero_plugin/hetero_executable_network.hpp
View File

@ -49,6 +49,9 @@ public:
InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs) override;
InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) override;
InferenceEngine::IInferRequestInternal::Ptr CreateInferRequest() override;

14
inference-engine/src/hetero_plugin/hetero_infer_request.cpp
View File

@ -16,12 +16,26 @@ using namespace HeteroPlugin;
using namespace InferenceEngine;
using namespace InferenceEngine::details;
HeteroInferRequest::HeteroInferRequest(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs,
const SubRequestsList& inferRequests,
const std::unordered_map<std::string, std::string>& subgraphInputToOutputBlobNames) :
IInferRequestInternal(inputs, outputs),
_inferRequests(inferRequests) {
CreateInferRequest(subgraphInputToOutputBlobNames);
}
HeteroInferRequest::HeteroInferRequest(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs,
const SubRequestsList& inferRequests,
const std::unordered_map<std::string, std::string>& subgraphInputToOutputBlobNames) :
IInferRequestInternal(networkInputs, networkOutputs),
_inferRequests(inferRequests) {
CreateInferRequest(subgraphInputToOutputBlobNames);
}
void HeteroInferRequest::CreateInferRequest(const std::unordered_map<std::string, std::string>& subgraphInputToOutputBlobNames) {
if (_networkOutputs.empty() || _networkInputs.empty()) {
IE_THROW() << "Internal error: no information about network's output/input";
}

10
inference-engine/src/hetero_plugin/hetero_infer_request.hpp
View File

@ -27,11 +27,16 @@ public:
};
using SubRequestsList = std::vector<SubRequestDesc>;
explicit HeteroInferRequest(InferenceEngine::InputsDataMap networkInputs,
HeteroInferRequest(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs,
const SubRequestsList &inferRequests,
const std::unordered_map<std::string, std::string>& blobNameMap);
HeteroInferRequest(const std::vector<std::shared_ptr<const ov::Node>>& networkInputs,
const std::vector<std::shared_ptr<const ov::Node>>& networkOutputs,
const SubRequestsList &inferRequests,
const std::unordered_map<std::string, std::string>& blobNameMap);
void InferImpl() override;
void SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr& blob) override;
@ -49,6 +54,9 @@ public:
SubRequestsList _inferRequests;
std::map<std::string, InferenceEngine::Blob::Ptr> _blobs;
std::map<std::string, InferenceEngine::IInferRequestInternal*> _subRequestFromBlobName;
private:
void CreateInferRequest(const std::unordered_map<std::string, std::string>& subgraphInputToOutputBlobNames);
};
} // namespace HeteroPlugin

93
inference-engine/src/inference_engine/include/openvino/runtime/infer_request.hpp
View File

@ -13,6 +13,7 @@
#include <memory>
#include <string>
#include "openvino/core/node_output.hpp"
#include "openvino/runtime/common.hpp"
#include "openvino/runtime/profiling_info.hpp"
#include "openvino/runtime/tensor.hpp"
@ -54,21 +55,103 @@ public:
/**
* @brief Sets input/output data to infer
*
* @note Memory allocation does not happen
* @param name Name of input or output tensor.
* @param tensor Reference to input or output tensor. The type of a tensor must match the network input precision
* and size.
* @param tensor Reference to input or output tensor. The type of a tensor must match the network input/output
* precision and size.
*/
void set_tensor(const std::string& name, const Tensor& tensor);
/**
* @brief Sets input/output data to infer
*
* @param port Port of input or output tensor.
* @param tensor Reference to input or output tensor. The type of a tensor must match the network input/output
* precision and size.
*/
void set_tensor(const ov::Output<const ov::Node>& port, const Tensor& tensor);
/**
* @brief Sets input/output data to infer
*
* @param port Port of input or output tensor.
* @param tensor Reference to input or output tensor. The type of a tensor must match the network input/output
* precision and size.
*/
void set_tensor(const ov::Output<ov::Node>& port, const Tensor& tensor);
/**
* @brief Sets input tensor to infer
*
* @param idx Index of input tensor.
* @param tensor Reference to input tensor. The type of a tensor must match the network input precision and size.
*/
void set_input_tensor(size_t idx, const Tensor& tensor);
/**
* @brief Sets input tensor to infer models with single input
*
* @param tensor Reference to input tensor. If model has several inputs, an exception is thrown.
*/
void set_input_tensor(const Tensor& tensor);
/**
* @brief Sets output tensor to infer
*
* @param idx Index of output tensor.
* @param tensor Reference to output tensor. The type of a tensor must match the network output precision and size.
*/
void set_output_tensor(size_t idx, const Tensor& tensor);
/**
* @brief Sets output tensor to infer models with single output
*
* @param tensor Reference to output tensor. If model has several outputs, an exception is thrown.
*/
void set_output_tensor(const Tensor& tensor);
/**
* @brief Gets input/output data for inference
* @brief Gets input/output tensor for inference
*
* @note Memory allocation does not happen
* @param name A name of tensor to get
* @return A Tensor with a name @p name. If a tensor is not found, an exception is thrown.
*/
Tensor get_tensor(const std::string& name);
/**
* @brief Gets input/output tensor for inference
*
* @param port Port of tensor to get
* @return A Tensor for the port @p port. If a tensor with specified @p port is not found, an exception is thrown.
*/
Tensor get_tensor(const ov::Output<const ov::Node>& port);
/**
* @brief Gets input/output tensor for inference
*
* @param port Port of tensor to get
* @return A Tensor for the port @p port. If a tensor with specified @p port is not found, an exception is thrown.
*/
Tensor get_tensor(const ov::Output<ov::Node>& port);
/**
* @brief Gets input tensor for inference
*
* @param idx An index of tensor to get
* @return A Tensor with an input index @p idx. If a tensor with specified @p idx is not found, an exception is
* thrown.
*/
Tensor get_input_tensor(size_t idx);
/**
* @brief Gets input tensor for inference
*
* @return An input Tensor for the model. If model has several inputs, an exception is thrown.
*/
Tensor get_input_tensor();
/**
* @brief Gets output tensor for inference
*
* @param idx An index of tensor to get
* @return A Tensor with an output index @p idx. If a tensor with specified @p idx is not found, an exception is
* thrown.
*/
Tensor get_output_tensor(size_t idx);
/**
* @brief Gets output tensor for inference
*
* @return An output Tensor for the model. If model has several outputs, an exception is thrown.
*/
Tensor get_output_tensor();
/**
* @brief Infers specified input(s) in synchronous mode

12
inference-engine/src/inference_engine/src/cpp/ie_executable_network.cpp
View File

@ -145,11 +145,11 @@ std::vector<ov::Output<const ov::Node>> ExecutableNetwork::inputs() const {
ov::Output<const ov::Node> ExecutableNetwork::input() const {
OV_EXEC_NET_CALL_STATEMENT({
const auto params = _impl->getInputs();
if (params.size() != 1) {
const auto inputs = _impl->getInputs();
if (inputs.size() != 1) {
throw ov::Exception("input() must be called on a function with exactly one parameter.");
}
return params.at(0);
return inputs.at(0);
});
}
@ -179,11 +179,11 @@ std::vector<ov::Output<const ov::Node>> ExecutableNetwork::outputs() const {
}
ov::Output<const ov::Node> ExecutableNetwork::output() const {
OV_EXEC_NET_CALL_STATEMENT({
const auto result = _impl->getOutputs();
if (result.size() != 1) {
const auto outputs = _impl->getOutputs();
if (outputs.size() != 1) {
throw ov::Exception("output() must be called on a function with exactly one parameter.");
}
return result.at(0);
return outputs.at(0);
});
}
ov::Output<const ov::Node> ExecutableNetwork::output(size_t i) const {

144
inference-engine/src/inference_engine/src/cpp/ie_infer_request.cpp
View File

@ -8,6 +8,7 @@
#include <memory>
#include <string>
#include "cpp_interfaces/interface/ie_iexecutable_network_internal.hpp"
#include "cpp_interfaces/interface/ie_iinfer_request_internal.hpp"
#include "ie_infer_async_request_base.hpp"
#include "ie_ngraph_utils.hpp"
@ -16,6 +17,24 @@
#include "openvino/runtime/infer_request.hpp"
#include "transformations/utils/utils.hpp"
namespace {
inline bool getPort(ov::Output<const ov::Node>& port,
const std::string& name,
const std::vector<std::vector<std::shared_ptr<const ov::Node>>>& ports) {
for (const auto& nodes : ports) {
for (const auto& node : nodes) {
const auto& names = node->get_output_tensor(0).get_names();
if (names.find(name) != names.end()) {
port = node->output(0);
return true;
}
}
}
return false;
}
} // namespace
namespace InferenceEngine {
#define INFER_REQ_CALL_STATEMENT(...) \
@ -204,6 +223,18 @@ bool InferRequest::operator==(const InferRequest& r) const noexcept {
} // namespace InferenceEngine
namespace {
std::string get_legacy_name_from_port(const ov::Output<const ov::Node>& port) {
ov::Output<ngraph::Node> p(std::const_pointer_cast<ov::Node>(port.get_node_shared_ptr()), port.get_index());
if (auto node = std::dynamic_pointer_cast<ov::op::v0::Result>(p.get_node_shared_ptr())) {
p = node->input_value(0);
}
return ngraph::op::util::create_ie_output_name(p);
}
} // namespace
namespace ov {
namespace runtime {
@ -213,23 +244,114 @@ InferRequest::InferRequest(const std::shared_ptr<void>& so, const ie::IInferRequ
OPENVINO_ASSERT(_impl != nullptr, "InferRequest was not initialized.");
}
void InferRequest::set_tensor(const ov::Output<const ov::Node>& port, const Tensor& tensor) {
OV_INFER_REQ_CALL_STATEMENT({ _impl->SetBlob(get_legacy_name_from_port(port), tensor._impl); });
}
void InferRequest::set_tensor(const ov::Output<ov::Node>& port, const Tensor& tensor) {
set_tensor(ov::Output<const ov::Node>(port.get_node(), port.get_index()), tensor);
}
void InferRequest::set_tensor(const std::string& name, const Tensor& tensor) {
OV_INFER_REQ_CALL_STATEMENT({ _impl->SetBlob(name, tensor._impl); })}
OV_INFER_REQ_CALL_STATEMENT({
ov::Output<const ov::Node> port;
OPENVINO_ASSERT(::getPort(port, name, {_impl->GetInputs(), _impl->GetOutputs()}),
"Port for tensor name " + name + " was not found.");
set_tensor(port, tensor);
});
}
void InferRequest::set_input_tensor(size_t idx, const Tensor& tensor) {
OV_INFER_REQ_CALL_STATEMENT({
const auto& inputs = _impl->GetInputs();
OPENVINO_ASSERT(inputs.size() > idx,
"Input port for index ",
idx,
" was not found! The model has only ",
inputs.size(),
" inputs.");
set_tensor(inputs.at(idx)->output(0), tensor);
});
}
void InferRequest::set_input_tensor(const Tensor& tensor) {
OV_INFER_REQ_CALL_STATEMENT({
const auto inputs = _impl->GetInputs();
OPENVINO_ASSERT(inputs.size() == 1,
"set_input_tensor() must be called on a function with exactly one parameter.");
set_tensor(inputs.at(0)->output(0), tensor);
});
}
void InferRequest::set_output_tensor(size_t idx, const Tensor& tensor) {
OV_INFER_REQ_CALL_STATEMENT({
const auto& outputs = _impl->GetOutputs();
OPENVINO_ASSERT(outputs.size() > idx,
"Output port for index ",
idx,
" was not found! The model has only ",
outputs.size(),
" outputs.");
set_tensor(outputs.at(idx)->output(0), tensor);
});
}
void InferRequest::set_output_tensor(const Tensor& tensor) {
OV_INFER_REQ_CALL_STATEMENT({
const auto outputs = _impl->GetOutputs();
OPENVINO_ASSERT(outputs.size() == 1,
"set_output_tensor() must be called on a function with exactly one parameter.");
set_tensor(outputs.at(0)->output(0), tensor);
});
}
Tensor InferRequest::get_tensor(const ov::Output<const ov::Node>& port) {
OV_INFER_REQ_CALL_STATEMENT({
const auto& name = get_legacy_name_from_port(port);
auto blob = _impl->GetBlob(name);
return {_so, blob};
});
}
Tensor InferRequest::get_tensor(const ov::Output<ov::Node>& port) {
return get_tensor(ov::Output<const ov::Node>(port.get_node(), port.get_index()));
}
Tensor InferRequest::get_tensor(const std::string& name) {
OV_INFER_REQ_CALL_STATEMENT({
auto blob = _impl->GetBlob(name);
const bool remoteBlobPassed = blob->is<ie::RemoteBlob>();
if (blob == nullptr) {
IE_THROW(NotAllocated) << "Internal tensor implementation with name `" << name << "` is not allocated!";
ov::Output<const ov::Node> port;
OPENVINO_ASSERT(::getPort(port, name, {_impl->GetInputs(), _impl->GetOutputs()}),
"Port for tensor name " + name + " was not found.");
return get_tensor(port);
});
}
if (!remoteBlobPassed && blob->buffer() == nullptr) {
IE_THROW(NotAllocated) << "Internal tensor implementation with name `" << name << "` is not allocated!";
Tensor InferRequest::get_input_tensor(size_t idx) {
OV_INFER_REQ_CALL_STATEMENT({ return get_tensor(_impl->GetInputs().at(idx)->output(0)); });
}
auto tensorDesc = blob->getTensorDesc();
auto dims = tensorDesc.getDims();
return {_so, blob};
})
Tensor InferRequest::get_output_tensor(size_t idx) {
OV_INFER_REQ_CALL_STATEMENT({ return get_tensor(_impl->GetOutputs().at(idx)->output(0)); });
}
Tensor InferRequest::get_input_tensor() {
OV_INFER_REQ_CALL_STATEMENT({
const auto inputs = _impl->GetInputs();
if (inputs.size() != 1) {
throw ov::Exception("get_input_tensor() must be called on a function with exactly one parameter.");
}
return get_tensor(inputs.at(0)->output(0));
});
}
Tensor InferRequest::get_output_tensor() {
OV_INFER_REQ_CALL_STATEMENT({
const auto outputs = _impl->GetOutputs();
if (outputs.size() != 1) {
throw ov::Exception("get_output_tensor() must be called on a function with exactly one parameter.");
}
return get_tensor(outputs.at(0)->output(0));
});
}
void InferRequest::infer() {

16
inference-engine/src/inference_engine/src/cpp_interfaces/interface/ie_iexecutable_network_internal.cpp
View File

@ -14,7 +14,9 @@
#include "cpp_interfaces/interface/ie_iinfer_request_internal.hpp"
#include "cpp_interfaces/interface/ie_iplugin_internal.hpp"
#include "ie_icore.hpp"
#include "ie_ngraph_utils.hpp"
#include "ie_parameter.hpp"
#include "openvino/core/node.hpp"
namespace InferenceEngine {
@ -56,7 +58,13 @@ ConstInputsDataMap IExecutableNetworkInternal::GetInputsInfo() const {
}
std::shared_ptr<IInferRequestInternal> IExecutableNetworkInternal::CreateInferRequest() {
auto asyncRequestImpl = CreateInferRequestImpl(_networkInputs, _networkOutputs);
std::shared_ptr<IInferRequestInternal> asyncRequestImpl;
try {
asyncRequestImpl = CreateInferRequestImpl(_parameters, _results);
} catch (const NotImplemented&) {
}
if (!asyncRequestImpl)
asyncRequestImpl = CreateInferRequestImpl(_networkInputs, _networkOutputs);
asyncRequestImpl->setPointerToExecutableNetworkInternal(shared_from_this());
return asyncRequestImpl;
}
@ -109,4 +117,10 @@ std::shared_ptr<IInferRequestInternal> IExecutableNetworkInternal::CreateInferRe
IE_THROW(NotImplemented);
}
std::shared_ptr<IInferRequestInternal> IExecutableNetworkInternal::CreateInferRequestImpl(
const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) {
IE_THROW(NotImplemented);
}
} // namespace InferenceEngine

47
inference-engine/src/inference_engine/src/cpp_interfaces/interface/ie_iinfer_request_internal.cpp
View File

@ -16,8 +16,10 @@
#include "ie_blob.h"
#include "ie_common.h"
#include "ie_compound_blob.h"
#include "ie_ngraph_utils.hpp"
#include "ie_preprocess.hpp"
#include "ie_remote_context.hpp"
#include "transformations/utils/utils.hpp"
namespace InferenceEngine {
@ -28,6 +30,43 @@ IInferRequestInternal::IInferRequestInternal(const InputsDataMap& networkInputs,
_networkInputs{copyInfo(networkInputs)},
_networkOutputs{copyInfo(networkOutputs)} {}
IInferRequestInternal::IInferRequestInternal(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs)
: _parameters(inputs),
_results(outputs) {
const auto& create_old_data = [](const ov::Output<const ov::Node>& output) -> InferenceEngine::DataPtr {
IE_SUPPRESS_DEPRECATED_START
auto name = ngraph::op::util::get_ie_output_name(output);
auto shape = output.get_partial_shape();
auto rank = shape.rank().is_static() ? shape.rank().get_length() : -1;
for (const auto& dim : shape) {
if (dim.is_static() && dim.get_length() == 0)
IE_THROW() << name << " has zero dimension which is not allowed";
}
const Layout rankLayout = rank < 0 ? Layout::BLOCKED : TensorDesc::getLayoutByRank(rank);
const auto precision = InferenceEngine::details::convertPrecision(output.get_element_type());
return std::make_shared<Data>(name, precision, shape, rankLayout);
IE_SUPPRESS_DEPRECATED_END
};
const auto& create_old_input_data =
[create_old_data](const ov::Output<const ov::Node>& output) -> InferenceEngine::InputInfo::Ptr {
auto info = std::make_shared<InferenceEngine::InputInfo>();
info->setInputData(create_old_data(output));
return info;
};
for (const auto& param : _parameters) {
const auto& input = create_old_input_data(param->output(0));
_networkInputs[input->name()] = input;
}
for (const auto& result : _results) {
auto input = result->input_value(0);
const auto& output = create_old_data(ov::Output<const ov::Node>(input.get_node(), input.get_index()));
_networkOutputs[output->getName()] = output;
}
}
void IInferRequestInternal::Infer() {
checkBlobs();
InferImpl();
@ -373,4 +412,12 @@ void* IInferRequestInternal::GetUserData() noexcept {
void IInferRequestInternal::SetUserData(void* userData) noexcept {
_userData = userData;
}
const std::vector<std::shared_ptr<const ov::Node>>& IInferRequestInternal::GetInputs() const {
return _parameters;
}
const std::vector<std::shared_ptr<const ov::Node>>& IInferRequestInternal::GetOutputs() const {
return _results;
}
} // namespace InferenceEngine

92
inference-engine/src/inference_engine/src/ie_core.cpp
View File

@ -542,6 +542,54 @@ public:
const std::map<std::string, std::string>& config) override {
auto parsed = parseDeviceNameIntoConfig(deviceName, config);
auto exec = GetCPPPluginByName(parsed._deviceName).import_model(networkModel, parsed._config);
if (isNewAPI()) {
// create getInputs() based on GetInputsInfo()
using namespace InferenceEngine::details;
if (exec->getInputs().empty()) {
const auto& inputsInfo = exec->GetInputsInfo();
OPENVINO_ASSERT(!inputsInfo.empty(), "inputsInfo is empty after network import");
std::vector<std::shared_ptr<const ov::Node>> params;
params.reserve(inputsInfo.size());
for (auto&& input : inputsInfo) {
auto param = std::make_shared<ov::op::v0::Parameter>(
convertPrecision(input.second->getPrecision()),
ov::PartialShape(input.second->getTensorDesc().getDims()));
param->set_friendly_name(input.first);
param->get_output_tensor(0).add_names({input.first});
params.emplace_back(std::move(param));
}
exec->setInputs(params);
}
if (exec->getOutputs().empty()) {
const auto& outputsInfo = exec->GetOutputsInfo();
OPENVINO_ASSERT(!outputsInfo.empty(), "outputsInfo is empty after network import");
std::vector<std::shared_ptr<const ov::Node>> results;
results.reserve(outputsInfo.size());
for (auto&& output : outputsInfo) {
auto fake_param = std::make_shared<ov::op::v0::Parameter>(
convertPrecision(output.second->getPrecision()),
ov::PartialShape(output.second->getTensorDesc().getDims()));
fake_param->set_friendly_name(output.first);
auto result = std::make_shared<ov::op::v0::Result>(fake_param);
result->get_output_tensor(0).add_names({output.first});
results.emplace_back(std::move(result));
}
exec->setOutputs(results);
}
// but for true support plugins need:
// 1. ensure order or parameters and results as in ov::Function
// 2. provide tensor names for inputs and outputs
// 3. precisions for getInputs and getOutputs should be taken from GetInputsInfo / GetOutputsInfo
// not from ngraph. Plugins should use SetExeNetworkInfo
}
return {{exec._so}, exec._ptr};
}
@ -1381,50 +1429,6 @@ ExecutableNetwork Core::import_model(std::istream& modelStream,
OV_ITT_SCOPED_TASK(ov::itt::domains::IE, "Core::import_model");
OV_CORE_CALL_STATEMENT({
auto exec = _impl->ImportNetwork(modelStream, deviceName, config);
// create getInputs() based on GetInputsInfo()
using namespace InferenceEngine::details;
if (exec->getInputs().empty()) {
const auto& inputsInfo = exec->GetInputsInfo();
OPENVINO_ASSERT(!inputsInfo.empty(), "inputsInfo is empty after network import");
std::vector<std::shared_ptr<const ov::Node>> params;
params.reserve(inputsInfo.size());
for (auto&& input : inputsInfo) {
auto param =
std::make_shared<ov::op::v0::Parameter>(convertPrecision(input.second->getPrecision()),
ov::PartialShape(input.second->getTensorDesc().getDims()));
param->get_output_tensor(0).add_names({input.first});
params.emplace_back(std::move(param));
}
exec->setInputs(params);
}
if (exec->getOutputs().empty()) {
const auto& outputsInfo = exec->GetOutputsInfo();
OPENVINO_ASSERT(!outputsInfo.empty(), "outputsInfo is empty after network import");
std::vector<std::shared_ptr<const ov::Node>> results;
results.reserve(outputsInfo.size());
for (auto&& output : outputsInfo) {
auto fake_param =
std::make_shared<ov::op::v0::Parameter>(convertPrecision(output.second->getPrecision()),
ov::PartialShape(output.second->getTensorDesc().getDims()));
auto result = std::make_shared<ov::op::v0::Result>(fake_param);
result->get_output_tensor(0).add_names({output.first});
results.emplace_back(std::move(result));
}
exec->setOutputs(results);
}
// but for true support plugins need:
// 1. ensure order or parameters and results as in ov::Function
// 2. provide tensor names for inputs and outputs
// 3. precisions for getInputs and getOutputs should be taken from GetInputsInfo / GetOutputsInfo
// not from ngraph. Plugins should use SetExeNetworkInfo
return {exec._so, exec._ptr};
});
}

2
inference-engine/src/legacy_api/src/ngraph_ops/nms_ie.cpp
View File

@ -162,7 +162,7 @@ int64_t op::NonMaxSuppressionIE3::max_boxes_output_from_input() const {
}
const auto max_output_boxes_input =
ov::as_type_ptr<op::Constant>(input_value(max_output_boxes_per_class_port).get_node_shared_ptr());
ov::as_type_ptr<const op::Constant>(input_value(max_output_boxes_per_class_port).get_node_shared_ptr());
max_output_boxes = max_output_boxes_input->cast_vector<int64_t>().at(0);
return max_output_boxes;

10
inference-engine/src/mkldnn_plugin/mkldnn_exec_network.cpp
View File

@ -25,11 +25,21 @@
#include <cstring>
#include <ngraph/opsets/opset1.hpp>
#include <transformations/utils/utils.hpp>
#include "cpp_interfaces/interface/ie_iplugin_internal.hpp"
#include "ie_icore.hpp"
using namespace MKLDNNPlugin;
using namespace InferenceEngine;
using namespace InferenceEngine::details;
InferenceEngine::IInferRequestInternal::Ptr
MKLDNNExecNetwork::CreateInferRequestImpl(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) {
if (!this->_plugin || !this->_plugin->GetCore() || !this->_plugin->GetCore()->isNewAPI())
return nullptr;
return std::make_shared<MKLDNNInferRequest>(inputs, outputs, std::static_pointer_cast<MKLDNNExecNetwork>(shared_from_this()));
}
InferenceEngine::IInferRequestInternal::Ptr
MKLDNNExecNetwork::CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs) {

4
inference-engine/src/mkldnn_plugin/mkldnn_exec_network.h
View File

@ -23,6 +23,10 @@ class MKLDNNExecNetwork: public InferenceEngine::ExecutableNetworkThreadSafeDefa
public:
typedef std::shared_ptr<MKLDNNExecNetwork> Ptr;
std::shared_ptr<InferenceEngine::IInferRequestInternal>
CreateInferRequestImpl(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) override;
std::shared_ptr<InferenceEngine::IInferRequestInternal>
CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs) override;

12
inference-engine/src/mkldnn_plugin/mkldnn_infer_request.cpp
View File

@ -29,6 +29,18 @@ MKLDNNPlugin::MKLDNNInferRequest::MKLDNNInferRequest(InferenceEngine::InputsData
MKLDNNExecNetwork::Ptr execNetwork_)
: IInferRequestInternal(networkInputs, networkOutputs)
, execNetwork(execNetwork_) {
CreateInferRequest();
}
MKLDNNPlugin::MKLDNNInferRequest::MKLDNNInferRequest(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs,
MKLDNNExecNetwork::Ptr execNetwork_)
: IInferRequestInternal(inputs, outputs)
, execNetwork(execNetwork_) {
CreateInferRequest();
}
void MKLDNNPlugin::MKLDNNInferRequest::CreateInferRequest() {
auto id = (execNetwork->_numRequests)++;
profilingTask = openvino::itt::handle("MKLDNN_INFER_" + execNetwork->_name + "_" + std::to_string(id));

5
inference-engine/src/mkldnn_plugin/mkldnn_infer_request.h
View File

@ -22,6 +22,10 @@ public:
InferenceEngine::OutputsDataMap networkOutputs,
std::shared_ptr<MKLDNNExecNetwork> execNetwork);
MKLDNNInferRequest(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs,
std::shared_ptr<MKLDNNExecNetwork> execNetwork);
~MKLDNNInferRequest();
void InferImpl() override;
@ -48,6 +52,7 @@ public:
void ThrowIfCanceled() const;
private:
void CreateInferRequest();
void PushInputData();
void PushStates();
void PullStates();

31
inference-engine/src/multi_device/multi_device_exec_network.cpp
View File

@ -372,6 +372,30 @@ std::shared_ptr<InferenceEngine::ICore> MultiDeviceExecutableNetwork::GetCore()
return _plugin->GetCore();
}
InferenceEngine::IInferRequestInternal::Ptr MultiDeviceExecutableNetwork::CreateInferRequestImpl(
const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) {
auto num = _numRequestsCreated++;
size_t sum = 0;
InferenceEngine::SoIInferRequestInternal request_to_share_blobs_with;
if (_workModeIsAUTO) {
return std::make_shared<MultiDeviceInferRequest>(inputs, outputs, request_to_share_blobs_with);
}
// borrowing device-specific blobs from the underlying requests for the device-agnostic, user-facing requests
// this allows to potentially save on the data-copy later (if the requests are scheduled in the same order)
for (const auto& device : _devicePrioritiesInitial) {
auto& dev_requests = _workerRequests[device.deviceName];
if ((num - sum) < dev_requests.size()) {
request_to_share_blobs_with = dev_requests.at(num - sum)._inferRequest;
break;
}
sum += dev_requests.size();
}
return std::make_shared<MultiDeviceInferRequest>(inputs, outputs, request_to_share_blobs_with);
}
InferenceEngine::IInferRequestInternal::Ptr MultiDeviceExecutableNetwork::CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs) {
auto num = _numRequestsCreated++;
@ -396,7 +420,12 @@ InferenceEngine::IInferRequestInternal::Ptr MultiDeviceExecutableNetwork::Create
}
IInferRequestInternal::Ptr MultiDeviceExecutableNetwork::CreateInferRequest() {
auto syncRequestImpl = CreateInferRequestImpl(_networkInputs, _networkOutputs);
IInferRequestInternal::Ptr syncRequestImpl;
if (this->_plugin && this->_plugin->GetCore() && GetCore()->isNewAPI())
syncRequestImpl = CreateInferRequestImpl(_parameters, _results);
if (!syncRequestImpl)
syncRequestImpl = CreateInferRequestImpl(_networkInputs, _networkOutputs);
syncRequestImpl->setPointerToExecutableNetworkInternal(shared_from_this());
return std::make_shared<MultiDeviceAsyncInferRequest>(std::static_pointer_cast<MultiDeviceInferRequest>(syncRequestImpl),
_needPerfCounters,

2
inference-engine/src/multi_device/multi_device_exec_network.hpp
View File

@ -131,6 +131,8 @@ public:
InferenceEngine::IInferRequestInternal::Ptr CreateInferRequest() override;
InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs) override;
InferenceEngine::IInferRequestInternal::Ptr CreateInferRequestImpl(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs) override;
std::shared_ptr<InferenceEngine::RemoteContext> GetContext() const override;
std::shared_ptr<InferenceEngine::ICore> GetCore() const;
~MultiDeviceExecutableNetwork() override;

16
inference-engine/src/multi_device/multi_device_infer_request.cpp
View File

@ -14,10 +14,21 @@ namespace MultiDevicePlugin {
using namespace InferenceEngine;
// ------------------------------MultiDeviceInferRequest----------------------------
MultiDeviceInferRequest::MultiDeviceInferRequest(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs,
const InferenceEngine::SoIInferRequestInternal & request_to_share_blobs_with)
: IInferRequestInternal(inputs, outputs) {
CreateInferRequest(request_to_share_blobs_with);
}
MultiDeviceInferRequest::MultiDeviceInferRequest(const InputsDataMap& networkInputs,
const OutputsDataMap& networkOutputs,
const SoIInferRequestInternal & request_to_share_blobs_with)
: IInferRequestInternal(networkInputs, networkOutputs) {
CreateInferRequest(request_to_share_blobs_with);
}
void MultiDeviceInferRequest::CreateInferRequest(const InferenceEngine::SoIInferRequestInternal& request_to_share_blobs_with) {
if (request_to_share_blobs_with) {
// borrow device-friendly blobs from the request
for (const auto &it : _networkInputs)
@ -27,7 +38,7 @@ MultiDeviceInferRequest::MultiDeviceInferRequest(const InputsDataMap& networkI
return;
}
// Allocate all input blobs
for (const auto &it : networkInputs) {
for (const auto &it : _networkInputs) {
Layout l = it.second->getLayout();
Precision p = it.second->getPrecision();
SizeVector dims = it.second->getTensorDesc().getDims();
@ -37,7 +48,7 @@ MultiDeviceInferRequest::MultiDeviceInferRequest(const InputsDataMap& networkI
_inputs[it.first]->allocate();
}
// Allocate all output blobs
for (const auto &it : networkOutputs) {
for (const auto &it : _networkOutputs) {
Layout l = it.second->getLayout();
Precision p = it.second->getPrecision();
SizeVector dims = it.second->getTensorDesc().getDims();
@ -47,7 +58,6 @@ MultiDeviceInferRequest::MultiDeviceInferRequest(const InputsDataMap& networkI
_outputs[it.first]->allocate();
}
}
void MultiDeviceInferRequest::SetBlobsToAnotherRequest(const SoIInferRequestInternal& req) {
for (const auto &it : _networkInputs) {
auto &name = it.first;

6
inference-engine/src/multi_device/multi_device_infer_request.hpp
View File

@ -24,10 +24,16 @@ public:
explicit MultiDeviceInferRequest(const InferenceEngine::InputsDataMap& networkInputs,
const InferenceEngine::OutputsDataMap& networkOutputs,
const InferenceEngine::SoIInferRequestInternal & request_to_share_blobs_with);
explicit MultiDeviceInferRequest(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs,
const InferenceEngine::SoIInferRequestInternal & request_to_share_blobs_with);
std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> GetPerformanceCounts() const override;
void InferImpl() override;
// Multi-Device impl specific: sets the data (blobs from the device-less requests to the specific device request)
void SetBlobsToAnotherRequest(const InferenceEngine::SoIInferRequestInternal& req);
private:
void CreateInferRequest(const InferenceEngine::SoIInferRequestInternal& request_to_share_blobs_with);
};
} // namespace MultiDevicePlugin

8
inference-engine/src/plugin_api/cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp
View File

@ -58,7 +58,13 @@ protected:
*/
template <typename AsyncInferRequestType = AsyncInferRequestThreadSafeDefault>
IInferRequestInternal::Ptr CreateAsyncInferRequestFromSync() {
auto syncRequestImpl = this->CreateInferRequestImpl(_networkInputs, _networkOutputs);
InferenceEngine::IInferRequestInternal::Ptr syncRequestImpl;
try {
syncRequestImpl = this->CreateInferRequestImpl(_parameters, _results);
} catch (const NotImplemented&) {
}
if (!syncRequestImpl)
syncRequestImpl = this->CreateInferRequestImpl(_networkInputs, _networkOutputs);
syncRequestImpl->setPointerToExecutableNetworkInternal(shared_from_this());
return std::make_shared<AsyncInferRequestType>(syncRequestImpl, _taskExecutor, _callbackExecutor);
}

3
inference-engine/src/plugin_api/cpp_interfaces/impl/ie_infer_async_request_thread_safe_default.hpp
View File

@ -145,7 +145,8 @@ public:
AsyncInferRequestThreadSafeDefault(const IInferRequestInternal::Ptr& request,
const ITaskExecutor::Ptr& taskExecutor,
const ITaskExecutor::Ptr& callbackExecutor)
: _syncRequest{request},
: IInferRequestInternal(*request),
_syncRequest{request},
_requestExecutor{taskExecutor},
_callbackExecutor{callbackExecutor},
_pipeline{{taskExecutor,

11
inference-engine/src/plugin_api/cpp_interfaces/interface/ie_iexecutable_network_internal.hpp
View File

@ -168,6 +168,17 @@ protected:
*/
virtual std::shared_ptr<IInferRequestInternal> CreateInferRequestImpl(InputsDataMap networkInputs,
OutputsDataMap networkOutputs);
/**
* @brief Creates an inference request internal implementation.
* @note The method is called by IExecutableNetworkInternal::CreateInferRequest as
* plugin-specific implementation.
* @param[in] inputs The function inputs
* @param[in] outputs The function outputs
* @return A shared pointer to inference request object.
*/
virtual std::shared_ptr<IInferRequestInternal> CreateInferRequestImpl(
const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs);
InferenceEngine::InputsDataMap _networkInputs; //!< Holds information about network inputs info
InferenceEngine::OutputsDataMap _networkOutputs; //!< Holds information about network outputs data

14
inference-engine/src/plugin_api/cpp_interfaces/interface/ie_iinfer_request_internal.hpp
View File

@ -13,6 +13,7 @@
#include "ie_common.h"
#include "ie_input_info.hpp"
#include "ie_preprocess_data.hpp"
#include "openvino/core/node_output.hpp"
#include "so_ptr.hpp"
namespace InferenceEngine {
@ -42,6 +43,14 @@ public:
*/
IInferRequestInternal(const InputsDataMap& networkInputs, const OutputsDataMap& networkOutputs);
/**
* @brief Constructs a new instance.
* @param[in] inputs The network inputs
* @param[in] outputs The network outputs
*/
IInferRequestInternal(const std::vector<std::shared_ptr<const ov::Node>>& networkInputs,
const std::vector<std::shared_ptr<const ov::Node>>& networkOutputs);
/**
* @brief Infers specified input(s) in synchronous mode
* @note blocks all method of InferRequest while request is ongoing (running or waiting in queue)
@ -190,6 +199,9 @@ public:
INFERENCE_ENGINE_DEPRECATED("The method will be removed")
void SetUserData(void* userData) noexcept;
const std::vector<std::shared_ptr<const ov::Node>>& GetInputs() const;
const std::vector<std::shared_ptr<const ov::Node>>& GetOutputs() const;
protected:
/**
* @brief Destroys the object.
@ -232,6 +244,8 @@ protected:
InferenceEngine::BlobMap _inputs; //!< A map of user passed blobs for network inputs
InferenceEngine::BlobMap _deviceInputs; //!< A map of actual network inputs, in plugin specific format
InferenceEngine::BlobMap _outputs; //!< A map of user passed blobs for network outputs
std::vector<std::shared_ptr<const ov::Node>> _parameters; //!< A vector of function inputs
std::vector<std::shared_ptr<const ov::Node>> _results; //!< A vector of function outputs
std::map<std::string, PreProcessDataPtr> _preProcData; //!< A map of pre-process data per input
int m_curBatch = -1; //!< Current batch value used in dynamic batching

10
inference-engine/src/transformations/include/transformations/utils/utils.hpp
View File

@ -49,14 +49,17 @@ bool has_op_with_type(const std::shared_ptr<const ngraph::Function> &function) {
}
return false;
}
inline std::string create_ie_output_name(const ngraph::Output<ngraph::Node>& output) {
inline std::string create_ie_output_name(const ngraph::Output<const ngraph::Node>& output) {
const auto& prev_layer = output.get_node_shared_ptr();
std::string out_name = prev_layer->get_friendly_name();
if (prev_layer->get_output_size() != 1)
out_name += "." + std::to_string(output.get_index());
return out_name;
}
inline std::string get_ie_output_name(const ngraph::Output<ngraph::Node>& output) {
inline std::string create_ie_output_name(const ngraph::Output<ngraph::Node>& output) {
return create_ie_output_name(ov::Output<const ngraph::Node>(output.get_node(), output.get_index()));
}
inline std::string get_ie_output_name(const ngraph::Output<const ngraph::Node>& output) {
NGRAPH_SUPPRESS_DEPRECATED_START
auto name = output.get_tensor().get_name();
NGRAPH_SUPPRESS_DEPRECATED_END
@ -65,6 +68,9 @@ inline std::string get_ie_output_name(const ngraph::Output<ngraph::Node>& output
}
return name;
}
inline std::string get_ie_output_name(const ngraph::Output<ngraph::Node>& output) {
return get_ie_output_name(ov::Output<const ngraph::Node>(output.get_node(), output.get_index()));
}
template <typename T>
bool has_constant_value(const std::shared_ptr<ngraph::opset4::Constant>& constant,

2
inference-engine/src/transformations/src/ngraph_ops/nms_ie_internal.cpp
View File

@ -80,7 +80,7 @@ int64_t op::internal::NonMaxSuppressionIEInternal::max_boxes_output_from_input()
}
const auto max_output_boxes_input =
ov::as_type_ptr<op::Constant>(input_value(max_output_boxes_per_class_port).get_node_shared_ptr());
ov::as_type_ptr<const op::Constant>(input_value(max_output_boxes_per_class_port).get_node_shared_ptr());
max_output_boxes = max_output_boxes_input->cast_vector<int64_t>().at(0);
return max_output_boxes;

12
inference-engine/src/vpu/myriad_plugin/myriad_executable_network.h
View File

@ -22,6 +22,8 @@
#include "myriad_executor.h"
#include "myriad_infer_request.h"
#include "myriad_async_infer_request.h"
#include "cpp_interfaces/interface/ie_iplugin_internal.hpp"
#include "ie_icore.hpp"
namespace vpu {
namespace MyriadPlugin {
@ -67,8 +69,14 @@ public:
if (!_isNetworkConstant && (_device == nullptr || !_device->isBooted())) {
IE_THROW() << "Can not create infer request: there is no available devices with platform ";
}
auto syncRequestImpl = std::make_shared<MyriadInferRequest>(_graphDesc, _networkInputs, _networkOutputs,
std::shared_ptr<MyriadInferRequest> syncRequestImpl;
if (this->_plugin && this->_plugin->GetCore() && this->_plugin->GetCore()->isNewAPI())
syncRequestImpl = std::make_shared<MyriadInferRequest>(_graphDesc, _parameters, _results,
_inputInfo, _outputInfo,
_graphMetaData.stagesMeta, _config, _log,
_executor, _constDatas, _isNetworkConstant);
if (!syncRequestImpl)
syncRequestImpl = std::make_shared<MyriadInferRequest>(_graphDesc, _networkInputs, _networkOutputs,
_inputInfo, _outputInfo,
_graphMetaData.stagesMeta, _config, _log,
_executor, _constDatas, _isNetworkConstant);

26
inference-engine/src/vpu/myriad_plugin/myriad_infer_request.cpp
View File

@ -28,6 +28,24 @@ using namespace InferenceEngine;
#define MEMCPY(dst, src, bytes) std::copy_n((src), (bytes), (dst))
MyriadInferRequest::MyriadInferRequest(GraphDesc &graphDesc,
const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs,
DataInfo& compilerInputsInfo,
DataInfo& compilerOutputsInfo,
const std::vector<StageMetaInfo> &blobMetaData,
const PluginConfiguration& myriadConfig,
const Logger::Ptr &log,
const MyriadExecutorPtr &executor,
std::map<std::string, ie::Blob::Ptr> constDatas,
bool isNetworkConstant = true) :
IInferRequestInternal(inputs, outputs), _executor(executor),
_log(log), _stagesMetaData(blobMetaData), _config(myriadConfig),
_inputInfo(compilerInputsInfo), _outputInfo(compilerOutputsInfo),
_graphDesc(graphDesc), _constDatas(constDatas), _isNetworkConstant(isNetworkConstant) {
CreateInferRequest();
}
MyriadInferRequest::MyriadInferRequest(GraphDesc &graphDesc,
InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs,
@ -43,6 +61,10 @@ MyriadInferRequest::MyriadInferRequest(GraphDesc &graphDesc,
_log(log), _stagesMetaData(blobMetaData), _config(myriadConfig),
_inputInfo(compilerInputsInfo), _outputInfo(compilerOutputsInfo),
_graphDesc(graphDesc), _constDatas(constDatas), _isNetworkConstant(isNetworkConstant) {
CreateInferRequest();
}
void MyriadInferRequest::CreateInferRequest() {
VPU_PROFILE(MyriadInferRequest);
const auto& ioStrides = _config.get<TensorStridesOption>();
@ -84,8 +106,8 @@ MyriadInferRequest::MyriadInferRequest(GraphDesc &graphDesc,
_outputs[networkOutput.first] = outputBlob;
}
inputBuffer .resize(compilerInputsInfo.totalSize);
resultBuffer.resize(compilerOutputsInfo.totalSize);
inputBuffer .resize(_inputInfo.totalSize);
resultBuffer.resize(_outputInfo.totalSize);
VPU_THROW_UNLESS(
!_networkOutputs.empty() && !(_networkInputs.empty() && !_isNetworkConstant),

15
inference-engine/src/vpu/myriad_plugin/myriad_infer_request.h
View File

@ -35,11 +35,12 @@ class MyriadInferRequest : public InferenceEngine::IInferRequestInternal {
std::vector<uint8_t> inputBuffer;
std::map<std::string, ie::Blob::Ptr> _constDatas;
bool _isNetworkConstant;
void CreateInferRequest();
public:
typedef std::shared_ptr<MyriadInferRequest> Ptr;
explicit MyriadInferRequest(GraphDesc &_graphDesc,
MyriadInferRequest(GraphDesc &_graphDesc,
InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs,
DataInfo& compilerInputsInfo,
@ -51,6 +52,18 @@ public:
std::map<std::string, ie::Blob::Ptr> constDatas,
bool isNetworkConstant);
MyriadInferRequest(GraphDesc &_graphDesc,
const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs,
DataInfo& compilerInputsInfo,
DataInfo& compilerOutputsInfo,
const std::vector<StageMetaInfo> &blobMetaData,
const PluginConfiguration &myriadConfig,
const Logger::Ptr &log,
const MyriadExecutorPtr &executor,
std::map<std::string, ie::Blob::Ptr> constDatas,
bool isNetworkConstant);
void InferImpl() override;
void InferAsync();
void GetResult();

23
inference-engine/tests/functional/inference_engine/ov_infer_request_test.cpp
View File

@ -16,12 +16,12 @@ using namespace InferenceEngine::details;
TEST(InferRequestOVTests, throwsOnUninitializedSetTensor) {
ov::runtime::InferRequest req;
ASSERT_THROW(req.set_tensor({}, {}), ov::Exception);
ASSERT_THROW(req.set_tensor("", {}), ov::Exception);
}
TEST(InferRequestOVTests, throwsOnUninitializedGetTensor) {
ov::runtime::InferRequest req;
ASSERT_THROW(req.get_tensor({}), ov::Exception);
ASSERT_THROW(req.get_tensor(""), ov::Exception);
}
TEST(InferRequestOVTests, throwsOnUninitializedInfer) {
@ -60,9 +60,26 @@ TEST(InferRequestOVTests, throwsOnUninitializedQueryState) {
ASSERT_THROW(req.query_state(), ov::Exception);
}
TEST(InferRequestOVTests, throwsOnUninitializedSetRemoteTensorWithName) {
ov::runtime::InferRequest req;
ov::runtime::RemoteTensor remote_tensor;
ASSERT_THROW(req.set_tensor("", remote_tensor), ov::Exception);
}
TEST(InferRequestOVTests, throwsOnUninitializedSetInputRemoteTensor) {
ov::runtime::InferRequest req;
ov::runtime::RemoteTensor remote_tensor;
ASSERT_THROW(req.set_input_tensor(0, remote_tensor), ov::Exception);
}
TEST(InferRequestOVTests, throwsOnUninitializedSetOutputRemoteTensor) {
ov::runtime::InferRequest req;
ov::runtime::RemoteTensor remote_tensor;
ASSERT_THROW(req.set_output_tensor(0, remote_tensor), ov::Exception);
}
TEST(InferRequestOVTests, throwsOnUninitializedSetRemoteTensor) {
ov::runtime::InferRequest req;
ov::runtime::RemoteTensor remote_tensor;
ASSERT_THROW(req.set_tensor({}, remote_tensor), ov::Exception);
ASSERT_THROW(req.set_tensor(ov::Output<const ov::Node>(), remote_tensor), ov::Exception);
}

7
inference-engine/tests/functional/plugin/cpu/shared_tests_instances/behavior/infer_request/infer_requset_dynamic.cpp
View File

@ -28,12 +28,14 @@ std::shared_ptr<ngraph::Function> getFunction1() {
auto params = ngraph::builder::makeParams(ngPrc, {inputShape});
params.front()->set_friendly_name("Param_1");
params.front()->get_output_tensor(0).set_names({"Tensor_1"});
params.front()->get_output_tensor(0).set_names({"input_tensor"});
auto in2add = ngraph::builder::makeConstant(ngPrc, {1, 4, 1, 1}, std::vector<float>{}, true);
auto add = ngraph::builder::makeEltwise(params[0], in2add, ngraph::helpers::EltwiseTypes::ADD);
auto relu1 = std::make_shared<ngraph::opset1::Relu>(add->output(0));
relu1->get_output_tensor(0).set_names({"relu1"});
auto relu2 = std::make_shared<ngraph::opset1::Relu>(add->output(0));
relu2->get_output_tensor(0).set_names({"relu2"});
ngraph::NodeVector results{relu1, relu2};
return std::make_shared<ngraph::Function>(results, params, "AddTwoOutputEdges");
@ -45,7 +47,7 @@ std::shared_ptr<ngraph::Function> getFunction2() {
auto params = ngraph::builder::makeParams(ngPrc, {inputShape});
params.front()->set_friendly_name("Param_1");
params.front()->get_output_tensor(0).set_names({"Tensor_1"});
params.front()->get_output_tensor(0).set_names({"input_tensor"});
auto split = ngraph::builder::makeSplit(params[0], ngPrc, 2, 1);
auto in2add = ngraph::builder::makeConstant(ngPrc, {1, 2, 1, 1}, std::vector<float>{}, true);
@ -57,6 +59,7 @@ std::shared_ptr<ngraph::Function> getFunction2() {
auto relu2 = std::make_shared<ngraph::opset1::Relu>(mult);
auto concat = std::make_shared<ngraph::opset1::Concat>(ngraph::OutputVector{relu1->output(0), relu2->output(0)}, 3);
concat->get_output_tensor(0).set_names({"concat"});
return std::make_shared<ngraph::Function>(concat, params, "SplitAddConcat");
}

1
inference-engine/tests/functional/plugin/gna/shared_tests_instances/skip_tests_config.cpp
View File

@ -62,6 +62,7 @@ std::vector<std::string> disabledTestPatterns() {
R"(.*OVExecutableNetworkBaseTest.*canLoadCorrectNetworkToGetExecutableWithIncorrectConfig.*)",
R"(.*OVExecutableNetworkBaseTest.*CanSetConfigToExecNet.*)",
R"(.*OVExecutableNetworkBaseTest.*CanGetInputsInfoAndCheck.*)",
R"(.*OVExecutableNetworkBaseTest.*getOutputsFromSplitFunctionWithSeveralOutputs.*)",
R"(.*OVClassHeteroExecutableNetworkGetMetricTest_TARGET_FALLBACK.*GetMetricNoThrow.*)",
R"(.*Behavior.*OVExecutableNetworkBaseTest.*get(Inputs|Outputs)FromFunctionWithSeveral(Inputs|Outputs).*)",
// TODO: Issue: 29577

1
inference-engine/tests/functional/plugin/shared/CMakeLists.txt
View File

@ -32,6 +32,7 @@ addIeTarget(
PUBLIC
pugixml::static
funcTestUtils
ngraph_test_util
ngraphFunctions
lptNgraphFunctions
sharedTestClasses

87
inference-engine/tests/functional/plugin/shared/include/behavior/infer_request/infer_request_dynamic.hpp
View File

@ -85,7 +85,7 @@ protected:
};
TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithoutSetShape) {
const std::string tensor_name = "Tensor_1";
const std::string tensor_name = "input_tensor";
std::map<std::string, ov::PartialShape> shapes;
shapes[tensor_name] = {ov::Dimension::dynamic(), 4, 20, 20};
ASSERT_NO_THROW(function->reshape(shapes));
@ -95,11 +95,11 @@ TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithoutSetShape) {
ov::runtime::InferRequest req;
ov::runtime::Tensor tensor;
ASSERT_NO_THROW(req = execNet.create_infer_request());
ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name()));
ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name()));
}
TEST_P(InferRequestDynamicTests, InferDynamicNetworkBoundWithoutSetShape) {
const std::string tensor_name = "Tensor_1";
const std::string tensor_name = "input_tensor";
std::map<std::string, ov::PartialShape> shapes;
shapes[tensor_name] = {ov::Dimension(0, 5), 4, 20, 20};
ASSERT_NO_THROW(function->reshape(shapes));
@ -109,12 +109,12 @@ TEST_P(InferRequestDynamicTests, InferDynamicNetworkBoundWithoutSetShape) {
ov::runtime::InferRequest req;
ov::runtime::Tensor tensor;
ASSERT_NO_THROW(req = execNet.create_infer_request());
ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name()));
ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name()));
}
TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithGetTensor) {
const std::string tensor_name = "Tensor_1";
const std::string tensor_name = "input_tensor";
const ov::Shape refShape = inOutShapes[0].first;
const ov::Shape refOutShape = inOutShapes[0].second;
std::map<std::string, ov::PartialShape> shapes;
@ -125,11 +125,10 @@ TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithGetTensor) {
// Create InferRequest
ov::runtime::InferRequest req;
ov::runtime::Tensor tensor, otensor;
const std::string outputname = ngraph::op::util::create_ie_output_name(
function->get_results().front()->input_value(0));
const std::string outputname = function->outputs().back().get_any_name();
ASSERT_NO_THROW(req = execNet.create_infer_request());
//ASSERT_NO_THROW(req.SetShape(tensor_name, {1, 4, 20, 20}));
ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name()));
ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name()));
ASSERT_NO_THROW(tensor.set_shape({1, 4, 20, 20}));
ASSERT_EQ(tensor.get_shape(), refShape);
ASSERT_NO_THROW(otensor = req.get_tensor(outputname));
@ -144,7 +143,7 @@ TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithGetTensor) {
}
TEST_P(InferRequestDynamicTests, InferUpperBoundNetworkWithGetTensor) {
const std::string tensor_name = "Tensor_1";
const std::string tensor_name = "input_tensor";
const ov::Shape refShape = inOutShapes[0].first;
const ov::Shape refOutShape = inOutShapes[0].second;
std::map<std::string, ov::PartialShape> shapes;
@ -155,14 +154,13 @@ TEST_P(InferRequestDynamicTests, InferUpperBoundNetworkWithGetTensor) {
// Create InferRequest
ov::runtime::InferRequest req;
ov::runtime::Tensor tensor, otensor;
const std::string outputname = ngraph::op::util::create_ie_output_name(
function->get_results().front()->input_value(0));
const std::string outputname = function->outputs().back().get_any_name();
ASSERT_NO_THROW(req = execNet.create_infer_request());
//ASSERT_NO_THROW(req.SetShape(tensor_name, {1, 4, 20, 20}));
ASSERT_NO_THROW(otensor = req.get_tensor(outputname));
ASSERT_EQ(0, otensor.get_size()); // output tensor is not allocated
ASSERT_EQ(function->output().get_element_type(), otensor.get_element_type()); // by it has type
ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name()));
ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name()));
ASSERT_NO_THROW(tensor.set_shape({1, 4, 20, 20}));
ASSERT_EQ(tensor.get_shape(), refShape);
ASSERT_NO_THROW(req.infer());
@ -172,7 +170,7 @@ TEST_P(InferRequestDynamicTests, InferUpperBoundNetworkWithGetTensor) {
}
TEST_P(InferRequestDynamicTests, InferFullyDynamicNetworkWithGetTensor) {
const std::string tensor_name = "Tensor_1";
const std::string tensor_name = "input_tensor";
const ov::Shape refShape = inOutShapes[0].first;
const ov::Shape refOutShape = inOutShapes[0].second;
std::map<std::string, ov::PartialShape> shapes;
@ -183,25 +181,24 @@ TEST_P(InferRequestDynamicTests, InferFullyDynamicNetworkWithGetTensor) {
// Create InferRequest
ov::runtime::InferRequest req;
ov::runtime::Tensor tensor, otensor;
const std::string outputname = ngraph::op::util::create_ie_output_name(
function->get_results().front()->input_value(0));
const std::string outputName = function->outputs().back().get_any_name();
ASSERT_NO_THROW(req = execNet.create_infer_request());
//ASSERT_NO_THROW(req.SetShape(tensor_name, {1, 4, 20, 20}));
ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name()));
ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name()));
ASSERT_NO_THROW(tensor.set_shape({1, 4, 20, 20}));
ASSERT_EQ(tensor.get_shape(), refShape);
ASSERT_NO_THROW(otensor = req.get_tensor(outputname));
ASSERT_NO_THROW(otensor = req.get_tensor(outputName));
ASSERT_EQ(0, otensor.get_size()); // output tensor is not allocated
ASSERT_EQ(function->output().get_element_type(), otensor.get_element_type()); // by it has type
ASSERT_NO_THROW(req.infer());
ASSERT_NO_THROW(req.start_async());
ASSERT_NO_THROW(req.wait());
ASSERT_NO_THROW(otensor = req.get_tensor(ngraph::op::util::create_ie_output_name(function->get_results().front()->input_value(0))));
ASSERT_NO_THROW(otensor = req.get_tensor(outputName));
ASSERT_EQ(otensor.get_shape(), refOutShape);
}
TEST_P(InferRequestDynamicTests, InferOutOfRangeShapeNetworkWithGetTensorLower) {
const std::string tensor_name = "Tensor_1";
const std::string tensor_name = "input_tensor";
const ov::Shape refShape = inOutShapes[0].first;
const ov::Shape refOutShape = inOutShapes[0].second;
std::map<std::string, ov::PartialShape> shapes;
@ -213,14 +210,14 @@ TEST_P(InferRequestDynamicTests, InferOutOfRangeShapeNetworkWithGetTensorLower)
ov::runtime::InferRequest req;
ov::runtime::Tensor tensor;
ASSERT_NO_THROW(req = execNet.create_infer_request());
ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name()));
ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name()));
ASSERT_NO_THROW(tensor.set_shape({1, 4, 20, 20}));
// Plugin may or may not throw in case if input tensor has dimensions that are out of bounds
//ASSERT_THROW(req.infer(), ov::Exception);
}
TEST_P(InferRequestDynamicTests, InferOutOfRangeShapeNetworkWithGetTensorUpper) {
const std::string tensor_name = "Tensor_1";
const std::string tensor_name = "input_tensor";
const ov::Shape refShape = inOutShapes[0].first;
const ov::Shape refOutShape = inOutShapes[0].second;
std::map<std::string, ov::PartialShape> shapes;
@ -232,14 +229,14 @@ TEST_P(InferRequestDynamicTests, InferOutOfRangeShapeNetworkWithGetTensorUpper)
ov::runtime::InferRequest req;
ov::runtime::Tensor tensor;
ASSERT_NO_THROW(req = execNet.create_infer_request());
ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name()));
ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name()));
ASSERT_NO_THROW(tensor.set_shape({3, 4, 20, 20}));
// Plugin may or may not throw in case if input tensor has dimensions that are out of bounds
// ASSERT_THROW(req.infer(), ov::Exception);
}
TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithGetTensor2times) {
const std::string tensor_name = "Tensor_1";
const std::string tensor_name = "input_tensor";
const ov::Shape refShape = inOutShapes[0].first;
const ov::Shape refShape2 = inOutShapes[1].first;
const ov::Shape refOutShape = inOutShapes[0].second;
@ -253,28 +250,29 @@ TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithGetTensor2times) {
ov::runtime::InferRequest req;
ov::runtime::Tensor tensor;
ASSERT_NO_THROW(req = execNet.create_infer_request());
ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name()));
ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name()));
ASSERT_NO_THROW(tensor.set_shape(refShape));
ASSERT_EQ(tensor.get_shape(), refShape);
ASSERT_NO_THROW(req.infer());
ASSERT_NO_THROW(req.start_async());
req.wait();
ASSERT_NO_THROW(tensor = req.get_tensor(ngraph::op::util::create_ie_output_name(function->get_results().front()->input_value(0))));
const std::string outputName = function->outputs().back().get_any_name();
ASSERT_NO_THROW(tensor = req.get_tensor(outputName));
ASSERT_EQ(tensor.get_shape(), refOutShape);
ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name()));
ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name()));
ASSERT_NO_THROW(tensor.set_shape(refShape2));
ASSERT_EQ(tensor.get_shape(), refShape2);
ASSERT_NO_THROW(req.infer());
ASSERT_NO_THROW(req.start_async());
req.wait();
ASSERT_NO_THROW(tensor = req.get_tensor(ngraph::op::util::create_ie_output_name(function->get_results().front()->input_value(0))));
ASSERT_NO_THROW(tensor = req.get_tensor(outputName));
ASSERT_EQ(tensor.get_shape(), refOutShape2);
}
TEST_P(InferRequestDynamicTests, GetSameTensor2times) {
const std::string tensor_name = "Tensor_1";
const std::string tensor_name = "input_tensor";
const ov::Shape refShape = inOutShapes[0].first;
std::map<std::string, ov::PartialShape> shapes;
shapes[tensor_name] = {ov::Dimension::dynamic(), 4, 20, 20};
@ -285,15 +283,15 @@ TEST_P(InferRequestDynamicTests, GetSameTensor2times) {
ov::runtime::InferRequest req;
ov::runtime::Tensor tensor;
ASSERT_NO_THROW(req = execNet.create_infer_request());
ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name()));
ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name()));
ASSERT_NO_THROW(tensor.set_shape(refShape));
ASSERT_EQ(tensor.get_shape(), refShape);
ASSERT_NO_THROW(tensor = req.get_tensor(function->get_parameters().back()->get_friendly_name()));
ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name()));
ASSERT_EQ(tensor.get_shape(), refShape);
}
TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithSetTensor) {
const std::string tensor_name = "Tensor_1";
const std::string tensor_name = "input_tensor";
const ov::Shape refShape = inOutShapes[0].first;
const ov::Shape refOutShape = inOutShapes[0].second;
std::map<std::string, ov::PartialShape> shapes;
@ -305,17 +303,18 @@ TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithSetTensor) {
ov::runtime::InferRequest req;
ov::runtime::Tensor tensor(ov::element::f32, refShape);
ASSERT_NO_THROW(req = execNet.create_infer_request());
ASSERT_NO_THROW(req.set_tensor(function->get_parameters().back()->get_friendly_name(), tensor));
ASSERT_NO_THROW(req.set_tensor(function->inputs().back().get_any_name(), tensor));
ASSERT_EQ(tensor.get_shape(), refShape);
ASSERT_NO_THROW(req.infer());
ASSERT_NO_THROW(req.start_async());
ASSERT_NO_THROW(req.wait());
ASSERT_NO_THROW(tensor = req.get_tensor(ngraph::op::util::create_ie_output_name(function->get_results().front()->input_value(0))));
const std::string outputName = function->outputs().back().get_any_name();
ASSERT_NO_THROW(tensor = req.get_tensor(outputName));
ASSERT_EQ(tensor.get_shape(), refOutShape);
}
TEST_P(InferRequestDynamicTests, InferFullyDynamicNetworkWithSetTensor) {
const std::string tensor_name = "Tensor_1";
const std::string tensor_name = "input_tensor";
const ov::Shape refShape = inOutShapes[0].first;
const ov::Shape refOutShape = inOutShapes[0].second;
std::map<std::string, ov::PartialShape> shapes;
@ -326,12 +325,11 @@ TEST_P(InferRequestDynamicTests, InferFullyDynamicNetworkWithSetTensor) {
// Create InferRequest
ov::runtime::InferRequest req;
ov::runtime::Tensor tensor(ov::element::f32, refShape), otensor;
const std::string outputname = ngraph::op::util::create_ie_output_name(
function->get_results().front()->input_value(0));
const std::string outputName = function->outputs().back().get_any_name();
ASSERT_NO_THROW(req = execNet.create_infer_request());
ASSERT_NO_THROW(req.set_tensor(function->get_parameters().back()->get_friendly_name(), tensor));
ASSERT_NO_THROW(req.set_tensor(function->inputs().back().get_any_name(), tensor));
ASSERT_EQ(tensor.get_shape(), refShape);
ASSERT_NO_THROW(otensor = req.get_tensor(outputname));
ASSERT_NO_THROW(otensor = req.get_tensor(outputName));
ASSERT_EQ(0, otensor.get_size()); // output tensor is not allocated
ASSERT_EQ(function->output().get_element_type(), otensor.get_element_type()); // by it has type
ASSERT_NO_THROW(req.infer());
@ -339,13 +337,13 @@ TEST_P(InferRequestDynamicTests, InferFullyDynamicNetworkWithSetTensor) {
ASSERT_NO_THROW(req.start_async());
ASSERT_NO_THROW(req.wait());
ASSERT_EQ(otensor.get_shape(), refOutShape);
ASSERT_NO_THROW(tensor = req.get_tensor(ngraph::op::util::create_ie_output_name(function->get_results().front()->input_value(0))));
ASSERT_NO_THROW(tensor = req.get_tensor(outputName));
ASSERT_EQ(tensor.get_shape(), refOutShape);
ASSERT_EQ(otensor.get_shape(), refOutShape);
}
TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithSetTensor2times) {
const std::string tensor_name = "Tensor_1";
const std::string tensor_name = "input_tensor";
const ov::Shape refShape = inOutShapes[0].first;
const ov::Shape refShape2 = inOutShapes[1].first;
const ov::Shape refOutShape = inOutShapes[0].second;
@ -353,6 +351,7 @@ TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithSetTensor2times) {
std::map<std::string, ov::PartialShape> shapes;
shapes[tensor_name] = {ov::Dimension::dynamic(), 4, 20, 20};
ASSERT_NO_THROW(function->reshape(shapes));
const std::string outputName = function->outputs().back().get_any_name();
// Load ov::Function to target plugins
auto execNet = ie->compile_model(function, targetDevice, configuration);
// Create InferRequest
@ -360,21 +359,21 @@ TEST_P(InferRequestDynamicTests, InferDynamicNetworkWithSetTensor2times) {
ov::runtime::Tensor tensor(ov::element::f32, refShape);
ASSERT_NO_THROW(req = execNet.create_infer_request());
ASSERT_NO_THROW(req.set_tensor(function->get_parameters().back()->get_friendly_name(), tensor));
ASSERT_NO_THROW(req.set_tensor(function->inputs().back().get_any_name(), tensor));
ASSERT_EQ(tensor.get_shape(), refShape);
ASSERT_NO_THROW(req.infer());
ASSERT_NO_THROW(req.start_async());
ASSERT_NO_THROW(req.wait());
ASSERT_NO_THROW(tensor = req.get_tensor(ngraph::op::util::create_ie_output_name(function->get_results().front()->input_value(0))));
ASSERT_NO_THROW(tensor = req.get_tensor(outputName));
ASSERT_EQ(tensor.get_shape(), refOutShape);
tensor = ov::runtime::Tensor(ov::element::f32, refShape2);
ASSERT_NO_THROW(req.set_tensor(function->get_parameters().back()->get_friendly_name(), tensor));
ASSERT_NO_THROW(req.set_tensor(function->inputs().back().get_any_name(), tensor));
ASSERT_EQ(tensor.get_shape(), refShape2);
ASSERT_NO_THROW(req.infer());
ASSERT_NO_THROW(req.start_async());
ASSERT_NO_THROW(req.wait());
ASSERT_NO_THROW(tensor = req.get_tensor(ngraph::op::util::create_ie_output_name(function->get_results().front()->input_value(0))));
ASSERT_NO_THROW(tensor = req.get_tensor(outputName));
ASSERT_EQ(tensor.get_shape(), refOutShape2);
}

33
inference-engine/tests/functional/plugin/shared/include/behavior/infer_request/inference_chaining.hpp
View File

@ -2,8 +2,9 @@
// SPDX-License-Identifier: Apache-2.0
//
#include <chrono>
#include <gtest/gtest.h>
#include <chrono>
#include <initializer_list>
#include <memory>
#include <string>
@ -12,6 +13,7 @@
#include "base/behavior_test_utils.hpp"
#include "openvino/core/attribute_visitor.hpp"
#include "openvino/core/function.hpp"
#include "openvino/core/node.hpp"
#include "openvino/core/partial_shape.hpp"
#include "openvino/core/rank.hpp"
@ -19,8 +21,6 @@
#include "openvino/core/type/element_type.hpp"
#include "openvino/core/type/element_type_traits.hpp"
#include "openvino/op/parameter.hpp"
#include "openvino/core/function.hpp"
#include "ngraph_functions/builders.hpp"
#include "openvino/runtime/infer_request.hpp"
#include "openvino/runtime/tensor.hpp"
@ -107,9 +107,9 @@ public:
// perform inference chaining
if (outputToInput) {
ASSERT_NO_THROW(r1.set_tensor("param_0", r0.get_tensor("result_0")));
ASSERT_NO_THROW(r1.set_tensor("input_tensor_0", r0.get_tensor("result_tensor_0")));
} else {
ASSERT_NO_THROW(r0.set_tensor("result_0", r1.get_tensor("param_0")));
ASSERT_NO_THROW(r0.set_tensor("result_tensor_0", r1.get_tensor("input_tensor_0")));
}
// create input tensors
@ -118,15 +118,15 @@ public:
ov::runtime::Tensor t2 = tensor(std::vector<float>{7.0f, 8.0f, 9.0f});
ov::runtime::Tensor t3 = tensor(std::vector<float>{2.0f, 3.0f, 2.0f});
ASSERT_NO_THROW(r0.set_tensor("param_0", t0));
ASSERT_NO_THROW(r0.set_tensor("param_1", t1));
ASSERT_NO_THROW(r0.set_tensor("param_2", t2));
ASSERT_NO_THROW(r1.set_tensor("param_1", t3));
ASSERT_NO_THROW(r0.set_tensor("input_tensor_0", t0));
ASSERT_NO_THROW(r0.set_tensor("input_tensor_1", t1));
ASSERT_NO_THROW(r0.set_tensor("input_tensor_2", t2));
ASSERT_NO_THROW(r1.set_tensor("input_tensor_1", t3));
ASSERT_NO_THROW(r2.set_tensor("param_0", t0));
ASSERT_NO_THROW(r2.set_tensor("param_1", t1));
ASSERT_NO_THROW(r2.set_tensor("param_2", t2));
ASSERT_NO_THROW(r2.set_tensor("param_3", t3));
ASSERT_NO_THROW(r2.set_tensor("input_tensor_0", t0));
ASSERT_NO_THROW(r2.set_tensor("input_tensor_1", t1));
ASSERT_NO_THROW(r2.set_tensor("input_tensor_2", t2));
ASSERT_NO_THROW(r2.set_tensor("input_tensor_3", t3));
ASSERT_NO_THROW(r0.infer());
ASSERT_NO_THROW(r1.infer());
@ -136,9 +136,9 @@ public:
std::vector<float> reference1 = {12.0f, 15.0f, 18.0f};
std::vector<float> reference2 = {24.0f, 45.0f, 36.0f};
auto rti = r0.get_tensor("result_0");
auto rt0 = r1.get_tensor("result_0");
auto rt1 = r2.get_tensor("result_0");
auto rti = r0.get_tensor("result_tensor_0");
auto rt0 = r1.get_tensor("result_tensor_0");
auto rt1 = r2.get_tensor("result_tensor_0");
for (size_t i = 0; i < reference1.size(); ++i) {
EXPECT_EQ(reference1[i], rti.data<float>()[i]);
@ -148,7 +148,6 @@ public:
}
};
TEST_P(OVInferenceChaining, StaticOutputToStaticInput) {
// Skip test according to plugin specific disabledTestPatterns() (if any)
SKIP_IF_CURRENT_TEST_IS_DISABLED()

10
inference-engine/tests/functional/plugin/shared/include/behavior/ov_executable_network/exec_graph_info.hpp
View File

@ -100,7 +100,7 @@ TEST_P(OVExecGraphImportExportTest, importExportedFunction) {
execNet.export_model(strm);
ov::runtime::ExecutableNetwork importedExecNet = core->import_model(strm, targetDevice, configuration);
ASSERT_EQ(function->inputs().size(), 2);
EXPECT_EQ(function->inputs().size(), 2);
EXPECT_EQ(function->inputs().size(), importedExecNet.inputs().size());
EXPECT_THROW(importedExecNet.input(), ov::Exception);
EXPECT_EQ(function->input(0).get_tensor().get_names(), importedExecNet.input(0).get_tensor().get_names());
@ -256,14 +256,14 @@ TEST_P(OVExecGraphImportExportTest, importExportedIENetwork) {
execNet.Export(strm);
ov::runtime::ExecutableNetwork importedExecNet = core->import_model(strm, targetDevice, configuration);
ASSERT_EQ(function->inputs().size(), 2);
EXPECT_EQ(function->inputs().size(), 2);
EXPECT_EQ(function->inputs().size(), importedExecNet.inputs().size());
EXPECT_THROW(importedExecNet.input(), ov::Exception);
EXPECT_NO_THROW(importedExecNet.input("data1").get_node());
EXPECT_NO_THROW(importedExecNet.input("data2").get_node());
EXPECT_NO_THROW(importedExecNet.input("param1").get_node());
EXPECT_NO_THROW(importedExecNet.input("param2").get_node());
ASSERT_EQ(function->outputs().size(), 2);
EXPECT_EQ(function->outputs().size(), 2);
EXPECT_EQ(function->outputs().size(), importedExecNet.outputs().size());
EXPECT_THROW(importedExecNet.output(), ov::Exception);
EXPECT_NE(function->output(0).get_tensor().get_names(),
@ -319,11 +319,11 @@ TEST_P(OVExecGraphImportExportTest, ieImportExportedFunction) {
execNet.export_model(strm);
InferenceEngine::ExecutableNetwork importedExecNet = ie->ImportNetwork(strm, targetDevice, configuration);
ASSERT_EQ(function->inputs().size(), 2);
EXPECT_EQ(function->inputs().size(), 2);
EXPECT_EQ(function->inputs().size(), importedExecNet.GetInputsInfo().size());
EXPECT_NO_THROW(importedExecNet.GetInputsInfo()["param1"]);
EXPECT_NO_THROW(importedExecNet.GetInputsInfo()["param2"]);
ASSERT_EQ(function->outputs().size(), 2);
EXPECT_EQ(function->outputs().size(), 2);
EXPECT_EQ(function->outputs().size(), importedExecNet.GetOutputsInfo().size());
EXPECT_NO_THROW(importedExecNet.GetOutputsInfo()["relu_op"]);
EXPECT_NO_THROW(importedExecNet.GetOutputsInfo()["concat_op"]);

272
inference-engine/tests/functional/plugin/shared/include/behavior/ov_executable_network/exec_network_base.hpp
View File

@ -2,15 +2,15 @@
// SPDX-License-Identifcorer: Apache-2.0
//
#include <fstream>
#include <exec_graph_info.hpp>
#include <fstream>
#include <transformations/serialize.hpp>
#include "base/ov_behavior_test_utils.hpp"
#include "common_test_utils/ngraph_test_utils.hpp"
#include "common_test_utils/file_utils.hpp"
#include "base/ov_behavior_test_utils.hpp"
#include "common_test_utils/file_utils.hpp"
#include "common_test_utils/ngraph_test_utils.hpp"
#include "functional_test_utils/plugin_cache.hpp"
#include "openvino/runtime/tensor.hpp"
namespace ov {
namespace test {
@ -46,6 +46,29 @@ public:
}
}
bool compareTensors(const ov::runtime::Tensor& t1, const ov::runtime::Tensor& t2) {
void* data1;
void* data2;
try {
data1 = t1.data();
} catch (const ov::Exception&) {
// Remote tensor
data1 = nullptr;
}
try {
data2 = t2.data();
} catch (const ov::Exception&) {
// Remote tensor
data2 = nullptr;
}
return t1.get_element_type() == t2.get_element_type() &&
t1.get_shape() == t2.get_shape() &&
t1.get_byte_size() == t2.get_byte_size() &&
t1.get_size() == t2.get_size() &&
t1.get_strides() == t2.get_strides() &&
data1 == data2;
}
protected:
std::shared_ptr<ov::runtime::Core> core = utils::PluginCache::get().core();
std::string targetDevice;
@ -54,37 +77,37 @@ protected:
};
TEST_P(OVExecutableNetworkBaseTest, canLoadCorrectNetworkToGetExecutable) {
ASSERT_NO_THROW(auto execNet = core->compile_model(function, targetDevice, configuration));
EXPECT_NO_THROW(auto execNet = core->compile_model(function, targetDevice, configuration));
}
TEST_P(OVExecutableNetworkBaseTest, canLoadCorrectNetworkToGetExecutableWithIncorrectConfig) {
std::map<std::string, std::string> incorrectConfig = {{"abc", "def"}};
ASSERT_ANY_THROW(auto execNet = core->compile_model(function, targetDevice, configuration));
EXPECT_ANY_THROW(auto execNet = core->compile_model(function, targetDevice, configuration));
}
TEST_P(OVExecutableNetworkBaseTest, canLoadCorrectNetworkToGetExecutableAndCreateInferRequest) {
auto execNet = core->compile_model(function, targetDevice, configuration);
ASSERT_NO_THROW(auto req = execNet.create_infer_request());
EXPECT_NO_THROW(auto req = execNet.create_infer_request());
}
TEST_P(OVExecutableNetworkBaseTest, checkGetExecGraphInfoIsNotNullptr) {
auto execNet = core->compile_model(function, targetDevice, configuration);
auto execGraph = execNet.get_runtime_function();
ASSERT_NE(execGraph, nullptr);
EXPECT_NE(execGraph, nullptr);
}
TEST_P(OVExecutableNetworkBaseTest, checkGetMetric) {
auto execNet = core->compile_model(function, targetDevice, configuration);
ASSERT_NO_THROW(execNet.get_metric(METRIC_KEY(SUPPORTED_CONFIG_KEYS)));
EXPECT_NO_THROW(execNet.get_metric(METRIC_KEY(SUPPORTED_CONFIG_KEYS)));
}
TEST_P(OVExecutableNetworkBaseTest, canLoadCorrectNetworkToGetExecutableAndCheckConfig) {
auto execNet = core->compile_model(function, targetDevice, configuration);
for (const auto& configItem : configuration) {
InferenceEngine::Parameter param;
ASSERT_NO_THROW(param = execNet.get_config(configItem.first));
ASSERT_FALSE(param.empty());
ASSERT_EQ(param, InferenceEngine::Parameter(configItem.second));
EXPECT_NO_THROW(param = execNet.get_config(configItem.first));
EXPECT_FALSE(param.empty());
EXPECT_EQ(param, InferenceEngine::Parameter(configItem.second));
}
}
@ -94,7 +117,7 @@ TEST_P(OVExecutableNetworkBaseTest, CanSetConfigToExecNet) {
for (const auto& confItem : configuration) {
config.insert({confItem.first, InferenceEngine::Parameter(confItem.second)});
}
ASSERT_NO_THROW(execNet.set_config(config));
EXPECT_NO_THROW(execNet.set_config(config));
}
TEST_P(OVExecutableNetworkBaseTest, CanSetConfigToExecNetWithIncorrectConfig) {
@ -104,7 +127,7 @@ TEST_P(OVExecutableNetworkBaseTest, CanSetConfigToExecNetWithIncorrectConfig) {
for (const auto& confItem : incorrectConfig) {
config.insert({confItem.first, InferenceEngine::Parameter(confItem.second)});
}
ASSERT_ANY_THROW(execNet.set_config(config));
EXPECT_ANY_THROW(execNet.set_config(config));
}
TEST_P(OVExecutableNetworkBaseTest, CanSetConfigToExecNetAndCheckConfigAndCheck) {
@ -116,37 +139,37 @@ TEST_P(OVExecutableNetworkBaseTest, CanSetConfigToExecNetAndCheckConfigAndCheck)
execNet.set_config(config);
for (const auto& configItem : configuration) {
InferenceEngine::Parameter param;
ASSERT_NO_THROW(param = execNet.get_config(configItem.first));
ASSERT_FALSE(param.empty());
ASSERT_EQ(param, InferenceEngine::Parameter(configItem.second));
EXPECT_NO_THROW(param = execNet.get_config(configItem.first));
EXPECT_FALSE(param.empty());
EXPECT_EQ(param, InferenceEngine::Parameter(configItem.second));
}
}
TEST_P(OVExecutableNetworkBaseTest, CanCreateTwoExeNetworks) {
std::vector<ov::runtime::ExecutableNetwork> vec;
for (auto i = 0; i < 2; i++) {
ASSERT_NO_THROW(vec.push_back(core->compile_model(function, targetDevice, configuration)));
ASSERT_NE(nullptr, function);
EXPECT_NO_THROW(vec.push_back(core->compile_model(function, targetDevice, configuration)));
EXPECT_NE(nullptr, function);
}
}
TEST_P(OVExecutableNetworkBaseTest, CanCreateTwoExeNetworksAndCheckFunction) {
std::vector<ov::runtime::ExecutableNetwork> vec;
for (auto i = 0; i < 2; i++) {
ASSERT_NO_THROW(vec.push_back(core->compile_model(function, targetDevice, configuration)));
ASSERT_NE(nullptr, vec[i].get_runtime_function());
ASSERT_NE(vec.begin()->get_runtime_function(), vec[i].get_runtime_function());
EXPECT_NO_THROW(vec.push_back(core->compile_model(function, targetDevice, configuration)));
EXPECT_NE(nullptr, vec[i].get_runtime_function());
EXPECT_NE(vec.begin()->get_runtime_function(), vec[i].get_runtime_function());
}
}
TEST_P(OVExecutableNetworkBaseTest, CanGetInputsInfo) {
auto execNet = core->compile_model(function, targetDevice, configuration);
ASSERT_NO_THROW(auto inInfo = execNet.inputs());
EXPECT_NO_THROW(auto inInfo = execNet.inputs());
}
TEST_P(OVExecutableNetworkBaseTest, CanGetOutputsInfo) {
auto execNet = core->compile_model(function, targetDevice, configuration);
ASSERT_NO_THROW(auto outInfo = execNet.outputs());
EXPECT_NO_THROW(auto outInfo = execNet.outputs());
}
TEST_P(OVExecutableNetworkBaseTest, CanGetInputsInfoAndCheck) {
@ -158,7 +181,8 @@ TEST_P(OVExecutableNetworkBaseTest, CanGetInputsInfoAndCheck) {
}
auto params = function->get_parameters();
for (const auto& param : params) {
ASSERT_NE(std::find(paramVec.begin(), paramVec.end(), *param->get_output_tensor(0).get_names().begin()), paramVec.end());
EXPECT_NE(std::find(paramVec.begin(), paramVec.end(), *param->get_output_tensor(0).get_names().begin()),
paramVec.end());
}
}
@ -171,7 +195,8 @@ TEST_P(OVExecutableNetworkBaseTest, CanGetOutputsInfoAndCheck) {
}
auto results = function->get_results();
for (const auto& param : results) {
ASSERT_NE(std::find(resVec.begin(), resVec.end(), *param->get_output_tensor(0).get_names().begin()), resVec.end());
EXPECT_NE(std::find(resVec.begin(), resVec.end(), *param->get_output_tensor(0).get_names().begin()),
resVec.end());
}
}
@ -179,7 +204,7 @@ TEST_P(OVExecutableNetworkBaseTest, CheckExecGraphInfoBeforeExecution) {
std::shared_ptr<const ov::Function> execGraph;
// Load CNNNetwork to target plugins
auto execNet = core->compile_model(function, targetDevice, configuration);
ASSERT_NO_THROW(execGraph = execNet.get_runtime_function());
EXPECT_NO_THROW(execGraph = execNet.get_runtime_function());
std::map<std::string, int> originalLayersMap;
for (const auto& layer : function->get_ops()) {
originalLayersMap[layer->get_friendly_name()] = 0;
@ -187,7 +212,7 @@ TEST_P(OVExecutableNetworkBaseTest, CheckExecGraphInfoBeforeExecution) {
int constCnt = 0;
std::shared_ptr<const ngraph::Function> getFunction = std::dynamic_pointer_cast<const ngraph::Function>(execGraph);
ASSERT_NE(getFunction, nullptr);
EXPECT_NE(getFunction, nullptr);
for (const auto& op : getFunction->get_ops()) {
const ov::RTMap& rtInfo = op->get_rt_info();
@ -202,7 +227,7 @@ TEST_P(OVExecutableNetworkBaseTest, CheckExecGraphInfoBeforeExecution) {
};
// Each layer from the execGraphInfo network must have PM data option set
ASSERT_EQ("not_executed", getExecValue(ExecGraphInfoSerialization::PERF_COUNTER));
EXPECT_EQ("not_executed", getExecValue(ExecGraphInfoSerialization::PERF_COUNTER));
// Parse origin layer names (fused/merged layers) from the executable graph
// and compare with layers from the original model
auto origFromExecLayer = getExecValue(ExecGraphInfoSerialization::ORIGINAL_NAMES);
@ -212,7 +237,7 @@ TEST_P(OVExecutableNetworkBaseTest, CheckExecGraphInfoBeforeExecution) {
auto origFromExecLayerSep = CommonTestUtils::splitStringByDelimiter(origFromExecLayer);
std::for_each(origFromExecLayerSep.begin(), origFromExecLayerSep.end(), [&](const std::string& op) {
auto origLayer = originalLayersMap.find(op);
ASSERT_NE(originalLayersMap.end(), origLayer) << op;
EXPECT_NE(originalLayersMap.end(), origLayer) << op;
origLayer->second++;
});
}
@ -223,7 +248,7 @@ TEST_P(OVExecutableNetworkBaseTest, CheckExecGraphInfoBeforeExecution) {
if ((layer.second == 0) && (constCnt > 0)) {
constCnt--;
} else {
ASSERT_GE(layer.second, 0);
EXPECT_GE(layer.second, 0);
}
}
}
@ -232,7 +257,7 @@ TEST_P(OVExecutableNetworkBaseTest, CheckExecGraphInfoAfterExecution) {
std::shared_ptr<const ov::Function> execGraph;
// Load CNNNetwork to target plugins
auto execNet = core->compile_model(function, targetDevice, configuration);
ASSERT_NO_THROW(execGraph = execNet.get_runtime_function());
EXPECT_NO_THROW(execGraph = execNet.get_runtime_function());
std::map<std::string, int> originalLayersMap;
for (const auto& layer : function->get_ops()) {
originalLayersMap[layer->get_friendly_name()] = 0;
@ -241,7 +266,7 @@ TEST_P(OVExecutableNetworkBaseTest, CheckExecGraphInfoAfterExecution) {
// Store all the layers from the executable graph information represented as CNNNetwork
bool hasOpWithValidTime = false;
auto getFunction = std::dynamic_pointer_cast<const ngraph::Function>(execGraph);
ASSERT_NE(nullptr, getFunction);
EXPECT_NE(nullptr, getFunction);
for (const auto& op : getFunction->get_ops()) {
const auto& rtInfo = op->get_rt_info();
@ -259,9 +284,10 @@ TEST_P(OVExecutableNetworkBaseTest, CheckExecGraphInfoAfterExecution) {
try {
float x = static_cast<float>(std::atof(getExecValue(ExecGraphInfoSerialization::PERF_COUNTER).c_str()));
std::cout << "TIME: " << x << std::endl;
ASSERT_GE(x, 0.0f);
EXPECT_GE(x, 0.0f);
hasOpWithValidTime = true;
} catch (std::exception &) {}
} catch (std::exception&) {
}
// Parse origin layer names (fused/merged layers) from the executable graph
// and compare with layers from the original model
@ -272,20 +298,20 @@ TEST_P(OVExecutableNetworkBaseTest, CheckExecGraphInfoAfterExecution) {
} else {
std::for_each(origFromExecLayerSep.begin(), origFromExecLayerSep.end(), [&](const std::string& layer) {
auto origLayer = originalLayersMap.find(layer);
ASSERT_NE(originalLayersMap.end(), origLayer) << layer;
EXPECT_NE(originalLayersMap.end(), origLayer) << layer;
origLayer->second++;
});
}
}
ASSERT_TRUE(hasOpWithValidTime);
EXPECT_TRUE(hasOpWithValidTime);
// All layers from the original IR must be present within ExecGraphInfo
for (auto& layer : originalLayersMap) {
if ((layer.second == 0) && (constCnt > 0)) {
constCnt--;
} else {
ASSERT_GE(layer.second, 0);
EXPECT_GE(layer.second, 0);
}
}
}
@ -295,10 +321,10 @@ TEST_P(OVExecutableNetworkBaseTest, canExport) {
std::string modelName = GetTestName().substr(0, CommonTestUtils::maxFileNameLength) + "_" + ts;
auto execNet = core->compile_model(function, targetDevice, configuration);
std::ofstream out(modelName, std::ios::out);
ASSERT_NO_THROW(execNet.export_model(out));
EXPECT_NO_THROW(execNet.export_model(out));
out.close();
ASSERT_TRUE(CommonTestUtils::fileExists(modelName + ".xml"));
ASSERT_TRUE(CommonTestUtils::fileExists(modelName + ".bin"));
EXPECT_TRUE(CommonTestUtils::fileExists(modelName + ".xml"));
EXPECT_TRUE(CommonTestUtils::fileExists(modelName + ".bin"));
CommonTestUtils::removeIRFiles(modelName + ".xml", modelName + ".bin");
}
@ -314,12 +340,25 @@ TEST_P(OVExecutableNetworkBaseTest, getInputFromFunctionWithSingleInput) {
ov::runtime::ExecutableNetwork execNet;
execNet = core->compile_model(function, targetDevice, configuration);
ASSERT_EQ(function->inputs().size(), 1);
EXPECT_EQ(function->inputs().size(), 1);
EXPECT_EQ(function->inputs().size(), execNet.inputs().size());
EXPECT_NO_THROW(execNet.input());
EXPECT_EQ(function->input().get_tensor().get_names(), execNet.input().get_tensor().get_names());
EXPECT_EQ(function->input().get_tensor().get_partial_shape(), execNet.input().get_tensor().get_partial_shape());
EXPECT_EQ(function->input().get_tensor().get_element_type(), execNet.input().get_tensor().get_element_type());
ov::runtime::InferRequest request = execNet.create_infer_request();
ov::runtime::Tensor tensor1, tensor2;
EXPECT_NO_THROW(tensor1 = request.get_tensor(execNet.input()));
EXPECT_NO_THROW(tensor2 = request.get_tensor(function->input()));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_tensor(execNet.input().get_any_name()));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_tensor(function->input().get_any_name()));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_input_tensor(0));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
}
TEST_P(OVExecutableNetworkBaseTest, getOutputFromFunctionWithSingleInput) {
@ -334,6 +373,18 @@ TEST_P(OVExecutableNetworkBaseTest, getOutputFromFunctionWithSingleInput) {
EXPECT_EQ(function->output().get_tensor().get_names(), execNet.output().get_tensor().get_names());
EXPECT_EQ(function->output().get_tensor().get_partial_shape(), execNet.output().get_tensor().get_partial_shape());
EXPECT_EQ(function->output().get_tensor().get_element_type(), execNet.output().get_tensor().get_element_type());
ov::runtime::InferRequest request = execNet.create_infer_request();
ov::runtime::Tensor tensor1, tensor2;
EXPECT_NO_THROW(tensor1 = request.get_tensor(execNet.output()));
EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output()));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_tensor(execNet.output().get_any_name()));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output().get_any_name()));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_output_tensor(0));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
}
TEST_P(OVExecutableNetworkBaseTest, getInputsFromFunctionWithSeveralInputs) {
@ -361,10 +412,10 @@ TEST_P(OVExecutableNetworkBaseTest, getInputsFromFunctionWithSeveralInputs) {
result2->set_friendly_name("result2");
function = std::make_shared<ngraph::Function>(ngraph::ResultVector{result1, result2},
ngraph::ParameterVector{param1, param2});
function->set_friendly_name("SingleRuLU");
function->set_friendly_name("SimpleReLU");
}
execNet = core->compile_model(function, targetDevice, configuration);
ASSERT_EQ(function->inputs().size(), 2);
EXPECT_EQ(function->inputs().size(), 2);
EXPECT_EQ(function->inputs().size(), execNet.inputs().size());
EXPECT_THROW(execNet.input(), ov::Exception);
EXPECT_EQ(function->input(0).get_tensor().get_names(), execNet.input(0).get_tensor().get_names());
@ -377,6 +428,34 @@ TEST_P(OVExecutableNetworkBaseTest, getInputsFromFunctionWithSeveralInputs) {
EXPECT_NE(function->input(1).get_node(), function->input("data1").get_node());
EXPECT_EQ(function->input(1).get_node(), function->input("data2").get_node());
EXPECT_NE(function->input(0).get_node(), function->input("data2").get_node());
ov::runtime::InferRequest request = execNet.create_infer_request();
ov::runtime::Tensor tensor1, tensor2;
EXPECT_NO_THROW(tensor1 = request.get_tensor(execNet.input(0)));
EXPECT_NO_THROW(tensor2 = request.get_tensor(function->input(0)));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_tensor(execNet.input(0).get_any_name()));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_tensor(function->input(0).get_any_name()));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_input_tensor(0));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor1 = request.get_tensor(execNet.input(1)));
try {
// To avoid case with remote tensors
tensor1.data();
EXPECT_FALSE(compareTensors(tensor1, tensor2));
} catch (const ov::Exception&) {
}
EXPECT_NO_THROW(tensor2 = request.get_tensor(function->input(1)));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_tensor(execNet.input(1).get_any_name()));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_tensor(function->input(1).get_any_name()));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_input_tensor(1));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
}
TEST_P(OVExecutableNetworkBaseTest, getOutputsFromFunctionWithSeveralOutputs) {
@ -404,10 +483,10 @@ TEST_P(OVExecutableNetworkBaseTest, getOutputsFromFunctionWithSeveralOutputs) {
result2->set_friendly_name("result2");
function = std::make_shared<ngraph::Function>(ngraph::ResultVector{result1, result2},
ngraph::ParameterVector{param1, param2});
function->set_friendly_name("SingleRuLU");
function->set_friendly_name("SimpleReLU");
}
execNet = core->compile_model(function, targetDevice, configuration);
ASSERT_EQ(function->outputs().size(), 2);
EXPECT_EQ(function->outputs().size(), 2);
EXPECT_EQ(function->outputs().size(), execNet.outputs().size());
EXPECT_THROW(execNet.output(), ov::Exception);
EXPECT_EQ(function->output(0).get_tensor().get_names(), execNet.output(0).get_tensor().get_names());
@ -420,12 +499,107 @@ TEST_P(OVExecutableNetworkBaseTest, getOutputsFromFunctionWithSeveralOutputs) {
EXPECT_NE(function->output(1).get_node(), function->output("relu").get_node());
EXPECT_EQ(function->output(1).get_node(), function->output("concat").get_node());
EXPECT_NE(function->output(0).get_node(), function->output("concat").get_node());
ov::runtime::InferRequest request = execNet.create_infer_request();
ov::runtime::Tensor tensor1, tensor2;
EXPECT_NO_THROW(tensor1 = request.get_tensor(execNet.output(0)));
EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output(0)));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_tensor(execNet.output(0).get_any_name()));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output(0).get_any_name()));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_output_tensor(0));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor1 = request.get_tensor(execNet.output(1)));
try {
// To avoid case with remote tensors
tensor1.data();
EXPECT_FALSE(compareTensors(tensor1, tensor2));
} catch (const ov::Exception&) {
}
EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output(1)));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_tensor(execNet.output(1).get_any_name()));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output(1).get_any_name()));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_output_tensor(1));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
}
TEST_P(OVExecutableNetworkBaseTest, getOutputsFromSplitFunctionWithSeveralOutputs) {
// Skip test according to plugin specific disabledTestPatterns() (if any)
SKIP_IF_CURRENT_TEST_IS_DISABLED()
ov::runtime::ExecutableNetwork execNet;
// Create simple function
{
auto param1 = std::make_shared<ov::opset8::Parameter>(ov::element::Type_t::f32, ngraph::Shape({1, 4, 24, 24}));
param1->set_friendly_name("param1");
param1->output(0).get_tensor().set_names({"data1"});
auto axis_node = ov::opset8::Constant::create(element::i64, Shape{}, {1});
auto split = std::make_shared<ov::opset8::Split>(param1, axis_node, 2);
split->set_friendly_name("split");
split->output(0).get_tensor().set_names({"tensor_split_1"});
split->output(1).get_tensor().set_names({"tensor_split_2"});
auto result1 = std::make_shared<ov::opset8::Result>(split->output(0));
result1->set_friendly_name("result1");
auto result2 = std::make_shared<ov::opset8::Result>(split->output(1));
result2->set_friendly_name("result2");
function = std::make_shared<ngraph::Function>(ngraph::ResultVector{result1, result2},
ngraph::ParameterVector{param1});
function->set_friendly_name("SingleSplit");
}
execNet = core->compile_model(function, targetDevice, configuration);
EXPECT_EQ(function->outputs().size(), 2);
EXPECT_EQ(function->outputs().size(), execNet.outputs().size());
EXPECT_THROW(execNet.output(), ov::Exception);
EXPECT_EQ(function->output(0).get_tensor().get_names(), execNet.output(0).get_tensor().get_names());
EXPECT_EQ(function->output(0).get_tensor().get_partial_shape(), execNet.output(0).get_tensor().get_partial_shape());
EXPECT_EQ(function->output(0).get_tensor().get_element_type(), execNet.output(0).get_tensor().get_element_type());
EXPECT_EQ(function->output(1).get_tensor().get_names(), execNet.output(1).get_tensor().get_names());
EXPECT_EQ(function->output(1).get_tensor().get_partial_shape(), execNet.output(1).get_tensor().get_partial_shape());
EXPECT_EQ(function->output(1).get_tensor().get_element_type(), execNet.output(1).get_tensor().get_element_type());
EXPECT_EQ(function->output(0).get_node(), function->output("tensor_split_1").get_node());
EXPECT_NE(function->output(1).get_node(), function->output("tensor_split_1").get_node());
EXPECT_EQ(function->output(1).get_node(), function->output("tensor_split_2").get_node());
EXPECT_NE(function->output(0).get_node(), function->output("tensor_split_2").get_node());
ov::runtime::InferRequest request = execNet.create_infer_request();
ov::runtime::Tensor tensor1, tensor2;
EXPECT_NO_THROW(tensor1 = request.get_tensor(execNet.output(0)));
EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output(0)));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_tensor(execNet.output(0).get_any_name()));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output(0).get_any_name()));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_output_tensor(0));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor1 = request.get_tensor(execNet.output(1)));
try {
// To avoid case with remote tensors
tensor1.data();
EXPECT_FALSE(compareTensors(tensor1, tensor2));
} catch (const ov::Exception&) {
}
EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output(1)));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_tensor(execNet.output(1).get_any_name()));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_tensor(function->output(1).get_any_name()));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
EXPECT_NO_THROW(tensor2 = request.get_output_tensor(1));
EXPECT_TRUE(compareTensors(tensor1, tensor2));
}
// Load correct network to Plugin to get executable network
TEST_P(OVExecutableNetworkBaseTest, precisionsAsInOriginalFunction) {
ov::runtime::ExecutableNetwork execNet;
ASSERT_NO_THROW(execNet = core->compile_model(function, targetDevice, configuration));
EXPECT_NO_THROW(execNet = core->compile_model(function, targetDevice, configuration));
EXPECT_EQ(function->get_parameters().size(), execNet.inputs().size());
auto ref_parameter = function->get_parameters().back();
@ -449,7 +623,7 @@ TEST_P(OVExecutableNetworkBaseTest, precisionsAsInOriginalIR) {
ov::pass::Serialize(m_out_xml_path_1, m_out_bin_path_1).run_on_function(function);
ov::runtime::ExecutableNetwork execNet;
ASSERT_NO_THROW(execNet = core->compile_model(m_out_xml_path_1, targetDevice, configuration));
EXPECT_NO_THROW(execNet = core->compile_model(m_out_xml_path_1, targetDevice, configuration));
EXPECT_EQ(function->get_parameters().size(), execNet.inputs().size());
auto ref_parameter = function->get_parameters().back();

3
inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/subgraph_builders.hpp
View File

@ -51,7 +51,7 @@ inline std::shared_ptr<ngraph::Function> makeSplitConvConcat(std::vector<size_t>
ngraph::element::Type_t ngPrc = ngraph::element::Type_t::f32) {
auto params = ngraph::builder::makeParams(ngPrc, {inputShape});
params.front()->set_friendly_name("Param_1");
params.front()->get_output_tensor(0).set_names({"Tensor_1"});
params.front()->get_output_tensor(0).set_names({"input_tensor"});
auto split = ngraph::builder::makeSplit(params[0], ngPrc, 2, 1);
auto conv1 = ngraph::builder::makeConvolution(split->output(0), ngPrc, {3, 3}, {1, 1}, {0, 0}, {0, 0}, {1, 1},
@ -63,6 +63,7 @@ inline std::shared_ptr<ngraph::Function> makeSplitConvConcat(std::vector<size_t>
auto relu2 = std::make_shared<ngraph::opset1::Relu>(conv2);
auto concat = std::make_shared<ngraph::opset1::Concat>(ngraph::OutputVector{relu1->output(0), relu2->output(0)}, 1);
concat->get_output_tensor(0).set_names({"concat_tensor"});
ngraph::ResultVector results{std::make_shared<ngraph::opset1::Result>(concat)};
std::shared_ptr<ngraph::Function> fnPtr = std::make_shared<ngraph::Function>(results, params);
fnPtr->set_friendly_name("SplitConvConcat");

4
ngraph/core/src/node_output.cpp
View File

@ -84,7 +84,7 @@ const std::unordered_set<std::string>& Output<Node>::get_names() const {
}
std::string Output<Node>::get_any_name() const {
return m_node->m_outputs.at(m_index).get_tensor_ptr()->get_any_name();
return get_tensor().get_any_name();
}
void Output<Node>::set_names(const std::unordered_set<std::string>& names) {
@ -100,7 +100,7 @@ const std::unordered_set<std::string>& Output<const Node>::get_names() const {
}
std::string Output<const Node>::get_any_name() const {
return m_node->m_outputs.at(m_index).get_tensor_ptr()->get_any_name();
return get_tensor().get_any_name();
}
bool Output<Node>::operator==(const Output& other) const {

91
ngraph/test/util/all_close.cpp Normal file
View File

@ -0,0 +1,91 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "all_close.hpp"
::testing::AssertionResult ov::test::all_close(const ov::runtime::Tensor& a,
const ov::runtime::Tensor& b,
float rtol,
float atol) {
if (a.get_element_type() != b.get_element_type()) {
return ::testing::AssertionFailure() << "Cannot compare tensors with different element types";
}
switch (a.get_element_type()) {
case ov::element::u8:
return all_close<ov::element_type_traits<ov::element::u8>::value_type>(
a,
b,
static_cast<ov::element_type_traits<ov::element::u8>::value_type>(rtol),
static_cast<ov::element_type_traits<ov::element::u8>::value_type>(atol));
case ov::element::u16:
return all_close<ov::element_type_traits<ov::element::u16>::value_type>(
a,
b,
static_cast<ov::element_type_traits<ov::element::u16>::value_type>(rtol),
static_cast<ov::element_type_traits<ov::element::u16>::value_type>(atol));
case ov::element::u32:
return all_close<ov::element_type_traits<ov::element::u32>::value_type>(
a,
b,
static_cast<ov::element_type_traits<ov::element::u32>::value_type>(rtol),
static_cast<ov::element_type_traits<ov::element::u32>::value_type>(atol));
case ov::element::u64:
return all_close<ov::element_type_traits<ov::element::u64>::value_type>(
a,
b,
static_cast<ov::element_type_traits<ov::element::u64>::value_type>(rtol),
static_cast<ov::element_type_traits<ov::element::u64>::value_type>(atol));
case ov::element::i8:
return all_close<ov::element_type_traits<ov::element::i8>::value_type>(
a,
b,
static_cast<ov::element_type_traits<ov::element::i8>::value_type>(rtol),
static_cast<ov::element_type_traits<ov::element::i8>::value_type>(atol));
case ov::element::i16:
return all_close<ov::element_type_traits<ov::element::i16>::value_type>(
a,
b,
static_cast<ov::element_type_traits<ov::element::i16>::value_type>(rtol),
static_cast<ov::element_type_traits<ov::element::i16>::value_type>(atol));
case ov::element::i32:
return all_close<ov::element_type_traits<ov::element::i32>::value_type>(
a,
b,
static_cast<ov::element_type_traits<ov::element::i32>::value_type>(rtol),
static_cast<ov::element_type_traits<ov::element::i32>::value_type>(atol));
case ov::element::i64:
return all_close<ov::element_type_traits<ov::element::i64>::value_type>(
a,
b,
static_cast<ov::element_type_traits<ov::element::i64>::value_type>(rtol),
static_cast<ov::element_type_traits<ov::element::i64>::value_type>(atol));
// case ov::element::bf16:
// return all_close<ov::element_type_traits<ov::element::bf16>::value_type>(
// a,
// b,
// static_cast<ov::element_type_traits<ov::element::bf16>::value_type>(rtol),
// static_cast<ov::element_type_traits<ov::element::bf16>::value_type>(atol));
// case ov::element::f16:
// return all_close<ov::element_type_traits<ov::element::f16>::value_type>(
// a,
// b,
// static_cast<ov::element_type_traits<ov::element::f16>::value_type>(rtol),
// static_cast<ov::element_type_traits<ov::element::f16>::value_type>(atol));
case ov::element::f32:
return all_close<ov::element_type_traits<ov::element::f32>::value_type>(
a,
b,
static_cast<ov::element_type_traits<ov::element::f32>::value_type>(rtol),
static_cast<ov::element_type_traits<ov::element::f32>::value_type>(atol));
case ov::element::f64:
return all_close<ov::element_type_traits<ov::element::f64>::value_type>(
a,
b,
static_cast<ov::element_type_traits<ov::element::f64>::value_type>(rtol),
static_cast<ov::element_type_traits<ov::element::f64>::value_type>(atol));
default:
return ::testing::AssertionFailure()
<< "Cannot compare tensors with unsupported element type: " << a.get_element_type();
}
}

33
ngraph/test/util/all_close.hpp
View File

@ -11,6 +11,7 @@
#include "gtest/gtest.h"
#include "ngraph/pass/manager.hpp"
#include "ngraph/type/element_type.hpp"
#include "openvino/runtime/tensor.hpp"
namespace ngraph {
namespace test {
@ -116,3 +117,35 @@ template <typename T>
}
} // namespace test
} // namespace ngraph
namespace ov {
namespace test {
/// \brief Same as numpy.allclose
/// \param a First tensor to compare
/// \param b Second tensor to compare
/// \param rtol Relative tolerance
/// \param atol Absolute tolerance
/// Returns true if shapes match and for all elements, |a_i-b_i| <= atol + rtol*|b_i|.
template <typename T>
::testing::AssertionResult all_close(const ov::runtime::Tensor& a,
const ov::runtime::Tensor& b,
T rtol = 1e-5f,
T atol = 1e-8f) {
auto a_t = std::make_shared<ngraph::runtime::HostTensor>(a.get_element_type(), a.get_shape(), a.data());
auto b_t = std::make_shared<ngraph::runtime::HostTensor>(b.get_element_type(), b.get_shape(), b.data());
return ngraph::test::all_close(a_t, b_t, rtol, atol);
}
/// \brief Same as numpy.allclose
/// \param a First tensor to compare
/// \param b Second tensor to compare
/// \param rtol Relative tolerance
/// \param atol Absolute tolerance
/// Returns true if shapes match and for all elements, |a_i-b_i| <= atol + rtol*|b_i|.
::testing::AssertionResult all_close(const ov::runtime::Tensor& a,
const ov::runtime::Tensor& b,
float rtol = 1e-5f,
float atol = 1e-8f);
} // namespace test
} // namespace ov