IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CPU] Migration to API 2.0 (#9456)

Browse Source
This commit is contained in:
Maxim Andronov 2022-01-19 14:45:28 +03:00 committed by GitHub
parent 81685c8d21
commit 204ca99dac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 854 additions and 361 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/plugins/intel_cpu/src/mkldnn_async_infer_request.cpp
View File

@ -9,7 +9,7 @@ MKLDNNPlugin::MKLDNNAsyncInferRequest::MKLDNNAsyncInferRequest(const InferenceEn
const InferenceEngine::ITaskExecutor::Ptr& taskExecutor,
const InferenceEngine::ITaskExecutor::Ptr& callbackExecutor)
: InferenceEngine::AsyncInferRequestThreadSafeDefault(inferRequest, taskExecutor, callbackExecutor) {
static_cast<MKLDNNInferRequest*>(inferRequest.get())->SetAsyncRequest(this);
static_cast<MKLDNNInferRequestBase*>(inferRequest.get())->SetAsyncRequest(this);
}
MKLDNNPlugin::MKLDNNAsyncInferRequest::~MKLDNNAsyncInferRequest() {

2
src/plugins/intel_cpu/src/mkldnn_exec_network.cpp
View File

@ -43,7 +43,7 @@ MKLDNNExecNetwork::CreateInferRequestImpl(const std::vector<std::shared_ptr<cons
InferenceEngine::IInferRequestInternal::Ptr
MKLDNNExecNetwork::CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs) {
return std::make_shared<MKLDNNInferRequest>(networkInputs, networkOutputs, std::static_pointer_cast<MKLDNNExecNetwork>(shared_from_this()));
return std::make_shared<MKLDNNLegacyInferRequest>(networkInputs, networkOutputs, std::static_pointer_cast<MKLDNNExecNetwork>(shared_from_this()));
}
struct ImmediateSerialExecutor : public ITaskExecutor {

2
src/plugins/intel_cpu/src/mkldnn_exec_network.h
View File

@ -47,7 +47,7 @@ public:
void Export(std::ostream& modelStream) override;
protected:
friend class MKLDNNInferRequest;
friend class MKLDNNInferRequestBase;
MKLDNNExtensionManager::Ptr extensionManager;
std::vector<InferenceEngine::IVariableStateInternal::Ptr> memoryStates;
const InferenceEngine::CNNNetwork _network;

2
src/plugins/intel_cpu/src/mkldnn_graph.cpp
View File

@ -847,7 +847,7 @@ inline void MKLDNNGraph::ExecuteNode(const MKLDNNNodePtr& node, const mkldnn::st
}
}
void MKLDNNGraph::Infer(MKLDNNInferRequest* request, int batch) {
void MKLDNNGraph::Infer(MKLDNNInferRequestBase* request, int batch) {
if (!IsReady()) {
IE_THROW() << "Wrong state. Topology is not ready.";
}

7
src/plugins/intel_cpu/src/mkldnn_graph.h
View File

@ -18,7 +18,7 @@
#include <atomic>
namespace MKLDNNPlugin {
class MKLDNNInferRequest;
class MKLDNNInferRequestBase;
class MKLDNNGraph {
public:
typedef std::shared_ptr<MKLDNNGraph> Ptr;
@ -57,7 +57,7 @@ public:
void PushInputData(const std::string& name, const InferenceEngine::Blob::Ptr &in);
void PullOutputData(InferenceEngine::BlobMap &out);
void Infer(MKLDNNInferRequest* request = nullptr, int batch = -1);
void Infer(MKLDNNInferRequestBase* request = nullptr, int batch = -1);
const std::vector<MKLDNNNodePtr>& GetNodes() const {
return graphNodes;
@ -233,8 +233,9 @@ protected:
void ExecuteNode(const MKLDNNNodePtr& node, const mkldnn::stream& stream) const;
void ExecuteConstantNodesOnly() const;
friend class MKLDNNInferRequestBase;
friend class MKLDNNLegacyInferRequest;
friend class MKLDNNInferRequest;
friend class MKLDNNGraphlessInferRequest;
friend std::shared_ptr<ngraph::Function> dump_graph_as_ie_ngraph_net(const MKLDNNGraph &graph);
private:

850
src/plugins/intel_cpu/src/mkldnn_infer_request.cpp
View File

@ -23,24 +23,10 @@
#include "utils/general_utils.h"
#include "utils/cpu_utils.hpp"
#include "memory_desc/dnnl_blocked_memory_desc.h"
#include <transformations/utils/utils.hpp>
#include <ie_ngraph_utils.hpp>
MKLDNNPlugin::MKLDNNInferRequest::MKLDNNInferRequest(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs,
MKLDNNExecNetwork::Ptr execNetwork_)
: IInferRequestInternal(networkInputs, networkOutputs)
, execNetwork(execNetwork_) {
CreateInferRequest();
}
MKLDNNPlugin::MKLDNNInferRequest::MKLDNNInferRequest(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs,
MKLDNNExecNetwork::Ptr execNetwork_)
: IInferRequestInternal(inputs, outputs)
, execNetwork(execNetwork_) {
CreateInferRequest();
}
void MKLDNNPlugin::MKLDNNInferRequest::CreateInferRequest() {
void MKLDNNPlugin::MKLDNNInferRequestBase::CreateInferRequest() {
auto id = (execNetwork->_numRequests)++;
profilingTask = openvino::itt::handle("MKLDNN_INFER_" + execNetwork->_name + "_" + std::to_string(id));
@ -48,14 +34,7 @@ void MKLDNNPlugin::MKLDNNInferRequest::CreateInferRequest() {
IE_THROW() << "No graph was found";
graph = &(execNetwork->GetGraph()._graph);
// Allocate all input blobs if shape is static, delay allocation otherwise
for (const auto& it : _networkInputs) {
MKLDNNInferRequest::GetBlob(it.first);
}
// Allocate all output blobs if shape is static, delay allocation otherwise
for (const auto& it : _networkOutputs) {
MKLDNNInferRequest::GetBlob(it.first);
}
initBlobs();
// Save all MemoryLayer data tensors. Will use insight about mechanics
// of MemoryLayer implementation. It uses output edge of MemoryLayer
@ -79,11 +58,11 @@ void MKLDNNPlugin::MKLDNNInferRequest::CreateInferRequest() {
}
}
MKLDNNPlugin::MKLDNNInferRequest::~MKLDNNInferRequest() {
MKLDNNPlugin::MKLDNNInferRequestBase::~MKLDNNInferRequestBase() {
--(execNetwork->_numRequests);
}
void MKLDNNPlugin::MKLDNNInferRequest::pushInput(const std::string& inputName, InferenceEngine::Blob::Ptr& inputBlob, InferenceEngine::Precision inPrec) {
void MKLDNNPlugin::MKLDNNInferRequestBase::pushInput(const std::string& inputName, InferenceEngine::Blob::Ptr& inputBlob, InferenceEngine::Precision inPrec) {
auto& tensorDesc = inputBlob->getTensorDesc();
bool needConvert = inPrec != tensorDesc.getPrecision();
@ -110,36 +89,7 @@ void MKLDNNPlugin::MKLDNNInferRequest::pushInput(const std::string& inputName, I
graph->PushInputData(inputName, needConvert ? iconv : inputBlob);
}
void MKLDNNPlugin::MKLDNNInferRequest::PushInputData() {
for (auto input : _inputs) {
auto inputName = input.first;
if (!_networkInputs[inputName]) {
IE_THROW() << "Input blobs map contains not registered during IInferencePlugin::LoadNetwork blob with name " << inputName;
}
auto inputBlob = input.second;
auto& inputTensorDesc = inputBlob->getTensorDesc();
auto inPrec = inputTensorDesc.getPrecision();
if (graph->hasMeanImageFor(inputName) && one_of(inPrec, InferenceEngine::Precision::U8, InferenceEngine::Precision::BOOL)) {
inPrec = InferenceEngine::Precision::FP32;
} else {
inPrec = normalizeToSupportedPrecision(inPrec);
}
if (inPrec == InferenceEngine::Precision::UNSPECIFIED) {
IE_THROW() << "Unsupported input precision " << inputTensorDesc.getPrecision();
}
// User can initialize input via setBlob API using tensorDesc with default (ANY) layout.
// Currently IE doesn't specify behavior in such scenario, so we assume real layout is equal to the network input.
if (inputTensorDesc.getLayout() == InferenceEngine::ANY) {
inputTensorDesc.setLayout(_networkInputs[inputName]->getLayout());
}
pushInput(inputName, inputBlob, inPrec);
}
}
void MKLDNNPlugin::MKLDNNInferRequest::PushStates() {
void MKLDNNPlugin::MKLDNNInferRequestBase::PushStates() {
for (auto &node : graph->GetNodes()) {
if (node->getType() == MemoryInput) {
auto cur_node = dynamic_cast<MKLDNNMemoryInputNode*>(node.get());
@ -161,7 +111,7 @@ void MKLDNNPlugin::MKLDNNInferRequest::PushStates() {
}
}
void MKLDNNPlugin::MKLDNNInferRequest::PullStates() {
void MKLDNNPlugin::MKLDNNInferRequestBase::PullStates() {
for (auto &node : graph->GetNodes()) {
if (node->getType() == MemoryInput) {
auto cur_node = dynamic_cast<MKLDNNMemoryInputNode*>(node.get());
@ -183,7 +133,7 @@ void MKLDNNPlugin::MKLDNNInferRequest::PullStates() {
}
}
void MKLDNNPlugin::MKLDNNInferRequest::redefineMemoryForInputNodes() {
void MKLDNNPlugin::MKLDNNInferRequestBase::redefineMemoryForInputNodes() {
const auto cpuInputNodes = graph->GetInputNodesMap();
for (const auto &blob : _inputs) {
@ -196,7 +146,7 @@ void MKLDNNPlugin::MKLDNNInferRequest::redefineMemoryForInputNodes() {
}
}
void MKLDNNPlugin::MKLDNNInferRequest::InferImpl() {
void MKLDNNPlugin::MKLDNNInferRequestBase::InferImpl() {
using namespace openvino::itt;
OV_ITT_SCOPED_TASK(itt::domains::MKLDNNPlugin, profilingTask);
auto graphLock = execNetwork->GetGraph();
@ -230,7 +180,7 @@ void MKLDNNPlugin::MKLDNNInferRequest::InferImpl() {
graph->PullOutputData(_outputs);
}
std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> MKLDNNPlugin::MKLDNNInferRequest::GetPerformanceCounts() const {
std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> MKLDNNPlugin::MKLDNNInferRequestBase::GetPerformanceCounts() const {
if (!graph || !graph->IsReady())
IE_THROW() << "Graph is not ready!";
std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> perfMap;
@ -238,134 +188,187 @@ std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> MKLDNNPlugin:
return perfMap;
}
InferenceEngine::Blob::Ptr MKLDNNPlugin::MKLDNNInferRequest::GetBlob(const std::string& name) {
OV_ITT_SCOPED_TASK(itt::domains::MKLDNNPlugin, "GetBlob");
if (!graph || !graph->IsReady())
IE_THROW() << "Graph is not ready!";
InferenceEngine::Blob::Ptr data;
const auto &inMap = graph->inputNodesMap;
auto input = inMap.find(name);
if (input != inMap.end()) {
// ROI blob is returned only if it was set previously.
auto it = _preProcData.find(name);
if (it != _preProcData.end()) {
data = it->second->getRoiBlob();
return data;
}
if (_inputs.find(name) == _inputs.end()) {
if (_networkInputs.find(name) != _networkInputs.end()) {
InferenceEngine::TensorDesc desc = _networkInputs[name]->getTensorDesc();
bool isDynamic = input->second->isDynamicNode();
_inputs[name] = make_blob_with_precision(desc);
_inputs[name]->allocate();
if (!isDynamic &&
desc == MemoryDescUtils::convertToTensorDesc(graph->getInputNodeByName(name)->getChildEdgesAtPort(0)[0]->getMemory().getDesc()) &&
graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end() && !graph->getProperty().batchLimit) {
externalPtr[name] = _inputs[name]->buffer();
}
} else {
IE_THROW() << "Blob with name: " << name << " exists in MKLDNN graph, but absents in network inputs";
}
}
data = _inputs[name];
checkBlob(data, name, true);
// check if preprocess required, but still wasn't set
auto preProcessedInput = std::find_if(std::begin(_networkInputs), std::end(_networkInputs),
[&](const std::pair<std::string, InferenceEngine::InputInfo::Ptr>& pair) {
return pair.first == name;
});
if (preProcessedInput != std::end(_networkInputs)) {
InferenceEngine::InputInfo::Ptr foundInput;
InferenceEngine::DataPtr foundOutput;
if (!findInputAndOutputBlobByName(name, foundInput, foundOutput)) {
IE_THROW() << "Blob with name: " << name << " absents in network inputs";
}
if (preProcessingRequired(foundInput, data)) {
_preProcData.emplace(name, InferenceEngine::CreatePreprocDataHelper());
_preProcData[name]->isApplicable(data, _inputs[name]);
_preProcData[name]->setRoiBlob(data);
}
}
}
if (graph->hasOutputWithName(name)) {
if (auto outNode = graph->getOutputNodeByName(name)) {
if (_outputs.find(name) == _outputs.end()) {
if (_networkOutputs.find(name) != _networkOutputs.end()) {
bool isDynamic = outNode->isDynamicNode();
const auto &desc = outNode->getParentEdgesAtPort(0)[0]->getMemory().getDesc();
if (!data) {
InferenceEngine::TensorDesc desc = _networkOutputs[name]->getTensorDesc();
desc.setPrecision(normalizeToSupportedPrecision(desc.getPrecision()));
data = make_blob_with_precision(desc);
data->allocate();
} else {
const auto &expectedTensorDesc = isDynamic ? InferenceEngine::TensorDesc(desc.getPrecision(),
InferenceEngine::TensorDesc::getLayoutByRank(
desc.getShape().getRank()))
: MemoryDescUtils::convertToTensorDesc(desc);
const auto &tensorDesc = data->getTensorDesc();
if (expectedTensorDesc.getPrecision() != normalizeToSupportedPrecision(tensorDesc.getPrecision())) {
IE_THROW(ParameterMismatch)
<< "Network input and output use the same name: " << name << " but expect blobs with different precision: "
<< tensorDesc.getPrecision() << " for input and " << expectedTensorDesc.getPrecision()
<< " for output.";
}
if (expectedTensorDesc.getDims() != tensorDesc.getDims()) {
IE_THROW(ParameterMismatch) << "Network input and output use the same name: " << name << " but expect blobs with different shapes.";
}
if (tensorDesc.getLayout() != InferenceEngine::Layout::ANY && expectedTensorDesc.getLayout() != InferenceEngine::Layout::ANY) {
if (tensorDesc.getLayout() != expectedTensorDesc.getLayout() &&
!(tensorDesc.getLayout() == InferenceEngine::Layout::BLOCKED &&
InferenceEngine::TensorDesc(tensorDesc.getPrecision(),
tensorDesc.getDims(),
tensorDesc.getBlockingDesc()).getLayout() == expectedTensorDesc.getLayout())) {
IE_THROW(ParameterMismatch) << "Network input and output use the same name: " << name << " but expect blobs" <<
" with different layouts.";
}
if (expectedTensorDesc.getBlockingDesc() != tensorDesc.getBlockingDesc())
IE_THROW(ParameterMismatch) << "Network input and output use the same name: " << name
<< " but expect blobs with different blocking descriptors.";
}
}
_outputs[name] = data;
if (!isDynamic && !externalPtr.count(name) && data->getTensorDesc() == MemoryDescUtils::convertToTensorDesc(desc) &&
!graph->getProperty().batchLimit) {
externalPtr[name] = data->buffer();
}
} else {
IE_THROW() << "Blob with name: " << name << " exists in MKLDNN graph, but absents in network outputs";
}
}
data = _outputs[name];
if (!outNode->isDynamicNode())
checkBlob(data, name, false);
} else {
IE_THROW() << "Output node with name: " << name << " has not been created";
}
}
if (!data) {
IE_THROW() << "Cannot find blob with name: " << name;
}
return data;
static inline void changeEdgePtr(const MKLDNNPlugin::MKLDNNEdgePtr &edge, void *newPtr) {
edge->getMemory().GetPrimitivePtr()->set_data_handle(newPtr);
}
void MKLDNNPlugin::MKLDNNInferRequest::SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr &data) {
OV_ITT_SCOPED_TASK(itt::domains::MKLDNNPlugin, "SetBlob");
void MKLDNNPlugin::MKLDNNInferRequestBase::changeDefaultPtr() {
for (auto& it : externalPtr) {
const auto& inputNodesMap = graph->GetInputNodesMap();
auto input = inputNodesMap.find(it.first);
if (input != inputNodesMap.end()) {
MKLDNNNodePtr inputNodePtr = input->second;
if (inputNodePtr->getChildEdgeAt(0)->getMemory().GetPrimitive().get_data_handle() == it.second)
continue;
auto& childEdges = inputNodePtr->getChildEdges();
// Input cannot be in-place with other primitives
bool canBeInPlace = true;
for (auto& childEdge : childEdges) {
auto ce = childEdge.lock();
if (!ce)
IE_THROW() << "Node " << inputNodePtr->getName() << " contains empty child edge";
auto& child = ce->getChild();
if (child->isConstant()) {
canBeInPlace = false;
break;
}
if (child->getType() == Concatenation) {
auto concat = dynamic_cast<MKLDNNConcatNode*>(child.get());
if (concat && concat->isOptimized()) {
canBeInPlace = false;
break;
}
}
// Cannot be in-place before split because split is using different ptrs without offsets
if (child->getType() == Split) {
canBeInPlace = false;
break;
}
if (child->isInPlace()) {
canBeInPlace = false;
break;
}
auto& edges = child->getChildEdges();
for (auto& edge : edges) {
auto e = edge.lock();
if (!e)
IE_THROW() << "Node " << child->getName() << " contains empty child edge";
if (e->getMemory().GetPrimitive().get_data_handle() == ce->getMemory().GetPrimitive().get_data_handle()) {
canBeInPlace = false;
break;
}
}
if (!canBeInPlace)
break;
}
if (canBeInPlace) {
for (auto& edge : childEdges) {
auto e = edge.lock();
if (!e)
IE_THROW() << "Node " << inputNodePtr->getName() << " contains empty child edge";
changeEdgePtr(e, it.second);
}
}
continue;
}
const auto& outputNodesMap = graph->GetOutputNodesMap();
auto output = outputNodesMap.find(it.first);
if (output != outputNodesMap.end()) {
auto parentEdge = output->second->getParentEdgeAt(0);
if (parentEdge->getMemory().GetPrimitive().get_data_handle() == it.second)
continue;
bool canBeInPlace = true;
void* defaultPtr = parentEdge->getMemory().GetPrimitivePtr()->get_data_handle();
// Cannot be in-place after concat because concat is using different ptrs without offsets
auto parent = parentEdge->getParent();
MKLDNNNodePtr previousParent;
do {
previousParent = parent;
if (parent->getChildEdges().size() != 1 || parent->isConstant() || parent->isInPlace()) {
canBeInPlace = false;
break;
}
auto& parentEdges = parent->getParentEdges();
for (auto& edge : parentEdges) {
auto e = edge.lock();
if (!e)
IE_THROW() << "Node " << parent->getName() << " contains empty parent edge";
if (e->getMemory().GetPrimitivePtr()->get_data_handle() == defaultPtr) {
parent = e->getParent();
break;
}
}
} while (previousParent != parent);
if (canBeInPlace)
changeEdgePtr(parentEdge, it.second);
continue;
}
IE_THROW() << "Cannot find input/output blob: " << it.first;
}
}
void MKLDNNPlugin::MKLDNNInferRequestBase::SetBatch(int new_batch) {
if (!graph->getProperty().enableDynamicBatch)
IE_THROW() << "Dynamic batch is not enabled.";
if (new_batch < 1 || new_batch > graph->getProperty().batchLimit) {
IE_THROW() << "Invalid dynamic batch size " << new_batch <<
" for this request.";
}
m_curBatch = new_batch;
for (const auto& node : graph->GetNodes()) {
node->setDynamicBatchLim(new_batch);
}
}
std::vector<InferenceEngine::IVariableStateInternal::Ptr> MKLDNNPlugin::MKLDNNInferRequestBase::QueryState() {
return memoryStates;
}
void MKLDNNPlugin::MKLDNNInferRequestBase::SetAsyncRequest(MKLDNNAsyncInferRequest* asyncRequest) {
_asyncRequest = asyncRequest;
}
void MKLDNNPlugin::MKLDNNInferRequestBase::ThrowIfCanceled() const {
if (_asyncRequest != nullptr) {
_asyncRequest->ThrowIfCanceled();
}
}
InferenceEngine::Precision
MKLDNNPlugin::MKLDNNInferRequestBase::normToInputSupportedPrec(const std::pair<const std::string, InferenceEngine::Blob::Ptr>& input) const {
const auto& inputTensorDesc = input.second->getTensorDesc();
auto inPrec = inputTensorDesc.getPrecision();
if (graph->hasMeanImageFor(input.first) && one_of(inPrec, InferenceEngine::Precision::U8, InferenceEngine::Precision::BOOL)) {
inPrec = InferenceEngine::Precision::FP32;
} else {
inPrec = normalizeToSupportedPrecision(inPrec);
}
if (inPrec == InferenceEngine::Precision::UNSPECIFIED) {
IE_THROW() << "Unsupported input precision " << inputTensorDesc.getPrecision();
}
return inPrec;
}
/* ========================================== MKLDNNLegacyInferRequest ========================================== */
MKLDNNPlugin::MKLDNNLegacyInferRequest::MKLDNNLegacyInferRequest(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs,
std::shared_ptr<MKLDNNExecNetwork> execNetwork)
: MKLDNNInferRequestBase(networkInputs, networkOutputs, execNetwork) {
CreateInferRequest();
}
void MKLDNNPlugin::MKLDNNLegacyInferRequest::initBlobs() {
for (const auto& it : _networkInputs) {
MKLDNNLegacyInferRequest::GetBlob(it.first);
}
for (const auto& it : _networkOutputs) {
MKLDNNLegacyInferRequest::GetBlob(it.first);
}
}
// TODO [DS] : restore API 1.0 version, after dynamism supporting will be removed from API 1.0
void MKLDNNPlugin::MKLDNNLegacyInferRequest::SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr &data) {
OV_ITT_SCOPED_TASK(itt::domains::MKLDNNPlugin, "SetBlobOldApi");
if (name.empty()) {
IE_THROW(NotFound) << "Failed to set blob with empty name";
}
@ -502,146 +505,379 @@ void MKLDNNPlugin::MKLDNNInferRequest::SetBlob(const std::string& name, const In
}
}
static inline void changeEdgePtr(const MKLDNNPlugin::MKLDNNEdgePtr &edge, void *newPtr) {
edge->getMemory().GetPrimitivePtr()->set_data_handle(newPtr);
}
// TODO [DS] : restore API 1.0 version, after dynamism supporting will be removed from API 1.0
InferenceEngine::Blob::Ptr MKLDNNPlugin::MKLDNNLegacyInferRequest::GetBlob(const std::string& name) {
OV_ITT_SCOPED_TASK(itt::domains::MKLDNNPlugin, "GetBlobOldApi");
void MKLDNNPlugin::MKLDNNInferRequest::changeDefaultPtr() {
for (auto& it : externalPtr) {
const auto& inputNodesMap = graph->GetInputNodesMap();
auto input = inputNodesMap.find(it.first);
if (input != inputNodesMap.end()) {
MKLDNNNodePtr inputNodePtr = input->second;
if (inputNodePtr->getChildEdgeAt(0)->getMemory().GetPrimitive().get_data_handle() == it.second)
continue;
auto& childEdges = inputNodePtr->getChildEdges();
// Input cannot be in-place with other primitives
bool canBeInPlace = true;
for (auto& childEdge : childEdges) {
auto ce = childEdge.lock();
if (!ce)
IE_THROW() << "Node " << inputNodePtr->getName() << " contains empty child edge";
if (!graph || !graph->IsReady())
IE_THROW() << "Graph is not ready!";
auto& child = ce->getChild();
InferenceEngine::Blob::Ptr data;
if (child->isConstant()) {
canBeInPlace = false;
break;
const auto &inMap = graph->inputNodesMap;
auto input = inMap.find(name);
if (input != inMap.end()) {
// ROI blob is returned only if it was set previously.
auto it = _preProcData.find(name);
if (it != _preProcData.end()) {
data = it->second->getRoiBlob();
return data;
}
if (_inputs.find(name) == _inputs.end()) {
if (_networkInputs.find(name) != _networkInputs.end()) {
InferenceEngine::TensorDesc desc = _networkInputs[name]->getTensorDesc();
bool isDynamic = input->second->isDynamicNode();
_inputs[name] = make_blob_with_precision(desc);
_inputs[name]->allocate();
if (!isDynamic &&
desc == MemoryDescUtils::convertToTensorDesc(graph->getInputNodeByName(name)->getChildEdgesAtPort(0)[0]->getMemory().getDesc()) &&
graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end() && !graph->getProperty().batchLimit) {
externalPtr[name] = _inputs[name]->buffer();
}
if (child->getType() == Concatenation) {
auto concat = dynamic_cast<MKLDNNConcatNode*>(child.get());
if (concat && concat->isOptimized()) {
canBeInPlace = false;
break;
}
}
// Cannot be in-place before split because split is using different ptrs without offsets
if (child->getType() == Split) {
canBeInPlace = false;
break;
}
if (child->isInPlace()) {
canBeInPlace = false;
break;
}
auto& edges = child->getChildEdges();
for (auto& edge : edges) {
auto e = edge.lock();
if (!e)
IE_THROW() << "Node " << child->getName() << " contains empty child edge";
if (e->getMemory().GetPrimitive().get_data_handle() == ce->getMemory().GetPrimitive().get_data_handle()) {
canBeInPlace = false;
break;
}
}
if (!canBeInPlace)
break;
} else {
IE_THROW() << "Blob with name: " << name << " exists in MKLDNN graph, but absents in network inputs";
}
if (canBeInPlace) {
for (auto& edge : childEdges) {
auto e = edge.lock();
if (!e)
IE_THROW() << "Node " << inputNodePtr->getName() << " contains empty child edge";
}
data = _inputs[name];
checkBlob(data, name, true);
changeEdgePtr(e, it.second);
// check if preprocess required, but still wasn't set
auto preProcessedInput = std::find_if(std::begin(_networkInputs), std::end(_networkInputs),
[&](const std::pair<std::string, InferenceEngine::InputInfo::Ptr>& pair) {
return pair.first == name;
});
if (preProcessedInput != std::end(_networkInputs)) {
InferenceEngine::InputInfo::Ptr foundInput;
InferenceEngine::DataPtr foundOutput;
if (!findInputAndOutputBlobByName(name, foundInput, foundOutput)) {
IE_THROW() << "Blob with name: " << name << " absents in network inputs";
}
if (preProcessingRequired(foundInput, data)) {
_preProcData.emplace(name, InferenceEngine::CreatePreprocDataHelper());
_preProcData[name]->isApplicable(data, _inputs[name]);
_preProcData[name]->setRoiBlob(data);
}
}
}
if (graph->hasOutputWithName(name)) {
if (auto outNode = graph->getOutputNodeByName(name)) {
if (_outputs.find(name) == _outputs.end()) {
if (_networkOutputs.find(name) != _networkOutputs.end()) {
bool isDynamic = outNode->isDynamicNode();
const auto &desc = outNode->getParentEdgesAtPort(0)[0]->getMemory().getDesc();
if (!data) {
InferenceEngine::TensorDesc desc = _networkOutputs[name]->getTensorDesc();
desc.setPrecision(normalizeToSupportedPrecision(desc.getPrecision()));
data = make_blob_with_precision(desc);
data->allocate();
} else {
const auto &expectedTensorDesc = isDynamic ? InferenceEngine::TensorDesc(desc.getPrecision(),
InferenceEngine::TensorDesc::getLayoutByRank(
desc.getShape().getRank()))
: MemoryDescUtils::convertToTensorDesc(desc);
const auto &tensorDesc = data->getTensorDesc();
if (expectedTensorDesc.getPrecision() != normalizeToSupportedPrecision(tensorDesc.getPrecision())) {
IE_THROW(ParameterMismatch)
<< "Network input and output use the same name: " << name << " but expect blobs with different precision: "
<< tensorDesc.getPrecision() << " for input and " << expectedTensorDesc.getPrecision()
<< " for output.";
}
if (expectedTensorDesc.getDims() != tensorDesc.getDims()) {
IE_THROW(ParameterMismatch) << "Network input and output use the same name: " << name << " but expect blobs with different shapes.";
}
if (tensorDesc.getLayout() != InferenceEngine::Layout::ANY && expectedTensorDesc.getLayout() != InferenceEngine::Layout::ANY) {
if (tensorDesc.getLayout() != expectedTensorDesc.getLayout() &&
!(tensorDesc.getLayout() == InferenceEngine::Layout::BLOCKED &&
InferenceEngine::TensorDesc(tensorDesc.getPrecision(),
tensorDesc.getDims(),
tensorDesc.getBlockingDesc()).getLayout() == expectedTensorDesc.getLayout())) {
IE_THROW(ParameterMismatch) << "Network input and output use the same name: " << name << " but expect blobs" <<
" with different layouts.";
}
if (expectedTensorDesc.getBlockingDesc() != tensorDesc.getBlockingDesc())
IE_THROW(ParameterMismatch) << "Network input and output use the same name: " << name
<< " but expect blobs with different blocking descriptors.";
}
}
_outputs[name] = data;
if (!isDynamic && !externalPtr.count(name) && data->getTensorDesc() == MemoryDescUtils::convertToTensorDesc(desc) &&
!graph->getProperty().batchLimit) {
externalPtr[name] = data->buffer();
}
} else {
IE_THROW() << "Blob with name: " << name << " exists in MKLDNN graph, but absents in network outputs";
}
}
continue;
data = _outputs[name];
if (!outNode->isDynamicNode())
checkBlob(data, name, false);
} else {
IE_THROW() << "Output node with name: " << name << " has not been created";
}
}
if (!data) {
IE_THROW() << "Cannot find blob with name: " << name;
}
return data;
}
void MKLDNNPlugin::MKLDNNLegacyInferRequest::PushInputData() {
for (auto input : _inputs) {
auto inputName = input.first;
if (!_networkInputs[inputName]) {
IE_THROW() << "Input blobs map contains not registered during IInferencePlugin::LoadNetwork blob with name " << inputName;
}
const auto& outputNodesMap = graph->GetOutputNodesMap();
auto output = outputNodesMap.find(it.first);
if (output != outputNodesMap.end()) {
auto parentEdge = output->second->getParentEdgeAt(0);
if (parentEdge->getMemory().GetPrimitive().get_data_handle() == it.second)
continue;
// User can initialize input via setBlob API using tensorDesc with default (ANY) layout.
// Currently IE doesn't specify behavior in such scenario, so we assume real layout is equal to the network input.
auto inputBlob = input.second;
if (inputBlob->getTensorDesc().getLayout() == InferenceEngine::ANY) {
inputBlob->getTensorDesc().setLayout(_networkInputs[inputName]->getLayout());
}
bool canBeInPlace = true;
void* defaultPtr = parentEdge->getMemory().GetPrimitivePtr()->get_data_handle();
// Cannot be in-place after concat because concat is using different ptrs without offsets
auto parent = parentEdge->getParent();
MKLDNNNodePtr previousParent;
do {
previousParent = parent;
if (parent->getChildEdges().size() != 1 || parent->isConstant() || parent->isInPlace()) {
canBeInPlace = false;
break;
pushInput(inputName, inputBlob, normToInputSupportedPrec(input));
}
}
/* ========================================== MKLDNNInferRequest ========================================== */
MKLDNNPlugin::MKLDNNInferRequest::MKLDNNInferRequest(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs,
MKLDNNExecNetwork::Ptr execNetwork)
: MKLDNNInferRequestBase(inputs, outputs, execNetwork) {
for (const std::shared_ptr<const ov::Node>& in : inputs) {
modelInputsMap[ngraph::op::util::get_ie_output_name(ngraph::Output<const ngraph::Node>(in))] = in;
}
for (const std::shared_ptr<const ov::Node>& out : outputs) {
modelOutputsMap[ngraph::op::util::get_ie_output_name(out->input_value(0))] = out;
}
CreateInferRequest();
}
void MKLDNNPlugin::MKLDNNInferRequest::initBlobs() {
for (const auto& it : modelInputsMap) {
MKLDNNInferRequest::GetBlob(it.first);
}
for (const auto& it : modelOutputsMap) {
MKLDNNInferRequest::GetBlob(it.first);
}
}
void MKLDNNPlugin::MKLDNNInferRequest::SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr &data) {
OV_ITT_SCOPED_TASK(itt::domains::MKLDNNPlugin, "SetBlobNewApi");
if (name.empty()) {
IE_THROW(NotFound) << "Failed to set blob with empty name";
}
if (!data)
IE_THROW(NotAllocated) << "Failed to set empty blob with name: \'" << name << "\'";
bool isInput = false;
const auto inputNodeItr = modelInputsMap.find(name);
const auto outputNodeItr = modelOutputsMap.find(name);
if (inputNodeItr != modelInputsMap.end()) {
if (!inputNodeItr->second) {
IE_THROW() << "Can't SetBlob with name: " << name << ", because has null pointer to input node";
}
isInput = true;
} else if (outputNodeItr != modelOutputsMap.end()) {
if (!outputNodeItr->second) {
IE_THROW() << "Can't SetBlob with name: " << name << ", because has null pointer to output node";
}
isInput = false;
} else {
IE_THROW(NotFound) << "Can't SetBlob with name: " << name << ", because input/output with this name doesn't exist";
}
const bool compoundBlobPassed = data->is<InferenceEngine::CompoundBlob>();
if (!compoundBlobPassed && data->buffer() == nullptr)
IE_THROW(NotAllocated) << "Input data was not allocated. Input name: \'" << name << "\'";
const auto &blobDesc = data->getTensorDesc();
if (isInput) {
const auto netInPrc = InferenceEngine::details::convertPrecision(inputNodeItr->second->get_output_element_type(0));
if (netInPrc != blobDesc.getPrecision()) {
IE_THROW(ParameterMismatch) << "Failed to set input blob with precision: "
<< blobDesc.getPrecision() << ", if CNNNetwork input blob precision is: " << netInPrc;
}
const auto shape = inputNodeItr->second->get_output_partial_shape(0);
const bool isDynamic = shape.is_dynamic();
if (!shape.compatible(ov::PartialShape(data->getTensorDesc().getDims()))) {
IE_THROW() << "Can't SetBlob with name: " << name << ", because model input and blob are incompatible";
}
if (!isDynamic && ngraph::shape_size(shape.to_shape()) != data->size()) {
IE_THROW() << "Can't SetBlob with name: " << name << ", because model input and blob have different size";
}
MemoryDescPtr actualDesc = graph->getInputNodeByName(name)->getBaseMemDescAtOutputPort(0);
if (!actualDesc->isDefined()) {
// we must define desc for dynamic case
// otherwise we got incorrect check on shape compatibility inside isCompatible
// because lower and upper bound will be compared
actualDesc = actualDesc->cloneWithNewDims(blobDesc.getLayout() == InferenceEngine::Layout::SCALAR ? InferenceEngine::SizeVector{1} :
blobDesc.getDims());
}
if (actualDesc->isCompatible(MemoryDescUtils::convertToCpuBlockedMemoryDesc(blobDesc)) &&
graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end() && !graph->getProperty().batchLimit) {
externalPtr[name] = data->buffer();
} else if (externalPtr.find(name) != externalPtr.end()) {
externalPtr.erase(name);
}
_inputs[name] = data;
} else {
if (compoundBlobPassed) {
IE_THROW(NotImplemented) << "cannot set compound blob: supported only for input pre-processing";
}
const auto netOutPrc = InferenceEngine::details::convertPrecision(outputNodeItr->second->get_input_element_type(0));
if (netOutPrc != blobDesc.getPrecision()) {
IE_THROW(ParameterMismatch) << "Failed to set input blob with precision: "
<< blobDesc.getPrecision() << ", if CNNNetwork output blob precision is: " << netOutPrc;
}
const auto shape = outputNodeItr->second->get_input_partial_shape(0);
const bool isDynamic = shape.is_dynamic();
if (!shape.compatible(ov::PartialShape(data->getTensorDesc().getDims()))) {
IE_THROW() << "Can't SetBlob with name: " << name << ", because model output and blob are incompatible";
}
if (!isDynamic && ngraph::shape_size(shape.to_shape()) != data->size()) {
IE_THROW() << "Can't SetBlob with name: " << name << ", because model output and blob have different size";
}
const auto &desc = graph->getOutputNodeByName(name)->getParentEdgesAtPort(0)[0]->getMemory().getDesc();
if (!isDynamic && blobDesc == MemoryDescUtils::convertToTensorDesc(desc) && !graph->getProperty().batchLimit) {
externalPtr[name] = data->buffer();
} else if (externalPtr.find(name) != externalPtr.end()) {
externalPtr.erase(name);
}
_outputs[name] = data;
}
}
InferenceEngine::Blob::Ptr MKLDNNPlugin::MKLDNNInferRequest::GetBlob(const std::string& name) {
OV_ITT_SCOPED_TASK(itt::domains::MKLDNNPlugin, "GetBlobNewApi");
if (!graph || !graph->IsReady())
IE_THROW() << "Graph is not ready!";
InferenceEngine::Blob::Ptr data;
const auto &inMap = graph->inputNodesMap;
auto input = inMap.find(name);
if (input != inMap.end()) {
if (_inputs.find(name) == _inputs.end()) {
auto inputNode = modelInputsMap.find(name);
if (inputNode != modelInputsMap.end()) {
if (!inputNode->second) {
IE_THROW() << "Can't GetBlob with name: " << name << ", because has null pointer to input node";
}
auto& parentEdges = parent->getParentEdges();
for (auto& edge : parentEdges) {
auto e = edge.lock();
if (!e)
IE_THROW() << "Node " << parent->getName() << " contains empty parent edge";
const auto shape = inputNode->second->get_output_partial_shape(0);
const bool isDynamic = shape.is_dynamic();
InferenceEngine::SizeVector dims;
if (isDynamic) {
dims = InferenceEngine::SizeVector(shape.rank().get_length(), 0);
} else {
dims = shape.to_shape();
}
if (e->getMemory().GetPrimitivePtr()->get_data_handle() == defaultPtr) {
parent = e->getParent();
break;
InferenceEngine::TensorDesc desc(InferenceEngine::details::convertPrecision(inputNode->second->get_output_element_type(0)),
dims, InferenceEngine::TensorDesc::getLayoutByRank(dims.size()));
_inputs[name] = make_blob_with_precision(desc);
_inputs[name]->allocate();
if (!isDynamic &&
desc == MemoryDescUtils::convertToTensorDesc(graph->getInputNodeByName(name)->getChildEdgesAtPort(0)[0]->getMemory().getDesc()) &&
graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end() && !graph->getProperty().batchLimit) {
externalPtr[name] = _inputs[name]->buffer();
}
} else {
IE_THROW() << "Blob with name: " << name << " exists in MKLDNN graph, but absents in network inputs";
}
}
data = _inputs[name];
}
const auto &outMap = graph->outputNodesMap;
auto output = outMap.find(name);
if (output != outMap.end()) {
if (_outputs.find(name) == _outputs.end()) {
auto outputNode = modelOutputsMap.find(name);
if (modelOutputsMap.find(name) != modelOutputsMap.end()) {
const auto shape = outputNode->second->get_input_partial_shape(0);
bool isDynamic = shape.is_dynamic();
if (!data) {
InferenceEngine::SizeVector dims;
if (isDynamic) {
dims = InferenceEngine::SizeVector(shape.rank().get_length(), 0);
} else {
dims = shape.to_shape();
}
InferenceEngine::TensorDesc desc(InferenceEngine::details::convertPrecision(outputNode->second->get_input_element_type(0)),
dims, InferenceEngine::TensorDesc::getLayoutByRank(dims.size()));
data = make_blob_with_precision(desc);
data->allocate();
} else {
if (!shape.compatible(ov::PartialShape(data->getTensorDesc().getDims()))) {
IE_THROW(ParameterMismatch) << "Network input and output use the same name: " << name << ", but expect blobs with different shapes.";
}
const auto netOutPrc = InferenceEngine::details::convertPrecision(outputNode->second->get_input_element_type(0));
if (netOutPrc != data->getTensorDesc().getPrecision()) {
IE_THROW(ParameterMismatch)
<< "Network input and output use the same name: " << name << " but expect blobs with different precision: "
<< data->getTensorDesc().getPrecision() << " for input and " << netOutPrc
<< " for output.";
}
}
} while (previousParent != parent);
if (canBeInPlace)
changeEdgePtr(parentEdge, it.second);
continue;
_outputs[name] = data;
if (!isDynamic && !externalPtr.count(name) &&
data->getTensorDesc() == MemoryDescUtils::convertToTensorDesc(output->second->getParentEdgesAtPort(0)[0]->getMemory().getDesc()) &&
!graph->getProperty().batchLimit) {
externalPtr[name] = data->buffer();
}
} else {
IE_THROW() << "Blob with name: " << name << " exists in MKLDNN graph, but absents in network outputs";
}
}
IE_THROW() << "Cannot find input/output blob: " << it.first;
}
}
void MKLDNNPlugin::MKLDNNInferRequest::SetBatch(int new_batch) {
if (!graph->getProperty().enableDynamicBatch)
IE_THROW() << "Dynamic batch is not enabled.";
if (new_batch < 1 || new_batch > graph->getProperty().batchLimit) {
IE_THROW() << "Invalid dynamic batch size " << new_batch <<
" for this request.";
data = _outputs[name];
}
m_curBatch = new_batch;
if (!data) {
IE_THROW() << "Cannot find blob with name: " << name;
}
for (const auto& node : graph->GetNodes()) {
node->setDynamicBatchLim(new_batch);
}
}
std::vector<InferenceEngine::IVariableStateInternal::Ptr> MKLDNNPlugin::MKLDNNInferRequest::QueryState() {
return memoryStates;
}
void MKLDNNPlugin::MKLDNNInferRequest::SetAsyncRequest(MKLDNNAsyncInferRequest* asyncRequest) {
_asyncRequest = asyncRequest;
}
void MKLDNNPlugin::MKLDNNInferRequest::ThrowIfCanceled() const {
if (_asyncRequest != nullptr) {
_asyncRequest->ThrowIfCanceled();
return data;
}
void MKLDNNPlugin::MKLDNNInferRequest::PushInputData() {
for (auto input : _inputs) {
auto inputName = input.first;
if (!modelInputsMap[inputName]) {
IE_THROW() << "Input blobs map contains not registered during IInferencePlugin::LoadNetwork blob with name " << inputName;
}
pushInput(inputName, input.second, normToInputSupportedPrec(input));
}
}

74
src/plugins/intel_cpu/src/mkldnn_infer_request.h
View File

@ -15,27 +15,14 @@ namespace MKLDNNPlugin {
class MKLDNNExecNetwork;
class MKLDNNAsyncInferRequest;
class MKLDNNInferRequest : public InferenceEngine::IInferRequestInternal {
class MKLDNNInferRequestBase : public InferenceEngine::IInferRequestInternal {
public:
typedef std::shared_ptr<MKLDNNInferRequest> Ptr;
explicit MKLDNNInferRequest(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs,
std::shared_ptr<MKLDNNExecNetwork> execNetwork);
MKLDNNInferRequest(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs,
std::shared_ptr<MKLDNNExecNetwork> execNetwork);
~MKLDNNInferRequest();
virtual ~MKLDNNInferRequestBase();
void InferImpl() override;
std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> GetPerformanceCounts() const override;
void SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr &data) override;
InferenceEngine::Blob::Ptr GetBlob(const std::string& name) override;
void SetBatch(int batch = -1) override;
std::vector<std::shared_ptr<InferenceEngine::IVariableStateInternal>> QueryState() override;
@ -51,21 +38,66 @@ public:
*/
void ThrowIfCanceled() const;
private:
protected:
MKLDNNInferRequestBase(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs,
std::shared_ptr<MKLDNNExecNetwork> execNetwork_)
: IInferRequestInternal(networkInputs, networkOutputs), execNetwork(execNetwork_) {}
MKLDNNInferRequestBase(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs,
std::shared_ptr<MKLDNNExecNetwork> execNetwork_)
: IInferRequestInternal(inputs, outputs), execNetwork(execNetwork_) {}
void CreateInferRequest();
void PushInputData();
InferenceEngine::Precision normToInputSupportedPrec(const std::pair<const std::string, InferenceEngine::Blob::Ptr>& input) const;
void pushInput(const std::string& inputName, InferenceEngine::Blob::Ptr& inputBlob, InferenceEngine::Precision dataType);
virtual void initBlobs() = 0;
virtual void PushInputData() = 0;
MKLDNNGraph* graph = nullptr;
std::unordered_map<std::string, void*> externalPtr;
private:
void PushStates();
void PullStates();
void redefineMemoryForInputNodes();
void pushInput(const std::string& inputName, InferenceEngine::Blob::Ptr& inputBlob, InferenceEngine::Precision dataType);
void changeDefaultPtr();
std::shared_ptr<MKLDNNExecNetwork> execNetwork;
MKLDNNGraph* graph = nullptr;
std::map<std::string, void*> externalPtr;
openvino::itt::handle_t profilingTask;
std::vector<std::shared_ptr<InferenceEngine::IVariableStateInternal>> memoryStates;
MKLDNNAsyncInferRequest* _asyncRequest = nullptr;
};
class MKLDNNLegacyInferRequest : public MKLDNNInferRequestBase {
public:
MKLDNNLegacyInferRequest(InferenceEngine::InputsDataMap networkInputs,
InferenceEngine::OutputsDataMap networkOutputs,
std::shared_ptr<MKLDNNExecNetwork> execNetwork);
void SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr &data) override;
InferenceEngine::Blob::Ptr GetBlob(const std::string& name) override;
private:
void PushInputData() override;
void initBlobs() override;
};
class MKLDNNInferRequest : public MKLDNNInferRequestBase {
public:
MKLDNNInferRequest(const std::vector<std::shared_ptr<const ov::Node>>& inputs,
const std::vector<std::shared_ptr<const ov::Node>>& outputs,
std::shared_ptr<MKLDNNExecNetwork> execNetwork);
void SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr &data) override;
InferenceEngine::Blob::Ptr GetBlob(const std::string& name) override;
private:
void PushInputData() override;
void initBlobs() override;
std::unordered_map<std::string, std::shared_ptr<const ov::Node>> modelInputsMap;
std::unordered_map<std::string, std::shared_ptr<const ov::Node>> modelOutputsMap;
};
} // namespace MKLDNNPlugin

8
src/tests/functional/plugin/cpu/shared_tests_instances/skip_tests_config.cpp
View File

@ -120,10 +120,6 @@ std::vector<std::string> disabledTestPatterns() {
R"(.*smoke_Hetero_BehaviorTests.*DynamicInputToDynamicOutput.*)",
R"(.*smoke_Auto_BehaviorTests.*DynamicOutputToDynamicInput.*)",
R"(.*smoke_Auto_BehaviorTests.*DynamicInputToDynamicOutput.*)",
// CPU dynamism: empty tensor returns size() == 1. Looks like layout is SCALAR
// Issue: CVS-66780
R"(.*smoke_BehaviorTests.*InferUpperBoundNetworkWithGetTensor.*)",
R"(.*smoke_BehaviorTests.*InferDynamicNetworkWithGetTensor.*)",
// TODO: Issue CVS-51680
R"(.*BehaviorTests.*canRun3SyncRequestsConsistentlyFromThreads.*CPU_THROUGHPUT.*)",
@ -173,7 +169,9 @@ std::vector<std::string> disabledTestPatterns() {
// Issue:
R"(.*smoke_VariadicSplit4D_CPU_zero_dims.*)",
// New API tensor tests
R"(.*OVInferRequestCheckTensorPrecision.*)",
R"(.*OVInferRequestCheckTensorPrecision.*type=i4.*)",
R"(.*OVInferRequestCheckTensorPrecision.*type=u1.*)",
R"(.*OVInferRequestCheckTensorPrecision.*type=u4.*)",
// Issue: 75022
R"(.*OVExecutableNetworkBaseTest.*LoadNetworkToDefaultDeviceNoThrow.*)",
R"(.*IEClassBasicTest.*LoadNetworkToDefaultDeviceNoThrow.*)",

2
src/tests/functional/plugin/cpu/single_layer_tests/broadcast.cpp
View File

@ -447,7 +447,7 @@ const std::vector<std::vector<ov::test::InputShape>> dynamicShapes1D = {
{ // Origin dynamic shapes
{-1},
{ // Dynamic shapes instances
{},
{1},
{1}
}
}

33
src/tests/functional/plugin/cpu/single_layer_tests/non_max_suppression.cpp
View File

@ -333,19 +333,32 @@ private:
const auto indeces_iter = actualOutputs.begin();
const auto scores_iter = actualOutputs.begin() + 1;
size_t selected_indices_size = indeces_iter->get_size();
const auto selected_indices_data = indeces_iter->data<int32_t>();
const auto selected_scores_data = scores_iter->data<float>();
for (size_t i = 0; i < selected_indices_size; i += 3) {
const int32_t batchId = selected_indices_data[i+0];
const int32_t classId = selected_indices_data[i+1];
const int32_t boxId = selected_indices_data[i+2];
const float score = selected_scores_data[i+2];
if (batchId == -1 || classId == -1 || boxId == -1)
break;
if (indeces_iter->get_element_type() == ov::element::i32) {
const auto selected_indices_data = indeces_iter->data<int32_t>();
for (size_t i = 0; i < selected_indices_size; i += 3) {
const int32_t batchId = selected_indices_data[i+0];
const int32_t classId = selected_indices_data[i+1];
const int32_t boxId = selected_indices_data[i+2];
const float score = selected_scores_data[i+2];
if (batchId == -1 || classId == -1 || boxId == -1)
break;
actualList.emplace_back(batchId, classId, boxId, coordList[batchId][boxId], score);
actualList.emplace_back(batchId, classId, boxId, coordList[batchId][boxId], score);
}
} else {
const auto selected_indices_data = indeces_iter->data<int64_t>();
for (size_t i = 0; i < selected_indices_size; i += 3) {
const int32_t batchId = selected_indices_data[i+0];
const int32_t classId = selected_indices_data[i+1];
const int32_t boxId = selected_indices_data[i+2];
const float score = selected_scores_data[i+2];
if (batchId == -1 || classId == -1 || boxId == -1)
break;
actualList.emplace_back(batchId, classId, boxId, coordList[batchId][boxId], score);
}
}
std::sort(actualList.begin(), actualList.end(), compareBox);
}

129
src/tests/functional/plugin/cpu/subgraph_tests/src/input_tensor_roi.cpp Normal file
View File

@ -0,0 +1,129 @@
// Copyright (C) 2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "ngraph_functions/builders.hpp"
#include "test_utils/cpu_test_utils.hpp"
#include "functional_test_utils/ov_plugin_cache.hpp"
using namespace ngraph;
using namespace ngraph::op;
using namespace InferenceEngine;
using namespace CPUTestUtils;
namespace SubgraphTestsDefinitions {
struct InputTensorROIParamType {
ov::PartialShape shape;
ov::element::Type type;
ov::Layout layout;
};
class InputTensorROI : public ::testing::TestWithParam<InputTensorROIParamType> {
public:
static std::string getTestCaseName(::testing::TestParamInfo<InputTensorROIParamType> obj) {
std::ostringstream result;
result << "type=" << obj.param.type << "_";
result << "shape=" << obj.param.shape << "_";
result << "layout=" << obj.param.layout.to_string();
return result.str();
}
protected:
std::shared_ptr<ov::Model>
create_test_function(element::Type type,
const ov::PartialShape & shape,
const ov::Layout & layout) {
ResultVector res;
ParameterVector params;
auto param = std::make_shared<opset8::Parameter>(type, shape);
param->set_friendly_name("input_0");
param->get_output_tensor(0).set_names({"tensor_input_0"});
param->set_layout(layout);
auto constant = opset8::Constant::create(type, {1}, {1});
auto add = std::make_shared<opset8::Add>(param, constant);
add->set_friendly_name("Add");
auto result = std::make_shared<opset8::Result>(add);
result->set_friendly_name("result_0");
result->get_output_tensor(0).set_names({"tensor_output_0"});
params.push_back(param);
res.push_back(result);
return std::make_shared<ov::Model>(res, params);
}
template<typename T>
void Run() {
std::shared_ptr<ov::runtime::Core> ie = ov::test::utils::PluginCache::get().core();
// Compile model
auto fn_shape = GetParam().shape;
auto model = create_test_function(GetParam().type, fn_shape, GetParam().layout);
auto compiled_model = ie->compile_model(model, "CPU");
// Create InferRequest
ov::runtime::InferRequest req = compiled_model.create_infer_request();
// Create input tensor
auto input_shape = Shape{ 1, 4, 4, 4 };
auto input_shape_size = ov::shape_size(input_shape);
std::vector<T> data(input_shape_size);
std::iota(data.begin(), data.end(), 0);
auto input_tensor = ov::runtime::Tensor(GetParam().type, input_shape, &data[0]);
// Set ROI
auto roi = ov::runtime::Tensor(input_tensor, { 0, 1, 1, 1 }, { 1, 3, 3, 3 });
req.set_tensor("tensor_input_0", roi);
// Infer
req.infer();
// Check result
auto actual_tensor = req.get_tensor("tensor_output_0");
auto* actual = actual_tensor.data<T>();
EXPECT_EQ(actual[0], 21 + 1);
EXPECT_EQ(actual[1], 22 + 1);
EXPECT_EQ(actual[2], 25 + 1);
EXPECT_EQ(actual[3], 26 + 1);
EXPECT_EQ(actual[4], 37 + 1);
EXPECT_EQ(actual[5], 38 + 1);
EXPECT_EQ(actual[6], 41 + 1);
EXPECT_EQ(actual[7], 42 + 1);
}
};
TEST_P(InputTensorROI, SetInputTensorROI) {
SKIP_IF_CURRENT_TEST_IS_DISABLED()
switch (GetParam().type) {
case ov::element::Type_t::f32: {
Run<float>();
break;
}
case ov::element::Type_t::u8: {
Run<uint8_t>();
break;
}
default:
break;
}
}
static InputTensorROI::ParamType InputTensorROIParams[] = {
{ ov::PartialShape{ 1, 2, 2, 2 }, element::f32, "NCHW" },
{ ov::PartialShape{ 1, 2, ov::Dimension::dynamic(), ov::Dimension::dynamic() }, element::f32, "NCHW" },
{ ov::PartialShape{ 1, 2, 2, 2 }, element::u8, "NCHW" },
{ ov::PartialShape{ 1, 2, ov::Dimension::dynamic(), ov::Dimension::dynamic() }, element::u8, "NCHW" },
};
INSTANTIATE_TEST_SUITE_P(smoke_InputTensorROI,
InputTensorROI,
::testing::ValuesIn(InputTensorROIParams),
InputTensorROI::getTestCaseName);
} // namespace SubgraphTestsDefinitions

8
src/tests/functional/plugin/shared/src/behavior/ov_infer_request/infer_request_dynamic.cpp
View File

@ -115,7 +115,7 @@ TEST_P(OVInferRequestDynamicTests, InferDynamicNetworkWithGetTensor) {
ASSERT_EQ(tensor.get_shape(), refShape);
OV_ASSERT_NO_THROW(otensor = req.get_tensor(outputname));
ASSERT_EQ(0, otensor.get_size()); // output tensor is not allocated
ASSERT_EQ(function->output().get_element_type(), otensor.get_element_type()); // by it has type
ASSERT_EQ(function->output(0).get_element_type(), otensor.get_element_type()); // by it has type
OV_ASSERT_NO_THROW(req.infer());
OV_ASSERT_NO_THROW(req.start_async());
OV_ASSERT_NO_THROW(req.wait());
@ -141,7 +141,7 @@ TEST_P(OVInferRequestDynamicTests, InferUpperBoundNetworkWithGetTensor) {
//OV_ASSERT_NO_THROW(req.SetShape(tensor_name, {1, 4, 20, 20}));
OV_ASSERT_NO_THROW(otensor = req.get_tensor(outputname));
ASSERT_EQ(0, otensor.get_size()); // output tensor is not allocated
ASSERT_EQ(function->output().get_element_type(), otensor.get_element_type()); // by it has type
ASSERT_EQ(function->output(0).get_element_type(), otensor.get_element_type()); // by it has type
OV_ASSERT_NO_THROW(tensor = req.get_tensor(function->inputs().back().get_any_name()));
OV_ASSERT_NO_THROW(tensor.set_shape({1, 4, 20, 20}));
ASSERT_EQ(tensor.get_shape(), refShape);
@ -171,7 +171,7 @@ TEST_P(OVInferRequestDynamicTests, InferFullyDynamicNetworkWithGetTensor) {
ASSERT_EQ(tensor.get_shape(), refShape);
OV_ASSERT_NO_THROW(otensor = req.get_tensor(outputName));
ASSERT_EQ(0, otensor.get_size()); // output tensor is not allocated
ASSERT_EQ(function->output().get_element_type(), otensor.get_element_type()); // by it has type
ASSERT_EQ(function->output(0).get_element_type(), otensor.get_element_type()); // by it has type
OV_ASSERT_NO_THROW(req.infer());
OV_ASSERT_NO_THROW(req.start_async());
OV_ASSERT_NO_THROW(req.wait());
@ -313,7 +313,7 @@ TEST_P(OVInferRequestDynamicTests, InferFullyDynamicNetworkWithSetTensor) {
ASSERT_EQ(tensor.get_shape(), refShape);
OV_ASSERT_NO_THROW(otensor = req.get_tensor(outputName));
ASSERT_EQ(0, otensor.get_size()); // output tensor is not allocated
ASSERT_EQ(function->output().get_element_type(), otensor.get_element_type()); // by it has type
ASSERT_EQ(function->output(0).get_element_type(), otensor.get_element_type()); // by it has type
OV_ASSERT_NO_THROW(req.infer());
ASSERT_EQ(otensor.get_shape(), refOutShape);
OV_ASSERT_NO_THROW(req.start_async());

49
src/tests/functional/shared_test_classes/src/single_layer/matrix_nms.cpp
View File

@ -140,8 +140,13 @@ void MatrixNmsLayerTest::compare(const std::vector<ov::runtime::Tensor> &expecte
const auto _dims = actual.get_shape();
if (_dims.size() == 1 && _dims[0] == numBatches) {
batchIndex = outputIndex;
auto buffer = reinterpret_cast<const int32_t*>(actual.data());
std::copy_n(buffer, numBatches, numPerBatch.begin());
if (actual.get_element_type() == ov::element::i32) {
auto buffer = actual.data<int32_t>();
std::copy_n(buffer, numBatches, numPerBatch.begin());
} else {
auto buffer = actual.data<int64_t>();
std::copy_n(buffer, numBatches, numPerBatch.begin());
}
}
}
@ -204,7 +209,28 @@ void MatrixNmsLayerTest::compare(const std::vector<ov::runtime::Tensor> &expecte
break;
}
if (m_outStaticShape) {
const auto iBuffer = static_cast<int*>(actual.data());
const auto iBuffer = actual.data<int32_t>();
for (size_t tailing = validNums; tailing < maxOutputBoxesPerBatch; tailing++) {
ASSERT_TRUE(iBuffer[actual_offset + tailing] == -1) << "Invalid default value: " << iBuffer[i] << " at index: " << i;
}
}
break;
}
case ov::element::i64: {
switch (expected.get_element_type()) {
case ov::element::i32:
LayerTestsUtils::LayerTestsCommon::Compare(reinterpret_cast<const int32_t*>(expectedBuffer) + expected_offset,
reinterpret_cast<const int64_t*>(actualBuffer) + actual_offset, validNums, 0);
break;
case ov::element::i64:
LayerTestsUtils::LayerTestsCommon::Compare(reinterpret_cast<const int64_t*>(expectedBuffer) + expected_offset,
reinterpret_cast<const int64_t*>(actualBuffer) + actual_offset, validNums, 0);
break;
default:
break;
}
if (m_outStaticShape) {
const auto iBuffer = actual.data<int64_t>();
for (size_t tailing = validNums; tailing < maxOutputBoxesPerBatch; tailing++) {
ASSERT_TRUE(iBuffer[actual_offset + tailing] == -1) << "Invalid default value: " << iBuffer[i] << " at index: " << i;
}
@ -248,6 +274,23 @@ void MatrixNmsLayerTest::compare(const std::vector<ov::runtime::Tensor> &expecte
}
break;
}
case ov::element::i64: {
switch (expected.get_element_type()) {
case ov::element::i32:
LayerTestsUtils::LayerTestsCommon::Compare(
reinterpret_cast<const int32_t*>(expectedBuffer),
reinterpret_cast<const int64_t*>(actualBuffer), size, 0);
break;
case ov::element::i64:
LayerTestsUtils::LayerTestsCommon::Compare(
reinterpret_cast<const int64_t*>(expectedBuffer),
reinterpret_cast<const int64_t*>(actualBuffer), size, 0);
break;
default:
break;
}
break;
}
default:
FAIL() << "Comparator for " << precision << " precision isn't supported";
}

47
src/tests/functional/shared_test_classes/src/single_layer/multiclass_nms.cpp
View File

@ -141,8 +141,13 @@ void MulticlassNmsLayerTest::compare(const std::vector<ov::runtime::Tensor> &exp
const auto _dims = actual.get_shape();
if (_dims.size() == 1 && _dims[0] == numBatches) {
batchIndex = outputIndex;
auto buffer = reinterpret_cast<const int32_t*>(actual.data());
std::copy_n(buffer, numBatches, numPerBatch.begin());
if (actual.get_element_type() == ov::element::i32) {
auto buffer = actual.data<int32_t>();
std::copy_n(buffer, numBatches, numPerBatch.begin());
} else {
auto buffer = actual.data<int64_t>();
std::copy_n(buffer, numBatches, numPerBatch.begin());
}
}
}
@ -203,7 +208,28 @@ void MulticlassNmsLayerTest::compare(const std::vector<ov::runtime::Tensor> &exp
break;
}
if (m_outStaticShape) {
const auto iBuffer = static_cast<int*>(actual.data());
const auto iBuffer = actual.data<int32_t>();
for (size_t tailing = validNums; tailing < maxOutputBoxesPerBatch; tailing++) {
ASSERT_TRUE(iBuffer[actual_offset + tailing] == -1) << "Invalid default value: " << iBuffer[i] << " at index: " << i;
}
}
break;
}
case ov::element::i64: {
switch (expected.get_element_type()) {
case ov::element::i32:
LayerTestsUtils::LayerTestsCommon::Compare(reinterpret_cast<const int32_t*>(expectedBuffer) + expected_offset,
reinterpret_cast<const int64_t*>(actualBuffer) + actual_offset, validNums, 0);
break;
case ov::element::i64:
LayerTestsUtils::LayerTestsCommon::Compare(reinterpret_cast<const int64_t*>(expectedBuffer) + expected_offset,
reinterpret_cast<const int64_t*>(actualBuffer) + actual_offset, validNums, 0);
break;
default:
break;
}
if (m_outStaticShape) {
const auto iBuffer = actual.data<int64_t>();
for (size_t tailing = validNums; tailing < maxOutputBoxesPerBatch; tailing++) {
ASSERT_TRUE(iBuffer[actual_offset + tailing] == -1) << "Invalid default value: " << iBuffer[i] << " at index: " << i;
}
@ -246,6 +272,21 @@ void MulticlassNmsLayerTest::compare(const std::vector<ov::runtime::Tensor> &exp
}
break;
}
case ov::element::i64: {
switch (expected.get_element_type()) {
case ov::element::i32:
LayerTestsUtils::LayerTestsCommon::Compare(reinterpret_cast<const int32_t*>(expectedBuffer), reinterpret_cast<const int64_t*>(actualBuffer),
size, 0);
break;
case ov::element::i64:
LayerTestsUtils::LayerTestsCommon::Compare(reinterpret_cast<const int64_t*>(expectedBuffer), reinterpret_cast<const int64_t*>(actualBuffer),
size, 0);
break;
default:
break;
}
break;
}
default:
FAIL() << "Comparator for " << precision << " precision isn't supported";
}