diff --git a/inference-engine/src/vpu/graph_transformer/src/frontend/ie_parsed_network.cpp b/inference-engine/src/vpu/graph_transformer/src/frontend/ie_parsed_network.cpp index 49ee5f1caae..a8c0bc0a8f5 100644 --- a/inference-engine/src/vpu/graph_transformer/src/frontend/ie_parsed_network.cpp +++ b/inference-engine/src/vpu/graph_transformer/src/frontend/ie_parsed_network.cpp @@ -27,7 +27,6 @@ IeParsedNetwork parseNetwork(const ie::CNNNetwork& network) { out.networkOutputs = network.getOutputsInfo(); env.log->trace("Got %d inputs and %d outputs", out.networkInputs.size(), out.networkOutputs.size()); - IE_ASSERT(!out.networkInputs.empty()); IE_ASSERT(!out.networkOutputs.empty()); env.log->trace("Perform topological sort"); diff --git a/inference-engine/src/vpu/myriad_plugin/myriad_executable_network.cpp b/inference-engine/src/vpu/myriad_plugin/myriad_executable_network.cpp index f2cd14bbb15..3ddcc2fcc45 100644 --- a/inference-engine/src/vpu/myriad_plugin/myriad_executable_network.cpp +++ b/inference-engine/src/vpu/myriad_plugin/myriad_executable_network.cpp @@ -7,6 +7,7 @@ #include #include +#include #include "exec_graph_info.hpp" #include #include @@ -25,7 +26,6 @@ namespace MyriadPlugin { ExecutableNetwork::ExecutableNetwork( std::shared_ptr mvnc, - std::vector& devicePool, const MyriadConfiguration& config, const std::shared_ptr core) : _config(config), @@ -40,10 +40,6 @@ ExecutableNetwork::ExecutableNetwork( defaultOutput(_config.pluginLogFilePath())); _executor = std::make_shared(_config.forceReset(), std::move(mvnc), logLevel, _log); - _device = _executor->openDevice(devicePool, _config); - - const auto& revision = _device->revision(); - _actualNumExecutors = config.compileConfig().numExecutors != -1 ? config.compileConfig().numExecutors : DefaultAllocation::numStreams(revision, config); _supportedMetrics = { METRIC_KEY(NETWORK_NAME), @@ -54,13 +50,19 @@ ExecutableNetwork::ExecutableNetwork( }; } +void ExecutableNetwork::openDevice(std::vector& devicePool) { + _device = _executor->openDevice(devicePool, _config); + const auto& revision = _device->revision(); + _actualNumExecutors = _config.compileConfig().numExecutors != -1 ? _config.compileConfig().numExecutors : DefaultAllocation::numStreams(revision, _config); +} + ExecutableNetwork::ExecutableNetwork( const ie::CNNNetwork& network, std::shared_ptr mvnc, std::vector& devicePool, const MyriadConfiguration& config, const std::shared_ptr core) : - ExecutableNetwork(std::move(mvnc), devicePool, config, core) { + ExecutableNetwork(std::move(mvnc), config, core) { VPU_PROFILE(ExecutableNetwork); const auto compilerLog = std::make_shared( @@ -68,11 +70,9 @@ ExecutableNetwork::ExecutableNetwork( _config.get(), defaultOutput(_config.compilerLogFilePath())); - if (_device == nullptr) - IE_THROW() << "No device was detected"; auto compiledGraph = compileNetwork( network, - _device->_platform, + NC_MYRIAD_X, _config, compilerLog, _core); @@ -84,12 +84,7 @@ ExecutableNetwork::ExecutableNetwork( _inputInfo = std::move(compiledGraph->inputInfo); _outputInfo = std::move(compiledGraph->outputInfo); - if (!_device->isBooted()) { - return; - } - const auto& networkName = network.getName(); - _executor->allocateGraph(_device, _graphDesc, _graphBlob, compiledGraph->blobHeader, compiledGraph->numActiveStages, networkName, _actualNumExecutors); if (_config.exclusiveAsyncRequests()) { ExecutorManager *executorManager = ExecutorManager::getInstance(); _taskExecutor = executorManager->getExecutor("MYRIAD"); @@ -100,6 +95,21 @@ ExecutableNetwork::ExecutableNetwork( idStream << networkName << "_TaskExecutorGetResult" << i; _taskExecutorGetResultIds.emplace(idStream.str()); } + if (_inputInfo.totalSize == 0) { + _isNetworkConstant = true; + const auto& nGraphFunc = network.getFunction(); + const auto& sortedLayers = nGraphFunc->get_ordered_ops(); + for (const auto& layer : sortedLayers) { + if (strcmp(layer->get_type_info().name, "Constant") == 0) { + const auto& constOp = std::dynamic_pointer_cast(layer); + auto name = constOp->get_friendly_name(); + _constDatas[name] = ie::details::shareWeights(constOp); + } + } + return; + } + openDevice(devicePool); + _executor->allocateGraph(_device, _graphDesc, _graphBlob, compiledGraph->blobHeader, compiledGraph->numActiveStages, networkName, _actualNumExecutors); } void ExecutableNetwork::Import(std::istream& strm, std::vector &devicePool, const MyriadConfiguration& configuration) { @@ -110,10 +120,6 @@ void ExecutableNetwork::Import(std::istream& strm, std::vector &devic strm.seekg(currentPos, strm.beg); strm.read(&_graphBlob[0], blobSize); - if (!_device->isBooted()) { - return; - } - std::string networkName = importedNetworkName; BlobReader blobReader; @@ -126,9 +132,8 @@ void ExecutableNetwork::Import(std::istream& strm, std::vector &devic _inputInfo = blobReader.getInputInfo(); _outputInfo = blobReader.getOutputInfo(); - + openDevice(devicePool); _executor->allocateGraph(_device, _graphDesc, _graphBlob, blobHeader, numStages, networkName, _actualNumExecutors); - _graphMetaData.stagesMeta.resize(numStages); for (auto &meta : _graphMetaData.stagesMeta) { meta.stageName = meta.stageType = meta.layerName = meta.layerType = "UNKNOWN"; @@ -147,9 +152,12 @@ void ExecutableNetwork::Import(std::istream& strm, std::vector &devic } } -ExecutableNetwork::ExecutableNetwork(std::istream& strm, std::shared_ptr mvnc, std::vector &devicePool, - const MyriadConfiguration& config, const std::shared_ptr core) : - ExecutableNetwork(std::move(mvnc), devicePool, config, core) { +ExecutableNetwork::ExecutableNetwork(std::istream& strm, + std::shared_ptr mvnc, + std::vector &devicePool, + const MyriadConfiguration& config, + const std::shared_ptr core) : + ExecutableNetwork(std::move(mvnc), config, core) { VPU_PROFILE(ExecutableNetwork); Import(strm, devicePool, config); } @@ -160,7 +168,7 @@ ExecutableNetwork::ExecutableNetwork( std::vector& devicePool, const MyriadConfiguration& config, const std::shared_ptr core) : - ExecutableNetwork(std::move(mvnc), devicePool, config, core) { + ExecutableNetwork(std::move(mvnc), config, core) { VPU_PROFILE(ExecutableNetwork); std::ifstream blobFile{blobFilename, std::ios::binary}; Import(blobFile, devicePool, config); diff --git a/inference-engine/src/vpu/myriad_plugin/myriad_executable_network.h b/inference-engine/src/vpu/myriad_plugin/myriad_executable_network.h index 51f9d10ad56..94ed3e76b11 100644 --- a/inference-engine/src/vpu/myriad_plugin/myriad_executable_network.h +++ b/inference-engine/src/vpu/myriad_plugin/myriad_executable_network.h @@ -44,7 +44,9 @@ public: virtual ~ExecutableNetwork() { try { - _executor->deallocateGraph(_device, _graphDesc); + if (_device != nullptr) { + _executor->deallocateGraph(_device, _graphDesc); + } } catch (...) { std::cerr << "ERROR ~ExecutableNetwork():\n" @@ -54,18 +56,19 @@ public: ie::IInferRequestInternal::Ptr CreateInferRequestImpl(ie::InputsDataMap networkInputs, ie::OutputsDataMap networkOutputs) override { - if (_device == nullptr || !_device->isBooted()) { + if (!_isNetworkConstant && (_device == nullptr || !_device->isBooted())) { IE_THROW() << "Can not create infer request: there is no available devices with platform " << _device->_platform; } return std::make_shared(_graphDesc, networkInputs, networkOutputs, _inputInfo, _outputInfo, - _graphMetaData.stagesMeta, _config, _log, _executor); + _graphMetaData.stagesMeta, _config, _log, _executor, + _constDatas, _isNetworkConstant); } ie::IInferRequestInternal::Ptr CreateInferRequest() override { - if (_device == nullptr || !_device->isBooted()) { + if (!_isNetworkConstant && (_device == nullptr || !_device->isBooted())) { IE_THROW() << "Can not create infer request: there is no available devices with platform " << _device->_platform; } @@ -73,7 +76,7 @@ public: auto syncRequestImpl = std::make_shared(_graphDesc, _networkInputs, _networkOutputs, _inputInfo, _outputInfo, _graphMetaData.stagesMeta, _config, _log, - _executor); + _executor, _constDatas, _isNetworkConstant); syncRequestImpl->setPointerToExecutableNetworkInternal(shared_from_this()); auto taskExecutorGetResult = getNextTaskExecutor(); return std::make_shared( @@ -84,6 +87,16 @@ public: model.write(_graphBlob.data(), _graphBlob.size()); } + void Export(const std::string &modelFileName) override { + std::ofstream modelFile(modelFileName, std::ios::out | std::ios::binary); + + if (modelFile.is_open()) { + Export(modelFile); + } else { + IE_THROW() << "The " << modelFileName << " file can not be opened for export"; + } + } + ie::Parameter GetMetric(const std::string &name) const override; ie::CNNNetwork GetExecGraphInfo() override; @@ -98,9 +111,11 @@ private: DevicePtr _device; GraphMetaInfo _graphMetaData; MyriadConfiguration _config; + bool _isNetworkConstant = false; const std::shared_ptr _core = nullptr; int _actualNumExecutors = 0; std::vector _supportedMetrics; + std::map _constDatas; DataInfo _inputInfo; DataInfo _outputInfo; @@ -109,9 +124,8 @@ private: std::queue _taskExecutorGetResultIds; ExecutableNetwork(std::shared_ptr mvnc, - std::vector &devicePool, - const MyriadConfiguration& config, - const std::shared_ptr core); + const MyriadConfiguration& config, + const std::shared_ptr core); ie::ITaskExecutor::Ptr getNextTaskExecutor() { std::string id = _taskExecutorGetResultIds.front(); @@ -124,6 +138,8 @@ private: return taskExecutor; } + + void openDevice(std::vector& devicePool); }; } // namespace MyriadPlugin diff --git a/inference-engine/src/vpu/myriad_plugin/myriad_infer_request.cpp b/inference-engine/src/vpu/myriad_plugin/myriad_infer_request.cpp index 020a3d9e580..a3662543a0f 100644 --- a/inference-engine/src/vpu/myriad_plugin/myriad_infer_request.cpp +++ b/inference-engine/src/vpu/myriad_plugin/myriad_infer_request.cpp @@ -33,11 +33,13 @@ MyriadInferRequest::MyriadInferRequest(GraphDesc &graphDesc, const std::vector &blobMetaData, const MyriadConfig& myriadConfig, const Logger::Ptr &log, - const MyriadExecutorPtr &executor) : + const MyriadExecutorPtr &executor, + std::map constDatas, + bool isNetworkConstant = true) : IInferRequestInternal(networkInputs, networkOutputs), _executor(executor), _log(log), _stagesMetaData(blobMetaData), _config(myriadConfig), _inputInfo(compilerInputsInfo), _outputInfo(compilerOutputsInfo), - _graphDesc(graphDesc) { + _graphDesc(graphDesc), _constDatas(constDatas), _isNetworkConstant(isNetworkConstant) { VPU_PROFILE(MyriadInferRequest); const auto& ioStrides = _config.compileConfig().ioStrides; @@ -83,7 +85,7 @@ MyriadInferRequest::MyriadInferRequest(GraphDesc &graphDesc, resultBuffer.resize(compilerOutputsInfo.totalSize); VPU_THROW_UNLESS( - !_networkOutputs.empty() && !_networkInputs.empty(), + !_networkOutputs.empty() && !(_networkInputs.empty() && !_isNetworkConstant), "No information about network's output/input"); } @@ -93,6 +95,9 @@ void MyriadInferRequest::InferImpl() { } void MyriadInferRequest::InferAsync() { + if (_isNetworkConstant) { + return; + } VPU_PROFILE(InferAsync); // execute input pre-processing @@ -104,7 +109,7 @@ void MyriadInferRequest::InferAsync() { auto getOffset = [&inputInfo] (const std::string& name) { const auto offsetIt = inputInfo.offset.find(name); IE_ASSERT(offsetIt != inputInfo.offset.end()) << "MyriadInferRequest::InferAsync()\n" - << "Input offset [" << name << "] is not provided."; + << "Input offset [" << name << "] is not provided."; return offsetIt->second; }; @@ -123,9 +128,9 @@ void MyriadInferRequest::InferAsync() { const auto byteSize = blob->byteSize(); const auto requiredSize = vpu::checked_cast(offset) + byteSize; IE_ASSERT(requiredSize <= inputBuffer.size()) << "MyriadInferRequest::InferAsync()\n" - << "Input offset is too big. " - << "Required size: " << requiredSize - << ", Input buffer size: " << inputBuffer.size(); + << "Input offset is too big. " + << "Required size: " << requiredSize + << ", Input buffer size: " << inputBuffer.size(); const auto foundBlob = getNetInputInfo(name); const auto vpuLayout = foundBlob->second->getTensorDesc().getLayout(); @@ -139,9 +144,8 @@ void MyriadInferRequest::InferAsync() { } _executor->queueInference(_graphDesc, inputBuffer.data(), - _inputInfo.totalSize, nullptr, 0); + _inputInfo.totalSize, nullptr, 0); } - static void copyBlobAccordingUpperBound( const Blob::Ptr& in, const Blob::Ptr& out) { @@ -199,10 +203,22 @@ void MyriadInferRequest::GetResult() { const auto getVpuLayout = [&networkOutputs] (const std::string& name){ const auto foundBlob = networkOutputs.find(name); IE_ASSERT(foundBlob != networkOutputs.end()) << "MyriadInferRequest::InferAsync()\n" - << "Output [" << name << "] is not provided."; + << "Output [" << name << "] is not provided."; return foundBlob->second->getTensorDesc().getLayout(); }; - + if (_isNetworkConstant) { + for (const auto& output : _outputs) { + const auto& ieBlobName = output.first; + const auto& ieBlob = output.second; + IE_ASSERT(_constDatas.find(ieBlobName) != _constDatas.end()) << + "Input [" << ieBlobName << "] is not provided."; + std::copy_n( + _constDatas[ieBlobName]->cbuffer().as(), + _constDatas[ieBlobName]->byteSize(), + ieBlob->buffer().as()); + } + return; + } // For networks with only one output if (_outputInfo.offset.size() == 1) { const auto& it = _outputs.begin(); @@ -224,12 +240,12 @@ void MyriadInferRequest::GetResult() { const auto resultOffset = [&](const std::string& name) { const auto offset_it = _outputInfo.offset.find(name); IE_ASSERT(offset_it != _outputInfo.offset.end()) << "MyriadInferRequest::InferAsync()\n" - << "Output offset [" << name << "] error."; + << "Output offset [" << name << "] error."; const auto offset = vpu::checked_cast(offset_it->second); IE_ASSERT(offset <= resultBuffer.size()) << "MyriadInferRequest::InferAsync()\n" - << "Input offset is too big." - << "Required offset: " << offset - << "Result buffer size: " << resultBuffer.size(); + << "Input offset is too big." + << "Required offset: " << offset + << "Result buffer size: " << resultBuffer.size(); return offset; }; diff --git a/inference-engine/src/vpu/myriad_plugin/myriad_infer_request.h b/inference-engine/src/vpu/myriad_plugin/myriad_infer_request.h index 9373f771fb9..194ce5fc990 100644 --- a/inference-engine/src/vpu/myriad_plugin/myriad_infer_request.h +++ b/inference-engine/src/vpu/myriad_plugin/myriad_infer_request.h @@ -34,6 +34,8 @@ class MyriadInferRequest : public InferenceEngine::IInferRequestInternal { GraphDesc _graphDesc; std::vector resultBuffer; std::vector inputBuffer; + std::map _constDatas; + bool _isNetworkConstant; public: typedef std::shared_ptr Ptr; @@ -46,7 +48,9 @@ public: const std::vector &blobMetaData, const MyriadConfig &myriadConfig, const Logger::Ptr &log, - const MyriadExecutorPtr &executor); + const MyriadExecutorPtr &executor, + std::map constDatas, + bool isNetworkConstant); void InferImpl() override; void InferAsync(); diff --git a/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp b/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp index bdb17b09505..d67ae3e6270 100644 --- a/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp +++ b/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp @@ -23,8 +23,6 @@ std::vector disabledTestPatterns() { R"(.*IEClassGetAvailableDevices.*)", // TODO: Issue: 40473 R"(.*TopKLayerTest.*mode=min.*sort=index.*)", - // TODO: Issue: 40961 - R"(.*(ConstantResultSubgraphTest).*)", // TODO: Issue: 42828 R"(.*DSR_NonMaxSuppression.*NBoxes=(5|20|200).*)", // TODO: Issue: 42721 diff --git a/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/subgraph_tests/constant_result.cpp b/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/subgraph_tests/constant_result.cpp index 919fcdef3d0..2b47c8eb466 100644 --- a/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/subgraph_tests/constant_result.cpp +++ b/inference-engine/tests/functional/plugin/myriad/shared_tests_instances/subgraph_tests/constant_result.cpp @@ -23,15 +23,7 @@ const std::vector shapes = { }; const std::vector precisions = { - Precision::U8, - Precision::I8, - Precision::U16, - Precision::I16, - Precision::I32, - Precision::U64, - Precision::I64, - Precision::FP32, - Precision::BOOL + Precision::FP32 }; INSTANTIATE_TEST_SUITE_P(smoke_Check, ConstantResultSubgraphTest,