IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Introduce model reader which works only with new API (#19077)

Browse Source 
* Introduce model reader which works only with new API

* Fix GNA compilation

* Removed old code

* Fixed Windows build

* Remove legacy headers from core_impl

* Fixed caching tests if plugin on legacy API call ReadNetwork
This commit is contained in:
Ilya Churaev
2023-08-21 15:42:36 +04:00
committed by GitHub
parent 19ff7fba3d
commit cbe744b717
13 changed files with 398 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/inference/dev_api/openvino/runtime/internal_properties.hpp
View File

@@ -47,11 +47,8 @@ static constexpr Property<std::string, PropertyMutability::WO> config_device_id{
* @brief The name for setting CPU affinity per thread option.
*
* It is passed to Core::get_property()
* PluginConfigParams::NO (no pinning for CPU inference threads)
* PluginConfigParams::YES, which is default on the conventional CPUs (pinning threads to cores, best for static
* benchmarks),
*
* the following options are implemented only for the TBB as a threading option
* The following options are implemented only for the TBB as a threading option
* ov::threading::IStreamsExecutor::ThreadBindingType::NUMA (pinning threads to NUMA nodes, best for real-life,
* contented cases) on the Windows and MacOS* this option behaves as YES
* ov::threading::IStreamsExecutor::ThreadBindingType::HYBRID_AWARE (let the runtime to do pinning to the cores types,

4
src/inference/src/cache_guard.hpp
View File

@@ -5,7 +5,7 @@
#pragma once
/**
* @brief This is a header file for the Inference Engine Cache Guard class C++ API
* @brief This is a header file for the OpenVINO Cache Guard class C++ API
*
* @file cache_guard.hpp
*/
@@ -69,7 +69,7 @@ private:
/**
* @brief This class holds a table of currently locked hashes
* Inference engine core will need to obtain a lock for a specific cache to get exclusive access to it
* OpenVINO core will need to obtain a lock for a specific cache to get exclusive access to it
* It is needed to avoid race situations when multiple threads try to to write to the same cache simultaneously
*
* Usage example:

6
src/inference/src/compilation_context.hpp
View File

@@ -12,12 +12,6 @@
#include "openvino/core/any.hpp"
namespace InferenceEngine {
class CNNNetwork;
}
namespace ov {
class Tensor;

74
src/inference/src/dev/converter_utils.cpp
View File

@@ -24,6 +24,7 @@
#include "ie_plugin_config.hpp"
#include "ie_version.hpp"
#include "iplugin_wrapper.hpp"
#include "legacy_op_extension.hpp"
#include "openvino/core/except.hpp"
#include "openvino/op/parameter.hpp"
#include "openvino/runtime/exception.hpp"
@@ -921,3 +922,76 @@ ov::SoPtr<ov::IRemoteContext> ov::legacy_convert::convert_remote_context(
}
return {std::make_shared<InferenceEngine::IRemoteContextWrapper>(context)};
}
namespace ov {
/*
* @brief Wrapper for old IE extensions to new API
*/
class ExtensionWrapper : public ov::LegacyOpExtension {
public:
ExtensionWrapper(const InferenceEngine::IExtensionPtr& ext, const std::string& opset, const std::string& name)
: m_ext(ext),
m_opset_name(opset),
m_type(name),
m_ext_type(m_type.c_str(), m_opset_name.c_str()) {}
~ExtensionWrapper() override = default;
const ov::DiscreteTypeInfo& get_type_info() const override {
return m_ext_type;
}
ngraph::OutputVector create(const ngraph::OutputVector& inputs, ngraph::AttributeVisitor& visitor) const override {
std::shared_ptr<ngraph::Node> node(m_ext->getOpSets().at(m_opset_name).create_insensitive(m_ext_type.name));
node->set_arguments(inputs);
if (node->visit_attributes(visitor)) {
node->constructor_validate_and_infer_types();
}
return node->outputs();
}
std::vector<ov::Extension::Ptr> get_attached_extensions() const override {
return {};
}
const InferenceEngine::IExtensionPtr& get_extension() const {
return m_ext;
}
private:
InferenceEngine::IExtensionPtr m_ext;
std::string m_opset_name;
std::string m_type;
ov::DiscreteTypeInfo m_ext_type;
};
} // namespace ov
std::vector<ov::Extension::Ptr> ov::legacy_convert::convert_extension(
const std::vector<InferenceEngine::IExtensionPtr>& exts) {
std::vector<ov::Extension::Ptr> extensions;
for (const auto& ext : exts) {
for (const auto& item : ext->getOpSets()) {
for (const auto& type_info : item.second.get_types_info()) {
extensions.emplace_back(std::make_shared<ov::ExtensionWrapper>(ext, item.first, type_info.name));
}
}
}
return extensions;
}
std::vector<InferenceEngine::IExtensionPtr> ov::legacy_convert::convert_extension(
const std::vector<ov::Extension::Ptr>& exts) {
std::vector<InferenceEngine::IExtensionPtr> extensions;
std::unordered_set<InferenceEngine::IExtensionPtr> existed_extensions;
for (const auto& ext : exts) {
if (const auto& wrapper = std::dynamic_pointer_cast<ov::ExtensionWrapper>(ext)) {
if (!existed_extensions.count(wrapper->get_extension())) {
extensions.emplace_back(wrapper->get_extension());
existed_extensions.insert(wrapper->get_extension());
}
}
}
return extensions;
}

5
src/inference/src/dev/converter_utils.hpp
View File

@@ -7,7 +7,9 @@
#include "cpp/ie_cnn_network.h"
#include "cpp_interfaces/interface/ie_iinfer_request_internal.hpp"
#include "cpp_interfaces/interface/ie_iplugin_internal.hpp"
#include "ie_iextension.h"
#include "ie_remote_blob.hpp"
#include "openvino/core/extension.hpp"
#include "openvino/core/model.hpp"
#include "openvino/runtime/iasync_infer_request.hpp"
#include "openvino/runtime/icompiled_model.hpp"
@@ -40,6 +42,9 @@ ov::SoPtr<::ov::IAsyncInferRequest> convert_infer_request(
std::shared_ptr<InferenceEngine::RemoteContext> convert_remote_context(const ov::SoPtr<ov::IRemoteContext>& context);
ov::SoPtr<ov::IRemoteContext> convert_remote_context(const std::shared_ptr<InferenceEngine::RemoteContext>& context);
std::vector<ov::Extension::Ptr> convert_extension(const std::vector<InferenceEngine::IExtensionPtr>& exts);
std::vector<InferenceEngine::IExtensionPtr> convert_extension(const std::vector<ov::Extension::Ptr>& exts);
} // namespace legacy_convert
} // namespace ov

56
src/inference/src/dev/core_impl.cpp
View File

@@ -6,21 +6,13 @@
#include <memory>
#include "any_copy.hpp"
#include "check_network_batchable.hpp"
#include "compilation_context.hpp"
#include "cpp_interfaces/interface/ie_iexecutable_network_internal.hpp"
#include "cpp_interfaces/interface/ie_internal_plugin_config.hpp"
#include "cpp_interfaces/interface/ie_iplugin_internal.hpp"
#include "dev/converter_utils.hpp"
#include "dev/icompiled_model_wrapper.hpp"
#include "file_utils.h"
#include "ie_network_reader.hpp"
#include "ie_ngraph_utils.hpp"
#include "iplugin_wrapper.hpp"
#include "dev/iplugin_wrapper.hpp"
#include "itt.hpp"
#include "ngraph/op/constant.hpp"
#include "ngraph/pass/constant_folding.hpp"
#include "model_reader.hpp"
#include "openvino/core/any.hpp"
#include "openvino/core/except.hpp"
#include "openvino/core/op_extension.hpp"
@@ -30,6 +22,7 @@
#include "openvino/pass/manager.hpp"
#include "openvino/runtime/device_id_parser.hpp"
#include "openvino/runtime/icompiled_model.hpp"
#include "openvino/runtime/internal_properties.hpp"
#include "openvino/runtime/itensor.hpp"
#include "openvino/runtime/make_tensor.hpp"
#include "openvino/runtime/remote_context.hpp"
@@ -39,11 +32,11 @@
#include "openvino/util/shared_object.hpp"
#include "ov_plugins.hpp"
#include "preprocessing/preprocessing.hpp"
#include "xml_parse_utils.h"
#ifdef PROXY_PLUGIN_ENABLED
# include "openvino/proxy/plugin.hpp"
# include "openvino/proxy/properties.hpp"
#endif
#include "xml_parse_utils.h"
ov::ICore::~ICore() = default;
@@ -607,7 +600,8 @@ ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const {
// Add registered extensions to new plugin
allowNotImplemented([&]() {
for (const auto& ext : extensions) {
auto old_extensions = ov::legacy_convert::convert_extension(extensions);
for (const auto& ext : old_extensions) {
plugin.add_extension(ext);
}
});
@@ -838,19 +832,16 @@ ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::compile_model(const std::string& mod
std::unique_ptr<CacheGuardEntry> lock = cacheGuard.get_hash_lock(cacheContent.blobId);
compiled_model =
load_model_from_cache(cacheContent, plugin, parsed._config, ov::SoPtr<ov::IRemoteContext>{}, [&]() {
auto cnnNetwork = ReadNetwork(model_path, std::string());
return compile_model_and_cache(cnnNetwork.getFunction(), plugin, parsed._config, {}, cacheContent);
auto model = read_model(model_path, std::string{});
return compile_model_and_cache(model, plugin, parsed._config, {}, cacheContent);
});
} else if (cacheManager) {
// this code path is enabled for AUTO / MULTI / BATCH devices which don't support
// import / export explicitly, but can redirect this functionality to actual HW plugin
compiled_model = plugin.compile_model(model_path, parsed._config);
} else {
auto cnnNetwork = ReadNetwork(model_path, std::string());
compiled_model = compile_model_with_preprocess(plugin,
cnnNetwork.getFunction(),
ov::SoPtr<ov::IRemoteContext>{},
parsed._config);
auto model = read_model(model_path, std::string());
compiled_model = compile_model_with_preprocess(plugin, model, ov::SoPtr<ov::IRemoteContext>{}, parsed._config);
}
return compiled_model;
}
@@ -874,8 +865,8 @@ ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::compile_model(const std::string& mod
std::unique_ptr<CacheGuardEntry> lock = cacheGuard.get_hash_lock(cacheContent.blobId);
compiled_model =
load_model_from_cache(cacheContent, plugin, parsed._config, ov::SoPtr<ov::IRemoteContext>{}, [&]() {
auto cnnNetwork = read_model(model_str, weights);
return compile_model_and_cache(cnnNetwork,
auto model = read_model(model_str, weights);
return compile_model_and_cache(model,
plugin,
parsed._config,
ov::SoPtr<ov::IRemoteContext>{},
@@ -1373,15 +1364,15 @@ void ov::CoreImpl::set_property_for_device(const ov::AnyMap& configMap, const st
});
}
}
void ov::CoreImpl::add_extensions_unsafe(const std::vector<ov::Extension::Ptr>& extensions) const {
for (const auto& ext : extensions) {
ov_extensions.emplace_back(ext);
void ov::CoreImpl::add_extensions_unsafe(const std::vector<ov::Extension::Ptr>& exts) const {
for (const auto& ext : exts) {
extensions.emplace_back(ext);
auto ext_obj = ext;
if (auto so_ext = std::dynamic_pointer_cast<ov::detail::SOExtension>(ext_obj))
ext_obj = so_ext->extension();
if (auto op_base_ext = std::dynamic_pointer_cast<ov::BaseOpExtension>(ext_obj)) {
for (const auto& attached_ext : op_base_ext->get_attached_extensions()) {
ov_extensions.emplace_back(attached_ext);
extensions.emplace_back(attached_ext);
}
}
}
@@ -1392,10 +1383,6 @@ void ov::CoreImpl::add_extension(const std::vector<ov::Extension::Ptr>& extensio
add_extensions_unsafe(extensions);
}
const std::vector<InferenceEngine::IExtensionPtr>& ov::CoreImpl::GetExtensions() const {
return extensions;
}
bool ov::CoreImpl::device_supports_model_caching(const std::string& device_name) const {
auto parsed = parseDeviceNameIntoConfig(device_name);
return device_supports_model_caching(get_plugin(parsed._deviceName));
@@ -1552,7 +1539,8 @@ void ov::CoreImpl::AddExtensionUnsafe(const InferenceEngine::IExtensionPtr& exte
plugin.second.add_extension(extension);
});
}
extensions.emplace_back(extension);
for (const auto& ext : ov::legacy_convert::convert_extension({extension}))
extensions.emplace_back(ext);
}
void ov::CoreImpl::CoreConfig::set_and_update(ov::AnyMap& config) {
@@ -1650,16 +1638,12 @@ void ov::CoreImpl::add_mutex(const std::string& dev_name) {
std::shared_ptr<ov::Model> ov::CoreImpl::read_model(const std::string& modelPath, const std::string& binPath) const {
OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::ReadTime, "CoreImpl::read_model from file");
return ReadNetwork(modelPath, binPath).getFunction();
return ov::util::read_model(modelPath, binPath, extensions, coreConfig.get_enable_mmap());
}
std::shared_ptr<ov::Model> ov::CoreImpl::read_model(const std::string& model,
const ov::Tensor& weights,
bool frontendMode) const {
InferenceEngine::Blob::Ptr blob;
if (weights) {
blob = tensor_to_blob(get_tensor_impl(weights));
}
OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::ReadTime, "CoreImpl::read_model from memory");
return ReadNetwork(model, blob, frontendMode).getFunction();
return ov::util::read_model(model, weights, extensions, frontendMode);
}

10
src/inference/src/dev/core_impl.hpp
View File

@@ -147,9 +147,7 @@ private:
std::shared_ptr<ov::threading::ExecutorManager> m_executor_manager;
mutable std::unordered_set<std::string> opsetNames;
// TODO: make extensions to be optional with conditional compilation
mutable std::vector<InferenceEngine::IExtensionPtr> extensions;
mutable std::vector<ov::Extension::Ptr> ov_extensions;
mutable std::vector<ov::Extension::Ptr> extensions;
std::map<std::string, PluginDescriptor> pluginRegistry;
@@ -312,12 +310,6 @@ public:
*/
void AddExtension(const InferenceEngine::IExtensionPtr& extension);
/**
* @brief Provides a list of extensions
* @return A list of registered extensions
*/
const std::vector<InferenceEngine::IExtensionPtr>& GetExtensions() const;
bool DeviceSupportsModelCaching(const std::string& deviceName) const override;
std::map<std::string, InferenceEngine::Version> GetVersions(const std::string& deviceName) const;

5
src/inference/src/dev/core_impl_ie.cpp
View File

@@ -53,8 +53,7 @@ InferenceEngine::CNNNetwork ov::CoreImpl::ReadNetwork(const std::string& modelPa
return InferenceEngine::details::ReadNetwork(modelPath,
binPath,
extensions,
ov_extensions,
is_new_api(),
isNewAPI(),
coreConfig.get_enable_mmap());
}
@@ -62,7 +61,7 @@ InferenceEngine::CNNNetwork ov::CoreImpl::ReadNetwork(const std::string& model,
const InferenceEngine::Blob::CPtr& weights,
bool frontendMode) const {
OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::ReadTime, "CoreImpl::ReadNetwork from memory");
return InferenceEngine::details::ReadNetwork(model, weights, extensions, ov_extensions, is_new_api(), frontendMode);
return InferenceEngine::details::ReadNetwork(model, weights, extensions, isNewAPI(), frontendMode);
}
ov::SoPtr<InferenceEngine::IExecutableNetworkInternal> ov::CoreImpl::LoadNetwork(

264
src/inference/src/ie_network_reader.cpp
View File

@@ -13,6 +13,7 @@
#include "cnn_network_ngraph_impl.hpp"
#include "cpp/ie_cnn_network.h"
#include "dev/converter_utils.hpp"
#include "file_utils.h"
#include "ie_api.h"
#include "ie_common.h"
@@ -36,46 +37,6 @@
#include "transformations/rt_info/old_api_map_order_attribute.hpp"
#include "transformations/utils/utils.hpp"
namespace ov {
/*
* @brief Wrapper for old IE extensions to new API
*/
class ExtensionWrapper : public ov::LegacyOpExtension {
public:
ExtensionWrapper(const InferenceEngine::IExtensionPtr& ext, const std::string& opset, const std::string& name)
: m_ext(ext),
m_opset_name(opset),
m_type(name),
m_ext_type(m_type.c_str(), m_opset_name.c_str()) {}
const ov::DiscreteTypeInfo& get_type_info() const override {
return m_ext_type;
}
ngraph::OutputVector create(const ngraph::OutputVector& inputs, ngraph::AttributeVisitor& visitor) const override {
std::shared_ptr<ngraph::Node> node(m_ext->getOpSets().at(m_opset_name).create_insensitive(m_ext_type.name));
node->set_arguments(inputs);
if (node->visit_attributes(visitor)) {
node->constructor_validate_and_infer_types();
}
return node->outputs();
}
std::vector<ov::Extension::Ptr> get_attached_extensions() const override {
return {};
}
private:
InferenceEngine::IExtensionPtr m_ext;
std::string m_opset_name;
std::string m_type;
ov::DiscreteTypeInfo m_ext_type;
};
} // namespace ov
namespace InferenceEngine {
#ifdef ENABLE_IR_V7_READER
@@ -255,175 +216,86 @@ namespace {
CNNNetwork convert_to_cnnnetwork(std::shared_ptr<ngraph::Function>& function,
const std::vector<IExtensionPtr>& exts,
bool newAPI,
bool is_new_api,
bool frontendMode = false) {
auto& rt_info = function->get_rt_info();
const bool is_ir = function->has_rt_info("version");
// only for IR cases we need preprocessing or postprocessing steps
if (is_ir) {
IR_READER_SCOPE(is_ir);
using namespace ov::preprocess;
PrePostProcessor prepost(function);
if (function->has_rt_info("version") && function->get_rt_info<int64_t>("version") == 11 && !is_new_api) {
IR_READER_SCOPE(ir11_old_api);
ov::preprocess::PrePostProcessor prepost(function);
const int64_t ir_version = function->get_rt_info<int64_t>("version");
const std::string& old_api_map_key_order = ov::OldApiMapOrder::get_type_info_static();
const std::string& old_api_map_key_type = ov::OldApiMapElementType::get_type_info_static();
if (ir_version == 10 && newAPI) {
IR_READER_SCOPE(ir10_new_api);
std::unordered_map<std::string, std::shared_ptr<ov::descriptor::Tensor>> leaf_names;
const auto inputs = function->inputs();
for (size_t i = 0; i < inputs.size(); ++i) {
if (!frontendMode) {
const auto ngraph_type = inputs[i].get_element_type();
const auto legacy_type = details::toLegacyType(ngraph_type, true);
prepost.input(i).tensor().set_element_type(legacy_type);
}
for (const auto& name : inputs[i].get_names()) {
OPENVINO_ASSERT(leaf_names.find(name) == leaf_names.end(),
"Model tensor names have collisions.",
" Please use MO to generate new IR version, it should allow to avoid the issue");
leaf_names.emplace(name, inputs[i].get_tensor_ptr());
bool need_validate_nodes_and_infer_types = false;
auto& parameters = function->get_parameters();
for (size_t i = 0; i < parameters.size(); ++i) {
const auto& parameter = parameters[i];
ov::RTMap& rtInfo = parameter->get_rt_info();
const auto it_type = rtInfo.find(old_api_map_key_type);
auto& pre_input = prepost.input(i);
if (it_type != rtInfo.end()) {
const auto old_api_map_type = it_type->second.as<ov::OldApiMapElementType>().value;
const auto param_type = parameter->get_element_type();
// In the following code we add Convert node from old_api_map_type to Parameter type
// using PrePostProcessor. As some plugins do not support uint8 type, Convert to uint8 leads
// to error, so for such case type is set directly to Parameter node instead of inserting Convert.
if ((param_type == ngraph::element::u8 && old_api_map_type.is_real())) {
parameter->set_element_type(old_api_map_type);
need_validate_nodes_and_infer_types = true;
} else {
pre_input.tensor().set_element_type(old_api_map_type);
}
OPENVINO_ASSERT(!old_api_map_type.is_dynamic(), "Old API map does not support dynamic type");
rtInfo.erase(it_type);
}
const auto outputs = function->outputs();
for (size_t i = 0; i < outputs.size(); ++i) {
if (!frontendMode) {
const auto ngraph_type = outputs[i].get_element_type();
const auto legacy_type = details::toLegacyType(ngraph_type, false);
prepost.output(i).tensor().set_element_type(legacy_type);
}
for (const auto& name : outputs[i].get_names()) {
auto tensor_it = leaf_names.find(name);
OPENVINO_ASSERT(tensor_it == leaf_names.end() || tensor_it->second == outputs[i].get_tensor_ptr(),
"Model tensor names have collisions.",
" Please use MO to generate new IR version, it should allow to avoid the issue");
leaf_names.emplace(name, outputs[i].get_tensor_ptr());
}
const auto it_order = rtInfo.find(old_api_map_key_order);
if (it_order != rtInfo.end()) {
const auto order = it_order->second.as<ov::OldApiMapOrder>().value;
pre_input.preprocess().convert_layout(order);
rtInfo.erase(it_order);
}
// in order to support the following scenarios for IR v10 cases:
// ov::Model f = ie.read_model(..);
// f.input("input_operation_name");
// f.output("output_operation_name");
// f.add_output("operation_name[].port_index]");
// f.reshape({ { "input_operation_name", ov::PartialShape{} } });
// we need to add operation names as tensor names for inputs and outputs
{
for (const auto& result : function->get_results()) {
auto res_name = ov::op::util::create_ie_output_name(result->input_value(0));
OPENVINO_ASSERT(
leaf_names.find(res_name) == leaf_names.end() ||
result->output(0).get_names().find(res_name) != result->output(0).get_names().end(),
"Model operation names have collisions with tensor names.",
" Please use MO to generate new IR version, it should allow to avoid the issue");
leaf_names.emplace(res_name, nullptr);
result->output(0).get_tensor().add_names({res_name});
}
for (const auto& param : function->get_parameters()) {
auto param_name = param->get_friendly_name();
OPENVINO_ASSERT(
leaf_names.find(param_name) == leaf_names.end() ||
param->output(0).get_names().find(param_name) != param->output(0).get_names().end(),
"Model operation names have collisions with tensor names.",
" Please use MO to generate new IR version, it should allow to avoid the issue");
leaf_names.emplace(param_name, nullptr);
param->output(0).get_tensor().add_names({param_name});
}
}
function = prepost.build();
// Set version to 10
rt_info["version"] = int64_t(10);
} else if (ir_version == 11 && !newAPI) {
IR_READER_SCOPE(ir11_old_api);
const std::string& old_api_map_key_order = ov::OldApiMapOrder::get_type_info_static();
const std::string& old_api_map_key_type = ov::OldApiMapElementType::get_type_info_static();
bool need_validate_nodes_and_infer_types = false;
auto& parameters = function->get_parameters();
for (size_t i = 0; i < parameters.size(); ++i) {
const auto& parameter = parameters[i];
ov::RTMap& rtInfo = parameter->get_rt_info();
const auto it_type = rtInfo.find(old_api_map_key_type);
auto& pre_input = prepost.input(i);
if (it_type != rtInfo.end()) {
const auto old_api_map_type = it_type->second.as<ov::OldApiMapElementType>().value;
const auto param_type = parameter->get_element_type();
// In the following code we add Convert node from old_api_map_type to Parameter type
// using PrePostProcessor. As some plugins do not support uint8 type, Convert to uint8 leads
// to error, so for such case type is set directly to Parameter node instead of inserting Convert.
if ((param_type == ngraph::element::u8 && old_api_map_type.is_real())) {
parameter->set_element_type(old_api_map_type);
need_validate_nodes_and_infer_types = true;
} else {
pre_input.tensor().set_element_type(old_api_map_type);
}
OPENVINO_ASSERT(!old_api_map_type.is_dynamic(), "Old API map does not support dynamic type");
rtInfo.erase(it_type);
}
const auto it_order = rtInfo.find(old_api_map_key_order);
if (it_order != rtInfo.end()) {
const auto order = it_order->second.as<ov::OldApiMapOrder>().value;
pre_input.preprocess().convert_layout(order);
rtInfo.erase(it_order);
}
}
auto& results = function->get_results();
for (size_t i = 0; i < results.size(); ++i) {
const auto& result = results[i];
ov::RTMap& rtInfo = result->get_rt_info();
const auto it = rtInfo.find(old_api_map_key_order);
if (it == rtInfo.end())
continue;
const auto order = it->second.as<ov::OldApiMapOrder>().value;
auto& post_output = prepost.output(i);
post_output.postprocess().convert_layout(order);
// remove old api once we applied it
rtInfo.erase(it);
}
if (need_validate_nodes_and_infer_types)
function->validate_nodes_and_infer_types();
// Set version to 10
rt_info["version"] = int64_t(10);
function = prepost.build();
}
auto& results = function->get_results();
for (size_t i = 0; i < results.size(); ++i) {
const auto& result = results[i];
ov::RTMap& rtInfo = result->get_rt_info();
const auto it = rtInfo.find(old_api_map_key_order);
if (it == rtInfo.end())
continue;
const auto order = it->second.as<ov::OldApiMapOrder>().value;
auto& post_output = prepost.output(i);
post_output.postprocess().convert_layout(order);
// remove old api once we applied it
rtInfo.erase(it);
}
if (need_validate_nodes_and_infer_types)
function->validate_nodes_and_infer_types();
// Set version to 10
function->set_rt_info<int64_t>(10, "version");
function = prepost.build();
}
OPENVINO_SUPPRESS_DEPRECATED_START
return CNNNetwork(std::make_shared<details::CNNNetworkNGraphImpl>(function, exts, newAPI));
return CNNNetwork(std::make_shared<details::CNNNetworkNGraphImpl>(function, exts, is_new_api));
OPENVINO_SUPPRESS_DEPRECATED_END
}
std::vector<ov::Extension::Ptr> wrap_old_extensions(const std::vector<InferenceEngine::IExtensionPtr>& exts) {
std::vector<ov::Extension::Ptr> extensions;
for (const auto& ext : exts) {
for (const auto& item : ext->getOpSets()) {
for (const auto& type_info : item.second.get_types_info()) {
extensions.emplace_back(std::make_shared<ov::ExtensionWrapper>(ext, item.first, type_info.name));
}
}
}
return extensions;
}
} // namespace
CNNNetwork details::ReadNetwork(const std::string& modelPath,
const std::string& binPath,
const std::vector<IExtensionPtr>& exts,
const std::vector<ov::Extension::Ptr>& ov_exts,
bool newAPI,
bool is_new_api,
bool enable_mmap) {
auto exts = ov::legacy_convert::convert_extension(ov_exts);
#ifdef ENABLE_IR_V7_READER
// IR v7 obsolete code
{
@@ -433,7 +305,6 @@ CNNNetwork details::ReadNetwork(const std::string& modelPath,
OPENVINO_SUPPRESS_DEPRECATED_START
if (static_cast<ICNNNetwork::Ptr>(cnnnetwork) != nullptr) {
OPENVINO_ASSERT(!newAPI, "Cannot read IR v7 from OpenVINO 2.0 API");
return cnnnetwork;
}
OPENVINO_SUPPRESS_DEPRECATED_END
@@ -467,14 +338,12 @@ CNNNetwork details::ReadNetwork(const std::string& modelPath,
FE = manager.load_by_model(params);
if (FE) {
FE->add_extension(ov_exts);
if (!exts.empty())
FE->add_extension(wrap_old_extensions(exts));
inputModel = FE->load(params);
}
if (inputModel) {
auto ngFunc = FE->convert(inputModel);
return convert_to_cnnnetwork(ngFunc, exts, newAPI);
return convert_to_cnnnetwork(ngFunc, exts, is_new_api);
}
const auto fileExt = modelPath.substr(modelPath.find_last_of(".") + 1);
@@ -489,12 +358,12 @@ CNNNetwork details::ReadNetwork(const std::string& modelPath,
CNNNetwork details::ReadNetwork(const std::string& model,
const Blob::CPtr& weights,
const std::vector<IExtensionPtr>& exts,
const std::vector<ov::Extension::Ptr>& ov_exts,
bool newAPI,
bool is_new_api,
bool frontendMode) {
std::istringstream modelStringStream(model);
std::istream& modelStream = modelStringStream;
auto exts = ov::legacy_convert::convert_extension(ov_exts);
#ifdef ENABLE_IR_V7_READER
// IR v7 obsolete code
@@ -504,7 +373,6 @@ CNNNetwork details::ReadNetwork(const std::string& model,
assertIfIRv7LikeModel(modelStream);
if (reader_irv7 && reader_irv7->supportModel(modelStream)) {
OPENVINO_ASSERT(!newAPI, "Cannot read IR v7 from OpenVINO 2.0 API");
if (weights)
return reader_irv7->read(modelStream, weights, exts);
return reader_irv7->read(modelStream, exts);
@@ -528,13 +396,11 @@ CNNNetwork details::ReadNetwork(const std::string& model,
FE = manager.load_by_model(params);
if (FE) {
FE->add_extension(ov_exts);
if (!exts.empty())
FE->add_extension(wrap_old_extensions(exts));
inputModel = FE->load(params);
}
if (inputModel) {
auto ngFunc = FE->convert(inputModel);
return convert_to_cnnnetwork(ngFunc, exts, newAPI, frontendMode);
return convert_to_cnnnetwork(ngFunc, exts, is_new_api, frontendMode);
}
IE_THROW(NetworkNotRead)

10
src/inference/src/ie_network_reader.hpp
View File

@@ -19,33 +19,27 @@ namespace details {
* @param modelPath path to IR file
* @param binPath path to bin file, if path is empty, will try to read bin file with the same name as xml and
* if bin file with the same name was not found, will load IR without weights.
* @param exts vector with extensions
* @param ov_exts vector with OpenVINO extensions
* @param newAPI Whether this function is called from OpenVINO 2.0 API
* @param enable_mmap boolean to enable/disable `mmap` use in Frontend
* @return CNNNetwork
*/
CNNNetwork ReadNetwork(const std::string& modelPath,
const std::string& binPath,
const std::vector<IExtensionPtr>& exts,
const std::vector<ov::Extension::Ptr>& ov_exts,
bool newAPI,
bool is_new_api,
bool enable_mmap);
/**
* @brief Reads IR xml and bin (with the same name) files
* @param model string with IR
* @param weights shared pointer to constant blob with weights
* @param exts vector with extensions
* @param ov_exts vector with OpenVINO extensions
* @param newAPI Whether this function is called from OpenVINO 2.0 API
* @param frontendMode read network without post-processing or other transformations
* @return CNNNetwork
*/
CNNNetwork ReadNetwork(const std::string& model,
const Blob::CPtr& weights,
const std::vector<IExtensionPtr>& exts,
const std::vector<ov::Extension::Ptr>& ov_exts,
bool newAPI,
bool is_new_api,
bool frontendMode = false);
} // namespace details

180
src/inference/src/model_reader.cpp Normal file
View File

@@ -0,0 +1,180 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "model_reader.hpp"
#include "cnn_network_ngraph_impl.hpp"
#include "itt.hpp"
#include "openvino/core/model.hpp"
#include "openvino/core/preprocess/pre_post_process.hpp"
#include "openvino/frontend/manager.hpp"
#include "openvino/util/file_util.hpp"
#include "transformations/utils/utils.hpp"
namespace {
void update_v10_model(std::shared_ptr<ov::Model>& model, bool frontendMode = false) {
// only for IR cases we need preprocessing or postprocessing steps
if (model->has_rt_info("version") && model->get_rt_info<int64_t>("version") == 10) {
IR_READER_SCOPE(ir10_new_api);
ov::preprocess::PrePostProcessor prepost(model);
std::unordered_map<std::string, std::shared_ptr<ov::descriptor::Tensor>> leaf_names;
const auto inputs = model->inputs();
for (size_t i = 0; i < inputs.size(); ++i) {
if (!frontendMode) {
const auto ov_type = inputs[i].get_element_type();
const auto legacy_type = InferenceEngine::details::toLegacyType(ov_type, true);
prepost.input(i).tensor().set_element_type(legacy_type);
}
for (const auto& name : inputs[i].get_names()) {
OPENVINO_ASSERT(leaf_names.find(name) == leaf_names.end(),
"Model tensor names have collisions.",
" Please use MO to generate new IR version, it should allow to avoid the issue");
leaf_names.emplace(name, inputs[i].get_tensor_ptr());
}
}
const auto outputs = model->outputs();
for (size_t i = 0; i < outputs.size(); ++i) {
if (!frontendMode) {
const auto ngraph_type = outputs[i].get_element_type();
const auto legacy_type = InferenceEngine::details::toLegacyType(ngraph_type, false);
prepost.output(i).tensor().set_element_type(legacy_type);
}
for (const auto& name : outputs[i].get_names()) {
auto tensor_it = leaf_names.find(name);
OPENVINO_ASSERT(tensor_it == leaf_names.end() || tensor_it->second == outputs[i].get_tensor_ptr(),
"Model tensor names have collisions.",
" Please use MO to generate new IR version, it should allow to avoid the issue");
leaf_names.emplace(name, outputs[i].get_tensor_ptr());
}
}
// in order to support the following scenarios for IR v10 cases:
// ov::Model f = ie.read_model(..);
// f.input("input_operation_name");
// f.output("output_operation_name");
// f.add_output("operation_name[].port_index]");
// f.reshape({ { "input_operation_name", ov::PartialShape{} } });
// we need to add operation names as tensor names for inputs and outputs
{
for (const auto& result : model->get_results()) {
auto res_name = ov::op::util::create_ie_output_name(result->input_value(0));
OPENVINO_ASSERT(leaf_names.find(res_name) == leaf_names.end() ||
result->output(0).get_names().find(res_name) != result->output(0).get_names().end(),
"Model operation names have collisions with tensor names.",
" Please use MO to generate new IR version, it should allow to avoid the issue");
leaf_names.emplace(res_name, nullptr);
result->output(0).get_tensor().add_names({res_name});
}
for (const auto& param : model->get_parameters()) {
auto param_name = param->get_friendly_name();
OPENVINO_ASSERT(leaf_names.find(param_name) == leaf_names.end() ||
param->output(0).get_names().find(param_name) != param->output(0).get_names().end(),
"Model operation names have collisions with tensor names.",
" Please use MO to generate new IR version, it should allow to avoid the issue");
leaf_names.emplace(param_name, nullptr);
param->output(0).get_tensor().add_names({param_name});
}
}
model = prepost.build();
}
}
} // namespace
namespace ov {
namespace util {
std::shared_ptr<ov::Model> read_model(const std::string& modelPath,
const std::string& binPath,
const std::vector<ov::Extension::Ptr>& extensions,
bool enable_mmap) {
// Fix unicode name
#if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32)
std::wstring model_path = ov::util::string_to_wstring(modelPath.c_str());
#else
std::string model_path = modelPath;
#endif
// Try to load with FrontEndManager
ov::frontend::FrontEndManager manager;
ov::frontend::FrontEnd::Ptr FE;
ov::frontend::InputModel::Ptr inputModel;
ov::AnyVector params{model_path};
if (!binPath.empty()) {
#if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32)
const std::wstring& weights_path = ov::util::string_to_wstring(binPath.c_str());
#else
const std::string& weights_path = binPath;
#endif
params.emplace_back(weights_path);
}
params.emplace_back(enable_mmap);
FE = manager.load_by_model(params);
if (FE) {
FE->add_extension(extensions);
inputModel = FE->load(params);
}
if (inputModel) {
auto model = FE->convert(inputModel);
update_v10_model(model);
return model;
}
const auto fileExt = modelPath.substr(modelPath.find_last_of(".") + 1);
std::string FEs;
for (const auto& fe_name : manager.get_available_front_ends())
FEs += fe_name + " ";
OPENVINO_THROW("Unable to read the model: ",
modelPath,
" Please check that model format: ",
fileExt,
" is supported and the model is correct.",
" Available frontends: ",
FEs);
}
std::shared_ptr<ov::Model> read_model(const std::string& model,
const ov::Tensor& weights,
const std::vector<ov::Extension::Ptr>& ov_exts,
bool frontendMode) {
std::istringstream modelStringStream(model);
std::istream& modelStream = modelStringStream;
// Try to load with FrontEndManager
ov::frontend::FrontEndManager manager;
ov::frontend::FrontEnd::Ptr FE;
ov::frontend::InputModel::Ptr inputModel;
ov::AnyVector params{&modelStream};
if (weights) {
std::shared_ptr<ngraph::runtime::AlignedBuffer> weights_buffer =
std::make_shared<ngraph::runtime::SharedBuffer<ov::Tensor>>(reinterpret_cast<char*>(weights.data()),
weights.get_byte_size(),
weights);
params.emplace_back(weights_buffer);
}
FE = manager.load_by_model(params);
if (FE) {
FE->add_extension(ov_exts);
inputModel = FE->load(params);
}
if (inputModel) {
auto model = FE->convert(inputModel);
update_v10_model(model);
return model;
}
OPENVINO_THROW("Unable to read the model. Please check if the model format is supported and model is correct.");
}
} // namespace util
} // namespace ov

45
src/inference/src/model_reader.hpp Normal file
View File

@@ -0,0 +1,45 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <string>
#include <vector>
#include "openvino/core/extension.hpp"
#include "openvino/core/model.hpp"
namespace ov {
namespace util {
/**
* @brief Reads model
* @param modelPath path to Model file
* @param binPath optional path for model weights. If empty for IR we will find bin file with the model name.
* if bin file with the same name was not found, will load IR without weights.
* @param extensions vector with OpenVINO extensions
* @param enable_mmap boolean to enable/disable `mmap` use in Frontend
* @return Shared pointer to ov::Model
*/
std::shared_ptr<ov::Model> read_model(const std::string& modelPath,
const std::string& binPath,
const std::vector<ov::Extension::Ptr>& extensions,
bool enable_mmap);
/**
* @brief Reads model
* @param model Serialized model representation
* @param weights constant Tensor with weights
* @param extensions vector with OpenVINO extensions
* @param frontendMode read network without post-processing or other transformations
* @return Shared pointer to ov::Model
*/
std::shared_ptr<ov::Model> read_model(const std::string& model,
const ov::Tensor& weights,
const std::vector<ov::Extension::Ptr>& extensions,
bool frontendMode = false);
} // namespace util
} // namespace ov

1
src/plugins/intel_gna/legacy/src/network_serializer_v7.hpp
View File

@@ -10,6 +10,7 @@
#include <vector>
#include "compilation_context.hpp"
#include "cpp/ie_cnn_network.h"
namespace InferenceEngine {
namespace Serialization {