From c3b22af3f7a530036514d22c1cfa66ca128efdb3 Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Wed, 8 Mar 2023 19:19:52 +0400 Subject: [PATCH] CoreImpl small refactoring (#16145) * Small CoreImpl refactoring * Removed cache_dirhandling from CPU plugin * clang-format * Fixed python tests * Fix * Fixed bugs in HETERO case * Fixed clang-format and warnings in auto plugin * Added import_export as capability for TEMPLATE plugin * Commented throw exception from loaded_from_cache * Fixed clang-formatof ro template plugin --- .../interface/ie_internal_plugin_config.hpp | 7 - src/inference/src/compilation_context.cpp | 19 +- src/inference/src/compilation_context.hpp | 2 +- src/inference/src/core.cpp | 62 +++ src/inference/src/dev/core_impl.cpp | 492 ++++++++---------- src/inference/src/dev/core_impl.hpp | 34 +- src/inference/src/dev/icompiled_model.cpp | 2 +- src/inference/src/dev/plugin.cpp | 4 + src/inference/src/dev/plugin.hpp | 2 + src/inference/src/ie_core.cpp | 34 +- .../unit/ie_compilation_context_test.cpp | 93 ++-- src/plugins/auto/plugin.cpp | 1 - src/plugins/hetero/executable_network.cpp | 10 +- src/plugins/intel_cpu/src/config.cpp | 3 - src/plugins/intel_cpu/src/config.h | 2 - src/plugins/intel_cpu/src/plugin.cpp | 1 - src/plugins/template/src/plugin.cpp | 4 +- 17 files changed, 387 insertions(+), 385 deletions(-) diff --git a/src/inference/dev_api/cpp_interfaces/interface/ie_internal_plugin_config.hpp b/src/inference/dev_api/cpp_interfaces/interface/ie_internal_plugin_config.hpp index 294ab6a18e0..d94aa978ffc 100644 --- a/src/inference/dev_api/cpp_interfaces/interface/ie_internal_plugin_config.hpp +++ b/src/inference/dev_api/cpp_interfaces/interface/ie_internal_plugin_config.hpp @@ -92,13 +92,6 @@ DECLARE_CONFIG_KEY(SMALL_CORE_OFFSET); */ DECLARE_CONFIG_KEY(CPU_RUNTIME_CACHE_CAPACITY); -/** - * @brief This key should be used to force disable export while loading network even if global cache dir is defined - * Used by HETERO plugin to disable automatic caching of subnetworks (set value to YES) - * @ingroup ie_dev_api_plugin_api - */ -DECLARE_CONFIG_KEY(FORCE_DISABLE_CACHE); - /** * @brief Internal device id for particular device (like GPU.0, GPU.1 etc) */ diff --git a/src/inference/src/compilation_context.cpp b/src/inference/src/compilation_context.cpp index 3c43f224941..b0e69110ac8 100644 --- a/src/inference/src/compilation_context.cpp +++ b/src/inference/src/compilation_context.cpp @@ -56,7 +56,7 @@ uint64_t calculate_td(const InferenceEngine::TensorDesc& td, uint64_t _seed) { namespace ov { -std::string NetworkCompilationContext::calculate_file_info(const std::string& filePath) { +std::string ModelCache::calculate_file_info(const std::string& filePath) { uint64_t seed = 0; auto absPath = filePath; if (filePath.size() > 0) { @@ -78,9 +78,8 @@ std::string NetworkCompilationContext::calculate_file_info(const std::string& fi return std::to_string(seed); } -std::string NetworkCompilationContext::compute_hash(const std::shared_ptr& model, - const ov::AnyMap& compileOptions) { - OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::IE_RT, "NetworkCompilationContext::compute_hash - Model"); +std::string ModelCache::compute_hash(const std::shared_ptr& model, const ov::AnyMap& compileOptions) { + OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::IE_RT, "ModelCache::compute_hash - Model"); OPENVINO_ASSERT(model); @@ -145,8 +144,8 @@ std::string NetworkCompilationContext::compute_hash(const std::shared_ptr& model, const ov::AnyMap& compileOptions); diff --git a/src/inference/src/core.cpp b/src/inference/src/core.cpp index 14e30e339a3..71e290930ab 100644 --- a/src/inference/src/core.cpp +++ b/src/inference/src/core.cpp @@ -27,10 +27,72 @@ std::string resolve_extension_path(const std::string& path) { return retvalue; } +ov::AnyMap flatten_sub_properties(const std::string& device, const ov::AnyMap& properties) { + ov::AnyMap result = properties; + bool isVirtualDev = device.find("AUTO") != std::string::npos || device.find("MULTI") != std::string::npos || + device.find("HETERO") != std::string::npos; + for (auto item = result.begin(); item != result.end();) { + auto parsed = ov::parseDeviceNameIntoConfig(item->first); + if (!item->second.is()) { + item++; + continue; + } + if (device == parsed._deviceName) { + // 1. flatten the secondary property for target device + for (auto&& sub_property : item->second.as()) { + // 1.1 1st level property overrides 2nd level property + if (result.find(sub_property.first) != result.end()) + continue; + result[sub_property.first] = sub_property.second; + } + item = result.erase(item); + } else if (isVirtualDev) { + // 2. keep the secondary property for the other virtual devices + item++; + } else { + // 3. remove the secondary property setting for other hardware device + item = result.erase(item); + } + } + return result; +} + } // namespace namespace ov { +#ifndef OPENVINO_STATIC_LIBRARY + +std::string findPluginXML(const std::string& xmlFile) { + std::string xmlConfigFile_ = xmlFile; + if (xmlConfigFile_.empty()) { + const auto ielibraryDir = ie::getInferenceEngineLibraryPath(); + + // plugins.xml can be found in either: + + // 1. openvino-X.Y.Z relative to libopenvino.so folder + std::ostringstream str; + str << "openvino-" << OPENVINO_VERSION_MAJOR << "." << OPENVINO_VERSION_MINOR << "." << OPENVINO_VERSION_PATCH; + const auto subFolder = ov::util::to_file_path(str.str()); + + // register plugins from default openvino-/plugins.xml config + ov::util::FilePath xmlConfigFileDefault = + FileUtils::makePath(FileUtils::makePath(ielibraryDir, subFolder), ov::util::to_file_path("plugins.xml")); + if (FileUtils::fileExist(xmlConfigFileDefault)) + return xmlConfigFile_ = ov::util::from_file_path(xmlConfigFileDefault); + + // 2. in folder with libopenvino.so + xmlConfigFileDefault = FileUtils::makePath(ielibraryDir, ov::util::to_file_path("plugins.xml")); + if (FileUtils::fileExist(xmlConfigFileDefault)) + return xmlConfigFile_ = ov::util::from_file_path(xmlConfigFileDefault); + + throw ov::Exception("Failed to find plugins.xml file"); + } + return xmlConfigFile_; +} + +#endif // OPENVINO_STATIC_LIBRARY + #define OV_CORE_CALL_STATEMENT(...) \ try { \ __VA_ARGS__; \ diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp index 9611cf9ce2b..0c6f0ee8536 100644 --- a/src/inference/src/dev/core_impl.cpp +++ b/src/inference/src/dev/core_impl.cpp @@ -193,33 +193,48 @@ ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const { // configuring { - if (device_supports_cache_dir(plugin)) { - auto cacheConfig = coreConfig.get_cache_config_for_device(deviceName); - if (cacheConfig._cacheManager) { - desc.defaultConfig[CONFIG_KEY(CACHE_DIR)] = cacheConfig._cacheDir; + // TODO: remove this block of code once GPU removes support of ov::cache_dir + // also, remove device_supports_cache_dir at all + { + OPENVINO_SUPPRESS_DEPRECATED_START + if (device_supports_cache_dir(plugin)) { + ov::AnyMap empty_map; + auto cacheConfig = coreConfig.get_cache_config_for_device(plugin, empty_map); + if (cacheConfig._cacheManager) { + desc.defaultConfig[CONFIG_KEY(CACHE_DIR)] = cacheConfig._cacheDir; + } + } else if (desc.defaultConfig.count(CONFIG_KEY(CACHE_DIR)) > 0) { + // Remove "CACHE_DIR" from config if it is not supported by plugin + desc.defaultConfig.erase(CONFIG_KEY(CACHE_DIR)); } - } else if (desc.defaultConfig.count(CONFIG_KEY(CACHE_DIR)) > 0) { - // Remove "CACHE_DIR" from config if it is not supported by plugin - desc.defaultConfig.erase(CONFIG_KEY(CACHE_DIR)); + OPENVINO_SUPPRESS_DEPRECATED_END } + allowNotImplemented([&]() { // Add device specific value to support device_name.device_id cases - auto supportedConfigKeys = - plugin.get_property(METRIC_KEY(SUPPORTED_CONFIG_KEYS), {}).as>(); - auto config_iter = std::find(supportedConfigKeys.begin(), - supportedConfigKeys.end(), - CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID)); - const bool supportsConfigDeviceID = config_iter != supportedConfigKeys.end(); - const std::string deviceKey = - supportsConfigDeviceID ? CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID) : CONFIG_KEY(DEVICE_ID); + { + auto supportedConfigKeys = + plugin.get_property(METRIC_KEY(SUPPORTED_CONFIG_KEYS), {}).as>(); + auto config_iter = std::find(supportedConfigKeys.begin(), + supportedConfigKeys.end(), + CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID)); + const bool supportsConfigDeviceID = config_iter != supportedConfigKeys.end(); + const std::string deviceKey = + supportsConfigDeviceID ? CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID) : CONFIG_KEY(DEVICE_ID); - for (auto pluginDesc : pluginRegistry) { - InferenceEngine::DeviceIDParser parser(pluginDesc.first); - if (pluginDesc.first.find(deviceName) != std::string::npos && !parser.getDeviceID().empty()) { - pluginDesc.second.defaultConfig[deviceKey] = parser.getDeviceID(); - plugin.set_property(pluginDesc.second.defaultConfig); + // here we can store values like GPU.0, GPU.1 and we need to set properties to plugin + // for each such .0, .1, .# device to make sure plugin can handle different settings for different + // device IDs + for (auto pluginDesc : pluginRegistry) { + InferenceEngine::DeviceIDParser parser(pluginDesc.first); + if (pluginDesc.first.find(deviceName) != std::string::npos && !parser.getDeviceID().empty()) { + pluginDesc.second.defaultConfig[deviceKey] = parser.getDeviceID(); + plugin.set_property(pluginDesc.second.defaultConfig); + } } } + + // set global device-id independent settings to plugin plugin.set_property(desc.defaultConfig); }); @@ -263,32 +278,21 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< apply_auto_batching(model, deviceName, config_with_batch); clean_properties(deviceName, config_with_batch, ov::auto_batch_timeout); - bool forceDisableCache = config_with_batch.count(CONFIG_KEY_INTERNAL(FORCE_DISABLE_CACHE)) > 0; auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch); - if (forceDisableCache) { - // remove this config key from parsed as plugins can throw unsupported exception - parsed._config.erase(CONFIG_KEY_INTERNAL(FORCE_DISABLE_CACHE)); - } auto plugin = get_plugin(parsed._deviceName); ov::SoPtr res; - auto cacheManager = - coreConfig.get_cache_config_for_device(parsed._deviceName, device_supports_cache_dir(plugin), parsed._config) - ._cacheManager; - auto cacheContent = CacheContent{cacheManager}; - if (!forceDisableCache && cacheManager && device_supports_import_export(plugin)) { - cacheContent.blobId = ov::NetworkCompilationContext::compute_hash( - model, - create_compile_config(plugin, parsed._deviceName, parsed._config)); - bool loadedFromCache = false; + auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; + if (cacheManager && device_supports_import_export(plugin)) { + CacheContent cacheContent{cacheManager}; + cacheContent.blobId = ov::ModelCache::compute_hash(model, create_compile_config(plugin, parsed._config)); auto lock = cacheGuard.get_hash_lock(cacheContent.blobId); - res = load_model_from_cache(cacheContent, plugin, parsed._config, {}, loadedFromCache); - if (!loadedFromCache) { - res = compile_model_impl(model, plugin, parsed._config, {}, cacheContent, forceDisableCache); - } + res = load_model_from_cache(cacheContent, plugin, parsed._config, ov::RemoteContext{}, [&]() { + return compile_model_and_cache(model, plugin, parsed._config, ov::RemoteContext{}, cacheContent); + }); } else { - res = compile_model_impl(model, plugin, parsed._config, {}, cacheContent, forceDisableCache); + res = compile_model_with_preprocess(plugin, model, ov::RemoteContext{}, parsed._config); } - return {res._ptr, res._so}; + return res; } ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr& model, @@ -309,31 +313,25 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::shared_ptr< auto plugin = get_plugin(parsed._deviceName); ov::SoPtr res; - auto cacheManager = - coreConfig.get_cache_config_for_device(parsed._deviceName, device_supports_cache_dir(plugin), parsed._config) - ._cacheManager; - auto cacheContent = CacheContent{cacheManager}; + auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; if (cacheManager && device_supports_import_export(plugin)) { - cacheContent.blobId = ov::NetworkCompilationContext::compute_hash( - model, - create_compile_config(plugin, parsed._deviceName, parsed._config)); - bool loadedFromCache = false; + CacheContent cacheContent{cacheManager}; + cacheContent.blobId = ov::ModelCache::compute_hash(model, create_compile_config(plugin, parsed._config)); auto lock = cacheGuard.get_hash_lock(cacheContent.blobId); - res = load_model_from_cache(cacheContent, plugin, parsed._config, context, loadedFromCache); - if (!loadedFromCache) { - res = compile_model_impl(model, plugin, parsed._config, context, cacheContent); - } + res = load_model_from_cache(cacheContent, plugin, parsed._config, context, [&]() { + return compile_model_and_cache(model, plugin, parsed._config, context, cacheContent); + }); } else { - res = compile_model_impl(model, plugin, parsed._config, context, cacheContent); + res = compile_model_with_preprocess(plugin, model, context, parsed._config); } return res; } -ov::SoPtr ov::CoreImpl::compile_model(ov::Plugin& plugin, - const std::shared_ptr& model, - const ov::RemoteContext& context, - const ov::AnyMap& config) const { - std::shared_ptr prepared_model = model; - ov::SoPtr compiled_model; + +ov::SoPtr ov::CoreImpl::compile_model_with_preprocess(ov::Plugin& plugin, + const std::shared_ptr& model, + const ov::RemoteContext& context, + const ov::AnyMap& config) const { + std::shared_ptr preprocessed_model = model; if (!is_new_api() && !std::dynamic_pointer_cast(plugin.m_ptr)) { ov::pass::Manager manager; @@ -341,15 +339,11 @@ ov::SoPtr ov::CoreImpl::compile_model(ov::Plugin& plugin, auto cloned_model = model->clone(); manager.run_passes(cloned_model); - prepared_model = cloned_model; + preprocessed_model = cloned_model; } - if (!context._impl) { - compiled_model = plugin.compile_model(prepared_model, config); - } else { - compiled_model = plugin.compile_model(prepared_model, context, config); - } - return compiled_model; + return context._impl ? plugin.compile_model(preprocessed_model, context, config) + : plugin.compile_model(preprocessed_model, config); } ov::SoPtr ov::CoreImpl::compile_model(const std::string& model_path, @@ -358,69 +352,63 @@ ov::SoPtr ov::CoreImpl::compile_model(const std::string& mod OV_ITT_SCOPE(FIRST_INFERENCE, ie::itt::domains::IE_LT, "Core::compile_model::Path"); auto parsed = parseDeviceNameIntoConfig(device_name, config); auto plugin = get_plugin(parsed._deviceName); - ov::SoPtr res; - auto cacheManager = - coreConfig.get_cache_config_for_device(parsed._deviceName, device_supports_cache_dir(plugin), parsed._config) - ._cacheManager; - auto cacheContent = CacheContent{cacheManager, model_path}; + ov::SoPtr compiled_model; + + auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; if (cacheManager && device_supports_import_export(plugin)) { - bool loadedFromCache = false; - cacheContent.blobId = ov::NetworkCompilationContext::compute_hash( - model_path, - create_compile_config(plugin, parsed._deviceName, parsed._config)); + CacheContent cacheContent{cacheManager, model_path}; + cacheContent.blobId = ov::ModelCache::compute_hash(model_path, create_compile_config(plugin, parsed._config)); auto lock = cacheGuard.get_hash_lock(cacheContent.blobId); - res = load_model_from_cache(cacheContent, plugin, parsed._config, {}, loadedFromCache); - if (!loadedFromCache) { + compiled_model = load_model_from_cache(cacheContent, plugin, parsed._config, ov::RemoteContext{}, [&]() { auto cnnNetwork = ReadNetwork(model_path, std::string()); - res = compile_model_impl(cnnNetwork.getFunction(), plugin, parsed._config, {}, cacheContent); - } + return compile_model_and_cache(cnnNetwork.getFunction(), plugin, parsed._config, {}, cacheContent); + }); } else if (cacheManager) { - res = plugin.compile_model(model_path, parsed._config); + // this code path is enabled for AUTO / MULTI / BATCH devices which don't support + // import / export explicitly, but can redirect this functionality to actual HW plugin + compiled_model = plugin.compile_model(model_path, parsed._config); } else { auto cnnNetwork = ReadNetwork(model_path, std::string()); - res = compile_model_impl(cnnNetwork.getFunction(), plugin, parsed._config, {}, cacheContent); + compiled_model = + compile_model_with_preprocess(plugin, cnnNetwork.getFunction(), ov::RemoteContext{}, parsed._config); } - return {res._ptr, res._so}; + return compiled_model; } ov::SoPtr ov::CoreImpl::compile_model(const std::string& model_str, const ov::Tensor& weights, const std::string& device_name, const ov::AnyMap& config) const { + OV_ITT_SCOPED_TASK(ov::itt::domains::IE, "Core::compile_model::from_memory"); auto parsed = parseDeviceNameIntoConfig(device_name, config); auto plugin = get_plugin(parsed._deviceName); - ov::SoPtr res; + ov::SoPtr compiled_model; - auto cacheManager = - coreConfig.get_cache_config_for_device(parsed._deviceName, device_supports_cache_dir(plugin), parsed._config) - ._cacheManager; - auto cacheContent = CacheContent{cacheManager}; + auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager; if (cacheManager && device_supports_import_export(plugin)) { - bool loadedFromCache = false; - cacheContent.blobId = ov::NetworkCompilationContext::compute_hash( - model_str, - weights, - create_compile_config(plugin, parsed._deviceName, parsed._config)); + CacheContent cacheContent{cacheManager}; + cacheContent.blobId = + ov::ModelCache::compute_hash(model_str, weights, create_compile_config(plugin, parsed._config)); auto lock = cacheGuard.get_hash_lock(cacheContent.blobId); - res = load_model_from_cache(cacheContent, plugin, parsed._config, {}, loadedFromCache); - if (!loadedFromCache) { + compiled_model = load_model_from_cache(cacheContent, plugin, parsed._config, ov::RemoteContext{}, [&]() { auto cnnNetwork = read_model(model_str, weights); - res = compile_model_impl(cnnNetwork, plugin, parsed._config, {}, cacheContent); - } + return compile_model_and_cache(cnnNetwork, plugin, parsed._config, {}, cacheContent); + }); } else { - auto cnnNetwork = read_model(model_str, weights); - res = compile_model_impl(cnnNetwork, plugin, parsed._config, {}, cacheContent); + auto model = read_model(model_str, weights); + compiled_model = compile_model_with_preprocess(plugin, model, ov::RemoteContext{}, parsed._config); } - return {res._ptr, res._so}; + return compiled_model; } ov::SoPtr ov::CoreImpl::import_model(std::istream& model, const std::string& device_name, const ov::AnyMap& config) const { + OV_ITT_SCOPED_TASK(ov::itt::domains::IE, "Core::import_model"); auto parsed = parseDeviceNameIntoConfig(device_name, config); - auto exec = get_plugin(parsed._deviceName).import_model(model, config); + auto compiled_model = get_plugin(parsed._deviceName).import_model(model, config); - return {exec._ptr, exec._so}; + return compiled_model; } ov::SupportedOpsMap ov::CoreImpl::query_model(const std::shared_ptr& model, @@ -474,10 +462,14 @@ ov::RemoteContext ov::CoreImpl::create_context(const std::string& device_name, c ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& device_name, const ov::AnyMap& config) const { std::vector supportedConfigKeys; + + // try to search against IE API 1.0' SUPPORTED_CONFIG_KEYS try { supportedConfigKeys = GetMetric(device_name, METRIC_KEY(SUPPORTED_CONFIG_KEYS)).as>(); } catch (ov::Exception&) { } + + // try to search against OV API 2.0' supported_properties try { for (auto&& property : ICore::get_property(device_name, ov::supported_properties)) { if (property.is_mutable()) { @@ -486,6 +478,7 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& device_name, } } catch (ov::Exception&) { } + ov::AnyMap supportedConfig; for (auto&& key : supportedConfigKeys) { auto itKey = config.find(key); @@ -493,6 +486,7 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& device_name, supportedConfig[key] = itKey->second; } } + for (auto&& config : config) { auto parsed = parseDeviceNameIntoConfig(config.first); if (device_name.find(parsed._deviceName) != std::string::npos) { @@ -509,6 +503,7 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& device_name, } } } + return supportedConfig; } @@ -555,9 +550,10 @@ void ov::CoreImpl::apply_auto_batching(const std::shared_ptr& m config.erase(batch_mode); if (disabled) return; - } else if (!coreConfig.flag_allow_auto_batching) { + } else if (!coreConfig.get_allow_auto_batch()) { return; } + // check whether if the Auto-Batching is applicable to the device auto device = ov::parseDeviceNameIntoConfig(deviceName); deviceNameWithoutBatch = deviceName; @@ -567,6 +563,7 @@ void ov::CoreImpl::apply_auto_batching(const std::shared_ptr& m auto it = std::find(metrics.begin(), metrics.end(), METRIC_KEY(OPTIMAL_BATCH_SIZE)); if (metrics.end() == it) return; + // if applicable, the Auto-Batching is implicitly enabled via the performance hints bool bTputInPlg = GetConfig(d, CONFIG_KEY(PERFORMANCE_HINT)).as() == CONFIG_VALUE(THROUGHPUT); const auto& mode = config.find(CONFIG_KEY(PERFORMANCE_HINT)); @@ -638,7 +635,7 @@ ov::Any ov::CoreImpl::get_property_for_core(const std::string& name) const { } else if (name == ov::cache_dir.name()) { return ov::Any(coreConfig.get_cache_dir()); } else if (name == ov::hint::allow_auto_batching.name()) { - const auto flag = coreConfig.flag_allow_auto_batching; + const auto flag = coreConfig.get_allow_auto_batch(); return decltype(ov::hint::allow_auto_batching)::value_type(flag); } @@ -663,11 +660,15 @@ ov::Any ov::CoreImpl::get_property(const std::string& device_name, "You can only get_property of the BATCH itself (without devices). " "get_property is also possible for the individual devices before creating the BATCH on top."); - if (device_name.empty()) { + auto parsed = parseDeviceNameIntoConfig(device_name, arguments); + + if (parsed._deviceName.empty()) { return get_property_for_core(name); + } else if (name == ov::cache_dir.name()) { + ov::AnyMap empty_map; + return coreConfig.get_cache_config_for_device(get_plugin(parsed._deviceName), empty_map)._cacheDir; } - auto parsed = parseDeviceNameIntoConfig(device_name, arguments); return get_plugin(parsed._deviceName).get_property(name, parsed._config); } @@ -714,7 +715,7 @@ std::vector ov::CoreImpl::get_registered_devices() const { } /** - * @brief Sets config values for a plugin or set of plugins + * @brief Sets property values for a plugin or set of plugins * @param deviceName A device name to set config to * If empty, config is set for all the plugins / plugin's meta-data * @note `deviceName` is not allowed in form of MULTI:CPU, HETERO:GPU,CPU, AUTO:CPU @@ -734,13 +735,20 @@ void ov::CoreImpl::set_property_for_device(const ov::AnyMap& configMap, const st std::lock_guard lock(get_mutex()); created_plugins.reserve(plugins.size()); + // TODO: keep only: + // coreConfig.set_and_update(config); + // once GPU remove support of ov::cache_dir + // CoreConfg::set_and_update will drop CACHE_DIR from config map + // and updates core config with new ov::cache_dir if (deviceName.empty()) { coreConfig.set_and_update(config); } else { + OPENVINO_SUPPRESS_DEPRECATED_START auto cache_it = config.find(CONFIG_KEY(CACHE_DIR)); if (cache_it != config.end()) { coreConfig.set_cache_dir_for_device((cache_it->second).as(), clearDeviceName); } + OPENVINO_SUPPRESS_DEPRECATED_END } auto base_desc = pluginRegistry.find(clearDeviceName); @@ -771,31 +779,40 @@ void ov::CoreImpl::set_property_for_device(const ov::AnyMap& configMap, const st } } } + for (auto& plugin : created_plugins) { allowNotImplemented([&]() { std::lock_guard lock(get_mutex(plugin.first)); auto configCopy = config; - if (device_supports_cache_dir(plugin.second)) { - auto cacheConfig = coreConfig.get_cache_config_for_device(deviceName); - if (cacheConfig._cacheManager) { - configCopy[CONFIG_KEY(CACHE_DIR)] = cacheConfig._cacheDir; + // TODO: remove once GPU remove explicit support of ov::cache_dir + { + OPENVINO_SUPPRESS_DEPRECATED_START + if (device_supports_cache_dir(plugin.second)) { + ov::AnyMap empty_map; + auto cacheConfig = coreConfig.get_cache_config_for_device(plugin.second, empty_map); + if (cacheConfig._cacheManager) { + configCopy[CONFIG_KEY(CACHE_DIR)] = cacheConfig._cacheDir; + } + } else if (configCopy.count(CONFIG_KEY(CACHE_DIR)) > 0) { + // Remove "CACHE_DIR" from config if it is not supported by plugin + configCopy.erase(CONFIG_KEY(CACHE_DIR)); } - } else if (configCopy.count(CONFIG_KEY(CACHE_DIR)) > 0) { - // Remove "CACHE_DIR" from config if it is not supported by plugin - configCopy.erase(CONFIG_KEY(CACHE_DIR)); + OPENVINO_SUPPRESS_DEPRECATED_END } // Add device specific value to support device_name.device_id cases - auto supportedConfigKeys = - plugin.second.get_property(METRIC_KEY(SUPPORTED_CONFIG_KEYS), {}).as>(); - auto config_iter = std::find(supportedConfigKeys.begin(), - supportedConfigKeys.end(), - CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID)); - const bool supportsConfigDeviceID = config_iter != supportedConfigKeys.end(); - const std::string deviceKey = - supportsConfigDeviceID ? CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID) : CONFIG_KEY(DEVICE_ID); + { + auto supportedConfigKeys = + plugin.second.get_property(METRIC_KEY(SUPPORTED_CONFIG_KEYS), {}).as>(); + auto config_iter = std::find(supportedConfigKeys.begin(), + supportedConfigKeys.end(), + CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID)); + const bool supportsConfigDeviceID = config_iter != supportedConfigKeys.end(); + const std::string deviceKey = + supportsConfigDeviceID ? CONFIG_KEY_INTERNAL(CONFIG_DEVICE_ID) : CONFIG_KEY(DEVICE_ID); - if (!parser.getDeviceID().empty()) { - configCopy[deviceKey] = parser.getDeviceID(); + if (!parser.getDeviceID().empty()) { + configCopy[deviceKey] = parser.getDeviceID(); + } } plugin.second.set_property(configCopy); }); @@ -846,23 +863,21 @@ bool ov::CoreImpl::device_supports_cache_dir(const ov::Plugin& plugin) const { return util::contains(plugin.get_property(ov::supported_properties), ov::cache_dir); } -ov::SoPtr ov::CoreImpl::compile_model_impl(const std::shared_ptr& model, - ov::Plugin& plugin, - const ov::AnyMap& parsedConfig, - const ov::RemoteContext& context, - const CacheContent& cacheContent, - bool forceDisableCache) const { - OV_ITT_SCOPED_TASK(ov::itt::domains::IE, "CoreImpl::compile_model_impl"); +ov::SoPtr ov::CoreImpl::compile_model_and_cache(const std::shared_ptr& model, + ov::Plugin& plugin, + const ov::AnyMap& parsedConfig, + const ov::RemoteContext& context, + const CacheContent& cacheContent) const { + OV_ITT_SCOPED_TASK(ov::itt::domains::IE, "CoreImpl::compile_model_and_cache"); ov::SoPtr execNetwork; - execNetwork = compile_model(plugin, model, context, parsedConfig); - if (!forceDisableCache && cacheContent.cacheManager && device_supports_import_export(plugin)) { + execNetwork = compile_model_with_preprocess(plugin, model, context, parsedConfig); + if (cacheContent.cacheManager && device_supports_import_export(plugin)) { try { // need to export network for further import from "cache" OV_ITT_SCOPE(FIRST_INFERENCE, InferenceEngine::itt::domains::IE_LT, "Core::compile_model::Export"); cacheContent.cacheManager->write_cache_entry(cacheContent.blobId, [&](std::ostream& networkStream) { - networkStream << ov::CompiledBlobHeader( - InferenceEngine::GetInferenceEngineVersion()->buildNumber, - ov::NetworkCompilationContext::calculate_file_info(cacheContent.modelPath)); + networkStream << ov::CompiledBlobHeader(InferenceEngine::GetInferenceEngineVersion()->buildNumber, + ov::ModelCache::calculate_file_info(cacheContent.modelPath)); execNetwork->export_model(networkStream); }); } catch (...) { @@ -873,12 +888,13 @@ ov::SoPtr ov::CoreImpl::compile_model_impl(const std::shared return execNetwork; } -ov::SoPtr ov::CoreImpl::load_model_from_cache(const CacheContent& cacheContent, - ov::Plugin& plugin, - const ov::AnyMap& config, - const ov::RemoteContext& context, - bool& networkIsImported) { - ov::SoPtr execNetwork; +ov::SoPtr ov::CoreImpl::load_model_from_cache( + const CacheContent& cacheContent, + ov::Plugin& plugin, + const ov::AnyMap& config, + const ov::RemoteContext& context, + std::function()> compile_model_lambda) { + ov::SoPtr compiled_model; struct HeaderException {}; OPENVINO_ASSERT(cacheContent.cacheManager != nullptr); @@ -886,7 +902,7 @@ ov::SoPtr ov::CoreImpl::load_model_from_cache(const CacheCon cacheContent.cacheManager->read_cache_entry(cacheContent.blobId, [&](std::istream& networkStream) { OV_ITT_SCOPE(FIRST_INFERENCE, InferenceEngine::itt::domains::IE_LT, - "Core::LoadNetworkFromCache::ReadStreamAndImport"); + "Core::load_model_from_cache::ReadStreamAndImport"); try { ov::CompiledBlobHeader header; networkStream >> header; @@ -894,8 +910,7 @@ ov::SoPtr ov::CoreImpl::load_model_from_cache(const CacheCon // Build number mismatch, don't use this cache throw InferenceEngine::NetworkNotRead("Version does not match"); } - if (header.getFileInfo() != - ov::NetworkCompilationContext::calculate_file_info(cacheContent.modelPath)) { + if (header.getFileInfo() != ov::ModelCache::calculate_file_info(cacheContent.modelPath)) { // Original file is changed, don't use cache throw InferenceEngine::NetworkNotRead("Original model file is changed"); } @@ -903,60 +918,64 @@ ov::SoPtr ov::CoreImpl::load_model_from_cache(const CacheCon throw HeaderException(); } - execNetwork = context._impl ? plugin.import_model(networkStream, context, config) - : plugin.import_model(networkStream, config); - networkIsImported = true; - execNetwork->loaded_from_cache(); + compiled_model = context._impl ? plugin.import_model(networkStream, context, config) + : plugin.import_model(networkStream, config); + compiled_model->loaded_from_cache(); }); } catch (const HeaderException&) { // For these exceptions just remove old cache and set that import didn't work cacheContent.cacheManager->remove_cache_entry(cacheContent.blobId); - networkIsImported = false; } catch (...) { cacheContent.cacheManager->remove_cache_entry(cacheContent.blobId); - networkIsImported = false; // TODO: temporary disabled by #54335. In future don't throw only for new 'blob_outdated' exception // throw; } - return execNetwork; + + // fallback scenario + if (!compiled_model) + compiled_model = compile_model_lambda(); + + return compiled_model; } -ov::AnyMap ov::CoreImpl::create_compile_config(const ov::Plugin& plugin, - const std::string& deviceFamily, - const ov::AnyMap& origConfig) const { - ov::AnyMap getMetricConfig; +ov::AnyMap ov::CoreImpl::create_compile_config(const ov::Plugin& plugin, const ov::AnyMap& origConfig) const { + ov::AnyMap property_config; ov::AnyMap compileConfig; - // 0. Move TARGET_FALLBACK key to getMetricConfig + // 0. Move TARGET_FALLBACK key to property_config auto targetFallbackIt = origConfig.find("TARGET_FALLBACK"); if (targetFallbackIt == origConfig.end()) { targetFallbackIt = origConfig.find(ov::device::priorities.name()); } if (targetFallbackIt != origConfig.end()) { - getMetricConfig[targetFallbackIt->first] = targetFallbackIt->second.as(); + property_config[targetFallbackIt->first] = targetFallbackIt->second.as(); } - // 1. Move DEVICE_ID key to getMetricConfig + // 1. Move DEVICE_ID key to property_config auto deviceIt = origConfig.find(ov::device::id.name()); if (deviceIt != origConfig.end()) { - getMetricConfig[deviceIt->first] = deviceIt->second.as(); + property_config[deviceIt->first] = deviceIt->second.as(); + } else { + // we likely need to extract default device_id from the plugin, + // but we suppose when we call plugin.get_property it will provide the answer + // for the default device (e.g. DEVICE_ID = 0 for GPU) } - // 2. Replace it with DEVICE_ARCHITECTURE value + // 2. Replace DEVICE_ID with DEVICE_ARCHITECTURE value to identify device if (device_supports_property(plugin, ov::device::architecture.name())) { - compileConfig[ov::device::architecture.name()] = plugin.get_property(ov::device::architecture, getMetricConfig); + compileConfig[ov::device::architecture.name()] = plugin.get_property(ov::device::architecture, property_config); } else { // Take device name if device does not support DEVICE_ARCHITECTURE metric - compileConfig[ov::device::architecture.name()] = deviceFamily; + compileConfig[ov::device::architecture.name()] = plugin.get_name(); } - // 3. Extract config keys which affect compile config + // 3. Extract config keys which affect compilation process if (device_supports_property(plugin, ov::caching_properties.name())) { auto cachingProps = plugin.get_property(ov::caching_properties); for (const auto& prop : cachingProps) { // origConfig values have higher priority than plugin parameters auto it = origConfig.find(prop); - compileConfig[prop] = it == origConfig.end() ? plugin.get_property(prop, {}) : it->second; + compileConfig[prop] = it == origConfig.end() ? plugin.get_property(prop, property_config) : it->second; } } return compileConfig; @@ -972,10 +991,9 @@ void ov::CoreImpl::AddExtensionUnsafe(const InferenceEngine::IExtensionPtr& exte // add extensions for already created plugins for (auto& plugin : plugins) { - try { + allowNotImplemented([&]() { plugin.second.add_extension(extension); - } catch (...) { - } + }); } extensions.emplace_back(extension); } @@ -984,9 +1002,11 @@ void ov::CoreImpl::CoreConfig::set_and_update(ov::AnyMap& config) { auto it = config.find(CONFIG_KEY(CACHE_DIR)); if (it != config.end()) { std::lock_guard lock(_cacheConfigMutex); - fill_config(_cacheConfig, it->second.as()); + // fill global cache config + _cacheConfig = CoreConfig::CacheConfig::create(it->second.as()); + // sets cache config per-device if it's set explicitly before for (auto& deviceCfg : _cacheConfigPerDevice) { - fill_config(deviceCfg.second, it->second.as()); + deviceCfg.second = CoreConfig::CacheConfig::create(it->second.as()); } config.erase(it); } @@ -1001,14 +1021,14 @@ void ov::CoreImpl::CoreConfig::set_and_update(ov::AnyMap& config) { it = config.find(ov::hint::allow_auto_batching.name()); if (it != config.end()) { auto flag = it->second.as(); - flag_allow_auto_batching = flag; + _flag_allow_auto_batching = flag; config.erase(it); } } void ov::CoreImpl::CoreConfig::set_cache_dir_for_device(const std::string& dir, const std::string& name) { std::lock_guard lock(_cacheConfigMutex); - fill_config(_cacheConfigPerDevice[name], dir); + _cacheConfigPerDevice[name] = CoreConfig::CacheConfig::create(dir); } std::string ov::CoreImpl::CoreConfig::get_cache_dir() const { @@ -1016,48 +1036,45 @@ std::string ov::CoreImpl::CoreConfig::get_cache_dir() const { return _cacheConfig._cacheDir; } +bool ov::CoreImpl::CoreConfig::get_allow_auto_batch() const { + return _flag_allow_auto_batching; +} + // Creating thread-safe copy of config including shared_ptr to ICacheManager // Passing empty or not-existing name will return global cache config ov::CoreImpl::CoreConfig::CacheConfig ov::CoreImpl::CoreConfig::get_cache_config_for_device( - const std::string& device_name, - bool device_supports_cache_dir, + const ov::Plugin& plugin, ov::AnyMap& parsedConfig) const { - if (parsedConfig.count(CONFIG_KEY(CACHE_DIR))) { - CoreConfig::CacheConfig tempConfig; - CoreConfig::fill_config(tempConfig, parsedConfig.at(CONFIG_KEY(CACHE_DIR)).as()); - if (!device_supports_cache_dir) { - parsedConfig.erase(CONFIG_KEY(CACHE_DIR)); + // cache_dir is enabled locally in compile_model only + if (parsedConfig.count(ov::cache_dir.name())) { + auto cache_dir_val = parsedConfig.at(ov::cache_dir.name()).as(); + auto tempConfig = CoreConfig::CacheConfig::create(cache_dir_val); + // if plugin does not explicitly support cache_dir, we need to remove it from config + if (!util::contains(plugin.get_property(ov::supported_properties), ov::cache_dir)) { + parsedConfig.erase(ov::cache_dir.name()); } return tempConfig; - } else { + } else { // cache_dir is set to Core globally or for the specific device std::lock_guard lock(_cacheConfigMutex); - if (_cacheConfigPerDevice.count(device_name) > 0) { - return _cacheConfigPerDevice.at(device_name); + if (_cacheConfigPerDevice.count(plugin.get_name()) > 0) { + return _cacheConfigPerDevice.at(plugin.get_name()); } else { return _cacheConfig; } } } -ov::CoreImpl::CoreConfig::CacheConfig ov::CoreImpl::CoreConfig::get_cache_config_for_device( - const std::string& device_name) const { - std::lock_guard lock(_cacheConfigMutex); - if (_cacheConfigPerDevice.count(device_name) > 0) { - return _cacheConfigPerDevice.at(device_name); - } else { - return _cacheConfig; - } -} +ov::CoreImpl::CoreConfig::CacheConfig ov::CoreImpl::CoreConfig::CacheConfig::create(const std::string& dir) { + std::shared_ptr cache_manager = nullptr; -void ov::CoreImpl::CoreConfig::fill_config(CacheConfig& config, const std::string& dir) { - config._cacheDir = dir; if (!dir.empty()) { FileUtils::createDirectoryRecursive(dir); - config._cacheManager = std::make_shared(dir); - } else { - config._cacheManager = nullptr; + cache_manager = std::make_shared(dir); } + + return {dir, cache_manager}; } + std::mutex& ov::CoreImpl::get_mutex(const std::string& dev_name) const { std::lock_guard lock(global_mutex); try { @@ -1066,95 +1083,12 @@ std::mutex& ov::CoreImpl::get_mutex(const std::string& dev_name) const { throw ov::Exception("Cannot get mutex for device: " + dev_name); } } + void ov::CoreImpl::add_mutex(const std::string& dev_name) { std::lock_guard lock(global_mutex); dev_mutexes[dev_name]; } -std::tuple ov::CoreImpl::CheckStatic(const InferenceEngine::CNNNetwork& network) { - bool res = true; - std::stringstream errMsg; - auto model = network.getFunction(); - if (model) { - for (const auto& input : model->inputs()) { - if (input.get_partial_shape().is_dynamic()) { - errMsg << "{ input:'"; - for (const auto& name : input.get_names()) { - errMsg << name << ","; - } - if (auto node = input.get_node_shared_ptr()) { - errMsg << node->get_friendly_name(); - } - errMsg << "', shape=" << input.get_partial_shape() << "} "; - res = false; - } - } - } - return {res, errMsg.str()}; -} - -#ifndef OPENVINO_STATIC_LIBRARY - -std::string ov::findPluginXML(const std::string& xmlFile) { - std::string xmlConfigFile_ = xmlFile; - if (xmlConfigFile_.empty()) { - const auto ielibraryDir = ie::getInferenceEngineLibraryPath(); - - // plugins.xml can be found in either: - - // 1. openvino-X.Y.Z relative to libopenvino.so folder - std::ostringstream str; - str << "openvino-" << OPENVINO_VERSION_MAJOR << "." << OPENVINO_VERSION_MINOR << "." << OPENVINO_VERSION_PATCH; - const auto subFolder = ov::util::to_file_path(str.str()); - - // register plugins from default openvino-/plugins.xml config - ov::util::FilePath xmlConfigFileDefault = - FileUtils::makePath(FileUtils::makePath(ielibraryDir, subFolder), ov::util::to_file_path("plugins.xml")); - if (FileUtils::fileExist(xmlConfigFileDefault)) - return xmlConfigFile_ = ov::util::from_file_path(xmlConfigFileDefault); - - // 2. in folder with libopenvino.so - xmlConfigFileDefault = FileUtils::makePath(ielibraryDir, ov::util::to_file_path("plugins.xml")); - if (FileUtils::fileExist(xmlConfigFileDefault)) - return xmlConfigFile_ = ov::util::from_file_path(xmlConfigFileDefault); - - throw ov::Exception("Failed to find plugins.xml file"); - } - return xmlConfigFile_; -} - -#endif - -ov::AnyMap ov::flatten_sub_properties(const std::string& device, const ov::AnyMap& properties) { - ov::AnyMap result = properties; - bool isVirtualDev = device.find("AUTO") != std::string::npos || device.find("MULTI") != std::string::npos || - device.find("HETERO") != std::string::npos; - for (auto item = result.begin(); item != result.end();) { - auto parsed = parseDeviceNameIntoConfig(item->first); - if (!item->second.is()) { - item++; - continue; - } - if (device == parsed._deviceName) { - // 1. flatten the scondary property for target device - for (auto&& sub_property : item->second.as()) { - // 1.1 1st level property overides 2nd level property - if (result.find(sub_property.first) != result.end()) - continue; - result[sub_property.first] = sub_property.second; - } - item = result.erase(item); - } else if (isVirtualDev) { - // 2. keep the secondary property for the other virtual devices - item++; - } else { - // 3. remove the secondary property setting for other hardware device - item = result.erase(item); - } - } - return result; -} - std::shared_ptr ov::CoreImpl::read_model(const std::string& modelPath, const std::string& binPath) const { OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::IE_RT, "CoreImpl::read_model from file"); return ReadNetwork(modelPath, binPath).getFunction(); diff --git a/src/inference/src/dev/core_impl.hpp b/src/inference/src/dev/core_impl.hpp index 7e223202f03..49c13d4f406 100644 --- a/src/inference/src/dev/core_impl.hpp +++ b/src/inference/src/dev/core_impl.hpp @@ -74,8 +74,6 @@ std::string findPluginXML(const std::string& xmlFile); #endif -ov::AnyMap flatten_sub_properties(const std::string& device, const ov::AnyMap& properties); - class CoreImpl : public InferenceEngine::ICore, public std::enable_shared_from_this { private: mutable std::map plugins; @@ -93,30 +91,33 @@ private: struct CacheConfig { std::string _cacheDir; std::shared_ptr _cacheManager; + + static CacheConfig create(const std::string& dir); }; - bool flag_allow_auto_batching = true; - + /** + * @brief Removes core-level properties from config and triggers new state for core config + * @param config - config to be updated + */ void set_and_update(ov::AnyMap& config); + OPENVINO_DEPRECATED("Don't use this method, it will be removed soon") void set_cache_dir_for_device(const std::string& dir, const std::string& name); std::string get_cache_dir() const; + bool get_allow_auto_batch() const; + // Creating thread-safe copy of config including shared_ptr to ICacheManager // Passing empty or not-existing name will return global cache config - CacheConfig get_cache_config_for_device(const std::string& device_name, - bool device_supports_cache_dir, + CacheConfig get_cache_config_for_device(const ov::Plugin& plugin, ov::AnyMap& parsedConfig) const; - CacheConfig get_cache_config_for_device(const std::string& device_name) const; - private: - static void fill_config(CacheConfig& config, const std::string& dir); - mutable std::mutex _cacheConfigMutex; CacheConfig _cacheConfig; std::map _cacheConfigPerDevice; + bool _flag_allow_auto_batching = true; }; struct CacheContent { @@ -172,32 +173,31 @@ private: const bool m_new_api; - ov::SoPtr compile_model_impl(const std::shared_ptr& model, + ov::SoPtr compile_model_and_cache(const std::shared_ptr& model, ov::Plugin& plugin, const ov::AnyMap& parsedConfig, const ov::RemoteContext& context, - const CacheContent& cacheContent, - bool forceDisableCache = false) const; + const CacheContent& cacheContent) const; static ov::SoPtr load_model_from_cache(const CacheContent& cacheContent, ov::Plugin& plugin, const ov::AnyMap& config, const ov::RemoteContext& context, - bool& networkIsImported); + std::function()> compile_model_lambda); bool device_supports_import_export(const ov::Plugin& plugin) const; bool device_supports_property(const ov::Plugin& plugin, const std::string& key) const; + OPENVINO_DEPRECATED("Don't use this method, it will be removed soon") bool device_supports_cache_dir(const ov::Plugin& plugin) const; - ov::SoPtr compile_model(ov::Plugin& plugin, + ov::SoPtr compile_model_with_preprocess(ov::Plugin& plugin, const std::shared_ptr& model, const ov::RemoteContext& context, const ov::AnyMap& config) const; ov::AnyMap create_compile_config(const ov::Plugin& plugin, - const std::string& deviceFamily, const ov::AnyMap& origConfig) const; // Legacy API @@ -273,8 +273,6 @@ public: bool isNewAPI() const override; - static std::tuple CheckStatic(const InferenceEngine::CNNNetwork& network); - InferenceEngine::RemoteContext::Ptr GetDefaultContext(const std::string& deviceName) override; ov::SoPtr LoadNetwork( diff --git a/src/inference/src/dev/icompiled_model.cpp b/src/inference/src/dev/icompiled_model.cpp index 96957e387eb..3ca57549ade 100644 --- a/src/inference/src/dev/icompiled_model.cpp +++ b/src/inference/src/dev/icompiled_model.cpp @@ -97,5 +97,5 @@ void ov::ICompiledModel::loaded_from_cache() { wrapper->get_executable_network()->loadedFromCache(); return; } - OPENVINO_NOT_IMPLEMENTED; + // OPENVINO_NOT_IMPLEMENTED; } diff --git a/src/inference/src/dev/plugin.cpp b/src/inference/src/dev/plugin.cpp index bd4d179f305..c18f39e043d 100644 --- a/src/inference/src/dev/plugin.cpp +++ b/src/inference/src/dev/plugin.cpp @@ -33,6 +33,10 @@ void ov::Plugin::set_name(const std::string& deviceName) { }); } +const std::string& ov::Plugin::get_name() const { + OV_PLUGIN_CALL_STATEMENT({ return m_ptr->get_device_name(); }); +} + void ov::Plugin::set_core(std::weak_ptr core) { OV_PLUGIN_CALL_STATEMENT({ m_ptr->set_core(core); diff --git a/src/inference/src/dev/plugin.hpp b/src/inference/src/dev/plugin.hpp index 34a5bebb032..be029c8b9a7 100644 --- a/src/inference/src/dev/plugin.hpp +++ b/src/inference/src/dev/plugin.hpp @@ -34,6 +34,8 @@ public: void set_name(const std::string& deviceName); + const std::string& get_name() const; + void set_core(std::weak_ptr core); const ov::Version get_version() const; diff --git a/src/inference/src/ie_core.cpp b/src/inference/src/ie_core.cpp index 283ff3cd650..da34182ad23 100644 --- a/src/inference/src/ie_core.cpp +++ b/src/inference/src/ie_core.cpp @@ -54,6 +54,32 @@ using namespace InferenceEngine::PluginConfigParams; using namespace InferenceEngine; using namespace std::placeholders; +namespace { + +std::tuple CheckStatic(const InferenceEngine::CNNNetwork& network) { + bool res = true; + std::stringstream errMsg; + auto model = network.getFunction(); + if (model) { + for (const auto& input : model->inputs()) { + if (input.get_partial_shape().is_dynamic()) { + errMsg << "{ input:'"; + for (const auto& name : input.get_names()) { + errMsg << name << ","; + } + if (auto node = input.get_node_shared_ptr()) { + errMsg << node->get_friendly_name(); + } + errMsg << "', shape=" << input.get_partial_shape() << "} "; + res = false; + } + } + } + return {res, errMsg.str()}; +} + +} // namespace + namespace InferenceEngine { class Core::Impl : public ov::CoreImpl { @@ -100,7 +126,7 @@ ExecutableNetwork Core::LoadNetwork(const CNNNetwork& network, const std::map& config) { - auto valid = ov::CoreImpl::CheckStatic(network); + auto valid = ::CheckStatic(network); OPENVINO_ASSERT(std::get<0>(valid), "InferenceEngine::Core::LoadNetwork doesn't support inputs having dynamic shapes. ", "Use ov::Core::compile_model API instead. Dynamic inputs are :", @@ -112,7 +138,7 @@ ExecutableNetwork Core::LoadNetwork(const CNNNetwork& network, ExecutableNetwork Core::LoadNetwork(const CNNNetwork& network, RemoteContext::Ptr context, const std::map& config) { - auto valid = ov::CoreImpl::CheckStatic(network); + auto valid = ::CheckStatic(network); OPENVINO_ASSERT(std::get<0>(valid), "InferenceEngine::Core::LoadNetwork doesn't support inputs having dynamic shapes. ", "Use ov::Core::compile_model API instead. Dynamic inputs are :", @@ -125,7 +151,7 @@ ExecutableNetwork Core::LoadNetwork(const std::string& modelPath, const std::string& deviceName, const std::map& config) { auto exec = _impl->LoadNetwork(modelPath, deviceName, config, [](const CNNNetwork& network) { - auto valid = ov::CoreImpl::CheckStatic(network); + auto valid = ::CheckStatic(network); OPENVINO_ASSERT(std::get<0>(valid), "InferenceEngine::Core::LoadNetwork doesn't support inputs having dynamic shapes. ", "Use ov::Core::compile_model API instead. Dynamic inputs are :", @@ -239,7 +265,7 @@ ExecutableNetwork Core::ImportNetwork(std::istream& networkModel, QueryNetworkResult Core::QueryNetwork(const CNNNetwork& network, const std::string& deviceName, const std::map& config) const { - auto valid = ov::CoreImpl::CheckStatic(network); + auto valid = ::CheckStatic(network); OPENVINO_ASSERT(std::get<0>(valid), "InferenceEngine::Core::QueryNetwork doesn't support inputs having dynamic shapes. ", "Use ov::Core::compile_model API instead. Dynamic inputs are :", diff --git a/src/inference/tests/unit/ie_compilation_context_test.cpp b/src/inference/tests/unit/ie_compilation_context_test.cpp index 4e035f6d26f..42906daab83 100644 --- a/src/inference/tests/unit/ie_compilation_context_test.cpp +++ b/src/inference/tests/unit/ie_compilation_context_test.cpp @@ -62,50 +62,46 @@ public: }; TEST_F(NetworkContext_CalcFileInfoTests, NoFile) { - ASSERT_NE(NetworkCompilationContext::calculate_file_info("notexisting.abc"), - NetworkCompilationContext::calculate_file_info("notexisting2.abc")); + ASSERT_NE(ModelCache::calculate_file_info("notexisting.abc"), ModelCache::calculate_file_info("notexisting2.abc")); std::string fileName(100, 'a'); std::string fileName2(fileName); - ASSERT_EQ(NetworkCompilationContext::calculate_file_info(fileName), - NetworkCompilationContext::calculate_file_info(fileName2)); + ASSERT_EQ(ModelCache::calculate_file_info(fileName), ModelCache::calculate_file_info(fileName2)); } TEST_F(NetworkContext_CalcFileInfoTests, ExistingFile) { - ASSERT_EQ(NetworkCompilationContext::calculate_file_info(m_fileName), - NetworkCompilationContext::calculate_file_info(m_fileName)); + ASSERT_EQ(ModelCache::calculate_file_info(m_fileName), ModelCache::calculate_file_info(m_fileName)); } TEST_F(NetworkContext_CalcFileInfoTests, ExistingDiffFiles) { - auto hash1 = NetworkCompilationContext::calculate_file_info(m_fileName); + auto hash1 = ModelCache::calculate_file_info(m_fileName); std::string newName = m_fileName + "2"; std::rename(m_fileName.c_str(), newName.c_str()); m_fileName = std::move(newName); - auto hash2 = NetworkCompilationContext::calculate_file_info(m_fileName); + auto hash2 = ModelCache::calculate_file_info(m_fileName); ASSERT_NE(hash1, hash2); } TEST_F(NetworkContext_CalcFileInfoTests, ExistingFile_sameAbsPath) { std::string file1 = m_fileName; std::string file2 = std::string(".") + CommonTestUtils::FileSeparator + m_fileName; - ASSERT_EQ(NetworkCompilationContext::calculate_file_info(file1), - NetworkCompilationContext::calculate_file_info(file2)) + ASSERT_EQ(ModelCache::calculate_file_info(file1), ModelCache::calculate_file_info(file2)) << "Hash of [" << file1 << "] is not equal to hash of [" << file2 << "]"; } TEST_F(NetworkContext_CalcFileInfoTests, DateModified) { - auto info1 = NetworkCompilationContext::calculate_file_info(m_fileName); + auto info1 = ModelCache::calculate_file_info(m_fileName); std::this_thread::sleep_for(std::chrono::seconds(2)); createFile(m_fileName); - auto info2 = NetworkCompilationContext::calculate_file_info(m_fileName); + auto info2 = ModelCache::calculate_file_info(m_fileName); ASSERT_NE(info1, info2); } TEST_F(NetworkContext_CalcFileInfoTests, SizeModified) { createFile(m_fileName, 1); - auto info1 = NetworkCompilationContext::calculate_file_info(m_fileName); + auto info1 = ModelCache::calculate_file_info(m_fileName); createFile(m_fileName, 2); - auto info2 = NetworkCompilationContext::calculate_file_info(m_fileName); + auto info2 = ModelCache::calculate_file_info(m_fileName); ASSERT_NE(info1, info2); } @@ -154,34 +150,32 @@ static void checkCustomRt(const std::function& emptyCb, auto& op2 = model2->get_ops().front()->get_rt_info(); emptyCb(op2); - ASSERT_NE(NetworkCompilationContext::compute_hash(model1, {}), NetworkCompilationContext::compute_hash(model2, {})); + ASSERT_NE(ModelCache::compute_hash(model1, {}), ModelCache::compute_hash(model2, {})); emptyCb(op1); - ASSERT_EQ(NetworkCompilationContext::compute_hash(model1, {}), NetworkCompilationContext::compute_hash(model2, {})); + ASSERT_EQ(ModelCache::compute_hash(model1, {}), ModelCache::compute_hash(model2, {})); nameCb(op1, "test"); - ASSERT_NE(NetworkCompilationContext::compute_hash(model1, {}), NetworkCompilationContext::compute_hash(model2, {})); + ASSERT_NE(ModelCache::compute_hash(model1, {}), ModelCache::compute_hash(model2, {})); nameCb(op2, "test"); - ASSERT_EQ(NetworkCompilationContext::compute_hash(model1, {}), NetworkCompilationContext::compute_hash(model2, {})); + ASSERT_EQ(ModelCache::compute_hash(model1, {}), ModelCache::compute_hash(model2, {})); nameCb(op1, "test2"); - ASSERT_NE(NetworkCompilationContext::compute_hash(model1, {}), NetworkCompilationContext::compute_hash(model2, {})); + ASSERT_NE(ModelCache::compute_hash(model1, {}), ModelCache::compute_hash(model2, {})); } TEST(NetworkContext, HashOfSame) { auto model1 = create_simple_function(); auto model2 = create_simple_function(); - ASSERT_EQ(NetworkCompilationContext::compute_hash(model1, {}), NetworkCompilationContext::compute_hash(model2, {})); + ASSERT_EQ(ModelCache::compute_hash(model1, {}), ModelCache::compute_hash(model2, {})); } TEST(NetworkContext, HashWithConfig) { auto net1 = create_simple_function(); auto net2 = create_simple_function(); - ASSERT_NE(NetworkCompilationContext::compute_hash(net1, {{"key", "value"}}), - NetworkCompilationContext::compute_hash(net2, {})); - ASSERT_EQ(NetworkCompilationContext::compute_hash(net1, {{"key", "value"}}), - NetworkCompilationContext::compute_hash(net2, {{"key", "value"}})); + ASSERT_NE(ModelCache::compute_hash(net1, {{"key", "value"}}), ModelCache::compute_hash(net2, {})); + ASSERT_EQ(ModelCache::compute_hash(net1, {{"key", "value"}}), ModelCache::compute_hash(net2, {{"key", "value"}})); } TEST(NetworkContext, HashWithPrimitivesPriority) { @@ -194,9 +188,9 @@ TEST(NetworkContext, HashWithPrimitivesPriority) { auto& op3 = net3->get_ops().front()->get_rt_info(); op3["PrimitivesPriority"] = "testPriority"; - ASSERT_NE(NetworkCompilationContext::compute_hash(net1, {}), NetworkCompilationContext::compute_hash(net2, {})); + ASSERT_NE(ModelCache::compute_hash(net1, {}), ModelCache::compute_hash(net2, {})); - ASSERT_EQ(NetworkCompilationContext::compute_hash(net2, {}), NetworkCompilationContext::compute_hash(net3, {})); + ASSERT_EQ(ModelCache::compute_hash(net2, {}), ModelCache::compute_hash(net3, {})); } TEST(NetworkContext, HashWithFusedNames) { @@ -229,9 +223,9 @@ TEST(NetworkContext, HashWithAffinity) { auto& op3 = net3->get_ops().front()->get_rt_info(); op3["affinity"] = "testAffinity"; - ASSERT_NE(NetworkCompilationContext::compute_hash(net1, {}), NetworkCompilationContext::compute_hash(net2, {})); + ASSERT_NE(ModelCache::compute_hash(net1, {}), ModelCache::compute_hash(net2, {})); - ASSERT_EQ(NetworkCompilationContext::compute_hash(net2, {}), NetworkCompilationContext::compute_hash(net3, {})); + ASSERT_EQ(ModelCache::compute_hash(net2, {}), ModelCache::compute_hash(net3, {})); } TEST(NetworkContext, HashWithFutureRt_string) { @@ -248,9 +242,9 @@ TEST(NetworkContext, HashWithFutureRt_string) { auto& op3 = net3->get_ops().front()->get_rt_info(); op3["someFutureKey"] = "olleh"; - ASSERT_EQ(NetworkCompilationContext::compute_hash(net1, {}), NetworkCompilationContext::compute_hash(net2, {})); + ASSERT_EQ(ModelCache::compute_hash(net1, {}), ModelCache::compute_hash(net2, {})); - ASSERT_NE(NetworkCompilationContext::compute_hash(net2, {}), NetworkCompilationContext::compute_hash(net3, {})); + ASSERT_NE(ModelCache::compute_hash(net2, {}), ModelCache::compute_hash(net3, {})); } TEST(NetworkContext, HashWithFutureRt_int64) { @@ -267,9 +261,9 @@ TEST(NetworkContext, HashWithFutureRt_int64) { auto& op3 = net3->get_ops().front()->get_rt_info(); op3["someFutureKey"] = int64_t(43); - ASSERT_EQ(NetworkCompilationContext::compute_hash(net1, {}), NetworkCompilationContext::compute_hash(net2, {})); + ASSERT_EQ(ModelCache::compute_hash(net1, {}), ModelCache::compute_hash(net2, {})); - ASSERT_NE(NetworkCompilationContext::compute_hash(net2, {}), NetworkCompilationContext::compute_hash(net3, {})); + ASSERT_NE(ModelCache::compute_hash(net2, {}), ModelCache::compute_hash(net3, {})); } TEST(NetworkContext, HashWithTensorNames) { @@ -292,9 +286,9 @@ TEST(NetworkContext, HashWithTensorNames) { fun1->input().set_names(names1); fun2->input().set_names(names2); - ASSERT_EQ(NetworkCompilationContext::compute_hash(fun1, {}), NetworkCompilationContext::compute_hash(fun2, {})); + ASSERT_EQ(ModelCache::compute_hash(fun1, {}), ModelCache::compute_hash(fun2, {})); - ASSERT_NE(NetworkCompilationContext::compute_hash(fun2, {}), NetworkCompilationContext::compute_hash(fun3, {})); + ASSERT_NE(ModelCache::compute_hash(fun2, {}), ModelCache::compute_hash(fun3, {})); } TEST(NetworkContext, HashWithDifferentResults) { @@ -303,8 +297,8 @@ TEST(NetworkContext, HashWithDifferentResults) { net2->remove_result(net2->get_results().front()); auto net3 = create_simple_function(); net3->remove_result(net3->get_results().front()); - ASSERT_NE(NetworkCompilationContext::compute_hash(net1, {}), NetworkCompilationContext::compute_hash(net2, {})); - ASSERT_EQ(NetworkCompilationContext::compute_hash(net2, {}), NetworkCompilationContext::compute_hash(net3, {})); + ASSERT_NE(ModelCache::compute_hash(net1, {}), ModelCache::compute_hash(net2, {})); + ASSERT_EQ(ModelCache::compute_hash(net2, {}), ModelCache::compute_hash(net3, {})); } // Verify all internal hash calculations are thread-safe (like ngraph::function serialization) @@ -318,8 +312,8 @@ TEST(NetworkContext, HashOfSameMultiThreading) { auto threadFun = [&](int& count) { do { count++; - auto hash1 = NetworkCompilationContext::compute_hash(net1, {}); - auto hash2 = NetworkCompilationContext::compute_hash(net2, {}); + auto hash1 = ModelCache::compute_hash(net1, {}); + auto hash2 = ModelCache::compute_hash(net2, {}); if (hash1 != hash2) { fail = true; break; @@ -337,17 +331,14 @@ TEST(NetworkContext, HashOfSameMultiThreading) { //////////////////////////////////////////// TEST(NetworkContext_ModelName, HashOfSame) { - ASSERT_EQ(NetworkCompilationContext::compute_hash("model1", {}), - NetworkCompilationContext::compute_hash("model1", {})); + ASSERT_EQ(ModelCache::compute_hash("model1", {}), ModelCache::compute_hash("model1", {})); - ASSERT_NE(NetworkCompilationContext::compute_hash("model1", {}), - NetworkCompilationContext::compute_hash("model2", {})); + ASSERT_NE(ModelCache::compute_hash("model1", {}), ModelCache::compute_hash("model2", {})); - ASSERT_NE(NetworkCompilationContext::compute_hash("model1", {{"key", "value"}}), - NetworkCompilationContext::compute_hash("model1", {})); + ASSERT_NE(ModelCache::compute_hash("model1", {{"key", "value"}}), ModelCache::compute_hash("model1", {})); - ASSERT_EQ(NetworkCompilationContext::compute_hash("model1", {{"key", "value"}}), - NetworkCompilationContext::compute_hash("model1", {{"key", "value"}})); + ASSERT_EQ(ModelCache::compute_hash("model1", {{"key", "value"}}), + ModelCache::compute_hash("model1", {{"key", "value"}})); } TEST(NetworkContext_ModelName, HashOfExistingFile) { @@ -359,13 +350,11 @@ TEST(NetworkContext_ModelName, HashOfExistingFile) { std::ofstream os(file1); os << "test"; } - ASSERT_EQ(NetworkCompilationContext::compute_hash(file1, {}), NetworkCompilationContext::compute_hash(file1, {})); + ASSERT_EQ(ModelCache::compute_hash(file1, {}), ModelCache::compute_hash(file1, {})); - ASSERT_EQ(NetworkCompilationContext::compute_hash(file1, {}), NetworkCompilationContext::compute_hash(file2, {})); + ASSERT_EQ(ModelCache::compute_hash(file1, {}), ModelCache::compute_hash(file2, {})); - ASSERT_NE(NetworkCompilationContext::compute_hash(file1, {{"key", "value"}}), - NetworkCompilationContext::compute_hash(file2, {})); + ASSERT_NE(ModelCache::compute_hash(file1, {{"key", "value"}}), ModelCache::compute_hash(file2, {})); - ASSERT_EQ(NetworkCompilationContext::compute_hash(file1, {{"key", "value"}}), - NetworkCompilationContext::compute_hash(file2, {{"key", "value"}})); + ASSERT_EQ(ModelCache::compute_hash(file1, {{"key", "value"}}), ModelCache::compute_hash(file2, {{"key", "value"}})); } diff --git a/src/plugins/auto/plugin.cpp b/src/plugins/auto/plugin.cpp index da5ade70d47..448829f32d9 100644 --- a/src/plugins/auto/plugin.cpp +++ b/src/plugins/auto/plugin.cpp @@ -516,7 +516,6 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons DeviceMap executableNetworkPerDevice; std::mutex load_mutex; std::vector loads; - std::once_flag readNetworkFlag; auto loadInferEngTask = [&](DeviceInformation& p) { auto tmpiter = fullConfig.find(CONFIG_KEY(ALLOW_AUTO_BATCHING)); diff --git a/src/plugins/hetero/executable_network.cpp b/src/plugins/hetero/executable_network.cpp index 9b0fb7b2029..6f37bdd7c3a 100644 --- a/src/plugins/hetero/executable_network.cpp +++ b/src/plugins/hetero/executable_network.cpp @@ -444,10 +444,12 @@ HeteroExecutableNetwork::HeteroExecutableNetwork(const InferenceEngine::CNNNetwo } for (auto&& network : _networks) { auto metaDevices = _heteroPlugin->GetDevicePlugins(network._device, _config); - metaDevices[network._device].emplace(CONFIG_KEY_INTERNAL(FORCE_DISABLE_CACHE), ""); - network._network = _heteroPlugin->GetCore()->LoadNetwork(network._clonedNetwork, - network._device, - metaDevices[network._device]); + + auto config = metaDevices[network._device]; + // disable caching for subgraphs, because the whole HERERO model is cached + config[ov::cache_dir.name()] = ""; + + network._network = _heteroPlugin->GetCore()->LoadNetwork(network._clonedNetwork, network._device, config); } } diff --git a/src/plugins/intel_cpu/src/config.cpp b/src/plugins/intel_cpu/src/config.cpp index e11ca103be8..071a9be9ba8 100644 --- a/src/plugins/intel_cpu/src/config.cpp +++ b/src/plugins/intel_cpu/src/config.cpp @@ -165,8 +165,6 @@ void Config::readProperties(const std::map &prop) { IE_THROW() << "Wrong value for property key " << ov::inference_precision.name() << ". Supported values: bf16, f32"; } - } else if (key == PluginConfigParams::KEY_CACHE_DIR) { - cache_dir = val; } else if (PluginConfigInternalParams::KEY_CPU_RUNTIME_CACHE_CAPACITY == key) { int val_i = -1; try { @@ -261,7 +259,6 @@ void Config::updateProperties() { _config.insert({ PluginConfigParams::KEY_PERFORMANCE_HINT, perfHintsConfig.ovPerfHint }); _config.insert({ PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, std::to_string(perfHintsConfig.ovPerfHintNumRequests) }); - _config.insert({PluginConfigParams::KEY_CACHE_DIR, cache_dir}); } } // namespace intel_cpu diff --git a/src/plugins/intel_cpu/src/config.h b/src/plugins/intel_cpu/src/config.h index 012fd0fd9f0..efd3c47a1b8 100644 --- a/src/plugins/intel_cpu/src/config.h +++ b/src/plugins/intel_cpu/src/config.h @@ -59,8 +59,6 @@ struct Config { bool manualEnforceBF16 = false; #endif - std::string cache_dir{}; - DenormalsOptMode denormalsOptMode = DenormalsOptMode::DO_Keep; void readProperties(const std::map &config); diff --git a/src/plugins/intel_cpu/src/plugin.cpp b/src/plugins/intel_cpu/src/plugin.cpp index 159134fbdb8..b93b6704c55 100644 --- a/src/plugins/intel_cpu/src/plugin.cpp +++ b/src/plugins/intel_cpu/src/plugin.cpp @@ -587,7 +587,6 @@ Parameter Engine::GetMetric(const std::string& name, const std::map rwProperties {RW_property(ov::num_streams.name()), diff --git a/src/plugins/template/src/plugin.cpp b/src/plugins/template/src/plugin.cpp index 6821534deec..2b0f7671273 100644 --- a/src/plugins/template/src/plugin.cpp +++ b/src/plugins/template/src/plugin.cpp @@ -265,8 +265,8 @@ ov::Any ov::template_plugin::Plugin::get_property(const std::string& name, const std::string arch = "TEMPLATE"; return decltype(ov::device::architecture)::value_type(arch); } else if (ov::device::capabilities == name) { - // TODO: fill actual list of supported capabilities: e.g. Template device supports only FP32 - std::vector capabilities = {ov::device::capability::FP32}; + // TODO: fill actual list of supported capabilities: e.g. Template device supports only FP32 and EXPORT_IMPORT + std::vector capabilities = {ov::device::capability::FP32, ov::device::capability::EXPORT_IMPORT}; return decltype(ov::device::capabilities)::value_type(capabilities); } else if (ov::range_for_async_infer_requests == name) { // TODO: fill with actual values