IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix Loaded from cache for new plugin API (#16301)

Browse Source 
* Extend template plugin tests

* Fixed loaded_from_cache for new API

* Added const

* Added ov::loaded_from_cache as supported property of CompiledModel

* Remove loaded_from_cache from core

* Reverted logic for old plugins

* Fixed comments

* Fixed build
This commit is contained in:
Ilya Churaev 2023-03-16 12:29:56 +04:00 committed by GitHub
parent 2194552dc5
commit fbc420093d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 134 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/inference/dev_api/openvino/runtime/icompiled_model.hpp
View File

@ -126,11 +126,6 @@ private:
friend ov::IExecutableNetworkWrapper;
friend InferenceEngine::ICompiledModelWrapper;
/**
* @brief function allows to mark that model was loaded from cache
*/
void loaded_from_cache();
// FIXME: Remove after removing IE API
std::vector<std::shared_ptr<const ov::Node>> _parameters;
std::vector<std::shared_ptr<const ov::Node>> _results;

1
src/inference/src/compiled_model.cpp
View File

@ -6,6 +6,7 @@
#include "openvino/core/except.hpp"
#include "openvino/runtime/icompiled_model.hpp"
#include "openvino/runtime/properties.hpp"
#define OV_COMPILED_MODEL_CALL_STATEMENT(...) \
OPENVINO_ASSERT(_impl != nullptr, "CompiledModel was not initialized."); \

8
src/inference/src/dev/core_impl.cpp
View File

@ -13,6 +13,7 @@
#include "cpp_interfaces/interface/ie_internal_plugin_config.hpp"
#include "cpp_interfaces/interface/ie_iplugin_internal.hpp"
#include "dev/converter_utils.hpp"
#include "dev/icompiled_model_wrapper.hpp"
#include "dev/make_tensor.hpp"
#include "file_utils.h"
#include "ie_itt.hpp"
@ -409,6 +410,9 @@ ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::import_model(std::istream& model,
OV_ITT_SCOPED_TASK(ov::itt::domains::IE, "Core::import_model");
auto parsed = parseDeviceNameIntoConfig(device_name, config);
auto compiled_model = get_plugin(parsed._deviceName).import_model(model, config);
if (auto wrapper = std::dynamic_pointer_cast<InferenceEngine::ICompiledModelWrapper>(compiled_model._ptr)) {
wrapper->get_executable_network()->loadedFromCache();
}
return compiled_model;
}
@ -920,7 +924,9 @@ ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::load_model_from_cache(
compiled_model = context._impl ? plugin.import_model(networkStream, context, config)
: plugin.import_model(networkStream, config);
compiled_model->loaded_from_cache();
if (auto wrapper = std::dynamic_pointer_cast<InferenceEngine::ICompiledModelWrapper>(compiled_model._ptr)) {
wrapper->get_executable_network()->loadedFromCache();
}
});
} catch (const HeaderException&) {
// For these exceptions just remove old cache and set that import didn't work

4
src/inference/src/dev/core_impl_ie.cpp
View File

@ -10,6 +10,7 @@
#include "cpp_interfaces/interface/ie_internal_plugin_config.hpp"
#include "cpp_interfaces/interface/ie_iplugin_internal.hpp"
#include "dev/converter_utils.hpp"
#include "dev/icompiled_model_wrapper.hpp"
#include "dev/make_tensor.hpp"
#include "ie_itt.hpp"
#include "ie_network_reader.hpp"
@ -128,6 +129,9 @@ InferenceEngine::SoExecutableNetworkInternal ov::CoreImpl::ImportNetwork(
const std::string& deviceName,
const std::map<std::string, std::string>& config) {
auto compiled_model = import_model(networkModel, deviceName, any_copy(config));
if (auto wrapper = std::dynamic_pointer_cast<InferenceEngine::ICompiledModelWrapper>(compiled_model._ptr)) {
wrapper->get_executable_network()->loadedFromCache();
}
return {ov::legacy_convert::convert_compiled_model(compiled_model._ptr), compiled_model._so};
}

9
src/inference/src/dev/icompiled_model.cpp
View File

@ -6,6 +6,7 @@
#include "icompiled_model_wrapper.hpp"
#include "openvino/core/model.hpp"
#include "openvino/runtime/properties.hpp"
#include "transformations/utils/utils.hpp"
ov::ICompiledModel::ICompiledModel(const std::shared_ptr<const ov::Model>& model,
@ -91,11 +92,3 @@ const std::shared_ptr<ov::threading::ITaskExecutor> ov::ICompiledModel::get_task
const std::shared_ptr<ov::threading::ITaskExecutor> ov::ICompiledModel::get_callback_executor() const {
return m_callback_executor;
}
void ov::ICompiledModel::loaded_from_cache() {
if (auto wrapper = dynamic_cast<InferenceEngine::ICompiledModelWrapper*>(this)) {
wrapper->get_executable_network()->loadedFromCache();
return;
}
// OPENVINO_NOT_IMPLEMENTED;
}

5
src/inference/src/dev/icompiled_model_wrapper.cpp
View File

@ -85,3 +85,8 @@ std::shared_ptr<InferenceEngine::IExecutableNetworkInternal>
InferenceEngine::ICompiledModelWrapper::get_executable_network() {
return m_model;
}
std::shared_ptr<const InferenceEngine::IExecutableNetworkInternal>
InferenceEngine::ICompiledModelWrapper::get_executable_network() const {
return m_model;
}

1
src/inference/src/dev/icompiled_model_wrapper.hpp
View File

@ -26,6 +26,7 @@ public:
ov::RemoteContext get_context() const override;
std::shared_ptr<InferenceEngine::IExecutableNetworkInternal> get_executable_network();
std::shared_ptr<const InferenceEngine::IExecutableNetworkInternal> get_executable_network() const;
private:
std::shared_ptr<InferenceEngine::IExecutableNetworkInternal> m_model;

13
src/plugins/template/src/compiled_model.cpp
View File

@ -19,10 +19,12 @@
ov::template_plugin::CompiledModel::CompiledModel(const std::shared_ptr<ov::Model>& model,
const std::shared_ptr<const ov::IPlugin>& plugin,
const std::shared_ptr<ov::threading::ITaskExecutor>& task_executor,
const Configuration& cfg)
const Configuration& cfg,
bool loaded_from_cache)
: ov::ICompiledModel(model, plugin, task_executor), // Disable default threads creation
_cfg(cfg),
m_model(model) {
m_model(model),
m_loaded_from_cache(loaded_from_cache) {
// TODO: if your plugin supports device ID (more that single instance of device can be on host machine)
// you should select proper device based on KEY_DEVICE_ID or automatic behavior
// In this case, _waitExecutor should also be created per device.
@ -95,6 +97,7 @@ ov::Any ov::template_plugin::CompiledModel::get_property(const std::string& name
const auto& default_ro_properties = []() {
std::vector<ov::PropertyName> ro_properties{ov::model_name,
ov::supported_properties,
ov::loaded_from_cache,
ov::optimal_number_of_infer_requests};
return ro_properties;
};
@ -129,12 +132,12 @@ ov::Any ov::template_plugin::CompiledModel::get_property(const std::string& name
} else if (ov::model_name == name) {
auto model_name = m_model->get_friendly_name();
return decltype(ov::model_name)::value_type(model_name);
} else if (ov::loaded_from_cache == name) {
return m_loaded_from_cache;
} else if (ov::optimal_number_of_infer_requests == name) {
unsigned int value = _cfg.streams_executor_config._streams;
return decltype(ov::optimal_number_of_infer_requests)::value_type(value);
}
if (ov::supported_properties == name) {
} else if (ov::supported_properties == name) {
auto ro_properties = default_ro_properties();
auto rw_properties = default_rw_properties();

4
src/plugins/template/src/compiled_model.hpp
View File

@ -26,7 +26,8 @@ public:
CompiledModel(const std::shared_ptr<ov::Model>& model,
const std::shared_ptr<const ov::IPlugin>& plugin,
const std::shared_ptr<ov::threading::ITaskExecutor>& task_executor,
const Configuration& cfg);
const Configuration& cfg,
bool loaded_from_cache = false);
// Methods from a base class ov::ICompiledModel
void export_model(std::ostream& model) const override;
@ -53,6 +54,7 @@ private:
mutable std::atomic<std::size_t> _requestId = {0};
Configuration _cfg;
std::shared_ptr<ov::Model> m_model;
const bool m_loaded_from_cache;
};
// ! [executable_network:header]

3
src/plugins/template/src/plugin.cpp
View File

@ -140,7 +140,8 @@ std::shared_ptr<ov::ICompiledModel> ov::template_plugin::Plugin::import_model(st
std::make_shared<CompiledModel>(ov_model,
shared_from_this(),
get_executor_manager()->get_idle_cpu_streams_executor(streamsExecutorConfig),
fullConfig);
fullConfig,
true);
return compiled_model;
}
// ! [plugin:import_model]

152
src/plugins/template/tests/functional/shared_tests_instances/behavior/ov_executable_network/properties.cpp
View File

@ -3,6 +3,7 @@
//
#include "behavior/ov_executable_network/properties.hpp"
#include "openvino/runtime/properties.hpp"
using namespace ov::test::behavior;
@ -10,117 +11,118 @@ using namespace ov::test::behavior;
namespace {
const std::vector<ov::AnyMap> inproperties = {
{ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")},
{ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")},
};
const std::vector<ov::AnyMap> hetero_inproperties = {
{ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")},
{ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")},
};
const std::vector<ov::AnyMap> multi_inproperties = {
{ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")},
{ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")},
};
const std::vector<ov::AnyMap> auto_inproperties = {
{ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")},
{ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")},
};
const std::vector<ov::AnyMap> auto_batch_inproperties = {
{ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")},
{{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , std::string(CommonTestUtils::DEVICE_TEMPLATE) + "(4)"},
{ov::auto_batch_timeout(-1)}},
{ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")},
{{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG), std::string(CommonTestUtils::DEVICE_TEMPLATE) + "(4)"},
{ov::auto_batch_timeout(-1)}},
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVCompiledModelPropertiesIncorrectTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::ValuesIn(inproperties)),
OVCompiledModelPropertiesIncorrectTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests,
OVCompiledModelPropertiesIncorrectTests,
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::ValuesIn(inproperties)),
OVCompiledModelPropertiesIncorrectTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVCompiledModelPropertiesIncorrectTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(hetero_inproperties)),
OVCompiledModelPropertiesIncorrectTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests,
OVCompiledModelPropertiesIncorrectTests,
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(hetero_inproperties)),
OVCompiledModelPropertiesIncorrectTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVCompiledModelPropertiesIncorrectTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(multi_inproperties)),
OVCompiledModelPropertiesIncorrectTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests,
OVCompiledModelPropertiesIncorrectTests,
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(multi_inproperties)),
OVCompiledModelPropertiesIncorrectTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests, OVCompiledModelPropertiesIncorrectTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(auto_inproperties)),
OVCompiledModelPropertiesIncorrectTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Auto_BehaviorTests,
OVCompiledModelPropertiesIncorrectTests,
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_AUTO),
::testing::ValuesIn(auto_inproperties)),
OVCompiledModelPropertiesIncorrectTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_AutoBatch_BehaviorTests, OVCompiledModelPropertiesIncorrectTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_BATCH),
::testing::ValuesIn(auto_batch_inproperties)),
OVCompiledModelPropertiesIncorrectTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_AutoBatch_BehaviorTests,
OVCompiledModelPropertiesIncorrectTests,
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_BATCH),
::testing::ValuesIn(auto_batch_inproperties)),
OVCompiledModelPropertiesIncorrectTests::getTestCaseName);
const std::vector<ov::AnyMap> default_properties = {
{ov::enable_profiling(true)},
{ov::device::id("0")},
{ov::enable_profiling(true)},
{{ov::loaded_from_cache.name(), false}},
{ov::device::id("0")},
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVCompiledModelPropertiesDefaultTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::ValuesIn(default_properties)),
OVCompiledModelPropertiesDefaultTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests,
OVCompiledModelPropertiesDefaultTests,
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::ValuesIn(default_properties)),
OVCompiledModelPropertiesDefaultTests::getTestCaseName);
const std::vector<ov::AnyMap> properties = {
{ov::enable_profiling(true)},
{ov::device::id("0")},
{ov::enable_profiling(true)},
{ov::device::id("0")},
};
const std::vector<ov::AnyMap> hetero_properties = {
{ov::device::priorities(CommonTestUtils::DEVICE_TEMPLATE), ov::enable_profiling(true)},
{ov::device::priorities(CommonTestUtils::DEVICE_TEMPLATE), ov::device::id("0")},
{ov::device::priorities(CommonTestUtils::DEVICE_TEMPLATE), ov::enable_profiling(true)},
{ov::device::priorities(CommonTestUtils::DEVICE_TEMPLATE), ov::device::id("0")},
};
const std::vector<ov::AnyMap> multi_properties = {
{ov::device::priorities(CommonTestUtils::DEVICE_TEMPLATE), ov::enable_profiling(true)},
{ov::device::priorities(CommonTestUtils::DEVICE_TEMPLATE), ov::device::id("0")},
{ov::device::priorities(CommonTestUtils::DEVICE_TEMPLATE), ov::enable_profiling(true)},
{ov::device::priorities(CommonTestUtils::DEVICE_TEMPLATE), ov::device::id("0")},
};
const std::vector<ov::AnyMap> auto_batch_properties = {
{{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , std::string(CommonTestUtils::DEVICE_TEMPLATE) + "(4)"}},
{{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , std::string(CommonTestUtils::DEVICE_TEMPLATE) + "(4)"}, {CONFIG_KEY(AUTO_BATCH_TIMEOUT) , "1"}},
{{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , std::string(CommonTestUtils::DEVICE_TEMPLATE) + "(4)"}, {ov::auto_batch_timeout(10)}},
{{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG), std::string(CommonTestUtils::DEVICE_TEMPLATE) + "(4)"}},
{{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG), std::string(CommonTestUtils::DEVICE_TEMPLATE) + "(4)"},
{CONFIG_KEY(AUTO_BATCH_TIMEOUT), "1"}},
{{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG), std::string(CommonTestUtils::DEVICE_TEMPLATE) + "(4)"},
{ov::auto_batch_timeout(10)}},
};
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVCompiledModelPropertiesTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::ValuesIn(properties)),
OVCompiledModelPropertiesTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests,
OVCompiledModelPropertiesTests,
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::ValuesIn(properties)),
OVCompiledModelPropertiesTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVCompiledModelPropertiesTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(hetero_properties)),
OVCompiledModelPropertiesTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests,
OVCompiledModelPropertiesTests,
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_HETERO),
::testing::ValuesIn(hetero_properties)),
OVCompiledModelPropertiesTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests, OVCompiledModelPropertiesTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(multi_properties)),
OVCompiledModelPropertiesTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_Multi_BehaviorTests,
OVCompiledModelPropertiesTests,
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_MULTI),
::testing::ValuesIn(multi_properties)),
OVCompiledModelPropertiesTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_AutoBatch_BehaviorTests, OVCompiledModelPropertiesTests,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_BATCH),
::testing::ValuesIn(auto_batch_properties)),
OVCompiledModelPropertiesTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_AutoBatch_BehaviorTests,
OVCompiledModelPropertiesTests,
::testing::Combine(::testing::Values(CommonTestUtils::DEVICE_BATCH),
::testing::ValuesIn(auto_batch_properties)),
OVCompiledModelPropertiesTests::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(
smoke_BehaviorTests, OVCompiledModelEmptyPropertiesTests, ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
OVCompiledModelEmptyPropertiesTests::getTestCaseName);
} // namespace
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests,
OVCompiledModelEmptyPropertiesTests,
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
OVCompiledModelEmptyPropertiesTests::getTestCaseName);
} // namespace

40
src/plugins/template/tests/functional/shared_tests_instances/behavior/ov_plugin/caching_tests.cpp
View File

@ -7,20 +7,30 @@
using namespace ov::test::behavior;
namespace {
static const std::vector<ov::element::Type> precisionsTemplate = {
ov::element::f32,
};
static const std::vector<ov::element::Type> precisionsTemplate = {
ov::element::f32,
};
static const std::vector<std::size_t> batchSizesTemplate = {
1, 2
};
static const std::vector<std::size_t> batchSizesTemplate = {1, 2};
INSTANTIATE_TEST_SUITE_P(smoke_Behavior_CachingSupportCase_Template, CompileModelCacheTestBase,
::testing::Combine(
::testing::ValuesIn(CompileModelCacheTestBase::getStandardFunctions()),
::testing::ValuesIn(precisionsTemplate),
::testing::ValuesIn(batchSizesTemplate),
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::Values(ov::AnyMap{})),
CompileModelCacheTestBase::getTestCaseName);
} // namespace
INSTANTIATE_TEST_SUITE_P(smoke_Behavior_CachingSupportCase_Template,
CompileModelCacheTestBase,
::testing::Combine(::testing::ValuesIn(CompileModelCacheTestBase::getStandardFunctions()),
::testing::ValuesIn(precisionsTemplate),
::testing::ValuesIn(batchSizesTemplate),
::testing::Values(CommonTestUtils::DEVICE_TEMPLATE),
::testing::Values(ov::AnyMap{})),
CompileModelCacheTestBase::getTestCaseName);
const std::vector<ov::AnyMap> TemplateConfigs = {
{ov::num_streams(2)},
};
const std::vector<std::string> TestTemplateTargets = {
CommonTestUtils::DEVICE_TEMPLATE,
};
INSTANTIATE_TEST_SUITE_P(smoke_CachingSupportCase_Template,
CompileModelLoadFromMemoryTestBase,
::testing::Combine(::testing::ValuesIn(TestTemplateTargets),
::testing::ValuesIn(TemplateConfigs)),
CompileModelLoadFromMemoryTestBase::getTestCaseName);
} // namespace