From 01967bdbffffa658b1cd32167003c7c7651d413c Mon Sep 17 00:00:00 2001 From: Eddy Kim Date: Tue, 19 Dec 2023 02:50:33 +0900 Subject: [PATCH] [GPU] Adding a switch to change the caching policy (#21443) * added a new property to enable full model caching * added a new public property cache_mode * updated according to clang-format guide * changed default caching policy to optimize_speed --- .../pyopenvino/core/properties/properties.cpp | 1 + .../include/openvino/runtime/properties.hpp | 43 +++++++++++++++++++ .../intel_gpu/src/plugin/compiled_model.cpp | 4 ++ src/plugins/intel_gpu/src/plugin/plugin.cpp | 4 ++ .../src/runtime/execution_config.cpp | 1 + .../intel_gpu/src/runtime/kernels_cache.cpp | 3 +- 6 files changed, 55 insertions(+), 1 deletion(-) diff --git a/src/bindings/python/src/pyopenvino/core/properties/properties.cpp b/src/bindings/python/src/pyopenvino/core/properties/properties.cpp index 23e10bedadc..e8fdc7ceeb8 100644 --- a/src/bindings/python/src/pyopenvino/core/properties/properties.cpp +++ b/src/bindings/python/src/pyopenvino/core/properties/properties.cpp @@ -24,6 +24,7 @@ void regmodule_properties(py::module m) { // Submodule properties - properties wrap_property_RW(m_properties, ov::enable_profiling, "enable_profiling"); wrap_property_RW(m_properties, ov::cache_dir, "cache_dir"); + wrap_property_RW(m_properties, ov::cache_mode, "cache_mode"); wrap_property_RW(m_properties, ov::auto_batch_timeout, "auto_batch_timeout"); wrap_property_RW(m_properties, ov::num_streams, "num_streams"); wrap_property_RW(m_properties, ov::inference_num_threads, "inference_num_threads"); diff --git a/src/inference/include/openvino/runtime/properties.hpp b/src/inference/include/openvino/runtime/properties.hpp index f28e46de6b6..aa7f8b2acde 100644 --- a/src/inference/include/openvino/runtime/properties.hpp +++ b/src/inference/include/openvino/runtime/properties.hpp @@ -618,6 +618,49 @@ static constexpr Property cache_dir{"CACHE_DIR"}; */ static constexpr Property loaded_from_cache{"LOADED_FROM_CACHE"}; +/** + * @brief Enum to define possible cache mode + * @ingroup ov_runtime_cpp_prop_api + */ +enum class CacheMode { + OPTIMIZE_SIZE = 0, //!< smaller cache size + OPTIMIZE_SPEED = 1, //!< faster loading time +}; + +/** @cond INTERNAL */ +inline std::ostream& operator<<(std::ostream& os, const CacheMode& mode) { + switch (mode) { + case CacheMode::OPTIMIZE_SIZE: + return os << "optimize_size"; + case CacheMode::OPTIMIZE_SPEED: + return os << "optimize_speed"; + default: + OPENVINO_THROW("Unsupported cache mode"); + } +} + +inline std::istream& operator>>(std::istream& is, CacheMode& mode) { + std::string str; + is >> str; + if (str == "OPTIMIZE_SIZE" || str == "optimize_size") { + mode = CacheMode::OPTIMIZE_SIZE; + } else if (str == "OPTIMIZE_SPEED" || str == "optimize_speed") { + mode = CacheMode::OPTIMIZE_SPEED; + } else { + OPENVINO_THROW("Unsupported cache mode: ", str); + } + return is; +} +/** @endcond */ + +/** + * @brief Read-write property to select the cache mode between optimize_size and optimize_speed. + * If optimize_size is selected, smaller cache files will be created. + * And if optimize_speed is selected, loading time will decrease but the cache file size will increase. + * @ingroup ov_runtime_cpp_prop_api + */ +static constexpr Property cache_mode{"CACHE_MODE"}; + /** * @brief Read-only property to provide information about a range for streams on platforms where streams are supported. * @ingroup ov_runtime_cpp_prop_api diff --git a/src/plugins/intel_gpu/src/plugin/compiled_model.cpp b/src/plugins/intel_gpu/src/plugin/compiled_model.cpp index 696e9d4d113..0f907c2d2cd 100644 --- a/src/plugins/intel_gpu/src/plugin/compiled_model.cpp +++ b/src/plugins/intel_gpu/src/plugin/compiled_model.cpp @@ -187,6 +187,9 @@ std::shared_ptr CompiledModel::create_infer_request() co // [ ov::Node::Input/ ov::Node::Output ] // [ ov::intel_gpu::Graph ] void CompiledModel::export_model(std::ostream& model) const { + if (m_config.get_property(ov::cache_mode) == ov::CacheMode::OPTIMIZE_SIZE) + return; + OV_ITT_SCOPED_TASK(itt::domains::intel_gpu_plugin, "CompiledModel::export_model"); OPENVINO_ASSERT(!m_graphs.empty(), "[GPU] Model not loaded"); @@ -263,6 +266,7 @@ ov::Any CompiledModel::get_property(const std::string& name) const { ov::PropertyName{ov::intel_gpu::enable_loop_unrolling.name(), PropertyMutability::RO}, ov::PropertyName{ov::intel_gpu::disable_winograd_convolution.name(), PropertyMutability::RO}, ov::PropertyName{ov::cache_dir.name(), PropertyMutability::RO}, + ov::PropertyName{ov::cache_mode.name(), PropertyMutability::RO}, ov::PropertyName{ov::hint::performance_mode.name(), PropertyMutability::RO}, ov::PropertyName{ov::hint::execution_mode.name(), PropertyMutability::RO}, ov::PropertyName{ov::compilation_num_threads.name(), PropertyMutability::RO}, diff --git a/src/plugins/intel_gpu/src/plugin/plugin.cpp b/src/plugins/intel_gpu/src/plugin/plugin.cpp index 5ed0e899364..767cb72e16a 100644 --- a/src/plugins/intel_gpu/src/plugin/plugin.cpp +++ b/src/plugins/intel_gpu/src/plugin/plugin.cpp @@ -311,6 +311,9 @@ std::shared_ptr Plugin::import_model(std::istream& model, config.set_user_property(preprocess_config(orig_config)); config.apply_user_properties(context_impl->get_engine().get_device_info()); + if (config.get_property(ov::cache_mode) == ov::CacheMode::OPTIMIZE_SIZE) + return nullptr; + cldnn::BinaryInputBuffer ib(model, context_impl->get_engine()); return std::make_shared(ib, shared_from_this(), context_impl, config); } @@ -555,6 +558,7 @@ std::vector Plugin::get_supported_properties() const { ov::PropertyName{ov::intel_gpu::enable_loop_unrolling.name(), PropertyMutability::RW}, ov::PropertyName{ov::intel_gpu::disable_winograd_convolution.name(), PropertyMutability::RW}, ov::PropertyName{ov::cache_dir.name(), PropertyMutability::RW}, + ov::PropertyName{ov::cache_mode.name(), PropertyMutability::RW}, ov::PropertyName{ov::hint::performance_mode.name(), PropertyMutability::RW}, ov::PropertyName{ov::hint::execution_mode.name(), PropertyMutability::RW}, ov::PropertyName{ov::compilation_num_threads.name(), PropertyMutability::RW}, diff --git a/src/plugins/intel_gpu/src/runtime/execution_config.cpp b/src/plugins/intel_gpu/src/runtime/execution_config.cpp index 8bcb853a4c3..55716a0be04 100644 --- a/src/plugins/intel_gpu/src/runtime/execution_config.cpp +++ b/src/plugins/intel_gpu/src/runtime/execution_config.cpp @@ -56,6 +56,7 @@ void ExecutionConfig::set_default() { std::make_tuple(ov::intel_gpu::enable_loop_unrolling, true), std::make_tuple(ov::intel_gpu::disable_winograd_convolution, false), std::make_tuple(ov::internal::exclusive_async_requests, false), + std::make_tuple(ov::cache_mode, ov::CacheMode::OPTIMIZE_SPEED), // Legacy API properties std::make_tuple(ov::intel_gpu::nv12_two_inputs, false), diff --git a/src/plugins/intel_gpu/src/runtime/kernels_cache.cpp b/src/plugins/intel_gpu/src/runtime/kernels_cache.cpp index 148a5f1b7a8..5b24c46abea 100644 --- a/src/plugins/intel_gpu/src/runtime/kernels_cache.cpp +++ b/src/plugins/intel_gpu/src/runtime/kernels_cache.cpp @@ -76,7 +76,8 @@ std::string kernels_cache::get_cache_path() const { } bool kernels_cache::is_cache_enabled() const { - if (!_config.get_property(ov::intel_gpu::allow_new_shape_infer)) { + if (!_config.get_property(ov::intel_gpu::allow_new_shape_infer) && + (_config.get_property(ov::cache_mode) == ov::CacheMode::OPTIMIZE_SPEED)) { return false; }