From c59746fd2739f84e2c5dcb2e8961935ce8debb52 Mon Sep 17 00:00:00 2001 From: Mateusz Bencer Date: Thu, 13 Jul 2023 21:17:58 +0200 Subject: [PATCH] Implementation of MMAP for ONNX FE (#18026) * Implementation of MMAP for ONNX FE * fix win offsets * added virtual dtor to MappedMemory * review remarks. part.1 * added disable mmap flag to MO * added additional checks to mmap * remove unnecessary const * fix pybind default value * Added args.disable_mmap = False to MO tests * fixed MO test * avoid global headers * fix casting for win * disable mmap for legacy frontends flow * review remarks * Fixed passing parameters * added doc to MappedMemory and load_mmap_object --- .../frontends/frontends.cmake | 11 - .../src/pyopenvino/frontend/frontend.cpp | 11 +- .../include/openvino/util/mmap_object.hpp | 61 ++++++ .../util}/src/os/lin/lin_mmap_object.cpp | 46 +++-- .../util}/src/os/win/win_mmap_object.cpp | 81 ++++---- src/frontends/ir/src/frontend.cpp | 12 +- src/frontends/ir/src/mmap_object.hpp | 26 --- .../frontend/include/onnx_import/onnx.hpp | 16 +- .../onnx/frontend/src/core/attribute.hpp | 24 ++- .../onnx/frontend/src/core/graph.cpp | 10 +- .../onnx/frontend/src/core/graph.hpp | 6 + src/frontends/onnx/frontend/src/core/node.cpp | 4 +- .../onnx/frontend/src/core/sparse_tensor.hpp | 8 +- .../onnx/frontend/src/core/tensor.hpp | 33 ++- src/frontends/onnx/frontend/src/editor.cpp | 30 ++- src/frontends/onnx/frontend/src/editor.hpp | 20 +- src/frontends/onnx/frontend/src/frontend.cpp | 13 +- .../onnx/frontend/src/input_model.cpp | 27 ++- .../onnx/frontend/src/input_model.hpp | 16 +- src/frontends/onnx/frontend/src/onnx.cpp | 10 +- .../onnx/frontend/src/utils/onnx_internal.cpp | 4 + .../onnx/frontend/src/utils/onnx_internal.hpp | 6 +- .../src/utils/tensor_external_data.cpp | 38 +++- .../src/utils/tensor_external_data.hpp | 18 +- src/frontends/onnx/tests/CMakeLists.txt | 1 - .../tests/onnx_import_external_data.in.cpp | 188 ----------------- .../onnx/tests/onnx_reader_external_data.cpp | 190 +++++++++++++++++- tools/mo/openvino/tools/mo/convert.py | 5 + tools/mo/openvino/tools/mo/convert_impl.py | 4 +- .../mo/unit_tests/mo/utils/cli_parser_test.py | 2 +- tools/ovc/openvino/tools/ovc/convert.py | 5 + .../tools/ovc/moc_frontend/pipeline.py | 11 +- .../unit_tests/ovc/utils/cli_parser_test.py | 2 +- 33 files changed, 555 insertions(+), 384 deletions(-) create mode 100644 src/common/util/include/openvino/util/mmap_object.hpp rename src/{frontends/ir => common/util}/src/os/lin/lin_mmap_object.cpp (56%) rename src/{frontends/ir => common/util}/src/os/win/win_mmap_object.cpp (50%) delete mode 100644 src/frontends/ir/src/mmap_object.hpp delete mode 100644 src/frontends/onnx/tests/onnx_import_external_data.in.cpp diff --git a/cmake/developer_package/frontends/frontends.cmake b/cmake/developer_package/frontends/frontends.cmake index 31111dba5ae..f27dd1fe5f0 100644 --- a/cmake/developer_package/frontends/frontends.cmake +++ b/cmake/developer_package/frontends/frontends.cmake @@ -112,17 +112,6 @@ macro(ov_add_frontend) endif() file(GLOB_RECURSE LIBRARY_SRC ${frontend_root_dir}/src/*.cpp) - if (WIN32) - # Remove linux specific files - file(GLOB_RECURSE LIN_FILES ${frontend_root_dir}/src/os/lin/*.cpp - ${frontend_root_dir}/src/os/lin/*.hpp) - list(REMOVE_ITEM LIBRARY_SRC "${LIN_FILES}") - else() - # Remove windows specific files - file(GLOB_RECURSE WIN_FILES ${frontend_root_dir}/src/os/win/*.cpp - ${frontend_root_dir}/src/os/win/*.hpp) - list(REMOVE_ITEM LIBRARY_SRC "${WIN_FILES}") - endif() file(GLOB_RECURSE LIBRARY_HEADERS ${frontend_root_dir}/src/*.hpp) file(GLOB_RECURSE LIBRARY_PUBLIC_HEADERS ${frontend_root_dir}/include/*.hpp) diff --git a/src/bindings/python/src/pyopenvino/frontend/frontend.cpp b/src/bindings/python/src/pyopenvino/frontend/frontend.cpp index 10541775f7e..085ec1e2b83 100644 --- a/src/bindings/python/src/pyopenvino/frontend/frontend.cpp +++ b/src/bindings/python/src/pyopenvino/frontend/frontend.cpp @@ -37,29 +37,32 @@ void regclass_frontend_FrontEnd(py::module m) { fem.def( "load", - [](FrontEnd& self, const py::object& py_obj) { + [](FrontEnd& self, const py::object& py_obj, const bool enable_mmap = true) { if (py::isinstance(py_obj, py::module_::import("pathlib").attr("Path")) || py::isinstance(py_obj) || py::isinstance(py_obj)) { // check if model path is either a string/pathlib.Path/bytes std::string model_path = Common::utils::convert_path_to_string(py_obj); - return self.load(model_path); + return self.load(model_path, enable_mmap); } else if (py::isinstance(py_obj, pybind11::module::import("io").attr("BytesIO"))) { // support of BytesIO py::buffer_info info = py::buffer(py_obj.attr("getbuffer")()).request(); MemoryBuffer mb(reinterpret_cast(info.ptr), info.size); std::istream _istream(&mb); - return self.load(&_istream); + return self.load(&_istream, enable_mmap); } else { // Extended for one argument only for this time - return self.load({Common::utils::py_object_to_any(py_obj)}); + return self.load({Common::utils::py_object_to_any(py_obj), enable_mmap}); } }, py::arg("path"), + py::arg("enable_mmap") = true, R"( Loads an input model. :param path: Object describing the model. It can be path to model file. :type path: Any + :param enable_mmap: Use mmap feature to map memory of a model's weights instead of reading directly. Optional. The default value is true. + :type enable_mmap: boolean :return: Loaded input model. :rtype: openvino.frontend.InputModel )"); diff --git a/src/common/util/include/openvino/util/mmap_object.hpp b/src/common/util/include/openvino/util/mmap_object.hpp new file mode 100644 index 00000000000..270a5caa59b --- /dev/null +++ b/src/common/util/include/openvino/util/mmap_object.hpp @@ -0,0 +1,61 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief A header file for definition of abstraction over platform specific shared memory map objects + * @file mmap_object.hpp + */ + +#pragma once + +#include +#include + +namespace ov { + +/** + * @brief This class represents a mapped memory. + * Instead of reading files, we can map the memory via mmap for Linux or MapViewOfFile for Windows. + * The MappedMemory class is a abstraction to handle such memory with os-dependent details. + */ +class MappedMemory { +public: + virtual char* data() noexcept = 0; + virtual size_t size() const noexcept = 0; + virtual ~MappedMemory() = default; +}; + +/** + * @brief Returns mapped memory for a file from provided path. + * Instead of reading files, we can map the memory via mmap for Linux + * in order to avoid time-consuming reading and reduce memory consumption. + * + * @param path Path to a file which memory will be mmaped. + * @param data_size Number of bytes to map (zero, if all file will be mmaped). + * @param offset Number of bytes to skip from beginning of the file. + * @return MappedMemory shared ptr object which keep mmaped memory and control the lifetime. + */ +std::shared_ptr load_mmap_object(const std::string& path, + const size_t data_size = 0, + const size_t offset = 0); + +#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT + +/** + * @brief Returns mapped memory for a file from provided path. + * Instead of reading files, we can map the memory via MapViewOfFile for Windows + * in order to avoid time-consuming reading and reduce memory consumption. + * + * @param path Path to a file which memory will be mmaped. + * @param data_size Number of bytes to map (zero, if all file will be mmaped). + * @param offset Number of bytes to skip from beginning of the file. + * @return MappedMemory shared ptr object which keep mmaped memory and control the lifetime. + */ +std::shared_ptr load_mmap_object(const std::wstring& path, + const size_t data_size = 0, + const size_t offset = 0); + +#endif // OPENVINO_ENABLE_UNICODE_PATH_SUPPORT + +} // namespace ov diff --git a/src/frontends/ir/src/os/lin/lin_mmap_object.cpp b/src/common/util/src/os/lin/lin_mmap_object.cpp similarity index 56% rename from src/frontends/ir/src/os/lin/lin_mmap_object.cpp rename to src/common/util/src/os/lin/lin_mmap_object.cpp index 82dfc722148..f8a603918b6 100644 --- a/src/frontends/ir/src/os/lin/lin_mmap_object.cpp +++ b/src/common/util/src/os/lin/lin_mmap_object.cpp @@ -8,12 +8,12 @@ #include #include +#include #include #include -#include "mmap_object.hpp" -#include "ngraph/runtime/shared_buffer.hpp" #include "openvino/util/file_util.hpp" +#include "openvino/util/mmap_object.hpp" namespace ov { @@ -55,7 +55,7 @@ public: } }; -class MapHolder { +class MapHolder : public MappedMemory { void* m_data = MAP_FAILED; size_t m_size = 0; HandleHolder m_handle; @@ -63,21 +63,29 @@ class MapHolder { public: MapHolder() = default; - void set(const std::string& path) { + void set(const std::string& path, const size_t data_size = 0, const size_t offset = 0) { int prot = PROT_READ; int mode = O_RDONLY; struct stat sb = {}; m_handle = HandleHolder(open(path.c_str(), mode)); - OPENVINO_ASSERT(m_handle.get() != -1, - "Can not open file ", - path, - " for mapping. Ensure that file exists and has appropriate permissions"); - OPENVINO_ASSERT(fstat(m_handle.get(), &sb) != -1, "Can not get file size for ", path); - m_size = sb.st_size; + if (m_handle.get() == -1) { + throw std::runtime_error("Can not open file " + path + + " for mapping. Ensure that file exists and has appropriate permissions"); + } + if (fstat(m_handle.get(), &sb) == -1) { + throw std::runtime_error("Can not get file size for " + path); + } + if (data_size + offset > static_cast(sb.st_size)) { + throw std::runtime_error("Can not map out of scope memory for " + path); + } + m_size = data_size > 0 ? data_size : sb.st_size - offset; if (m_size > 0) { - m_data = mmap(nullptr, m_size, prot, MAP_PRIVATE, m_handle.get(), 0); - OPENVINO_ASSERT(m_data != MAP_FAILED, "Can not create file mapping for ", path, ", err=", strerror(errno)); + m_data = mmap(nullptr, m_size, prot, MAP_PRIVATE, m_handle.get(), offset); + if (m_data == MAP_FAILED) { + throw std::runtime_error("Can not create file mapping for " + path + ", err=" + std::strerror(errno)); + } } else { + m_size = 0; m_data = MAP_FAILED; } } @@ -88,21 +96,21 @@ public: } } - char* data() noexcept { + char* data() noexcept override { return static_cast(m_data); } - size_t size() const noexcept { + size_t size() const noexcept override { return m_size; } }; -std::shared_ptr load_mmap_object(const std::string& path) { +std::shared_ptr load_mmap_object(const std::string& path, + const size_t data_size, + const size_t offset) { auto holder = std::make_shared(); - holder->set(path); - return std::make_shared>>(holder->data(), - holder->size(), - holder); + holder->set(path, data_size, offset); + return holder; } } // namespace ov diff --git a/src/frontends/ir/src/os/win/win_mmap_object.cpp b/src/common/util/src/os/win/win_mmap_object.cpp similarity index 50% rename from src/frontends/ir/src/os/win/win_mmap_object.cpp rename to src/common/util/src/os/win/win_mmap_object.cpp index 5a99c1c63b3..1e5728d6c5d 100644 --- a/src/frontends/ir/src/os/win/win_mmap_object.cpp +++ b/src/common/util/src/os/win/win_mmap_object.cpp @@ -2,10 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "mmap_object.hpp" -#include "ngraph/runtime/shared_buffer.hpp" +#include "openvino/util/mmap_object.hpp" #include "openvino/util/file_util.hpp" +#include // clang-format-off #ifndef NOMINMAX # define NOMINMAX @@ -50,7 +50,7 @@ public: } }; -class MapHolder { +class MapHolder : public ov::MappedMemory { public: MapHolder() = default; @@ -60,57 +60,74 @@ public: } } - void set(const std::string& path) { + void set(const std::string& path, const size_t data_size=0, const size_t offset=0) { // Note that file can't be changed (renamed/deleted) until it's unmapped. FILE_SHARE_DELETE flag allow // rename/deletion, but it doesn't work with FAT32 filesystem (works on NTFS) auto h = ::CreateFileA(path.c_str(), GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); - map(path, h); + map(path, h, data_size, offset); } #ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT - void set(const std::wstring& path) { + void set(const std::wstring& path, const size_t data_size=0, const size_t offset=0) { // Note that file can't be changed (renamed/deleted) until it's unmapped. FILE_SHARE_DELETE flag allow // rename/deletion, but it doesn't work with FAT32 filesystem (works on NTFS) auto h = ::CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); - map(ov::util::wstring_to_string(path), h); + map(ov::util::wstring_to_string(path), h, data_size, offset); } #endif - char* data() noexcept { + char* data() noexcept override { return static_cast(m_data); } - size_t size() const noexcept { + size_t size() const noexcept override { return m_size; } private: - void map(const std::string& path, HANDLE h) { - OPENVINO_ASSERT(h != INVALID_HANDLE_VALUE, - "Can not open file ", - path, - " for mapping. Ensure that file exists and has appropriate permissions"); + void map(const std::string& path, HANDLE h, const size_t data_size=0, const size_t offset=0) { + if(h == INVALID_HANDLE_VALUE) { + throw std::runtime_error("Can not open file " + path + " for mapping. Ensure that file exists and has appropriate permissions"); + } m_handle = HandleHolder(h); - SYSTEM_INFO SystemInfo; - GetSystemInfo(&SystemInfo); + SYSTEM_INFO sys_info; + GetSystemInfo(&sys_info); DWORD map_mode = FILE_MAP_READ; DWORD access = PAGE_READONLY; LARGE_INTEGER file_size_large; - OPENVINO_ASSERT(::GetFileSizeEx(m_handle.get(), &file_size_large) != 0, "Can not get file size for ", path); + if(::GetFileSizeEx(m_handle.get(), &file_size_large) == 0) { + throw std::runtime_error("Can not get file size for " + path); + } + if (data_size + offset > static_cast(file_size_large.QuadPart)) { + throw std::runtime_error("Can not map out of scope memory for " + path); + } + m_size = data_size > 0 ? data_size : static_cast(file_size_large.QuadPart) - offset; + + // the offset must be a multiple of the allocation granularity + const size_t alloc_gran = sys_info.dwAllocationGranularity; + const size_t map_view_start = (offset / alloc_gran) * alloc_gran; + const size_t map_view_size = (offset%alloc_gran) + m_size; + const size_t file_map_size = offset + m_size; + const size_t view_delta = offset - map_view_start; - m_size = static_cast(file_size_large.QuadPart); if (m_size > 0) { m_mapping = - HandleHolder(::CreateFileMapping(m_handle.get(), 0, access, m_size >> 32, m_size & 0xffffffff, 0)); - OPENVINO_ASSERT(m_mapping.get() != INVALID_HANDLE_VALUE, "Can not create file mapping for ", path); + HandleHolder(::CreateFileMapping(m_handle.get(), 0, access, file_map_size >> 32, file_map_size & 0xffffffff, 0)); + if(m_mapping.get() == INVALID_HANDLE_VALUE) { + throw std::runtime_error("Can not create file mapping for " + path); + } m_data = ::MapViewOfFile(m_mapping.get(), map_mode, - 0, // offset_align >> 32, - 0, // offset_align & 0xffffffff, - m_size); - OPENVINO_ASSERT(m_data, "Can not create map view for ", path); + map_view_start >> 32, // offset_align >> 32, + map_view_start & 0xffffffff, // offset_align & 0xffffffff, + map_view_size); + if(!m_data) { + throw std::runtime_error("Can not create map view for " + path); + } + // move pointer to the expected data (after alignment with allocation granularity) + m_data = reinterpret_cast(m_data) + view_delta; } else { m_data = nullptr; } @@ -123,22 +140,18 @@ private: HandleHolder m_mapping; }; -std::shared_ptr load_mmap_object(const std::string& path) { +std::shared_ptr load_mmap_object(const std::string& path, const size_t data_size, const size_t offset) { auto holder = std::make_shared(); - holder->set(path); - return std::make_shared>>(holder->data(), - holder->size(), - holder); + holder->set(path, data_size, offset); + return holder; } #ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT -std::shared_ptr load_mmap_object(const std::wstring& path) { +std::shared_ptr load_mmap_object(const std::wstring& path, const size_t data_size, const size_t offset) { auto holder = std::make_shared(); - holder->set(path); - return std::make_shared>>(holder->data(), - holder->size(), - holder); + holder->set(path, data_size, offset); + return holder; } #endif diff --git a/src/frontends/ir/src/frontend.cpp b/src/frontends/ir/src/frontend.cpp index bd9ad9fccdd..b7a565ab6cd 100644 --- a/src/frontends/ir/src/frontend.cpp +++ b/src/frontends/ir/src/frontend.cpp @@ -8,12 +8,12 @@ #include #include "input_model.hpp" -#include "mmap_object.hpp" #include "ngraph/runtime/aligned_buffer.hpp" #include "ngraph/runtime/shared_buffer.hpp" #include "openvino/core/any.hpp" #include "openvino/core/so_extension.hpp" #include "openvino/util/file_util.hpp" +#include "openvino/util/mmap_object.hpp" #include "xml_parse_utils.h" using namespace ov; @@ -201,9 +201,13 @@ InputModel::Ptr FrontEnd::load_impl(const std::vector& variants) const } } if (!weights_path.empty()) { - if (enable_mmap) - weights = ov::load_mmap_object(weights_path); - else { + if (enable_mmap) { + auto mapped_memory = ov::load_mmap_object(weights_path); + weights = + std::make_shared>>(mapped_memory->data(), + mapped_memory->size(), + mapped_memory); + } else { std::ifstream bin_stream; bin_stream.open(weights_path.c_str(), std::ios::binary); if (!bin_stream.is_open()) diff --git a/src/frontends/ir/src/mmap_object.hpp b/src/frontends/ir/src/mmap_object.hpp deleted file mode 100644 index 6b96fce7ee7..00000000000 --- a/src/frontends/ir/src/mmap_object.hpp +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -/** - * @brief A header file for definition of abstraction over platform specific shared memory map objects - * @file mmap_object.hpp - */ - -#pragma once - -#include - -#include "ngraph/runtime/aligned_buffer.hpp" - -namespace ov { - -std::shared_ptr load_mmap_object(const std::string& path); - -#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT - -std::shared_ptr load_mmap_object(const std::wstring& path); - -#endif // OPENVINO_ENABLE_UNICODE_PATH_SUPPORT - -} // namespace ov diff --git a/src/frontends/onnx/frontend/include/onnx_import/onnx.hpp b/src/frontends/onnx/frontend/include/onnx_import/onnx.hpp index 0899c48685f..9e1746b4122 100644 --- a/src/frontends/onnx/frontend/include/onnx_import/onnx.hpp +++ b/src/frontends/onnx/frontend/include/onnx_import/onnx.hpp @@ -45,14 +45,17 @@ bool is_operator_supported(const std::string& op_name, std::int64_t version, con /// \note If stream parsing fails or the ONNX model contains unsupported ops, /// the function throws an ngraph_error exception. /// -/// \param[in] stream The input stream (e.g. file stream, memory stream, etc). -/// \param[in] model_path The path to the imported onnx model. -/// It is required if the imported model uses data saved in external -/// files. +/// \param[in] stream The input stream (e.g. file stream, memory stream, etc). +/// \param[in] model_path The path to the imported onnx model. +/// It is required if the imported model uses data saved in external +/// files. +/// \param[in] enable_mmap Enable mapping files with external weights instead of reading. /// /// \return An nGraph function that represents a single output from the created graph. ONNX_IMPORTER_API -std::shared_ptr import_onnx_model(std::istream& stream, const std::string& model_path = ""); +std::shared_ptr import_onnx_model(std::istream& stream, + const std::string& model_path = "", + bool enable_mmap = false); /// \brief Imports and converts an ONNX model from the input file /// to an nGraph Function representation. @@ -62,10 +65,11 @@ std::shared_ptr import_onnx_model(std::istream& stream, const std::str /// /// \param[in] file_path The path to a file containing the ONNX model /// (relative or absolute). +/// \param[in] enable_mmap Enable mapping files with external weights instead of reading. /// /// \return An nGraph function that represents a single output from the created graph. ONNX_IMPORTER_API -std::shared_ptr import_onnx_model(const std::string& file_path); +std::shared_ptr import_onnx_model(const std::string& file_path, bool enable_mmap = false); } // namespace onnx_import } // namespace ngraph diff --git a/src/frontends/onnx/frontend/src/core/attribute.hpp b/src/frontends/onnx/frontend/src/core/attribute.hpp index 80e7cf10df4..0cf23654a34 100644 --- a/src/frontends/onnx/frontend/src/core/attribute.hpp +++ b/src/frontends/onnx/frontend/src/core/attribute.hpp @@ -200,9 +200,12 @@ public: }; Attribute() = delete; - explicit Attribute(const ONNX_NAMESPACE::AttributeProto& attribute_proto, const std::string& model_dir) + explicit Attribute(const ONNX_NAMESPACE::AttributeProto& attribute_proto, + const std::string& model_dir, + const bool enable_mmap) : m_attribute_proto{&attribute_proto}, - m_model_dir{model_dir} {} + m_model_dir{model_dir}, + m_enable_mmap{enable_mmap} {} Attribute(Attribute&&) noexcept = default; Attribute(const Attribute&) = default; @@ -253,10 +256,10 @@ public: return get_type() == Type::graph_array; } Tensor get_tensor() const { - return Tensor{m_attribute_proto->t(), m_model_dir}; + return Tensor{m_attribute_proto->t(), m_model_dir, m_enable_mmap}; } SparseTensor get_sparse_tensor() const { - return SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir}; + return SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir, m_enable_mmap}; } float get_float() const { return m_attribute_proto->f(); @@ -274,7 +277,7 @@ public: const auto& tensors = m_attribute_proto->tensors(); ret.reserve(tensors.size()); for (const auto& tensor : tensors) - ret.emplace_back(tensor, m_model_dir); + ret.emplace_back(tensor, m_model_dir, m_enable_mmap); return ret; } @@ -283,7 +286,7 @@ public: const auto& sparse_tensors = m_attribute_proto->sparse_tensors(); ret.reserve(sparse_tensors.size()); for (const auto& tensor : sparse_tensors) - ret.emplace_back(tensor, m_model_dir); + ret.emplace_back(tensor, m_model_dir, m_enable_mmap); return ret; } @@ -315,7 +318,7 @@ public: template ::value, bool>::type = true> T get_value() const { if (is_tensor()) { - return Tensor{m_attribute_proto->t(), m_model_dir}; + return Tensor{m_attribute_proto->t(), m_model_dir, m_enable_mmap}; } throw error::attribute::InvalidData{m_attribute_proto->type()}; } @@ -323,7 +326,7 @@ public: template >::value, bool>::type = true> T get_value() const { if (is_tensor()) { - return {Tensor{m_attribute_proto->t(), m_model_dir}}; + return {Tensor{m_attribute_proto->t(), m_model_dir, m_enable_mmap}}; } else if (is_tensor_array()) { return get_tensor_array(); } @@ -333,7 +336,7 @@ public: template ::value, bool>::type = true> T get_value() const { if (is_sparse_tensor()) { - return SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir}; + return SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir, m_enable_mmap}; } throw error::attribute::InvalidData{m_attribute_proto->type()}; } @@ -341,7 +344,7 @@ public: template >::value, bool>::type = true> T get_value() const { if (is_sparse_tensor()) { - return {SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir}}; + return {SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir, m_enable_mmap}}; } else if (is_sparse_tensor_array()) { return get_sparse_tensor_array(); } @@ -353,6 +356,7 @@ public: private: const ONNX_NAMESPACE::AttributeProto* m_attribute_proto; std::string m_model_dir; + const bool m_enable_mmap; }; } // namespace onnx_import diff --git a/src/frontends/onnx/frontend/src/core/graph.cpp b/src/frontends/onnx/frontend/src/core/graph.cpp index ac8cd3907c4..5d0eb68dfed 100644 --- a/src/frontends/onnx/frontend/src/core/graph.cpp +++ b/src/frontends/onnx/frontend/src/core/graph.cpp @@ -145,16 +145,19 @@ ov::frontend::ExtensionHolder subgraph_required_extensions( Graph::Graph(const std::string& model_dir, const std::shared_ptr& model_proto, + const bool enable_mmap, ov::frontend::ExtensionHolder extensions) - : Graph(model_dir, model_proto, common::make_unique(), std::move(extensions)) {} + : Graph(model_dir, model_proto, common::make_unique(), enable_mmap, std::move(extensions)) {} Graph::Graph(const std::string& model_dir, const std::shared_ptr& model_proto, std::unique_ptr&& cache, + const bool enable_mmap, ov::frontend::ExtensionHolder extensions) : m_cache{std::move(cache)}, m_extensions{std::move(extensions)}, - m_model_dir{model_dir} { + m_model_dir{model_dir}, + m_enable_mmap{enable_mmap} { const auto ops_bridge = detail::init_ops_bridge(m_extensions.conversions); m_model = common::make_unique(model_proto, detail::build_model_opset(*model_proto, ops_bridge)); @@ -165,7 +168,7 @@ Graph::Graph(const std::string& model_dir, // Process all initializers in the graph for (const auto& initializer_tensor : m_model->get_graph().initializer()) { if (initializer_tensor.has_name()) { - Tensor tensor = Tensor{initializer_tensor, m_model_dir}; + Tensor tensor = Tensor{initializer_tensor, m_model_dir, enable_mmap}; std::shared_ptr ng_constant; // For each initializer create a Constant node and store it in cache try { @@ -477,6 +480,7 @@ Subgraph::Subgraph(const std::shared_ptr& model_prot : Graph(parent_graph->model_dir(), model_proto, common::make_unique(), + parent_graph->mmap_enabled(), detail::subgraph_required_extensions(parent_graph->get_extensions())), m_parent_graph(parent_graph) {} diff --git a/src/frontends/onnx/frontend/src/core/graph.hpp b/src/frontends/onnx/frontend/src/core/graph.hpp index ef8b8fb2bd1..ba251b6f5dd 100644 --- a/src/frontends/onnx/frontend/src/core/graph.hpp +++ b/src/frontends/onnx/frontend/src/core/graph.hpp @@ -23,6 +23,7 @@ class Graph : public std::enable_shared_from_this { public: Graph(const std::string& model_dir, const std::shared_ptr& model_proto, + const bool enable_mmap, ov::frontend::ExtensionHolder extensions = {}); Graph() = delete; @@ -40,6 +41,9 @@ public: const std::string& model_dir() const { return m_model_dir; } + bool mmap_enabled() const { + return m_enable_mmap; + } const ParameterVector& get_ng_parameters() const { return m_parameters; } @@ -57,6 +61,7 @@ protected: Graph(const std::string& model_dir, const std::shared_ptr& model, std::unique_ptr&& cache, + const bool enable_mmap, ov::frontend::ExtensionHolder extensions = {}); void set_friendly_names(const Node& onnx_node, const OutputVector& ng_subgraph_outputs) const; @@ -77,6 +82,7 @@ protected: private: std::vector m_nodes; std::string m_model_dir; + bool m_enable_mmap; }; /// \brief Representation of ONNX subgraph. It is used for example by ONNX Loop op. diff --git a/src/frontends/onnx/frontend/src/core/node.cpp b/src/frontends/onnx/frontend/src/core/node.cpp index 29857e6efd9..1679a5bb043 100644 --- a/src/frontends/onnx/frontend/src/core/node.cpp +++ b/src/frontends/onnx/frontend/src/core/node.cpp @@ -26,7 +26,7 @@ public: const auto& attributes = node_proto.attribute(); m_attributes.reserve(attributes.size()); for (const auto& attr_proto : attributes) { - m_attributes.emplace_back(attr_proto, m_graph->model_dir()); + m_attributes.emplace_back(attr_proto, m_graph->model_dir(), m_graph->mmap_enabled()); const auto& attribute = m_attributes.back(); if (attribute.is_graph()) m_subgraphs.insert({attribute.get_name(), std::make_shared(attribute.get_subgraph(m_graph))}); @@ -43,7 +43,7 @@ public: m_output_names{std::begin(node_proto.output()), std::end(node_proto.output())}, m_subgraphs(subgraphs) { for (const auto& attr_proto : node_proto.attribute()) { - m_attributes.emplace_back(attr_proto, m_graph->model_dir()); + m_attributes.emplace_back(attr_proto, m_graph->model_dir(), m_graph->mmap_enabled()); } } diff --git a/src/frontends/onnx/frontend/src/core/sparse_tensor.hpp b/src/frontends/onnx/frontend/src/core/sparse_tensor.hpp index c2874a435d0..9182feab412 100644 --- a/src/frontends/onnx/frontend/src/core/sparse_tensor.hpp +++ b/src/frontends/onnx/frontend/src/core/sparse_tensor.hpp @@ -17,9 +17,11 @@ namespace onnx_import { class SparseTensor { public: SparseTensor() = delete; - explicit SparseTensor(const ONNX_NAMESPACE::SparseTensorProto& sparse_tensor, const std::string& model_dir) - : m_values{sparse_tensor.values(), model_dir}, - m_indices{sparse_tensor.indices(), model_dir}, + SparseTensor(const ONNX_NAMESPACE::SparseTensorProto& sparse_tensor, + const std::string& model_dir, + const bool enable_mmap) + : m_values{sparse_tensor.values(), model_dir, enable_mmap}, + m_indices{sparse_tensor.indices(), model_dir, enable_mmap}, m_shape{std::begin(sparse_tensor.dims()), std::end(sparse_tensor.dims())} { if (m_shape == Shape{0}) { // It's possible to construct a sparse tensor in ONNX with "dims: 0" property diff --git a/src/frontends/onnx/frontend/src/core/tensor.hpp b/src/frontends/onnx/frontend/src/core/tensor.hpp index 2f824949a32..406f92b2bae 100644 --- a/src/frontends/onnx/frontend/src/core/tensor.hpp +++ b/src/frontends/onnx/frontend/src/core/tensor.hpp @@ -108,10 +108,11 @@ public: }; Tensor() = delete; - explicit Tensor(const ONNX_NAMESPACE::TensorProto& tensor, const std::string& model_dir) + explicit Tensor(const ONNX_NAMESPACE::TensorProto& tensor, const std::string& model_dir, const bool enable_mmap) : m_tensor_proto{&tensor}, m_shape{std::begin(tensor.dims()), std::end(tensor.dims())}, - m_model_dir{model_dir} { + m_model_dir{model_dir}, + m_enable_mmap{enable_mmap} { if (m_shape == Shape{0}) { // It's possible to construct a tensor in ONNX with "dims: 0" property // Such tensor contains a scalar. This results in a Shape{0} stored in m_shape. @@ -239,9 +240,16 @@ private: std::shared_ptr constant{nullptr}; size_t data_size = get_data_size(); if (has_external_data()) { - auto external_data = load_external_data(); - constant = std::make_shared(type, m_shape, external_data.data()); - if (constant->get_byte_size() != external_data.size()) { + const auto ext_data = detail::TensorExternalData(*m_tensor_proto); + if (m_enable_mmap) { + constant = std::make_shared(type, + m_shape, + ext_data.load_external_mmap_data(m_model_dir)); + } else { + constant = + std::make_shared(type, m_shape, ext_data.load_external_data(m_model_dir)); + } + if (constant->get_byte_size() != ov::shape_size(m_shape) * type.size()) { throw error::invalid_external_data( "The size of the external data file does not match the byte size of an initializer '" + get_name() + "' in the model"); @@ -288,14 +296,16 @@ private: ONNX_NAMESPACE::TensorProto_DataLocation::TensorProto_DataLocation_EXTERNAL; } - std::string load_external_data() const { - const auto tensor_external_data = detail::TensorExternalData(*m_tensor_proto); - return tensor_external_data.load_external_data(m_model_dir); - } - template std::vector get_external_data() const { - return detail::__get_raw_data(load_external_data(), m_tensor_proto->data_type()); + const auto ext_data = detail::TensorExternalData(*m_tensor_proto); + std::shared_ptr buffer = nullptr; + if (m_enable_mmap) { + buffer = ext_data.load_external_mmap_data(m_model_dir); + } else { + buffer = ext_data.load_external_data(m_model_dir); + } + return std::vector(buffer->get_ptr(), buffer->get_ptr() + buffer->size()); } const void* get_data_ptr() const { @@ -339,6 +349,7 @@ private: const ONNX_NAMESPACE::TensorProto* m_tensor_proto; Shape m_shape; std::string m_model_dir; + bool m_enable_mmap = true; }; inline std::ostream& operator<<(std::ostream& outs, const Tensor& tensor) { diff --git a/src/frontends/onnx/frontend/src/editor.cpp b/src/frontends/onnx/frontend/src/editor.cpp index d9054b5a319..678e6dc3247 100644 --- a/src/frontends/onnx/frontend/src/editor.cpp +++ b/src/frontends/onnx/frontend/src/editor.cpp @@ -315,17 +315,23 @@ struct onnx_editor::ONNXModelEditor::Impl { #endif }; -onnx_editor::ONNXModelEditor::ONNXModelEditor(const std::string& model_path, frontend::ExtensionHolder extensions) - : m_extensions{std::move(extensions)}, - m_model_path{model_path}, +onnx_editor::ONNXModelEditor::ONNXModelEditor(const std::string& model_path, + const bool enable_mmap, + frontend::ExtensionHolder extensions) + : m_model_path{model_path}, + m_enable_mmap{enable_mmap}, + m_extensions{std::move(extensions)}, m_pimpl{new ONNXModelEditor::Impl{model_path}, [](Impl* impl) { delete impl; }} {} #if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) -onnx_editor::ONNXModelEditor::ONNXModelEditor(const std::wstring& model_path, frontend::ExtensionHolder extensions) +onnx_editor::ONNXModelEditor::ONNXModelEditor(const std::wstring& model_path, + const bool enable_mmap, + frontend::ExtensionHolder extensions) : m_extensions{std::move(extensions)}, m_model_path{ov::util::wstring_to_string(model_path)}, + m_enable_mmap{enable_mmap}, m_pimpl{new ONNXModelEditor::Impl{model_path}, [](Impl* impl) { delete impl; }} {} @@ -333,9 +339,11 @@ onnx_editor::ONNXModelEditor::ONNXModelEditor(const std::wstring& model_path, fr onnx_editor::ONNXModelEditor::ONNXModelEditor(std::istream& model_stream, const std::string& model_path, + const bool enable_mmap, frontend::ExtensionHolder extensions) - : m_extensions{std::move(extensions)}, - m_model_path{model_path}, + : m_model_path{model_path}, + m_enable_mmap{enable_mmap}, + m_extensions{std::move(extensions)}, m_pimpl{new ONNXModelEditor::Impl{model_stream}, [](Impl* impl) { delete impl; }} {} @@ -522,7 +530,10 @@ std::string onnx_editor::ONNXModelEditor::model_string() const { } std::shared_ptr onnx_editor::ONNXModelEditor::get_function() const { - return ngraph::onnx_import::detail::import_onnx_model(m_pimpl->m_model_proto, m_model_path, m_extensions); + return ngraph::onnx_import::detail::import_onnx_model(m_pimpl->m_model_proto, + m_model_path, + m_enable_mmap, + m_extensions); } void onnx_editor::ONNXModelEditor::set_input_values( @@ -713,7 +724,10 @@ std::vector onnx_editor::ONNXModelEditor::get_output_ports(const Ed } std::shared_ptr onnx_editor::ONNXModelEditor::decode() { - return ngraph::onnx_import::detail::decode_to_framework_nodes(m_pimpl->m_model_proto, m_model_path, m_extensions); + return ngraph::onnx_import::detail::decode_to_framework_nodes(m_pimpl->m_model_proto, + m_model_path, + m_enable_mmap, + m_extensions); } void onnx_editor::ONNXModelEditor::add_output(const OutputEdge& output_edge) const { diff --git a/src/frontends/onnx/frontend/src/editor.hpp b/src/frontends/onnx/frontend/src/editor.hpp index 17683d5df15..9d4a7e461e8 100644 --- a/src/frontends/onnx/frontend/src/editor.hpp +++ b/src/frontends/onnx/frontend/src/editor.hpp @@ -31,9 +31,13 @@ public: /// is parsed and loaded into the m_model_proto member variable. /// /// \param model_path Path to the file containing the model. - ONNXModelEditor(const std::string& model_path, frontend::ExtensionHolder extensions = {}); + /// \param enable_mmap Enable mapping files with external weights instead of reading. + /// \param extensions Holder for custom extensions (like custom ops). + explicit ONNXModelEditor(const std::string& model_path, + const bool enable_mmap = false, + frontend::ExtensionHolder extensions = {}); #if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) - ONNXModelEditor(const std::wstring& model_path, frontend::ExtensionHolder extensions = {}); + ONNXModelEditor(const std::wstring& model_path, const bool enable_mmap, frontend::ExtensionHolder extensions = {}); #endif /// \brief Creates an editor from a model stream. The stream is parsed and loaded @@ -42,9 +46,12 @@ public: /// \param model_stream The stream containing the model. /// \param model_path Path to the file containing the model. This information can be used /// for ONNX external weights feature support. - ONNXModelEditor(std::istream& model_stream, - const std::string& path = {}, - frontend::ExtensionHolder extensions = {}); + /// \param enable_mmap Enable mapping files with external weights instead of reading. + /// \param extensions Holder for custom extensions (like custom ops). + explicit ONNXModelEditor(std::istream& model_stream, + const std::string& path = {}, + const bool enable_mmap = false, + frontend::ExtensionHolder extensions = {}); /// \brief Modifies the in-memory representation of the model by setting /// custom input types for all inputs specified in the provided map. @@ -296,8 +303,9 @@ public: private: void update_mapper_if_needed() const; - frontend::ExtensionHolder m_extensions; const std::string m_model_path; + const bool m_enable_mmap; + frontend::ExtensionHolder m_extensions; struct Impl; std::unique_ptr m_pimpl; diff --git a/src/frontends/onnx/frontend/src/frontend.cpp b/src/frontends/onnx/frontend/src/frontend.cpp index 4381f02f22c..dbc7f1c6036 100644 --- a/src/frontends/onnx/frontend/src/frontend.cpp +++ b/src/frontends/onnx/frontend/src/frontend.cpp @@ -56,29 +56,32 @@ InputModel::Ptr FrontEnd::load_impl(const std::vector& variants) const if (variants.empty()) { return nullptr; } + // enable mmap by default + const bool enable_mmap = variants[variants.size() - 1].is() ? variants[variants.size() - 1].as() : true; + if (variants[0].is()) { const auto path = variants[0].as(); - return std::make_shared(path, m_extensions); + return std::make_shared(path, enable_mmap, m_extensions); } #if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) if (variants[0].is()) { const auto path = variants[0].as(); - return std::make_shared(path, m_extensions); + return std::make_shared(path, enable_mmap, m_extensions); } #endif if (variants[0].is()) { const auto stream = variants[0].as(); if (variants.size() > 1 && variants[1].is()) { const auto path = variants[1].as(); - return std::make_shared(*stream, path, m_extensions); + return std::make_shared(*stream, path, enable_mmap, m_extensions); } #if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) if (variants.size() > 1 && variants[1].is()) { const auto path = variants[1].as(); - return std::make_shared(*stream, path, m_extensions); + return std::make_shared(*stream, path, enable_mmap, m_extensions); } #endif - return std::make_shared(*stream, m_extensions); + return std::make_shared(*stream, enable_mmap, m_extensions); } return nullptr; } diff --git a/src/frontends/onnx/frontend/src/input_model.cpp b/src/frontends/onnx/frontend/src/input_model.cpp index 41706c0dc20..5f274d590c5 100644 --- a/src/frontends/onnx/frontend/src/input_model.cpp +++ b/src/frontends/onnx/frontend/src/input_model.cpp @@ -16,23 +16,30 @@ using namespace ov::frontend::onnx; NGRAPH_SUPPRESS_DEPRECATED_START -InputModel::InputModel(const std::string& path, frontend::ExtensionHolder extensions) - : m_editor{std::make_shared(path, std::move(extensions))} {} +InputModel::InputModel(const std::string& path, const bool enable_mmap, frontend::ExtensionHolder extensions) + : m_editor{std::make_shared(path, enable_mmap, std::move(extensions))} {} #if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) -InputModel::InputModel(const std::wstring& path, frontend::ExtensionHolder extensions) - : m_editor{std::make_shared(path, std::move(extensions))} {} +InputModel::InputModel(const std::wstring& path, const bool enable_mmap, frontend::ExtensionHolder extensions) + : m_editor{std::make_shared(path, enable_mmap, std::move(extensions))} {} #endif -InputModel::InputModel(std::istream& model_stream, frontend::ExtensionHolder extensions) - : m_editor{std::make_shared(model_stream, "", std::move(extensions))} {} +InputModel::InputModel(std::istream& model_stream, const bool enable_mmap, frontend::ExtensionHolder extensions) + : m_editor{std::make_shared(model_stream, "", enable_mmap, std::move(extensions))} {} -InputModel::InputModel(std::istream& model_stream, const std::string& path, frontend::ExtensionHolder extensions) - : m_editor{std::make_shared(model_stream, path, std::move(extensions))} {} +InputModel::InputModel(std::istream& model_stream, + const std::string& path, + const bool enable_mmap, + frontend::ExtensionHolder extensions) + : m_editor{std::make_shared(model_stream, path, enable_mmap, std::move(extensions))} { +} #ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT -InputModel::InputModel(std::istream& model_stream, const std::wstring& path, frontend::ExtensionHolder extensions) - : InputModel(model_stream, ov::util::wstring_to_string(path), std::move(extensions)) {} +InputModel::InputModel(std::istream& model_stream, + const std::wstring& path, + const bool enable_mmap, + frontend::ExtensionHolder extensions) + : InputModel(model_stream, ov::util::wstring_to_string(path), enable_mmap, std::move(extensions)) {} #endif std::vector InputModel::get_inputs() const { diff --git a/src/frontends/onnx/frontend/src/input_model.hpp b/src/frontends/onnx/frontend/src/input_model.hpp index 1fd0b61ffd2..be093bac9da 100644 --- a/src/frontends/onnx/frontend/src/input_model.hpp +++ b/src/frontends/onnx/frontend/src/input_model.hpp @@ -16,16 +16,22 @@ namespace onnx { class InputModel : public ov::frontend::InputModel { public: - InputModel(const std::string& path, ExtensionHolder extensions = {}); + InputModel(const std::string& path, const bool enable_mmap = false, ExtensionHolder extensions = {}); #if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) - InputModel(const std::wstring& path, ExtensionHolder extensions = {}); + InputModel(const std::wstring& path, const bool enable_mmap = false, ExtensionHolder extensions = {}); #endif - InputModel(std::istream& model_stream, ExtensionHolder extensions = {}); + InputModel(std::istream& model_stream, const bool enable_mmap = false, ExtensionHolder extensions = {}); // The path can be required even if the model is passed as a stream because it is necessary // for ONNX external data feature - InputModel(std::istream& model_stream, const std::string& path, ExtensionHolder extensions = {}); + InputModel(std::istream& model_stream, + const std::string& path, + const bool enable_mmap = false, + ExtensionHolder extensions = {}); #ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT - InputModel(std::istream& model_stream, const std::wstring& path, ExtensionHolder extensions = {}); + InputModel(std::istream& model_stream, + const std::wstring& path, + const bool enable_mmap = false, + ExtensionHolder extensions = {}); #endif std::vector get_inputs() const override; diff --git a/src/frontends/onnx/frontend/src/onnx.cpp b/src/frontends/onnx/frontend/src/onnx.cpp index 227d28f803b..68ed2b4424b 100644 --- a/src/frontends/onnx/frontend/src/onnx.cpp +++ b/src/frontends/onnx/frontend/src/onnx.cpp @@ -22,14 +22,16 @@ const auto legacy_conversion_extension = std::make_shared import_onnx_model(std::istream& stream, const std::string& model_path) { +std::shared_ptr import_onnx_model(std::istream& stream, + const std::string& model_path, + const bool enable_mmap) { const auto model_proto = std::make_shared(onnx_common::parse_from_istream(stream)); ov::frontend::ExtensionHolder extensions; extensions.conversions.push_back(legacy_conversion_extension); - return detail::import_onnx_model(model_proto, model_path, std::move(extensions)); + return detail::import_onnx_model(model_proto, model_path, enable_mmap, std::move(extensions)); } -std::shared_ptr import_onnx_model(const std::string& file_path) { +std::shared_ptr import_onnx_model(const std::string& file_path, const bool enable_mmap) { std::ifstream model_stream{file_path, std::ios::in | std::ios::binary}; if (!model_stream.is_open()) { @@ -37,7 +39,7 @@ std::shared_ptr import_onnx_model(const std::string& file_path) { ". Could not open the file."); }; - return import_onnx_model(model_stream, file_path); + return import_onnx_model(model_stream, file_path, enable_mmap); } std::set get_supported_operators(std::int64_t version, const std::string& domain) { diff --git a/src/frontends/onnx/frontend/src/utils/onnx_internal.cpp b/src/frontends/onnx/frontend/src/utils/onnx_internal.cpp index b3eb6f9a6b1..e812d02c236 100644 --- a/src/frontends/onnx/frontend/src/utils/onnx_internal.cpp +++ b/src/frontends/onnx/frontend/src/utils/onnx_internal.cpp @@ -89,11 +89,13 @@ void convert_decoded_function(std::shared_ptr function) { std::shared_ptr import_onnx_model(std::shared_ptr model_proto, const std::string& model_path, + const bool enable_mmap, ov::frontend::ExtensionHolder extensions) { apply_transformations(*model_proto); NGRAPH_SUPPRESS_DEPRECATED_START Graph graph{file_util::get_directory(ov::util::get_absolute_file_path(model_path)), model_proto, + enable_mmap, std::move(extensions)}; NGRAPH_SUPPRESS_DEPRECATED_END return graph.convert(); @@ -101,11 +103,13 @@ std::shared_ptr import_onnx_model(std::shared_ptr decode_to_framework_nodes(std::shared_ptr model_proto, const std::string& model_path, + const bool enable_mmap, ov::frontend::ExtensionHolder extensions) { apply_transformations(*model_proto); NGRAPH_SUPPRESS_DEPRECATED_START auto graph = std::make_shared(file_util::get_directory(ov::util::get_absolute_file_path(model_path)), model_proto, + enable_mmap, extensions); NGRAPH_SUPPRESS_DEPRECATED_END return graph->decode(); diff --git a/src/frontends/onnx/frontend/src/utils/onnx_internal.hpp b/src/frontends/onnx/frontend/src/utils/onnx_internal.hpp index d415efe8aeb..18cb9a8ce34 100644 --- a/src/frontends/onnx/frontend/src/utils/onnx_internal.hpp +++ b/src/frontends/onnx/frontend/src/utils/onnx_internal.hpp @@ -29,12 +29,13 @@ namespace detail { /// \param model_proto Reference to a GraphProto object. /// \param model_path The path to the imported onnx model. /// It is required if the imported model uses data saved in external files. +/// \param enable_mmap Enable mapping files with external weights instead of reading. /// \param extensions An object containing a collection of frontend extensions to use during the import process -/// /// \return An nGraph function that represents a single output from the created /// graph. std::shared_ptr import_onnx_model(std::shared_ptr model_proto, const std::string& model_path, + const bool enable_mmap, ov::frontend::ExtensionHolder extensions = {}); /// \brief Decode ONNX model to nGraph function with ONNXFrameworkNode(s) @@ -42,11 +43,12 @@ std::shared_ptr import_onnx_model(std::shared_ptr decode_to_framework_nodes(std::shared_ptr model_proto, const std::string& model_path, + const bool enable_mmap, ov::frontend::ExtensionHolder extensions = {}); /// \brief Converts a nGraph function (onnx model decoded to function with ONNXFrameworkNode(s)) diff --git a/src/frontends/onnx/frontend/src/utils/tensor_external_data.cpp b/src/frontends/onnx/frontend/src/utils/tensor_external_data.cpp index b12696192c5..174b2f1e834 100644 --- a/src/frontends/onnx/frontend/src/utils/tensor_external_data.cpp +++ b/src/frontends/onnx/frontend/src/utils/tensor_external_data.cpp @@ -29,9 +29,29 @@ TensorExternalData::TensorExternalData(const ONNX_NAMESPACE::TensorProto& tensor m_sha1_digest = entry.value(); } } + if (m_sha1_digest.size() > 0) { + OPENVINO_WARN << "SHA1 checksum is not supported"; + } } -std::string TensorExternalData::load_external_data(const std::string& model_dir) const { +Buffer TensorExternalData::load_external_mmap_data(const std::string& model_dir) const { + NGRAPH_SUPPRESS_DEPRECATED_START + auto full_path = file_util::path_join(model_dir, m_data_location); + NGRAPH_SUPPRESS_DEPRECATED_END + const int64_t file_size = ov::util::file_size(full_path); + if (file_size <= 0 || m_offset + m_data_length > static_cast(file_size)) { + throw error::invalid_external_data{*this}; + } + auto mapped_memory = ov::load_mmap_object(full_path, m_data_length, m_offset); + if (m_data_length > mapped_memory->size() || mapped_memory->size() == 0) { + throw error::invalid_external_data{*this}; + } + return std::make_shared>>(mapped_memory->data(), + mapped_memory->size(), + mapped_memory); +} + +Buffer TensorExternalData::load_external_data(const std::string& model_dir) const { NGRAPH_SUPPRESS_DEPRECATED_START auto full_path = file_util::path_join(model_dir, m_data_location); @@ -57,16 +77,16 @@ std::string TensorExternalData::load_external_data(const std::string& model_dir) // default value of m_offset is 0 external_data_stream.seekg(m_offset, std::ios::beg); - if (m_sha1_digest.size() > 0) { - OPENVINO_WARN << "SHA1 checksum is not supported"; - } - - std::string read_data; - read_data.resize(read_data_length); - external_data_stream.read(&read_data[0], read_data_length); + auto read_data = std::make_shared(read_data_length); + external_data_stream.read(read_data->get_ptr(), read_data_length); external_data_stream.close(); - return read_data; + auto buffer = std::make_shared>>( + read_data->get_ptr(), + read_data->size(), + read_data); + + return buffer; } std::string TensorExternalData::to_string() const { diff --git a/src/frontends/onnx/frontend/src/utils/tensor_external_data.hpp b/src/frontends/onnx/frontend/src/utils/tensor_external_data.hpp index 62f8c2317d1..66eddfc86d9 100644 --- a/src/frontends/onnx/frontend/src/utils/tensor_external_data.hpp +++ b/src/frontends/onnx/frontend/src/utils/tensor_external_data.hpp @@ -6,9 +6,14 @@ #include +#include "ngraph/runtime/shared_buffer.hpp" +#include "openvino/util/mmap_object.hpp" + namespace ngraph { namespace onnx_import { namespace detail { +template +using Buffer = std::shared_ptr>>; /// \brief Helper class used to load tensor data from external files class TensorExternalData { public: @@ -20,8 +25,17 @@ public: /// \note If reading data from external files fails, /// the invalid_external_data exception is thrown. /// - /// \return External binary data loaded into a std::string - std::string load_external_data(const std::string& model_dir) const; + /// \return External binary data loaded into the SharedBuffer + Buffer load_external_data(const std::string& model_dir) const; + + /// \brief Map (mmap for lin, MapViewOfFile for win) external data from tensor passed to constructor + /// + /// \note If read data from external file fails, + /// \note If reading data from external files fails, + /// the invalid_external_data exception is thrown. + /// + /// \return External binary data loaded into the SharedBuffer + Buffer load_external_mmap_data(const std::string& model_dir) const; /// \brief Represets parameter of external data as string /// diff --git a/src/frontends/onnx/tests/CMakeLists.txt b/src/frontends/onnx/tests/CMakeLists.txt index 6dcc113f6a0..080876ab103 100644 --- a/src/frontends/onnx/tests/CMakeLists.txt +++ b/src/frontends/onnx/tests/CMakeLists.txt @@ -63,7 +63,6 @@ set(MULTI_TEST_SRC onnx_import_convpool.in.cpp onnx_import_deprecated.in.cpp onnx_import_dyn_shapes.in.cpp - onnx_import_external_data.in.cpp onnx_import_org_openvino.in.cpp onnx_import_org_pytorch.in.cpp onnx_import_reshape.in.cpp diff --git a/src/frontends/onnx/tests/onnx_import_external_data.in.cpp b/src/frontends/onnx/tests/onnx_import_external_data.in.cpp deleted file mode 100644 index 179f97269e6..00000000000 --- a/src/frontends/onnx/tests/onnx_import_external_data.in.cpp +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright (C) 2018-2023 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "common_test_utils/file_utils.hpp" -#include "default_opset.hpp" -#include "engines_util/test_case.hpp" -#include "engines_util/test_engines.hpp" -#include "gtest/gtest.h" -#include "ngraph/file_util.hpp" -#include "ngraph/type/element_type.hpp" -#include "onnx_import/onnx.hpp" -#include "util/test_control.hpp" -#include "util/test_tools.hpp" -#include "util/type_prop.hpp" - -using namespace ngraph; -using namespace ngraph::onnx_import; -using namespace ngraph::test; - -OPENVINO_SUPPRESS_DEPRECATED_START - -static std::string s_manifest = "${MANIFEST}"; -static std::string s_device = test::backend_name_to_device("${BACKEND_NAME}"); - -NGRAPH_TEST(${BACKEND_NAME}, onnx_external_data) { - const auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/external_data/external_data.onnx")); - - auto test_case = test::TestCase(function, s_device); - test_case.add_input({1.f, 2.f, 3.f, 4.f}); - test_case.add_expected_output(Shape{2, 2}, {3.f, 6.f, 9.f, 12.f}); - - test_case.run(); -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_external_data_from_stream) { - std::string path = file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/external_data/external_data.onnx"); - std::ifstream stream{path, std::ios::in | std::ios::binary}; - ASSERT_TRUE(stream.is_open()); - const auto function = onnx_import::import_onnx_model(stream, path); - - auto test_case = test::TestCase(function, s_device); - test_case.add_input({1.f, 2.f, 3.f, 4.f}); - test_case.add_expected_output(Shape{2, 2}, {3.f, 6.f, 9.f, 12.f}); - - test_case.run(); - - stream.close(); -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_external_data_incorrect_size_exception) { - try { - auto function = onnx_import::import_onnx_model( - file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/external_data/external_data_incorrect_data_shape.onnx")); - FAIL() << "Incorrect size of external data not detected"; - } catch (const ngraph_error& error) { - EXPECT_PRED_FORMAT2( - testing::IsSubstring, - std::string( - "The size of the external data file does not match the byte size of an initializer 'A' in the model"), - error.what()); - } catch (...) { - FAIL() << "Importing onnx model failed for unexpected reason"; - } -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_external_data_optional_fields) { - const auto function = - onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/external_data/external_data_optional_fields.onnx")); - - auto test_case = test::TestCase(function, s_device); - test_case.add_input({1.f, 2.f, 3.f, 4.f}); - test_case.add_expected_output(Shape{2, 2}, {3.f, 6.f, 9.f, 12.f}); - - test_case.run(); -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_external_data_in_different_paths) { - auto function = - onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/external_data/external_data_different_paths.onnx")); - - auto test_case = test::TestCase(function, s_device); - // first input: {3.f}, second: {1.f, 2.f, 5.f} read from external files - test_case.add_input({2.f, 7.f, 7.f}); - - test_case.add_expected_output({2.f, 4.f, 5.f}); - test_case.run(); -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_external_two_tensors_data_in_the_same_file) { - auto function = onnx_import::import_onnx_model( - file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/external_data/external_data_two_tensors_data_in_the_same_file.onnx")); - - auto test_case = test::TestCase(function, s_device); - // first input: {3, 2, 1}, second: {1, 2, 3} read from external file - test_case.add_input({2, 3, 1}); - - test_case.add_expected_output({3, 3, 3}); - test_case.run(); -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_external_invalid_external_data_exception) { - try { - auto function = onnx_import::import_onnx_model( - file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/external_data/external_data_file_not_found.onnx")); - FAIL() << "Incorrect path to external data not detected"; - } catch (const ngraph_error& error) { - EXPECT_PRED_FORMAT2(testing::IsSubstring, - std::string("not_existed_file.data, offset: 4096, data_length: 16)"), - error.what()); - } catch (...) { - FAIL() << "Importing onnx model failed for unexpected reason"; - } -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_external_invalid_up_dir_path) { - try { - auto function = onnx_import::import_onnx_model( - file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/external_data/inner_scope/external_data_file_in_up_dir.onnx")); - FAIL() << "Incorrect path to external data not detected"; - } catch (const ngraph_error& error) { - EXPECT_PRED_FORMAT2(testing::IsSubstring, - std::string("tensor.data, offset: 4096, " - "data_length: 16)"), - error.what()); - } catch (...) { - FAIL() << "Importing onnx model failed for unexpected reason"; - } -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_external_invalid_data_length) { - try { - auto function = onnx_import::import_onnx_model( - file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/external_data/external_data_invalid_data_length.onnx")); - FAIL() << "Incorrect path to external data not detected"; - } catch (const ngraph_error& error) { - EXPECT_PRED_FORMAT2(testing::IsSubstring, - std::string("tensor.data, offset: 0, " - "data_length: 30000000000)"), - error.what()); - } catch (...) { - FAIL() << "Importing onnx model failed for unexpected reason"; - } -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_external_data_sanitize_path) { - const auto function = - onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/external_data/external_data_sanitize_test.onnx")); - - auto test_case = test::TestCase(function, s_device); - test_case.add_input({1.f, 2.f, 3.f, 4.f}); - test_case.add_expected_output(Shape{2, 2}, {3.f, 6.f, 9.f, 12.f}); - - test_case.run(); -} - -NGRAPH_TEST(${BACKEND_NAME}, onnx_external_data_in_constant_node) { - const auto function = - onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(), - SERIALIZED_ZOO, - "onnx/external_data/external_data_in_constant_node.onnx")); - - auto test_case = test::TestCase(function, s_device); - test_case.add_input({3.f, 5.f, 8.f, 13.f}); - test_case.add_expected_output(Shape{2, 2}, {4.f, 7.f, 11.f, 17.f}); - - test_case.run(); -} diff --git a/src/frontends/onnx/tests/onnx_reader_external_data.cpp b/src/frontends/onnx/tests/onnx_reader_external_data.cpp index 0e71b248d1f..ee90dbd1a73 100644 --- a/src/frontends/onnx/tests/onnx_reader_external_data.cpp +++ b/src/frontends/onnx/tests/onnx_reader_external_data.cpp @@ -4,19 +4,23 @@ #include #include -#include #include #include -#include -#include #include #include #include #include "common_test_utils/file_utils.hpp" #include "common_test_utils/unicode_utils.hpp" +#include "engines_util/test_case.hpp" +#include "ie_blob.h" +#include "ie_core.hpp" +#include "ngraph/ngraph.hpp" +#include "openvino/frontend/manager.hpp" +#include "openvino/openvino.hpp" +// API 1.0 tests TEST(ONNX_Reader_Tests, ImportModelWithExternalDataFromFile) { InferenceEngine::Core ie; auto cnnNetwork = ie.ReadNetwork( @@ -114,3 +118,183 @@ TEST(ONNX_Reader_Tests, ImportModelWithExternalDataFromWstringNamedFile) { ASSERT_TRUE(external_data_node_const->get_vector() == (std::vector{1, 2, 3, 4})); } #endif + +// API 2.0 tests + +class OnnxFeMmapFixture : public ::testing::TestWithParam {}; + +TEST_P(OnnxFeMmapFixture, onnx_external_data) { + const auto path = + CommonTestUtils::getModelFromTestModelZoo(std::string(ONNX_TEST_MODELS) + "external_data/external_data.onnx"); + ov::Core core; + core.set_property(ov::enable_mmap(GetParam())); + const auto model = core.read_model(path); + auto test_case = ngraph::test::TestCase(model); + test_case.add_input({1.f, 2.f, 3.f, 4.f}); + test_case.add_expected_output({2, 2}, {3.f, 6.f, 9.f, 12.f}); + + test_case.run(); +} + +TEST_P(OnnxFeMmapFixture, onnx_external_data_from_stream) { + const auto path = + CommonTestUtils::getModelFromTestModelZoo(std::string(ONNX_TEST_MODELS) + "external_data/external_data.onnx"); + std::ifstream stream{path, std::ios::in | std::ios::binary}; + std::istream* is = &stream; + ASSERT_TRUE(stream.is_open()); + + auto fem = ov::frontend::FrontEndManager(); + auto frontend = fem.load_by_framework("onnx"); + const bool enable_mmap = GetParam(); + const auto in_model = frontend->load(is, path, enable_mmap); + const auto model = frontend->convert(in_model); + + auto test_case = ngraph::test::TestCase(model); + test_case.add_input({1.f, 2.f, 3.f, 4.f}); + test_case.add_expected_output(ov::Shape{2, 2}, {3.f, 6.f, 9.f, 12.f}); + + test_case.run(); + + stream.close(); +} + +TEST_P(OnnxFeMmapFixture, onnx_external_data_incorrect_size_exception) { + try { + const auto path = CommonTestUtils::getModelFromTestModelZoo( + std::string(ONNX_TEST_MODELS) + "external_data/external_data_incorrect_data_shape.onnx"); + ov::Core core; + core.set_property(ov::enable_mmap(GetParam())); + const auto model = core.read_model(path); + FAIL() << "Incorrect size of external data not detected"; + } catch (const ov::Exception& ex) { + EXPECT_PRED_FORMAT2( + testing::IsSubstring, + std::string( + "The size of the external data file does not match the byte size of an initializer 'A' in the model"), + ex.what()); + } catch (...) { + FAIL() << "Importing onnx model failed for unexpected reason"; + } +} + +TEST_P(OnnxFeMmapFixture, onnx_external_data_optional_fields) { + const auto path = CommonTestUtils::getModelFromTestModelZoo(std::string(ONNX_TEST_MODELS) + + "external_data/external_data_optional_fields.onnx"); + ov::Core core; + core.set_property(ov::enable_mmap(GetParam())); + const auto model = core.read_model(path); + auto test_case = ngraph::test::TestCase(model); + test_case.add_input({1.f, 2.f, 3.f, 4.f}); + test_case.add_expected_output(ov::Shape{2, 2}, {3.f, 6.f, 9.f, 12.f}); + + test_case.run(); +} + +TEST_P(OnnxFeMmapFixture, onnx_external_data_in_different_paths) { + const auto path = CommonTestUtils::getModelFromTestModelZoo(std::string(ONNX_TEST_MODELS) + + "external_data/external_data_different_paths.onnx"); + ov::Core core; + core.set_property(ov::enable_mmap(GetParam())); + const auto model = core.read_model(path); + auto test_case = ngraph::test::TestCase(model); + // first input: {3.f}, second: {1.f, 2.f, 5.f} read from external files + test_case.add_input({2.f, 7.f, 7.f}); + + test_case.add_expected_output({2.f, 4.f, 5.f}); + test_case.run(); +} + +TEST_P(OnnxFeMmapFixture, onnx_external_two_tensors_data_in_the_same_file) { + const auto path = CommonTestUtils::getModelFromTestModelZoo( + std::string(ONNX_TEST_MODELS) + "external_data/external_data_two_tensors_data_in_the_same_file.onnx"); + ov::Core core; + core.set_property(ov::enable_mmap(GetParam())); + const auto model = core.read_model(path); + auto test_case = ngraph::test::TestCase(model); + // first input: {3, 2, 1}, second: {1, 2, 3} read from external file + test_case.add_input({2, 3, 1}); + + test_case.add_expected_output({3, 3, 3}); + test_case.run(); +} + +TEST_P(OnnxFeMmapFixture, onnx_external_invalid_external_data_exception) { + try { + const auto path = CommonTestUtils::getModelFromTestModelZoo(std::string(ONNX_TEST_MODELS) + + "external_data/external_data_file_not_found.onnx"); + ov::Core core; + core.set_property(ov::enable_mmap(GetParam())); + core.read_model(path); + FAIL() << "Incorrect path to external data not detected"; + } catch (const ov::Exception& ex) { + EXPECT_PRED_FORMAT2(testing::IsSubstring, + std::string("not_existed_file.data, offset: 4096, data_length: 16)"), + ex.what()); + } catch (...) { + FAIL() << "Importing onnx model failed for unexpected reason"; + } +} + +TEST_P(OnnxFeMmapFixture, onnx_external_invalid_up_dir_path) { + try { + const auto path = CommonTestUtils::getModelFromTestModelZoo( + std::string(ONNX_TEST_MODELS) + "external_data/inner_scope/external_data_file_in_up_dir.onnx"); + ov::Core core; + core.set_property(ov::enable_mmap(GetParam())); + core.read_model(path); + FAIL() << "Incorrect path to external data not detected"; + } catch (const ov::Exception& ex) { + EXPECT_PRED_FORMAT2(testing::IsSubstring, + std::string("tensor.data, offset: 4096, " + "data_length: 16)"), + ex.what()); + } catch (...) { + FAIL() << "Importing onnx model failed for unexpected reason"; + } +} + +TEST_P(OnnxFeMmapFixture, onnx_external_invalid_data_length) { + try { + const auto path = CommonTestUtils::getModelFromTestModelZoo( + std::string(ONNX_TEST_MODELS) + "external_data/external_data_invalid_data_length.onnx"); + ov::Core core; + core.set_property(ov::enable_mmap(GetParam())); + core.read_model(path); + FAIL() << "Incorrect path to external data not detected"; + } catch (const ov::Exception& ex) { + EXPECT_PRED_FORMAT2(testing::IsSubstring, + std::string("tensor.data, offset: 0, " + "data_length: 30000000000)"), + ex.what()); + } catch (...) { + FAIL() << "Importing onnx model failed for unexpected reason"; + } +} + +TEST_P(OnnxFeMmapFixture, onnx_external_data_sanitize_path) { + const auto path = CommonTestUtils::getModelFromTestModelZoo(std::string(ONNX_TEST_MODELS) + + "external_data/external_data_sanitize_test.onnx"); + ov::Core core; + core.set_property(ov::enable_mmap(GetParam())); + const auto model = core.read_model(path); + auto test_case = ngraph::test::TestCase(model); + test_case.add_input({1.f, 2.f, 3.f, 4.f}); + test_case.add_expected_output(ov::Shape{2, 2}, {3.f, 6.f, 9.f, 12.f}); + + test_case.run(); +} + +TEST_P(OnnxFeMmapFixture, onnx_external_data_in_constant_node) { + const auto path = CommonTestUtils::getModelFromTestModelZoo(std::string(ONNX_TEST_MODELS) + + "external_data/external_data_in_constant_node.onnx"); + ov::Core core; + core.set_property(ov::enable_mmap(GetParam())); + const auto model = core.read_model(path); + auto test_case = ngraph::test::TestCase(model); + test_case.add_input({3.f, 5.f, 8.f, 13.f}); + test_case.add_expected_output(ov::Shape{2, 2}, {4.f, 7.f, 11.f, 17.f}); + + test_case.run(); +} + +INSTANTIATE_TEST_SUITE_P(OnnxFeMMapReadModel, OnnxFeMmapFixture, ::testing::Bool()); diff --git a/tools/mo/openvino/tools/mo/convert.py b/tools/mo/openvino/tools/mo/convert.py index 7b2c92747b7..367e9527ccc 100644 --- a/tools/mo/openvino/tools/mo/convert.py +++ b/tools/mo/openvino/tools/mo/convert.py @@ -40,6 +40,7 @@ def convert_model( version: bool = None, progress: bool = False, stream_output: bool = False, + enable_mmap: bool = False, # PaddlePaddle-specific parameters: example_output: Any = None, @@ -268,6 +269,10 @@ def convert_model( Enable model conversion progress display. :param stream_output: Switch model conversion progress display to a multiline mode. + :param enable_mmap: + Force reading weights instead of mapping weights files into memory. + Currently, mmap feature is provided only for ONNX models + that do not require fallback to the legacy ONNX frontend for the conversion. PaddlePaddle-specific parameters: :param example_output: diff --git a/tools/mo/openvino/tools/mo/convert_impl.py b/tools/mo/openvino/tools/mo/convert_impl.py index 37042a28b10..53f085cf47c 100644 --- a/tools/mo/openvino/tools/mo/convert_impl.py +++ b/tools/mo/openvino/tools/mo/convert_impl.py @@ -444,7 +444,9 @@ def read_model(fem: FrontEndManager, path_to_xml: str): # avoid segfault during object destruction. So fe must # be destructed before fem object explicitly. fe = fem.load_by_framework(framework="ir") - function = fe.convert(fe.load(path_to_xml)) + # *.xml/.*bin files are temporary created in the legacy scenario, so we cannot map the memory + enable_mmap = False + function = fe.convert(fe.load(path_to_xml, enable_mmap)) return function diff --git a/tools/mo/unit_tests/mo/utils/cli_parser_test.py b/tools/mo/unit_tests/mo/utils/cli_parser_test.py index 1b4b0892eec..ae522598c0a 100644 --- a/tools/mo/unit_tests/mo/utils/cli_parser_test.py +++ b/tools/mo/unit_tests/mo/utils/cli_parser_test.py @@ -2040,7 +2040,7 @@ class TestConvertModelParamsParsing(unittest.TestCase): 'log_level', 'input', 'output', 'mean_values', 'scale_values', 'source_layout', 'target_layout', 'layout', 'compress_to_fp16', 'transform', 'extensions', 'batch', 'silent', 'version', 'progress', 'stream_output', - 'transformations_config', 'example_input'}, + 'transformations_config', 'example_input', 'enable_mmap'}, 'Caffe*-specific parameters:': {'input_proto', 'caffe_parser_path', 'k', 'disable_omitting_optional', 'enable_flattening_nested_params'}, 'TensorFlow*-specific parameters:': {'input_model_is_text', 'input_checkpoint', 'input_meta_graph', diff --git a/tools/ovc/openvino/tools/ovc/convert.py b/tools/ovc/openvino/tools/ovc/convert.py index 6ef08c86d21..2b13f49bf5c 100644 --- a/tools/ovc/openvino/tools/ovc/convert.py +++ b/tools/ovc/openvino/tools/ovc/convert.py @@ -45,6 +45,7 @@ def convert_model( version: bool = None, progress: bool = False, stream_output: bool = False, + enable_mmap: bool = True, # PaddlePaddle-specific parameters: example_output: Any = None, @@ -273,6 +274,10 @@ def convert_model( Enable model conversion progress display. :param stream_output: Switch model conversion progress display to a multiline mode. + :param enable_mmap: + Force reading weights instead of mapping weights files into memory. + Currently, mmap feature is provided only for ONNX models + that do not require fallback to the legacy ONNX frontend for the conversion. PaddlePaddle-specific parameters: :param example_output: diff --git a/tools/ovc/openvino/tools/ovc/moc_frontend/pipeline.py b/tools/ovc/openvino/tools/ovc/moc_frontend/pipeline.py index 4a7498e0fe9..b72130a2a87 100644 --- a/tools/ovc/openvino/tools/ovc/moc_frontend/pipeline.py +++ b/tools/ovc/openvino/tools/ovc/moc_frontend/pipeline.py @@ -46,18 +46,19 @@ def moc_pipeline(argv: argparse.Namespace, moc_front_end: FrontEnd): :return: converted nGraph function ready for serialization """ input_checkpoint = getattr(argv, 'input_checkpoint', None) + enable_mmap = getattr(argv, 'enable_mmap', True) if argv.input_model and input_checkpoint: # frozen format with v1 checkpoints - input_model = moc_front_end.load([argv.input_model, argv.input_checkpoint]) + input_model = moc_front_end.load([argv.input_model, argv.input_checkpoint], enable_mmap) elif argv.input_model: - input_model = moc_front_end.load(argv.input_model) + input_model = moc_front_end.load(argv.input_model, enable_mmap) elif argv.saved_model_dir: if argv.saved_model_tags: - input_model = moc_front_end.load([argv.saved_model_dir, argv.saved_model_tags]) + input_model = moc_front_end.load([argv.saved_model_dir, argv.saved_model_tags], enable_mmap) else: - input_model = moc_front_end.load(argv.saved_model_dir) + input_model = moc_front_end.load(argv.saved_model_dir, enable_mmap) elif argv.input_meta_graph: - input_model = moc_front_end.load(argv.input_meta_graph) + input_model = moc_front_end.load(argv.input_meta_graph, enable_mmap) if argv.output: # Simulate original behavior with freezing model # While freezing we do a cutting of model, to keep similar behavior we diff --git a/tools/ovc/unit_tests/ovc/utils/cli_parser_test.py b/tools/ovc/unit_tests/ovc/utils/cli_parser_test.py index 409fcf4ee27..64ed15c8b77 100644 --- a/tools/ovc/unit_tests/ovc/utils/cli_parser_test.py +++ b/tools/ovc/unit_tests/ovc/utils/cli_parser_test.py @@ -2073,7 +2073,7 @@ class TestConvertModelParamsParsing(unittest.TestCase): 'log_level', 'input', 'output', 'mean_values', 'scale_values', 'source_layout', 'target_layout', 'layout', 'compress_to_fp16', 'transform', 'extensions', 'batch', 'silent', 'version', 'progress', 'stream_output', - 'transformations_config', 'example_input'}, + 'transformations_config', 'example_input', 'enable_mmap'}, 'Caffe*-specific parameters:': {'input_proto', 'caffe_parser_path', 'k', 'disable_omitting_optional', 'enable_flattening_nested_params'}, 'TensorFlow*-specific parameters:': {'input_model_is_text', 'input_checkpoint', 'input_meta_graph',