IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[ONNX] Implemented cache for MMAP handles (#19130)

Browse Source 
* Implemented cache for MMAP handles

* Applied comments and fixed behavior when m_data_length==0

* Aligned data length calculation with no-MMAP implementation

* Removed unused code
This commit is contained in:
Georgy Krivoruchko 2023-08-14 04:56:59 -07:00 committed by GitHub
parent e5c4350d92
commit 46f428eeac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 114 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/common/util/include/openvino/util/mmap_object.hpp
View File

@ -32,13 +32,9 @@ public:
* in order to avoid time-consuming reading and reduce memory consumption. * in order to avoid time-consuming reading and reduce memory consumption.
* *
* @param path Path to a file which memory will be mmaped. * @param path Path to a file which memory will be mmaped.
* @param data_size Number of bytes to map (zero, if all file will be mmaped).
* @param offset Number of bytes to skip from beginning of the file.
* @return MappedMemory shared ptr object which keep mmaped memory and control the lifetime. * @return MappedMemory shared ptr object which keep mmaped memory and control the lifetime.
*/ */
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::string& path, std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::string& path);
const size_t data_size = 0,
const size_t offset = 0);
#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT #ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT
@ -48,13 +44,9 @@ std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::string& path,
* in order to avoid time-consuming reading and reduce memory consumption. * in order to avoid time-consuming reading and reduce memory consumption.
* *
* @param path Path to a file which memory will be mmaped. * @param path Path to a file which memory will be mmaped.
* @param data_size Number of bytes to map (zero, if all file will be mmaped).
* @param offset Number of bytes to skip from beginning of the file.
* @return MappedMemory shared ptr object which keep mmaped memory and control the lifetime. * @return MappedMemory shared ptr object which keep mmaped memory and control the lifetime.
*/ */
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::wstring& path, std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::wstring& path);
const size_t data_size = 0,
const size_t offset = 0);
#endif // OPENVINO_ENABLE_UNICODE_PATH_SUPPORT #endif // OPENVINO_ENABLE_UNICODE_PATH_SUPPORT

25
src/common/util/src/os/lin/lin_mmap_object.cpp
View File

@ -63,7 +63,7 @@ class MapHolder : public MappedMemory {
public: public:
MapHolder() = default; MapHolder() = default;
void set(const std::string& path, const size_t data_size = 0, const size_t offset = 0) { void set(const std::string& path) {
int prot = PROT_READ; int prot = PROT_READ;
int mode = O_RDONLY; int mode = O_RDONLY;
struct stat sb = {}; struct stat sb = {};
@ -75,24 +75,13 @@ public:
if (fstat(m_handle.get(), &sb) == -1) { if (fstat(m_handle.get(), &sb) == -1) {
throw std::runtime_error("Can not get file size for " + path); throw std::runtime_error("Can not get file size for " + path);
} }
if (data_size + offset > static_cast<size_t>(sb.st_size)) { m_size = sb.st_size;
throw std::runtime_error("Can not map out of scope memory for " + path); if (m_size > 0) {
} m_data = mmap(nullptr, m_size, prot, MAP_PRIVATE, m_handle.get(), 0);
m_size = data_size > 0 ? data_size : sb.st_size - offset;
const size_t page_size = sysconf(_SC_PAGE_SIZE);
// align offset with page size
const size_t aligned_offset = (offset / page_size) * page_size;
const size_t aligned_size = (offset % page_size) + m_size;
// move pointer to the expected data (after alignment with page size)
const size_t offset_delta = offset - aligned_offset;
if (aligned_size > 0) {
m_data = mmap(nullptr, aligned_size, prot, MAP_PRIVATE, m_handle.get(), aligned_offset);
if (m_data == MAP_FAILED) { if (m_data == MAP_FAILED) {
throw std::runtime_error("Can not create file mapping for " + path + ", err=" + std::strerror(errno)); throw std::runtime_error("Can not create file mapping for " + path + ", err=" + std::strerror(errno));
} }
m_data = reinterpret_cast<char*>(m_data) + offset_delta;
} else { } else {
m_size = 0;
m_data = MAP_FAILED; m_data = MAP_FAILED;
} }
} }
@ -112,11 +101,9 @@ public:
} }
}; };
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::string& path, std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::string& path) {
const size_t data_size,
const size_t offset) {
auto holder = std::make_shared<MapHolder>(); auto holder = std::make_shared<MapHolder>();
holder->set(path, data_size, offset); holder->set(path);
return holder; return holder;
} }

64
src/common/util/src/os/win/win_mmap_object.cpp
View File

@ -2,10 +2,11 @@
// SPDX-License-Identifier: Apache-2.0 // SPDX-License-Identifier: Apache-2.0
// //
#include "openvino/util/mmap_object.hpp"
#include "openvino/util/file_util.hpp"
#include <stdexcept> #include <stdexcept>
#include "openvino/util/file_util.hpp"
#include "openvino/util/mmap_object.hpp"
// clang-format-off // clang-format-off
#ifndef NOMINMAX #ifndef NOMINMAX
# define NOMINMAX # define NOMINMAX
@ -60,19 +61,19 @@ public:
} }
} }
void set(const std::string& path, const size_t data_size=0, const size_t offset=0) { void set(const std::string& path) {
// Note that file can't be changed (renamed/deleted) until it's unmapped. FILE_SHARE_DELETE flag allow // Note that file can't be changed (renamed/deleted) until it's unmapped. FILE_SHARE_DELETE flag allow
// rename/deletion, but it doesn't work with FAT32 filesystem (works on NTFS) // rename/deletion, but it doesn't work with FAT32 filesystem (works on NTFS)
auto h = ::CreateFileA(path.c_str(), GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); auto h = ::CreateFileA(path.c_str(), GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
map(path, h, data_size, offset); map(path, h);
} }
#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT #ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT
void set(const std::wstring& path, const size_t data_size=0, const size_t offset=0) { void set(const std::wstring& path) {
// Note that file can't be changed (renamed/deleted) until it's unmapped. FILE_SHARE_DELETE flag allow // Note that file can't be changed (renamed/deleted) until it's unmapped. FILE_SHARE_DELETE flag allow
// rename/deletion, but it doesn't work with FAT32 filesystem (works on NTFS) // rename/deletion, but it doesn't work with FAT32 filesystem (works on NTFS)
auto h = ::CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); auto h = ::CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
map(ov::util::wstring_to_string(path), h, data_size, offset); map(ov::util::wstring_to_string(path), h);
} }
#endif #endif
@ -84,50 +85,37 @@ public:
} }
private: private:
void map(const std::string& path, HANDLE h, const size_t data_size=0, const size_t offset=0) { void map(const std::string& path, HANDLE h) {
if(h == INVALID_HANDLE_VALUE) { if (h == INVALID_HANDLE_VALUE) {
throw std::runtime_error("Can not open file " + path + " for mapping. Ensure that file exists and has appropriate permissions"); throw std::runtime_error("Can not open file " + path +
" for mapping. Ensure that file exists and has appropriate permissions");
} }
m_handle = HandleHolder(h); m_handle = HandleHolder(h);
SYSTEM_INFO sys_info;
GetSystemInfo(&sys_info);
DWORD map_mode = FILE_MAP_READ; DWORD map_mode = FILE_MAP_READ;
DWORD access = PAGE_READONLY; DWORD access = PAGE_READONLY;
LARGE_INTEGER file_size_large; LARGE_INTEGER file_size_large;
if(::GetFileSizeEx(m_handle.get(), &file_size_large) == 0) { if (::GetFileSizeEx(m_handle.get(), &file_size_large) == 0) {
throw std::runtime_error("Can not get file size for " + path); throw std::runtime_error("Can not get file size for " + path);
} }
if (data_size + offset > static_cast<size_t>(file_size_large.QuadPart)) {
throw std::runtime_error("Can not map out of scope memory for " + path);
}
m_size = data_size > 0 ? data_size : static_cast<size_t>(file_size_large.QuadPart) - offset;
// the offset must be a multiple of the allocation granularity
const size_t alloc_gran = sys_info.dwAllocationGranularity;
const size_t map_view_start = (offset / alloc_gran) * alloc_gran;
const size_t map_view_size = (offset%alloc_gran) + m_size;
const size_t file_map_size = offset + m_size;
const size_t view_delta = offset - map_view_start;
m_size = static_cast<uint64_t>(file_size_large.QuadPart);
if (m_size > 0) { if (m_size > 0) {
m_mapping = m_mapping =
HandleHolder(::CreateFileMapping(m_handle.get(), 0, access, file_map_size >> 32, file_map_size & 0xffffffff, 0)); HandleHolder(::CreateFileMapping(m_handle.get(), 0, access, m_size >> 32, m_size & 0xffffffff, 0));
if(m_mapping.get() == INVALID_HANDLE_VALUE) { if (m_mapping.get() == INVALID_HANDLE_VALUE) {
throw std::runtime_error("Can not create file mapping for " + path); throw std::runtime_error("Can not create file mapping for " + path);
} }
m_data = ::MapViewOfFile(m_mapping.get(), m_data = ::MapViewOfFile(m_mapping.get(),
map_mode, map_mode,
map_view_start >> 32, // offset_align >> 32, 0, // offset_align >> 32,
map_view_start & 0xffffffff, // offset_align & 0xffffffff, 0, // offset_align & 0xffffffff,
map_view_size); m_size);
if(!m_data) { if (!m_data) {
throw std::runtime_error("Can not create map view for " + path); throw std::runtime_error("Can not create map view for " + path);
} }
// move pointer to the expected data (after alignment with allocation granularity)
m_data = reinterpret_cast<char*>(m_data) + view_delta;
} else { } else {
m_data = nullptr; m_data = nullptr;
} }
@ -140,17 +128,17 @@ private:
HandleHolder m_mapping; HandleHolder m_mapping;
}; };
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::string& path, const size_t data_size, const size_t offset) { std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::string& path) {
auto holder = std::make_shared<MapHolder>(); auto holder = std::make_shared<MapHolder>();
holder->set(path, data_size, offset); holder->set(path);
return holder; return holder;
} }
#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT #ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::wstring& path, const size_t data_size, const size_t offset) { std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::wstring& path) {
auto holder = std::make_shared<MapHolder>(); auto holder = std::make_shared<MapHolder>();
holder->set(path, data_size, offset); holder->set(path);
return holder; return holder;
} }

26
src/frontends/onnx/frontend/src/core/attribute.hpp
View File

@ -200,12 +200,12 @@ public:
}; };
Attribute() = delete; Attribute() = delete;
explicit Attribute(const ONNX_NAMESPACE::AttributeProto& attribute_proto, Attribute(const ONNX_NAMESPACE::AttributeProto& attribute_proto,
const std::string& model_dir, const std::string& model_dir,
const bool enable_mmap) detail::MappedMemoryHandles mmap_cache)
: m_attribute_proto{&attribute_proto}, : m_attribute_proto{&attribute_proto},
m_model_dir{model_dir}, m_model_dir{model_dir},
m_enable_mmap{enable_mmap} {} m_mmap_cache{mmap_cache} {}
Attribute(Attribute&&) noexcept = default; Attribute(Attribute&&) noexcept = default;
Attribute(const Attribute&) = default; Attribute(const Attribute&) = default;
@ -256,10 +256,10 @@ public:
return get_type() == Type::graph_array; return get_type() == Type::graph_array;
} }
Tensor get_tensor() const { Tensor get_tensor() const {
return Tensor{m_attribute_proto->t(), m_model_dir, m_enable_mmap}; return Tensor{m_attribute_proto->t(), m_model_dir, m_mmap_cache};
} }
SparseTensor get_sparse_tensor() const { SparseTensor get_sparse_tensor() const {
return SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir, m_enable_mmap}; return SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir, m_mmap_cache};
} }
float get_float() const { float get_float() const {
return m_attribute_proto->f(); return m_attribute_proto->f();
@ -277,7 +277,7 @@ public:
const auto& tensors = m_attribute_proto->tensors(); const auto& tensors = m_attribute_proto->tensors();
ret.reserve(tensors.size()); ret.reserve(tensors.size());
for (const auto& tensor : tensors) for (const auto& tensor : tensors)
ret.emplace_back(tensor, m_model_dir, m_enable_mmap); ret.emplace_back(tensor, m_model_dir, m_mmap_cache);
return ret; return ret;
} }
@ -286,7 +286,7 @@ public:
const auto& sparse_tensors = m_attribute_proto->sparse_tensors(); const auto& sparse_tensors = m_attribute_proto->sparse_tensors();
ret.reserve(sparse_tensors.size()); ret.reserve(sparse_tensors.size());
for (const auto& tensor : sparse_tensors) for (const auto& tensor : sparse_tensors)
ret.emplace_back(tensor, m_model_dir, m_enable_mmap); ret.emplace_back(tensor, m_model_dir, m_mmap_cache);
return ret; return ret;
} }
@ -318,7 +318,7 @@ public:
template <typename T, typename std::enable_if<std::is_same<T, Tensor>::value, bool>::type = true> template <typename T, typename std::enable_if<std::is_same<T, Tensor>::value, bool>::type = true>
T get_value() const { T get_value() const {
if (is_tensor()) { if (is_tensor()) {
return Tensor{m_attribute_proto->t(), m_model_dir, m_enable_mmap}; return Tensor{m_attribute_proto->t(), m_model_dir, m_mmap_cache};
} }
throw error::attribute::InvalidData{m_attribute_proto->type()}; throw error::attribute::InvalidData{m_attribute_proto->type()};
} }
@ -326,7 +326,7 @@ public:
template <typename T, typename std::enable_if<std::is_same<T, std::vector<Tensor>>::value, bool>::type = true> template <typename T, typename std::enable_if<std::is_same<T, std::vector<Tensor>>::value, bool>::type = true>
T get_value() const { T get_value() const {
if (is_tensor()) { if (is_tensor()) {
return {Tensor{m_attribute_proto->t(), m_model_dir, m_enable_mmap}}; return {Tensor{m_attribute_proto->t(), m_model_dir, m_mmap_cache}};
} else if (is_tensor_array()) { } else if (is_tensor_array()) {
return get_tensor_array(); return get_tensor_array();
} }
@ -336,7 +336,7 @@ public:
template <typename T, typename std::enable_if<std::is_same<T, SparseTensor>::value, bool>::type = true> template <typename T, typename std::enable_if<std::is_same<T, SparseTensor>::value, bool>::type = true>
T get_value() const { T get_value() const {
if (is_sparse_tensor()) { if (is_sparse_tensor()) {
return SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir, m_enable_mmap}; return SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir, m_mmap_cache};
} }
throw error::attribute::InvalidData{m_attribute_proto->type()}; throw error::attribute::InvalidData{m_attribute_proto->type()};
} }
@ -344,7 +344,7 @@ public:
template <typename T, typename std::enable_if<std::is_same<T, std::vector<SparseTensor>>::value, bool>::type = true> template <typename T, typename std::enable_if<std::is_same<T, std::vector<SparseTensor>>::value, bool>::type = true>
T get_value() const { T get_value() const {
if (is_sparse_tensor()) { if (is_sparse_tensor()) {
return {SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir, m_enable_mmap}}; return {SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir, m_mmap_cache}};
} else if (is_sparse_tensor_array()) { } else if (is_sparse_tensor_array()) {
return get_sparse_tensor_array(); return get_sparse_tensor_array();
} }
@ -356,7 +356,7 @@ public:
private: private:
const ONNX_NAMESPACE::AttributeProto* m_attribute_proto; const ONNX_NAMESPACE::AttributeProto* m_attribute_proto;
std::string m_model_dir; std::string m_model_dir;
const bool m_enable_mmap; detail::MappedMemoryHandles m_mmap_cache;
}; };
} // namespace onnx_import } // namespace onnx_import

12
src/frontends/onnx/frontend/src/core/graph.cpp
View File

@ -123,19 +123,19 @@ ov::frontend::ExtensionHolder subgraph_required_extensions(
Graph::Graph(const std::string& model_dir, Graph::Graph(const std::string& model_dir,
const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model_proto, const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model_proto,
const bool enable_mmap, detail::MappedMemoryHandles mmap_cache,
ov::frontend::ExtensionHolder extensions) ov::frontend::ExtensionHolder extensions)
: Graph(model_dir, model_proto, common::make_unique<GraphCache>(), enable_mmap, std::move(extensions)) {} : Graph(model_dir, model_proto, common::make_unique<GraphCache>(), mmap_cache, std::move(extensions)) {}
Graph::Graph(const std::string& model_dir, Graph::Graph(const std::string& model_dir,
const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model_proto, const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model_proto,
std::unique_ptr<GraphCache>&& cache, std::unique_ptr<GraphCache>&& cache,
const bool enable_mmap, detail::MappedMemoryHandles mmap_cache,
ov::frontend::ExtensionHolder extensions) ov::frontend::ExtensionHolder extensions)
: m_cache{std::move(cache)}, : m_cache{std::move(cache)},
m_extensions{std::move(extensions)}, m_extensions{std::move(extensions)},
m_model_dir{model_dir}, m_model_dir{model_dir},
m_enable_mmap{enable_mmap}, m_mmap_cache{mmap_cache},
m_ops_bridge{detail::init_ops_bridge(m_extensions.conversions)} { m_ops_bridge{detail::init_ops_bridge(m_extensions.conversions)} {
m_model = common::make_unique<Model>(model_proto, detail::build_model_opset(*model_proto, m_ops_bridge)); m_model = common::make_unique<Model>(model_proto, detail::build_model_opset(*model_proto, m_ops_bridge));
@ -146,7 +146,7 @@ Graph::Graph(const std::string& model_dir,
// Process all initializers in the graph // Process all initializers in the graph
for (const auto& initializer_tensor : m_model->get_graph().initializer()) { for (const auto& initializer_tensor : m_model->get_graph().initializer()) {
if (initializer_tensor.has_name()) { if (initializer_tensor.has_name()) {
Tensor tensor = Tensor{initializer_tensor, m_model_dir, enable_mmap}; Tensor tensor = Tensor{initializer_tensor, m_model_dir, m_mmap_cache};
std::shared_ptr<default_opset::Constant> ng_constant; std::shared_ptr<default_opset::Constant> ng_constant;
// For each initializer create a Constant node and store it in cache // For each initializer create a Constant node and store it in cache
try { try {
@ -458,7 +458,7 @@ Subgraph::Subgraph(const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model_prot
: Graph(parent_graph->model_dir(), : Graph(parent_graph->model_dir(),
model_proto, model_proto,
common::make_unique<GraphCache>(), common::make_unique<GraphCache>(),
parent_graph->mmap_enabled(), parent_graph->get_mmap_cache(),
detail::subgraph_required_extensions(parent_graph->get_extensions())), detail::subgraph_required_extensions(parent_graph->get_extensions())),
m_parent_graph(parent_graph) {} m_parent_graph(parent_graph) {}

11
src/frontends/onnx/frontend/src/core/graph.hpp
View File

@ -18,6 +18,7 @@
#include "openvino/core/deprecated.hpp" #include "openvino/core/deprecated.hpp"
#include "openvino/frontend/extension/holder.hpp" #include "openvino/frontend/extension/holder.hpp"
#include "ops_bridge.hpp" #include "ops_bridge.hpp"
#include "utils/tensor_external_data.hpp"
namespace ngraph { namespace ngraph {
namespace onnx_import { namespace onnx_import {
@ -25,7 +26,7 @@ class Graph : public std::enable_shared_from_this<Graph> {
public: public:
Graph(const std::string& model_dir, Graph(const std::string& model_dir,
const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model_proto, const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model_proto,
const bool enable_mmap, detail::MappedMemoryHandles mmap_cache,
ov::frontend::ExtensionHolder extensions = {}); ov::frontend::ExtensionHolder extensions = {});
Graph() = delete; Graph() = delete;
@ -43,8 +44,8 @@ public:
const std::string& model_dir() const { const std::string& model_dir() const {
return m_model_dir; return m_model_dir;
} }
bool mmap_enabled() const { detail::MappedMemoryHandles get_mmap_cache() const {
return m_enable_mmap; return m_mmap_cache;
} }
const ParameterVector& get_ng_parameters() const { const ParameterVector& get_ng_parameters() const {
return m_parameters; return m_parameters;
@ -65,7 +66,7 @@ protected:
Graph(const std::string& model_dir, Graph(const std::string& model_dir,
const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model, const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model,
std::unique_ptr<GraphCache>&& cache, std::unique_ptr<GraphCache>&& cache,
const bool enable_mmap, detail::MappedMemoryHandles mmap_cache,
ov::frontend::ExtensionHolder extensions = {}); ov::frontend::ExtensionHolder extensions = {});
OPENVINO_SUPPRESS_DEPRECATED_START OPENVINO_SUPPRESS_DEPRECATED_START
@ -92,7 +93,7 @@ private:
std::vector<Node> m_nodes; std::vector<Node> m_nodes;
OPENVINO_SUPPRESS_DEPRECATED_END OPENVINO_SUPPRESS_DEPRECATED_END
std::string m_model_dir; std::string m_model_dir;
bool m_enable_mmap; detail::MappedMemoryHandles m_mmap_cache;
OperatorsBridge m_ops_bridge; OperatorsBridge m_ops_bridge;
}; };

4
src/frontends/onnx/frontend/src/core/node.cpp
View File

@ -26,7 +26,7 @@ public:
const auto& attributes = node_proto.attribute(); const auto& attributes = node_proto.attribute();
m_attributes.reserve(attributes.size()); m_attributes.reserve(attributes.size());
for (const auto& attr_proto : attributes) { for (const auto& attr_proto : attributes) {
m_attributes.emplace_back(attr_proto, m_graph->model_dir(), m_graph->mmap_enabled()); m_attributes.emplace_back(attr_proto, m_graph->model_dir(), m_graph->get_mmap_cache());
const auto& attribute = m_attributes.back(); const auto& attribute = m_attributes.back();
if (attribute.is_graph()) if (attribute.is_graph())
m_subgraphs.insert({attribute.get_name(), std::make_shared<Subgraph>(attribute.get_subgraph(m_graph))}); m_subgraphs.insert({attribute.get_name(), std::make_shared<Subgraph>(attribute.get_subgraph(m_graph))});
@ -43,7 +43,7 @@ public:
m_output_names{std::begin(node_proto.output()), std::end(node_proto.output())}, m_output_names{std::begin(node_proto.output()), std::end(node_proto.output())},
m_subgraphs(subgraphs) { m_subgraphs(subgraphs) {
for (const auto& attr_proto : node_proto.attribute()) { for (const auto& attr_proto : node_proto.attribute()) {
m_attributes.emplace_back(attr_proto, m_graph->model_dir(), m_graph->mmap_enabled()); m_attributes.emplace_back(attr_proto, m_graph->model_dir(), m_graph->get_mmap_cache());
} }
} }

6
src/frontends/onnx/frontend/src/core/sparse_tensor.hpp
View File

@ -19,9 +19,9 @@ public:
SparseTensor() = delete; SparseTensor() = delete;
SparseTensor(const ONNX_NAMESPACE::SparseTensorProto& sparse_tensor, SparseTensor(const ONNX_NAMESPACE::SparseTensorProto& sparse_tensor,
const std::string& model_dir, const std::string& model_dir,
const bool enable_mmap) detail::MappedMemoryHandles mmap_cache)
: m_values{sparse_tensor.values(), model_dir, enable_mmap}, : m_values{sparse_tensor.values(), model_dir, mmap_cache},
m_indices{sparse_tensor.indices(), model_dir, enable_mmap}, m_indices{sparse_tensor.indices(), model_dir, mmap_cache},
m_shape{std::begin(sparse_tensor.dims()), std::end(sparse_tensor.dims())} { m_shape{std::begin(sparse_tensor.dims()), std::end(sparse_tensor.dims())} {
if (m_shape == Shape{0}) { if (m_shape == Shape{0}) {
// It's possible to construct a sparse tensor in ONNX with "dims: 0" property // It's possible to construct a sparse tensor in ONNX with "dims: 0" property

21
src/frontends/onnx/frontend/src/core/tensor.hpp
View File

@ -108,11 +108,13 @@ public:
}; };
Tensor() = delete; Tensor() = delete;
explicit Tensor(const ONNX_NAMESPACE::TensorProto& tensor, const std::string& model_dir, const bool enable_mmap) Tensor(const ONNX_NAMESPACE::TensorProto& tensor,
const std::string& model_dir,
detail::MappedMemoryHandles mmap_cache)
: m_tensor_proto{&tensor}, : m_tensor_proto{&tensor},
m_shape{std::begin(tensor.dims()), std::end(tensor.dims())}, m_shape{std::begin(tensor.dims()), std::end(tensor.dims())},
m_model_dir{model_dir}, m_model_dir{model_dir},
m_enable_mmap{enable_mmap} { m_mmap_cache{mmap_cache} {
if (m_shape == Shape{0}) { if (m_shape == Shape{0}) {
// It's possible to construct a tensor in ONNX with "dims: 0" property // It's possible to construct a tensor in ONNX with "dims: 0" property
// Such tensor contains a scalar. This results in a Shape{0} stored in m_shape. // Such tensor contains a scalar. This results in a Shape{0} stored in m_shape.
@ -241,10 +243,11 @@ private:
size_t data_size = get_data_size(); size_t data_size = get_data_size();
if (has_external_data()) { if (has_external_data()) {
const auto ext_data = detail::TensorExternalData(*m_tensor_proto); const auto ext_data = detail::TensorExternalData(*m_tensor_proto);
if (m_enable_mmap) { if (m_mmap_cache) {
constant = std::make_shared<ngraph::op::Constant>(type, constant =
m_shape, std::make_shared<ngraph::op::Constant>(type,
ext_data.load_external_mmap_data(m_model_dir)); m_shape,
ext_data.load_external_mmap_data(m_model_dir, m_mmap_cache));
} else { } else {
constant = constant =
std::make_shared<ngraph::op::Constant>(type, m_shape, ext_data.load_external_data(m_model_dir)); std::make_shared<ngraph::op::Constant>(type, m_shape, ext_data.load_external_data(m_model_dir));
@ -301,8 +304,8 @@ private:
const auto ext_data = detail::TensorExternalData(*m_tensor_proto); const auto ext_data = detail::TensorExternalData(*m_tensor_proto);
OPENVINO_SUPPRESS_DEPRECATED_START OPENVINO_SUPPRESS_DEPRECATED_START
std::shared_ptr<ngraph::runtime::AlignedBuffer> buffer = nullptr; std::shared_ptr<ngraph::runtime::AlignedBuffer> buffer = nullptr;
if (m_enable_mmap) { if (m_mmap_cache) {
buffer = ext_data.load_external_mmap_data(m_model_dir); buffer = ext_data.load_external_mmap_data(m_model_dir, m_mmap_cache);
} else { } else {
buffer = ext_data.load_external_data(m_model_dir); buffer = ext_data.load_external_data(m_model_dir);
} }
@ -351,7 +354,7 @@ private:
const ONNX_NAMESPACE::TensorProto* m_tensor_proto; const ONNX_NAMESPACE::TensorProto* m_tensor_proto;
Shape m_shape; Shape m_shape;
std::string m_model_dir; std::string m_model_dir;
bool m_enable_mmap = true; detail::MappedMemoryHandles m_mmap_cache;
}; };
inline std::ostream& operator<<(std::ostream& outs, const Tensor& tensor) { inline std::ostream& operator<<(std::ostream& outs, const Tensor& tensor) {

13
src/frontends/onnx/frontend/src/editor.cpp
View File

@ -319,7 +319,8 @@ onnx_editor::ONNXModelEditor::ONNXModelEditor(const std::string& model_path,
const bool enable_mmap, const bool enable_mmap,
frontend::ExtensionHolder extensions) frontend::ExtensionHolder extensions)
: m_model_path{model_path}, : m_model_path{model_path},
m_enable_mmap{enable_mmap}, m_mmap_cache{enable_mmap ? std::make_shared<std::map<std::string, std::shared_ptr<ov::MappedMemory>>>()
: nullptr},
m_extensions{std::move(extensions)}, m_extensions{std::move(extensions)},
m_pimpl{new ONNXModelEditor::Impl{model_path}, [](Impl* impl) { m_pimpl{new ONNXModelEditor::Impl{model_path}, [](Impl* impl) {
delete impl; delete impl;
@ -331,7 +332,8 @@ onnx_editor::ONNXModelEditor::ONNXModelEditor(const std::wstring& model_path,
frontend::ExtensionHolder extensions) frontend::ExtensionHolder extensions)
: m_extensions{std::move(extensions)}, : m_extensions{std::move(extensions)},
m_model_path{ov::util::wstring_to_string(model_path)}, m_model_path{ov::util::wstring_to_string(model_path)},
m_enable_mmap{enable_mmap}, m_mmap_cache{enable_mmap ? std::make_shared<std::map<std::string, std::shared_ptr<ov::MappedMemory>>>()
: nullptr},
m_pimpl{new ONNXModelEditor::Impl{model_path}, [](Impl* impl) { m_pimpl{new ONNXModelEditor::Impl{model_path}, [](Impl* impl) {
delete impl; delete impl;
}} {} }} {}
@ -342,7 +344,8 @@ onnx_editor::ONNXModelEditor::ONNXModelEditor(std::istream& model_stream,
const bool enable_mmap, const bool enable_mmap,
frontend::ExtensionHolder extensions) frontend::ExtensionHolder extensions)
: m_model_path{model_path}, : m_model_path{model_path},
m_enable_mmap{enable_mmap}, m_mmap_cache{enable_mmap ? std::make_shared<std::map<std::string, std::shared_ptr<ov::MappedMemory>>>()
: nullptr},
m_extensions{std::move(extensions)}, m_extensions{std::move(extensions)},
m_pimpl{new ONNXModelEditor::Impl{model_stream}, [](Impl* impl) { m_pimpl{new ONNXModelEditor::Impl{model_stream}, [](Impl* impl) {
delete impl; delete impl;
@ -533,7 +536,7 @@ std::shared_ptr<Model> onnx_editor::ONNXModelEditor::get_function() const {
OPENVINO_SUPPRESS_DEPRECATED_START OPENVINO_SUPPRESS_DEPRECATED_START
return ngraph::onnx_import::detail::import_onnx_model(m_pimpl->m_model_proto, return ngraph::onnx_import::detail::import_onnx_model(m_pimpl->m_model_proto,
m_model_path, m_model_path,
m_enable_mmap, m_mmap_cache,
m_extensions); m_extensions);
OPENVINO_SUPPRESS_DEPRECATED_END OPENVINO_SUPPRESS_DEPRECATED_END
} }
@ -728,7 +731,7 @@ std::vector<std::string> onnx_editor::ONNXModelEditor::get_output_ports(const Ed
std::shared_ptr<Model> onnx_editor::ONNXModelEditor::decode() { std::shared_ptr<Model> onnx_editor::ONNXModelEditor::decode() {
return ngraph::onnx_import::detail::decode_to_framework_nodes(m_pimpl->m_model_proto, return ngraph::onnx_import::detail::decode_to_framework_nodes(m_pimpl->m_model_proto,
m_model_path, m_model_path,
m_enable_mmap, m_mmap_cache,
m_extensions); m_extensions);
} }

3
src/frontends/onnx/frontend/src/editor.hpp
View File

@ -18,6 +18,7 @@
#include "openvino/frontend/extension/holder.hpp" #include "openvino/frontend/extension/holder.hpp"
#include "openvino/frontend/extension/progress_reporter.hpp" #include "openvino/frontend/extension/progress_reporter.hpp"
#include "openvino/frontend/extension/telemetry.hpp" #include "openvino/frontend/extension/telemetry.hpp"
#include "utils/tensor_external_data.hpp"
namespace ov { namespace ov {
namespace onnx_editor { namespace onnx_editor {
@ -305,7 +306,7 @@ private:
void update_mapper_if_needed() const; void update_mapper_if_needed() const;
const std::string m_model_path; const std::string m_model_path;
const bool m_enable_mmap; ngraph::onnx_import::detail::MappedMemoryHandles m_mmap_cache;
frontend::ExtensionHolder m_extensions; frontend::ExtensionHolder m_extensions;
struct Impl; struct Impl;

6
src/frontends/onnx/frontend/src/onnx.cpp
View File

@ -30,7 +30,11 @@ std::shared_ptr<Function> import_onnx_model(std::istream& stream,
ov::frontend::ExtensionHolder extensions; ov::frontend::ExtensionHolder extensions;
extensions.conversions.push_back(legacy_conversion_extension); extensions.conversions.push_back(legacy_conversion_extension);
OPENVINO_SUPPRESS_DEPRECATED_START OPENVINO_SUPPRESS_DEPRECATED_START
const auto model = detail::import_onnx_model(model_proto, model_path, enable_mmap, std::move(extensions)); const auto model = detail::import_onnx_model(
model_proto,
model_path,
enable_mmap ? std::make_shared<std::map<std::string, std::shared_ptr<ov::MappedMemory>>>() : nullptr,
std::move(extensions));
OPENVINO_SUPPRESS_DEPRECATED_END OPENVINO_SUPPRESS_DEPRECATED_END
const auto error_message = common::collect_translation_exceptions(model); const auto error_message = common::collect_translation_exceptions(model);
NGRAPH_CHECK(error_message.empty(), error_message); NGRAPH_CHECK(error_message.empty(), error_message);

8
src/frontends/onnx/frontend/src/utils/onnx_internal.cpp
View File

@ -91,13 +91,13 @@ void convert_decoded_function(std::shared_ptr<Function> function) {
std::shared_ptr<Function> import_onnx_model(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto, std::shared_ptr<Function> import_onnx_model(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto,
const std::string& model_path, const std::string& model_path,
const bool enable_mmap, detail::MappedMemoryHandles mmap_cache,
ov::frontend::ExtensionHolder extensions) { ov::frontend::ExtensionHolder extensions) {
apply_transformations(*model_proto); apply_transformations(*model_proto);
NGRAPH_SUPPRESS_DEPRECATED_START NGRAPH_SUPPRESS_DEPRECATED_START
Graph graph{file_util::get_directory(ov::util::get_absolute_file_path(model_path)), Graph graph{file_util::get_directory(ov::util::get_absolute_file_path(model_path)),
model_proto, model_proto,
enable_mmap, mmap_cache,
std::move(extensions)}; std::move(extensions)};
NGRAPH_SUPPRESS_DEPRECATED_END NGRAPH_SUPPRESS_DEPRECATED_END
return graph.convert(); return graph.convert();
@ -105,13 +105,13 @@ std::shared_ptr<Function> import_onnx_model(std::shared_ptr<ONNX_NAMESPACE::Mode
std::shared_ptr<Function> decode_to_framework_nodes(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto, std::shared_ptr<Function> decode_to_framework_nodes(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto,
const std::string& model_path, const std::string& model_path,
const bool enable_mmap, detail::MappedMemoryHandles mmap_cache,
ov::frontend::ExtensionHolder extensions) { ov::frontend::ExtensionHolder extensions) {
apply_transformations(*model_proto); apply_transformations(*model_proto);
NGRAPH_SUPPRESS_DEPRECATED_START NGRAPH_SUPPRESS_DEPRECATED_START
auto graph = std::make_shared<Graph>(file_util::get_directory(ov::util::get_absolute_file_path(model_path)), auto graph = std::make_shared<Graph>(file_util::get_directory(ov::util::get_absolute_file_path(model_path)),
model_proto, model_proto,
enable_mmap, mmap_cache,
extensions); extensions);
NGRAPH_SUPPRESS_DEPRECATED_END NGRAPH_SUPPRESS_DEPRECATED_END
return graph->decode(); return graph->decode();

5
src/frontends/onnx/frontend/src/utils/onnx_internal.hpp
View File

@ -10,6 +10,7 @@
#include "legacy_conversion_extension.hpp" #include "legacy_conversion_extension.hpp"
#include "ngraph/function.hpp" #include "ngraph/function.hpp"
#include "openvino/frontend/extension/holder.hpp" #include "openvino/frontend/extension/holder.hpp"
#include "utils/tensor_external_data.hpp"
namespace ONNX_NAMESPACE { namespace ONNX_NAMESPACE {
class ModelProto; class ModelProto;
@ -35,7 +36,7 @@ namespace detail {
/// graph. /// graph.
std::shared_ptr<Function> import_onnx_model(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto, std::shared_ptr<Function> import_onnx_model(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto,
const std::string& model_path, const std::string& model_path,
const bool enable_mmap, detail::MappedMemoryHandles mmap_cache,
ov::frontend::ExtensionHolder extensions = {}); ov::frontend::ExtensionHolder extensions = {});
/// \brief Decode ONNX model to nGraph function with ONNXFrameworkNode(s) /// \brief Decode ONNX model to nGraph function with ONNXFrameworkNode(s)
@ -48,7 +49,7 @@ std::shared_ptr<Function> import_onnx_model(std::shared_ptr<ONNX_NAMESPACE::Mode
/// \return A nGraph function with ONNXFrameworkNodes /// \return A nGraph function with ONNXFrameworkNodes
std::shared_ptr<Function> decode_to_framework_nodes(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto, std::shared_ptr<Function> decode_to_framework_nodes(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto,
const std::string& model_path, const std::string& model_path,
const bool enable_mmap, detail::MappedMemoryHandles mmap_cache,
ov::frontend::ExtensionHolder extensions = {}); ov::frontend::ExtensionHolder extensions = {});
/// \brief Converts a nGraph function (onnx model decoded to function with ONNXFrameworkNode(s)) /// \brief Converts a nGraph function (onnx model decoded to function with ONNXFrameworkNode(s))

19
src/frontends/onnx/frontend/src/utils/tensor_external_data.cpp
View File

@ -35,7 +35,8 @@ TensorExternalData::TensorExternalData(const ONNX_NAMESPACE::TensorProto& tensor
} }
} }
Buffer<ov::MappedMemory> TensorExternalData::load_external_mmap_data(const std::string& model_dir) const { Buffer<ov::MappedMemory> TensorExternalData::load_external_mmap_data(const std::string& model_dir,
MappedMemoryHandles cache) const {
NGRAPH_SUPPRESS_DEPRECATED_START NGRAPH_SUPPRESS_DEPRECATED_START
auto full_path = file_util::path_join(model_dir, m_data_location); auto full_path = file_util::path_join(model_dir, m_data_location);
NGRAPH_SUPPRESS_DEPRECATED_END NGRAPH_SUPPRESS_DEPRECATED_END
@ -43,13 +44,21 @@ Buffer<ov::MappedMemory> TensorExternalData::load_external_mmap_data(const std::
if (file_size <= 0 || m_offset + m_data_length > static_cast<uint64_t>(file_size)) { if (file_size <= 0 || m_offset + m_data_length > static_cast<uint64_t>(file_size)) {
throw error::invalid_external_data{*this}; throw error::invalid_external_data{*this};
} }
auto mapped_memory = ov::load_mmap_object(full_path, m_data_length, m_offset); auto cached_mapped_memory = cache->find(full_path);
std::shared_ptr<ov::MappedMemory> mapped_memory;
if (cached_mapped_memory != cache->end()) {
mapped_memory = cached_mapped_memory->second;
} else {
mapped_memory = ov::load_mmap_object(full_path);
(*cache)[full_path] = mapped_memory;
}
if (m_data_length > mapped_memory->size() || mapped_memory->size() == 0) { if (m_data_length > mapped_memory->size() || mapped_memory->size() == 0) {
throw error::invalid_external_data{*this}; throw error::invalid_external_data{*this};
} }
return std::make_shared<ngraph::runtime::SharedBuffer<std::shared_ptr<ov::MappedMemory>>>(mapped_memory->data(), return std::make_shared<ngraph::runtime::SharedBuffer<std::shared_ptr<ov::MappedMemory>>>(
mapped_memory->size(), mapped_memory->data() + m_offset,
mapped_memory); m_data_length > 0 ? m_data_length : static_cast<uint64_t>(file_size) - m_offset,
mapped_memory);
} }
Buffer<ngraph::runtime::AlignedBuffer> TensorExternalData::load_external_data(const std::string& model_dir) const { Buffer<ngraph::runtime::AlignedBuffer> TensorExternalData::load_external_data(const std::string& model_dir) const {

3
src/frontends/onnx/frontend/src/utils/tensor_external_data.hpp
View File

@ -15,6 +15,7 @@ namespace detail {
OPENVINO_SUPPRESS_DEPRECATED_START OPENVINO_SUPPRESS_DEPRECATED_START
template <class T> template <class T>
using Buffer = std::shared_ptr<ngraph::runtime::SharedBuffer<std::shared_ptr<T>>>; using Buffer = std::shared_ptr<ngraph::runtime::SharedBuffer<std::shared_ptr<T>>>;
using MappedMemoryHandles = std::shared_ptr<std::map<std::string, std::shared_ptr<ov::MappedMemory>>>;
/// \brief Helper class used to load tensor data from external files /// \brief Helper class used to load tensor data from external files
class TensorExternalData { class TensorExternalData {
public: public:
@ -36,7 +37,7 @@ public:
/// the invalid_external_data exception is thrown. /// the invalid_external_data exception is thrown.
/// ///
/// \return External binary data loaded into the SharedBuffer /// \return External binary data loaded into the SharedBuffer
Buffer<ov::MappedMemory> load_external_mmap_data(const std::string& model_dir) const; Buffer<ov::MappedMemory> load_external_mmap_data(const std::string& model_dir, MappedMemoryHandles cache) const;
/// \brief Represets parameter of external data as string /// \brief Represets parameter of external data as string
/// ///