IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[ONNX] Implemented cache for MMAP handles (#19130)

Browse Source 
* Implemented cache for MMAP handles

* Applied comments and fixed behavior when m_data_length==0

* Aligned data length calculation with no-MMAP implementation

* Removed unused code
This commit is contained in:
Georgy Krivoruchko 2023-08-14 04:56:59 -07:00 committed by GitHub
parent e5c4350d92
commit 46f428eeac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 114 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/common/util/include/openvino/util/mmap_object.hpp
View File

@ -32,13 +32,9 @@ public:
* in order to avoid time-consuming reading and reduce memory consumption.
*
* @param path Path to a file which memory will be mmaped.
* @param data_size Number of bytes to map (zero, if all file will be mmaped).
* @param offset Number of bytes to skip from beginning of the file.
* @return MappedMemory shared ptr object which keep mmaped memory and control the lifetime.
*/
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::string& path,
const size_t data_size = 0,
const size_t offset = 0);
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::string& path);
#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT
@ -48,13 +44,9 @@ std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::string& path,
* in order to avoid time-consuming reading and reduce memory consumption.
*
* @param path Path to a file which memory will be mmaped.
* @param data_size Number of bytes to map (zero, if all file will be mmaped).
* @param offset Number of bytes to skip from beginning of the file.
* @return MappedMemory shared ptr object which keep mmaped memory and control the lifetime.
*/
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::wstring& path,
const size_t data_size = 0,
const size_t offset = 0);
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::wstring& path);
#endif // OPENVINO_ENABLE_UNICODE_PATH_SUPPORT

25
src/common/util/src/os/lin/lin_mmap_object.cpp
View File

@ -63,7 +63,7 @@ class MapHolder : public MappedMemory {
public:
MapHolder() = default;
void set(const std::string& path, const size_t data_size = 0, const size_t offset = 0) {
void set(const std::string& path) {
int prot = PROT_READ;
int mode = O_RDONLY;
struct stat sb = {};
@ -75,24 +75,13 @@ public:
if (fstat(m_handle.get(), &sb) == -1) {
throw std::runtime_error("Can not get file size for " + path);
}
if (data_size + offset > static_cast<size_t>(sb.st_size)) {
throw std::runtime_error("Can not map out of scope memory for " + path);
}
m_size = data_size > 0 ? data_size : sb.st_size - offset;
const size_t page_size = sysconf(_SC_PAGE_SIZE);
// align offset with page size
const size_t aligned_offset = (offset / page_size) * page_size;
const size_t aligned_size = (offset % page_size) + m_size;
// move pointer to the expected data (after alignment with page size)
const size_t offset_delta = offset - aligned_offset;
if (aligned_size > 0) {
m_data = mmap(nullptr, aligned_size, prot, MAP_PRIVATE, m_handle.get(), aligned_offset);
m_size = sb.st_size;
if (m_size > 0) {
m_data = mmap(nullptr, m_size, prot, MAP_PRIVATE, m_handle.get(), 0);
if (m_data == MAP_FAILED) {
throw std::runtime_error("Can not create file mapping for " + path + ", err=" + std::strerror(errno));
}
m_data = reinterpret_cast<char*>(m_data) + offset_delta;
} else {
m_size = 0;
m_data = MAP_FAILED;
}
}
@ -112,11 +101,9 @@ public:
}
};
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::string& path,
const size_t data_size,
const size_t offset) {
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::string& path) {
auto holder = std::make_shared<MapHolder>();
holder->set(path, data_size, offset);
holder->set(path);
return holder;
}

60
src/common/util/src/os/win/win_mmap_object.cpp
View File

@ -2,10 +2,11 @@
// SPDX-License-Identifier: Apache-2.0
//
#include "openvino/util/mmap_object.hpp"
#include "openvino/util/file_util.hpp"
#include <stdexcept>
#include "openvino/util/file_util.hpp"
#include "openvino/util/mmap_object.hpp"
// clang-format-off
#ifndef NOMINMAX
# define NOMINMAX
@ -60,19 +61,19 @@ public:
}
}
void set(const std::string& path, const size_t data_size=0, const size_t offset=0) {
void set(const std::string& path) {
// Note that file can't be changed (renamed/deleted) until it's unmapped. FILE_SHARE_DELETE flag allow
// rename/deletion, but it doesn't work with FAT32 filesystem (works on NTFS)
auto h = ::CreateFileA(path.c_str(), GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
map(path, h, data_size, offset);
map(path, h);
}
#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT
void set(const std::wstring& path, const size_t data_size=0, const size_t offset=0) {
void set(const std::wstring& path) {
// Note that file can't be changed (renamed/deleted) until it's unmapped. FILE_SHARE_DELETE flag allow
// rename/deletion, but it doesn't work with FAT32 filesystem (works on NTFS)
auto h = ::CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
map(ov::util::wstring_to_string(path), h, data_size, offset);
map(ov::util::wstring_to_string(path), h);
}
#endif
@ -84,50 +85,37 @@ public:
}
private:
void map(const std::string& path, HANDLE h, const size_t data_size=0, const size_t offset=0) {
if(h == INVALID_HANDLE_VALUE) {
throw std::runtime_error("Can not open file " + path + " for mapping. Ensure that file exists and has appropriate permissions");
void map(const std::string& path, HANDLE h) {
if (h == INVALID_HANDLE_VALUE) {
throw std::runtime_error("Can not open file " + path +
" for mapping. Ensure that file exists and has appropriate permissions");
}
m_handle = HandleHolder(h);
SYSTEM_INFO sys_info;
GetSystemInfo(&sys_info);
DWORD map_mode = FILE_MAP_READ;
DWORD access = PAGE_READONLY;
LARGE_INTEGER file_size_large;
if(::GetFileSizeEx(m_handle.get(), &file_size_large) == 0) {
if (::GetFileSizeEx(m_handle.get(), &file_size_large) == 0) {
throw std::runtime_error("Can not get file size for " + path);
}
if (data_size + offset > static_cast<size_t>(file_size_large.QuadPart)) {
throw std::runtime_error("Can not map out of scope memory for " + path);
}
m_size = data_size > 0 ? data_size : static_cast<size_t>(file_size_large.QuadPart) - offset;
// the offset must be a multiple of the allocation granularity
const size_t alloc_gran = sys_info.dwAllocationGranularity;
const size_t map_view_start = (offset / alloc_gran) * alloc_gran;
const size_t map_view_size = (offset%alloc_gran) + m_size;
const size_t file_map_size = offset + m_size;
const size_t view_delta = offset - map_view_start;
m_size = static_cast<uint64_t>(file_size_large.QuadPart);
if (m_size > 0) {
m_mapping =
HandleHolder(::CreateFileMapping(m_handle.get(), 0, access, file_map_size >> 32, file_map_size & 0xffffffff, 0));
if(m_mapping.get() == INVALID_HANDLE_VALUE) {
HandleHolder(::CreateFileMapping(m_handle.get(), 0, access, m_size >> 32, m_size & 0xffffffff, 0));
if (m_mapping.get() == INVALID_HANDLE_VALUE) {
throw std::runtime_error("Can not create file mapping for " + path);
}
m_data = ::MapViewOfFile(m_mapping.get(),
map_mode,
map_view_start >> 32, // offset_align >> 32,
map_view_start & 0xffffffff, // offset_align & 0xffffffff,
map_view_size);
if(!m_data) {
0, // offset_align >> 32,
0, // offset_align & 0xffffffff,
m_size);
if (!m_data) {
throw std::runtime_error("Can not create map view for " + path);
}
// move pointer to the expected data (after alignment with allocation granularity)
m_data = reinterpret_cast<char*>(m_data) + view_delta;
} else {
m_data = nullptr;
}
@ -140,17 +128,17 @@ private:
HandleHolder m_mapping;
};
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::string& path, const size_t data_size, const size_t offset) {
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::string& path) {
auto holder = std::make_shared<MapHolder>();
holder->set(path, data_size, offset);
holder->set(path);
return holder;
}
#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::wstring& path, const size_t data_size, const size_t offset) {
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::wstring& path) {
auto holder = std::make_shared<MapHolder>();
holder->set(path, data_size, offset);
holder->set(path);
return holder;
}

24
src/frontends/onnx/frontend/src/core/attribute.hpp
View File

@ -200,12 +200,12 @@ public:
};
Attribute() = delete;
explicit Attribute(const ONNX_NAMESPACE::AttributeProto& attribute_proto,
Attribute(const ONNX_NAMESPACE::AttributeProto& attribute_proto,
const std::string& model_dir,
const bool enable_mmap)
detail::MappedMemoryHandles mmap_cache)
: m_attribute_proto{&attribute_proto},
m_model_dir{model_dir},
m_enable_mmap{enable_mmap} {}
m_mmap_cache{mmap_cache} {}
Attribute(Attribute&&) noexcept = default;
Attribute(const Attribute&) = default;
@ -256,10 +256,10 @@ public:
return get_type() == Type::graph_array;
}
Tensor get_tensor() const {
return Tensor{m_attribute_proto->t(), m_model_dir, m_enable_mmap};
return Tensor{m_attribute_proto->t(), m_model_dir, m_mmap_cache};
}
SparseTensor get_sparse_tensor() const {
return SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir, m_enable_mmap};
return SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir, m_mmap_cache};
}
float get_float() const {
return m_attribute_proto->f();
@ -277,7 +277,7 @@ public:
const auto& tensors = m_attribute_proto->tensors();
ret.reserve(tensors.size());
for (const auto& tensor : tensors)
ret.emplace_back(tensor, m_model_dir, m_enable_mmap);
ret.emplace_back(tensor, m_model_dir, m_mmap_cache);
return ret;
}
@ -286,7 +286,7 @@ public:
const auto& sparse_tensors = m_attribute_proto->sparse_tensors();
ret.reserve(sparse_tensors.size());
for (const auto& tensor : sparse_tensors)
ret.emplace_back(tensor, m_model_dir, m_enable_mmap);
ret.emplace_back(tensor, m_model_dir, m_mmap_cache);
return ret;
}
@ -318,7 +318,7 @@ public:
template <typename T, typename std::enable_if<std::is_same<T, Tensor>::value, bool>::type = true>
T get_value() const {
if (is_tensor()) {
return Tensor{m_attribute_proto->t(), m_model_dir, m_enable_mmap};
return Tensor{m_attribute_proto->t(), m_model_dir, m_mmap_cache};
}
throw error::attribute::InvalidData{m_attribute_proto->type()};
}
@ -326,7 +326,7 @@ public:
template <typename T, typename std::enable_if<std::is_same<T, std::vector<Tensor>>::value, bool>::type = true>
T get_value() const {
if (is_tensor()) {
return {Tensor{m_attribute_proto->t(), m_model_dir, m_enable_mmap}};
return {Tensor{m_attribute_proto->t(), m_model_dir, m_mmap_cache}};
} else if (is_tensor_array()) {
return get_tensor_array();
}
@ -336,7 +336,7 @@ public:
template <typename T, typename std::enable_if<std::is_same<T, SparseTensor>::value, bool>::type = true>
T get_value() const {
if (is_sparse_tensor()) {
return SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir, m_enable_mmap};
return SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir, m_mmap_cache};
}
throw error::attribute::InvalidData{m_attribute_proto->type()};
}
@ -344,7 +344,7 @@ public:
template <typename T, typename std::enable_if<std::is_same<T, std::vector<SparseTensor>>::value, bool>::type = true>
T get_value() const {
if (is_sparse_tensor()) {
return {SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir, m_enable_mmap}};
return {SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir, m_mmap_cache}};
} else if (is_sparse_tensor_array()) {
return get_sparse_tensor_array();
}
@ -356,7 +356,7 @@ public:
private:
const ONNX_NAMESPACE::AttributeProto* m_attribute_proto;
std::string m_model_dir;
const bool m_enable_mmap;
detail::MappedMemoryHandles m_mmap_cache;
};
} // namespace onnx_import

12
src/frontends/onnx/frontend/src/core/graph.cpp
View File

@ -123,19 +123,19 @@ ov::frontend::ExtensionHolder subgraph_required_extensions(
Graph::Graph(const std::string& model_dir,
const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model_proto,
const bool enable_mmap,
detail::MappedMemoryHandles mmap_cache,
ov::frontend::ExtensionHolder extensions)
: Graph(model_dir, model_proto, common::make_unique<GraphCache>(), enable_mmap, std::move(extensions)) {}
: Graph(model_dir, model_proto, common::make_unique<GraphCache>(), mmap_cache, std::move(extensions)) {}
Graph::Graph(const std::string& model_dir,
const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model_proto,
std::unique_ptr<GraphCache>&& cache,
const bool enable_mmap,
detail::MappedMemoryHandles mmap_cache,
ov::frontend::ExtensionHolder extensions)
: m_cache{std::move(cache)},
m_extensions{std::move(extensions)},
m_model_dir{model_dir},
m_enable_mmap{enable_mmap},
m_mmap_cache{mmap_cache},
m_ops_bridge{detail::init_ops_bridge(m_extensions.conversions)} {
m_model = common::make_unique<Model>(model_proto, detail::build_model_opset(*model_proto, m_ops_bridge));
@ -146,7 +146,7 @@ Graph::Graph(const std::string& model_dir,
// Process all initializers in the graph
for (const auto& initializer_tensor : m_model->get_graph().initializer()) {
if (initializer_tensor.has_name()) {
Tensor tensor = Tensor{initializer_tensor, m_model_dir, enable_mmap};
Tensor tensor = Tensor{initializer_tensor, m_model_dir, m_mmap_cache};
std::shared_ptr<default_opset::Constant> ng_constant;
// For each initializer create a Constant node and store it in cache
try {
@ -458,7 +458,7 @@ Subgraph::Subgraph(const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model_prot
: Graph(parent_graph->model_dir(),
model_proto,
common::make_unique<GraphCache>(),
parent_graph->mmap_enabled(),
parent_graph->get_mmap_cache(),
detail::subgraph_required_extensions(parent_graph->get_extensions())),
m_parent_graph(parent_graph) {}

11
src/frontends/onnx/frontend/src/core/graph.hpp
View File

@ -18,6 +18,7 @@
#include "openvino/core/deprecated.hpp"
#include "openvino/frontend/extension/holder.hpp"
#include "ops_bridge.hpp"
#include "utils/tensor_external_data.hpp"
namespace ngraph {
namespace onnx_import {
@ -25,7 +26,7 @@ class Graph : public std::enable_shared_from_this<Graph> {
public:
Graph(const std::string& model_dir,
const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model_proto,
const bool enable_mmap,
detail::MappedMemoryHandles mmap_cache,
ov::frontend::ExtensionHolder extensions = {});
Graph() = delete;
@ -43,8 +44,8 @@ public:
const std::string& model_dir() const {
return m_model_dir;
}
bool mmap_enabled() const {
return m_enable_mmap;
detail::MappedMemoryHandles get_mmap_cache() const {
return m_mmap_cache;
}
const ParameterVector& get_ng_parameters() const {
return m_parameters;
@ -65,7 +66,7 @@ protected:
Graph(const std::string& model_dir,
const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model,
std::unique_ptr<GraphCache>&& cache,
const bool enable_mmap,
detail::MappedMemoryHandles mmap_cache,
ov::frontend::ExtensionHolder extensions = {});
OPENVINO_SUPPRESS_DEPRECATED_START
@ -92,7 +93,7 @@ private:
std::vector<Node> m_nodes;
OPENVINO_SUPPRESS_DEPRECATED_END
std::string m_model_dir;
bool m_enable_mmap;
detail::MappedMemoryHandles m_mmap_cache;
OperatorsBridge m_ops_bridge;
};

4
src/frontends/onnx/frontend/src/core/node.cpp
View File

@ -26,7 +26,7 @@ public:
const auto& attributes = node_proto.attribute();
m_attributes.reserve(attributes.size());
for (const auto& attr_proto : attributes) {
m_attributes.emplace_back(attr_proto, m_graph->model_dir(), m_graph->mmap_enabled());
m_attributes.emplace_back(attr_proto, m_graph->model_dir(), m_graph->get_mmap_cache());
const auto& attribute = m_attributes.back();
if (attribute.is_graph())
m_subgraphs.insert({attribute.get_name(), std::make_shared<Subgraph>(attribute.get_subgraph(m_graph))});
@ -43,7 +43,7 @@ public:
m_output_names{std::begin(node_proto.output()), std::end(node_proto.output())},
m_subgraphs(subgraphs) {
for (const auto& attr_proto : node_proto.attribute()) {
m_attributes.emplace_back(attr_proto, m_graph->model_dir(), m_graph->mmap_enabled());
m_attributes.emplace_back(attr_proto, m_graph->model_dir(), m_graph->get_mmap_cache());
}
}

6
src/frontends/onnx/frontend/src/core/sparse_tensor.hpp
View File

@ -19,9 +19,9 @@ public:
SparseTensor() = delete;
SparseTensor(const ONNX_NAMESPACE::SparseTensorProto& sparse_tensor,
const std::string& model_dir,
const bool enable_mmap)
: m_values{sparse_tensor.values(), model_dir, enable_mmap},
m_indices{sparse_tensor.indices(), model_dir, enable_mmap},
detail::MappedMemoryHandles mmap_cache)
: m_values{sparse_tensor.values(), model_dir, mmap_cache},
m_indices{sparse_tensor.indices(), model_dir, mmap_cache},
m_shape{std::begin(sparse_tensor.dims()), std::end(sparse_tensor.dims())} {
if (m_shape == Shape{0}) {
// It's possible to construct a sparse tensor in ONNX with "dims: 0" property

19
src/frontends/onnx/frontend/src/core/tensor.hpp
View File

@ -108,11 +108,13 @@ public:
};
Tensor() = delete;
explicit Tensor(const ONNX_NAMESPACE::TensorProto& tensor, const std::string& model_dir, const bool enable_mmap)
Tensor(const ONNX_NAMESPACE::TensorProto& tensor,
const std::string& model_dir,
detail::MappedMemoryHandles mmap_cache)
: m_tensor_proto{&tensor},
m_shape{std::begin(tensor.dims()), std::end(tensor.dims())},
m_model_dir{model_dir},
m_enable_mmap{enable_mmap} {
m_mmap_cache{mmap_cache} {
if (m_shape == Shape{0}) {
// It's possible to construct a tensor in ONNX with "dims: 0" property
// Such tensor contains a scalar. This results in a Shape{0} stored in m_shape.
@ -241,10 +243,11 @@ private:
size_t data_size = get_data_size();
if (has_external_data()) {
const auto ext_data = detail::TensorExternalData(*m_tensor_proto);
if (m_enable_mmap) {
constant = std::make_shared<ngraph::op::Constant>(type,
if (m_mmap_cache) {
constant =
std::make_shared<ngraph::op::Constant>(type,
m_shape,
ext_data.load_external_mmap_data(m_model_dir));
ext_data.load_external_mmap_data(m_model_dir, m_mmap_cache));
} else {
constant =
std::make_shared<ngraph::op::Constant>(type, m_shape, ext_data.load_external_data(m_model_dir));
@ -301,8 +304,8 @@ private:
const auto ext_data = detail::TensorExternalData(*m_tensor_proto);
OPENVINO_SUPPRESS_DEPRECATED_START
std::shared_ptr<ngraph::runtime::AlignedBuffer> buffer = nullptr;
if (m_enable_mmap) {
buffer = ext_data.load_external_mmap_data(m_model_dir);
if (m_mmap_cache) {
buffer = ext_data.load_external_mmap_data(m_model_dir, m_mmap_cache);
} else {
buffer = ext_data.load_external_data(m_model_dir);
}
@ -351,7 +354,7 @@ private:
const ONNX_NAMESPACE::TensorProto* m_tensor_proto;
Shape m_shape;
std::string m_model_dir;
bool m_enable_mmap = true;
detail::MappedMemoryHandles m_mmap_cache;
};
inline std::ostream& operator<<(std::ostream& outs, const Tensor& tensor) {

13
src/frontends/onnx/frontend/src/editor.cpp
View File

@ -319,7 +319,8 @@ onnx_editor::ONNXModelEditor::ONNXModelEditor(const std::string& model_path,
const bool enable_mmap,
frontend::ExtensionHolder extensions)
: m_model_path{model_path},
m_enable_mmap{enable_mmap},
m_mmap_cache{enable_mmap ? std::make_shared<std::map<std::string, std::shared_ptr<ov::MappedMemory>>>()
: nullptr},
m_extensions{std::move(extensions)},
m_pimpl{new ONNXModelEditor::Impl{model_path}, [](Impl* impl) {
delete impl;
@ -331,7 +332,8 @@ onnx_editor::ONNXModelEditor::ONNXModelEditor(const std::wstring& model_path,
frontend::ExtensionHolder extensions)
: m_extensions{std::move(extensions)},
m_model_path{ov::util::wstring_to_string(model_path)},
m_enable_mmap{enable_mmap},
m_mmap_cache{enable_mmap ? std::make_shared<std::map<std::string, std::shared_ptr<ov::MappedMemory>>>()
: nullptr},
m_pimpl{new ONNXModelEditor::Impl{model_path}, [](Impl* impl) {
delete impl;
}} {}
@ -342,7 +344,8 @@ onnx_editor::ONNXModelEditor::ONNXModelEditor(std::istream& model_stream,
const bool enable_mmap,
frontend::ExtensionHolder extensions)
: m_model_path{model_path},
m_enable_mmap{enable_mmap},
m_mmap_cache{enable_mmap ? std::make_shared<std::map<std::string, std::shared_ptr<ov::MappedMemory>>>()
: nullptr},
m_extensions{std::move(extensions)},
m_pimpl{new ONNXModelEditor::Impl{model_stream}, [](Impl* impl) {
delete impl;
@ -533,7 +536,7 @@ std::shared_ptr<Model> onnx_editor::ONNXModelEditor::get_function() const {
OPENVINO_SUPPRESS_DEPRECATED_START
return ngraph::onnx_import::detail::import_onnx_model(m_pimpl->m_model_proto,
m_model_path,
m_enable_mmap,
m_mmap_cache,
m_extensions);
OPENVINO_SUPPRESS_DEPRECATED_END
}
@ -728,7 +731,7 @@ std::vector<std::string> onnx_editor::ONNXModelEditor::get_output_ports(const Ed
std::shared_ptr<Model> onnx_editor::ONNXModelEditor::decode() {
return ngraph::onnx_import::detail::decode_to_framework_nodes(m_pimpl->m_model_proto,
m_model_path,
m_enable_mmap,
m_mmap_cache,
m_extensions);
}

3
src/frontends/onnx/frontend/src/editor.hpp
View File

@ -18,6 +18,7 @@
#include "openvino/frontend/extension/holder.hpp"
#include "openvino/frontend/extension/progress_reporter.hpp"
#include "openvino/frontend/extension/telemetry.hpp"
#include "utils/tensor_external_data.hpp"
namespace ov {
namespace onnx_editor {
@ -305,7 +306,7 @@ private:
void update_mapper_if_needed() const;
const std::string m_model_path;
const bool m_enable_mmap;
ngraph::onnx_import::detail::MappedMemoryHandles m_mmap_cache;
frontend::ExtensionHolder m_extensions;
struct Impl;

6
src/frontends/onnx/frontend/src/onnx.cpp
View File

@ -30,7 +30,11 @@ std::shared_ptr<Function> import_onnx_model(std::istream& stream,
ov::frontend::ExtensionHolder extensions;
extensions.conversions.push_back(legacy_conversion_extension);
OPENVINO_SUPPRESS_DEPRECATED_START
const auto model = detail::import_onnx_model(model_proto, model_path, enable_mmap, std::move(extensions));
const auto model = detail::import_onnx_model(
model_proto,
model_path,
enable_mmap ? std::make_shared<std::map<std::string, std::shared_ptr<ov::MappedMemory>>>() : nullptr,
std::move(extensions));
OPENVINO_SUPPRESS_DEPRECATED_END
const auto error_message = common::collect_translation_exceptions(model);
NGRAPH_CHECK(error_message.empty(), error_message);

8
src/frontends/onnx/frontend/src/utils/onnx_internal.cpp
View File

@ -91,13 +91,13 @@ void convert_decoded_function(std::shared_ptr<Function> function) {
std::shared_ptr<Function> import_onnx_model(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto,
const std::string& model_path,
const bool enable_mmap,
detail::MappedMemoryHandles mmap_cache,
ov::frontend::ExtensionHolder extensions) {
apply_transformations(*model_proto);
NGRAPH_SUPPRESS_DEPRECATED_START
Graph graph{file_util::get_directory(ov::util::get_absolute_file_path(model_path)),
model_proto,
enable_mmap,
mmap_cache,
std::move(extensions)};
NGRAPH_SUPPRESS_DEPRECATED_END
return graph.convert();
@ -105,13 +105,13 @@ std::shared_ptr<Function> import_onnx_model(std::shared_ptr<ONNX_NAMESPACE::Mode
std::shared_ptr<Function> decode_to_framework_nodes(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto,
const std::string& model_path,
const bool enable_mmap,
detail::MappedMemoryHandles mmap_cache,
ov::frontend::ExtensionHolder extensions) {
apply_transformations(*model_proto);
NGRAPH_SUPPRESS_DEPRECATED_START
auto graph = std::make_shared<Graph>(file_util::get_directory(ov::util::get_absolute_file_path(model_path)),
model_proto,
enable_mmap,
mmap_cache,
extensions);
NGRAPH_SUPPRESS_DEPRECATED_END
return graph->decode();

5
src/frontends/onnx/frontend/src/utils/onnx_internal.hpp
View File

@ -10,6 +10,7 @@
#include "legacy_conversion_extension.hpp"
#include "ngraph/function.hpp"
#include "openvino/frontend/extension/holder.hpp"
#include "utils/tensor_external_data.hpp"
namespace ONNX_NAMESPACE {
class ModelProto;
@ -35,7 +36,7 @@ namespace detail {
/// graph.
std::shared_ptr<Function> import_onnx_model(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto,
const std::string& model_path,
const bool enable_mmap,
detail::MappedMemoryHandles mmap_cache,
ov::frontend::ExtensionHolder extensions = {});
/// \brief Decode ONNX model to nGraph function with ONNXFrameworkNode(s)
@ -48,7 +49,7 @@ std::shared_ptr<Function> import_onnx_model(std::shared_ptr<ONNX_NAMESPACE::Mode
/// \return A nGraph function with ONNXFrameworkNodes
std::shared_ptr<Function> decode_to_framework_nodes(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto,
const std::string& model_path,
const bool enable_mmap,
detail::MappedMemoryHandles mmap_cache,
ov::frontend::ExtensionHolder extensions = {});
/// \brief Converts a nGraph function (onnx model decoded to function with ONNXFrameworkNode(s))

17
src/frontends/onnx/frontend/src/utils/tensor_external_data.cpp
View File

@ -35,7 +35,8 @@ TensorExternalData::TensorExternalData(const ONNX_NAMESPACE::TensorProto& tensor
}
}
Buffer<ov::MappedMemory> TensorExternalData::load_external_mmap_data(const std::string& model_dir) const {
Buffer<ov::MappedMemory> TensorExternalData::load_external_mmap_data(const std::string& model_dir,
MappedMemoryHandles cache) const {
NGRAPH_SUPPRESS_DEPRECATED_START
auto full_path = file_util::path_join(model_dir, m_data_location);
NGRAPH_SUPPRESS_DEPRECATED_END
@ -43,12 +44,20 @@ Buffer<ov::MappedMemory> TensorExternalData::load_external_mmap_data(const std::
if (file_size <= 0 || m_offset + m_data_length > static_cast<uint64_t>(file_size)) {
throw error::invalid_external_data{*this};
}
auto mapped_memory = ov::load_mmap_object(full_path, m_data_length, m_offset);
auto cached_mapped_memory = cache->find(full_path);
std::shared_ptr<ov::MappedMemory> mapped_memory;
if (cached_mapped_memory != cache->end()) {
mapped_memory = cached_mapped_memory->second;
} else {
mapped_memory = ov::load_mmap_object(full_path);
(*cache)[full_path] = mapped_memory;
}
if (m_data_length > mapped_memory->size() || mapped_memory->size() == 0) {
throw error::invalid_external_data{*this};
}
return std::make_shared<ngraph::runtime::SharedBuffer<std::shared_ptr<ov::MappedMemory>>>(mapped_memory->data(),
mapped_memory->size(),
return std::make_shared<ngraph::runtime::SharedBuffer<std::shared_ptr<ov::MappedMemory>>>(
mapped_memory->data() + m_offset,
m_data_length > 0 ? m_data_length : static_cast<uint64_t>(file_size) - m_offset,
mapped_memory);
}

3
src/frontends/onnx/frontend/src/utils/tensor_external_data.hpp
View File

@ -15,6 +15,7 @@ namespace detail {
OPENVINO_SUPPRESS_DEPRECATED_START
template <class T>
using Buffer = std::shared_ptr<ngraph::runtime::SharedBuffer<std::shared_ptr<T>>>;
using MappedMemoryHandles = std::shared_ptr<std::map<std::string, std::shared_ptr<ov::MappedMemory>>>;
/// \brief Helper class used to load tensor data from external files
class TensorExternalData {
public:
@ -36,7 +37,7 @@ public:
/// the invalid_external_data exception is thrown.
///
/// \return External binary data loaded into the SharedBuffer
Buffer<ov::MappedMemory> load_external_mmap_data(const std::string& model_dir) const;
Buffer<ov::MappedMemory> load_external_mmap_data(const std::string& model_dir, MappedMemoryHandles cache) const;
/// \brief Represets parameter of external data as string
///