IntenseWebs/openvino
3
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implementation of MMAP for ONNX FE (#18026)

Browse Source 
* Implementation of MMAP for ONNX FE

* fix win offsets

* added virtual dtor to MappedMemory

* review remarks. part.1

* added disable mmap flag to MO

* added additional checks to mmap

* remove unnecessary const

* fix pybind default value

* Added args.disable_mmap = False to MO tests

* fixed MO test

* avoid global headers

* fix casting for win

* disable mmap for legacy frontends flow

* review remarks

* Fixed passing parameters

* added doc to MappedMemory and load_mmap_object
This commit is contained in:
Mateusz Bencer authored and GitHub committed 2023-07-13 19:17:58 +00:00
1 parent 96d16b141d
commit c59746fd27
33 files changed
+555 -384
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cmake/developer_package/frontends/frontends.cmake
-11
View File
@@ -112,17 +112,6 @@ macro(ov_add_frontend)
endif()
file(GLOB_RECURSE LIBRARY_SRC ${frontend_root_dir}/src/*.cpp)
if (WIN32)
# Remove linux specific files
file(GLOB_RECURSE LIN_FILES ${frontend_root_dir}/src/os/lin/*.cpp
${frontend_root_dir}/src/os/lin/*.hpp)
list(REMOVE_ITEM LIBRARY_SRC "${LIN_FILES}")
else()
# Remove windows specific files
file(GLOB_RECURSE WIN_FILES ${frontend_root_dir}/src/os/win/*.cpp
${frontend_root_dir}/src/os/win/*.hpp)
list(REMOVE_ITEM LIBRARY_SRC "${WIN_FILES}")
endif()
file(GLOB_RECURSE LIBRARY_HEADERS ${frontend_root_dir}/src/*.hpp)
file(GLOB_RECURSE LIBRARY_PUBLIC_HEADERS ${frontend_root_dir}/include/*.hpp)
src/bindings/python/src/pyopenvino/frontend/frontend.cpp
+7 -4
View File
@@ -37,29 +37,32 @@ void regclass_frontend_FrontEnd(py::module m) {
fem.def(
"load",
[](FrontEnd& self, const py::object& py_obj) {
[](FrontEnd& self, const py::object& py_obj, const bool enable_mmap = true) {
if (py::isinstance(py_obj, py::module_::import("pathlib").attr("Path")) ||
py::isinstance<py::str>(py_obj) || py::isinstance<py::bytes>(py_obj)) {
// check if model path is either a string/pathlib.Path/bytes
std::string model_path = Common::utils::convert_path_to_string(py_obj);
return self.load(model_path);
return self.load(model_path, enable_mmap);
} else if (py::isinstance(py_obj, pybind11::module::import("io").attr("BytesIO"))) {
// support of BytesIO
py::buffer_info info = py::buffer(py_obj.attr("getbuffer")()).request();
MemoryBuffer mb(reinterpret_cast<char*>(info.ptr), info.size);
std::istream _istream(&mb);
return self.load(&_istream);
return self.load(&_istream, enable_mmap);
} else {
// Extended for one argument only for this time
return self.load({Common::utils::py_object_to_any(py_obj)});
return self.load({Common::utils::py_object_to_any(py_obj), enable_mmap});
}
},
py::arg("path"),
py::arg("enable_mmap") = true,
R"(
Loads an input model.
:param path: Object describing the model. It can be path to model file.
:type path: Any
:param enable_mmap: Use mmap feature to map memory of a model's weights instead of reading directly. Optional. The default value is true.
:type enable_mmap: boolean
:return: Loaded input model.
:rtype: openvino.frontend.InputModel
)");
src/common/util/include/openvino/util/mmap_object.hpp
+61
View File
@@ -0,0 +1,61 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
/**
* @brief A header file for definition of abstraction over platform specific shared memory map objects
* @file mmap_object.hpp
*/
#pragma once
#include <memory>
#include <string>
namespace ov {
/**
* @brief This class represents a mapped memory.
* Instead of reading files, we can map the memory via mmap for Linux or MapViewOfFile for Windows.
* The MappedMemory class is a abstraction to handle such memory with os-dependent details.
*/
class MappedMemory {
public:
virtual char* data() noexcept = 0;
virtual size_t size() const noexcept = 0;
virtual ~MappedMemory() = default;
};
/**
* @brief Returns mapped memory for a file from provided path.
* Instead of reading files, we can map the memory via mmap for Linux
* in order to avoid time-consuming reading and reduce memory consumption.
*
* @param path Path to a file which memory will be mmaped.
* @param data_size Number of bytes to map (zero, if all file will be mmaped).
* @param offset Number of bytes to skip from beginning of the file.
* @return MappedMemory shared ptr object which keep mmaped memory and control the lifetime.
*/
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::string& path,
const size_t data_size = 0,
const size_t offset = 0);
#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT
/**
* @brief Returns mapped memory for a file from provided path.
* Instead of reading files, we can map the memory via MapViewOfFile for Windows
* in order to avoid time-consuming reading and reduce memory consumption.
*
* @param path Path to a file which memory will be mmaped.
* @param data_size Number of bytes to map (zero, if all file will be mmaped).
* @param offset Number of bytes to skip from beginning of the file.
* @return MappedMemory shared ptr object which keep mmaped memory and control the lifetime.
*/
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::wstring& path,
const size_t data_size = 0,
const size_t offset = 0);
#endif // OPENVINO_ENABLE_UNICODE_PATH_SUPPORT
} // namespace ov
src/frontends/ir/src/os/lin/lin_mmap_object.cpp → src/common/util/src/os/lin/lin_mmap_object.cpp
+27 -19
View File
@@ -8,12 +8,12 @@
#include <sys/stat.h>
#include <unistd.h>
#include <cstring>
#include <iostream>
#include <sstream>
#include "mmap_object.hpp"
#include "ngraph/runtime/shared_buffer.hpp"
#include "openvino/util/file_util.hpp"
#include "openvino/util/mmap_object.hpp"
namespace ov {
@@ -55,7 +55,7 @@ public:
}
};
class MapHolder {
class MapHolder : public MappedMemory {
void* m_data = MAP_FAILED;
size_t m_size = 0;
HandleHolder m_handle;
@@ -63,21 +63,29 @@ class MapHolder {
public:
MapHolder() = default;
void set(const std::string& path) {
void set(const std::string& path, const size_t data_size = 0, const size_t offset = 0) {
int prot = PROT_READ;
int mode = O_RDONLY;
struct stat sb = {};
m_handle = HandleHolder(open(path.c_str(), mode));
OPENVINO_ASSERT(m_handle.get() != -1,
"Can not open file ",
path,
" for mapping. Ensure that file exists and has appropriate permissions");
OPENVINO_ASSERT(fstat(m_handle.get(), &sb) != -1, "Can not get file size for ", path);
m_size = sb.st_size;
if (m_handle.get() == -1) {
throw std::runtime_error("Can not open file " + path +
" for mapping. Ensure that file exists and has appropriate permissions");
}
if (fstat(m_handle.get(), &sb) == -1) {
throw std::runtime_error("Can not get file size for " + path);
}
if (data_size + offset > static_cast<size_t>(sb.st_size)) {
throw std::runtime_error("Can not map out of scope memory for " + path);
}
m_size = data_size > 0 ? data_size : sb.st_size - offset;
if (m_size > 0) {
m_data = mmap(nullptr, m_size, prot, MAP_PRIVATE, m_handle.get(), 0);
OPENVINO_ASSERT(m_data != MAP_FAILED, "Can not create file mapping for ", path, ", err=", strerror(errno));
m_data = mmap(nullptr, m_size, prot, MAP_PRIVATE, m_handle.get(), offset);
if (m_data == MAP_FAILED) {
throw std::runtime_error("Can not create file mapping for " + path + ", err=" + std::strerror(errno));
}
} else {
m_size = 0;
m_data = MAP_FAILED;
}
}
@@ -88,21 +96,21 @@ public:
}
}
char* data() noexcept {
char* data() noexcept override {
return static_cast<char*>(m_data);
}
size_t size() const noexcept {
size_t size() const noexcept override {
return m_size;
}
};
std::shared_ptr<ngraph::runtime::AlignedBuffer> load_mmap_object(const std::string& path) {
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::string& path,
const size_t data_size,
const size_t offset) {
auto holder = std::make_shared<MapHolder>();
holder->set(path);
return std::make_shared<ngraph::runtime::SharedBuffer<std::shared_ptr<MapHolder>>>(holder->data(),
holder->size(),
holder);
holder->set(path, data_size, offset);
return holder;
}
} // namespace ov
src/frontends/ir/src/os/win/win_mmap_object.cpp → src/common/util/src/os/win/win_mmap_object.cpp
+47 -34
View File
@@ -2,10 +2,10 @@
// SPDX-License-Identifier: Apache-2.0
//
#include "mmap_object.hpp"
#include "ngraph/runtime/shared_buffer.hpp"
#include "openvino/util/mmap_object.hpp"
#include "openvino/util/file_util.hpp"
#include <stdexcept>
// clang-format-off
#ifndef NOMINMAX
# define NOMINMAX
@@ -50,7 +50,7 @@ public:
}
};
class MapHolder {
class MapHolder : public ov::MappedMemory {
public:
MapHolder() = default;
@@ -60,57 +60,74 @@ public:
}
}
void set(const std::string& path) {
void set(const std::string& path, const size_t data_size=0, const size_t offset=0) {
// Note that file can't be changed (renamed/deleted) until it's unmapped. FILE_SHARE_DELETE flag allow
// rename/deletion, but it doesn't work with FAT32 filesystem (works on NTFS)
auto h = ::CreateFileA(path.c_str(), GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
map(path, h);
map(path, h, data_size, offset);
}
#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT
void set(const std::wstring& path) {
void set(const std::wstring& path, const size_t data_size=0, const size_t offset=0) {
// Note that file can't be changed (renamed/deleted) until it's unmapped. FILE_SHARE_DELETE flag allow
// rename/deletion, but it doesn't work with FAT32 filesystem (works on NTFS)
auto h = ::CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
map(ov::util::wstring_to_string(path), h);
map(ov::util::wstring_to_string(path), h, data_size, offset);
}
#endif
char* data() noexcept {
char* data() noexcept override {
return static_cast<char*>(m_data);
}
size_t size() const noexcept {
size_t size() const noexcept override {
return m_size;
}
private:
void map(const std::string& path, HANDLE h) {
OPENVINO_ASSERT(h != INVALID_HANDLE_VALUE,
"Can not open file ",
path,
" for mapping. Ensure that file exists and has appropriate permissions");
void map(const std::string& path, HANDLE h, const size_t data_size=0, const size_t offset=0) {
if(h == INVALID_HANDLE_VALUE) {
throw std::runtime_error("Can not open file " + path + " for mapping. Ensure that file exists and has appropriate permissions");
}
m_handle = HandleHolder(h);
SYSTEM_INFO SystemInfo;
GetSystemInfo(&SystemInfo);
SYSTEM_INFO sys_info;
GetSystemInfo(&sys_info);
DWORD map_mode = FILE_MAP_READ;
DWORD access = PAGE_READONLY;
LARGE_INTEGER file_size_large;
OPENVINO_ASSERT(::GetFileSizeEx(m_handle.get(), &file_size_large) != 0, "Can not get file size for ", path);
if(::GetFileSizeEx(m_handle.get(), &file_size_large) == 0) {
throw std::runtime_error("Can not get file size for " + path);
}
if (data_size + offset > static_cast<size_t>(file_size_large.QuadPart)) {
throw std::runtime_error("Can not map out of scope memory for " + path);
}
m_size = data_size > 0 ? data_size : static_cast<size_t>(file_size_large.QuadPart) - offset;
// the offset must be a multiple of the allocation granularity
const size_t alloc_gran = sys_info.dwAllocationGranularity;
const size_t map_view_start = (offset / alloc_gran) * alloc_gran;
const size_t map_view_size = (offset%alloc_gran) + m_size;
const size_t file_map_size = offset + m_size;
const size_t view_delta = offset - map_view_start;
m_size = static_cast<uint64_t>(file_size_large.QuadPart);
if (m_size > 0) {
m_mapping =
HandleHolder(::CreateFileMapping(m_handle.get(), 0, access, m_size >> 32, m_size & 0xffffffff, 0));
OPENVINO_ASSERT(m_mapping.get() != INVALID_HANDLE_VALUE, "Can not create file mapping for ", path);
HandleHolder(::CreateFileMapping(m_handle.get(), 0, access, file_map_size >> 32, file_map_size & 0xffffffff, 0));
if(m_mapping.get() == INVALID_HANDLE_VALUE) {
throw std::runtime_error("Can not create file mapping for " + path);
}
m_data = ::MapViewOfFile(m_mapping.get(),
map_mode,
0, // offset_align >> 32,
0, // offset_align & 0xffffffff,
m_size);
OPENVINO_ASSERT(m_data, "Can not create map view for ", path);
map_view_start >> 32, // offset_align >> 32,
map_view_start & 0xffffffff, // offset_align & 0xffffffff,
map_view_size);
if(!m_data) {
throw std::runtime_error("Can not create map view for " + path);
}
// move pointer to the expected data (after alignment with allocation granularity)
m_data = reinterpret_cast<char*>(m_data) + view_delta;
} else {
m_data = nullptr;
}
@@ -123,22 +140,18 @@ private:
HandleHolder m_mapping;
};
std::shared_ptr<ngraph::runtime::AlignedBuffer> load_mmap_object(const std::string& path) {
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::string& path, const size_t data_size, const size_t offset) {
auto holder = std::make_shared<MapHolder>();
holder->set(path);
return std::make_shared<ngraph::runtime::SharedBuffer<std::shared_ptr<MapHolder>>>(holder->data(),
holder->size(),
holder);
holder->set(path, data_size, offset);
return holder;
}
#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT
std::shared_ptr<ngraph::runtime::AlignedBuffer> load_mmap_object(const std::wstring& path) {
std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::wstring& path, const size_t data_size, const size_t offset) {
auto holder = std::make_shared<MapHolder>();
holder->set(path);
return std::make_shared<ngraph::runtime::SharedBuffer<std::shared_ptr<MapHolder>>>(holder->data(),
holder->size(),
holder);
holder->set(path, data_size, offset);
return holder;
}
#endif
src/frontends/ir/src/frontend.cpp
+8 -4
View File
@@ -8,12 +8,12 @@
#include <vector>
#include "input_model.hpp"
#include "mmap_object.hpp"
#include "ngraph/runtime/aligned_buffer.hpp"
#include "ngraph/runtime/shared_buffer.hpp"
#include "openvino/core/any.hpp"
#include "openvino/core/so_extension.hpp"
#include "openvino/util/file_util.hpp"
#include "openvino/util/mmap_object.hpp"
#include "xml_parse_utils.h"
using namespace ov;
@@ -201,9 +201,13 @@ InputModel::Ptr FrontEnd::load_impl(const std::vector<ov::Any>& variants) const
}
}
if (!weights_path.empty()) {
if (enable_mmap)
weights = ov::load_mmap_object(weights_path);
else {
if (enable_mmap) {
auto mapped_memory = ov::load_mmap_object(weights_path);
weights =
std::make_shared<ngraph::runtime::SharedBuffer<std::shared_ptr<MappedMemory>>>(mapped_memory->data(),
mapped_memory->size(),
mapped_memory);
} else {
std::ifstream bin_stream;
bin_stream.open(weights_path.c_str(), std::ios::binary);
if (!bin_stream.is_open())
src/frontends/ir/src/mmap_object.hpp
-26
View File
@@ -1,26 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
/**
* @brief A header file for definition of abstraction over platform specific shared memory map objects
* @file mmap_object.hpp
*/
#pragma once
#include <memory>
#include "ngraph/runtime/aligned_buffer.hpp"
namespace ov {
std::shared_ptr<ngraph::runtime::AlignedBuffer> load_mmap_object(const std::string& path);
#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT
std::shared_ptr<ngraph::runtime::AlignedBuffer> load_mmap_object(const std::wstring& path);
#endif // OPENVINO_ENABLE_UNICODE_PATH_SUPPORT
} // namespace ov
src/frontends/onnx/frontend/include/onnx_import/onnx.hpp
+10 -6
View File
@@ -45,14 +45,17 @@ bool is_operator_supported(const std::string& op_name, std::int64_t version, con
/// \note If stream parsing fails or the ONNX model contains unsupported ops,
/// the function throws an ngraph_error exception.
///
/// \param[in] stream The input stream (e.g. file stream, memory stream, etc).
/// \param[in] model_path The path to the imported onnx model.
/// It is required if the imported model uses data saved in external
/// files.
/// \param[in] stream The input stream (e.g. file stream, memory stream, etc).
/// \param[in] model_path The path to the imported onnx model.
/// It is required if the imported model uses data saved in external
/// files.
/// \param[in] enable_mmap Enable mapping files with external weights instead of reading.
///
/// \return An nGraph function that represents a single output from the created graph.
ONNX_IMPORTER_API
std::shared_ptr<Function> import_onnx_model(std::istream& stream, const std::string& model_path = "");
std::shared_ptr<Function> import_onnx_model(std::istream& stream,
const std::string& model_path = "",
bool enable_mmap = false);
/// \brief Imports and converts an ONNX model from the input file
/// to an nGraph Function representation.
@@ -62,10 +65,11 @@ std::shared_ptr<Function> import_onnx_model(std::istream& stream, const std::str
///
/// \param[in] file_path The path to a file containing the ONNX model
/// (relative or absolute).
/// \param[in] enable_mmap Enable mapping files with external weights instead of reading.
///
/// \return An nGraph function that represents a single output from the created graph.
ONNX_IMPORTER_API
std::shared_ptr<Function> import_onnx_model(const std::string& file_path);
std::shared_ptr<Function> import_onnx_model(const std::string& file_path, bool enable_mmap = false);
} // namespace onnx_import
} // namespace ngraph
src/frontends/onnx/frontend/src/core/attribute.hpp
+14 -10
View File
@@ -200,9 +200,12 @@ public:
};
Attribute() = delete;
explicit Attribute(const ONNX_NAMESPACE::AttributeProto& attribute_proto, const std::string& model_dir)
explicit Attribute(const ONNX_NAMESPACE::AttributeProto& attribute_proto,
const std::string& model_dir,
const bool enable_mmap)
: m_attribute_proto{&attribute_proto},
m_model_dir{model_dir} {}
m_model_dir{model_dir},
m_enable_mmap{enable_mmap} {}
Attribute(Attribute&&) noexcept = default;
Attribute(const Attribute&) = default;
@@ -253,10 +256,10 @@ public:
return get_type() == Type::graph_array;
}
Tensor get_tensor() const {
return Tensor{m_attribute_proto->t(), m_model_dir};
return Tensor{m_attribute_proto->t(), m_model_dir, m_enable_mmap};
}
SparseTensor get_sparse_tensor() const {
return SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir};
return SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir, m_enable_mmap};
}
float get_float() const {
return m_attribute_proto->f();
@@ -274,7 +277,7 @@ public:
const auto& tensors = m_attribute_proto->tensors();
ret.reserve(tensors.size());
for (const auto& tensor : tensors)
ret.emplace_back(tensor, m_model_dir);
ret.emplace_back(tensor, m_model_dir, m_enable_mmap);
return ret;
}
@@ -283,7 +286,7 @@ public:
const auto& sparse_tensors = m_attribute_proto->sparse_tensors();
ret.reserve(sparse_tensors.size());
for (const auto& tensor : sparse_tensors)
ret.emplace_back(tensor, m_model_dir);
ret.emplace_back(tensor, m_model_dir, m_enable_mmap);
return ret;
}
@@ -315,7 +318,7 @@ public:
template <typename T, typename std::enable_if<std::is_same<T, Tensor>::value, bool>::type = true>
T get_value() const {
if (is_tensor()) {
return Tensor{m_attribute_proto->t(), m_model_dir};
return Tensor{m_attribute_proto->t(), m_model_dir, m_enable_mmap};
}
throw error::attribute::InvalidData{m_attribute_proto->type()};
}
@@ -323,7 +326,7 @@ public:
template <typename T, typename std::enable_if<std::is_same<T, std::vector<Tensor>>::value, bool>::type = true>
T get_value() const {
if (is_tensor()) {
return {Tensor{m_attribute_proto->t(), m_model_dir}};
return {Tensor{m_attribute_proto->t(), m_model_dir, m_enable_mmap}};
} else if (is_tensor_array()) {
return get_tensor_array();
}
@@ -333,7 +336,7 @@ public:
template <typename T, typename std::enable_if<std::is_same<T, SparseTensor>::value, bool>::type = true>
T get_value() const {
if (is_sparse_tensor()) {
return SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir};
return SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir, m_enable_mmap};
}
throw error::attribute::InvalidData{m_attribute_proto->type()};
}
@@ -341,7 +344,7 @@ public:
template <typename T, typename std::enable_if<std::is_same<T, std::vector<SparseTensor>>::value, bool>::type = true>
T get_value() const {
if (is_sparse_tensor()) {
return {SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir}};
return {SparseTensor{m_attribute_proto->sparse_tensor(), m_model_dir, m_enable_mmap}};
} else if (is_sparse_tensor_array()) {
return get_sparse_tensor_array();
}
@@ -353,6 +356,7 @@ public:
private:
const ONNX_NAMESPACE::AttributeProto* m_attribute_proto;
std::string m_model_dir;
const bool m_enable_mmap;
};
} // namespace onnx_import
src/frontends/onnx/frontend/src/core/graph.cpp
+7 -3
View File
@@ -145,16 +145,19 @@ ov::frontend::ExtensionHolder subgraph_required_extensions(
Graph::Graph(const std::string& model_dir,
const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model_proto,
const bool enable_mmap,
ov::frontend::ExtensionHolder extensions)
: Graph(model_dir, model_proto, common::make_unique<GraphCache>(), std::move(extensions)) {}
: Graph(model_dir, model_proto, common::make_unique<GraphCache>(), enable_mmap, std::move(extensions)) {}
Graph::Graph(const std::string& model_dir,
const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model_proto,
std::unique_ptr<GraphCache>&& cache,
const bool enable_mmap,
ov::frontend::ExtensionHolder extensions)
: m_cache{std::move(cache)},
m_extensions{std::move(extensions)},
m_model_dir{model_dir} {
m_model_dir{model_dir},
m_enable_mmap{enable_mmap} {
const auto ops_bridge = detail::init_ops_bridge(m_extensions.conversions);
m_model = common::make_unique<Model>(model_proto, detail::build_model_opset(*model_proto, ops_bridge));
@@ -165,7 +168,7 @@ Graph::Graph(const std::string& model_dir,
// Process all initializers in the graph
for (const auto& initializer_tensor : m_model->get_graph().initializer()) {
if (initializer_tensor.has_name()) {
Tensor tensor = Tensor{initializer_tensor, m_model_dir};
Tensor tensor = Tensor{initializer_tensor, m_model_dir, enable_mmap};
std::shared_ptr<default_opset::Constant> ng_constant;
// For each initializer create a Constant node and store it in cache
try {
@@ -477,6 +480,7 @@ Subgraph::Subgraph(const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model_prot
: Graph(parent_graph->model_dir(),
model_proto,
common::make_unique<GraphCache>(),
parent_graph->mmap_enabled(),
detail::subgraph_required_extensions(parent_graph->get_extensions())),
m_parent_graph(parent_graph) {}
src/frontends/onnx/frontend/src/core/graph.hpp
+6
View File
@@ -23,6 +23,7 @@ class Graph : public std::enable_shared_from_this<Graph> {
public:
Graph(const std::string& model_dir,
const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model_proto,
const bool enable_mmap,
ov::frontend::ExtensionHolder extensions = {});
Graph() = delete;
@@ -40,6 +41,9 @@ public:
const std::string& model_dir() const {
return m_model_dir;
}
bool mmap_enabled() const {
return m_enable_mmap;
}
const ParameterVector& get_ng_parameters() const {
return m_parameters;
}
@@ -57,6 +61,7 @@ protected:
Graph(const std::string& model_dir,
const std::shared_ptr<ONNX_NAMESPACE::ModelProto>& model,
std::unique_ptr<GraphCache>&& cache,
const bool enable_mmap,
ov::frontend::ExtensionHolder extensions = {});
void set_friendly_names(const Node& onnx_node, const OutputVector& ng_subgraph_outputs) const;
@@ -77,6 +82,7 @@ protected:
private:
std::vector<Node> m_nodes;
std::string m_model_dir;
bool m_enable_mmap;
};
/// \brief Representation of ONNX subgraph. It is used for example by ONNX Loop op.
src/frontends/onnx/frontend/src/core/node.cpp
+2 -2
View File
@@ -26,7 +26,7 @@ public:
const auto& attributes = node_proto.attribute();
m_attributes.reserve(attributes.size());
for (const auto& attr_proto : attributes) {
m_attributes.emplace_back(attr_proto, m_graph->model_dir());
m_attributes.emplace_back(attr_proto, m_graph->model_dir(), m_graph->mmap_enabled());
const auto& attribute = m_attributes.back();
if (attribute.is_graph())
m_subgraphs.insert({attribute.get_name(), std::make_shared<Subgraph>(attribute.get_subgraph(m_graph))});
@@ -43,7 +43,7 @@ public:
m_output_names{std::begin(node_proto.output()), std::end(node_proto.output())},
m_subgraphs(subgraphs) {
for (const auto& attr_proto : node_proto.attribute()) {
m_attributes.emplace_back(attr_proto, m_graph->model_dir());
m_attributes.emplace_back(attr_proto, m_graph->model_dir(), m_graph->mmap_enabled());
}
}
src/frontends/onnx/frontend/src/core/sparse_tensor.hpp
+5 -3
View File
@@ -17,9 +17,11 @@ namespace onnx_import {
class SparseTensor {
public:
SparseTensor() = delete;
explicit SparseTensor(const ONNX_NAMESPACE::SparseTensorProto& sparse_tensor, const std::string& model_dir)
: m_values{sparse_tensor.values(), model_dir},
m_indices{sparse_tensor.indices(), model_dir},
SparseTensor(const ONNX_NAMESPACE::SparseTensorProto& sparse_tensor,
const std::string& model_dir,
const bool enable_mmap)
: m_values{sparse_tensor.values(), model_dir, enable_mmap},
m_indices{sparse_tensor.indices(), model_dir, enable_mmap},
m_shape{std::begin(sparse_tensor.dims()), std::end(sparse_tensor.dims())} {
if (m_shape == Shape{0}) {
// It's possible to construct a sparse tensor in ONNX with "dims: 0" property
src/frontends/onnx/frontend/src/core/tensor.hpp
+22 -11
View File
@@ -108,10 +108,11 @@ public:
};
Tensor() = delete;
explicit Tensor(const ONNX_NAMESPACE::TensorProto& tensor, const std::string& model_dir)
explicit Tensor(const ONNX_NAMESPACE::TensorProto& tensor, const std::string& model_dir, const bool enable_mmap)
: m_tensor_proto{&tensor},
m_shape{std::begin(tensor.dims()), std::end(tensor.dims())},
m_model_dir{model_dir} {
m_model_dir{model_dir},
m_enable_mmap{enable_mmap} {
if (m_shape == Shape{0}) {
// It's possible to construct a tensor in ONNX with "dims: 0" property
// Such tensor contains a scalar. This results in a Shape{0} stored in m_shape.
@@ -239,9 +240,16 @@ private:
std::shared_ptr<default_opset::Constant> constant{nullptr};
size_t data_size = get_data_size();
if (has_external_data()) {
auto external_data = load_external_data();
constant = std::make_shared<ngraph::op::Constant>(type, m_shape, external_data.data());
if (constant->get_byte_size() != external_data.size()) {
const auto ext_data = detail::TensorExternalData(*m_tensor_proto);
if (m_enable_mmap) {
constant = std::make_shared<ngraph::op::Constant>(type,
m_shape,
ext_data.load_external_mmap_data(m_model_dir));
} else {
constant =
std::make_shared<ngraph::op::Constant>(type, m_shape, ext_data.load_external_data(m_model_dir));
}
if (constant->get_byte_size() != ov::shape_size(m_shape) * type.size()) {
throw error::invalid_external_data(
"The size of the external data file does not match the byte size of an initializer '" + get_name() +
"' in the model");
@@ -288,14 +296,16 @@ private:
ONNX_NAMESPACE::TensorProto_DataLocation::TensorProto_DataLocation_EXTERNAL;
}
std::string load_external_data() const {
const auto tensor_external_data = detail::TensorExternalData(*m_tensor_proto);
return tensor_external_data.load_external_data(m_model_dir);
}
template <typename T>
std::vector<T> get_external_data() const {
return detail::__get_raw_data<T>(load_external_data(), m_tensor_proto->data_type());
const auto ext_data = detail::TensorExternalData(*m_tensor_proto);
std::shared_ptr<ngraph::runtime::AlignedBuffer> buffer = nullptr;
if (m_enable_mmap) {
buffer = ext_data.load_external_mmap_data(m_model_dir);
} else {
buffer = ext_data.load_external_data(m_model_dir);
}
return std::vector<T>(buffer->get_ptr<char>(), buffer->get_ptr<char>() + buffer->size());
}
const void* get_data_ptr() const {
@@ -339,6 +349,7 @@ private:
const ONNX_NAMESPACE::TensorProto* m_tensor_proto;
Shape m_shape;
std::string m_model_dir;
bool m_enable_mmap = true;
};
inline std::ostream& operator<<(std::ostream& outs, const Tensor& tensor) {
src/frontends/onnx/frontend/src/editor.cpp
+22 -8
View File
@@ -315,17 +315,23 @@ struct onnx_editor::ONNXModelEditor::Impl {
#endif
};
onnx_editor::ONNXModelEditor::ONNXModelEditor(const std::string& model_path, frontend::ExtensionHolder extensions)
: m_extensions{std::move(extensions)},
m_model_path{model_path},
onnx_editor::ONNXModelEditor::ONNXModelEditor(const std::string& model_path,
const bool enable_mmap,
frontend::ExtensionHolder extensions)
: m_model_path{model_path},
m_enable_mmap{enable_mmap},
m_extensions{std::move(extensions)},
m_pimpl{new ONNXModelEditor::Impl{model_path}, [](Impl* impl) {
delete impl;
}} {}
#if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32)
onnx_editor::ONNXModelEditor::ONNXModelEditor(const std::wstring& model_path, frontend::ExtensionHolder extensions)
onnx_editor::ONNXModelEditor::ONNXModelEditor(const std::wstring& model_path,
const bool enable_mmap,
frontend::ExtensionHolder extensions)
: m_extensions{std::move(extensions)},
m_model_path{ov::util::wstring_to_string(model_path)},
m_enable_mmap{enable_mmap},
m_pimpl{new ONNXModelEditor::Impl{model_path}, [](Impl* impl) {
delete impl;
}} {}
@@ -333,9 +339,11 @@ onnx_editor::ONNXModelEditor::ONNXModelEditor(const std::wstring& model_path, fr
onnx_editor::ONNXModelEditor::ONNXModelEditor(std::istream& model_stream,
const std::string& model_path,
const bool enable_mmap,
frontend::ExtensionHolder extensions)
: m_extensions{std::move(extensions)},
m_model_path{model_path},
: m_model_path{model_path},
m_enable_mmap{enable_mmap},
m_extensions{std::move(extensions)},
m_pimpl{new ONNXModelEditor::Impl{model_stream}, [](Impl* impl) {
delete impl;
}} {}
@@ -522,7 +530,10 @@ std::string onnx_editor::ONNXModelEditor::model_string() const {
}
std::shared_ptr<Model> onnx_editor::ONNXModelEditor::get_function() const {
return ngraph::onnx_import::detail::import_onnx_model(m_pimpl->m_model_proto, m_model_path, m_extensions);
return ngraph::onnx_import::detail::import_onnx_model(m_pimpl->m_model_proto,
m_model_path,
m_enable_mmap,
m_extensions);
}
void onnx_editor::ONNXModelEditor::set_input_values(
@@ -713,7 +724,10 @@ std::vector<std::string> onnx_editor::ONNXModelEditor::get_output_ports(const Ed
}
std::shared_ptr<Model> onnx_editor::ONNXModelEditor::decode() {
return ngraph::onnx_import::detail::decode_to_framework_nodes(m_pimpl->m_model_proto, m_model_path, m_extensions);
return ngraph::onnx_import::detail::decode_to_framework_nodes(m_pimpl->m_model_proto,
m_model_path,
m_enable_mmap,
m_extensions);
}
void onnx_editor::ONNXModelEditor::add_output(const OutputEdge& output_edge) const {
src/frontends/onnx/frontend/src/editor.hpp
+14 -6
View File
@@ -31,9 +31,13 @@ public:
/// is parsed and loaded into the m_model_proto member variable.
///
/// \param model_path Path to the file containing the model.
ONNXModelEditor(const std::string& model_path, frontend::ExtensionHolder extensions = {});
/// \param enable_mmap Enable mapping files with external weights instead of reading.
/// \param extensions Holder for custom extensions (like custom ops).
explicit ONNXModelEditor(const std::string& model_path,
const bool enable_mmap = false,
frontend::ExtensionHolder extensions = {});
#if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32)
ONNXModelEditor(const std::wstring& model_path, frontend::ExtensionHolder extensions = {});
ONNXModelEditor(const std::wstring& model_path, const bool enable_mmap, frontend::ExtensionHolder extensions = {});
#endif
/// \brief Creates an editor from a model stream. The stream is parsed and loaded
@@ -42,9 +46,12 @@ public:
/// \param model_stream The stream containing the model.
/// \param model_path Path to the file containing the model. This information can be used
/// for ONNX external weights feature support.
ONNXModelEditor(std::istream& model_stream,
const std::string& path = {},
frontend::ExtensionHolder extensions = {});
/// \param enable_mmap Enable mapping files with external weights instead of reading.
/// \param extensions Holder for custom extensions (like custom ops).
explicit ONNXModelEditor(std::istream& model_stream,
const std::string& path = {},
const bool enable_mmap = false,
frontend::ExtensionHolder extensions = {});
/// \brief Modifies the in-memory representation of the model by setting
/// custom input types for all inputs specified in the provided map.
@@ -296,8 +303,9 @@ public:
private:
void update_mapper_if_needed() const;
frontend::ExtensionHolder m_extensions;
const std::string m_model_path;
const bool m_enable_mmap;
frontend::ExtensionHolder m_extensions;
struct Impl;
std::unique_ptr<Impl, void (*)(Impl*)> m_pimpl;
src/frontends/onnx/frontend/src/frontend.cpp
+8 -5
View File
@@ -56,29 +56,32 @@ InputModel::Ptr FrontEnd::load_impl(const std::vector<ov::Any>& variants) const
if (variants.empty()) {
return nullptr;
}
// enable mmap by default
const bool enable_mmap = variants[variants.size() - 1].is<bool>() ? variants[variants.size() - 1].as<bool>() : true;
if (variants[0].is<std::string>()) {
const auto path = variants[0].as<std::string>();
return std::make_shared<InputModel>(path, m_extensions);
return std::make_shared<InputModel>(path, enable_mmap, m_extensions);
}
#if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32)
if (variants[0].is<std::wstring>()) {
const auto path = variants[0].as<std::wstring>();
return std::make_shared<InputModel>(path, m_extensions);
return std::make_shared<InputModel>(path, enable_mmap, m_extensions);
}
#endif
if (variants[0].is<std::istream*>()) {
const auto stream = variants[0].as<std::istream*>();
if (variants.size() > 1 && variants[1].is<std::string>()) {
const auto path = variants[1].as<std::string>();
return std::make_shared<InputModel>(*stream, path, m_extensions);
return std::make_shared<InputModel>(*stream, path, enable_mmap, m_extensions);
}
#if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32)
if (variants.size() > 1 && variants[1].is<std::wstring>()) {
const auto path = variants[1].as<std::wstring>();
return std::make_shared<InputModel>(*stream, path, m_extensions);
return std::make_shared<InputModel>(*stream, path, enable_mmap, m_extensions);
}
#endif
return std::make_shared<InputModel>(*stream, m_extensions);
return std::make_shared<InputModel>(*stream, enable_mmap, m_extensions);
}
return nullptr;
}
src/frontends/onnx/frontend/src/input_model.cpp
+17 -10
View File
@@ -16,23 +16,30 @@ using namespace ov::frontend::onnx;
NGRAPH_SUPPRESS_DEPRECATED_START
InputModel::InputModel(const std::string& path, frontend::ExtensionHolder extensions)
: m_editor{std::make_shared<onnx_editor::ONNXModelEditor>(path, std::move(extensions))} {}
InputModel::InputModel(const std::string& path, const bool enable_mmap, frontend::ExtensionHolder extensions)
: m_editor{std::make_shared<onnx_editor::ONNXModelEditor>(path, enable_mmap, std::move(extensions))} {}
#if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32)
InputModel::InputModel(const std::wstring& path, frontend::ExtensionHolder extensions)
: m_editor{std::make_shared<onnx_editor::ONNXModelEditor>(path, std::move(extensions))} {}
InputModel::InputModel(const std::wstring& path, const bool enable_mmap, frontend::ExtensionHolder extensions)
: m_editor{std::make_shared<onnx_editor::ONNXModelEditor>(path, enable_mmap, std::move(extensions))} {}
#endif
InputModel::InputModel(std::istream& model_stream, frontend::ExtensionHolder extensions)
: m_editor{std::make_shared<onnx_editor::ONNXModelEditor>(model_stream, "", std::move(extensions))} {}
InputModel::InputModel(std::istream& model_stream, const bool enable_mmap, frontend::ExtensionHolder extensions)
: m_editor{std::make_shared<onnx_editor::ONNXModelEditor>(model_stream, "", enable_mmap, std::move(extensions))} {}
InputModel::InputModel(std::istream& model_stream, const std::string& path, frontend::ExtensionHolder extensions)
: m_editor{std::make_shared<onnx_editor::ONNXModelEditor>(model_stream, path, std::move(extensions))} {}
InputModel::InputModel(std::istream& model_stream,
const std::string& path,
const bool enable_mmap,
frontend::ExtensionHolder extensions)
: m_editor{std::make_shared<onnx_editor::ONNXModelEditor>(model_stream, path, enable_mmap, std::move(extensions))} {
}
#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT
InputModel::InputModel(std::istream& model_stream, const std::wstring& path, frontend::ExtensionHolder extensions)
: InputModel(model_stream, ov::util::wstring_to_string(path), std::move(extensions)) {}
InputModel::InputModel(std::istream& model_stream,
const std::wstring& path,
const bool enable_mmap,
frontend::ExtensionHolder extensions)
: InputModel(model_stream, ov::util::wstring_to_string(path), enable_mmap, std::move(extensions)) {}
#endif
std::vector<ov::frontend::Place::Ptr> InputModel::get_inputs() const {
src/frontends/onnx/frontend/src/input_model.hpp
+11 -5
View File
@@ -16,16 +16,22 @@ namespace onnx {
class InputModel : public ov::frontend::InputModel {
public:
InputModel(const std::string& path, ExtensionHolder extensions = {});
InputModel(const std::string& path, const bool enable_mmap = false, ExtensionHolder extensions = {});
#if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32)
InputModel(const std::wstring& path, ExtensionHolder extensions = {});
InputModel(const std::wstring& path, const bool enable_mmap = false, ExtensionHolder extensions = {});
#endif
InputModel(std::istream& model_stream, ExtensionHolder extensions = {});
InputModel(std::istream& model_stream, const bool enable_mmap = false, ExtensionHolder extensions = {});
// The path can be required even if the model is passed as a stream because it is necessary
// for ONNX external data feature
InputModel(std::istream& model_stream, const std::string& path, ExtensionHolder extensions = {});
InputModel(std::istream& model_stream,
const std::string& path,
const bool enable_mmap = false,
ExtensionHolder extensions = {});
#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT
InputModel(std::istream& model_stream, const std::wstring& path, ExtensionHolder extensions = {});
InputModel(std::istream& model_stream,
const std::wstring& path,
const bool enable_mmap = false,
ExtensionHolder extensions = {});
#endif
std::vector<ov::frontend::Place::Ptr> get_inputs() const override;
src/frontends/onnx/frontend/src/onnx.cpp
+6 -4
View File
@@ -22,14 +22,16 @@ const auto legacy_conversion_extension = std::make_shared<ngraph::onnx_import::L
namespace ngraph {
namespace onnx_import {
std::shared_ptr<Function> import_onnx_model(std::istream& stream, const std::string& model_path) {
std::shared_ptr<Function> import_onnx_model(std::istream& stream,
const std::string& model_path,
const bool enable_mmap) {
const auto model_proto = std::make_shared<ONNX_NAMESPACE::ModelProto>(onnx_common::parse_from_istream(stream));
ov::frontend::ExtensionHolder extensions;
extensions.conversions.push_back(legacy_conversion_extension);
return detail::import_onnx_model(model_proto, model_path, std::move(extensions));
return detail::import_onnx_model(model_proto, model_path, enable_mmap, std::move(extensions));
}
std::shared_ptr<Function> import_onnx_model(const std::string& file_path) {
std::shared_ptr<Function> import_onnx_model(const std::string& file_path, const bool enable_mmap) {
std::ifstream model_stream{file_path, std::ios::in | std::ios::binary};
if (!model_stream.is_open()) {
@@ -37,7 +39,7 @@ std::shared_ptr<Function> import_onnx_model(const std::string& file_path) {
". Could not open the file.");
};
return import_onnx_model(model_stream, file_path);
return import_onnx_model(model_stream, file_path, enable_mmap);
}
std::set<std::string> get_supported_operators(std::int64_t version, const std::string& domain) {
src/frontends/onnx/frontend/src/utils/onnx_internal.cpp
+4
View File
@@ -89,11 +89,13 @@ void convert_decoded_function(std::shared_ptr<Function> function) {
std::shared_ptr<Function> import_onnx_model(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto,
const std::string& model_path,
const bool enable_mmap,
ov::frontend::ExtensionHolder extensions) {
apply_transformations(*model_proto);
NGRAPH_SUPPRESS_DEPRECATED_START
Graph graph{file_util::get_directory(ov::util::get_absolute_file_path(model_path)),
model_proto,
enable_mmap,
std::move(extensions)};
NGRAPH_SUPPRESS_DEPRECATED_END
return graph.convert();
@@ -101,11 +103,13 @@ std::shared_ptr<Function> import_onnx_model(std::shared_ptr<ONNX_NAMESPACE::Mode
std::shared_ptr<Function> decode_to_framework_nodes(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto,
const std::string& model_path,
const bool enable_mmap,
ov::frontend::ExtensionHolder extensions) {
apply_transformations(*model_proto);
NGRAPH_SUPPRESS_DEPRECATED_START
auto graph = std::make_shared<Graph>(file_util::get_directory(ov::util::get_absolute_file_path(model_path)),
model_proto,
enable_mmap,
extensions);
NGRAPH_SUPPRESS_DEPRECATED_END
return graph->decode();
src/frontends/onnx/frontend/src/utils/onnx_internal.hpp
+4 -2
View File
@@ -29,12 +29,13 @@ namespace detail {
/// \param model_proto Reference to a GraphProto object.
/// \param model_path The path to the imported onnx model.
/// It is required if the imported model uses data saved in external files.
/// \param enable_mmap Enable mapping files with external weights instead of reading.
/// \param extensions An object containing a collection of frontend extensions to use during the import process
///
/// \return An nGraph function that represents a single output from the created
/// graph.
std::shared_ptr<Function> import_onnx_model(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto,
const std::string& model_path,
const bool enable_mmap,
ov::frontend::ExtensionHolder extensions = {});
/// \brief Decode ONNX model to nGraph function with ONNXFrameworkNode(s)
@@ -42,11 +43,12 @@ std::shared_ptr<Function> import_onnx_model(std::shared_ptr<ONNX_NAMESPACE::Mode
/// \param model_proto Reference to a GraphProto object.
/// \param model_path The path to the imported onnx model.
/// It is required if the imported model uses data saved in external files.
/// \param enable_mmap Enable mapping files with external weights instead of reading.
/// \param extensions An object containing a collection of frontend extensions to use during the import process
///
/// \return A nGraph function with ONNXFrameworkNodes
std::shared_ptr<Function> decode_to_framework_nodes(std::shared_ptr<ONNX_NAMESPACE::ModelProto> model_proto,
const std::string& model_path,
const bool enable_mmap,
ov::frontend::ExtensionHolder extensions = {});
/// \brief Converts a nGraph function (onnx model decoded to function with ONNXFrameworkNode(s))
src/frontends/onnx/frontend/src/utils/tensor_external_data.cpp
+29 -9
View File
@@ -29,9 +29,29 @@ TensorExternalData::TensorExternalData(const ONNX_NAMESPACE::TensorProto& tensor
m_sha1_digest = entry.value();
}
}
if (m_sha1_digest.size() > 0) {
OPENVINO_WARN << "SHA1 checksum is not supported";
}
}
std::string TensorExternalData::load_external_data(const std::string& model_dir) const {
Buffer<ov::MappedMemory> TensorExternalData::load_external_mmap_data(const std::string& model_dir) const {
NGRAPH_SUPPRESS_DEPRECATED_START
auto full_path = file_util::path_join(model_dir, m_data_location);
NGRAPH_SUPPRESS_DEPRECATED_END
const int64_t file_size = ov::util::file_size(full_path);
if (file_size <= 0 || m_offset + m_data_length > static_cast<uint64_t>(file_size)) {
throw error::invalid_external_data{*this};
}
auto mapped_memory = ov::load_mmap_object(full_path, m_data_length, m_offset);
if (m_data_length > mapped_memory->size() || mapped_memory->size() == 0) {
throw error::invalid_external_data{*this};
}
return std::make_shared<ngraph::runtime::SharedBuffer<std::shared_ptr<ov::MappedMemory>>>(mapped_memory->data(),
mapped_memory->size(),
mapped_memory);
}
Buffer<ngraph::runtime::AlignedBuffer> TensorExternalData::load_external_data(const std::string& model_dir) const {
NGRAPH_SUPPRESS_DEPRECATED_START
auto full_path = file_util::path_join(model_dir, m_data_location);
@@ -57,16 +77,16 @@ std::string TensorExternalData::load_external_data(const std::string& model_dir)
// default value of m_offset is 0
external_data_stream.seekg(m_offset, std::ios::beg);
if (m_sha1_digest.size() > 0) {
OPENVINO_WARN << "SHA1 checksum is not supported";
}
std::string read_data;
read_data.resize(read_data_length);
external_data_stream.read(&read_data[0], read_data_length);
auto read_data = std::make_shared<ngraph::runtime::AlignedBuffer>(read_data_length);
external_data_stream.read(read_data->get_ptr<char>(), read_data_length);
external_data_stream.close();
return read_data;
auto buffer = std::make_shared<ngraph::runtime::SharedBuffer<std::shared_ptr<ngraph::runtime::AlignedBuffer>>>(
read_data->get_ptr<char>(),
read_data->size(),
read_data);
return buffer;
}
std::string TensorExternalData::to_string() const {
src/frontends/onnx/frontend/src/utils/tensor_external_data.hpp
+16 -2
View File
@@ -6,9 +6,14 @@
#include <onnx/onnx_pb.h>
#include "ngraph/runtime/shared_buffer.hpp"
#include "openvino/util/mmap_object.hpp"
namespace ngraph {
namespace onnx_import {
namespace detail {
template <class T>
using Buffer = std::shared_ptr<ngraph::runtime::SharedBuffer<std::shared_ptr<T>>>;
/// \brief Helper class used to load tensor data from external files
class TensorExternalData {
public:
@@ -20,8 +25,17 @@ public:
/// \note If reading data from external files fails,
/// the invalid_external_data exception is thrown.
///
/// \return External binary data loaded into a std::string
std::string load_external_data(const std::string& model_dir) const;
/// \return External binary data loaded into the SharedBuffer
Buffer<ngraph::runtime::AlignedBuffer> load_external_data(const std::string& model_dir) const;
/// \brief Map (mmap for lin, MapViewOfFile for win) external data from tensor passed to constructor
///
/// \note If read data from external file fails,
/// \note If reading data from external files fails,
/// the invalid_external_data exception is thrown.
///
/// \return External binary data loaded into the SharedBuffer
Buffer<ov::MappedMemory> load_external_mmap_data(const std::string& model_dir) const;
/// \brief Represets parameter of external data as string
///
src/frontends/onnx/tests/CMakeLists.txt
-1
View File
@@ -63,7 +63,6 @@ set(MULTI_TEST_SRC
onnx_import_convpool.in.cpp
onnx_import_deprecated.in.cpp
onnx_import_dyn_shapes.in.cpp
onnx_import_external_data.in.cpp
onnx_import_org_openvino.in.cpp
onnx_import_org_pytorch.in.cpp
onnx_import_reshape.in.cpp
src/frontends/onnx/tests/onnx_import_external_data.in.cpp
-188
View File
@@ -1,188 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "common_test_utils/file_utils.hpp"
#include "default_opset.hpp"
#include "engines_util/test_case.hpp"
#include "engines_util/test_engines.hpp"
#include "gtest/gtest.h"
#include "ngraph/file_util.hpp"
#include "ngraph/type/element_type.hpp"
#include "onnx_import/onnx.hpp"
#include "util/test_control.hpp"
#include "util/test_tools.hpp"
#include "util/type_prop.hpp"
using namespace ngraph;
using namespace ngraph::onnx_import;
using namespace ngraph::test;
OPENVINO_SUPPRESS_DEPRECATED_START
static std::string s_manifest = "${MANIFEST}";
static std::string s_device = test::backend_name_to_device("${BACKEND_NAME}");
NGRAPH_TEST(${BACKEND_NAME}, onnx_external_data) {
const auto function = onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(),
SERIALIZED_ZOO,
"onnx/external_data/external_data.onnx"));
auto test_case = test::TestCase(function, s_device);
test_case.add_input<float>({1.f, 2.f, 3.f, 4.f});
test_case.add_expected_output<float>(Shape{2, 2}, {3.f, 6.f, 9.f, 12.f});
test_case.run();
}
NGRAPH_TEST(${BACKEND_NAME}, onnx_external_data_from_stream) {
std::string path = file_util::path_join(CommonTestUtils::getExecutableDirectory(),
SERIALIZED_ZOO,
"onnx/external_data/external_data.onnx");
std::ifstream stream{path, std::ios::in | std::ios::binary};
ASSERT_TRUE(stream.is_open());
const auto function = onnx_import::import_onnx_model(stream, path);
auto test_case = test::TestCase(function, s_device);
test_case.add_input<float>({1.f, 2.f, 3.f, 4.f});
test_case.add_expected_output<float>(Shape{2, 2}, {3.f, 6.f, 9.f, 12.f});
test_case.run();
stream.close();
}
NGRAPH_TEST(${BACKEND_NAME}, onnx_external_data_incorrect_size_exception) {
try {
auto function = onnx_import::import_onnx_model(
file_util::path_join(CommonTestUtils::getExecutableDirectory(),
SERIALIZED_ZOO,
"onnx/external_data/external_data_incorrect_data_shape.onnx"));
FAIL() << "Incorrect size of external data not detected";
} catch (const ngraph_error& error) {
EXPECT_PRED_FORMAT2(
testing::IsSubstring,
std::string(
"The size of the external data file does not match the byte size of an initializer 'A' in the model"),
error.what());
} catch (...) {
FAIL() << "Importing onnx model failed for unexpected reason";
}
}
NGRAPH_TEST(${BACKEND_NAME}, onnx_external_data_optional_fields) {
const auto function =
onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(),
SERIALIZED_ZOO,
"onnx/external_data/external_data_optional_fields.onnx"));
auto test_case = test::TestCase(function, s_device);
test_case.add_input<float>({1.f, 2.f, 3.f, 4.f});
test_case.add_expected_output<float>(Shape{2, 2}, {3.f, 6.f, 9.f, 12.f});
test_case.run();
}
NGRAPH_TEST(${BACKEND_NAME}, onnx_external_data_in_different_paths) {
auto function =
onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(),
SERIALIZED_ZOO,
"onnx/external_data/external_data_different_paths.onnx"));
auto test_case = test::TestCase(function, s_device);
// first input: {3.f}, second: {1.f, 2.f, 5.f} read from external files
test_case.add_input<float>({2.f, 7.f, 7.f});
test_case.add_expected_output<float>({2.f, 4.f, 5.f});
test_case.run();
}
NGRAPH_TEST(${BACKEND_NAME}, onnx_external_two_tensors_data_in_the_same_file) {
auto function = onnx_import::import_onnx_model(
file_util::path_join(CommonTestUtils::getExecutableDirectory(),
SERIALIZED_ZOO,
"onnx/external_data/external_data_two_tensors_data_in_the_same_file.onnx"));
auto test_case = test::TestCase(function, s_device);
// first input: {3, 2, 1}, second: {1, 2, 3} read from external file
test_case.add_input<int32_t>({2, 3, 1});
test_case.add_expected_output<int32_t>({3, 3, 3});
test_case.run();
}
NGRAPH_TEST(${BACKEND_NAME}, onnx_external_invalid_external_data_exception) {
try {
auto function = onnx_import::import_onnx_model(
file_util::path_join(CommonTestUtils::getExecutableDirectory(),
SERIALIZED_ZOO,
"onnx/external_data/external_data_file_not_found.onnx"));
FAIL() << "Incorrect path to external data not detected";
} catch (const ngraph_error& error) {
EXPECT_PRED_FORMAT2(testing::IsSubstring,
std::string("not_existed_file.data, offset: 4096, data_length: 16)"),
error.what());
} catch (...) {
FAIL() << "Importing onnx model failed for unexpected reason";
}
}
NGRAPH_TEST(${BACKEND_NAME}, onnx_external_invalid_up_dir_path) {
try {
auto function = onnx_import::import_onnx_model(
file_util::path_join(CommonTestUtils::getExecutableDirectory(),
SERIALIZED_ZOO,
"onnx/external_data/inner_scope/external_data_file_in_up_dir.onnx"));
FAIL() << "Incorrect path to external data not detected";
} catch (const ngraph_error& error) {
EXPECT_PRED_FORMAT2(testing::IsSubstring,
std::string("tensor.data, offset: 4096, "
"data_length: 16)"),
error.what());
} catch (...) {
FAIL() << "Importing onnx model failed for unexpected reason";
}
}
NGRAPH_TEST(${BACKEND_NAME}, onnx_external_invalid_data_length) {
try {
auto function = onnx_import::import_onnx_model(
file_util::path_join(CommonTestUtils::getExecutableDirectory(),
SERIALIZED_ZOO,
"onnx/external_data/external_data_invalid_data_length.onnx"));
FAIL() << "Incorrect path to external data not detected";
} catch (const ngraph_error& error) {
EXPECT_PRED_FORMAT2(testing::IsSubstring,
std::string("tensor.data, offset: 0, "
"data_length: 30000000000)"),
error.what());
} catch (...) {
FAIL() << "Importing onnx model failed for unexpected reason";
}
}
NGRAPH_TEST(${BACKEND_NAME}, onnx_external_data_sanitize_path) {
const auto function =
onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(),
SERIALIZED_ZOO,
"onnx/external_data/external_data_sanitize_test.onnx"));
auto test_case = test::TestCase(function, s_device);
test_case.add_input<float>({1.f, 2.f, 3.f, 4.f});
test_case.add_expected_output<float>(Shape{2, 2}, {3.f, 6.f, 9.f, 12.f});
test_case.run();
}
NGRAPH_TEST(${BACKEND_NAME}, onnx_external_data_in_constant_node) {
const auto function =
onnx_import::import_onnx_model(file_util::path_join(CommonTestUtils::getExecutableDirectory(),
SERIALIZED_ZOO,
"onnx/external_data/external_data_in_constant_node.onnx"));
auto test_case = test::TestCase(function, s_device);
test_case.add_input<float>({3.f, 5.f, 8.f, 13.f});
test_case.add_expected_output<float>(Shape{2, 2}, {4.f, 7.f, 11.f, 17.f});
test_case.run();
}
src/frontends/onnx/tests/onnx_reader_external_data.cpp
+187 -3
View File
@@ -4,19 +4,23 @@
#include <file_utils.h>
#include <gtest/gtest.h>
#include <ie_blob.h>
#include <algorithm>
#include <fstream>
#include <ie_core.hpp>
#include <ngraph/ngraph.hpp>
#include <set>
#include <streambuf>
#include <string>
#include "common_test_utils/file_utils.hpp"
#include "common_test_utils/unicode_utils.hpp"
#include "engines_util/test_case.hpp"
#include "ie_blob.h"
#include "ie_core.hpp"
#include "ngraph/ngraph.hpp"
#include "openvino/frontend/manager.hpp"
#include "openvino/openvino.hpp"
// API 1.0 tests
TEST(ONNX_Reader_Tests, ImportModelWithExternalDataFromFile) {
InferenceEngine::Core ie;
auto cnnNetwork = ie.ReadNetwork(
@@ -114,3 +118,183 @@ TEST(ONNX_Reader_Tests, ImportModelWithExternalDataFromWstringNamedFile) {
ASSERT_TRUE(external_data_node_const->get_vector<float>() == (std::vector<float>{1, 2, 3, 4}));
}
#endif
// API 2.0 tests
class OnnxFeMmapFixture : public ::testing::TestWithParam<bool> {};
TEST_P(OnnxFeMmapFixture, onnx_external_data) {
const auto path =
CommonTestUtils::getModelFromTestModelZoo(std::string(ONNX_TEST_MODELS) + "external_data/external_data.onnx");
ov::Core core;
core.set_property(ov::enable_mmap(GetParam()));
const auto model = core.read_model(path);
auto test_case = ngraph::test::TestCase(model);
test_case.add_input<float>({1.f, 2.f, 3.f, 4.f});
test_case.add_expected_output<float>({2, 2}, {3.f, 6.f, 9.f, 12.f});
test_case.run();
}
TEST_P(OnnxFeMmapFixture, onnx_external_data_from_stream) {
const auto path =
CommonTestUtils::getModelFromTestModelZoo(std::string(ONNX_TEST_MODELS) + "external_data/external_data.onnx");
std::ifstream stream{path, std::ios::in | std::ios::binary};
std::istream* is = &stream;
ASSERT_TRUE(stream.is_open());
auto fem = ov::frontend::FrontEndManager();
auto frontend = fem.load_by_framework("onnx");
const bool enable_mmap = GetParam();
const auto in_model = frontend->load(is, path, enable_mmap);
const auto model = frontend->convert(in_model);
auto test_case = ngraph::test::TestCase(model);
test_case.add_input<float>({1.f, 2.f, 3.f, 4.f});
test_case.add_expected_output<float>(ov::Shape{2, 2}, {3.f, 6.f, 9.f, 12.f});
test_case.run();
stream.close();
}
TEST_P(OnnxFeMmapFixture, onnx_external_data_incorrect_size_exception) {
try {
const auto path = CommonTestUtils::getModelFromTestModelZoo(
std::string(ONNX_TEST_MODELS) + "external_data/external_data_incorrect_data_shape.onnx");
ov::Core core;
core.set_property(ov::enable_mmap(GetParam()));
const auto model = core.read_model(path);
FAIL() << "Incorrect size of external data not detected";
} catch (const ov::Exception& ex) {
EXPECT_PRED_FORMAT2(
testing::IsSubstring,
std::string(
"The size of the external data file does not match the byte size of an initializer 'A' in the model"),
ex.what());
} catch (...) {
FAIL() << "Importing onnx model failed for unexpected reason";
}
}
TEST_P(OnnxFeMmapFixture, onnx_external_data_optional_fields) {
const auto path = CommonTestUtils::getModelFromTestModelZoo(std::string(ONNX_TEST_MODELS) +
"external_data/external_data_optional_fields.onnx");
ov::Core core;
core.set_property(ov::enable_mmap(GetParam()));
const auto model = core.read_model(path);
auto test_case = ngraph::test::TestCase(model);
test_case.add_input<float>({1.f, 2.f, 3.f, 4.f});
test_case.add_expected_output<float>(ov::Shape{2, 2}, {3.f, 6.f, 9.f, 12.f});
test_case.run();
}
TEST_P(OnnxFeMmapFixture, onnx_external_data_in_different_paths) {
const auto path = CommonTestUtils::getModelFromTestModelZoo(std::string(ONNX_TEST_MODELS) +
"external_data/external_data_different_paths.onnx");
ov::Core core;
core.set_property(ov::enable_mmap(GetParam()));
const auto model = core.read_model(path);
auto test_case = ngraph::test::TestCase(model);
// first input: {3.f}, second: {1.f, 2.f, 5.f} read from external files
test_case.add_input<float>({2.f, 7.f, 7.f});
test_case.add_expected_output<float>({2.f, 4.f, 5.f});
test_case.run();
}
TEST_P(OnnxFeMmapFixture, onnx_external_two_tensors_data_in_the_same_file) {
const auto path = CommonTestUtils::getModelFromTestModelZoo(
std::string(ONNX_TEST_MODELS) + "external_data/external_data_two_tensors_data_in_the_same_file.onnx");
ov::Core core;
core.set_property(ov::enable_mmap(GetParam()));
const auto model = core.read_model(path);
auto test_case = ngraph::test::TestCase(model);
// first input: {3, 2, 1}, second: {1, 2, 3} read from external file
test_case.add_input<int32_t>({2, 3, 1});
test_case.add_expected_output<int32_t>({3, 3, 3});
test_case.run();
}
TEST_P(OnnxFeMmapFixture, onnx_external_invalid_external_data_exception) {
try {
const auto path = CommonTestUtils::getModelFromTestModelZoo(std::string(ONNX_TEST_MODELS) +
"external_data/external_data_file_not_found.onnx");
ov::Core core;
core.set_property(ov::enable_mmap(GetParam()));
core.read_model(path);
FAIL() << "Incorrect path to external data not detected";
} catch (const ov::Exception& ex) {
EXPECT_PRED_FORMAT2(testing::IsSubstring,
std::string("not_existed_file.data, offset: 4096, data_length: 16)"),
ex.what());
} catch (...) {
FAIL() << "Importing onnx model failed for unexpected reason";
}
}
TEST_P(OnnxFeMmapFixture, onnx_external_invalid_up_dir_path) {
try {
const auto path = CommonTestUtils::getModelFromTestModelZoo(
std::string(ONNX_TEST_MODELS) + "external_data/inner_scope/external_data_file_in_up_dir.onnx");
ov::Core core;
core.set_property(ov::enable_mmap(GetParam()));
core.read_model(path);
FAIL() << "Incorrect path to external data not detected";
} catch (const ov::Exception& ex) {
EXPECT_PRED_FORMAT2(testing::IsSubstring,
std::string("tensor.data, offset: 4096, "
"data_length: 16)"),
ex.what());
} catch (...) {
FAIL() << "Importing onnx model failed for unexpected reason";
}
}
TEST_P(OnnxFeMmapFixture, onnx_external_invalid_data_length) {
try {
const auto path = CommonTestUtils::getModelFromTestModelZoo(
std::string(ONNX_TEST_MODELS) + "external_data/external_data_invalid_data_length.onnx");
ov::Core core;
core.set_property(ov::enable_mmap(GetParam()));
core.read_model(path);
FAIL() << "Incorrect path to external data not detected";
} catch (const ov::Exception& ex) {
EXPECT_PRED_FORMAT2(testing::IsSubstring,
std::string("tensor.data, offset: 0, "
"data_length: 30000000000)"),
ex.what());
} catch (...) {
FAIL() << "Importing onnx model failed for unexpected reason";
}
}
TEST_P(OnnxFeMmapFixture, onnx_external_data_sanitize_path) {
const auto path = CommonTestUtils::getModelFromTestModelZoo(std::string(ONNX_TEST_MODELS) +
"external_data/external_data_sanitize_test.onnx");
ov::Core core;
core.set_property(ov::enable_mmap(GetParam()));
const auto model = core.read_model(path);
auto test_case = ngraph::test::TestCase(model);
test_case.add_input<float>({1.f, 2.f, 3.f, 4.f});
test_case.add_expected_output<float>(ov::Shape{2, 2}, {3.f, 6.f, 9.f, 12.f});
test_case.run();
}
TEST_P(OnnxFeMmapFixture, onnx_external_data_in_constant_node) {
const auto path = CommonTestUtils::getModelFromTestModelZoo(std::string(ONNX_TEST_MODELS) +
"external_data/external_data_in_constant_node.onnx");
ov::Core core;
core.set_property(ov::enable_mmap(GetParam()));
const auto model = core.read_model(path);
auto test_case = ngraph::test::TestCase(model);
test_case.add_input<float>({3.f, 5.f, 8.f, 13.f});
test_case.add_expected_output<float>(ov::Shape{2, 2}, {4.f, 7.f, 11.f, 17.f});
test_case.run();
}
INSTANTIATE_TEST_SUITE_P(OnnxFeMMapReadModel, OnnxFeMmapFixture, ::testing::Bool());
tools/mo/openvino/tools/mo/convert.py
+5
View File
@@ -40,6 +40,7 @@ def convert_model(
version: bool = None,
progress: bool = False,
stream_output: bool = False,
enable_mmap: bool = False,
# PaddlePaddle-specific parameters:
example_output: Any = None,
@@ -268,6 +269,10 @@ def convert_model(
Enable model conversion progress display.
:param stream_output:
Switch model conversion progress display to a multiline mode.
:param enable_mmap:
Force reading weights instead of mapping weights files into memory.
Currently, mmap feature is provided only for ONNX models
that do not require fallback to the legacy ONNX frontend for the conversion.
PaddlePaddle-specific parameters:
:param example_output:
tools/mo/openvino/tools/mo/convert_impl.py
+3 -1
View File
@@ -444,7 +444,9 @@ def read_model(fem: FrontEndManager, path_to_xml: str):
# avoid segfault during object destruction. So fe must
# be destructed before fem object explicitly.
fe = fem.load_by_framework(framework="ir")
function = fe.convert(fe.load(path_to_xml))
# *.xml/.*bin files are temporary created in the legacy scenario, so we cannot map the memory
enable_mmap = False
function = fe.convert(fe.load(path_to_xml, enable_mmap))
return function
tools/mo/unit_tests/mo/utils/cli_parser_test.py
+1 -1
View File
@@ -2040,7 +2040,7 @@ class TestConvertModelParamsParsing(unittest.TestCase):
'log_level', 'input', 'output', 'mean_values', 'scale_values', 'source_layout',
'target_layout', 'layout', 'compress_to_fp16', 'transform', 'extensions',
'batch', 'silent', 'version', 'progress', 'stream_output',
'transformations_config', 'example_input'},
'transformations_config', 'example_input', 'enable_mmap'},
'Caffe*-specific parameters:': {'input_proto', 'caffe_parser_path', 'k', 'disable_omitting_optional',
'enable_flattening_nested_params'},
'TensorFlow*-specific parameters:': {'input_model_is_text', 'input_checkpoint', 'input_meta_graph',
tools/ovc/openvino/tools/ovc/convert.py
+5
View File
@@ -45,6 +45,7 @@ def convert_model(
version: bool = None,
progress: bool = False,
stream_output: bool = False,
enable_mmap: bool = True,
# PaddlePaddle-specific parameters:
example_output: Any = None,
@@ -273,6 +274,10 @@ def convert_model(
Enable model conversion progress display.
:param stream_output:
Switch model conversion progress display to a multiline mode.
:param enable_mmap:
Force reading weights instead of mapping weights files into memory.
Currently, mmap feature is provided only for ONNX models
that do not require fallback to the legacy ONNX frontend for the conversion.
PaddlePaddle-specific parameters:
:param example_output:
tools/ovc/openvino/tools/ovc/moc_frontend/pipeline.py
+6 -5
View File
@@ -46,18 +46,19 @@ def moc_pipeline(argv: argparse.Namespace, moc_front_end: FrontEnd):
:return: converted nGraph function ready for serialization
"""
input_checkpoint = getattr(argv, 'input_checkpoint', None)
enable_mmap = getattr(argv, 'enable_mmap', True)
if argv.input_model and input_checkpoint:
# frozen format with v1 checkpoints
input_model = moc_front_end.load([argv.input_model, argv.input_checkpoint])
input_model = moc_front_end.load([argv.input_model, argv.input_checkpoint], enable_mmap)
elif argv.input_model:
input_model = moc_front_end.load(argv.input_model)
input_model = moc_front_end.load(argv.input_model, enable_mmap)
elif argv.saved_model_dir:
if argv.saved_model_tags:
input_model = moc_front_end.load([argv.saved_model_dir, argv.saved_model_tags])
input_model = moc_front_end.load([argv.saved_model_dir, argv.saved_model_tags], enable_mmap)
else:
input_model = moc_front_end.load(argv.saved_model_dir)
input_model = moc_front_end.load(argv.saved_model_dir, enable_mmap)
elif argv.input_meta_graph:
input_model = moc_front_end.load(argv.input_meta_graph)
input_model = moc_front_end.load(argv.input_meta_graph, enable_mmap)
if argv.output:
# Simulate original behavior with freezing model
# While freezing we do a cutting of model, to keep similar behavior we
tools/ovc/unit_tests/ovc/utils/cli_parser_test.py
+1 -1
View File
@@ -2073,7 +2073,7 @@ class TestConvertModelParamsParsing(unittest.TestCase):
'log_level', 'input', 'output', 'mean_values', 'scale_values', 'source_layout',
'target_layout', 'layout', 'compress_to_fp16', 'transform', 'extensions',
'batch', 'silent', 'version', 'progress', 'stream_output',
'transformations_config', 'example_input'},
'transformations_config', 'example_input', 'enable_mmap'},
'Caffe*-specific parameters:': {'input_proto', 'caffe_parser_path', 'k', 'disable_omitting_optional',
'enable_flattening_nested_params'},
'TensorFlow*-specific parameters:': {'input_model_is_text', 'input_checkpoint', 'input_meta_graph',