Enable MapAllocator in IR Frontend (#12673)

* Enable MapAllocator in IR Frontend

* Fix `ov_infer_request_ppp` test

With `mmap()`ing of IR, .bin can't be deleted until unmapping.
And it shows that there was a leak in test

* Add comment to Win `CreateFile()` regarding
FILE_SHARE_DELETE

* Unmap .bin file before IR files deletion

Wait ov::Model deletion to trigger .bin file unmapping
before IR files deletion

* ClangFormat

* Add `use_map_allocator` switch in FE

In case of direct use of FE (e.g. via MO), `mmap()` is OFF.
But in case of use FE via Core, `mmap()` is ON.

src/bindings/c/tests/ov_infer_request_test.cpp

@@ -164,6 +164,7 @@ protected:
         OV_EXPECT_OK(ov_preprocess_input_model_info_set_layout(input_model, model_layout));
         ov_layout_free(model_layout);
 
+        ov_model_free(model);  // clean before assigning built model
         OV_EXPECT_OK(ov_preprocess_prepostprocessor_build(preprocess, &model));
         EXPECT_NE(nullptr, model);
 

src/bindings/python/tests/test_graph/test_manager.py

@@ -48,9 +48,22 @@ def test_constant_folding():
 
 
 # request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request
-def test_serialize_seperate_paths_kwargs(request, tmp_path):
+@pytest.fixture
-    core = Core()
+def prepare_ir_paths(request, tmp_path):
     xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path)
+
+    yield xml_path, bin_path
+    
+    # IR Files deletion should be done after `Model` is destructed.
+    # It may be achieved by splitting scopes (`Model` will be destructed 
+    # just after test scope finished), or by calling `del Model`
+    os.remove(xml_path)
+    os.remove(bin_path)
+
+
+def test_serialize_separate_paths_kwargs(prepare_ir_paths):
+    core = Core()
+
     shape = [2, 2]
     parameter_a = ops.parameter(shape, dtype=np.float32, name="A")
     parameter_b = ops.parameter(shape, dtype=np.float32, name="B")
@@ -58,6 +71,7 @@ def test_serialize_seperate_paths_kwargs(request, tmp_path):
     model = (parameter_a + parameter_b) * parameter_c
     func = Model(model, [parameter_a, parameter_b, parameter_c], "Model")
 
+    xml_path, bin_path = prepare_ir_paths
     pass_manager = Manager()
     pass_manager.register_pass(Serialize(path_to_xml=xml_path, path_to_bin=bin_path))
     pass_manager.run_passes(func)
@@ -67,14 +81,10 @@ def test_serialize_seperate_paths_kwargs(request, tmp_path):
     assert func.get_parameters() == res_model.get_parameters()
     assert func.get_ordered_ops() == res_model.get_ordered_ops()
 
-    os.remove(xml_path)
-    os.remove(bin_path)
 
-
+def test_serialize_separate_paths_args(prepare_ir_paths):
-# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request
-def test_serialize_seperate_paths_args(request, tmp_path):
     core = Core()
-    xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path)
+
     shape = [2, 2]
     parameter_a = ops.parameter(shape, dtype=np.float32, name="A")
     parameter_b = ops.parameter(shape, dtype=np.float32, name="B")
@@ -83,6 +93,7 @@ def test_serialize_seperate_paths_args(request, tmp_path):
     model = ((parameter_a + parameter_b) * parameter_c) / parameter_d
     func = Model(model, [parameter_a, parameter_b, parameter_c, parameter_d], "Model")
 
+    xml_path, bin_path = prepare_ir_paths
     pass_manager = Manager()
     pass_manager.register_pass(Serialize(xml_path, bin_path))
     pass_manager.run_passes(func)
@@ -92,20 +103,17 @@ def test_serialize_seperate_paths_args(request, tmp_path):
     assert func.get_parameters() == res_model.get_parameters()
     assert func.get_ordered_ops() == res_model.get_ordered_ops()
 
-    os.remove(xml_path)
-    os.remove(bin_path)
 
-
+def test_serialize_pass_mixed_args_kwargs(prepare_ir_paths):
-# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request
-def test_serialize_pass_mixed_args_kwargs(request, tmp_path):
     core = Core()
-    xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path)
+    
     shape = [3, 2]
     parameter_a = ops.parameter(shape, dtype=np.float32, name="A")
     parameter_b = ops.parameter(shape, dtype=np.float32, name="B")
     model = parameter_a - parameter_b
     func = Model(model, [parameter_a, parameter_b], "Model")
 
+    xml_path, bin_path = prepare_ir_paths
     pass_manager = Manager()
     pass_manager.register_pass(Serialize(xml_path, path_to_bin=bin_path))
     pass_manager.run_passes(func)
@@ -115,14 +123,11 @@ def test_serialize_pass_mixed_args_kwargs(request, tmp_path):
     assert func.get_parameters() == res_model.get_parameters()
     assert func.get_ordered_ops() == res_model.get_ordered_ops()
 
-    os.remove(xml_path)
-    os.remove(bin_path)
 
-
+def test_serialize_pass_mixed_args_kwargs_v2(prepare_ir_paths):
-# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request
-def test_serialize_pass_mixed_args_kwargs_v2(request, tmp_path):
     core = Core()
-    xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path)
+    
+    xml_path, bin_path = prepare_ir_paths
     model = create_model()
     pass_manager = Manager()
     pass_manager.register_pass(Serialize(path_to_xml=xml_path, path_to_bin=bin_path))
@@ -133,9 +138,6 @@ def test_serialize_pass_mixed_args_kwargs_v2(request, tmp_path):
     assert model.get_parameters() == res_model.get_parameters()
     assert model.get_ordered_ops() == res_model.get_ordered_ops()
 
-    os.remove(xml_path)
-    os.remove(bin_path)
-
 
 # request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request
 def test_serialize_pass_wrong_num_of_args(request, tmp_path):
@@ -147,14 +149,13 @@ def test_serialize_pass_wrong_num_of_args(request, tmp_path):
     assert "Invoked with:" in str(e.value)
 
 
-# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request
+def test_serialize_results(prepare_ir_paths):
-def test_serialize_results(request, tmp_path):
     core = Core()
     node_constant = ops.constant(np.array([[0.0, 0.1, -0.1], [-2.5, 2.5, 3.0]], dtype=np.float32))
     node_ceil = ops.ceiling(node_constant)
     func = Model(node_ceil, [], "Model")
 
-    xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path)
+    xml_path, bin_path = prepare_ir_paths
     pass_manager = Manager()
     pass_manager.register_pass(Serialize(path_to_xml=xml_path, path_to_bin=bin_path))
     pass_manager.run_passes(func)
@@ -165,14 +166,32 @@ def test_serialize_results(request, tmp_path):
 
     assert const == new_const
 
-    os.remove(xml_path)
+    import gc
-    os.remove(bin_path)
+    gc.collect()    # CVS-106805
 
 
-# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request
+@pytest.mark.xfail(reason="CVS-106805. Delete test after fix")
-def test_default_version(request, tmp_path):
+def test_serialize_results_fails(prepare_ir_paths):
     core = Core()
-    xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path)
+    node_constant = ops.constant(np.array([[0.0, 0.1, -0.1], [-2.5, 2.5, 3.0]], dtype=np.float32))
+    func = Model(node_constant, [], "Model")
+
+    xml_path, bin_path = prepare_ir_paths
+    pass_manager = Manager()
+    pass_manager.register_pass(Serialize(path_to_xml=xml_path, path_to_bin=bin_path))
+    pass_manager.run_passes(func)
+
+    res_model = core.read_model(model=xml_path, weights=bin_path)
+    const = func.get_results()
+    new_const = res_model.get_results()
+
+    assert const == new_const
+
+
+def test_default_version(prepare_ir_paths):
+    core = Core()
+
+    xml_path, bin_path = prepare_ir_paths
     model = create_model()
     pass_manager = Manager()
     pass_manager.register_pass(Serialize(xml_path, bin_path))
@@ -183,14 +202,11 @@ def test_default_version(request, tmp_path):
     assert model.get_parameters() == res_model.get_parameters()
     assert model.get_ordered_ops() == res_model.get_ordered_ops()
 
-    os.remove(xml_path)
-    os.remove(bin_path)
 
-
+def test_default_version_IR_V11_seperate_paths(prepare_ir_paths):
-# request - https://docs.pytest.org/en/7.1.x/reference/reference.html#request
-def test_default_version_IR_V11_seperate_paths(request, tmp_path):
     core = Core()
-    xml_path, bin_path = create_filename_for_test(request.node.name, tmp_path)
+
+    xml_path, bin_path = prepare_ir_paths
     model = create_model()
     pass_manager = Manager()
     pass_manager.register_pass(Serialize(path_to_xml=xml_path, path_to_bin=bin_path, version=Version.IR_V11))
@@ -200,6 +216,3 @@ def test_default_version_IR_V11_seperate_paths(request, tmp_path):
 
     assert model.get_parameters() == res_model.get_parameters()
     assert model.get_ordered_ops() == res_model.get_ordered_ops()
-
-    os.remove(xml_path)
-    os.remove(bin_path)

src/core/tests/frontend/mock_frontend.cpp

@@ -134,7 +134,9 @@ public:
     }
 
     bool supported_impl(const std::vector<ov::Any>& variants) const override {
-        if (variants.size() == 1 && variants[0].is<std::string>()) {
+        // Last boolean flag in `variants` (if presented) is reserved for FE configuration
+        size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is<bool>() ? 1 : 0;
+        if (variants.size() == 1 + extra_variants_num && variants[0].is<std::string>()) {
             std::string command = variants[0].as<std::string>();
             FRONT_END_GENERAL_CHECK(command != "throw_now", "Test exception");
         }
@@ -146,8 +148,10 @@ public:
     }
 
     InputModel::Ptr load_impl(const std::vector<ov::Any>& variants) const override {
+        // Last boolean flag in `variants` (if presented) is reserved for FE configuration
+        size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is<bool>() ? 1 : 0;
         auto input_model = std::make_shared<InputModelMock>();
-        if (variants.size() == 1 && variants[0].is<std::string>()) {
+        if (variants.size() == 1 + extra_variants_num && variants[0].is<std::string>()) {
             std::string command = variants[0].as<std::string>();
             if (command == "throw_now") {
                 OPENVINO_THROW("Test throw load input model");

src/frontends/ir/src/frontend.cpp

@@ -61,10 +61,12 @@ size_t get_ir_version(std::istream& model) {
 }  // namespace
 
 bool FrontEnd::supported_impl(const std::vector<ov::Any>& variants) const {
+    // Last boolean flag in `variants` (if presented) is reserved for FE configuration
+    size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is<bool>() ? 1 : 0;
     std::ifstream local_model_stream;
     std::istream* provided_model_stream = nullptr;
 
-    if (variants.empty() || variants.size() > 3) {
+    if (variants.empty() || variants.size() > 3 + extra_variants_num) {
         return false;
     }
 
@@ -181,6 +183,8 @@ InputModel::Ptr FrontEnd::load_impl(const std::vector<ov::Any>& variants) const
             weights = variant.as<std::shared_ptr<ngraph::runtime::AlignedBuffer>>();
         }
     }
+    bool use_map_allocator =
+        variants[variants.size() - 1].is<bool>() ? variants[variants.size() - 1].as<bool>() : false;
 
     // Find weights if only path to xml was provided
     if (weights_path.empty()) {
@@ -198,27 +202,31 @@ InputModel::Ptr FrontEnd::load_impl(const std::vector<ov::Any>& variants) const
         }
     }
     if (!weights_path.empty()) {
-        std::ifstream bin_stream;
+        if (use_map_allocator)
-        bin_stream.open(weights_path, std::ios::binary);
+            weights = ov::load_mmap_object(weights_path);
-        if (!bin_stream.is_open())
+        else {
+            std::ifstream bin_stream;
+            bin_stream.open(weights_path, std::ios::binary);
+            if (!bin_stream.is_open())
 #if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32)
-            IE_THROW() << "Weights file " + ov::util::wstring_to_string(weights_path) + " cannot be opened!";
+                IE_THROW() << "Weights file " + ov::util::wstring_to_string(weights_path) + " cannot be opened!";
 #else
-            IE_THROW() << "Weights file " + weights_path + " cannot be opened!";
+                IE_THROW() << "Weights file " + weights_path + " cannot be opened!";
 #endif
 
-        bin_stream.seekg(0, std::ios::end);
+            bin_stream.seekg(0, std::ios::end);
-        size_t file_size = bin_stream.tellg();
+            size_t file_size = bin_stream.tellg();
-        bin_stream.seekg(0, std::ios::beg);
+            bin_stream.seekg(0, std::ios::beg);
 
-        auto aligned_weights_buffer = std::make_shared<ngraph::runtime::AlignedBuffer>(file_size);
+            auto aligned_weights_buffer = std::make_shared<ngraph::runtime::AlignedBuffer>(file_size);
-        bin_stream.read(aligned_weights_buffer->get_ptr<char>(), aligned_weights_buffer->size());
+            bin_stream.read(aligned_weights_buffer->get_ptr<char>(), aligned_weights_buffer->size());
-        bin_stream.close();
+            bin_stream.close();
 
-        weights = std::make_shared<ngraph::runtime::SharedBuffer<std::shared_ptr<ngraph::runtime::AlignedBuffer>>>(
+            weights = std::make_shared<ngraph::runtime::SharedBuffer<std::shared_ptr<ngraph::runtime::AlignedBuffer>>>(
-            aligned_weights_buffer->get_ptr<char>(),
+                aligned_weights_buffer->get_ptr<char>(),
-            aligned_weights_buffer->size(),
+                aligned_weights_buffer->size(),
-            aligned_weights_buffer);
+                aligned_weights_buffer);
+        }
     }
 
     return create_input_model();

src/frontends/ir/src/os/win/win_mmap_object.cpp

@@ -58,12 +58,16 @@ public:
     }
 
     void set(const std::string& path) {
+        // Note that file can't be changed (renamed/deleted) until it's unmapped. FILE_SHARE_DELETE flag allow 
+        // rename/deletion, but it doesn't work with FAT32 filesystem (works on NTFS)
         auto h = ::CreateFileA(path.c_str(), GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
         map(path, h);
     }
 
 #ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT
     void set(const std::wstring& path) {
+        // Note that file can't be changed (renamed/deleted) until it's unmapped. FILE_SHARE_DELETE flag allow 
+        // rename/deletion, but it doesn't work with FAT32 filesystem (works on NTFS)
         auto h = ::CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
         map(ov::util::wstring_to_string(path), h);
     }

src/frontends/paddle/src/frontend.cpp

@@ -350,8 +350,10 @@ void FrontEnd::fuse_fakequantize_ops(const std::vector<std::shared_ptr<Model>>&
 }
 
 bool FrontEnd::supported_impl(const std::vector<ov::Any>& variants) const {
+    // Last boolean flag in `variants` (if presented) is reserved for FE configuration
+    size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is<bool>() ? 1 : 0;
     // FrontEnd can only load model specified by one path, one file or two files.
-    if (variants.empty() || variants.size() > 2)
+    if (variants.empty() || variants.size() > 2 + extra_variants_num)
         return false;
 
     // Validating first path, it must contain a model
@@ -389,7 +391,9 @@ bool FrontEnd::supported_impl(const std::vector<ov::Any>& variants) const {
 }
 
 InputModel::Ptr FrontEnd::load_impl(const std::vector<ov::Any>& variants) const {
-    if (variants.size() == 1) {
+    // Last boolean flag in `variants` (if presented) is reserved for FE configuration
+    size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is<bool>() ? 1 : 0;
+    if (variants.size() == 1 + extra_variants_num) {
         // The case when folder with __model__ and weight files is provided or .pdmodel file
         if (variants[0].is<std::string>()) {
             std::string m_path = variants[0].as<std::string>();
@@ -407,7 +411,7 @@ InputModel::Ptr FrontEnd::load_impl(const std::vector<ov::Any>& variants) const
             auto p_model_stream = variants[0].as<std::istream*>();
             return std::make_shared<InputModel>(std::vector<std::istream*>{p_model_stream}, m_telemetry);
         }
-    } else if (variants.size() == 2) {
+    } else if (variants.size() == 2 + extra_variants_num) {
         // The case when .pdmodel and .pdparams files are provided
         std::ifstream model_stream;
         std::ifstream weights_stream;

src/frontends/pytorch/src/frontend.cpp

@@ -141,15 +141,19 @@ void FrontEnd::add_extension(const std::shared_ptr<ov::Extension>& extension) {
 }
 
 bool FrontEnd::supported_impl(const std::vector<ov::Any>& variants) const {
+    // Last boolean flag in `variants` (if presented) is reserved for FE configuration
+    size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is<bool>() ? 1 : 0;
     // Currently PyTorch FrontEnd only support TorchDecoder as input
-    if (variants.size() != 1 || !variants[0].is<std::shared_ptr<IDecoder>>())
+    if (variants.size() != 1 + extra_variants_num || !variants[0].is<std::shared_ptr<IDecoder>>())
         return false;
     auto decoder = variants[0].as<std::shared_ptr<IDecoder>>();
     return decoder && std::dynamic_pointer_cast<TorchDecoder>(decoder);
 }
 
 ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector<ov::Any>& variants) const {
-    FRONT_END_GENERAL_CHECK(variants.size() == 1,
+    // Last boolean flag in `variants` (if presented) is reserved for FE configuration
+    size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is<bool>() ? 1 : 0;
+    FRONT_END_GENERAL_CHECK(variants.size() == 1 + extra_variants_num,
                             "PyTorch Frontend supports exactly one parameter in model representation, got ",
                             std::to_string(variants.size()),
                             " instead.");

src/frontends/tensorflow/src/frontend.cpp

@@ -77,8 +77,10 @@ FrontEnd::FrontEnd() : m_op_translators(tensorflow::op::get_supported_ops()) {}
 
 /// \brief Check if FrontEndTensorflow can recognize model from given parts
 bool FrontEnd::supported_impl(const std::vector<ov::Any>& variants) const {
+    // Last boolean flag in `variants` (if presented) is reserved for FE configuration
+    size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is<bool>() ? 1 : 0;
     // TODO: Support other TensorFlow formats: SavedModel, .meta, checkpoint, pbtxt
-    if (variants.size() != 1)
+    if (variants.size() != 1 + extra_variants_num)
         return false;
 
     if (variants[0].is<std::string>()) {
@@ -115,7 +117,10 @@ bool FrontEnd::supported_impl(const std::vector<ov::Any>& variants) const {
 
 ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector<ov::Any>& variants) const {
     // TODO: Support other TensorFlow formats: SavedModel, .meta, checkpoint, pbtxt
-    FRONT_END_GENERAL_CHECK(variants.size() == 1,
+
+    // Last boolean flag in `variants` (if presented) is reserved for FE configuration
+    size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is<bool>() ? 1 : 0;
+    FRONT_END_GENERAL_CHECK(variants.size() == 1 + extra_variants_num,
                             "[TensorFlow Frontend] Internal error or inconsistent input model: the frontend supports "
                             "only frozen binary protobuf format.");
 

src/frontends/tensorflow_lite/src/frontend.cpp

@@ -48,7 +48,9 @@ FrontEnd::FrontEnd() {
 
 /// \brief Check if FrontEndTensorflowLite can recognize model from given parts
 bool FrontEnd::supported_impl(const std::vector<ov::Any>& variants) const {
-    if (variants.size() != 1)
+    // Last boolean flag in `variants` (if presented) is reserved for FE configuration
+    size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is<bool>() ? 1 : 0;
+    if (variants.size() != 1 + extra_variants_num)
         return false;
 
     if (variants[0].is<std::string>()) {
@@ -71,7 +73,9 @@ bool FrontEnd::supported_impl(const std::vector<ov::Any>& variants) const {
 }
 
 ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector<ov::Any>& variants) const {
-    if (variants.size() == 1) {
+    // Last boolean flag in `variants` (if presented) is reserved for FE configuration
+    size_t extra_variants_num = variants.size() > 0 && variants[variants.size() - 1].is<bool>() ? 1 : 0;
+    if (variants.size() == 1 + extra_variants_num) {
         if (variants[0].is<std::string>()) {
             std::string suffix = ".tflite";
             std::string model_path = variants[0].as<std::string>();

src/inference/src/ie_network_reader.cpp

@@ -457,6 +457,7 @@ CNNNetwork details::ReadNetwork(const std::string& modelPath,
 #endif
         params.emplace_back(weights_path);
     }
+    params.emplace_back(/*use_ir_frontend_map_allocator=*/true);
 
     FE = manager.load_by_model(params);
     if (FE) {