[PYTHON API] release GIL (#10810)

* AsyncInferQueue nogil update + refactoring

* nogil in compiled model

* nogil in Core

* fix refactoring

* nogil in infer_request

* add tests

* Fix code style

* update test with incrementing reference counting

* try to fix code style

* fix code style

* release gil in reshape and preprocessing

* make args optional in test

* fix code style

* add docs about GIL

* try to link doc string with docs

* Apply suggestions from code review

Co-authored-by: Jan Iwaszkiewicz <jan.iwaszkiewicz@intel.com>

* Fix docs

* docs refactoring

* Apply review comments

* Fix code style

Co-authored-by: Jan Iwaszkiewicz <jan.iwaszkiewicz@intel.com>
Co-authored-by: Anastasia Kuporosova <anastasia.kuporosova@intel.com>

docs/OV_Runtime_UG/Python_API_exclusives.md

@@ -75,3 +75,36 @@ Another feature of `AsyncInferQueue` is ability of setting callbacks. When callb
 The callback of `AsyncInferQueue` is uniform for every job. When executed, GIL is acquired to ensure safety of data manipulation inside the function.
 
 @snippet docs/snippets/ov_python_exclusives.py asyncinferqueue_set_callback
+
+
+### Releasing the GIL
+
+Some functions in Python API release the Global Lock Interpreter (GIL) while running work-intensive code. It can help you to achieve more parallelism in your application using Python threads. For more information about GIL please refer to the Python documentation.
+
+@snippet docs/snippets/ov_python_exclusives.py releasing_gil
+
+> **NOTE**: While GIL is released functions can still modify and/or operate on Python objects in C++, thus there is no reference counting. User is responsible for thread safety if sharing of these objects with other thread occurs. It can affects your code only if multiple threads are spawned in Python.:
+
+#### List of functions that release the GIL
+- openvino.runtime.AsyncInferQueue.start_async
+- openvino.runtime.AsyncInferQueue.is_ready
+- openvino.runtime.AsyncInferQueue.wait_all
+- openvino.runtime.AsyncInferQueue.get_idle_request_id
+- openvino.runtime.CompiledModel.create_infer_request
+- openvino.runtime.CompiledModel.infer_new_request
+- openvino.runtime.CompiledModel.__call__
+- openvino.runtime.CompiledModel.export
+- openvino.runtime.CompiledModel.get_runtime_model
+- openvino.runtime.Core.compile_model
+- openvino.runtime.Core.read_model
+- openvino.runtime.Core.import_model
+- openvino.runtime.Core.query_model
+- openvino.runtime.Core.get_available_devices
+- openvino.runtime.InferRequest.infer
+- openvino.runtime.InferRequest.start_async
+- openvino.runtime.InferRequest.wait
+- openvino.runtime.InferRequest.wait_for
+- openvino.runtime.InferRequest.get_profiling_info
+- openvino.runtime.InferRequest.query_state
+- openvino.runtime.Model.reshape
+- openvino.preprocess.PrePostProcessor.build

docs/snippets/ov_python_exclusives.py

@@ -131,3 +131,39 @@ infer_queue.wait_all()
 
 assert all(data_done)
 #! [asyncinferqueue_set_callback]
+
+#! [releasing_gil]
+import openvino.runtime as ov
+import cv2 as cv
+from threading import Thread
+
+input_data = []
+
+# Processing input data will be done in a separate thread
+# while compilation of the model and creation of the infer request
+# is going to be executed in the main thread.
+def prepare_data(input, image_path):
+    image = cv.imread(image_path)
+    h, w = list(input.shape)[-2:]
+    image = cv.resize(image, (h, w))
+    image = image.transpose((2, 0, 1))
+    image = np.expand_dims(image, 0)
+    input_data.append(image)
+
+core = ov.Core()
+model = core.read_model("model.xml")
+# Create thread with prepare_data function as target and start it
+thread = Thread(target=prepare_data, args=[model.input(), "path/to/image"])
+thread.start()
+# The GIL will be released in compile_model.
+# It allows a thread above to start the job,
+# while main thread is running in the background.
+compiled = core.compile_model(model, "GPU")
+# After returning from compile_model, the main thread acquires the GIL
+# and starts create_infer_request which releases it once again.
+request = compiled.create_infer_request()
+# Join the thread to make sure the input_data is ready
+thread.join()
+# running the inference
+request.infer(input_data)
+#! [releasing_gil]

src/bindings/python/src/pyopenvino/core/async_infer_queue.cpp

@@ -147,7 +147,6 @@ void regclass_AsyncInferQueue(py::module m) {
                 if (jobs == 0) {
                     jobs = (size_t)Common::get_optimal_number_of_requests(model);
                 }
-
                 std::vector<InferRequestWrapper> requests;
                 std::queue<size_t> idle_handles;
                 std::vector<py::object> user_ids(jobs);
@@ -161,7 +160,6 @@ void regclass_AsyncInferQueue(py::module m) {
                     requests.push_back(request);
                     idle_handles.push(handle);
                 }
-
                 return new AsyncInferQueue(requests, idle_handles, user_ids);
             }),
             py::arg("model"),
@@ -212,71 +210,61 @@ void regclass_AsyncInferQueue(py::module m) {
             :type inputs: dict[Union[int, str, openvino.runtime.ConstOutput] : openvino.runtime.Tensor]
             :param userdata: Any data that will be passed to a callback
             :rtype: None
+
+            GIL is released while waiting for the next available InferRequest.
         )");
 
-    cls.def(
+    cls.def("is_ready",
-        "is_ready",
+            &AsyncInferQueue::_is_ready,
-        [](AsyncInferQueue& self) {
+            R"(
-            return self._is_ready();
-        },
-        R"(
             One of 'flow control' functions.
             Returns True if any free request in the pool, otherwise False.
 
-            Function releases GIL, other threads can work while this function waits.
+            GIL is released while running this function.
 
             :return: If there is at least one free InferRequest in a pool, returns True.
             :rtype: bool
     )");
 
-    cls.def(
+    cls.def("wait_all",
-        "wait_all",
+            &AsyncInferQueue::wait_all,
-        [](AsyncInferQueue& self) {
+            R"(
-            return self.wait_all();
+            One of 'flow control' functions. Blocking call.
-        },
+            Waits for all InferRequests in a pool to finish scheduled work.
-        R"(
-        One of 'flow control' functions. Blocking call.
-        Waits for all InferRequests in a pool to finish scheduled work. 
 
-        Function releases GIL, other threads can work while this function waits.
+            GIL is released while running this function.
-    )");
+        )");
 
-    cls.def(
+    cls.def("get_idle_request_id",
-        "get_idle_request_id",
+            &AsyncInferQueue::get_idle_request_id,
-        [](AsyncInferQueue& self) {
+            R"(
-            return self.get_idle_request_id();
+            Returns next free id of InferRequest from queue's pool.
-        },
+            Function waits for any request to complete and then returns this request's id.
-        R"(
-        Returns next free id of InferRequest from queue's pool.
-        Function waits for any request to complete and then returns this request's id.
 
-        Function releases GIL, other threads can work while this function waits.
+            GIL is released while running this function.
 
-        :rtype: int
+            :rtype: int
-    )");
+        )");
 
-    cls.def(
+    cls.def("set_callback",
-        "set_callback",
+            &AsyncInferQueue::set_custom_callbacks,
-        [](AsyncInferQueue& self, py::function callback) {
+            R"(
-            self.set_custom_callbacks(callback);
+            Sets unified callback on all InferRequests from queue's pool.
-        },
+            Signature of such function should have two arguments, where
-        R"(
+            first one is InferRequest object and second one is userdata
-        Sets unified callback on all InferRequests from queue's pool.
+            connected to InferRequest from the AsyncInferQueue's pool.
-        The signature of such function should have two arguments, where
-        the first one is InferRequest object and the second one is userdata
-        connected to InferRequest from the AsyncInferQueue's pool.
 
-        .. code-block:: python
+            .. code-block:: python
 
-            def f(request, userdata):
+                def f(request, userdata):
-                result = request.output_tensors[0]
+                    result = request.output_tensors[0]
-                print(result + userdata)
+                    print(result + userdata)
 
-            async_infer_queue.set_callback(f)
+                async_infer_queue.set_callback(f)
 
-        :param callback: Any Python defined function that matches callback's requirements.
+            :param callback: Any Python defined function that matches callback's requirements.
-        :type callback: function
+            :type callback: function
-    )");
+        )");
 
     cls.def(
         "__len__",

src/bindings/python/src/pyopenvino/core/compiled_model.cpp

@@ -33,6 +33,7 @@ void regclass_CompiledModel(py::module m) {
         [](ov::CompiledModel& self) {
             return std::make_shared<InferRequestWrapper>(self.create_infer_request(), self.inputs(), self.outputs());
         },
+        py::call_guard<py::gil_scoped_release>(),
         R"(
             Creates an inference request object used to infer the compiled model.
             The created request has allocated input and output tensors.
@@ -47,7 +48,10 @@ void regclass_CompiledModel(py::module m) {
             auto request = self.create_infer_request();
             // Update inputs if there are any
             Common::set_request_tensors(request, inputs);
-            request.infer();
+            {
+                py::gil_scoped_release release;
+                request.infer();
+            }
             return Common::outputs_to_dict(self.outputs(), request);
         },
         py::arg("inputs"),
@@ -59,6 +63,8 @@ void regclass_CompiledModel(py::module m) {
             It is advised to use a dedicated InferRequest class for performance,
             optimizing workflows, and creating advanced pipelines.
 
+            GIL is released during the inference.
+
             :param inputs: Data to set on input tensors.
             :type inputs: Dict[Union[int, str, openvino.runtime.ConstOutput], openvino.runtime.Tensor]
             :return: Dictionary of results from output tensors with ports as keys.
@@ -72,9 +78,12 @@ void regclass_CompiledModel(py::module m) {
             self.export_model(_stream);
             return py::bytes(_stream.str());
         },
+        py::call_guard<py::gil_scoped_release>(),
         R"(
             Exports the compiled model to bytes/output stream.
 
+            GIL is released while running this function.
+
             :return: Bytes object that contains this compiled model.
             :rtype: bytes
 
@@ -99,7 +108,10 @@ void regclass_CompiledModel(py::module m) {
                                      (std::string)(py::repr(model_stream)) + "` provided");
             }
             std::stringstream _stream;
-            self.export_model(_stream);
+            {
+                py::gil_scoped_release release;
+                self.export_model(_stream);
+            }
             model_stream.attr("flush")();
             model_stream.attr("write")(py::bytes(_stream.str()));
             model_stream.attr("seek")(0);  // Always rewind stream!
@@ -114,6 +126,8 @@ void regclass_CompiledModel(py::module m) {
             Function performs flushing of the stream, writes to it, and then rewinds
             the stream to the beginning (using seek(0)).
 
+            GIL is released while running this function.
+
             :param model_stream: A stream object to which the model will be serialized.
             :type model_stream: io.BytesIO
             :rtype: None
@@ -165,6 +179,7 @@ void regclass_CompiledModel(py::module m) {
 
     cls.def("get_runtime_model",
             &ov::CompiledModel::get_runtime_model,
+            py::call_guard<py::gil_scoped_release>(),
             R"(
                 Gets runtime model information from a device.
 

src/bindings/python/src/pyopenvino/core/core.cpp

@@ -77,6 +77,7 @@ void regclass_Core(py::module m) {
            const std::map<std::string, std::string>& properties) {
             return self.compile_model(model, device_name, {properties.begin(), properties.end()});
         },
+        py::call_guard<py::gil_scoped_release>(),
         py::arg("model"),
         py::arg("device_name"),
         py::arg("config") = py::dict(),
@@ -85,6 +86,8 @@ void regclass_Core(py::module m) {
             Users can create as many compiled models as they need, and use them simultaneously
             (up to the limitation of the hardware resources).
 
+            GIL is released while running this function.
+
             :param model: Model acquired from read_model function.
             :type model: openvino.runtime.Model
             :param device_name: Name of the device which will load the model.
@@ -102,6 +105,7 @@ void regclass_Core(py::module m) {
            const std::map<std::string, std::string>& config) {
             return self.compile_model(model, ov::AnyMap{config.begin(), config.end()});
         },
+        py::call_guard<py::gil_scoped_release>(),
         py::arg("model"),
         py::arg("config") = py::dict(),
         R"(
@@ -109,6 +113,8 @@ void regclass_Core(py::module m) {
             selected by AUTO plugin. Users can create as many compiled models as they need, and use
             them simultaneously (up to the limitation of the hardware resources).
 
+            GIL is released while running this function.
+
             :param model: Model acquired from read_model function.
             :type model: openvino.runtime.Model
             :param properties: Optional dict of pairs: (property name, property value) relevant only for this load operation.
@@ -125,6 +131,7 @@ void regclass_Core(py::module m) {
            const std::map<std::string, std::string>& config) {
             return self.compile_model(model_path, device_name, {config.begin(), config.end()});
         },
+        py::call_guard<py::gil_scoped_release>(),
         py::arg("model_path"),
         py::arg("device_name"),
         py::arg("properties") = py::dict(),
@@ -133,6 +140,8 @@ void regclass_Core(py::module m) {
             This can be more efficient than using read_model + compile_model(model_in_memory_object) flow,
             especially for cases when caching is enabled and cached model is available.
 
+            GIL is released while running this function.
+
             :param model_path: A path to a model in IR / ONNX / PDPD format.
             :type model_path: str
             :param device_name: Name of the device to load the model to.
@@ -148,6 +157,7 @@ void regclass_Core(py::module m) {
         [](ov::Core& self, const std::string& model_path, const std::map<std::string, std::string>& properties) {
             return self.compile_model(model_path, ov::AnyMap{properties.begin(), properties.end()});
         },
+        py::call_guard<py::gil_scoped_release>(),
         py::arg("model_path"),
         py::arg("config") = py::dict(),
         R"(
@@ -155,6 +165,8 @@ void regclass_Core(py::module m) {
             This can be more efficient than using read_model + compile_model(model_in_memory_object) flow,
             especially for cases when caching is enabled and cached model is available.
 
+            GIL is released while running this function.
+
             :param model_path: A path to a model in IR / ONNX / PDPD format.
             :type model_path: str
             :param properties: Optional dict of pairs: (property name, property value) relevant only for this load operation.
@@ -178,25 +190,26 @@ void regclass_Core(py::module m) {
     cls.def(
         "read_model",
         [](ov::Core& self, py::bytes model, py::bytes weights) {
+            std::string ir(model);
             // works on view in order to omit copying bytes into string
             py::buffer_info info(py::buffer(weights).request());
             size_t bin_size = static_cast<size_t>(info.size);
+            ov::Tensor tensor(ov::element::Type_t::u8, {bin_size});
             // if weights are not empty
             if (bin_size) {
                 const uint8_t* bin = reinterpret_cast<const uint8_t*>(info.ptr);
-                ov::Tensor tensor(ov::element::Type_t::u8, {bin_size});
                 std::memcpy(tensor.data(), bin, bin_size);
-                return self.read_model(model, tensor);
             }
-            // create empty tensor of type u8
+            py::gil_scoped_release release;
-            ov::Tensor tensor(ov::element::Type_t::u8, {});
+            return self.read_model(ir, tensor);
-            return self.read_model(model, tensor);
         },
         py::arg("model"),
         py::arg("weights") = py::bytes(),
         R"(
             Reads models from IR / ONNX / PDPD formats.
 
+            GIL is released while running this function.
+
             :param model: Bytes with model in IR / ONNX / PDPD format.
             :type model: bytes
             :param weights: Bytes with tensor's data.
@@ -208,11 +221,14 @@ void regclass_Core(py::module m) {
     cls.def(
         "read_model",
         (std::shared_ptr<ov::Model>(ov::Core::*)(const std::string&, const std::string&) const) & ov::Core::read_model,
+        py::call_guard<py::gil_scoped_release>(),
         py::arg("model"),
         py::arg("weights") = "",
         R"(
             Reads models from IR / ONNX / PDPD formats.
 
+            GIL is released while running this function.
+
             :param model: A path to a model in IR / ONNX / PDPD format.
             :type model: str
             :param weights: A path to a data file For IR format (*.bin): if path is empty,
@@ -228,11 +244,14 @@ void regclass_Core(py::module m) {
     cls.def(
         "read_model",
         (std::shared_ptr<ov::Model>(ov::Core::*)(const std::string&, const ov::Tensor&) const) & ov::Core::read_model,
+        py::call_guard<py::gil_scoped_release>(),
         py::arg("model"),
         py::arg("weights"),
         R"(
             Reads models from IR / ONNX / PDPD formats.
 
+            GIL is released while running this function.
+
             :param model: A string with model in IR / ONNX / PDPD format.
             :type model: str
             :param weights: Tensor with weights. Reading ONNX / PDPD models doesn't support
@@ -244,14 +263,19 @@ void regclass_Core(py::module m) {
 
     cls.def(
         "read_model",
-        [](ov::Core& self, py::object model, py::object weights) {
+        [](ov::Core& self, py::object model_path, py::object weights_path) {
-            return self.read_model(py::str(model), py::str(weights));
+            std::string model_path_cpp{py::str(model_path)};
+            std::string weights_path_cpp{py::str(weights_path)};
+            py::gil_scoped_release release;
+            return self.read_model(model_path_cpp, weights_path_cpp);
         },
         py::arg("model"),
         py::arg("weights") = "",
         R"(
             Reads models from IR / ONNX / PDPD formats.
 
+            GIL is released while running this function.
+
             :param model: A string with model in IR / ONNX / PDPD format.
             :type model: str
             :param weights: A path to a data file For IR format (*.bin): if path is empty,
@@ -273,12 +297,15 @@ void regclass_Core(py::module m) {
             _stream << model_stream;
             return self.import_model(_stream, device_name, {properties.begin(), properties.end()});
         },
+        py::call_guard<py::gil_scoped_release>(),
         py::arg("model_stream"),
         py::arg("device_name"),
         py::arg("properties") = py::none(),
         R"(
             Imports a compiled model from a previously exported one.
 
+            GIL is released while running this function.
+
             :param model_stream: Input stream, containing a model previously exported, using export_model method.
             :type model_stream: bytes
             :param device_name: Name of device to which compiled model is imported.
@@ -319,6 +346,7 @@ void regclass_Core(py::module m) {
             _stream << model_stream
                            .attr("read")()  // alternative: model_stream.attr("get_value")()
                            .cast<std::string>();
+            py::gil_scoped_release release;
             return self.import_model(_stream, device_name, {properties.begin(), properties.end()});
         },
         py::arg("model_stream"),
@@ -330,6 +358,8 @@ void regclass_Core(py::module m) {
             Advanced version of `import_model`. It utilizes, streams from standard
             Python library `io`.
 
+            GIL is released while running this function.
+
 
             :param model_stream: Input stream, containing a model previously exported, using export_model method.
             :type model_stream: io.BytesIO
@@ -419,12 +449,15 @@ void regclass_Core(py::module m) {
            const std::map<std::string, std::string>& properties) {
             return self.query_model(model, device_name, {properties.begin(), properties.end()});
         },
+        py::call_guard<py::gil_scoped_release>(),
         py::arg("model"),
         py::arg("device_name"),
         py::arg("properties") = py::dict(),
         R"(
             Query device if it supports specified model with specified properties.
 
+            GIL is released while running this function.
+
             :param model: Model object to query.
             :type model: openvino.runtime.Model
             :param device_name: A name of a device to query.
@@ -468,9 +501,12 @@ void regclass_Core(py::module m) {
 
     cls.def_property_readonly("available_devices",
                               &ov::Core::get_available_devices,
+                              py::call_guard<py::gil_scoped_release>(),
                               R"(
                                     Returns devices available for inference Core objects goes over all registered plugins.
 
+                                    GIL is released while running this function.
+
                                     :returns: A list of devices. The devices are returned as: CPU, GPU.0, GPU.1, MYRIAD...
                                         If there more than one device of specific type, they are enumerated with .# suffix.
                                         Such enumerated device can later be used as a device name in all Core methods like:

src/bindings/python/src/pyopenvino/core/infer_request.cpp

@@ -164,9 +164,12 @@ void regclass_InferRequest(py::module m) {
             // Update inputs if there are any
             Common::set_request_tensors(self._request, inputs);
             // Call Infer function
-            self._start_time = Time::now();
+            {
-            self._request.infer();
+                py::gil_scoped_release release;
-            self._end_time = Time::now();
+                self._start_time = Time::now();
+                self._request.infer();
+                self._end_time = Time::now();
+            }
             return Common::outputs_to_dict(self._outputs, self._request);
         },
         py::arg("inputs"),
@@ -175,6 +178,8 @@ void regclass_InferRequest(py::module m) {
             Blocks all methods of InferRequest while request is running.
             Calling any method will lead to throwing exceptions.
 
+            GIL is released while running the inference.
+
             :param inputs: Data to set on input tensors.
             :type inputs: Dict[Union[int, str, openvino.runtime.ConstOutput], openvino.runtime.Tensor]
             :return: Dictionary of results from output tensors with ports as keys.
@@ -203,8 +208,7 @@ void regclass_InferRequest(py::module m) {
             Starts inference of specified input(s) in asynchronous mode.
             Returns immediately. Inference starts also immediately.
 
-            This function releases the GIL, so another Python thread can
+            GIL is released while running the inference.
-            work while this function runs in the background.
 
             Calling any method on this InferRequest while the request is
             running will lead to throwing exceptions.
@@ -234,7 +238,7 @@ void regclass_InferRequest(py::module m) {
             Waits for the result to become available. 
             Blocks until the result becomes available.
 
-            Function releases GIL, other threads can work while this function waits.
+            GIL is released while running this function.
         )");
 
     cls.def(
@@ -249,7 +253,7 @@ void regclass_InferRequest(py::module m) {
             Blocks until specified timeout has elapsed or
             the result becomes available, whichever comes first.
 
-            Function releases GIL, other threads can work while this function waits.
+            GIL is released while running this function.
 
             :param timeout: Maximum duration in milliseconds (ms) of blocking call.
             :type timeout: int
@@ -512,11 +516,14 @@ void regclass_InferRequest(py::module m) {
         [](InferRequestWrapper& self) {
             return self._request.get_profiling_info();
         },
+        py::call_guard<py::gil_scoped_release>(),
         R"(
             Queries performance is measured per layer to get feedback on what
             is the most time-consuming operation, not all plugins provide
             meaningful data.
 
+            GIL is released while running this function.
+
             :return: List of profiling information for operations in model.
             :rtype: List[openvino.runtime.ProfilingInfo]
         )");
@@ -526,9 +533,12 @@ void regclass_InferRequest(py::module m) {
         [](InferRequestWrapper& self) {
             return self._request.query_state();
         },
+        py::call_guard<py::gil_scoped_release>(),
         R"(
             Gets state control interface for given infer request.
 
+            GIL is released while running this function.
+
             :return: List of VariableState objects.
             :rtype: List[openvino.runtime.VariableState]
         )");
@@ -615,9 +625,12 @@ void regclass_InferRequest(py::module m) {
         [](InferRequestWrapper& self) {
             return self._request.get_profiling_info();
         },
+        py::call_guard<py::gil_scoped_release>(),
         R"(
             Performance is measured per layer to get feedback on the most time-consuming operation.
             Not all plugins provide meaningful data!
+
+            GIL is released while running this function.
             
             :return: Inference time.
             :rtype: List[openvino.runtime.ProfilingInfo]

src/bindings/python/src/pyopenvino/graph/model.cpp

@@ -263,8 +263,13 @@ void regclass_graph_Model(py::module m) {
         [](ov::Model& self, const ov::PartialShape& partial_shape) {
             self.reshape(partial_shape);
         },
+        py::call_guard<py::gil_scoped_release>(),
         py::arg("partial_shape"),
         R"(
+                Reshape model input.
+
+                GIL is released while running this function.
+
                 :param partial_shape: New shape.
                 :type partial_shape: openvino.runtime.PartialShape
                 :return : void
@@ -273,10 +278,16 @@ void regclass_graph_Model(py::module m) {
     model.def(
         "reshape",
         [](ov::Model& self, const py::list& partial_shape) {
-            self.reshape(Common::partial_shape_from_list(partial_shape));
+            ov::PartialShape new_shape(Common::partial_shape_from_list(partial_shape));
+            py::gil_scoped_release release;
+            self.reshape(new_shape);
         },
         py::arg("partial_shape"),
         R"(
+                Reshape model input.
+
+                GIL is released while running this function.
+
                 :param partial_shape: New shape.
                 :type partial_shape: list
                 :return : void
@@ -285,10 +296,16 @@ void regclass_graph_Model(py::module m) {
     model.def(
         "reshape",
         [](ov::Model& self, const py::tuple& partial_shape) {
-            self.reshape(Common::partial_shape_from_list(partial_shape.cast<py::list>()));
+            ov::PartialShape new_shape(Common::partial_shape_from_list(partial_shape.cast<py::list>()));
+            py::gil_scoped_release release;
+            self.reshape(new_shape);
         },
         py::arg("partial_shape"),
         R"(
+                Reshape model input.
+
+                GIL is released while running this function.
+
                 :param partial_shape: New shape.
                 :type partial_shape: tuple
                 :return : void
@@ -299,8 +316,13 @@ void regclass_graph_Model(py::module m) {
         [](ov::Model& self, const std::string& partial_shape) {
             self.reshape(Common::partial_shape_from_str(partial_shape));
         },
+        py::call_guard<py::gil_scoped_release>(),
         py::arg("partial_shape"),
         R"(
+                Reshape model input.
+
+                GIL is released while running this function.
+
                 :param partial_shape: New shape.
                 :type partial_shape: str
                 :return : void
@@ -337,6 +359,7 @@ void regclass_graph_Model(py::module m) {
                 }
                 new_shapes.insert(new_shape);
             }
+            py::gil_scoped_release release;
             self.reshape(new_shapes);
         },
         py::arg("partial_shapes"),
@@ -368,6 +391,10 @@ void regclass_graph_Model(py::module m) {
                 '1..10' - to define bounded dimension
                 '..10' or '1..' to define dimension with only lower or only upper limit
 
+            Reshape model input.
+
+            GIL is released while running this function.
+
             :param partial_shapes: New shapes.
             :type partial_shapes: Dict[keys, values]
         )");

src/bindings/python/src/pyopenvino/graph/preprocess/pre_post_process.cpp

@@ -480,7 +480,7 @@ void regclass_graph_PrePostProcessor(py::module m) {
         },
         py::arg("output_index"));
 
-    proc.def("build", &ov::preprocess::PrePostProcessor::build);
+    proc.def("build", &ov::preprocess::PrePostProcessor::build, py::call_guard<py::gil_scoped_release>());
 
     proc.def("__str__", [](const ov::preprocess::PrePostProcessor& self) -> std::string {
         std::stringstream ss;

src/bindings/python/tests/test_inference_engine/test_nogil.py

@@ -0,0 +1,206 @@
+# Copyright (C) 2018-2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import time
+import pytest
+import os
+import io
+from threading import Thread
+import numpy as np
+
+from openvino.runtime import Core, Model, AsyncInferQueue, PartialShape, Layout, opset8 as ops, serialize
+from openvino.preprocess import PrePostProcessor
+
+
+# check if func releases the GIL and doens't increment reference counters of args while GIL is released
+def check_gil_released_safe(func, args=[]):  # noqa: B006
+    global gil_released
+    gil_released = False
+
+    def detect_gil():
+        global gil_released
+        # while sleeping main thread acquires GIL and runs func, which will release GIL
+        time.sleep(0.000001)
+        # increment reference counting of args while running func
+        args_ = args  # noqa: F841 'assigned to but never used'
+        gil_released = True
+    thread = Thread(target=detect_gil)
+    thread.start()
+    func(*args)
+    if not gil_released:
+        pytest.xfail(reason="Depend on condition race")
+    thread.join()
+
+
+device = os.environ.get("TEST_DEVICE") if os.environ.get("TEST_DEVICE") else "CPU"
+core = Core()
+core.set_property({"PERF_COUNT": "YES"})
+param = ops.parameter([224, 224])
+model = Model(ops.relu(param), [param])
+compiled = core.compile_model(model, device)
+infer_queue = AsyncInferQueue(compiled, 1)
+user_stream = io.BytesIO()
+
+
+# AsyncInferQueue
+
+def test_gil_released_async_infer_queue_start_async():
+    infer_queue.start_async()
+    check_gil_released_safe(infer_queue.start_async)
+
+
+def test_gil_released_async_infer_queue_is_ready():
+    infer_queue.start_async()
+    check_gil_released_safe(infer_queue.is_ready)
+
+
+def test_gil_released_async_infer_queue_wait_all():
+    infer_queue.start_async()
+    check_gil_released_safe(infer_queue.wait_all)
+
+
+def test_gil_released_async_infer_queue_get_idle_request_id():
+    infer_queue.start_async()
+    check_gil_released_safe(infer_queue.get_idle_request_id)
+
+
+# CompiledModel
+
+def test_gil_released_create_infer_request():
+    check_gil_released_safe(compiled.create_infer_request)
+
+
+def test_gil_released_infer_new_request():
+    check_gil_released_safe(compiled)
+
+
+def test_gil_released_export():
+    check_gil_released_safe(compiled.export_model)
+
+
+def test_gil_released_export_advanced():
+    check_gil_released_safe(compiled.export_model, [user_stream])
+
+
+def test_gil_released_get_runtime_model():
+    check_gil_released_safe(compiled.get_runtime_model)
+
+
+# Core
+
+def test_compile_model(device):
+    check_gil_released_safe(core.compile_model, [model, device])
+
+
+def test_read_model_from_bytes():
+    ir = bytes(b"""<net name="relu_model" version="11">
+    <layers>
+        <layer id="0" name="x" type="Parameter" version="opset1">
+            <data element_type="f32" shape="10"/>
+            <output>
+                <port id="0" precision="FP32">
+                    <dim>10</dim>
+                </port>
+            </output>
+        </layer>
+        <layer id="1" name="relu" type="ReLU" version="opset1">
+            <input>
+                <port id="0">
+                    <dim>10</dim>
+                </port>
+            </input>
+            <output>
+                <port id="1" precision="FP32">
+                    <dim>10</dim>
+                </port>
+            </output>
+        </layer>
+        <layer id="2" name="result" type="Result" version="opset1">
+            <input>
+                <port id="0">
+                    <dim>10</dim>
+                </port>
+            </input>
+        </layer>
+    </layers>
+    <edges>
+        <edge from-layer="0" from-port="0" to-layer="1" to-port="0"/>
+        <edge from-layer="1" from-port="1" to-layer="2" to-port="0"/>
+    </edges>
+</net>""")
+    check_gil_released_safe(core.read_model, [ir])
+
+
+def test_read_model_from_path():
+    from pathlib import Path
+    model_path = "relu.xml"
+    bin_path = "relu.bin"
+    serialize(model, model_path, bin_path)
+    check_gil_released_safe(core.read_model, [Path(model_path)])
+    os.remove(model_path)
+    os.remove(bin_path)
+
+
+def test_import_model(device):
+    check_gil_released_safe(core.import_model, [user_stream, device])
+
+
+def test_query_model(device):
+    check_gil_released_safe(core.query_model, [model, device])
+
+
+def test_get_available_devices(device):
+    check_gil_released_safe(getattr, [core, "available_devices"])
+
+
+# InferRequest
+
+request = compiled.create_infer_request()
+
+
+def test_infer():
+    data = [np.random.normal(size=list(compiled.input().shape))]
+    check_gil_released_safe(request.infer, [data])
+
+
+def test_start_async():
+    data = [np.random.normal(size=list(compiled.input().shape))]
+    check_gil_released_safe(request.start_async, [data])
+    request.wait()
+
+
+def test_wait():
+    data = [np.random.normal(size=list(compiled.input().shape))]
+    request.start_async(data)
+    check_gil_released_safe(request.wait)
+
+
+def test_wait_for():
+    data = [np.random.normal(size=list(compiled.input().shape))]
+    request.start_async(data)
+    check_gil_released_safe(request.wait_for, [1])
+    request.wait()
+
+
+def test_get_profiling_info():
+    check_gil_released_safe(request.get_profiling_info)
+
+
+def test_query_state():
+    check_gil_released_safe(request.query_state)
+
+
+# Preprocessing
+
+def test_pre_post_process_build():
+    p = PrePostProcessor(model)
+    p.input().model().set_layout(Layout("NC"))
+    check_gil_released_safe(p.build)
+
+
+def test_model_reshape():
+    check_gil_released_safe(model.reshape, [PartialShape([128, 128])])
+    check_gil_released_safe(model.reshape, [[164, 164]])
+    check_gil_released_safe(model.reshape, [(178, 178)])
+    check_gil_released_safe(model.reshape, ["194, 194"])
+    check_gil_released_safe(model.reshape, [{0: [224, 224]}])