From 1bd1eca8d990605476e82cb95ca1aabe32a6ab22 Mon Sep 17 00:00:00 2001 From: Sebastian Golebiewski Date: Tue, 7 Mar 2023 20:01:56 +0100 Subject: [PATCH] [DOCS] `Integrate OpenVINO with your application` article - add TF Lite (#16020) * fix snippets * add tflite * update tab directives * Update integrate_with_your_application.md --- .../integrate_with_your_application.md | 594 ++++++++++-------- .../images}/IMPLEMENT_PIPELINE_with_API_C.svg | 0 docs/snippets/src/main.c | 10 +- docs/snippets/src/main.cpp | 7 +- docs/snippets/src/main.py | 5 +- 5 files changed, 349 insertions(+), 267 deletions(-) rename docs/{OV_Runtime_UG/img => _static/images}/IMPLEMENT_PIPELINE_with_API_C.svg (100%) diff --git a/docs/OV_Runtime_UG/integrate_with_your_application.md b/docs/OV_Runtime_UG/integrate_with_your_application.md index 525a7ee9051..858c0525f9b 100644 --- a/docs/OV_Runtime_UG/integrate_with_your_application.md +++ b/docs/OV_Runtime_UG/integrate_with_your_application.md @@ -11,360 +11,428 @@ openvino_docs_OV_UG_Python_API_exclusives openvino_docs_MO_DG_TensorFlow_Frontend -@endsphinxdirective +Following these steps, you can implement a typical OpenVINO™ Runtime inference +pipeline in your application. Before proceeding, make sure you have +:doc:`installed OpenVINO Runtime ` and set environment variables (run ``/setupvars.sh`` for Linux or ``setupvars.bat`` for Windows, otherwise, the ``OpenVINO_DIR`` variable won't be configured properly to pass ``find_package`` calls). -Following these steps, you can implement a typical OpenVINO™ Runtime inference pipeline in your application. Before proceeding, make sure you have [installed OpenVINO Runtime](../install_guides/installing-openvino-runtime.md) and set environment variables (run `/setupvars.sh` for Linux or `setupvars.bat` for Windows, otherwise, the `OpenVINO_DIR` variable won't be configured properly to pass `find_package` calls). +.. image:: _static/images/IMPLEMENT_PIPELINE_with_API_C.svg -![ie_api_use_cpp] - -## Step 1. Create OpenVINO™ Runtime Core +Step 1. Create OpenVINO Runtime Core +#################################### Include next files to work with OpenVINO™ Runtime: -@sphinxtabset +.. tab-set:: -@sphinxtab{C++} + .. tab-item:: C++ + :sync: cpp -@snippet docs/snippets/src/main.cpp include + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [include] -@endsphinxtab + .. tab-item:: Python + :sync: py -@sphinxtab{Python} + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [import] -@snippet docs/snippets/src/main.py import + .. tab-item:: C + :sync: c -@endsphinxtab + .. doxygensnippet:: docs/snippets/src/main.c + :language: cpp + :fragment: [include] -@sphinxtab{C} - -@snippet docs/snippets/src/main.c include - -@endsphinxtab - -@endsphinxtabset Use the following code to create OpenVINO™ Core to manage available devices and read model objects: -@sphinxtabset +.. tab-set:: -@sphinxtab{C++} + .. tab-item:: C++ + :sync: cpp -@snippet docs/snippets/src/main.cpp part1 + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part1] -@endsphinxtab + .. tab-item:: Python + :sync: py -@sphinxtab{Python} + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part1] -@snippet docs/snippets/src/main.py part1 + .. tab-item:: C + :sync: c -@endsphinxtab - -@sphinxtab{C} - -@snippet docs/snippets/src/main.c part1 - -@endsphinxtab - -@endsphinxtabset - -## Step 2. Compile the Model - -`ov::CompiledModel` class represents a device specific compiled model. `ov::CompiledModel` allows you to get information inputs or output ports by a tensor name or index. This approach is aligned with the majority of frameworks. - -Compile the model for a specific device using `ov::Core::compile_model()`: - -@sphinxdirective - -.. tab:: C++ - - .. tab:: IR - - .. doxygensnippet:: docs/snippets/src/main.cpp - :language: cpp - :fragment: [part2_1] - - .. tab:: ONNX - - .. doxygensnippet:: docs/snippets/src/main.cpp - :language: cpp - :fragment: [part2_2] - - .. tab:: PaddlePaddle - - .. doxygensnippet:: docs/snippets/src/main.cpp - :language: cpp - :fragment: [part2_3] - - .. tab:: TensorFlow - - .. doxygensnippet:: docs/snippets/src/main.cpp - :language: cpp - :fragment: [part2_4] - - .. tab:: ov::Model - - .. doxygensnippet:: docs/snippets/src/main.cpp - :language: cpp - :fragment: [part2_5] - -.. tab:: Python - - .. tab:: IR - - .. doxygensnippet:: docs/snippets/src/main.py - :language: python - :fragment: [part2_1] - - .. tab:: ONNX - - .. doxygensnippet:: docs/snippets/src/main.py - :language: python - :fragment: [part2_2] - - .. tab:: PaddlePaddle - - .. doxygensnippet:: docs/snippets/src/main.py - :language: python - :fragment: [part2_3] - - .. tab:: TensorFlow - - .. doxygensnippet:: docs/snippets/src/main.py - :language: python - :fragment: [part2_4] - - .. tab:: ov::Model - - .. doxygensnippet:: docs/snippets/src/main.py - :language: python - :fragment: [part2_5] - -.. tab:: C - - .. tab:: IR - - .. doxygensnippet:: docs/snippets/src/main.c - :language: cpp - :fragment: [part2_1] - - .. tab:: ONNX - - .. doxygensnippet:: docs/snippets/src/main.c - :language: cpp - :fragment: [part2_2] - - .. tab:: PaddlePaddle - - .. doxygensnippet:: docs/snippets/src/main.c - :language: cpp - :fragment: [part2_3] - - .. tab:: TensorFlow - - .. doxygensnippet:: docs/snippets/src/main.c - :language: cpp - :fragment: [part2_4] - - .. tab:: ov::Model - - .. doxygensnippet:: docs/snippets/src/main.c - :language: cpp - :fragment: [part2_5] - -@endsphinxdirective + .. doxygensnippet:: docs/snippets/src/main.c + :language: cpp + :fragment: [part1] -The `ov::Model` object represents any models inside the OpenVINO™ Runtime. -For more details please read article about [OpenVINO™ Model representation](model_representation.md). +Step 2. Compile the Model +######################### + +``ov::CompiledModel`` class represents a device specific compiled model. ``ov::CompiledModel`` allows you to get information inputs or output ports by a tensor name or index. This approach is aligned with the majority of frameworks. + +Compile the model for a specific device using ``ov::Core::compile_model()``: + +.. tab-set:: + + .. tab-item:: C++ + :sync: cpp + + .. tab-set:: + + .. tab-item:: IR + :sync: ir + + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part2_1] + + .. tab-item:: ONNX + :sync: onnx + + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part2_2] + + .. tab-item:: PaddlePaddle + :sync: pdpd + + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part2_3] + + .. tab-item:: TensorFlow + :sync: tf + + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part2_4] + + .. tab-item:: TensorFlow Lite + :sync: tflite + + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part2_5] + + .. tab-item:: ov::Model + :sync: ovmodel + + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part2_6] + + .. tab-item:: Python + :sync: py + + .. tab-set:: + + .. tab-item:: IR + :sync: ir + + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part2_1] + + .. tab-item:: ONNX + :sync: onnx + + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part2_2] + + .. tab-item:: PaddlePaddle + :sync: pdpd + + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part2_3] + + .. tab-item:: TensorFlow + :sync: tf + + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part2_4] + + .. tab-item:: TensorFlow Lite + :sync: tflite + + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part2_5] + + .. tab-item:: ov::Model + :sync: ovmodel + + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part2_6] + + .. tab-item:: C + :sync: c + + .. tab-set:: + + .. tab-item:: IR + :sync: ir + + .. doxygensnippet:: docs/snippets/src/main.c + :language: cpp + :fragment: [part2_1] + + .. tab-item:: ONNX + :sync: onnx + + .. doxygensnippet:: docs/snippets/src/main.c + :language: cpp + :fragment: [part2_2] + + .. tab-item:: PaddlePaddle + :sync: pdpd + + .. doxygensnippet:: docs/snippets/src/main.c + :language: cpp + :fragment: [part2_3] + + .. tab-item:: TensorFlow + :sync: tf + + .. doxygensnippet:: docs/snippets/src/main.c + :language: cpp + :fragment: [part2_4] + + .. tab-item:: TensorFlow Lite + :sync: tflite + + .. doxygensnippet:: docs/snippets/src/main.c + :language: cpp + :fragment: [part2_5] + + .. tab-item:: ov::Model + :sync: ovmodel + + .. doxygensnippet:: docs/snippets/src/main.c + :language: cpp + :fragment: [part2_6] + + +The ``ov::Model`` object represents any models inside the OpenVINO™ Runtime. +For more details please read article about :doc:`OpenVINO™ Model representation `. The code above creates a compiled model associated with a single hardware device from the model object. It is possible to create as many compiled models as needed and use them simultaneously (up to the limitation of the hardware resources). -To learn how to change the device configuration, read the [Query device properties](./supported_plugins/config_properties.md) article. +To learn how to change the device configuration, read the :doc:`Query device properties ` article. -## Step 3. Create an Inference Request +Step 3. Create an Inference Request +################################### -`ov::InferRequest` class provides methods for model inference in OpenVINO™ Runtime. Create an infer request using the following code (see [InferRequest detailed documentation](./ov_infer_request.md) for more details): +``ov::InferRequest`` class provides methods for model inference in OpenVINO™ Runtime. +Create an infer request using the following code (see +:doc:`InferRequest detailed documentation ` for more details): -@sphinxtabset +.. tab-set:: -@sphinxtab{C++} + .. tab-item:: C++ + :sync: cpp -@snippet docs/snippets/src/main.cpp part3 + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part3] -@endsphinxtab + .. tab-item:: Python + :sync: py -@sphinxtab{Python} + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part3] -@snippet docs/snippets/src/main.py part3 + .. tab-item:: C + :sync: c -@endsphinxtab + .. doxygensnippet:: docs/snippets/src/main.c + :language: cpp + :fragment: [part3] -@sphinxtab{C} -@snippet docs/snippets/src/main.c part3 +Step 4. Set Inputs +#################### -@endsphinxtab +You can use external memory to create ``ov::Tensor`` and use the ``ov::InferRequest::set_input_tensor`` method to put this tensor on the device: -@endsphinxtabset +.. tab-set:: -## Step 4. Set Inputs + .. tab-item:: C++ + :sync: cpp -You can use external memory to create `ov::Tensor` and use the `ov::InferRequest::set_input_tensor` method to put this tensor on the device: + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part4] -@sphinxtabset + .. tab-item:: Python + :sync: py -@sphinxtab{C++} + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part4] -@snippet docs/snippets/src/main.cpp part4 + .. tab-item:: C + :sync: c -@endsphinxtab + .. doxygensnippet:: docs/snippets/src/main.c + :language: cpp + :fragment: [part4] -@sphinxtab{Python} -@snippet docs/snippets/src/main.py part4 +Step 5. Start Inference +####################### -@endsphinxtab +OpenVINO™ Runtime supports inference in either synchronous or asynchronous mode. Using the Async API can improve application's overall frame-rate: instead of waiting for inference to complete, the app can keep working on the host while the accelerator is busy. You can use ``ov::InferRequest::start_async`` to start model inference in the asynchronous mode and call ``ov::InferRequest::wait`` to wait for the inference results: -@sphinxtab{C} +.. tab-set:: -@snippet docs/snippets/src/main.c part4 + .. tab-item:: C++ + :sync: cpp -@endsphinxtab + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part5] -@endsphinxtabset + .. tab-item:: Python + :sync: py -## Step 5. Start Inference + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part5] -OpenVINO™ Runtime supports inference in either synchronous or asynchronous mode. Using the Async API can improve application's overall frame-rate: instead of waiting for inference to complete, the app can keep working on the host while the accelerator is busy. You can use `ov::InferRequest::start_async` to start model inference in the asynchronous mode and call `ov::InferRequest::wait` to wait for the inference results: + .. tab-item:: C + :sync: c -@sphinxtabset + .. doxygensnippet:: docs/snippets/src/main.c + :language: cpp + :fragment: [part5] -@sphinxtab{C++} -@snippet docs/snippets/src/main.cpp part5 +This section demonstrates a simple pipeline. To get more information about other ways to perform inference, read the dedicated +:doc:`"Run inference" section `. -@endsphinxtab - -@sphinxtab{Python} - -@snippet docs/snippets/src/main.py part5 - -@endsphinxtab - -@sphinxtab{C} - -@snippet docs/snippets/src/main.c part5 - -@endsphinxtab - -@endsphinxtabset - -This section demonstrates a simple pipeline. To get more information about other ways to perform inference, read the dedicated ["Run inference" section](./ov_infer_request.md). - -## Step 6. Process the Inference Results +Step 6. Process the Inference Results +##################################### Go over the output tensors and process the inference results. -@sphinxtabset +.. tab-set:: -@sphinxtab{C++} + .. tab-item:: C++ + :sync: cpp -@snippet docs/snippets/src/main.cpp part6 + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part6] -@endsphinxtab + .. tab-item:: Python + :sync: py -@sphinxtab{Python} + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part6] -@snippet docs/snippets/src/main.py part6 + .. tab-item:: C + :sync: c -@endsphinxtab + .. doxygensnippet:: docs/snippets/src/main.c + :language: cpp + :fragment: [part6] -@sphinxtab{C} -@snippet docs/snippets/src/main.c part6 - -@endsphinxtab - -@endsphinxtabset - -## Step 7. Release the allocated objects (only for C) +Step 7. Release the allocated objects (only for C) +################################################## To avoid memory leak, applications developed with C API need to release the allocated objects in order. -@sphinxtabset +.. tab-set:: -@sphinxtab{C} + .. tab-item:: C + :sync: c -@snippet docs/snippets/src/main.c part8 + .. doxygensnippet:: docs/snippets/src/main.c + :language: cpp + :fragment: [part8] -@endsphinxtab -@endsphinxtabset - -## Step 8. Link and Build Your Application with OpenVINO™ Runtime (example) +Step 8. Link and Build Your Application with OpenVINO™ Runtime (example) +######################################################################## This step may differ for different projects. In this example, a C++ & C application is used, together with CMake for project configuration. -### Create Structure for project: +Create Structure for project: ++++++++++++++++++++++++++++++ -@sphinxtabset +.. tab-set:: -@sphinxtab{C++} + .. tab-item:: C++ + :sync: cpp -@snippet docs/snippets/src/main.cpp part7 + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part7] -@endsphinxtab + .. tab-item:: C + :sync: c -@sphinxtab{C} - -@snippet docs/snippets/src/main.c part7 - -@endsphinxtab - -@endsphinxtabset - -### Create Cmake Script - -For details on additional CMake build options, refer to the [CMake page](https://cmake.org/cmake/help/latest/manual/cmake.1.html#manual:cmake(1)). - -@sphinxtabset - -@sphinxtab{C++} - -@snippet snippets/CMakeLists.txt cmake:integration_example_cpp - -@endsphinxtab - -@sphinxtab{C} - -@snippet snippets/CMakeLists.txt cmake:integration_example_c - -@endsphinxtab - -@endsphinxtabset + .. doxygensnippet:: docs/snippets/src/main.c + :language: cpp + :fragment: [part7] -### Build Project +Create Cmake Script +++++++++++++++++++++ + +For details on additional CMake build options, refer to the `CMake page `__. + +.. tab-set:: + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/snippets/CMakeLists.txt + :language: cpp + :fragment: [cmake:integration_example_cpp] + + .. tab-item:: C + :sync: c + + .. doxygensnippet:: docs/snippets/CMakeLists.txt + :language: cpp + :fragment: [cmake:integration_example_c] + + +Build Project +++++++++++++++++++++ To build your project using CMake with the default build tools currently available on your machine, execute the following commands: -```sh -cd build/ -cmake ../project -cmake --build . -``` +.. code-block:: sh -## Additional Resources - - - See the [OpenVINO Samples](Samples_Overview.md) page or the [Open Model Zoo Demos](https://docs.openvino.ai/nightly/omz_demos.html) page for specific examples of how OpenVINO pipelines are implemented for applications like image classification, text prediction, and many others. - - [OpenVINO™ Runtime Preprocessing](./preprocessing_overview.md) - - [Using Encrypted Models with OpenVINO™](./protecting_model_guide.md) - - [Open Model Zoo Demos](https://docs.openvino.ai/nightly/omz_demos.html) + cd build/ + cmake ../project + cmake --build . -[ie_api_flow_cpp]: img/BASIC_IE_API_workflow_Cpp.svg -[ie_api_use_cpp]: img/IMPLEMENT_PIPELINE_with_API_C.svg -[ie_api_flow_python]: img/BASIC_IE_API_workflow_Python.svg -[ie_api_use_python]: img/IMPLEMENT_PIPELINE_with_API_Python.svg +Additional Resources +#################### + +* See the :doc:`OpenVINO Samples ` page or the `Open Model Zoo Demos `__ page for specific examples of how OpenVINO pipelines are implemented for applications like image classification, text prediction, and many others. +* :doc:`OpenVINO™ Runtime Preprocessing ` +* :doc:`Using Encrypted Models with OpenVINO ` +* `Open Model Zoo Demos `__ + +@endsphinxdirective diff --git a/docs/OV_Runtime_UG/img/IMPLEMENT_PIPELINE_with_API_C.svg b/docs/_static/images/IMPLEMENT_PIPELINE_with_API_C.svg similarity index 100% rename from docs/OV_Runtime_UG/img/IMPLEMENT_PIPELINE_with_API_C.svg rename to docs/_static/images/IMPLEMENT_PIPELINE_with_API_C.svg diff --git a/docs/snippets/src/main.c b/docs/snippets/src/main.c index 5bc07e063be..1f6e641ac1c 100644 --- a/docs/snippets/src/main.c +++ b/docs/snippets/src/main.c @@ -36,14 +36,20 @@ ov_compiled_model_t* compiled_model = NULL; ov_core_compile_model_from_file(core, "model.pb", "AUTO", 0, &compiled_model); //! [part2_4] } - +{ //! [part2_5] +ov_compiled_model_t* compiled_model = NULL; +ov_core_compile_model_from_file(core, "model.tflite", "AUTO", 0, &compiled_model); +//! [part2_5] +} + +//! [part2_6] // Construct a model ov_model_t* model = NULL; ov_core_read_model(core, "model.xml", NULL, &model); ov_compiled_model_t* compiled_model = NULL; ov_core_compile_model(core, model, "AUTO", 0, &compiled_model); -//! [part2_5] +//! [part2_6] //! [part3] diff --git a/docs/snippets/src/main.cpp b/docs/snippets/src/main.cpp index 93d5f583bda..048a0ccd35d 100644 --- a/docs/snippets/src/main.cpp +++ b/docs/snippets/src/main.cpp @@ -34,6 +34,11 @@ ov::CompiledModel compiled_model = core.compile_model("model.pb", "AUTO"); } { //! [part2_5] +ov::CompiledModel compiled_model = core.compile_model("model.tflite", "AUTO"); +//! [part2_5] +} +{ +//! [part2_6] auto create_model = []() { std::shared_ptr model; // To construct a model, please follow @@ -42,7 +47,7 @@ auto create_model = []() { }; std::shared_ptr model = create_model(); compiled_model = core.compile_model(model, "AUTO"); -//! [part2_5] +//! [part2_6] } //! [part3] diff --git a/docs/snippets/src/main.py b/docs/snippets/src/main.py index 56f600a3b8a..fe14021bbab 100644 --- a/docs/snippets/src/main.py +++ b/docs/snippets/src/main.py @@ -23,6 +23,9 @@ compiled_model = core.compile_model("model.pdmodel", "AUTO") compiled_model = core.compile_model("model.pb", "AUTO") #! [part2_4] #! [part2_5] +compiled_model = core.compile_model("model.tflite", "AUTO") +#! [part2_5] +#! [part2_6] def create_model(): # This example shows how to create ov::Function # @@ -34,7 +37,7 @@ def create_model(): model = create_model() compiled_model = core.compile_model(model, "AUTO") -#! [part2_5] +#! [part2_6] #! [part3] infer_request = compiled_model.create_infer_request()