update links (#19563)

* 2023.1 link fix

* 2023.1 link fix

* 2023.1 link fix

* 2023.1 link fix

.ci/azure/linux.yml

@@ -32,13 +32,13 @@ resources:
     type: github
     endpoint: openvinotoolkit
     name: openvinotoolkit/openvino_contrib
-    ref: master
+    ref: releases/2023/1
 
   - repository: testdata
     type: github
     endpoint: openvinotoolkit
     name: openvinotoolkit/testdata
-    ref: master
+    ref: releases/2023/1
 
 variables:
   - group: github
@@ -365,9 +365,6 @@ jobs:
   - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_inference_unit_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-InferenceUnit.xml
     displayName: 'Inference Unit Tests'
 
-  - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVProxyTests.xml
-    displayName: 'OV Proxy Plugin Tests'
-
   - script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_hetero_func_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVHeteroFuncTests.xml
     displayName: 'OV Hetero Func Tests'
 

.ci/azure/linux_conditional_compilation.yml

@@ -35,6 +35,7 @@ resources:
     type: github
     endpoint: openvinotoolkit
     name: openvinotoolkit/testdata
+    ref: releases/2023/1
 
 variables:
   - group: github

.ci/azure/linux_coverity.yml

@@ -4,7 +4,7 @@ resources:
     type: github
     endpoint: openvinotoolkit
     name: openvinotoolkit/openvino_contrib
-    ref: master
+    ref: releases/2023/1
 
 variables:
   - group: github

.ci/azure/linux_cuda.yml

@@ -33,20 +33,17 @@ pr:
 
 resources:
   repositories:
-  - repository: openvino
-    type: github
-    endpoint: openvinotoolkit
-    name: openvinotoolkit/openvino
-
   - repository: openvino_contrib
     type: github
     endpoint: openvinotoolkit
     name: openvinotoolkit/openvino_contrib
+    ref: releases/2023/1
 
   - repository: testdata
     type: github
     endpoint: openvinotoolkit
     name: openvinotoolkit/testdata
+    ref: releases/2023/1
 
 jobs:
 - job: CUDAPlugin_Lin

.ci/azure/linux_debian.yml

@@ -34,7 +34,7 @@ resources:
     type: github
     endpoint: openvinotoolkit
     name: openvinotoolkit/testdata
-    ref: master
+    ref: releases/2023/1
 
 jobs:
 - job: Lin_Debian
@@ -278,12 +278,6 @@ jobs:
       LD_LIBRARY_PATH: $(INSTALL_TEST_DIR)
     displayName: 'OV Core UT'
 
-  - script: |
-      $(INSTALL_TEST_DIR)/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVProxyTests.xml
-    env:
-      LD_LIBRARY_PATH: $(INSTALL_TEST_DIR)
-    displayName: 'OV Proxy Tests'
-
   - script: |
       $(INSTALL_TEST_DIR)/ov_hetero_func_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVHeteroFuncTests.xml
     env:

.ci/azure/linux_lohika.yml

@@ -4,7 +4,7 @@
 #    type: github
 #    endpoint: openvinotoolkit
 #    name: openvinotoolkit/testdata
-#    ref: master
+#    ref: releases/2023/1
 
 jobs:
 - job: Lin_lohika

.ci/azure/mac.yml

@@ -35,13 +35,13 @@ resources:
     type: github
     endpoint: openvinotoolkit
     name: openvinotoolkit/openvino_contrib
-    ref: master
+    ref: releases/2023/1
 
   - repository: testdata
     type: github
     endpoint: openvinotoolkit
     name: openvinotoolkit/testdata
-    ref: master
+    ref: releases/2023/1
 
 variables:
   - group: github
@@ -185,10 +185,6 @@ jobs:
     displayName: 'OV Core UT'
     enabled: 'false'
 
-  - script: $(SETUPVARS) && $(INSTALL_TEST_DIR)/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVProxyTests.xml
-    displayName: 'OV Proxy Plugin Tests'
-    enabled: 'false'
-
   - script: $(SETUPVARS) && $(INSTALL_TEST_DIR)/ov_hetero_func_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-OVHeteroFuncTests.xml
     displayName: 'OV Hetero Func Tests'
     enabled: 'false'

.ci/azure/windows.yml

@@ -32,13 +32,13 @@ resources:
     type: github
     endpoint: openvinotoolkit
     name: openvinotoolkit/openvino_contrib
-    ref: master
+    ref: releases/2023/1
 
   - repository: testdata
     type: github
     endpoint: openvinotoolkit
     name: openvinotoolkit/testdata
-    ref: master
+    ref: releases/2023/1
 
 jobs:
 - job: Win
@@ -261,9 +261,6 @@ jobs:
   - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_inference_unit_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-InferenceUnit.xml
     displayName: 'Inference Unit Tests'
 
-  - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-OVProxyTests.xml
-    displayName: 'OV Proxy Plugin Tests'
-
   - script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_hetero_func_tests --gtest_print_time=1 --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-OVHeteroFuncTests.xml
     displayName: 'OV Hetero Func Tests'
 

.ci/azure/windows_conditional_compilation.yml

@@ -35,6 +35,7 @@ resources:
     type: github
     endpoint: openvinotoolkit
     name: openvinotoolkit/testdata
+    ref: releases/2023/1
 
 variables:
   - group: github

.github/workflows/coverage.yml

@@ -101,9 +101,6 @@ jobs:
       - name: Run OV core unit tests
         run: ${{ github.workspace }}/bin/intel64/Release/ov_core_unit_tests
 
-      - name: Run OV Proxy plugin tests
-        run: ${{ github.workspace }}/bin/intel64/Release/ov_proxy_plugin_tests
-
       - name: Run OV Hetero Func tests
         run: ${{ github.workspace }}/bin/intel64/Release/ov_hetero_func_tests
 

.github/workflows/linux.yml

@@ -451,11 +451,6 @@ jobs:
           source ${{ env.INSTALL_DIR }}/setupvars.sh
           ${{ env.INSTALL_TEST_DIR }}/ov_auto_batch_func_tests --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_batch_func_tests.xml
 
-      - name: Proxy Plugin Tests
-        run: |
-          source ${{ env.INSTALL_DIR }}/setupvars.sh
-          ${{ env.INSTALL_TEST_DIR }}/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVProxyTests.xml
-
       - name: Hetero Func Tests
         run: |
           source ${{ env.INSTALL_DIR }}/setupvars.sh

.github/workflows/windows.yml

@@ -640,11 +640,6 @@ jobs:
         run: |
           call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_auto_batch_func_tests --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_batch_func_tests.xml
 
-      - name: Proxy Plugin Tests
-        shell: cmd
-        run: |
-          call "${{ env.INSTALL_DIR }}\\setupvars.bat" && ${{ env.INSTALL_TEST_DIR }}/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVProxyTests.xml
-
       - name: Hetero Func Tests
         shell: cmd
         run: |

README.md

@@ -68,24 +68,24 @@ The OpenVINO™ Runtime can infer models on different hardware devices. This sec
     <tbody>
         <tr>
             <td rowspan=2>CPU</td>
-            <td> <a href="https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_CPU.html#doxid-openvino-docs-o-v-u-g-supported-plugins-c-p-u">Intel CPU</a></tb>
+            <td> <a href="https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_supported_plugins_CPU.html#doxid-openvino-docs-o-v-u-g-supported-plugins-c-p-u">Intel CPU</a></tb>
             <td><b><i><a href="./src/plugins/intel_cpu">openvino_intel_cpu_plugin</a></i></b></td>
             <td>Intel Xeon with Intel® Advanced Vector Extensions 2 (Intel® AVX2), Intel® Advanced Vector Extensions 512 (Intel® AVX-512), and AVX512_BF16, Intel Core Processors with Intel AVX2, Intel Atom Processors with Intel® Streaming SIMD Extensions (Intel® SSE)</td>
         </tr>
         <tr>
-            <td> <a href="https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_CPU.html">ARM CPU</a></tb>
+            <td> <a href="https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_supported_plugins_CPU.html">ARM CPU</a></tb>
             <td><b><i><a href="https://github.com/openvinotoolkit/openvino_contrib/tree/master/modules/arm_plugin">openvino_arm_cpu_plugin</a></i></b></td>
             <td>Raspberry Pi™ 4 Model B, Apple® Mac mini with M1 chip, NVIDIA® Jetson Nano™, Android™ devices
         </tr>
         <tr>
             <td>GPU</td>
-            <td><a href="https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_GPU.html#doxid-openvino-docs-o-v-u-g-supported-plugins-g-p-u">Intel GPU</a></td>
+            <td><a href="https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_supported_plugins_GPU.html#doxid-openvino-docs-o-v-u-g-supported-plugins-g-p-u">Intel GPU</a></td>
             <td><b><i><a href="./src/plugins/intel_gpu">openvino_intel_gpu_plugin</a></i></b></td>
             <td>Intel Processor Graphics, including Intel HD Graphics and Intel Iris Graphics</td>
         </tr>
         <tr>
             <td>GNA</td>
-            <td><a href="https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_GNA.html#doxid-openvino-docs-o-v-u-g-supported-plugins-g-n-a">Intel GNA</a></td>
+            <td><a href="https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_supported_plugins_GNA.html#doxid-openvino-docs-o-v-u-g-supported-plugins-g-n-a">Intel GNA</a></td>
             <td><b><i><a href="./src/plugins/intel_gna">openvino_intel_gna_plugin</a></i></b></td>
             <td>Intel Speech Enabling Developer Kit, Amazon Alexa* Premium Far-Field Developer Kit, Intel Pentium Silver J5005 Processor, Intel Pentium Silver N5000 Processor, Intel Celeron J4005 Processor, Intel Celeron J4105 Processor, Intel Celeron Processor N4100, Intel Celeron Processor N4000, Intel Core i3-8121U Processor, Intel Core i7-1065G7 Processor, Intel Core i7-1060G7 Processor, Intel Core i5-1035G4 Processor, Intel Core i5-1035G7 Processor, Intel Core i5-1035G1 Processor, Intel Core i5-1030G7 Processor, Intel Core i5-1030G4 Processor, Intel Core i3-1005G1 Processor, Intel Core i3-1000G1 Processor, Intel Core i3-1000G4 Processor</td>
         </tr>
@@ -103,22 +103,22 @@ OpenVINO™ Toolkit also contains several plugins which simplify loading models
     </thead>
     <tbody>
         <tr>
-            <td><a href="https://docs.openvino.ai/2023.0/openvino_docs_IE_DG_supported_plugins_AUTO.html#doxid-openvino-docs-i-e-d-g-supported-plugins-a-u-t-o">Auto</a></td>
+            <td><a href="https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_supported_plugins_AUTO.html">Auto</a></td>
             <td><b><i><a href="./src/plugins/auto">openvino_auto_plugin</a></i></b></td>
             <td>Auto plugin enables selecting Intel device for inference automatically</td>
         </tr>
         <tr>
-            <td><a href="https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Automatic_Batching.html">Auto Batch</a></td>
+            <td><a href="https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_Automatic_Batching.html">Auto Batch</a></td>
             <td><b><i><a href="./src/plugins/auto_batch">openvino_auto_batch_plugin</a></i></b></td>
             <td>Auto batch plugin performs on-the-fly automatic batching (i.e. grouping inference requests together) to improve device utilization, with no programming effort from the user</td>
         </tr>
         <tr>
-            <td><a href="https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Hetero_execution.html#doxid-openvino-docs-o-v-u-g-hetero-execution">Hetero</a></td>
+            <td><a href="https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_Hetero_execution.html#doxid-openvino-docs-o-v-u-g-hetero-execution">Hetero</a></td>
             <td><b><i><a href="./src/plugins/hetero">openvino_hetero_plugin</a></i></b></td>
             <td>Heterogeneous execution enables automatic inference splitting between several devices</td>
         </tr>
         <tr>
-            <td><a href="https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Running_on_multiple_devices.html#doxid-openvino-docs-o-v-u-g-running-on-multiple-devices">Multi</a></td>
+            <td><a href="https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_Running_on_multiple_devices.html#doxid-openvino-docs-o-v-u-g-running-on-multiple-devices">Multi</a></td>
             <td><b><i><a href="./src/plugins/auto">openvino_auto_plugin</a></i></b></td>
             <td>Multi plugin enables simultaneous inference of the same model on several devices in parallel</td>
         </tr>
@@ -155,10 +155,9 @@ The list of OpenVINO tutorials:
 ## System requirements
 
 The system requirements vary depending on platform and are available on dedicated pages:
-- [Linux](https://docs.openvino.ai/2023.0/openvino_docs_install_guides_installing_openvino_linux_header.html)
-- [Windows](https://docs.openvino.ai/2023.0/openvino_docs_install_guides_installing_openvino_windows_header.html)
-- [macOS](https://docs.openvino.ai/2023.0/openvino_docs_install_guides_installing_openvino_macos_header.html)
-- [Raspbian](https://docs.openvino.ai/2023.0/openvino_docs_install_guides_installing_openvino_raspbian.html)
+- [Linux](https://docs.openvino.ai/2023.1/openvino_docs_install_guides_installing_openvino_linux_header.html)
+- [Windows](https://docs.openvino.ai/2023.1/openvino_docs_install_guides_installing_openvino_windows_header.html)
+- [macOS](https://docs.openvino.ai/2023.1/openvino_docs_install_guides_installing_openvino_macos_header.html)
 
 ## How to build
 
@@ -196,7 +195,7 @@ Report questions, issues and suggestions, using:
 \* Other names and brands may be claimed as the property of others.
 
 [Open Model Zoo]:https://github.com/openvinotoolkit/open_model_zoo
-[OpenVINO™ Runtime]:https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_OV_Runtime_User_Guide.html
-[Model Optimizer]:https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html
-[Post-Training Optimization Tool]:https://docs.openvino.ai/2023.0/pot_introduction.html
+[OpenVINO™ Runtime]:https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_OV_Runtime_User_Guide.html
+[Model Optimizer]:https://docs.openvino.ai/2023.1/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html
+[Post-Training Optimization Tool]:https://docs.openvino.ai/2023.1/pot_introduction.html
 [Samples]:https://github.com/openvinotoolkit/openvino/tree/master/samples

cmake/developer_package/frontends/frontends.cmake

@@ -284,8 +284,7 @@ macro(ov_add_frontend)
 
             if(OV_FRONTEND_LINKABLE_FRONTEND)
                 set(export_set EXPORT OpenVINOTargets)
-                set(archive_dest ARCHIVE DESTINATION ${OV_CPACK_ARCHIVEDIR}
-                                 COMPONENT ${lib_component})
+                set(archive_dest ARCHIVE DESTINATION ${OV_CPACK_ARCHIVEDIR} COMPONENT ${lib_component})
                 set(namelink NAMELINK_COMPONENT ${dev_component})
             else()
                 set(namelink NAMELINK_SKIP)
@@ -295,6 +294,12 @@ macro(ov_add_frontend)
                     ${archive_dest}
                     LIBRARY DESTINATION ${OV_CPACK_LIBRARYDIR} COMPONENT ${lib_component}
                     ${namelink})
+
+            # export to build tree
+            if(OV_FRONTEND_LINKABLE_FRONTEND)
+                export(TARGETS ${TARGET_NAME} NAMESPACE openvino::
+                       APPEND FILE "${CMAKE_BINARY_DIR}/OpenVINOTargets.cmake")
+            endif()
         else()
             ov_install_static_lib(${TARGET_NAME} ${OV_CPACK_COMP_CORE})
         endif()
@@ -306,9 +311,8 @@ macro(ov_add_frontend)
                     COMPONENT ${dev_component}
                     FILES_MATCHING PATTERN "*.hpp")
 
+            # public target name
             set_target_properties(${TARGET_NAME} PROPERTIES EXPORT_NAME frontend::${OV_FRONTEND_NAME})
-            export(TARGETS ${TARGET_NAME} NAMESPACE openvino::
-                   APPEND FILE "${CMAKE_BINARY_DIR}/OpenVINOTargets.cmake")
         endif()
     else()
         # skipped frontend has to be installed in static libraries case

cmake/developer_package/packaging/common-libraries.cmake

@@ -84,7 +84,11 @@ macro(ov_define_component_include_rules)
     unset(OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL)
     set(OV_CPACK_COMP_CORE_C_DEV_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL})
     # licensing
-    set(OV_CPACK_COMP_LICENSING_EXCLUDE_ALL EXCLUDE_FROM_ALL)
+    if(CPACK_GENERATOR STREQUAL "CONAN")
+        unset(OV_CPACK_COMP_LICENSING_EXCLUDE_ALL)
+    else()
+        set(OV_CPACK_COMP_LICENSING_EXCLUDE_ALL EXCLUDE_FROM_ALL)
+    endif()
     # samples
     set(OV_CPACK_COMP_CPP_SAMPLES_EXCLUDE_ALL EXCLUDE_FROM_ALL)
     set(OV_CPACK_COMP_C_SAMPLES_EXCLUDE_ALL ${OV_CPACK_COMP_CPP_SAMPLES_EXCLUDE_ALL})

cmake/developer_package/packaging/packaging.cmake

@@ -24,6 +24,10 @@ macro(ov_install_static_lib target comp)
 
         install(TARGETS ${target} EXPORT OpenVINOTargets
                 ARCHIVE DESTINATION ${OV_CPACK_ARCHIVEDIR} COMPONENT ${comp} ${ARGN})
+
+        # export to local tree to build against static build tree
+        export(TARGETS ${target} NAMESPACE openvino::
+               APPEND FILE "${CMAKE_BINARY_DIR}/OpenVINOTargets.cmake")
     endif()
 endmacro()
 

cmake/developer_package/plugins/plugins.cmake

@@ -358,7 +358,7 @@ function(ov_generate_plugins_hpp)
                          "${plugins_hpp_in}"
                          "${IEDevScripts_DIR}/plugins/create_plugins_hpp.cmake"
                        COMMENT
-                         "Generate ov_plugins.hpp for build"
+                         "Generate ov_plugins.hpp"
                        VERBATIM)
 
     # for some reason dependency on source files does not work

cmake/features.cmake

@@ -5,7 +5,7 @@
 #
 # Common cmake options
 #
-ov_option (ENABLE_PROXY "Proxy plugin for OpenVINO Runtime" ON)
+ov_option (ENABLE_PROXY "Proxy plugin for OpenVINO Runtime" OFF)
 
 ie_dependent_option (ENABLE_INTEL_CPU "CPU plugin for OpenVINO Runtime" ON "RISCV64 OR X86 OR X86_64 OR AARCH64 OR ARM" OFF)
 

cmake/packaging/debian.cmake

@@ -90,7 +90,7 @@ macro(ov_cpack_settings)
         # - 2022.1.1, 2022.2 do not have debian packages enabled, distributed only as archives
         # - 2022.3 is the first release where Debian updated packages are introduced, others 2022.3.X are LTS
         2022.3.0 2022.3.1 2022.3.2 2022.3.3 2022.3.4 2022.3.5
-        2023.0.0 2023.0.1
+        2023.0.0 2023.0.1 2023.0.2 2023.0.3
         )
 
     #

cmake/packaging/rpm.cmake

@@ -76,7 +76,7 @@ macro(ov_cpack_settings)
         # - 2022.1.1, 2022.2 do not have rpm packages enabled, distributed only as archives
         # - 2022.3 is the first release where RPM updated packages are introduced, others 2022.3.X are LTS
         2022.3.0 2022.3.1 2022.3.2 2022.3.3 2022.3.4 2022.3.5
-        2023.0.0 2023.0.1
+        2023.0.0 2023.0.1 2023.0.2 2023.0.3
         )
 
     find_host_program(rpmlint_PROGRAM NAMES rpmlint DOC "Path to rpmlint")

conanfile.txt

@@ -2,12 +2,10 @@
 ade/0.1.2a
 onetbb/[>=2021.2.1]
 pugixml/[>=1.10]
-protobuf/3.21.9
+protobuf/3.21.12
 ittapi/[>=3.23.0]
 zlib/[>=1.2.8]
 opencl-icd-loader/2023.04.17
-opencl-clhpp-headers/2023.04.17
-opencl-headers/2023.04.17
 xbyak/[>=6.62]
 snappy/[>=1.1.7]
 gflags/2.2.2

docs/Documentation/model_introduction.md

@@ -3,7 +3,8 @@
 @sphinxdirective
 
 .. meta::
-   :description: Preparing models for OpenVINO Runtime. Learn how to convert and compile models from different frameworks or read them directly.
+   :description: Preparing models for OpenVINO Runtime. Learn about the methods 
+                 used to read, convert and compile models from different frameworks.
 
 
 .. toctree::
@@ -15,41 +16,45 @@
    omz_tools_downloader
 
 
-Every deep learning workflow begins with obtaining a model. You can choose to prepare a custom one, use a ready-made solution and adjust it to your needs, or even download and run a pre-trained network from an online database, such as `TensorFlow Hub <https://tfhub.dev/>`__, `Hugging Face <https://huggingface.co/>`__, `Torchvision models <https://pytorch.org/hub/>`__.
+Every deep learning workflow begins with obtaining a model. You can choose to prepare a custom one, use a ready-made solution and adjust it to your needs, or even download and run a pre-trained network from an online database, such as `TensorFlow Hub <https://tfhub.dev/>`__, `Hugging Face <https://huggingface.co/>`__, or `Torchvision models <https://pytorch.org/hub/>`__.
 
-:doc:`OpenVINO™ supports several model formats <Supported_Model_Formats>` and allows converting them to it's own, `openvino.runtime.Model <api/ie_python_api/_autosummary/openvino.runtime.Model.html>`__ (`ov.Model <api/ie_python_api/_autosummary/openvino.runtime.Model.html>`__ ), providing a tool dedicated to this task.
+Import a model using ``read_model()``
+#################################################
 
-There are several options to convert a model from original framework to OpenVINO model format (``ov.Model``).
+Model files (not Python objects) from :doc:`ONNX, PaddlePaddle, TensorFlow and TensorFlow Lite <Supported_Model_Formats>`  (check :doc:`TensorFlow Frontend Capabilities and Limitations <openvino_docs_MO_DG_TensorFlow_Frontend>`) do not require a separate step for model conversion, that is ``mo.convert_model``.
 
-The ``read_model()`` method reads a model from a file and produces ``ov.Model``. If the file is in one of the supported original framework file formats, it is converted automatically to OpenVINO Intermediate Representation. If the file is already in the OpenVINO IR format, it is read "as-is", without any conversion involved. ``ov.Model`` can be serialized to IR using the ``ov.serialize()`` method. The serialized IR can be further optimized using :doc:`Neural Network Compression Framework (NNCF) <ptq_introduction>` that applies post-training quantization methods.
+The ``read_model()`` method reads a model from a file and produces `openvino.runtime.Model <api/ie_python_api/_autosummary/openvino.runtime.Model.html>`__. If the file is in one of the supported original framework :doc:`file formats <Supported_Model_Formats>`, the method runs internal conversion to an OpenVINO model format. If the file is already in the :doc:`OpenVINO IR format <openvino_ir>`, it is read "as-is", without any conversion involved.
 
-Convert a model in Python
-######################################
+You can also convert a model from original framework to `openvino.runtime.Model <api/ie_python_api/_autosummary/openvino.runtime.Model.html>`__ using ``convert_model()`` method. More details about ``convert_model()`` are provided in :doc:`model conversion guide <openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide>` .
 
-Model conversion API, specifically, the ``mo.convert_model()`` method converts a model from original framework to ``ov.Model``. ``mo.convert_model()`` returns ``ov.Model`` object in memory so the ``read_model()`` method is not required. The resulting ``ov.Model`` can be inferred in the same training environment (python script or Jupiter Notebook). ``mo.convert_model()`` provides a convenient way to quickly switch from framework-based code to OpenVINO-based code in your inference application. In addition to model files, ``mo.convert_model()`` can take OpenVINO extension objects constructed directly in Python for easier conversion of operations that are not supported in OpenVINO. The ``mo.convert_model()`` method also has a set of parameters to :doc:`cut the model <openvino_docs_MO_DG_prepare_model_convert_model_Cutting_Model>`, :doc:`set input shapes or layout <openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model>`, :doc:`add preprocessing <openvino_docs_MO_DG_Additional_Optimization_Use_Cases>`, etc.
+``ov.Model`` can be saved to IR using the ``ov.save_model()`` method. The saved IR can be further optimized using :doc:`Neural Network Compression Framework (NNCF) <ptq_introduction>` that applies post-training quantization methods.
 
-.. image:: _static/images/model_conversion_diagram.svg
-   :alt: model conversion diagram
+.. note::
 
-Convert a model with ``mo`` command-line tool
-#############################################
+   ``convert_model()`` also allows you to perform input/output cut, add pre-processing or add custom Python conversion extensions.
 
-Another option to convert a model is to use ``mo`` command-line tool. ``mo`` is a cross-platform tool that facilitates the transition between training and deployment environments, performs static model analysis, and adjusts deep learning models for optimal execution on end-point target devices in the same measure, as the ``mo.convert_model`` method.
+Convert a model with Python using ``mo.convert_model()``
+###########################################################
 
-``mo`` requires the use of a pre-trained deep learning model in one of the supported formats: TensorFlow, TensorFlow Lite, PaddlePaddle, or ONNX. ``mo`` converts the model to the OpenVINO Intermediate Representation format (IR), which needs to be read with the ``ov.read_model()`` method. Then, you can compile and infer the ``ov.Model`` later with :doc:`OpenVINO™ Runtime <openvino_docs_OV_UG_OV_Runtime_User_Guide>`.
+Model conversion API, specifically, the ``mo.convert_model()`` method converts a model from original framework to ``ov.Model``. ``mo.convert_model()`` returns ``ov.Model`` object in memory so the ``read_model()`` method is not required. The resulting ``ov.Model`` can be inferred in the same training environment (python script or Jupiter Notebook). ``mo.convert_model()`` provides a convenient way to quickly switch from framework-based code to OpenVINO-based code in your inference application. 
 
+In addition to model files, ``mo.convert_model()`` can take OpenVINO extension objects constructed directly in Python for easier conversion of operations that are not supported in OpenVINO. The ``mo.convert_model()`` method also has a set of parameters to :doc:`cut the model <openvino_docs_MO_DG_prepare_model_convert_model_Cutting_Model>`, :doc:`set input shapes or layout <openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model>`, :doc:`add preprocessing <openvino_docs_MO_DG_Additional_Optimization_Use_Cases>`, etc.
 
-The figure below illustrates the typical workflow for deploying a trained deep learning model:
-
-.. image:: _static/images/BASIC_FLOW_MO_simplified.svg
-
-where IR is a pair of files describing the model:
+The figure below illustrates the typical workflow for deploying a trained deep learning model, where IR is a pair of files describing the model:
 
 * ``.xml`` - Describes the network topology.
 * ``.bin`` - Contains the weights and biases binary data.
 
+.. image:: _static/images/model_conversion_diagram.svg
+   :alt: model conversion diagram
 
-Model files (not Python objects) from ONNX, PaddlePaddle, TensorFlow and TensorFlow Lite  (check :doc:`TensorFlow Frontend Capabilities and Limitations <openvino_docs_MO_DG_TensorFlow_Frontend>`) do not require a separate step for model conversion, that is ``mo.convert_model``. OpenVINO provides C++ and Python APIs for importing the models to OpenVINO Runtime directly by just calling the ``read_model`` method. 
+
+Convert a model using ``mo`` command-line tool
+#################################################
+
+Another option to convert a model is to use ``mo`` command-line tool. ``mo`` is a cross-platform tool that facilitates the transition between training and deployment environments, performs static model analysis, and adjusts deep learning models for optimal execution on end-point target devices in the same measure, as the ``mo.convert_model()`` method.
+
+``mo`` requires the use of a pre-trained deep learning model in one of the supported formats: TensorFlow, TensorFlow Lite, PaddlePaddle, or ONNX. ``mo`` converts the model to the OpenVINO Intermediate Representation format (IR), which needs to be read with the ``ov.read_model()`` method. Then, you can compile and infer the ``ov.Model`` later with :doc:`OpenVINO™ Runtime <openvino_docs_OV_UG_OV_Runtime_User_Guide>`.
 
 The results of both ``mo`` and ``mo.convert_model()`` conversion methods described above are the same. You can choose one of them, depending on what is most convenient for you. Keep in mind that there should not be any differences in the results of model conversion if the same set of parameters is used.
 

docs/Documentation/openvino_workflow.md

@@ -17,6 +17,7 @@
    Running Inference <openvino_docs_OV_UG_OV_Runtime_User_Guide>
    Deployment on a Local System  <openvino_deployment_guide>
    Deployment on a Model Server <ovms_what_is_openvino_model_server>
+   pytorch_2_0_torch_compile
    
 
 | :doc:`Model Preparation <openvino_docs_model_processing_introduction>`

docs/Documentation/torch_compile.md

@@ -0,0 +1,157 @@
+# PyTorch Deployment via "torch.compile" {#pytorch_2_0_torch_compile}
+
+@sphinxdirective
+
+
+The ``torch.compile`` feature enables you to use OpenVINO for PyTorch-native applications. 
+It speeds up PyTorch code by JIT-compiling it into optimized kernels.
+By default, Torch code runs in eager-mode, but with the use of ``torch.compile`` it goes through the following steps:
+
+1. **Graph acquisition** - the model is rewritten as blocks of subgraphs that are either:
+
+   * compiled by TorchDynamo and "flattened",
+   * falling back to the eager-mode, due to unsupported Python constructs (like control-flow code).
+
+2. **Graph lowering** - all PyTorch operations are decomposed into their constituent kernels specific to the chosen backend.
+3. **Graph compilation** - the kernels call their corresponding low-level device-specific operations.
+
+
+
+How to Use
+#################
+
+To use ``torch.compile``, you need to add an import statement and define one of the two available backends:
+
+| ``openvino``
+|   With this backend, Torch FX subgraphs are directly converted to OpenVINO representation without any additional PyTorch based tracing/scripting.
+
+| ``openvino_ts``
+|   With this backend, Torch FX subgraphs are first traced/scripted with PyTorch Torchscript, and then converted to OpenVINO representation.
+
+
+.. tab-set::
+
+   .. tab-item:: openvino
+      :sync: backend-openvino
+
+      .. code-block:: console
+
+         import openvino.torch 
+         ...
+         model = torch.compile(model, backend='openvino')
+
+      Execution diagram:
+
+      .. image:: _static/images/torch_compile_backend_openvino.svg
+         :width: 992px
+         :height: 720px
+         :scale: 60%
+         :align: center
+
+   .. tab-item:: openvino_ts
+      :sync: backend-openvino-ts
+
+      .. code-block:: console
+
+         import openvino.torch
+         ...
+         model = torch.compile(model, backend='openvino_ts')
+
+      Execution diagram:
+
+      .. image:: _static/images/torch_compile_backend_openvino_ts.svg
+         :width: 1088px
+         :height: 720px
+         :scale: 60%
+         :align: center
+
+
+Environment Variables
++++++++++++++++++++++++++++
+
+* **OPENVINO_TORCH_BACKEND_DEVICE**: enables selecting a specific hardware device to run the application. 
+  By default, the OpenVINO backend for ``torch.compile`` runs PyTorch applications using the CPU. Setting 
+  this variable to GPU.0, for example, will make the application use the integrated graphics processor instead.
+* **OPENVINO_TORCH_MODEL_CACHING**: enables saving the optimized model files to a hard drive, after the first application run.
+  This makes them available for the following application executions, reducing the first-inference latency.
+  By default, this variable is set to ``False``. Setting it to ``True`` enables caching.
+* **OPENVINO_TORCH_CACHE_DIR**: enables defining a custom directory for the model files (if model caching set to ``True``).
+  By default, the OpenVINO IR is saved in the ``cache`` sub-directory, created in the application's root directory. 
+
+Windows support
+++++++++++++++++++++++++++
+
+Currently, PyTorch does not support ``torch.compile`` feature on Windows officially. However, it can be accessed by running
+the below instructions:
+
+1. Install the PyTorch nightly wheel file - `2.1.0.dev20230713 <https://download.pytorch.org/whl/nightly/cpu/torch-2.1.0.dev20230713%2Bcpu-cp38-cp38-win_amd64.whl>`__ ,
+2. Update the file at ``<python_env_root>/Lib/site-packages/torch/_dynamo/eval_frames.py``
+3. Find the function called ``check_if_dynamo_supported()``:
+
+   .. code-block:: console
+
+      def check_if_dynamo_supported():
+          if sys.platform == "win32":
+              raise RuntimeError("Windows not yet supported for torch.compile")
+          if sys.version_info >= (3, 11):
+              raise RuntimeError("Python 3.11+ not yet supported for torch.compile")
+
+4. Put in comments the first two lines in this function, so it looks like this:
+
+   .. code-block:: console
+
+      def check_if_dynamo_supported():
+       #if sys.platform == "win32":
+       #    raise RuntimeError("Windows not yet supported for torch.compile")
+       if sys.version_info >= (3, 11):
+           `raise RuntimeError("Python 3.11+ not yet supported for torch.compile")
+
+
+Support for Automatic1111 Stable Diffusion WebUI
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+Automatic1111 Stable Diffusion WebUI is an open-source repository that hosts a browser-based interface for the Stable Diffusion 
+based image generation. It allows users to create realistic and creative images from text prompts. 
+Stable Diffusion WebUI is supported on Intel CPUs, Intel integrated GPUs, and Intel discrete GPUs by leveraging OpenVINO 
+``torch.compile`` capability. Detailed instructions are available in 
+`Stable Diffusion WebUI repository. <https://github.com/openvinotoolkit/stable-diffusion-webui/wiki/Installation-on-Intel-Silicon>`__
+
+
+Architecture
+#################
+
+The ``torch.compile`` feature is part of PyTorch 2.0, and is based on:
+
+* **TorchDynamo** - a Python-level JIT that hooks into the frame evaluation API in CPython,
+  (PEP 523) to dynamically modify Python bytecode right before it is executed (PyTorch operators 
+  that cannot be extracted to FX graph are executed in the native Python environment). 
+  It maintains the eager-mode capabilities using 
+  `Guards <https://pytorch.org/docs/stable/dynamo/guards-overview.html>`__ to ensure the 
+  generated graphs are valid.
+
+* **AOTAutograd** - generates the backward graph corresponding to the forward graph captured by TorchDynamo.
+* **PrimTorch** - decomposes complicated PyTorch operations into simpler and more elementary ops.
+* **TorchInductor** - a deep learning compiler that generates fast code for multiple accelerators and backends.
+
+
+
+
+When the PyTorch module is wrapped with ``torch.compile``, TorchDynamo traces the module and 
+rewrites Python bytecode to extract sequences of PyTorch operations into an FX Graph,
+which can be optimized by the OpenVINO backend. The Torch FX graphs are first converted to 
+inlined FX graphs and the graph partitioning module traverses inlined FX graph to identify 
+operators supported by OpenVINO. 
+
+All the supported operators are clustered into OpenVINO submodules, converted to the OpenVINO 
+graph using OpenVINO's PyTorch decoder, and executed in an optimized manner using OpenVINO runtime. 
+All unsupported operators fall back to the native PyTorch runtime on CPU. If the subgraph 
+fails during OpenVINO conversion, the subgraph falls back to PyTorch's default inductor backend.
+
+
+
+Additional Resources
+############################
+
+* `PyTorch 2.0 documentation <https://pytorch.org/docs/stable/index.html>`_
+
+@endsphinxdirective

docs/Extensibility_UG/frontend_extensions.md

@@ -301,11 +301,19 @@ This mapping also specifies the input name "X" and output name "Out".
 
 The last step is to register this custom operation by following:
 
-
 .. doxygensnippet:: docs/snippets/ov_extensions.cpp
    :language: cpp
    :fragment: [frontend_extension_framework_map_macro_add_extension]
 
+.. important::
+
+   To map an operation on a specific framework, you have to link to a respective 
+   frontend (``openvino::frontend::onnx``, ``openvino::frontend::tensorflow``, ``openvino::frontend::paddle``) in the ``CMakeLists.txt`` file:
+
+   .. code-block:: sh
+
+      target_link_libraries(${TARGET_NAME} PRIVATE openvino::frontend::onnx)
+
 
 Mapping to Multiple Operations with ConversionExtension
 #######################################################

docs/IE_PLUGIN_DG/Intro.md

@@ -94,7 +94,7 @@ Detailed Guides
 API References
 ##############
 
-*  `OpenVINO Plugin API <https://docs.openvino.ai/2023.0/groupov_dev_api.html>`__
-*  `OpenVINO Transformation API <https://docs.openvino.ai/2023.0/groupie_transformation_api.html>`__
+*  `OpenVINO Plugin API <https://docs.openvino.ai/2023.1/groupov_dev_api.html>`__
+*  `OpenVINO Transformation API <https://docs.openvino.ai/2023.1/groupie_transformation_api.html>`__
 
 @endsphinxdirective

docs/IE_PLUGIN_DG/dev_api_references.md

@@ -15,7 +15,7 @@
 
 The guides below provides extra API references needed for OpenVINO plugin development:
 
-*  `OpenVINO Plugin API <https://docs.openvino.ai/2023.0/groupov_dev_api.html>`__
-*  `OpenVINO Transformation API <https://docs.openvino.ai/2023.0/groupie_transformation_api.html>`__
+*  `OpenVINO Plugin API <https://docs.openvino.ai/2023.1/groupov_dev_api.html>`__
+*  `OpenVINO Transformation API <https://docs.openvino.ai/2023.1/groupie_transformation_api.html>`__
 
 @endsphinxdirective

docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md

@@ -44,18 +44,21 @@ To convert a model to OpenVINO model format (``ov.Model``), you can use the foll
 
 If the out-of-the-box conversion (only the ``input_model`` parameter is specified) is not successful, use the parameters mentioned below to override input shapes and cut the model:
 
-- model conversion API provides two parameters to override original input shapes for model conversion: ``input`` and ``input_shape``.
-  For more information about these parameters, refer to the :doc:`Setting Input Shapes <openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model>` guide.
+- ``input`` and ``input_shape`` - the model conversion API parameters used to override original input shapes for model conversion,
+
+  For more information about the parameters, refer to the :doc:`Setting Input Shapes <openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model>` guide.
+
+- ``input`` and ``output`` - the model conversion API parameters used to define new inputs and outputs of the converted model to cut off unwanted parts (such as unsupported operations and training sub-graphs),
 
-- To cut off unwanted parts of a model (such as unsupported operations and training sub-graphs),
-  use the ``input`` and ``output`` parameters to define new inputs and outputs of the converted model.
   For a more detailed description, refer to the :doc:`Cutting Off Parts of a Model <openvino_docs_MO_DG_prepare_model_convert_model_Cutting_Model>` guide.
 
-You can also insert additional input pre-processing sub-graphs into the converted model by using
-the ``mean_values``, ``scales_values``, ``layout``, and other parameters described
-in the :doc:`Embedding Preprocessing Computation <openvino_docs_MO_DG_Additional_Optimization_Use_Cases>` article.
+- ``mean_values``, ``scales_values``, ``layout`` - the parameters used to insert additional input pre-processing sub-graphs into the converted model,
 
-The ``compress_to_fp16`` compression parameter in ``mo`` command-line tool allows generating IR with constants (for example, weights for convolutions and matrix multiplications) compressed to ``FP16`` data type. For more details, refer to the :doc:`Compression of a Model to FP16 <openvino_docs_MO_DG_FP16_Compression>` guide.
+  For more details, see the :doc:`Embedding Preprocessing Computation <openvino_docs_MO_DG_Additional_Optimization_Use_Cases>` article.
+
+- ``compress_to_fp16`` - a compression parameter in ``mo`` command-line tool, which allows generating IR with constants (for example, weights for convolutions and matrix multiplications) compressed to ``FP16`` data type.
+
+  For more details, refer to the :doc:`Compression of a Model to FP16 <openvino_docs_MO_DG_FP16_Compression>` guide.
 
 To get the full list of conversion parameters, run the following command:
 

docs/MO_DG/prepare_model/MO_Python_API.md

@@ -19,7 +19,7 @@ Example of converting a PyTorch model directly from memory:
    :force:
 
    import torchvision
-   
+
    model = torchvision.models.resnet50(pretrained=True)
    ov_model = convert_model(model)
 
@@ -36,7 +36,7 @@ Example of using native Python classes to set ``input_shape``, ``mean_values`` a
    :force:
 
    from openvino.runtime import PartialShape, Layout
-   
+
    ov_model = convert_model(model, input_shape=PartialShape([1,3,100,100]), mean_values=[127, 127, 127], layout=Layout("NCHW"))
 
 Example of using strings for setting ``input_shape``, ``mean_values`` and ``layout``:
@@ -74,7 +74,7 @@ Example of using ``InputCutInfo`` to freeze an input with value:
    :force:
 
    from openvino.tools.mo import convert_model, InputCutInfo
-   
+
    ov_model = convert_model(model, input=InputCutInfo("input_name", [3], np.float32, [0.5, 2.1, 3.4]))
 
 To set parameters for models with multiple inputs, use ``list`` of parameters.
@@ -104,7 +104,7 @@ Example of using the ``Layout`` class to set the layout of a model input:
 
    from openvino.runtime import Layout
    from openvino.tools.mo import convert_model
-   
+
    ov_model = convert_model(model, source_layout=Layout("NCHW"))
 
 To set both source and destination layouts in the ``layout`` parameter, use the ``LayoutMap`` class. ``LayoutMap`` accepts two parameters: ``source_layout`` and ``target_layout``.
@@ -117,7 +117,7 @@ Example of using the ``LayoutMap`` class to change the layout of a model input:
    :force:
 
    from openvino.tools.mo import convert_model, LayoutMap
-   
+
    ov_model = convert_model(model, layout=LayoutMap("NCHW", "NHWC"))
 
-@endsphinxdirective
+@endsphinxdirective

docs/MO_DG/prepare_model/convert_model/Convert_Model_From_ONNX.md

@@ -21,11 +21,32 @@ This page provides instructions on model conversion from the ONNX format to the
 
 Model conversion process assumes you have an ONNX model that was directly downloaded from a public repository or converted from any framework that supports exporting to the ONNX format.
 
-To convert an ONNX model, run model conversion with the path to the input model ``.onnx`` file:
+.. tab-set::
 
-.. code-block:: sh
+   .. tab-item:: Python
+      :sync: py
+
+      To convert an ONNX model, run ``convert_model()`` method with the path to the ``<INPUT_MODEL>.onnx`` file:
+
+      .. code-block:: py
+         :force:
+
+         ov_model = convert_model("<INPUT_MODEL>.onnx")
+         compiled_model = core.compile_model(ov_model, "AUTO")
+
+      .. important::
+
+         The ``convert_model()`` method returns ``ov.Model`` that you can optimize, compile, or save to a file for subsequent use.
+
+   .. tab-item:: CLI
+      :sync: cli
+
+      You can use ``mo`` command-line tool to convert a model to IR. The obtained IR can then be read by ``read_model()`` and inferred.
+
+      .. code-block:: sh
+
+         mo --input_model <INPUT_MODEL>.onnx
 
-   mo --input_model <INPUT_MODEL>.onnx
 
 There are no ONNX specific parameters, so only framework-agnostic parameters are available to convert your model. For details, see the *General Conversion Parameters* section in the :doc:`Converting a Model to Intermediate Representation (IR) <openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model>` guide.
 

docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Paddle.md

@@ -32,60 +32,58 @@ To convert a PaddlePaddle model, use the ``mo`` script and specify the path to t
 Converting PaddlePaddle Model From Memory Using Python API
 ##########################################################
 
-Model conversion API supports passing PaddlePaddle models directly from memory.
-
-Following PaddlePaddle model formats are supported:
+Model conversion API supports passing the following PaddlePaddle models directly from memory:
 
 * ``paddle.hapi.model.Model``
 * ``paddle.fluid.dygraph.layers.Layer``
 * ``paddle.fluid.executor.Executor``
 
-Converting certain PaddlePaddle models may require setting ``example_input`` or ``example_output``. Below examples show how to execute such the conversion.
+When you convert certain PaddlePaddle models, you may need to set the ``example_input`` or ``example_output`` parameters first. Below you will find examples that show how to convert aforementioned model formats using the parameters.
 
-* Example of converting ``paddle.hapi.model.Model`` format model:
+* ``paddle.hapi.model.Model``
 
   .. code-block:: py
      :force:
 
      import paddle
      from openvino.tools.mo import convert_model
-     
+
      # create a paddle.hapi.model.Model format model
      resnet50 = paddle.vision.models.resnet50()
      x = paddle.static.InputSpec([1,3,224,224], 'float32', 'x')
      y = paddle.static.InputSpec([1,1000], 'float32', 'y')
- 
+
      model = paddle.Model(resnet50, x, y)
- 
+
      # convert to OpenVINO IR format
      ov_model = convert_model(model)
- 
+
      # optional: serialize OpenVINO IR to *.xml & *.bin
      from openvino.runtime import serialize
      serialize(ov_model, "ov_model.xml", "ov_model.bin")
 
-* Example of converting ``paddle.fluid.dygraph.layers.Layer`` format model:
+* ``paddle.fluid.dygraph.layers.Layer``
 
-  ``example_input`` is required while ``example_output`` is optional, which accept the following formats:
+  ``example_input`` is required while ``example_output`` is optional, and accept the following formats:
 
   ``list`` with tensor(``paddle.Tensor``) or InputSpec(``paddle.static.input.InputSpec``)
 
   .. code-block:: py
      :force:
-  
+
      import paddle
      from openvino.tools.mo import convert_model
-   
+
      # create a paddle.fluid.dygraph.layers.Layer format model
      model = paddle.vision.models.resnet50()
      x = paddle.rand([1,3,224,224])
- 
+
      # convert to OpenVINO IR format
      ov_model = convert_model(model, example_input=[x])
 
-* Example of converting ``paddle.fluid.executor.Executor`` format model:
+* ``paddle.fluid.executor.Executor``
 
-  ``example_input`` and ``example_output`` are required, which accept the following formats:
+  ``example_input`` and ``example_output`` are required, and accept the following formats:
 
   ``list`` or ``tuple`` with variable(``paddle.static.data``)
 
@@ -94,86 +92,37 @@ Converting certain PaddlePaddle models may require setting ``example_input`` or
 
      import paddle
      from openvino.tools.mo import convert_model
- 
+
      paddle.enable_static()
- 
+
      # create a paddle.fluid.executor.Executor format model
      x = paddle.static.data(name="x", shape=[1,3,224])
      y = paddle.static.data(name="y", shape=[1,3,224])
      relu = paddle.nn.ReLU()
      sigmoid = paddle.nn.Sigmoid()
      y = sigmoid(relu(x))
- 
+
      exe = paddle.static.Executor(paddle.CPUPlace())
      exe.run(paddle.static.default_startup_program())
- 
+
      # convert to OpenVINO IR format
      ov_model = convert_model(exe, example_input=[x], example_output=[y])
 
+
+.. important::
+
+   The ``convert_model()`` method returns ``ov.Model`` that you can optimize, compile, or save to a file for subsequent use.
+
+
 Supported PaddlePaddle Layers
 #############################
 
 For the list of supported standard layers, refer to the :doc:`Supported Operations <openvino_resources_supported_operations_frontend>` page.
 
-Officially Supported PaddlePaddle Models
-########################################
-
-The following PaddlePaddle models have been officially validated and confirmed to work (as of OpenVINO 2022.1):
-
-.. list-table::
-   :widths: 20 25 55
-   :header-rows: 1
-
-   * - Model Name
-     - Model Type
-     - Description
-   * - ppocr-det
-     - optical character recognition
-     - Models are exported from `PaddleOCR <https://github.com/PaddlePaddle/PaddleOCR/tree/release/2.1/>`_. Refer to `READ.md <https://github.com/PaddlePaddle/PaddleOCR/tree/release/2.1/#pp-ocr-20-series-model-listupdate-on-dec-15>`_.
-   * - ppocr-rec
-     - optical character recognition
-     - Models are exported from `PaddleOCR <https://github.com/PaddlePaddle/PaddleOCR/tree/release/2.1/>`_. Refer to `READ.md <https://github.com/PaddlePaddle/PaddleOCR/tree/release/2.1/#pp-ocr-20-series-model-listupdate-on-dec-15>`_.
-   * - ResNet-50
-     - classification
-     - Models are exported from `PaddleClas <https://github.com/PaddlePaddle/PaddleClas/tree/release/2.1/>`_. Refer to `getting_started_en.md <https://github.com/PaddlePaddle/PaddleClas/blob/release/2.1/docs/en/tutorials/getting_started_en.md#4-use-the-inference-model-to-predict>`_.
-   * - MobileNet v2
-     - classification
-     - Models are exported from `PaddleClas <https://github.com/PaddlePaddle/PaddleClas/tree/release/2.1/>`_. Refer to `getting_started_en.md <https://github.com/PaddlePaddle/PaddleClas/blob/release/2.1/docs/en/tutorials/getting_started_en.md#4-use-the-inference-model-to-predict>`_.
-   * - MobileNet v3
-     - classification
-     - Models are exported from `PaddleClas <https://github.com/PaddlePaddle/PaddleClas/tree/release/2.1/>`_. Refer to `getting_started_en.md <https://github.com/PaddlePaddle/PaddleClas/blob/release/2.1/docs/en/tutorials/getting_started_en.md#4-use-the-inference-model-to-predict>`_.
-   * - BiSeNet v2
-     - semantic segmentation
-     - Models are exported from `PaddleSeg <https://github.com/PaddlePaddle/PaddleSeg/tree/release/2.1>`_. Refer to `model_export.md <https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.1/docs/model_export.md#>`_.
-   * - DeepLab v3 plus
-     - semantic segmentation
-     - Models are exported from `PaddleSeg <https://github.com/PaddlePaddle/PaddleSeg/tree/release/2.1>`_. Refer to `model_export.md <https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.1/docs/model_export.md#>`_.
-   * - Fast-SCNN
-     - semantic segmentation
-     - Models are exported from `PaddleSeg <https://github.com/PaddlePaddle/PaddleSeg/tree/release/2.1>`_. Refer to `model_export.md <https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.1/docs/model_export.md#>`_.
-   * - OCRNET
-     - semantic segmentation
-     - Models are exported from `PaddleSeg <https://github.com/PaddlePaddle/PaddleSeg/tree/release/2.1>`_. Refer to `model_export.md <https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.1/docs/model_export.md#>`_.
-   * - Yolo v3
-     - detection
-     - Models are exported from `PaddleDetection <https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.1>`_. Refer to `EXPORT_MODEL.md <https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.1/deploy/EXPORT_MODEL.md#>`_.
-   * - ppyolo
-     - detection
-     - Models are exported from `PaddleDetection <https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.1>`_. Refer to `EXPORT_MODEL.md <https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.1/deploy/EXPORT_MODEL.md#>`_.
-   * - MobileNetv3-SSD
-     - detection
-     - Models are exported from `PaddleDetection <https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.2>`_. Refer to `EXPORT_MODEL.md <https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.2/deploy/EXPORT_MODEL.md#>`_.
-   * - U-Net
-     - semantic segmentation
-     - Models are exported from `PaddleSeg <https://github.com/PaddlePaddle/PaddleSeg/tree/release/2.3>`_. Refer to `model_export.md <https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.3/docs/model_export.md#>`_.
-   * - BERT
-     - language representation
-     -  Models are exported from `PaddleNLP <https://github.com/PaddlePaddle/PaddleNLP/tree/v2.1.1>`_. Refer to `README.md <https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/language_model/bert#readme>`_.   
-
 Frequently Asked Questions (FAQ)
 ################################
 
-When model conversion API is unable to run to completion due to typographical errors, incorrectly used options, or other issues, it provides explanatory messages. They describe the potential cause of the problem and give a link to the :doc:`Model Optimizer FAQ <openvino_docs_MO_DG_prepare_model_Model_Optimizer_FAQ>`, which provides instructions on how to resolve most issues. The FAQ also includes links to relevant sections in :doc:`Convert a Model <openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide>` to help you understand what went wrong.
+The model conversion API displays explanatory messages for typographical errors, incorrectly used options, or other issues. They describe the potential cause of the problem and give a link to the :doc:`Model Optimizer FAQ <openvino_docs_MO_DG_prepare_model_Model_Optimizer_FAQ>`, which provides instructions on how to resolve most issues. The FAQ also includes links to relevant sections in :doc:`Convert a Model <openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide>` to help you understand what went wrong.
 
 Additional Resources
 ####################

docs/MO_DG/prepare_model/convert_model/Convert_Model_From_PyTorch.md

@@ -33,8 +33,8 @@ Following PyTorch model formats are supported:
 
 Converting certain PyTorch models may require model tracing, which needs ``input_shape`` or ``example_input`` parameters to be set.
 
-``example_input`` is used as example input for model tracing.
-``input_shape`` is used for constructing a float zero-filled torch.Tensor for model tracing.
+* ``example_input`` is used as example input for model tracing.
+* ``input_shape`` is used for constructing a float zero-filled torch.Tensor for model tracing.
 
 Example of using ``example_input``:
 
@@ -56,6 +56,10 @@ Example of using ``example_input``:
 * ``list`` or ``tuple`` with tensors (``openvino.runtime.Tensor`` / ``torch.Tensor`` / ``np.ndarray``)
 * ``dictionary`` where key is the input name, value is the tensor (``openvino.runtime.Tensor`` / ``torch.Tensor`` / ``np.ndarray``)
 
+.. important::
+
+   The ``convert_model()`` method returns ``ov.Model`` that you can optimize, compile, or save to a file for subsequent use.
+
 Exporting a PyTorch Model to ONNX Format
 ########################################
 

docs/MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md

@@ -299,6 +299,10 @@ Model conversion API supports passing TensorFlow/TensorFlow2 models directly fro
      checkpoint.restore(save_path)
      ov_model = convert_model(checkpoint)
 
+.. important::
+
+   The ``convert_model()`` method returns ``ov.Model`` that you can optimize, compile, or save to a file for subsequent use.
+
 Supported TensorFlow and TensorFlow 2 Keras Layers
 ##################################################
 

docs/MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow_Lite.md

@@ -13,7 +13,11 @@ To convert a TensorFlow Lite model, use the ``mo`` script and specify the path t
 
    mo --input_model <INPUT_MODEL>.tflite
 
-.. note:: TensorFlow Lite models are supported via FrontEnd API. You may skip conversion to IR and read models directly by OpenVINO runtime API. Refer to the :doc:`inference example <openvino_docs_OV_UG_Integrate_OV_with_your_application>` for more details. Using ``convert_model`` is still necessary in more complex cases, such as new custom inputs/outputs in model pruning, adding pre-processing, or using Python conversion extensions.
+TensorFlow Lite models are supported via FrontEnd API. You may skip conversion to IR and read models directly by OpenVINO runtime API. Refer to the :doc:`inference example <openvino_docs_OV_UG_Integrate_OV_with_your_application>` for more details. Using ``convert_model`` is still necessary in more complex cases, such as new custom inputs/outputs in model pruning, adding pre-processing, or using Python conversion extensions.
+
+.. important::
+
+   The ``convert_model()`` method returns ``ov.Model`` that you can optimize, compile, or save to a file for subsequent use.
 
 Supported TensorFlow Lite Layers
 ###################################

docs/MO_DG/prepare_model/convert_model/Cutting_Model.md

@@ -22,7 +22,7 @@ The following examples are the situations when model cutting is useful or even r
 Model conversion API parameters
 ###############################
 
-Model conversion API provides command line options ``input`` and ``output`` to specify new entry and exit nodes, while ignoring the rest of the model:
+Model conversion API provides ``input`` and ``output`` command-line options to specify new entry and exit nodes, while ignoring the rest of the model:
 
 * ``input`` option accepts a list of layer names of the input model that should be treated as new entry points to the model. See the full list of accepted types for input on :doc:`Model Conversion Python API <openvino_docs_MO_DG_Python_API>` page.
 * ``output`` option accepts a list of layer names of the input model that should be treated as new exit points from the model.

docs/MO_DG/prepare_model/convert_model/supported_model_formats.md

@@ -17,31 +17,545 @@
    openvino_docs_MO_DG_prepare_model_convert_model_tutorials
 
 .. meta::
-   :description: In OpenVINO, ONNX, PaddlePaddle, TensorFlow and TensorFlow Lite 
-                 models do not require any prior conversion, while MxNet, Caffe and Kaldi do.
+   :description: Learn about supported model formats and the methods used to convert, read, and compile them in OpenVINO™.
 
 
-**OpenVINO IR (Intermediate Representation)** - the proprietary format of OpenVINO™, benefiting from the full extent of its features.
+**OpenVINO IR (Intermediate Representation)** - the proprietary and default format of OpenVINO, benefiting from the full extent of its features. All other supported model formats, as listed below, are converted to :doc:`OpenVINO IR <openvino_ir>` to enable inference. Consider storing your model in this format to minimize first-inference latency, perform model optimization, and, in some cases, save space on your drive. 
 
-**ONNX, PaddlePaddle, TensorFlow, TensorFlow Lite** - formats supported directly, which means they can be used with 
-OpenVINO Runtime without any prior conversion. For a guide on how to run inference on ONNX, PaddlePaddle, or TensorFlow, 
-see how to :doc:`Integrate OpenVINO™ with Your Application <openvino_docs_OV_UG_Integrate_OV_with_your_application>`.
+**PyTorch, TensorFlow, ONNX, and PaddlePaddle** - can be used with OpenVINO Runtime API directly, 
+which means you do not need to save them as OpenVINO IR before including them in your application.
+OpenVINO can read, compile, and convert them automatically, as part of its pipeline.
 
-**MXNet, Caffe, Kaldi** - legacy formats that need to be converted to OpenVINO IR before running inference. 
-The model conversion in some cases may involve intermediate steps. OpenVINO is currently proceeding 
-**to deprecate these formats** and **remove their support entirely in the future**.
+In the Python API, these options are provided as three separate methods: 
+``read_model()``, ``compile_model()``, and ``convert_model()``.
+The ``convert_model()`` method enables you to perform additional adjustments 
+to the model, such as setting shapes, changing model input types or layouts, 
+cutting parts of the model, freezing inputs, etc. For a detailed description 
+of the conversion process, see the 
+:doc:`model conversion guide <openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide>`.
+
+Here are code examples of how to use these methods with different model formats:
+
+.. tab-set::
+
+   .. tab-item:: PyTorch
+      :sync: torch
+
+      .. tab-set::
+
+         .. tab-item:: Python
+            :sync: py
+
+            * The ``convert_model()`` method:
+
+              This is the only method applicable to PyTorch models.
+
+              .. dropdown:: List of supported formats:
+
+                 * **Python objects**:
+
+                   * ``torch.nn.Module``
+                   * ``torch.jit.ScriptModule``
+                   * ``torch.jit.ScriptFunction``
+
+              .. code-block:: py
+                 :force:
+
+                 model = torchvision.models.resnet50(pretrained=True)
+                 ov_model = convert_model(model)
+                 compiled_model = core.compile_model(ov_model, "AUTO")
+
+              For more details on conversion, refer to the 
+              :doc:`guide <openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_PyTorch>` 
+              and an example `tutorial <https://docs.openvino.ai/nightly/notebooks/102-pytorch-onnx-to-openvino-with-output.html>`__ 
+              on this topic.
+
+   .. tab-item:: TensorFlow
+      :sync: tf
+
+      .. tab-set::
+
+         .. tab-item:: Python
+            :sync: py
+
+            * The ``convert_model()`` method:
+
+              When you use the ``convert_model()`` method, you have more control and you can specify additional adjustments for ``ov.Model``. The ``read_model()`` and ``compile_model()`` methods are easier to use, however, they do not have such capabilities. With ``ov.Model`` you can choose to optimize, compile and run inference on it or serialize it into a file for subsequent use.
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * SavedModel - ``<SAVED_MODEL_DIRECTORY>`` or ``<INPUT_MODEL>.pb``
+                   * Checkpoint - ``<INFERENCE_GRAPH>.pb`` or ``<INFERENCE_GRAPH>.pbtxt``
+                   * MetaGraph - ``<INPUT_META_GRAPH>.meta``
+
+                 * **Python objects**:
+
+                   * ``tf.keras.Model``
+                   * ``tf.keras.layers.Layer``
+                   * ``tf.Module``
+                   * ``tf.compat.v1.Graph``
+                   * ``tf.compat.v1.GraphDef``
+                   * ``tf.function``
+                   * ``tf.compat.v1.session``
+                   * ``tf.train.checkpoint``
+
+              .. code-block:: py
+                 :force:
+
+                 ov_model = convert_model("saved_model.pb")
+                 compiled_model = core.compile_model(ov_model, "AUTO")
+
+              For more details on conversion, refer to the 
+              :doc:`guide <openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow>` 
+              and an example `tutorial <https://docs.openvino.ai/nightly/notebooks/101-tensorflow-to-openvino-with-output.html>`__ 
+              on this topic. 
+
+            * The ``read_model()`` and ``compile_model()`` methods:
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * SavedModel - ``<SAVED_MODEL_DIRECTORY>`` or ``<INPUT_MODEL>.pb``
+                   * Checkpoint - ``<INFERENCE_GRAPH>.pb`` or ``<INFERENCE_GRAPH>.pbtxt``
+                   * MetaGraph - ``<INPUT_META_GRAPH>.meta``
+
+              .. code-block:: py
+                 :force:
+
+                 ov_model = read_model("saved_model.pb")
+                 compiled_model = core.compile_model(ov_model, "AUTO")
+
+              For a guide on how to run inference, see how to 
+              :doc:`Integrate OpenVINO™ with Your Application <openvino_docs_OV_UG_Integrate_OV_with_your_application>`. 
+              For TensorFlow format, see :doc:`TensorFlow Frontend Capabilities and Limitations <openvino_docs_MO_DG_TensorFlow_Frontend>`.
+
+         .. tab-item:: C++
+            :sync: cpp
+
+            * The ``compile_model()`` method:
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * SavedModel - ``<SAVED_MODEL_DIRECTORY>`` or ``<INPUT_MODEL>.pb``
+                   * Checkpoint - ``<INFERENCE_GRAPH>.pb`` or ``<INFERENCE_GRAPH>.pbtxt``
+                   * MetaGraph - ``<INPUT_META_GRAPH>.meta``
+
+              .. code-block:: cpp
+
+                 ov::CompiledModel compiled_model = core.compile_model("saved_model.pb", "AUTO");
+
+              For a guide on how to run inference, see how to 
+              :doc:`Integrate OpenVINO™ with Your Application <openvino_docs_OV_UG_Integrate_OV_with_your_application>`.
+
+         .. tab-item:: C
+            :sync: c
+
+            * The ``compile_model()`` method:
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * SavedModel - ``<SAVED_MODEL_DIRECTORY>`` or ``<INPUT_MODEL>.pb``
+                   * Checkpoint - ``<INFERENCE_GRAPH>.pb`` or ``<INFERENCE_GRAPH>.pbtxt``
+                   * MetaGraph - ``<INPUT_META_GRAPH>.meta``
+
+              .. code-block:: c
+
+                 ov_compiled_model_t* compiled_model = NULL;
+                 ov_core_compile_model_from_file(core, "saved_model.pb", "AUTO", 0, &compiled_model);
+
+              For a guide on how to run inference, see how to 
+              :doc:`Integrate OpenVINO™ with Your Application <openvino_docs_OV_UG_Integrate_OV_with_your_application>`.
+
+         .. tab-item:: CLI
+            :sync: cli
+
+            You can use ``mo`` command-line tool to convert a model to IR. The obtained IR can then be read by ``read_model()`` and inferred.
+
+            .. code-block:: sh
+
+               mo --input_model <INPUT_MODEL>.pb
+
+            For details on the conversion, refer to the
+            :doc:`article <openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow>`.
+
+   .. tab-item:: TensorFlow Lite
+      :sync: tflite
+
+      .. tab-set::
+
+         .. tab-item:: Python
+            :sync: py
+
+            * The ``convert_model()`` method:
+
+              When you use the ``convert_model()`` method, you have more control and you can specify additional adjustments for ``ov.Model``. The ``read_model()`` and ``compile_model()`` methods are easier to use, however, they do not have such capabilities. With ``ov.Model`` you can choose to optimize, compile and run inference on it or serialize it into a file for subsequent use.
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * ``<INPUT_MODEL>.tflite``
+
+              .. code-block:: py
+                 :force:
+
+                 ov_model = convert_model("<INPUT_MODEL>.tflite")
+                 compiled_model = core.compile_model(ov_model, "AUTO")
+
+              For more details on conversion, refer to the 
+              :doc:`guide <openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow>` 
+              and an example `tutorial <https://docs.openvino.ai/nightly/notebooks/119-tflite-to-openvino-with-output.html>`__ 
+              on this topic.
 
 
-Refer to the following articles for details on conversion for different formats and models:
+            * The ``read_model()`` method:
 
-* :doc:`How to convert ONNX <openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_ONNX>`
-* :doc:`How to convert PaddlePaddle <openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Paddle>`
-* :doc:`How to convert TensorFlow <openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow>`
-* :doc:`How to convert TensorFlow Lite <openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow_Lite>`
-* :doc:`How to convert MXNet <openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_MxNet>`
-* :doc:`How to convert Caffe <openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Caffe>`
-* :doc:`How to convert Kaldi <openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Kaldi>`
+              .. dropdown:: List of supported formats:
 
-* :doc:`Conversion examples for specific models <openvino_docs_MO_DG_prepare_model_convert_model_tutorials>`
+                 * **Files**:
 
-@endsphinxdirective
+                   * ``<INPUT_MODEL>.tflite``
+
+              .. code-block:: py
+                 :force:
+
+                 ov_model = read_model("<INPUT_MODEL>.tflite")
+                 compiled_model = core.compile_model(ov_model, "AUTO")
+
+            * The ``compile_model()`` method:
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * ``<INPUT_MODEL>.tflite``
+
+              .. code-block:: py
+                 :force:
+
+                 compiled_model = core.compile_model("<INPUT_MODEL>.tflite", "AUTO")
+
+              For a guide on how to run inference, see how to 
+              :doc:`Integrate OpenVINO™ with Your Application <openvino_docs_OV_UG_Integrate_OV_with_your_application>`.
+
+
+         .. tab-item:: C++
+            :sync: cpp
+
+            * The ``compile_model()`` method:
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * ``<INPUT_MODEL>.tflite``
+
+              .. code-block:: cpp
+
+                 ov::CompiledModel compiled_model = core.compile_model("<INPUT_MODEL>.tflite", "AUTO");
+
+              For a guide on how to run inference, see how to 
+              :doc:`Integrate OpenVINO™ with Your Application <openvino_docs_OV_UG_Integrate_OV_with_your_application>`.
+
+         .. tab-item:: C
+            :sync: c
+
+            * The ``compile_model()`` method:
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * ``<INPUT_MODEL>.tflite``
+
+              .. code-block:: c
+
+                 ov_compiled_model_t* compiled_model = NULL;
+                 ov_core_compile_model_from_file(core, "<INPUT_MODEL>.tflite", "AUTO", 0, &compiled_model);
+
+              For a guide on how to run inference, see how to 
+              :doc:`Integrate OpenVINO™ with Your Application <openvino_docs_OV_UG_Integrate_OV_with_your_application>`.
+
+         .. tab-item:: CLI
+            :sync: cli
+
+            * The ``convert_model()`` method:
+
+              You can use ``mo`` command-line tool to convert a model to IR. The obtained IR can then be read by ``read_model()`` and inferred.
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * ``<INPUT_MODEL>.tflite``
+
+              .. code-block:: sh
+
+                 mo --input_model <INPUT_MODEL>.tflite
+
+              For details on the conversion, refer to the 
+              :doc:`article <openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow_Lite>`.
+
+   .. tab-item:: ONNX
+      :sync: onnx
+
+      .. tab-set::
+
+         .. tab-item:: Python
+            :sync: py
+
+            * The ``convert_model()`` method:
+
+              When you use the ``convert_model()`` method, you have more control and you can specify additional adjustments for ``ov.Model``. The ``read_model()`` and ``compile_model()`` methods are easier to use, however, they do not have such capabilities. With ``ov.Model`` you can choose to optimize, compile and run inference on it or serialize it into a file for subsequent use.
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * ``<INPUT_MODEL>.onnx``
+
+              .. code-block:: py
+                 :force:
+
+                 ov_model = convert_model("<INPUT_MODEL>.onnx")
+                 compiled_model = core.compile_model(ov_model, "AUTO")
+
+              For more details on conversion, refer to the 
+              :doc:`guide <openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_ONNX>` 
+              and an example `tutorial <https://docs.openvino.ai/nightly/notebooks/102-pytorch-onnx-to-openvino-with-output.html>`__ 
+              on this topic.
+
+
+            * The ``read_model()`` method:
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * ``<INPUT_MODEL>.onnx``
+
+              .. code-block:: py
+                 :force:
+
+                 ov_model = read_model("<INPUT_MODEL>.onnx")
+                 compiled_model = core.compile_model(ov_model, "AUTO")
+
+            * The ``compile_model()`` method:
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * ``<INPUT_MODEL>.onnx``
+
+              .. code-block:: py
+                 :force:
+
+                 compiled_model = core.compile_model("<INPUT_MODEL>.onnx", "AUTO")
+
+              For a guide on how to run inference, see how to :doc:`Integrate OpenVINO™ with Your Application <openvino_docs_OV_UG_Integrate_OV_with_your_application>`.
+
+
+         .. tab-item:: C++
+            :sync: cpp
+
+            * The ``compile_model()`` method:
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * ``<INPUT_MODEL>.onnx``
+
+              .. code-block:: cpp
+
+                 ov::CompiledModel compiled_model = core.compile_model("<INPUT_MODEL>.onnx", "AUTO");
+
+              For a guide on how to run inference, see how to :doc:`Integrate OpenVINO™ with Your Application <openvino_docs_OV_UG_Integrate_OV_with_your_application>`.
+
+         .. tab-item:: C
+            :sync: c
+
+            * The ``compile_model()`` method:
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * ``<INPUT_MODEL>.onnx``
+
+              .. code-block:: c
+
+                 ov_compiled_model_t* compiled_model = NULL;
+                 ov_core_compile_model_from_file(core, "<INPUT_MODEL>.onnx", "AUTO", 0, &compiled_model);
+
+              For details on the conversion, refer to the :doc:`article <openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_ONNX>`
+
+         .. tab-item:: CLI
+            :sync: cli
+
+            * The ``convert_model()`` method:
+
+              You can use ``mo`` command-line tool to convert a model to IR. The obtained IR can then be read by ``read_model()`` and inferred.
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * ``<INPUT_MODEL>.onnx``
+
+              .. code-block:: sh
+
+                 mo --input_model <INPUT_MODEL>.onnx
+
+              For details on the conversion, refer to the
+              :doc:`article <openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_ONNX>`
+
+   .. tab-item:: PaddlePaddle
+      :sync: pdpd
+
+      .. tab-set::
+
+         .. tab-item:: Python
+            :sync: py
+
+            * The ``convert_model()`` method:
+
+              When you use the ``convert_model()`` method, you have more control and you can specify additional adjustments for ``ov.Model``. The ``read_model()`` and ``compile_model()`` methods are easier to use, however, they do not have such capabilities. With ``ov.Model`` you can choose to optimize, compile and run inference on it or serialize it into a file for subsequent use.
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * ``<INPUT_MODEL>.pdmodel``
+
+                 * **Python objects**:
+
+                   * ``paddle.hapi.model.Model``
+                   * ``paddle.fluid.dygraph.layers.Layer``
+                   * ``paddle.fluid.executor.Executor``
+
+              .. code-block:: py
+                 :force:
+
+                 ov_model = convert_model("<INPUT_MODEL>.pdmodel")
+                 compiled_model = core.compile_model(ov_model, "AUTO")
+
+              For more details on conversion, refer to the 
+              :doc:`guide <openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Paddle>` 
+              and an example `tutorial <https://docs.openvino.ai/nightly/notebooks/103-paddle-to-openvino-classification-with-output.html>`__ 
+              on this topic.
+
+            * The ``read_model()`` method:
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * ``<INPUT_MODEL>.pdmodel``
+
+              .. code-block:: py
+                 :force:
+
+                 ov_model = read_model("<INPUT_MODEL>.pdmodel")
+                 compiled_model = core.compile_model(ov_model, "AUTO")
+
+            * The ``compile_model()`` method:
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * ``<INPUT_MODEL>.pdmodel``
+
+              .. code-block:: py
+                 :force:
+
+                 compiled_model = core.compile_model("<INPUT_MODEL>.pdmodel", "AUTO")
+
+              For a guide on how to run inference, see how to 
+              :doc:`Integrate OpenVINO™ with Your Application <openvino_docs_OV_UG_Integrate_OV_with_your_application>`.
+
+         .. tab-item:: C++
+            :sync: cpp
+
+            * The ``compile_model()`` method:
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * ``<INPUT_MODEL>.pdmodel``
+
+              .. code-block:: cpp
+
+                 ov::CompiledModel compiled_model = core.compile_model("<INPUT_MODEL>.pdmodel", "AUTO");
+
+              For a guide on how to run inference, see how to 
+              :doc:`Integrate OpenVINO™ with Your Application <openvino_docs_OV_UG_Integrate_OV_with_your_application>`.
+
+         .. tab-item:: C
+            :sync: c
+
+            * The ``compile_model()`` method:
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * ``<INPUT_MODEL>.pdmodel``
+
+              .. code-block:: c
+
+                 ov_compiled_model_t* compiled_model = NULL;
+                 ov_core_compile_model_from_file(core, "<INPUT_MODEL>.pdmodel", "AUTO", 0, &compiled_model);
+
+              For a guide on how to run inference, see how to 
+              :doc:`Integrate OpenVINO™ with Your Application <openvino_docs_OV_UG_Integrate_OV_with_your_application>`.
+
+         .. tab-item:: CLI
+            :sync: cli
+
+            * The ``convert_model()`` method:
+
+              You can use ``mo`` command-line tool to convert a model to IR. The obtained IR can then be read by ``read_model()`` and inferred.
+
+              .. dropdown:: List of supported formats:
+
+                 * **Files**:
+
+                   * ``<INPUT_MODEL>.pdmodel``
+
+              .. code-block:: sh
+
+                 mo --input_model <INPUT_MODEL>.pdmodel
+
+              For details on the conversion, refer to the
+              :doc:`article <openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Paddle>`.
+
+
+**MXNet, Caffe, and Kaldi** are legacy formats that need to be converted explicitly to OpenVINO IR or ONNX before running inference. 
+As OpenVINO is currently proceeding **to deprecate these formats** and **remove their support entirely in the future**, 
+converting them to ONNX for use with OpenVINO should be considered the default path.
+
+
+.. note::
+
+   If you want to keep working with the legacy formats the old way, refer to a previous 
+   `OpenVINO LTS version and its documentation. <https://docs.openvino.ai/2022.3/Supported_Model_Formats.html>.
+    
+   OpenVINO versions of 2023 are mostly compatible with the old instructions, 
+   through a deprecated MO tool, installed with the deprecated OpenVINO Developer Tools package.
+
+   `OpenVINO 2023.0 <https://docs.openvino.ai/2023.0/Supported_Model_Formats.html> is the last
+   release officially supporting the MO conversion process for the legacy formats.
+
+
+@endsphinxdirective

docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_RetinaNet_From_Tensorflow.md

@@ -10,7 +10,7 @@
 This tutorial explains how to convert a RetinaNet model to the Intermediate Representation (IR).
 
 `Public RetinaNet model <https://github.com/fizyr/keras-retinanet>`__ does not contain pretrained TensorFlow weights.
-To convert this model to the TensorFlow format, follow the `Reproduce Keras to TensorFlow Conversion tutorial <https://docs.openvino.ai/2023.0/omz_models_model_retinanet_tf.html>`__. 
+To convert this model to the TensorFlow format, follow the `Reproduce Keras to TensorFlow Conversion tutorial <https://docs.openvino.ai/2023.1/omz_models_model_retinanet_tf.html>`__. 
 
 After converting the model to TensorFlow format, run the following command:
 

docs/OV_Runtime_UG/deployment/deployment_intro.md

@@ -26,7 +26,7 @@ Local Deployment Options
 
   - using Debian / RPM packages - a recommended way for Linux operating systems;
   - using PIP package manager on PyPI - the default approach for Python-based applications;
-  - using Docker images - if the application should be deployed as a Docker image, use a pre-built OpenVINO™ Runtime Docker image as a base image in the Dockerfile for the application container image. For more information about OpenVINO Docker images, refer to :doc:`Installing OpenVINO on Linux from Docker <openvino_docs_install_guides_installing_openvino_docker_linux>`
+  - using Docker images - if the application should be deployed as a Docker image, use a pre-built OpenVINO™ Runtime Docker image as a base image in the Dockerfile for the application container image. For more information about OpenVINO Docker images, refer to :doc:`Installing OpenVINO from Docker <openvino_docs_install_guides_installing_openvino_docker>`
 
 Furthermore, to customize your OpenVINO Docker image, use the `Docker CI Framework <https://github.com/openvinotoolkit/docker_ci>`__ to generate a Dockerfile and built the image.
 

docs/OV_Runtime_UG/integrate_with_your_application.md

@@ -437,9 +437,9 @@ To build your project using CMake with the default build tools currently availab
 Additional Resources
 ####################
 
-* See the :doc:`OpenVINO Samples <openvino_docs_OV_UG_Samples_Overview>` page or the `Open Model Zoo Demos <https://docs.openvino.ai/2023.0/omz_demos.html>`__ page for specific examples of how OpenVINO pipelines are implemented for applications like image classification, text prediction, and many others.
+* See the :doc:`OpenVINO Samples <openvino_docs_OV_UG_Samples_Overview>` page or the `Open Model Zoo Demos <https://docs.openvino.ai/2023.1/omz_demos.html>`__ page for specific examples of how OpenVINO pipelines are implemented for applications like image classification, text prediction, and many others.
 * :doc:`OpenVINO™ Runtime Preprocessing <openvino_docs_OV_UG_Preprocessing_Overview>`
 * :doc:`Using Encrypted Models with OpenVINO <openvino_docs_OV_UG_protecting_model_guide>`
-* `Open Model Zoo Demos <https://docs.openvino.ai/2023.0/omz_demos.html>`__
+* `Open Model Zoo Demos <https://docs.openvino.ai/2023.1/omz_demos.html>`__
 
 @endsphinxdirective

docs/OV_Runtime_UG/migration_ov_2_0/graph_construction.md

@@ -20,7 +20,7 @@ nGraph API
    .. tab-item:: Python
       :sync: py
 
-      .. doxygensnippet:: docs/snippets/ngraph.py
+      .. doxygensnippet:: docs/snippets/ngraph_snippet.py
          :language: Python
          :fragment: ngraph:graph
 

docs/OV_Runtime_UG/model_state_intro.md

@@ -163,9 +163,21 @@ Example of Creating Model OpenVINO API
 
 In the following example, the ``SinkVector`` is used to create the ``ov::Model``. For a model with states, except inputs and outputs, the ``Assign`` nodes should also point to the ``Model`` to avoid deleting it during graph transformations. Use the constructor to do it, as shown in the example, or with the special ``add_sinks(const SinkVector& sinks)`` method. After deleting the node from the graph with the ``delete_sink()`` method, a sink can be deleted from ``ov::Model``.
 
-.. doxygensnippet:: docs/snippets/ov_model_with_state_infer.cpp
-   :language: cpp
-   :fragment: [model_create]
+.. tab-set::
+
+   .. tab-item:: C++
+      :sync: cpp
+
+      .. doxygensnippet:: docs/snippets/ov_model_with_state_infer.cpp
+         :language: cpp
+         :fragment: [model_create]
+
+   .. tab-item:: Python
+      :sync: py
+
+      .. doxygensnippet:: docs/snippets/ov_model_with_state_infer.py
+         :language: python
+         :fragment: ov:model_create
 
 .. _openvino-state-api:
 
@@ -189,10 +201,22 @@ Based on the IR from the previous section, the example below demonstrates infere
 
 One infer request and one thread will be used in this example. Using several threads is possible if there are several independent sequences. Then, each sequence can be processed in its own infer request. Inference of one sequence in several infer requests is not recommended. In one infer request, a state will be saved automatically between inferences, but if the first step is done in one infer request and the second in another, a state should be set in a new infer request manually (using the ``ov::IVariableState::set_state`` method).
 
+.. tab-set::
+
+   .. tab-item:: C++
+      :sync: cpp
+      
+      .. doxygensnippet:: docs/snippets/ov_model_with_state_infer.cpp
+         :language: cpp
+         :fragment: [part1]
+    
+   .. tab-item:: Python
+      :sync: py
+
+      .. doxygensnippet:: docs/snippets/ov_model_with_state_infer.py
+         :language: python
+         :fragment: ov:part1
 
-.. doxygensnippet:: docs/snippets/ov_model_with_state_infer.cpp
-   :language: cpp
-   :fragment: [part1]
 
 
 For more elaborate examples demonstrating how to work with models with states, 

docs/OV_Runtime_UG/ov_dynamic_shapes.md

@@ -62,7 +62,7 @@ Model input dimensions can be specified as dynamic using the model.reshape metho
    
    Some models may already have dynamic shapes out of the box and do not require additional configuration. This can either be because it was generated with dynamic shapes from the source framework, or because it was converted with Model Conversion API to use dynamic shapes. For more information, see the Dynamic Dimensions “Out of the Box” section.
 
-The examples below show how to set dynamic dimensions with a model that has a static ``[1, 3, 224, 224]`` input shape (such as `mobilenet-v2 <https://docs.openvino.ai/2023.0/omz_models_model_mobilenet_v2.html>`__). The first example shows how to change the first dimension (batch size) to be dynamic. In the second example, the third and fourth dimensions (height and width) are set as dynamic.
+The examples below show how to set dynamic dimensions with a model that has a static ``[1, 3, 224, 224]`` input shape (such as `mobilenet-v2 <https://docs.openvino.ai/2023.1/omz_models_model_mobilenet_v2.html>`__). The first example shows how to change the first dimension (batch size) to be dynamic. In the second example, the third and fourth dimensions (height and width) are set as dynamic.
 
 .. tab-set::
 
@@ -175,7 +175,7 @@ The lower and/or upper bounds of a dynamic dimension can also be specified. They
     .. tab-item:: C
         :sync: c
         
-        The dimension bounds can be coded as arguments for `ov_dimension <https://docs.openvino.ai/2023.0/structov_dimension.html#doxid-structov-dimension>`__, as shown in these examples:
+        The dimension bounds can be coded as arguments for `ov_dimension <https://docs.openvino.ai/2023.1/structov_dimension.html#doxid-structov-dimension>`__, as shown in these examples:
 
         .. doxygensnippet:: docs/snippets/ov_dynamic_shapes.c
            :language: cpp

docs/OV_Runtime_UG/preprocessing_usecase_save.md

@@ -110,8 +110,8 @@ Additional Resources
 * :doc:`Layout API overview <openvino_docs_OV_UG_Layout_Overview>`
 * :doc:`Model Optimizer - Optimize Preprocessing Computation <openvino_docs_MO_DG_Additional_Optimization_Use_Cases>`
 * :doc:`Model Caching Overview <openvino_docs_OV_UG_Model_caching_overview>`
-* The `ov::preprocess::PrePostProcessor <https://docs.openvino.ai/2023.0/classov_1_1preprocess_1_1PrePostProcessor.html#doxid-classov-1-1preprocess-1-1-pre-post-processor>`__ C++ class documentation
-* The `ov::pass::Serialize <https://docs.openvino.ai/2023.0/classov_1_1pass_1_1Serialize.html#doxid-classov-1-1pass-1-1-serialize.html>`__ - pass to serialize model to XML/BIN
-* The `ov::set_batch <https://docs.openvino.ai/2023.0/namespaceov.html#doxid-namespaceov-1a3314e2ff91fcc9ffec05b1a77c37862b.html>`__ - update batch dimension for a given model
+* The `ov::preprocess::PrePostProcessor <https://docs.openvino.ai/2023.1/classov_1_1preprocess_1_1PrePostProcessor.html#doxid-classov-1-1preprocess-1-1-pre-post-processor>`__ C++ class documentation
+* The `ov::pass::Serialize <https://docs.openvino.ai/2023.1/classov_1_1pass_1_1Serialize.html#doxid-classov-1-1pass-1-1-serialize.html>`__ - pass to serialize model to XML/BIN
+* The `ov::set_batch <https://docs.openvino.ai/2023.1/namespaceov.html#doxid-namespaceov-1a3314e2ff91fcc9ffec05b1a77c37862b.html>`__ - update batch dimension for a given model
 
 @endsphinxdirective

docs/OV_Runtime_UG/supported_plugins/Device_Plugins.md

@@ -2,6 +2,12 @@
 
 @sphinxdirective
 
+.. meta::
+   :description: The list of types of devices and corresponding plugins which 
+                 are compatible with OpenVINO Runtime and support inference 
+                 of deep learning models.
+
+
 .. toctree::
    :maxdepth: 1
    :hidden:
@@ -9,13 +15,9 @@
    openvino_docs_OV_UG_query_api
    openvino_docs_OV_UG_supported_plugins_CPU
    openvino_docs_OV_UG_supported_plugins_GPU
+   openvino_docs_OV_UG_supported_plugins_NPU
    openvino_docs_OV_UG_supported_plugins_GNA
 
-.. meta::
-   :description: The list of types of devices and corresponding plugins which 
-                 are compatible with OpenVINO Runtime and support inference 
-                 of deep learning models.
-
 
 OpenVINO™ Runtime can infer deep learning models using the following device types:
 

docs/OV_Runtime_UG/supported_plugins/GNA.md

@@ -18,8 +18,11 @@ For more details on how to configure a system to use GNA, see the :doc:`GNA conf
 
 .. note::
 
-   Intel's GNA is being discontinued and Intel® Core™ Ultra (formerly known as Meteor Lake) will be the last generation of hardware to include it.
-   Consider Intel's new Visual Processing Unit as a low-power solution for offloading neural network computation, for processors offering the technology.
+   Intel's GNA is being discontinued and Intel® Core™ Ultra (formerly known as Meteor Lake) 
+   will be the last generation of hardware to include it.
+   For this reason, OpenVINO 2023.2 will also be the last version supporting the GNA plugin.
+   Consider Intel's new Visual Processing Unit as a low-power solution for offloading 
+   neural network computation, for processors offering the technology.
    
 
 Intel® GNA Generational Differences

docs/OV_Runtime_UG/supported_plugins/NPU.md

@@ -0,0 +1,28 @@
+# NPU Device {#openvino_docs_OV_UG_supported_plugins_NPU}
+
+@sphinxdirective
+
+.. meta::
+   :description: The NPU plugin in the Intel® Distribution of OpenVINO™ toolkit 
+                 aims at high performance inference of neural 
+                 networks on the low-power NPU processing device.
+
+
+NPU is a new generation of low-power processing unit dedicated to processing neural networks. 
+The NPU plugin is a core part of the OpenVINO™ toolkit. For its in-depth description, see:
+
+..
+   - `NPU plugin developer documentation <    cmake_options_for_custom_compilation.md ???       >`__.
+   - `NPU plugin source files <              ???                  >`__.
+
+
+
+
+
+
+
+
+
+
+
+@endsphinxdirective

docs/_static/css/homepage_style.css

@@ -167,3 +167,7 @@ h1 {
         max-width: 100%;
     }
 }
+
+.sd-row {
+    --sd-gutter-x: 0rem!important;
+}

docs/_static/images/BASIC_FLOW_MO_simplified.svg

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:53f6f30af6d39d91d7f3f4e3bbd086e3dbc94e5ac97233d56a90826579759e7f
-size 104225

docs/_static/images/model_conversion_diagram.svg

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc27fc105f73d9fb6e1a5436843c090c0748486632084aa27611222b2738a108
-size 187128
+oid sha256:894cc0f49385b304f7129b31e616cfc47dd188e910fca8d726b006bcbb3082f3
+size 252381

docs/_static/images/torch_compile_backend_openvino.svg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e26fe889ada0e02a3bbc03e451a7e1d4b06037723349971efff1d721b5e13f6
+size 117253

docs/_static/images/torch_compile_backend_openvino_ts.svg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c5ba73be918d2105b54e2afe36aa99e8fc5313489afc08f1a92ff75580667ba
+size 173373

docs/benchmarks/performance_benchmarks.md

@@ -13,7 +13,7 @@
 
    openvino_docs_performance_benchmarks_faq
    OpenVINO Accuracy <openvino_docs_performance_int8_vs_fp32>
-   Performance Data Spreadsheet (download xlsx) <https://docs.openvino.ai/2023.0/_static/benchmarks_files/OV-2023.0-Performance-Data.xlsx> 
+   Performance Data Spreadsheet (download xlsx) <https://docs.openvino.ai/2023.1/_static/benchmarks_files/OV-2023.0-Performance-Data.xlsx> 
    openvino_docs_MO_DG_Getting_Performance_Numbers
 
 

docs/dev/cmake_options_for_custom_compilation.md

@@ -189,8 +189,8 @@ In this case OpenVINO CMake scripts take `TBBROOT` environment variable into acc
 [pugixml]:https://pugixml.org/
 [ONNX]:https://onnx.ai/
 [protobuf]:https://github.com/protocolbuffers/protobuf
-[deployment manager]:https://docs.openvino.ai/2023.0/openvino_docs_install_guides_deployment_manager_tool.html
-[OpenVINO Runtime Introduction]:https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Integrate_OV_with_your_application.html
+[deployment manager]:https://docs.openvino.ai/2023.1/openvino_docs_install_guides_deployment_manager_tool.html
+[OpenVINO Runtime Introduction]:https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_Integrate_OV_with_your_application.html
 [PDPD]:https://github.com/PaddlePaddle/Paddle
 [TensorFlow]:https://www.tensorflow.org/
 [TensorFlow Lite]:https://www.tensorflow.org/lite

docs/dev/debug_capabilities.md

@@ -2,7 +2,7 @@
 
 OpenVINO components provides different debug capabilities, to get more information please read:
 
-* [OpenVINO Model Debug Capabilities](https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Model_Representation.html#model-debug-capabilities)
+* [OpenVINO Model Debug Capabilities](https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_Model_Representation.html#model-debug-capabilities)
 * [OpenVINO Pass Manager Debug Capabilities](#todo)
 
 ## See also

docs/gapi/face_beautification.md

@@ -24,8 +24,8 @@ This sample requires:
 * OpenCV 4.2 or higher built with `Intel® Distribution of OpenVINO™ Toolkit <https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit.html>`__ (building with `Intel® TBB <https://www.threadingbuildingblocks.org/intel-tbb-tutorial>`__ is a plus)
 * The following pre-trained models from the :doc:`Open Model Zoo <omz_models_group_intel>`
 
-  * `face-detection-adas-0001 <https://docs.openvino.ai/2023.0/omz_models_model_face_detection_adas_0001.html#doxid-omz-models-model-face-detection-adas-0001>`__
-  * `facial-landmarks-35-adas-0002 <https://docs.openvino.ai/2023.0/omz_models_model_facial_landmarks_35_adas_0002.html#doxid-omz-models-model-facial-landmarks-35-adas-0002>`__
+  * `face-detection-adas-0001 <https://docs.openvino.ai/nightly/omz_models_model_face_detection_adas_0001.html#doxid-omz-models-model-face-detection-adas-0001>`__
+  * `facial-landmarks-35-adas-0002 <https://docs.openvino.ai/nightly/omz_models_model_facial_landmarks_35_adas_0002.html#doxid-omz-models-model-facial-landmarks-35-adas-0002>`__
 
 To download the models from the Open Model Zoo, use the :doc:`Model Downloader <omz_tools_downloader>` tool.
 

docs/gapi/gapi_face_analytics_pipeline.md

@@ -24,9 +24,9 @@ This sample requires:
 * OpenCV 4.2 or higher built with `Intel® Distribution of OpenVINO™ Toolkit <https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit.html>`__ (building with `Intel® TBB <https://www.threadingbuildingblocks.org/intel-tbb-tutorial>`__ is a plus)
 * The following pre-trained models from the :doc:`Open Model Zoo <omz_models_group_intel>`
 
-  * `face-detection-adas-0001 <https://docs.openvino.ai/2023.0/omz_models_model_face_detection_adas_0001.html#doxid-omz-models-model-face-detection-adas-0001>`__
-  * `age-gender-recognition-retail-0013 <https://docs.openvino.ai/2023.0/omz_models_model_age_gender_recognition_retail_0013.html#doxid-omz-models-model-age-gender-recognition-retail-0013>`__
-  * `emotions-recognition-retail-0003 <https://docs.openvino.ai/2023.0/omz_models_model_emotions_recognition_retail_0003.html#doxid-omz-models-model-emotions-recognition-retail-0003>`__
+  * `face-detection-adas-0001 <https://docs.openvino.ai/2023.1/omz_models_model_face_detection_adas_0001.html#doxid-omz-models-model-face-detection-adas-0001>`__
+  * `age-gender-recognition-retail-0013 <https://docs.openvino.ai/2023.1/omz_models_model_age_gender_recognition_retail_0013.html#doxid-omz-models-model-age-gender-recognition-retail-0013>`__
+  * `emotions-recognition-retail-0003 <https://docs.openvino.ai/2023.1/omz_models_model_emotions_recognition_retail_0003.html#doxid-omz-models-model-emotions-recognition-retail-0003>`__
 
 To download the models from the Open Model Zoo, use the :doc:`Model Downloader <omz_tools_downloader>` tool.
 
@@ -42,7 +42,7 @@ Starting with version 4.2, OpenCV offers a solution to this problem. OpenCV G-AP
 Pipeline Overview
 #################
 
-Our sample application is based on `Interactive Face Detection <https://docs.openvino.ai/2023.0/omz_demos_interactive_face_detection_demo_cpp.html#doxid-omz-demos-interactive-face-detection-demo-cpp>`__ demo from Open Model Zoo. A simplified pipeline consists of the following steps:
+Our sample application is based on `Interactive Face Detection <https://docs.openvino.ai/2023.1/omz_demos_interactive_face_detection_demo_cpp.html#doxid-omz-demos-interactive-face-detection-demo-cpp>`__ demo from Open Model Zoo. A simplified pipeline consists of the following steps:
 
 1. Image acquisition and decode
 2. Detection with preprocessing

docs/get_started/get_started_demos.md

@@ -30,7 +30,7 @@ To use sample applications, install OpenVINO Runtime via one of the following di
 
 * Archive files (recommended) - :doc:`Linux <openvino_docs_install_guides_installing_openvino_from_archive_linux>` | :doc:`Windows <openvino_docs_install_guides_installing_openvino_from_archive_windows>` | :doc:`macOS <openvino_docs_install_guides_installing_openvino_from_archive_macos>`
 * :doc:`APT <openvino_docs_install_guides_installing_openvino_apt>` or :doc:`YUM <openvino_docs_install_guides_installing_openvino_yum>` for Linux
-* Docker image - :doc:`Linux <openvino_docs_install_guides_installing_openvino_docker_linux>`
+* :doc:`Docker image <openvino_docs_install_guides_installing_openvino_docker>`
 * `Build from source <https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build.md>`__
 
 Make sure that you also `install OpenCV <https://github.com/opencv/opencv/wiki/BuildOpenCV4OpenVINO>`__ , as it's required for running sample applications.

docs/home.rst

@@ -23,9 +23,10 @@ OpenVINO 2023.0
            <div class="splide__track">
          		<ul class="splide__list">
          			<li class="splide__slide">An open-source toolkit for optimizing and deploying deep learning models.<br>Boost your AI deep-learning inference performance!</li>
-			         <li class="splide__slide">Even more integrations in 2023.0!<br>Load TensorFlow, TensorFlow Lite, and PyTorch models directly, without manual conversion.<br><a href="https://docs.openvino.ai/2023.0/Supported_Model_Formats.html">See the supported model formats...</a></li>
-         			<li class="splide__slide">CPU inference has become even better. ARM processors are supported and thread scheduling is available on 12th gen Intel® Core and up.<br><a href="https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_OV_Runtime_User_Guide.html">See how to run OpenVINO on various devices...</a></li>
-         			<li class="splide__slide">Post-training optimization and quantization-aware training now in one tool!<br><a href="https://docs.openvino.ai/2023.0/openvino_docs_model_optimization_guide.html">See the new NNCF capabilities...</a></li>
+			         <li class="splide__slide">Even more integrations in 2023.0!<br>Load TensorFlow, TensorFlow Lite, and PyTorch models directly, without manual conversion.<br><a href="https://docs.openvino.ai/2023.1/Supported_Model_Formats.html">See the supported model formats...</a></li>
+         			<li class="splide__slide">CPU inference has become even better. ARM processors are supported and thread scheduling is available on 12th gen Intel® Core and up.<br><a href="https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_OV_Runtime_User_Guide.html">See how to run OpenVINO on various devices...</a></li>
+         			<li class="splide__slide">Post-training optimization and quantization-aware training now in one tool!<br><a href="https://docs.openvino.ai/2023.1/openvino_docs_model_optimization_guide.html">See the new NNCF capabilities...</a></li>
+                  <li class="splide__slide">OpenVINO is enabled in the PyTorch 2.0 torch.compile() backend.<br><a href="https://docs.openvino.ai/2023.1/pytorch_2_0_torch_compile.html">See how it works...</a></li>
          		</ul>
            </div>
          </section>
@@ -83,6 +84,13 @@ OpenVINO 2023.0
 
       Reach for performance with post-training and training-time compression with NNCF
 
+   .. grid-item-card:: PyTorch 2.0 - torch.compile() backend
+      :link: pytorch_2_0_torch_compile
+      :link-alt: torch.compile 
+      :link-type: doc
+
+      Optimize generation of the graph model with PyTorch 2.0 torch.compile() backend
+
 
 Feature Overview
 ##############################

docs/install_guides/configurations-for-intel-gpu.md

@@ -147,7 +147,7 @@ What’s Next?
 ############
 
 * :doc:`GPU Device <openvino_docs_OV_UG_supported_plugins_GPU>`
-* :doc:`Install Intel® Distribution of OpenVINO™ toolkit for Linux from a Docker Image <openvino_docs_install_guides_installing_openvino_docker_linux>`
+* :doc:`Install Intel® Distribution of OpenVINO™ toolkit from a Docker Image <openvino_docs_install_guides_installing_openvino_docker>`
 * `Docker CI framework for Intel® Distribution of OpenVINO™ toolkit <https://github.com/openvinotoolkit/docker_ci/blob/master/README.md>`__
 * `Get Started with DockerHub CI for Intel® Distribution of OpenVINO™ toolkit <https://github.com/openvinotoolkit/docker_ci/blob/master/get-started.md>`__
 * `Dockerfiles with Intel® Distribution of OpenVINO™ toolkit <https://github.com/openvinotoolkit/docker_ci/blob/master/dockerfiles/README.md>`__

docs/install_guides/configurations-for-intel-npu.md

@@ -0,0 +1,71 @@
+# Configurations for Intel® NPU with OpenVINO™ {#openvino_docs_install_guides_configurations_for_intel_npu}
+
+@sphinxdirective
+
+.. meta::
+   :description: Learn how to provide additional configuration for Intel® 
+                 NPU to work with the OpenVINO™ toolkit on your system.
+
+
+
+Drivers and Dependencies
+########################
+
+
+The Intel® NPU device requires a proper driver to be installed on the system.
+
+
+
+Linux
+####################
+
+Prerequisites
+++++++++++++++++++++
+
+Ensure that make, gcc, and Linux kernel headers are installed. Use the following command to install the required software:
+
+.. code-block:: sh
+
+   sudo apt-get install gcc make linux-headers-generic
+
+
+Configuration steps
+++++++++++++++++++++
+
+
+
+
+
+
+
+
+
+
+
+Windows
+####################
+
+Intel® NPU driver for Windows is available through Windows Update.
+
+
+
+
+What’s Next?
+####################
+
+Now you are ready to try out OpenVINO™. You can use the following tutorials to write your applications using Python and C++.
+
+* Developing in Python:
+
+  * `Start with tensorflow models with OpenVINO™ <notebooks/101-tensorflow-to-openvino-with-output.html>`__
+  * `Start with ONNX and PyTorch models with OpenVINO™ <notebooks/102-pytorch-onnx-to-openvino-with-output.html>`__
+  * `Start with PaddlePaddle models with OpenVINO™ <notebooks/103-paddle-to-openvino-classification-with-output.html>`__
+
+* Developing in C++:
+
+  * :doc:`Image Classification Async C++ Sample <openvino_inference_engine_samples_classification_sample_async_README>`
+  * :doc:`Hello Classification C++ Sample <openvino_inference_engine_samples_hello_classification_README>`
+  * :doc:`Hello Reshape SSD C++ Sample <openvino_inference_engine_samples_hello_reshape_ssd_README>`
+
+@endsphinxdirective
+

docs/install_guides/configurations-header.md

@@ -13,6 +13,7 @@
    :hidden:
  
    For GPU <openvino_docs_install_guides_configurations_for_intel_gpu>
+   For NPU <openvino_docs_install_guides_configurations_for_intel_npu>
    For GNA <openvino_docs_install_guides_configurations_for_intel_gna>
 
 

docs/install_guides/installing-openvino-docker-linux.md

@@ -1,4 +1,4 @@
-# Install Intel® Distribution of OpenVINO™ toolkit for Linux from a Docker Image {#openvino_docs_install_guides_installing_openvino_docker_linux}
+# Install Intel® Distribution of OpenVINO™ toolkit from a Docker Image {#openvino_docs_install_guides_installing_openvino_docker}
 
 @sphinxdirective
 
@@ -6,31 +6,37 @@
    :description: Learn how to use a prebuilt Docker image or create an image 
                  manually to install OpenVINO™ Runtime on Linux and Windows operating systems.
 
+This guide presents infromation on how to use a pre-built Docker image/create an image manually to install OpenVINO™ Runtime.
 
-Supported operating systems for the Docker Base image: 
+Supported host operating systems for the Docker Base image: 
 
-- Ubuntu 22.04 LTS
-- Ubuntu 20.04 LTS
-- RedHat UBI 8
+- Linux
 - Windows (WSL2)
+- macOS (CPU exectuion only)
 
-.. important::
+You can get started easily with pre-built and published docker images, which are available at:
 
-   While Windows is listed as a supported system, there is no dedicated Docker Image for it. To work with Windows, use Windows Subsystem for Linux (WSL2).
+* `Docker Hub <https://hub.docker.com/u/openvino>`__
+* `Red Hat Quay.io <https://quay.io/organization/openvino>`__
+* `Red Hat Ecosystem Catalog (runtime image) <https://catalog.redhat.com/software/containers/intel/openvino-runtime/606ff4d7ecb5241699188fb3>`__
+* `Red Hat Ecosystem Catalog (development image) <https://catalog.redhat.com/software/containers/intel/openvino-dev/613a450dc9bc35f21dc4a1f7>`__
+* `Azure Marketplace <https://azuremarketplace.microsoft.com/en-us/marketplace/apps/intel_corporation.openvino>`__
 
-The `Docker CI framework <https://github.com/openvinotoolkit/docker_ci/>`__ can generate a Dockerfile, build, test, and deploy an image using the Intel® Distribution of OpenVINO™ toolkit. You can reuse available Dockerfiles, add your layer and customize the OpenVINO™ image to your needs. You can get started easily with pre-built and published docker images. Details on how to get started can be found `here <https://github.com/openvinotoolkit/docker_ci/blob/master/get-started.md>`__.
+You can use the `available Dockerfiles on GitHub <https://github.com/openvinotoolkit/docker_ci/tree/master/dockerfiles>`__ or generate a Dockerfile with your settings via `DockerHub CI framework <https://github.com/openvinotoolkit/docker_ci/>`__ , which can generate a Dockerfile, build, test, and deploy an image using the Intel® Distribution of OpenVINO™ toolkit. You can reuse available Dockerfiles, add your layer and customize the OpenVINO™ image to your needs. Docker CI repository includes guides on how to how to `get started with docker images <https://github.com/openvinotoolkit/docker_ci/blob/master/get-started.md>`__ and how to use `OpenVINO™ Toolkit containers with GPU accelerators. <https://github.com/openvinotoolkit/docker_ci/blob/master/docs/accelerators.md>`__
 
-To start using them, the following conditions must be met:
+To start using Dockerfiles the following conditions must be met:
 
 - Linux OS or Windows (under :ref:`Windows Subsystem for Linux (WSL2) <wsl-install>`)
 - Installed docker engine or compatible container engine
 - Permissions to run containers (sudo or docker group membership)
 
-OpenVINO's `Docker <https://docs.docker.com/>`__ and `Bare Metal <https://docs.openvino.ai/2023.0/ovms_docs_deploying_server.html#doxid-ovms-docs-deploying-server>`__ distributions are identical, so the documentation applies to both.
+.. note::
+
+   OpenVINO's `Docker <https://docs.docker.com/>`__ and :doc:`Bare Metal <openvino_docs_install_guides_overview>` distributions are identical, so the documentation applies to both. 
 
 .. note:: 
 
-   The OpenVINO development environment in a docker container is also available in the `notebook repository <https://github.com/openvinotoolkit/openvino_notebooks>`__ . It can be implemented in `OpenShift RedHat OpenData Science (RHODS) <https://github.com/openvinotoolkit/operator/blob/main/docs/notebook_in_rhods.md>`__.
+   OpenVINO development environment in a docker container is also available in the `notebook repository <https://github.com/openvinotoolkit/openvino_notebooks>`__ . It can be implemented in `OpenShift RedHat OpenData Science (RHODS) <https://github.com/openvinotoolkit/operator/blob/main/docs/notebook_in_rhods.md>`__.
 
 More information about Docker CI for Intel® Distribution of OpenVINO™ toolset can be found `here <https://github.com/openvinotoolkit/docker_ci/blob/master/README.md>`__
 

docs/install_guides/installing-openvino-linux-header.md

@@ -18,7 +18,7 @@
    Use YUM <openvino_docs_install_guides_installing_openvino_yum>
    Use Conda Forge <openvino_docs_install_guides_installing_openvino_conda>
    Use Homebrew <openvino_docs_install_guides_installing_openvino_brew>
-   Use Docker <openvino_docs_install_guides_installing_openvino_docker_linux>
+   Use Docker <openvino_docs_install_guides_installing_openvino_docker>
 
 
 If you want to install OpenVINO™ Runtime on your Linux machine, these are your options: 
@@ -29,7 +29,7 @@ If you want to install OpenVINO™ Runtime on your Linux machine, these are your
 * :doc:`Install OpenVINO Runtime using YUM <openvino_docs_install_guides_installing_openvino_yum>`
 * :doc:`Install OpenVINO Runtime using Conda Forge <openvino_docs_install_guides_installing_openvino_conda>`
 * :doc:`Install OpenVINO Runtime using Homebrew <openvino_docs_install_guides_installing_openvino_brew>`
-* :doc:`Install OpenVINO using Docker <openvino_docs_install_guides_installing_openvino_docker_linux>`
+* :doc:`Install OpenVINO using Docker <openvino_docs_install_guides_installing_openvino_docker>`
 
 
 

docs/install_guides/installing-openvino-pip.md

@@ -111,16 +111,16 @@ Now that you've installed OpenVINO Runtime, you're ready to run your own machine
 .. image:: https://user-images.githubusercontent.com/15709723/127752390-f6aa371f-31b5-4846-84b9-18dd4f662406.gif
    :width: 400
 
-Try the `Python Quick Start Example <https://docs.openvino.ai/2022.3/notebooks/201-vision-monodepth-with-output.html>`__ to estimate depth in a scene using an OpenVINO monodepth model in a Jupyter Notebook inside your web browser.
+Try the `Python Quick Start Example <https://docs.openvino.ai/2023.1/notebooks/201-vision-monodepth-with-output.html>`__ to estimate depth in a scene using an OpenVINO monodepth model in a Jupyter Notebook inside your web browser.
 
 Get started with Python
 +++++++++++++++++++++++
 
 Visit the :doc:`Tutorials <tutorials>` page for more Jupyter Notebooks to get you started with OpenVINO, such as:
 
-* `OpenVINO Python API Tutorial <https://docs.openvino.ai/2022.3/notebooks/002-openvino-api-with-output.html>`__
-* `Basic image classification program with Hello Image Classification <https://docs.openvino.ai/2022.3/notebooks/001-hello-world-with-output.html>`__
-* `Convert a PyTorch model and use it for image background removal <https://docs.openvino.ai/2022.3/notebooks/205-vision-background-removal-with-output.html>`__
+* `OpenVINO Python API Tutorial <https://docs.openvino.ai/2023.1/notebooks/002-openvino-api-with-output.html>`__
+* `Basic image classification program with Hello Image Classification <https://docs.openvino.ai/2023.1/notebooks/001-hello-world-with-output.html>`__
+* `Convert a PyTorch model and use it for image background removal <https://docs.openvino.ai/2023.1/notebooks/205-vision-background-removal-with-output.html>`__
 
 Run OpenVINO on accelerated devices
 +++++++++++++++++++++++++++++++++++

docs/install_guides/installing-openvino-windows-header.md

@@ -15,7 +15,7 @@
    Use Archive <openvino_docs_install_guides_installing_openvino_from_archive_windows>
    Use PyPI <openvino_docs_install_guides_installing_openvino_pip>
    Use Conda Forge <openvino_docs_install_guides_installing_openvino_conda>
-   Use Docker <openvino_docs_install_guides_installing_openvino_docker_linux>
+   Use Docker <openvino_docs_install_guides_installing_openvino_docker>
 
 
 If you want to install OpenVINO™ Runtime on Windows, you have the following options:
@@ -23,7 +23,7 @@ If you want to install OpenVINO™ Runtime on Windows, you have the following op
 * :doc:`Install OpenVINO Runtime from an Archive File <openvino_docs_install_guides_installing_openvino_from_archive_windows>`
 * :doc:`Install OpenVINO Runtime using PyPI <openvino_docs_install_guides_installing_openvino_pip>`
 * :doc:`Install OpenVINO Runtime using Conda Forge <openvino_docs_install_guides_installing_openvino_conda>`
-* :doc:`Install OpenVINO using Docker <openvino_docs_install_guides_installing_openvino_docker_linux>`
+* :doc:`Install OpenVINO using Docker <openvino_docs_install_guides_installing_openvino_docker>`
 
 
 @endsphinxdirective

docs/install_guides/pre-release-note.md

@@ -1,2 +0,0 @@
-
-> **NOTE**: This version is pre-release software and has not undergone full release validation or qualification. No support is offered on pre-release software and APIs/behavior are subject to change.  It should NOT be incorporated into any production software/solution and instead should be used only for early testing and integration while awaiting a final release version of this software.

docs/install_guides/pypi-openvino-dev.md

@@ -1,8 +1,5 @@
 # OpenVINO™ Development Tools 
 
-<!--- The note below is intended for master branch only for pre-release purpose. Remove it for official releases. --->
-> **NOTE**: This version is pre-release software and has not undergone full release validation or qualification. No support is offered on pre-release software and APIs/behavior are subject to change. It should NOT be incorporated into any production software/solution and instead should be used only for early testing and integration while awaiting a final release version of this software.
-
 Intel® Distribution of OpenVINO™ toolkit is an open-source toolkit for optimizing and deploying AI inference. It can be used to develop applications and solutions based on deep learning tasks, such as: emulation of human vision, automatic speech recognition, natural language processing, recommendation systems, etc. It provides high-performance and rich deployment options, from edge to cloud.
 
 OpenVINO™ Development Tools enables you to download models from Open Model Zoo, convert your own models to OpenVINO IR, as well as optimize and tune pre-trained deep learning models. See [What's in the Package](#whats-in-the-package) for more information.
@@ -119,15 +116,15 @@ For example, to install and configure the components for working with TensorFlow
 
 | Component        | Console Script                                                                   | Description                                                                                                                                                                                                                                                                                                   |  
 |------------------|---------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| [Model conversion API](https://docs.openvino.ai/nightly/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html) | `mo` |**Model conversion API** imports, converts, and optimizes models that were trained in popular frameworks to a format usable by OpenVINO components. <br>Supported frameworks include Caffe\*, TensorFlow\*, MXNet\*, PaddlePaddle\*, and ONNX\*.                                               |
-| [Benchmark Tool](https://docs.openvino.ai/nightly/openvino_inference_engine_tools_benchmark_tool_README.html)| `benchmark_app` | **Benchmark Application** allows you to estimate deep learning inference performance on supported devices for synchronous and asynchronous modes.                                              |
-| [Accuracy Checker](https://docs.openvino.ai/nightly/omz_tools_accuracy_checker.html) and <br> [Annotation Converter](https://docs.openvino.ai/nightly/omz_tools_accuracy_checker_annotation_converters.html) | `accuracy_check` <br> `convert_annotation` |**Accuracy Checker**  is a deep learning accuracy validation tool that allows you to collect accuracy metrics against popular datasets. The main advantages of the tool are the flexibility of configuration and a set of supported datasets, preprocessing, postprocessing, and metrics. <br> **Annotation Converter** is a utility that prepares datasets for evaluation with Accuracy Checker.                                             |
-| [Post-Training Optimization Tool](https://docs.openvino.ai/nightly/pot_introduction.html)| `pot` |**Post-Training Optimization Tool** allows you to optimize trained models with advanced capabilities, such as quantization and low-precision optimizations, without the need to retrain or fine-tune models.                                            |
-| [Model Downloader and other Open Model Zoo tools](https://docs.openvino.ai/nightly/omz_tools_downloader.html)| `omz_downloader` <br> `omz_converter` <br> `omz_quantizer` <br> `omz_info_dumper`| **Model Downloader** is a tool for getting access to the collection of high-quality and extremely fast pre-trained deep learning [public](@ref omz_models_group_public) and [Intel](@ref omz_models_group_intel)-trained models. These free pre-trained models can be used to speed up the development and production deployment process without training your own models. The tool downloads model files from online sources and, if necessary, patches them to make them more usable with model conversion API. A number of additional tools are also provided to automate the process of working with downloaded models:<br> **Model Converter** is a tool for converting Open Model Zoo models that are stored in an original deep learning framework format into the OpenVINO Intermediate Representation (IR) using model conversion API. <br> **Model Quantizer** is a tool for automatic quantization of full-precision models in the IR format into low-precision versions using the Post-Training Optimization Tool. <br> **Model Information Dumper** is a helper utility for dumping information about the models to a stable, machine-readable format.                                          |      
+| [Model conversion API](https://docs.openvino.ai/2023.1/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html) | `mo` |**Model conversion API** imports, converts, and optimizes models that were trained in popular frameworks to a format usable by OpenVINO components. <br>Supported frameworks include Caffe\*, TensorFlow\*, MXNet\*, PaddlePaddle\*, and ONNX\*.                                               |
+| [Benchmark Tool](https://docs.openvino.ai/2023.1/openvino_inference_engine_tools_benchmark_tool_README.html)| `benchmark_app` | **Benchmark Application** allows you to estimate deep learning inference performance on supported devices for synchronous and asynchronous modes.                                              |
+| [Accuracy Checker](https://docs.openvino.ai/2023.1/omz_tools_accuracy_checker.html) and <br> [Annotation Converter](https://docs.openvino.ai/2023.1/omz_tools_accuracy_checker_annotation_converters.html) | `accuracy_check` <br> `convert_annotation` |**Accuracy Checker**  is a deep learning accuracy validation tool that allows you to collect accuracy metrics against popular datasets. The main advantages of the tool are the flexibility of configuration and a set of supported datasets, preprocessing, postprocessing, and metrics. <br> **Annotation Converter** is a utility that prepares datasets for evaluation with Accuracy Checker.                                             |
+| [Post-Training Optimization Tool](https://docs.openvino.ai/2023.1/pot_introduction.html)| `pot` |**Post-Training Optimization Tool** allows you to optimize trained models with advanced capabilities, such as quantization and low-precision optimizations, without the need to retrain or fine-tune models.                                            |
+| [Model Downloader and other Open Model Zoo tools](https://docs.openvino.ai/2023.1/omz_tools_downloader.html)| `omz_downloader` <br> `omz_converter` <br> `omz_quantizer` <br> `omz_info_dumper`| **Model Downloader** is a tool for getting access to the collection of high-quality and extremely fast pre-trained deep learning [public](@ref omz_models_group_public) and [Intel](@ref omz_models_group_intel)-trained models. These free pre-trained models can be used to speed up the development and production deployment process without training your own models. The tool downloads model files from online sources and, if necessary, patches them to make them more usable with model conversion API. A number of additional tools are also provided to automate the process of working with downloaded models:<br> **Model Converter** is a tool for converting Open Model Zoo models that are stored in an original deep learning framework format into the OpenVINO Intermediate Representation (IR) using model conversion API. <br> **Model Quantizer** is a tool for automatic quantization of full-precision models in the IR format into low-precision versions using the Post-Training Optimization Tool. <br> **Model Information Dumper** is a helper utility for dumping information about the models to a stable, machine-readable format.                                          |      
 
 ## Troubleshooting
 
-For general troubleshooting steps and issues, see [Troubleshooting Guide for OpenVINO Installation](https://docs.openvino.ai/2023.0/openvino_docs_get_started_guide_troubleshooting.html). The following sections also provide explanations to several error messages. 
+For general troubleshooting steps and issues, see [Troubleshooting Guide for OpenVINO Installation](https://docs.openvino.ai/2023.1/openvino_docs_get_started_guide_troubleshooting.html). The following sections also provide explanations to several error messages. 
 
 ### Errors with Installing via PIP for Users in China
 

docs/install_guides/pypi-openvino-rt.md

@@ -1,11 +1,8 @@
 # OpenVINO™ Runtime
 
-<!--- The note below is intended for master branch only for pre-release purpose. Remove it for official releases. --->
-> **NOTE**: This version is pre-release software and has not undergone full release validation or qualification. No support is offered on pre-release software and APIs/behavior are subject to change. It should NOT be incorporated into any production software/solution and instead should be used only for early testing and integration while awaiting a final release version of this software.
-
 Intel® Distribution of OpenVINO™ toolkit is an open-source toolkit for optimizing and deploying AI inference. It can be used to develop applications and solutions based on deep learning tasks, such as: emulation of human vision, automatic speech recognition, natural language processing, recommendation systems, etc. It provides high-performance and rich deployment options, from edge to cloud.
 
-If you have already finished developing your models and converting them to the OpenVINO model format, you can install OpenVINO Runtime to deploy your applications on various devices. The [OpenVINO™ Runtime](https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_OV_Runtime_User_Guide.html) Python package includes a set of libraries for an easy inference integration with your products.
+If you have already finished developing your models and converting them to the OpenVINO model format, you can install OpenVINO Runtime to deploy your applications on various devices. The [OpenVINO™ Runtime](https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_OV_Runtime_User_Guide.html) Python package includes a set of libraries for an easy inference integration with your products.
 
 ## System Requirements
 
@@ -72,7 +69,7 @@ If installation was successful, you will see the list of available devices.
 
 ## Troubleshooting
 
-For general troubleshooting steps and issues, see [Troubleshooting Guide for OpenVINO Installation](https://docs.openvino.ai/2023.0/openvino_docs_get_started_guide_troubleshooting.html). The following sections also provide explanations to several error messages. 
+For general troubleshooting steps and issues, see [Troubleshooting Guide for OpenVINO Installation](https://docs.openvino.ai/2023.1/openvino_docs_get_started_guide_troubleshooting.html). The following sections also provide explanations to several error messages. 
 
 ### Errors with Installing via PIP for Users in China
 

docs/nbdoc/consts.py

@@ -8,7 +8,7 @@ repo_owner = "openvinotoolkit"
 
 repo_name = "openvino_notebooks"
 
-artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/"
+artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/"
 
 blacklisted_extensions = ['.xml', '.bin']
 

docs/notebooks/001-hello-world-with-output.rst

@@ -1,20 +1,35 @@
 Hello Image Classification
 ==========================
 
+
+
 This basic introduction to OpenVINO™ shows how to do inference with an
 image classification model.
 
 A pre-trained `MobileNetV3
-model <https://docs.openvino.ai/2023.0/omz_models_model_mobilenet_v3_small_1_0_224_tf.html>`__
+model <https://docs.openvino.ai/2023.1/omz_models_model_mobilenet_v3_small_1_0_224_tf.html>`__
 from `Open Model
 Zoo <https://github.com/openvinotoolkit/open_model_zoo/>`__ is used in
 this tutorial. For more information about how OpenVINO IR models are
 created, refer to the `TensorFlow to
 OpenVINO <101-tensorflow-classification-to-openvino-with-output.html>`__
-tutorial.
+tutorial. 
 
-Imports
--------
+
+
+.. _top:
+
+**Table of contents**:
+
+- `Imports <#imports>`__
+- `Download the Model and data samples <#download-the-model-and-data-samples>`__
+- `Select inference device <#select-inference-device>`__
+- `Load the Model <#load-the-model>`__
+- `Load an Image <#load-an-image>`__
+- `Do Inference <#do-inference>`__
+
+Imports `⇑ <#top>`__
+############################################
 
 .. code:: ipython3
 
@@ -29,8 +44,8 @@ Imports
     sys.path.append("../utils")
     from notebook_utils import download_file
 
-Download the Model and data samples
------------------------------------
+Download the Model and data samples `⇑ <#top>`__
+########################################################################
 
 .. code:: ipython3
 
@@ -63,10 +78,10 @@ Download the Model and data samples
     artifacts/v3-small_224_1.0_float.bin:   0%|          | 0.00/4.84M [00:00<?, ?B/s]
 
 
-Select inference device
------------------------
+Select inference device `⇑ <#top>`__
+############################################################
 
-select device from dropdown list for running inference using OpenVINO
+Select device from dropdown list for running inference using OpenVINO:
 
 .. code:: ipython3
 
@@ -91,8 +106,8 @@ select device from dropdown list for running inference using OpenVINO
 
 
 
-Load the Model
---------------
+Load the Model `⇑ <#top>`__
+###################################################
 
 .. code:: ipython3
 
@@ -102,8 +117,8 @@ Load the Model
     
     output_layer = compiled_model.output(0)
 
-Load an Image
--------------
+Load an Image `⇑ <#top>`__
+##################################################
 
 .. code:: ipython3
 
@@ -122,8 +137,8 @@ Load an Image
 .. image:: 001-hello-world-with-output_files/001-hello-world-with-output_10_0.png
 
 
-Do Inference
-------------
+Do Inference `⇑ <#top>`__
+#################################################
 
 .. code:: ipython3
 

docs/notebooks/001-hello-world-with-output_files/index.html

@@ -1,7 +1,7 @@
 <html>
-<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/001-hello-world-with-output_files/</title></head>
+<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/001-hello-world-with-output_files/</title></head>
 <body bgcolor="white">
-<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/001-hello-world-with-output_files/</h1><hr><pre><a href="../">../</a>
-<a href="001-hello-world-with-output_10_0.png">001-hello-world-with-output_10_0.png</a>               12-Jul-2023 00:11              387941
+<h1>Index of /projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/001-hello-world-with-output_files/</h1><hr><pre><a href="../">../</a>
+<a href="001-hello-world-with-output_10_0.png">001-hello-world-with-output_10_0.png</a>               16-Aug-2023 01:31              387941
 </pre><hr></body>
 </html>

docs/notebooks/002-openvino-api-with-output.rst

@@ -4,29 +4,27 @@ OpenVINO™ Runtime API Tutorial
 This notebook explains the basics of the OpenVINO Runtime API. It
 covers:
 
--  `Loading OpenVINO Runtime and Showing
-   Info <#Loading-OpenVINO-Runtime-and-Showing-Info>`__
--  `Loading a Model <#Loading-a-Model>`__
+-  `Loading OpenVINO Runtime and Showing Info <#loading-openvino-runtime-and-showing-info>`__
+-  `Loading a Model <#loading-a-model>`__
 
-   -  `OpenVINO IR Model <#OpenVINO-IR-Model>`__
-   -  `ONNX Model <#ONNX-Model>`__
-   -  `PaddlePaddle Model <#PaddlePaddle-Model>`__
-   -  `TensorFlow Model <#TensorFlow-Model>`__
-   -  `TensorFlow Lite Model <#TensorFlow-Lite-Model>`__
+   -  `OpenVINO IR Model <#openvino-ir-model>`__
+   -  `ONNX Model <#onnx-model>`__
+   -  `PaddlePaddle Model <#paddlepaddle-model>`__
+   -  `TensorFlow Model <#tensorflow-model>`__
+   -  `TensorFlow Lite Model <#tensorflow-lite-model>`__
 
--  `Getting Information about a
-   Model <#Getting-Information-about-a-Model>`__
+-  `Getting Information about a Model <#getting-information-about-a-model>`__
 
-   -  `Model Inputs <#Model-Inputs>`__
-   -  `Model Outputs <#Model-Outputs>`__
+   -  `Model Inputs <#model-inputs>`__
+   -  `Model Outputs <#model-outputs>`__
 
--  `Doing Inference on a Model <#Doing-Inference-on-a-Model>`__
--  `Reshaping and Resizing <#Reshaping-and-Resizing>`__
+-  `Doing Inference on a Model <#doing-inference-on-a-model>`__
+-  `Reshaping and Resizing <#reshaping-and-resizing>`__
 
-   -  `Change Image Size <#Change-Image-Size>`__
-   -  `Change Batch Size <#Change-Batch-Size>`__
+   -  `Change Image Size <#change-image-size>`__
+   -  `Change Batch Size <#change-batch-size>`__
 
--  `Caching a Model <#Caching-a-Model>`__
+-  `Caching a Model <#caching-a-model>`__
 
 The notebook is divided into sections with headers. The next cell
 contains global requirements installation and imports. Each section is
@@ -54,12 +52,12 @@ same.
 
 .. parsed-literal::
 
-    Requirement already satisfied: requests in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2.31.0)
-    Requirement already satisfied: tqdm in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.65.0)
-    Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (3.2.0)
-    Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (3.4)
-    Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (1.26.16)
-    Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (2023.5.7)
+    Requirement already satisfied: requests in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2.31.0)
+    Requirement already satisfied: tqdm in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.66.1)
+    Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (3.2.0)
+    Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (3.4)
+    Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (1.26.16)
+    Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (2023.7.22)
 
 
 Loading OpenVINO Runtime and Showing Info
@@ -106,7 +104,7 @@ After initializing OpenVINO Runtime, first read the model file with
 ``compile_model()`` method.
 
 `OpenVINO™ supports several model
-formats <https://docs.openvino.ai/2023.0/Supported_Model_Formats.html#doxid-supported-model-formats>`__
+formats <https://docs.openvino.ai/2023.1/Supported_Model_Formats.html#doxid-supported-model-formats>`__
 and enables developers to convert them to its own OpenVINO IR format
 using a tool dedicated to this task.
 
@@ -116,7 +114,7 @@ OpenVINO IR Model
 An OpenVINO IR (Intermediate Representation) model consists of an
 ``.xml`` file, containing information about network topology, and a
 ``.bin`` file, containing the weights and biases binary data. Models in
-OpenVINO IR format are obtained by using Model Optimizer tool. The
+OpenVINO IR format are obtained by using model conversion API. The
 ``read_model()`` function expects the ``.bin`` weights file to have the
 same filename and be located in the same directory as the ``.xml`` file:
 ``model_weights_file == Path(model_xml).with_suffix(".bin")``. If this
@@ -124,16 +122,16 @@ is the case, specifying the weights file is optional. If the weights
 file has a different filename, it can be specified using the ``weights``
 parameter in ``read_model()``.
 
-The OpenVINO `Model
-Optimizer <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html#doxid-openvino-docs-m-o-d-g-deep-learning-model-optimizer-dev-guide>`__
-tool is used to convert models to OpenVINO IR format. Model Optimizer
-reads the original model and creates an OpenVINO IR model (.xml and .bin
-files) so inference can be performed without delays due to format
-conversion. Optionally, Model Optimizer can adjust the model to be more
-suitable for inference, for example, by alternating input shapes,
-embedding preprocessing and cutting training parts off. For information
-on how to convert your existing TensorFlow, PyTorch or ONNX model to
-OpenVINO IR format with Model Optimizer, refer to the
+The OpenVINO `model conversion
+API <https://docs.openvino.ai/2023.1/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html#doxid-openvino-docs-m-o-d-g-deep-learning-model-optimizer-dev-guide>`__
+tool is used to convert models to OpenVINO IR format. Model conversion
+API reads the original model and creates an OpenVINO IR model (``.xml``
+and ``.bin`` files) so inference can be performed without delays due to
+format conversion. Optionally, model conversion API can adjust the model
+to be more suitable for inference, for example, by alternating input
+shapes, embedding preprocessing and cutting training parts off. For
+information on how to convert your existing TensorFlow, PyTorch or ONNX
+model to OpenVINO IR format with model conversion API, refer to the
 `tensorflow-to-openvino <101-tensorflow-classification-to-openvino-with-output.html>`__
 and
 `pytorch-onnx-to-openvino <102-pytorch-onnx-to-openvino-with-output.html>`__
@@ -165,7 +163,7 @@ notebooks.
 
 .. parsed-literal::
 
-    PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
+    PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
 
 
 
@@ -212,7 +210,7 @@ points to the filename of an ONNX model.
 
 .. parsed-literal::
 
-    PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/segmentation.onnx')
+    PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/segmentation.onnx')
 
 
 
@@ -268,7 +266,7 @@ without any conversion step. Pass the filename with extension to
 
 .. parsed-literal::
 
-    PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/inference.pdiparams')
+    PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/inference.pdiparams')
 
 
 
@@ -292,12 +290,16 @@ TensorFlow Model
 ~~~~~~~~~~~~~~~~
 
 TensorFlow models saved in frozen graph format can also be passed to
-``read_model`` starting in OpenVINO 2022.3. 
+``read_model`` starting in OpenVINO 2022.3.
 
 .. note::
 
-   * Directly loading TensorFlow models is available as a preview feature in the OpenVINO 2022.3 release. Fully functional support will be provided in the upcoming 2023 releases.
-   * Currently support is limited to only frozen graph inference format. Other TensorFlow model formats must be converted to OpenVINO IR using `Model Optimizer <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow.html>`__.
+   Directly loading TensorFlow models is available as a
+   preview feature in the OpenVINO 2022.3 release. Fully functional
+   support will be provided in the upcoming 2023 releases. Currently
+   support is limited to only frozen graph inference format. Other
+   TensorFlow model formats must be converted to OpenVINO IR using
+   `model conversion API <https://docs.openvino.ai/2023.1/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow.html>`__.
 
 
 .. code:: ipython3
@@ -318,7 +320,7 @@ TensorFlow models saved in frozen graph format can also be passed to
 
 .. parsed-literal::
 
-    PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.pb')
+    PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.pb')
 
 
 
@@ -370,7 +372,7 @@ It is pre-trained model optimized to work with TensorFlow Lite.
 
 .. parsed-literal::
 
-    PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.tflite')
+    PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.tflite')
 
 
 
@@ -395,7 +397,7 @@ Getting Information about a Model
 The OpenVINO Model instance stores information about the model.
 Information about the inputs and outputs of the model are in
 ``model.inputs`` and ``model.outputs``. These are also properties of the
-CompiledModel instance. While using ``model.inputs`` and
+``CompiledModel`` instance. While using ``model.inputs`` and
 ``model.outputs`` in the cells below, you can also use
 ``compiled_model.inputs`` and ``compiled_model.outputs``.
 
@@ -419,7 +421,7 @@ CompiledModel instance. While using ``model.inputs`` and
 
 .. parsed-literal::
 
-    PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
+    PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
 
 
 
@@ -563,13 +565,15 @@ classes (``C``). The output is returned as 32-bit floating point.
 Doing Inference on a Model
 --------------------------
 
-   **NOTE** this notebook demonstrates only the basic synchronous
-   inference API. For an async inference example, please refer to `Async
-   API notebook <115-async-api-with-output.html>`__
+.. note::
+
+   This notebook demonstrates only the basic synchronous
+   inference API. For an async inference example, please refer to 
+   `Async API notebook <115-async-api-with-output.html>`__
 
 The diagram below shows a typical inference pipeline with OpenVINO
 
-.. figure:: https://docs.openvino.ai/2023.0/_images/IMPLEMENT_PIPELINE_with_API_C.svg
+.. figure:: https://docs.openvino.ai/2023.1/_images/IMPLEMENT_PIPELINE_with_API_C.svg
    :alt: image.png
 
    image.png
@@ -581,8 +585,8 @@ on a model, first create an inference request by calling the
 ``compiled_model`` that was loaded with ``compile_model()``. Then, call
 the ``infer()`` method of ``InferRequest``. It expects one argument:
 ``inputs``. This is a dictionary that maps input layer names to input
-data or list of input data in np.ndarray format, where the position of
-the input tensor corresponds to input index. If a model has a single
+data or list of input data in ``np.ndarray`` format, where the position
+of the input tensor corresponds to input index. If a model has a single
 input, wrapping to a dictionary or list can be omitted.
 
 .. code:: ipython3
@@ -612,7 +616,7 @@ input, wrapping to a dictionary or list can be omitted.
 
 .. parsed-literal::
 
-    PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
+    PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
 
 
 
@@ -707,10 +711,10 @@ add the ``N`` dimension (where ``N``\ = 1) by calling the
 **Do inference**
 
 Now that the input data is in the right shape, run inference. The
-CompiledModel inference result is a dictionary where keys are the Output
-class instances (the same keys in ``compiled_model.outputs`` that can
-also be obtained with ``compiled_model.output(index)``) and values -
-predicted result in np.array format.
+``CompiledModel`` inference result is a dictionary where keys are the
+Output class instances (the same keys in ``compiled_model.outputs`` that
+can also be obtained with ``compiled_model.output(index)``) and values -
+predicted result in ``np.array`` format.
 
 .. code:: ipython3
 
@@ -797,7 +801,7 @@ input shape.
 
 .. parsed-literal::
 
-    PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/segmentation.bin')
+    PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/segmentation.bin')
 
 
 
@@ -926,7 +930,9 @@ model will be loaded to the GPU. After running this cell once, the model
 will be cached, so subsequent runs of this cell will load the model from
 the cache.
 
-*Note: Model Caching is also available on CPU devices*
+.. note::
+
+   Model Caching is also available on CPU devices
 
 .. code:: ipython3
 
@@ -948,7 +954,7 @@ the cache.
 
 .. parsed-literal::
 
-    PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
+    PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
 
 
 

docs/notebooks/003-hello-segmentation-with-output.rst

@@ -1,17 +1,34 @@
 Hello Image Segmentation
 ========================
 
+
+
 A very basic introduction to using segmentation models with OpenVINO™.
 
 In this tutorial, a pre-trained
-`road-segmentation-adas-0001 <https://docs.openvino.ai/2023.0/omz_models_model_road_segmentation_adas_0001.html>`__
+`road-segmentation-adas-0001 <https://docs.openvino.ai/2023.1/omz_models_model_road_segmentation_adas_0001.html>`__
 model from the `Open Model
 Zoo <https://github.com/openvinotoolkit/open_model_zoo/>`__ is used.
 ADAS stands for Advanced Driver Assistance Services. The model
 recognizes four classes: background, road, curb and mark.
 
-Imports
--------
+
+
+.. _top:
+
+**Table of contents**:
+
+- `Imports <#imports>`__
+- `Download model weights <#download-model-weights>`__
+- `Select inference device <#select-inference-device>`__
+- `Load the Model <#load-the-model>`__
+- `Load an Image <#load-an-image>`__
+- `Do Inference <#do-inference>`__
+- `Prepare Data for Visualization <#prepare-data-for-visualization>`__
+- `Visualize data <#visualize-data>`__
+
+Imports `⇑ <#top>`__
+#########################################
 
 .. code:: ipython3
 
@@ -24,8 +41,9 @@ Imports
     sys.path.append("../utils")
     from notebook_utils import segmentation_map_to_image, download_file
 
-Download model weights
-----------------------
+Download model weights `⇑ <#top>`__
+#############################################################################################################################
+
 
 .. code:: ipython3
 
@@ -40,8 +58,8 @@ Download model weights
     model_xml_path = base_model_dir / model_xml_name
     
     if not model_xml_path.exists():
-        model_xml_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/road-segmentation-adas-0001/FP32/road-segmentation-adas-0001.xml"
-        model_bin_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/road-segmentation-adas-0001/FP32/road-segmentation-adas-0001.bin"
+        model_xml_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.1/models_bin/1/road-segmentation-adas-0001/FP32/road-segmentation-adas-0001.xml"
+        model_bin_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.1/models_bin/1/road-segmentation-adas-0001/FP32/road-segmentation-adas-0001.bin"
     
         download_file(model_xml_url, model_xml_name, base_model_dir)
         download_file(model_bin_url, model_bin_name, base_model_dir)
@@ -61,10 +79,11 @@ Download model weights
     model/road-segmentation-adas-0001.bin:   0%|          | 0.00/720k [00:00<?, ?B/s]
 
 
-Select inference device
------------------------
+Select inference device `⇑ <#top>`__
+#############################################################################################################################
 
-select device from dropdown list for running inference using OpenVINO
+
+Select device from dropdown list for running inference using OpenVINO:
 
 .. code:: ipython3
 
@@ -89,8 +108,9 @@ select device from dropdown list for running inference using OpenVINO
 
 
 
-Load the Model
---------------
+Load the Model `⇑ <#top>`__
+#############################################################################################################################
+
 
 .. code:: ipython3
 
@@ -102,11 +122,10 @@ Load the Model
     input_layer_ir = compiled_model.input(0)
     output_layer_ir = compiled_model.output(0)
 
-Load an Image
--------------
+Load an Image `⇑ <#top>`__
+#############################################################################################################################
 
-A sample image from the `Mapillary
-Vistas <https://www.mapillary.com/dataset/vistas>`__ dataset is
+A sample image from the `Mapillary Vistas <https://www.mapillary.com/dataset/vistas>`__ dataset is
 provided.
 
 .. code:: ipython3
@@ -134,7 +153,7 @@ provided.
 
 .. parsed-literal::
 
-    <matplotlib.image.AxesImage at 0x7f021436e0a0>
+    <matplotlib.image.AxesImage at 0x7f1f2c248430>
 
 
 
@@ -142,8 +161,9 @@ provided.
 .. image:: 003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_10_1.png
 
 
-Do Inference
-------------
+Do Inference `⇑ <#top>`__
+#############################################################################################################################
+
 
 .. code:: ipython3
 
@@ -159,7 +179,7 @@ Do Inference
 
 .. parsed-literal::
 
-    <matplotlib.image.AxesImage at 0x7f02142bce50>
+    <matplotlib.image.AxesImage at 0x7f1f2c18bfd0>
 
 
 
@@ -167,8 +187,9 @@ Do Inference
 .. image:: 003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_12_1.png
 
 
-Prepare Data for Visualization
-------------------------------
+Prepare Data for Visualization `⇑ <#top>`__
+#############################################################################################################################
+
 
 .. code:: ipython3
 
@@ -185,8 +206,9 @@ Prepare Data for Visualization
     # Create an image with mask.
     image_with_mask = cv2.addWeighted(resized_mask, alpha, rgb_image, 1 - alpha, 0)
 
-Visualize data
---------------
+Visualize data `⇑ <#top>`__
+#############################################################################################################################
+
 
 .. code:: ipython3
 

docs/notebooks/003-hello-segmentation-with-output_files/index.html

@@ -1,9 +1,9 @@
 <html>
-<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/003-hello-segmentation-with-output_files/</title></head>
+<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/003-hello-segmentation-with-output_files/</title></head>
 <body bgcolor="white">
-<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/003-hello-segmentation-with-output_files/</h1><hr><pre><a href="../">../</a>
-<a href="003-hello-segmentation-with-output_10_1.png">003-hello-segmentation-with-output_10_1.png</a>        12-Jul-2023 00:11              249032
-<a href="003-hello-segmentation-with-output_12_1.png">003-hello-segmentation-with-output_12_1.png</a>        12-Jul-2023 00:11               20550
-<a href="003-hello-segmentation-with-output_16_0.png">003-hello-segmentation-with-output_16_0.png</a>        12-Jul-2023 00:11              260045
+<h1>Index of /projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/003-hello-segmentation-with-output_files/</h1><hr><pre><a href="../">../</a>
+<a href="003-hello-segmentation-with-output_10_1.png">003-hello-segmentation-with-output_10_1.png</a>        16-Aug-2023 01:31              249032
+<a href="003-hello-segmentation-with-output_12_1.png">003-hello-segmentation-with-output_12_1.png</a>        16-Aug-2023 01:31               20550
+<a href="003-hello-segmentation-with-output_16_0.png">003-hello-segmentation-with-output_16_0.png</a>        16-Aug-2023 01:31              260045
 </pre><hr></body>
 </html>

docs/notebooks/004-hello-detection-with-output.rst

@@ -1,11 +1,13 @@
 Hello Object Detection
 ======================
 
+
+
 A very basic introduction to using object detection models with
 OpenVINO™.
 
 The
-`horizontal-text-detection-0001 <https://docs.openvino.ai/2023.0/omz_models_model_horizontal_text_detection_0001.html>`__
+`horizontal-text-detection-0001 <https://docs.openvino.ai/2023.1/omz_models_model_horizontal_text_detection_0001.html>`__
 model from `Open Model
 Zoo <https://github.com/openvinotoolkit/open_model_zoo/>`__ is used. It
 detects horizontal text in images and returns a blob of data in the
@@ -14,10 +16,24 @@ shape of ``[100, 5]``. Each detected text box is stored in the
 ``(x_min, y_min)`` are the coordinates of the top left bounding box
 corner, ``(x_max, y_max)`` are the coordinates of the bottom right
 bounding box corner and ``conf`` is the confidence for the predicted
-class.
+class. 
 
-Imports
--------
+
+
+.. _top:
+
+**Table of contents**: 
+
+- `Imports <#imports>`__ 
+- `Download model weights <#download-model-weights>`__ 
+- `Select inference device <#select-inference-device>`__ 
+- `Load the Model <#load-the-model>`__ 
+- `Load an Image <#load-an-image>`__ 
+- `Do Inference <#do-inference>`__ 
+- `Visualize Results <#visualize-results>`__
+
+Imports `⇑ <#top>`__
+########################################
 
 .. code:: ipython3
 
@@ -31,8 +47,8 @@ Imports
     sys.path.append("../utils")
     from notebook_utils import download_file
 
-Download model weights
-----------------------
+Download model weights `⇑ <#top>`__
+#######################################################
 
 .. code:: ipython3
 
@@ -67,10 +83,10 @@ Download model weights
     model/horizontal-text-detection-0001.bin:   0%|          | 0.00/7.39M [00:00<?, ?B/s]
 
 
-Select inference device
------------------------
+Select inference device `⇑ <#top>`__
+###########################################################
 
-select device from dropdown list for running inference using OpenVINO
+Select device from dropdown list for running inference using OpenVINO:
 
 .. code:: ipython3
 
@@ -95,8 +111,8 @@ select device from dropdown list for running inference using OpenVINO
 
 
 
-Load the Model
---------------
+Load the Model `⇑ <#top>`__
+###############################################
 
 .. code:: ipython3
 
@@ -108,8 +124,8 @@ Load the Model
     input_layer_ir = compiled_model.input(0)
     output_layer_ir = compiled_model.output("boxes")
 
-Load an Image
--------------
+Load an Image `⇑ <#top>`__
+##############################################
 
 .. code:: ipython3
 
@@ -132,8 +148,8 @@ Load an Image
 .. image:: 004-hello-detection-with-output_files/004-hello-detection-with-output_10_0.png
 
 
-Do Inference
-------------
+Do Inference `⇑ <#top>`__
+##############################################
 
 .. code:: ipython3
 
@@ -143,8 +159,8 @@ Do Inference
     # Remove zero only boxes.
     boxes = boxes[~np.all(boxes == 0, axis=1)]
 
-Visualize Results
------------------
+Visualize Results `⇑ <#top>`__
+##################################################
 
 .. code:: ipython3
 

docs/notebooks/004-hello-detection-with-output_files/index.html

@@ -1,8 +1,8 @@
 <html>
-<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/004-hello-detection-with-output_files/</title></head>
+<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/004-hello-detection-with-output_files/</title></head>
 <body bgcolor="white">
-<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/004-hello-detection-with-output_files/</h1><hr><pre><a href="../">../</a>
-<a href="004-hello-detection-with-output_10_0.png">004-hello-detection-with-output_10_0.png</a>           12-Jul-2023 00:11              305482
-<a href="004-hello-detection-with-output_15_0.png">004-hello-detection-with-output_15_0.png</a>           12-Jul-2023 00:11              457214
+<h1>Index of /projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/004-hello-detection-with-output_files/</h1><hr><pre><a href="../">../</a>
+<a href="004-hello-detection-with-output_10_0.png">004-hello-detection-with-output_10_0.png</a>           16-Aug-2023 01:31              305482
+<a href="004-hello-detection-with-output_15_0.png">004-hello-detection-with-output_15_0.png</a>           16-Aug-2023 01:31              457214
 </pre><hr></body>
 </html>

docs/notebooks/101-tensorflow-classification-to-openvino-with-output.rst

@@ -1,18 +1,46 @@
 Convert a TensorFlow Model to OpenVINO™
 =======================================
 
-This short tutorial shows how to convert a TensorFlow
-`MobileNetV3 <https://docs.openvino.ai/2023.0/omz_models_model_mobilenet_v3_small_1_0_224_tf.html>`__
-image classification model to OpenVINO `Intermediate
-Representation <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_IR_and_opsets.html>`__
-(OpenVINO IR) format, using `Model
-Optimizer <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
-After creating the OpenVINO IR, load the model in `OpenVINO
-Runtime <https://docs.openvino.ai/nightly/openvino_docs_OV_UG_OV_Runtime_User_Guide.html>`__
-and do inference with a sample image.
 
-Imports
--------
+
+| This short tutorial shows how to convert a TensorFlow
+  `MobileNetV3 <https://docs.openvino.ai/2023.1/omz_models_model_mobilenet_v3_small_1_0_224_tf.html>`__
+  image classification model to OpenVINO `Intermediate
+  Representation <https://docs.openvino.ai/2023.1/openvino_docs_MO_DG_IR_and_opsets.html>`__
+  (OpenVINO IR) format, using `model conversion
+  API <https://docs.openvino.ai/2023.1/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
+  After creating the OpenVINO IR, load the model in `OpenVINO
+  Runtime <https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_OV_Runtime_User_Guide.html>`__
+  and do inference with a sample image.
+
+
+
+| .. _top:
+
+**Table of contents**:
+
+- `Imports <#imports>`__
+- `Settings <#settings>`__
+- `Download model <#download-model>`__
+- `Convert a Model to OpenVINO IR Format <#convert-a-model-to-openvino-ir-format>`__
+
+  - `Convert a TensorFlow Model to OpenVINO IR Format <#convert-a-tensorflow-model-to-openvino-ir-format>`__
+
+- `Test Inference on the Converted Model <#test-inference-on-the-converted-model>`__
+
+  - `Load the Model <#load-the-model>`__
+
+- `Select inference device <#select-inference-device>`__
+
+  - `Get Model Information <#get-model-information>`__
+  - `Load an Image <#load-an-image>`__
+  - `Do Inference <#do-inference>`__
+
+- `Timing <#timing>`__
+
+Imports `⇑ <#top>`__
+###############################################################################################################################
+
 
 .. code:: ipython3
 
@@ -29,14 +57,15 @@ Imports
 
 .. parsed-literal::
 
-    2023-07-11 22:24:31.659014: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
-    2023-07-11 22:24:31.694466: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
+    2023-08-15 22:26:34.199621: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
+    2023-08-15 22:26:34.233464: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
     To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
-    2023-07-11 22:24:32.210417: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
+    2023-08-15 22:26:34.746193: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
 
 
-Settings
---------
+Settings `⇑ <#top>`__
+###############################################################################################################################
+
 
 .. code:: ipython3
 
@@ -48,8 +77,9 @@ Settings
     
     ir_path = Path("model/v3-small_224_1.0_float.xml")
 
-Download model
---------------
+Download model `⇑ <#top>`__
+###############################################################################################################################
+
 
 Load model using `tf.keras.applications
 api <https://www.tensorflow.org/api_docs/python/tf/keras/applications/MobileNetV3Small>`__
@@ -68,7 +98,7 @@ and save it to the disk.
 
 .. parsed-literal::
 
-    2023-07-11 22:24:33.117393: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
+    2023-08-15 22:26:35.659386: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
     Skipping registering GPU devices...
 
 
@@ -79,9 +109,9 @@ and save it to the disk.
 
 .. parsed-literal::
 
-    2023-07-11 22:24:37.315661: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'inputs' with dtype float and shape [?,1,1,1024]
+    2023-08-15 22:26:39.846021: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'inputs' with dtype float and shape [?,1,1,1024]
     	 [[{{node inputs}}]]
-    2023-07-11 22:24:40.473876: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'inputs' with dtype float and shape [?,1,1,1024]
+    2023-08-15 22:26:42.992490: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'inputs' with dtype float and shape [?,1,1,1024]
     	 [[{{node inputs}}]]
     WARNING:absl:Found untraced functions such as _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op while saving (showing 5 of 54). These functions will not be directly callable after loading.
 
@@ -96,25 +126,27 @@ and save it to the disk.
     INFO:tensorflow:Assets written to: model/v3-small_224_1.0_float/assets
 
 
-Convert a Model to OpenVINO IR Format
--------------------------------------
+Convert a Model to OpenVINO IR Format `⇑ <#top>`__
+###############################################################################################################################
 
-Convert a TensorFlow Model to OpenVINO IR Format
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Call the OpenVINO Model Optimizer Python API to convert the TensorFlow
-model to OpenVINO IR. ``mo.convert_model`` function accept path to saved
+Convert a TensorFlow Model to OpenVINO IR Format `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+
+Use the model conversion Python API to convert the TensorFlow model to
+OpenVINO IR. The ``mo.convert_model`` function accept path to saved
 model directory and returns OpenVINO Model class instance which
-represents this model. Obtained model is ready to use and loading on
-device using ``compile_model`` or can be saved on disk using
-``serialize`` function. See the `Model Optimizer Developer
-Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow.html>`__
-for more information about Model Optimizer and TensorFlow models
-conversion.
+represents this model. Obtained model is ready to use and to be loaded
+on a device using ``compile_model`` or can be saved on a disk using the
+``serialize`` function. See the
+`tutorial <https://docs.openvino.ai/2023.1/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow.html>`__
+for more information about using model conversion API with TensorFlow
+models.
 
 .. code:: ipython3
 
-    # Run Model Optimizer if the IR model file does not exist
+    # Run model conversion API if the IR model file does not exist
     if not ir_path.exists():
         print("Exporting TensorFlow model to IR... This may take a few minutes.")
         ov_model = mo.convert_model(saved_model_dir=model_path, input_shape=[[1, 224, 224, 3]], compress_to_fp16=True)
@@ -128,21 +160,24 @@ conversion.
     Exporting TensorFlow model to IR... This may take a few minutes.
 
 
-Test Inference on the Converted Model
--------------------------------------
+Test Inference on the Converted Model `⇑ <#top>`__
+###############################################################################################################################
+
+
+Load the Model `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
-Load the Model
-~~~~~~~~~~~~~~
 
 .. code:: ipython3
 
     core = Core()
     model = core.read_model(ir_path)
 
-Select inference device
------------------------
+Select inference device `⇑ <#top>`__
+###############################################################################################################################
 
-select device from dropdown list for running inference using OpenVINO
+
+Select device from dropdown list for running inference using OpenVINO:
 
 .. code:: ipython3
 
@@ -170,8 +205,9 @@ select device from dropdown list for running inference using OpenVINO
 
     compiled_model = core.compile_model(model=model, device_name=device.value)
 
-Get Model Information
-~~~~~~~~~~~~~~~~~~~~~
+Get Model Information `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 .. code:: ipython3
 
@@ -179,8 +215,9 @@ Get Model Information
     output_key = compiled_model.output(0)
     network_input_shape = input_key.shape 
 
-Load an Image
-~~~~~~~~~~~~~
+Load an Image `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 Load an image, resize it, and convert it to the input shape of the
 network.
@@ -203,8 +240,9 @@ network.
 .. image:: 101-tensorflow-classification-to-openvino-with-output_files/101-tensorflow-classification-to-openvino-with-output_18_0.png
 
 
-Do Inference
-~~~~~~~~~~~~
+Do Inference `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 .. code:: ipython3
 
@@ -228,13 +266,14 @@ Do Inference
 
 
 
-Timing
-------
+Timing `⇑ <#top>`__
+###############################################################################################################################
+
 
 Measure the time it takes to do inference on thousand images. This gives
 an indication of performance. For more accurate benchmarking, use the
 `Benchmark
-Tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_tool_README.html>`__
+Tool <https://docs.openvino.ai/2023.1/openvino_inference_engine_tools_benchmark_tool_README.html>`__
 in OpenVINO. Note that many optimizations are possible to improve the
 performance.
 
@@ -258,5 +297,5 @@ performance.
 
 .. parsed-literal::
 
-    IR model in OpenVINO Runtime/CPU: 0.0010 seconds per image, FPS: 998.88
+    IR model in OpenVINO Runtime/CPU: 0.0010 seconds per image, FPS: 988.20
 

docs/notebooks/101-tensorflow-classification-to-openvino-with-output_files/index.html

@@ -1,7 +1,7 @@
 <html>
-<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/101-tensorflow-classification-to-openvino-with-output_files/</title></head>
+<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/101-tensorflow-classification-to-openvino-with-output_files/</title></head>
 <body bgcolor="white">
-<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/101-tensorflow-classification-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
-<a href="101-tensorflow-classification-to-openvino-with-output_18_0.png">101-tensorflow-classification-to-openvino-with-..&gt;</a> 12-Jul-2023 00:11              387941
+<h1>Index of /projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/101-tensorflow-classification-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
+<a href="101-tensorflow-classification-to-openvino-with-output_18_0.png">101-tensorflow-classification-to-openvino-with-..&gt;</a> 16-Aug-2023 01:31              387941
 </pre><hr></body>
 </html>

docs/notebooks/102-pytorch-onnx-to-openvino-with-output.rst

@@ -1,6 +1,8 @@
 Convert a PyTorch Model to ONNX and OpenVINO™ IR
 ================================================
 
+
+
 This tutorial demonstrates step-by-step instructions on how to do
 inference on a PyTorch semantic segmentation model, using OpenVINO
 Runtime.
@@ -28,16 +30,47 @@ all 80 classes, the segmentation model has been trained on 20 classes
 from the `PASCAL VOC <http://host.robots.ox.ac.uk/pascal/VOC/>`__
 dataset: **background, aeroplane, bicycle, bird, boat, bottle, bus, car,
 cat, chair, cow, dining table, dog, horse, motorbike, person, potted
-plant, sheep, sofa, train, tvmonitor**
+plant, sheep, sofa, train, tv monitor**
 
 More information about the model is available in the `torchvision
 documentation <https://pytorch.org/vision/main/models/lraspp.html>`__
 
-Preparation
------------
 
-Imports
-~~~~~~~
+
+.. _top:
+
+**Table of contents**: 
+
+- `Preparation <#preparation>`__
+
+  - `Imports <#imports>`__
+  - `Settings <#settings>`__
+  - `Load Model <#load-model>`__
+
+- `ONNX Model Conversion <#onnx-model-conversion>`__
+
+  - `Convert PyTorch model to ONNX <#convert-pytorch-model-to-onnx>`__
+  - `Convert ONNX Model to OpenVINO IR Format <#convert-onnx-model-to-openvino-ir-format>`__
+
+- `Show Results <#show-results>`__
+
+  - `Load and Preprocess an Input Image <#load-and-preprocess-an-input-image>`__
+  - `Load the OpenVINO IR Network and Run Inference on the ONNX model <#load-the-openvino-ir-network-and-run-inference-on-the-onnx-model>`__
+
+    - `1. ONNX Model in OpenVINO Runtime <#onnx-model-in-openvino-runtime>`__
+    - `Select an inference device <#select-an-inference-device>`__
+    - `2. OpenVINO IR Model in OpenVINO Runtime <#openvino-ir-model-in-openvino-runtime>`__
+    - `Select the inference device <#select-the-inference-device>`__
+
+- `PyTorch Comparison <#pytorch-comparison>`__ 
+- `Performance Comparison <#performance-comparison>`__ 
+- `References <#references>`__
+
+Preparation `⇑ <#top>`__
+########################################################################
+
+Imports `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
 .. code:: ipython3
 
@@ -55,8 +88,8 @@ Imports
     sys.path.append("../utils")
     from notebook_utils import segmentation_map_to_image, viz_result_image, SegmentationMap, Label, download_file
 
-Settings
-~~~~~~~~
+Settings `⇑ <#top>`__
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
 Set a name for the model, then define width and height of the image that
 will be used by the network during inference. According to the input
@@ -77,16 +110,15 @@ transforms function, the model is pre-trained on images with a height of
         onnx_path.parent.mkdir()
     ir_path = onnx_path.with_suffix(".xml")
 
-Load Model
-~~~~~~~~~~
+Load Model `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
 Generally, PyTorch models represent an instance of ``torch.nn.Module``
 class, initialized by a state dictionary with model weights. Typical
-steps for getting a pre-trained model: 
-
-1. Create instance of model class
+steps for getting a pre-trained model: 1. Create instance of model class
 2. Load checkpoint state dict, which contains pre-trained model weights
-3. Turn model to evaluation for switching some operations to inference mode
+3. Turn model to evaluation for switching some operations to inference
+mode
 
 The ``torchvision`` module provides a ready to use set of functions for
 model class initialization. We will use
@@ -129,11 +161,11 @@ have not downloaded the model before.
     Loaded PyTorch LRASPP MobileNetV3 model
 
 
-ONNX Model Conversion
----------------------
+ONNX Model Conversion `⇑ <#top>`__
+################################################################################
 
-Convert PyTorch model to ONNX
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Convert PyTorch model to ONNX `⇑ <#top>`__
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
 OpenVINO supports PyTorch models that are exported in ONNX format. We
 will use the ``torch.onnx.export`` function to obtain the ONNX model,
@@ -172,14 +204,13 @@ line of the output will read:
     ONNX model exported to model/lraspp_mobilenet_v3_large.onnx.
 
 
-Convert ONNX Model to OpenVINO IR Format
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Convert ONNX Model to OpenVINO IR Format `⇑ <#top>`__
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
-Use Model Optimizer to convert the ONNX model to OpenVINO IR with
-``FP16`` precision. The models are saved inside the current directory.
-For more information about Model Optimizer, see the `Model Optimizer
-Developer
-Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
+To convert the ONNX model to OpenVINO IR with ``FP16`` precision, use
+model conversion API. The models are saved inside the current directory.
+For more information on how to convert models, see this
+`page <https://docs.openvino.ai/2023.1/openvino_docs_model_processing_introduction.html>`__.
 
 .. code:: ipython3
 
@@ -199,14 +230,14 @@ Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_O
     Exporting ONNX model to IR... This may take a few minutes.
 
 
-Show Results
-------------
+Show Results `⇑ <#top>`__
+######################################################################
 
 Confirm that the segmentation results look as expected by comparing
 model predictions on the ONNX, OpenVINO IR and PyTorch models.
 
-Load and Preprocess an Input Image
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Load and Preprocess an Input Image `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
 Images need to be normalized before propagating through the network.
 
@@ -237,8 +268,8 @@ Images need to be normalized before propagating through the network.
     input_image = np.expand_dims(np.transpose(resized_image, (2, 0, 1)), 0)
     normalized_input_image = np.expand_dims(np.transpose(normalized_image, (2, 0, 1)), 0)
 
-Load the OpenVINO IR Network and Run Inference on the ONNX model
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Load the OpenVINO IR Network and Run Inference on the ONNX model `⇑ <#top>`__
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
 OpenVINO Runtime can load ONNX models directly. First, load the ONNX
 model, do inference and show the results. Then, load the model that was
@@ -246,8 +277,8 @@ converted to OpenVINO Intermediate Representation (OpenVINO IR) with
 Model Optimizer and do inference on that model, and show the results on
 an image.
 
-1. ONNX Model in OpenVINO Runtime
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+1. ONNX Model in OpenVINO Runtime `⇑ <#top>`__
+------------------------------------------------------------------------------------------
 
 .. code:: ipython3
 
@@ -257,10 +288,10 @@ an image.
     # Read model to OpenVINO Runtime
     model_onnx = core.read_model(model=onnx_path)
 
-Select inference device
-^^^^^^^^^^^^^^^^^^^^^^^
+Select an inference device `⇑ <#top>`__
+...................................................................................
 
-select device from dropdown list for running inference using OpenVINO
+Select a device from dropdown list for running inference using OpenVINO:
 
 .. code:: ipython3
 
@@ -339,13 +370,13 @@ be applied to each label for more convenient visualization.
 
 
 
-2. OpenVINO IR Model in OpenVINO Runtime
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+2. OpenVINO IR Model in OpenVINO Runtime `⇑ <#top>`__
+------------------------------------------------------------------------------------------------------
 
-Select inference device
-^^^^^^^^^^^^^^^^^^^^^^^
+Select the inference device `⇑ <#top>`__
+.....................................................................................
 
-select device from dropdown list for running inference using OpenVINO
+Select a device from dropdown list for running inference using OpenVINO:
 
 .. code:: ipython3
 
@@ -389,8 +420,8 @@ select device from dropdown list for running inference using OpenVINO
 
 
 
-PyTorch Comparison
-------------------
+PyTorch Comparison `⇑ <#top>`__
+############################################################################
 
 Do inference on the PyTorch model to verify that the output visually
 looks the same as the output on the ONNX/OpenVINO IR models.
@@ -415,13 +446,13 @@ looks the same as the output on the ONNX/OpenVINO IR models.
 
 
 
-Performance Comparison
-----------------------
+Performance Comparison `⇑ <#top>`__
+################################################################################
 
 Measure the time it takes to do inference on twenty images. This gives
 an indication of performance. For more accurate benchmarking, use the
 `Benchmark
-Tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_tool_README.html>`__.
+Tool <https://docs.openvino.ai/2023.1/openvino_inference_engine_tools_benchmark_tool_README.html>`__.
 Keep in mind that many optimizations are possible to improve the
 performance.
 
@@ -488,9 +519,9 @@ performance.
 
 .. parsed-literal::
 
-    PyTorch model on CPU: 0.039 seconds per image, FPS: 25.80
-    ONNX model in OpenVINO Runtime/CPU: 0.031 seconds per image, FPS: 31.95
-    OpenVINO IR model in OpenVINO Runtime/CPU: 0.031 seconds per image, FPS: 32.67
+    PyTorch model on CPU: 0.037 seconds per image, FPS: 27.19
+    ONNX model in OpenVINO Runtime/CPU: 0.031 seconds per image, FPS: 32.33
+    OpenVINO IR model in OpenVINO Runtime/CPU: 0.032 seconds per image, FPS: 31.72
 
 
 **Show Device Information**
@@ -508,8 +539,8 @@ performance.
     CPU: Intel(R) Core(TM) i9-10920X CPU @ 3.50GHz
 
 
-References
-----------
+References `⇑ <#top>`__
+######################################################################
 
 -  `Torchvision <https://pytorch.org/vision/stable/index.html>`__
 -  `Pytorch ONNX
@@ -517,7 +548,7 @@ References
 -  `PIP install openvino-dev <https://pypi.org/project/openvino-dev/>`__
 -  `OpenVINO ONNX
    support <https://docs.openvino.ai/2021.4/openvino_docs_IE_DG_ONNX_Support.html>`__
--  `Model Optimizer
-   Documentation <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html>`__
--  `Model Optimizer Pytorch conversion
-   guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_PyTorch.html>`__
+-  `Model Conversion API
+   documentation <https://docs.openvino.ai/2023.1/openvino_docs_model_processing_introduction.html>`__
+-  `Converting Pytorch
+   model <https://docs.openvino.ai/2023.1/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_PyTorch.html>`__

docs/notebooks/102-pytorch-onnx-to-openvino-with-output_files/index.html

@@ -1,9 +1,9 @@
 <html>
-<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/102-pytorch-onnx-to-openvino-with-output_files/</title></head>
+<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/102-pytorch-onnx-to-openvino-with-output_files/</title></head>
 <body bgcolor="white">
-<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/102-pytorch-onnx-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
-<a href="102-pytorch-onnx-to-openvino-with-output_21_0.png">102-pytorch-onnx-to-openvino-with-output_21_0.png</a>  12-Jul-2023 00:11              465692
-<a href="102-pytorch-onnx-to-openvino-with-output_26_0.png">102-pytorch-onnx-to-openvino-with-output_26_0.png</a>  12-Jul-2023 00:11              465695
-<a href="102-pytorch-onnx-to-openvino-with-output_28_0.png">102-pytorch-onnx-to-openvino-with-output_28_0.png</a>  12-Jul-2023 00:11              465692
+<h1>Index of /projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/102-pytorch-onnx-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
+<a href="102-pytorch-onnx-to-openvino-with-output_21_0.png">102-pytorch-onnx-to-openvino-with-output_21_0.png</a>  16-Aug-2023 01:31              465692
+<a href="102-pytorch-onnx-to-openvino-with-output_26_0.png">102-pytorch-onnx-to-openvino-with-output_26_0.png</a>  16-Aug-2023 01:31              465695
+<a href="102-pytorch-onnx-to-openvino-with-output_28_0.png">102-pytorch-onnx-to-openvino-with-output_28_0.png</a>  16-Aug-2023 01:31              465692
 </pre><hr></body>
 </html>

docs/notebooks/102-pytorch-to-openvino-with-output.rst

@@ -1,6 +1,8 @@
 Convert a PyTorch Model to OpenVINO™ IR
 =======================================
 
+
+
 This tutorial demonstrates step-by-step instructions on how to do
 inference on a PyTorch classification model using OpenVINO Runtime.
 Starting from OpenVINO 2023.0 release, OpenVINO supports direct PyTorch
@@ -27,12 +29,49 @@ network design spaces that parametrize populations of networks. The
 overall process is analogous to the classic manual design of networks
 but elevated to the design space level. The RegNet design space provides
 simple and fast networks that work well across a wide range of flop
-regimes.
+regimes. 
 
-Prerequisites
--------------
 
-Install notebook dependecies
+
+.. _top:
+
+**Table of contents**:
+
+- `Prerequisites <#prerequisites>`__
+- `Load PyTorch Model <#load-pytorch-model>`__
+
+  - `Prepare Input Data <#prepare-input-data>`__
+  - `Run PyTorch Model Inference <#run-pytorch-model-inference>`__
+  - `Benchmark PyTorch Model Inference <#benchmark-pytorch-model-inference>`__
+
+- `Convert PyTorch Model to OpenVINO Intermediate Representation <#convert-pytorch-model-to-openvino-intermediate-representation>`__
+
+  - `Select inference device <#select-inference-device>`__
+  - `Run OpenVINO Model Inference <#run-openvino-model-inference>`__
+  - `Benchmark OpenVINO Model Inference <#benchmark-openvino-model-inference>`__
+
+- `Convert PyTorch Model with Static Input Shape <#convert-pytorch-model-with-static-input-shape>`__
+
+  - `Select inference device <#select-inference-device>`__
+  - `Run OpenVINO Model Inference with Static Input Shape <#run-openvino-model-inference-with-static-input-shape>`__
+  - `Benchmark OpenVINO Model Inference with Static Input Shape <#benchmark-openvino-model-inference-with-static-input-shape>`__
+
+- `Convert TorchScript Model to OpenVINO Intermediate Representation <#convert-torchscript-model-to-openvino-intermediate-representation>`__
+
+  - `Scripted Model <#scripted-model>`__
+  - `Benchmark Scripted Model Inference <#benchmark-scripted-model-inference>`__
+  - `Convert PyTorch Scripted Model to OpenVINO Intermediate Representation <#convert-pytorch-scripted-model-to-openvino-intermediate-representation>`__
+  - `Benchmark OpenVINO Model Inference Converted From Scripted Model <#benchmark-openvino-model-inference-converted-from-scripted-model>`__
+  - `Traced Model <#traced-model>`__
+  - `Benchmark Traced Model Inference <#benchmark-traced-model-inference>`__
+  - `Convert PyTorch Traced Model to OpenVINO Intermediate Representation <#convert-pytorch-traced-model-to-openvino-intermediate-representation>`__
+  - `Benchmark OpenVINO Model Inference Converted From Traced Model <#benchmark-openvino-model-inference-converted-from-traced-model>`__
+
+Prerequisites `⇑ <#top>`__
+###############################################################################################################################
+
+
+Install notebook dependencies
 
 .. code:: ipython3
 
@@ -64,8 +103,9 @@ Download input data and label map
     
     imagenet_classes = labels_file.open("r").read().splitlines()
 
-Load PyTorch Model
-------------------
+Load PyTorch Model `⇑ <#top>`__
+###############################################################################################################################
+
 
 Generally, PyTorch models represent an instance of the
 ``torch.nn.Module`` class, initialized by a state dictionary with model
@@ -73,7 +113,8 @@ weights. Typical steps for getting a pre-trained model:
 
 1. Create an instance of a model class
 2. Load checkpoint state dict, which contains pre-trained model weights
-3. Turn the model to evaluation for switching some operations to inference mode
+3. Turn the model to evaluation for switching some operations to
+   inference mode
 
 The ``torchvision`` module provides a ready-to-use set of functions for
 model class initialization. We will use
@@ -94,8 +135,9 @@ enum ``RegNet_Y_800MF_Weights.DEFAULT``.
     # switch model to inference mode 
     model.eval();
 
-Prepare Input Data
-~~~~~~~~~~~~~~~~~~
+Prepare Input Data `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 The code below demonstrates how to preprocess input data using a
 model-specific transforms module from ``torchvision``. After
@@ -116,8 +158,9 @@ the first dimension.
     # Add batch dimension to image tensor
     input_tensor = img_transformed.unsqueeze(0)
 
-Run PyTorch Model Inference
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Run PyTorch Model Inference `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 The model returns a vector of probabilities in raw logits format,
 softmax can be applied to get normalized values in the [0, 1] range. For
@@ -172,8 +215,9 @@ can be reused later.
     5: hamper - 2.35%
 
 
-Benchmark PyTorch Model Inference
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Benchmark PyTorch Model Inference `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 .. code:: ipython3
 
@@ -185,39 +229,46 @@ Benchmark PyTorch Model Inference
 
 .. parsed-literal::
 
-    13.5 ms ± 3.76 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
+    13.2 ms ± 27.7 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
 
 
-Convert PyTorch Model to OpenVINO Intermediate Representation
--------------------------------------------------------------
+Convert PyTorch Model to OpenVINO Intermediate Representation. `⇑ <#top>`__
+###############################################################################################################################
 
 Starting from the 2023.0 release OpenVINO supports direct PyTorch models
 conversion to OpenVINO Intermediate Representation (IR) format. Model
 Optimizer Python API should be used for these purposes. More details
 regarding PyTorch model conversion can be found in OpenVINO
-`documentation <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_PyTorch.html>`__
+`documentation <https://docs.openvino.ai/2023.1/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_PyTorch.html>`__
 
-   **Note**: Please, take into account that direct support PyTorch
+.. note::
+
+   Please, take into account that direct support PyTorch
    models conversion is an experimental feature. Model coverage will be
    increased in the next releases. For cases, when PyTorch model
    conversion failed, you still can try to export the model to ONNX
-   format. Please refer to this
+   format. Please, refer to this
    `tutorial <102-pytorch-to-openvino-with-output.html>`__
    which explains how to convert PyTorch model to ONNX, then to OpenVINO
 
+
 The ``convert_model`` function accepts the PyTorch model object and
 returns the ``openvino.runtime.Model`` instance ready to load on a
 device using ``core.compile_model`` or save on disk for next usage using
 ``openvino.runtime.serialize``. Optionally, we can provide additional
 parameters, such as:
 
-* ``compress_to_fp16`` - flag to perform model weights compression into FP16 data format. It may reduce the required space for model storage on disk and give speedup for inference devices, where FP16 calculation is supported. 
-* ``example_input`` - input data sample which can be used for model tracing.
-* ``input_shape`` - the shape of input tensor for conversion
+-  ``compress_to_fp16`` - flag to perform model weights compression into
+   FP16 data format. It may reduce the required space for model storage
+   on disk and give speedup for inference devices, where FP16
+   calculation is supported.
+-  ``example_input`` - input data sample which can be used for model
+   tracing.
+-  ``input_shape`` - the shape of input tensor for conversion
 
-and any other advanced options supported by Model Optimizer Python API.
+and any other advanced options supported by model conversion Python API.
 More details can be found on this
-`page <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Python_API.html>`__
+`page <https://docs.openvino.ai/2023.1/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__
 
 .. code:: ipython3
 
@@ -250,10 +301,11 @@ More details can be found on this
 
 
 
-Select inference device
-~~~~~~~~~~~~~~~~~~~~~~~
+Select inference device `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
-select device from dropdown list for running inference using OpenVINO
+
+Select device from dropdown list for running inference using OpenVINO:
 
 .. code:: ipython3
 
@@ -298,8 +350,9 @@ select device from dropdown list for running inference using OpenVINO
 
 
 
-Run OpenVINO Model Inference
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Run OpenVINO Model Inference `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 .. code:: ipython3
 
@@ -329,8 +382,9 @@ Run OpenVINO Model Inference
     5: hamper - 2.35%
 
 
-Benchmark OpenVINO Model Inference
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Benchmark OpenVINO Model Inference `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 .. code:: ipython3
 
@@ -341,11 +395,12 @@ Benchmark OpenVINO Model Inference
 
 .. parsed-literal::
 
-    3.03 ms ± 5.57 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
+    3.03 ms ± 45.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
 
 
-Convert PyTorch Model with Static Input Shape
----------------------------------------------
+Convert PyTorch Model with Static Input Shape `⇑ <#top>`__
+###############################################################################################################################
+
 
 The default conversion path preserves dynamic input shapes, in order if
 you want to convert the model with static shapes, you can explicitly
@@ -377,10 +432,11 @@ reshaping example please check the following
 
 
 
-Select inference device
-~~~~~~~~~~~~~~~~~~~~~~~
+Select inference device `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
-select device from dropdown list for running inference using OpenVINO
+
+Select device from dropdown list for running inference using OpenVINO:
 
 .. code:: ipython3
 
@@ -420,8 +476,9 @@ Now, we can see that input of our converted model is tensor of shape [1,
 3, 224, 224] instead of [?, 3, ?, ?] reported by previously converted
 model.
 
-Run OpenVINO Model Inference with Static Input Shape
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Run OpenVINO Model Inference with Static Input Shape `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 .. code:: ipython3
 
@@ -451,8 +508,8 @@ Run OpenVINO Model Inference with Static Input Shape
     5: hamper - 2.35%
 
 
-Benchmark OpenVINO Model Inference with Static Input Shape
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Benchmark OpenVINO Model Inference with Static Input Shape `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
 .. code:: ipython3
 
@@ -463,11 +520,11 @@ Benchmark OpenVINO Model Inference with Static Input Shape
 
 .. parsed-literal::
 
-    2.79 ms ± 26.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
+    2.77 ms ± 12.7 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
 
 
-Convert TorchScript Model to OpenVINO Intermediate Representation
------------------------------------------------------------------
+Convert TorchScript Model to OpenVINO Intermediate Representation. `⇑ <#top>`__
+###############################################################################################################################
 
 TorchScript is a way to create serializable and optimizable models from
 PyTorch code. Any TorchScript program can be saved from a Python process
@@ -487,8 +544,9 @@ There are 2 possible ways to convert the PyTorch model to TorchScript:
 
 Let’s consider both approaches and their conversion into OpenVINO IR.
 
-Scriped Model
-~~~~~~~~~~~~~
+Scripted Model `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 ``torch.jit.script`` inspects model source code and compiles it to
 ``ScriptModule``. After compilation model can be used for inference or
@@ -540,8 +598,9 @@ Reference <https://pytorch.org/docs/stable/jit_language_reference.html#language-
     5: hamper - 2.35%
 
 
-Benchmark Scripted Model Inference
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Benchmark Scripted Model Inference `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 .. code:: ipython3
 
@@ -552,14 +611,12 @@ Benchmark Scripted Model Inference
 
 .. parsed-literal::
 
-    12.7 ms ± 13.4 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)
+    12.6 ms ± 17.6 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)
 
 
-Convert PyTorch Scripted Model to OpenVINO Intermediate Representation
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The conversion step for the scripted model to OpenVINO IR is similar to
-the original PyTorch model.
+Convert PyTorch Scripted Model to OpenVINO Intermediate
+Representation `⇑ <#top>`__ The conversion step for the scripted model to
+OpenVINO IR is similar to the original PyTorch model.
 
 .. code:: ipython3
 
@@ -595,8 +652,9 @@ the original PyTorch model.
     5: hamper - 2.35%
 
 
-Benchmark OpenVINO Model Inference Converted From Scripted Model
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Benchmark OpenVINO Model Inference Converted From Scripted Model `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 .. code:: ipython3
 
@@ -607,11 +665,12 @@ Benchmark OpenVINO Model Inference Converted From Scripted Model
 
 .. parsed-literal::
 
-    3.1 ms ± 4.02 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
+    3.07 ms ± 5.58 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
 
 
-Traced Model
-~~~~~~~~~~~~
+Traced Model `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 Using ``torch.jit.trace``, you can turn an existing module or Python
 function into a TorchScript ``ScriptFunction`` or ``ScriptModule``. You
@@ -619,10 +678,10 @@ must provide example inputs, and model will be executed, recording the
 operations performed on all the tensors.
 
 -  The resulting recording of a standalone function produces
-   ScriptFunction.
+   ``ScriptFunction``.
 
--  The resulting recording of nn.Module.forward or nn.Module produces
-   ScriptModule.
+-  The resulting recording of ``nn.Module.forward`` or ``nn.Module``
+   produces ``ScriptModule``.
 
 In the same way like scripted model, traced model can be used for
 inference or saved on disk using ``torch.jit.save`` function and after
@@ -667,8 +726,9 @@ original PyTorch model code definitions.
     5: hamper - 2.35%
 
 
-Benchmark Traced Model Inference
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Benchmark Traced Model Inference `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 .. code:: ipython3
 
@@ -679,14 +739,12 @@ Benchmark Traced Model Inference
 
 .. parsed-literal::
 
-    12.7 ms ± 39.6 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)
+    12.7 ms ± 61.1 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)
 
 
 Convert PyTorch Traced Model to OpenVINO Intermediate Representation
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The conversion step for a traced model to OpenVINO IR is similar to the
-original PyTorch model.
+`⇑ <#top>`__ The conversion step for a traced model to OpenVINO IR is
+similar to the original PyTorch model.
 
 .. code:: ipython3
 
@@ -722,8 +780,8 @@ original PyTorch model.
     5: hamper - 2.35%
 
 
-Benchmark OpenVINO Model Inference Converted From Traced Model
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Benchmark OpenVINO Model Inference Converted From Traced Model `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
 .. code:: ipython3
 
@@ -734,5 +792,5 @@ Benchmark OpenVINO Model Inference Converted From Traced Model
 
 .. parsed-literal::
 
-    3.08 ms ± 9.07 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
+    3.05 ms ± 6.85 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
 

docs/notebooks/102-pytorch-to-openvino-with-output_files/index.html

@@ -1,20 +1,20 @@
 <html>
-<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/102-pytorch-to-openvino-with-output_files/</title></head>
+<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/102-pytorch-to-openvino-with-output_files/</title></head>
 <body bgcolor="white">
-<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/102-pytorch-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
-<a href="102-pytorch-to-openvino-with-output_11_0.jpg">102-pytorch-to-openvino-with-output_11_0.jpg</a>       12-Jul-2023 00:11               54874
-<a href="102-pytorch-to-openvino-with-output_11_0.png">102-pytorch-to-openvino-with-output_11_0.png</a>       12-Jul-2023 00:11              542516
-<a href="102-pytorch-to-openvino-with-output_20_0.jpg">102-pytorch-to-openvino-with-output_20_0.jpg</a>       12-Jul-2023 00:11               54874
-<a href="102-pytorch-to-openvino-with-output_20_0.png">102-pytorch-to-openvino-with-output_20_0.png</a>       12-Jul-2023 00:11              542516
-<a href="102-pytorch-to-openvino-with-output_31_0.jpg">102-pytorch-to-openvino-with-output_31_0.jpg</a>       12-Jul-2023 00:11               54874
-<a href="102-pytorch-to-openvino-with-output_31_0.png">102-pytorch-to-openvino-with-output_31_0.png</a>       12-Jul-2023 00:11              542516
-<a href="102-pytorch-to-openvino-with-output_35_0.jpg">102-pytorch-to-openvino-with-output_35_0.jpg</a>       12-Jul-2023 00:11               54874
-<a href="102-pytorch-to-openvino-with-output_35_0.png">102-pytorch-to-openvino-with-output_35_0.png</a>       12-Jul-2023 00:11              542516
-<a href="102-pytorch-to-openvino-with-output_39_0.jpg">102-pytorch-to-openvino-with-output_39_0.jpg</a>       12-Jul-2023 00:11               54874
-<a href="102-pytorch-to-openvino-with-output_39_0.png">102-pytorch-to-openvino-with-output_39_0.png</a>       12-Jul-2023 00:11              542516
-<a href="102-pytorch-to-openvino-with-output_43_0.jpg">102-pytorch-to-openvino-with-output_43_0.jpg</a>       12-Jul-2023 00:11               54874
-<a href="102-pytorch-to-openvino-with-output_43_0.png">102-pytorch-to-openvino-with-output_43_0.png</a>       12-Jul-2023 00:11              542516
-<a href="102-pytorch-to-openvino-with-output_47_0.jpg">102-pytorch-to-openvino-with-output_47_0.jpg</a>       12-Jul-2023 00:11               54874
-<a href="102-pytorch-to-openvino-with-output_47_0.png">102-pytorch-to-openvino-with-output_47_0.png</a>       12-Jul-2023 00:11              542516
+<h1>Index of /projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/102-pytorch-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
+<a href="102-pytorch-to-openvino-with-output_11_0.jpg">102-pytorch-to-openvino-with-output_11_0.jpg</a>       16-Aug-2023 01:31               54874
+<a href="102-pytorch-to-openvino-with-output_11_0.png">102-pytorch-to-openvino-with-output_11_0.png</a>       16-Aug-2023 01:31              542516
+<a href="102-pytorch-to-openvino-with-output_20_0.jpg">102-pytorch-to-openvino-with-output_20_0.jpg</a>       16-Aug-2023 01:31               54874
+<a href="102-pytorch-to-openvino-with-output_20_0.png">102-pytorch-to-openvino-with-output_20_0.png</a>       16-Aug-2023 01:31              542516
+<a href="102-pytorch-to-openvino-with-output_31_0.jpg">102-pytorch-to-openvino-with-output_31_0.jpg</a>       16-Aug-2023 01:31               54874
+<a href="102-pytorch-to-openvino-with-output_31_0.png">102-pytorch-to-openvino-with-output_31_0.png</a>       16-Aug-2023 01:31              542516
+<a href="102-pytorch-to-openvino-with-output_35_0.jpg">102-pytorch-to-openvino-with-output_35_0.jpg</a>       16-Aug-2023 01:31               54874
+<a href="102-pytorch-to-openvino-with-output_35_0.png">102-pytorch-to-openvino-with-output_35_0.png</a>       16-Aug-2023 01:31              542516
+<a href="102-pytorch-to-openvino-with-output_39_0.jpg">102-pytorch-to-openvino-with-output_39_0.jpg</a>       16-Aug-2023 01:31               54874
+<a href="102-pytorch-to-openvino-with-output_39_0.png">102-pytorch-to-openvino-with-output_39_0.png</a>       16-Aug-2023 01:31              542516
+<a href="102-pytorch-to-openvino-with-output_43_0.jpg">102-pytorch-to-openvino-with-output_43_0.jpg</a>       16-Aug-2023 01:31               54874
+<a href="102-pytorch-to-openvino-with-output_43_0.png">102-pytorch-to-openvino-with-output_43_0.png</a>       16-Aug-2023 01:31              542516
+<a href="102-pytorch-to-openvino-with-output_47_0.jpg">102-pytorch-to-openvino-with-output_47_0.jpg</a>       16-Aug-2023 01:31               54874
+<a href="102-pytorch-to-openvino-with-output_47_0.png">102-pytorch-to-openvino-with-output_47_0.png</a>       16-Aug-2023 01:31              542516
 </pre><hr></body>
 </html>

docs/notebooks/103-paddle-to-openvino-classification-with-output.rst

@@ -1,6 +1,8 @@
 Convert a PaddlePaddle Model to OpenVINO™ IR
 ============================================
 
+
+
 This notebook shows how to convert a MobileNetV3 model from
 `PaddleHub <https://github.com/PaddlePaddle/PaddleHub>`__, pre-trained
 on the `ImageNet <https://www.image-net.org>`__ dataset, to OpenVINO IR.
@@ -14,11 +16,32 @@ IR model.
 Source of the
 `model <https://www.paddlepaddle.org.cn/hubdetail?name=mobilenet_v3_large_imagenet_ssld&en_category=ImageClassification>`__.
 
-Preparation
------------
 
-Imports
-~~~~~~~
+
+.. _top:
+
+**Table of contents**:
+
+- `Preparation <#preparation>`__
+
+  - `Imports <#imports>`__
+  - `Settings <#settings>`__
+
+- `Show Inference on PaddlePaddle Model <#show-inference-on-paddlepaddle-model>`__
+- `Convert the Model to OpenVINO IR Format <#convert-the-model-to-openvino-ir-format>`__
+- `Select inference device <#select-inference-device>`__
+- `Show Inference on OpenVINO Model <#show-inference-on-openvino-model>`__
+- `Timing and Comparison <#timing-and-comparison>`__
+- `Select inference device <#select-inference-device>`__
+- `References <#references>`__
+
+Preparation `⇑ <#top>`__
+###############################################################################################################################
+
+
+Imports `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 .. code:: ipython3
 
@@ -61,18 +84,19 @@ Imports
 
 .. parsed-literal::
 
-    2023-07-11 22:26:05 INFO: Loading faiss with AVX2 support.
-    2023-07-11 22:26:05 INFO: Successfully loaded faiss with AVX2 support.
+    2023-08-15 22:28:07 INFO: Loading faiss with AVX2 support.
+    2023-08-15 22:28:07 INFO: Successfully loaded faiss with AVX2 support.
 
 
-Settings
-~~~~~~~~
+Settings `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 Set ``IMAGE_FILENAME`` to the filename of an image to use. Set
 ``MODEL_NAME`` to the PaddlePaddle model to download from PaddleHub.
 ``MODEL_NAME`` will also be the base name for the IR model. The notebook
 is tested with the
-`mobilenet_v3_large_x1_0 <https://github.com/PaddlePaddle/PaddleClas/blob/release/2.5/docs/en/models/Mobile_en.md>`__
+`MobileNetV3_large_x1_0 <https://github.com/PaddlePaddle/PaddleClas/blob/release/2.5/docs/en/models/Mobile_en.md>`__
 model. Other models may use different preprocessing methods and
 therefore require some modification to get the same results on the
 original and converted model.
@@ -110,8 +134,9 @@ PaddleHub. This may take a while.
     Model Extracted to "./model".
 
 
-Show Inference on PaddlePaddle Model
-------------------------------------
+Show Inference on PaddlePaddle Model `⇑ <#top>`__
+###############################################################################################################################
+
 
 In the next cell, we load the model, load and display an image, do
 inference on that image, and then show the top three prediction results.
@@ -130,7 +155,7 @@ inference on that image, and then show the top three prediction results.
 
 .. parsed-literal::
 
-    [2023/07/11 22:26:25] ppcls WARNING: The current running environment does not support the use of GPU. CPU has been used instead.
+    [2023/08/15 22:28:34] ppcls WARNING: The current running environment does not support the use of GPU. CPU has been used instead.
     Labrador retriever, 0.75138
     German short-haired pointer, 0.02373
     Great Dane, 0.01848
@@ -182,8 +207,8 @@ the same method.
 
 It is useful to show the output of the ``process_image()`` function, to
 see the effect of cropping and resizing. Because of the normalization,
-the colors will look strange, and matplotlib will warn about clipping
-values.
+the colors will look strange, and ``matplotlib`` will warn about
+clipping values.
 
 .. code:: ipython3
 
@@ -196,7 +221,7 @@ values.
 
 .. parsed-literal::
 
-    2023-07-11 22:26:25 WARNING: Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
+    2023-08-15 22:28:34 WARNING: Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
 
 
 .. parsed-literal::
@@ -208,7 +233,7 @@ values.
 
 .. parsed-literal::
 
-    <matplotlib.image.AxesImage at 0x7fd87c6ee3a0>
+    <matplotlib.image.AxesImage at 0x7f28506c8c70>
 
 
 
@@ -232,8 +257,9 @@ OpenVINO model.
             partition = line.split("\n")[0].partition(" ")
             class_id_map[int(partition[0])] = str(partition[-1])
 
-Convert the Model to OpenVINO IR Format
----------------------------------------
+Convert the Model to OpenVINO IR Format `⇑ <#top>`__
+###############################################################################################################################
+
 
 Call the OpenVINO Model Optimizer Python API to convert the PaddlePaddle
 model to OpenVINO IR, with FP32 precision. ``mo.convert_model`` function
@@ -241,7 +267,7 @@ accept path to PaddlePaddle model and returns OpenVINO Model class
 instance which represents this model. Obtained model is ready to use and
 loading on device using ``compile_model`` or can be saved on disk using
 ``serialize`` function. See the `Model Optimizer Developer
-Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__
+Guide <https://docs.openvino.ai/2023.1/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__
 for more information about Model Optimizer.
 
 .. code:: ipython3
@@ -256,10 +282,11 @@ for more information about Model Optimizer.
     else:
         print(f"{model_xml} already exists.")
 
-Select inference device
------------------------
+Select inference device `⇑ <#top>`__
+###############################################################################################################################
 
-select device from dropdown list for running inference using OpenVINO
+
+Select device from dropdown list for running inference using OpenVINO:
 
 .. code:: ipython3
 
@@ -284,8 +311,9 @@ select device from dropdown list for running inference using OpenVINO
 
 
 
-Show Inference on OpenVINO Model
---------------------------------
+Show Inference on OpenVINO Model `⇑ <#top>`__
+###############################################################################################################################
+
 
 Load the IR model, get model information, load the image, do inference,
 convert the inference to a meaningful result, and show the output. See
@@ -332,14 +360,15 @@ information.
 .. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_23_1.png
 
 
-Timing and Comparison
----------------------
+Timing and Comparison `⇑ <#top>`__
+###############################################################################################################################
+
 
 Measure the time it takes to do inference on fifty images and compare
 the result. The timing information gives an indication of performance.
 For a fair comparison, we include the time it takes to process the
 image. For more accurate benchmarking, use the `OpenVINO benchmark
-tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_tool_README.html>`__.
+tool <https://docs.openvino.ai/2023.1/openvino_inference_engine_tools_benchmark_tool_README.html>`__.
 Note that many optimizations are possible to improve the performance.
 
 .. code:: ipython3
@@ -386,7 +415,7 @@ Note that many optimizations are possible to improve the performance.
 
 .. parsed-literal::
 
-    PaddlePaddle model on CPU: 0.0074 seconds per image, FPS: 135.48
+    PaddlePaddle model on CPU: 0.0071 seconds per image, FPS: 141.47
     
     PaddlePaddle result:
     Labrador retriever, 0.75138
@@ -400,10 +429,11 @@ Note that many optimizations are possible to improve the performance.
 .. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_27_1.png
 
 
-Select inference device
------------------------
+Select inference device `⇑ <#top>`__
+###############################################################################################################################
 
-select device from dropdown list for running inference using OpenVINO
+
+Select device from dropdown list for running inference using OpenVINO:
 
 .. code:: ipython3
 
@@ -448,7 +478,7 @@ select device from dropdown list for running inference using OpenVINO
 
 .. parsed-literal::
 
-    OpenVINO IR model in OpenVINO Runtime (AUTO): 0.0031 seconds per image, FPS: 326.50
+    OpenVINO IR model in OpenVINO Runtime (AUTO): 0.0030 seconds per image, FPS: 337.97
     
     OpenVINO result:
     Labrador retriever, 0.75138
@@ -462,11 +492,12 @@ select device from dropdown list for running inference using OpenVINO
 .. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_30_1.png
 
 
-References
-----------
+References `⇑ <#top>`__
+###############################################################################################################################
+
 
 -  `PaddleClas <https://github.com/PaddlePaddle/PaddleClas>`__
 -  `OpenVINO PaddlePaddle
-   support <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Paddle.html>`__
+   support <https://docs.openvino.ai/2023.1/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Paddle.html>`__
 -  `OpenVINO Model Optimizer
-   Documentation <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html>`__
+   Documentation <https://docs.openvino.ai/2023.1/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__

docs/notebooks/103-paddle-to-openvino-classification-with-output_files/index.html

@@ -1,11 +1,11 @@
 <html>
-<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/103-paddle-to-openvino-classification-with-output_files/</title></head>
+<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/103-paddle-to-openvino-classification-with-output_files/</title></head>
 <body bgcolor="white">
-<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/103-paddle-to-openvino-classification-with-output_files/</h1><hr><pre><a href="../">../</a>
-<a href="103-paddle-to-openvino-classification-with-output_15_3.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 12-Jul-2023 00:11              120883
-<a href="103-paddle-to-openvino-classification-with-output_23_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 12-Jul-2023 00:11              224886
-<a href="103-paddle-to-openvino-classification-with-output_27_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 12-Jul-2023 00:11              224886
-<a href="103-paddle-to-openvino-classification-with-output_30_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 12-Jul-2023 00:11              224886
-<a href="103-paddle-to-openvino-classification-with-output_8_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 12-Jul-2023 00:11              224886
+<h1>Index of /projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/103-paddle-to-openvino-classification-with-output_files/</h1><hr><pre><a href="../">../</a>
+<a href="103-paddle-to-openvino-classification-with-output_15_3.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 16-Aug-2023 01:31              120883
+<a href="103-paddle-to-openvino-classification-with-output_23_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 16-Aug-2023 01:31              224886
+<a href="103-paddle-to-openvino-classification-with-output_27_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 16-Aug-2023 01:31              224886
+<a href="103-paddle-to-openvino-classification-with-output_30_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 16-Aug-2023 01:31              224886
+<a href="103-paddle-to-openvino-classification-with-output_8_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 16-Aug-2023 01:31              224886
 </pre><hr></body>
 </html>

docs/notebooks/104-model-tools-with-output.rst

@@ -1,37 +1,60 @@
 Working with Open Model Zoo Models
 ==================================
 
+
+
 This tutorial shows how to download a model from `Open Model
 Zoo <https://github.com/openvinotoolkit/open_model_zoo>`__, convert it
 to OpenVINO™ IR format, show information about the model, and benchmark
-the model.
+the model. 
+
+.. _top:
+
+**Table of contents**:
+
+- `OpenVINO and Open Model Zoo Tools <#openvino-and-open-model-zoo-tools>`__
+- `Preparation <#preparation>`__
+
+  - `Model Name <#model-name>`__
+  - `Imports <#imports>`__
+  - `Settings and Configuration <#settings-and-configuration>`__
+
+- `Download a Model from Open Model Zoo <#download-a-model-from-open-model-zoo>`__
+- `Convert a Model to OpenVINO IR format <#convert-a-model-to-openvino-ir-format>`__
+- `Get Model Information <#get-model-information>`__
+- `Run Benchmark Tool <#run-benchmark-tool>`__
+
+  - `Benchmark with Different Settings <#benchmark-with-different-settings>`__
+
+OpenVINO and Open Model Zoo Tools `⇑ <#top>`__
+###############################################################################################################################
 
-OpenVINO and Open Model Zoo Tools
----------------------------------
 
 OpenVINO and Open Model Zoo tools are listed in the table below.
 
 +------------+--------------+-----------------------------------------+
 | Tool       | Command      | Description                             |
 +============+==============+=========================================+
-| Model      | omz_download | Download models from Open Model Zoo.    |
-| Downloader | er           |                                         |
+| Model      | ``omz_downlo | Download models from Open Model Zoo.    |
+| Downloader | ader``       |                                         |
 +------------+--------------+-----------------------------------------+
-| Model      | omz_converte | Convert Open Model Zoo models to        |
-| Converter  | r            | OpenVINO’s IR format.                   |
+| Model      | ``omz_conver | Convert Open Model Zoo models to        |
+| Converter  | ter``        | OpenVINO’s IR format.                   |
 +------------+--------------+-----------------------------------------+
-| Info       | omz_info_dum | Print information about Open Model Zoo  |
-| Dumper     | per          | models.                                 |
+| Info       | ``omz_info_d | Print information about Open Model Zoo  |
+| Dumper     | umper``      | models.                                 |
 +------------+--------------+-----------------------------------------+
-| Benchmark  | benchmark_ap | Benchmark model performance by          |
-| Tool       | p            | computing inference time.               |
+| Benchmark  | ``benchmark_ | Benchmark model performance by          |
+| Tool       | app``        | computing inference time.               |
 +------------+--------------+-----------------------------------------+
 
-Preparation
------------
+Preparation `⇑ <#top>`__
+###############################################################################################################################
+
+
+Model Name `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
-Model Name
-~~~~~~~~~~
 
 Set ``model_name`` to the name of the Open Model Zoo model to use in
 this notebook. Refer to the list of
@@ -46,8 +69,9 @@ pre-trained models for a full list of models that can be used. Set
     # model_name = "resnet-50-pytorch"
     model_name = "mobilenet-v2-pytorch"
 
-Imports
-~~~~~~~
+Imports `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 .. code:: ipython3
 
@@ -61,8 +85,9 @@ Imports
     sys.path.append("../utils")
     from notebook_utils import DeviceNotFoundAlert, NotebookAlert
 
-Settings and Configuration
-~~~~~~~~~~~~~~~~~~~~~~~~~~
+Settings and Configuration `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 Set the file and directory paths. By default, this notebook downloads
 models from Open Model Zoo to the ``open_model_zoo_models`` directory in
@@ -100,8 +125,9 @@ The following settings can be changed:
     base_model_dir: model, omz_cache_dir: cache, gpu_availble: False
 
 
-Download a Model from Open Model Zoo
-------------------------------------
+Download a Model from Open Model Zoo `⇑ <#top>`__
+###############################################################################################################################
+
 
 Specify, display and run the Model Downloader command to download the
 model.
@@ -140,8 +166,9 @@ Downloading mobilenet-v2-pytorch…
     
 
 
-Convert a Model to OpenVINO IR format
--------------------------------------
+Convert a Model to OpenVINO IR format `⇑ <#top>`__
+###############################################################################################################################
+
 
 Specify, display and run the Model Converter command to convert the
 model to OpenVINO IR format. Model conversion may take a while. The
@@ -177,25 +204,26 @@ Converting mobilenet-v2-pytorch…
 .. parsed-literal::
 
     ========== Converting mobilenet-v2-pytorch to ONNX
-    Conversion to ONNX command: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/internal_scripts/pytorch_to_onnx.py --model-name=mobilenet_v2 --weights=model/public/mobilenet-v2-pytorch/mobilenet_v2-b0353104.pth --import-module=torchvision.models --input-shape=1,3,224,224 --output-file=model/public/mobilenet-v2-pytorch/mobilenet-v2.onnx --input-names=data --output-names=prob
+    Conversion to ONNX command: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/internal_scripts/pytorch_to_onnx.py --model-name=mobilenet_v2 --weights=model/public/mobilenet-v2-pytorch/mobilenet_v2-b0353104.pth --import-module=torchvision.models --input-shape=1,3,224,224 --output-file=model/public/mobilenet-v2-pytorch/mobilenet-v2.onnx --input-names=data --output-names=prob
     
     ONNX check passed successfully.
     
     ========== Converting mobilenet-v2-pytorch to IR (FP16)
-    Conversion command: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/bin/mo --framework=onnx --output_dir=/tmp/tmpe5yh3lmf --model_name=mobilenet-v2-pytorch --input=data '--mean_values=data[123.675,116.28,103.53]' '--scale_values=data[58.624,57.12,57.375]' --reverse_input_channels --output=prob --input_model=model/public/mobilenet-v2-pytorch/mobilenet-v2.onnx '--layout=data(NCHW)' '--input_shape=[1, 3, 224, 224]' --compress_to_fp16=True
+    Conversion command: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/bin/mo --framework=onnx --output_dir=/tmp/tmp3q4nxrwu --model_name=mobilenet-v2-pytorch --input=data '--mean_values=data[123.675,116.28,103.53]' '--scale_values=data[58.624,57.12,57.375]' --reverse_input_channels --output=prob --input_model=model/public/mobilenet-v2-pytorch/mobilenet-v2.onnx '--layout=data(NCHW)' '--input_shape=[1, 3, 224, 224]' --compress_to_fp16=True
     
     [ INFO ] Generated IR will be compressed to FP16. If you get lower accuracy, please consider disabling compression by removing argument --compress_to_fp16 or set it to false --compress_to_fp16=False.
-    Find more information about compression to FP16 at https://docs.openvino.ai/latest/openvino_docs_MO_DG_FP16_Compression.html
+    Find more information about compression to FP16 at https://docs.openvino.ai/2023.1/openvino_docs_MO_DG_FP16_Compression.html
     [ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11.
-    Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/latest/openvino_2_0_transition_guide.html
+    Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/2023.1/openvino_2_0_transition_guide.html
     [ SUCCESS ] Generated IR version 11 model.
-    [ SUCCESS ] XML file: /tmp/tmpe5yh3lmf/mobilenet-v2-pytorch.xml
-    [ SUCCESS ] BIN file: /tmp/tmpe5yh3lmf/mobilenet-v2-pytorch.bin
+    [ SUCCESS ] XML file: /tmp/tmp3q4nxrwu/mobilenet-v2-pytorch.xml
+    [ SUCCESS ] BIN file: /tmp/tmp3q4nxrwu/mobilenet-v2-pytorch.bin
     
 
 
-Get Model Information
----------------------
+Get Model Information `⇑ <#top>`__
+###############################################################################################################################
+
 
 The Info Dumper prints the following information for Open Model Zoo
 models:
@@ -240,8 +268,8 @@ information in a dictionary.
       'description': 'MobileNet V2 is image classification model pre-trained on ImageNet dataset. This is a PyTorch* implementation of MobileNetV2 architecture as described in the paper "Inverted Residuals and Linear Bottlenecks: Mobile Networks for Classification, Detection and Segmentation" <https://arxiv.org/abs/1801.04381>.\nThe model input is a blob that consists of a single image of "1, 3, 224, 224" in "RGB" order.\nThe model output is typical object classifier for the 1000 different classifications matching with those in the ImageNet database.',
       'framework': 'pytorch',
       'license_url': 'https://raw.githubusercontent.com/pytorch/vision/master/LICENSE',
-      'accuracy_config': '/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/models/public/mobilenet-v2-pytorch/accuracy-check.yml',
-      'model_config': '/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/models/public/mobilenet-v2-pytorch/model.yml',
+      'accuracy_config': '/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/models/public/mobilenet-v2-pytorch/accuracy-check.yml',
+      'model_config': '/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/models/public/mobilenet-v2-pytorch/model.yml',
       'precisions': ['FP16', 'FP32'],
       'quantization_output_precisions': ['FP16-INT8', 'FP32-INT8'],
       'subdirectory': 'public/mobilenet-v2-pytorch',
@@ -273,8 +301,9 @@ file.
     model/public/mobilenet-v2-pytorch/FP16/mobilenet-v2-pytorch.xml exists: True
 
 
-Run Benchmark Tool
-------------------
+Run Benchmark Tool `⇑ <#top>`__
+###############################################################################################################################
+
 
 By default, Benchmark Tool runs inference for 60 seconds in asynchronous
 mode on CPU. It returns inference speed as latency (milliseconds per
@@ -321,7 +350,7 @@ seconds…
     [ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
     [Step 4/11] Reading model files
     [ INFO ] Loading model files
-    [ INFO ] Read model took 30.53 ms
+    [ INFO ] Read model took 29.61 ms
     [ INFO ] Original model I/O parameters:
     [ INFO ] Model inputs:
     [ INFO ]     data (node: data) : f32 / [N,C,H,W] / [1,3,224,224]
@@ -335,7 +364,7 @@ seconds…
     [ INFO ] Model outputs:
     [ INFO ]     prob (node: prob) : f32 / [...] / [1,1000]
     [Step 7/11] Loading the model to the device
-    [ INFO ] Compile model took 149.89 ms
+    [ INFO ] Compile model took 154.76 ms
     [Step 8/11] Querying optimal runtime parameters
     [ INFO ] Model:
     [ INFO ]   NETWORK_NAME: torch_jit
@@ -357,21 +386,22 @@ seconds…
     [ INFO ] Fill input 'data' with random values 
     [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
     [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
-    [ INFO ] First inference took 6.66 ms
+    [ INFO ] First inference took 7.60 ms
     [Step 11/11] Dumping statistics report
     [ INFO ] Execution Devices:['CPU']
-    [ INFO ] Count:            20136 iterations
-    [ INFO ] Duration:         15007.99 ms
+    [ INFO ] Count:            20076 iterations
+    [ INFO ] Duration:         15004.20 ms
     [ INFO ] Latency:
-    [ INFO ]    Median:        4.33 ms
-    [ INFO ]    Average:       4.34 ms
-    [ INFO ]    Min:           3.20 ms
-    [ INFO ]    Max:           11.84 ms
-    [ INFO ] Throughput:   1341.69 FPS
+    [ INFO ]    Median:        4.34 ms
+    [ INFO ]    Average:       4.35 ms
+    [ INFO ]    Min:           2.53 ms
+    [ INFO ]    Max:           11.71 ms
+    [ INFO ] Throughput:   1338.03 FPS
 
 
-Benchmark with Different Settings
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Benchmark with Different Settings `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 The ``benchmark_app`` tool displays logging information that is not
 always necessary. A more compact result is achieved when the output is
@@ -395,7 +425,9 @@ In the next cell, define the ``benchmark_model()`` function that calls
 ``benchmark_app``. This makes it easy to try different combinations. In
 the cell below that, you display available devices on the system.
 
-   **Note**: In this notebook, ``benchmark_app`` runs for 15 seconds to
+.. note::
+
+   In this notebook, ``benchmark_app`` runs for 15 seconds to
    give a quick indication of performance. For more accurate
    performance, it is recommended to run inference for at least one
    minute by setting the ``t`` parameter to 60 or higher, and run
@@ -404,6 +436,7 @@ the cell below that, you display available devices on the system.
    command prompt where you have activated the ``openvino_env``
    environment.
 
+
 .. code:: ipython3
 
     def benchmark_model(model_xml, device="CPU", seconds=60, api="async", batch=1):
@@ -456,9 +489,9 @@ Benchmark command:
 
     command ended
     Traceback (most recent call last):
-      File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 327, in main
+      File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 327, in main
         benchmark.set_allow_auto_batching(False)
-      File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 63, in set_allow_auto_batching
+      File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 63, in set_allow_auto_batching
         self.core.set_property({'ALLOW_AUTO_BATCHING': flag})
     RuntimeError: Check 'false' failed at src/inference/src/core.cpp:238:
 
@@ -484,9 +517,9 @@ Benchmark command:
     Check 'false' failed at src/plugins/auto/src/plugin_config.cpp:55:
     property: ALLOW_AUTO_BATCHING: not supported
     Traceback (most recent call last):
-      File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 327, in main
+      File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 327, in main
         benchmark.set_allow_auto_batching(False)
-      File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 63, in set_allow_auto_batching
+      File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 63, in set_allow_auto_batching
         self.core.set_property({'ALLOW_AUTO_BATCHING': flag})
     RuntimeError: Check 'false' failed at src/inference/src/core.cpp:238:
     Check 'false' failed at src/plugins/auto/src/plugin_config.cpp:55:
@@ -495,9 +528,7 @@ Benchmark command:
 
 .. code:: ipython3
 
-    benchmark_model(model_path, device="GPU", seconds=15, api="async")
-
-
+   benchmark_model(model_path, device="GPU", seconds=15, api="async")
 
 .. raw:: html
 
@@ -506,8 +537,7 @@ Benchmark command:
 
 .. code:: ipython3
 
-    benchmark_model(model_path, device="MULTI:CPU,GPU", seconds=15, api="async")
-
+   benchmark_model(model_path, device="MULTI:CPU,GPU", seconds=15, api="async")
 
 
 .. raw:: html

docs/notebooks/105-language-quantize-bert-with-output.rst

@@ -1,6 +1,8 @@
 Quantize NLP models with Post-Training Quantization ​in NNCF
 ============================================================
 
+
+
 This tutorial demonstrates how to apply ``INT8`` quantization to the
 Natural Language Processing model known as
 `BERT <https://en.wikipedia.org/wiki/BERT_(language_model)>`__, using
@@ -22,12 +24,31 @@ and datasets. It consists of the following steps:
 -  Compare the performance of the original, converted and quantized
    models.
 
+
+
+.. _top:
+
+**Table of contents**:
+
+- `Imports <#imports>`__
+- `Settings <#settings>`__
+- `Prepare the Model <#prepare-the-model>`__
+- `Prepare the Dataset <#prepare-the-dataset>`__
+- `Optimize model using NNCF Post-training Quantization API <#optimize-model-using-nncf-post-training-quantization-api>`__
+- `Load and Test OpenVINO Model <#load-and-test-openvino-model>`__
+
+  - `Select inference device <#select-inference-device>`__
+
+- `Compare F1-score of FP32 and INT8 models <#compare-f1-score-of-fp32-and-int8-models>`__
+- `Compare Performance of the Original, Converted and Quantized Models <#compare-performance-of-the-original,-converted-and-quantized-models>`__
+
 .. code:: ipython3
 
     !pip install -q "nncf>=2.5.0" datasets evaluate
 
-Imports
--------
+Imports `⇑ <#top>`__
+###############################################################################################################################
+
 
 .. code:: ipython3
 
@@ -56,10 +77,10 @@ Imports
 
 .. parsed-literal::
 
-    2023-07-11 22:27:10.887837: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
-    2023-07-11 22:27:10.921844: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
+    2023-08-15 22:29:19.942802: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
+    2023-08-15 22:29:19.975605: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
     To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
-    2023-07-11 22:27:11.494944: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
+    2023-08-15 22:29:20.517786: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
 
 
 .. parsed-literal::
@@ -67,8 +88,9 @@ Imports
     INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino
 
 
-Settings
---------
+Settings `⇑ <#top>`__
+###############################################################################################################################
+
 
 .. code:: ipython3
 
@@ -82,13 +104,15 @@ Settings
     os.makedirs(DATA_DIR, exist_ok=True)
     os.makedirs(MODEL_DIR, exist_ok=True)
 
-Prepare the Model
------------------
+Prepare the Model `⇑ <#top>`__
+###############################################################################################################################
+
 
 Perform the following:
 
-- Download and unpack pre-trained BERT model for MRPC by PyTorch.
-- Convert the model to the OpenVINO Intermediate Representation (OpenVINO IR)
+-  Download and unpack pre-trained BERT model for MRPC by PyTorch.
+-  Convert the model to the OpenVINO Intermediate Representation
+   (OpenVINO IR)
 
 .. code:: ipython3
 
@@ -107,12 +131,12 @@ Convert the original PyTorch model to the OpenVINO Intermediate
 Representation.
 
 From OpenVINO 2023.0, we can directly convert a model from the PyTorch
-format to the OpenVINO IR format using Model Optimizer. Following
+format to the OpenVINO IR format using model conversion API. Following
 PyTorch model formats are supported:
 
--  torch.nn.Module
--  torch.jit.ScriptModule
--  torch.jit.ScriptFunction
+-  ``torch.nn.Module``
+-  ``torch.jit.ScriptModule``
+-  ``torch.jit.ScriptFunction``
 
 .. code:: ipython3
 
@@ -142,17 +166,16 @@ PyTorch model formats are supported:
 
 .. parsed-literal::
 
-    /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/annotations.py:309: UserWarning: TorchScript will treat type annotations of Tensor dtype-specific subtypes as if they are normal Tensors. dtype constraints are not enforced in compilation either.
+    /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/annotations.py:309: UserWarning: TorchScript will treat type annotations of Tensor dtype-specific subtypes as if they are normal Tensors. dtype constraints are not enforced in compilation either.
       warnings.warn("TorchScript will treat type annotations of Tensor "
 
 
-Prepare the Dataset
--------------------
+Prepare the Dataset `⇑ <#top>`__
+###############################################################################################################################
 
-We download the `General Language Understanding Evaluation
-(GLUE) <https://gluebenchmark.com/>`__ dataset for the MRPC task from
-HuggingFace datasets. Then, we tokenize the data with a pre-trained BERT
-tokenizer from HuggingFace.
+We download the `General Language Understanding Evaluation (GLUE) <https://gluebenchmark.com/>`__ dataset
+for the MRPC task from HuggingFace datasets. Then, we tokenize the data
+with a pre-trained BERT tokenizer from HuggingFace.
 
 .. code:: ipython3
 
@@ -171,15 +194,9 @@ tokenizer from HuggingFace.
     
     data_source = create_data_source()
 
+Optimize model using NNCF Post-training Quantization API `⇑ <#top>`__
+###############################################################################################################################
 
-.. parsed-literal::
-
-    Found cached dataset glue (/opt/home/k8sworker/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)
-    Loading cached processed dataset at /opt/home/k8sworker/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-b5f4c739eb2a4a9f.arrow
-
-
-Optimize model using NNCF Post-training Quantization API
---------------------------------------------------------
 
 `NNCF <https://github.com/openvinotoolkit/nncf>`__ provides a suite of
 advanced algorithms for Neural Networks inference optimization in
@@ -387,8 +404,8 @@ The optimization process contains the following steps:
 
 .. parsed-literal::
 
-    Statistics collection: 100%|██████████| 300/300 [00:24<00:00, 12.02it/s]
-    Biases correction: 100%|██████████| 74/74 [00:25<00:00,  2.94it/s]
+    Statistics collection: 100%|██████████| 300/300 [00:24<00:00, 12.04it/s]
+    Biases correction: 100%|██████████| 74/74 [00:25<00:00,  2.95it/s]
 
 
 .. code:: ipython3
@@ -396,20 +413,22 @@ The optimization process contains the following steps:
     compressed_model_xml = Path(MODEL_DIR) / "quantized_bert_mrpc.xml"
     ov.serialize(quantized_model, compressed_model_xml)
 
-Load and Test OpenVINO Model
-----------------------------
+Load and Test OpenVINO Model `⇑ <#top>`__
+###############################################################################################################################
+
 
 To load and test converted model, perform the following:
 
-* Load the model and compile it for selected device.
-* Prepare the input.
-* Run the inference.
-* Get the answer from the model output.
+-  Load the model and compile it for selected device.
+-  Prepare the input.
+-  Run the inference.
+-  Get the answer from the model output.
 
-Select inference device
-~~~~~~~~~~~~~~~~~~~~~~~
+Select inference device `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
-select device from dropdown list for running inference using OpenVINO
+
+Select device from dropdown list for running inference using OpenVINO:
 
 .. code:: ipython3
 
@@ -465,8 +484,9 @@ changing ``sample_idx`` to another value (from 0 to 407).
     The same meaning: yes
 
 
-Compare F1-score of FP32 and INT8 models
-----------------------------------------
+Compare F1-score of FP32 and INT8 models `⇑ <#top>`__
+###############################################################################################################################
+
 
 .. code:: ipython3
 
@@ -509,8 +529,8 @@ Compare F1-score of FP32 and INT8 models
     F1 score: 0.8995
 
 
-Compare Performance of the Original, Converted and Quantized Models
--------------------------------------------------------------------
+Compare Performance of the Original, Converted and Quantized Models. `⇑ <#top>`__
+###############################################################################################################################
 
 Compare the original PyTorch model with OpenVINO converted and quantized
 models (``FP32``, ``INT8``) to see the difference in performance. It is
@@ -562,17 +582,24 @@ Frames Per Second (FPS) for images.
 
 .. parsed-literal::
 
-    PyTorch model on CPU: 0.071 seconds per sentence, SPS: 14.09
-    IR FP32 model in OpenVINO Runtime/AUTO: 0.022 seconds per sentence, SPS: 45.98
-    OpenVINO IR INT8 model in OpenVINO Runtime/AUTO: 0.010 seconds per sentence, SPS: 98.77
+    We strongly recommend passing in an `attention_mask` since your input_ids may be padded. See https://huggingface.co/docs/transformers/troubleshooting#incorrect-output-when-padding-tokens-arent-masked.
+
+
+.. parsed-literal::
+
+    PyTorch model on CPU: 0.070 seconds per sentence, SPS: 14.22
+    IR FP32 model in OpenVINO Runtime/AUTO: 0.021 seconds per sentence, SPS: 48.42
+    OpenVINO IR INT8 model in OpenVINO Runtime/AUTO: 0.010 seconds per sentence, SPS: 98.01
 
 
 Finally, measure the inference performance of OpenVINO ``FP32`` and
-``INT8`` models. For this purpose, use `Benchmark
-Tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_tool_README.html>`__
+``INT8`` models. For this purpose, use 
+`Benchmark Tool <https://docs.openvino.ai/2023.1/openvino_inference_engine_tools_benchmark_tool_README.html>`__
 in OpenVINO.
 
-   **Note**: The ``benchmark_app`` tool is able to measure the
+.. note::
+
+   The ``benchmark_app`` tool is able to measure the
    performance of the OpenVINO Intermediate Representation (OpenVINO IR)
    models only. For more accurate performance, run ``benchmark_app`` in
    a terminal/command prompt after closing other applications. Run
@@ -581,6 +608,7 @@ in OpenVINO.
    Run ``benchmark_app --help`` to see an overview of all command-line
    options.
 
+
 .. code:: ipython3
 
     # Inference FP32 model (OpenVINO IR)
@@ -600,9 +628,9 @@ in OpenVINO.
     [ ERROR ] Check 'false' failed at src/inference/src/core.cpp:84:
     Device with "device" name is not registered in the OpenVINO Runtime
     Traceback (most recent call last):
-      File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 103, in main
+      File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 103, in main
         benchmark.print_version_info()
-      File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 48, in print_version_info
+      File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 48, in print_version_info
         for device, version in self.core.get_versions(self.device).items():
     RuntimeError: Check 'false' failed at src/inference/src/core.cpp:84:
     Device with "device" name is not registered in the OpenVINO Runtime
@@ -628,9 +656,9 @@ in OpenVINO.
     [ ERROR ] Check 'false' failed at src/inference/src/core.cpp:84:
     Device with "device" name is not registered in the OpenVINO Runtime
     Traceback (most recent call last):
-      File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 103, in main
+      File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 103, in main
         benchmark.print_version_info()
-      File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 48, in print_version_info
+      File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-475/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 48, in print_version_info
         for device, version in self.core.get_versions(self.device).items():
     RuntimeError: Check 'false' failed at src/inference/src/core.cpp:84:
     Device with "device" name is not registered in the OpenVINO Runtime

docs/notebooks/106-auto-device-with-output.rst

@@ -2,19 +2,19 @@ Automatic Device Selection with OpenVINO™
 =========================================
 
 The `Auto
-device <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__
+device <https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__
 (or AUTO in short) selects the most suitable device for inference by
 considering the model precision, power efficiency and processing
 capability of the available `compute
-devices <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_Supported_Devices.html>`__.
+devices <https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_supported_plugins_Supported_Devices.html>`__.
 The model precision (such as ``FP32``, ``FP16``, ``INT8``, etc.) is the
 first consideration to filter out the devices that cannot run the
 network efficiently.
 
 Next, if dedicated accelerators are available, these devices are
 preferred (for example, integrated and discrete
-`GPU <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_GPU.html#doxid-openvino-docs-o-v-u-g-supported-plugins-g-p-u>`__).
-`CPU <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_CPU.html>`__
+`GPU <https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_supported_plugins_GPU.html#doxid-openvino-docs-o-v-u-g-supported-plugins-g-p-u>`__).
+`CPU <https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_supported_plugins_CPU.html>`__
 is used as the default “fallback device”. Keep in mind that AUTO makes
 this selection only once, during the loading of a model.
 
@@ -25,10 +25,40 @@ immediately on the CPU and then transparently shifts inference to the
 GPU, once it is ready. This dramatically reduces the time to execute
 first inference.
 
-.. image:: https://camo.githubusercontent.com/cc526c3f5fc992cc7176d097894303248adbd04b4d158bd98e65edc8270af5fc/68747470733a2f2f757365722d696d616765732e67697468756275736572636f6e74656e742e636f6d2f31353730393732332f3136313435313834372d37353965326264622d373062632d343633642d393831382d3430306330636366336331362e706e67
+.. figure:: https://user-images.githubusercontent.com/15709723/161451847-759e2bdb-70bc-463d-9818-400c0ccf3c16.png
+   :alt: auto
+
+   auto
+
+
+
+.. _top:
+
+**Table of contents**:
+
+- `Import modules and create Core <#import-modules-and-create-core>`__
+- `Convert the model to OpenVINO IR format <#convert-the-model-to-openvino-ir-format>`__
+- `(1) Simplify selection logic <#simplify-selection-logic>`__
+
+  - `Default behavior of Core::compile_model API without device_name <#default-behavior-of-core::compile_model-api-without-device_name>`__
+  - `Explicitly pass AUTO as device_name to Core::compile_model API <#explicitly-pass-auto-as-device_name-to-core::compile_model-api>`__
+
+- `(2) Improve the first inference latency <#improve-the-first-inference-latency>`__
+
+  - `Load an Image <#load-an-image>`__
+  - `Load the model to GPU device and perform inference <#load-the-model-to-gpu-device-and-perform-inference>`__
+  - `Load the model using AUTO device and do inference <#load-the-model-using-auto-device-and-do-inference>`__
+
+- `(3) Achieve different performance for different targets <#achieve-different-performance-for-different-targets>`__
+
+  - `Class and callback definition <#class-and-callback-definition>`__
+  - `Inference with THROUGHPUT hint <#inference-with-throughput-hint>`__
+  - `Inference with LATENCY hint <#inference-with-latency-hint>`__
+  - `Difference in FPS and latency <#difference-in-fps-and-latency>`__
+
+Import modules and create Core `⇑ <#top>`__
+###############################################################################################################################
 
-Import modules and create Core
-------------------------------
 
 .. code:: ipython3
 
@@ -50,28 +80,27 @@ Import modules and create Core
    device to have meaningful results.
 
 
-Convert the model to OpenVINO IR format
----------------------------------------
+Convert the model to OpenVINO IR format `⇑ <#top>`__
+###############################################################################################################################
+
 
 This tutorial uses
 `resnet50 <https://pytorch.org/vision/main/models/generated/torchvision.models.resnet50.html#resnet50>`__
 model from
 `torchvision <https://pytorch.org/vision/main/index.html?highlight=torchvision#module-torchvision>`__
 library. ResNet 50 is image classification model pre-trained on ImageNet
-dataset described in paper `“Deep Residual Learning for Image
-Recognition” <https://arxiv.org/abs/1512.03385>`__. From OpenVINO
+dataset described in paper `“Deep Residual Learning for Image Recognition” <https://arxiv.org/abs/1512.03385>`__. From OpenVINO
 2023.0, we can directly convert a model from the PyTorch format to the
-OpenVINO IR format using Model Optimizer. To convert model, we should
-provide model object instance into ``mo.convert_model`` function,
+OpenVINO IR format using model conversion API. To convert model, we
+should provide model object instance into ``mo.convert_model`` function,
 optionally, we can specify input shape for conversion (by default models
 from PyTorch converted with dynamic input shapes). ``mo.convert_model``
-returns openvino.runtime.Model object ready to be loaded on device with
-``openvino.runtime.Core().compile_model`` or serialized for next usage
-with ``openvino.runtime.serialize``.
+returns openvino.runtime.Model object ready to be loaded on a device
+with ``openvino.runtime.Core().compile_model`` or serialized for next
+usage with ``openvino.runtime.serialize``.
 
-For more information about Model Optimizer, see the `Model Optimizer
-Developer
-Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
+For more information about model conversion API, see this
+`page <https://docs.openvino.ai/2023.1/openvino_docs_model_processing_introduction.html>`__.
 
 .. code:: ipython3
 
@@ -99,14 +128,15 @@ Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_O
     IR model saved to model/resnet50.xml
 
 
-(1) Simplify selection logic
-----------------------------
+(1) Simplify selection logic `⇑ <#top>`__
+###############################################################################################################################
+ 
 
-Default behavior of Core::compile_model API without device_name
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Default behavior of Core::compile_model API without device_name `⇑ <#top>`__ 
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
-By default, ``compile_model`` API will select **AUTO** as
-``device_name`` if no device is specified.
+By default, ``compile_model`` API will select **AUTO** as ``device_name`` if no
+device is specified.
 
 .. code:: ipython3
 
@@ -137,11 +167,11 @@ By default, ``compile_model`` API will select **AUTO** as
     Deleted compiled_model
 
 
-Explicitly pass AUTO as device_name to Core::compile_model API
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Explicitly pass AUTO as device_name to Core::compile_model API `⇑ <#top>`__ 
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
-It is optional, but passing AUTO explicitly as ``device_name`` may
-improve readability of your code.
+It is optional, but passing AUTO explicitly as
+``device_name`` may improve readability of your code.
 
 .. code:: ipython3
 
@@ -171,13 +201,13 @@ improve readability of your code.
     Deleted compiled_model
 
 
-(2) Improve the first inference latency
----------------------------------------
+(2) Improve the first inference latency `⇑ <#top>`__
+###############################################################################################################################
 
-One of the benefits of using AUTO device selection is reducing FIL
-(first inference latency). FIL is the model compilation time combined
-with the first inference execution time. Using the CPU device explicitly
-will produce the shortest first inference latency, as the OpenVINO graph
+One of the benefits of using AUTO device selection is reducing FIL (first inference
+latency). FIL is the model compilation time combined with the first
+inference execution time. Using the CPU device explicitly will produce
+the shortest first inference latency, as the OpenVINO graph
 representation loads quickly on CPU, using just-in-time (JIT)
 compilation. The challenge is with GPU devices since OpenCL graph
 complication to GPU-optimized kernels takes a few seconds to complete.
@@ -185,11 +215,12 @@ This initialization time may be intolerable for some applications. To
 avoid this delay, the AUTO uses CPU transparently as the first inference
 device until GPU is ready.
 
-Load an Image
-~~~~~~~~~~~~~
+Load an Image `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
-torchvision library provides model specific input transformation
-function, we will reuse it for preparing input data.
+Torchvision library provides model specific
+input transformation function, we will reuse it for preparing input
+data.
 
 .. code:: ipython3
 
@@ -209,8 +240,9 @@ function, we will reuse it for preparing input data.
 
 
 
-Load the model to GPU device and perform inference
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Load the model to GPU device and perform inference `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 .. code:: ipython3
 
@@ -236,8 +268,8 @@ Load the model to GPU device and perform inference
     A GPU device is not available. Available devices are: ['CPU']
 
 
-Load the model using AUTO device and do inference
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Load the model using AUTO device and do inference `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
 When GPU is the best available device, the first few inferences will be
 executed on CPU until GPU is ready.
@@ -268,8 +300,8 @@ executed on CPU until GPU is ready.
     # Deleted model will wait for compiling on the selected device to complete.
     del compiled_model
 
-(3) Achieve different performance for different targets
--------------------------------------------------------
+(3) Achieve different performance for different targets `⇑ <#top>`__
+###############################################################################################################################
 
 It is an advantage to define **performance hints** when using Automatic
 Device Selection. By specifying a **THROUGHPUT** or **LATENCY** hint,
@@ -280,14 +312,13 @@ hints do not require any device-specific settings and they are
 completely portable between devices – meaning AUTO can configure the
 performance hint on whichever device is being used.
 
-For more information, refer to the `Performance
-Hints <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_AUTO.html#performance-hints>`__
-section of `Automatic Device
-Selection <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__
+For more information, refer to the `Performance Hints <https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_supported_plugins_AUTO.html#performance-hints>`__
+section of `Automatic Device Selection <https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__
 article.
 
-Class and callback definition
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Class and callback definition `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 .. code:: ipython3
 
@@ -385,8 +416,9 @@ Class and callback definition
     metrics_update_interval = 10
     metrics_update_num = 6
 
-Inference with THROUGHPUT hint
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Inference with THROUGHPUT hint `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 Loop for inference and update the FPS/Latency every
 @metrics_update_interval seconds.
@@ -424,17 +456,18 @@ Loop for inference and update the FPS/Latency every
 
     Compiling Model for AUTO device with THROUGHPUT hint
     Start inference,  6 groups of FPS/latency will be measured over  10s intervals
-    throughput:  190.70fps, latency:  29.76ms, time interval: 10.02s
-    throughput:  191.95fps, latency:  30.48ms, time interval: 10.00s
-    throughput:  192.78fps, latency:  30.40ms, time interval: 10.00s
-    throughput:  191.39fps, latency:  30.62ms, time interval: 10.00s
-    throughput:  192.18fps, latency:  30.44ms, time interval: 10.03s
-    throughput:  191.33fps, latency:  30.62ms, time interval: 10.00s
+    throughput:  189.24fps, latency:  30.04ms, time interval: 10.00s
+    throughput:  192.12fps, latency:  30.48ms, time interval: 10.01s
+    throughput:  191.27fps, latency:  30.64ms, time interval: 10.00s
+    throughput:  190.87fps, latency:  30.69ms, time interval: 10.01s
+    throughput:  189.50fps, latency:  30.89ms, time interval: 10.02s
+    throughput:  190.30fps, latency:  30.79ms, time interval: 10.01s
     Done
 
 
-Inference with LATENCY hint
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Inference with LATENCY hint `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 Loop for inference and update the FPS/Latency for each
 @metrics_update_interval seconds
@@ -473,17 +506,18 @@ Loop for inference and update the FPS/Latency for each
 
     Compiling Model for AUTO Device with LATENCY hint
     Start inference,  6 groups fps/latency will be out with  10s interval
-    throughput:  136.99fps, latency:  6.75ms, time interval: 10.00s
-    throughput:  140.91fps, latency:  6.74ms, time interval: 10.01s
-    throughput:  140.83fps, latency:  6.74ms, time interval: 10.00s
-    throughput:  140.90fps, latency:  6.74ms, time interval: 10.00s
-    throughput:  140.83fps, latency:  6.74ms, time interval: 10.00s
-    throughput:  140.85fps, latency:  6.74ms, time interval: 10.00s
+    throughput:  138.76fps, latency:  6.68ms, time interval: 10.00s
+    throughput:  141.79fps, latency:  6.70ms, time interval: 10.00s
+    throughput:  142.39fps, latency:  6.68ms, time interval: 10.00s
+    throughput:  142.30fps, latency:  6.68ms, time interval: 10.00s
+    throughput:  142.30fps, latency:  6.68ms, time interval: 10.01s
+    throughput:  142.53fps, latency:  6.67ms, time interval: 10.00s
     Done
 
 
-Difference in FPS and latency
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Difference in FPS and latency `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 .. code:: ipython3
 

docs/notebooks/106-auto-device-with-output_files/106-auto-device-with-output_25_0.png

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:03224259b8d1a7dd982c9d5b2baae8bdca495cbe008f83bc5b0a0fd3ecc47ab6
-size 27172
+oid sha256:7858463aadc74dfe6af6906da15e208305234a13a8463c4f4e4632630fffde70
+size 27107

docs/notebooks/106-auto-device-with-output_files/106-auto-device-with-output_26_0.png

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d962a84b77491e4c1f9bdcb4e0c58f6aae55f11f8e59ae747077b166e706fd76
-size 40994
+oid sha256:48a1f70dac1b326af8f205247fc377c9fb72286526c2d19507ada4b520a779ed
+size 39987

docs/notebooks/106-auto-device-with-output_files/index.html

@@ -1,10 +1,10 @@
 <html>
-<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/106-auto-device-with-output_files/</title></head>
+<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/106-auto-device-with-output_files/</title></head>
 <body bgcolor="white">
-<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/106-auto-device-with-output_files/</h1><hr><pre><a href="../">../</a>
-<a href="106-auto-device-with-output_12_0.jpg">106-auto-device-with-output_12_0.jpg</a>               12-Jul-2023 00:11              121563
-<a href="106-auto-device-with-output_12_0.png">106-auto-device-with-output_12_0.png</a>               12-Jul-2023 00:11              869661
-<a href="106-auto-device-with-output_25_0.png">106-auto-device-with-output_25_0.png</a>               12-Jul-2023 00:11               27172
-<a href="106-auto-device-with-output_26_0.png">106-auto-device-with-output_26_0.png</a>               12-Jul-2023 00:11               40994
+<h1>Index of /projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/106-auto-device-with-output_files/</h1><hr><pre><a href="../">../</a>
+<a href="106-auto-device-with-output_12_0.jpg">106-auto-device-with-output_12_0.jpg</a>               16-Aug-2023 01:31              121563
+<a href="106-auto-device-with-output_12_0.png">106-auto-device-with-output_12_0.png</a>               16-Aug-2023 01:31              869661
+<a href="106-auto-device-with-output_25_0.png">106-auto-device-with-output_25_0.png</a>               16-Aug-2023 01:31               27107
+<a href="106-auto-device-with-output_26_0.png">106-auto-device-with-output_26_0.png</a>               16-Aug-2023 01:31               39987
 </pre><hr></body>
 </html>

docs/notebooks/107-speech-recognition-quantization-data2vec-with-output.rst

@@ -19,8 +19,28 @@ steps:
 -  Compare performance of the original and quantized models.
 -  Compare Accuracy of the Original and Quantized Models.
 
-Download and prepare model
---------------------------
+
+
+.. _top:
+
+**Table of contents**:
+
+- `Download and prepare model <#download-and-prepare-model>`__
+
+  - `Obtain Pytorch model representation <#obtain-pytorch-model-representation>`__
+  - `Convert model to OpenVINO Intermediate Representation <#convert-model-to-openvino-intermediate-representation>`__
+  - `Prepare inference data <#prepare-inference-data>`__
+
+- `Check model inference result <#check-model-inference-result>`__
+- `Validate model accuracy on dataset <#validate-model-accuracy-on-dataset>`__
+- `Quantization <#quantization>`__
+- `Check INT8 model inference result <#check-int8-model-inference-result>`__
+- `Compare Performance of the Original and Quantized Models <#compare-performance-of-the-original-and-quantized-models>`__
+- `Compare Accuracy of the Original and Quantized Models <#compare-accuracy-of-the-original-and-quantized-models>`__
+
+Download and prepare model `⇑ <#top>`__
+###############################################################################################################################
+
 
 data2vec is a framework for self-supervised representation learning for
 images, speech, and text as described in `data2vec: A General Framework
@@ -38,8 +58,9 @@ In our case, we will use ``data2vec-audio-base-960h`` model, which was
 finetuned on 960 hours of audio from LibriSpeech Automatic Speech
 Recognition corpus and distributed as part of HuggingFace transformers.
 
-Obtain Pytorch model representation
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Obtain Pytorch model representation `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 For instantiating PyTorch model class, we should use
 ``Data2VecAudioForCTC.from_pretrained`` method with providing model ID
@@ -53,7 +74,7 @@ model specific pre- and post-processing steps.
 
 .. code:: ipython3
 
-    !pip install -q 'openvino-dev>=2023.0.0' 'nncf>=2.5.0'
+    !pip install -q "openvino-dev>=2023.0.0" "nncf>=2.5.0"
     !pip install -q soundfile librosa transformers onnx
 
 .. code:: ipython3
@@ -63,8 +84,9 @@ model specific pre- and post-processing steps.
     processor = Wav2Vec2Processor.from_pretrained("facebook/data2vec-audio-base-960h")
     model = Data2VecAudioForCTC.from_pretrained("facebook/data2vec-audio-base-960h")
 
-Convert model to OpenVINO Intermediate Representation
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Convert model to OpenVINO Intermediate Representation `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 .. code:: ipython3
 
@@ -134,11 +156,12 @@ Convert model to OpenVINO Intermediate Representation
     Read IR model from model/data2vec-audo-base.xml
 
 
-Prepare inference data
-~~~~~~~~~~~~~~~~~~~~~~
+Prepare inference data `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 For demonstration purposes, we will use short dummy version of
-librispeach dataset - ``patrickvonplaten/librispeech_asr_dummy`` to
+LibriSpeech dataset - ``patrickvonplaten/librispeech_asr_dummy`` to
 speed up model evaluation. Model accuracy can be different from reported
 in the paper. For reproducing original accuracy, use ``librispeech_asr``
 dataset.
@@ -174,8 +197,9 @@ dataset.
     Loading cached processed dataset at /home/adrian/.cache/huggingface/datasets/patrickvonplaten___librispeech_asr_dummy/clean/2.1.0/f2c70a4d03ab4410954901bde48c54b85ca1b7f9bf7d616e7e2a72b5ee6ddbfc/cache-4e0f4916cd205b24.arrow
 
 
-Check model inference result
-----------------------------
+Check model inference result `⇑ <#top>`__
+###############################################################################################################################
+
 
 The code below is used for running model inference on a single sample
 from the dataset. It contains the following steps:
@@ -247,8 +271,9 @@ For reference, see the same function provided for OpenVINO model.
 
 
 
-Validate model accuracy on dataset
-----------------------------------
+Validate model accuracy on dataset `⇑ <#top>`__
+###############################################################################################################################
+
 
 For model accuracy evaluation, `Word Error
 Rate <https://en.wikipedia.org/wiki/Word_error_rate>`__ metric can be
@@ -307,8 +332,9 @@ library.
     [OpenVino]  Word Error Rate: 0.0383
 
 
-Quantization
-------------
+Quantization `⇑ <#top>`__
+###############################################################################################################################
+
 
 `NNCF <https://github.com/openvinotoolkit/nncf>`__ provides a suite of
 advanced algorithms for Neural Networks inference optimization in
@@ -318,8 +344,10 @@ Create a quantized model from the pre-trained ``FP16`` model and the
 calibration dataset. The optimization process contains the following
 steps:
 
+
 1. Create a Dataset for quantization.
-2. Run ``nncf.quantize`` for getting an optimized model. The ``nncf.quantize`` function provides an interface for model quantization. It requires an instance of the  OpenVINO Model and quantization dataset. Optionally, some additional parameters for the configuration quantization process (number of samples for quantization, preset, ignored scope, etc.) can be provided. For more accurate results, we should keep the operation in the postprocessing subgraph in floating point precision, using the ``ignored_scope`` parameter. ``advanced_parameters`` can be used to specify advanced quantization parameters for fine-tuning the quantization algorithm. In this tutorial we pass range estimator parameters for activations. For more information, see `Tune quantization parameters <https://docs.openvino.ai/2023.0/basic_quantization_flow.html#tune-quantization-parameters>`__.
+2. Run ``nncf.quantize`` for getting an optimized model. The ``nncf.quantize`` function provides an interface for model quantization. It requires an instance of the OpenVINO Model and quantization dataset. Optionally, some additional parameters for the configuration quantization process (number of samples for quantization, preset, ignored scope, etc.) can be provided. For more accurate results, we should keep the operation in the postprocessing subgraph in floating point precision, using the ``ignored_scope`` parameter. ``advanced_parameters`` can be used to specify advanced quantization parameters for fine-tuning the quantization algorithm. In this tutorial we pass range estimator parameters for activations. For more information see 
+`Tune quantization parameters <https://docs.openvino.ai/2023.1/basic_quantization_flow.html#tune-quantization-parameters>`__.
 3. Serialize OpenVINO IR model using ``openvino.runtime.serialize`` function.
 
 .. code:: ipython3
@@ -589,8 +617,9 @@ saved using ``serialize`` function.
     quantized_model_path = Path(f"{MODEL_NAME}_openvino_model/{MODEL_NAME}_quantized.xml")
     serialize(quantized_model, str(quantized_model_path))
 
-Check INT8 model inference result
----------------------------------
+Check INT8 model inference result `⇑ <#top>`__
+###############################################################################################################################
+
 
 ``INT8`` model is the same in usage like the original one. We need to
 read it, using the ``core.read_model`` method and load on the device,
@@ -628,15 +657,17 @@ using ``core.compile_model``. After that, we can reuse the same
 
 
 
-Compare Performance of the Original and Quantized Models
---------------------------------------------------------
+Compare Performance of the Original and Quantized Models `⇑ <#top>`__
+###############################################################################################################################
 
 `Benchmark
-Tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_tool_README.html>`__
+Tool <https://docs.openvino.ai/2023.1/openvino_inference_engine_tools_benchmark_tool_README.html>`__
 is used to measure the inference performance of the ``FP16`` and
 ``INT8`` models.
 
-   **NOTE**: For more accurate performance, it is recommended to run
+.. note::
+
+   For more accurate performance, it is recommended to run
    ``benchmark_app`` in a terminal/command prompt after closing other
    applications. Run ``benchmark_app -m model.xml -d CPU`` to benchmark
    async inference on CPU for one minute. Change ``CPU`` to ``GPU`` to
@@ -791,8 +822,9 @@ is used to measure the inference performance of the ``FP16`` and
     [ INFO ] Throughput:   38.24 FPS
 
 
-Compare Accuracy of the Original and Quantized Models
------------------------------------------------------
+Compare Accuracy of the Original and Quantized Models `⇑ <#top>`__
+###############################################################################################################################
+
 
 Finally, calculate WER metric for the ``INT8`` model representation and
 compare it with the ``FP16`` result.

docs/notebooks/109-latency-tricks-with-output.rst

@@ -11,15 +11,17 @@ simulate a camera application that provides frames one by one.
 
 The performance tips applied in this notebook could be summarized in the
 following figure. Some of the steps below can be applied to any device
-at any stage, e.g., “shared_memory”; some can be used only to specific
-devices, e.g., “inference num threads” to CPU. As the number of
+at any stage, e.g., ``shared_memory``; some can be used only to specific
+devices, e.g., ``INFERENCE_NUM_THREADS`` to CPU. As the number of
 potential configurations is vast, we recommend looking at the steps
 below and then apply a trial-and-error approach. You can incorporate
 many hints simultaneously, like more inference threads + shared memory.
 It should give even better performance, but we recommend testing it
 anyway.
 
-   **NOTE**: We especially recommend trying
+.. note::
+
+   We especially recommend trying
    ``OpenVINO IR model + CPU + shared memory in latency mode`` or
    ``OpenVINO IR model + CPU + shared memory + more inference threads``.
 
@@ -27,17 +29,50 @@ The quantization and pre-post-processing API are not included here as
 they change the precision (quantization) or processing graph
 (prepostprocessor). You can find examples of how to apply them to
 optimize performance on OpenVINO IR files in
-`111-detection-quantization <../111-detection-quantization>`__ and
-`118-optimize-preprocessing <../118-optimize-preprocessing>`__.
+`111-detection-quantization <111-yolov5-quantization-migration-with-output.html>`__ and
+`118-optimize-preprocessing <118-optimize-preprocessing-with-output.html>`__.
 
 |image0|
 
-   **NOTE**: Many of the steps presented below will give you better
+.. note::
+
+   Many of the steps presented below will give you better
    performance. However, some of them may not change anything if they
    are strongly dependent on either the hardware or the model. Please
    run this notebook on your computer with your model to learn which of
    them makes sense in your case.
 
+   All the following tricks were run with OpenVINO 2022.3. Future
+   versions of OpenVINO may include various optimizations that may
+   result in different performance.
+
+A similar notebook focused on the throughput mode is available
+`here <109-throughput-tricks-with-output.html>`__.
+
+
+
+.. _top:
+
+**Table of contents**:
+
+- `Data <#data>`__
+- `Model <#model>`__
+- `Hardware <#hardware>`__
+- `Helper functions <#helper-functions>`__
+- `Optimizations <#optimizations>`__
+
+  - `PyTorch model <#pytorch-model>`__
+  - `ONNX model <#onnx-model>`__
+  - `OpenVINO IR model <#openvino-ir-model>`__
+  - `OpenVINO IR model on GPU <#openvino-ir-model-on-gpu>`__
+  - `OpenVINO IR model + more inference threads <#openvino-ir-model-+-more-inference-threads>`__
+  - `OpenVINO IR model in latency mode <#openvino-ir-model-in-latency-mode>`__
+  - `OpenVINO IR model in latency mode + shared memory <#openvino-ir-model-in-latency-mode-+-shared-memory>`__
+  - `Other tricks <#other-tricks>`__
+
+- `Performance comparison <#performance-comparison>`__
+- `Conclusions <#conclusions>`__
+
 Prerequisites
 -------------
 
@@ -58,8 +93,9 @@ Prerequisites
     sys.path.append("../utils")
     import notebook_utils as utils
 
-Data
-----
+Data `⇑ <#top>`__
+###############################################################################################################################
+
 
 We will use the same image of the dog sitting on a bicycle for all
 experiments below. The image is resized and preprocessed to fulfill the
@@ -94,12 +130,13 @@ requirements of this particular object detection model.
 
 .. parsed-literal::
 
-    <DisplayHandle display_id=5ab5e600f8b1dde3f55ff40ec675d04c>
+    <DisplayHandle display_id=c492bd5a5310fae2dfdea013f2ab99ab>
 
 
 
-Model
------
+Model `⇑ <#top>`__
+###############################################################################################################################
+
 
 We decided to go with
 `YOLOv5n <https://github.com/ultralytics/yolov5>`__, one of the
@@ -155,18 +192,21 @@ PyTorch Hub and small enough to see the difference in performance.
     Adding AutoShape... 
 
 
-Hardware
---------
+Hardware `⇑ <#top>`__
+###############################################################################################################################
+
 
 The code below lists the available hardware we will use in the
 benchmarking process.
 
-   **NOTE**: The hardware you have is probably completely different from
+.. note::
+
+   The hardware you have is probably completely different from
    ours. It means you can see completely different results.
 
 .. code:: ipython3
 
-    import openvino.runtime as ov
+    import openvino as ov
     
     # initialize OpenVINO
     core = ov.Core()
@@ -182,8 +222,9 @@ benchmarking process.
     CPU: Intel(R) Core(TM) i9-10920X CPU @ 3.50GHz
 
 
-Helper functions
-----------------
+Helper functions `⇑ <#top>`__
+###############################################################################################################################
+
 
 We’re defining a benchmark model function to use for all optimized
 models below. It runs inference 1000 times, averages the latency time,
@@ -317,15 +358,17 @@ the image.
     
         utils.show_array(output_img)
 
-Optimizations
--------------
+Optimizations `⇑ <#top>`__
+###############################################################################################################################
+
 
 Below, we present the performance tricks for faster inference in the
 latency mode. We release resources after every benchmarking to be sure
 the same amount of resource is available for every experiment.
 
-PyTorch model
-~~~~~~~~~~~~~
+PyTorch model `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 First, we’re benchmarking the original PyTorch model without any
 optimizations applied. We will treat it as our baseline.
@@ -346,12 +389,13 @@ optimizations applied. We will treat it as our baseline.
 
 .. parsed-literal::
 
-    PyTorch model on CPU. First inference time: 0.0286 seconds
-    PyTorch model on CPU: 0.0202 seconds per image (49.58 FPS)
+    PyTorch model on CPU. First inference time: 0.0227 seconds
+    PyTorch model on CPU: 0.0191 seconds per image (52.34 FPS)
 
 
-ONNX model
-~~~~~~~~~~
+ONNX model `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 The first optimization is exporting the PyTorch model to ONNX and
 running it in OpenVINO. It’s possible, thanks to the ONNX frontend. It
@@ -395,18 +439,19 @@ Representation (IR) to leverage the OpenVINO Runtime.
 
 .. parsed-literal::
 
-    ONNX model on CPU. First inference time: 0.0173 seconds
-    ONNX model on CPU: 0.0133 seconds per image (75.16 FPS)
+    ONNX model on CPU. First inference time: 0.0182 seconds
+    ONNX model on CPU: 0.0135 seconds per image (74.31 FPS)
 
 
-OpenVINO IR model
-~~~~~~~~~~~~~~~~~
+OpenVINO IR model `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 Let’s convert the ONNX model to OpenVINO Intermediate Representation
 (IR) FP16 and run it. Reducing the precision is one of the well-known
 methods for faster inference provided the hardware that supports lower
 precision, such as FP16 or even INT8. If the hardware doesn’t support
-lower precisions, the model will be inferred in FP32 automatically. We
+lower precision, the model will be inferred in FP32 automatically. We
 could also use quantization (INT8), but we should experience a little
 accuracy drop. That’s why we skip that step in this notebook.
 
@@ -433,12 +478,13 @@ accuracy drop. That’s why we skip that step in this notebook.
 
 .. parsed-literal::
 
-    OpenVINO model on CPU. First inference time: 0.0163 seconds
-    OpenVINO model on CPU: 0.0132 seconds per image (75.64 FPS)
+    OpenVINO model on CPU. First inference time: 0.0157 seconds
+    OpenVINO model on CPU: 0.0134 seconds per image (74.40 FPS)
 
 
-OpenVINO IR model on GPU
-~~~~~~~~~~~~~~~~~~~~~~~~
+OpenVINO IR model on GPU `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 Usually, a GPU device is faster than a CPU, so let’s run the above model
 on the GPU. Please note you need to have an Intel GPU and `install
@@ -461,13 +507,14 @@ execution.
     
         del ov_gpu_model  # release resources
 
-OpenVINO IR model + more inference threads
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+OpenVINO IR model + more inference threads `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 There is a possibility to add a config for any device (CPU in this
 case). We will increase the number of threads to an equal number of our
 cores. It should help us a lot. There are `more
-options <https://docs.openvino.ai/2023.0/groupov_runtime_cpp_prop_api.html>`__
+options <https://docs.openvino.ai/2023.1/groupov_runtime_cpp_prop_api.html>`__
 to be changed, so it’s worth playing with them to see what works best in
 our case.
 
@@ -491,14 +538,15 @@ our case.
 .. parsed-literal::
 
     OpenVINO model + more threads on CPU. First inference time: 0.0151 seconds
-    OpenVINO model + more threads on CPU: 0.0132 seconds per image (75.68 FPS)
+    OpenVINO model + more threads on CPU: 0.0134 seconds per image (74.36 FPS)
 
 
-OpenVINO IR model in latency mode
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+OpenVINO IR model in latency mode `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 OpenVINO offers a virtual device called
-`AUTO <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__,
+`AUTO <https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__,
 which can select the best device for us based on a performance hint.
 There are three different hints: ``LATENCY``, ``THROUGHPUT``, and
 ``CUMULATIVE_THROUGHPUT``. As this notebook is focused on the latency
@@ -520,12 +568,13 @@ devices as well.
 
 .. parsed-literal::
 
-    OpenVINO model on AUTO. First inference time: 0.0156 seconds
-    OpenVINO model on AUTO: 0.0135 seconds per image (73.93 FPS)
+    OpenVINO model on AUTO. First inference time: 0.0159 seconds
+    OpenVINO model on AUTO: 0.0136 seconds per image (73.57 FPS)
 
 
-OpenVINO IR model in latency mode + shared memory
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+OpenVINO IR model in latency mode + shared memory `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 
 OpenVINO is a C++ toolkit with Python wrappers (API). The default
 behavior in the Python API is copying the input to the additional buffer
@@ -554,21 +603,24 @@ performance!
 
 .. parsed-literal::
 
-    OpenVINO model + shared memory on AUTO. First inference time: 0.0144 seconds
-    OpenVINO model + shared memory on AUTO: 0.0053 seconds per image (187.92 FPS)
+    OpenVINO model + shared memory on AUTO. First inference time: 0.0139 seconds
+    OpenVINO model + shared memory on AUTO: 0.0054 seconds per image (184.61 FPS)
 
 
-Other tricks
-~~~~~~~~~~~~
+Other tricks `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
-There are other tricks for performance improvement, especially
-quantization and prepostprocessing. To get even more from your model,
-please visit
-`111-detection-quantization <../111-detection-quantization>`__ and
-`118-optimize-preprocessing <../118-optimize-preprocessing>`__.
 
-Performance comparison
-----------------------
+There are other tricks for performance improvement, such as quantization
+and pre-post-processing or dedicated to throughput mode. To get even
+more from your model, please visit
+`111-detection-quantization <111-yolov5-quantization-migration-with-output.html>`__,
+`118-optimize-preprocessing <118-optimize-preprocessing-with-output.html>`__, and
+`109-throughput-tricks <109-latency-tricks-with-output.html>`__.
+
+Performance comparison `⇑ <#top>`__
+###############################################################################################################################
+
 
 The following graphical comparison is valid for the selected model and
 hardware simultaneously. If you cannot see any improvement between some
@@ -604,14 +656,15 @@ steps, just skip them.
 .. image:: 109-latency-tricks-with-output_files/109-latency-tricks-with-output_30_0.png
 
 
-Conclusions
------------
+Conclusions `⇑ <#top>`__
+###############################################################################################################################
+
 
 We already showed the steps needed to improve the performance of an
 object detection model. Even if you experience much better performance
 after running this notebook, please note this may not be valid for every
 hardware or every model. For the most accurate results, please use
 ``benchmark_app`` `command-line
-tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_samples_benchmark_app_README.html>`__.
+tool <https://docs.openvino.ai/2023.1/openvino_inference_engine_samples_benchmark_app_README.html>`__.
 Note that ``benchmark_app`` cannot measure the impact of some tricks
 above, e.g., shared memory.

docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_30_0.png

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3683f4df707fff21d7f1e5329acbdfb9d00031ded856582d893c05a67f726d4e
-size 57103
+oid sha256:9d03578132292067edf38c21858cb3b2ed61780d3ff47f1962330823f4abee26
+size 56954

docs/notebooks/109-latency-tricks-with-output_files/index.html

@@ -1,14 +1,14 @@
 <html>
-<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/109-latency-tricks-with-output_files/</title></head>
+<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/109-latency-tricks-with-output_files/</title></head>
 <body bgcolor="white">
-<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/109-latency-tricks-with-output_files/</h1><hr><pre><a href="../">../</a>
-<a href="109-latency-tricks-with-output_14_0.jpg">109-latency-tricks-with-output_14_0.jpg</a>            12-Jul-2023 00:11              162715
-<a href="109-latency-tricks-with-output_17_0.jpg">109-latency-tricks-with-output_17_0.jpg</a>            12-Jul-2023 00:11              162715
-<a href="109-latency-tricks-with-output_19_0.jpg">109-latency-tricks-with-output_19_0.jpg</a>            12-Jul-2023 00:11              162756
-<a href="109-latency-tricks-with-output_23_0.jpg">109-latency-tricks-with-output_23_0.jpg</a>            12-Jul-2023 00:11              162756
-<a href="109-latency-tricks-with-output_25_0.jpg">109-latency-tricks-with-output_25_0.jpg</a>            12-Jul-2023 00:11              162756
-<a href="109-latency-tricks-with-output_27_0.jpg">109-latency-tricks-with-output_27_0.jpg</a>            12-Jul-2023 00:11              162756
-<a href="109-latency-tricks-with-output_30_0.png">109-latency-tricks-with-output_30_0.png</a>            12-Jul-2023 00:11               57103
-<a href="109-latency-tricks-with-output_4_0.jpg">109-latency-tricks-with-output_4_0.jpg</a>             12-Jul-2023 00:11              155828
+<h1>Index of /projects/ov-notebook/0.1.0-latest/20230815220807/dist/rst_files/109-latency-tricks-with-output_files/</h1><hr><pre><a href="../">../</a>
+<a href="109-latency-tricks-with-output_14_0.jpg">109-latency-tricks-with-output_14_0.jpg</a>            16-Aug-2023 01:31              162715
+<a href="109-latency-tricks-with-output_17_0.jpg">109-latency-tricks-with-output_17_0.jpg</a>            16-Aug-2023 01:31              162715
+<a href="109-latency-tricks-with-output_19_0.jpg">109-latency-tricks-with-output_19_0.jpg</a>            16-Aug-2023 01:31              162756
+<a href="109-latency-tricks-with-output_23_0.jpg">109-latency-tricks-with-output_23_0.jpg</a>            16-Aug-2023 01:31              162756
+<a href="109-latency-tricks-with-output_25_0.jpg">109-latency-tricks-with-output_25_0.jpg</a>            16-Aug-2023 01:31              162756
+<a href="109-latency-tricks-with-output_27_0.jpg">109-latency-tricks-with-output_27_0.jpg</a>            16-Aug-2023 01:31              162756
+<a href="109-latency-tricks-with-output_30_0.png">109-latency-tricks-with-output_30_0.png</a>            16-Aug-2023 01:31               56954
+<a href="109-latency-tricks-with-output_4_0.jpg">109-latency-tricks-with-output_4_0.jpg</a>             16-Aug-2023 01:31              155828
 </pre><hr></body>
 </html>

docs/notebooks/109-throughput-tricks-with-output.rst

@@ -0,0 +1,727 @@
+Performance tricks in OpenVINO for throughput mode
+==================================================
+
+
+
+The goal of this notebook is to provide a step-by-step tutorial for
+improving performance for inferencing in a throughput mode. High
+throughput is especially desired in applications when the results are
+not expected to appear as soon as possible but to lower the whole
+processing time. This notebook assumes computer vision workflow and uses
+`YOLOv5n <https://github.com/ultralytics/yolov5>`__ model. We will
+simulate a video processing application that has access to all frames at
+once (e.g. video editing).
+
+The performance tips applied in this notebook could be summarized in the
+following figure. Some of the steps below can be applied to any device
+at any stage, e.g., batch size; some can be used only to specific
+devices, e.g., inference threads number to CPU. As the number of
+potential configurations is vast, we recommend looking at the steps
+below and then apply a trial-and-error approach. You can incorporate
+many hints simultaneously, like more inference threads + async
+processing. It should give even better performance, but we recommend
+testing it anyway.
+
+The quantization and pre-post-processing API are not included here as
+they change the precision (quantization) or processing graph
+(prepostprocessor). You can find examples of how to apply them to
+optimize performance on OpenVINO IR files in
+`111-detection-quantization <111-yolov5-quantization-migration-with-output.html>`__ and
+`118-optimize-preprocessing <118-optimize-preprocessing-with-output.html>`__.
+
+|image0|
+
+.. note::
+
+   Many of the steps presented below will give you better
+   performance. However, some of them may not change anything if they
+   are strongly dependent on either the hardware or the model. Please
+   run this notebook on your computer with your model to learn which of
+   them makes sense in your case.
+
+   All the following tricks were run with OpenVINO 2022.3. Future
+   versions of OpenVINO may include various optimizations that may
+   result in different performance.
+
+A similar notebook focused on the latency mode is available
+`here <109-latency-tricks-with-output.html>`__.
+
+
+
+.. _top:
+
+**Table of contents**:
+
+- `Data <#data>`__
+- `Model <#model>`__
+- `Hardware <#hardware>`__
+- `Helper functions <#helper-functions>`__
+- `Optimizations <#optimizations>`__
+
+  - `PyTorch model <#pytorch-model>`__
+  - `OpenVINO IR model <#openvino-ir-model>`__
+  - `OpenVINO IR model + bigger batch <#openvino-ir-model-+-bigger-batch>`__
+  - `OpenVINO IR model in throughput mode <#openvino-ir-model-in-throughput-mode>`__
+  - `OpenVINO IR model in throughput mode on GPU <#openvino-ir-model-in-throughput-mode-on-gpu>`__
+  - `OpenVINO IR model in throughput mode on AUTO <#openvino-ir-model-in-throughput-mode-on-auto>`__
+  - `OpenVINO IR model in cumulative throughput mode on AUTO <#openvino-ir-model-in-cumulative-throughput-mode-on-auto>`__
+  - `OpenVINO IR model in cumulative throughput mode on AUTO + asynchronous processing <#openvino-ir-model-in-cumulative-throughput-mode-on-auto-+-asynchronous-processing>`__
+  - `Other tricks <#other-tricks>`__
+
+- `Performance comparison <#performance-comparison>`__
+- `Conclusions <#conclusions>`__
+
+Prerequisites
+-------------
+
+.. |image0| image:: https://github.com/openvinotoolkit/openvino_notebooks/assets/4547501/e1a6e230-7c80-491a-8732-02515c556f1b
+
+.. code:: ipython3
+
+    !pip install -q seaborn ultralytics
+
+.. code:: ipython3
+
+    import sys
+    import time
+    from pathlib import Path
+    from typing import Any, List, Tuple
+    
+    sys.path.append("../utils")
+    import notebook_utils as utils
+
+Data `⇑ <#top>`__
+###############################################################################################################################
+
+
+We will use the same image of the dog sitting on a bicycle copied 1000
+times to simulate the video with 1000 frames (about 33s). The image is
+resized and preprocessed to fulfill the requirements of this particular
+object detection model.
+
+.. code:: ipython3
+
+    import numpy as np
+    import cv2
+    
+    FRAMES_NUMBER = 1024
+    
+    IMAGE_WIDTH = 640
+    IMAGE_HEIGHT = 480
+    
+    # load image
+    image = utils.load_image("../data/image/coco_bike.jpg")
+    image = cv2.resize(image, dsize=(IMAGE_WIDTH, IMAGE_HEIGHT), interpolation=cv2.INTER_AREA)
+    
+    # preprocess it for YOLOv5
+    input_image = image / 255.0
+    input_image = np.transpose(input_image, axes=(2, 0, 1))
+    input_image = np.expand_dims(input_image, axis=0)
+    
+    # simulate video with many frames
+    video_frames = np.tile(input_image, (FRAMES_NUMBER, 1, 1, 1, 1))
+    
+    # show the image
+    utils.show_array(image)
+
+
+
+.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_4_0.jpg
+
+
+
+
+.. parsed-literal::
+
+    <DisplayHandle display_id=5216512dd310a9e1c0f8296b3aca9633>
+
+
+
+Model `⇑ <#top>`__
+###############################################################################################################################
+
+
+We decided to go with
+`YOLOv5n <https://github.com/ultralytics/yolov5>`__, one of the
+state-of-the-art object detection models, easily available through the
+PyTorch Hub and small enough to see the difference in performance.
+
+.. code:: ipython3
+
+    import torch
+    from IPython.utils import io
+    
+    # directory for all models
+    base_model_dir = Path("model")
+    
+    model_name = "yolov5n"
+    model_path = base_model_dir / model_name
+    
+    # load YOLOv5n from PyTorch Hub
+    pytorch_model = torch.hub.load("ultralytics/yolov5", "custom", path=model_path, device="cpu", skip_validation=True)
+    # don't print full model architecture
+    with io.capture_output():
+        pytorch_model.eval()
+
+
+.. parsed-literal::
+
+    Using cache found in /opt/home/k8sworker/.cache/torch/hub/ultralytics_yolov5_master
+    YOLOv5 🚀 2023-4-21 Python-3.8.10 torch-1.13.1+cpu CPU
+    
+    Fusing layers... 
+    YOLOv5n summary: 213 layers, 1867405 parameters, 0 gradients
+    Adding AutoShape... 
+
+
+.. parsed-literal::
+
+    requirements: /opt/home/k8sworker/.cache/torch/hub/requirements.txt not found, check failed.
+
+
+Hardware `⇑ <#top>`__
+###############################################################################################################################
+
+
+The code below lists the available hardware we will use in the
+benchmarking process.
+
+.. note::
+
+   The hardware you have is probably completely different from
+   ours. It means you can see completely different results.
+
+.. code:: ipython3
+
+    import openvino as ov
+    
+    # initialize OpenVINO
+    core = ov.Core()
+    
+    # print available devices
+    for device in core.available_devices:
+        device_name = core.get_property(device, "FULL_DEVICE_NAME")
+        print(f"{device}: {device_name}")
+
+
+.. parsed-literal::
+
+    CPU: Intel(R) Core(TM) i9-10920X CPU @ 3.50GHz
+
+
+Helper functions `⇑ <#top>`__
+###############################################################################################################################
+
+
+We’re defining a benchmark model function to use for all optimizations
+below. It runs inference for 1000 frames and prints average frames per
+second (FPS).
+
+.. code:: ipython3
+
+    from openvino.runtime import AsyncInferQueue
+    
+    
+    def benchmark_model(model: Any, frames: np.ndarray, async_queue: AsyncInferQueue = None, benchmark_name: str = "OpenVINO model", device_name: str = "CPU") -> float:
+        """
+        Helper function for benchmarking the model. It measures the time and prints results.
+        """
+        # measure the first inference separately - it may be slower as it contains also initialization
+        start = time.perf_counter()
+        model(frames[0])
+        if async_queue:
+            async_queue.wait_all()
+        end = time.perf_counter()
+        first_infer_time = end - start
+        print(f"{benchmark_name} on {device_name}. First inference time: {first_infer_time :.4f} seconds")
+    
+        # benchmarking
+        start = time.perf_counter()
+        for batch in frames:
+            model(batch)
+        # wait for all threads if async processing
+        if async_queue:
+            async_queue.wait_all()
+        end = time.perf_counter()
+    
+        # elapsed time
+        infer_time = end - start
+    
+        # print second per image and FPS
+        mean_infer_time = infer_time / FRAMES_NUMBER
+        mean_fps = FRAMES_NUMBER / infer_time
+        print(f"{benchmark_name} on {device_name}: {mean_infer_time :.4f} seconds per image ({mean_fps :.2f} FPS)")
+    
+        return mean_fps
+
+The following functions aim to post-process results and draw boxes on
+the image.
+
+.. code:: ipython3
+
+    # https://gist.github.com/AruniRC/7b3dadd004da04c80198557db5da4bda
+    classes = [
+        "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant",
+        "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra",
+        "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite",
+        "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork",
+        "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut",
+        "cake", "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard",
+        "cell phone", "microwave", "oven", "oaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear",
+        "hair drier", "toothbrush"
+    ]
+    
+    # Colors for the classes above (Rainbow Color Map).
+    colors = cv2.applyColorMap(
+        src=np.arange(0, 255, 255 / len(classes), dtype=np.float32).astype(np.uint8),
+        colormap=cv2.COLORMAP_RAINBOW,
+    ).squeeze()
+    
+    
+    def postprocess(detections: np.ndarray) -> List[Tuple]:
+        """
+        Postprocess the raw results from the model.
+        """
+        # candidates - probability > 0.25
+        detections = detections[detections[..., 4] > 0.25]
+    
+        boxes = []
+        labels = []
+        scores = []
+        for obj in detections:
+            xmin, ymin, ww, hh = obj[:4]
+            score = obj[4]
+            label = np.argmax(obj[5:])
+            # Create a box with pixels coordinates from the box with normalized coordinates [0,1].
+            boxes.append(
+                tuple(map(int, (xmin - ww // 2, ymin - hh // 2, ww, hh)))
+            )
+            labels.append(int(label))
+            scores.append(float(score))
+    
+        # Apply non-maximum suppression to get rid of many overlapping entities.
+        # See https://paperswithcode.com/method/non-maximum-suppression
+        # This algorithm returns indices of objects to keep.
+        indices = cv2.dnn.NMSBoxes(
+            bboxes=boxes, scores=scores, score_threshold=0.25, nms_threshold=0.5
+        )
+    
+        # If there are no boxes.
+        if len(indices) == 0:
+            return []
+    
+        # Filter detected objects.
+        return [(labels[idx], scores[idx], boxes[idx]) for idx in indices.flatten()]
+    
+    
+    def draw_boxes(img: np.ndarray, boxes):
+        """
+        Draw detected boxes on the image.
+        """
+        for label, score, box in boxes:
+            # Choose color for the label.
+            color = tuple(map(int, colors[label]))
+            # Draw a box.
+            x2 = box[0] + box[2]
+            y2 = box[1] + box[3]
+            cv2.rectangle(img=img, pt1=box[:2], pt2=(x2, y2), color=color, thickness=2)
+    
+            # Draw a label name inside the box.
+            cv2.putText(
+                img=img,
+                text=f"{classes[label]} {score:.2f}",
+                org=(box[0] + 10, box[1] + 20),
+                fontFace=cv2.FONT_HERSHEY_COMPLEX,
+                fontScale=img.shape[1] / 1200,
+                color=color,
+                thickness=1,
+                lineType=cv2.LINE_AA,
+            )
+    
+    
+    def show_result(results: np.ndarray):
+        """
+        Postprocess the raw results, draw boxes and show the image.
+        """
+        output_img = image.copy()
+    
+        detections = postprocess(results)
+        draw_boxes(output_img, detections)
+    
+        utils.show_array(output_img)
+
+Optimizations `⇑ <#top>`__
+###############################################################################################################################
+
+
+Below, we present the performance tricks for faster inference in the
+throughput mode. We release resources after every benchmarking to be
+sure the same amount of resource is available for every experiment.
+
+PyTorch model `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+
+First, we’re benchmarking the original PyTorch model without any
+optimizations applied. We will treat it as our baseline.
+
+.. code:: ipython3
+
+    import torch
+    
+    with torch.no_grad():
+        result = pytorch_model(torch.as_tensor(video_frames[0])).detach().numpy()[0]
+        show_result(result)
+        pytorch_fps = benchmark_model(pytorch_model, frames=torch.as_tensor(video_frames).float(), benchmark_name="PyTorch model")
+
+
+
+.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_14_0.jpg
+
+
+.. parsed-literal::
+
+    PyTorch model on CPU. First inference time: 0.0266 seconds
+    PyTorch model on CPU: 0.0200 seconds per image (49.99 FPS)
+
+
+OpenVINO IR model `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+
+The first optimization is exporting the PyTorch model to OpenVINO
+Intermediate Representation (IR) FP16 and running it. Reducing the
+precision is one of the well-known methods for faster inference provided
+the hardware that supports lower precision, such as FP16 or even INT8.
+If the hardware doesn’t support lower precision, the model will be
+inferred in FP32 automatically. We could also use quantization (INT8),
+but we should experience a little accuracy drop. That’s why we skip that
+step in this notebook.
+
+.. code:: ipython3
+
+    from openvino.tools import mo
+    
+    onnx_path = base_model_dir / Path(f"{model_name}_{IMAGE_WIDTH}_{IMAGE_HEIGHT}").with_suffix(".onnx")
+    
+    # export PyTorch model to ONNX if it doesn't already exist
+    if not onnx_path.exists():
+        dummy_input = torch.randn(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH)
+        torch.onnx.export(pytorch_model, dummy_input, onnx_path)
+    
+    # convert ONNX model to IR, use FP16
+    ov_model = mo.convert_model(onnx_path, compress_to_fp16=True)
+
+.. code:: ipython3
+
+    ov_cpu_model = core.compile_model(ov_model, device_name="CPU")
+    
+    result = ov_cpu_model(video_frames[0])[ov_cpu_model.output(0)][0]
+    show_result(result)
+    ov_cpu_fps = benchmark_model(model=ov_cpu_model, frames=video_frames, benchmark_name="OpenVINO model")
+    
+    del ov_cpu_model  # release resources
+
+
+
+.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_17_0.jpg
+
+
+.. parsed-literal::
+
+    OpenVINO model on CPU. First inference time: 0.0195 seconds
+    OpenVINO model on CPU: 0.0073 seconds per image (136.92 FPS)
+
+
+OpenVINO IR model + bigger batch `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+
+Batch processing often gives higher throughput as more inputs are
+processed at once. To use bigger batches (than 1), we must convert the
+model again, specifying a new input shape, and reshape input frames. In
+our case, a batch size equal to 4 is the best choice, but optimal batch
+size is very device-specific and depends on many factors, e.g.,
+inference precision. We recommend trying various sizes for other
+hardware and model.
+
+.. code:: ipython3
+
+    batch_size = 4
+    
+    onnx_batch_path = base_model_dir / Path(f"{model_name}_{IMAGE_WIDTH}_{IMAGE_HEIGHT}_batch_{batch_size}").with_suffix(".onnx")
+    
+    if not onnx_batch_path.exists():
+        dummy_input = torch.randn(batch_size, 3, IMAGE_HEIGHT, IMAGE_WIDTH)
+        torch.onnx.export(pytorch_model, dummy_input, onnx_batch_path)
+    
+    # export the model with the bigger batch size
+    ov_batch_model = mo.convert_model(onnx_batch_path, compress_to_fp16=True)
+
+
+.. parsed-literal::
+
+    /opt/home/k8sworker/.cache/torch/hub/ultralytics_yolov5_master/models/common.py:514: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
+      y = self.model(im, augment=augment, visualize=visualize) if augment or visualize else self.model(im)
+    /opt/home/k8sworker/.cache/torch/hub/ultralytics_yolov5_master/models/yolo.py:64: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
+      if self.dynamic or self.grid[i].shape[2:4] != x[i].shape[2:4]:
+
+
+.. code:: ipython3
+
+    ov_cpu_batch_model = core.compile_model(ov_batch_model, device_name="CPU")
+    
+    batched_video_frames = video_frames.reshape([-1, batch_size, 3, IMAGE_HEIGHT, IMAGE_WIDTH])
+    
+    result = ov_cpu_batch_model(batched_video_frames[0])[ov_cpu_batch_model.output(0)][0]
+    show_result(result)
+    ov_cpu_batch_fps = benchmark_model(model=ov_cpu_batch_model, frames=batched_video_frames, benchmark_name="OpenVINO model + bigger batch")
+    
+    del ov_cpu_batch_model  # release resources
+
+
+
+.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_20_0.jpg
+
+
+.. parsed-literal::
+
+    OpenVINO model + bigger batch on CPU. First inference time: 0.0590 seconds
+    OpenVINO model + bigger batch on CPU: 0.0069 seconds per image (143.96 FPS)
+
+
+OpenVINO IR model in throughput mode `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+
+OpenVINO allows specifying a performance hint changing the internal
+configuration of the device. There are three different hints:
+``LATENCY``, ``THROUGHPUT``, and ``CUMULATIVE_THROUGHPUT``. As this
+notebook is focused on the throughput mode, we will use the latter two.
+The hints can be used with other devices as well. Throughput mode
+implicitly triggers using the `Automatic
+Batching <https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_Automatic_Batching.html>`__
+feature, which sets the batch size to the optimal level.
+
+.. code:: ipython3
+
+    ov_cpu_through_model = core.compile_model(ov_model, device_name="CPU", config={"PERFORMANCE_HINT": "THROUGHPUT"})
+    
+    result = ov_cpu_through_model(video_frames[0])[ov_cpu_through_model.output(0)][0]
+    show_result(result)
+    ov_cpu_through_fps = benchmark_model(model=ov_cpu_through_model, frames=video_frames, benchmark_name="OpenVINO model", device_name="CPU (THROUGHPUT)")
+    
+    del ov_cpu_through_model  # release resources
+
+
+
+.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_22_0.jpg
+
+
+.. parsed-literal::
+
+    OpenVINO model on CPU (THROUGHPUT). First inference time: 0.0226 seconds
+    OpenVINO model on CPU (THROUGHPUT): 0.0117 seconds per image (85.50 FPS)
+
+
+OpenVINO IR model in throughput mode on GPU `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+
+Usually, a GPU device provides more frames per second than a CPU, so
+let’s run the above model on the GPU. Please note you need to have an
+Intel GPU and `install
+drivers <https://github.com/openvinotoolkit/openvino_notebooks/wiki/Ubuntu#1-install-python-git-and-gpu-drivers-optional>`__
+to be able to run this step. In addition, offloading to the GPU helps
+reduce CPU load and memory consumption, allowing it to be left for
+routine processes. If you cannot observe a higher throughput on GPU, it
+may be because the model is too light to benefit from massive parallel
+execution.
+
+.. code:: ipython3
+
+    ov_gpu_fps = 0.0
+    if "GPU" in core.available_devices:
+        # compile for GPU
+        ov_gpu_model = core.compile_model(ov_model, device_name="GPU", config={"PERFORMANCE_HINT": "THROUGHPUT"})
+    
+        result = ov_gpu_model(video_frames[0])[ov_gpu_model.output(0)][0]
+        show_result(result)
+        ov_gpu_fps = benchmark_model(model=ov_gpu_model, frames=video_frames, benchmark_name="OpenVINO model", device_name="GPU (THROUGHPUT)")
+    
+        del ov_gpu_model  # release resources
+
+OpenVINO IR model in throughput mode on AUTO `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+
+OpenVINO offers a virtual device called
+`AUTO <https://docs.openvino.ai/2023.1/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__,
+which can select the best device for us based on the aforementioned
+performance hint.
+
+.. code:: ipython3
+
+    ov_auto_model = core.compile_model(ov_model, device_name="AUTO", config={"PERFORMANCE_HINT": "THROUGHPUT"})
+    
+    result = ov_auto_model(video_frames[0])[ov_auto_model.output(0)][0]
+    show_result(result)
+    ov_auto_fps = benchmark_model(model=ov_auto_model, frames=video_frames, benchmark_name="OpenVINO model", device_name="AUTO (THROUGHPUT)")
+    
+    del ov_auto_model  # release resources
+
+
+
+.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_26_0.jpg
+
+
+.. parsed-literal::
+
+    OpenVINO model on AUTO (THROUGHPUT). First inference time: 0.0257 seconds
+    OpenVINO model on AUTO (THROUGHPUT): 0.0215 seconds per image (46.61 FPS)
+
+
+OpenVINO IR model in cumulative throughput mode on AUTO `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+
+The AUTO device in throughput mode will select the best, but one
+physical device to bring the highest throughput. However, if we have
+more Intel devices like CPU, iGPUs, and dGPUs in one machine, we may
+benefit from them all. To do so, we must use cumulative throughput to
+activate all devices.
+
+.. code:: ipython3
+
+    ov_auto_cumulative_model = core.compile_model(ov_model, device_name="AUTO", config={"PERFORMANCE_HINT": "CUMULATIVE_THROUGHPUT"})
+    
+    result = ov_auto_cumulative_model(video_frames[0])[ov_auto_cumulative_model.output(0)][0]
+    show_result(result)
+    ov_auto_cumulative_fps = benchmark_model(model=ov_auto_cumulative_model, frames=video_frames, benchmark_name="OpenVINO model", device_name="AUTO (CUMULATIVE THROUGHPUT)")
+
+
+
+.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_28_0.jpg
+
+
+.. parsed-literal::
+
+    OpenVINO model on AUTO (CUMULATIVE THROUGHPUT). First inference time: 0.0268 seconds
+    OpenVINO model on AUTO (CUMULATIVE THROUGHPUT): 0.0216 seconds per image (46.25 FPS)
+
+
+OpenVINO IR model in cumulative throughput mode on AUTO + asynchronous processing `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+Asynchronous mode means that OpenVINO immediately returns from an
+inference call and doesn’t wait for the result. It requires more
+concurrent code to be written, but should offer better processing time
+utilization e.g. we can run some pre- or post-processing code while
+waiting for the result. Although we could use async processing directly
+(start_async() function), it’s recommended to use AsyncInferQueue, which
+is an easier approach to achieve the same outcome. This class
+automatically spawns the pool of InferRequest objects (also called
+“jobs”) and provides synchronization mechanisms to control the flow of
+the pipeline.
+
+.. note::
+
+   Asynchronous processing cannot guarantee outputs to be in
+   the same order as inputs, so be careful in case of applications when
+   the order of frames matters, e.g., videos.
+
+.. code:: ipython3
+
+    from openvino.runtime import AsyncInferQueue
+    
+    
+    def callback(infer_request, info):
+        result = infer_request.get_output_tensor(0).data[0]
+        show_result(result)
+        pass
+    
+    infer_queue = AsyncInferQueue(ov_auto_cumulative_model)
+    infer_queue.set_callback(callback)  # set callback to post-process (show) results
+    
+    infer_queue.start_async(video_frames[0])
+    infer_queue.wait_all()
+    
+    # don't show output for the remaining frames
+    infer_queue.set_callback(lambda x, y: {})
+    ov_async_model = benchmark_model(model=infer_queue.start_async, frames=video_frames, async_queue=infer_queue, benchmark_name="OpenVINO model in asynchronous processing", device_name="AUTO (CUMULATIVE THROUGHPUT)")
+    
+    del infer_queue  # release resources
+
+
+
+.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_30_0.jpg
+
+
+.. parsed-literal::
+
+    OpenVINO model in asynchronous processing on AUTO (CUMULATIVE THROUGHPUT). First inference time: 0.0239 seconds
+    OpenVINO model in asynchronous processing on AUTO (CUMULATIVE THROUGHPUT): 0.0041 seconds per image (245.46 FPS)
+
+
+Other tricks `⇑ <#top>`__
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+
+There are other tricks for performance improvement, such as advanced
+options, quantization and pre-post-processing or dedicated to latency
+mode. To get even more from your model, please visit `advanced
+throughput
+options <https://docs.openvino.ai/2023.1/openvino_docs_deployment_optimization_guide_tput_advanced.html>`__,
+`109-latency-tricks <109-latency-tricks-with-output.html>`__,
+`111-detection-quantization <111-yolov5-quantization-migration-with-output.html>`__, and
+`118-optimize-preprocessing <118-optimize-preprocessing-with-output.html>`__.
+
+Performance comparison `⇑ <#top>`__
+###############################################################################################################################
+
+
+The following graphical comparison is valid for the selected model and
+hardware simultaneously. If you cannot see any improvement between some
+steps, just skip them.
+
+.. code:: ipython3
+
+    %matplotlib inline
+
+.. code:: ipython3
+
+    from matplotlib import pyplot as plt
+    
+    labels = ["PyTorch model", "OpenVINO IR model", "OpenVINO IR model + bigger batch", "OpenVINO IR model in throughput mode", "OpenVINO IR model in throughput mode on GPU",
+              "OpenVINO IR model in throughput mode on AUTO", "OpenVINO IR model in cumulative throughput mode on AUTO", "OpenVINO IR model in cumulative throughput mode on AUTO + asynchronous processing"]
+    
+    fps = [pytorch_fps, ov_cpu_fps, ov_cpu_batch_fps, ov_cpu_through_fps, ov_gpu_fps, ov_auto_fps, ov_auto_cumulative_fps, ov_async_model]
+    
+    bar_colors = colors[::10] / 255.0
+    
+    fig, ax = plt.subplots(figsize=(16, 8))
+    ax.bar(labels, fps, color=bar_colors)
+    
+    ax.set_ylabel("Throughput [FPS]")
+    ax.set_title("Performance difference")
+    
+    plt.xticks(rotation='vertical')
+    plt.show()
+
+
+
+.. image:: 109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_33_0.png
+
+
+Conclusions `⇑ <#top>`__
+###############################################################################################################################
+
+
+We already showed the steps needed to improve the throughput of an
+object detection model. Even if you experience much better performance
+after running this notebook, please note this may not be valid for every
+hardware or every model. For the most accurate results, please use
+``benchmark_app`` `command-line
+tool <https://docs.openvino.ai/2023.1/openvino_inference_engine_samples_benchmark_app_README.html>`__.
+Note that ``benchmark_app`` cannot measure the impact of some tricks
+above.

docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_14_0.jpg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84e4f91c248768c2ea746240e307041396099f0d52fdb89b0179fa72e353894a
+size 162715

docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_17_0.jpg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcd7da923bb0f72430eaf7b4770175320f1f3219aaca2d460c54fa9ef07e51c2
+size 162756

docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_20_0.jpg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcd7da923bb0f72430eaf7b4770175320f1f3219aaca2d460c54fa9ef07e51c2
+size 162756

docs/notebooks/109-throughput-tricks-with-output_files/109-throughput-tricks-with-output_22_0.jpg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcd7da923bb0f72430eaf7b4770175320f1f3219aaca2d460c54fa9ef07e51c2
+size 162756