IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[DOCS] Updating Interactive Tutorials (#18504)

Browse Source 
* update-notebooks

* Update docs/nbdoc/nbdoc.py

Co-authored-by: bstankix <bartoszx.stankiewicz@intel.com>

* Update docs/nbdoc/nbdoc.py

Co-authored-by: bstankix <bartoszx.stankiewicz@intel.com>

---------

Co-authored-by: bstankix <bartoszx.stankiewicz@intel.com>
This commit is contained in:
Sebastian Golebiewski
2023-07-14 10:57:06 +02:00
committed by GitHub
parent 92ecccc1b9
commit 6761a29af5
273 changed files with 8964 additions and 3464 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
docs/nbdoc/consts.py
View File

@@ -8,7 +8,7 @@ repo_owner = "openvinotoolkit"
repo_name = "openvino_notebooks"
artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/"
artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/"
blacklisted_extensions = ['.xml', '.bin']
@@ -20,7 +20,7 @@ section_names = ["Getting Started", "Convert & Optimize",
binder_template = """
This tutorial is also available as a Jupyter notebook that can be cloned directly from GitHub.
See the |installation_link| for instructions to run this tutorial locally on Windows, Linux or macOS.
To run without installing anything, click the launch binder button.
To run without installing anything, click the "launch binder" button.
|binder_link| |github_link|
@@ -36,6 +36,52 @@ To run without installing anything, click the launch binder button.
<a href="https://github.com/{{ owner }}/{{ repo }}" target="_blank"><img src="https://badgen.net/badge/icon/github?icon=github&label" alt="Github"></a>
\n
"""
colab_template = """
This tutorial is also available as a Jupyter notebook that can be cloned directly from GitHub.
See the |installation_link| for instructions to run this tutorial locally on Windows, Linux or macOS.
To run without installing anything, click the "Open in Colab" button.
|colab_link| |github_link|
.. |installation_link| raw:: html
<a href="https://github.com/{{ owner }}/{{ repo }}#-installation-guide" target="_blank">installation guide</a>
.. |colab_link| raw:: html
<a href="https://colab.research.google.com/github/{{ owner }}/{{ repo }}/blob/main/{{ folder }}/{{ notebook }}/{{ notebook }}.ipynb" target="_blank"><img src="https://camo.githubusercontent.com/84f0493939e0c4de4e6dbe113251b4bfb5353e57134ffd9fcab6b8714514d4d1/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667" alt="Google Colab"></a>
.. |github_link| raw:: html
<a href="https://github.com/{{ owner }}/{{ repo }}" target="_blank"><img src="https://badgen.net/badge/icon/github?icon=github&label" alt="Github"></a>
\n
"""
binder_colab_template = """
This tutorial is also available as a Jupyter notebook that can be cloned directly from GitHub.
See the |installation_link| for instructions to run this tutorial locally on Windows, Linux or macOS.
To run without installing anything, click the "launch binder" or "Open in Colab" button.
|binder_link| |colab_link| |github_link|
.. |installation_link| raw:: html
<a href="https://github.com/{{ owner }}/{{ repo }}#-installation-guide" target="_blank">installation guide</a>
.. |binder_link| raw:: html
<a href="https://mybinder.org/v2/gh/{{ owner }}/{{ repo }}/HEAD?filepath={{ folder }}%2F{{ notebook }}%2F{{ notebook }}.ipynb" target="_blank"><img src="https://mybinder.org/badge_logo.svg" alt="Binder"></a>
.. |colab_link| raw:: html
<a href="https://colab.research.google.com/github/{{ owner }}/{{ repo }}/blob/main/{{ folder }}/{{ notebook }}/{{ notebook }}.ipynb" target="_blank"><img src="https://camo.githubusercontent.com/84f0493939e0c4de4e6dbe113251b4bfb5353e57134ffd9fcab6b8714514d4d1/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667" alt="Google Colab"></a>
.. |github_link| raw:: html
<a href="https://github.com/{{ owner }}/{{ repo }}" target="_blank"><img src="https://badgen.net/badge/icon/github?icon=github&label" alt="Github"></a>
\n
"""
no_binder_template = """

52
docs/nbdoc/nbdoc.py
View File

@@ -11,6 +11,8 @@ from utils import (
from consts import (
artifacts_link,
binder_template,
colab_template,
binder_colab_template,
blacklisted_extensions,
notebooks_docs,
notebooks_path,
@@ -98,24 +100,44 @@ class NbProcessor:
"repo": repo_name,
"folder": repo_directory,
}
self.colab_data = {
"owner": repo_owner,
"repo": repo_name,
"folder": repo_directory,
}
def fetch_binder_list(self, file_format: str = 'txt') -> list:
def fetch_binder_list(self, file) -> list:
"""Function that fetches list of notebooks with binder buttons
:param file_format: Format of file containing list of notebooks with button. Defaults to 'txt'
:type file_format: str
:return: List of notebooks conaining binder buttons
:return: List of notebooks containing binder buttons
:rtype: list
"""
list_of_buttons = glob(f"{self.nb_path}/*.{file_format}")
list_of_buttons = glob(f"{self.nb_path}/{file}")
if list_of_buttons:
with open(list_of_buttons[0]) as file:
list_of_buttons = file.read().splitlines()
return list_of_buttons
else:
return []
def add_binder(self, buttons_list: list, template_with_binder: str = binder_template, template_without_binder: str = no_binder_template):
def fetch_colab_list(self, file) -> list:
"""Function that fetches list of notebooks with colab buttons
:param file_format: Format of file containing list of notebooks with button. Defaults to 'lst'
:type file_format: str
:return: List of notebooks containing colab buttons
:rtype: list
"""
list_of_cbuttons = glob(f"{self.nb_path}/{file}")
if list_of_cbuttons:
with open(list_of_cbuttons[0]) as file:
list_of_cbuttons = file.read().splitlines()
return list_of_cbuttons
return []
def add_binder(self, buttons_list: list, cbuttons_list: list, template_with_colab_and_binder: str = binder_colab_template, template_with_binder: str = binder_template, template_with_colab: str = colab_template, template_without_binder: str = no_binder_template):
"""Function working as an example how to add binder button to existing rst files
:param buttons_list: List of notebooks that work on Binder.
@@ -127,17 +149,18 @@ class NbProcessor:
:raises FileNotFoundError: In case of failure of adding content, error will appear
"""
for notebook in [
nb for nb in os.listdir(self.nb_path) if verify_notebook_name(nb)
]:
if '-'.join(notebook.split('-')[:-2]) in buttons_list:
button_text = create_content(
template_with_binder, self.binder_data, notebook)
if not add_content_below(button_text, f"{self.nb_path}/{notebook}"):
raise FileNotFoundError("Unable to modify file")
notebook_item = '-'.join(notebook.split('-')[:-2])
if notebook_item in buttons_list:
template = template_with_colab_and_binder if notebook_item in cbuttons_list else template_with_binder
else:
button_text = create_content(
template_without_binder, self.binder_data, notebook)
template = template_with_colab if notebook_item in cbuttons_list else template_without_binder
button_text = create_content(template, self.binder_data, notebook)
if not add_content_below(button_text, f"{self.nb_path}/{notebook}"):
raise FileNotFoundError("Unable to modify file")
@@ -170,8 +193,9 @@ def main():
shutil.copytree(sourcedir, outdir)
# Step 3. Run processing on downloaded file
nbp = NbProcessor(outdir)
buttons_list = nbp.fetch_binder_list('txt')
nbp.add_binder(buttons_list)
buttons_list = nbp.fetch_binder_list('notebooks_with_binder_buttons.txt')
cbuttons_list = nbp.fetch_colab_list('notebooks_with_colab_buttons.txt')
nbp.add_binder(buttons_list, cbuttons_list)
nbp.render_rst(outdir.joinpath(notebooks_docs))

39
docs/notebooks/001-hello-world-with-output.rst
View File

@@ -10,7 +10,7 @@ from `Open Model
Zoo <https://github.com/openvinotoolkit/open_model_zoo/>`__ is used in
this tutorial. For more information about how OpenVINO IR models are
created, refer to the `TensorFlow to
OpenVINO <101-tensorflow-to-openvino-with-output.html>`__
OpenVINO <101-tensorflow-classification-to-openvino-with-output.html>`__
tutorial.
Imports
@@ -52,7 +52,6 @@ Download the Model and data samples
.. parsed-literal::
artifacts/v3-small_224_1.0_float.xml: 0%| | 0.00/294k [00:00<?, ?B/s]
@@ -64,14 +63,42 @@ Download the Model and data samples
artifacts/v3-small_224_1.0_float.bin: 0%| | 0.00/4.84M [00:00<?, ?B/s]
Select inference device
-----------------------
select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
Load the Model
--------------
.. code:: ipython3
ie = Core()
model = ie.read_model(model=model_xml_path)
compiled_model = ie.compile_model(model=model, device_name="CPU")
core = Core()
model = core.read_model(model=model_xml_path)
compiled_model = core.compile_model(model=model, device_name=device.value)
output_layer = compiled_model.output(0)
@@ -92,7 +119,7 @@ Load an Image
.. image:: 001-hello-world-with-output_files/001-hello-world-with-output_8_0.png
.. image:: 001-hello-world-with-output_files/001-hello-world-with-output_10_0.png
Do Inference

0
docs/notebooks/001-hello-world-with-output_files/001-hello-world-with-output_8_0.png → docs/notebooks/001-hello-world-with-output_files/001-hello-world-with-output_10_0.png
View File

6
docs/notebooks/001-hello-world-with-output_files/index.html
View File

@@ -1,7 +1,7 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/001-hello-world-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/001-hello-world-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/001-hello-world-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="001-hello-world-with-output_8_0.png">001-hello-world-with-output_8_0.png</a> 22-Jun-2023 00:06 387941
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/001-hello-world-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="001-hello-world-with-output_10_0.png">001-hello-world-with-output_10_0.png</a> 12-Jul-2023 00:11 387941
</pre><hr></body>
</html>

167
docs/notebooks/002-openvino-api-with-output.rst
View File

@@ -28,21 +28,40 @@ covers:
- `Caching a Model <#Caching-a-Model>`__
The notebook is divided into sections with headers. Each section is
standalone and does not depend on any previous sections except for the
next cell with imports. A segmentation and classification OpenVINO IR
model and a segmentation ONNX model are provided as examples. These
model files can be replaced with your own models. The exact outputs will
be different, but the process is the same.
The notebook is divided into sections with headers. The next cell
contains global requirements installation and imports. Each section is
standalone and does not depend on any previous sections. A segmentation
and classification OpenVINO IR model and a segmentation ONNX model are
provided as examples. These model files can be replaced with your own
models. The exact outputs will be different, but the process is the
same.
.. code:: ipython3
# Required imports. Please execute this cell first.
import sys
!pip install -q "openvino>=2023.0.0"
!pip install requests tqdm
# Fetch `notebook_utils` module
import urllib.request
urllib.request.urlretrieve(
url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py',
filename='notebook_utils.py'
)
sys.path.append('../utils')
from notebook_utils import download_file
.. parsed-literal::
Requirement already satisfied: requests in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (2.31.0)
Requirement already satisfied: tqdm in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (4.65.0)
Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (3.2.0)
Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (3.4)
Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (1.26.16)
Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests) (2023.5.7)
Loading OpenVINO Runtime and Showing Info
-----------------------------------------
@@ -52,13 +71,13 @@ Initialize OpenVINO Runtime with Core()
from openvino.runtime import Core
ie = Core()
core = Core()
OpenVINO Runtime can load a network on a device. A device in this
context means a CPU, an Intel GPU, a Neural Compute Stick 2, etc. The
``available_devices`` property shows the available devices in your
system. The “FULL_DEVICE_NAME” option to ``ie.get_property()`` shows the
name of the device.
system. The “FULL_DEVICE_NAME” option to ``core.get_property()`` shows
the name of the device.
In this notebook, the CPU device is used. To use an integrated GPU, use
``device_name="GPU"`` instead. Be aware that loading a network on GPU
@@ -67,10 +86,10 @@ be faster.
.. code:: ipython3
devices = ie.available_devices
devices = core.available_devices
for device in devices:
device_name = ie.get_property(device, "FULL_DEVICE_NAME")
device_name = core.get_property(device, "FULL_DEVICE_NAME")
print(f"{device}: {device_name}")
@@ -115,7 +134,7 @@ suitable for inference, for example, by alternating input shapes,
embedding preprocessing and cutting training parts off. For information
on how to convert your existing TensorFlow, PyTorch or ONNX model to
OpenVINO IR format with Model Optimizer, refer to the
`tensorflow-to-openvino <101-tensorflow-to-openvino-with-output.html>`__
`tensorflow-to-openvino <101-tensorflow-classification-to-openvino-with-output.html>`__
and
`pytorch-onnx-to-openvino <102-pytorch-onnx-to-openvino-with-output.html>`__
notebooks.
@@ -146,7 +165,7 @@ notebooks.
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
@@ -154,11 +173,11 @@ notebooks.
from openvino.runtime import Core
ie = Core()
core = Core()
classification_model_xml = "model/classification.xml"
model = ie.read_model(model=classification_model_xml)
compiled_model = ie.compile_model(model=model, device_name="CPU")
model = core.read_model(model=classification_model_xml)
compiled_model = core.compile_model(model=model, device_name="CPU")
ONNX Model
~~~~~~~~~~
@@ -193,7 +212,7 @@ points to the filename of an ONNX model.
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/segmentation.onnx')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/segmentation.onnx')
@@ -201,11 +220,11 @@ points to the filename of an ONNX model.
from openvino.runtime import Core
ie = Core()
core = Core()
onnx_model_path = "model/segmentation.onnx"
model_onnx = ie.read_model(model=onnx_model_path)
compiled_model_onnx = ie.compile_model(model=model_onnx, device_name="CPU")
model_onnx = core.read_model(model=onnx_model_path)
compiled_model_onnx = core.compile_model(model=model_onnx, device_name="CPU")
The ONNX model can be exported to OpenVINO IR with ``serialize()``:
@@ -249,7 +268,7 @@ without any conversion step. Pass the filename with extension to
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/inference.pdiparams')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/inference.pdiparams')
@@ -257,11 +276,11 @@ without any conversion step. Pass the filename with extension to
from openvino.runtime import Core
ie = Core()
core = Core()
paddle_model_path = 'model/inference.pdmodel'
model_paddle = ie.read_model(model=paddle_model_path)
compiled_model_paddle = ie.compile_model(model=model_paddle, device_name="CPU")
model_paddle = core.read_model(model=paddle_model_path)
compiled_model_paddle = core.compile_model(model=model_paddle, device_name="CPU")
.. code:: ipython3
@@ -273,13 +292,13 @@ TensorFlow Model
~~~~~~~~~~~~~~~~
TensorFlow models saved in frozen graph format can also be passed to
``read_model`` starting in OpenVINO 2022.3. > **NOTE**: Directly loading
TensorFlow models is available as a preview feature in the OpenVINO
2022.3 release. Fully functional support will be provided in the
upcoming 2023 releases. > Currently support is limited to only frozen
graph inference format. Other TensorFlow model formats must be converted
to OpenVINO IR using `Model
Optimizer <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow.html>`__.
``read_model`` starting in OpenVINO 2022.3.
.. note::
* Directly loading TensorFlow models is available as a preview feature in the OpenVINO 2022.3 release. Fully functional support will be provided in the upcoming 2023 releases.
* Currently support is limited to only frozen graph inference format. Other TensorFlow model formats must be converted to OpenVINO IR using `Model Optimizer <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow.html>`__.
.. code:: ipython3
@@ -299,7 +318,7 @@ Optimizer <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_con
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.pb')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.pb')
@@ -307,11 +326,11 @@ Optimizer <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_con
from openvino.runtime import Core
ie = Core()
core = Core()
tf_model_path = "model/classification.pb"
model_tf = ie.read_model(model=tf_model_path)
compiled_model_tf = ie.compile_model(model=model_tf, device_name="CPU")
model_tf = core.read_model(model=tf_model_path)
compiled_model_tf = core.compile_model(model=model_tf, device_name="CPU")
.. code:: ipython3
@@ -351,7 +370,7 @@ It is pre-trained model optimized to work with TensorFlow Lite.
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.tflite')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.tflite')
@@ -361,8 +380,8 @@ It is pre-trained model optimized to work with TensorFlow Lite.
ie = Core()
model_tflite = ie.read_model(tflite_model_path)
compiled_model_tflite = ie.compile_model(model=model_tflite, device_name="CPU")
model_tflite = core.read_model(tflite_model_path)
compiled_model_tflite = core.compile_model(model=model_tflite, device_name="CPU")
.. code:: ipython3
@@ -400,7 +419,7 @@ CompiledModel instance. While using ``model.inputs`` and
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
@@ -414,9 +433,9 @@ dictionary.
from openvino.runtime import Core
ie = Core()
core = Core()
classification_model_xml = "model/classification.xml"
model = ie.read_model(model=classification_model_xml)
model = core.read_model(model=classification_model_xml)
model.inputs
@@ -486,7 +505,7 @@ Model Outputs
ie = Core()
classification_model_xml = "model/classification.xml"
model = ie.read_model(model=classification_model_xml)
model = core.read_model(model=classification_model_xml)
model.outputs
@@ -566,6 +585,11 @@ data or list of input data in np.ndarray format, where the position of
the input tensor corresponds to input index. If a model has a single
input, wrapping to a dictionary or list can be omitted.
.. code:: ipython3
# Install opencv package for image handling
!pip install -q opencv-python
**Load the network**
.. code:: ipython3
@@ -588,7 +612,7 @@ input, wrapping to a dictionary or list can be omitted.
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
@@ -598,8 +622,8 @@ input, wrapping to a dictionary or list can be omitted.
ie = Core()
classification_model_xml = "model/classification.xml"
model = ie.read_model(model=classification_model_xml)
compiled_model = ie.compile_model(model=model, device_name="CPU")
model = core.read_model(model=classification_model_xml)
compiled_model = core.compile_model(model=model, device_name="CPU")
input_layer = compiled_model.input(0)
output_layer = compiled_model.output(0)
@@ -613,12 +637,21 @@ the input layout of the network.
import cv2
image_filename = "../data/image/coco_hollywood.jpg"
image = cv2.imread(image_filename)
image_filename = download_file(
"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_hollywood.jpg",
directory="data"
)
image = cv2.imread(str(image_filename))
image.shape
.. parsed-literal::
data/coco_hollywood.jpg: 0%| | 0.00/485k [00:00<?, ?B/s]
.. parsed-literal::
@@ -764,7 +797,7 @@ input shape.
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/segmentation.bin')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/segmentation.bin')
@@ -772,9 +805,9 @@ input shape.
from openvino.runtime import Core, PartialShape
ie = Core()
core = Core()
segmentation_model_xml = "model/segmentation.xml"
segmentation_model = ie.read_model(model=segmentation_model_xml)
segmentation_model = core.read_model(model=segmentation_model_xml)
segmentation_input_layer = segmentation_model.input(0)
segmentation_output_layer = segmentation_model.output(0)
@@ -784,7 +817,7 @@ input shape.
new_shape = PartialShape([1, 3, 544, 544])
segmentation_model.reshape({segmentation_input_layer.any_name: new_shape})
segmentation_compiled_model = ie.compile_model(model=segmentation_model, device_name="CPU")
segmentation_compiled_model = core.compile_model(model=segmentation_model, device_name="CPU")
# help(segmentation_compiled_model)
print("~~~~ RESHAPED MODEL ~~~~")
print(f"model input shape: {segmentation_input_layer.shape}")
@@ -824,13 +857,15 @@ set ``new_shape = (2,3,544,544)`` in the cell above.
.. code:: ipython3
from openvino.runtime import Core, PartialShape
segmentation_model_xml = "model/segmentation.xml"
segmentation_model = ie.read_model(model=segmentation_model_xml)
segmentation_model = core.read_model(model=segmentation_model_xml)
segmentation_input_layer = segmentation_model.input(0)
segmentation_output_layer = segmentation_model.output(0)
new_shape = PartialShape([2, 3, 544, 544])
segmentation_model.reshape({segmentation_input_layer.any_name: new_shape})
segmentation_compiled_model = ie.compile_model(model=segmentation_model, device_name="CPU")
segmentation_compiled_model = core.compile_model(model=segmentation_model, device_name="CPU")
print(f"input shape: {segmentation_input_layer.shape}")
print(f"output shape: {segmentation_output_layer.shape}")
@@ -851,14 +886,14 @@ input image through the network to see the result:
import numpy as np
from openvino.runtime import Core, PartialShape
ie = Core()
core = Core()
segmentation_model_xml = "model/segmentation.xml"
segmentation_model = ie.read_model(model=segmentation_model_xml)
segmentation_model = core.read_model(model=segmentation_model_xml)
segmentation_input_layer = segmentation_model.input(0)
segmentation_output_layer = segmentation_model.output(0)
new_shape = PartialShape([2, 3, 544, 544])
segmentation_model.reshape({segmentation_input_layer.any_name: new_shape})
segmentation_compiled_model = ie.compile_model(model=segmentation_model, device_name="CPU")
segmentation_compiled_model = core.compile_model(model=segmentation_model, device_name="CPU")
input_data = np.random.rand(2, 3, 544, 544)
output = segmentation_compiled_model([input_data])
@@ -878,7 +913,7 @@ Caching a Model
For some devices, like GPU, loading a model can take some time. Model
Caching solves this issue by caching the model in a cache directory. If
``ie.compile_model(model=net, device_name=device_name, config=config_dict)``
``core.compile_model(model=net, device_name=device_name, config=config_dict)``
is set, caching will be used. This option checks if a model exists in
the cache. If so, it loads it from the cache. If not, it loads the model
regularly, and stores it in the cache, so that the next time the model
@@ -913,7 +948,7 @@ the cache.
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
@@ -924,11 +959,11 @@ the cache.
from openvino.runtime import Core
ie = Core()
core = Core()
device_name = "GPU"
if device_name in ie.available_devices:
if device_name in core.available_devices:
cache_path = Path("model/model_cache")
cache_path.mkdir(exist_ok=True)
# Enable caching for OpenVINO Runtime. To disable caching set enable_caching = False
@@ -936,10 +971,10 @@ the cache.
config_dict = {"CACHE_DIR": str(cache_path)} if enable_caching else {}
classification_model_xml = "model/classification.xml"
model = ie.read_model(model=classification_model_xml)
model = core.read_model(model=classification_model_xml)
start_time = time.perf_counter()
compiled_model = ie.compile_model(model=model, device_name=device_name, config=config_dict)
compiled_model = core.compile_model(model=model, device_name=device_name, config=config_dict)
end_time = time.perf_counter()
print(f"Loading the network to the {device_name} device took {end_time-start_time:.2f} seconds.")
@@ -949,9 +984,9 @@ measure the time it takes now.
.. code:: ipython3
if device_name in ie.available_devices:
if device_name in core.available_devices:
del compiled_model
start_time = time.perf_counter()
compiled_model = ie.compile_model(model=model, device_name=device_name, config=config_dict)
compiled_model = core.compile_model(model=model, device_name=device_name, config=config_dict)
end_time = time.perf_counter()
print(f"Loading the network to the {device_name} device took {end_time-start_time:.2f} seconds.")

44
docs/notebooks/003-hello-segmentation-with-output.rst
View File

@@ -61,15 +61,43 @@ Download model weights
model/road-segmentation-adas-0001.bin: 0%| | 0.00/720k [00:00<?, ?B/s]
Select inference device
-----------------------
select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
ie = Core()
device = widgets.Dropdown(
options=ie.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
Load the Model
--------------
.. code:: ipython3
ie = Core()
core = Core()
model = ie.read_model(model=model_xml_path)
compiled_model = ie.compile_model(model=model, device_name="CPU")
model = core.read_model(model=model_xml_path)
compiled_model = core.compile_model(model=model, device_name=device.value)
input_layer_ir = compiled_model.input(0)
output_layer_ir = compiled_model.output(0)
@@ -106,12 +134,12 @@ provided.
.. parsed-literal::
<matplotlib.image.AxesImage at 0x7f40900e8ac0>
<matplotlib.image.AxesImage at 0x7f021436e0a0>
.. image:: 003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_8_1.png
.. image:: 003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_10_1.png
Do Inference
@@ -131,12 +159,12 @@ Do Inference
.. parsed-literal::
<matplotlib.image.AxesImage at 0x7f4090039a30>
<matplotlib.image.AxesImage at 0x7f02142bce50>
.. image:: 003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_10_1.png
.. image:: 003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_12_1.png
Prepare Data for Visualization
@@ -179,5 +207,5 @@ Visualize data
.. image:: 003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_14_0.png
.. image:: 003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_16_0.png

4
docs/notebooks/003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_10_1.png
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d9d40b840eea67857e42a78dd89b65799215e806e4f8d23b0523b0c163cb453b
size 20550
oid sha256:76113c575caa9c8a8aca45d3ec6ebd7a4b513dadffd8e9e63861a7a041d7e5de
size 249032

3
docs/notebooks/003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_12_1.png Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d9d40b840eea67857e42a78dd89b65799215e806e4f8d23b0523b0c163cb453b
size 20550

0
docs/notebooks/003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_14_0.png → docs/notebooks/003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_16_0.png
View File

3
docs/notebooks/003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_8_1.png
View File

@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:76113c575caa9c8a8aca45d3ec6ebd7a4b513dadffd8e9e63861a7a041d7e5de
size 249032

10
docs/notebooks/003-hello-segmentation-with-output_files/index.html
View File

@@ -1,9 +1,9 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/003-hello-segmentation-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/003-hello-segmentation-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/003-hello-segmentation-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="003-hello-segmentation-with-output_10_1.png">003-hello-segmentation-with-output_10_1.png</a> 22-Jun-2023 00:06 20550
<a href="003-hello-segmentation-with-output_14_0.png">003-hello-segmentation-with-output_14_0.png</a> 22-Jun-2023 00:06 260045
<a href="003-hello-segmentation-with-output_8_1.png">003-hello-segmentation-with-output_8_1.png</a> 22-Jun-2023 00:06 249032
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/003-hello-segmentation-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="003-hello-segmentation-with-output_10_1.png">003-hello-segmentation-with-output_10_1.png</a> 12-Jul-2023 00:11 249032
<a href="003-hello-segmentation-with-output_12_1.png">003-hello-segmentation-with-output_12_1.png</a> 12-Jul-2023 00:11 20550
<a href="003-hello-segmentation-with-output_16_0.png">003-hello-segmentation-with-output_16_0.png</a> 12-Jul-2023 00:11 260045
</pre><hr></body>
</html>

32
docs/notebooks/004-hello-detection-with-output.rst
View File

@@ -67,6 +67,34 @@ Download model weights
model/horizontal-text-detection-0001.bin: 0%| | 0.00/7.39M [00:00<?, ?B/s]
Select inference device
-----------------------
select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
ie = Core()
device = widgets.Dropdown(
options=ie.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
Load the Model
--------------
@@ -101,7 +129,7 @@ Load an Image
.. image:: 004-hello-detection-with-output_files/004-hello-detection-with-output_8_0.png
.. image:: 004-hello-detection-with-output_files/004-hello-detection-with-output_10_0.png
Do Inference
@@ -174,5 +202,5 @@ Visualize Results
.. image:: 004-hello-detection-with-output_files/004-hello-detection-with-output_13_0.png
.. image:: 004-hello-detection-with-output_files/004-hello-detection-with-output_15_0.png

0
docs/notebooks/004-hello-detection-with-output_files/004-hello-detection-with-output_8_0.png → docs/notebooks/004-hello-detection-with-output_files/004-hello-detection-with-output_10_0.png
View File

0
docs/notebooks/004-hello-detection-with-output_files/004-hello-detection-with-output_13_0.png → docs/notebooks/004-hello-detection-with-output_files/004-hello-detection-with-output_15_0.png
View File

8
docs/notebooks/004-hello-detection-with-output_files/index.html
View File

@@ -1,8 +1,8 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/004-hello-detection-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/004-hello-detection-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/004-hello-detection-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="004-hello-detection-with-output_13_0.png">004-hello-detection-with-output_13_0.png</a> 22-Jun-2023 00:06 457214
<a href="004-hello-detection-with-output_8_0.png">004-hello-detection-with-output_8_0.png</a> 22-Jun-2023 00:06 305482
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/004-hello-detection-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="004-hello-detection-with-output_10_0.png">004-hello-detection-with-output_10_0.png</a> 12-Jul-2023 00:11 305482
<a href="004-hello-detection-with-output_15_0.png">004-hello-detection-with-output_15_0.png</a> 12-Jul-2023 00:11 457214
</pre><hr></body>
</html>

63
docs/notebooks/101-tensorflow-to-openvino-with-output.rst → docs/notebooks/101-tensorflow-classification-to-openvino-with-output.rst
View File

@@ -29,10 +29,10 @@ Imports
.. parsed-literal::
2023-06-21 22:25:42.619122: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-06-21 22:25:42.653691: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2023-07-11 22:24:31.659014: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-07-11 22:24:31.694466: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-06-21 22:25:43.164083: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2023-07-11 22:24:32.210417: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
Settings
@@ -68,7 +68,7 @@ and save it to the disk.
.. parsed-literal::
2023-06-21 22:25:44.061927: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
2023-07-11 22:24:33.117393: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
@@ -79,9 +79,9 @@ and save it to the disk.
.. parsed-literal::
2023-06-21 22:25:48.238616: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'inputs' with dtype float and shape [?,1,1,1024]
2023-07-11 22:24:37.315661: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'inputs' with dtype float and shape [?,1,1,1024]
[[{{node inputs}}]]
2023-06-21 22:25:51.395055: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'inputs' with dtype float and shape [?,1,1,1024]
2023-07-11 22:24:40.473876: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'inputs' with dtype float and shape [?,1,1,1024]
[[{{node inputs}}]]
WARNING:absl:Found untraced functions such as _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op while saving (showing 5 of 54). These functions will not be directly callable after loading.
@@ -112,25 +112,20 @@ Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert
for more information about Model Optimizer and TensorFlow models
conversion.
.. code:: ipython3
# Construct the command for Model Optimizer.
ov_model = mo.convert_model(saved_model_dir=model_path, input_shape=[[1, 224, 224, 3]], compress_to_fp16=True)
serialize(ov_model, ir_path)
.. code:: ipython3
# Run Model Optimizer if the IR model file does not exist
if not ir_path.exists():
print("Exporting TensorFlow model to IR... This may take a few minutes.")
! $mo_command
ov_model = mo.convert_model(saved_model_dir=model_path, input_shape=[[1, 224, 224, 3]], compress_to_fp16=True)
serialize(ov_model, ir_path)
else:
print(f"IR model {ir_path} already exists.")
.. parsed-literal::
IR model model/v3-small_224_1.0_float.xml already exists.
Exporting TensorFlow model to IR... This may take a few minutes.
Test Inference on the Converted Model
@@ -141,9 +136,39 @@ Load the Model
.. code:: ipython3
ie = Core()
model = ie.read_model(ir_path)
compiled_model = ie.compile_model(model=model, device_name="CPU")
core = Core()
model = core.read_model(ir_path)
Select inference device
-----------------------
select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
.. code:: ipython3
compiled_model = core.compile_model(model=model, device_name=device.value)
Get Model Information
~~~~~~~~~~~~~~~~~~~~~
@@ -175,7 +200,7 @@ network.
.. image:: 101-tensorflow-to-openvino-with-output_files/101-tensorflow-to-openvino-with-output_16_0.png
.. image:: 101-tensorflow-classification-to-openvino-with-output_files/101-tensorflow-classification-to-openvino-with-output_18_0.png
Do Inference
@@ -233,5 +258,5 @@ performance.
.. parsed-literal::
IR model in OpenVINO Runtime/CPU: 0.0010 seconds per image, FPS: 1010.84
IR model in OpenVINO Runtime/CPU: 0.0010 seconds per image, FPS: 998.88

0
docs/notebooks/101-tensorflow-to-openvino-with-output_files/101-tensorflow-to-openvino-with-output_16_0.png → docs/notebooks/101-tensorflow-classification-to-openvino-with-output_files/101-tensorflow-classification-to-openvino-with-output_18_0.png
View File

7
docs/notebooks/101-tensorflow-classification-to-openvino-with-output_files/index.html Normal file
View File

@@ -0,0 +1,7 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/101-tensorflow-classification-to-openvino-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/101-tensorflow-classification-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="101-tensorflow-classification-to-openvino-with-output_18_0.png">101-tensorflow-classification-to-openvino-with-..&gt;</a> 12-Jul-2023 00:11 387941
</pre><hr></body>
</html>

7
docs/notebooks/101-tensorflow-to-openvino-with-output_files/index.html
View File

@@ -1,7 +0,0 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/101-tensorflow-to-openvino-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/101-tensorflow-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="101-tensorflow-to-openvino-with-output_16_0.png">101-tensorflow-to-openvino-with-output_16_0.png</a> 22-Jun-2023 00:06 387941
</pre><hr></body>
</html>

140
docs/notebooks/102-pytorch-onnx-to-openvino-with-output.rst
View File

@@ -49,7 +49,6 @@ Imports
import cv2
import numpy as np
import torch
from IPython.display import Markdown, display
from torchvision.models.segmentation import lraspp_mobilenet_v3_large, LRASPP_MobileNet_V3_Large_Weights
from openvino.runtime import Core
@@ -83,10 +82,11 @@ Load Model
Generally, PyTorch models represent an instance of ``torch.nn.Module``
class, initialized by a state dictionary with model weights. Typical
steps for getting a pre-trained model: 1. Create instance of model class
steps for getting a pre-trained model:
1. Create instance of model class
2. Load checkpoint state dict, which contains pre-trained model weights
3. Turn model to evaluation for switching some operations to inference
mode
3. Turn model to evaluation for switching some operations to inference mode
The ``torchvision`` module provides a ready to use set of functions for
model class initialization. We will use
@@ -181,39 +181,15 @@ For more information about Model Optimizer, see the `Model Optimizer
Developer
Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
Executing this command may take a while. There may be some errors or
warnings in the output. When Model Optimization is successful, the last
lines of the output will include:
``[ SUCCESS ] Generated IR version 11 model.``
.. code:: ipython3
# Construct the command for Model Optimizer.
mo_command = f"""mo
--input_model "{onnx_path}"
--compress_to_fp16
--output_dir "{ir_path.parent}"
"""
mo_command = " ".join(mo_command.split())
print("Model Optimizer command to convert the ONNX model to OpenVINO:")
display(Markdown(f"`{mo_command}`"))
.. parsed-literal::
Model Optimizer command to convert the ONNX model to OpenVINO:
``mo --input_model "model/lraspp_mobilenet_v3_large.onnx" --compress_to_fp16 --output_dir "model"``
.. code:: ipython3
from openvino.tools import mo
from openvino.runtime import serialize
if not ir_path.exists():
print("Exporting ONNX model to IR... This may take a few minutes.")
mo_result = %sx $mo_command
print("\n".join(mo_result))
ov_model = mo.convert_model(onnx_path, compress_to_fp16=True)
serialize(ov_model, ir_path)
else:
print(f"IR model {ir_path} already exists.")
@@ -221,13 +197,6 @@ lines of the output will include:
.. parsed-literal::
Exporting ONNX model to IR... This may take a few minutes.
[ INFO ] Generated IR will be compressed to FP16. If you get lower accuracy, please consider disabling compression by removing argument --compress_to_fp16 or set it to false --compress_to_fp16=False.
Find more information about compression to FP16 at https://docs.openvino.ai/latest/openvino_docs_MO_DG_FP16_Compression.html
[ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11.
Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/latest/openvino_2_0_transition_guide.html
[ SUCCESS ] Generated IR version 11 model.
[ SUCCESS ] XML file: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/102-pytorch-onnx-to-openvino/model/lraspp_mobilenet_v3_large.xml
[ SUCCESS ] BIN file: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/102-pytorch-onnx-to-openvino/model/lraspp_mobilenet_v3_large.bin
Show Results
@@ -282,15 +251,46 @@ an image.
.. code:: ipython3
# Load the network to OpenVINO Runtime.
ie = Core()
model_onnx = ie.read_model(model=onnx_path)
compiled_model_onnx = ie.compile_model(model=model_onnx, device_name="CPU")
# Instantiate OpenVINO Core
core = Core()
output_layer_onnx = compiled_model_onnx.output(0)
# Read model to OpenVINO Runtime
model_onnx = core.read_model(model=onnx_path)
# Run inference on the input image.
res_onnx = compiled_model_onnx([normalized_input_image])[output_layer_onnx]
Select inference device
^^^^^^^^^^^^^^^^^^^^^^^
select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
.. code:: ipython3
# Load model on device
compiled_model_onnx = core.compile_model(model=model_onnx, device_name=device.value)
# Run inference on the input image
res_onnx = compiled_model_onnx([normalized_input_image])[0]
Model predicts probabilities for how well each pixel corresponds to a
specific label. To get the label with highest probability for each
@@ -335,19 +335,37 @@ be applied to each label for more convenient visualization.
.. image:: 102-pytorch-onnx-to-openvino-with-output_files/102-pytorch-onnx-to-openvino-with-output_20_0.png
.. image:: 102-pytorch-onnx-to-openvino-with-output_files/102-pytorch-onnx-to-openvino-with-output_21_0.png
2. OpenVINO IR Model in OpenVINO Runtime
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Select inference device
^^^^^^^^^^^^^^^^^^^^^^^
select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
.. code:: ipython3
# Load the network in OpenVINO Runtime.
ie = Core()
model_ir = ie.read_model(model=ir_path)
compiled_model_ir = ie.compile_model(model=model_ir, device_name="CPU")
core = Core()
model_ir = core.read_model(model=ir_path)
compiled_model_ir = core.compile_model(model=model_ir, device_name=device.value)
# Get input and output layers.
output_layer_ir = compiled_model_ir.output(0)
@@ -367,7 +385,7 @@ be applied to each label for more convenient visualization.
.. image:: 102-pytorch-onnx-to-openvino-with-output_files/102-pytorch-onnx-to-openvino-with-output_23_0.png
.. image:: 102-pytorch-onnx-to-openvino-with-output_files/102-pytorch-onnx-to-openvino-with-output_26_0.png
@@ -393,7 +411,7 @@ looks the same as the output on the ONNX/OpenVINO IR models.
.. image:: 102-pytorch-onnx-to-openvino-with-output_files/102-pytorch-onnx-to-openvino-with-output_25_0.png
.. image:: 102-pytorch-onnx-to-openvino-with-output_files/102-pytorch-onnx-to-openvino-with-output_28_0.png
@@ -422,6 +440,7 @@ performance.
f"FPS: {num_images/time_torch:.2f}"
)
compiled_model_onnx = core.compile_model(model=model_onnx, device_name="CPU")
start = time.perf_counter()
for _ in range(num_images):
compiled_model_onnx([normalized_input_image])
@@ -432,6 +451,7 @@ performance.
f"seconds per image, FPS: {num_images/time_onnx:.2f}"
)
compiled_model_ir = core.compile_model(model=model_ir, device_name="CPU")
start = time.perf_counter()
for _ in range(num_images):
compiled_model_ir([input_image])
@@ -442,8 +462,8 @@ performance.
f"seconds per image, FPS: {num_images/time_ir:.2f}"
)
if "GPU" in ie.available_devices:
compiled_model_onnx_gpu = ie.compile_model(model=model_onnx, device_name="GPU")
if "GPU" in core.available_devices:
compiled_model_onnx_gpu = core.compile_model(model=model_onnx, device_name="GPU")
start = time.perf_counter()
for _ in range(num_images):
compiled_model_onnx_gpu([input_image])
@@ -454,7 +474,7 @@ performance.
f"seconds per image, FPS: {num_images/time_onnx_gpu:.2f}"
)
compiled_model_ir_gpu = ie.compile_model(model=model_ir, device_name="GPU")
compiled_model_ir_gpu = core.compile_model(model=model_ir, device_name="GPU")
start = time.perf_counter()
for _ in range(num_images):
compiled_model_ir_gpu([input_image])
@@ -468,18 +488,18 @@ performance.
.. parsed-literal::
PyTorch model on CPU: 0.037 seconds per image, FPS: 27.22
ONNX model in OpenVINO Runtime/CPU: 0.025 seconds per image, FPS: 39.64
OpenVINO IR model in OpenVINO Runtime/CPU: 0.031 seconds per image, FPS: 31.95
PyTorch model on CPU: 0.039 seconds per image, FPS: 25.80
ONNX model in OpenVINO Runtime/CPU: 0.031 seconds per image, FPS: 31.95
OpenVINO IR model in OpenVINO Runtime/CPU: 0.031 seconds per image, FPS: 32.67
**Show Device Information**
.. code:: ipython3
devices = ie.available_devices
devices = core.available_devices
for device in devices:
device_name = ie.get_property(device, "FULL_DEVICE_NAME")
device_name = core.get_property(device, "FULL_DEVICE_NAME")
print(f"{device}: {device_name}")

0
docs/notebooks/102-pytorch-onnx-to-openvino-with-output_files/102-pytorch-onnx-to-openvino-with-output_20_0.png → docs/notebooks/102-pytorch-onnx-to-openvino-with-output_files/102-pytorch-onnx-to-openvino-with-output_21_0.png
View File

0
docs/notebooks/102-pytorch-onnx-to-openvino-with-output_files/102-pytorch-onnx-to-openvino-with-output_23_0.png → docs/notebooks/102-pytorch-onnx-to-openvino-with-output_files/102-pytorch-onnx-to-openvino-with-output_26_0.png
View File

0
docs/notebooks/102-pytorch-onnx-to-openvino-with-output_files/102-pytorch-onnx-to-openvino-with-output_25_0.png → docs/notebooks/102-pytorch-onnx-to-openvino-with-output_files/102-pytorch-onnx-to-openvino-with-output_28_0.png
View File

10
docs/notebooks/102-pytorch-onnx-to-openvino-with-output_files/index.html
View File

@@ -1,9 +1,9 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/102-pytorch-onnx-to-openvino-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/102-pytorch-onnx-to-openvino-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/102-pytorch-onnx-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="102-pytorch-onnx-to-openvino-with-output_20_0.png">102-pytorch-onnx-to-openvino-with-output_20_0.png</a> 22-Jun-2023 00:06 465692
<a href="102-pytorch-onnx-to-openvino-with-output_23_0.png">102-pytorch-onnx-to-openvino-with-output_23_0.png</a> 22-Jun-2023 00:06 465695
<a href="102-pytorch-onnx-to-openvino-with-output_25_0.png">102-pytorch-onnx-to-openvino-with-output_25_0.png</a> 22-Jun-2023 00:06 465692
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/102-pytorch-onnx-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="102-pytorch-onnx-to-openvino-with-output_21_0.png">102-pytorch-onnx-to-openvino-with-output_21_0.png</a> 12-Jul-2023 00:11 465692
<a href="102-pytorch-onnx-to-openvino-with-output_26_0.png">102-pytorch-onnx-to-openvino-with-output_26_0.png</a> 12-Jul-2023 00:11 465695
<a href="102-pytorch-onnx-to-openvino-with-output_28_0.png">102-pytorch-onnx-to-openvino-with-output_28_0.png</a> 12-Jul-2023 00:11 465692
</pre><hr></body>
</html>

738
docs/notebooks/102-pytorch-to-openvino-with-output.rst Normal file
View File

@@ -0,0 +1,738 @@
Convert a PyTorch Model to OpenVINO™ IR
=======================================
This tutorial demonstrates step-by-step instructions on how to do
inference on a PyTorch classification model using OpenVINO Runtime.
Starting from OpenVINO 2023.0 release, OpenVINO supports direct PyTorch
model conversion without an intermediate step to convert them into ONNX
format. In order, if you try to use the lower OpenVINO version or prefer
to use ONNX, please check this
`tutorial <102-pytorch-to-openvino-with-output.html>`__.
In this tutorial, we will use the
`RegNetY_800MF <https://arxiv.org/abs/2003.13678>`__ model from
`torchvision <https://pytorch.org/vision/stable/index.html>`__ to
demonstrate how to convert PyTorch models to OpenVINO Intermediate
Representation.
The RegNet model was proposed in `Designing Network Design
Spaces <https://arxiv.org/abs/2003.13678>`__ by Ilija Radosavovic, Raj
Prateek Kosaraju, Ross Girshick, Kaiming He, Piotr Dollár. The authors
design search spaces to perform Neural Architecture Search (NAS). They
first start from a high dimensional search space and iteratively reduce
the search space by empirically applying constraints based on the
best-performing models sampled by the current search space. Instead of
focusing on designing individual network instances, authors design
network design spaces that parametrize populations of networks. The
overall process is analogous to the classic manual design of networks
but elevated to the design space level. The RegNet design space provides
simple and fast networks that work well across a wide range of flop
regimes.
Prerequisites
-------------
Install notebook dependecies
.. code:: ipython3
!pip install -q "openvino-dev>=2023.0.0"
Download input data and label map
.. code:: ipython3
import requests
from pathlib import Path
from PIL import Image
MODEL_DIR = Path("model")
DATA_DIR = Path("data")
MODEL_DIR.mkdir(exist_ok=True)
DATA_DIR.mkdir(exist_ok=True)
MODEL_NAME = "regnet_y_800mf"
image = Image.open(requests.get("https://farm9.staticflickr.com/8225/8511402100_fea15da1c5_z.jpg", stream=True).raw)
labels_file = DATA_DIR / "imagenet_2012.txt"
if not labels_file.exists():
resp = requests.get("https://raw.githubusercontent.com/openvinotoolkit/open_model_zoo/master/data/dataset_classes/imagenet_2012.txt")
with labels_file.open("wb") as f:
f.write(resp.content)
imagenet_classes = labels_file.open("r").read().splitlines()
Load PyTorch Model
------------------
Generally, PyTorch models represent an instance of the
``torch.nn.Module`` class, initialized by a state dictionary with model
weights. Typical steps for getting a pre-trained model:
1. Create an instance of a model class
2. Load checkpoint state dict, which contains pre-trained model weights
3. Turn the model to evaluation for switching some operations to inference mode
The ``torchvision`` module provides a ready-to-use set of functions for
model class initialization. We will use
``torchvision.models.regnet_y_800mf``. You can directly pass pre-trained
model weights to the model initialization function using the weights
enum ``RegNet_Y_800MF_Weights.DEFAULT``.
.. code:: ipython3
import torchvision
# get default weights using available weights Enum for model
weights = torchvision.models.RegNet_Y_800MF_Weights.DEFAULT
# create model topology and load weights
model = torchvision.models.regnet_y_800mf(weights=weights)
# switch model to inference mode
model.eval();
Prepare Input Data
~~~~~~~~~~~~~~~~~~
The code below demonstrates how to preprocess input data using a
model-specific transforms module from ``torchvision``. After
transformation, we should concatenate images into batched tensor, in our
case, we will run the model with batch 1, so we just unsqueeze input on
the first dimension.
.. code:: ipython3
import torch
# Initialize the Weight Transforms
preprocess = weights.transforms()
# Apply it to the input image
img_transformed = preprocess(image)
# Add batch dimension to image tensor
input_tensor = img_transformed.unsqueeze(0)
Run PyTorch Model Inference
~~~~~~~~~~~~~~~~~~~~~~~~~~~
The model returns a vector of probabilities in raw logits format,
softmax can be applied to get normalized values in the [0, 1] range. For
a demonstration that the output of the original model and OpenVINO
converted is the same, we defined a common postprocessing function which
can be reused later.
.. code:: ipython3
import numpy as np
from scipy.special import softmax
# Perform model inference on input tensor
result = model(input_tensor)
# Postprocessing function for getting results in the same way for both PyTorch model inference and OpenVINO
def postprocess_result(output_tensor:np.ndarray, top_k:int = 5):
"""
Posprocess model results. This function applied sofrmax on output tensor and returns specified top_k number of labels with highest probability
Parameters:
output_tensor (np.ndarray): model output tensor with probabilities
top_k (int, *optional*, default 5): number of labels with highest probability for return
Returns:
topk_labels: label ids for selected top_k scores
topk_scores: selected top_k highest scores predicted by model
"""
softmaxed_scores = softmax(output_tensor, -1)[0]
topk_labels = np.argsort(softmaxed_scores)[-top_k:][::-1]
topk_scores = softmaxed_scores[topk_labels]
return topk_labels, topk_scores
# Postprocess results
top_labels, top_scores = postprocess_result(result.detach().numpy())
# Show results
display(image)
for idx, (label, score) in enumerate(zip(top_labels, top_scores)):
_, predicted_label = imagenet_classes[label].split(" ", 1)
print(f"{idx + 1}: {predicted_label} - {score * 100 :.2f}%")
.. image:: 102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_11_0.png
.. parsed-literal::
1: tiger cat - 25.91%
2: Egyptian cat - 10.26%
3: computer keyboard, keypad - 9.22%
4: tabby, tabby cat - 9.09%
5: hamper - 2.35%
Benchmark PyTorch Model Inference
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. code:: ipython3
%%timeit
# Run model inference
model(input_tensor)
.. parsed-literal::
13.5 ms ± 3.76 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
Convert PyTorch Model to OpenVINO Intermediate Representation
-------------------------------------------------------------
Starting from the 2023.0 release OpenVINO supports direct PyTorch models
conversion to OpenVINO Intermediate Representation (IR) format. Model
Optimizer Python API should be used for these purposes. More details
regarding PyTorch model conversion can be found in OpenVINO
`documentation <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_PyTorch.html>`__
**Note**: Please, take into account that direct support PyTorch
models conversion is an experimental feature. Model coverage will be
increased in the next releases. For cases, when PyTorch model
conversion failed, you still can try to export the model to ONNX
format. Please refer to this
`tutorial <102-pytorch-to-openvino-with-output.html>`__
which explains how to convert PyTorch model to ONNX, then to OpenVINO
The ``convert_model`` function accepts the PyTorch model object and
returns the ``openvino.runtime.Model`` instance ready to load on a
device using ``core.compile_model`` or save on disk for next usage using
``openvino.runtime.serialize``. Optionally, we can provide additional
parameters, such as:
* ``compress_to_fp16`` - flag to perform model weights compression into FP16 data format. It may reduce the required space for model storage on disk and give speedup for inference devices, where FP16 calculation is supported.
* ``example_input`` - input data sample which can be used for model tracing.
* ``input_shape`` - the shape of input tensor for conversion
and any other advanced options supported by Model Optimizer Python API.
More details can be found on this
`page <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Python_API.html>`__
.. code:: ipython3
from openvino.tools import mo
from openvino.runtime import Core, serialize
# Create OpenVINO Core object instance
core = Core()
# Convert model to openvino.runtime.Model object
ov_model = mo.convert_model(model)
# Save openvino.runtime.Model object on disk
serialize(ov_model, MODEL_DIR / f"{MODEL_NAME}_dynamic.xml")
ov_model
.. parsed-literal::
<Model: 'Model30'
inputs[
<ConstOutput: names[x, x.1, 1] shape[?,3,?,?] type: f32>
]
outputs[
<ConstOutput: names[x.21, 401] shape[?,1000] type: f32>
]>
Select inference device
~~~~~~~~~~~~~~~~~~~~~~~
select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
.. code:: ipython3
# Load OpenVINO model on device
compiled_model = core.compile_model(ov_model, device.value)
compiled_model
.. parsed-literal::
<CompiledModel:
inputs[
<ConstOutput: names[x, x.1, 1] shape[?,3,?,?] type: f32>
]
outputs[
<ConstOutput: names[x.21, 401] shape[?,1000] type: f32>
]>
Run OpenVINO Model Inference
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. code:: ipython3
# Run model inference
result = compiled_model(input_tensor)[0]
# Posptorcess results
top_labels, top_scores = postprocess_result(result)
# Show results
display(image)
for idx, (label, score) in enumerate(zip(top_labels, top_scores)):
_, predicted_label = imagenet_classes[label].split(" ", 1)
print(f"{idx + 1}: {predicted_label} - {score * 100 :.2f}%")
.. image:: 102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_20_0.png
.. parsed-literal::
1: tiger cat - 25.91%
2: Egyptian cat - 10.26%
3: computer keyboard, keypad - 9.22%
4: tabby, tabby cat - 9.09%
5: hamper - 2.35%
Benchmark OpenVINO Model Inference
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. code:: ipython3
%%timeit
compiled_model(input_tensor)
.. parsed-literal::
3.03 ms ± 5.57 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
Convert PyTorch Model with Static Input Shape
---------------------------------------------
The default conversion path preserves dynamic input shapes, in order if
you want to convert the model with static shapes, you can explicitly
specify it during conversion using the ``input_shape`` parameter or
reshape the model into the desired shape after conversion. For the model
reshaping example please check the following
`tutorial <002-openvino-api-with-output.html>`__.
.. code:: ipython3
# Convert model to openvino.runtime.Model object
ov_model = mo.convert_model(model, input_shape=[[1,3,224,224]])
# Save openvino.runtime.Model object on disk
serialize(ov_model, MODEL_DIR / f"{MODEL_NAME}_static.xml")
ov_model
.. parsed-literal::
<Model: 'Model38'
inputs[
<ConstOutput: names[x, x.1, 1] shape[1,3,224,224] type: f32>
]
outputs[
<ConstOutput: names[355] shape[1,1000] type: f32>
]>
Select inference device
~~~~~~~~~~~~~~~~~~~~~~~
select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
.. code:: ipython3
# Load OpenVINO model on device
compiled_model = core.compile_model(ov_model, device.value)
compiled_model
.. parsed-literal::
<CompiledModel:
inputs[
<ConstOutput: names[x, x.1, 1] shape[1,3,224,224] type: f32>
]
outputs[
<ConstOutput: names[355] shape[1,1000] type: f32>
]>
Now, we can see that input of our converted model is tensor of shape [1,
3, 224, 224] instead of [?, 3, ?, ?] reported by previously converted
model.
Run OpenVINO Model Inference with Static Input Shape
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. code:: ipython3
# Run model inference
result = compiled_model(input_tensor)[0]
# Posptorcess results
top_labels, top_scores = postprocess_result(result)
# Show results
display(image)
for idx, (label, score) in enumerate(zip(top_labels, top_scores)):
_, predicted_label = imagenet_classes[label].split(" ", 1)
print(f"{idx + 1}: {predicted_label} - {score * 100 :.2f}%")
.. image:: 102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_31_0.png
.. parsed-literal::
1: tiger cat - 25.91%
2: Egyptian cat - 10.26%
3: computer keyboard, keypad - 9.22%
4: tabby, tabby cat - 9.09%
5: hamper - 2.35%
Benchmark OpenVINO Model Inference with Static Input Shape
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. code:: ipython3
%%timeit
compiled_model(input_tensor)
.. parsed-literal::
2.79 ms ± 26.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
Convert TorchScript Model to OpenVINO Intermediate Representation
-----------------------------------------------------------------
TorchScript is a way to create serializable and optimizable models from
PyTorch code. Any TorchScript program can be saved from a Python process
and loaded in a process where there is no Python dependency. More
details about TorchScript can be found in `PyTorch
documentation <https://pytorch.org/docs/stable/jit.html>`__.
There are 2 possible ways to convert the PyTorch model to TorchScript:
- ``torch.jit.script`` - Scripting a function or ``nn.Module`` will
inspect the source code, compile it as TorchScript code using the
TorchScript compiler, and return a ``ScriptModule`` or
``ScriptFunction``.
- ``torch.jit.trace`` - Trace a function and return an executable or
``ScriptFunction`` that will be optimized using just-in-time
compilation.
Let’s consider both approaches and their conversion into OpenVINO IR.
Scriped Model
~~~~~~~~~~~~~
``torch.jit.script`` inspects model source code and compiles it to
``ScriptModule``. After compilation model can be used for inference or
saved on disk using the ``torch.jit.save`` function and after that
restored with ``torch.jit.load`` in any other environment without the
original PyTorch model code definitions.
TorchScript itself is a subset of the Python language, so not all
features in Python work, but TorchScript provides enough functionality
to compute on tensors and do control-dependent operations. For a
complete guide, see the `TorchScript Language
Reference <https://pytorch.org/docs/stable/jit_language_reference.html#language-reference>`__.
.. code:: ipython3
# Get model path
scripted_model_path = MODEL_DIR / f"{MODEL_NAME}_scripted.pth"
# Compile and save model if it has not been compiled before or load compiled model
if not scripted_model_path.exists():
scripted_model = torch.jit.script(model)
torch.jit.save(scripted_model, scripted_model_path)
else:
scripted_model = torch.jit.load(scripted_model_path)
# Run scripted model inference
result = scripted_model(input_tensor)
# Postprocess results
top_labels, top_scores = postprocess_result(result.detach().numpy())
# Show results
display(image)
for idx, (label, score) in enumerate(zip(top_labels, top_scores)):
_, predicted_label = imagenet_classes[label].split(" ", 1)
print(f"{idx + 1}: {predicted_label} - {score * 100 :.2f}%")
.. image:: 102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_35_0.png
.. parsed-literal::
1: tiger cat - 25.91%
2: Egyptian cat - 10.26%
3: computer keyboard, keypad - 9.22%
4: tabby, tabby cat - 9.09%
5: hamper - 2.35%
Benchmark Scripted Model Inference
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. code:: ipython3
%%timeit
scripted_model(input_tensor)
.. parsed-literal::
12.7 ms ± 13.4 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)
Convert PyTorch Scripted Model to OpenVINO Intermediate Representation
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The conversion step for the scripted model to OpenVINO IR is similar to
the original PyTorch model.
.. code:: ipython3
# Convert model to openvino.runtime.Model object
ov_model = mo.convert_model(scripted_model)
# Load OpenVINO model on device
compiled_model = core.compile_model(ov_model, device.value)
# Run OpenVINO model inference
result = compiled_model(input_tensor, device.value)[0]
# Postprocess results
top_labels, top_scores = postprocess_result(result)
# Show results
display(image)
for idx, (label, score) in enumerate(zip(top_labels, top_scores)):
_, predicted_label = imagenet_classes[label].split(" ", 1)
print(f"{idx + 1}: {predicted_label} - {score * 100 :.2f}%")
.. image:: 102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_39_0.png
.. parsed-literal::
1: tiger cat - 25.91%
2: Egyptian cat - 10.26%
3: computer keyboard, keypad - 9.22%
4: tabby, tabby cat - 9.09%
5: hamper - 2.35%
Benchmark OpenVINO Model Inference Converted From Scripted Model
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. code:: ipython3
%%timeit
compiled_model(input_tensor)
.. parsed-literal::
3.1 ms ± 4.02 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
Traced Model
~~~~~~~~~~~~
Using ``torch.jit.trace``, you can turn an existing module or Python
function into a TorchScript ``ScriptFunction`` or ``ScriptModule``. You
must provide example inputs, and model will be executed, recording the
operations performed on all the tensors.
- The resulting recording of a standalone function produces
ScriptFunction.
- The resulting recording of nn.Module.forward or nn.Module produces
ScriptModule.
In the same way like scripted model, traced model can be used for
inference or saved on disk using ``torch.jit.save`` function and after
that restored with ``torch.jit.load`` in any other environment without
original PyTorch model code definitions.
.. code:: ipython3
# Get model path
traced_model_path = MODEL_DIR / f"{MODEL_NAME}_traced.pth"
# Trace and save model if it has not been traced before or load traced model
if not traced_model_path.exists():
traced_model = torch.jit.trace(model, example_inputs=input_tensor)
torch.jit.save(traced_model, traced_model_path)
else:
traced_model = torch.jit.load(traced_model_path)
# Run traced model inference
result = traced_model(input_tensor)
# Postprocess results
top_labels, top_scores = postprocess_result(result.detach().numpy())
# Show results
display(image)
for idx, (label, score) in enumerate(zip(top_labels, top_scores)):
_, predicted_label = imagenet_classes[label].split(" ", 1)
print(f"{idx + 1}: {predicted_label} - {score * 100 :.2f}%")
.. image:: 102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_43_0.png
.. parsed-literal::
1: tiger cat - 25.91%
2: Egyptian cat - 10.26%
3: computer keyboard, keypad - 9.22%
4: tabby, tabby cat - 9.09%
5: hamper - 2.35%
Benchmark Traced Model Inference
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. code:: ipython3
%%timeit
traced_model(input_tensor)
.. parsed-literal::
12.7 ms ± 39.6 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)
Convert PyTorch Traced Model to OpenVINO Intermediate Representation
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The conversion step for a traced model to OpenVINO IR is similar to the
original PyTorch model.
.. code:: ipython3
# Convert model to openvino.runtime.Model object
ov_model = mo.convert_model(traced_model)
# Load OpenVINO model on device
compiled_model = core.compile_model(ov_model, device.value)
# Run OpenVINO model inference
result = compiled_model(input_tensor)[0]
# Postprocess results
top_labels, top_scores = postprocess_result(result)
# Show results
display(image)
for idx, (label, score) in enumerate(zip(top_labels, top_scores)):
_, predicted_label = imagenet_classes[label].split(" ", 1)
print(f"{idx + 1}: {predicted_label} - {score * 100 :.2f}%")
.. image:: 102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_47_0.png
.. parsed-literal::
1: tiger cat - 25.91%
2: Egyptian cat - 10.26%
3: computer keyboard, keypad - 9.22%
4: tabby, tabby cat - 9.09%
5: hamper - 2.35%
Benchmark OpenVINO Model Inference Converted From Traced Model
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. code:: ipython3
%%timeit
compiled_model(input_tensor)[0]
.. parsed-literal::
3.08 ms ± 9.07 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)

3
docs/notebooks/102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_11_0.jpg Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:723665126ea8a89da96968eb9e069f401f5842819f56ccc6dfee899098bc79be
size 54874

3
docs/notebooks/102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_11_0.png Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:dacfb67af6d9ec0554aaa658a1d7b831d1445404397f743f4712eb3187980e31
size 542516

3
docs/notebooks/102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_20_0.jpg Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:723665126ea8a89da96968eb9e069f401f5842819f56ccc6dfee899098bc79be
size 54874

3
docs/notebooks/102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_20_0.png Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:dacfb67af6d9ec0554aaa658a1d7b831d1445404397f743f4712eb3187980e31
size 542516

3
docs/notebooks/102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_31_0.jpg Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:723665126ea8a89da96968eb9e069f401f5842819f56ccc6dfee899098bc79be
size 54874

3
docs/notebooks/102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_31_0.png Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:dacfb67af6d9ec0554aaa658a1d7b831d1445404397f743f4712eb3187980e31
size 542516

3
docs/notebooks/102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_35_0.jpg Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:723665126ea8a89da96968eb9e069f401f5842819f56ccc6dfee899098bc79be
size 54874

3
docs/notebooks/102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_35_0.png Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:dacfb67af6d9ec0554aaa658a1d7b831d1445404397f743f4712eb3187980e31
size 542516

3
docs/notebooks/102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_39_0.jpg Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:723665126ea8a89da96968eb9e069f401f5842819f56ccc6dfee899098bc79be
size 54874

3
docs/notebooks/102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_39_0.png Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:dacfb67af6d9ec0554aaa658a1d7b831d1445404397f743f4712eb3187980e31
size 542516

3
docs/notebooks/102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_43_0.jpg Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:723665126ea8a89da96968eb9e069f401f5842819f56ccc6dfee899098bc79be
size 54874

3
docs/notebooks/102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_43_0.png Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:dacfb67af6d9ec0554aaa658a1d7b831d1445404397f743f4712eb3187980e31
size 542516

3
docs/notebooks/102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_47_0.jpg Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:723665126ea8a89da96968eb9e069f401f5842819f56ccc6dfee899098bc79be
size 54874

3
docs/notebooks/102-pytorch-to-openvino-with-output_files/102-pytorch-to-openvino-with-output_47_0.png Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:dacfb67af6d9ec0554aaa658a1d7b831d1445404397f743f4712eb3187980e31
size 542516

20
docs/notebooks/102-pytorch-to-openvino-with-output_files/index.html Normal file
View File

@@ -0,0 +1,20 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/102-pytorch-to-openvino-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/102-pytorch-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="102-pytorch-to-openvino-with-output_11_0.jpg">102-pytorch-to-openvino-with-output_11_0.jpg</a> 12-Jul-2023 00:11 54874
<a href="102-pytorch-to-openvino-with-output_11_0.png">102-pytorch-to-openvino-with-output_11_0.png</a> 12-Jul-2023 00:11 542516
<a href="102-pytorch-to-openvino-with-output_20_0.jpg">102-pytorch-to-openvino-with-output_20_0.jpg</a> 12-Jul-2023 00:11 54874
<a href="102-pytorch-to-openvino-with-output_20_0.png">102-pytorch-to-openvino-with-output_20_0.png</a> 12-Jul-2023 00:11 542516
<a href="102-pytorch-to-openvino-with-output_31_0.jpg">102-pytorch-to-openvino-with-output_31_0.jpg</a> 12-Jul-2023 00:11 54874
<a href="102-pytorch-to-openvino-with-output_31_0.png">102-pytorch-to-openvino-with-output_31_0.png</a> 12-Jul-2023 00:11 542516
<a href="102-pytorch-to-openvino-with-output_35_0.jpg">102-pytorch-to-openvino-with-output_35_0.jpg</a> 12-Jul-2023 00:11 54874
<a href="102-pytorch-to-openvino-with-output_35_0.png">102-pytorch-to-openvino-with-output_35_0.png</a> 12-Jul-2023 00:11 542516
<a href="102-pytorch-to-openvino-with-output_39_0.jpg">102-pytorch-to-openvino-with-output_39_0.jpg</a> 12-Jul-2023 00:11 54874
<a href="102-pytorch-to-openvino-with-output_39_0.png">102-pytorch-to-openvino-with-output_39_0.png</a> 12-Jul-2023 00:11 542516
<a href="102-pytorch-to-openvino-with-output_43_0.jpg">102-pytorch-to-openvino-with-output_43_0.jpg</a> 12-Jul-2023 00:11 54874
<a href="102-pytorch-to-openvino-with-output_43_0.png">102-pytorch-to-openvino-with-output_43_0.png</a> 12-Jul-2023 00:11 542516
<a href="102-pytorch-to-openvino-with-output_47_0.jpg">102-pytorch-to-openvino-with-output_47_0.jpg</a> 12-Jul-2023 00:11 54874
<a href="102-pytorch-to-openvino-with-output_47_0.png">102-pytorch-to-openvino-with-output_47_0.png</a> 12-Jul-2023 00:11 542516
</pre><hr></body>
</html>

105
docs/notebooks/103-paddle-to-openvino-classification-with-output.rst
View File

@@ -25,7 +25,7 @@ Imports
import sys
if sys.version_info.minor > 7:
!pip install -q "paddlepaddle==2.5.0rc0"
!pip install -q "paddlepaddle>=2.5.0"
else:
!pip install -q "paddlepaddle==2.4.2"
@@ -61,8 +61,8 @@ Imports
.. parsed-literal::
2023-06-21 22:26:35 INFO: Loading faiss with AVX2 support.
2023-06-21 22:26:35 INFO: Successfully loaded faiss with AVX2 support.
2023-07-11 22:26:05 INFO: Loading faiss with AVX2 support.
2023-07-11 22:26:05 INFO: Successfully loaded faiss with AVX2 support.
Settings
@@ -130,16 +130,7 @@ inference on that image, and then show the top three prediction results.
.. parsed-literal::
[2023/06/21 22:27:03] ppcls WARNING: The current running environment does not support the use of GPU. CPU has been used instead.
.. parsed-literal::
W0621 22:27:03.734274 1205526 analysis_config.cc:971] It is detected that mkldnn and memory_optimize_pass are enabled at the same time, but they are not supported yet. Currently, memory_optimize_pass is explicitly disabled
.. parsed-literal::
[2023/07/11 22:26:25] ppcls WARNING: The current running environment does not support the use of GPU. CPU has been used instead.
Labrador retriever, 0.75138
German short-haired pointer, 0.02373
Great Dane, 0.01848
@@ -148,7 +139,7 @@ inference on that image, and then show the top three prediction results.
.. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_8_3.png
.. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_8_1.png
``classifier.predict()`` takes an image file name, reads the image,
@@ -205,7 +196,7 @@ values.
.. parsed-literal::
2023-06-21 22:27:04 WARNING: Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
2023-07-11 22:26:25 WARNING: Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
.. parsed-literal::
@@ -217,7 +208,7 @@ values.
.. parsed-literal::
<matplotlib.image.AxesImage at 0x7fedb78da400>
<matplotlib.image.AxesImage at 0x7fd87c6ee3a0>
@@ -265,6 +256,34 @@ for more information about Model Optimizer.
else:
print(f"{model_xml} already exists.")
Select inference device
-----------------------
select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
ie = Core()
device = widgets.Dropdown(
options=ie.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
Show Inference on OpenVINO Model
--------------------------------
@@ -277,9 +296,9 @@ information.
.. code:: ipython3
# Load OpenVINO Runtime and OpenVINO IR model
ie = Core()
model = ie.read_model(model_xml)
compiled_model = ie.compile_model(model=model, device_name="CPU")
core = Core()
model = core.read_model(model_xml)
compiled_model = core.compile_model(model=model, device_name="CPU")
# Get model output
output_layer = compiled_model.output(0)
@@ -291,11 +310,11 @@ information.
input_image = process_image(np.array(image))[None,]
# Do inference
ie_result = compiled_model([input_image])[output_layer][0]
ov_result = compiled_model([input_image])[output_layer][0]
# find the top three values
top_indices = np.argsort(ie_result)[-3:][::-1]
top_scores = ie_result[top_indices]
top_indices = np.argsort(ov_result)[-3:][::-1]
top_scores = ov_result[top_indices]
# Convert the inference results to class names, using the same labels as the PaddlePaddle classifier
for index, softmax_probability in zip(top_indices, top_scores):
@@ -310,7 +329,7 @@ information.
.. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_21_1.png
.. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_23_1.png
Timing and Comparison
@@ -331,9 +350,13 @@ Note that many optimizations are possible to improve the performance.
.. code:: ipython3
# Show CPU information
ie = Core()
print(f"CPU: {ie.get_property('CPU', 'FULL_DEVICE_NAME')}")
# Show device information
core = Core()
devices = core.available_devices
for device_name in devices:
device_full_name = core.get_property(device_name, "FULL_DEVICE_NAME")
print(f"{device_name}: {device_full_name}")
.. parsed-literal::
@@ -363,7 +386,7 @@ Note that many optimizations are possible to improve the performance.
.. parsed-literal::
PaddlePaddle model on CPU: 0.0073 seconds per image, FPS: 137.70
PaddlePaddle model on CPU: 0.0074 seconds per image, FPS: 135.48
PaddlePaddle result:
Labrador retriever, 0.75138
@@ -374,13 +397,31 @@ Note that many optimizations are possible to improve the performance.
.. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_25_1.png
.. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_27_1.png
Select inference device
-----------------------
select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
.. code:: ipython3
# Show inference speed on OpenVINO IR model
compiled_model = ie.compile_model(model=model, device_name="CPU")
compiled_model = ie.compile_model(model=model, device_name=device.value)
output_layer = compiled_model.output(0)
@@ -395,7 +436,7 @@ Note that many optimizations are possible to improve the performance.
time_ir = end - start
print(
f"OpenVINO IR model in OpenVINO Runtime (CPU): {time_ir/num_images:.4f} "
f"OpenVINO IR model in OpenVINO Runtime ({device.value}): {time_ir/num_images:.4f} "
f"seconds per image, FPS: {num_images/time_ir:.2f}"
)
print()
@@ -407,7 +448,7 @@ Note that many optimizations are possible to improve the performance.
.. parsed-literal::
OpenVINO IR model in OpenVINO Runtime (CPU): 0.0027 seconds per image, FPS: 364.96
OpenVINO IR model in OpenVINO Runtime (AUTO): 0.0031 seconds per image, FPS: 326.50
OpenVINO result:
Labrador retriever, 0.75138
@@ -418,7 +459,7 @@ Note that many optimizations are possible to improve the performance.
.. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_26_1.png
.. image:: 103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_30_1.png
References

0
docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_21_1.png → docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_23_1.png
View File

0
docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_25_1.png → docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_27_1.png
View File

0
docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_26_1.png → docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_30_1.png
View File

0
docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_8_3.png → docs/notebooks/103-paddle-to-openvino-classification-with-output_files/103-paddle-to-openvino-classification-with-output_8_1.png
View File

14
docs/notebooks/103-paddle-to-openvino-classification-with-output_files/index.html
View File

@@ -1,11 +1,11 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/103-paddle-to-openvino-classification-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/103-paddle-to-openvino-classification-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/103-paddle-to-openvino-classification-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="103-paddle-to-openvino-classification-with-output_15_3.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 22-Jun-2023 00:06 120883
<a href="103-paddle-to-openvino-classification-with-output_21_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 22-Jun-2023 00:06 224886
<a href="103-paddle-to-openvino-classification-with-output_25_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 22-Jun-2023 00:06 224886
<a href="103-paddle-to-openvino-classification-with-output_26_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 22-Jun-2023 00:06 224886
<a href="103-paddle-to-openvino-classification-with-output_8_3.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 22-Jun-2023 00:06 224886
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/103-paddle-to-openvino-classification-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="103-paddle-to-openvino-classification-with-output_15_3.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 12-Jul-2023 00:11 120883
<a href="103-paddle-to-openvino-classification-with-output_23_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 12-Jul-2023 00:11 224886
<a href="103-paddle-to-openvino-classification-with-output_27_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 12-Jul-2023 00:11 224886
<a href="103-paddle-to-openvino-classification-with-output_30_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 12-Jul-2023 00:11 224886
<a href="103-paddle-to-openvino-classification-with-output_8_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 12-Jul-2023 00:11 224886
</pre><hr></body>
</html>

40
docs/notebooks/104-model-tools-with-output.rst
View File

@@ -177,20 +177,20 @@ Converting mobilenet-v2-pytorch…
.. parsed-literal::
========== Converting mobilenet-v2-pytorch to ONNX
Conversion to ONNX command: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/internal_scripts/pytorch_to_onnx.py --model-name=mobilenet_v2 --weights=model/public/mobilenet-v2-pytorch/mobilenet_v2-b0353104.pth --import-module=torchvision.models --input-shape=1,3,224,224 --output-file=model/public/mobilenet-v2-pytorch/mobilenet-v2.onnx --input-names=data --output-names=prob
Conversion to ONNX command: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/internal_scripts/pytorch_to_onnx.py --model-name=mobilenet_v2 --weights=model/public/mobilenet-v2-pytorch/mobilenet_v2-b0353104.pth --import-module=torchvision.models --input-shape=1,3,224,224 --output-file=model/public/mobilenet-v2-pytorch/mobilenet-v2.onnx --input-names=data --output-names=prob
ONNX check passed successfully.
========== Converting mobilenet-v2-pytorch to IR (FP16)
Conversion command: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/bin/mo --framework=onnx --output_dir=/tmp/tmpt97jmp4t --model_name=mobilenet-v2-pytorch --input=data '--mean_values=data[123.675,116.28,103.53]' '--scale_values=data[58.624,57.12,57.375]' --reverse_input_channels --output=prob --input_model=model/public/mobilenet-v2-pytorch/mobilenet-v2.onnx '--layout=data(NCHW)' '--input_shape=[1, 3, 224, 224]' --compress_to_fp16=True
Conversion command: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/bin/mo --framework=onnx --output_dir=/tmp/tmpe5yh3lmf --model_name=mobilenet-v2-pytorch --input=data '--mean_values=data[123.675,116.28,103.53]' '--scale_values=data[58.624,57.12,57.375]' --reverse_input_channels --output=prob --input_model=model/public/mobilenet-v2-pytorch/mobilenet-v2.onnx '--layout=data(NCHW)' '--input_shape=[1, 3, 224, 224]' --compress_to_fp16=True
[ INFO ] Generated IR will be compressed to FP16. If you get lower accuracy, please consider disabling compression by removing argument --compress_to_fp16 or set it to false --compress_to_fp16=False.
Find more information about compression to FP16 at https://docs.openvino.ai/latest/openvino_docs_MO_DG_FP16_Compression.html
[ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11.
Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/latest/openvino_2_0_transition_guide.html
[ SUCCESS ] Generated IR version 11 model.
[ SUCCESS ] XML file: /tmp/tmpt97jmp4t/mobilenet-v2-pytorch.xml
[ SUCCESS ] BIN file: /tmp/tmpt97jmp4t/mobilenet-v2-pytorch.bin
[ SUCCESS ] XML file: /tmp/tmpe5yh3lmf/mobilenet-v2-pytorch.xml
[ SUCCESS ] BIN file: /tmp/tmpe5yh3lmf/mobilenet-v2-pytorch.bin
@@ -240,8 +240,8 @@ information in a dictionary.
'description': 'MobileNet V2 is image classification model pre-trained on ImageNet dataset. This is a PyTorch* implementation of MobileNetV2 architecture as described in the paper "Inverted Residuals and Linear Bottlenecks: Mobile Networks for Classification, Detection and Segmentation" <https://arxiv.org/abs/1801.04381>.\nThe model input is a blob that consists of a single image of "1, 3, 224, 224" in "RGB" order.\nThe model output is typical object classifier for the 1000 different classifications matching with those in the ImageNet database.',
'framework': 'pytorch',
'license_url': 'https://raw.githubusercontent.com/pytorch/vision/master/LICENSE',
'accuracy_config': '/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/models/public/mobilenet-v2-pytorch/accuracy-check.yml',
'model_config': '/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/models/public/mobilenet-v2-pytorch/model.yml',
'accuracy_config': '/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/models/public/mobilenet-v2-pytorch/accuracy-check.yml',
'model_config': '/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/models/public/mobilenet-v2-pytorch/model.yml',
'precisions': ['FP16', 'FP32'],
'quantization_output_precisions': ['FP16-INT8', 'FP32-INT8'],
'subdirectory': 'public/mobilenet-v2-pytorch',
@@ -321,7 +321,7 @@ seconds…
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 29.56 ms
[ INFO ] Read model took 30.53 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] data (node: data) : f32 / [N,C,H,W] / [1,3,224,224]
@@ -335,7 +335,7 @@ seconds…
[ INFO ] Model outputs:
[ INFO ] prob (node: prob) : f32 / [...] / [1,1000]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 165.63 ms
[ INFO ] Compile model took 149.89 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
@@ -357,17 +357,17 @@ seconds…
[ INFO ] Fill input 'data' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 6.54 ms
[ INFO ] First inference took 6.66 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 20190 iterations
[ INFO ] Duration: 15003.81 ms
[ INFO ] Count: 20136 iterations
[ INFO ] Duration: 15007.99 ms
[ INFO ] Latency:
[ INFO ] Median: 4.31 ms
[ INFO ] Average: 4.32 ms
[ INFO ] Min: 2.52 ms
[ INFO ] Max: 11.83 ms
[ INFO ] Throughput: 1345.66 FPS
[ INFO ] Median: 4.33 ms
[ INFO ] Average: 4.34 ms
[ INFO ] Min: 3.20 ms
[ INFO ] Max: 11.84 ms
[ INFO ] Throughput: 1341.69 FPS
Benchmark with Different Settings
@@ -456,9 +456,9 @@ Benchmark command:
command ended
Traceback (most recent call last):
File "/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 327, in main
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 327, in main
benchmark.set_allow_auto_batching(False)
File "/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 63, in set_allow_auto_batching
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 63, in set_allow_auto_batching
self.core.set_property({'ALLOW_AUTO_BATCHING': flag})
RuntimeError: Check 'false' failed at src/inference/src/core.cpp:238:
@@ -484,9 +484,9 @@ Benchmark command:
Check 'false' failed at src/plugins/auto/src/plugin_config.cpp:55:
property: ALLOW_AUTO_BATCHING: not supported
Traceback (most recent call last):
File "/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 327, in main
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 327, in main
benchmark.set_allow_auto_batching(False)
File "/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 63, in set_allow_auto_batching
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 63, in set_allow_auto_batching
self.core.set_property({'ALLOW_AUTO_BATCHING': flag})
RuntimeError: Check 'false' failed at src/inference/src/core.cpp:238:
Check 'false' failed at src/plugins/auto/src/plugin_config.cpp:55:

235
docs/notebooks/105-language-quantize-bert-with-output.rst
View File

@@ -24,7 +24,7 @@ and datasets. It consists of the following steps:
.. code:: ipython3
!pip install -q nncf datasets evaluate
!pip install -q "nncf>=2.5.0" datasets evaluate
Imports
-------
@@ -56,10 +56,10 @@ Imports
.. parsed-literal::
2023-06-21 22:27:49.466196: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-06-21 22:27:49.499821: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2023-07-11 22:27:10.887837: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-07-11 22:27:10.921844: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-06-21 22:27:50.040301: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2023-07-11 22:27:11.494944: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
.. parsed-literal::
@@ -85,9 +85,10 @@ Settings
Prepare the Model
-----------------
Perform the following: - Download and unpack pre-trained BERT model for
MRPC by PyTorch. - Convert the model to the OpenVINO Intermediate
Representation (OpenVINO IR)
Perform the following:
- Download and unpack pre-trained BERT model for MRPC by PyTorch.
- Convert the model to the OpenVINO Intermediate Representation (OpenVINO IR)
.. code:: ipython3
@@ -141,7 +142,7 @@ PyTorch model formats are supported:
.. parsed-literal::
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/annotations.py:309: UserWarning: TorchScript will treat type annotations of Tensor dtype-specific subtypes as if they are normal Tensors. dtype constraints are not enforced in compilation either.
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/annotations.py:309: UserWarning: TorchScript will treat type annotations of Tensor dtype-specific subtypes as if they are normal Tensors. dtype constraints are not enforced in compilation either.
warnings.warn("TorchScript will treat type annotations of Tensor "
@@ -171,12 +172,6 @@ tokenizer from HuggingFace.
data_source = create_data_source()
.. parsed-literal::
Downloading readme: 0.00B [00:00, ?B/s]
.. parsed-literal::
Found cached dataset glue (/opt/home/k8sworker/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)
@@ -392,8 +387,8 @@ The optimization process contains the following steps:
.. parsed-literal::
Statistics collection: 100%|██████████| 300/300 [00:24<00:00, 12.11it/s]
Biases correction: 100%|██████████| 74/74 [00:25<00:00, 2.93it/s]
Statistics collection: 100%|██████████| 300/300 [00:24<00:00, 12.02it/s]
Biases correction: 100%|██████████| 74/74 [00:25<00:00, 2.94it/s]
.. code:: ipython3
@@ -404,14 +399,44 @@ The optimization process contains the following steps:
Load and Test OpenVINO Model
----------------------------
To load and test converted model, perform the following: \* Load the
model and compile it for CPU. \* Prepare the input. \* Run the
inference. \* Get the answer from the model output.
To load and test converted model, perform the following:
* Load the model and compile it for selected device.
* Prepare the input.
* Run the inference.
* Get the answer from the model output.
Select inference device
~~~~~~~~~~~~~~~~~~~~~~~
select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
.. code:: ipython3
# Compile the model for a specific device.
compiled_quantized_model = core.compile_model(model=quantized_model, device_name="CPU")
compiled_quantized_model = core.compile_model(model=quantized_model, device_name=device.value)
output_layer = compiled_quantized_model.outputs[0]
The Data Source returns a pair of sentences (indicated by
@@ -450,7 +475,7 @@ Compare F1-score of FP32 and INT8 models
Evaluate the model on GLUE dataset.
Returns F1 score metric.
"""
compiled_model = core.compile_model(model, device_name='CPU')
compiled_model = core.compile_model(model, device_name=device.value)
output_layer = compiled_model.output(0)
metric = evaluate.load('glue', 'mrpc')
@@ -495,7 +520,7 @@ Frames Per Second (FPS) for images.
.. code:: ipython3
# Compile the model for a specific device.
compiled_model = core.compile_model(model=model, device_name="CPU")
compiled_model = core.compile_model(model=model, device_name=device.value)
.. code:: ipython3
@@ -520,7 +545,7 @@ Frames Per Second (FPS) for images.
end = time.perf_counter()
time_ir = end - start
print(
f"IR FP32 model in OpenVINO Runtime/CPU: {time_ir / num_samples:.3f} "
f"IR FP32 model in OpenVINO Runtime/{device.value}: {time_ir / num_samples:.3f} "
f"seconds per sentence, SPS: {num_samples / time_ir:.2f}"
)
@@ -530,16 +555,16 @@ Frames Per Second (FPS) for images.
end = time.perf_counter()
time_ir = end - start
print(
f"OpenVINO IR INT8 model in OpenVINO Runtime/CPU: {time_ir / num_samples:.3f} "
f"OpenVINO IR INT8 model in OpenVINO Runtime/{device.value}: {time_ir / num_samples:.3f} "
f"seconds per sentence, SPS: {num_samples / time_ir:.2f}"
)
.. parsed-literal::
PyTorch model on CPU: 0.071 seconds per sentence, SPS: 14.07
IR FP32 model in OpenVINO Runtime/CPU: 0.021 seconds per sentence, SPS: 48.22
OpenVINO IR INT8 model in OpenVINO Runtime/CPU: 0.010 seconds per sentence, SPS: 97.87
PyTorch model on CPU: 0.071 seconds per sentence, SPS: 14.09
IR FP32 model in OpenVINO Runtime/AUTO: 0.022 seconds per sentence, SPS: 45.98
OpenVINO IR INT8 model in OpenVINO Runtime/AUTO: 0.010 seconds per sentence, SPS: 98.77
Finally, measure the inference performance of OpenVINO ``FP32`` and
@@ -559,7 +584,7 @@ in OpenVINO.
.. code:: ipython3
# Inference FP32 model (OpenVINO IR)
! benchmark_app -m $ir_model_xml -shape [1,128],[1,128],[1,128] -d CPU -api sync
! benchmark_app -m $ir_model_xml -shape [1,128],[1,128],[1,128] -d device.value -api sync
.. parsed-literal::
@@ -567,81 +592,27 @@ in OpenVINO.
[Step 1/11] Parsing and validating input arguments
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ WARNING ] Default duration 120 seconds is used for unknown device device.value
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.LATENCY.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 56.68 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] 1 , input_ids (node: Parameter_2) : i64 / [...] / [1,?]
[ INFO ] 2 , attention_mask , attention_mask.1 (node: Parameter_3) : i64 / [...] / [1,?]
[ INFO ] token_type_ids , 3 (node: Parameter_4) : i64 / [...] / [1,?]
[ INFO ] Model outputs:
[ INFO ] logits , 707 (node: aten::linear_1192) : f32 / [...] / [1,2]
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[ INFO ] Reshaping model: '1': [1,128], '2': [1,128], '3': [1,128]
[ INFO ] Reshape model took 57.63 ms
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] 1 , input_ids (node: Parameter_2) : i64 / [...] / [1,128]
[ INFO ] 2 , attention_mask , attention_mask.1 (node: Parameter_3) : i64 / [...] / [1,128]
[ INFO ] token_type_ids , 3 (node: Parameter_4) : i64 / [...] / [1,128]
[ INFO ] Model outputs:
[ INFO ] logits , 707 (node: aten::linear_1192) : f32 / [...] / [1,2]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 266.53 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: Model0
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 1
[ INFO ] NUM_STREAMS: 1
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] INFERENCE_NUM_THREADS: 12
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.LATENCY
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input '1'!. This input will be filled with random values!
[ WARNING ] No input files were given for input '2'!. This input will be filled with random values!
[ WARNING ] No input files were given for input '3'!. This input will be filled with random values!
[ INFO ] Fill input '1' with random values
[ INFO ] Fill input '2' with random values
[ INFO ] Fill input '3' with random values
[Step 10/11] Measuring performance (Start inference synchronously, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 33.80 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 3293 iterations
[ INFO ] Duration: 60005.54 ms
[ INFO ] Latency:
[ INFO ] Median: 18.09 ms
[ INFO ] Average: 18.12 ms
[ INFO ] Min: 17.73 ms
[ INFO ] Max: 22.74 ms
[ INFO ] Throughput: 55.27 FPS
[ ERROR ] Check 'false' failed at src/inference/src/core.cpp:84:
Device with "device" name is not registered in the OpenVINO Runtime
Traceback (most recent call last):
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 103, in main
benchmark.print_version_info()
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 48, in print_version_info
for device, version in self.core.get_versions(self.device).items():
RuntimeError: Check 'false' failed at src/inference/src/core.cpp:84:
Device with "device" name is not registered in the OpenVINO Runtime
.. code:: ipython3
# Inference INT8 model (OpenVINO IR)
! benchmark_app -m $compressed_model_xml -shape [1,128],[1,128],[1,128] -d CPU -api sync
! benchmark_app -m $compressed_model_xml -shape [1,128],[1,128],[1,128] -d device.value -api sync
.. parsed-literal::
@@ -649,73 +620,19 @@ in OpenVINO.
[Step 1/11] Parsing and validating input arguments
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ WARNING ] Default duration 120 seconds is used for unknown device device.value
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.LATENCY.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 53.69 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] input_ids , 1 (node: Parameter_2) : i64 / [...] / [1,?]
[ INFO ] attention_mask.1 , 2 , attention_mask (node: Parameter_3) : i64 / [...] / [1,?]
[ INFO ] 3 , token_type_ids (node: Parameter_4) : i64 / [...] / [1,?]
[ INFO ] Model outputs:
[ INFO ] logits , 707 (node: aten::linear_1192) : f32 / [...] / [1,2]
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[ INFO ] Reshaping model: '1': [1,128], '2': [1,128], '3': [1,128]
[ INFO ] Reshape model took 61.50 ms
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] input_ids , 1 (node: Parameter_2) : i64 / [...] / [1,128]
[ INFO ] attention_mask.1 , 2 , attention_mask (node: Parameter_3) : i64 / [...] / [1,128]
[ INFO ] 3 , token_type_ids (node: Parameter_4) : i64 / [...] / [1,128]
[ INFO ] Model outputs:
[ INFO ] logits , 707 (node: aten::linear_1192) : f32 / [...] / [1,2]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 473.56 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: Model0
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 1
[ INFO ] NUM_STREAMS: 1
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] INFERENCE_NUM_THREADS: 12
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.LATENCY
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input '1'!. This input will be filled with random values!
[ WARNING ] No input files were given for input '2'!. This input will be filled with random values!
[ WARNING ] No input files were given for input '3'!. This input will be filled with random values!
[ INFO ] Fill input '1' with random values
[ INFO ] Fill input '2' with random values
[ INFO ] Fill input '3' with random values
[Step 10/11] Measuring performance (Start inference synchronously, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 17.89 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 6496 iterations
[ INFO ] Duration: 60002.09 ms
[ INFO ] Latency:
[ INFO ] Median: 9.09 ms
[ INFO ] Average: 9.13 ms
[ INFO ] Min: 8.45 ms
[ INFO ] Max: 12.68 ms
[ INFO ] Throughput: 109.96 FPS
[ ERROR ] Check 'false' failed at src/inference/src/core.cpp:84:
Device with "device" name is not registered in the OpenVINO Runtime
Traceback (most recent call last):
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 103, in main
benchmark.print_version_info()
File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 48, in print_version_info
for device, version in self.core.get_versions(self.device).items():
RuntimeError: Check 'false' failed at src/inference/src/core.cpp:84:
Device with "device" name is not registered in the OpenVINO Runtime

34
docs/notebooks/106-auto-device-with-output.rst
View File

@@ -25,13 +25,7 @@ immediately on the CPU and then transparently shifts inference to the
GPU, once it is ready. This dramatically reduces the time to execute
first inference.
.. raw:: html
<center>
.. raw:: html
</center>
.. image:: https://camo.githubusercontent.com/cc526c3f5fc992cc7176d097894303248adbd04b4d158bd98e65edc8270af5fc/68747470733a2f2f757365722d696d616765732e67697468756275736572636f6e74656e742e636f6d2f31353730393732332f3136313435313834372d37353965326264622d373062632d343633642d393831382d3430306330636366336331362e706e67
Import modules and create Core
------------------------------
@@ -266,7 +260,7 @@ executed on CPU until GPU is ready.
.. parsed-literal::
Time to load model using AUTO device and get first inference: 0.19 seconds.
Time to load model using AUTO device and get first inference: 0.18 seconds.
.. code:: ipython3
@@ -430,12 +424,12 @@ Loop for inference and update the FPS/Latency every
Compiling Model for AUTO device with THROUGHPUT hint
Start inference, 6 groups of FPS/latency will be measured over 10s intervals
throughput: 190.61fps, latency: 29.81ms, time interval: 10.00s
throughput: 194.35fps, latency: 30.08ms, time interval: 10.00s
throughput: 194.90fps, latency: 30.02ms, time interval: 10.01s
throughput: 192.30fps, latency: 30.47ms, time interval: 10.00s
throughput: 192.76fps, latency: 30.38ms, time interval: 10.02s
throughput: 194.26fps, latency: 30.10ms, time interval: 10.00s
throughput: 190.70fps, latency: 29.76ms, time interval: 10.02s
throughput: 191.95fps, latency: 30.48ms, time interval: 10.00s
throughput: 192.78fps, latency: 30.40ms, time interval: 10.00s
throughput: 191.39fps, latency: 30.62ms, time interval: 10.00s
throughput: 192.18fps, latency: 30.44ms, time interval: 10.03s
throughput: 191.33fps, latency: 30.62ms, time interval: 10.00s
Done
@@ -479,12 +473,12 @@ Loop for inference and update the FPS/Latency for each
Compiling Model for AUTO Device with LATENCY hint
Start inference, 6 groups fps/latency will be out with 10s interval
throughput: 138.65fps, latency: 6.69ms, time interval: 10.00s
throughput: 142.09fps, latency: 6.68ms, time interval: 10.00s
throughput: 142.03fps, latency: 6.68ms, time interval: 10.00s
throughput: 142.07fps, latency: 6.68ms, time interval: 10.00s
throughput: 141.79fps, latency: 6.69ms, time interval: 10.00s
throughput: 142.19fps, latency: 6.67ms, time interval: 10.00s
throughput: 136.99fps, latency: 6.75ms, time interval: 10.00s
throughput: 140.91fps, latency: 6.74ms, time interval: 10.01s
throughput: 140.83fps, latency: 6.74ms, time interval: 10.00s
throughput: 140.90fps, latency: 6.74ms, time interval: 10.00s
throughput: 140.83fps, latency: 6.74ms, time interval: 10.00s
throughput: 140.85fps, latency: 6.74ms, time interval: 10.00s
Done

3
docs/notebooks/106-auto-device-with-output_files/106-auto-device-with-output_12_0.jpg Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:5a58493845ebd3e98186df7a1ea042b20545bb3c2a5b4a326163da6c9eb5e7d9
size 121563

4
docs/notebooks/106-auto-device-with-output_files/106-auto-device-with-output_12_0.png
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d8a9811dc8ab8b5f2ef6171ba757ccf9bbe33d0f137824ad0c32380bd74c5ff4
size 928896
oid sha256:84112f4f04be0a3445174f4dcf7e300fc12dbd50fd8a5e2d98b4082402dda6de
size 869661

4
docs/notebooks/106-auto-device-with-output_files/106-auto-device-with-output_25_0.png
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:512f1331af7db846d6d7f59ede72bde479afc77892515952f57288aa0b028cd9
size 26607
oid sha256:03224259b8d1a7dd982c9d5b2baae8bdca495cbe008f83bc5b0a0fd3ecc47ab6
size 27172

4
docs/notebooks/106-auto-device-with-output_files/106-auto-device-with-output_26_0.png
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b3cf4ed07da8d66040ad109a3f88329d9b29011f6ec2586df67ce9719fc2841b
size 41162
oid sha256:d962a84b77491e4c1f9bdcb4e0c58f6aae55f11f8e59ae747077b166e706fd76
size 40994

11
docs/notebooks/106-auto-device-with-output_files/index.html
View File

@@ -1,9 +1,10 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/106-auto-device-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/106-auto-device-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/106-auto-device-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="106-auto-device-with-output_12_0.png">106-auto-device-with-output_12_0.png</a> 22-Jun-2023 00:06 928896
<a href="106-auto-device-with-output_25_0.png">106-auto-device-with-output_25_0.png</a> 22-Jun-2023 00:06 26607
<a href="106-auto-device-with-output_26_0.png">106-auto-device-with-output_26_0.png</a> 22-Jun-2023 00:06 41162
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/106-auto-device-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="106-auto-device-with-output_12_0.jpg">106-auto-device-with-output_12_0.jpg</a> 12-Jul-2023 00:11 121563
<a href="106-auto-device-with-output_12_0.png">106-auto-device-with-output_12_0.png</a> 12-Jul-2023 00:11 869661
<a href="106-auto-device-with-output_25_0.png">106-auto-device-with-output_25_0.png</a> 12-Jul-2023 00:11 27172
<a href="106-auto-device-with-output_26_0.png">106-auto-device-with-output_26_0.png</a> 12-Jul-2023 00:11 40994
</pre><hr></body>
</html>

20
docs/notebooks/107-speech-recognition-quantization-data2vec-with-output.rst
View File

@@ -316,21 +316,11 @@ OpenVINO with minimal accuracy drop.
Create a quantized model from the pre-trained ``FP16`` model and the
calibration dataset. The optimization process contains the following
steps: 1. Create a Dataset for quantization. 2. Run ``nncf.quantize``
for getting an optimized model. The ``nncf.quantize`` function provides
an interface for model quantization. It requires an instance of the
OpenVINO Model and quantization dataset. Optionally, some additional
parameters for the configuration quantization process (number of samples
for quantization, preset, ignored scope, etc.) can be provided. For more
accurate results, we should keep the operation in the postprocessing
subgraph in floating point precision, using the ``ignored_scope``
parameter. ``advanced_parameters`` can be used to specify advanced
quantization parameters for fine-tuning the quantization algorithm. In
this tutorial we pass range estimator parameters for activations. For
more information see `Tune quantization
parameters <https://docs.openvino.ai/2023.0/basic_quantization_flow.html#tune-quantization-parameters>`__.
3. Serialize OpenVINO IR model using ``openvino.runtime.serialize``
function.
steps:
1. Create a Dataset for quantization.
2. Run ``nncf.quantize`` for getting an optimized model. The ``nncf.quantize`` function provides an interface for model quantization. It requires an instance of the OpenVINO Model and quantization dataset. Optionally, some additional parameters for the configuration quantization process (number of samples for quantization, preset, ignored scope, etc.) can be provided. For more accurate results, we should keep the operation in the postprocessing subgraph in floating point precision, using the ``ignored_scope`` parameter. ``advanced_parameters`` can be used to specify advanced quantization parameters for fine-tuning the quantization algorithm. In this tutorial we pass range estimator parameters for activations. For more information, see `Tune quantization parameters <https://docs.openvino.ai/2023.0/basic_quantization_flow.html#tune-quantization-parameters>`__.
3. Serialize OpenVINO IR model using ``openvino.runtime.serialize`` function.
.. code:: ipython3

52
docs/notebooks/108-gpu-device-with-output.rst
View File

@@ -39,6 +39,30 @@ instructions <https://docs.openvino.ai/2023.0/openvino_docs_install_guides_confi
to configure OpenVINO to work with your GPU. Then, read on to learn how
to accelerate inference with GPUs in OpenVINO!
Install required packages
~~~~~~~~~~~~~~~~~~~~~~~~~
.. code:: ipython3
!pip install -q "openvino-dev>=2023.0.0"
!pip install -q tensorflow
# Fetch `notebook_utils` module
import urllib.request
urllib.request.urlretrieve(
url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py',
filename='notebook_utils.py'
)
.. parsed-literal::
('notebook_utils.py', <http.client.HTTPMessage at 0x7f2d0c5baa40>)
Checking GPUs with Query Device
-------------------------------
@@ -1050,7 +1074,7 @@ Load and Preprocess Video Frames
.. code:: ipython3
# Load video
video_file = "../data/video/Coco Walking in Berkeley.mp4"
video_file = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/video/Coco%20Walking%20in%20Berkeley.mp4"
video = cv2.VideoCapture(video_file)
framebuf = []
@@ -1259,19 +1283,13 @@ of a basic object detection application that uses a GPU and displays the
detected bounding boxes.
To read more about any of these topics, feel free to visit their
corresponding documentation: \* `GPU
Plugin <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_GPU.html>`__
\* `AUTO
Plugin <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__
\* `Model
Caching <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Model_caching_overview.html>`__
\* `MULTI Device
Mode <https://docs.openvino.ai/nightly/openvino_docs_OV_UG_Running_on_multiple_devices.html>`__
\* `Query Device
Properties <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_query_api.html>`__
\* `Configurations for GPUs with
OpenVINO <https://docs.openvino.ai/2023.0/openvino_docs_install_guides_configurations_for_intel_gpu.html>`__
\* `Benchmark Python
Tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_tool_README.html>`__
\* `Asynchronous
Inferencing <https://docs.openvino.ai/2023.0/openvino_docs_ie_plugin_dg_async_infer_request.html>`__
corresponding documentation:
* `GPU Plugin <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_GPU.html>`__
* `AUTO Plugin <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__
* `Model Caching <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Model_caching_overview.html>`__
* `MULTI Device Mode <https://docs.openvino.ai/nightly/openvino_docs_OV_UG_Running_on_multiple_devices.html>`__
* `Query Device Properties <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_query_api.html>`__
* `Configurations for GPUs with OpenVINO <https://docs.openvino.ai/2023.0/openvino_docs_install_guides_configurations_for_intel_gpu.html>`__
* `Benchmark Python Tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_tool_README.html>`__
* `Asynchronous Inferencing <https://docs.openvino.ai/2023.0/openvino_docs_ie_plugin_dg_async_infer_request.html>`__

24
docs/notebooks/109-latency-tricks-with-output.rst
View File

@@ -94,7 +94,7 @@ requirements of this particular object detection model.
.. parsed-literal::
<DisplayHandle display_id=18f2331c8bf61fd35d93bb638f7ea64b>
<DisplayHandle display_id=5ab5e600f8b1dde3f55ff40ec675d04c>
@@ -346,8 +346,8 @@ optimizations applied. We will treat it as our baseline.
.. parsed-literal::
PyTorch model on CPU. First inference time: 0.0241 seconds
PyTorch model on CPU: 0.0195 seconds per image (51.21 FPS)
PyTorch model on CPU. First inference time: 0.0286 seconds
PyTorch model on CPU: 0.0202 seconds per image (49.58 FPS)
ONNX model
@@ -395,8 +395,8 @@ Representation (IR) to leverage the OpenVINO Runtime.
.. parsed-literal::
ONNX model on CPU. First inference time: 0.0167 seconds
ONNX model on CPU: 0.0132 seconds per image (76.03 FPS)
ONNX model on CPU. First inference time: 0.0173 seconds
ONNX model on CPU: 0.0133 seconds per image (75.16 FPS)
OpenVINO IR model
@@ -433,8 +433,8 @@ accuracy drop. That’s why we skip that step in this notebook.
.. parsed-literal::
OpenVINO model on CPU. First inference time: 0.0155 seconds
OpenVINO model on CPU: 0.0132 seconds per image (75.91 FPS)
OpenVINO model on CPU. First inference time: 0.0163 seconds
OpenVINO model on CPU: 0.0132 seconds per image (75.64 FPS)
OpenVINO IR model on GPU
@@ -491,7 +491,7 @@ our case.
.. parsed-literal::
OpenVINO model + more threads on CPU. First inference time: 0.0151 seconds
OpenVINO model + more threads on CPU: 0.0131 seconds per image (76.12 FPS)
OpenVINO model + more threads on CPU: 0.0132 seconds per image (75.68 FPS)
OpenVINO IR model in latency mode
@@ -520,8 +520,8 @@ devices as well.
.. parsed-literal::
OpenVINO model on AUTO. First inference time: 0.0155 seconds
OpenVINO model on AUTO: 0.0135 seconds per image (73.89 FPS)
OpenVINO model on AUTO. First inference time: 0.0156 seconds
OpenVINO model on AUTO: 0.0135 seconds per image (73.93 FPS)
OpenVINO IR model in latency mode + shared memory
@@ -554,8 +554,8 @@ performance!
.. parsed-literal::
OpenVINO model + shared memory on AUTO. First inference time: 0.0132 seconds
OpenVINO model + shared memory on AUTO: 0.0053 seconds per image (187.78 FPS)
OpenVINO model + shared memory on AUTO. First inference time: 0.0144 seconds
OpenVINO model + shared memory on AUTO: 0.0053 seconds per image (187.92 FPS)
Other tricks

4
docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_30_0.png
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:eeffdc620ca9a96149c05b5fa8815c1e0f0f6d1775de63ae1893c051cd690977
size 56942
oid sha256:3683f4df707fff21d7f1e5329acbdfb9d00031ded856582d893c05a67f726d4e
size 57103

20
docs/notebooks/109-latency-tricks-with-output_files/index.html
View File

@@ -1,14 +1,14 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/109-latency-tricks-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/109-latency-tricks-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/109-latency-tricks-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="109-latency-tricks-with-output_14_0.jpg">109-latency-tricks-with-output_14_0.jpg</a> 22-Jun-2023 00:06 162715
<a href="109-latency-tricks-with-output_17_0.jpg">109-latency-tricks-with-output_17_0.jpg</a> 22-Jun-2023 00:06 162715
<a href="109-latency-tricks-with-output_19_0.jpg">109-latency-tricks-with-output_19_0.jpg</a> 22-Jun-2023 00:06 162756
<a href="109-latency-tricks-with-output_23_0.jpg">109-latency-tricks-with-output_23_0.jpg</a> 22-Jun-2023 00:06 162756
<a href="109-latency-tricks-with-output_25_0.jpg">109-latency-tricks-with-output_25_0.jpg</a> 22-Jun-2023 00:06 162756
<a href="109-latency-tricks-with-output_27_0.jpg">109-latency-tricks-with-output_27_0.jpg</a> 22-Jun-2023 00:06 162756
<a href="109-latency-tricks-with-output_30_0.png">109-latency-tricks-with-output_30_0.png</a> 22-Jun-2023 00:06 56942
<a href="109-latency-tricks-with-output_4_0.jpg">109-latency-tricks-with-output_4_0.jpg</a> 22-Jun-2023 00:06 155828
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/109-latency-tricks-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="109-latency-tricks-with-output_14_0.jpg">109-latency-tricks-with-output_14_0.jpg</a> 12-Jul-2023 00:11 162715
<a href="109-latency-tricks-with-output_17_0.jpg">109-latency-tricks-with-output_17_0.jpg</a> 12-Jul-2023 00:11 162715
<a href="109-latency-tricks-with-output_19_0.jpg">109-latency-tricks-with-output_19_0.jpg</a> 12-Jul-2023 00:11 162756
<a href="109-latency-tricks-with-output_23_0.jpg">109-latency-tricks-with-output_23_0.jpg</a> 12-Jul-2023 00:11 162756
<a href="109-latency-tricks-with-output_25_0.jpg">109-latency-tricks-with-output_25_0.jpg</a> 12-Jul-2023 00:11 162756
<a href="109-latency-tricks-with-output_27_0.jpg">109-latency-tricks-with-output_27_0.jpg</a> 12-Jul-2023 00:11 162756
<a href="109-latency-tricks-with-output_30_0.png">109-latency-tricks-with-output_30_0.png</a> 12-Jul-2023 00:11 57103
<a href="109-latency-tricks-with-output_4_0.jpg">109-latency-tricks-with-output_4_0.jpg</a> 12-Jul-2023 00:11 155828
</pre><hr></body>
</html>

115
docs/notebooks/110-ct-scan-live-inference-with-output.rst
View File

@@ -110,18 +110,34 @@ is a command-line application that can be run in the notebook with
.. code:: ipython3
ie = Core()
core = Core()
# By default, benchmark on MULTI:CPU,GPU if a GPU is available, otherwise on CPU.
device = "MULTI:CPU,GPU" if "GPU" in ie.available_devices else "CPU"
# Uncomment one of the options below to benchmark on other devices.
# device = "GPU"
# device = "CPU"
# device = "AUTO"
device_list = ["MULTI:CPU,GPU" if "GPU" in core.available_devices else "AUTO"]
import ipywidgets as widgets
device = widgets.Dropdown(
options=core.available_devices + device_list,
value=device_list[0],
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
.. code:: ipython3
# Benchmark model
! benchmark_app -m $MODEL_PATH -d $device -t 15 -api sync
! benchmark_app -m $MODEL_PATH -d $device.value -t 15 -api sync
.. parsed-literal::
@@ -133,15 +149,15 @@ is a command-line application that can be run in the notebook with
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] AUTO
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.LATENCY.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.LATENCY.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 13.79 ms
[ INFO ] Read model took 13.92 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] input.1 (node: input.1) : f32 / [...] / [1,1,512,512]
@@ -155,39 +171,46 @@ is a command-line application that can be run in the notebook with
[ INFO ] Model outputs:
[ INFO ] 153 (node: 153) : f32 / [...] / [1,1,512,512]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 163.85 ms
[ INFO ] Compile model took 181.67 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] PERFORMANCE_HINT: PerformanceMode.LATENCY
[ INFO ] NETWORK_NAME: pretrained_unet_kits19
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 1
[ INFO ] NUM_STREAMS: 1
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] INFERENCE_NUM_THREADS: 12
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.LATENCY
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] MODEL_PRIORITY: Priority.MEDIUM
[ INFO ] MULTI_DEVICE_PRIORITIES: CPU
[ INFO ] CPU:
[ INFO ] CPU_BIND_THREAD: YES
[ INFO ] CPU_THREADS_NUM: 0
[ INFO ] CPU_THROUGHPUT_STREAMS: 1
[ INFO ] DEVICE_ID:
[ INFO ] DUMP_EXEC_GRAPH_AS_DOT:
[ INFO ] DYN_BATCH_ENABLED: NO
[ INFO ] DYN_BATCH_LIMIT: 0
[ INFO ] ENFORCE_BF16: NO
[ INFO ] EXCLUSIVE_ASYNC_REQUESTS: NO
[ INFO ] NETWORK_NAME: pretrained_unet_kits19
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 1
[ INFO ] PERFORMANCE_HINT: LATENCY
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] PERF_COUNT: NO
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'input.1'!. This input will be filled with random values!
[ INFO ] Fill input 'input.1' with random values
[Step 10/11] Measuring performance (Start inference synchronously, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 25.59 ms
[ INFO ] First inference took 27.22 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 1419 iterations
[ INFO ] Duration: 15009.20 ms
[ INFO ] Count: 1431 iterations
[ INFO ] Duration: 15005.09 ms
[ INFO ] Latency:
[ INFO ] Median: 10.34 ms
[ INFO ] Average: 10.38 ms
[ INFO ] Min: 10.13 ms
[ INFO ] Max: 14.24 ms
[ INFO ] Throughput: 96.71 FPS
[ INFO ] Median: 10.25 ms
[ INFO ] Average: 10.29 ms
[ INFO ] Min: 9.99 ms
[ INFO ] Max: 15.67 ms
[ INFO ] Throughput: 97.57 FPS
Download and Prepare Data
@@ -257,12 +280,10 @@ to perform asynchronous inference. It can be instantiated with compiled
model and a number of jobs - parallel execution threads. If you don’t
pass a number of jobs or pass ``0``, then OpenVINO will pick the optimal
number based on your device and heuristics. After acquiring the
inference queue, there are two jobs to do: - Preprocess the data and
push it to the inference queue. The preprocessing steps will remain the
same. - Tell the inference queue what to do with the model output after
the inference is finished. It is represented by the ``callback`` python
function that takes an inference result and data that we passed to the
inference queue along with the prepared input data
inference queue, there are two jobs to do:
- Preprocess the data and push it to the inference queue. The preprocessing steps will remain the same.
- Tell the inference queue what to do with the model output after the inference is finished. It is represented by the ``callback`` python function that takes an inference result and data that we passed to the inference queue along with the prepared input data
Everything else will be handled by the ``AsyncInferQueue`` instance.
@@ -279,9 +300,9 @@ to see the implementation.
.. code:: ipython3
ie = Core()
core = Core()
segmentation_model = SegmentationModel(
ie=ie, model_path=Path(MODEL_PATH), sigmoid=True, rotate_and_flip=True
ie=core, model_path=Path(MODEL_PATH), sigmoid=True, rotate_and_flip=True
)
image_paths = sorted(case_path.glob("imaging_frames/*jpg"))
@@ -319,8 +340,16 @@ Specify device
.. code:: ipython3
# Possible options for device include "CPU", "GPU", "AUTO", "MULTI".
device = "MULTI:CPU,GPU" if "GPU" in ie.available_devices else "CPU"
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
Setting callback function
~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -367,7 +396,7 @@ Create asynchronous inference queue and perform it
from openvino.runtime import AsyncInferQueue
load_start_time = time.perf_counter()
compiled_model = ie.compile_model(segmentation_model.net, device)
compiled_model = core.compile_model(segmentation_model.net, device.value)
# Create asynchronous inference queue with optimal number of infer requests
infer_queue = AsyncInferQueue(compiled_model)
infer_queue.set_callback(completion_callback)
@@ -403,7 +432,7 @@ Create asynchronous inference queue and perform it
.. parsed-literal::
Loaded model to CPU in 0.17 seconds.
Total time to infer all frames: 3.412s
Time per frame: 0.050177s (19.929 FPS)
Loaded model to Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') in 0.18 seconds.
Total time to infer all frames: 3.416s
Time per frame: 0.050229s (19.909 FPS)

6
docs/notebooks/110-ct-scan-live-inference-with-output_files/index.html
View File

@@ -1,7 +1,7 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/110-ct-scan-live-inference-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/110-ct-scan-live-inference-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/110-ct-scan-live-inference-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="110-ct-scan-live-inference-with-output_21_0.png">110-ct-scan-live-inference-with-output_21_0.png</a> 22-Jun-2023 00:06 48780
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/110-ct-scan-live-inference-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="110-ct-scan-live-inference-with-output_21_0.png">110-ct-scan-live-inference-with-output_21_0.png</a> 12-Jul-2023 00:11 48780
</pre><hr></body>
</html>

73
docs/notebooks/110-ct-segmentation-quantize-nncf-with-output.rst
View File

@@ -146,10 +146,10 @@ Imports
.. parsed-literal::
2023-06-21 22:40:32.613237: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-06-21 22:40:32.647235: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2023-07-11 22:37:38.702892: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-07-11 22:37:38.737776: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-06-21 22:40:33.170121: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2023-07-11 22:37:39.282688: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
.. parsed-literal::
@@ -416,21 +416,26 @@ this notebook.
.. parsed-literal::
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:179: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:179: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
if x_e.shape[-i - 1] != x_0.shape[-i - 1]:
`NNCF <https://github.com/openvinotoolkit/nncf>`__ provides a suite of
advanced algorithms for Neural Networks inference optimization in
OpenVINO with minimal accuracy drop. > **Note**: NNCF Post-training
Quantization is available in OpenVINO 2023.0 release.
OpenVINO with minimal accuracy drop.
.. note::
NNCF Post-training Quantization is available in OpenVINO 2023.0 release.
Create a quantized model from the pre-trained ``FP32`` model and the
calibration dataset. The optimization process contains the following
steps: 1. Create a Dataset for quantization. 2. Run ``nncf.quantize``
for getting an optimized model. 3. Export the quantized model to ONNX
and then convert to OpenVINO IR model. 4. Serialize the INT8 model using
``openvino.runtime.serialize`` function for benchmarking.
steps:
1. Create a Dataset for quantization.
2. Run ``nncf.quantize`` for getting an optimized model.
3. Export the quantized model to ONNX and then convert to OpenVINO IR model.
4. Serialize the INT8 model using ``openvino.runtime.serialize`` function for benchmarking.
.. code:: ipython3
@@ -474,13 +479,13 @@ model and save it.
.. parsed-literal::
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:338: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:338: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
return self._level_low.item()
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:346: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:346: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
return self._level_high.item()
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:179: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:179: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
if x_e.shape[-i - 1] != x_0.shape[-i - 1]:
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/quantize_functions.py:140: FutureWarning: 'torch.onnx._patch_torch._graph_op' is deprecated in version 1.13 and will be removed in version 1.14. Please note 'g.op()' is to be removed from torch.Graph. Please open a GitHub issue if you need this functionality..
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/quantize_functions.py:140: FutureWarning: 'torch.onnx._patch_torch._graph_op' is deprecated in version 1.13 and will be removed in version 1.14. Please note 'g.op()' is to be removed from torch.Graph. Please open a GitHub issue if you need this functionality..
output = g.op(
@@ -581,7 +586,7 @@ be run in the notebook with ``! benchmark_app`` or
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.LATENCY.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 22.91 ms
[ INFO ] Read model took 22.47 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] 1 , x (node: Parameter_2) : f32 / [...] / [1,1,512,512]
@@ -595,7 +600,7 @@ be run in the notebook with ``! benchmark_app`` or
[ INFO ] Model outputs:
[ INFO ] 169 (node: aten::_convolution_861) : f32 / [...] / [1,1,512,512]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 89.56 ms
[ INFO ] Compile model took 89.66 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: Model0
@@ -617,17 +622,17 @@ be run in the notebook with ``! benchmark_app`` or
[ INFO ] Fill input '1' with random values
[Step 10/11] Measuring performance (Start inference synchronously, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 55.95 ms
[ INFO ] First inference took 56.61 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 427 iterations
[ INFO ] Duration: 15009.56 ms
[ INFO ] Duration: 15001.35 ms
[ INFO ] Latency:
[ INFO ] Median: 34.91 ms
[ INFO ] Average: 34.93 ms
[ INFO ] Min: 34.59 ms
[ INFO ] Max: 36.22 ms
[ INFO ] Throughput: 28.64 FPS
[ INFO ] Median: 34.89 ms
[ INFO ] Average: 34.92 ms
[ INFO ] Min: 34.57 ms
[ INFO ] Max: 39.09 ms
[ INFO ] Throughput: 28.66 FPS
.. code:: ipython3
@@ -653,7 +658,7 @@ be run in the notebook with ``! benchmark_app`` or
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.LATENCY.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 32.53 ms
[ INFO ] Read model took 32.41 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] x.1 (node: x.1) : f32 / [...] / [1,1,512,512]
@@ -667,7 +672,7 @@ be run in the notebook with ``! benchmark_app`` or
[ INFO ] Model outputs:
[ INFO ] 578 (node: 578) : f32 / [...] / [1,1,512,512]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 145.14 ms
[ INFO ] Compile model took 144.74 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
@@ -689,17 +694,17 @@ be run in the notebook with ``! benchmark_app`` or
[ INFO ] Fill input 'x.1' with random values
[Step 10/11] Measuring performance (Start inference synchronously, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 31.93 ms
[ INFO ] First inference took 32.29 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 984 iterations
[ INFO ] Duration: 15002.92 ms
[ INFO ] Count: 985 iterations
[ INFO ] Duration: 15013.03 ms
[ INFO ] Latency:
[ INFO ] Median: 15.02 ms
[ INFO ] Median: 14.99 ms
[ INFO ] Average: 15.04 ms
[ INFO ] Min: 14.70 ms
[ INFO ] Max: 18.85 ms
[ INFO ] Throughput: 66.57 FPS
[ INFO ] Min: 14.76 ms
[ INFO ] Max: 17.68 ms
[ INFO ] Throughput: 66.70 FPS
Visually Compare Inference Results
@@ -782,7 +787,7 @@ seed is displayed to enable reproducing specific runs of this cell.
.. parsed-literal::
Visualizing results with seed 1687380135
Visualizing results with seed 1689107962
@@ -860,7 +865,7 @@ performs inference, and displays the results on the frames loaded in
.. parsed-literal::
Loaded model to CPU in 0.13 seconds.
Total time for 68 frames: 3.26 seconds, fps:21.19
Total time for 68 frames: 3.28 seconds, fps:21.05
References

4
docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/110-ct-segmentation-quantize-nncf-with-output_37_1.png
View File

@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:e80df41d53a6af0956c94d93705d3cb2fb637c0b92bb41e1dddbbd305a0fffc1
size 377021
oid sha256:960b979d5c041593481e188c2f1f14c72c314c8b2338c263ad450e6ad52e3600
size 385435

10
docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/index.html
View File

@@ -1,9 +1,9 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/110-ct-segmentation-quantize-nncf-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/110-ct-segmentation-quantize-nncf-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/110-ct-segmentation-quantize-nncf-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="110-ct-segmentation-quantize-nncf-with-output_15_1.png">110-ct-segmentation-quantize-nncf-with-output_1..&gt;</a> 22-Jun-2023 00:06 158997
<a href="110-ct-segmentation-quantize-nncf-with-output_37_1.png">110-ct-segmentation-quantize-nncf-with-output_3..&gt;</a> 22-Jun-2023 00:06 377021
<a href="110-ct-segmentation-quantize-nncf-with-output_42_0.jpg">110-ct-segmentation-quantize-nncf-with-output_4..&gt;</a> 22-Jun-2023 00:06 73812
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/110-ct-segmentation-quantize-nncf-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="110-ct-segmentation-quantize-nncf-with-output_15_1.png">110-ct-segmentation-quantize-nncf-with-output_1..&gt;</a> 12-Jul-2023 00:11 158997
<a href="110-ct-segmentation-quantize-nncf-with-output_37_1.png">110-ct-segmentation-quantize-nncf-with-output_3..&gt;</a> 12-Jul-2023 00:11 385435
<a href="110-ct-segmentation-quantize-nncf-with-output_42_0.jpg">110-ct-segmentation-quantize-nncf-with-output_4..&gt;</a> 12-Jul-2023 00:11 73812
</pre><hr></body>
</html>

150
docs/notebooks/111-yolov5-quantization-migration-with-output.rst
View File

@@ -67,18 +67,20 @@ Conversion of the YOLOv5 model to OpenVINO
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
There are three variables provided for easy run through all the notebook
cells. \* ``IMAGE_SIZE`` - the image size for model input. \*
``MODEL_NAME`` - the model you want to use. It can be either yolov5s,
yolov5m or yolov5l and so on. \* ``MODEL_PATH`` - to the path of the
model directory in the YOLOv5 repository.
cells.
* ``IMAGE_SIZE`` - the image size for model input.
* ``MODEL_NAME`` - the model you want to use. It can be either yolov5s, yolov5m or yolov5l and so on.
* ``MODEL_PATH`` - to the path of the model directory in the YOLOv5 repository.
YoloV5 ``export.py`` scripts support multiple model formats for
conversion. ONNX is also represented among supported formats. We need to
specify ``--include ONNX`` parameter for exporting. As the result,
directory with the ``{MODEL_NAME}`` name will be created with the
following content: \* ``{MODEL_NAME}.pt`` - the downloaded pre-trained
weight. \* ``{MODEL_NAME}.onnx`` - the Open Neural Network Exchange
(ONNX) is an open format, built to represent machine learning models.
following content:
* ``{MODEL_NAME}.pt`` - the downloaded pre-trained weight.
* ``{MODEL_NAME}.onnx`` - the Open Neural Network Exchange (ONNX) is an open format, built to represent machine learning models.
.. code:: ipython3
@@ -109,7 +111,7 @@ weight. \* ``{MODEL_NAME}.onnx`` - the Open Neural Network Exchange
YOLOv5 🚀 v7.0-0-g915bbf2 Python-3.8.10 torch-1.13.1+cpu CPU
Downloading https://github.com/ultralytics/yolov5/releases/download/v7.0/yolov5m.pt to yolov5m/yolov5m.pt...
100%|██████████████████████████████████████| 40.8M/40.8M [00:10<00:00, 4.09MB/s]
100%|██████████████████████████████████████| 40.8M/40.8M [00:09<00:00, 4.52MB/s]
Fusing layers...
YOLOv5m summary: 290 layers, 21172173 parameters, 0 gradients
@@ -119,8 +121,8 @@ weight. \* ``{MODEL_NAME}.onnx`` - the Open Neural Network Exchange
ONNX: starting export with onnx 1.14.0...
ONNX: export success ✅ 1.2s, saved as yolov5m/yolov5m.onnx (81.2 MB)
Export complete (13.4s)
Results saved to /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/yolov5m
Export complete (12.2s)
Results saved to /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/yolov5m
Detect: python detect.py --weights yolov5m/yolov5m.onnx
Validate: python val.py --weights yolov5m/yolov5m.onnx
PyTorch Hub: model = torch.hub.load('ultralytics/yolov5', 'custom', 'yolov5m/yolov5m.onnx')
@@ -135,12 +137,10 @@ API <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Python_API.html>`__
``openvino.tools.mo.convert_model`` function to convert ONNX model to
OpenVINO Model, then it can be seralized using
``openvino.runtime.serialize``\ As the result, directory with the
``{MODEL_DIR}`` name will be created with the following content: \*
``{MODEL_NAME}_fp32.xml``, ``{MODEL_NAME}_fp32.bin`` - OpenVINO
Intermediate Representation (IR) model format with FP32 precision
generated by Model Optimizer. \* ``{MODEL_NAME}_fp16.xml``,
``{MODEL_NAME}_fp16.bin`` - OpenVINO Intermediate Representation (IR)
model format with FP32 precision generated by Model Optimizer.
``{MODEL_DIR}`` name will be created with the following content:
* ``{MODEL_NAME}_fp32.xml``, ``{MODEL_NAME}_fp32.bin`` - OpenVINO Intermediate Representation (IR) model format with FP32 precision generated by Model Optimizer.
* ``{MODEL_NAME}_fp16.xml``, ``{MODEL_NAME}_fp16.bin`` - OpenVINO Intermediate Representation (IR) model format with FP32 precision generated by Model Optimizer.
.. code:: ipython3
@@ -229,8 +229,8 @@ first.
.. parsed-literal::
Unzipping datasets/coco128.zip...
Scanning /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017... 126 images, 2 backgrounds, 0 corrupt: 100%|██████████| 128/128 00:00
New cache created: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017.cache
Scanning /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017... 126 images, 2 backgrounds, 0 corrupt: 100%|██████████| 128/128 00:00
New cache created: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017.cache
Create YOLOv5 DataLoader class for POT
@@ -304,7 +304,7 @@ index. Any implementation should override the following methods:
.. parsed-literal::
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/offline_transformations/__init__.py:10: FutureWarning: The module is private and following namespace `offline_transformations` will be removed in the future.
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/offline_transformations/__init__.py:10: FutureWarning: The module is private and following namespace `offline_transformations` will be removed in the future.
warnings.warn(
@@ -483,8 +483,8 @@ device for inferece and can be saved on disk using
.. parsed-literal::
Statistics collection: 43%|████▎ | 128/300 [00:30<00:41, 4.18it/s]
Biases correction: 100%|██████████| 82/82 [00:10<00:00, 7.71it/s]
Statistics collection: 43%|████▎ | 128/300 [00:30<00:41, 4.17it/s]
Biases correction: 100%|██████████| 82/82 [00:10<00:00, 7.86it/s]
Compare accuracy FP32 and INT8 models
@@ -551,10 +551,10 @@ same directory, where model located.
.. parsed-literal::
Forcing --batch-size 1 square inference (1,3,640,640) for non-PyTorch models
val: Scanning /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017.cache... 126 images, 2 backgrounds, 0 corrupt: 100%|██████████| 128/128 00:00
val: Scanning /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017.cache... 126 images, 2 backgrounds, 0 corrupt: 100%|██████████| 128/128 00:00
Class Images Instances P R mAP50 mAP50-95: 100%|██████████| 128/128 00:05
all 128 929 0.726 0.687 0.769 0.554
Speed: 0.2ms pre-process, 35.3ms inference, 3.1ms NMS per image at shape (1, 3, 640, 640)
Speed: 0.2ms pre-process, 35.3ms inference, 3.2ms NMS per image at shape (1, 3, 640, 640)
Results saved to yolov5/runs/val/exp
@@ -598,10 +598,10 @@ same directory, where model located.
.. parsed-literal::
Forcing --batch-size 1 square inference (1,3,640,640) for non-PyTorch models
val: Scanning /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017.cache... 126 images, 2 backgrounds, 0 corrupt: 100%|██████████| 128/128 00:00
val: Scanning /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017.cache... 126 images, 2 backgrounds, 0 corrupt: 100%|██████████| 128/128 00:00
Class Images Instances P R mAP50 mAP50-95: 100%|██████████| 128/128 00:03
all 128 929 0.761 0.677 0.773 0.548
Speed: 0.2ms pre-process, 17.1ms inference, 3.3ms NMS per image at shape (1, 3, 640, 640)
Speed: 0.2ms pre-process, 17.3ms inference, 3.3ms NMS per image at shape (1, 3, 640, 640)
Results saved to yolov5/runs/val/exp2
@@ -645,10 +645,10 @@ same directory, where model located.
.. parsed-literal::
Forcing --batch-size 1 square inference (1,3,640,640) for non-PyTorch models
val: Scanning /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017.cache... 126 images, 2 backgrounds, 0 corrupt: 100%|██████████| 128/128 00:00
val: Scanning /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017.cache... 126 images, 2 backgrounds, 0 corrupt: 100%|██████████| 128/128 00:00
Class Images Instances P R mAP50 mAP50-95: 100%|██████████| 128/128 00:03
all 128 929 0.742 0.684 0.766 0.546
Speed: 0.2ms pre-process, 17.0ms inference, 3.2ms NMS per image at shape (1, 3, 640, 640)
Speed: 0.2ms pre-process, 17.1ms inference, 3.3ms NMS per image at shape (1, 3, 640, 640)
Results saved to yolov5/runs/val/exp3
@@ -744,9 +744,9 @@ images.
'YOLOv5 🚀 v7.0-0-g915bbf2 Python-3.8.10 torch-1.13.1+cpu CPU',
'',
'Loading yolov5m/FP32_openvino_model for OpenVINO inference...',
'image 1/2 /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/bus.jpg: 640x640 4 persons, 1 bus, 64.1ms',
'image 2/2 /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/zidane.jpg: 640x640 3 persons, 2 ties, 60.5ms',
'Speed: 1.6ms pre-process, 62.3ms inference, 1.4ms NMS per image at shape (1, 3, 640, 640)',
'image 1/2 /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/bus.jpg: 640x640 4 persons, 1 bus, 56.6ms',
'image 2/2 /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/zidane.jpg: 640x640 3 persons, 2 ties, 43.1ms',
'Speed: 1.4ms pre-process, 49.8ms inference, 1.2ms NMS per image at shape (1, 3, 640, 640)',
'Results saved to \x1b[1mruns/detect/exp\x1b[0m']
@@ -771,9 +771,9 @@ images.
'YOLOv5 🚀 v7.0-0-g915bbf2 Python-3.8.10 torch-1.13.1+cpu CPU',
'',
'Loading yolov5m/POT_INT8_openvino_model for OpenVINO inference...',
'image 1/2 /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/bus.jpg: 640x640 4 persons, 1 bus, 36.4ms',
'image 2/2 /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/zidane.jpg: 640x640 3 persons, 1 tie, 30.9ms',
'Speed: 1.6ms pre-process, 33.6ms inference, 1.5ms NMS per image at shape (1, 3, 640, 640)',
'image 1/2 /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/bus.jpg: 640x640 4 persons, 1 bus, 38.2ms',
'image 2/2 /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/zidane.jpg: 640x640 3 persons, 1 tie, 33.4ms',
'Speed: 1.6ms pre-process, 35.8ms inference, 1.4ms NMS per image at shape (1, 3, 640, 640)',
'Results saved to \x1b[1mruns/detect/exp2\x1b[0m']
@@ -798,9 +798,9 @@ images.
'YOLOv5 🚀 v7.0-0-g915bbf2 Python-3.8.10 torch-1.13.1+cpu CPU',
'',
'Loading yolov5m/NNCF_INT8_openvino_model for OpenVINO inference...',
'image 1/2 /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/bus.jpg: 640x640 4 persons, 1 bus, 44.1ms',
'image 2/2 /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/zidane.jpg: 640x640 3 persons, 2 ties, 35.7ms',
'Speed: 1.6ms pre-process, 39.9ms inference, 1.8ms NMS per image at shape (1, 3, 640, 640)',
'image 1/2 /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/bus.jpg: 640x640 4 persons, 1 bus, 37.5ms',
'image 2/2 /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/zidane.jpg: 640x640 3 persons, 2 ties, 32.4ms',
'Speed: 1.5ms pre-process, 35.0ms inference, 1.3ms NMS per image at shape (1, 3, 640, 640)',
'Results saved to \x1b[1mruns/detect/exp3\x1b[0m']
@@ -864,7 +864,7 @@ Benchmark
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 34.71 ms
[ INFO ] Read model took 34.56 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] images (node: images) : f32 / [...] / [1,3,640,640]
@@ -878,7 +878,7 @@ Benchmark
[ INFO ] Model outputs:
[ INFO ] output0 (node: output0) : f32 / [...] / [1,25200,85]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 364.66 ms
[ INFO ] Compile model took 382.88 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
@@ -900,17 +900,17 @@ Benchmark
[ INFO ] Fill input 'images' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 103.67 ms
[ INFO ] First inference took 106.76 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 456 iterations
[ INFO ] Duration: 15243.71 ms
[ INFO ] Count: 450 iterations
[ INFO ] Duration: 15298.04 ms
[ INFO ] Latency:
[ INFO ] Median: 200.52 ms
[ INFO ] Average: 199.48 ms
[ INFO ] Min: 167.24 ms
[ INFO ] Max: 219.12 ms
[ INFO ] Throughput: 29.91 FPS
[ INFO ] Median: 204.03 ms
[ INFO ] Average: 202.87 ms
[ INFO ] Min: 140.18 ms
[ INFO ] Max: 217.69 ms
[ INFO ] Throughput: 29.42 FPS
.. code:: ipython3
@@ -941,7 +941,7 @@ Benchmark
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 34.33 ms
[ INFO ] Read model took 39.27 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] images (node: images) : f32 / [...] / [1,3,640,640]
@@ -955,7 +955,7 @@ Benchmark
[ INFO ] Model outputs:
[ INFO ] output0 (node: output0) : f32 / [...] / [1,25200,85]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 388.69 ms
[ INFO ] Compile model took 409.09 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
@@ -977,17 +977,17 @@ Benchmark
[ INFO ] Fill input 'images' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 104.81 ms
[ INFO ] First inference took 103.11 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 462 iterations
[ INFO ] Duration: 15265.35 ms
[ INFO ] Count: 456 iterations
[ INFO ] Duration: 15294.88 ms
[ INFO ] Latency:
[ INFO ] Median: 198.22 ms
[ INFO ] Average: 197.86 ms
[ INFO ] Min: 167.44 ms
[ INFO ] Max: 229.61 ms
[ INFO ] Throughput: 30.26 FPS
[ INFO ] Median: 201.67 ms
[ INFO ] Average: 200.70 ms
[ INFO ] Min: 124.86 ms
[ INFO ] Max: 218.07 ms
[ INFO ] Throughput: 29.81 FPS
.. code:: ipython3
@@ -1018,7 +1018,7 @@ Benchmark
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 68.51 ms
[ INFO ] Read model took 47.44 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] images (node: images) : f32 / [...] / [1,3,640,640]
@@ -1032,7 +1032,7 @@ Benchmark
[ INFO ] Model outputs:
[ INFO ] output0 (node: output0) : f32 / [...] / [1,25200,85]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 706.48 ms
[ INFO ] Compile model took 705.56 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
@@ -1054,17 +1054,17 @@ Benchmark
[ INFO ] Fill input 'images' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 49.42 ms
[ INFO ] First inference took 50.77 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 1416 iterations
[ INFO ] Duration: 15057.61 ms
[ INFO ] Count: 1422 iterations
[ INFO ] Duration: 15093.38 ms
[ INFO ] Latency:
[ INFO ] Median: 63.81 ms
[ INFO ] Average: 63.58 ms
[ INFO ] Min: 47.42 ms
[ INFO ] Max: 84.90 ms
[ INFO ] Throughput: 94.04 FPS
[ INFO ] Median: 63.57 ms
[ INFO ] Average: 63.51 ms
[ INFO ] Min: 44.75 ms
[ INFO ] Max: 86.46 ms
[ INFO ] Throughput: 94.21 FPS
.. code:: ipython3
@@ -1095,7 +1095,7 @@ Benchmark
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 60.65 ms
[ INFO ] Read model took 53.05 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] images (node: images) : f32 / [...] / [1,3,640,640]
@@ -1109,7 +1109,7 @@ Benchmark
[ INFO ] Model outputs:
[ INFO ] output0 (node: output0) : f32 / [...] / [1,25200,85]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 715.09 ms
[ INFO ] Compile model took 714.97 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
@@ -1131,17 +1131,17 @@ Benchmark
[ INFO ] Fill input 'images' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 50.63 ms
[ INFO ] First inference took 53.03 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 1422 iterations
[ INFO ] Duration: 15094.14 ms
[ INFO ] Duration: 15073.87 ms
[ INFO ] Latency:
[ INFO ] Median: 63.67 ms
[ INFO ] Average: 63.52 ms
[ INFO ] Min: 47.40 ms
[ INFO ] Max: 86.32 ms
[ INFO ] Throughput: 94.21 FPS
[ INFO ] Median: 63.63 ms
[ INFO ] Average: 63.46 ms
[ INFO ] Min: 53.07 ms
[ INFO ] Max: 85.38 ms
[ INFO ] Throughput: 94.34 FPS
References

8
docs/notebooks/111-yolov5-quantization-migration-with-output_files/index.html
View File

@@ -1,8 +1,8 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/111-yolov5-quantization-migration-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/111-yolov5-quantization-migration-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/111-yolov5-quantization-migration-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="111-yolov5-quantization-migration-with-output_33_0.png">111-yolov5-quantization-migration-with-output_3..&gt;</a> 22-Jun-2023 00:06 33667
<a href="111-yolov5-quantization-migration-with-output_39_0.png">111-yolov5-quantization-migration-with-output_3..&gt;</a> 22-Jun-2023 00:06 770524
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/111-yolov5-quantization-migration-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="111-yolov5-quantization-migration-with-output_33_0.png">111-yolov5-quantization-migration-with-output_3..&gt;</a> 12-Jul-2023 00:11 33667
<a href="111-yolov5-quantization-migration-with-output_39_0.png">111-yolov5-quantization-migration-with-output_3..&gt;</a> 12-Jul-2023 00:11 770524
</pre><hr></body>
</html>

177
docs/notebooks/112-pytorch-post-training-quantization-nncf-with-output.rst
View File

@@ -88,10 +88,10 @@ Imports
.. parsed-literal::
2023-06-21 22:46:53.036679: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-06-21 22:46:53.070884: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2023-07-11 22:44:00.778229: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-07-11 22:44:00.812197: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-06-21 22:46:53.615742: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2023-07-11 22:44:01.354577: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
.. parsed-literal::
@@ -104,8 +104,8 @@ Settings
.. code:: ipython3
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Using {device} device")
torch_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Using {torch_device} device")
MODEL_DIR = Path("model")
OUTPUT_DIR = Path("output")
@@ -142,7 +142,7 @@ Settings
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/112-pytorch-post-training-quantization-nncf/model/resnet50_fp32.pth')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/112-pytorch-post-training-quantization-nncf/model/resnet50_fp32.pth')
@@ -291,13 +291,13 @@ Validation function
# Switch to evaluate mode.
if not isinstance(model, CompiledModel):
model.eval()
model.to(device)
model.to(torch_device)
with torch.no_grad():
end = time.time()
for i, (images, target) in enumerate(val_loader):
images = images.to(device)
target = target.to(device)
images = images.to(torch_device)
target = target.to(torch_device)
# Compute the output.
if isinstance(model, CompiledModel):
@@ -342,7 +342,7 @@ values.
# Update the last FC layer for Tiny ImageNet number of classes.
NUM_CLASSES = 200
model.fc = torch.nn.Linear(in_features=2048, out_features=NUM_CLASSES, bias=True)
model.to(device)
model.to(torch_device)
if model_path.exists():
checkpoint = torch.load(str(model_path), map_location="cpu")
model.load_state_dict(checkpoint["state_dict"], strict=True)
@@ -421,15 +421,15 @@ I. Evaluate the loaded model
.. parsed-literal::
Test: [ 0/79] Time 0.292 (0.292) Acc@1 81.25 (81.25) Acc@5 92.19 (92.19)
Test: [10/79] Time 0.234 (0.243) Acc@1 56.25 (66.97) Acc@5 86.72 (87.50)
Test: [ 0/79] Time 0.253 (0.253) Acc@1 81.25 (81.25) Acc@5 92.19 (92.19)
Test: [10/79] Time 0.238 (0.240) Acc@1 56.25 (66.97) Acc@5 86.72 (87.50)
Test: [20/79] Time 0.235 (0.239) Acc@1 67.97 (64.29) Acc@5 85.16 (87.35)
Test: [30/79] Time 0.233 (0.239) Acc@1 53.12 (62.37) Acc@5 77.34 (85.33)
Test: [40/79] Time 0.233 (0.238) Acc@1 67.19 (60.86) Acc@5 90.62 (84.51)
Test: [50/79] Time 0.284 (0.239) Acc@1 60.16 (60.80) Acc@5 88.28 (84.42)
Test: [60/79] Time 0.236 (0.240) Acc@1 66.41 (60.46) Acc@5 86.72 (83.79)
Test: [70/79] Time 0.233 (0.239) Acc@1 52.34 (60.21) Acc@5 80.47 (83.33)
* Acc@1 60.740 Acc@5 83.960 Total time: 18.662
Test: [30/79] Time 0.234 (0.238) Acc@1 53.12 (62.37) Acc@5 77.34 (85.33)
Test: [40/79] Time 0.235 (0.238) Acc@1 67.19 (60.86) Acc@5 90.62 (84.51)
Test: [50/79] Time 0.232 (0.240) Acc@1 60.16 (60.80) Acc@5 88.28 (84.42)
Test: [60/79] Time 0.251 (0.240) Acc@1 66.41 (60.46) Acc@5 86.72 (83.79)
Test: [70/79] Time 0.237 (0.240) Acc@1 52.34 (60.21) Acc@5 80.47 (83.33)
* Acc@1 60.740 Acc@5 83.960 Total time: 18.698
Test accuracy of FP32 model: 60.740
@@ -498,16 +498,16 @@ Guide <https://docs.openvino.ai/2023.0/basic_qauntization_flow.html#doxid-basic-
.. parsed-literal::
Test: [ 0/79] Time 0.427 (0.427) Acc@1 81.25 (81.25) Acc@5 92.19 (92.19)
Test: [10/79] Time 0.420 (0.421) Acc@1 55.47 (67.33) Acc@5 87.50 (87.86)
Test: [20/79] Time 0.417 (0.419) Acc@1 67.97 (64.32) Acc@5 88.28 (87.69)
Test: [30/79] Time 0.421 (0.420) Acc@1 52.34 (62.53) Acc@5 78.91 (85.56)
Test: [40/79] Time 0.422 (0.420) Acc@1 67.97 (61.01) Acc@5 90.62 (84.76)
Test: [50/79] Time 0.421 (0.421) Acc@1 58.59 (60.85) Acc@5 87.50 (84.60)
Test: [60/79] Time 0.423 (0.421) Acc@1 66.41 (60.50) Acc@5 87.50 (83.94)
Test: [70/79] Time 0.422 (0.421) Acc@1 50.00 (60.22) Acc@5 78.12 (83.45)
* Acc@1 60.730 Acc@5 84.060 Total time: 32.959
Accuracy of initialized INT8 model: 60.730
Test: [ 0/79] Time 0.438 (0.438) Acc@1 81.25 (81.25) Acc@5 90.62 (90.62)
Test: [10/79] Time 0.428 (0.429) Acc@1 57.81 (67.26) Acc@5 86.72 (87.36)
Test: [20/79] Time 0.430 (0.429) Acc@1 67.97 (64.21) Acc@5 85.94 (87.43)
Test: [30/79] Time 0.429 (0.429) Acc@1 52.34 (62.45) Acc@5 75.78 (85.28)
Test: [40/79] Time 0.429 (0.429) Acc@1 68.75 (60.88) Acc@5 89.84 (84.47)
Test: [50/79] Time 0.429 (0.429) Acc@1 59.38 (60.81) Acc@5 87.50 (84.34)
Test: [60/79] Time 0.428 (0.429) Acc@1 66.41 (60.49) Acc@5 85.94 (83.68)
Test: [70/79] Time 0.434 (0.429) Acc@1 54.69 (60.27) Acc@5 78.91 (83.24)
* Acc@1 60.810 Acc@5 83.880 Total time: 33.587
Accuracy of initialized INT8 model: 60.810
It should be noted that the inference time for the quantized PyTorch
@@ -549,63 +549,88 @@ command may take a while.
.. parsed-literal::
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:338: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:338: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
return self._level_low.item()
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:346: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:346: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
return self._level_high.item()
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/quantize_functions.py:140: FutureWarning: 'torch.onnx._patch_torch._graph_op' is deprecated in version 1.13 and will be removed in version 1.14. Please note 'g.op()' is to be removed from torch.Graph. Please open a GitHub issue if you need this functionality..
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/quantize_functions.py:140: FutureWarning: 'torch.onnx._patch_torch._graph_op' is deprecated in version 1.13 and will be removed in version 1.14. Please note 'g.op()' is to be removed from torch.Graph. Please open a GitHub issue if you need this functionality..
output = g.op(
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/_patch_torch.py:81: UserWarning: The shape inference of org.openvinotoolkit::FakeQuantize type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/_patch_torch.py:81: UserWarning: The shape inference of org.openvinotoolkit::FakeQuantize type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
_C._jit_pass_onnx_node_shape_type_inference(
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:687: UserWarning: The shape inference of org.openvinotoolkit::FakeQuantize type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:687: UserWarning: The shape inference of org.openvinotoolkit::FakeQuantize type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
_C._jit_pass_onnx_graph_shape_type_inference(
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:1178: UserWarning: The shape inference of org.openvinotoolkit::FakeQuantize type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:1178: UserWarning: The shape inference of org.openvinotoolkit::FakeQuantize type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
_C._jit_pass_onnx_graph_shape_type_inference(
Select inference device for OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
Evaluate the FP32 and INT8 models.
.. code:: ipython3
core = Core()
fp32_compiled_model = core.compile_model(model_ir)
fp32_compiled_model = core.compile_model(model_ir, device.value)
acc1 = validate(val_loader, fp32_compiled_model)
print(f"Accuracy of FP32 IR model: {acc1:.3f}")
.. parsed-literal::
Test: [ 0/79] Time 0.200 (0.200) Acc@1 81.25 (81.25) Acc@5 92.19 (92.19)
Test: [10/79] Time 0.140 (0.145) Acc@1 56.25 (66.97) Acc@5 86.72 (87.50)
Test: [20/79] Time 0.140 (0.143) Acc@1 67.97 (64.29) Acc@5 85.16 (87.35)
Test: [30/79] Time 0.140 (0.142) Acc@1 53.12 (62.37) Acc@5 77.34 (85.33)
Test: [40/79] Time 0.138 (0.141) Acc@1 67.19 (60.86) Acc@5 90.62 (84.51)
Test: [50/79] Time 0.140 (0.141) Acc@1 60.16 (60.80) Acc@5 88.28 (84.42)
Test: [60/79] Time 0.140 (0.141) Acc@1 66.41 (60.46) Acc@5 86.72 (83.79)
Test: [70/79] Time 0.138 (0.141) Acc@1 52.34 (60.21) Acc@5 80.47 (83.33)
* Acc@1 60.740 Acc@5 83.960 Total time: 11.002
Test: [ 0/79] Time 0.205 (0.205) Acc@1 81.25 (81.25) Acc@5 92.19 (92.19)
Test: [10/79] Time 0.137 (0.143) Acc@1 56.25 (66.97) Acc@5 86.72 (87.50)
Test: [20/79] Time 0.145 (0.141) Acc@1 67.97 (64.29) Acc@5 85.16 (87.35)
Test: [30/79] Time 0.137 (0.140) Acc@1 53.12 (62.37) Acc@5 77.34 (85.33)
Test: [40/79] Time 0.137 (0.140) Acc@1 67.19 (60.86) Acc@5 90.62 (84.51)
Test: [50/79] Time 0.139 (0.140) Acc@1 60.16 (60.80) Acc@5 88.28 (84.42)
Test: [60/79] Time 0.135 (0.139) Acc@1 66.41 (60.46) Acc@5 86.72 (83.79)
Test: [70/79] Time 0.139 (0.139) Acc@1 52.34 (60.21) Acc@5 80.47 (83.33)
* Acc@1 60.740 Acc@5 83.960 Total time: 10.882
Accuracy of FP32 IR model: 60.740
.. code:: ipython3
int8_compiled_model = core.compile_model(quantized_model_ir)
int8_compiled_model = core.compile_model(quantized_model_ir, device.value)
acc1 = validate(val_loader, int8_compiled_model)
print(f"Accuracy of INT8 IR model: {acc1:.3f}")
.. parsed-literal::
Test: [ 0/79] Time 0.186 (0.186) Acc@1 82.03 (82.03) Acc@5 92.97 (92.97)
Test: [10/79] Time 0.083 (0.095) Acc@1 57.81 (67.90) Acc@5 86.72 (88.28)
Test: [20/79] Time 0.079 (0.088) Acc@1 67.19 (64.88) Acc@5 85.16 (87.72)
Test: [30/79] Time 0.081 (0.085) Acc@1 52.34 (62.80) Acc@5 78.12 (85.61)
Test: [40/79] Time 0.082 (0.084) Acc@1 67.97 (61.32) Acc@5 89.84 (84.60)
Test: [50/79] Time 0.081 (0.084) Acc@1 60.16 (61.09) Acc@5 87.50 (84.47)
Test: [60/79] Time 0.079 (0.083) Acc@1 67.97 (60.71) Acc@5 88.28 (83.80)
Test: [70/79] Time 0.079 (0.082) Acc@1 53.12 (60.35) Acc@5 80.47 (83.37)
* Acc@1 60.880 Acc@5 83.950 Total time: 6.438
Accuracy of INT8 IR model: 60.880
Test: [ 0/79] Time 0.187 (0.187) Acc@1 81.25 (81.25) Acc@5 90.62 (90.62)
Test: [10/79] Time 0.082 (0.091) Acc@1 57.81 (66.90) Acc@5 85.94 (87.78)
Test: [20/79] Time 0.080 (0.086) Acc@1 67.97 (64.06) Acc@5 83.59 (87.31)
Test: [30/79] Time 0.081 (0.084) Acc@1 52.34 (62.17) Acc@5 78.12 (85.36)
Test: [40/79] Time 0.079 (0.083) Acc@1 67.97 (60.80) Acc@5 89.84 (84.38)
Test: [50/79] Time 0.077 (0.082) Acc@1 60.16 (60.71) Acc@5 87.50 (84.30)
Test: [60/79] Time 0.079 (0.082) Acc@1 67.19 (60.43) Acc@5 87.50 (83.72)
Test: [70/79] Time 0.082 (0.081) Acc@1 53.12 (60.17) Acc@5 80.47 (83.31)
* Acc@1 60.730 Acc@5 83.930 Total time: 6.353
Accuracy of INT8 IR model: 60.730
IV. Compare performance of INT8 model and FP32 model in OpenVINO
@@ -627,6 +652,19 @@ throughput (frames per second) values.
to benchmark on GPU. Run ``benchmark_app --help`` to see an overview
of all command-line options.
.. code:: ipython3
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
.. code:: ipython3
def parse_benchmark_output(benchmark_output: str):
@@ -636,46 +674,47 @@ throughput (frames per second) values.
print('Benchmark FP32 model (OpenVINO IR)')
benchmark_output = ! benchmark_app -m "$fp32_ir_path" -d CPU -api async -t 15 -shape "[1, 3, 512, 512]"
benchmark_output = ! benchmark_app -m "$fp32_ir_path" -d $device.value -api async -t 15 -shape "[1, 3, 512, 512]"
parse_benchmark_output(benchmark_output)
print('Benchmark INT8 model (OpenVINO IR)')
benchmark_output = ! benchmark_app -m "$int8_ir_path" -d CPU -api async -t 15 -shape "[1, 3, 512, 512]"
benchmark_output = ! benchmark_app -m "$int8_ir_path" -d $device.value -api async -t 15 -shape "[1, 3, 512, 512]"
parse_benchmark_output(benchmark_output)
print('Benchmark FP32 model (OpenVINO IR) synchronously')
benchmark_output = ! benchmark_app -m "$fp32_ir_path" -d CPU -api sync -t 15 -shape "[1, 3, 512, 512]"
benchmark_output = ! benchmark_app -m "$fp32_ir_path" -d $device.value -api sync -t 15 -shape "[1, 3, 512, 512]"
parse_benchmark_output(benchmark_output)
print('Benchmark INT8 model (OpenVINO IR) synchronously')
benchmark_output = ! benchmark_app -m "$int8_ir_path" -d CPU -api sync -t 15 -shape "[1, 3, 512, 512]"
benchmark_output = ! benchmark_app -m "$int8_ir_path" -d $device.value -api sync -t 15 -shape "[1, 3, 512, 512]"
parse_benchmark_output(benchmark_output)
.. parsed-literal::
Benchmark FP32 model (OpenVINO IR)
[ INFO ] Throughput: 37.97 FPS
[ INFO ] Throughput: 37.61 FPS
Benchmark INT8 model (OpenVINO IR)
[ INFO ] Throughput: 157.50 FPS
[ INFO ] Throughput: 159.33 FPS
Benchmark FP32 model (OpenVINO IR) synchronously
[ INFO ] Throughput: 39.17 FPS
[ INFO ] Throughput: 39.06 FPS
Benchmark INT8 model (OpenVINO IR) synchronously
[ INFO ] Throughput: 140.62 FPS
[ INFO ] Throughput: 140.24 FPS
Show CPU Information for reference:
Show device Information for reference:
.. code:: ipython3
ie = Core()
ie.get_property("CPU", "FULL_DEVICE_NAME")
core = Core()
devices = core.available_devices
for device_name in devices:
device_full_name = core.get_property(device_name, "FULL_DEVICE_NAME")
print(f"{device_name}: {device_full_name}")
.. parsed-literal::
'Intel(R) Core(TM) i9-10920X CPU @ 3.50GHz'
CPU: Intel(R) Core(TM) i9-10920X CPU @ 3.50GHz

178
docs/notebooks/113-image-classification-quantization-with-output.rst
View File

@@ -10,11 +10,14 @@ backend for performing model quantization in NNCF, if you interested how
to apply quantization on PyTorch model, please check this
`tutorial <112-pytorch-post-training-quantization-nncf-with-output.html>`__.
This tutorial consists of the following steps: - Prepare the model for
quantization. - Define a data loading functionality. - Perform
quantization. - Compare accuracy of the original and quantized models. -
Compare performance of the original and quantized models. - Compare
results on one picture.
This tutorial consists of the following steps:
- Prepare the model for quantization.
- Define a data loading functionality.
- Perform quantization.
- Compare accuracy of the original and quantized models.
- Compare performance of the original and quantized models.
- Compare results on one picture.
.. code:: ipython3
@@ -31,10 +34,11 @@ results on one picture.
Prepare the Model
-----------------
Model preparation stage has the following steps: - Download a PyTorch
model - Convert model to OpenVINO Intermediate Representation format
(IR) using Model Optimizer Python API - Serialize converted model on
disk
Model preparation stage has the following steps:
- Download a PyTorch model
- Convert model to OpenVINO Intermediate Representation format (IR) using Model Optimizer Python API
- Serialize converted model on disk
.. code:: ipython3
@@ -51,10 +55,10 @@ disk
Cloning into 'pytorch-cifar-models'...
remote: Enumerating objects: 282, done.
remote: Counting objects: 100% (281/281), done.
remote: Compressing objects: 100% (96/96), done.
remote: Total 282 (delta 135), reused 269 (delta 128), pack-reused 1
remote: Compressing objects: 100% (95/95), done.
remote: Total 282 (delta 136), reused 269 (delta 129), pack-reused 1
Receiving objects: 100% (282/282), 9.22 MiB | 4.20 MiB/s, done.
Resolving deltas: 100% (135/135), done.
Resolving deltas: 100% (136/136), done.
.. code:: ipython3
@@ -184,8 +188,8 @@ about supported parameters can be found on this
.. parsed-literal::
Statistics collection: 100%|██████████| 300/300 [00:08<00:00, 35.76it/s]
Biases correction: 100%|██████████| 36/36 [00:01<00:00, 20.36it/s]
Statistics collection: 100%|██████████| 300/300 [00:08<00:00, 35.03it/s]
Biases correction: 100%|██████████| 36/36 [00:01<00:00, 23.98it/s]
Serialize an OpenVINO IR model
@@ -217,13 +221,43 @@ Compare Accuracy of the Original and Quantized Models
total += 1
return correct / total
Select inference device
~~~~~~~~~~~~~~~~~~~~~~~
select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
from openvino.runtime import Core
core = Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
.. code:: ipython3
from openvino.runtime import Core
core = Core()
compiled_model = core.compile_model(ov_model)
optimized_compiled_model = core.compile_model(quant_ov_model)
compiled_model = core.compile_model(ov_model, device.value)
optimized_compiled_model = core.compile_model(quant_ov_model, device.value)
orig_accuracy = test_accuracy(compiled_model, val_loader)
optimized_accuracy = test_accuracy(optimized_compiled_model, val_loader)
@@ -271,7 +305,7 @@ Tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_
.. code:: ipython3
# Inference FP16 model (OpenVINO IR)
!benchmark_app -m "model/mobilenet_v2.xml" -d CPU -api async -t 15
!benchmark_app -m "model/mobilenet_v2.xml" -d $device.value -api async -t 15
.. parsed-literal::
@@ -283,15 +317,15 @@ Tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] AUTO
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 25.72 ms
[ INFO ] Read model took 7.88 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] x.1 , x , 1 (node: Parameter_2) : f32 / [...] / [1,3,32,32]
@@ -305,45 +339,52 @@ Tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_
[ INFO ] Model outputs:
[ INFO ] 223 (node: aten::linear_928) : f32 / [...] / [1,10]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 144.38 ms
[ INFO ] Compile model took 164.70 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] NETWORK_NAME: Model0
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 12
[ INFO ] NUM_STREAMS: 12
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] INFERENCE_NUM_THREADS: 24
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] MODEL_PRIORITY: Priority.MEDIUM
[ INFO ] MULTI_DEVICE_PRIORITIES: CPU
[ INFO ] CPU:
[ INFO ] CPU_BIND_THREAD: YES
[ INFO ] CPU_THREADS_NUM: 0
[ INFO ] CPU_THROUGHPUT_STREAMS: 12
[ INFO ] DEVICE_ID:
[ INFO ] DUMP_EXEC_GRAPH_AS_DOT:
[ INFO ] DYN_BATCH_ENABLED: NO
[ INFO ] DYN_BATCH_LIMIT: 0
[ INFO ] ENFORCE_BF16: NO
[ INFO ] EXCLUSIVE_ASYNC_REQUESTS: NO
[ INFO ] NETWORK_NAME: Model0
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 12
[ INFO ] PERFORMANCE_HINT: THROUGHPUT
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] PERF_COUNT: NO
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input '1'!. This input will be filled with random values!
[ INFO ] Fill input '1' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 12 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 3.09 ms
[ INFO ] First inference took 3.15 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 90984 iterations
[ INFO ] Duration: 15002.82 ms
[ INFO ] Count: 89880 iterations
[ INFO ] Duration: 15002.85 ms
[ INFO ] Latency:
[ INFO ] Median: 1.78 ms
[ INFO ] Average: 1.80 ms
[ INFO ] Min: 1.10 ms
[ INFO ] Max: 6.27 ms
[ INFO ] Throughput: 6064.46 FPS
[ INFO ] Median: 1.80 ms
[ INFO ] Average: 1.82 ms
[ INFO ] Min: 1.13 ms
[ INFO ] Max: 7.45 ms
[ INFO ] Throughput: 5990.86 FPS
.. code:: ipython3
# Inference INT8 model (OpenVINO IR)
!benchmark_app -m "model/quantized_mobilenet_v2.xml" -d CPU -api async -t 15
!benchmark_app -m "model/quantized_mobilenet_v2.xml" -d $device.value -api async -t 15
.. parsed-literal::
@@ -355,61 +396,68 @@ Tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] AUTO
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 34.93 ms
[ INFO ] Read model took 15.06 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] x , 1 , x.1 (node: Parameter_2) : f32 / [...] / [1,3,32,32]
[ INFO ] 1 , x.1 , x (node: Parameter_2) : f32 / [...] / [1,3,32,32]
[ INFO ] Model outputs:
[ INFO ] 223 (node: aten::linear_928) : f32 / [...] / [1,10]
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] x , 1 , x.1 (node: Parameter_2) : u8 / [N,C,H,W] / [1,3,32,32]
[ INFO ] 1 , x.1 , x (node: Parameter_2) : u8 / [N,C,H,W] / [1,3,32,32]
[ INFO ] Model outputs:
[ INFO ] 223 (node: aten::linear_928) : f32 / [...] / [1,10]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 247.66 ms
[ INFO ] Compile model took 314.58 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] NETWORK_NAME: Model0
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 12
[ INFO ] NUM_STREAMS: 12
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] INFERENCE_NUM_THREADS: 24
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] MODEL_PRIORITY: Priority.MEDIUM
[ INFO ] MULTI_DEVICE_PRIORITIES: CPU
[ INFO ] CPU:
[ INFO ] CPU_BIND_THREAD: YES
[ INFO ] CPU_THREADS_NUM: 0
[ INFO ] CPU_THROUGHPUT_STREAMS: 12
[ INFO ] DEVICE_ID:
[ INFO ] DUMP_EXEC_GRAPH_AS_DOT:
[ INFO ] DYN_BATCH_ENABLED: NO
[ INFO ] DYN_BATCH_LIMIT: 0
[ INFO ] ENFORCE_BF16: NO
[ INFO ] EXCLUSIVE_ASYNC_REQUESTS: NO
[ INFO ] NETWORK_NAME: Model0
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 12
[ INFO ] PERFORMANCE_HINT: THROUGHPUT
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] PERF_COUNT: NO
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input '1'!. This input will be filled with random values!
[ INFO ] Fill input '1' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 12 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 1.68 ms
[ INFO ] First inference took 1.86 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 183924 iterations
[ INFO ] Duration: 15000.75 ms
[ INFO ] Count: 181056 iterations
[ INFO ] Duration: 15001.51 ms
[ INFO ] Latency:
[ INFO ] Median: 0.92 ms
[ INFO ] Average: 0.94 ms
[ INFO ] Min: 0.64 ms
[ INFO ] Max: 3.03 ms
[ INFO ] Throughput: 12260.99 FPS
[ INFO ] Median: 0.93 ms
[ INFO ] Average: 0.96 ms
[ INFO ] Min: 0.62 ms
[ INFO ] Max: 3.36 ms
[ INFO ] Throughput: 12069.19 FPS
Compare results on four pictures
@@ -503,5 +551,5 @@ Compare results on four pictures
.. image:: 113-image-classification-quantization-with-output_files/113-image-classification-quantization-with-output_26_2.png
.. image:: 113-image-classification-quantization-with-output_files/113-image-classification-quantization-with-output_28_2.png

0
docs/notebooks/113-image-classification-quantization-with-output_files/113-image-classification-quantization-with-output_26_2.png → docs/notebooks/113-image-classification-quantization-with-output_files/113-image-classification-quantization-with-output_28_2.png
View File

6
docs/notebooks/113-image-classification-quantization-with-output_files/index.html
View File

@@ -1,7 +1,7 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/113-image-classification-quantization-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/113-image-classification-quantization-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/113-image-classification-quantization-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="113-image-classification-quantization-with-output_26_2.png">113-image-classification-quantization-with-outp..&gt;</a> 22-Jun-2023 00:06 14855
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/113-image-classification-quantization-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="113-image-classification-quantization-with-output_28_2.png">113-image-classification-quantization-with-outp..&gt;</a> 12-Jul-2023 00:11 14855
</pre><hr></body>
</html>

39
docs/notebooks/115-async-api-with-output.rst
View File

@@ -16,7 +16,11 @@ Imports
.. code:: ipython3
import sys
!pip install -q 'openvino-dev>=2023.0.0'
!pip install -q opencv-python matplotlib
.. code:: ipython3
import cv2
import time
import numpy as np
@@ -24,7 +28,14 @@ Imports
import openvino.runtime as ov
from IPython import display
import matplotlib.pyplot as plt
sys.path.append("../utils")
# Fetch the notebook utils script from the openvino_notebooks repo
import urllib.request
urllib.request.urlretrieve(
url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py',
filename='notebook_utils.py'
)
import notebook_utils as utils
Prepare model and data processing
@@ -136,7 +147,7 @@ Get the test video
.. code:: ipython3
video_path = "../data/video/CEO Pat Gelsinger on Leading Intel.mp4"
video_path = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/video/CEO%20Pat%20Gelsinger%20on%20Leading%20Intel.mp4'
How to improve the throughput of video processing
-------------------------------------------------
@@ -151,6 +162,8 @@ Let us see how video processing works with the default approach. Using
the synchronous approach, the frame is captured with OpenCV and then
immediately processed:
.. image:: https://camo.githubusercontent.com/b77ae49e3c46a0fe3931ad02a7767d6def49aa7b2a7a8ad620a4e12ae31d43a5/68747470733a2f2f757365722d696d616765732e67697468756275736572636f6e74656e742e636f6d2f39313233373932342f3136383435323537332d64333534656135622d373936362d343465352d383133642d6639303533626534333338612e706e67
::
while(true) {
@@ -237,13 +250,11 @@ Test performance in Sync Mode
.. image:: 115-async-api-with-output_files/115-async-api-with-output_14_0.png
.. parsed-literal::
Source ended
average throuput in sync mode: 36.81 fps
average throuput in sync mode: 38.25 fps
Async Mode
@@ -256,6 +267,8 @@ do other things in parallel (for example, populating inputs or
scheduling other requests) rather than wait for the current inference to
complete first.
.. image:: https://camo.githubusercontent.com/7bcadb7cf72aefc84d74d8e971a31dab1710bb4303fef814619df638dff8be92/68747470733a2f2f757365722d696d616765732e67697468756275736572636f6e74656e742e636f6d2f39313233373932342f3136383435323537322d63326666316335392d643437302d346238352d623166362d6236653164616339353430652e706e67
In the example below, inference is applied to the results of the video
decoding. So it is possible to keep multiple infer requests, and while
the current request is processed, the input frame for the next is being
@@ -365,14 +378,10 @@ Test the performance in Async Mode
print(f"average throuput in async mode: {async_fps:.2f} fps")
.. image:: 115-async-api-with-output_files/115-async-api-with-output_18_0.png
.. parsed-literal::
Source ended
average throuput in async mode: 72.96 fps
average throuput in async mode: 71.88 fps
Compare the performance
@@ -400,9 +409,6 @@ Compare the performance
.. image:: 115-async-api-with-output_files/115-async-api-with-output_20_0.png
AsyncInferQueue
---------------
@@ -502,10 +508,7 @@ Test the performance with AsyncInferQueue
.. image:: 115-async-api-with-output_files/115-async-api-with-output_26_0.png
.. parsed-literal::
average throughput in async mode with async infer queue: 102.11 fps
average throughput in async mode with async infer queue: 104.69 fps

0
docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_14_0.png → docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_15_0.png
View File

0
docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_18_0.png → docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_19_0.png
View File

3
docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_20_0.png
View File

@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:4959e1b241bf2e8076746dfc5428e583a79b2f87d251a2b0d53dae9ee1cb4b20
size 30500

0
docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_26_0.png → docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_27_0.png
View File

12
docs/notebooks/115-async-api-with-output_files/index.html
View File

@@ -1,10 +1,10 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/115-async-api-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/115-async-api-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/115-async-api-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="115-async-api-with-output_14_0.png">115-async-api-with-output_14_0.png</a> 22-Jun-2023 00:06 4307
<a href="115-async-api-with-output_18_0.png">115-async-api-with-output_18_0.png</a> 22-Jun-2023 00:06 4307
<a href="115-async-api-with-output_20_0.png">115-async-api-with-output_20_0.png</a> 22-Jun-2023 00:06 30500
<a href="115-async-api-with-output_26_0.png">115-async-api-with-output_26_0.png</a> 22-Jun-2023 00:06 4307
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/115-async-api-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="115-async-api-with-output_15_0.png">115-async-api-with-output_15_0.png</a> 12-Jul-2023 00:11 4307
<a href="115-async-api-with-output_19_0.png">115-async-api-with-output_19_0.png</a> 12-Jul-2023 00:11 4307
<a href="115-async-api-with-output_21_0.png">115-async-api-with-output_21_0.png</a> 12-Jul-2023 00:11 30421
<a href="115-async-api-with-output_27_0.png">115-async-api-with-output_27_0.png</a> 12-Jul-2023 00:11 4307
</pre><hr></body>
</html>

59
docs/notebooks/116-sparsity-optimization-with-output.rst
View File

@@ -44,10 +44,10 @@ Imports
.. parsed-literal::
2023-06-21 22:54:03.983988: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-06-21 22:54:04.017706: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2023-07-11 22:51:18.663091: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-07-11 22:51:18.697477: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-06-21 22:54:04.587651: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2023-07-11 22:51:19.248196: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
.. parsed-literal::
@@ -92,6 +92,7 @@ model card on Hugging Face.
.. parsed-literal::
Compiling the model...
Set CACHE_DIR to /opt/home/k8sworker/.cache/huggingface/hub/models--OpenVINO--bert-base-uncased-sst2-int8-unstructured80/snapshots/dc44eb46300882463d50ee847e0f6485bad3cdad/model_cache
Xformers is not installed correctly. If you want to use memory_efficient_attention to accelerate training use the following command to install Xformers
pip install xformers.
@@ -174,7 +175,7 @@ as an example. It is recommended to tune based on your applications.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 67.89 ms
[ INFO ] Read model took 79.41 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] input_ids (node: input_ids) : i64 / [...] / [?,?]
@@ -185,7 +186,7 @@ as an example. It is recommended to tune based on your applications.
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[ INFO ] Reshaping model: 'input_ids': [1,64], 'attention_mask': [1,64], 'token_type_ids': [1,64]
[ INFO ] Reshape model took 25.89 ms
[ INFO ] Reshape model took 26.38 ms
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] input_ids (node: input_ids) : i64 / [...] / [1,64]
@@ -194,7 +195,7 @@ as an example. It is recommended to tune based on your applications.
[ INFO ] Model outputs:
[ INFO ] logits (node: logits) : f32 / [...] / [1,2]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 1229.91 ms
[ INFO ] Compile model took 1252.95 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
@@ -220,17 +221,17 @@ as an example. It is recommended to tune based on your applications.
[ INFO ] Fill input 'token_type_ids' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 4 inference requests, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 29.99 ms
[ INFO ] First inference took 26.89 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 8908 iterations
[ INFO ] Duration: 60048.39 ms
[ INFO ] Count: 8944 iterations
[ INFO ] Duration: 60031.11 ms
[ INFO ] Latency:
[ INFO ] Median: 26.80 ms
[ INFO ] Average: 26.84 ms
[ INFO ] Min: 25.30 ms
[ INFO ] Max: 40.79 ms
[ INFO ] Throughput: 148.35 FPS
[ INFO ] Median: 26.64 ms
[ INFO ] Average: 26.68 ms
[ INFO ] Min: 25.23 ms
[ INFO ] Max: 40.15 ms
[ INFO ] Throughput: 148.99 FPS
Benchmark quantized sparse inference performance
@@ -281,7 +282,7 @@ for which a layer will be enabled.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 72.33 ms
[ INFO ] Read model took 69.37 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] input_ids (node: input_ids) : i64 / [...] / [?,?]
@@ -292,7 +293,7 @@ for which a layer will be enabled.
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[ INFO ] Reshaping model: 'input_ids': [1,64], 'attention_mask': [1,64], 'token_type_ids': [1,64]
[ INFO ] Reshape model took 26.21 ms
[ INFO ] Reshape model took 25.96 ms
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] input_ids (node: input_ids) : i64 / [...] / [1,64]
@@ -301,7 +302,7 @@ for which a layer will be enabled.
[ INFO ] Model outputs:
[ INFO ] logits (node: logits) : f32 / [...] / [1,2]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 1287.76 ms
[ INFO ] Compile model took 1250.91 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
@@ -327,17 +328,17 @@ for which a layer will be enabled.
[ INFO ] Fill input 'token_type_ids' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 4 inference requests, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 31.87 ms
[ INFO ] First inference took 32.79 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 8896 iterations
[ INFO ] Duration: 60049.93 ms
[ INFO ] Count: 8972 iterations
[ INFO ] Duration: 60029.38 ms
[ INFO ] Latency:
[ INFO ] Median: 26.76 ms
[ INFO ] Average: 26.80 ms
[ INFO ] Min: 24.55 ms
[ INFO ] Max: 42.25 ms
[ INFO ] Throughput: 148.14 FPS
[ INFO ] Median: 26.59 ms
[ INFO ] Average: 26.63 ms
[ INFO ] Min: 25.42 ms
[ INFO ] Max: 41.12 ms
[ INFO ] Throughput: 149.46 FPS
When this might be helpful
@@ -349,7 +350,7 @@ requests in parallel asynchronously. It is especially helpful with a
small sequence length, for example, 32 and lower.
For more details about asynchronous inference with OpenVINO, refer to
the following documentation: - `Deployment Optimization
Guide <https://docs.openvino.ai/2023.0/openvino_docs_deployment_optimization_guide_common.html#doxid-openvino-docs-deployment-optimization-guide-common-1async-api>`__
- `Inference Request
API <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Infer_request.html#doxid-openvino-docs-o-v-u-g-infer-request-1in-out-tensors>`__
the following documentation:
- `Deployment Optimization Guide <https://docs.openvino.ai/2023.0/openvino_docs_deployment_optimization_guide_common.html#doxid-openvino-docs-deployment-optimization-guide-common-1async-api>`__
- `Inference Request API <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Infer_request.html#doxid-openvino-docs-o-v-u-g-infer-request-1in-out-tensors>`__

8
docs/notebooks/117-model-server-with-output_files/index.html
View File

@@ -1,8 +1,8 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/117-model-server-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/117-model-server-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/117-model-server-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="117-model-server-with-output_20_1.png">117-model-server-with-output_20_1.png</a> 22-Jun-2023 00:06 112408
<a href="117-model-server-with-output_25_1.png">117-model-server-with-output_25_1.png</a> 22-Jun-2023 00:06 232667
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/117-model-server-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="117-model-server-with-output_20_1.png">117-model-server-with-output_20_1.png</a> 12-Jul-2023 00:11 112408
<a href="117-model-server-with-output_25_1.png">117-model-server-with-output_25_1.png</a> 12-Jul-2023 00:11 232667
</pre><hr></body>
</html>

111
docs/notebooks/118-optimize-preprocessing-with-output.rst
View File

@@ -13,12 +13,14 @@ API, see this
and
`details <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Preprocessing_Details.html>`__
This tutorial include following steps: - Downloading the model. - Setup
preprocessing with ModelOptimizer, loading the model and inference with
original image. - Setup preprocessing with Preprocessing API, loading
the model and inference with original image. - Fitting image to the
model input type and inference with prepared image. - Comparing results
on one picture. - Comparing performance.
This tutorial include following steps:
- Downloading the model.
- Setup preprocessing with ModelOptimizer, loading the model and inference with original image.
- Setup preprocessing with Preprocessing API, loading the model and inference with original image.
- Fitting image to the model input type and inference with prepared image.
- Comparing results on one picture.
- Comparing performance.
Settings
--------
@@ -41,10 +43,10 @@ Imports
.. parsed-literal::
2023-06-21 22:56:19.365436: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-06-21 22:56:19.399309: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2023-07-11 22:53:33.947684: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-07-11 22:53:33.981920: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-06-21 22:56:19.937411: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2023-07-11 22:53:34.528705: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
Setup image and device
@@ -53,8 +55,29 @@ Setup image and device
.. code:: ipython3
image_path = "../data/image/coco.jpg"
device = "CPU"
# device = "GPU"
.. code:: ipython3
import ipywidgets as widgets
core = Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
Downloading the model
~~~~~~~~~~~~~~~~~~~~~
@@ -88,7 +111,7 @@ and save it to the disk.
.. parsed-literal::
2023-06-21 22:56:21.198595: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
2023-07-11 22:53:35.902963: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
@@ -137,7 +160,7 @@ Check the original parameters of image
.. image:: 118-optimize-preprocessing-with-output_files/118-optimize-preprocessing-with-output_11_1.png
.. image:: 118-optimize-preprocessing-with-output_files/118-optimize-preprocessing-with-output_12_1.png
Convert model to OpenVINO IR and setup preprocessing steps with Model Optimizer
@@ -157,11 +180,11 @@ of work on processing the input data before propagating it through the
network. These conversions will be inserted as additional input
pre-processing sub-graphs into the converted model.
Setup the following conversions: - mean normalization with
``mean_values`` parameter. - scale with ``scale_values``. - color
conversion, the color format of example image will be ``BGR``, but the
model required ``RGB`` format, so add ``reverse_input_channels=True`` to
process the image into the desired format.
Setup the following conversions:
- mean normalization with ``mean_values`` parameter
- scale with ``scale_values``
- color conversion, the color format of example image will be ``BGR``, but the model required ``RGB`` format, so add ``reverse_input_channels=True`` to process the image into the desired format
Also converting of layout could be specified with ``layout`` option.
More information and parameters described in the `Embedding
@@ -227,7 +250,7 @@ Compile model and perform inerence
.. code:: ipython3
compiled_model_mo_pp = core.compile_model(model=ov_model_mo_preprocess, device_name=device)
compiled_model_mo_pp = core.compile_model(model=ov_model_mo_preprocess, device_name=device.value)
output_layer = compiled_model_mo_pp.output(0)
@@ -236,12 +259,11 @@ Compile model and perform inerence
Setup preprocessing steps with Preprocessing API and perform inference
----------------------------------------------------------------------
Intuitively, preprocessing API consists of the following parts: - Tensor
- declares user data format, like shape, layout, precision, color format
from actual user’s data. - Steps - describes sequence of preprocessing
steps which need to be applied to user data. - Model - specifies model
data format. Usually, precision and shape are already known for model,
only additional information, like layout can be specified.
Intuitively, preprocessing API consists of the following parts:
- Tensor - declares user data format, like shape, layout, precision, color format from actual user’s data.
- Steps - describes sequence of preprocessing steps which need to be applied to user data.
- Model - specifies model data format. Usually, precision and shape are already known for model, only additional information, like layout can be specified.
Graph modifications of a model shall be performed after the model is
read from a drive and before it is loaded on the actual device.
@@ -300,10 +322,11 @@ the following
`page <https://docs.openvino.ai/2023.0/classov_1_1preprocess_1_1InputTensorInfo.html#doxid-classov-1-1preprocess-1-1-input-tensor-info-1a98fb73ff9178c8c71d809ddf8927faf5>`__
for more information about parameters for overriding.
Below is all the specified input information: - Precision is ``U8``
(unsigned 8-bit integer). - Size is non-fixed, setup of one determined
shape size can be done with ``.set_shape([1, 577, 800, 3])`` - Layout is
``“NHWC”``. It means, for example: height=577, width=800, channels=3.
Below is all the specified input information:
- Precision is ``U8``(unsigned 8-bit integer).
- Size is non-fixed, setup of one determined shape size can be done with ``.set_shape([1, 577, 800, 3])``.
- Layout is ``“NHWC”``. It means, for example: height=577, width=800, channels=3.
The height and width are necessary for resizing, and channels are needed
for mean/scale normalization.
@@ -322,7 +345,7 @@ for mean/scale normalization.
.. parsed-literal::
<openvino._pyopenvino.preprocess.InputTensorInfo at 0x7f97f45dc730>
<openvino._pyopenvino.preprocess.InputTensorInfo at 0x7efb4037f5b0>
@@ -351,7 +374,7 @@ may be specified is input data
.. parsed-literal::
<openvino._pyopenvino.preprocess.InputModelInfo at 0x7f97f462f770>
<openvino._pyopenvino.preprocess.InputModelInfo at 0x7efc3610ac70>
@@ -362,14 +385,12 @@ Now, the sequence of preprocessing steps can be defined. For more
information about preprocessing steps, see
`here <https://docs.openvino.ai/2023.0/api/ie_python_api/_autosummary/openvino.preprocess.PreProcessSteps.html>`__.
Perform the following: - Convert ``U8`` to ``FP32`` precision. - Resize
to height/width of a model. Be aware that if a model accepts dynamic
size, for example, ``{?, 3, ?, ?}`` resize will not know how to resize
the picture. Therefore, in this case, target height/ width should be
specified. For more details, see also the
`PreProcessSteps.resize() <https://docs.openvino.ai/2023.0/classov_1_1preprocess_1_1PreProcessSteps.html#doxid-classov-1-1preprocess-1-1-pre-process-steps-1a40dab78be1222fee505ed6a13400efe6>`__.
- Subtract mean from each channel. - Divide each pixel data to
appropriate scale value.
Perform the following:
- Convert ``U8`` to ``FP32`` precision.
- Resize to height/width of a model. Be aware that if a model accepts dynamic size, for example, ``{?, 3, ?, ?}`` resize will not know how to resize the picture. Therefore, in this case, target height/ width should be specified. For more details, see also the `PreProcessSteps.resize() <https://docs.openvino.ai/2023.0/classov_1_1preprocess_1_1PreProcessSteps.html#doxid-classov-1-1preprocess-1-1-pre-process-steps-1a40dab78be1222fee505ed6a13400efe6>`__.
- Subtract mean from each channel.
- Divide each pixel data to appropriate scale value.
There is no need to specify conversion layout. If layouts are different,
then such conversion will be added explicitly.
@@ -388,7 +409,7 @@ then such conversion will be added explicitly.
.. parsed-literal::
<openvino._pyopenvino.preprocess.PreProcessSteps at 0x7f97b8346af0>
<openvino._pyopenvino.preprocess.PreProcessSteps at 0x7efd2c4155b0>
@@ -429,7 +450,7 @@ Load model and perform inference
return input_tensor
compiled_model_with_preprocess_api = core.compile_model(model=ppp_model, device_name=device)
compiled_model_with_preprocess_api = core.compile_model(model=ppp_model, device_name=device.value)
ppp_output_layer = compiled_model_with_preprocess_api.output(0)
@@ -445,7 +466,7 @@ Load the model
.. code:: ipython3
model = core.read_model(model=ir_path)
compiled_model = core.compile_model(model=model, device_name=device)
compiled_model = core.compile_model(model=model, device_name=device.value)
Load image and fit it to model input
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -600,7 +621,7 @@ Compare performance
.. parsed-literal::
IR model in OpenVINO Runtime/CPU with preprocessing API: 0.0189 seconds per image, FPS: 52.77
IR model in OpenVINO Runtime/CPU with preprocessing API: 0.0153 seconds per image, FPS: 65.26
IR model in OpenVINO Runtime/CPU with preprocessing API: 0.0175 seconds per image, FPS: 57.09
IR model in OpenVINO Runtime/CPU with preprocessing API: 0.0200 seconds per image, FPS: 49.90
IR model in OpenVINO Runtime/CPU with preprocessing API: 0.0153 seconds per image, FPS: 65.52
IR model in OpenVINO Runtime/CPU with preprocessing API: 0.0187 seconds per image, FPS: 53.59

0
docs/notebooks/118-optimize-preprocessing-with-output_files/118-optimize-preprocessing-with-output_11_1.png → docs/notebooks/118-optimize-preprocessing-with-output_files/118-optimize-preprocessing-with-output_12_1.png
View File

6
docs/notebooks/118-optimize-preprocessing-with-output_files/index.html
View File

@@ -1,7 +1,7 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/118-optimize-preprocessing-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/118-optimize-preprocessing-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/118-optimize-preprocessing-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="118-optimize-preprocessing-with-output_11_1.png">118-optimize-preprocessing-with-output_11_1.png</a> 22-Jun-2023 00:06 387941
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/118-optimize-preprocessing-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="118-optimize-preprocessing-with-output_12_1.png">118-optimize-preprocessing-with-output_12_1.png</a> 12-Jul-2023 00:11 387941
</pre><hr></body>
</html>

90
docs/notebooks/119-tflite-to-openvino-with-output.rst
View File

@@ -15,16 +15,36 @@ After creating the OpenVINO IR, load the model in `OpenVINO
Runtime <https://docs.openvino.ai/nightly/openvino_docs_OV_UG_OV_Runtime_User_Guide.html>`__
and do inference with a sample image.
Preparation
-----------
Install requirements
~~~~~~~~~~~~~~~~~~~~
.. code:: ipython3
!pip install -q "openvino-dev>=2023.0.0"
!pip install -q opencv-python requests tqdm
# Fetch `notebook_utils` module
import urllib.request
urllib.request.urlretrieve(
url='https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py',
filename='notebook_utils.py'
);
Imports
~~~~~~~
.. code:: ipython3
import sys
from pathlib import Path
import numpy as np
from PIL import Image
from openvino.runtime import Core, serialize
from openvino.tools import mo
sys.path.append("../utils")
from notebook_utils import download_file
from notebook_utils import download_file, load_image
Download TFLite model
---------------------
@@ -50,7 +70,7 @@ Download TFLite model
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/119-tflite-to-openvino/model/efficientnet_lite0_fp32_2.tflite')
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/119-tflite-to-openvino/model/efficientnet_lite0_fp32_2.tflite')
@@ -105,10 +125,39 @@ on `TensorFlow Hub <https://tfhub.dev/>`__.
.. code:: ipython3
image = Image.open("../data/image/coco_bricks.png")
image = load_image("https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_bricks.png")
# load_image reads the image in BGR format, [:,:,::-1] reshape transfroms it to RGB
image = Image.fromarray(image[:,:,::-1])
resized_image = image.resize((224, 224))
input_tensor = np.expand_dims((np.array(resized_image).astype(np.float32) - 127) / 128, 0)
Select inference device
~~~~~~~~~~~~~~~~~~~~~~~
select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
.. code:: ipython3
compiled_model = core.compile_model(ov_model)
@@ -116,7 +165,8 @@ on `TensorFlow Hub <https://tfhub.dev/>`__.
.. code:: ipython3
imagenet_classes = open("../data/datasets/imagenet/imagenet_2012.txt").read().splitlines()
imagenet_classes_file_path = download_file("https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/datasets/imagenet/imagenet_2012.txt")
imagenet_classes = open(imagenet_classes_file_path).read().splitlines()
top1_predicted_cls_id = np.argmax(predicted_scores)
top1_predicted_score = predicted_scores[0][top1_predicted_cls_id]
@@ -127,7 +177,13 @@ on `TensorFlow Hub <https://tfhub.dev/>`__.
.. image:: 119-tflite-to-openvino-with-output_files/119-tflite-to-openvino-with-output_11_0.png
.. parsed-literal::
imagenet_2012.txt: 0%| | 0.00/30.9k [00:00<?, ?B/s]
.. image:: 119-tflite-to-openvino-with-output_files/119-tflite-to-openvino-with-output_16_1.png
.. parsed-literal::
@@ -177,7 +233,7 @@ GPU.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 9.15 ms
[ INFO ] Read model took 21.86 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] images (node: images) : f32 / [...] / [1,224,224,3]
@@ -191,7 +247,7 @@ GPU.
[ INFO ] Model outputs:
[ INFO ] Softmax (node: 61) : f32 / [...] / [1,1000]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 149.12 ms
[ INFO ] Compile model took 183.81 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: TensorFlow_Lite_Frontend_IR
@@ -213,15 +269,15 @@ GPU.
[ INFO ] Fill input 'images' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 7.30 ms
[ INFO ] First inference took 7.41 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 17514 iterations
[ INFO ] Duration: 15008.85 ms
[ INFO ] Count: 17364 iterations
[ INFO ] Duration: 15009.20 ms
[ INFO ] Latency:
[ INFO ] Median: 4.99 ms
[ INFO ] Average: 5.00 ms
[ INFO ] Min: 3.71 ms
[ INFO ] Max: 15.12 ms
[ INFO ] Throughput: 1166.91 FPS
[ INFO ] Median: 5.04 ms
[ INFO ] Average: 5.05 ms
[ INFO ] Min: 3.22 ms
[ INFO ] Max: 15.36 ms
[ INFO ] Throughput: 1156.89 FPS

3
docs/notebooks/119-tflite-to-openvino-with-output_files/119-tflite-to-openvino-with-output_11_0.png
View File

@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3b2bcf5770d51d67341faebf7ff45bc93104f1296fb5d7b2594d87299f1ce153
size 612081

3
docs/notebooks/119-tflite-to-openvino-with-output_files/119-tflite-to-openvino-with-output_16_1.jpg Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a11e349455a5b61c5f93ca1017f5f92b84c29282264891abfcfe160e72e0624c
size 68170

3
docs/notebooks/119-tflite-to-openvino-with-output_files/119-tflite-to-openvino-with-output_16_1.png Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9000b06fe3a0537c489dc4c87fcb6ca0bae5eecd37ad736dc7e4c6e35f766ab8
size 621006

7
docs/notebooks/119-tflite-to-openvino-with-output_files/index.html
View File

@@ -1,7 +1,8 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/119-tflite-to-openvino-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/119-tflite-to-openvino-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/119-tflite-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="119-tflite-to-openvino-with-output_11_0.png">119-tflite-to-openvino-with-output_11_0.png</a> 22-Jun-2023 00:06 612081
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/119-tflite-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="119-tflite-to-openvino-with-output_16_1.jpg">119-tflite-to-openvino-with-output_16_1.jpg</a> 12-Jul-2023 00:11 68170
<a href="119-tflite-to-openvino-with-output_16_1.png">119-tflite-to-openvino-with-output_16_1.png</a> 12-Jul-2023 00:11 621006
</pre><hr></body>
</html>

652
docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output.rst Normal file
View File

@@ -0,0 +1,652 @@
Convert a TensorFlow Object Detection Model to OpenVINO™
========================================================
`TensorFlow <https://www.tensorflow.org/>`__, or TF for short, is an
open-source framework for machine learning.
The `TensorFlow Object Detection
API <https://github.com/tensorflow/models/tree/master/research/object_detection>`__
is an open-source computer vision framework built on top of TensorFlow.
It is used for building object detection and image segmentation models
that can localize multiple objects in the same image. TensorFlow Object
Detection API supports various architectures and models, which can be
found and downloaded from the `TensorFlow
Hub <https://tfhub.dev/tensorflow/collections/object_detection/1>`__.
This tutorial shows how to convert a TensorFlow `Faster R-CNN with
Resnet-50
V1 <https://tfhub.dev/tensorflow/faster_rcnn/resnet50_v1_640x640/1>`__
object detection model to OpenVINO `Intermediate
Representation <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_IR_and_opsets.html>`__
(OpenVINO IR) format, using `Model
Optimizer <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
After creating the OpenVINO IR, load the model in `OpenVINO
Runtime <https://docs.openvino.ai/nightly/openvino_docs_OV_UG_OV_Runtime_User_Guide.html>`__
and do inference with a sample image.
Prerequisites
-------------
Install required packages:
.. code:: ipython3
!pip install -q "openvino-dev>=2023.0.0" "numpy>=1.21.0" "opencv-python" "matplotlib>=3.4,<3.5.3"
The notebook uses utility functions. The cell below will download the
``notebook_utils`` Python module from GitHub.
.. code:: ipython3
# Fetch the notebook utils script from the openvino_notebooks repo
import urllib.request
urllib.request.urlretrieve(
url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/main/notebooks/utils/notebook_utils.py",
filename="notebook_utils.py",
);
Imports
-------
.. code:: ipython3
# Standard python modules
from pathlib import Path
# External modules and dependencies
import cv2
import matplotlib.pyplot as plt
import numpy as np
# Notebook utils module
from notebook_utils import download_file
# OpenVINO modules
from openvino.runtime import Core, serialize
from openvino.tools import mo
Settings
--------
Define model related variables and create corresponding directories:
.. code:: ipython3
# Create directories for models files
model_dir = Path("model")
model_dir.mkdir(exist_ok=True)
# Create directory for TensorFlow model
tf_model_dir = model_dir / "tf"
tf_model_dir.mkdir(exist_ok=True)
# Create directory for OpenVINO IR model
ir_model_dir = model_dir / "ir"
ir_model_dir.mkdir(exist_ok=True)
model_name = "faster_rcnn_resnet50_v1_640x640"
openvino_ir_path = ir_model_dir / f"{model_name}.xml"
tf_model_url = "https://tfhub.dev/tensorflow/faster_rcnn/resnet50_v1_640x640/1?tf-hub-format=compressed"
tf_model_archive_filename = f"{model_name}.tar.gz"
Download Model from TensorFlow Hub
----------------------------------
Download archive with TensorFlow Object Detection model
(`faster_rcnn_resnet50_v1_640x640 <https://tfhub.dev/tensorflow/faster_rcnn/resnet50_v1_640x640/1>`__)
from TensorFlow Hub:
.. code:: ipython3
download_file(
url=tf_model_url,
filename=tf_model_archive_filename,
directory=tf_model_dir
)
.. parsed-literal::
model/tf/faster_rcnn_resnet50_v1_640x640.tar.gz: 0%| | 0.00/101M [00:00<?, ?B/s]
.. parsed-literal::
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/120-tensorflow-object-detection-to-openvino/model/tf/faster_rcnn_resnet50_v1_640x640.tar.gz')
Extract TensorFlow Object Detection model from the downloaded archive:
.. code:: ipython3
import tarfile
with tarfile.open(tf_model_dir / tf_model_archive_filename) as file:
file.extractall(path=tf_model_dir)
Convert Model to OpenVINO IR
----------------------------
OpenVINO Model Optimizer Python API can be used to convert the
TensorFlow model to OpenVINO IR.
``mo.convert_model`` function accept path to TensorFlow model and
returns OpenVINO Model class instance which represents this model. Also
we need to provide model input shape (``input_shape``) that is described
at `model overview page on TensorFlow
Hub <https://tfhub.dev/tensorflow/faster_rcnn/resnet50_v1_640x640/1>`__.
Optionally, we can apply compression to FP16 model weigths using
``compress_to_fp16=True`` option and integrate preprocessing using this
approach.
The converted model is ready to load on a device using ``compile_model``
or saved on disk using the ``serialize`` function to reduce loading time
when the model is run in the future.
See the `Model Optimizer Developer
Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__
for more information about Model Optimizer and TensorFlow `models
suport <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow.html>`__.
.. code:: ipython3
ov_model = mo.convert_model(
saved_model_dir=tf_model_dir,
input_shape=[[1, 255, 255, 3]]
)
# Save converted OpenVINO IR model to the corresponding directory
serialize(ov_model, openvino_ir_path)
Test Inference on the Converted Model
-------------------------------------
Select inference device
-----------------------
select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
import ipywidgets as widgets
core = Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
Load the Model
~~~~~~~~~~~~~~
.. code:: ipython3
core = Core()
openvino_ir_model = core.read_model(openvino_ir_path)
compiled_model = core.compile_model(model=openvino_ir_model, device_name=device.value)
Get Model Information
~~~~~~~~~~~~~~~~~~~~~
Faster R-CNN with Resnet-50 V1 object detection model has one input - a
three-channel image of variable size. The input tensor shape is
``[1, height, width, 3]`` with values in ``[0, 255]``.
Model output dictionary contains several tensors:
- ``num_detections`` - the number of detections in ``[N]`` format.
- ``detection_boxes`` - bounding box coordinates for all ``N`` detections in ``[ymin, xmin, ymax, xmax]`` format.
- ``detection_classes`` - ``N`` detection class indexes size from the label file.
- ``detection_scores``- ``N`` detection scores (confidence) for each detected class.
- ``raw_detection_boxes`` - decoded detection boxes without Non-Max suppression.
- ``raw_detection_scores`` - class score logits for raw detection boxes.
- ``detection_anchor_indices`` - the anchor indices of the detections after NMS.
- ``detection_multiclass_scores`` - class score distribution (including background) for detection boxes in the image including background class.
In this tutorial we will mostly use ``detection_boxes``,
``detection_classes``, ``detection_scores`` tensors. It is important to
mention, that values of these tensors correspond to each other and are
ordered by the highest detection score: the first detection box
corresponds to the first detection class and to the first (and highest)
detection score.
See the `model overview page on TensorFlow
Hub <https://tfhub.dev/tensorflow/faster_rcnn/resnet50_v1_640x640/1>`__
for more information about model inputs, outputs and their formats.
.. code:: ipython3
model_inputs = compiled_model.inputs
model_input = compiled_model.input(0)
model_outputs = compiled_model.outputs
print("Model inputs count:", len(model_inputs))
print("Model input:", model_input)
print("Model outputs count:", len(model_outputs))
print("Model outputs:")
for output in model_outputs:
print(" ", output)
.. parsed-literal::
Model inputs count: 1
Model input: <ConstOutput: names[input_tensor] shape[1,255,255,3] type: u8>
Model outputs count: 8
Model outputs:
<ConstOutput: names[detection_anchor_indices] shape[1,?] type: f32>
<ConstOutput: names[detection_boxes] shape[1,?,..8] type: f32>
<ConstOutput: names[detection_classes] shape[1,?] type: f32>
<ConstOutput: names[detection_multiclass_scores] shape[1,?,..182] type: f32>
<ConstOutput: names[detection_scores] shape[1,?] type: f32>
<ConstOutput: names[num_detections] shape[1] type: f32>
<ConstOutput: names[raw_detection_boxes] shape[1,300,4] type: f32>
<ConstOutput: names[raw_detection_scores] shape[1,300,91] type: f32>
Get an Image for Test Inference
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Load and save an image:
.. code:: ipython3
image_path = Path("./data/coco_bike.jpg")
download_file(
url="https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_bike.jpg",
filename=image_path.name,
directory=image_path.parent,
)
.. parsed-literal::
data/coco_bike.jpg: 0%| | 0.00/182k [00:00<?, ?B/s]
.. parsed-literal::
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/120-tensorflow-object-detection-to-openvino/data/coco_bike.jpg')
Read the image, resize and convert it to the input shape of the network:
.. code:: ipython3
# Read the image
image = cv2.imread(filename=str(image_path))
# The network expects images in RGB format
image = cv2.cvtColor(image, code=cv2.COLOR_BGR2RGB)
# Resize the image to the network input shape
resized_image = cv2.resize(src=image, dsize=(255, 255))
# Transpose the image to the network input shape
network_input_image = np.expand_dims(resized_image, 0)
# Show the image
plt.imshow(image)
.. parsed-literal::
<matplotlib.image.AxesImage at 0x7f877c319b50>
.. image:: 120-tensorflow-object-detection-to-openvino-with-output_files/120-tensorflow-object-detection-to-openvino-with-output_25_1.png
Perform Inference
~~~~~~~~~~~~~~~~~
.. code:: ipython3
inference_result = compiled_model(network_input_image)
After model inference on the test image, object detection data can be
extracted from the result. For further model result visualization
``detection_boxes``, ``detection_classes`` and ``detection_scores``
outputs will be used.
.. code:: ipython3
_, detection_boxes, detection_classes, _, detection_scores, num_detections, _, _ = model_outputs
image_detection_boxes = inference_result[detection_boxes]
print("image_detection_boxes:", image_detection_boxes)
image_detection_classes = inference_result[detection_classes]
print("image_detection_classes:", image_detection_classes)
image_detection_scores = inference_result[detection_scores]
print("image_detection_scores:", image_detection_scores)
image_num_detections = inference_result[num_detections]
print("image_detections_num:", image_num_detections)
# Alternatively, inference result data can be extracted by model output name with `.get()` method
assert (inference_result[detection_boxes] == inference_result.get("detection_boxes")).all(), "extracted inference result data should be equal"
.. parsed-literal::
image_detection_boxes: [[[0.16453631 0.54612625 0.89533776 0.85469896]
[0.6721994 0.01249559 0.98444635 0.53168815]
[0.4910983 0.01171527 0.98045075 0.88644964]
...
[0.5012431 0.5489591 0.6030575 0.61094964]
[0.45808432 0.3619884 0.8841141 0.83722156]
[0.4652153 0.02054662 0.48204365 0.0438836 ]]]
image_detection_classes: [[18. 2. 2. 3. 2. 8. 2. 2. 3. 2. 4. 4. 2. 4. 16. 1. 1. 27.
2. 8. 62. 2. 2. 4. 4. 2. 41. 18. 4. 2. 4. 18. 2. 2. 4. 27.
2. 2. 27. 2. 1. 1. 16. 2. 2. 2. 16. 2. 2. 4. 2. 1. 33. 4.
15. 2. 3. 2. 2. 1. 2. 1. 4. 2. 3. 11. 4. 35. 40. 4. 1. 62.
2. 2. 4. 36. 4. 36. 1. 31. 77. 2. 36. 1. 51. 1. 34. 3. 90. 2.
3. 2. 1. 2. 2. 1. 1. 2. 1. 4. 18. 2. 2. 3. 31. 1. 41. 1.
2. 2. 33. 41. 3. 31. 1. 3. 36. 27. 27. 15. 4. 4. 15. 3. 2. 37.
1. 35. 27. 4. 36. 88. 4. 2. 3. 15. 2. 4. 2. 1. 3. 3. 27. 4.
4. 44. 16. 1. 1. 23. 4. 3. 1. 4. 4. 62. 15. 36. 77. 3. 28. 1.
35. 27. 2. 27. 75. 36. 8. 28. 3. 4. 36. 35. 44. 4. 3. 1. 2. 1.
1. 35. 87. 1. 84. 1. 1. 1. 15. 1. 3. 1. 35. 1. 1. 1. 1. 62.
15. 1. 44. 15. 1. 41. 62. 1. 4. 43. 15. 4. 3. 4. 16. 35. 2. 33.
3. 14. 62. 34. 41. 2. 35. 4. 18. 3. 15. 1. 27. 87. 1. 4. 19. 21.
27. 1. 3. 2. 1. 27. 15. 4. 3. 1. 38. 1. 2. 15. 38. 4. 15. 1.
3. 3. 62. 84. 20. 58. 2. 4. 41. 20. 88. 15. 1. 19. 31. 62. 31. 4.
14. 1. 8. 18. 15. 2. 4. 2. 2. 2. 31. 84. 2. 15. 28. 3. 27. 18.
15. 1. 31. 41. 1. 28. 3. 1. 8. 15. 1. 16.]]
image_detection_scores: [[0.9808771 0.9418091 0.9318733 0.8789291 0.8423196 0.5888979
0.5630133 0.53731316 0.4974923 0.48222807 0.4673298 0.4398691
0.39919445 0.33909947 0.3190495 0.27470118 0.24837914 0.23406433
0.23351488 0.22481255 0.22016802 0.20236589 0.19338816 0.14771679
0.14576106 0.14285511 0.12738948 0.12668392 0.12027147 0.10873836
0.10812037 0.09577218 0.09060974 0.08950701 0.08673717 0.08170561
0.08120535 0.0789713 0.06743153 0.06118729 0.06112184 0.05309067
0.05216556 0.05023476 0.04783678 0.04460874 0.04213375 0.04042179
0.04019568 0.03522961 0.03165065 0.0310733 0.03000823 0.02873152
0.02782036 0.02706797 0.0266978 0.02341437 0.02291683 0.02147149
0.02130841 0.02099001 0.02032206 0.01978395 0.01961209 0.01902091
0.01893682 0.01863261 0.01858075 0.01846547 0.01823624 0.0176264
0.01760109 0.01703349 0.01584588 0.01582033 0.01547665 0.01527787
0.01522782 0.01430391 0.01428877 0.01422195 0.0141238 0.01411421
0.0135575 0.01288707 0.01269312 0.01218521 0.01160688 0.01143213
0.01142005 0.01137567 0.0111644 0.01107758 0.0109348 0.01073039
0.0106188 0.01016685 0.01010454 0.00983268 0.00977985 0.00967134
0.00965687 0.00964259 0.00962718 0.00956944 0.00950549 0.00937742
0.00927729 0.00916896 0.00897371 0.00891221 0.00866699 0.00863667
0.00855941 0.00836656 0.00835135 0.00816708 0.00795946 0.00793826
0.00789131 0.00781442 0.00773429 0.00767627 0.00765273 0.00752015
0.00749519 0.00744095 0.00715925 0.00700314 0.00692652 0.00655058
0.00643994 0.00641626 0.00629459 0.00628646 0.00627907 0.00612065
0.00593393 0.00582955 0.00582755 0.00570769 0.00569362 0.00564996
0.00563695 0.00558055 0.00557034 0.00551842 0.00549368 0.00544169
0.00544044 0.00542281 0.00540061 0.00525593 0.00524985 0.00515946
0.00515553 0.00511156 0.00489827 0.00484957 0.00472266 0.00465891
0.00464309 0.00463513 0.00459531 0.00456809 0.0045585 0.00455432
0.00443505 0.00443078 0.00440637 0.00422725 0.00416438 0.0041492
0.00413432 0.00413151 0.00409415 0.00409274 0.00407757 0.00405691
0.00396555 0.00393284 0.00391471 0.00388586 0.00385833 0.00385633
0.00385035 0.00379386 0.00378297 0.00378109 0.00377772 0.00370916
0.00364531 0.00363934 0.00358231 0.00354156 0.0035037 0.00348796
0.00344136 0.00340937 0.00334414 0.00330951 0.00329006 0.00321436
0.00320603 0.00312488 0.00309948 0.00307925 0.00307775 0.00306451
0.00303381 0.00302188 0.00299367 0.00299316 0.00298596 0.00296609
0.00293693 0.00288884 0.0028709 0.00283928 0.00283312 0.00281894
0.00276538 0.00276278 0.00270719 0.00268026 0.00258883 0.00258464
0.00254383 0.00253249 0.00250638 0.00250605 0.00250558 0.0025017
0.00249729 0.00248757 0.00246982 0.00243592 0.0024358 0.00235382
0.0023404 0.00233721 0.00233374 0.00233181 0.0023271 0.00230558
0.00230428 0.00229607 0.00227586 0.00226048 0.00223509 0.00222384
0.00220214 0.00219295 0.00219229 0.00218538 0.00218472 0.00217254
0.00216129 0.00214788 0.00213485 0.00213233 0.00208789 0.00206768
0.00206485 0.00206409 0.00204371 0.00203812 0.00201267 0.00200125
0.00199629 0.00199346 0.00198402 0.00192943 0.00191091 0.0019036
0.0018943 0.00188735 0.00188038 0.00186264 0.00179476 0.00177307
0.00176998 0.00176099 0.0017542 0.00174639 0.00171193 0.0017064
0.00169167 0.00168484 0.00167157 0.00166569 0.00166213 0.00166009
0.00164244 0.00164076 0.00163557 0.00162898 0.00160348 0.00159898]]
image_detections_num: [300.]
Inference Result Visualization
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Define utility functions to visualize the inference results
.. code:: ipython3
import random
from typing import Optional
def add_detection_box(box: np.ndarray, image: np.ndarray, label: Optional[str] = None) -> np.ndarray:
"""
Helper function for adding single bounding box to the image
Parameters
----------
box : np.ndarray
Bounding box coordinates in format [ymin, xmin, ymax, xmax]
image : np.ndarray
The image to which detection box is added
label : str, optional
Detection box label string, if not provided will not be added to result image (default is None)
Returns
-------
np.ndarray
NumPy array including both image and detection box
"""
ymin, xmin, ymax, xmax = box
point1, point2 = (int(xmin), int(ymin)), (int(xmax), int(ymax))
box_color = [random.randint(0, 255) for _ in range(3)]
line_thickness = round(0.002 * (image.shape[0] + image.shape[1]) / 2) + 1
cv2.rectangle(img=image, pt1=point1, pt2=point2, color=box_color, thickness=line_thickness, lineType=cv2.LINE_AA)
if label:
font_thickness = max(line_thickness - 1, 1)
font_face = 0
font_scale = line_thickness / 3
font_color = (255, 255, 255)
text_size = cv2.getTextSize(text=label, fontFace=font_face, fontScale=font_scale, thickness=font_thickness)[0]
# Calculate rectangle coordinates
rectangle_point1 = point1
rectangle_point2 = (point1[0] + text_size[0], point1[1] - text_size[1] - 3)
# Add filled rectangle
cv2.rectangle(img=image, pt1=rectangle_point1, pt2=rectangle_point2, color=box_color, thickness=-1, lineType=cv2.LINE_AA)
# Calculate text position
text_position = point1[0], point1[1] - 3
# Add text with label to filled rectangle
cv2.putText(img=image, text=label, org=text_position, fontFace=font_face, fontScale=font_scale, color=font_color, thickness=font_thickness, lineType=cv2.LINE_AA)
return image
.. code:: ipython3
from typing import Dict
from openvino.runtime.utils.data_helpers import OVDict
def visualize_inference_result(inference_result: OVDict, image: np.ndarray, labels_map: Dict, detections_limit: Optional[int] = None):
"""
Helper function for visualizing inference result on the image
Parameters
----------
inference_result : OVDict
Result of the compiled model inference on the test image
image : np.ndarray
Original image to use for visualization
labels_map : Dict
Dictionary with mappings of detection classes numbers and its names
detections_limit : int, optional
Number of detections to show on the image, if not provided all detections will be shown (default is None)
"""
detection_boxes: np.ndarray = inference_result.get("detection_boxes")
detection_classes: np.ndarray = inference_result.get("detection_classes")
detection_scores: np.ndarray = inference_result.get("detection_scores")
num_detections: np.ndarray = inference_result.get("num_detections")
detections_limit = int(
min(detections_limit, num_detections[0])
if detections_limit is not None
else num_detections[0]
)
# Normalize detection boxes coordinates to original image size
original_image_height, original_image_width, _ = image.shape
normalized_detection_boxex = detection_boxes[::] * [
original_image_height,
original_image_width,
original_image_height,
original_image_width,
]
image_with_detection_boxex = np.copy(image)
for i in range(detections_limit):
detected_class_name = labels_map[int(detection_classes[0, i])]
score = detection_scores[0, i]
label = f"{detected_class_name} {score:.2f}"
add_detection_box(
box=normalized_detection_boxex[0, i],
image=image_with_detection_boxex,
label=label,
)
plt.imshow(image_with_detection_boxex)
TensorFlow Object Detection model
(`faster_rcnn_resnet50_v1_640x640 <https://tfhub.dev/tensorflow/faster_rcnn/resnet50_v1_640x640/1>`__)
used in this notebook was trained on `COCO
2017 <https://cocodataset.org/>`__ dataset with 91 classes. For better
visualization experience we can use COCO dataset labels with human
readable class names instead of class numbers or indexes.
We can download COCO dataset classes labels from `Open Model
Zoo <https://github.com/openvinotoolkit/open_model_zoo/>`__:
.. code:: ipython3
coco_labels_file_path = Path("./data/coco_91cl.txt")
download_file(
url="https://raw.githubusercontent.com/openvinotoolkit/open_model_zoo/master/data/dataset_classes/coco_91cl.txt",
filename=coco_labels_file_path.name,
directory=coco_labels_file_path.parent,
)
.. parsed-literal::
data/coco_91cl.txt: 0%| | 0.00/421 [00:00<?, ?B/s]
.. parsed-literal::
PosixPath('/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-448/.workspace/scm/ov-notebook/notebooks/120-tensorflow-object-detection-to-openvino/data/coco_91cl.txt')
Then we need to create dictionary ``coco_labels_map`` with mappings
between detection classes numbers and its names from the downloaded
file:
.. code:: ipython3
with open(coco_labels_file_path, "r") as file:
coco_labels = file.read().strip().split("\n")
coco_labels_map = dict(enumerate(coco_labels, 1))
print(coco_labels_map)
.. parsed-literal::
{1: 'person', 2: 'bicycle', 3: 'car', 4: 'motorcycle', 5: 'airplan', 6: 'bus', 7: 'train', 8: 'truck', 9: 'boat', 10: 'traffic light', 11: 'fire hydrant', 12: 'street sign', 13: 'stop sign', 14: 'parking meter', 15: 'bench', 16: 'bird', 17: 'cat', 18: 'dog', 19: 'horse', 20: 'sheep', 21: 'cow', 22: 'elephant', 23: 'bear', 24: 'zebra', 25: 'giraffe', 26: 'hat', 27: 'backpack', 28: 'umbrella', 29: 'shoe', 30: 'eye glasses', 31: 'handbag', 32: 'tie', 33: 'suitcase', 34: 'frisbee', 35: 'skis', 36: 'snowboard', 37: 'sports ball', 38: 'kite', 39: 'baseball bat', 40: 'baseball glove', 41: 'skateboard', 42: 'surfboard', 43: 'tennis racket', 44: 'bottle', 45: 'plate', 46: 'wine glass', 47: 'cup', 48: 'fork', 49: 'knife', 50: 'spoon', 51: 'bowl', 52: 'banana', 53: 'apple', 54: 'sandwich', 55: 'orange', 56: 'broccoli', 57: 'carrot', 58: 'hot dog', 59: 'pizza', 60: 'donut', 61: 'cake', 62: 'chair', 63: 'couch', 64: 'potted plant', 65: 'bed', 66: 'mirror', 67: 'dining table', 68: 'window', 69: 'desk', 70: 'toilet', 71: 'door', 72: 'tv', 73: 'laptop', 74: 'mouse', 75: 'remote', 76: 'keyboard', 77: 'cell phone', 78: 'microwave', 79: 'oven', 80: 'toaster', 81: 'sink', 82: 'refrigerator', 83: 'blender', 84: 'book', 85: 'clock', 86: 'vase', 87: 'scissors', 88: 'teddy bear', 89: 'hair drier', 90: 'toothbrush', 91: 'hair brush'}
Finally, we are ready to visualize model inference results on the
original test image:
.. code:: ipython3
visualize_inference_result(
inference_result=inference_result,
image=image,
labels_map=coco_labels_map,
detections_limit=5,
)
.. image:: 120-tensorflow-object-detection-to-openvino-with-output_files/120-tensorflow-object-detection-to-openvino-with-output_38_0.png
Next Steps
----------
This section contains suggestions on how to additionally improve the
performance of your application using OpenVINO.
Async inference pipeline
~~~~~~~~~~~~~~~~~~~~~~~~
The key advantage of the Async API is that when a device is busy with
inference, the application can perform other tasks in parallel (for
example, populating inputs or scheduling other requests) rather than
wait for the current inference to complete first. To understand how to
perform async inference using openvino, refer to the `Async API
tutorial <115-async-api-with-output.html>`__.
Integration preprocessing to model
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Preprocessing API enables making preprocessing a part of the model
reducing application code and dependency on additional image processing
libraries. The main advantage of Preprocessing API is that preprocessing
steps will be integrated into the execution graph and will be performed
on a selected device (CPU/GPU etc.) rather than always being executed on
CPU as part of an application. This will improve selected device
utilization.
For more information, refer to the `Optimize Preprocessing
tutorial <118-optimize-preprocessing-with-output.html>`__
and to the overview of `Preprocessing
API <https://docs.openvino.ai/2023.0/openvino_docs_OV_Runtime_UG_Preprocessing_Overview.html>`__.

3
docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output_files/120-tensorflow-object-detection-to-openvino-with-output_25_1.png Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:899a1126af7e881b5b8ad2182133a5334f3c98031a9d1b5d9285a76b44a162fc
size 395346

3
docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output_files/120-tensorflow-object-detection-to-openvino-with-output_38_0.png Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:59da4a179a69c8527e526a4b705f9d2ac5225cb90e2489353e36340dd12b481c
size 391541

8
docs/notebooks/120-tensorflow-object-detection-to-openvino-with-output_files/index.html Normal file
View File

@@ -0,0 +1,8 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/120-tensorflow-object-detection-to-openvino-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/120-tensorflow-object-detection-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="120-tensorflow-object-detection-to-openvino-with-output_25_1.png">120-tensorflow-object-detection-to-openvino-wit..&gt;</a> 12-Jul-2023 00:11 395346
<a href="120-tensorflow-object-detection-to-openvino-with-output_38_0.png">120-tensorflow-object-detection-to-openvino-wit..&gt;</a> 12-Jul-2023 00:11 391541
</pre><hr></body>
</html>

80
docs/notebooks/201-vision-monodepth-with-output.rst
View File

File diff suppressed because one or more lines are too long

0
docs/notebooks/201-vision-monodepth-with-output_files/201-vision-monodepth-with-output_14_0.png → docs/notebooks/201-vision-monodepth-with-output_files/201-vision-monodepth-with-output_18_0.png
View File

6
docs/notebooks/201-vision-monodepth-with-output_files/index.html
View File

@@ -1,7 +1,7 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/201-vision-monodepth-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/201-vision-monodepth-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/201-vision-monodepth-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="201-vision-monodepth-with-output_14_0.png">201-vision-monodepth-with-output_14_0.png</a> 22-Jun-2023 00:06 959858
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/201-vision-monodepth-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="201-vision-monodepth-with-output_18_0.png">201-vision-monodepth-with-output_18_0.png</a> 12-Jul-2023 00:11 959858
</pre><hr></body>
</html>

109
docs/notebooks/202-vision-superresolution-image-with-output.rst
View File

@@ -17,6 +17,15 @@ pp. 2777-2784, doi: 10.1109/ICPR.2018.8545760.
Preparation
-----------
Install requirements
~~~~~~~~~~~~~~~~~~~~
.. code:: ipython3
!pip install -q 'openvino>=2023.0.0'
!pip install -q opencv-python
!pip install -q pillow matplotlib
Imports
~~~~~~~
@@ -24,9 +33,7 @@ Imports
import os
import time
import requests
from pathlib import Path
import sys
import cv2
import matplotlib.pyplot as plt
@@ -37,16 +44,47 @@ Imports
from PIL import Image
from openvino.runtime import Core
sys.path.append("../utils")
from notebook_utils import download_file
.. code:: ipython3
# Define a download file helper function
def download_file(url: str, path: Path) -> None:
"""Download file."""
import urllib.request
path.parent.mkdir(parents=True, exist_ok=True)
urllib.request.urlretrieve(url, path)
Settings
~~~~~~~~
Select inference device
^^^^^^^^^^^^^^^^^^^^^^^
select device from dropdown list for running inference using OpenVINO
.. code:: ipython3
# Device to use for inference. For example, "CPU", or "GPU".
DEVICE = 'CPU'
import ipywidgets as widgets
core = Core()
device = widgets.Dropdown(
options=core.available_devices + ["AUTO"],
value='AUTO',
description='Device:',
disabled=False,
)
device
.. parsed-literal::
Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')
.. code:: ipython3
# 1032: 4x superresolution, 1033: 3x superresolution
model_name = 'single-image-super-resolution-1032'
@@ -64,24 +102,11 @@ Settings
model_xml_url = base_url + model_xml_name
model_bin_url = base_url + model_bin_name
download_file(model_xml_url, model_xml_name, base_model_dir)
download_file(model_bin_url, model_bin_name, base_model_dir)
download_file(model_xml_url, model_xml_path)
download_file(model_bin_url, model_bin_path)
else:
print(f'{model_name} already downloaded to {base_model_dir}')
.. parsed-literal::
model/single-image-super-resolution-1032.xml: 0%| | 0.00/89.3k [00:00<?, ?B/s]
.. parsed-literal::
model/single-image-super-resolution-1032.bin: 0%| | 0.00/58.4k [00:00<?, ?B/s]
Functions
~~~~~~~~~
@@ -122,24 +147,6 @@ Functions
return textim.get()
def load_image(path: str) -> np.ndarray:
"""
Loads an image from `path` and returns it as BGR numpy array.
:param path: path to an image filename or url
:return: image as numpy array, with BGR channel order
"""
if path.startswith("http"):
# Set User-Agent to Mozilla because some websites block requests
# with User-Agent Python.
response = requests.get(path, headers={"User-Agent": "Mozilla/5.0"})
array = np.asarray(bytearray(response.content), dtype="uint8")
image = cv2.imdecode(array, -1) # Loads the image as BGR.
else:
image = cv2.imread(path)
return image
def convert_result_to_image(result) -> np.ndarray:
"""
Convert network result of floating point numbers to image with integer
@@ -177,7 +184,7 @@ about the network inputs and outputs.
ie = Core()
model = ie.read_model(model=model_xml_path)
compiled_model = ie.compile_model(model=model, device_name=DEVICE)
compiled_model = ie.compile_model(model=model, device_name=device.value)
# Network inputs and outputs are dictionaries. Get the keys for the
# dictionaries.
@@ -218,11 +225,13 @@ Load and Show the Input Image
.. code:: ipython3
IMAGE_PATH = Path("../data/image/tower.jpg")
IMAGE_PATH = Path("./data/tower.jpg")
OUTPUT_PATH = Path("output/")
os.makedirs(str(OUTPUT_PATH), exist_ok=True)
full_image = load_image(str(IMAGE_PATH))
download_file('https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/tower.jpg', IMAGE_PATH)
full_image = cv2.imread(str(IMAGE_PATH))
# Uncomment these lines to load a raw image as BGR.
# import rawpy
@@ -239,7 +248,7 @@ Load and Show the Input Image
.. image:: 202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_10_1.png
.. image:: 202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_15_1.png
Superresolution on a Crop of the Image
@@ -291,7 +300,7 @@ as the crop size.
.. image:: 202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_12_1.png
.. image:: 202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_17_1.png
Reshape/Resize Crop for Model Input
@@ -357,7 +366,7 @@ Show the bicubic image and the enhanced superresolution image.
.. image:: 202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_18_1.png
.. image:: 202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_23_1.png
Save Superresolution and Bicubic Image Crop
@@ -425,7 +434,7 @@ Write Animated GIF with Bicubic/Superresolution Comparison
.. image:: 202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_22_1.png
.. image:: 202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_27_1.png
:width: 960px
@@ -437,7 +446,9 @@ This may take a while. For the video, the superresolution and bicubic
image are resized by a factor of 2 to improve processing speed. This
gives an indication of the superresolution effect. The video is saved as
an ``.avi`` file. You can click on the link to download the video, or
open it directly from the images directory, and play it locally.
open it directly from the ``output/`` directory, and play it locally. >
Note: If you run the example in Google Colab, download video files using
the ``Files`` tool.
.. code:: ipython3
@@ -670,8 +681,8 @@ as total time to process each patch.
.. parsed-literal::
Processed 42 patches in 5.06 seconds. Total patches per second (including processing): 8.29.
Inference patches per second: 16.98
Processed 42 patches in 4.73 seconds. Total patches per second (including processing): 8.87.
Inference patches per second: 17.65
Save superresolution image and the bicubic image

0
docs/notebooks/202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_10_1.png → docs/notebooks/202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_15_1.png
View File

0
docs/notebooks/202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_12_1.png → docs/notebooks/202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_17_1.png
View File

0
docs/notebooks/202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_18_1.png → docs/notebooks/202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_23_1.png
View File

0
docs/notebooks/202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_22_1.png → docs/notebooks/202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_27_1.png
View File

12
docs/notebooks/202-vision-superresolution-image-with-output_files/index.html
View File

@@ -1,10 +1,10 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/202-vision-superresolution-image-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/202-vision-superresolution-image-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/202-vision-superresolution-image-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="202-vision-superresolution-image-with-output_10_1.png">202-vision-superresolution-image-with-output_10..&gt;</a> 22-Jun-2023 00:06 272963
<a href="202-vision-superresolution-image-with-output_12_1.png">202-vision-superresolution-image-with-output_12..&gt;</a> 22-Jun-2023 00:06 356735
<a href="202-vision-superresolution-image-with-output_18_1.png">202-vision-superresolution-image-with-output_18..&gt;</a> 22-Jun-2023 00:06 2896276
<a href="202-vision-superresolution-image-with-output_22_1.png">202-vision-superresolution-image-with-output_22..&gt;</a> 22-Jun-2023 00:06 3207711
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230711220806/dist/rst_files/202-vision-superresolution-image-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="202-vision-superresolution-image-with-output_15_1.png">202-vision-superresolution-image-with-output_15..&gt;</a> 12-Jul-2023 00:11 272963
<a href="202-vision-superresolution-image-with-output_17_1.png">202-vision-superresolution-image-with-output_17..&gt;</a> 12-Jul-2023 00:11 356735
<a href="202-vision-superresolution-image-with-output_23_1.png">202-vision-superresolution-image-with-output_23..&gt;</a> 12-Jul-2023 00:11 2896276
<a href="202-vision-superresolution-image-with-output_27_1.png">202-vision-superresolution-image-with-output_27..&gt;</a> 12-Jul-2023 00:11 3207711
</pre><hr></body>
</html>

Some files were not shown because too many files have changed in this diff Show More