IntenseWebs/openvino
3
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

updating-tutorials (#18210)

Browse Source
This commit is contained in:
Sebastian Golebiewski authored and GitHub committed 2023-06-26 09:41:49 +02:00
1 parent bf299c807e
commit 9e91076a63
221 files changed
+8057 -7032
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

docs/nbdoc/consts.py
+1 -1
View File
@@ -8,7 +8,7 @@ repo_owner = "openvinotoolkit"
repo_name = "openvino_notebooks"
artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/"
artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/"
blacklisted_extensions = ['.xml', '.bin']
docs/notebooks/001-hello-world-with-output.rst
+45 -4
View File
@@ -5,7 +5,7 @@ This basic introduction to OpenVINO™ shows how to do inference with an
image classification model.
A pre-trained `MobileNetV3
model <https://docs.openvino.ai/latest/omz_models_model_mobilenet_v3_small_1_0_224_tf.html>`__
model <https://docs.openvino.ai/2023.0/omz_models_model_mobilenet_v3_small_1_0_224_tf.html>`__
from `Open Model
Zoo <https://github.com/openvinotoolkit/open_model_zoo/>`__ is used in
this tutorial. For more information about how OpenVINO IR models are
@@ -18,10 +18,51 @@ Imports
.. code:: ipython3
from pathlib import Path
import sys
import cv2
import matplotlib.pyplot as plt
import numpy as np
from openvino.runtime import Core
sys.path.append("../utils")
from notebook_utils import download_file
Download the Model and data samples
-----------------------------------
.. code:: ipython3
base_artifacts_dir = Path('./artifacts').expanduser()
model_name = "v3-small_224_1.0_float"
model_xml_name = f'{model_name}.xml'
model_bin_name = f'{model_name}.bin'
model_xml_path = base_artifacts_dir / model_xml_name
base_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/mobelinet-v3-tf/FP32/'
if not model_xml_path.exists():
download_file(base_url + model_xml_name, model_xml_name, base_artifacts_dir)
download_file(base_url + model_bin_name, model_bin_name, base_artifacts_dir)
else:
print(f'{model_name} already downloaded to {base_artifacts_dir}')
.. parsed-literal::
artifacts/v3-small_224_1.0_float.xml: 0%| | 0.00/294k [00:00<?, ?B/s]
.. parsed-literal::
artifacts/v3-small_224_1.0_float.bin: 0%| | 0.00/4.84M [00:00<?, ?B/s]
Load the Model
--------------
@@ -29,7 +70,7 @@ Load the Model
.. code:: ipython3
ie = Core()
model = ie.read_model(model="model/v3-small_224_1.0_float.xml")
model = ie.read_model(model=model_xml_path)
compiled_model = ie.compile_model(model=model, device_name="CPU")
output_layer = compiled_model.output(0)
@@ -47,11 +88,11 @@ Load an Image
# Reshape to model input shape.
input_image = np.expand_dims(input_image, 0)
plt.imshow(image)
plt.imshow(image);
.. image:: 001-hello-world-with-output_files/001-hello-world-with-output_6_0.png
.. image:: 001-hello-world-with-output_files/001-hello-world-with-output_8_0.png
Do Inference
docs/notebooks/001-hello-world-with-output_files/001-hello-world-with-output_6_0.png → docs/notebooks/001-hello-world-with-output_files/001-hello-world-with-output_8_0.png
View File
File renamed without changes.
docs/notebooks/001-hello-world-with-output_files/index.html
+3 -3
View File
@@ -1,7 +1,7 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/001-hello-world-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/001-hello-world-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/001-hello-world-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="001-hello-world-with-output_6_0.png">001-hello-world-with-output_6_0.png</a> 30-May-2023 00:09 387941
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/001-hello-world-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="001-hello-world-with-output_8_0.png">001-hello-world-with-output_8_0.png</a> 22-Jun-2023 00:06 387941
</pre><hr></body>
</html>
docs/notebooks/002-openvino-api-with-output.rst
+276 -13
View File
@@ -12,6 +12,7 @@ covers:
- `ONNX Model <#ONNX-Model>`__
- `PaddlePaddle Model <#PaddlePaddle-Model>`__
- `TensorFlow Model <#TensorFlow-Model>`__
- `TensorFlow Lite Model <#TensorFlow-Lite-Model>`__
- `Getting Information about a
Model <#Getting-Information-about-a-Model>`__
@@ -28,11 +29,19 @@ covers:
- `Caching a Model <#Caching-a-Model>`__
The notebook is divided into sections with headers. Each section is
standalone and does not depend on previous sections. A segmentation and
classification OpenVINO IR model and a segmentation ONNX model are
provided as examples. These model files can be replaced with your own
models. The exact outputs will be different, but the process is the
same.
standalone and does not depend on any previous sections except for the
next cell with imports. A segmentation and classification OpenVINO IR
model and a segmentation ONNX model are provided as examples. These
model files can be replaced with your own models. The exact outputs will
be different, but the process is the same.
.. code:: ipython3
# Required imports. Please execute this cell first.
import sys
sys.path.append('../utils')
from notebook_utils import download_file
Loading OpenVINO Runtime and Showing Info
-----------------------------------------
@@ -78,7 +87,7 @@ After initializing OpenVINO Runtime, first read the model file with
``compile_model()`` method.
`OpenVINO™ supports several model
formats <https://docs.openvino.ai/latest/Supported_Model_Formats.html#doxid-supported-model-formats>`__
formats <https://docs.openvino.ai/2023.0/Supported_Model_Formats.html#doxid-supported-model-formats>`__
and enables developers to convert them to its own OpenVINO IR format
using a tool dedicated to this task.
@@ -97,7 +106,7 @@ file has a different filename, it can be specified using the ``weights``
parameter in ``read_model()``.
The OpenVINO `Model
Optimizer <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html#doxid-openvino-docs-m-o-d-g-deep-learning-model-optimizer-dev-guide>`__
Optimizer <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html#doxid-openvino-docs-m-o-d-g-deep-learning-model-optimizer-dev-guide>`__
tool is used to convert models to OpenVINO IR format. Model Optimizer
reads the original model and creates an OpenVINO IR model (.xml and .bin
files) so inference can be performed without delays due to format
@@ -111,6 +120,36 @@ and
`pytorch-onnx-to-openvino <102-pytorch-onnx-to-openvino-with-output.html>`__
notebooks.
.. code:: ipython3
ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/'
ir_model_name_xml = 'classification.xml'
ir_model_name_bin = 'classification.bin'
download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory='model')
download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory='model')
.. parsed-literal::
model/classification.xml: 0%| | 0.00/179k [00:00<?, ?B/s]
.. parsed-literal::
model/classification.bin: 0%| | 0.00/4.84M [00:00<?, ?B/s]
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
.. code:: ipython3
from openvino.runtime import Core
@@ -136,6 +175,28 @@ Reading and loading an ONNX model, which is a single ``.onnx`` file,
works the same way as with an OpenVINO IR model. The ``model`` argument
points to the filename of an ONNX model.
.. code:: ipython3
onnx_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/segmentation.onnx'
onnx_model_name = 'segmentation.onnx'
download_file(onnx_model_url, filename=onnx_model_name, directory='model')
.. parsed-literal::
model/segmentation.onnx: 0%| | 0.00/4.41M [00:00<?, ?B/s]
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/segmentation.onnx')
.. code:: ipython3
from openvino.runtime import Core
@@ -162,12 +223,42 @@ models saved for inference can also be passed to OpenVINO Runtime
without any conversion step. Pass the filename with extension to
``read_model`` and exported an OpenVINO IR with ``serialize``
.. code:: ipython3
paddle_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/'
paddle_model_name = 'inference.pdmodel'
paddle_params_name = 'inference.pdiparams'
download_file(paddle_model_url + paddle_model_name, filename=paddle_model_name, directory='model')
download_file(paddle_model_url + paddle_params_name, filename=paddle_params_name, directory='model')
.. parsed-literal::
model/inference.pdmodel: 0%| | 0.00/1.03M [00:00<?, ?B/s]
.. parsed-literal::
model/inference.pdiparams: 0%| | 0.00/21.0M [00:00<?, ?B/s]
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/inference.pdiparams')
.. code:: ipython3
from openvino.runtime import Core
ie = Core()
paddle_model_path = "model/inference.pdmodel"
paddle_model_path = 'model/inference.pdmodel'
model_paddle = ie.read_model(model=paddle_model_path)
compiled_model_paddle = ie.compile_model(model=model_paddle, device_name="CPU")
@@ -188,7 +279,29 @@ TensorFlow models is available as a preview feature in the OpenVINO
upcoming 2023 releases. > Currently support is limited to only frozen
graph inference format. Other TensorFlow model formats must be converted
to OpenVINO IR using `Model
Optimizer <https://docs.openvino.ai/latest/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow.html>`__.
Optimizer <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow.html>`__.
.. code:: ipython3
pb_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/classification.pb'
pb_model_name = 'classification.pb'
download_file(pb_model_url, filename=pb_model_name, directory='model')
.. parsed-literal::
model/classification.pb: 0%| | 0.00/9.88M [00:00<?, ?B/s]
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.pb')
.. code:: ipython3
@@ -206,6 +319,57 @@ Optimizer <https://docs.openvino.ai/latest/openvino_docs_MO_DG_prepare_model_con
serialize(model_tf, xml_path="model/exported_tf_model.xml")
TensorFlow Lite Model
~~~~~~~~~~~~~~~~~~~~~
`TFLite <https://www.tensorflow.org/lite>`__ models saved for inference
can also be passed to OpenVINO Runtime. Pass the filename with extension
``.tflite`` to ``read_model`` and exported an OpenVINO IR with
``serialize``.
This tutorial uses the image classification model
`inception_v4_quant <https://tfhub.dev/tensorflow/lite-model/inception_v4_quant/1/default/1>`__.
It is pre-trained model optimized to work with TensorFlow Lite.
.. code:: ipython3
from pathlib import Path
tflite_model_url = 'https://tfhub.dev/tensorflow/lite-model/inception_v4_quant/1/default/1?lite-format=tflite'
tflite_model_path = Path('model/classification.tflite')
download_file(tflite_model_url, filename=tflite_model_path.name, directory=tflite_model_path.parent)
.. parsed-literal::
model/classification.tflite: 0%| | 0.00/40.9M [00:00<?, ?B/s]
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.tflite')
.. code:: ipython3
from openvino.runtime import Core
ie = Core()
model_tflite = ie.read_model(tflite_model_path)
compiled_model_tflite = ie.compile_model(model=model_tflite, device_name="CPU")
.. code:: ipython3
from openvino.runtime import serialize
serialize(model_tflite, xml_path="model/exported_tflite_model.xml")
Getting Information about a Model
---------------------------------
@@ -216,6 +380,30 @@ CompiledModel instance. While using ``model.inputs`` and
``model.outputs`` in the cells below, you can also use
``compiled_model.inputs`` and ``compiled_model.outputs``.
.. code:: ipython3
ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/'
ir_model_name_xml = 'classification.xml'
ir_model_name_bin = 'classification.bin'
download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory='model')
download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory='model')
.. parsed-literal::
'model/classification.xml' already exists.
'model/classification.bin' already exists.
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
Model Inputs
~~~~~~~~~~~~
@@ -362,7 +550,7 @@ Doing Inference on a Model
The diagram below shows a typical inference pipeline with OpenVINO
.. figure:: https://docs.openvino.ai/latest/_images/IMPLEMENT_PIPELINE_with_API_C.svg
.. figure:: https://docs.openvino.ai/2023.0/_images/IMPLEMENT_PIPELINE_with_API_C.svg
:alt: image.png
image.png
@@ -380,6 +568,30 @@ input, wrapping to a dictionary or list can be omitted.
**Load the network**
.. code:: ipython3
ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/'
ir_model_name_xml = 'classification.xml'
ir_model_name_bin = 'classification.bin'
download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory='model')
download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory='model')
.. parsed-literal::
'model/classification.xml' already exists.
'model/classification.bin' already exists.
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
.. code:: ipython3
from openvino.runtime import Core
@@ -526,6 +738,36 @@ model accuracy may also suffer if you reshape the model input shape.
First check the input shape of the model, then reshape it to the new
input shape.
.. code:: ipython3
ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/'
ir_model_name_xml = 'segmentation.xml'
ir_model_name_bin = 'segmentation.bin'
download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory='model')
download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory='model')
.. parsed-literal::
model/segmentation.xml: 0%| | 0.00/1.38M [00:00<?, ?B/s]
.. parsed-literal::
model/segmentation.bin: 0%| | 0.00/1.09M [00:00<?, ?B/s]
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/segmentation.bin')
.. code:: ipython3
from openvino.runtime import Core, PartialShape
@@ -582,9 +824,6 @@ set ``new_shape = (2,3,544,544)`` in the cell above.
.. code:: ipython3
from openvino.runtime import Core, PartialShape
ie = Core()
segmentation_model_xml = "model/segmentation.xml"
segmentation_model = ie.read_model(model=segmentation_model_xml)
segmentation_input_layer = segmentation_model.input(0)
@@ -654,6 +893,30 @@ the cache.
*Note: Model Caching is also available on CPU devices*
.. code:: ipython3
ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/002-example-models/'
ir_model_name_xml = 'classification.xml'
ir_model_name_bin = 'classification.bin'
download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory='model')
download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory='model')
.. parsed-literal::
'model/classification.xml' already exists.
'model/classification.bin' already exists.
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/002-openvino-api/model/classification.bin')
.. code:: ipython3
import time
docs/notebooks/003-hello-segmentation-with-output.rst
+45 -8
View File
@@ -4,7 +4,7 @@ Hello Image Segmentation
A very basic introduction to using segmentation models with OpenVINO™.
In this tutorial, a pre-trained
`road-segmentation-adas-0001 <https://docs.openvino.ai/latest/omz_models_model_road_segmentation_adas_0001.html>`__
`road-segmentation-adas-0001 <https://docs.openvino.ai/2023.0/omz_models_model_road_segmentation_adas_0001.html>`__
model from the `Open Model
Zoo <https://github.com/openvinotoolkit/open_model_zoo/>`__ is used.
ADAS stands for Advanced Driver Assistance Services. The model
@@ -22,7 +22,44 @@ Imports
from openvino.runtime import Core
sys.path.append("../utils")
from notebook_utils import segmentation_map_to_image
from notebook_utils import segmentation_map_to_image, download_file
Download model weights
----------------------
.. code:: ipython3
from pathlib import Path
base_model_dir = Path("./model").expanduser()
model_name = "road-segmentation-adas-0001"
model_xml_name = f'{model_name}.xml'
model_bin_name = f'{model_name}.bin'
model_xml_path = base_model_dir / model_xml_name
if not model_xml_path.exists():
model_xml_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/road-segmentation-adas-0001/FP32/road-segmentation-adas-0001.xml"
model_bin_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/road-segmentation-adas-0001/FP32/road-segmentation-adas-0001.bin"
download_file(model_xml_url, model_xml_name, base_model_dir)
download_file(model_bin_url, model_bin_name, base_model_dir)
else:
print(f'{model_name} already downloaded to {base_model_dir}')
.. parsed-literal::
model/road-segmentation-adas-0001.xml: 0%| | 0.00/389k [00:00<?, ?B/s]
.. parsed-literal::
model/road-segmentation-adas-0001.bin: 0%| | 0.00/720k [00:00<?, ?B/s]
Load the Model
--------------
@@ -31,7 +68,7 @@ Load the Model
ie = Core()
model = ie.read_model(model="model/road-segmentation-adas-0001.xml")
model = ie.read_model(model=model_xml_path)
compiled_model = ie.compile_model(model=model, device_name="CPU")
input_layer_ir = compiled_model.input(0)
@@ -69,12 +106,12 @@ provided.
.. parsed-literal::
<matplotlib.image.AxesImage at 0x7f02b820d640>
<matplotlib.image.AxesImage at 0x7f40900e8ac0>
.. image:: 003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_6_1.png
.. image:: 003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_8_1.png
Do Inference
@@ -94,12 +131,12 @@ Do Inference
.. parsed-literal::
<matplotlib.image.AxesImage at 0x7f02b80ef6d0>
<matplotlib.image.AxesImage at 0x7f4090039a30>
.. image:: 003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_8_1.png
.. image:: 003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_10_1.png
Prepare Data for Visualization
@@ -142,5 +179,5 @@ Visualize data
.. image:: 003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_12_0.png
.. image:: 003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_14_0.png
docs/notebooks/003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_10_1.png
+3
View File
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d9d40b840eea67857e42a78dd89b65799215e806e4f8d23b0523b0c163cb453b
size 20550
docs/notebooks/003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_12_0.png → docs/notebooks/003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_14_0.png
View File
File renamed without changes.
docs/notebooks/003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_6_1.png
-3
View File
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:76113c575caa9c8a8aca45d3ec6ebd7a4b513dadffd8e9e63861a7a041d7e5de
size 249032
docs/notebooks/003-hello-segmentation-with-output_files/003-hello-segmentation-with-output_8_1.png
+2 -2
View File
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d9d40b840eea67857e42a78dd89b65799215e806e4f8d23b0523b0c163cb453b
size 20550
oid sha256:76113c575caa9c8a8aca45d3ec6ebd7a4b513dadffd8e9e63861a7a041d7e5de
size 249032
docs/notebooks/003-hello-segmentation-with-output_files/index.html
+5 -5
View File
@@ -1,9 +1,9 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/003-hello-segmentation-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/003-hello-segmentation-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/003-hello-segmentation-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="003-hello-segmentation-with-output_12_0.png">003-hello-segmentation-with-output_12_0.png</a> 30-May-2023 00:08 260045
<a href="003-hello-segmentation-with-output_6_1.png">003-hello-segmentation-with-output_6_1.png</a> 30-May-2023 00:08 249032
<a href="003-hello-segmentation-with-output_8_1.png">003-hello-segmentation-with-output_8_1.png</a> 30-May-2023 00:08 20550
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/003-hello-segmentation-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="003-hello-segmentation-with-output_10_1.png">003-hello-segmentation-with-output_10_1.png</a> 22-Jun-2023 00:06 20550
<a href="003-hello-segmentation-with-output_14_0.png">003-hello-segmentation-with-output_14_0.png</a> 22-Jun-2023 00:06 260045
<a href="003-hello-segmentation-with-output_8_1.png">003-hello-segmentation-with-output_8_1.png</a> 22-Jun-2023 00:06 249032
</pre><hr></body>
</html>
docs/notebooks/004-hello-detection-with-output.rst
+45 -4
View File
@@ -5,7 +5,7 @@ A very basic introduction to using object detection models with
OpenVINO™.
The
`horizontal-text-detection-0001 <https://docs.openvino.ai/latest/omz_models_model_horizontal_text_detection_0001.html>`__
`horizontal-text-detection-0001 <https://docs.openvino.ai/2023.0/omz_models_model_horizontal_text_detection_0001.html>`__
model from `Open Model
Zoo <https://github.com/openvinotoolkit/open_model_zoo/>`__ is used. It
detects horizontal text in images and returns a blob of data in the
@@ -25,6 +25,47 @@ Imports
import matplotlib.pyplot as plt
import numpy as np
from openvino.runtime import Core
from pathlib import Path
import sys
sys.path.append("../utils")
from notebook_utils import download_file
Download model weights
----------------------
.. code:: ipython3
base_model_dir = Path("./model").expanduser()
model_name = "horizontal-text-detection-0001"
model_xml_name = f'{model_name}.xml'
model_bin_name = f'{model_name}.bin'
model_xml_path = base_model_dir / model_xml_name
model_bin_path = base_model_dir / model_bin_name
if not model_xml_path.exists():
model_xml_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.xml"
model_bin_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.bin"
download_file(model_xml_url, model_xml_name, base_model_dir)
download_file(model_bin_url, model_bin_name, base_model_dir)
else:
print(f'{model_name} already downloaded to {base_model_dir}')
.. parsed-literal::
model/horizontal-text-detection-0001.xml: 0%| | 0.00/680k [00:00<?, ?B/s]
.. parsed-literal::
model/horizontal-text-detection-0001.bin: 0%| | 0.00/7.39M [00:00<?, ?B/s]
Load the Model
--------------
@@ -33,7 +74,7 @@ Load the Model
ie = Core()
model = ie.read_model(model="model/horizontal-text-detection-0001.xml")
model = ie.read_model(model=model_xml_path)
compiled_model = ie.compile_model(model=model, device_name="CPU")
input_layer_ir = compiled_model.input(0)
@@ -60,7 +101,7 @@ Load an Image
.. image:: 004-hello-detection-with-output_files/004-hello-detection-with-output_6_0.png
.. image:: 004-hello-detection-with-output_files/004-hello-detection-with-output_8_0.png
Do Inference
@@ -133,5 +174,5 @@ Visualize Results
.. image:: 004-hello-detection-with-output_files/004-hello-detection-with-output_11_0.png
.. image:: 004-hello-detection-with-output_files/004-hello-detection-with-output_13_0.png
docs/notebooks/004-hello-detection-with-output_files/004-hello-detection-with-output_11_0.png → docs/notebooks/004-hello-detection-with-output_files/004-hello-detection-with-output_13_0.png
View File
File renamed without changes.
docs/notebooks/004-hello-detection-with-output_files/004-hello-detection-with-output_6_0.png → docs/notebooks/004-hello-detection-with-output_files/004-hello-detection-with-output_8_0.png
View File
File renamed without changes.
docs/notebooks/004-hello-detection-with-output_files/index.html
+4 -4
View File
@@ -1,8 +1,8 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/004-hello-detection-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/004-hello-detection-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/004-hello-detection-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="004-hello-detection-with-output_11_0.png">004-hello-detection-with-output_11_0.png</a> 30-May-2023 00:09 457214
<a href="004-hello-detection-with-output_6_0.png">004-hello-detection-with-output_6_0.png</a> 30-May-2023 00:09 305482
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/004-hello-detection-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="004-hello-detection-with-output_13_0.png">004-hello-detection-with-output_13_0.png</a> 22-Jun-2023 00:06 457214
<a href="004-hello-detection-with-output_8_0.png">004-hello-detection-with-output_8_0.png</a> 22-Jun-2023 00:06 305482
</pre><hr></body>
</html>
docs/notebooks/101-tensorflow-to-openvino-with-output.rst
+26 -62
View File
@@ -2,13 +2,13 @@ Convert a TensorFlow Model to OpenVINO™
=======================================
This short tutorial shows how to convert a TensorFlow
`MobileNetV3 <https://docs.openvino.ai/latest/omz_models_model_mobilenet_v3_small_1_0_224_tf.html>`__
`MobileNetV3 <https://docs.openvino.ai/2023.0/omz_models_model_mobilenet_v3_small_1_0_224_tf.html>`__
image classification model to OpenVINO `Intermediate
Representation <https://docs.openvino.ai/latest/openvino_docs_MO_DG_IR_and_opsets.html>`__
Representation <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_IR_and_opsets.html>`__
(OpenVINO IR) format, using `Model
Optimizer <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
Optimizer <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
After creating the OpenVINO IR, load the model in `OpenVINO
Runtime <https://docs.openvino.ai/latest/openvino_docs_IE_DG_Deep_Learning_Inference_Engine_DevGuide.html>`__
Runtime <https://docs.openvino.ai/nightly/openvino_docs_OV_UG_OV_Runtime_User_Guide.html>`__
and do inference with a sample image.
Imports
@@ -23,16 +23,16 @@ Imports
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
from IPython.display import Markdown
from openvino.runtime import Core
from openvino.runtime import Core, serialize
from openvino.tools import mo
.. parsed-literal::
2023-05-29 22:25:46.984005: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-05-29 22:25:47.018849: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2023-06-21 22:25:42.619122: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-06-21 22:25:42.653691: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-05-29 22:25:47.534965: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2023-06-21 22:25:43.164083: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
Settings
@@ -68,7 +68,7 @@ and save it to the disk.
.. parsed-literal::
2023-05-29 22:25:48.396946: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
2023-06-21 22:25:44.061927: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
@@ -79,9 +79,9 @@ and save it to the disk.
.. parsed-literal::
2023-05-29 22:25:52.606419: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'inputs' with dtype float and shape [?,1,1,1024]
2023-06-21 22:25:48.238616: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'inputs' with dtype float and shape [?,1,1,1024]
[[{{node inputs}}]]
2023-05-29 22:25:55.768588: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'inputs' with dtype float and shape [?,1,1,1024]
2023-06-21 22:25:51.395055: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'inputs' with dtype float and shape [?,1,1,1024]
[[{{node inputs}}]]
WARNING:absl:Found untraced functions such as _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op while saving (showing 5 of 54). These functions will not be directly callable after loading.
@@ -102,51 +102,21 @@ Convert a Model to OpenVINO IR Format
Convert a TensorFlow Model to OpenVINO IR Format
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Use Model Optimizer to convert a TensorFlow model to OpenVINO IR with
``FP16`` precision. The models are saved to the current directory. Add
mean values to the model and scale the output with the standard
deviation with ``--scale_values``. With these options, it is not
necessary to normalize input data before propagating it through the
network. The original model expects input images in ``RGB`` format. The
converted model also expects images in ``RGB`` format. If you want the
converted model to work with ``BGR`` images, use the
``--reverse-input-channels`` option. For more information about Model
Optimizer, including a description of the command-line options, see the
`Model Optimizer Developer
Guide <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
For information about the model, including input shape, expected color
order and mean values, refer to the `model
documentation <https://docs.openvino.ai/latest/omz_models_model_mobilenet_v3_small_1_0_224_tf.html>`__.
First construct the command for Model Optimizer, and then execute this
command in the notebook by prepending the command with an ``!``. There
may be some errors or warnings in the output. When model optimization is
successful, the last lines of the output will include
``[ SUCCESS ] Generated IR version 11 model.``
Call the OpenVINO Model Optimizer Python API to convert the TensorFlow
model to OpenVINO IR. ``mo.convert_model`` function accept path to saved
model directory and returns OpenVINO Model class instance which
represents this model. Obtained model is ready to use and loading on
device using ``compile_model`` or can be saved on disk using
``serialize`` function. See the `Model Optimizer Developer
Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow.html>`__
for more information about Model Optimizer and TensorFlow models
conversion.
.. code:: ipython3
# Construct the command for Model Optimizer.
mo_command = f"""mo
--saved_model_dir "{model_path}"
--input_shape "[1,224,224,3]"
--model_name "{model_path.name}"
--compress_to_fp16
--output_dir "{model_path.parent}"
"""
mo_command = " ".join(mo_command.split())
print("Model Optimizer command to convert TensorFlow to OpenVINO:")
display(Markdown(f"`{mo_command}`"))
.. parsed-literal::
Model Optimizer command to convert TensorFlow to OpenVINO:
``mo --saved_model_dir "model/v3-small_224_1.0_float" --input_shape "[1,224,224,3]" --model_name "v3-small_224_1.0_float" --compress_to_fp16 --output_dir "model"``
ov_model = mo.convert_model(saved_model_dir=model_path, input_shape=[[1, 224, 224, 3]], compress_to_fp16=True)
serialize(ov_model, ir_path)
.. code:: ipython3
@@ -160,13 +130,7 @@ successful, the last lines of the output will include
.. parsed-literal::
Exporting TensorFlow model to IR... This may take a few minutes.
Check for a new version of Intel(R) Distribution of OpenVINO(TM) toolkit here https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit/download.html?cid=other&source=prod&campid=ww_2023_bu_IOTG_OpenVINO-2022-3&content=upg_all&medium=organic or on https://github.com/openvinotoolkit/openvino
[ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11.
Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/latest/openvino_2_0_transition_guide.html
[ SUCCESS ] Generated IR version 11 model.
[ SUCCESS ] XML file: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/101-tensorflow-to-openvino/model/v3-small_224_1.0_float.xml
[ SUCCESS ] BIN file: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/101-tensorflow-to-openvino/model/v3-small_224_1.0_float.bin
IR model model/v3-small_224_1.0_float.xml already exists.
Test Inference on the Converted Model
@@ -245,7 +209,7 @@ Timing
Measure the time it takes to do inference on thousand images. This gives
an indication of performance. For more accurate benchmarking, use the
`Benchmark
Tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_tool_README.html>`__
Tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_tool_README.html>`__
in OpenVINO. Note that many optimizations are possible to improve the
performance.
@@ -269,5 +233,5 @@ performance.
.. parsed-literal::
IR model in OpenVINO Runtime/CPU: 0.0010 seconds per image, FPS: 1032.55
IR model in OpenVINO Runtime/CPU: 0.0010 seconds per image, FPS: 1010.84
docs/notebooks/101-tensorflow-to-openvino-with-output_files/index.html
+3 -3
View File
@@ -1,7 +1,7 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/101-tensorflow-to-openvino-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/101-tensorflow-to-openvino-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/101-tensorflow-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="101-tensorflow-to-openvino-with-output_16_0.png">101-tensorflow-to-openvino-with-output_16_0.png</a> 30-May-2023 00:08 387941
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/101-tensorflow-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="101-tensorflow-to-openvino-with-output_16_0.png">101-tensorflow-to-openvino-with-output_16_0.png</a> 22-Jun-2023 00:06 387941
</pre><hr></body>
</html>
docs/notebooks/102-pytorch-onnx-to-openvino-with-output.rst
+11 -10
View File
@@ -179,7 +179,7 @@ Use Model Optimizer to convert the ONNX model to OpenVINO IR with
``FP16`` precision. The models are saved inside the current directory.
For more information about Model Optimizer, see the `Model Optimizer
Developer
Guide <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
Executing this command may take a while. There may be some errors or
warnings in the output. When Model Optimization is successful, the last
@@ -221,12 +221,13 @@ lines of the output will include:
.. parsed-literal::
Exporting ONNX model to IR... This may take a few minutes.
Check for a new version of Intel(R) Distribution of OpenVINO(TM) toolkit here https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit/download.html?cid=other&source=prod&campid=ww_2023_bu_IOTG_OpenVINO-2022-3&content=upg_all&medium=organic or on https://github.com/openvinotoolkit/openvino
[ INFO ] Generated IR will be compressed to FP16. If you get lower accuracy, please consider disabling compression by removing argument --compress_to_fp16 or set it to false --compress_to_fp16=False.
Find more information about compression to FP16 at https://docs.openvino.ai/latest/openvino_docs_MO_DG_FP16_Compression.html
[ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11.
Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/latest/openvino_2_0_transition_guide.html
[ SUCCESS ] Generated IR version 11 model.
[ SUCCESS ] XML file: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/102-pytorch-onnx-to-openvino/model/lraspp_mobilenet_v3_large.xml
[ SUCCESS ] BIN file: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/102-pytorch-onnx-to-openvino/model/lraspp_mobilenet_v3_large.bin
[ SUCCESS ] XML file: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/102-pytorch-onnx-to-openvino/model/lraspp_mobilenet_v3_large.xml
[ SUCCESS ] BIN file: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/102-pytorch-onnx-to-openvino/model/lraspp_mobilenet_v3_large.bin
Show Results
@@ -402,7 +403,7 @@ Performance Comparison
Measure the time it takes to do inference on twenty images. This gives
an indication of performance. For more accurate benchmarking, use the
`Benchmark
Tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_tool_README.html>`__.
Tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_tool_README.html>`__.
Keep in mind that many optimizations are possible to improve the
performance.
@@ -467,9 +468,9 @@ performance.
.. parsed-literal::
PyTorch model on CPU: 0.039 seconds per image, FPS: 25.80
ONNX model in OpenVINO Runtime/CPU: 0.031 seconds per image, FPS: 32.10
OpenVINO IR model in OpenVINO Runtime/CPU: 0.031 seconds per image, FPS: 32.29
PyTorch model on CPU: 0.037 seconds per image, FPS: 27.22
ONNX model in OpenVINO Runtime/CPU: 0.025 seconds per image, FPS: 39.64
OpenVINO IR model in OpenVINO Runtime/CPU: 0.031 seconds per image, FPS: 31.95
**Show Device Information**
@@ -497,6 +498,6 @@ References
- `OpenVINO ONNX
support <https://docs.openvino.ai/2021.4/openvino_docs_IE_DG_ONNX_Support.html>`__
- `Model Optimizer
Documentation <https://docs.openvino.ai/latest/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html>`__
Documentation <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html>`__
- `Model Optimizer Pytorch conversion
guide <https://docs.openvino.ai/latest/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_PyTorch.html>`__
guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_PyTorch.html>`__
docs/notebooks/102-pytorch-onnx-to-openvino-with-output_files/index.html
+5 -5
View File
@@ -1,9 +1,9 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/102-pytorch-onnx-to-openvino-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/102-pytorch-onnx-to-openvino-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/102-pytorch-onnx-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="102-pytorch-onnx-to-openvino-with-output_20_0.png">102-pytorch-onnx-to-openvino-with-output_20_0.png</a> 30-May-2023 00:09 465692
<a href="102-pytorch-onnx-to-openvino-with-output_23_0.png">102-pytorch-onnx-to-openvino-with-output_23_0.png</a> 30-May-2023 00:09 465695
<a href="102-pytorch-onnx-to-openvino-with-output_25_0.png">102-pytorch-onnx-to-openvino-with-output_25_0.png</a> 30-May-2023 00:09 465692
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/102-pytorch-onnx-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="102-pytorch-onnx-to-openvino-with-output_20_0.png">102-pytorch-onnx-to-openvino-with-output_20_0.png</a> 22-Jun-2023 00:06 465692
<a href="102-pytorch-onnx-to-openvino-with-output_23_0.png">102-pytorch-onnx-to-openvino-with-output_23_0.png</a> 22-Jun-2023 00:06 465695
<a href="102-pytorch-onnx-to-openvino-with-output_25_0.png">102-pytorch-onnx-to-openvino-with-output_25_0.png</a> 22-Jun-2023 00:06 465692
</pre><hr></body>
</html>
docs/notebooks/103-paddle-to-openvino-classification-with-output.rst
+30 -63
View File
@@ -6,7 +6,7 @@ This notebook shows how to convert a MobileNetV3 model from
on the `ImageNet <https://www.image-net.org>`__ dataset, to OpenVINO IR.
It also shows how to perform classification inference on a sample image,
using `OpenVINO
Runtime <https://docs.openvino.ai/latest/openvino_docs_IE_DG_Deep_Learning_Inference_Engine_DevGuide.html>`__
Runtime <https://docs.openvino.ai/nightly/openvino_docs_OV_UG_OV_Runtime_User_Guide.html>`__
and compares the results of the
`PaddlePaddle <https://github.com/PaddlePaddle/Paddle>`__ model with the
IR model.
@@ -22,7 +22,12 @@ Imports
.. code:: ipython3
!pip install -q "paddlepaddle==2.5.0rc0"
import sys
if sys.version_info.minor > 7:
!pip install -q "paddlepaddle==2.5.0rc0"
else:
!pip install -q "paddlepaddle==2.4.2"
.. code:: ipython3
@@ -48,7 +53,6 @@ Imports
import matplotlib.pyplot as plt
import numpy as np
from paddleclas import PaddleClas
from IPython.display import Markdown, display
from PIL import Image
from openvino.runtime import Core
sys.path.append("../utils")
@@ -57,8 +61,8 @@ Imports
.. parsed-literal::
2023-05-29 22:27:09 INFO: Loading faiss with AVX2 support.
2023-05-29 22:27:09 INFO: Successfully loaded faiss with AVX2 support.
2023-06-21 22:26:35 INFO: Loading faiss with AVX2 support.
2023-06-21 22:26:35 INFO: Successfully loaded faiss with AVX2 support.
Settings
@@ -126,12 +130,12 @@ inference on that image, and then show the top three prediction results.
.. parsed-literal::
[2023/05/29 22:27:44] ppcls WARNING: The current running environment does not support the use of GPU. CPU has been used instead.
[2023/06/21 22:27:03] ppcls WARNING: The current running environment does not support the use of GPU. CPU has been used instead.
.. parsed-literal::
W0529 22:27:44.770324 2561117 analysis_config.cc:971] It is detected that mkldnn and memory_optimize_pass are enabled at the same time, but they are not supported yet. Currently, memory_optimize_pass is explicitly disabled
W0621 22:27:03.734274 1205526 analysis_config.cc:971] It is detected that mkldnn and memory_optimize_pass are enabled at the same time, but they are not supported yet. Currently, memory_optimize_pass is explicitly disabled
.. parsed-literal::
@@ -201,7 +205,7 @@ values.
.. parsed-literal::
2023-05-29 22:27:45 WARNING: Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
2023-06-21 22:27:04 WARNING: Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
.. parsed-literal::
@@ -213,7 +217,7 @@ values.
.. parsed-literal::
<matplotlib.image.AxesImage at 0x7f68a9799eb0>
<matplotlib.image.AxesImage at 0x7fedb78da400>
@@ -240,64 +244,27 @@ OpenVINO model.
Convert the Model to OpenVINO IR Format
---------------------------------------
Call the OpenVINO Model Optimizer tool to convert the PaddlePaddle model
to OpenVINO IR, with FP32 precision. The models are saved to the current
directory. You can add the mean values to the model with
``--mean_values`` and scale the output with the standard deviation with
``--scale_values``. With these options, it is not necessary to normalize
input data before propagating it through the network. However, to get
the exact same output as the PaddlePaddle model, it is necessary to
preprocess in the image in the same way. Therefore, for this tutorial,
you do not add the mean and scale values to the model, and you use the
``process_image`` function, as described in the previous section, to
ensure that both the IR and the PaddlePaddle model use the same
preprocessing methods. It is explained how to get the mean and scale
values of the PaddleGAN model, so you can add them to the Model
Optimizer command if you want. See the `PyTorch/ONNX to
OpenVINO <102-pytorch-onnx-to-openvino-with-output.html>`__
notebook, where these options are used.
Run ``! mo --help`` in a code cell to show an overview of command line
options for Model Optimizer. See the `Model Optimizer Developer
Guide <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__
Call the OpenVINO Model Optimizer Python API to convert the PaddlePaddle
model to OpenVINO IR, with FP32 precision. ``mo.convert_model`` function
accept path to PaddlePaddle model and returns OpenVINO Model class
instance which represents this model. Obtained model is ready to use and
loading on device using ``compile_model`` or can be saved on disk using
``serialize`` function. See the `Model Optimizer Developer
Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__
for more information about Model Optimizer.
In the next cell, we first construct the command for Model Optimizer,
and then execute this command in the notebook by prepending the command
with a ``!``. When Model Optimization is successful, the last lines of
the output include ``[ SUCCESS ] Generated IR version 11 model``.
.. code:: ipython3
from openvino.tools import mo
from openvino.runtime import serialize
model_xml = Path(MODEL_NAME).with_suffix('.xml')
if not model_xml.exists():
mo_command = f'mo --input_model model/MobileNetV3_large_x1_0_infer/inference.pdmodel --model_name {MODEL_NAME}'
display(Markdown(f"Model Optimizer command to convert the ONNX model to IR: `{mo_command}`"))
display(Markdown("_Converting model to IR. This may take a few minutes..._"))
! $mo_command
ov_model = mo.convert_model("model/MobileNetV3_large_x1_0_infer/inference.pdmodel")
serialize(ov_model, str(model_xml))
else:
print(f"{model_xml} already exists.")
Model Optimizer command to convert the ONNX model to IR:
``mo --input_model model/MobileNetV3_large_x1_0_infer/inference.pdmodel --model_name MobileNetV3_large_x1_0``
*Converting model to IR. This may take a few minutes…*
.. parsed-literal::
Check for a new version of Intel(R) Distribution of OpenVINO(TM) toolkit here https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit/download.html?cid=other&source=prod&campid=ww_2023_bu_IOTG_OpenVINO-2022-3&content=upg_all&medium=organic or on https://github.com/openvinotoolkit/openvino
[ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11.
Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/latest/openvino_2_0_transition_guide.html
[ SUCCESS ] Generated IR version 11 model.
[ SUCCESS ] XML file: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/103-paddle-to-openvino/MobileNetV3_large_x1_0.xml
[ SUCCESS ] BIN file: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/103-paddle-to-openvino/MobileNetV3_large_x1_0.bin
Show Inference on OpenVINO Model
--------------------------------
@@ -353,7 +320,7 @@ Measure the time it takes to do inference on fifty images and compare
the result. The timing information gives an indication of performance.
For a fair comparison, we include the time it takes to process the
image. For more accurate benchmarking, use the `OpenVINO benchmark
tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_tool_README.html>`__.
tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_tool_README.html>`__.
Note that many optimizations are possible to improve the performance.
.. code:: ipython3
@@ -396,7 +363,7 @@ Note that many optimizations are possible to improve the performance.
.. parsed-literal::
PaddlePaddle model on CPU: 0.0067 seconds per image, FPS: 148.81
PaddlePaddle model on CPU: 0.0073 seconds per image, FPS: 137.70
PaddlePaddle result:
Labrador retriever, 0.75138
@@ -440,7 +407,7 @@ Note that many optimizations are possible to improve the performance.
.. parsed-literal::
OpenVINO IR model in OpenVINO Runtime (CPU): 0.0029 seconds per image, FPS: 342.28
OpenVINO IR model in OpenVINO Runtime (CPU): 0.0027 seconds per image, FPS: 364.96
OpenVINO result:
Labrador retriever, 0.75138
@@ -459,6 +426,6 @@ References
- `PaddleClas <https://github.com/PaddlePaddle/PaddleClas>`__
- `OpenVINO PaddlePaddle
support <https://docs.openvino.ai/latest/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Paddle.html>`__
support <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Paddle.html>`__
- `OpenVINO Model Optimizer
Documentation <https://docs.openvino.ai/latest/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html>`__
Documentation <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html>`__
docs/notebooks/103-paddle-to-openvino-classification-with-output_files/index.html
+7 -7
View File
@@ -1,11 +1,11 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/103-paddle-to-openvino-classification-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/103-paddle-to-openvino-classification-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/103-paddle-to-openvino-classification-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="103-paddle-to-openvino-classification-with-output_15_3.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 30-May-2023 00:09 120883
<a href="103-paddle-to-openvino-classification-with-output_21_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 30-May-2023 00:09 224886
<a href="103-paddle-to-openvino-classification-with-output_25_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 30-May-2023 00:09 224886
<a href="103-paddle-to-openvino-classification-with-output_26_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 30-May-2023 00:09 224886
<a href="103-paddle-to-openvino-classification-with-output_8_3.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 30-May-2023 00:09 224886
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/103-paddle-to-openvino-classification-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="103-paddle-to-openvino-classification-with-output_15_3.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 22-Jun-2023 00:06 120883
<a href="103-paddle-to-openvino-classification-with-output_21_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 22-Jun-2023 00:06 224886
<a href="103-paddle-to-openvino-classification-with-output_25_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 22-Jun-2023 00:06 224886
<a href="103-paddle-to-openvino-classification-with-output_26_1.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 22-Jun-2023 00:06 224886
<a href="103-paddle-to-openvino-classification-with-output_8_3.png">103-paddle-to-openvino-classification-with-outp..&gt;</a> 22-Jun-2023 00:06 224886
</pre><hr></body>
</html>
docs/notebooks/104-model-tools-with-output.rst
+80 -102
View File
@@ -103,72 +103,6 @@ The following settings can be changed:
Download a Model from Open Model Zoo
------------------------------------
.. code:: ipython3
import shutil
if Path("open_model_zoo").exists():
shutil.rmtree("open_model_zoo")
!git clone https://github.com/openvinotoolkit/open_model_zoo.git
%cd open_model_zoo
!git checkout aa02473c20c1b62763de5385229ffde476e8119c
%cd tools/model_tools
!pip install .
%cd ../../../
.. parsed-literal::
Cloning into 'open_model_zoo'...
remote: Enumerating objects: 103165, done.
remote: Counting objects: 100% (889/889), done.
remote: Compressing objects: 100% (525/525), done.
remote: Total 103165 (delta 291), reused 825 (delta 268), pack-reused 102276
Receiving objects: 100% (103165/103165), 303.37 MiB | 3.87 MiB/s, done.
Resolving deltas: 100% (70208/70208), done.
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/104-model-tools/open_model_zoo
Note: switching to 'aa02473c20c1b62763de5385229ffde476e8119c'.
You are in 'detached HEAD' state. You can look around, make experimental
changes and commit them, and you can discard any commits you make in this
state without impacting any branches by switching back to a branch.
If you want to create a new branch to retain commits you create, you may
do so (now or later) by using -c with the switch command. Example:
git switch -c <new-branch-name>
Or undo this operation with:
git switch -
Turn off this advice by setting config variable advice.detachedHead to false
HEAD is now at aa02473c2 Merge pull request #3621 from eaidova/ea/fix_imports_order
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/104-model-tools/open_model_zoo/tools/model_tools
Processing /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/104-model-tools/open_model_zoo/tools/model_tools
Installing build dependencies ... - \ | done
Getting requirements to build wheel ... - done
Preparing metadata (pyproject.toml) ... - done
Requirement already satisfied: openvino-telemetry>=2022.1.0 in /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from omz-tools==1.0.2) (2022.3.0)
Requirement already satisfied: pyyaml>=5.4.1 in /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from omz-tools==1.0.2) (6.0)
Requirement already satisfied: requests>=2.25.1 in /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from omz-tools==1.0.2) (2.31.0)
Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->omz-tools==1.0.2) (3.1.0)
Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->omz-tools==1.0.2) (3.4)
Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->omz-tools==1.0.2) (1.26.16)
Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->omz-tools==1.0.2) (2023.5.7)
Building wheels for collected packages: omz-tools
Building wheel for omz-tools (pyproject.toml) ... - \ | / - \ done
Created wheel for omz-tools: filename=omz_tools-1.0.2-py3-none-any.whl size=3364798 sha256=3180da2c121c85b9de8ca19bcdd1b0d774d203a28f153a893b993431c328c30b
Stored in directory: /tmp/pip-ephem-wheel-cache-tlfkktz2/wheels/fa/a1/e1/1a73fbbacdfff3d1371ea61c64d2882179d286cc1c8521f629
Successfully built omz-tools
Installing collected packages: omz-tools
Successfully installed omz-tools-1.0.2
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/104-model-tools
Specify, display and run the Model Downloader command to download the
model.
@@ -243,21 +177,21 @@ Converting mobilenet-v2-pytorch…
.. parsed-literal::
========== Converting mobilenet-v2-pytorch to ONNX
Conversion to ONNX command: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/internal_scripts/pytorch_to_onnx.py --model-name=mobilenet_v2 --weights=model/public/mobilenet-v2-pytorch/mobilenet_v2-b0353104.pth --import-module=torchvision.models --input-shape=1,3,224,224 --output-file=model/public/mobilenet-v2-pytorch/mobilenet-v2.onnx --input-names=data --output-names=prob
Conversion to ONNX command: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/internal_scripts/pytorch_to_onnx.py --model-name=mobilenet_v2 --weights=model/public/mobilenet-v2-pytorch/mobilenet_v2-b0353104.pth --import-module=torchvision.models --input-shape=1,3,224,224 --output-file=model/public/mobilenet-v2-pytorch/mobilenet-v2.onnx --input-names=data --output-names=prob
Traceback (most recent call last):
File "/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/internal_scripts/pytorch_to_onnx.py", line 187, in <module>
main()
File "/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/internal_scripts/pytorch_to_onnx.py", line 179, in main
model = load_model(args.model_name, args.weights,
File "/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/internal_scripts/pytorch_to_onnx.py", line 126, in load_model
with prepend_to_path(model_paths):
File "/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/internal_scripts/pytorch_to_onnx.py", line 49, in __enter__
sys.path = self._preprended_paths + sys.path
TypeError: unsupported operand type(s) for +: 'NoneType' and 'list'
ONNX check passed successfully.
========== Converting mobilenet-v2-pytorch to IR (FP16)
Conversion command: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/bin/mo --framework=onnx --output_dir=/tmp/tmpt97jmp4t --model_name=mobilenet-v2-pytorch --input=data '--mean_values=data[123.675,116.28,103.53]' '--scale_values=data[58.624,57.12,57.375]' --reverse_input_channels --output=prob --input_model=model/public/mobilenet-v2-pytorch/mobilenet-v2.onnx '--layout=data(NCHW)' '--input_shape=[1, 3, 224, 224]' --compress_to_fp16=True
[ INFO ] Generated IR will be compressed to FP16. If you get lower accuracy, please consider disabling compression by removing argument --compress_to_fp16 or set it to false --compress_to_fp16=False.
Find more information about compression to FP16 at https://docs.openvino.ai/latest/openvino_docs_MO_DG_FP16_Compression.html
[ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11.
Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/latest/openvino_2_0_transition_guide.html
[ SUCCESS ] Generated IR version 11 model.
[ SUCCESS ] XML file: /tmp/tmpt97jmp4t/mobilenet-v2-pytorch.xml
[ SUCCESS ] BIN file: /tmp/tmpt97jmp4t/mobilenet-v2-pytorch.bin
FAILED:
mobilenet-v2-pytorch
Get Model Information
@@ -306,8 +240,8 @@ information in a dictionary.
'description': 'MobileNet V2 is image classification model pre-trained on ImageNet dataset. This is a PyTorch* implementation of MobileNetV2 architecture as described in the paper "Inverted Residuals and Linear Bottlenecks: Mobile Networks for Classification, Detection and Segmentation" <https://arxiv.org/abs/1801.04381>.\nThe model input is a blob that consists of a single image of "1, 3, 224, 224" in "RGB" order.\nThe model output is typical object classifier for the 1000 different classifications matching with those in the ImageNet database.',
'framework': 'pytorch',
'license_url': 'https://raw.githubusercontent.com/pytorch/vision/master/LICENSE',
'accuracy_config': '/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/models/public/mobilenet-v2-pytorch/accuracy-check.yml',
'model_config': '/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/models/public/mobilenet-v2-pytorch/model.yml',
'accuracy_config': '/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/models/public/mobilenet-v2-pytorch/accuracy-check.yml',
'model_config': '/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/models/public/mobilenet-v2-pytorch/model.yml',
'precisions': ['FP16', 'FP32'],
'quantization_output_precisions': ['FP16-INT8', 'FP32-INT8'],
'subdirectory': 'public/mobilenet-v2-pytorch',
@@ -336,7 +270,7 @@ file.
.. parsed-literal::
model/public/mobilenet-v2-pytorch/FP16/mobilenet-v2-pytorch.xml exists: False
model/public/mobilenet-v2-pytorch/FP16/mobilenet-v2-pytorch.xml exists: True
Run Benchmark Tool
@@ -376,24 +310,64 @@ seconds…
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to THROUGHPUT.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ ERROR ] Model file /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/104-model-tools/model/public/mobilenet-v2-pytorch/FP16/mobilenet-v2-pytorch.xml cannot be opened!
Traceback (most recent call last):
File "/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 368, in main
model = benchmark.read_model(args.path_to_model)
File "/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 69, in read_model
return self.core.read_model(model_filename, weights_filename)
RuntimeError: Model file /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/104-model-tools/model/public/mobilenet-v2-pytorch/FP16/mobilenet-v2-pytorch.xml cannot be opened!
[ INFO ] Read model took 29.56 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] data (node: data) : f32 / [N,C,H,W] / [1,3,224,224]
[ INFO ] Model outputs:
[ INFO ] prob (node: prob) : f32 / [...] / [1,1000]
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] data (node: data) : u8 / [N,C,H,W] / [1,3,224,224]
[ INFO ] Model outputs:
[ INFO ] prob (node: prob) : f32 / [...] / [1,1000]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 165.63 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 6
[ INFO ] NUM_STREAMS: 6
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] INFERENCE_NUM_THREADS: 24
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'data'!. This input will be filled with random values!
[ INFO ] Fill input 'data' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 6.54 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 20190 iterations
[ INFO ] Duration: 15003.81 ms
[ INFO ] Latency:
[ INFO ] Median: 4.31 ms
[ INFO ] Average: 4.32 ms
[ INFO ] Min: 2.52 ms
[ INFO ] Max: 11.83 ms
[ INFO ] Throughput: 1345.66 FPS
Benchmark with Different Settings
@@ -482,11 +456,11 @@ Benchmark command:
command ended
Traceback (most recent call last):
File "/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 368, in main
model = benchmark.read_model(args.path_to_model)
File "/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 69, in read_model
return self.core.read_model(model_filename, weights_filename)
RuntimeError: Model file /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/104-model-tools/model/public/mobilenet-v2-pytorch/FP16/mobilenet-v2-pytorch.xml cannot be opened!
File "/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 327, in main
benchmark.set_allow_auto_batching(False)
File "/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 63, in set_allow_auto_batching
self.core.set_property({'ALLOW_AUTO_BATCHING': flag})
RuntimeError: Check 'false' failed at src/inference/src/core.cpp:238:
.. code:: ipython3
@@ -507,12 +481,16 @@ Benchmark command:
.. parsed-literal::
command ended
Check 'false' failed at src/plugins/auto/src/plugin_config.cpp:55:
property: ALLOW_AUTO_BATCHING: not supported
Traceback (most recent call last):
File "/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 368, in main
model = benchmark.read_model(args.path_to_model)
File "/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 69, in read_model
return self.core.read_model(model_filename, weights_filename)
RuntimeError: Model file /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/104-model-tools/model/public/mobilenet-v2-pytorch/FP16/mobilenet-v2-pytorch.xml cannot be opened!
File "/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 327, in main
benchmark.set_allow_auto_batching(False)
File "/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/benchmark.py", line 63, in set_allow_auto_batching
self.core.set_property({'ALLOW_AUTO_BATCHING': flag})
RuntimeError: Check 'false' failed at src/inference/src/core.cpp:238:
Check 'false' failed at src/plugins/auto/src/plugin_config.cpp:55:
property: ALLOW_AUTO_BATCHING: not supported
.. code:: ipython3
docs/notebooks/105-language-quantize-bert-with-output.rst
+310 -186
View File
@@ -5,7 +5,7 @@ This tutorial demonstrates how to apply ``INT8`` quantization to the
Natural Language Processing model known as
`BERT <https://en.wikipedia.org/wiki/BERT_(language_model)>`__, using
the `Post-Training Quantization
API <https://docs.openvino.ai/latest/nncf_ptq_introduction.html>`__
API <https://docs.openvino.ai/nightly/basic_quantization_flow.html>`__
(NNCF library). A fine-tuned `HuggingFace
BERT <https://huggingface.co/transformers/model_doc/bert.html>`__
`PyTorch <https://pytorch.org/>`__ model, trained on the `Microsoft
@@ -42,11 +42,11 @@ Imports
import numpy as np
import torch
from openvino import runtime as ov
from openvino.tools import mo
from openvino.runtime import serialize, Model
from openvino.runtime import serialize, Model, PartialShape
import nncf
from nncf.parameters import ModelType
from transformers import BertForSequenceClassification, BertTokenizer
from openvino.tools.mo import convert_model
import datasets
import evaluate
@@ -56,8 +56,10 @@ Imports
.. parsed-literal::
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/offline_transformations/__init__.py:10: FutureWarning: The module is private and following namespace `offline_transformations` will be removed in the future, use `openvino.runtime.passes` instead!
warnings.warn(
2023-06-21 22:27:49.466196: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-06-21 22:27:49.499821: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-06-21 22:27:50.040301: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
.. parsed-literal::
@@ -84,9 +86,8 @@ Prepare the Model
-----------------
Perform the following: - Download and unpack pre-trained BERT model for
MRPC by PyTorch. - Convert the model to the ONNX. - Run Model Optimizer
to convert the model from the ONNX representation to the OpenVINO
Intermediate Representation (OpenVINO IR)
MRPC by PyTorch. - Convert the model to the OpenVINO Intermediate
Representation (OpenVINO IR)
.. code:: ipython3
@@ -101,67 +102,56 @@ Intermediate Representation (OpenVINO IR)
model/MRPC.zip: 0%| | 0.00/387M [00:00<?, ?B/s]
Convert the original PyTorch model to the ONNX representation.
Convert the original PyTorch model to the OpenVINO Intermediate
Representation.
From OpenVINO 2023.0, we can directly convert a model from the PyTorch
format to the OpenVINO IR format using Model Optimizer. Following
PyTorch model formats are supported:
- torch.nn.Module
- torch.jit.ScriptModule
- torch.jit.ScriptFunction
.. code:: ipython3
BATCH_SIZE = 1
MAX_SEQ_LENGTH = 128
def export_model_to_onnx(model, path):
with torch.no_grad():
default_input = torch.ones(1, MAX_SEQ_LENGTH, dtype=torch.int64)
inputs = {
"input_ids": default_input,
"attention_mask": default_input,
"token_type_ids": default_input,
}
torch.onnx.export(
model,
(inputs["input_ids"], inputs["attention_mask"], inputs["token_type_ids"]),
path,
opset_version=11,
input_names=["input_ids", "attention_mask", "token_type_ids"],
output_names=["output"]
)
print("ONNX model saved to {}".format(path))
input_shape = PartialShape([1, -1])
ir_model_xml = Path(MODEL_DIR) / "bert_mrpc.xml"
core = ov.Core()
torch_model = BertForSequenceClassification.from_pretrained(PRETRAINED_MODEL_DIR)
onnx_model_path = Path(MODEL_DIR) / "bert_mrpc.onnx"
if not onnx_model_path.exists():
export_model_to_onnx(torch_model, onnx_model_path)
torch_model.eval
input_info = [("input_ids", input_shape, np.int64),("attention_mask", input_shape, np.int64),("token_type_ids", input_shape, np.int64)]
default_input = torch.ones(1, MAX_SEQ_LENGTH, dtype=torch.int64)
inputs = {
"input_ids": default_input,
"attention_mask": default_input,
"token_type_ids": default_input,
}
# Convert the PyTorch model to OpenVINO IR FP32.
if not ir_model_xml.exists():
model = convert_model(torch_model, example_input=inputs, input=input_info)
serialize(model, str(ir_model_xml))
else:
model = core.read_model(ir_model_xml)
.. parsed-literal::
ONNX model saved to model/bert_mrpc.onnx
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/annotations.py:309: UserWarning: TorchScript will treat type annotations of Tensor dtype-specific subtypes as if they are normal Tensors. dtype constraints are not enforced in compilation either.
warnings.warn("TorchScript will treat type annotations of Tensor "
Convert the ONNX Model to OpenVINO IR
-------------------------------------
Use Model Optimizer Python API to convert the model to OpenVINO IR with
``FP32`` precision. For more information about Model Optimizer Python
API, see the `Model Optimizer Developer
Guide <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Python_API.html>`__.
.. code:: ipython3
ir_model_xml = onnx_model_path.with_suffix(".xml")
# Convert the ONNX model to OpenVINO IR FP32.
if not ir_model_xml.exists():
model = mo.convert_model(onnx_model_path)
serialize(model, str(ir_model_xml))
Prepare the Dataset
-------------------
We download the General Language Understanding Evaluation (GLUE) dataset
for the MRPC task from HuggingFace datasets. Then, we tokenize the data
with a pre-trained BERT tokenizer from HuggingFace.
We download the `General Language Understanding Evaluation
(GLUE) <https://gluebenchmark.com/>`__ dataset for the MRPC task from
HuggingFace datasets. Then, we tokenize the data with a pre-trained BERT
tokenizer from HuggingFace.
.. code:: ipython3
@@ -178,28 +168,19 @@ with a pre-trained BERT tokenizer from HuggingFace.
return processed_dataset
data_source = create_data_source()
.. parsed-literal::
[ WARNING ] Found cached dataset glue (/opt/home/k8sworker/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)
Downloading readme: 0.00B [00:00, ?B/s]
.. parsed-literal::
[ WARNING ] Found cached dataset glue (/opt/home/k8sworker/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)
.. parsed-literal::
[ WARNING ] Loading cached processed dataset at /opt/home/k8sworker/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-b5f4c739eb2a4a9f.arrow
.. parsed-literal::
[ WARNING ] Loading cached processed dataset at /opt/home/k8sworker/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-b5f4c739eb2a4a9f.arrow
Found cached dataset glue (/opt/home/k8sworker/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)
Loading cached processed dataset at /opt/home/k8sworker/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-b5f4c739eb2a4a9f.arrow
Optimize model using NNCF Post-training Quantization API
@@ -210,10 +191,6 @@ advanced algorithms for Neural Networks inference optimization in
OpenVINO with minimal accuracy drop. We will use 8-bit quantization in
post-training mode (without the fine-tuning pipeline) to optimize BERT.
**Note**: NNCF Post-training Quantization is available as a preview
feature in OpenVINO 2022.3 release. Fully functional support will be
provided in the next releases.
The optimization process contains the following steps:
1. Create a Dataset for quantization
@@ -223,11 +200,7 @@ The optimization process contains the following steps:
.. code:: ipython3
# Load the network in OpenVINO Runtime.
core = ov.Core()
model = core.read_model(ir_model_xml)
INPUT_NAMES = [x.any_name for x in model.inputs]
INPUT_NAMES = [key for key in inputs.keys()]
def transform_fn(data_item):
"""
@@ -236,11 +209,10 @@ The optimization process contains the following steps:
This function should be passed when the data item cannot be used as model's input.
"""
inputs = {
name: np.asarray(data_item[name], dtype=np.int64) for name in INPUT_NAMES
name: np.asarray([data_item[name]], dtype=np.int64) for name in INPUT_NAMES
}
return inputs
calibration_dataset = nncf.Dataset(data_source, transform_fn)
# Quantize the model. By specifying model_type, we specify additional transformer patterns in the model.
quantized_model = nncf.quantize(model, calibration_dataset,
@@ -249,23 +221,184 @@ The optimization process contains the following steps:
.. parsed-literal::
INFO:openvino.tools.pot.pipeline.pipeline:Inference Engine version: 2022.3.0-9052-9752fafe8eb-releases/2022/3
INFO:openvino.tools.pot.pipeline.pipeline:Model Optimizer version: 2022.3.0-9052-9752fafe8eb-releases/2022/3
INFO:openvino.tools.pot.pipeline.pipeline:Post-Training Optimization Tool version: 2022.3.0-9052-9752fafe8eb-releases/2022/3
INFO:openvino.tools.pot.statistics.collector:Start computing statistics for algorithms : DefaultQuantization
INFO:openvino.tools.pot.statistics.collector:Computing statistics finished
INFO:openvino.tools.pot.pipeline.pipeline:Start algorithm: DefaultQuantization
INFO:openvino.tools.pot.algorithms.quantization.default.algorithm:Start computing statistics for algorithm : ActivationChannelAlignment
INFO:openvino.tools.pot.algorithms.quantization.default.algorithm:Computing statistics finished
INFO:openvino.tools.pot.algorithms.quantization.default.algorithm:Start computing statistics for algorithms : MinMaxQuantization,FastBiasCorrection
INFO:openvino.tools.pot.algorithms.quantization.default.algorithm:Computing statistics finished
INFO:openvino.tools.pot.pipeline.pipeline:Finished: DefaultQuantization
===========================================================================
INFO:nncf:202 ignored nodes was found by types in the NNCFGraph
INFO:nncf:24 ignored nodes was found by name in the NNCFGraph
INFO:nncf:Not adding activation input quantizer for operation: 22 aten::rsub_16
INFO:nncf:Not adding activation input quantizer for operation: 25 aten::rsub_17
INFO:nncf:Not adding activation input quantizer for operation: 30 aten::mul_18
INFO:nncf:Not adding activation input quantizer for operation: 11 aten::add_40
INFO:nncf:Not adding activation input quantizer for operation: 14 aten::add__46
INFO:nncf:Not adding activation input quantizer for operation: 17 aten::layer_norm_48
20 aten::layer_norm_49
23 aten::layer_norm_50
INFO:nncf:Not adding activation input quantizer for operation: 36 aten::add_108
INFO:nncf:Not adding activation input quantizer for operation: 55 aten::softmax_109
INFO:nncf:Not adding activation input quantizer for operation: 74 aten::matmul_110
INFO:nncf:Not adding activation input quantizer for operation: 26 aten::add_126
INFO:nncf:Not adding activation input quantizer for operation: 31 aten::layer_norm_128
47 aten::layer_norm_129
66 aten::layer_norm_130
INFO:nncf:Not adding activation input quantizer for operation: 85 aten::add_140
INFO:nncf:Not adding activation input quantizer for operation: 103 aten::layer_norm_142
133 aten::layer_norm_143
171 aten::layer_norm_144
INFO:nncf:Not adding activation input quantizer for operation: 38 aten::add_202
INFO:nncf:Not adding activation input quantizer for operation: 57 aten::softmax_203
INFO:nncf:Not adding activation input quantizer for operation: 76 aten::matmul_204
INFO:nncf:Not adding activation input quantizer for operation: 209 aten::add_220
INFO:nncf:Not adding activation input quantizer for operation: 236 aten::layer_norm_222
250 aten::layer_norm_223
267 aten::layer_norm_224
INFO:nncf:Not adding activation input quantizer for operation: 287 aten::add_234
INFO:nncf:Not adding activation input quantizer for operation: 316 aten::layer_norm_236
342 aten::layer_norm_237
364 aten::layer_norm_238
INFO:nncf:Not adding activation input quantizer for operation: 39 aten::add_296
INFO:nncf:Not adding activation input quantizer for operation: 58 aten::softmax_297
INFO:nncf:Not adding activation input quantizer for operation: 77 aten::matmul_298
INFO:nncf:Not adding activation input quantizer for operation: 221 aten::add_314
INFO:nncf:Not adding activation input quantizer for operation: 242 aten::layer_norm_316
259 aten::layer_norm_317
279 aten::layer_norm_318
INFO:nncf:Not adding activation input quantizer for operation: 300 aten::add_328
INFO:nncf:Not adding activation input quantizer for operation: 326 aten::layer_norm_330
348 aten::layer_norm_331
370 aten::layer_norm_332
INFO:nncf:Not adding activation input quantizer for operation: 40 aten::add_390
INFO:nncf:Not adding activation input quantizer for operation: 59 aten::softmax_391
INFO:nncf:Not adding activation input quantizer for operation: 78 aten::matmul_392
INFO:nncf:Not adding activation input quantizer for operation: 223 aten::add_408
INFO:nncf:Not adding activation input quantizer for operation: 243 aten::layer_norm_410
260 aten::layer_norm_411
280 aten::layer_norm_412
INFO:nncf:Not adding activation input quantizer for operation: 302 aten::add_422
INFO:nncf:Not adding activation input quantizer for operation: 328 aten::layer_norm_424
350 aten::layer_norm_425
372 aten::layer_norm_426
INFO:nncf:Not adding activation input quantizer for operation: 41 aten::add_484
INFO:nncf:Not adding activation input quantizer for operation: 60 aten::softmax_485
INFO:nncf:Not adding activation input quantizer for operation: 79 aten::matmul_486
INFO:nncf:Not adding activation input quantizer for operation: 225 aten::add_502
INFO:nncf:Not adding activation input quantizer for operation: 244 aten::layer_norm_504
261 aten::layer_norm_505
281 aten::layer_norm_506
INFO:nncf:Not adding activation input quantizer for operation: 304 aten::add_516
INFO:nncf:Not adding activation input quantizer for operation: 330 aten::layer_norm_518
352 aten::layer_norm_519
374 aten::layer_norm_520
INFO:nncf:Not adding activation input quantizer for operation: 42 aten::add_578
INFO:nncf:Not adding activation input quantizer for operation: 61 aten::softmax_579
INFO:nncf:Not adding activation input quantizer for operation: 80 aten::matmul_580
INFO:nncf:Not adding activation input quantizer for operation: 227 aten::add_596
INFO:nncf:Not adding activation input quantizer for operation: 245 aten::layer_norm_598
262 aten::layer_norm_599
282 aten::layer_norm_600
INFO:nncf:Not adding activation input quantizer for operation: 306 aten::add_610
INFO:nncf:Not adding activation input quantizer for operation: 332 aten::layer_norm_612
354 aten::layer_norm_613
376 aten::layer_norm_614
INFO:nncf:Not adding activation input quantizer for operation: 43 aten::add_672
INFO:nncf:Not adding activation input quantizer for operation: 62 aten::softmax_673
INFO:nncf:Not adding activation input quantizer for operation: 81 aten::matmul_674
INFO:nncf:Not adding activation input quantizer for operation: 229 aten::add_690
INFO:nncf:Not adding activation input quantizer for operation: 246 aten::layer_norm_692
263 aten::layer_norm_693
283 aten::layer_norm_694
INFO:nncf:Not adding activation input quantizer for operation: 308 aten::add_704
INFO:nncf:Not adding activation input quantizer for operation: 334 aten::layer_norm_706
356 aten::layer_norm_707
378 aten::layer_norm_708
INFO:nncf:Not adding activation input quantizer for operation: 44 aten::add_766
INFO:nncf:Not adding activation input quantizer for operation: 63 aten::softmax_767
INFO:nncf:Not adding activation input quantizer for operation: 82 aten::matmul_768
INFO:nncf:Not adding activation input quantizer for operation: 231 aten::add_784
INFO:nncf:Not adding activation input quantizer for operation: 247 aten::layer_norm_786
264 aten::layer_norm_787
284 aten::layer_norm_788
INFO:nncf:Not adding activation input quantizer for operation: 310 aten::add_798
INFO:nncf:Not adding activation input quantizer for operation: 336 aten::layer_norm_800
358 aten::layer_norm_801
380 aten::layer_norm_802
INFO:nncf:Not adding activation input quantizer for operation: 45 aten::add_860
INFO:nncf:Not adding activation input quantizer for operation: 64 aten::softmax_861
INFO:nncf:Not adding activation input quantizer for operation: 83 aten::matmul_862
INFO:nncf:Not adding activation input quantizer for operation: 233 aten::add_878
INFO:nncf:Not adding activation input quantizer for operation: 248 aten::layer_norm_880
265 aten::layer_norm_881
285 aten::layer_norm_882
INFO:nncf:Not adding activation input quantizer for operation: 312 aten::add_892
INFO:nncf:Not adding activation input quantizer for operation: 338 aten::layer_norm_894
360 aten::layer_norm_895
382 aten::layer_norm_896
INFO:nncf:Not adding activation input quantizer for operation: 46 aten::add_954
INFO:nncf:Not adding activation input quantizer for operation: 65 aten::softmax_955
INFO:nncf:Not adding activation input quantizer for operation: 84 aten::matmul_956
INFO:nncf:Not adding activation input quantizer for operation: 235 aten::add_972
INFO:nncf:Not adding activation input quantizer for operation: 249 aten::layer_norm_974
266 aten::layer_norm_975
286 aten::layer_norm_976
INFO:nncf:Not adding activation input quantizer for operation: 314 aten::add_986
INFO:nncf:Not adding activation input quantizer for operation: 340 aten::layer_norm_988
362 aten::layer_norm_989
384 aten::layer_norm_990
INFO:nncf:Not adding activation input quantizer for operation: 35 aten::add_1048
INFO:nncf:Not adding activation input quantizer for operation: 54 aten::softmax_1049
INFO:nncf:Not adding activation input quantizer for operation: 73 aten::matmul_1050
INFO:nncf:Not adding activation input quantizer for operation: 215 aten::add_1066
INFO:nncf:Not adding activation input quantizer for operation: 240 aten::layer_norm_1068
257 aten::layer_norm_1069
277 aten::layer_norm_1070
INFO:nncf:Not adding activation input quantizer for operation: 296 aten::add_1080
INFO:nncf:Not adding activation input quantizer for operation: 322 aten::layer_norm_1082
344 aten::layer_norm_1083
366 aten::layer_norm_1084
INFO:nncf:Not adding activation input quantizer for operation: 37 aten::add_1142
INFO:nncf:Not adding activation input quantizer for operation: 56 aten::softmax_1143
INFO:nncf:Not adding activation input quantizer for operation: 75 aten::matmul_1144
INFO:nncf:Not adding activation input quantizer for operation: 218 aten::add_1160
INFO:nncf:Not adding activation input quantizer for operation: 241 aten::layer_norm_1162
258 aten::layer_norm_1163
278 aten::layer_norm_1164
INFO:nncf:Not adding activation input quantizer for operation: 298 aten::add_1174
INFO:nncf:Not adding activation input quantizer for operation: 324 aten::layer_norm_1176
346 aten::layer_norm_1177
368 aten::layer_norm_1178
.. parsed-literal::
Statistics collection: 100%|██████████| 300/300 [00:24<00:00, 12.11it/s]
Biases correction: 100%|██████████| 74/74 [00:25<00:00, 2.93it/s]
.. code:: ipython3
compressed_model_xml = 'quantized_bert_mrpc.xml'
compressed_model_xml = Path(MODEL_DIR) / "quantized_bert_mrpc.xml"
ov.serialize(quantized_model, compressed_model_xml)
Load and Test OpenVINO Model
@@ -277,21 +410,9 @@ inference. \* Get the answer from the model output.
.. code:: ipython3
core = ov.Core()
# Read the model from files.
model = core.read_model(model=compressed_model_xml)
# Assign dynamic shapes to every input layer.
for input_layer in model.inputs:
input_shape = input_layer.partial_shape
input_shape[1] = -1
model.reshape({input_layer: input_shape})
# Compile the model for a specific device.
compiled_model_int8 = core.compile_model(model=model, device_name="CPU")
output_layer = compiled_model_int8.outputs[0]
compiled_quantized_model = core.compile_model(model=quantized_model, device_name="CPU")
output_layer = compiled_quantized_model.outputs[0]
The Data Source returns a pair of sentences (indicated by
``sample_idx``) and the inference compares these sentences and outputs
@@ -304,7 +425,7 @@ changing ``sample_idx`` to another value (from 0 to 407).
sample = data_source[sample_idx]
inputs = {k: torch.unsqueeze(torch.tensor(sample[k]), 0) for k in ['input_ids', 'token_type_ids', 'attention_mask']}
result = compiled_model_int8(inputs)[output_layer]
result = compiled_quantized_model(inputs)[output_layer]
result = np.argmax(result)
print(f"Text 1: {sample['sentence1']}")
@@ -333,7 +454,6 @@ Compare F1-score of FP32 and INT8 models
output_layer = compiled_model.output(0)
metric = evaluate.load('glue', 'mrpc')
INPUT_NAMES = [x.any_name for x in compiled_model.inputs]
for batch in dataset:
inputs = [
np.expand_dims(np.asarray(batch[key], dtype=np.int64), 0) for key in INPUT_NAMES
@@ -359,9 +479,9 @@ Compare F1-score of FP32 and INT8 models
.. parsed-literal::
Checking the accuracy of the original model:
F1 score: 0.8927
F1 score: 0.9019
Checking the accuracy of the quantized model:
F1 score: 0.9014
F1 score: 0.8995
Compare Performance of the Original, Converted and Quantized Models
@@ -374,20 +494,8 @@ Frames Per Second (FPS) for images.
.. code:: ipython3
model = core.read_model(model=ir_model_xml)
# Assign dynamic shapes to every input layer.
dynamic_shapes = {}
for input_layer in model.inputs:
input_shape = input_layer.partial_shape
input_shape[1] = -1
dynamic_shapes[input_layer] = input_shape
model.reshape(dynamic_shapes)
# Compile the model for a specific device.
compiled_model_fp32 = core.compile_model(model=model, device_name="CPU")
compiled_model = core.compile_model(model=model, device_name="CPU")
.. code:: ipython3
@@ -408,7 +516,7 @@ Frames Per Second (FPS) for images.
start = time.perf_counter()
for _ in range(num_samples):
compiled_model_fp32(inputs)
compiled_model(inputs)
end = time.perf_counter()
time_ir = end - start
print(
@@ -418,7 +526,7 @@ Frames Per Second (FPS) for images.
start = time.perf_counter()
for _ in range(num_samples):
compiled_model_int8(inputs)
compiled_quantized_model(inputs)
end = time.perf_counter()
time_ir = end - start
print(
@@ -429,14 +537,14 @@ Frames Per Second (FPS) for images.
.. parsed-literal::
PyTorch model on CPU: 0.072 seconds per sentence, SPS: 13.80
IR FP32 model in OpenVINO Runtime/CPU: 0.020 seconds per sentence, SPS: 50.10
OpenVINO IR INT8 model in OpenVINO Runtime/CPU: 0.009 seconds per sentence, SPS: 113.99
PyTorch model on CPU: 0.071 seconds per sentence, SPS: 14.07
IR FP32 model in OpenVINO Runtime/CPU: 0.021 seconds per sentence, SPS: 48.22
OpenVINO IR INT8 model in OpenVINO Runtime/CPU: 0.010 seconds per sentence, SPS: 97.87
Finally, measure the inference performance of OpenVINO ``FP32`` and
``INT8`` models. For this purpose, use `Benchmark
Tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_tool_README.html>`__
Tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_tool_README.html>`__
in OpenVINO.
**Note**: The ``benchmark_app`` tool is able to measure the
@@ -451,7 +559,7 @@ in OpenVINO.
.. code:: ipython3
# Inference FP32 model (OpenVINO IR)
! benchmark_app -m $ir_model_xml -d CPU -api sync
! benchmark_app -m $ir_model_xml -shape [1,128],[1,128],[1,128] -d CPU -api sync
.. parsed-literal::
@@ -460,39 +568,41 @@ in OpenVINO.
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to LATENCY.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.LATENCY.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 170.14 ms
[ INFO ] Read model took 56.68 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] input_ids (node: input_ids) : i64 / [...] / [1,128]
[ INFO ] attention_mask (node: attention_mask) : i64 / [...] / [1,128]
[ INFO ] token_type_ids (node: token_type_ids) : i64 / [...] / [1,128]
[ INFO ] 1 , input_ids (node: Parameter_2) : i64 / [...] / [1,?]
[ INFO ] 2 , attention_mask , attention_mask.1 (node: Parameter_3) : i64 / [...] / [1,?]
[ INFO ] token_type_ids , 3 (node: Parameter_4) : i64 / [...] / [1,?]
[ INFO ] Model outputs:
[ INFO ] output (node: output) : f32 / [...] / [1,2]
[ INFO ] logits , 707 (node: aten::linear_1192) : f32 / [...] / [1,2]
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[ INFO ] Reshaping model: '1': [1,128], '2': [1,128], '3': [1,128]
[ INFO ] Reshape model took 57.63 ms
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] input_ids (node: input_ids) : i64 / [...] / [1,128]
[ INFO ] attention_mask (node: attention_mask) : i64 / [...] / [1,128]
[ INFO ] token_type_ids (node: token_type_ids) : i64 / [...] / [1,128]
[ INFO ] 1 , input_ids (node: Parameter_2) : i64 / [...] / [1,128]
[ INFO ] 2 , attention_mask , attention_mask.1 (node: Parameter_3) : i64 / [...] / [1,128]
[ INFO ] token_type_ids , 3 (node: Parameter_4) : i64 / [...] / [1,128]
[ INFO ] Model outputs:
[ INFO ] output (node: output) : f32 / [...] / [1,2]
[ INFO ] logits , 707 (node: aten::linear_1192) : f32 / [...] / [1,2]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 216.41 ms
[ INFO ] Compile model took 266.53 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
[ INFO ] NETWORK_NAME: Model0
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 1
[ INFO ] NUM_STREAMS: 1
[ INFO ] AFFINITY: Affinity.CORE
@@ -500,32 +610,38 @@ in OpenVINO.
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.LATENCY
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'input_ids'!. This input will be filled with random values!
[ WARNING ] No input files were given for input 'attention_mask'!. This input will be filled with random values!
[ WARNING ] No input files were given for input 'token_type_ids'!. This input will be filled with random values!
[ INFO ] Fill input 'input_ids' with random values
[ INFO ] Fill input 'attention_mask' with random values
[ INFO ] Fill input 'token_type_ids' with random values
[ WARNING ] No input files were given for input '1'!. This input will be filled with random values!
[ WARNING ] No input files were given for input '2'!. This input will be filled with random values!
[ WARNING ] No input files were given for input '3'!. This input will be filled with random values!
[ INFO ] Fill input '1' with random values
[ INFO ] Fill input '2' with random values
[ INFO ] Fill input '3' with random values
[Step 10/11] Measuring performance (Start inference synchronously, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 30.61 ms
[ INFO ] First inference took 33.80 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 3076 iterations
[ INFO ] Duration: 60012.39 ms
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 3293 iterations
[ INFO ] Duration: 60005.54 ms
[ INFO ] Latency:
[ INFO ] Median: 19.42 ms
[ INFO ] Average: 19.42 ms
[ INFO ] Min: 18.70 ms
[ INFO ] Max: 22.31 ms
[ INFO ] Throughput: 51.49 FPS
[ INFO ] Median: 18.09 ms
[ INFO ] Average: 18.12 ms
[ INFO ] Min: 17.73 ms
[ INFO ] Max: 22.74 ms
[ INFO ] Throughput: 55.27 FPS
.. code:: ipython3
# Inference INT8 model (OpenVINO IR)
! benchmark_app -m $compressed_model_xml -d CPU -api sync
! benchmark_app -m $compressed_model_xml -shape [1,128],[1,128],[1,128] -d CPU -api sync
.. parsed-literal::
@@ -534,39 +650,41 @@ in OpenVINO.
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to LATENCY.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.LATENCY.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 130.18 ms
[ INFO ] Read model took 53.69 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] input_ids , input_ids:0 (node: input_ids) : i64 / [...] / [1,128]
[ INFO ] attention_mask (node: attention_mask) : i64 / [...] / [1,128]
[ INFO ] token_type_ids:0 , token_type_ids (node: token_type_ids) : i64 / [...] / [1,128]
[ INFO ] input_ids , 1 (node: Parameter_2) : i64 / [...] / [1,?]
[ INFO ] attention_mask.1 , 2 , attention_mask (node: Parameter_3) : i64 / [...] / [1,?]
[ INFO ] 3 , token_type_ids (node: Parameter_4) : i64 / [...] / [1,?]
[ INFO ] Model outputs:
[ INFO ] output (node: output) : f32 / [...] / [1,2]
[ INFO ] logits , 707 (node: aten::linear_1192) : f32 / [...] / [1,2]
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[ INFO ] Reshaping model: '1': [1,128], '2': [1,128], '3': [1,128]
[ INFO ] Reshape model took 61.50 ms
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] input_ids , input_ids:0 (node: input_ids) : i64 / [...] / [1,128]
[ INFO ] attention_mask (node: attention_mask) : i64 / [...] / [1,128]
[ INFO ] token_type_ids:0 , token_type_ids (node: token_type_ids) : i64 / [...] / [1,128]
[ INFO ] input_ids , 1 (node: Parameter_2) : i64 / [...] / [1,128]
[ INFO ] attention_mask.1 , 2 , attention_mask (node: Parameter_3) : i64 / [...] / [1,128]
[ INFO ] 3 , token_type_ids (node: Parameter_4) : i64 / [...] / [1,128]
[ INFO ] Model outputs:
[ INFO ] output (node: output) : f32 / [...] / [1,2]
[ INFO ] logits , 707 (node: aten::linear_1192) : f32 / [...] / [1,2]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 387.50 ms
[ INFO ] Compile model took 473.56 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
[ INFO ] NETWORK_NAME: Model0
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 1
[ INFO ] NUM_STREAMS: 1
[ INFO ] AFFINITY: Affinity.CORE
@@ -574,24 +692,30 @@ in OpenVINO.
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.LATENCY
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'input_ids'!. This input will be filled with random values!
[ WARNING ] No input files were given for input 'attention_mask'!. This input will be filled with random values!
[ WARNING ] No input files were given for input 'token_type_ids'!. This input will be filled with random values!
[ INFO ] Fill input 'input_ids' with random values
[ INFO ] Fill input 'attention_mask' with random values
[ INFO ] Fill input 'token_type_ids' with random values
[ WARNING ] No input files were given for input '1'!. This input will be filled with random values!
[ WARNING ] No input files were given for input '2'!. This input will be filled with random values!
[ WARNING ] No input files were given for input '3'!. This input will be filled with random values!
[ INFO ] Fill input '1' with random values
[ INFO ] Fill input '2' with random values
[ INFO ] Fill input '3' with random values
[Step 10/11] Measuring performance (Start inference synchronously, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 14.96 ms
[ INFO ] First inference took 17.89 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 6830 iterations
[ INFO ] Duration: 60003.55 ms
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 6496 iterations
[ INFO ] Duration: 60002.09 ms
[ INFO ] Latency:
[ INFO ] Median: 8.74 ms
[ INFO ] Average: 8.70 ms
[ INFO ] Min: 7.54 ms
[ INFO ] Max: 11.06 ms
[ INFO ] Throughput: 114.43 FPS
[ INFO ] Median: 9.09 ms
[ INFO ] Average: 9.13 ms
[ INFO ] Min: 8.45 ms
[ INFO ] Max: 12.68 ms
[ INFO ] Throughput: 109.96 FPS
docs/notebooks/106-auto-device-with-output.rst
+87 -165
View File
@@ -2,21 +2,19 @@ Automatic Device Selection with OpenVINO™
=========================================
The `Auto
device <https://docs.openvino.ai/latest/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__
device <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__
(or AUTO in short) selects the most suitable device for inference by
considering the model precision, power efficiency and processing
capability of the available `compute
devices <https://docs.openvino.ai/latest/openvino_docs_OV_UG_supported_plugins_Supported_Devices.html>`__.
devices <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_Supported_Devices.html>`__.
The model precision (such as ``FP32``, ``FP16``, ``INT8``, etc.) is the
first consideration to filter out the devices that cannot run the
network efficiently.
Next, if dedicated accelerators are available, these devices are
preferred (for example, integrated and discrete
`GPU <https://docs.openvino.ai/latest/openvino_docs_OV_UG_supported_plugins_GPU.html#doxid-openvino-docs-o-v-u-g-supported-plugins-g-p-u>`__
or
`VPU <https://docs.openvino.ai/latest/openvino_docs_OV_UG_supported_plugins_VPU.html>`__).
`CPU <https://docs.openvino.ai/latest/openvino_docs_OV_UG_supported_plugins_CPU.html>`__
`GPU <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_GPU.html#doxid-openvino-docs-o-v-u-g-supported-plugins-g-p-u>`__).
`CPU <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_CPU.html>`__
is used as the default “fallback device”. Keep in mind that AUTO makes
this selection only once, during the loading of a model.
@@ -35,79 +33,15 @@ first inference.
</center>
Download and convert the model
------------------------------
This tutorial uses the
`bvlc_googlenet <https://github.com/BVLC/caffe/tree/master/models/bvlc_googlenet>`__
model. The bvlc_googlenet model is the first of the
`Inception <https://github.com/tensorflow/tpu/tree/master/models/experimental/inception>`__
family of models designed to perform image classification. Like other
Inception models, bvlc_googlenet was pre-trained on the
`ImageNet <https://image-net.org/>`__ data set. For more details about
this family of models, see the `research
paper <https://arxiv.org/abs/1512.00567>`__.
.. code:: ipython3
import sys
from pathlib import Path
from openvino.tools import mo
from openvino.runtime import serialize
from IPython.display import Markdown, display
sys.path.append("../utils")
import notebook_utils as utils
base_model_dir = Path("./model").expanduser()
model_name = "bvlc_googlenet"
caffemodel_name = f'{model_name}.caffemodel'
prototxt_name = f'{model_name}.prototxt'
caffemodel_path = base_model_dir / caffemodel_name
prototxt_path = base_model_dir / prototxt_name
if not caffemodel_path.exists() or not prototxt_path.exists():
caffemodel_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2022.1/googlenet-v1/bvlc_googlenet.caffemodel"
prototxt_url = "https://raw.githubusercontent.com/BVLC/caffe/88c96189bcbf3853b93e2b65c7b5e4948f9d5f67/models/bvlc_googlenet/deploy.prototxt"
utils.download_file(caffemodel_url, caffemodel_name, base_model_dir)
utils.download_file(prototxt_url, prototxt_name, base_model_dir)
else:
print(f'{caffemodel_name} and {prototxt_name} already downloaded to {base_model_dir}')
# postprocessing of model
text = prototxt_path.read_text()
text = text.replace('dim: 10', 'dim: 1')
res = prototxt_path.write_text(text)
.. parsed-literal::
model/bvlc_googlenet.caffemodel: 0%| | 0.00/51.1M [00:00<?, ?B/s]
.. parsed-literal::
model/bvlc_googlenet.prototxt: 0%| | 0.00/2.19k [00:00<?, ?B/s]
Import modules and create Core
------------------------------
.. code:: ipython3
import cv2
import matplotlib.pyplot as plt
import numpy as np
from openvino.runtime import Core, CompiledModel, AsyncInferQueue, InferRequest
import sys
import time
import sys
from IPython.display import Markdown, display
from openvino.runtime import Core, CompiledModel, AsyncInferQueue, InferRequest
ie = Core()
@@ -125,41 +59,50 @@ Import modules and create Core
Convert the model to OpenVINO IR format
---------------------------------------
Use Model Optimizer to convert the Caffe model to OpenVINO IR with
``FP16`` precision. The models are saved to the ``model/ir_model/``
directory. For more information about Model Optimizer, see the `Model
Optimizer Developer
Guide <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
This tutorial uses
`resnet50 <https://pytorch.org/vision/main/models/generated/torchvision.models.resnet50.html#resnet50>`__
model from
`torchvision <https://pytorch.org/vision/main/index.html?highlight=torchvision#module-torchvision>`__
library. ResNet 50 is image classification model pre-trained on ImageNet
dataset described in paper `“Deep Residual Learning for Image
Recognition” <https://arxiv.org/abs/1512.03385>`__. From OpenVINO
2023.0, we can directly convert a model from the PyTorch format to the
OpenVINO IR format using Model Optimizer. To convert model, we should
provide model object instance into ``mo.convert_model`` function,
optionally, we can specify input shape for conversion (by default models
from PyTorch converted with dynamic input shapes). ``mo.convert_model``
returns openvino.runtime.Model object ready to be loaded on device with
``openvino.runtime.Core().compile_model`` or serialized for next usage
with ``openvino.runtime.serialize``.
For more information about Model Optimizer, see the `Model Optimizer
Developer
Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
.. code:: ipython3
ir_model_path = base_model_dir / 'ir_model' / f'{model_name}.xml'
model = None
import torchvision
from pathlib import Path
from openvino.tools import mo
from openvino.runtime import serialize
if not ir_model_path.exists():
model = mo.convert_model(input_model=base_model_dir / caffemodel_name,
input_proto=base_model_dir / prototxt_name,
input_shape=[1, 3, 224, 224],
layout="NCHW",
mean_values=[104.0,117.0,123.0],
output="prob",
compress_to_fp16=True)
serialize(model, str(ir_model_path))
print("IR model saved to {}".format(ir_model_path))
base_model_dir = Path("./model")
base_model_dir.mkdir(exist_ok=True)
model_path = base_model_dir / "resnet50.xml"
if not model_path.exists():
pt_model = torchvision.models.resnet50(weights="DEFAULT")
ov_model = mo.convert_model(pt_model, input_shape=[[1,3,224,224]], compress_to_fp16=True)
serialize(ov_model, str(model_path))
print("IR model saved to {}".format(model_path))
else:
print("Read IR model from {}".format(ir_model_path))
model = ie.read_model(ir_model_path)
print("Read IR model from {}".format(model_path))
ov_model = ie.read_model(model_path)
.. parsed-literal::
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/numpy/lib/function_base.py:959: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.
return array(a, order=order, subok=subok, copy=True)
.. parsed-literal::
IR model saved to model/ir_model/bvlc_googlenet.xml
IR model saved to model/resnet50.xml
(1) Simplify selection logic
@@ -177,7 +120,7 @@ By default, ``compile_model`` API will select **AUTO** as
ie.set_property("AUTO", {"LOG_LEVEL":"LOG_INFO"})
# Load the model onto the target device.
compiled_model = ie.compile_model(model=model)
compiled_model = ie.compile_model(ov_model)
if isinstance(compiled_model, CompiledModel):
print("Successfully compiled model without a device_name.")
@@ -185,15 +128,6 @@ By default, ``compile_model`` API will select **AUTO** as
.. parsed-literal::
[22:35:25.7084]I[plugin.cpp:402][AUTO] load with CNN network
[22:35:25.7136]I[plugin.cpp:422][AUTO] device:CPU, config:EXCLUSIVE_ASYNC_REQUESTS=NO
[22:35:25.7137]I[plugin.cpp:422][AUTO] device:CPU, config:PERFORMANCE_HINT=LATENCY
[22:35:25.7137]I[plugin.cpp:422][AUTO] device:CPU, config:PERFORMANCE_HINT_NUM_REQUESTS=0
[22:35:25.7137]I[plugin.cpp:422][AUTO] device:CPU, config:PERF_COUNT=NO
[22:35:25.7137]I[plugin.cpp:435][AUTO] device:CPU, priority:0
[22:35:25.7141]I[auto_schedule.cpp:103][AUTO] ExecutableNetwork start
[22:35:25.7145]I[auto_schedule.cpp:146][AUTO] select device:CPU
[22:35:25.8945]I[auto_schedule.cpp:188][AUTO] device:CPU loading Network finished
Successfully compiled model without a device_name.
@@ -206,8 +140,6 @@ By default, ``compile_model`` API will select **AUTO** as
.. parsed-literal::
[22:35:25.9051]I[auto_schedule.cpp:509][AUTO] ExecutableNetwork end
[22:35:25.9052]I[multi_schedule.cpp:254][AUTO] CPU:infer:0
Deleted compiled_model
@@ -222,7 +154,7 @@ improve readability of your code.
# Set LOG_LEVEL to LOG_NONE.
ie.set_property("AUTO", {"LOG_LEVEL":"LOG_NONE"})
compiled_model = ie.compile_model(model=model, device_name="AUTO")
compiled_model = ie.compile_model(model=ov_model, device_name="AUTO")
if isinstance(compiled_model, CompiledModel):
print("Successfully compiled model using AUTO.")
@@ -257,34 +189,30 @@ compilation. The challenge is with GPU devices since OpenCL graph
complication to GPU-optimized kernels takes a few seconds to complete.
This initialization time may be intolerable for some applications. To
avoid this delay, the AUTO uses CPU transparently as the first inference
device until GPU is ready. ### Load an Image
device until GPU is ready.
Load an Image
~~~~~~~~~~~~~
torchvision library provides model specific input transformation
function, we will reuse it for preparing input data.
.. code:: ipython3
# For demonstration purposes, load the model to CPU and get inputs for buffer preparation.
compiled_model = ie.compile_model(model=model, device_name="CPU")
from PIL import Image
input_layer_ir = next(iter(compiled_model.inputs))
image = Image.open("../data/image/coco.jpg")
input_transform = torchvision.models.ResNet50_Weights.DEFAULT.transforms()
# Read image in BGR format.
image = cv2.imread("../data/image/coco.jpg")
# N, C, H, W = batch size, number of channels, height, width.
N, C, H, W = input_layer_ir.shape
# Resize image to the input size expected by the model.
resized_image = cv2.resize(image, (W, H))
# Reshape to match the input shape expected by the model.
input_image = np.expand_dims(resized_image.transpose(2, 0, 1), 0)
plt.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
del compiled_model
input_tensor = input_transform(image)
input_tensor = input_tensor.unsqueeze(0).numpy()
image
.. image:: 106-auto-device-with-output_files/106-auto-device-with-output_14_0.png
.. image:: 106-auto-device-with-output_files/106-auto-device-with-output_12_0.png
Load the model to GPU device and perform inference
@@ -297,14 +225,10 @@ Load the model to GPU device and perform inference
else :
# Start time.
gpu_load_start_time = time.perf_counter()
compiled_model = ie.compile_model(model=model, device_name="GPU") # load to GPU
# Get input and output nodes.
input_layer = compiled_model.input(0)
output_layer = compiled_model.output(0)
compiled_model = ie.compile_model(model=ov_model, device_name="GPU") # load to GPU
# Execute the first inference.
results = compiled_model([input_image])[output_layer]
results = compiled_model(input_tensor)[0]
# Measure time to the first inference.
gpu_fil_end_time = time.perf_counter()
@@ -328,14 +252,10 @@ executed on CPU until GPU is ready.
# Start time.
auto_load_start_time = time.perf_counter()
compiled_model = ie.compile_model(model=model) # The device_name is AUTO by default.
# Get input and output nodes.
input_layer = compiled_model.input(0)
output_layer = compiled_model.output(0)
compiled_model = ie.compile_model(model=ov_model) # The device_name is AUTO by default.
# Execute the first inference.
results = compiled_model([input_image])[output_layer]
results = compiled_model(input_tensor)[0]
# Measure time to the first inference.
@@ -346,7 +266,7 @@ executed on CPU until GPU is ready.
.. parsed-literal::
Time to load model using AUTO device and get first inference: 0.15 seconds.
Time to load model using AUTO device and get first inference: 0.19 seconds.
.. code:: ipython3
@@ -367,9 +287,9 @@ completely portable between devices – meaning AUTO can configure the
performance hint on whichever device is being used.
For more information, refer to the `Performance
Hints <https://docs.openvino.ai/latest/openvino_docs_OV_UG_supported_plugins_AUTO.html#performance-hints>`__
Hints <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_AUTO.html#performance-hints>`__
section of `Automatic Device
Selection <https://docs.openvino.ai/latest/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__
Selection <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__
article.
Class and callback definition
@@ -484,7 +404,7 @@ Loop for inference and update the FPS/Latency every
print("Compiling Model for AUTO device with THROUGHPUT hint")
sys.stdout.flush()
compiled_model = ie.compile_model(model=model, config={"PERFORMANCE_HINT":"THROUGHPUT"})
compiled_model = ie.compile_model(model=ov_model, config={"PERFORMANCE_HINT":"THROUGHPUT"})
infer_queue = AsyncInferQueue(compiled_model, 0) # Setting to 0 will query optimal number by default.
infer_queue.set_callback(completion_callback)
@@ -493,7 +413,7 @@ Loop for inference and update the FPS/Latency every
sys.stdout.flush()
while THROUGHPUT_hint_context.feed_inference:
infer_queue.start_async({input_layer_ir.any_name: input_image}, THROUGHPUT_hint_context)
infer_queue.start_async(input_tensor, THROUGHPUT_hint_context)
infer_queue.wait_all()
@@ -510,12 +430,12 @@ Loop for inference and update the FPS/Latency every
Compiling Model for AUTO device with THROUGHPUT hint
Start inference, 6 groups of FPS/latency will be measured over 10s intervals
throughput: 461.74fps, latency: 24.68ms, time interval: 10.01s
throughput: 470.76fps, latency: 24.89ms, time interval: 10.00s
throughput: 470.13fps, latency: 24.96ms, time interval: 10.01s
throughput: 470.19fps, latency: 24.89ms, time interval: 10.00s
throughput: 471.13fps, latency: 24.87ms, time interval: 10.00s
throughput: 469.51fps, latency: 24.92ms, time interval: 10.00s
throughput: 190.61fps, latency: 29.81ms, time interval: 10.00s
throughput: 194.35fps, latency: 30.08ms, time interval: 10.00s
throughput: 194.90fps, latency: 30.02ms, time interval: 10.01s
throughput: 192.30fps, latency: 30.47ms, time interval: 10.00s
throughput: 192.76fps, latency: 30.38ms, time interval: 10.02s
throughput: 194.26fps, latency: 30.10ms, time interval: 10.00s
Done
@@ -532,7 +452,7 @@ Loop for inference and update the FPS/Latency for each
print("Compiling Model for AUTO Device with LATENCY hint")
sys.stdout.flush()
compiled_model = ie.compile_model(model=model, config={"PERFORMANCE_HINT":"LATENCY"})
compiled_model = ie.compile_model(model=ov_model, config={"PERFORMANCE_HINT":"LATENCY"})
# Setting to 0 will query optimal number by default.
infer_queue = AsyncInferQueue(compiled_model, 0)
@@ -542,7 +462,7 @@ Loop for inference and update the FPS/Latency for each
sys.stdout.flush()
while LATENCY_hint_context.feed_inference:
infer_queue.start_async({input_layer_ir.any_name: input_image}, LATENCY_hint_context)
infer_queue.start_async(input_tensor, LATENCY_hint_context)
infer_queue.wait_all()
@@ -559,12 +479,12 @@ Loop for inference and update the FPS/Latency for each
Compiling Model for AUTO Device with LATENCY hint
Start inference, 6 groups fps/latency will be out with 10s interval
throughput: 250.83fps, latency: 3.62ms, time interval: 10.00s
throughput: 253.12fps, latency: 3.70ms, time interval: 10.00s
throughput: 250.90fps, latency: 3.73ms, time interval: 10.00s
throughput: 249.98fps, latency: 3.74ms, time interval: 10.00s
throughput: 248.29fps, latency: 3.77ms, time interval: 10.00s
throughput: 255.00fps, latency: 3.67ms, time interval: 10.00s
throughput: 138.65fps, latency: 6.69ms, time interval: 10.00s
throughput: 142.09fps, latency: 6.68ms, time interval: 10.00s
throughput: 142.03fps, latency: 6.68ms, time interval: 10.00s
throughput: 142.07fps, latency: 6.68ms, time interval: 10.00s
throughput: 141.79fps, latency: 6.69ms, time interval: 10.00s
throughput: 142.19fps, latency: 6.67ms, time interval: 10.00s
Done
@@ -573,6 +493,8 @@ Difference in FPS and latency
.. code:: ipython3
import matplotlib.pyplot as plt
TPUT = 0
LAT = 1
labels = ["THROUGHPUT hint", "LATENCY hint"]
@@ -600,7 +522,7 @@ Difference in FPS and latency
.. image:: 106-auto-device-with-output_files/106-auto-device-with-output_27_0.png
.. image:: 106-auto-device-with-output_files/106-auto-device-with-output_25_0.png
.. code:: ipython3
@@ -634,5 +556,5 @@ Difference in FPS and latency
.. image:: 106-auto-device-with-output_files/106-auto-device-with-output_28_0.png
.. image:: 106-auto-device-with-output_files/106-auto-device-with-output_26_0.png
docs/notebooks/106-auto-device-with-output_files/106-auto-device-with-output_12_0.png
+3
View File
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d8a9811dc8ab8b5f2ef6171ba757ccf9bbe33d0f137824ad0c32380bd74c5ff4
size 928896
docs/notebooks/106-auto-device-with-output_files/106-auto-device-with-output_25_0.png
+3
View File
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:512f1331af7db846d6d7f59ede72bde479afc77892515952f57288aa0b028cd9
size 26607
docs/notebooks/106-auto-device-with-output_files/106-auto-device-with-output_26_0.png
+3
View File
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b3cf4ed07da8d66040ad109a3f88329d9b29011f6ec2586df67ce9719fc2841b
size 41162
docs/notebooks/106-auto-device-with-output_files/106-auto-device-with-output_27_0.png
-3
View File
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b7098141604597456849e891e131217de0721f817667cb44993edff0ad8d20da
size 26785
docs/notebooks/106-auto-device-with-output_files/106-auto-device-with-output_28_0.png
-3
View File
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:7a8f5540b29277e803adfa05426264388f2d5872cde0830dbace8ed9e0dc114c
size 37899
docs/notebooks/106-auto-device-with-output_files/index.html
+5 -5
View File
@@ -1,9 +1,9 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/106-auto-device-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/106-auto-device-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/106-auto-device-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="106-auto-device-with-output_14_0.png">106-auto-device-with-output_14_0.png</a> 30-May-2023 00:09 387941
<a href="106-auto-device-with-output_27_0.png">106-auto-device-with-output_27_0.png</a> 30-May-2023 00:09 26785
<a href="106-auto-device-with-output_28_0.png">106-auto-device-with-output_28_0.png</a> 30-May-2023 00:09 37899
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/106-auto-device-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="106-auto-device-with-output_12_0.png">106-auto-device-with-output_12_0.png</a> 22-Jun-2023 00:06 928896
<a href="106-auto-device-with-output_25_0.png">106-auto-device-with-output_25_0.png</a> 22-Jun-2023 00:06 26607
<a href="106-auto-device-with-output_26_0.png">106-auto-device-with-output_26_0.png</a> 22-Jun-2023 00:06 41162
</pre><hr></body>
</html>
docs/notebooks/107-speech-recognition-quantization-data2vec-with-output.rst
+327 -311
View File
@@ -1,13 +1,12 @@
Quantize Speech Recognition Models with OpenVINO™ Post-Training Optimization Tool ​
===================================================================================
Quantize Speech Recognition Models using NNCF PTQ API
=====================================================
This tutorial demonstrates how to apply ``INT8`` quantization to the
speech recognition model, known as
`Data2Vec <https://arxiv.org/abs/2202.03555>`__, using the
`Post-Training Optimization Tool API (POT
API) <https://docs.openvino.ai/latest/pot_compression_api_README.html>`__
(part of the `OpenVINO Toolkit <https://docs.openvino.ai/>`__). This
notebook uses a fine-tuned
This tutorial demonstrates how to use the NNCF (Neural Network
Compression Framework) 8-bit quantization in post-training mode (without
the fine-tuning pipeline) to optimize the speech recognition model,
known as `Data2Vec <https://arxiv.org/abs/2202.03555>`__ for the
high-speed inference via OpenVINO™ Toolkit. This notebook uses a
fine-tuned
`data2vec-audio-base-960h <https://huggingface.co/facebook/data2vec-audio-base-960h>`__
`PyTorch <https://pytorch.org/>`__ model trained on the `LibriSpeech ASR
corpus <https://www.openslr.org/12>`__. The tutorial is designed to be
@@ -16,9 +15,9 @@ steps:
- Download and prepare model.
- Define data loading and accuracy validation functionality.
- Prepare the model for quantization.
- Run optimization pipeline.
- Prepare the model for quantization and quantize.
- Compare performance of the original and quantized models.
- Compare Accuracy of the Original and Quantized Models.
Download and prepare model
--------------------------
@@ -54,7 +53,8 @@ model specific pre- and post-processing steps.
.. code:: ipython3
!pip install -q soundfile librosa
!pip install -q 'openvino-dev>=2023.0.0' 'nncf>=2.5.0'
!pip install -q soundfile librosa transformers onnx
.. code:: ipython3
@@ -86,7 +86,7 @@ Convert model to OpenVINO Intermediate Representation
def export_model_to_onnx(model, path):
# switch model to evaluation mode
# switch model to evaluation mode
model.eval()
# disallow gradient propagation for reducing memory during export
with torch.no_grad():
@@ -128,18 +128,10 @@ Convert model to OpenVINO Intermediate Representation
ov_model = core.read_model(ir_model_path)
.. parsed-literal::
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/data2vec/modeling_data2vec_audio.py:427: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len):
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/data2vec/modeling_data2vec_audio.py:466: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim):
.. parsed-literal::
ONNX model saved to model/data2vec-audo-base.onnx
IR model saved to model/data2vec-audo-base.xml
Read IR model from model/data2vec-audo-base.xml
Prepare inference data
@@ -178,8 +170,8 @@ dataset.
.. parsed-literal::
[ WARNING ] Found cached dataset librispeech_asr_dummy (/opt/home/k8sworker/.cache/huggingface/datasets/patrickvonplaten___librispeech_asr_dummy/clean/2.1.0/f2c70a4d03ab4410954901bde48c54b85ca1b7f9bf7d616e7e2a72b5ee6ddbfc)
[ WARNING ] Loading cached processed dataset at /opt/home/k8sworker/.cache/huggingface/datasets/patrickvonplaten___librispeech_asr_dummy/clean/2.1.0/f2c70a4d03ab4410954901bde48c54b85ca1b7f9bf7d616e7e2a72b5ee6ddbfc/cache-c14398002c490f3f.arrow
Found cached dataset librispeech_asr_dummy (/home/adrian/.cache/huggingface/datasets/patrickvonplaten___librispeech_asr_dummy/clean/2.1.0/f2c70a4d03ab4410954901bde48c54b85ca1b7f9bf7d616e7e2a72b5ee6ddbfc)
Loading cached processed dataset at /home/adrian/.cache/huggingface/datasets/patrickvonplaten___librispeech_asr_dummy/clean/2.1.0/f2c70a4d03ab4410954901bde48c54b85ca1b7f9bf7d616e7e2a72b5ee6ddbfc/cache-4e0f4916cd205b24.arrow
Check model inference result
@@ -219,6 +211,8 @@ For reference, see the same function provided for OpenVINO model.
.. code:: ipython3
core = Core()
pt_transcription = torch_infer(model, dataset[0])
compiled_model = core.compile_model(ov_model)
ov_transcription = ov_infer(compiled_model, dataset[0])
@@ -235,9 +229,9 @@ For reference, see the same function provided for OpenVINO model.
.. parsed-literal::
[Reference]: A MAN SAID TO THE UNIVERSE SIR I EXIST
[PyTorch]: A MAN SAID TO THE UNIVERSE SIR I EXIST
[OpenVINO FP16]: A MAN SAID TO THE UNIVERSE SIR I EXIST
[Reference]: MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL
[PyTorch]: MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL
[OpenVINO FP16]: MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL
@@ -246,7 +240,7 @@ For reference, see the same function provided for OpenVINO model.
<audio controls="controls" >
<source src="data:audio/wav;base64,UklGRmRFAgBXQVZFZm10IBAAAAABAAEAgD4AAAB9AAACABAAZGF0YUBFAgDn//f/uf/6/9v/FgBCAGAAnADq/+r/LwD3/67/JQDYAEoA3v8MAOn/xf8HAIYA7P+M/xoALQDG/6v/hAB8AKr/lP/3/ywALQDDAFkA0//s/w8A9P/8/44AcAD8/zIAPQAMACEAKgC7/1D/q/8hACEALQBzAHMAAwC4/+T/w/+J/97/PwBgAOb/ZQBOAMb/gf+u/1MABADZ/9T/1//h/8D/jf/O/6r/pf/e/7D/4//f/+b/fP8P/7b/GQD1/7D/tv9WAMn/j//b/9v/wf++/+T/j//9/2wA9f9L/2//0f/G/1b/rf+HAD8Avf/D/xQA2//e/x0AZ/9x/z0ALAAWAMn/HwC2/8j/bv9y/48AbgDx/03/RwD0/4L/rf+2/3L/hf9AAPf/5/+UAF0APf8c/7b/y/+o////AQAqAC0AAwDT/9f/uf8y/3b/mv8XAIMAWQAHAMD/HQCy/3T/h/8SAJIAQgD3/xwAYADu/7v/oP/3//z/wf/c/8n/UAAiAA8A5/9f/9//XQD5/9b/PwB+ALn/gv/1/xIA9P84AC8Atf80AD0ABABZ//n/WwDc//L/DgA1AOT/AQDR/4H/n//8/+7/vf/1/14ALQCM/6X/zv/p/wwAKgAUAC8A6v/Q/7L/TgBNAA8AFADh//H/QAAfAND/4f8kADUAlP8sAGkAHQDy/0AAfwAvAAcAJADk/wYAmQAsAGEAIgBDANn/AwB+ACIAawAhAOH/9P8iADIA6f/n/w4AKQC6AKcARQASANv/3P9//9n/bgB5AHcAhABKAOT/xv8RAHn/ef9FAJQAYQBOAI4ALQCo/6P/kv/J/yoARwCZAF4ATgD6//T/+f/U/wkA1/8XADgAOgBHAD8ABwCB/1z/2f/3/1IAKgAfADQAPQDj/67/8v8AAMb/4f9NAH8AUAD6/8n/cf+y//n/AwDu/wQAeQD5/1b/tf8sABIAif/X//L/+v/0/+//8f8dABwAq/+g/+n/MAAwAPr/4/8LAAcAov+i/wEACwAAAO7/9//m//n/pf9j/9z/EQAOAPT/RQB8ABYAxf+X//f/HQAXAIYAdwBLACoAOwD1/4r/DgAkALL/LwCiAGMA0f8GACkAaf/u/1gAMgBwAKUAqADT/4r/IQDn/7D/bv/q/zgAvf98AHsAyf+g/5X/UP81/xQApQARAEgAcQBOAOf/bv+u/5//3/8MAFIAsgBNAEoAHQBx/2f/gf/8/+z/7/+cAF0A0P/e//H/DgD5/xkA/f98/yIAdwAZABIAXgC9ABkAkv8nAIwA/f9B/6D/WAAwAGgAwwBeACwACQCb/y//Sf8WACUABACVAHkBMAHk/63/oP/z/g//WAC7AHcA/gAyAWsAqP/J////nf+N/x8A2wDFAHsA0wBhAOT/qv/A/9H/yP+iAJkATgDX/xoADAAX/x3/hgAJATsAMgBYAPL/mP8fAL7/rv8PAEcA1//u/80AhgDT/zP/Wf+4/+H/MAAtAFsAlwA/ALP/Rv8BAB0Akv/U/3cAzgCHACUAgv9y/+H/4f9W/wkA7ACSAK7/lP8ZALn/OP9m/9v/KQBsAIEAYADh/0oAFgAY/wr/SADxAL7/xf/VAIwAbv/D/ywAd/8+//z/GgCf/2wA6AD1/4//GgD9/yP/mP/CABEAn/8lAIoA2f+u/6oALQDn/8X/yP9J/2n///8lACIAWQA4AOH/y//J/73/cf/e/wwAHQAiACEAJwA3AF0Ay/9k/63/+v+J/4T//f9rAAEAcv/B/9D/5/92/6r/zP+a//3/FgDI/5//NQBHAHn/gv9HAMv/bP9s/9z/AAD5/2UA/f+r/+b/8f+K/2b/6v9YAIX/tv9YAGMAAQA4/7b/5P8aABYA4/9oAFsAEgCU/7L/WwBjAMb/u/8RAIwAVgCf/9v/7P8EAIz/Xv9SAN0AVgDD/8n/aAA0AIT/9/9gAKgAMACr/xwAQAAwANf/zv9sAFYAMgDx/9v/aAApAIn/Jf/u/5IAJQAcAJUAlAAPAJT/3v9SAPT/0P+q/+f/l//y/2MAFAA/AGMAHQCU/wYAOgDD/3//HwDX/5T/LwCXAJUANwB/AAAAs/+l/3H/tv8wAGMAQABHACQAHQD6/yUA1//G/6L/Tf/R/2MAvwCnAHkAQgDA/6v/zv8ZAP//7P8vADAAQgBdADAA8v8EAFYAEgCr/yIAWQD//9v/WACGADIAFAAtABQAHwBbAPL/jP8LACkADABmAMsAugA6AI4ARQDe/+b//P8kABIANACnAOMAUAA9AAAA2/82/5v/YwBdAM0AvQCUAPr/zv/k/5X/n//x/yIAOwAUAD8ANADZ/77/ov+r/43/AQA7AOT/6v9IAJr/Tf95/+f/rv/9/6EA2//b/ycAyP/C/k3/NAD5/97/iQCJAPn/5P9k/w//Ov8fAPX/Yf8cALAAXgDf/5X/oP9O/5X/tf9s/14AxgBhABIARQBgAGH/Kv+l/8D/SgBKAAMABABTAIcAuP+V/zcA+v+Q/5T/TgCnAFUA//+V/8P/TgD8/7b/OACqAF0Ao//3/xQA8v/O/8P/tv/1/5oAlQDq/wEARwCj/x3/j/86ANb/8f9HACIAKQD1/0cABACQ/5D/jP8JAP//IQB7AAsA3P+E/4f/yf+q/+n/GQA9AAAAwP93AGMAkP8j/4r/zP+9/x8AvwClAF0ADgC1/7P/2/9LAL3/zP98AHsADwADAIcABgBc/33/7v/b/x8AHwAUAPf/NwBxANn/BwBKABwAhP+o/zgAbgAHAOP/9/9pAIcA2//c/yUAKQDG/2P/FgCqAE0A3P/Z/z0AFgABAAQA3P8DAFYACQCw/0UAhAAhALn/MgAhAP//GgBOAAsA4f8yAEMA8v/6/3sAJADT/97/RQA9ACoATQAJAKX/yf8LALv/w/9mAH8AsP+S/97/JADu/9v/EQD5/w4A/P8WAD0AIgASAKv/a/++/y0AJQDO/yQAYQDe/5r/0f/L/9n/NQAqAKL/9P+vACIASP+o/2MA6v89/6j/lwA/ANf/DAA4ACIA+f8fAAcA9/9AAPT/nf8lAIcADABm/9z/FgB6/1z/q//O/7j/RwCSABwACwD9/8P/VP99/xcACwAPAF0AhABHAA4AJACd/97+Xv/5/xkAKQCiAI8Arf+C/43/qP+U/9D/7P8PACwAKQA6ABcAAQCJ/6j/6f9DAHAAWAD0/6f/0//b/+T/7P9CAL7/sP+Y/+H/GgDb/+H/tf/e/7v/KQCcAEcArf+2/7P/Zv+f/00AjADs/9//y//h/wYA/f/Z/8b/9f/0/wEALABLAB8A1v+B/9H/KQAMAAcACwBIADcA7P/n/wQAMgADANb/aACOAPz/5//5/67/rf/6/+r/mv8tAGUAu/8WALIANACd/+b/NwDZ//n/bAAZAOH/SwAqANT/6f+yAF0Aiv/M/0UAFgCU/+b/XQBLAAsA8v9AAGgA+v+U//L/PQD3/7b/UACqAFkAXgBHANT/1/93ACoAoP8hAI8AJwDj/4kAaQDc/9n/CwDW/8z/SACBACkALQCvAHQAAQC1/wQA9/+r//r/YQCqACkAPQBQAE4A1P+P/+P/2f+1/yEAlAAXABIASwBzAMv/xf8UALD/5P9eAF0A5/8DAP3/8f/A/z8AEgDf/yEAJwBhACwATgAAAMH/gf/c/xcAVgBKAEIAhwDM/5f/w/8sAL3/zP8UADoA/P89AGYA3/8RAA4A4f+X/1UAzQBpAAYAXgCDAPr/vf/L/+f/wP/8/wsAKgBdADoAoP9N/w4AQwDJ/8H/JABsACQA7//Q/xEAZgADAJj/BAB8ADAAyP/m/zAA1P/y/6f/tf9FADsADACb/wYAAQC7/6L/1v8RAC8AFgD1/xoANQA7AJX/4/9SAEIAov+n//H/8f/8//n/1//W/0cA4f9y/8j/MgDJ/6X/zP8UACEAMgDv/3//+f8JAG//SP/c/00A/P/p/3wAWAADALv/tf/Q/5j/u//y/+b/AwBoADQAp//A/yUAy/+J/yUANwC+/8j/RwD0/53/3P8PAO//4/8tAAcANQAvANb/mv/T/yIA9P+5/xEAWQAlAAsADwA0APf/tf/B//z/JQAXANH/IQAvANn/DADn/wsA3//T/6r/6f8iALb/qv/u/1AA8f/f/7b/7P/n/63/2f/v/zoA5v8BAOb/JwADALj/u//I/0oADAD5/+r/awAhAND/zv/3/xIA0P8wACQAWQDy/x0A8f/9//r/DgAwAIf/BwBzAIQApf/k/yUA0P+S//r/XgDW/9b/zP8HAOn/NwAHAKD/vf8fABEAXP8BACIA2f9//xQAlwDy/5//FABlAO//xf/O/xkA7v9TAFIA1v8nACcArv8q/7v/RwAkAAEAYQBmAHAAFgCw/97/CwAyAOb/NwCoAGMASgBDAFUAAADF//T/DAA9AHwAhAAqAB8AJADj/4z/3v89AMH/6f+BALAA6f+K/yIALQDb/8n/HwDm/63//f9NANv/sP8lAPr/Yf+n/4QA7v9I/6L/JACu/4r/5P/A/5X/zP/s/6r/BgApAO//vf/b/yoAGQD//+r/HAA4AOb/zv8HAOP/u/+a/6j/EQA6ABkA6f8cABcA6v/b/8z/w//9/87/w/9HANYAcADR/wAA7P+K/0b/qv/0/xIASgBOAAkAHQAlAKj/X/93/9D/6v8BAGgAcABmADAAqv+V/63/QgAcAJj/GgB+AFgAIgD//+f/oP+r/xwADwCGAO4AfwC+/73/MAAqAJr/1P9oAHcAcABpACkA3v/X/6j/gv/A/2YA9P/f/10AyABOALL/zv/j/6f/dP9HAKEAhwCb/63/JwAJAA8A0/8dANb/uf+q/wAA9P84ACUA0f/s/xEAWQDF/6P/0P8XABQAQwAvLine truncated
<source src="data:audio/wav;base64,UklGRgTcAgBXQVZFZm10IBAAAAABAAEAgD4AAAB9AAACABAAZGF0YeDbAgDJAK8ApwCxAIgAGQAHABwAUgCaAKoAsQCSACYA3/8mAFUABwApAGEAPQBPAJgAmgBKACQAKQA9AFwAsQDYANUA2ADBAJgAWgA2AAoAFwBhAIAAgwCiAGcA/v/i/9//3//0/zgAbACdAIYAbAA9AAcA0P/a/wcAHAAUAF8AcwBsAEoAGQC2/4X/u/8KAGQAogCLAEoAQAAkANX/0v/i/9z/8f8wAE8APQBVADsAFADx/8r/tv/X/xcAPQCaAJgAaQAzAOL/lP+z/8j/wP8HADgACgA2AHEAHgAAABwADwAHAEUAVQA7ABkADwACALH/j/+7/7b/q/8kAEUAEgAMACEA6f/f/wIAFwAXABwAHgAuACEAAgD7/xcA+//7/ykAKwArABcA/v/K/9f/7v8KAAAA5P+k/6b/5P8MAPT/vf/D/5z/lP/X/+z/AgBVAFUAJgAhAPH/0P/X/9D/pP/D//T/vf+p/8P/eP94/3D/T/9e/5L/xf/x/1wAHgAFAAcAMwDi/6H/u//F/+n/CgAwAAUA0v+4/5//l/+k/9D/AgAPADsAIQDX/73/yv+4/7H/0v/A/9//BwAUADgAMwDa/43/rv/a/wIAQwBSACYABQDc/+f/0P/N//7//v/S/+f//v8HAE8AiwBFAAcA/v/i/9//MAA2AO7/3P/N/8j/9v8hAEMAUgA2AB4AHABIAE8AQAA4ADgAQAAcAAUABQD5/7v/yv8PAPb/9P8eAE0AaQCDAHYAIQAKAAoAEgAmAE8AUgAuABkAMwBKAEMAKwAAAPb/DwDp/yQArwCSADMAQAA2ANL/5/8HAAAAMAAuAOL/BQBKABcAVwDQAEUAyP/Q/9//4v8XABcAMwBsAGQAnwCaAD0Akv9j/43/n/8eAJUApwCLAFcAFADp/7j/iv96/8P/AgBFAIAArABhAPv/nP9z/5f/1f9fAMsAqgA7AAwA5/+Z/5n/DwA2ACsASgBxAEgA7v/c/6n/Xv9h/5L/9v8ZAB4AGQAMAN//gP96/4X/yP8KAFoAcwBfAPT/gP9c/x7/Mv+N//H/AAA7AC4A7v/N/5L/Ov81/5T/2v9PAIYAQwAAAPb/pv+K/9z/2v/X/zgAiABPAC4AGQDA/2j/T/+U/+L/+f8uADsA+f+U/43/n/+C/6n/CgBIAEUAJgAZANX/tv+c/7b/wP/i/9r/5P8PAOf/wP+H/0z/Qv/V/ysAVwCNACkAof+F/5n/q//I//v/EgAUABkAJAAXAMP/ev9w/33/qf9VAMEAiwApAAUAxf9w/43/yv/X/wIANgB5AIgAHgCu/5T/hf9Z/8j/JAAXAAcAFAAcABIAEgAPAAoAzf+2//T/AgAFADgARQDp/+T/HgD0/9z/GQDf/67/8f87AEMAMwBsAE0AKwAcAEgALgAXAAcAEgAeAFIAeQBsABcA8f8MADgAXwCQAG4AKwAcAEUApwC3ANAArwBkAPT/7P8uAFwAiwCfAJoAWgBXAHsAtwCYADMANgBfAC4AXAC0AIYA/v/p//v/AAASAEgAbABNAC4ADwA9AGkAfgBFACQAEgCK/4L/DAArAOz/MwBaADAAAgAcACQA7v/7/xIAOwBSAFcAzgCxAE8AQAACAI//gv/u/wIACgBVAGQAFAACAAoAxf/F//T/yv/u/xwAPQBcAJgAeQAwAPb/hf99/+T/HgAUAEoA+f9Z/4r/9P/i//T/BQCp/2j/0P8UAE8AgwBDAOf/OABNAPv/+f8AAOT/vf8CAA8A0P/S/83/BQAcAOT//v84ACYA0P/S/8X/nP/A//v/QABSAFwALgAUAOn/pP+S/9r/JAAPAAAAAADa/5L/kv/Q/4D/ff/c/wcA0v/i/9L/ev9W/4f/h/+N/9r/yP+U/7H/yv+Z/8X/xf+u/43/pP+7/8r/yv+A/4f/uP/p/9//AgC4/17/Vv+u/+T/+//n/+f/5P+U/2H/l//D/43/qf/X/9r/rv8wAGwACgDf/9X/xf+x/+L/AgDi/67/zf/n//H/BQAFAO7/sf96/5T/9P82AE0ATQArANr/s//k/83/0v+9/4f/jf/I/83/3P8cAAcA7P/c//v/6f/5/wwAHABKAHEAQwA2ADAA0v/X/xQAJgA7AC4A+//n//b/JgA2AFIALgDX/73/yv/n/yYAOwBVAJAAQwAmAG4AewCAAEgADADN/9L/MwC8AO8AqgAuAOL/7v/5/0AAUgAcAPv/9P/p/wIAOwA9ABkA8f/u/6n/5P8UADgAXABVAOn/3/8HAMP/5/9hAGcA2v+4//T/rv+2/0MAZwAFAOn/MwArADAAXwBaABQA9v/x//T/+/8HABQACgDk//7/GQAUADgAQwAkABIAPQA7AAoAHgBAAPb/FwBfAAUA3/8SACQAFwA2AA8A1/+c/7H/xf8HAE8AMAAMANf/yv+4/9D/NgBPAAAA6f/u//n/BwAPABQA0v/D/6v/w//s/xkAIQA2ABQA4v+x/9D/4v/K/9z/DwDx/8j/FwBVADsAAABDAC4ADAAzAGQARQAUAC4AUgA4AP7/7P/K/+7/s//n/wwAEgDi/+T/FAC9/4f/sf/X/4D/Xv9m/8X/zf/N/wUA5/+K/5T/3P/S/6H/qf/F/6H/q//N/+L/0v/f/+L/7v+7/9L/AgDp/+n/9v/7//b/IQAKAAIABwAkADAASgA2ACkA0v/D/9X/QwBhAIYAlQACAKT/pP/A/+n/XwCiAG4AAgA9AMr/RQAkAAcAKQDc/wAApACNACEA+/8XABkA1//K//v/KwBhALQAsQBnAMr/3//k/8P/xf82AGcAZwB7AGcAgwBNABkABwDs/9z/OACsAH4AbgBnAEMAOABuAK8AiACVAJAAfgBzAIMApAB+ADYAEgA7ACEAbADdALcAaQAuACYAIQBSAKIAuQCnAD0AEgBAAFwADwA4AHYARQAhAGEAiACSAEMA+f/0/9f/2v9KALwAewD2/xIAKwDa/+L/AAD7/+n/FACGAJ8AfgBXAD0A1f+r/+z/NgBIAF8AUgAuADMAcQBpABwA9v/0/zMAWgBsAF8AIQD+/+n/AAACAOn/DwBSAEoA+f/u/wAA3P/S/w8AMwDa/9L/9v/+/8r/yP/f/7j/q/+U/8r/GQA7APT/0P+c/z//Xv/F/w8AFwDp/7j/hf9o/2j/s//u/67/qf/D/6v/jf+m/7H/qf+X/5T/yv/A/1n/Vv+r/5n/ff/F/8D/tv/f/xkAAADx//v/0P/7/+n/5//c/7b/vf/F/67/lP+f/4L/Wf+S/6H/mf+K/6n/nP+F/33/rv+4/3D/Xv96/6n/cP+Z/8r/kv9e/33/vf+4/4X/aP+X/6v/sf/D/7v/pP9z/33/l/+2/9r/2v/n/9X/l/9h/3P/mf/s//b//v8rABkA4v8HABkA5P/a/9r/wP/S/xIAFAAMAOf/nP9r/5n/rv/s/ysAAADF/8j/ff9U/4f/qf+U/5f/3P/7/yEADwDK/7j/n/+k/xcAXAAuACQALgDx/7j/1f/7//H/BwBDAFcATQBFABkA7P+Z/6T/8f84AH4AqgCdAC4AKwAkADAADwBAAF8AQAB5ANAA6gCiADgA8f/N/+7/TwCfAMMAhgAuAC4ALgAPAFwAXwAeAA8ATQBpAHsAnQBsABIAzf/K/wUAcwCfAIsAeQA2AAAABQAkADgAFAAAAOL/HgAzAEgARQAwACQA3//i/wAAQwAzAEUAQwAhAPv/HABSACQABwAAADAAQABDAE0AGQDx//n/7v8hAE0AXABVADYALgAmAA8AEgAAAAIAKwBuAFoAMwAmAD0ABwApAHMAVwAKAO7/BQDi/yQAYQBSABcAKQBKAEMASgAuAPH/tv/p/w8AIQAcAE0AYQAcAB4AGQD0/8D/5/8MACsAUgBcACQA2v/A/7j/9P8PABQAFwAkACQAOAAmAO7/7v/X/8X/9v8hADAADAAcAA8ADwDx/9D/3//x/+L/6f8uADAAFAAMAMr/df89/27/2v8rACkADwAKAOn/zf8FAPv/u//D/+T/5//x/wUAKwBnAAIAsf/A/+L/4v8XAH4AVwAKAAcA9v/f//H//v8PAAwA/v8eAFIAKQAeAPn/7P8MADgAYQBVAEMAMwD2/w8AJAAHAPb/CgApACkADAAhAAoAyP/u/xQAMwAwAAcA1f+u/2j/iv/A/wIASABNADAABwAwADMAKQAwABIAzf8FAEAAgwCdAIAAFADx/8X/q//x/yYAJAAzAHMAXwAhAA8AGQASAAcAFABVAGQALgA4AFoANgBFAG4AcQA7AO7/AABAADgASABaADYAAAAkAFoAUgBAAEAAAACm/7v/xf8uAIAAkAB7AGkAMAD+/+T/5//x/wIAKQA4AGkAYQBIAFcAWgAXAPn/7v/p/+T/8f/0//b/HABNAEMAIQDx/+z/BQDs//b/JgBKAE0AfgCSAIgAOAAUANr/rv+c/+T/JABNAHkAXwBIABQA5P/A/9X/9P8UAEgAbgBpAGQAHgD+/wcA6f/N/+L//v/2/xwAPQBsAEUA8f+7/7j/mf+K/7v/+//x//T/EgAFAMj/pP/I/8X/mf+X/+L/rv+k/9//BQDD/4//lP99/4L/pv/n/9r/of+P/73/u//N/+f/vf+2//b/+f/x//T/5P+U/6b/s/+Z/67/0v/i/9X/5P/I/8D/pP+c/7j/yP/p//H/6f/S/8j/xf+r/5n/j/99/7P/yP/F/6v/rv+r/8D/4v/V/+n/yv+u/8D/7P/0/9///v/A/9z/5//i/wIAJADS/5T/yv/V/8r/AgAzAPH/rv/S//T/5P8FAAoABwDS/9z/DwAeAO7/yP/A/6b/yP8KADMA9v/S/6n/of+r/+n/AgD+/+z/xf/I/9//7P/2/9z/q/+m/8P/6f/uLine truncated
Your browser does not support the audio element.
</audio>
@@ -313,286 +307,297 @@ library.
[OpenVino] Word Error Rate: 0.0383
Prepare quantization pipeline
-----------------------------
Quantization
------------
Post-Training Optimization Tool designed to accelerate the inference of
DL models by converting them into a more hardware-friendly
representation by applying specific methods that do not require
re-training, for example, post-training quantization. For more details
about the low-precision flow in OpenVINO™, refer to the `Low Precision
Optimization
Guide <https://docs.openvino.ai/2020.4/pot_docs_LowPrecisionOptimizationGuide.html>`__.
`NNCF <https://github.com/openvinotoolkit/nncf>`__ provides a suite of
advanced algorithms for Neural Networks inference optimization in
OpenVINO with minimal accuracy drop.
`The Python POT
API <https://docs.openvino.ai/2020.4/pot_compression_api_README.html>`__
provides simple interfaces for implementing custom model inference with
data loading and pre-processing on an arbitrary dataset and implementing
custom accuracy metrics to make it possible to use optimization
algorithms from the POT. The Python POT API represented by ``Pipeline``
class for creating and configuring the optimization pipeline and
applying it to the model. The ``Pipeline`` class depends on the
implementation of the following model specific interfaces which should
be implemented according to the custom DL model:
- ``Engine`` is responsible for model inference and provides
statistical data and accuracy metrics for the model.
- ``DataLoader`` is responsible for the dataset loading, including the
data pre-processing.
- ``Metric`` is responsible for calculating the accuracy metric for the
model.
The diagram below shows relationships between the classes:
.. figure:: https://docs.openvino.ai/2020.4/custom_optimization_pipeline.png
:alt: pot pipeline
pot pipeline
Define DataLoader class
~~~~~~~~~~~~~~~~~~~~~~~
Define ``DataLoader`` based on POT API, as it will be used to collect
statistics for quantization and run model evaluation. Data22Vec model
accepts a raw waveform of the speech signal as input and produces
vocabulary class estimations as output. We already have prepared dataset
above for accuracy measurement. It will serve as data source for
quantization. DataLoader class encapsulates logic for iteration over
dataset samples and gets input data and label by index using
``__getitem__`` method.
Create a quantized model from the pre-trained ``FP16`` model and the
calibration dataset. The optimization process contains the following
steps: 1. Create a Dataset for quantization. 2. Run ``nncf.quantize``
for getting an optimized model. The ``nncf.quantize`` function provides
an interface for model quantization. It requires an instance of the
OpenVINO Model and quantization dataset. Optionally, some additional
parameters for the configuration quantization process (number of samples
for quantization, preset, ignored scope, etc.) can be provided. For more
accurate results, we should keep the operation in the postprocessing
subgraph in floating point precision, using the ``ignored_scope``
parameter. ``advanced_parameters`` can be used to specify advanced
quantization parameters for fine-tuning the quantization algorithm. In
this tutorial we pass range estimator parameters for activations. For
more information see `Tune quantization
parameters <https://docs.openvino.ai/2023.0/basic_quantization_flow.html#tune-quantization-parameters>`__.
3. Serialize OpenVINO IR model using ``openvino.runtime.serialize``
function.
.. code:: ipython3
from openvino.tools.pot import Metric, DataLoader, IEEngine, load_model, save_model, compress_model_weights, create_pipeline
import nncf
from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters
from nncf.quantization.range_estimator import AggregatorType
from nncf.quantization.range_estimator import RangeEstimatorParameters
from nncf.quantization.range_estimator import StatisticsCollectorParameters
from nncf.quantization.range_estimator import StatisticsType
from nncf.parameters import ModelType
class LibriSpeechDataLoader(DataLoader):
def transform_fn(data_item):
"""
Extract the model's input from the data item.
The data item here is the data item that is returned from the data source per iteration.
This function should be passed when the data item cannot be used as model's input.
"""
return np.array(data_item["input_values"])
# Required methods
def __init__(self, dataset, sample_limit=None):
"""Constructor
:param config: data loader specific config
"""
super().__init__({})
self._ds = dataset
self.sample_limit = None
def __len__(self):
"""Returns size of the dataset"""
return self.sample_limit or len(self._ds)
def __getitem__(self, index):
"""
Returns annotation, data and metadata at the specified index.
Possible formats:
(index, annotation), data
(index, annotation), data, metadata
"""
if self.sample_limit is not None and index >= self.sample_limit:
raise StopIteration
sample = self._ds[index]
inputs = {'inputs': np.array(sample['input_values'])}
label = [sample['text']]
return inputs, label
calibration_dataset = nncf.Dataset(dataset, transform_fn)
quantized_model = nncf.quantize(
ov_model,
calibration_dataset,
model_type=ModelType.TRANSFORMER, # specify additional transformer patterns in the model
subset_size=len(dataset),
ignored_scope=nncf.IgnoredScope(
names=[
"/data2vec_audio/encoder/layers.3/feed_forward/intermediate_dense/MatMul",
"/data2vec_audio/feature_extractor/conv_layers.2/conv/Conv",
"/data2vec_audio/encoder/layers.3/Add_1",
"/data2vec_audio/encoder/layers.2/feed_forward/intermediate_dense/MatMul",
"/data2vec_audio/feature_extractor/conv_layers.0/conv/Conv",
"/data2vec_audio/encoder/layers.4/Add_1",
"/data2vec_audio/encoder/layers.4/feed_forward/intermediate_dense/MatMul",
"/data2vec_audio/encoder/layers.4/final_layer_norm/Div",
"/data2vec_audio/encoder/layers.4/feed_forward/output_dense/MatMul",
"/data2vec_audio/encoder/layers.8/attention/MatMul_1",
"/data2vec_audio/feature_extractor/conv_layers.1/conv/Conv",
"/data2vec_audio/encoder/layers.2/Add_1",
"/data2vec_audio/feature_extractor/conv_layers.0/layer_norm/Div",
"/data2vec_audio/encoder/layers.1/feed_forward/intermediate_dense/MatMul",
"/data2vec_audio/encoder/layers.1/Add_1",
"/data2vec_audio/feature_extractor/conv_layers.3/layer_norm/Div"
],
),
advanced_parameters=AdvancedQuantizationParameters(
activations_range_estimator_params=RangeEstimatorParameters(
min=StatisticsCollectorParameters(
statistics_type=StatisticsType.MIN,
aggregator_type=AggregatorType.MIN
),
max=StatisticsCollectorParameters(
statistics_type=StatisticsType.QUANTILE,
aggregator_type=AggregatorType.MEAN,
quantile_outlier_prob=0.0001
),
)
)
)
.. parsed-literal::
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/offline_transformations/__init__.py:10: FutureWarning: The module is private and following namespace `offline_transformations` will be removed in the future, use `openvino.runtime.passes` instead!
warnings.warn(
Define Evaluation Metric class
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
In this step, the ``Metric`` interface for WER metric is implemented. To
make the metric compatible with running inside POT Pipeline, we should
inherit it from ``openvino.tools.pot.Metric`` class and override
following properties and methods: \* ``value`` - returns the accuracy
metric value for the last model output. \* ``avg_value`` - returns the
average accuracy metric value for all model outputs. \* ``attributes`` -
returns a dictionary of metric attributes: ``direction`` - metric
growing direction (``higher-better`` or ``higher-worse``), ``type`` -
type of metric. \* ``update(output, annotation)`` - calculates and
updates the accuracy metric value using last model output and
annotation. \* ``reset()`` - resets collected accuracy metric.
.. code:: ipython3
class WERMetric(Metric):
def __init__(self):
super().__init__()
self._name = "WER"
INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino
INFO:nncf:16 ignored nodes was found by name in the NNCFGraph
INFO:nncf:220 ignored nodes was found by types in the NNCFGraph
INFO:nncf:24 ignored nodes was found by name in the NNCFGraph
INFO:nncf:Not adding activation input quantizer for operation: 2 /data2vec_audio/feature_extractor/conv_layers.0/conv/Conv
INFO:nncf:Not adding activation input quantizer for operation: 4 /data2vec_audio/feature_extractor/conv_layers.0/layer_norm/Div
5 /data2vec_audio/feature_extractor/conv_layers.0/layer_norm/Mul
6 /data2vec_audio/feature_extractor/conv_layers.0/layer_norm/Add_1
def reset(self):
"""
Resets collected matches
"""
self._wer = WordErrorRate()
self._last_result = None
INFO:nncf:Not adding activation input quantizer for operation: 9 /data2vec_audio/feature_extractor/conv_layers.1/conv/Conv
INFO:nncf:Not adding activation input quantizer for operation: 11 /data2vec_audio/feature_extractor/conv_layers.1/layer_norm/Div
12 /data2vec_audio/feature_extractor/conv_layers.1/layer_norm/Mul
13 /data2vec_audio/feature_extractor/conv_layers.1/layer_norm/Add_1
def get_attributes(self):
"""
Returns a dictionary of metric attributes {metric_name: {attribute_name: value}}.
Required attributes: 'direction': 'higher-better' or 'higher-worse'
'type': metric type
"""
return {self._name: {"direction": "higher-worse", "type": "WER"}}
INFO:nncf:Not adding activation input quantizer for operation: 16 /data2vec_audio/feature_extractor/conv_layers.2/conv/Conv
INFO:nncf:Not adding activation input quantizer for operation: 18 /data2vec_audio/feature_extractor/conv_layers.2/layer_norm/Div
19 /data2vec_audio/feature_extractor/conv_layers.2/layer_norm/Mul
20 /data2vec_audio/feature_extractor/conv_layers.2/layer_norm/Add_1
@property
def value(self):
"""Returns accuracy metric value for the last model output."""
return {self._name: self._last_result if self._last_result is not None else self._wer.compute().item()}
INFO:nncf:Not adding activation input quantizer for operation: 25 /data2vec_audio/feature_extractor/conv_layers.3/layer_norm/Div
26 /data2vec_audio/feature_extractor/conv_layers.3/layer_norm/Mul
27 /data2vec_audio/feature_extractor/conv_layers.3/layer_norm/Add_1
@property
def avg_value(self):
"""Returns accuracy metric value for all model outputs."""
return {self._name: self._wer.compute().item()}
INFO:nncf:Not adding activation input quantizer for operation: 32 /data2vec_audio/feature_extractor/conv_layers.4/layer_norm/Div
33 /data2vec_audio/feature_extractor/conv_layers.4/layer_norm/Mul
34 /data2vec_audio/feature_extractor/conv_layers.4/layer_norm/Add_1
def update(self, output, target):
"""
Updates prediction matches.
INFO:nncf:Not adding activation input quantizer for operation: 39 /data2vec_audio/feature_extractor/conv_layers.5/layer_norm/Div
40 /data2vec_audio/feature_extractor/conv_layers.5/layer_norm/Mul
41 /data2vec_audio/feature_extractor/conv_layers.5/layer_norm/Add_1
:param output: model output
:param target: annotations
"""
res = output[0]
predicted_ids = np.argmax(res, axis=-1)
predicted_transcription = processor.batch_decode(torch.from_numpy(predicted_ids))
res = []
for pred, gt in zip(predicted_transcription, target):
res.append(self._wer.forward([pred], gt).item())
self._last_result = res
return res
Define quantization configuration and optimization pipeline
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The code below defines a configuration for the quantization pipeline and
runs it. To keep example minimalistic, built-in ``IEEngine``
implementation of ``Engine`` interface from the POT API for model
inference is used here. We will use DefaultQuantization algorithm with
``performance`` preset and additional specification of quantization
algorithm for activations. For information about configuration
parameters, refer to `POT
documentation <https://docs.openvino.ai/latest/pot_compression_algorithms_quantization_default_README.html>`__.
Our model architecture is transformer-based, so
``model_type: transformer`` should be selected. For better accuracy,
part of layers should be kept in floating point representation using
``ignored`` parameter. The ignored layers can be selected using
`AccuracyAwareQuantization <https://docs.openvino.ai/latest/pot_accuracyaware_usage.html>`__
algorithm, which aim to find layers that have the most significant
impact on accuracy drop and revert them back to floating point
precision. This process can be time consuming, that is why we keep this
experiment out of this tutorial and reuse its result, using
DefaultQuantization algorithm. > **NOTE**: Consider increasing
``stat_subset_size`` to get more precise results. A suggested value is
``300`` or more, as it will take longer time to process.
.. code:: ipython3
model_config = {"model_name": "data2vec_base", "model": ir_model_path, "weights": ir_model_path.with_suffix(".bin")}
INFO:nncf:Not adding activation input quantizer for operation: 46 /data2vec_audio/feature_extractor/conv_layers.6/layer_norm/Div
47 /data2vec_audio/feature_extractor/conv_layers.6/layer_norm/Mul
48 /data2vec_audio/feature_extractor/conv_layers.6/layer_norm/Add_1
engine_config = {"device": "CPU"}
INFO:nncf:Not adding activation input quantizer for operation: 52 /data2vec_audio/feature_projection/layer_norm/Div
53 /data2vec_audio/feature_projection/layer_norm/Mul
54 /data2vec_audio/feature_projection/layer_norm/Add_1
algorithms = [
{
"name": "DefaultQuantization",
"params": {
"target_device": "ANY",
"model_type": "transformer",
"preset": "performance",
"stat_subset_size": 300,
"activations": {
"range_estimator": {
"min": {
"aggregator": "min",
"type": "min"
},
"max": {
"aggregator": "mean",
"type": "quantile",
"outlier_prob": 0.0001
}
}
},
"ignored": {
"scope": [
"/data2vec_audio/encoder/layers.3/feed_forward/intermediate_dense/MatMul",
"/data2vec_audio/feature_extractor/conv_layers.2/conv/Conv",
"/data2vec_audio/encoder/layers.3/Add_1",
"/data2vec_audio/encoder/layers.2/feed_forward/intermediate_dense/MatMul",
"/data2vec_audio/feature_extractor/conv_layers.0/conv/Conv",
"/data2vec_audio/encoder/layers.4/Add_1",
"/data2vec_audio/encoder/layers.4/feed_forward/intermediate_dense/MatMul",
"/data2vec_audio/encoder/layers.4/final_layer_norm/Div",
"/data2vec_audio/encoder/layers.4/feed_forward/output_dense/MatMul",
"/data2vec_audio/encoder/layers.8/attention/MatMul_1",
"/data2vec_audio/feature_extractor/conv_layers.1/conv/Conv",
"/data2vec_audio/encoder/layers.2/Add_1",
"/data2vec_audio/feature_extractor/conv_layers.0/layer_norm/Div",
"/data2vec_audio/encoder/layers.1/feed_forward/intermediate_dense/MatMul",
"/data2vec_audio/encoder/layers.1/Add_1",
"/data2vec_audio/feature_extractor/conv_layers.3/layer_norm/Div"
]
}
}
}
]
INFO:nncf:Not adding activation input quantizer for operation: 70 /data2vec_audio/encoder/pos_conv_embed/layers.0/layer_norm/Div
INFO:nncf:Not adding activation input quantizer for operation: 124 /data2vec_audio/encoder/pos_conv_embed/layers.1/layer_norm/Div
INFO:nncf:Not adding activation input quantizer for operation: 178 /data2vec_audio/encoder/pos_conv_embed/layers.2/layer_norm/Div
INFO:nncf:Not adding activation input quantizer for operation: 221 /data2vec_audio/encoder/pos_conv_embed/layers.3/layer_norm/Div
INFO:nncf:Not adding activation input quantizer for operation: 262 /data2vec_audio/encoder/pos_conv_embed/layers.4/layer_norm/Div
INFO:nncf:Not adding activation input quantizer for operation: 57 /data2vec_audio/encoder/Add
INFO:nncf:Not adding activation input quantizer for operation: 59 /data2vec_audio/encoder/layer_norm/Div
61 /data2vec_audio/encoder/layer_norm/Mul
63 /data2vec_audio/encoder/layer_norm/Add_1
# Step 1: Load the model.
model = load_model(model_config=model_config)
INFO:nncf:Not adding activation input quantizer for operation: 65 /data2vec_audio/encoder/layers.0/Add
INFO:nncf:Not adding activation input quantizer for operation: 71 /data2vec_audio/encoder/layers.0/layer_norm/Div
83 /data2vec_audio/encoder/layers.0/layer_norm/Mul
94 /data2vec_audio/encoder/layers.0/layer_norm/Add_1
# Step 2: Initialize the data loader.
data_loader = LibriSpeechDataLoader(dataset)
INFO:nncf:Not adding activation input quantizer for operation: 104 /data2vec_audio/encoder/layers.0/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 115 /data2vec_audio/encoder/layers.0/final_layer_norm/Div
121 /data2vec_audio/encoder/layers.0/final_layer_norm/Mul
125 /data2vec_audio/encoder/layers.0/final_layer_norm/Add_1
# Step 3 (Optional. Required for AccuracyAwareQuantization): Initialize the metric.
metric = WERMetric()
INFO:nncf:Not adding activation input quantizer for operation: 128 /data2vec_audio/encoder/layers.1/Add
INFO:nncf:Not adding activation input quantizer for operation: 135 /data2vec_audio/encoder/layers.1/layer_norm/Div
147 /data2vec_audio/encoder/layers.1/layer_norm/Mul
158 /data2vec_audio/encoder/layers.1/layer_norm/Add_1
# Step 4: Initialize the engine for metric calculation and statistics collection.
engine = IEEngine(config=engine_config, data_loader=data_loader, metric=metric)
INFO:nncf:Not adding activation input quantizer for operation: 169 /data2vec_audio/encoder/layers.1/feed_forward/intermediate_dense/MatMul
180 /data2vec_audio/encoder/layers.1/feed_forward/intermediate_dense/Add
# Step 5: Create a pipeline of compression algorithms.
pipeline = create_pipeline(algo_config=algorithms, engine=engine)
Run model quantization
----------------------
Now, when all parts of compression pipeline are collected, we can start
quantization. >\ **NOTE**: quantization process is time and memory
consuming. It may takes several minutes depending on your hardware
configuration.
.. code:: ipython3
import time
INFO:nncf:Not adding activation input quantizer for operation: 168 /data2vec_audio/encoder/layers.1/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 179 /data2vec_audio/encoder/layers.1/final_layer_norm/Div
185 /data2vec_audio/encoder/layers.1/final_layer_norm/Mul
189 /data2vec_audio/encoder/layers.1/final_layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 192 /data2vec_audio/encoder/layers.2/Add
INFO:nncf:Not adding activation input quantizer for operation: 199 /data2vec_audio/encoder/layers.2/layer_norm/Div
211 /data2vec_audio/encoder/layers.2/layer_norm/Mul
222 /data2vec_audio/encoder/layers.2/layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 233 /data2vec_audio/encoder/layers.2/feed_forward/intermediate_dense/MatMul
244 /data2vec_audio/encoder/layers.2/feed_forward/intermediate_dense/Add
INFO:nncf:Not adding activation input quantizer for operation: 232 /data2vec_audio/encoder/layers.2/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 243 /data2vec_audio/encoder/layers.2/final_layer_norm/Div
249 /data2vec_audio/encoder/layers.2/final_layer_norm/Mul
253 /data2vec_audio/encoder/layers.2/final_layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 256 /data2vec_audio/encoder/layers.3/Add
INFO:nncf:Not adding activation input quantizer for operation: 263 /data2vec_audio/encoder/layers.3/layer_norm/Div
275 /data2vec_audio/encoder/layers.3/layer_norm/Mul
286 /data2vec_audio/encoder/layers.3/layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 297 /data2vec_audio/encoder/layers.3/feed_forward/intermediate_dense/MatMul
307 /data2vec_audio/encoder/layers.3/feed_forward/intermediate_dense/Add
INFO:nncf:Not adding activation input quantizer for operation: 296 /data2vec_audio/encoder/layers.3/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 306 /data2vec_audio/encoder/layers.3/final_layer_norm/Div
311 /data2vec_audio/encoder/layers.3/final_layer_norm/Mul
314 /data2vec_audio/encoder/layers.3/final_layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 316 /data2vec_audio/encoder/layers.4/Add
INFO:nncf:Not adding activation input quantizer for operation: 322 /data2vec_audio/encoder/layers.4/layer_norm/Div
333 /data2vec_audio/encoder/layers.4/layer_norm/Mul
343 /data2vec_audio/encoder/layers.4/layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 353 /data2vec_audio/encoder/layers.4/feed_forward/intermediate_dense/MatMul
363 /data2vec_audio/encoder/layers.4/feed_forward/intermediate_dense/Add
INFO:nncf:Not adding activation input quantizer for operation: 371 /data2vec_audio/encoder/layers.4/feed_forward/output_dense/MatMul
377 /data2vec_audio/encoder/layers.4/feed_forward/output_dense/Add
INFO:nncf:Not adding activation input quantizer for operation: 352 /data2vec_audio/encoder/layers.4/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 362 /data2vec_audio/encoder/layers.4/final_layer_norm/Div
367 /data2vec_audio/encoder/layers.4/final_layer_norm/Mul
370 /data2vec_audio/encoder/layers.4/final_layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 372 /data2vec_audio/encoder/layers.5/Add
INFO:nncf:Not adding activation input quantizer for operation: 378 /data2vec_audio/encoder/layers.5/layer_norm/Div
389 /data2vec_audio/encoder/layers.5/layer_norm/Mul
399 /data2vec_audio/encoder/layers.5/layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 408 /data2vec_audio/encoder/layers.5/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 418 /data2vec_audio/encoder/layers.5/final_layer_norm/Div
423 /data2vec_audio/encoder/layers.5/final_layer_norm/Mul
426 /data2vec_audio/encoder/layers.5/final_layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 428 /data2vec_audio/encoder/layers.6/Add
INFO:nncf:Not adding activation input quantizer for operation: 434 /data2vec_audio/encoder/layers.6/layer_norm/Div
445 /data2vec_audio/encoder/layers.6/layer_norm/Mul
455 /data2vec_audio/encoder/layers.6/layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 464 /data2vec_audio/encoder/layers.6/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 474 /data2vec_audio/encoder/layers.6/final_layer_norm/Div
479 /data2vec_audio/encoder/layers.6/final_layer_norm/Mul
482 /data2vec_audio/encoder/layers.6/final_layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 484 /data2vec_audio/encoder/layers.7/Add
INFO:nncf:Not adding activation input quantizer for operation: 490 /data2vec_audio/encoder/layers.7/layer_norm/Div
501 /data2vec_audio/encoder/layers.7/layer_norm/Mul
511 /data2vec_audio/encoder/layers.7/layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 520 /data2vec_audio/encoder/layers.7/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 530 /data2vec_audio/encoder/layers.7/final_layer_norm/Div
535 /data2vec_audio/encoder/layers.7/final_layer_norm/Mul
538 /data2vec_audio/encoder/layers.7/final_layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 540 /data2vec_audio/encoder/layers.8/Add
INFO:nncf:Not adding activation input quantizer for operation: 546 /data2vec_audio/encoder/layers.8/layer_norm/Div
557 /data2vec_audio/encoder/layers.8/layer_norm/Mul
567 /data2vec_audio/encoder/layers.8/layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 576 /data2vec_audio/encoder/layers.8/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 586 /data2vec_audio/encoder/layers.8/final_layer_norm/Div
591 /data2vec_audio/encoder/layers.8/final_layer_norm/Mul
594 /data2vec_audio/encoder/layers.8/final_layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 596 /data2vec_audio/encoder/layers.9/Add
INFO:nncf:Not adding activation input quantizer for operation: 602 /data2vec_audio/encoder/layers.9/layer_norm/Div
613 /data2vec_audio/encoder/layers.9/layer_norm/Mul
623 /data2vec_audio/encoder/layers.9/layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 632 /data2vec_audio/encoder/layers.9/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 642 /data2vec_audio/encoder/layers.9/final_layer_norm/Div
647 /data2vec_audio/encoder/layers.9/final_layer_norm/Mul
650 /data2vec_audio/encoder/layers.9/final_layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 652 /data2vec_audio/encoder/layers.10/Add
INFO:nncf:Not adding activation input quantizer for operation: 658 /data2vec_audio/encoder/layers.10/layer_norm/Div
669 /data2vec_audio/encoder/layers.10/layer_norm/Mul
679 /data2vec_audio/encoder/layers.10/layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 688 /data2vec_audio/encoder/layers.10/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 698 /data2vec_audio/encoder/layers.10/final_layer_norm/Div
703 /data2vec_audio/encoder/layers.10/final_layer_norm/Mul
706 /data2vec_audio/encoder/layers.10/final_layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 708 /data2vec_audio/encoder/layers.11/Add
INFO:nncf:Not adding activation input quantizer for operation: 714 /data2vec_audio/encoder/layers.11/layer_norm/Div
725 /data2vec_audio/encoder/layers.11/layer_norm/Mul
735 /data2vec_audio/encoder/layers.11/layer_norm/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 744 /data2vec_audio/encoder/layers.11/Add_1
INFO:nncf:Not adding activation input quantizer for operation: 754 /data2vec_audio/encoder/layers.11/final_layer_norm/Div
759 /data2vec_audio/encoder/layers.11/final_layer_norm/Mul
762 /data2vec_audio/encoder/layers.11/final_layer_norm/Add_1
# Step 6: Run compression pipeline
print(f"Quantizing model with {algorithms[0]['params']['preset']} preset and {algorithms[0]['name']}")
start_time = time.perf_counter()
compressed_model = pipeline.run(model=model)
end_time = time.perf_counter()
print(f"Quantization finished in {end_time - start_time:.2f} seconds")
.. parsed-literal::
Quantizing model with performance preset and DefaultQuantization
Quantization finished in 112.66 seconds
Statistics collection: 100%|██████████| 73/73 [00:37<00:00, 1.93it/s]
Biases correction: 100%|██████████| 74/74 [00:16<00:00, 4.60it/s]
After quantization is finished, compressed model representation can be
saved using ``save_model`` function.
saved using ``serialize`` function.
.. code:: ipython3
# Step 7 (Optional): Compress model weights to quantized precision
# in order to reduce the size of the final .bin file.
compress_model_weights(model=compressed_model)
# Step 8: Save the compressed model to the desired path.
compressed_model_paths = save_model(model=compressed_model, save_path=MODEL_DIR, model_name="quantized_data2vec_base")
compressed_model_path = compressed_model_paths[0]["model"]
MODEL_NAME = 'quantized_data2vec_base'
quantized_model_path = Path(f"{MODEL_NAME}_openvino_model/{MODEL_NAME}_quantized.xml")
serialize(quantized_model, str(quantized_model_path))
Check INT8 model inference result
---------------------------------
@@ -604,8 +609,7 @@ using ``core.compile_model``. After that, we can reuse the same
.. code:: ipython3
ov_int8_model = core.read_model(compressed_model_path)
int8_compiled_model = core.compile_model(ov_int8_model)
int8_compiled_model = core.compile_model(quantized_model)
.. code:: ipython3
@@ -617,8 +621,8 @@ using ``core.compile_model``. After that, we can reuse the same
.. parsed-literal::
[Reference]: A MAN SAID TO THE UNIVERSE SIR I EXIST
[OpenVINO INT8]: A MAN SAID TO THE UNIVERSE SIR I EXIST
[Reference]: MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL
[OpenVINO INT8]: MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL
@@ -627,7 +631,7 @@ using ``core.compile_model``. After that, we can reuse the same
<audio controls="controls" >
<source src="data:audio/wav;base64,UklGRmRFAgBXQVZFZm10IBAAAAABAAEAgD4AAAB9AAACABAAZGF0YUBFAgDn//f/uf/6/9v/FgBCAGAAnADq/+r/LwD3/67/JQDYAEoA3v8MAOn/xf8HAIYA7P+M/xoALQDG/6v/hAB8AKr/lP/3/ywALQDDAFkA0//s/w8A9P/8/44AcAD8/zIAPQAMACEAKgC7/1D/q/8hACEALQBzAHMAAwC4/+T/w/+J/97/PwBgAOb/ZQBOAMb/gf+u/1MABADZ/9T/1//h/8D/jf/O/6r/pf/e/7D/4//f/+b/fP8P/7b/GQD1/7D/tv9WAMn/j//b/9v/wf++/+T/j//9/2wA9f9L/2//0f/G/1b/rf+HAD8Avf/D/xQA2//e/x0AZ/9x/z0ALAAWAMn/HwC2/8j/bv9y/48AbgDx/03/RwD0/4L/rf+2/3L/hf9AAPf/5/+UAF0APf8c/7b/y/+o////AQAqAC0AAwDT/9f/uf8y/3b/mv8XAIMAWQAHAMD/HQCy/3T/h/8SAJIAQgD3/xwAYADu/7v/oP/3//z/wf/c/8n/UAAiAA8A5/9f/9//XQD5/9b/PwB+ALn/gv/1/xIA9P84AC8Atf80AD0ABABZ//n/WwDc//L/DgA1AOT/AQDR/4H/n//8/+7/vf/1/14ALQCM/6X/zv/p/wwAKgAUAC8A6v/Q/7L/TgBNAA8AFADh//H/QAAfAND/4f8kADUAlP8sAGkAHQDy/0AAfwAvAAcAJADk/wYAmQAsAGEAIgBDANn/AwB+ACIAawAhAOH/9P8iADIA6f/n/w4AKQC6AKcARQASANv/3P9//9n/bgB5AHcAhABKAOT/xv8RAHn/ef9FAJQAYQBOAI4ALQCo/6P/kv/J/yoARwCZAF4ATgD6//T/+f/U/wkA1/8XADgAOgBHAD8ABwCB/1z/2f/3/1IAKgAfADQAPQDj/67/8v8AAMb/4f9NAH8AUAD6/8n/cf+y//n/AwDu/wQAeQD5/1b/tf8sABIAif/X//L/+v/0/+//8f8dABwAq/+g/+n/MAAwAPr/4/8LAAcAov+i/wEACwAAAO7/9//m//n/pf9j/9z/EQAOAPT/RQB8ABYAxf+X//f/HQAXAIYAdwBLACoAOwD1/4r/DgAkALL/LwCiAGMA0f8GACkAaf/u/1gAMgBwAKUAqADT/4r/IQDn/7D/bv/q/zgAvf98AHsAyf+g/5X/UP81/xQApQARAEgAcQBOAOf/bv+u/5//3/8MAFIAsgBNAEoAHQBx/2f/gf/8/+z/7/+cAF0A0P/e//H/DgD5/xkA/f98/yIAdwAZABIAXgC9ABkAkv8nAIwA/f9B/6D/WAAwAGgAwwBeACwACQCb/y//Sf8WACUABACVAHkBMAHk/63/oP/z/g//WAC7AHcA/gAyAWsAqP/J////nf+N/x8A2wDFAHsA0wBhAOT/qv/A/9H/yP+iAJkATgDX/xoADAAX/x3/hgAJATsAMgBYAPL/mP8fAL7/rv8PAEcA1//u/80AhgDT/zP/Wf+4/+H/MAAtAFsAlwA/ALP/Rv8BAB0Akv/U/3cAzgCHACUAgv9y/+H/4f9W/wkA7ACSAK7/lP8ZALn/OP9m/9v/KQBsAIEAYADh/0oAFgAY/wr/SADxAL7/xf/VAIwAbv/D/ywAd/8+//z/GgCf/2wA6AD1/4//GgD9/yP/mP/CABEAn/8lAIoA2f+u/6oALQDn/8X/yP9J/2n///8lACIAWQA4AOH/y//J/73/cf/e/wwAHQAiACEAJwA3AF0Ay/9k/63/+v+J/4T//f9rAAEAcv/B/9D/5/92/6r/zP+a//3/FgDI/5//NQBHAHn/gv9HAMv/bP9s/9z/AAD5/2UA/f+r/+b/8f+K/2b/6v9YAIX/tv9YAGMAAQA4/7b/5P8aABYA4/9oAFsAEgCU/7L/WwBjAMb/u/8RAIwAVgCf/9v/7P8EAIz/Xv9SAN0AVgDD/8n/aAA0AIT/9/9gAKgAMACr/xwAQAAwANf/zv9sAFYAMgDx/9v/aAApAIn/Jf/u/5IAJQAcAJUAlAAPAJT/3v9SAPT/0P+q/+f/l//y/2MAFAA/AGMAHQCU/wYAOgDD/3//HwDX/5T/LwCXAJUANwB/AAAAs/+l/3H/tv8wAGMAQABHACQAHQD6/yUA1//G/6L/Tf/R/2MAvwCnAHkAQgDA/6v/zv8ZAP//7P8vADAAQgBdADAA8v8EAFYAEgCr/yIAWQD//9v/WACGADIAFAAtABQAHwBbAPL/jP8LACkADABmAMsAugA6AI4ARQDe/+b//P8kABIANACnAOMAUAA9AAAA2/82/5v/YwBdAM0AvQCUAPr/zv/k/5X/n//x/yIAOwAUAD8ANADZ/77/ov+r/43/AQA7AOT/6v9IAJr/Tf95/+f/rv/9/6EA2//b/ycAyP/C/k3/NAD5/97/iQCJAPn/5P9k/w//Ov8fAPX/Yf8cALAAXgDf/5X/oP9O/5X/tf9s/14AxgBhABIARQBgAGH/Kv+l/8D/SgBKAAMABABTAIcAuP+V/zcA+v+Q/5T/TgCnAFUA//+V/8P/TgD8/7b/OACqAF0Ao//3/xQA8v/O/8P/tv/1/5oAlQDq/wEARwCj/x3/j/86ANb/8f9HACIAKQD1/0cABACQ/5D/jP8JAP//IQB7AAsA3P+E/4f/yf+q/+n/GQA9AAAAwP93AGMAkP8j/4r/zP+9/x8AvwClAF0ADgC1/7P/2/9LAL3/zP98AHsADwADAIcABgBc/33/7v/b/x8AHwAUAPf/NwBxANn/BwBKABwAhP+o/zgAbgAHAOP/9/9pAIcA2//c/yUAKQDG/2P/FgCqAE0A3P/Z/z0AFgABAAQA3P8DAFYACQCw/0UAhAAhALn/MgAhAP//GgBOAAsA4f8yAEMA8v/6/3sAJADT/97/RQA9ACoATQAJAKX/yf8LALv/w/9mAH8AsP+S/97/JADu/9v/EQD5/w4A/P8WAD0AIgASAKv/a/++/y0AJQDO/yQAYQDe/5r/0f/L/9n/NQAqAKL/9P+vACIASP+o/2MA6v89/6j/lwA/ANf/DAA4ACIA+f8fAAcA9/9AAPT/nf8lAIcADABm/9z/FgB6/1z/q//O/7j/RwCSABwACwD9/8P/VP99/xcACwAPAF0AhABHAA4AJACd/97+Xv/5/xkAKQCiAI8Arf+C/43/qP+U/9D/7P8PACwAKQA6ABcAAQCJ/6j/6f9DAHAAWAD0/6f/0//b/+T/7P9CAL7/sP+Y/+H/GgDb/+H/tf/e/7v/KQCcAEcArf+2/7P/Zv+f/00AjADs/9//y//h/wYA/f/Z/8b/9f/0/wEALABLAB8A1v+B/9H/KQAMAAcACwBIADcA7P/n/wQAMgADANb/aACOAPz/5//5/67/rf/6/+r/mv8tAGUAu/8WALIANACd/+b/NwDZ//n/bAAZAOH/SwAqANT/6f+yAF0Aiv/M/0UAFgCU/+b/XQBLAAsA8v9AAGgA+v+U//L/PQD3/7b/UACqAFkAXgBHANT/1/93ACoAoP8hAI8AJwDj/4kAaQDc/9n/CwDW/8z/SACBACkALQCvAHQAAQC1/wQA9/+r//r/YQCqACkAPQBQAE4A1P+P/+P/2f+1/yEAlAAXABIASwBzAMv/xf8UALD/5P9eAF0A5/8DAP3/8f/A/z8AEgDf/yEAJwBhACwATgAAAMH/gf/c/xcAVgBKAEIAhwDM/5f/w/8sAL3/zP8UADoA/P89AGYA3/8RAA4A4f+X/1UAzQBpAAYAXgCDAPr/vf/L/+f/wP/8/wsAKgBdADoAoP9N/w4AQwDJ/8H/JABsACQA7//Q/xEAZgADAJj/BAB8ADAAyP/m/zAA1P/y/6f/tf9FADsADACb/wYAAQC7/6L/1v8RAC8AFgD1/xoANQA7AJX/4/9SAEIAov+n//H/8f/8//n/1//W/0cA4f9y/8j/MgDJ/6X/zP8UACEAMgDv/3//+f8JAG//SP/c/00A/P/p/3wAWAADALv/tf/Q/5j/u//y/+b/AwBoADQAp//A/yUAy/+J/yUANwC+/8j/RwD0/53/3P8PAO//4/8tAAcANQAvANb/mv/T/yIA9P+5/xEAWQAlAAsADwA0APf/tf/B//z/JQAXANH/IQAvANn/DADn/wsA3//T/6r/6f8iALb/qv/u/1AA8f/f/7b/7P/n/63/2f/v/zoA5v8BAOb/JwADALj/u//I/0oADAD5/+r/awAhAND/zv/3/xIA0P8wACQAWQDy/x0A8f/9//r/DgAwAIf/BwBzAIQApf/k/yUA0P+S//r/XgDW/9b/zP8HAOn/NwAHAKD/vf8fABEAXP8BACIA2f9//xQAlwDy/5//FABlAO//xf/O/xkA7v9TAFIA1v8nACcArv8q/7v/RwAkAAEAYQBmAHAAFgCw/97/CwAyAOb/NwCoAGMASgBDAFUAAADF//T/DAA9AHwAhAAqAB8AJADj/4z/3v89AMH/6f+BALAA6f+K/yIALQDb/8n/HwDm/63//f9NANv/sP8lAPr/Yf+n/4QA7v9I/6L/JACu/4r/5P/A/5X/zP/s/6r/BgApAO//vf/b/yoAGQD//+r/HAA4AOb/zv8HAOP/u/+a/6j/EQA6ABkA6f8cABcA6v/b/8z/w//9/87/w/9HANYAcADR/wAA7P+K/0b/qv/0/xIASgBOAAkAHQAlAKj/X/93/9D/6v8BAGgAcABmADAAqv+V/63/QgAcAJj/GgB+AFgAIgD//+f/oP+r/xwADwCGAO4AfwC+/73/MAAqAJr/1P9oAHcAcABpACkA3v/X/6j/gv/A/2YA9P/f/10AyABOALL/zv/j/6f/dP9HAKEAhwCb/63/JwAJAA8A0/8dANb/uf+q/wAA9P84ACUA0f/s/xEAWQDF/6P/0P8XABQAQwAvLine truncated
<source src="data:audio/wav;base64,UklGRgTcAgBXQVZFZm10IBAAAAABAAEAgD4AAAB9AAACABAAZGF0YeDbAgDJAK8ApwCxAIgAGQAHABwAUgCaAKoAsQCSACYA3/8mAFUABwApAGEAPQBPAJgAmgBKACQAKQA9AFwAsQDYANUA2ADBAJgAWgA2AAoAFwBhAIAAgwCiAGcA/v/i/9//3//0/zgAbACdAIYAbAA9AAcA0P/a/wcAHAAUAF8AcwBsAEoAGQC2/4X/u/8KAGQAogCLAEoAQAAkANX/0v/i/9z/8f8wAE8APQBVADsAFADx/8r/tv/X/xcAPQCaAJgAaQAzAOL/lP+z/8j/wP8HADgACgA2AHEAHgAAABwADwAHAEUAVQA7ABkADwACALH/j/+7/7b/q/8kAEUAEgAMACEA6f/f/wIAFwAXABwAHgAuACEAAgD7/xcA+//7/ykAKwArABcA/v/K/9f/7v8KAAAA5P+k/6b/5P8MAPT/vf/D/5z/lP/X/+z/AgBVAFUAJgAhAPH/0P/X/9D/pP/D//T/vf+p/8P/eP94/3D/T/9e/5L/xf/x/1wAHgAFAAcAMwDi/6H/u//F/+n/CgAwAAUA0v+4/5//l/+k/9D/AgAPADsAIQDX/73/yv+4/7H/0v/A/9//BwAUADgAMwDa/43/rv/a/wIAQwBSACYABQDc/+f/0P/N//7//v/S/+f//v8HAE8AiwBFAAcA/v/i/9//MAA2AO7/3P/N/8j/9v8hAEMAUgA2AB4AHABIAE8AQAA4ADgAQAAcAAUABQD5/7v/yv8PAPb/9P8eAE0AaQCDAHYAIQAKAAoAEgAmAE8AUgAuABkAMwBKAEMAKwAAAPb/DwDp/yQArwCSADMAQAA2ANL/5/8HAAAAMAAuAOL/BQBKABcAVwDQAEUAyP/Q/9//4v8XABcAMwBsAGQAnwCaAD0Akv9j/43/n/8eAJUApwCLAFcAFADp/7j/iv96/8P/AgBFAIAArABhAPv/nP9z/5f/1f9fAMsAqgA7AAwA5/+Z/5n/DwA2ACsASgBxAEgA7v/c/6n/Xv9h/5L/9v8ZAB4AGQAMAN//gP96/4X/yP8KAFoAcwBfAPT/gP9c/x7/Mv+N//H/AAA7AC4A7v/N/5L/Ov81/5T/2v9PAIYAQwAAAPb/pv+K/9z/2v/X/zgAiABPAC4AGQDA/2j/T/+U/+L/+f8uADsA+f+U/43/n/+C/6n/CgBIAEUAJgAZANX/tv+c/7b/wP/i/9r/5P8PAOf/wP+H/0z/Qv/V/ysAVwCNACkAof+F/5n/q//I//v/EgAUABkAJAAXAMP/ev9w/33/qf9VAMEAiwApAAUAxf9w/43/yv/X/wIANgB5AIgAHgCu/5T/hf9Z/8j/JAAXAAcAFAAcABIAEgAPAAoAzf+2//T/AgAFADgARQDp/+T/HgD0/9z/GQDf/67/8f87AEMAMwBsAE0AKwAcAEgALgAXAAcAEgAeAFIAeQBsABcA8f8MADgAXwCQAG4AKwAcAEUApwC3ANAArwBkAPT/7P8uAFwAiwCfAJoAWgBXAHsAtwCYADMANgBfAC4AXAC0AIYA/v/p//v/AAASAEgAbABNAC4ADwA9AGkAfgBFACQAEgCK/4L/DAArAOz/MwBaADAAAgAcACQA7v/7/xIAOwBSAFcAzgCxAE8AQAACAI//gv/u/wIACgBVAGQAFAACAAoAxf/F//T/yv/u/xwAPQBcAJgAeQAwAPb/hf99/+T/HgAUAEoA+f9Z/4r/9P/i//T/BQCp/2j/0P8UAE8AgwBDAOf/OABNAPv/+f8AAOT/vf8CAA8A0P/S/83/BQAcAOT//v84ACYA0P/S/8X/nP/A//v/QABSAFwALgAUAOn/pP+S/9r/JAAPAAAAAADa/5L/kv/Q/4D/ff/c/wcA0v/i/9L/ev9W/4f/h/+N/9r/yP+U/7H/yv+Z/8X/xf+u/43/pP+7/8r/yv+A/4f/uP/p/9//AgC4/17/Vv+u/+T/+//n/+f/5P+U/2H/l//D/43/qf/X/9r/rv8wAGwACgDf/9X/xf+x/+L/AgDi/67/zf/n//H/BQAFAO7/sf96/5T/9P82AE0ATQArANr/s//k/83/0v+9/4f/jf/I/83/3P8cAAcA7P/c//v/6f/5/wwAHABKAHEAQwA2ADAA0v/X/xQAJgA7AC4A+//n//b/JgA2AFIALgDX/73/yv/n/yYAOwBVAJAAQwAmAG4AewCAAEgADADN/9L/MwC8AO8AqgAuAOL/7v/5/0AAUgAcAPv/9P/p/wIAOwA9ABkA8f/u/6n/5P8UADgAXABVAOn/3/8HAMP/5/9hAGcA2v+4//T/rv+2/0MAZwAFAOn/MwArADAAXwBaABQA9v/x//T/+/8HABQACgDk//7/GQAUADgAQwAkABIAPQA7AAoAHgBAAPb/FwBfAAUA3/8SACQAFwA2AA8A1/+c/7H/xf8HAE8AMAAMANf/yv+4/9D/NgBPAAAA6f/u//n/BwAPABQA0v/D/6v/w//s/xkAIQA2ABQA4v+x/9D/4v/K/9z/DwDx/8j/FwBVADsAAABDAC4ADAAzAGQARQAUAC4AUgA4AP7/7P/K/+7/s//n/wwAEgDi/+T/FAC9/4f/sf/X/4D/Xv9m/8X/zf/N/wUA5/+K/5T/3P/S/6H/qf/F/6H/q//N/+L/0v/f/+L/7v+7/9L/AgDp/+n/9v/7//b/IQAKAAIABwAkADAASgA2ACkA0v/D/9X/QwBhAIYAlQACAKT/pP/A/+n/XwCiAG4AAgA9AMr/RQAkAAcAKQDc/wAApACNACEA+/8XABkA1//K//v/KwBhALQAsQBnAMr/3//k/8P/xf82AGcAZwB7AGcAgwBNABkABwDs/9z/OACsAH4AbgBnAEMAOABuAK8AiACVAJAAfgBzAIMApAB+ADYAEgA7ACEAbADdALcAaQAuACYAIQBSAKIAuQCnAD0AEgBAAFwADwA4AHYARQAhAGEAiACSAEMA+f/0/9f/2v9KALwAewD2/xIAKwDa/+L/AAD7/+n/FACGAJ8AfgBXAD0A1f+r/+z/NgBIAF8AUgAuADMAcQBpABwA9v/0/zMAWgBsAF8AIQD+/+n/AAACAOn/DwBSAEoA+f/u/wAA3P/S/w8AMwDa/9L/9v/+/8r/yP/f/7j/q/+U/8r/GQA7APT/0P+c/z//Xv/F/w8AFwDp/7j/hf9o/2j/s//u/67/qf/D/6v/jf+m/7H/qf+X/5T/yv/A/1n/Vv+r/5n/ff/F/8D/tv/f/xkAAADx//v/0P/7/+n/5//c/7b/vf/F/67/lP+f/4L/Wf+S/6H/mf+K/6n/nP+F/33/rv+4/3D/Xv96/6n/cP+Z/8r/kv9e/33/vf+4/4X/aP+X/6v/sf/D/7v/pP9z/33/l/+2/9r/2v/n/9X/l/9h/3P/mf/s//b//v8rABkA4v8HABkA5P/a/9r/wP/S/xIAFAAMAOf/nP9r/5n/rv/s/ysAAADF/8j/ff9U/4f/qf+U/5f/3P/7/yEADwDK/7j/n/+k/xcAXAAuACQALgDx/7j/1f/7//H/BwBDAFcATQBFABkA7P+Z/6T/8f84AH4AqgCdAC4AKwAkADAADwBAAF8AQAB5ANAA6gCiADgA8f/N/+7/TwCfAMMAhgAuAC4ALgAPAFwAXwAeAA8ATQBpAHsAnQBsABIAzf/K/wUAcwCfAIsAeQA2AAAABQAkADgAFAAAAOL/HgAzAEgARQAwACQA3//i/wAAQwAzAEUAQwAhAPv/HABSACQABwAAADAAQABDAE0AGQDx//n/7v8hAE0AXABVADYALgAmAA8AEgAAAAIAKwBuAFoAMwAmAD0ABwApAHMAVwAKAO7/BQDi/yQAYQBSABcAKQBKAEMASgAuAPH/tv/p/w8AIQAcAE0AYQAcAB4AGQD0/8D/5/8MACsAUgBcACQA2v/A/7j/9P8PABQAFwAkACQAOAAmAO7/7v/X/8X/9v8hADAADAAcAA8ADwDx/9D/3//x/+L/6f8uADAAFAAMAMr/df89/27/2v8rACkADwAKAOn/zf8FAPv/u//D/+T/5//x/wUAKwBnAAIAsf/A/+L/4v8XAH4AVwAKAAcA9v/f//H//v8PAAwA/v8eAFIAKQAeAPn/7P8MADgAYQBVAEMAMwD2/w8AJAAHAPb/CgApACkADAAhAAoAyP/u/xQAMwAwAAcA1f+u/2j/iv/A/wIASABNADAABwAwADMAKQAwABIAzf8FAEAAgwCdAIAAFADx/8X/q//x/yYAJAAzAHMAXwAhAA8AGQASAAcAFABVAGQALgA4AFoANgBFAG4AcQA7AO7/AABAADgASABaADYAAAAkAFoAUgBAAEAAAACm/7v/xf8uAIAAkAB7AGkAMAD+/+T/5//x/wIAKQA4AGkAYQBIAFcAWgAXAPn/7v/p/+T/8f/0//b/HABNAEMAIQDx/+z/BQDs//b/JgBKAE0AfgCSAIgAOAAUANr/rv+c/+T/JABNAHkAXwBIABQA5P/A/9X/9P8UAEgAbgBpAGQAHgD+/wcA6f/N/+L//v/2/xwAPQBsAEUA8f+7/7j/mf+K/7v/+//x//T/EgAFAMj/pP/I/8X/mf+X/+L/rv+k/9//BQDD/4//lP99/4L/pv/n/9r/of+P/73/u//N/+f/vf+2//b/+f/x//T/5P+U/6b/s/+Z/67/0v/i/9X/5P/I/8D/pP+c/7j/yP/p//H/6f/S/8j/xf+r/5n/j/99/7P/yP/F/6v/rv+r/8D/4v/V/+n/yv+u/8D/7P/0/9///v/A/9z/5//i/wIAJADS/5T/yv/V/8r/AgAzAPH/rv/S//T/5P8FAAoABwDS/9z/DwAeAO7/yP/A/6b/yP8KADMA9v/S/6n/of+r/+n/AgD+/+z/xf/I/9//7P/2/9z/q/+m/8P/6f/uLine truncated
Your browser does not support the audio element.
</audio>
@@ -652,7 +656,7 @@ is used to measure the inference performance of the ``FP16`` and
.. code:: ipython3
# Inference FP16 model (OpenVINO IR)
! benchmark_app -m $ir_model_path -shape "[1,30480]" -d CPU -api async -t 15
! benchmark_app -m $ir_model_path -shape [1,30480] -d CPU -api async -t 15
.. parsed-literal::
@@ -661,18 +665,18 @@ is used to measure the inference performance of the ``FP16`` and
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to THROUGHPUT.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 195.30 ms
[ INFO ] Read model took 34.19 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] inputs (node: inputs) : f32 / [...] / [?,?]
@@ -681,46 +685,52 @@ is used to measure the inference performance of the ``FP16`` and
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[ INFO ] Reshaping model: 'inputs': [1,30480]
[ INFO ] Reshape model took 26.05 ms
[ INFO ] Reshape model took 14.25 ms
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] inputs (node: inputs) : f32 / [...] / [1,30480]
[ INFO ] Model outputs:
[ INFO ] logits (node: logits) : f32 / [...] / [1,95,32]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 1059.76 ms
[ INFO ] Compile model took 510.06 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 6
[ INFO ] NUM_STREAMS: 6
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] INFERENCE_NUM_THREADS: 24
[ INFO ] AFFINITY: Affinity.HYBRID_AWARE
[ INFO ] INFERENCE_NUM_THREADS: 18
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'inputs'!. This input will be filled with random values!
[ INFO ] Fill input 'inputs' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 76.31 ms
[ INFO ] First inference took 110.84 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 630 iterations
[ INFO ] Duration: 15178.11 ms
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 360 iterations
[ INFO ] Duration: 15162.56 ms
[ INFO ] Latency:
[ INFO ] Median: 144.29 ms
[ INFO ] Average: 144.12 ms
[ INFO ] Min: 80.09 ms
[ INFO ] Max: 165.85 ms
[ INFO ] Throughput: 41.51 FPS
[ INFO ] Median: 219.94 ms
[ INFO ] Average: 251.64 ms
[ INFO ] Min: 154.25 ms
[ INFO ] Max: 378.39 ms
[ INFO ] Throughput: 23.74 FPS
.. code:: ipython3
# Inference INT8 model (OpenVINO IR)
! benchmark_app -m $compressed_model_path -shape "[1,30480]" -d CPU -api async -t 15
! benchmark_app -m $quantized_model_path -shape [1,30480] -d CPU -api async -t 15
.. parsed-literal::
@@ -729,18 +739,18 @@ is used to measure the inference performance of the ``FP16`` and
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to THROUGHPUT.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 169.92 ms
[ INFO ] Read model took 67.14 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] inputs (node: inputs) : f32 / [...] / [?,?]
@@ -749,40 +759,46 @@ is used to measure the inference performance of the ``FP16`` and
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[ INFO ] Reshaping model: 'inputs': [1,30480]
[ INFO ] Reshape model took 29.82 ms
[ INFO ] Reshape model took 32.77 ms
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] inputs (node: inputs) : f32 / [...] / [1,30480]
[ INFO ] Model outputs:
[ INFO ] logits (node: logits) : f32 / [...] / [1,95,32]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 902.94 ms
[ INFO ] Compile model took 731.37 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 6
[ INFO ] NUM_STREAMS: 6
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] INFERENCE_NUM_THREADS: 24
[ INFO ] AFFINITY: Affinity.HYBRID_AWARE
[ INFO ] INFERENCE_NUM_THREADS: 18
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'inputs'!. This input will be filled with random values!
[ INFO ] Fill input 'inputs' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 55.47 ms
[ INFO ] First inference took 72.68 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 906 iterations
[ INFO ] Duration: 15179.80 ms
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 582 iterations
[ INFO ] Duration: 15220.80 ms
[ INFO ] Latency:
[ INFO ] Median: 99.60 ms
[ INFO ] Average: 100.18 ms
[ INFO ] Min: 64.89 ms
[ INFO ] Max: 126.69 ms
[ INFO ] Throughput: 59.68 FPS
[ INFO ] Median: 135.06 ms
[ INFO ] Average: 156.14 ms
[ INFO ] Min: 124.62 ms
[ INFO ] Max: 232.68 ms
[ INFO ] Throughput: 38.24 FPS
Compare Accuracy of the Original and Quantized Models
@@ -807,5 +823,5 @@ compare it with the ``FP16`` result.
.. parsed-literal::
[OpenVino FP16] Word Error Rate: 0.03826
[OpenVino INT8] Word Error Rate: 0.0504
[OpenVino INT8] Word Error Rate: 0.0487
docs/notebooks/107-speech-recognition-quantization-wav2vec2-with-output.rst
-699
View File
@@ -1,699 +0,0 @@
Quantize Speech Recognition Models with OpenVINO™ Post-Training Optimization Tool ​
===================================================================================
This tutorial demonstrates how to apply ``INT8`` quantization to the
speech recognition model, known as
`Wav2Vec2 <https://huggingface.co/docs/transformers/model_doc/wav2vec2>`__,
using the `Post-Training Optimization Tool API (POT
API) <https://docs.openvino.ai/latest/pot_compression_api_README.html>`__
(part of the `OpenVINO Toolkit <https://docs.openvino.ai/>`__). This
notebook uses a fine-tuned
`Wav2Vec2-Base-960h <https://huggingface.co/facebook/wav2vec2-base-960h>`__
`PyTorch <https://pytorch.org/>`__ model trained on the `LibriSpeech ASR
corpus <https://www.openslr.org/12>`__. The tutorial is designed to be
extendable to custom models and datasets. It consists of the following
steps:
- Download and prepare the Wav2Vec2 model and LibriSpeech dataset.
- Define data loading and accuracy validation functionality.
- Prepare the model for quantization.
- Run optimization pipeline.
- Compare performance of the original and quantized models.
Imports
-------
.. code:: ipython3
import os
import sys
import time
import re
import numpy as np
import torch
import tarfile
from pathlib import Path
from itertools import groupby
import soundfile as sf
import IPython.display as ipd
from transformers import Wav2Vec2ForCTC
from openvino.runtime import Core
from openvino.tools.pot import Metric, DataLoader, IEEngine, \
load_model, save_model, compress_model_weights, create_pipeline
sys.path.append("../utils")
from notebook_utils import download_file
.. parsed-literal::
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/offline_transformations/__init__.py:10: FutureWarning: The module is private and following namespace `offline_transformations` will be removed in the future, use `openvino.runtime.passes` instead!
warnings.warn(
Settings
--------
.. code:: ipython3
# Set the data and model directories, model source URL and model filename.
DATA_DIR = "../data/datasets/librispeech"
MODEL_DIR = "model"
os.makedirs(DATA_DIR, exist_ok=True)
os.makedirs(MODEL_DIR, exist_ok=True)
Prepare the Model
-----------------
Perform the following: - Download and unpack a pre-trained Wav2Vec2
model. - Convert the model to ONNX. - Run Model Optimizer to convert the
model from the ONNX representation to the OpenVINO Intermediate
Representation (OpenVINO IR)
.. code:: ipython3
download_file("https://huggingface.co/facebook/wav2vec2-base-960h/resolve/main/pytorch_model.bin", directory=Path(MODEL_DIR) / 'pytorch', show_progress=True)
download_file("https://huggingface.co/facebook/wav2vec2-base-960h/resolve/main/config.json", directory=Path(MODEL_DIR) / 'pytorch', show_progress=False)
.. parsed-literal::
model/pytorch/pytorch_model.bin: 0%| | 0.00/360M [00:00<?, ?B/s]
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/107-speech-recognition-quantization/model/pytorch/config.json')
Import all dependencies to load the original PyTorch model and convert
it to the ONNX representation.
.. code:: ipython3
BATCH_SIZE = 1
MAX_SEQ_LENGTH = 30480
def export_model_to_onnx(model, path):
with torch.no_grad():
default_input = torch.zeros([1, MAX_SEQ_LENGTH], dtype=torch.float)
inputs = {
"inputs": default_input
}
symbolic_names = {0: "batch_size", 1: "sequence_len"}
torch.onnx.export(
model,
(inputs["inputs"]),
path,
opset_version=11,
input_names=["inputs"],
output_names=["logits"],
dynamic_axes={
"inputs": symbolic_names,
"logits": symbolic_names,
},
)
print("ONNX model saved to {}".format(path))
torch_model = Wav2Vec2ForCTC.from_pretrained(Path(MODEL_DIR) / 'pytorch')
onnx_model_path = Path(MODEL_DIR) / "wav2vec2_base.onnx"
if not onnx_model_path.exists():
export_model_to_onnx(torch_model, onnx_model_path)
.. parsed-literal::
Some weights of Wav2Vec2ForCTC were not initialized from the model checkpoint at model/pytorch and are newly initialized: ['wav2vec2.masked_spec_embed']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
.. parsed-literal::
ONNX model saved to model/wav2vec2_base.onnx
Convert the ONNX Model to OpenVINO IR
-------------------------------------
.. code:: ipython3
ir_model_xml = onnx_model_path.with_suffix(".xml")
ir_model_bin = onnx_model_path.with_suffix(".bin")
if not ir_model_xml.exists():
!mo --input_model $onnx_model_path --output_dir $MODEL_DIR --input_shape "[1,-1]" --compress_to_fp16
.. parsed-literal::
Check for a new version of Intel(R) Distribution of OpenVINO(TM) toolkit here https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit/download.html?cid=other&source=prod&campid=ww_2023_bu_IOTG_OpenVINO-2022-3&content=upg_all&medium=organic or on https://github.com/openvinotoolkit/openvino
[ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11.
Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/latest/openvino_2_0_transition_guide.html
[ SUCCESS ] Generated IR version 11 model.
[ SUCCESS ] XML file: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/107-speech-recognition-quantization/model/wav2vec2_base.xml
[ SUCCESS ] BIN file: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/107-speech-recognition-quantization/model/wav2vec2_base.bin
Prepare LibriSpeech Dataset
---------------------------
Use the code below to download and unpack the archive with ‘test-clean’
subset of LibriSpeech Dataset
.. code:: ipython3
download_file("http://openslr.elda.org/resources/12/test-clean.tar.gz", directory=DATA_DIR, show_progress=True)
if not os.path.exists(f'{DATA_DIR}/LibriSpeech'):
with tarfile.open(f"{DATA_DIR}/test-clean.tar.gz") as tar:
tar.extractall(path=DATA_DIR)
.. parsed-literal::
../data/datasets/librispeech/test-clean.tar.gz: 0%| | 0.00/331M [00:00<?, ?B/s]
Define DataLoader for POT
-------------------------
Define ``DataLoader`` based on POT API, as it will be used to collect
statistics for quantization and run model evaluation. Wav2Vec2 model
accepts a raw waveform of the speech signal as input and produces
vocabulary class estimations as output. Since the dataset contains audio
files in FLAC format, use the ‘soundfile’ package to convert them to
waveform.
**NOTE**: Consider increasing ``samples_limit`` to get more precise
results. A suggested value is ``300`` or more, as it will take longer
time to process.
.. code:: ipython3
class LibriSpeechDataLoader(DataLoader):
samples_limit = 4
@staticmethod
def read_flac(file_name):
speech, samplerate = sf.read(file_name)
assert samplerate == 16000, "read_flac: only 16kHz supported!"
return speech
# Required methods
def __init__(self, config):
"""Constructor
:param config: data loader specific config
"""
super().__init__(config)
self._data_dir = config["data_source"]
self._ds = []
self._prepare_dataset()
def __len__(self):
"""Returns size of the dataset"""
return len(self._ds)
def __getitem__(self, index):
"""
Returns annotation, data and metadata at the specified index.
Possible formats:
(index, annotation), data
(index, annotation), data, metadata
"""
label = self._ds[index][0]
inputs = {'inputs': np.expand_dims(self._ds[index][1], axis=0)}
return label, inputs
# Methods specific to the current implementation
def _prepare_dataset(self):
pattern = re.compile(r'([0-9\-]+)\s+(.+)')
data_folder = Path(self._data_dir)
txts = list(data_folder.glob('**/*.txt'))
counter = 0
for txt in txts:
content = txt.open().readlines()
for line in content:
res = pattern.search(line)
if not res:
continue
name = res.group(1)
transcript = res.group(2)
fname = txt.parent / name
fname = fname.with_suffix('.flac')
identifier = str(fname.relative_to(data_folder))
self._ds.append(((counter, transcript.upper()), LibriSpeechDataLoader.read_flac(os.path.join(self._data_dir, identifier))))
counter += 1
if counter >= self.samples_limit:
# Limit exceeded
return
Define WER Metric Calculation
-----------------------------
In this step, the ``Metric`` interface for WER metric is implemented. It
is used for validating the accuracy of the model. ``WER`` stands for
Word Error Rate. For more details, refer to the `Wiki
page <https://en.wikipedia.org/wiki/Word_error_rate>`__.
.. code:: ipython3
class MetricWER(Metric):
alphabet = [
"<pad>", "<s>", "</s>", "<unk>", "|",
"e", "t", "a", "o", "n", "i", "h", "s", "r", "d", "l", "u",
"m", "w", "c", "f", "g", "y", "p", "b", "v", "k", "'", "x", "j", "q", "z"]
words_delimiter = '|'
pad_token = '<pad>'
@staticmethod
def decode_logits(logits):
decoding_vocab = dict(enumerate(MetricWER.alphabet))
token_ids = np.squeeze(np.argmax(logits, -1))
tokens = [decoding_vocab[idx] for idx in token_ids]
tokens = [token_group[0] for token_group in groupby(tokens)]
tokens = [t for t in tokens if t != MetricWER.pad_token]
res_string = ''.join([t if t != MetricWER.words_delimiter else ' ' for t in tokens]).strip()
res_string = ' '.join(res_string.split(' '))
res_string = res_string.lower()
return res_string
# Required methods
def __init__(self):
super().__init__()
self._name = "WER"
self._sum_score = 0
self._sum_words = 0
self._cur_score = 0
self._decoding_vocab = dict(enumerate(self.alphabet))
@property
def value(self):
"""Returns accuracy metric value for the last model output."""
return {self._name: self._cur_score}
@property
def avg_value(self):
"""Returns accuracy metric value for all model outputs."""
return {self._name: self._sum_score / self._sum_words if self._sum_words != 0 else 0}
def update(self, output, target):
"""
Updates prediction matches.
:param output: model output
:param target: annotations
"""
decoded = [self.decode_logits(i) for i in output]
target = [i.lower() for i in target]
assert len(output) == len(target), "sizes of output and target mismatch!"
for i in range(len(output)):
self._get_metric_per_sample(decoded[i], target[i])
def reset(self):
"""
Resets collected matches
"""
self._sum_score = 0
self._sum_words = 0
def get_attributes(self):
"""
Returns a dictionary of metric attributes {metric_name: {attribute_name: value}}.
Required attributes: 'direction': 'higher-better' or 'higher-worse'
'type': metric type
"""
return {self._name: {"direction": "higher-worse", "type": "WER"}}
# Methods specific to the current implementation
def _get_metric_per_sample(self, annotation, prediction):
cur_score = self._editdistance_eval(annotation.split(), prediction.split())
cur_words = len(annotation.split())
self._sum_score += cur_score
self._sum_words += cur_words
self._cur_score = cur_score / cur_words
result = cur_score / cur_words if cur_words != 0 else 0
return result
def _editdistance_eval(self, source, target):
n, m = len(source), len(target)
distance = np.zeros((n + 1, m + 1), dtype=int)
distance[:, 0] = np.arange(0, n + 1)
distance[0, :] = np.arange(0, m + 1)
for i in range(1, n + 1):
for j in range(1, m + 1):
cost = 0 if source[i - 1] == target[j - 1] else 1
distance[i][j] = min(distance[i - 1][j] + 1,
distance[i][j - 1] + 1,
distance[i - 1][j - 1] + cost)
return distance[n][m]
Run Quantization Pipeline
-------------------------
Use the code below to define a configuration for the quantization
pipeline and run it. Keep in mind that built-in ``IEEngine``
implementation of ``Engine`` interface from the POT API for model
inference is used here.
.. code:: ipython3
model_config = {"model_name": "wav2vec2_base", "model": ir_model_xml, "weights": ir_model_bin}
engine_config = {"device": "CPU"}
dataset_config = {"data_source": os.path.join(DATA_DIR, "LibriSpeech/test-clean")}
algorithms = [
{
"name": "DefaultQuantization",
"params": {
"target_device": "ANY",
"model_type": "transformer",
"preset": "performance",
"stat_subset_size": 300,
"activations": {
"range_estimator": {
"min": {
"aggregator": "min",
"type": "min"
},
"max": {
"aggregator": "mean",
"type": "quantile",
"outlier_prob": 0.0001
}
}
},
"ignored": {
"scope": ["214"]
}
}
}
]
# Step 1: Load the model.
model = load_model(model_config=model_config)
# Step 2: Initialize the data loader.
data_loader = LibriSpeechDataLoader(config=dataset_config)
# Step 3 (Optional. Required for AccuracyAwareQuantization): Initialize the metric.
metric = MetricWER()
# Step 4: Initialize the engine for metric calculation and statistics collection.
engine = IEEngine(config=engine_config, data_loader=data_loader, metric=metric)
# Step 5: Create a pipeline of compression algorithms.
pipeline = create_pipeline(algo_config=algorithms, engine=engine)
# Step 6 (Optional): Evaluate the original model. Print the results.
start_time = time.perf_counter()
fp_results = pipeline.evaluate(model=model)
end_time = time.perf_counter()
print(f"Evaluation finished in {end_time - start_time:.2f} seconds")
if fp_results:
print("FP16 model results:")
for name, value in fp_results.items():
print(f"{name}: {value:.5f}")
.. parsed-literal::
Evaluation finished in 7.96 seconds
FP16 model results:
WER: 0.03704
.. code:: ipython3
# Step 7: Execute the pipeline.
print(f"Quantizing model with {algorithms[0]['params']['preset']} preset and {algorithms[0]['name']}")
start_time = time.perf_counter()
compressed_model = pipeline.run(model=model)
end_time = time.perf_counter()
print(f"Quantization finished in {end_time - start_time:.2f} seconds")
# Step 8 (Optional): Compress model weights to quantized precision
# in order to reduce the size of the final .bin file.
compress_model_weights(model=compressed_model)
# Step 9: Save the compressed model to the desired path.
compressed_model_paths = save_model(model=compressed_model, save_path=MODEL_DIR, model_name="quantized_wav2vec2_base")
compressed_model_xml = compressed_model_paths[0]["model"]
.. parsed-literal::
Quantizing model with performance preset and DefaultQuantization
Quantization finished in 68.16 seconds
.. code:: ipython3
# Step 10 (Optional): Evaluate the compressed model and print the results.
int_results = pipeline.evaluate(model=compressed_model)
if int_results:
print("INT8 model results:")
for name, value in int_results.items():
print(f"{name}: {value:.5f}")
.. parsed-literal::
INT8 model results:
WER: 0.05556
Model Usage Example with Inference Pipeline
-------------------------------------------
Both initial (``FP16``) and quantized (``INT8``) models are exactly the
same in use.
Start with taking one example from the dataset to show inference steps
for it.
.. code:: ipython3
audio = LibriSpeechDataLoader.read_flac(f'{DATA_DIR}/LibriSpeech/test-clean/121/127105/121-127105-0017.flac')
ipd.Audio(audio, rate=16000)
.. raw:: html
<audio controls="controls" >
<source src="data:audio/wav;base64,UklGRgRRAQBXQVZFZm10IBAAAAABAAEAgD4AAAB9AAACABAAZGF0YeBQAQAGAP7//P/+//7//v/+/wAAAgACAAIAAAAAAP7//v/+//z//v/+/wAAAAACAAIAAgD+//z//v/+/wAA/v/8//r/+v8AAP7//P/6//7//v/+//7/AAAEAAIAAgAAAAAAAAAAAAAA/v8AAAAAAAACAAAAAAAAAAAA+v/+//7//v8AAAAAAAACAAIAAAD+/wAA/v/8//z//P8AAAAAAgACAPz//P/8//r/+v/4//z//v/+/wAAAAAAAPz/AAACAAAAAAACAAIAAAAAAAQABAAEAAQAAAD+//z//P8AAP7//v/+/wAAAAAAAPz/+v/4//j/9v/4//b/8f/z//P/9v/6/wAA/v/6//z/AAD+//7//v8AAAAAAAAAAAIA/v/+/wAAAgACAAAAAAAAAAAAAAAAAP7/AAD+//7//v/+//7/AAAAAAAAAAAAAP7//v/+//7//P/+//7/AAD8//7//v/6//j/+P/6//r/9v/z//P/9v/2//j/+v/+//r/8//2//b/9v/z//b/8//z//H/8//x/+//7//t/+v/7//t/+3/7f/t/+3/7f/t/+3/6f/r/+v/6//t/+//8f/2//r//v/+//z//v8CAAAAAgAEAAgACAAIAAgABgAEAAQABAAEAAIAAAAEAAIAAAACAAQABgACAAYABgACAP7/AAACAAIAAgD+/wAABAACAPz/AAAAAAAAAAAAAPz//P/+//7//v/+/wAA+v/6//j//P/6//r//P8AAAAA/v/8/wAAAAD+/wAAAAACAAQAAAD+//7/AAAAAAAAAAAAAAIAAgACAAIAAgAAAAAAAAAAAAAA/P/+/wIAAgACAAIAAAAAAAAAAgACAP7//P/8//r/+v/4//P/7//t/+v/7f/p/+n/7//x/+3/7f/t/+//7f/x/+//7f/r/+//8f/x/+//8f/2//b/9v/2//b/9v/z//b/+P/4//j//P/6//z/AAAAAP7/AAD+//7//v/+/wIABAD+//7//v8AAAQABgAGAAYACAAKAAYABgAEAAAAAAAAAAAAAAAAAAIAAgACAAIAAgAEAAIA/v/+/wAAAAACAAIAAgACAAIA/v/8/wAAAAAAAAAAAAACAAIABAAEAP7/AAACAAQABAAGAAgACgAKAAgACgAIAAoACAAGAAoABgAGAAYACgAKAAoABAACAAIAAgACAAIAAgACAAIABAD+//7//v8AAP7//v/+/wIAAAD+/wIAAAD+/wIAAgACAAAAAAAAAAIAAAAAAAIAAgACAAIAAgAAAAAABAAEAAgACAAKAA0ACgAIAAgABgAGAAYABgAKAAoACgAPABEAEwARABEAEQARAA8ADQANAA0ACgAKAAoADQAKAAoACgAKAAgABgAEAAAAAAACAAAA/P/8//7//P/+//7/AAAAAAIABAAEAAYACAAIAAoACgAKAAgABgAIAAoACgAKAAYABgAEAAIAAgACAAAA/v/+/wAAAAD+//7/AAAAAP7/AAAAAAAAAAACAAQABAAGAAYABgAEAAQAAAAAAP7//v8CAAIAAAAAAP7//v/8//z//v/+//7//v8AAAAAAgACAAIAAAAAAAAAAAAAAAAAAAACAAIAAgACAAAA/v/+//7//v/+//7/AAAAAAAAAgACAAIAAAAAAAAAAAAAAAAAAgACAAAAAAAAAP7//v8AAP7//v/+/wAAAAAAAAQAAgACAAAAAAAAAAAAAAAAAAIAAgACAAIAAgAAAP7//v/8//j/+P/4//j/+P/z//H/8f/x/+//7f/r/+v/6f/p/+n/5v/m/+n/5v/m/+b/6f/m/+n/6//t/+//8//z//j/+P/2//j/+P/4//b/+P/8//z//v/+/wIAAAD+//z//v8CAP7/AAAAAAAAAAAAAAIAAAACAAIAAAACAAIAAgACAAIAAAACAP7/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAP7//v/+//7//v/+/wAAAAAAAP7//v/8//z//P/8//z//P/8//z//P/8//z//P/6//r/+v/+//7//v8AAAAAAgAEAAIAAAAAAAAAAAAAAAAAAAACAAIAAAAAAP7//P/8//z//P/4//j/+P/4//r//v/8//r/+v/6//7/AAAAAAAAAgAAAAAAAgAAAAAA/v8CAP7/AAAAAAIAAgACAAIAAgACAPz//P/+//7/AAACAAQACAAIAAgACAAIAAIAAgAAAAAAAAAAAAIAAAACAAYAAAD8//z//P/+//7//v8AAAAAAgACAAIAAgAAAP7/AAAAAAAAAAACAAIAAgAEAAQAAAD+//7//v/+//7/AAACAAIAAgACAAQABAAGAAYAAgACAAIABAAGAAYAAgACAAQAAgD+//7//v8AAAAAAgACAAIAAgAAAPz//v/+//7//v/+/wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAgACAAIAAgD+//7/AAAAAP7/AAAAAAIAAgACAAIAAgAAAAAAAAAAAAAAAAACAAIAAgACAAIAAgAAAAAAAAAAAAAAAAACAAIAAgACAAQAAgACAAIAAgACAAIABAAGAAYABgAGAAYABAAEAAQABAACAAIABgAGAAIA/v/+//z//P/+//7//v/+//7/AAAAAAIAAgACAAAAAAAAAP7//v/+/wAAAAAAAAAAAAAAAP7//v/+//7//v/+/wIAAgACAAIAAgAAAAAAAAAAAP7//v8CAAIAAgAAAAAAAAD+//7//v/+/wAAAAACAAIAAgACAAIAAgAAAAAAAAAAAAAAAAACAAIAAAAAAAAA/v/+//7//v/+//7/AAAAAAAAAAAAAAAAAAAAAAAA/v/+//7/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD+/wAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP7//v/+//7//v/+/wAAAAAAAAAAAAD+//7//v/+//7//v8AAAAAAAAAAAAAAAD+//7//v/+//7//v8AAAAAAAAAAAAA/v/+//7//v/+//7//v8AAAAAAAAAAAAA/v/+//7//v/+//7/AAAAAAAAAAAAAAAA/v/+//7//v/+//7/AAAAAAAAAAAAAPz//P/8//z//P/8//z//P/8//z/+v/6//r//P/8//z/+v/4//j//P/8//z//P/8//z//P/+/wAAAAAAAAIAAgAEAAQABAAAAAAAAAAAAAAAAAACAAIAAgAAAAAAAAD+//7//v8AAAAAAAAAAAAAAgACAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/v/+//7/AAD+//7/AAAAAAAAAgACAAIAAAAAAAAAAAD+//7/AAAAAAAAAAAAAP7//v/+/wAAAAAAAAIAAAAAAAIAAgACAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/v8CAAAAAgACAAAAAAAAAAAAAAAAAAAAAgACAAAAAAAAAAAAAAAAAPz/AAAAAAAAAAAAAAIAAAD+/wAA/v8AAAAAAAAAAP7/AAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/v/+/wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/v8AAAIAAAAAAAAAAgAAAP7//v/+/wAAAAAEAAAAAAAAAAAA/v8AAAAAAAACAAAAAgD+/wAAAAAAAAIA/v/8Line truncated
Your browser does not support the audio element.
</audio>
Next, load the quantized model to the inference pipeline.
.. code:: ipython3
ie = Core()
model = ie.read_model(compressed_model_xml)
compiled_model = ie.compile_model(model=model, device_name='CPU')
input_data = np.expand_dims(audio, axis=0)
output_layer = compiled_model.outputs[0]
Next, make a prediction.
.. code:: ipython3
predictions = compiled_model([input_data])[output_layer]
Now, you just need to decode predicted probabilities to text, using
tokenizer from ``MetricWER`` class.
Alternatively, use a built-in ``Wav2Vec2Processor`` tokenizer from the
``transformers`` package.
.. code:: ipython3
predicted_text = MetricWER.decode_logits(predictions)
predicted_text
.. parsed-literal::
'it was almost the tone of hope everybody will stay'
Compare Performance of the Original and Quantized Models
--------------------------------------------------------
Finally, use `Benchmark
Tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_tool_README.html>`__
to measure the inference performance of the ``FP16`` and ``INT8``
models.
**NOTE**: For more accurate performance, it is recommended to run
``benchmark_app`` in a terminal/command prompt after closing other
applications. Run ``benchmark_app -m model.xml -d CPU`` to benchmark
async inference on CPU for one minute. Change ``CPU`` to ``GPU`` to
benchmark on GPU. Run ``benchmark_app --help`` to see an overview of
all command-line options.
.. code:: ipython3
# Inference FP16 model (OpenVINO IR)
! benchmark_app -m $ir_model_xml -shape "[1,30480]" -d CPU -api async
.. parsed-literal::
[Step 1/11] Parsing and validating input arguments
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 186.10 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] inputs (node: inputs) : f32 / [...] / [1,?]
[ INFO ] Model outputs:
[ INFO ] logits (node: logits) : f32 / [...] / [1,?,32]
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[ INFO ] Reshaping model: 'inputs': [1,30480]
[ INFO ] Reshape model took 21.34 ms
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] inputs (node: inputs) : f32 / [...] / [1,30480]
[ INFO ] Model outputs:
[ INFO ] logits (node: logits) : f32 / [...] / [1,95,32]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 1044.11 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 6
[ INFO ] NUM_STREAMS: 6
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] INFERENCE_NUM_THREADS: 24
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'inputs'!. This input will be filled with random values!
[ INFO ] Fill input 'inputs' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 66.29 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 2760 iterations
[ INFO ] Duration: 60133.52 ms
[ INFO ] Latency:
[ INFO ] Median: 130.35 ms
[ INFO ] Average: 130.50 ms
[ INFO ] Min: 117.02 ms
[ INFO ] Max: 144.07 ms
[ INFO ] Throughput: 45.90 FPS
.. code:: ipython3
# Inference INT8 model (OpenVINO IR)
! benchmark_app -m $compressed_model_xml -shape "[1,30480]" -d CPU -api async
.. parsed-literal::
[Step 1/11] Parsing and validating input arguments
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 127.98 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] inputs (node: inputs) : f32 / [...] / [1,?]
[ INFO ] Model outputs:
[ INFO ] logits (node: logits) : f32 / [...] / [1,?,32]
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[ INFO ] Reshaping model: 'inputs': [1,30480]
[ INFO ] Reshape model took 23.75 ms
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] inputs (node: inputs) : f32 / [...] / [1,30480]
[ INFO ] Model outputs:
[ INFO ] logits (node: logits) : f32 / [...] / [1,95,32]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 939.21 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 6
[ INFO ] NUM_STREAMS: 6
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] INFERENCE_NUM_THREADS: 24
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'inputs'!. This input will be filled with random values!
[ INFO ] Fill input 'inputs' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 38.22 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 7668 iterations
[ INFO ] Duration: 60054.52 ms
[ INFO ] Latency:
[ INFO ] Median: 46.41 ms
[ INFO ] Average: 46.87 ms
[ INFO ] Min: 37.61 ms
[ INFO ] Max: 238.12 ms
[ INFO ] Throughput: 127.68 FPS
docs/notebooks/108-gpu-device-with-output.rst
+19 -19
View File
@@ -32,10 +32,10 @@ cards <https://www.intel.com/content/www/us/en/products/details/discrete-gpus/ar
and `Intel® Data Center GPU Flex
Series <https://www.intel.com/content/www/us/en/products/details/discrete-gpus/data-center-gpu/flex-series.html>`__.
To get started, first `install
OpenVINO <https://docs.openvino.ai/latest/openvino_docs_install_guides_overview.html>`__
OpenVINO <https://docs.openvino.ai/2023.0/openvino_docs_install_guides_overview.html>`__
on a system equipped with one or more Intel GPUs. Follow the `GPU
configuration
instructions <https://docs.openvino.ai/latest/openvino_docs_install_guides_configurations_for_intel_gpu.html>`__
instructions <https://docs.openvino.ai/2023.0/openvino_docs_install_guides_configurations_for_intel_gpu.html>`__
to configure OpenVINO to work with your GPU. Then, read on to learn how
to accelerate inference with GPUs in OpenVINO!
@@ -75,12 +75,12 @@ the system has a CPU, an integrated and discrete GPU, we should expect
to see a list like this: ``['CPU', 'GPU.0', 'GPU.1']``. To simplify its
use, the “GPU.0” can also be addressed with just “GPU”. For more
details, see the `Device Naming
Convention <https://docs.openvino.ai/latest/openvino_docs_OV_UG_supported_plugins_GPU.html#device-naming-convention>`__
Convention <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_GPU.html#device-naming-convention>`__
section.
If the GPUs are installed correctly on the system and still do not
appear in the list, follow the steps described
`here <https://docs.openvino.ai/latest/openvino_docs_install_guides_configurations_for_intel_gpu.html>`__
`here <https://docs.openvino.ai/2023.0/openvino_docs_install_guides_configurations_for_intel_gpu.html>`__
to configure your GPU drivers to work with OpenVINO. Once we have the
GPUs working with OpenVINO, we can proceed with the next sections.
@@ -192,7 +192,7 @@ the key properties are:
speed up compilation time.
To learn more about devices and properties, see the `Query Device
Properties <https://docs.openvino.ai/latest/openvino_docs_OV_UG_query_api.html>`__
Properties <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_query_api.html>`__
page.
Compiling a Model on GPU
@@ -201,7 +201,7 @@ Compiling a Model on GPU
Now, we know how to list the GPUs in the system and check their
properties. We can easily use one for compiling and running models with
OpenVINO `GPU
plugin <https://docs.openvino.ai/latest/openvino_docs_OV_UG_supported_plugins_GPU.html>`__.
plugin <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_GPU.html>`__.
Download and Convert a Model
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -277,7 +277,7 @@ Use Model Optimizer to convert the model to OpenVINO IR with ``FP16``
precision. The models are saved to the ``model/ir_model/`` directory.
For more information about Model Optimizer, see the `Model Optimizer
Developer
Guide <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
.. code:: ipython3
@@ -337,7 +337,7 @@ the ``available_devices`` method are valid device specifiers. You may
also use “AUTO”, which will automatically select the best device for
inference (which is often the GPU). To learn more about AUTO plugin,
visit the `Automatic Device
Selection <https://docs.openvino.ai/latest/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__
Selection <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__
page as well as the `AUTO device
tutorial <https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/106-auto-device>`__.
@@ -407,7 +407,7 @@ compile times with caching enabled and disabled as follows:
The actual time improvements will depend on the environment as well as
the model being used but it is definitely something to consider when
optimizing an application. To read more about this, see the `Model
Caching <https://docs.openvino.ai/latest/openvino_docs_OV_UG_Model_caching_overview.html>`__
Caching <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Model_caching_overview.html>`__
docs.
Throughput and Latency Performance Hints
@@ -446,7 +446,7 @@ Using Multiple GPUs with Multi-Device and Cumulative Throughput
The latency and throughput hints mentioned above are great and can make
a difference when used adequately but they usually use just one device,
either due to the `AUTO
plugin <https://docs.openvino.ai/latest/openvino_docs_OV_UG_supported_plugins_AUTO.html#how-auto-works>`__
plugin <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_AUTO.html#how-auto-works>`__
or by manual specification of the device name as above. When we have
multiple devices, such as an integrated and discrete GPU, we may use
both at the same time to improve the utilization of the resources. In
@@ -478,7 +478,7 @@ manually specify devices to use. Below is an example showing how to use
how to set up an asynchronous pipeline that takes advantage of
parallelism to increase throughput.** To learn more, see
`Asynchronous
Inferencing <https://docs.openvino.ai/latest/openvino_docs_ie_plugin_dg_async_infer_request.html>`__
Inferencing <https://docs.openvino.ai/2023.0/openvino_docs_ie_plugin_dg_async_infer_request.html>`__
in OpenVINO as well as the `Asynchronous Inference
notebook <https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/115-async-api>`__.
@@ -502,7 +502,7 @@ Note that benchmark_app only requires the model path to run but both the
device and hint arguments will be useful to us. For more advanced
usages, the tool itself has other options that can be checked by running
``benchmark_app -h`` or reading the
`docs <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_tool_README.html>`__.
`docs <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_tool_README.html>`__.
The following example shows how to benchmark a simple model, using a GPU
with a latency focus:
@@ -1260,18 +1260,18 @@ detected bounding boxes.
To read more about any of these topics, feel free to visit their
corresponding documentation: \* `GPU
Plugin <https://docs.openvino.ai/latest/openvino_docs_OV_UG_supported_plugins_GPU.html>`__
Plugin <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_GPU.html>`__
\* `AUTO
Plugin <https://docs.openvino.ai/latest/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__
Plugin <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__
\* `Model
Caching <https://docs.openvino.ai/latest/openvino_docs_OV_UG_Model_caching_overview.html>`__
Caching <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Model_caching_overview.html>`__
\* `MULTI Device
Mode <https://docs.openvino.ai/nightly/openvino_docs_OV_UG_Running_on_multiple_devices.html>`__
\* `Query Device
Properties <https://docs.openvino.ai/latest/openvino_docs_OV_UG_query_api.html>`__
Properties <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_query_api.html>`__
\* `Configurations for GPUs with
OpenVINO <https://docs.openvino.ai/latest/openvino_docs_install_guides_configurations_for_intel_gpu.html>`__
OpenVINO <https://docs.openvino.ai/2023.0/openvino_docs_install_guides_configurations_for_intel_gpu.html>`__
\* `Benchmark Python
Tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_tool_README.html>`__
Tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_tool_README.html>`__
\* `Asynchronous
Inferencing <https://docs.openvino.ai/latest/openvino_docs_ie_plugin_dg_async_infer_request.html>`__
Inferencing <https://docs.openvino.ai/2023.0/openvino_docs_ie_plugin_dg_async_infer_request.html>`__
docs/notebooks/109-latency-tricks-with-output.rst
+16 -16
View File
@@ -94,7 +94,7 @@ requirements of this particular object detection model.
.. parsed-literal::
<DisplayHandle display_id=6f2409e74123f40cdda4989b2a2a5922>
<DisplayHandle display_id=18f2331c8bf61fd35d93bb638f7ea64b>
@@ -346,8 +346,8 @@ optimizations applied. We will treat it as our baseline.
.. parsed-literal::
PyTorch model on CPU. First inference time: 0.0288 seconds
PyTorch model on CPU: 0.0205 seconds per image (48.90 FPS)
PyTorch model on CPU. First inference time: 0.0241 seconds
PyTorch model on CPU: 0.0195 seconds per image (51.21 FPS)
ONNX model
@@ -395,8 +395,8 @@ Representation (IR) to leverage the OpenVINO Runtime.
.. parsed-literal::
ONNX model on CPU. First inference time: 0.0165 seconds
ONNX model on CPU: 0.0091 seconds per image (109.62 FPS)
ONNX model on CPU. First inference time: 0.0167 seconds
ONNX model on CPU: 0.0132 seconds per image (76.03 FPS)
OpenVINO IR model
@@ -433,8 +433,8 @@ accuracy drop. That’s why we skip that step in this notebook.
.. parsed-literal::
OpenVINO model on CPU. First inference time: 0.0152 seconds
OpenVINO model on CPU: 0.0092 seconds per image (109.13 FPS)
OpenVINO model on CPU. First inference time: 0.0155 seconds
OpenVINO model on CPU: 0.0132 seconds per image (75.91 FPS)
OpenVINO IR model on GPU
@@ -467,7 +467,7 @@ OpenVINO IR model + more inference threads
There is a possibility to add a config for any device (CPU in this
case). We will increase the number of threads to an equal number of our
cores. It should help us a lot. There are `more
options <https://docs.openvino.ai/latest/groupov_runtime_cpp_prop_api.html>`__
options <https://docs.openvino.ai/2023.0/groupov_runtime_cpp_prop_api.html>`__
to be changed, so it’s worth playing with them to see what works best in
our case.
@@ -490,15 +490,15 @@ our case.
.. parsed-literal::
OpenVINO model + more threads on CPU. First inference time: 0.0143 seconds
OpenVINO model + more threads on CPU: 0.0094 seconds per image (106.13 FPS)
OpenVINO model + more threads on CPU. First inference time: 0.0151 seconds
OpenVINO model + more threads on CPU: 0.0131 seconds per image (76.12 FPS)
OpenVINO IR model in latency mode
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
OpenVINO offers a virtual device called
`AUTO <https://docs.openvino.ai/latest/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__,
`AUTO <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_AUTO.html>`__,
which can select the best device for us based on a performance hint.
There are three different hints: ``LATENCY``, ``THROUGHPUT``, and
``CUMULATIVE_THROUGHPUT``. As this notebook is focused on the latency
@@ -520,8 +520,8 @@ devices as well.
.. parsed-literal::
OpenVINO model on AUTO. First inference time: 0.0134 seconds
OpenVINO model on AUTO: 0.0095 seconds per image (105.46 FPS)
OpenVINO model on AUTO. First inference time: 0.0155 seconds
OpenVINO model on AUTO: 0.0135 seconds per image (73.89 FPS)
OpenVINO IR model in latency mode + shared memory
@@ -554,8 +554,8 @@ performance!
.. parsed-literal::
OpenVINO model + shared memory on AUTO. First inference time: 0.0140 seconds
OpenVINO model + shared memory on AUTO: 0.0055 seconds per image (181.99 FPS)
OpenVINO model + shared memory on AUTO. First inference time: 0.0132 seconds
OpenVINO model + shared memory on AUTO: 0.0053 seconds per image (187.78 FPS)
Other tricks
@@ -612,6 +612,6 @@ object detection model. Even if you experience much better performance
after running this notebook, please note this may not be valid for every
hardware or every model. For the most accurate results, please use
``benchmark_app`` `command-line
tool <https://docs.openvino.ai/latest/openvino_inference_engine_samples_benchmark_app_README.html>`__.
tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_samples_benchmark_app_README.html>`__.
Note that ``benchmark_app`` cannot measure the impact of some tricks
above, e.g., shared memory.
docs/notebooks/109-latency-tricks-with-output_files/109-latency-tricks-with-output_30_0.png
+2 -2
View File
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:0244df28a761b0553005731eb6cc5dbae3cfff25cc66f4ce0cd4335882bf8120
size 57110
oid sha256:eeffdc620ca9a96149c05b5fa8815c1e0f0f6d1775de63ae1893c051cd690977
size 56942
docs/notebooks/109-latency-tricks-with-output_files/index.html
+10 -10
View File
@@ -1,14 +1,14 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/109-latency-tricks-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/109-latency-tricks-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/109-latency-tricks-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="109-latency-tricks-with-output_14_0.jpg">109-latency-tricks-with-output_14_0.jpg</a> 30-May-2023 00:08 162715
<a href="109-latency-tricks-with-output_17_0.jpg">109-latency-tricks-with-output_17_0.jpg</a> 30-May-2023 00:08 162715
<a href="109-latency-tricks-with-output_19_0.jpg">109-latency-tricks-with-output_19_0.jpg</a> 30-May-2023 00:08 162756
<a href="109-latency-tricks-with-output_23_0.jpg">109-latency-tricks-with-output_23_0.jpg</a> 30-May-2023 00:08 162756
<a href="109-latency-tricks-with-output_25_0.jpg">109-latency-tricks-with-output_25_0.jpg</a> 30-May-2023 00:08 162756
<a href="109-latency-tricks-with-output_27_0.jpg">109-latency-tricks-with-output_27_0.jpg</a> 30-May-2023 00:08 162756
<a href="109-latency-tricks-with-output_30_0.png">109-latency-tricks-with-output_30_0.png</a> 30-May-2023 00:08 57110
<a href="109-latency-tricks-with-output_4_0.jpg">109-latency-tricks-with-output_4_0.jpg</a> 30-May-2023 00:08 155828
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/109-latency-tricks-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="109-latency-tricks-with-output_14_0.jpg">109-latency-tricks-with-output_14_0.jpg</a> 22-Jun-2023 00:06 162715
<a href="109-latency-tricks-with-output_17_0.jpg">109-latency-tricks-with-output_17_0.jpg</a> 22-Jun-2023 00:06 162715
<a href="109-latency-tricks-with-output_19_0.jpg">109-latency-tricks-with-output_19_0.jpg</a> 22-Jun-2023 00:06 162756
<a href="109-latency-tricks-with-output_23_0.jpg">109-latency-tricks-with-output_23_0.jpg</a> 22-Jun-2023 00:06 162756
<a href="109-latency-tricks-with-output_25_0.jpg">109-latency-tricks-with-output_25_0.jpg</a> 22-Jun-2023 00:06 162756
<a href="109-latency-tricks-with-output_27_0.jpg">109-latency-tricks-with-output_27_0.jpg</a> 22-Jun-2023 00:06 162756
<a href="109-latency-tricks-with-output_30_0.png">109-latency-tricks-with-output_30_0.png</a> 22-Jun-2023 00:06 56942
<a href="109-latency-tricks-with-output_4_0.jpg">109-latency-tricks-with-output_4_0.jpg</a> 22-Jun-2023 00:06 155828
</pre><hr></body>
</html>
docs/notebooks/110-ct-scan-live-inference-with-output.rst
+48 -19
View File
@@ -63,15 +63,38 @@ trained or optimized yourself, adjust the model paths.
.. code:: ipython3
# The directory that contains the IR model (xml and bin) files.
MODEL_PATH = "pretrained_model/quantized_unet_kits19.xml"
models_dir = Path('pretrained_model')
ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/kidney-segmentation-kits19/FP16-INT8/'
ir_model_name_xml = 'quantized_unet_kits19.xml'
ir_model_name_bin = 'quantized_unet_kits19.bin'
download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory=models_dir)
download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory=models_dir)
MODEL_PATH = models_dir / ir_model_name_xml
# Uncomment the next line to use the FP16 model instead of the quantized model.
# MODEL_PATH = "pretrained_model/unet_kits19.xml"
.. parsed-literal::
pretrained_model/quantized_unet_kits19.xml: 0%| | 0.00/280k [00:00<?, ?B/s]
.. parsed-literal::
pretrained_model/quantized_unet_kits19.bin: 0%| | 0.00/1.90M [00:00<?, ?B/s]
Benchmark Model Performance
---------------------------
To measure the inference performance of the IR model, use `Benchmark
Tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_tool_README.html>`__
Tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_tool_README.html>`__
- an inference performance measurement tool in OpenVINO. Benchmark tool
is a command-line application that can be run in the notebook with
``! benchmark_app`` or ``%sx benchmark_app`` commands.
@@ -107,18 +130,18 @@ is a command-line application that can be run in the notebook with
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to LATENCY.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.LATENCY.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 31.42 ms
[ INFO ] Read model took 13.79 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] input.1 (node: input.1) : f32 / [...] / [1,1,512,512]
@@ -132,7 +155,7 @@ is a command-line application that can be run in the notebook with
[ INFO ] Model outputs:
[ INFO ] 153 (node: 153) : f32 / [...] / [1,1,512,512]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 209.10 ms
[ INFO ] Compile model took 163.85 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: pretrained_unet_kits19
@@ -143,22 +166,28 @@ is a command-line application that can be run in the notebook with
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.LATENCY
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'input.1'!. This input will be filled with random values!
[ INFO ] Fill input 'input.1' with random values
[Step 10/11] Measuring performance (Start inference synchronously, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 23.91 ms
[ INFO ] First inference took 25.59 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 1437 iterations
[ INFO ] Duration: 15010.38 ms
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 1419 iterations
[ INFO ] Duration: 15009.20 ms
[ INFO ] Latency:
[ INFO ] Median: 10.20 ms
[ INFO ] Average: 10.25 ms
[ INFO ] Min: 9.93 ms
[ INFO ] Max: 13.01 ms
[ INFO ] Throughput: 98.02 FPS
[ INFO ] Median: 10.34 ms
[ INFO ] Average: 10.38 ms
[ INFO ] Min: 10.13 ms
[ INFO ] Max: 14.24 ms
[ INFO ] Throughput: 96.71 FPS
Download and Prepare Data
@@ -223,7 +252,7 @@ Caching, refer to the `OpenVINO API
tutorial <002-openvino-api-with-output.html>`__.
We will use
`AsyncInferQueue <https://docs.openvino.ai/latest/openvino_docs_OV_UG_Python_API_exclusives.html#asyncinferqueue>`__
`AsyncInferQueue <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Python_API_exclusives.html#asyncinferqueue>`__
to perform asynchronous inference. It can be instantiated with compiled
model and a number of jobs - parallel execution threads. If you don’t
pass a number of jobs or pass ``0``, then OpenVINO will pick the optimal
@@ -374,7 +403,7 @@ Create asynchronous inference queue and perform it
.. parsed-literal::
Loaded model to CPU in 0.20 seconds.
Total time to infer all frames: 3.085s
Time per frame: 0.045371s (22.040 FPS)
Loaded model to CPU in 0.17 seconds.
Total time to infer all frames: 3.412s
Time per frame: 0.050177s (19.929 FPS)
docs/notebooks/110-ct-scan-live-inference-with-output_files/index.html
+3 -3
View File
@@ -1,7 +1,7 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/110-ct-scan-live-inference-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/110-ct-scan-live-inference-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/110-ct-scan-live-inference-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="110-ct-scan-live-inference-with-output_21_0.png">110-ct-scan-live-inference-with-output_21_0.png</a> 30-May-2023 00:09 48780
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/110-ct-scan-live-inference-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="110-ct-scan-live-inference-with-output_21_0.png">110-ct-scan-live-inference-with-output_21_0.png</a> 22-Jun-2023 00:06 48780
</pre><hr></body>
</html>
docs/notebooks/110-ct-segmentation-quantize-nncf-with-output.rst
+66 -52
View File
@@ -14,9 +14,9 @@ scratch; the data is from
This third tutorial in the series shows how to:
- Convert an Original model to OpenVINO IR with `Model
Optimizer <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__,
Optimizer <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__,
using `Model Optimizer Python
API <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Python_API.html>`__
API <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Python_API.html>`__
- Quantize a PyTorch model with NNCF
- Evaluate the F1 score metric of the original model and the quantized
model
@@ -146,10 +146,10 @@ Imports
.. parsed-literal::
2023-05-29 22:49:24.630818: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-05-29 22:49:24.666920: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2023-06-21 22:40:32.613237: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-06-21 22:40:32.647235: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-05-29 22:49:25.225999: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2023-06-21 22:40:33.170121: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
.. parsed-literal::
@@ -157,12 +157,6 @@ Imports
INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino
.. parsed-literal::
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/offline_transformations/__init__.py:10: FutureWarning: The module is private and following namespace `offline_transformations` will be removed in the future, use `openvino.runtime.passes` instead!
warnings.warn(
Settings
--------
@@ -192,7 +186,7 @@ notebook <pytorch-monai-training.ipynb>`__.
.. code:: ipython3
state_dict_url = "https://github.com/helena-intel/openvino_notebooks/raw/110-nncf/notebooks/110-ct-segmentation-quantize/pretrained_model/unet_kits19_state_dict.pth"
state_dict_url = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/kidney-segmentation-kits19/unet_kits19_state_dict.pth"
state_dict_file = download_file(state_dict_url, directory="pretrained_model")
state_dict = torch.load(state_dict_file, map_location=torch.device("cpu"))
@@ -422,13 +416,8 @@ this notebook.
.. parsed-literal::
WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11.
[ WARNING ] Please fix your imports. Module tensorflow.python.training.tracking.base has been moved to tensorflow.python.trackable.base. The old module will be deleted in version 2.11.
.. parsed-literal::
[ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s.
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:179: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
if x_e.shape[-i - 1] != x_0.shape[-i - 1]:
`NNCF <https://github.com/openvinotoolkit/nncf>`__ provides a suite of
@@ -482,6 +471,19 @@ model and save it.
int8_ir_model = mo.convert_model(input_model=int8_onnx_path)
serialize(int8_ir_model, str(int8_ir_path))
.. parsed-literal::
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:338: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
return self._level_low.item()
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:346: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
return self._level_high.item()
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:179: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
if x_e.shape[-i - 1] != x_0.shape[-i - 1]:
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/quantize_functions.py:140: FutureWarning: 'torch.onnx._patch_torch._graph_op' is deprecated in version 1.13 and will be removed in version 1.14. Please note 'g.op()' is to be removed from torch.Graph. Please open a GitHub issue if you need this functionality..
output = g.op(
This notebook demonstrates post-training quantization with NNCF.
NNCF also supports quantization-aware training, and other algorithms
@@ -535,7 +537,7 @@ Compare Performance of the FP32 IR Model and Quantized Models
To measure the inference performance of the ``FP32`` and ``INT8``
models, we use `Benchmark
Tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_tool_README.html>`__
Tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_tool_README.html>`__
- OpenVINO’s inference performance measurement tool. Benchmark tool is a
command line application, part of OpenVINO development tools, that can
be run in the notebook with ``! benchmark_app`` or
@@ -568,35 +570,35 @@ be run in the notebook with ``! benchmark_app`` or
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to LATENCY.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.LATENCY.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 10.98 ms
[ INFO ] Read model took 22.91 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] input_0 (node: input_0) : f32 / [...] / [1,1,512,512]
[ INFO ] 1 , x (node: Parameter_2) : f32 / [...] / [1,1,512,512]
[ INFO ] Model outputs:
[ INFO ] 238 (node: 238) : f32 / [...] / [1,1,512,512]
[ INFO ] 169 (node: aten::_convolution_861) : f32 / [...] / [1,1,512,512]
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] input_0 (node: input_0) : f32 / [N,C,H,W] / [1,1,512,512]
[ INFO ] 1 , x (node: Parameter_2) : f32 / [N,C,H,W] / [1,1,512,512]
[ INFO ] Model outputs:
[ INFO ] 238 (node: 238) : f32 / [...] / [1,1,512,512]
[ INFO ] 169 (node: aten::_convolution_861) : f32 / [...] / [1,1,512,512]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 88.50 ms
[ INFO ] Compile model took 89.56 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
[ INFO ] NETWORK_NAME: Model0
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 1
[ INFO ] NUM_STREAMS: 1
[ INFO ] AFFINITY: Affinity.CORE
@@ -604,21 +606,27 @@ be run in the notebook with ``! benchmark_app`` or
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.LATENCY
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'input_0'!. This input will be filled with random values!
[ INFO ] Fill input 'input_0' with random values
[ WARNING ] No input files were given for input '1'!. This input will be filled with random values!
[ INFO ] Fill input '1' with random values
[Step 10/11] Measuring performance (Start inference synchronously, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 52.29 ms
[ INFO ] First inference took 55.95 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 427 iterations
[ INFO ] Duration: 15001.20 ms
[ INFO ] Duration: 15009.56 ms
[ INFO ] Latency:
[ INFO ] Median: 34.91 ms
[ INFO ] Average: 34.93 ms
[ INFO ] Min: 34.61 ms
[ INFO ] Max: 37.31 ms
[ INFO ] Min: 34.59 ms
[ INFO ] Max: 36.22 ms
[ INFO ] Throughput: 28.64 FPS
@@ -634,18 +642,18 @@ be run in the notebook with ``! benchmark_app`` or
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to LATENCY.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.LATENCY.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 24.51 ms
[ INFO ] Read model took 32.53 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] x.1 (node: x.1) : f32 / [...] / [1,1,512,512]
@@ -659,7 +667,7 @@ be run in the notebook with ``! benchmark_app`` or
[ INFO ] Model outputs:
[ INFO ] 578 (node: 578) : f32 / [...] / [1,1,512,512]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 149.51 ms
[ INFO ] Compile model took 145.14 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
@@ -670,22 +678,28 @@ be run in the notebook with ``! benchmark_app`` or
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.LATENCY
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'x.1'!. This input will be filled with random values!
[ INFO ] Fill input 'x.1' with random values
[Step 10/11] Measuring performance (Start inference synchronously, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 29.36 ms
[ INFO ] First inference took 31.93 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 995 iterations
[ INFO ] Duration: 15002.63 ms
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 984 iterations
[ INFO ] Duration: 15002.92 ms
[ INFO ] Latency:
[ INFO ] Median: 14.84 ms
[ INFO ] Average: 14.88 ms
[ INFO ] Min: 14.49 ms
[ INFO ] Max: 15.59 ms
[ INFO ] Throughput: 67.38 FPS
[ INFO ] Median: 15.02 ms
[ INFO ] Average: 15.04 ms
[ INFO ] Min: 14.70 ms
[ INFO ] Max: 18.85 ms
[ INFO ] Throughput: 66.57 FPS
Visually Compare Inference Results
@@ -768,7 +782,7 @@ seed is displayed to enable reproducing specific runs of this cell.
.. parsed-literal::
Visualizing results with seed 1685393451
Visualizing results with seed 1687380135
@@ -845,8 +859,8 @@ performs inference, and displays the results on the frames loaded in
.. parsed-literal::
Loaded model to CPU in 0.14 seconds.
Total time for 68 frames: 2.89 seconds, fps:23.84
Loaded model to CPU in 0.13 seconds.
Total time for 68 frames: 3.26 seconds, fps:21.19
References
docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/110-ct-segmentation-quantize-nncf-with-output_37_1.png
+2 -2
View File
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:45c8a85092a351c7636d6f17427ac38f29e9951137de1237e839411d503e019d
size 395128
oid sha256:e80df41d53a6af0956c94d93705d3cb2fb637c0b92bb41e1dddbbd305a0fffc1
size 377021
docs/notebooks/110-ct-segmentation-quantize-nncf-with-output_files/index.html
+5 -5
View File
@@ -1,9 +1,9 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/110-ct-segmentation-quantize-nncf-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/110-ct-segmentation-quantize-nncf-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/110-ct-segmentation-quantize-nncf-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="110-ct-segmentation-quantize-nncf-with-output_15_1.png">110-ct-segmentation-quantize-nncf-with-output_1..&gt;</a> 30-May-2023 00:09 158997
<a href="110-ct-segmentation-quantize-nncf-with-output_37_1.png">110-ct-segmentation-quantize-nncf-with-output_3..&gt;</a> 30-May-2023 00:09 395128
<a href="110-ct-segmentation-quantize-nncf-with-output_42_0.jpg">110-ct-segmentation-quantize-nncf-with-output_4..&gt;</a> 30-May-2023 00:09 73812
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/110-ct-segmentation-quantize-nncf-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="110-ct-segmentation-quantize-nncf-with-output_15_1.png">110-ct-segmentation-quantize-nncf-with-output_1..&gt;</a> 22-Jun-2023 00:06 158997
<a href="110-ct-segmentation-quantize-nncf-with-output_37_1.png">110-ct-segmentation-quantize-nncf-with-output_3..&gt;</a> 22-Jun-2023 00:06 377021
<a href="110-ct-segmentation-quantize-nncf-with-output_42_0.jpg">110-ct-segmentation-quantize-nncf-with-output_4..&gt;</a> 22-Jun-2023 00:06 73812
</pre><hr></body>
</html>
docs/notebooks/111-yolov5-quantization-migration-with-output.rst
+142 -92
View File
@@ -3,9 +3,9 @@ Migrate quantization from POT API to NNCF API
This tutorial demonstrates how to migrate quantization pipeline written
using the OpenVINO `Post-Training Optimization Tool
(POT) <https://docs.openvino.ai/latest/pot_introduction.html>`__ to
(POT) <https://docs.openvino.ai/2023.0/pot_introduction.html>`__ to
`NNCF Post-Training Quantization
API <https://docs.openvino.ai/latest/nncf_ptq_introduction.html>`__.
API <https://docs.openvino.ai/nightly/basic_quantization_flow.html>`__.
This tutorial is based on `Ultralytics
Yolov5 <https://github.com/ultralytics/yolov5>`__ model and additionally
it compares model accuracy between the FP32 precision and quantized INT8
@@ -31,7 +31,8 @@ Download the YOLOv5 model
.. code:: ipython3
!pip install -q psutil "seaborn>=0.11.0"
!pip install -q 'openvino-dev>=2023.0.0' 'nncf>=2.5.0'
!pip install -q psutil "seaborn>=0.11.0" matplotlib numpy onnx
.. code:: ipython3
@@ -108,7 +109,7 @@ weight. \* ``{MODEL_NAME}.onnx`` - the Open Neural Network Exchange
YOLOv5 🚀 v7.0-0-g915bbf2 Python-3.8.10 torch-1.13.1+cpu CPU
Downloading https://github.com/ultralytics/yolov5/releases/download/v7.0/yolov5m.pt to yolov5m/yolov5m.pt...
100%|██████████████████████████████████████| 40.8M/40.8M [00:09<00:00, 4.34MB/s]
100%|██████████████████████████████████████| 40.8M/40.8M [00:10<00:00, 4.09MB/s]
Fusing layers...
YOLOv5m summary: 290 layers, 21172173 parameters, 0 gradients
@@ -116,10 +117,10 @@ weight. \* ``{MODEL_NAME}.onnx`` - the Open Neural Network Exchange
PyTorch: starting from yolov5m/yolov5m.pt with output shape (1, 25200, 85) (40.8 MB)
ONNX: starting export with onnx 1.14.0...
ONNX: export success ✅ 1.0s, saved as yolov5m/yolov5m.onnx (81.2 MB)
ONNX: export success ✅ 1.2s, saved as yolov5m/yolov5m.onnx (81.2 MB)
Export complete (12.3s)
Results saved to /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/yolov5m
Export complete (13.4s)
Results saved to /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/yolov5m
Detect: python detect.py --weights yolov5m/yolov5m.onnx
Validate: python val.py --weights yolov5m/yolov5m.onnx
PyTorch Hub: model = torch.hub.load('ultralytics/yolov5', 'custom', 'yolov5m/yolov5m.onnx')
@@ -128,9 +129,9 @@ weight. \* ``{MODEL_NAME}.onnx`` - the Open Neural Network Exchange
Convert the ONNX model to OpenVINO Intermediate Representation (IR)
model generated by `Model
Optimizer <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html#doxid-openvino-docs-m-o-d-g-deep-learning-model-optimizer-dev-guide>`__.
Optimizer <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html#doxid-openvino-docs-m-o-d-g-deep-learning-model-optimizer-dev-guide>`__.
We will use `Model Optimizer Python
API <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Python_API.html>`__
API <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Python_API.html>`__
``openvino.tools.mo.convert_model`` function to convert ONNX model to
OpenVINO Model, then it can be seralized using
``openvino.runtime.serialize``\ As the result, directory with the
@@ -228,8 +229,8 @@ first.
.. parsed-literal::
Unzipping datasets/coco128.zip...
Scanning /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017... 126 images, 2 backgrounds, 0 corrupt: 100%|██████████| 128/128 00:00
New cache created: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017.cache
Scanning /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017... 126 images, 2 backgrounds, 0 corrupt: 100%|██████████| 128/128 00:00
New cache created: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017.cache
Create YOLOv5 DataLoader class for POT
@@ -303,10 +304,24 @@ index. Any implementation should override the following methods:
.. parsed-literal::
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/offline_transformations/__init__.py:10: FutureWarning: The module is private and following namespace `offline_transformations` will be removed in the future, use `openvino.runtime.passes` instead!
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/offline_transformations/__init__.py:10: FutureWarning: The module is private and following namespace `offline_transformations` will be removed in the future.
warnings.warn(
.. parsed-literal::
[ DEBUG ] Creating converter from 7 to 5
[ DEBUG ] Creating converter from 5 to 7
[ DEBUG ] Creating converter from 7 to 5
[ DEBUG ] Creating converter from 5 to 7
.. parsed-literal::
Post-training Optimization Tool is deprecated and will be removed in the future. Please use Neural Network Compression Framework instead: https://github.com/openvinotoolkit/nncf
Nevergrad package could not be imported. If you are planning to use any hyperparameter optimization algo, consider installing it using pip. This implies advanced usage of the tool. Note that nevergrad is compatible only with Python 3.7+
Create NNCF Dataset
~~~~~~~~~~~~~~~~~~~
@@ -410,7 +425,7 @@ Quantization parameters ``preset``, ``model_type``, ``subset_size``,
``fast_bias_correction``, ``ignored_scope`` are arguments of function.
More details about supported parameters and formats can be found in NNCF
Post-Training Quantization
`documentation <https://docs.openvino.ai/latest/basic_qauntization_flow.html#tune-quantization-parameters>`__.
`documentation <https://docs.openvino.ai/2023.0/basic_qauntization_flow.html#tune-quantization-parameters>`__.
NNCF also expect providing model object in inference framework format,
in our case ``openvino.runtime.Model`` instance created using
``core.read_model`` or ``openvino.tools.mo.convert_model``.
@@ -465,6 +480,13 @@ device for inferece and can be saved on disk using
nncf_int8_path = f"{MODEL_PATH}/NNCF_INT8_openvino_model/{MODEL_NAME}_int8.xml"
serialize(quantized_model, nncf_int8_path)
.. parsed-literal::
Statistics collection: 43%|████▎ | 128/300 [00:30<00:41, 4.18it/s]
Biases correction: 100%|██████████| 82/82 [00:10<00:00, 7.71it/s]
Compare accuracy FP32 and INT8 models
-------------------------------------
@@ -529,10 +551,10 @@ same directory, where model located.
.. parsed-literal::
Forcing --batch-size 1 square inference (1,3,640,640) for non-PyTorch models
val: Scanning /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017.cache... 126 images, 2 backgrounds, 0 corrupt: 100%|██████████| 128/128 00:00
val: Scanning /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017.cache... 126 images, 2 backgrounds, 0 corrupt: 100%|██████████| 128/128 00:00
Class Images Instances P R mAP50 mAP50-95: 100%|██████████| 128/128 00:05
all 128 929 0.726 0.687 0.769 0.554
Speed: 0.2ms pre-process, 35.7ms inference, 3.1ms NMS per image at shape (1, 3, 640, 640)
Speed: 0.2ms pre-process, 35.3ms inference, 3.1ms NMS per image at shape (1, 3, 640, 640)
Results saved to yolov5/runs/val/exp
@@ -561,6 +583,13 @@ same directory, where model located.
print(f"mAP@.5:.95 = {pot_int8_ap_full}")
.. parsed-literal::
YOLOv5 🚀 v7.0-0-g915bbf2 Python-3.8.10 torch-1.13.1+cpu CPU
Loading yolov5/yolov5m/POT_INT8_openvino_model for OpenVINO inference...
.. parsed-literal::
Checking the accuracy of the POT int8 model:
@@ -568,21 +597,18 @@ same directory, where model located.
.. parsed-literal::
YOLOv5 🚀 v7.0-0-g915bbf2 Python-3.8.10 torch-1.13.1+cpu CPU
Loading yolov5/yolov5m/POT_INT8_openvino_model for OpenVINO inference...
Forcing --batch-size 1 square inference (1,3,640,640) for non-PyTorch models
val: Scanning /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017.cache... 126 images, 2 backgrounds, 0 corrupt: 100%|██████████| 128/128 00:00
val: Scanning /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017.cache... 126 images, 2 backgrounds, 0 corrupt: 100%|██████████| 128/128 00:00
Class Images Instances P R mAP50 mAP50-95: 100%|██████████| 128/128 00:03
all 128 929 0.732 0.681 0.765 0.545
Speed: 0.2ms pre-process, 16.9ms inference, 3.4ms NMS per image at shape (1, 3, 640, 640)
all 128 929 0.761 0.677 0.773 0.548
Speed: 0.2ms pre-process, 17.1ms inference, 3.3ms NMS per image at shape (1, 3, 640, 640)
Results saved to yolov5/runs/val/exp2
.. parsed-literal::
mAP@.5 = 0.7647204368480177
mAP@.5:.95 = 0.5454870669362748
mAP@.5 = 0.7726143212109754
mAP@.5:.95 = 0.5482902837946336
.. code:: ipython3
@@ -619,17 +645,17 @@ same directory, where model located.
.. parsed-literal::
Forcing --batch-size 1 square inference (1,3,640,640) for non-PyTorch models
val: Scanning /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017.cache... 126 images, 2 backgrounds, 0 corrupt: 100%|██████████| 128/128 00:00
val: Scanning /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/datasets/coco128/labels/train2017.cache... 126 images, 2 backgrounds, 0 corrupt: 100%|██████████| 128/128 00:00
Class Images Instances P R mAP50 mAP50-95: 100%|██████████| 128/128 00:03
all 128 929 0.732 0.681 0.765 0.545
Speed: 0.2ms pre-process, 17.4ms inference, 3.4ms NMS per image at shape (1, 3, 640, 640)
all 128 929 0.742 0.684 0.766 0.546
Speed: 0.2ms pre-process, 17.0ms inference, 3.2ms NMS per image at shape (1, 3, 640, 640)
Results saved to yolov5/runs/val/exp3
.. parsed-literal::
mAP@.5 = 0.7647204368480177
mAP@.5:.95 = 0.5454870669362748
mAP@.5 = 0.7660170261123679
mAP@.5:.95 = 0.5460759842467641
Compare Average Precision of quantized INT8 model with original FP32
@@ -718,9 +744,9 @@ images.
'YOLOv5 🚀 v7.0-0-g915bbf2 Python-3.8.10 torch-1.13.1+cpu CPU',
'',
'Loading yolov5m/FP32_openvino_model for OpenVINO inference...',
'image 1/2 /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/bus.jpg: 640x640 4 persons, 1 bus, 57.2ms',
'image 2/2 /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/zidane.jpg: 640x640 3 persons, 2 ties, 47.7ms',
'Speed: 1.3ms pre-process, 52.4ms inference, 1.5ms NMS per image at shape (1, 3, 640, 640)',
'image 1/2 /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/bus.jpg: 640x640 4 persons, 1 bus, 64.1ms',
'image 2/2 /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/zidane.jpg: 640x640 3 persons, 2 ties, 60.5ms',
'Speed: 1.6ms pre-process, 62.3ms inference, 1.4ms NMS per image at shape (1, 3, 640, 640)',
'Results saved to \x1b[1mruns/detect/exp\x1b[0m']
@@ -745,9 +771,9 @@ images.
'YOLOv5 🚀 v7.0-0-g915bbf2 Python-3.8.10 torch-1.13.1+cpu CPU',
'',
'Loading yolov5m/POT_INT8_openvino_model for OpenVINO inference...',
'image 1/2 /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/bus.jpg: 640x640 4 persons, 1 bus, 34.3ms',
'image 2/2 /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/zidane.jpg: 640x640 3 persons, 2 ties, 31.4ms',
'Speed: 1.3ms pre-process, 32.9ms inference, 1.6ms NMS per image at shape (1, 3, 640, 640)',
'image 1/2 /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/bus.jpg: 640x640 4 persons, 1 bus, 36.4ms',
'image 2/2 /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/zidane.jpg: 640x640 3 persons, 1 tie, 30.9ms',
'Speed: 1.6ms pre-process, 33.6ms inference, 1.5ms NMS per image at shape (1, 3, 640, 640)',
'Results saved to \x1b[1mruns/detect/exp2\x1b[0m']
@@ -772,9 +798,9 @@ images.
'YOLOv5 🚀 v7.0-0-g915bbf2 Python-3.8.10 torch-1.13.1+cpu CPU',
'',
'Loading yolov5m/NNCF_INT8_openvino_model for OpenVINO inference...',
'image 1/2 /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/bus.jpg: 640x640 4 persons, 1 bus, 34.0ms',
'image 2/2 /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/zidane.jpg: 640x640 3 persons, 2 ties, 35.4ms',
'Speed: 1.3ms pre-process, 34.7ms inference, 1.6ms NMS per image at shape (1, 3, 640, 640)',
'image 1/2 /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/bus.jpg: 640x640 4 persons, 1 bus, 44.1ms',
'image 2/2 /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/111-yolov5-quantization-migration/yolov5/data/images/zidane.jpg: 640x640 3 persons, 2 ties, 35.7ms',
'Speed: 1.6ms pre-process, 39.9ms inference, 1.8ms NMS per image at shape (1, 3, 640, 640)',
'Results saved to \x1b[1mruns/detect/exp3\x1b[0m']
@@ -827,18 +853,18 @@ Benchmark
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to THROUGHPUT.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 47.87 ms
[ INFO ] Read model took 34.71 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] images (node: images) : f32 / [...] / [1,3,640,640]
@@ -852,7 +878,7 @@ Benchmark
[ INFO ] Model outputs:
[ INFO ] output0 (node: output0) : f32 / [...] / [1,25200,85]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 398.92 ms
[ INFO ] Compile model took 364.66 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
@@ -863,22 +889,28 @@ Benchmark
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'images'!. This input will be filled with random values!
[ INFO ] Fill input 'images' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 100.44 ms
[ INFO ] First inference took 103.67 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 444 iterations
[ INFO ] Duration: 15259.22 ms
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 456 iterations
[ INFO ] Duration: 15243.71 ms
[ INFO ] Latency:
[ INFO ] Median: 206.13 ms
[ INFO ] Average: 205.44 ms
[ INFO ] Min: 169.54 ms
[ INFO ] Max: 220.58 ms
[ INFO ] Throughput: 29.10 FPS
[ INFO ] Median: 200.52 ms
[ INFO ] Average: 199.48 ms
[ INFO ] Min: 167.24 ms
[ INFO ] Max: 219.12 ms
[ INFO ] Throughput: 29.91 FPS
.. code:: ipython3
@@ -898,18 +930,18 @@ Benchmark
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to THROUGHPUT.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 48.70 ms
[ INFO ] Read model took 34.33 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] images (node: images) : f32 / [...] / [1,3,640,640]
@@ -923,7 +955,7 @@ Benchmark
[ INFO ] Model outputs:
[ INFO ] output0 (node: output0) : f32 / [...] / [1,25200,85]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 466.54 ms
[ INFO ] Compile model took 388.69 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
@@ -934,22 +966,28 @@ Benchmark
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'images'!. This input will be filled with random values!
[ INFO ] Fill input 'images' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 95.11 ms
[ INFO ] First inference took 104.81 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 456 iterations
[ INFO ] Duration: 15341.96 ms
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 462 iterations
[ INFO ] Duration: 15265.35 ms
[ INFO ] Latency:
[ INFO ] Median: 203.47 ms
[ INFO ] Average: 201.45 ms
[ INFO ] Min: 163.81 ms
[ INFO ] Max: 214.79 ms
[ INFO ] Throughput: 29.72 FPS
[ INFO ] Median: 198.22 ms
[ INFO ] Average: 197.86 ms
[ INFO ] Min: 167.44 ms
[ INFO ] Max: 229.61 ms
[ INFO ] Throughput: 30.26 FPS
.. code:: ipython3
@@ -969,18 +1007,18 @@ Benchmark
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to THROUGHPUT.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 35.81 ms
[ INFO ] Read model took 68.51 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] images (node: images) : f32 / [...] / [1,3,640,640]
@@ -994,7 +1032,7 @@ Benchmark
[ INFO ] Model outputs:
[ INFO ] output0 (node: output0) : f32 / [...] / [1,25200,85]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 730.24 ms
[ INFO ] Compile model took 706.48 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
@@ -1005,22 +1043,28 @@ Benchmark
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'images'!. This input will be filled with random values!
[ INFO ] Fill input 'images' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 44.88 ms
[ INFO ] First inference took 49.42 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 1410 iterations
[ INFO ] Duration: 15055.41 ms
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 1416 iterations
[ INFO ] Duration: 15057.61 ms
[ INFO ] Latency:
[ INFO ] Median: 63.99 ms
[ INFO ] Average: 63.88 ms
[ INFO ] Min: 52.37 ms
[ INFO ] Max: 85.59 ms
[ INFO ] Throughput: 93.65 FPS
[ INFO ] Median: 63.81 ms
[ INFO ] Average: 63.58 ms
[ INFO ] Min: 47.42 ms
[ INFO ] Max: 84.90 ms
[ INFO ] Throughput: 94.04 FPS
.. code:: ipython3
@@ -1040,18 +1084,18 @@ Benchmark
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to THROUGHPUT.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 35.14 ms
[ INFO ] Read model took 60.65 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] images (node: images) : f32 / [...] / [1,3,640,640]
@@ -1065,7 +1109,7 @@ Benchmark
[ INFO ] Model outputs:
[ INFO ] output0 (node: output0) : f32 / [...] / [1,25200,85]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 728.24 ms
[ INFO ] Compile model took 715.09 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
@@ -1076,22 +1120,28 @@ Benchmark
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'images'!. This input will be filled with random values!
[ INFO ] Fill input 'images' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 44.67 ms
[ INFO ] First inference took 50.63 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 1410 iterations
[ INFO ] Duration: 15086.46 ms
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 1422 iterations
[ INFO ] Duration: 15094.14 ms
[ INFO ] Latency:
[ INFO ] Median: 64.06 ms
[ INFO ] Average: 63.99 ms
[ INFO ] Min: 46.71 ms
[ INFO ] Max: 86.28 ms
[ INFO ] Throughput: 93.46 FPS
[ INFO ] Median: 63.67 ms
[ INFO ] Average: 63.52 ms
[ INFO ] Min: 47.40 ms
[ INFO ] Max: 86.32 ms
[ INFO ] Throughput: 94.21 FPS
References
@@ -1099,8 +1149,8 @@ References
- `Ultralytics YOLOv5 <https://github.com/ultralytics/yolov5>`__
- `OpenVINO Post-training Optimization
Tool <https://docs.openvino.ai/latest/pot_introduction.html>`__
Tool <https://docs.openvino.ai/2023.0/pot_introduction.html>`__
- `NNCF Post-training
quantization <https://docs.openvino.ai/latest/nncf_ptq_introduction.html>`__
quantization <https://docs.openvino.ai/nightly/basic_quantization_flow.html>`__
- `Model
Optimizer <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html#doxid-openvino-docs-m-o-d-g-deep-learning-model-optimizer-dev-guide>`__
Optimizer <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html#doxid-openvino-docs-m-o-d-g-deep-learning-model-optimizer-dev-guide>`__
docs/notebooks/111-yolov5-quantization-migration-with-output_files/111-yolov5-quantization-migration-with-output_33_0.png
+2 -2
View File
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:4f425198a6bd63de3965dd680133d01367400db27a6532b337e669e226a9de10
size 32366
oid sha256:a7d7f50fd1250dd0675bd2dbaf57b27692cbc4648c9bd8caf0a90c9f303b7049
size 33667
docs/notebooks/111-yolov5-quantization-migration-with-output_files/111-yolov5-quantization-migration-with-output_39_0.png
+2 -2
View File
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:7e203ecda608c2cd3be4c3e43cfee86ec29fa538b3efc951e2dda7858fc057c6
size 720871
oid sha256:60c7a0fd81574ebcc9d02c1966c0c6dd30204e7c7de7cea02227faefc0b27708
size 770524
docs/notebooks/111-yolov5-quantization-migration-with-output_files/index.html
+4 -4
View File
@@ -1,8 +1,8 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/111-yolov5-quantization-migration-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/111-yolov5-quantization-migration-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/111-yolov5-quantization-migration-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="111-yolov5-quantization-migration-with-output_33_0.png">111-yolov5-quantization-migration-with-output_3..&gt;</a> 30-May-2023 00:08 32366
<a href="111-yolov5-quantization-migration-with-output_39_0.png">111-yolov5-quantization-migration-with-output_3..&gt;</a> 30-May-2023 00:08 720871
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/111-yolov5-quantization-migration-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="111-yolov5-quantization-migration-with-output_33_0.png">111-yolov5-quantization-migration-with-output_3..&gt;</a> 22-Jun-2023 00:06 33667
<a href="111-yolov5-quantization-migration-with-output_39_0.png">111-yolov5-quantization-migration-with-output_3..&gt;</a> 22-Jun-2023 00:06 770524
</pre><hr></body>
</html>
docs/notebooks/112-pytorch-post-training-quantization-nncf-with-output.rst
+69 -51
View File
@@ -88,12 +88,10 @@ Imports
.. parsed-literal::
2023-05-29 22:55:16.336185: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-05-29 22:55:16.370677: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2023-06-21 22:46:53.036679: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-06-21 22:46:53.070884: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-05-29 22:55:16.922906: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/offline_transformations/__init__.py:10: FutureWarning: The module is private and following namespace `offline_transformations` will be removed in the future, use `openvino.runtime.passes` instead!
warnings.warn(
2023-06-21 22:46:53.615742: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
.. parsed-literal::
@@ -144,7 +142,7 @@ Settings
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/112-pytorch-post-training-quantization-nncf/model/resnet50_fp32.pth')
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/112-pytorch-post-training-quantization-nncf/model/resnet50_fp32.pth')
@@ -423,15 +421,15 @@ I. Evaluate the loaded model
.. parsed-literal::
Test: [ 0/79] Time 0.257 (0.257) Acc@1 81.25 (81.25) Acc@5 92.19 (92.19)
Test: [10/79] Time 0.230 (0.235) Acc@1 56.25 (66.97) Acc@5 86.72 (87.50)
Test: [20/79] Time 0.231 (0.236) Acc@1 67.97 (64.29) Acc@5 85.16 (87.35)
Test: [30/79] Time 0.230 (0.239) Acc@1 53.12 (62.37) Acc@5 77.34 (85.33)
Test: [40/79] Time 0.232 (0.239) Acc@1 67.19 (60.86) Acc@5 90.62 (84.51)
Test: [50/79] Time 0.231 (0.238) Acc@1 60.16 (60.80) Acc@5 88.28 (84.42)
Test: [60/79] Time 0.253 (0.237) Acc@1 66.41 (60.46) Acc@5 86.72 (83.79)
Test: [70/79] Time 0.232 (0.236) Acc@1 52.34 (60.21) Acc@5 80.47 (83.33)
* Acc@1 60.740 Acc@5 83.960 Total time: 18.431
Test: [ 0/79] Time 0.292 (0.292) Acc@1 81.25 (81.25) Acc@5 92.19 (92.19)
Test: [10/79] Time 0.234 (0.243) Acc@1 56.25 (66.97) Acc@5 86.72 (87.50)
Test: [20/79] Time 0.235 (0.239) Acc@1 67.97 (64.29) Acc@5 85.16 (87.35)
Test: [30/79] Time 0.233 (0.239) Acc@1 53.12 (62.37) Acc@5 77.34 (85.33)
Test: [40/79] Time 0.233 (0.238) Acc@1 67.19 (60.86) Acc@5 90.62 (84.51)
Test: [50/79] Time 0.284 (0.239) Acc@1 60.16 (60.80) Acc@5 88.28 (84.42)
Test: [60/79] Time 0.236 (0.240) Acc@1 66.41 (60.46) Acc@5 86.72 (83.79)
Test: [70/79] Time 0.233 (0.239) Acc@1 52.34 (60.21) Acc@5 80.47 (83.33)
* Acc@1 60.740 Acc@5 83.960 Total time: 18.662
Test accuracy of FP32 model: 60.740
@@ -445,7 +443,7 @@ layers. The framework is designed so that modifications to your original
training code are minor. Quantization is the simplest scenario and
requires a few modifications. For more information about NNCF Post
Training Quantization (PTQ) API, refer to the `Basic Quantization Flow
Guide <https://docs.openvino.ai/latest/basic_qauntization_flow.html#doxid-basic-qauntization-flow>`__.
Guide <https://docs.openvino.ai/2023.0/basic_qauntization_flow.html#doxid-basic-qauntization-flow>`__.
1. Create a transformation function that accepts a sample from the
dataset and returns data suitable for model inference. This enables
@@ -482,6 +480,9 @@ Guide <https://docs.openvino.ai/latest/basic_qauntization_flow.html#doxid-basic-
INFO:nncf:Collecting tensor statistics |████████████████| 3 / 3
INFO:nncf:Compiling and loading torch extension: quantized_functions_cpu...
INFO:nncf:Finished loading torch extension: quantized_functions_cpu
INFO:nncf:BatchNorm statistics adaptation |█████ | 1 / 3
INFO:nncf:BatchNorm statistics adaptation |██████████ | 2 / 3
INFO:nncf:BatchNorm statistics adaptation |████████████████| 3 / 3
3. Evaluate the new model on the validation set after initialization of
@@ -497,16 +498,16 @@ Guide <https://docs.openvino.ai/latest/basic_qauntization_flow.html#doxid-basic-
.. parsed-literal::
Test: [ 0/79] Time 0.385 (0.385) Acc@1 80.47 (80.47) Acc@5 91.41 (91.41)
Test: [10/79] Time 0.380 (0.380) Acc@1 52.34 (66.48) Acc@5 85.94 (87.50)
Test: [20/79] Time 0.377 (0.379) Acc@1 68.75 (63.95) Acc@5 85.94 (87.20)
Test: [30/79] Time 0.382 (0.379) Acc@1 51.56 (62.22) Acc@5 73.44 (85.23)
Test: [40/79] Time 0.380 (0.379) Acc@1 67.19 (60.63) Acc@5 89.84 (84.34)
Test: [50/79] Time 0.377 (0.379) Acc@1 61.72 (60.66) Acc@5 87.50 (84.24)
Test: [60/79] Time 0.372 (0.379) Acc@1 64.84 (60.32) Acc@5 85.94 (83.71)
Test: [70/79] Time 0.380 (0.378) Acc@1 50.78 (60.00) Acc@5 79.69 (83.27)
* Acc@1 60.570 Acc@5 83.850 Total time: 29.644
Accuracy of initialized INT8 model: 60.570
Test: [ 0/79] Time 0.427 (0.427) Acc@1 81.25 (81.25) Acc@5 92.19 (92.19)
Test: [10/79] Time 0.420 (0.421) Acc@1 55.47 (67.33) Acc@5 87.50 (87.86)
Test: [20/79] Time 0.417 (0.419) Acc@1 67.97 (64.32) Acc@5 88.28 (87.69)
Test: [30/79] Time 0.421 (0.420) Acc@1 52.34 (62.53) Acc@5 78.91 (85.56)
Test: [40/79] Time 0.422 (0.420) Acc@1 67.97 (61.01) Acc@5 90.62 (84.76)
Test: [50/79] Time 0.421 (0.421) Acc@1 58.59 (60.85) Acc@5 87.50 (84.60)
Test: [60/79] Time 0.423 (0.421) Acc@1 66.41 (60.50) Acc@5 87.50 (83.94)
Test: [70/79] Time 0.422 (0.421) Acc@1 50.00 (60.22) Acc@5 78.12 (83.45)
* Acc@1 60.730 Acc@5 84.060 Total time: 32.959
Accuracy of initialized INT8 model: 60.730
It should be noted that the inference time for the quantized PyTorch
@@ -524,7 +525,7 @@ benchmarking.
For more information about Model Optimizer, refer to the `Model
Optimizer Developer
Guide <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
Before converting models export them to ONNX. Executing the following
command may take a while.
@@ -545,6 +546,23 @@ command may take a while.
serialize(quantized_model_ir, str(int8_ir_path))
.. parsed-literal::
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:338: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
return self._level_low.item()
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:346: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
return self._level_high.item()
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/quantize_functions.py:140: FutureWarning: 'torch.onnx._patch_torch._graph_op' is deprecated in version 1.13 and will be removed in version 1.14. Please note 'g.op()' is to be removed from torch.Graph. Please open a GitHub issue if you need this functionality..
output = g.op(
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/_patch_torch.py:81: UserWarning: The shape inference of org.openvinotoolkit::FakeQuantize type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
_C._jit_pass_onnx_node_shape_type_inference(
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:687: UserWarning: The shape inference of org.openvinotoolkit::FakeQuantize type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
_C._jit_pass_onnx_graph_shape_type_inference(
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:1178: UserWarning: The shape inference of org.openvinotoolkit::FakeQuantize type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
_C._jit_pass_onnx_graph_shape_type_inference(
Evaluate the FP32 and INT8 models.
.. code:: ipython3
@@ -557,15 +575,15 @@ Evaluate the FP32 and INT8 models.
.. parsed-literal::
Test: [ 0/79] Time 0.168 (0.168) Acc@1 81.25 (81.25) Acc@5 92.19 (92.19)
Test: [10/79] Time 0.118 (0.122) Acc@1 56.25 (66.97) Acc@5 86.72 (87.50)
Test: [20/79] Time 0.116 (0.120) Acc@1 67.97 (64.29) Acc@5 85.16 (87.35)
Test: [30/79] Time 0.118 (0.119) Acc@1 53.12 (62.37) Acc@5 77.34 (85.33)
Test: [40/79] Time 0.117 (0.119) Acc@1 67.19 (60.86) Acc@5 90.62 (84.51)
Test: [50/79] Time 0.117 (0.119) Acc@1 60.16 (60.80) Acc@5 88.28 (84.42)
Test: [60/79] Time 0.118 (0.119) Acc@1 66.41 (60.46) Acc@5 86.72 (83.79)
Test: [70/79] Time 0.118 (0.119) Acc@1 52.34 (60.21) Acc@5 80.47 (83.33)
* Acc@1 60.740 Acc@5 83.960 Total time: 9.280
Test: [ 0/79] Time 0.200 (0.200) Acc@1 81.25 (81.25) Acc@5 92.19 (92.19)
Test: [10/79] Time 0.140 (0.145) Acc@1 56.25 (66.97) Acc@5 86.72 (87.50)
Test: [20/79] Time 0.140 (0.143) Acc@1 67.97 (64.29) Acc@5 85.16 (87.35)
Test: [30/79] Time 0.140 (0.142) Acc@1 53.12 (62.37) Acc@5 77.34 (85.33)
Test: [40/79] Time 0.138 (0.141) Acc@1 67.19 (60.86) Acc@5 90.62 (84.51)
Test: [50/79] Time 0.140 (0.141) Acc@1 60.16 (60.80) Acc@5 88.28 (84.42)
Test: [60/79] Time 0.140 (0.141) Acc@1 66.41 (60.46) Acc@5 86.72 (83.79)
Test: [70/79] Time 0.138 (0.141) Acc@1 52.34 (60.21) Acc@5 80.47 (83.33)
* Acc@1 60.740 Acc@5 83.960 Total time: 11.002
Accuracy of FP32 IR model: 60.740
@@ -578,16 +596,16 @@ Evaluate the FP32 and INT8 models.
.. parsed-literal::
Test: [ 0/79] Time 0.116 (0.116) Acc@1 80.47 (80.47) Acc@5 91.41 (91.41)
Test: [10/79] Time 0.076 (0.082) Acc@1 54.69 (66.83) Acc@5 85.94 (87.71)
Test: [20/79] Time 0.077 (0.079) Acc@1 69.53 (63.95) Acc@5 85.94 (87.28)
Test: [30/79] Time 0.078 (0.079) Acc@1 51.56 (62.17) Acc@5 73.44 (85.26)
Test: [40/79] Time 0.079 (0.079) Acc@1 68.75 (60.75) Acc@5 89.84 (84.30)
Test: [50/79] Time 0.078 (0.078) Acc@1 60.94 (60.71) Acc@5 87.50 (84.15)
Test: [60/79] Time 0.078 (0.078) Acc@1 64.84 (60.35) Acc@5 85.94 (83.64)
Test: [70/79] Time 0.077 (0.078) Acc@1 51.56 (60.05) Acc@5 79.69 (83.24)
* Acc@1 60.580 Acc@5 83.830 Total time: 6.118
Accuracy of INT8 IR model: 60.580
Test: [ 0/79] Time 0.186 (0.186) Acc@1 82.03 (82.03) Acc@5 92.97 (92.97)
Test: [10/79] Time 0.083 (0.095) Acc@1 57.81 (67.90) Acc@5 86.72 (88.28)
Test: [20/79] Time 0.079 (0.088) Acc@1 67.19 (64.88) Acc@5 85.16 (87.72)
Test: [30/79] Time 0.081 (0.085) Acc@1 52.34 (62.80) Acc@5 78.12 (85.61)
Test: [40/79] Time 0.082 (0.084) Acc@1 67.97 (61.32) Acc@5 89.84 (84.60)
Test: [50/79] Time 0.081 (0.084) Acc@1 60.16 (61.09) Acc@5 87.50 (84.47)
Test: [60/79] Time 0.079 (0.083) Acc@1 67.97 (60.71) Acc@5 88.28 (83.80)
Test: [70/79] Time 0.079 (0.082) Acc@1 53.12 (60.35) Acc@5 80.47 (83.37)
* Acc@1 60.880 Acc@5 83.950 Total time: 6.438
Accuracy of INT8 IR model: 60.880
IV. Compare performance of INT8 model and FP32 model in OpenVINO
@@ -595,7 +613,7 @@ IV. Compare performance of INT8 model and FP32 model in OpenVINO
Finally, measure the inference performance of the ``FP32`` and ``INT8``
models, using `Benchmark
Tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_tool_README.html>`__
Tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_tool_README.html>`__
- an inference performance measurement tool in OpenVINO. By default,
Benchmark Tool runs inference for 60 seconds in asynchronous mode on
CPU. It returns inference speed as latency (milliseconds per image) and
@@ -637,13 +655,13 @@ throughput (frames per second) values.
.. parsed-literal::
Benchmark FP32 model (OpenVINO IR)
[ INFO ] Throughput: 37.57 FPS
[ INFO ] Throughput: 37.97 FPS
Benchmark INT8 model (OpenVINO IR)
[ INFO ] Throughput: 157.46 FPS
[ INFO ] Throughput: 157.50 FPS
Benchmark FP32 model (OpenVINO IR) synchronously
[ INFO ] Throughput: 38.73 FPS
[ INFO ] Throughput: 39.17 FPS
Benchmark INT8 model (OpenVINO IR) synchronously
[ INFO ] Throughput: 140.53 FPS
[ INFO ] Throughput: 140.62 FPS
Show CPU Information for reference:
docs/notebooks/113-image-classification-quantization-with-output.rst
+246 -301
View File
@@ -2,62 +2,44 @@ Quantization of Image Classification Models
===========================================
This tutorial demonstrates how to apply ``INT8`` quantization to Image
Classification model using `Post-training Optimization Tool
API <../../compression/api/README.md>`__. It also assumes that OpenVINO™
is already installed and it uses the Mobilenet V2 model, trained on
Cifar10 dataset. The code is designed to be extendable to custom models
and datasets.
Classification model using
`NNCF <https://github.com/openvinotoolkit/nncf>`__. It uses the
Mobilenet V2 model, trained on Cifar10 dataset. The code is designed to
be extendable to custom models and datasets. The tutorial uses OpenVINO
backend for performing model quantization in NNCF, if you interested how
to apply quantization on PyTorch model, please check this
`tutorial <112-pytorch-post-training-quantization-nncf-with-output.html>`__.
This tutorial consists of the following steps: - Prepare the model for
quantization. - Define a data loading and an accuracy validation
functionality. - Run optimization pipeline. - Compare accuracy of the
original and quantized models. - Compare performance of the original and
quantized models. - Compare results on one picture.
quantization. - Define a data loading functionality. - Perform
quantization. - Compare accuracy of the original and quantized models. -
Compare performance of the original and quantized models. - Compare
results on one picture.
.. code:: ipython3
import os
from pathlib import Path
import sys
import matplotlib.pyplot as plt
import numpy as np
from openvino.tools.pot.api import DataLoader, Metric
from openvino.tools.pot.engines.ie_engine import IEEngine
from openvino.tools.pot.graph import load_model, save_model
from openvino.tools.pot.graph.model_utils import compress_model_weights
from openvino.tools.pot.pipeline.initializer import create_pipeline
from openvino.runtime import Core
from torchvision import transforms
from torchvision.datasets import CIFAR10
.. parsed-literal::
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/offline_transformations/__init__.py:10: FutureWarning: The module is private and following namespace `offline_transformations` will be removed in the future, use `openvino.runtime.passes` instead!
warnings.warn(
.. code:: ipython3
# Set the data and model directories
DATA_DIR = '../data/datasets/cifar10'
MODEL_DIR = 'model'
DATA_DIR = Path('../data/datasets/cifar10')
MODEL_DIR = Path('model')
model_repo = 'pytorch-cifar-models'
os.makedirs(DATA_DIR, exist_ok=True)
os.makedirs(MODEL_DIR, exist_ok=True)
DATA_DIR.mkdir(exist_ok=True)
MODEL_DIR.mkdir(exist_ok=True)
Prepare the Model
-----------------
Model preparation stage has the following steps: - Download a PyTorch
model from Torchvision repository - Convert it to ONNX format - Run
Model Optimizer to convert ONNX to OpenVINO Intermediate Representation
(OpenVINO IR)
model - Convert model to OpenVINO Intermediate Representation format
(IR) using Model Optimizer Python API - Serialize converted model on
disk
.. code:: ipython3
import sys
if not Path(model_repo).exists():
!git clone https://github.com/chenyaofo/pytorch-cifar-models.git
@@ -69,10 +51,10 @@ Model Optimizer to convert ONNX to OpenVINO Intermediate Representation
Cloning into 'pytorch-cifar-models'...
remote: Enumerating objects: 282, done.
remote: Counting objects: 100% (281/281), done.
remote: Compressing objects: 100% (95/95), done.
remote: Total 282 (delta 136), reused 269 (delta 129), pack-reused 1
Receiving objects: 100% (282/282), 9.22 MiB | 4.06 MiB/s, done.
Resolving deltas: 100% (136/136), done.
remote: Compressing objects: 100% (96/96), done.
remote: Total 282 (delta 135), reused 269 (delta 128), pack-reused 1
Receiving objects: 100% (282/282), 9.22 MiB | 4.20 MiB/s, done.
Resolving deltas: 100% (135/135), done.
.. code:: ipython3
@@ -81,43 +63,53 @@ Model Optimizer to convert ONNX to OpenVINO Intermediate Representation
model = cifar10_mobilenetv2_x1_0(pretrained=True)
OpenVINO support PyTorch models via conversion to OpenVINO Intermediate
Representation format. Model Optimizer Python API should be used for
conversion. ``mo.convert_model`` accept PyTorch model instance and
convert it into ``openvino.runtime.Model`` representation of model in
OpenVINO. Optionally, you may specify ``example_input`` which serves as
helper for model tracing and ``input_shape`` for conversion model with
static shape. Conveted model is ready to be loaded on device for
inference and can be saved on disk for next usage via ``serialize``
function. More details about Model Optimizer Python API can be found on
this
`page <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Python_API.html>`__.
.. code:: ipython3
import torch
from openvino.tools import mo
from openvino.runtime import serialize
model.eval()
dummy_input = torch.randn(1, 3, 32, 32)
ov_model = mo.convert_model(model, input_shape=[1,3,32,32])
onnx_model_path = Path(MODEL_DIR) / 'mobilenet_v2.onnx'
ir_model_xml = onnx_model_path.with_suffix('.xml')
ir_model_bin = onnx_model_path.with_suffix('.bin')
torch.onnx.export(model, dummy_input, onnx_model_path)
# Run Model Optimizer to convert ONNX to OpenVINO IR.
!mo --compress_to_fp16 -m $onnx_model_path --output_dir $MODEL_DIR
serialize(ov_model, MODEL_DIR / "mobilenet_v2.xml")
Prepare Dataset
---------------
.. parsed-literal::
Check for a new version of Intel(R) Distribution of OpenVINO(TM) toolkit here https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit/download.html?cid=other&source=prod&campid=ww_2023_bu_IOTG_OpenVINO-2022-3&content=upg_all&medium=organic or on https://github.com/openvinotoolkit/openvino
[ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11.
Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/latest/openvino_2_0_transition_guide.html
[ SUCCESS ] Generated IR version 11 model.
[ SUCCESS ] XML file: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/113-image-classification-quantization/model/mobilenet_v2.xml
[ SUCCESS ] BIN file: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/113-image-classification-quantization/model/mobilenet_v2.bin
Define Data Loader
------------------
In this step, the ``DataLoader`` interface from POT API is implemented.
We will use `CIFAR10 <https://www.cs.toronto.edu/~kriz/cifar.html>`__
dataset from
`torchvision <https://pytorch.org/vision/stable/generated/torchvision.datasets.CIFAR10.html>`__.
Preprocessing for model obtained from training
`config <https://github.com/chenyaofo/image-classification-codebase/blob/master/conf/cifar10.conf>`__
.. code:: ipython3
import torch
from torchvision import transforms
from torchvision.datasets import CIFAR10
transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.247, 0.243, 0.261))])
dataset = CIFAR10(root=DATA_DIR, train=False, transform=transform, download=True)
val_loader = torch.utils.data.DataLoader(
dataset,
batch_size=1,
shuffle=False,
num_workers=0,
pin_memory=True,
)
.. parsed-literal::
@@ -136,182 +128,129 @@ In this step, the ``DataLoader`` interface from POT API is implemented.
Extracting ../data/datasets/cifar10/cifar-10-python.tar.gz to ../data/datasets/cifar10
.. code:: ipython3
Perform Quantization
--------------------
# Create a DataLoader from a CIFAR10 dataset.
class CifarDataLoader(DataLoader):
def __init__(self, config):
"""
Initialize config and dataset.
:param config: created config with DATA_DIR path.
"""
super().__init__(config)
self.dataset = dataset
def __len__(self):
return len(self.dataset)
def __getitem__(self, index):
"""
Return one sample of index, label and picture.
:param index: index of the taken sample.
"""
image, label = self.dataset[index]
return (index, label), image.numpy()
def load_data(self, dataset):
"""
Load dataset in needed format.
:param dataset: downloaded dataset.
"""
pictures, labels, indexes = [], [], []
for idx, sample in enumerate(dataset):
pictures.append(sample[0])
labels.append(sample[1])
indexes.append(idx)
return indexes, pictures, labels
`NNCF <https://github.com/openvinotoolkit/nncf>`__ provides a suite of
advanced algorithms for Neural Networks inference optimization in
OpenVINO with minimal accuracy drop. We will use 8-bit quantization in
post-training mode (without the fine-tuning pipeline) to optimize
MobilenetV2. The optimization process contains the following steps:
Define Accuracy Metric Calculation
----------------------------------
1. Create a Dataset for quantization.
2. Run ``nncf.quantize`` for getting an optimized model.
3. Serialize an OpenVINO IR model, using the
``openvino.runtime.serialize`` function.
In this step, the ``Metric`` interface for accuracy Top-1 metric is
implemented. It is used for validating accuracy of quantized model.
Create Datset for Validation
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
NNCF is compatible with ``torch.utils.data.DataLoader`` interface. For
performing quantization it should be passed into ``nncf.Dataset`` object
with transformation function, which prepares input data to fit into
model during quantization, in our case, to pick input tensor from pair
(input tensor and label) and convert PyTorch tensor to numpy.
.. code:: ipython3
# Custom implementation of classification accuracy metric.
import nncf
class Accuracy(Metric):
def transform_fn(data_item):
image_tensor = data_item[0]
return image_tensor.numpy()
# Required methods
def __init__(self, top_k=1):
super().__init__()
self._top_k = top_k
self._name = 'accuracy@top{}'.format(self._top_k)
self._matches = []
@property
def value(self):
""" Returns accuracy metric value for the last model output. """
return {self._name: self._matches[-1]}
@property
def avg_value(self):
""" Returns accuracy metric value for all model outputs. """
return {self._name: np.ravel(self._matches).mean()}
def update(self, output, target):
""" Updates prediction matches.
:param output: model output
:param target: annotations
"""
if len(output) > 1:
raise Exception('The accuracy metric cannot be calculated '
'for a model with multiple outputs')
if isinstance(target, dict):
target = list(target.values())
predictions = np.argsort(output[0], axis=1)[:, -self._top_k:]
match = [float(t in predictions[i]) for i, t in enumerate(target)]
self._matches.append(match)
def reset(self):
""" Resets collected matches """
self._matches = []
def get_attributes(self):
"""
Returns a dictionary of metric attributes {metric_name: {attribute_name: value}}.
Required attributes: 'direction': 'higher-better' or 'higher-worse'
'type': metric type
"""
return {self._name: {'direction': 'higher-better',
'type': 'accuracy'}}
Run Quantization Pipeline and compare the accuracy of the original and quantized models
---------------------------------------------------------------------------------------
In this step, define a configuration for the quantization pipeline and
run it.
**NOTE**: Use built-in ``IEEngine`` implementation of the ``Engine``
interface from the POT API for model inference. ``IEEngine`` is built
on top of OpenVINO Python API for inference and provides basic
functionality for inference of simple models. If you have a more
complicated inference flow for your model/models, you should create
your own implementation of ``Engine`` interface, for example, by
inheriting from ``IEEngine`` and extending it.
.. code:: ipython3
model_config = {
'model_name': 'mobilenet_v2',
'model': ir_model_xml,
'weights': ir_model_bin
}
engine_config = {'device': 'CPU'}
dataset_config = {
'data_source': DATA_DIR
}
algorithms = [
{
'name': 'DefaultQuantization',
'params': {
'target_device': 'CPU',
'preset': 'performance',
'stat_subset_size': 300
}
}
]
# Steps 1-7: Model optimization
# Step 1: Load the model.
model = load_model(model_config)
# Step 2: Initialize the data loader.
data_loader = CifarDataLoader(dataset_config)
# Step 3 (Optional. Required for AccuracyAwareQuantization): Initialize the metric.
metric = Accuracy(top_k=1)
# Step 4: Initialize the engine for metric calculation and statistics collection.
engine = IEEngine(engine_config, data_loader, metric)
# Step 5: Create a pipeline of compression algorithms.
pipeline = create_pipeline(algorithms, engine)
# Step 6: Execute the pipeline.
compressed_model = pipeline.run(model)
# Step 7 (Optional): Compress model weights quantized precision
# in order to reduce the size of final .bin file.
compress_model_weights(compressed_model)
# Step 8: Save the compressed model to the desired path.
compressed_model_paths = save_model(model=compressed_model, save_path=MODEL_DIR, model_name="quantized_mobilenet_v2"
)
compressed_model_xml = compressed_model_paths[0]["model"]
compressed_model_bin = Path(compressed_model_paths[0]["model"]).with_suffix(".bin")
# Step 9: Compare accuracy of the original and quantized models.
metric_results = pipeline.evaluate(model)
if metric_results:
for name, value in metric_results.items():
print(f"Accuracy of the original model: {name}: {value}")
metric_results = pipeline.evaluate(compressed_model)
if metric_results:
for name, value in metric_results.items():
print(f"Accuracy of the optimized model: {name}: {value}")
quantization_dataset = nncf.Dataset(val_loader, transform_fn)
.. parsed-literal::
Accuracy of the original model: accuracy@top1: 0.9348
Accuracy of the optimized model: accuracy@top1: 0.9348
INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino
Run nncf.quantize for Getting an Optimized Model
------------------------------------------------
``nncf.quantize`` function accepts model and prepared quantization
dataset for performing basic quantization. Optionally, additional
parameters like ``subset_size``, ``preset``, ``ignored_scope`` can be
provided to improve quantization result if applicable. More details
about supported parameters can be found on this
`page <https://docs.openvino.ai/2023.0/basic_quantization_flow.html#tune-quantization-parameters>`__
.. code:: ipython3
quant_ov_model = nncf.quantize(ov_model, quantization_dataset)
.. parsed-literal::
Statistics collection: 100%|██████████| 300/300 [00:08<00:00, 35.76it/s]
Biases correction: 100%|██████████| 36/36 [00:01<00:00, 20.36it/s]
Serialize an OpenVINO IR model
------------------------------
Similar to ``mo.convert_model``, quantized model is
``openvino.runtime.Model`` object which ready to be loaded into device
and can be serialized on disk using ``openvino.runtime.serialize``.
.. code:: ipython3
serialize(quant_ov_model, MODEL_DIR / "quantized_mobilenet_v2.xml")
Compare Accuracy of the Original and Quantized Models
-----------------------------------------------------
.. code:: ipython3
from tqdm.notebook import tqdm
import numpy as np
def test_accuracy(ov_model, data_loader):
correct = 0
total = 0
for (batch_imgs, batch_labels) in tqdm(data_loader):
result = ov_model(batch_imgs)[0]
top_label = np.argmax(result)
correct += top_label == batch_labels.numpy()
total += 1
return correct / total
.. code:: ipython3
from openvino.runtime import Core
core = Core()
compiled_model = core.compile_model(ov_model)
optimized_compiled_model = core.compile_model(quant_ov_model)
orig_accuracy = test_accuracy(compiled_model, val_loader)
optimized_accuracy = test_accuracy(optimized_compiled_model, val_loader)
.. parsed-literal::
0%| | 0/10000 [00:00<?, ?it/s]
.. parsed-literal::
0%| | 0/10000 [00:00<?, ?it/s]
.. code:: ipython3
print(f"Accuracy of the original model: {orig_accuracy[0] * 100 :.2f}%")
print(f"Accuracy of the optimized model: {optimized_accuracy[0] * 100 :.2f}%")
.. parsed-literal::
Accuracy of the original model: 93.61%
Accuracy of the optimized model: 93.51%
Compare Performance of the Original and Quantized Models
@@ -319,7 +258,7 @@ Compare Performance of the Original and Quantized Models
Finally, measure the inference performance of the ``FP32`` and ``INT8``
models, using `Benchmark
Tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_tool_README.html>`__
Tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_tool_README.html>`__
- an inference performance measurement tool in OpenVINO.
**NOTE**: For more accurate performance, it is recommended to run
@@ -332,7 +271,7 @@ Tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_
.. code:: ipython3
# Inference FP16 model (OpenVINO IR)
!benchmark_app -m $ir_model_xml -d CPU -api async
!benchmark_app -m "model/mobilenet_v2.xml" -d CPU -api async -t 15
.. parsed-literal::
@@ -341,35 +280,35 @@ Tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to THROUGHPUT.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 31.89 ms
[ INFO ] Read model took 25.72 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] input.1 (node: input.1) : f32 / [...] / [1,3,32,32]
[ INFO ] x.1 , x , 1 (node: Parameter_2) : f32 / [...] / [1,3,32,32]
[ INFO ] Model outputs:
[ INFO ] 536 (node: 536) : f32 / [...] / [1,10]
[ INFO ] 223 (node: aten::linear_928) : f32 / [...] / [1,10]
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] input.1 (node: input.1) : u8 / [N,C,H,W] / [1,3,32,32]
[ INFO ] x.1 , x , 1 (node: Parameter_2) : u8 / [N,C,H,W] / [1,3,32,32]
[ INFO ] Model outputs:
[ INFO ] 536 (node: 536) : f32 / [...] / [1,10]
[ INFO ] 223 (node: aten::linear_928) : f32 / [...] / [1,10]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 176.27 ms
[ INFO ] Compile model took 144.38 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
[ INFO ] NETWORK_NAME: Model0
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 12
[ INFO ] NUM_STREAMS: 12
[ INFO ] AFFINITY: Affinity.CORE
@@ -377,28 +316,34 @@ Tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'input.1'!. This input will be filled with random values!
[ INFO ] Fill input 'input.1' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 12 inference requests, limits: 60000 ms duration)
[ WARNING ] No input files were given for input '1'!. This input will be filled with random values!
[ INFO ] Fill input '1' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 12 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 3.08 ms
[ INFO ] First inference took 3.09 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 338496 iterations
[ INFO ] Duration: 60002.33 ms
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 90984 iterations
[ INFO ] Duration: 15002.82 ms
[ INFO ] Latency:
[ INFO ] Median: 1.97 ms
[ INFO ] Average: 1.97 ms
[ INFO ] Min: 1.11 ms
[ INFO ] Max: 20.45 ms
[ INFO ] Throughput: 5641.38 FPS
[ INFO ] Median: 1.78 ms
[ INFO ] Average: 1.80 ms
[ INFO ] Min: 1.10 ms
[ INFO ] Max: 6.27 ms
[ INFO ] Throughput: 6064.46 FPS
.. code:: ipython3
# Inference INT8 model (OpenVINO IR)
!benchmark_app -m $compressed_model_xml -d CPU -api async
!benchmark_app -m "model/quantized_mobilenet_v2.xml" -d CPU -api async -t 15
.. parsed-literal::
@@ -407,35 +352,35 @@ Tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to THROUGHPUT.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 18.32 ms
[ INFO ] Read model took 34.93 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] input.1 (node: input.1) : f32 / [...] / [1,3,32,32]
[ INFO ] x , 1 , x.1 (node: Parameter_2) : f32 / [...] / [1,3,32,32]
[ INFO ] Model outputs:
[ INFO ] 536 (node: 536) : f32 / [...] / [1,10]
[ INFO ] 223 (node: aten::linear_928) : f32 / [...] / [1,10]
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] input.1 (node: input.1) : u8 / [N,C,H,W] / [1,3,32,32]
[ INFO ] x , 1 , x.1 (node: Parameter_2) : u8 / [N,C,H,W] / [1,3,32,32]
[ INFO ] Model outputs:
[ INFO ] 536 (node: 536) : f32 / [...] / [1,10]
[ INFO ] 223 (node: aten::linear_928) : f32 / [...] / [1,10]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 256.00 ms
[ INFO ] Compile model took 247.66 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
[ INFO ] NETWORK_NAME: Model0
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 12
[ INFO ] NUM_STREAMS: 12
[ INFO ] AFFINITY: Affinity.CORE
@@ -443,57 +388,49 @@ Tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'input.1'!. This input will be filled with random values!
[ INFO ] Fill input 'input.1' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 12 inference requests, limits: 60000 ms duration)
[ WARNING ] No input files were given for input '1'!. This input will be filled with random values!
[ INFO ] Fill input '1' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 12 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 1.56 ms
[ INFO ] First inference took 1.68 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 761820 iterations
[ INFO ] Duration: 60001.53 ms
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 183924 iterations
[ INFO ] Duration: 15000.75 ms
[ INFO ] Latency:
[ INFO ] Median: 0.90 ms
[ INFO ] Average: 0.92 ms
[ INFO ] Min: 0.62 ms
[ INFO ] Max: 5.38 ms
[ INFO ] Throughput: 12696.68 FPS
[ INFO ] Median: 0.92 ms
[ INFO ] Average: 0.94 ms
[ INFO ] Min: 0.64 ms
[ INFO ] Max: 3.03 ms
[ INFO ] Throughput: 12260.99 FPS
Compare results on four pictures.
---------------------------------
Compare results on four pictures
--------------------------------
.. code:: ipython3
ie = Core()
# Read and load a float model.
float_model = ie.read_model(
model=ir_model_xml, weights=ir_model_bin
)
float_compiled_model = ie.compile_model(model=float_model, device_name="CPU")
# Read and load a quantized model.
quantized_model = ie.read_model(
model=compressed_model_xml, weights=compressed_model_bin
)
quantized_compiled_model = ie.compile_model(model=quantized_model, device_name="CPU")
.. code:: ipython3
# Define all possible labels from the CIFAR10 dataset.
# Define all possible labels from the CIFAR10 dataset
labels_names = ["airplane", "automobile", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck"]
all_pictures = []
all_labels = []
# Get all pictures and their labels.
for i, batch in enumerate(data_loader):
all_pictures.append(batch[1])
all_labels.append(batch[0][1])
for i, batch in enumerate(val_loader):
all_pictures.append(batch[0].numpy())
all_labels.append(batch[1].item())
.. code:: ipython3
import matplotlib.pyplot as plt
def plot_pictures(indexes: list, all_pictures=all_pictures, all_labels=all_labels):
"""Plot 4 pictures.
:param indexes: a list of indexes of pictures to be displayed.
@@ -533,7 +470,7 @@ Compare results on four pictures.
predicted_labels = []
for idx in indexes:
assert idx < 10000, 'Cannot get such index, there are only 10000'
result = model([all_pictures[idx][None,]])[output_key]
result = model(all_pictures[idx])[output_key]
result = labels_names[np.argmax(result[0])]
predicted_labels.append(result)
return predicted_labels
@@ -544,13 +481,21 @@ Compare results on four pictures.
plot_pictures(indexes_to_infer)
results_float = infer_on_pictures(float_compiled_model, indexes_to_infer)
results_quanized = infer_on_pictures(quantized_compiled_model, indexes_to_infer)
results_float = infer_on_pictures(compiled_model, indexes_to_infer)
results_quanized = infer_on_pictures(optimized_compiled_model, indexes_to_infer)
print(f"Labels for picture from float model : {results_float}.")
print(f"Labels for picture from quantized model : {results_quanized}.")
.. parsed-literal::
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
.. parsed-literal::
Labels for picture from float model : ['frog', 'dog', 'ship', 'horse'].
@@ -558,5 +503,5 @@ Compare results on four pictures.
.. image:: 113-image-classification-quantization-with-output_files/113-image-classification-quantization-with-output_22_1.png
.. image:: 113-image-classification-quantization-with-output_files/113-image-classification-quantization-with-output_26_2.png
docs/notebooks/113-image-classification-quantization-with-output_files/113-image-classification-quantization-with-output_22_1.png → docs/notebooks/113-image-classification-quantization-with-output_files/113-image-classification-quantization-with-output_26_2.png
View File
File renamed without changes.
docs/notebooks/113-image-classification-quantization-with-output_files/index.html
+3 -3
View File
@@ -1,7 +1,7 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/113-image-classification-quantization-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/113-image-classification-quantization-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/113-image-classification-quantization-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="113-image-classification-quantization-with-output_22_1.png">113-image-classification-quantization-with-outp..&gt;</a> 30-May-2023 00:09 14855
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/113-image-classification-quantization-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="113-image-classification-quantization-with-output_26_2.png">113-image-classification-quantization-with-outp..&gt;</a> 22-Jun-2023 00:06 14855
</pre><hr></body>
</html>
docs/notebooks/114-quantization-simplified-mode-with-output.rst
-418
View File
@@ -1,418 +0,0 @@
INT8 Quantization with Post-training Optimization Tool (POT) in Simplified Mode tutorial
========================================================================================
This tutorial shows how to quantize a
`ResNet20 <https://github.com/chenyaofo/pytorch-cifar-models>`__ image
classification model, trained on
`CIFAR10 <http://pytorch.org/vision/main/generated/torchvision.datasets.CIFAR10.html>`__
dataset, using the Post-Training Optimization Tool (POT) in Simplified
Mode.
Simplified Mode is designed to make the data preparation step easier,
before model optimization. The mode is represented by an implementation
of the engine interface in the POT API in OpenVINO™. It enables reading
data from an arbitrary folder specified by the user. Currently,
Simplified Mode is available only for image data in PNG or JPEG formats,
stored in a single folder.
**NOTE:** This mode cannot be used with the accuracy-aware method. It
is not possible to control accuracy after optimization using this
mode. However, Simplified Mode can be useful for estimating
performance improvements when optimizing models.
This tutorial includes the following steps:
- Downloading and saving the CIFAR10 dataset.
- Preparing the model for quantization.
- Compressing the prepared model.
- Measuring and comparing the performance of the original and quantized
models.
- Demonstrating the use of the quantized model for image
classification.
.. code:: ipython3
import os
from pathlib import Path
import warnings
import torch
from torchvision import transforms as T
from torchvision.datasets import CIFAR10
import matplotlib.pyplot as plt
import numpy as np
from openvino.runtime import Core, Tensor
warnings.filterwarnings("ignore")
# Set the data and model directories
MODEL_DIR = 'model'
CALIB_DIR = 'calib'
CIFAR_DIR = '../data/datasets/cifar10'
CALIB_SET_SIZE = 300
MODEL_NAME = 'resnet20'
os.makedirs(MODEL_DIR, exist_ok=True)
os.makedirs(CALIB_DIR, exist_ok=True)
Prepare the calibration dataset
-------------------------------
The following steps are required to prepare the calibration dataset: -
Download the CIFAR10 dataset from `Torchvision.datasets
repository <https://pytorch.org/vision/stable/datasets.html>`__. - Save
the selected number of elements from this dataset as ``.png`` images in
a separate folder.
.. code:: ipython3
transform = T.Compose([T.ToTensor()])
dataset = CIFAR10(root=CIFAR_DIR, train=False, transform=transform, download=True)
.. parsed-literal::
Files already downloaded and verified
.. code:: ipython3
pil_converter = T.ToPILImage(mode="RGB")
for idx, info in enumerate(dataset):
im = info[0]
if idx >= CALIB_SET_SIZE:
break
label = info[1]
pil_converter(im.squeeze(0)).save(Path(CALIB_DIR) / f'{label}_{idx}.png')
Prepare the Model
-----------------
Model preparation includes the following steps: - Download PyTorch model
from Torchvision repository. - Convert the model to ONNX format. - Run
Model Optimizer to convert ONNX to OpenVINO Intermediate Representation
(OpenVINO IR).
.. code:: ipython3
model = torch.hub.load("chenyaofo/pytorch-cifar-models", "cifar10_resnet20", pretrained=True, skip_validation=True)
dummy_input = torch.randn(1, 3, 32, 32)
onnx_model_path = Path(MODEL_DIR) / '{}.onnx'.format(MODEL_NAME)
ir_model_xml = onnx_model_path.with_suffix('.xml')
ir_model_bin = onnx_model_path.with_suffix('.bin')
torch.onnx.export(model, dummy_input, onnx_model_path)
.. parsed-literal::
Using cache found in /opt/home/k8sworker/.cache/torch/hub/chenyaofo_pytorch-cifar-models_master
Now, convert this model into the OpenVINO IR using Model Optimizer:
.. code:: ipython3
!mo -m $onnx_model_path --output_dir $MODEL_DIR
.. parsed-literal::
Check for a new version of Intel(R) Distribution of OpenVINO(TM) toolkit here https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit/download.html?cid=other&source=prod&campid=ww_2023_bu_IOTG_OpenVINO-2022-3&content=upg_all&medium=organic or on https://github.com/openvinotoolkit/openvino
[ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11.
Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/latest/openvino_2_0_transition_guide.html
[ SUCCESS ] Generated IR version 11 model.
[ SUCCESS ] XML file: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/114-quantization-simplified-mode/model/resnet20.xml
[ SUCCESS ] BIN file: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/114-quantization-simplified-mode/model/resnet20.bin
Compression stage
-----------------
Compress the model with the following command:
``pot -q default -m <path_to_xml> -w <path_to_bin> --engine simplified --data-source <path_to_data>``
.. code:: ipython3
!pot -q default -m $ir_model_xml -w $ir_model_bin --engine simplified --data-source $CALIB_DIR --output-dir compressed --direct-dump --name $MODEL_NAME
.. parsed-literal::
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/offline_transformations/__init__.py:10: FutureWarning: The module is private and following namespace `offline_transformations` will be removed in the future, use `openvino.runtime.passes` instead!
warnings.warn(
INFO:openvino.tools.pot.app.run:Output log dir: compressed
INFO:openvino.tools.pot.app.run:Creating pipeline:
Algorithm: DefaultQuantization
Parameters:
preset : performance
stat_subset_size : 300
target_device : ANY
model_type : None
dump_intermediate_model : False
inplace_statistics : True
exec_log_dir : compressed
===========================================================================
INFO:openvino.tools.pot.data_loaders.image_loader:Layout value is set [N,C,H,W]
INFO:openvino.tools.pot.pipeline.pipeline:Inference Engine version: 2022.3.0-9052-9752fafe8eb-releases/2022/3
INFO:openvino.tools.pot.pipeline.pipeline:Model Optimizer version: 2022.3.0-9052-9752fafe8eb-releases/2022/3
INFO:openvino.tools.pot.pipeline.pipeline:Post-Training Optimization Tool version: 2022.3.0-9052-9752fafe8eb-releases/2022/3
INFO:openvino.tools.pot.statistics.collector:Start computing statistics for algorithms : DefaultQuantization
INFO:openvino.tools.pot.statistics.collector:Computing statistics finished
INFO:openvino.tools.pot.pipeline.pipeline:Start algorithm: DefaultQuantization
INFO:openvino.tools.pot.algorithms.quantization.default.algorithm:Start computing statistics for algorithm : ActivationChannelAlignment
INFO:openvino.tools.pot.algorithms.quantization.default.algorithm:Computing statistics finished
INFO:openvino.tools.pot.algorithms.quantization.default.algorithm:Start computing statistics for algorithms : MinMaxQuantization,FastBiasCorrection
INFO:openvino.tools.pot.algorithms.quantization.default.algorithm:Computing statistics finished
INFO:openvino.tools.pot.pipeline.pipeline:Finished: DefaultQuantization
===========================================================================
Compare Performance of the Original and Quantized Models
--------------------------------------------------------
Finally, measure the inference performance of the ``FP32`` and ``INT8``
models, using `Benchmark
Tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_tool_README.html>`__
- an inference performance measurement tool in OpenVINO.
**NOTE**: For more accurate performance, it is recommended to run
benchmark_app in a terminal/command prompt after closing other
applications. Run ``benchmark_app -m model.xml -d CPU`` to benchmark
async inference on CPU for one minute. Change CPU to GPU to benchmark
on GPU. Run ``benchmark_app --help`` to see an overview of all
command-line options.
.. code:: ipython3
optimized_model_path = Path('compressed/optimized')
optimized_model_xml = optimized_model_path / '{}.xml'.format(MODEL_NAME)
optimized_model_bin = optimized_model_path / '{}.bin'.format(MODEL_NAME)
.. code:: ipython3
# Inference FP32 model (OpenVINO IR)
!benchmark_app -m $ir_model_xml -d CPU -api async
.. parsed-literal::
[Step 1/11] Parsing and validating input arguments
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 6.18 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] input.1 (node: input.1) : f32 / [...] / [1,3,32,32]
[ INFO ] Model outputs:
[ INFO ] 208 (node: 208) : f32 / [...] / [1,10]
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] input.1 (node: input.1) : u8 / [N,C,H,W] / [1,3,32,32]
[ INFO ] Model outputs:
[ INFO ] 208 (node: 208) : f32 / [...] / [1,10]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 75.31 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 12
[ INFO ] NUM_STREAMS: 12
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] INFERENCE_NUM_THREADS: 24
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'input.1'!. This input will be filled with random values!
[ INFO ] Fill input 'input.1' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 12 inference requests, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 1.08 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 969060 iterations
[ INFO ] Duration: 60000.75 ms
[ INFO ] Latency:
[ INFO ] Median: 0.69 ms
[ INFO ] Average: 0.71 ms
[ INFO ] Min: 0.41 ms
[ INFO ] Max: 12.50 ms
[ INFO ] Throughput: 16150.80 FPS
.. code:: ipython3
# Inference INT8 model (OpenVINO IR)
!benchmark_app -m $optimized_model_xml -d CPU -api async
.. parsed-literal::
[Step 1/11] Parsing and validating input arguments
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 9.50 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] input.1 (node: input.1) : f32 / [...] / [1,3,32,32]
[ INFO ] Model outputs:
[ INFO ] 208 (node: 208) : f32 / [...] / [1,10]
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] input.1 (node: input.1) : u8 / [N,C,H,W] / [1,3,32,32]
[ INFO ] Model outputs:
[ INFO ] 208 (node: 208) : f32 / [...] / [1,10]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 117.27 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 12
[ INFO ] NUM_STREAMS: 12
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] INFERENCE_NUM_THREADS: 24
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'input.1'!. This input will be filled with random values!
[ INFO ] Fill input 'input.1' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 12 inference requests, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 0.68 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 1587024 iterations
[ INFO ] Duration: 60000.59 ms
[ INFO ] Latency:
[ INFO ] Median: 0.35 ms
[ INFO ] Average: 0.36 ms
[ INFO ] Min: 0.22 ms
[ INFO ] Max: 13.28 ms
[ INFO ] Throughput: 26450.14 FPS
Demonstration of the results
----------------------------
This section demonstrates how to use the compressed model by running the
optimized model on a subset of images from the CIFAR10 dataset and shows
predictions, using the model.
The first step is to load the model:
.. code:: ipython3
ie = Core()
compiled_model = ie.compile_model(str(optimized_model_xml))
.. code:: ipython3
# Define all possible labels from the CIFAR10 dataset.
labels_names = ["airplane", "automobile", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck"]
all_images = []
all_labels = []
# Get all images and their labels.
for batch in dataset:
all_images.append(torch.unsqueeze(batch[0], 0))
all_labels.append(batch[1])
The code below defines the function that shows the images and their
labels, using the indexes and two lists created in the previous step:
.. code:: ipython3
def plot_pictures(indexes: list, images=all_images, labels=all_labels):
"""Plot images with the specified indexes.
:param indexes: a list of indexes of images to be displayed.
:param images: a list of images from the dataset.
:param labels: a list of labels for each image.
"""
num_pics = len(indexes)
_, axarr = plt.subplots(1, num_pics)
for idx, im_idx in enumerate(indexes):
assert idx < 10000, 'Cannot get such index, there are only 10000'
pic = np.rollaxis(images[im_idx].squeeze().numpy(), 0, 3)
axarr[idx].imshow(pic)
axarr[idx].set_title(labels_names[labels[im_idx]])
Use the code below, to define a function that uses the optimized model
to obtain predictions for the selected images:
.. code:: ipython3
def infer_on_images(net, indexes: list, images=all_images):
""" Inference model on a set of images.
:param net: model on which do inference
:param indexes: a list of indexes of images to infer on.
:param images: a list of images from the dataset.
"""
predicted_labels = []
infer_request = net.create_infer_request()
for idx in indexes:
assert idx < 10000, 'Cannot get such index, there are only 10000'
input_tensor = Tensor(array=images[idx].detach().numpy(), shared_memory=True)
infer_request.set_input_tensor(input_tensor)
infer_request.start_async()
infer_request.wait()
output = infer_request.get_output_tensor()
result = list(output.data)
result = labels_names[np.argmax(result[0])]
predicted_labels.append(result)
return predicted_labels
.. code:: ipython3
indexes_to_infer = [0, 1, 2] # to plot specify indexes
plot_pictures(indexes_to_infer)
results_quanized = infer_on_images(compiled_model, indexes_to_infer)
print(f"Image labels using the quantized model : {results_quanized}.")
.. parsed-literal::
Image labels using the quantized model : ['cat', 'ship', 'ship'].
.. image:: 114-quantization-simplified-mode-with-output_files/114-quantization-simplified-mode-with-output_22_1.png
docs/notebooks/114-quantization-simplified-mode-with-output_files/114-quantization-simplified-mode-with-output_22_1.png
-3
View File
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:bb6fad57e6e22313d4c934b4fdf1f6dc41ddf15d28a0f347cb194d1698433d26
size 18766
docs/notebooks/114-quantization-simplified-mode-with-output_files/index.html
-7
View File
@@ -1,7 +0,0 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/114-quantization-simplified-mode-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/114-quantization-simplified-mode-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="114-quantization-simplified-mode-with-output_22_1.png">114-quantization-simplified-mode-with-output_22..&gt;</a> 30-May-2023 00:08 18766
</pre><hr></body>
</html>
docs/notebooks/115-async-api-with-output.rst
+4 -4
View File
@@ -243,7 +243,7 @@ Test performance in Sync Mode
.. parsed-literal::
Source ended
average throuput in sync mode: 41.15 fps
average throuput in sync mode: 36.81 fps
Async Mode
@@ -372,7 +372,7 @@ Test the performance in Async Mode
.. parsed-literal::
Source ended
average throuput in async mode: 71.19 fps
average throuput in async mode: 72.96 fps
Compare the performance
@@ -407,7 +407,7 @@ AsyncInferQueue
---------------
Asynchronous mode pipelines can be supported with the
`AsyncInferQueue <https://docs.openvino.ai/latest/openvino_docs_OV_UG_Python_API_exclusives.html#asyncinferqueue>`__
`AsyncInferQueue <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Python_API_exclusives.html#asyncinferqueue>`__
wrapper class. This class automatically spawns the pool of InferRequest
objects (also called “jobs”) and provides synchronization mechanisms to
control the flow of the pipeline. It is a simpler way to manage the
@@ -507,5 +507,5 @@ Test the performance with AsyncInferQueue
.. parsed-literal::
average throughput in async mode with async infer queue: 111.22 fps
average throughput in async mode with async infer queue: 102.11 fps
docs/notebooks/115-async-api-with-output_files/115-async-api-with-output_20_0.png
+2 -2
View File
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:ff21ce3891ef2571506610d671a32b99a694bb6afa66d47820cac5d4476b6b41
size 30403
oid sha256:4959e1b241bf2e8076746dfc5428e583a79b2f87d251a2b0d53dae9ee1cb4b20
size 30500
docs/notebooks/115-async-api-with-output_files/index.html
+6 -6
View File
@@ -1,10 +1,10 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/115-async-api-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/115-async-api-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/115-async-api-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="115-async-api-with-output_14_0.png">115-async-api-with-output_14_0.png</a> 30-May-2023 00:08 4307
<a href="115-async-api-with-output_18_0.png">115-async-api-with-output_18_0.png</a> 30-May-2023 00:08 4307
<a href="115-async-api-with-output_20_0.png">115-async-api-with-output_20_0.png</a> 30-May-2023 00:08 30403
<a href="115-async-api-with-output_26_0.png">115-async-api-with-output_26_0.png</a> 30-May-2023 00:08 4307
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/115-async-api-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="115-async-api-with-output_14_0.png">115-async-api-with-output_14_0.png</a> 22-Jun-2023 00:06 4307
<a href="115-async-api-with-output_18_0.png">115-async-api-with-output_18_0.png</a> 22-Jun-2023 00:06 4307
<a href="115-async-api-with-output_20_0.png">115-async-api-with-output_20_0.png</a> 22-Jun-2023 00:06 30500
<a href="115-async-api-with-output_26_0.png">115-async-api-with-output_26_0.png</a> 22-Jun-2023 00:06 4307
</pre><hr></body>
</html>
docs/notebooks/116-sparsity-optimization-with-output.rst
+50 -46
View File
@@ -12,7 +12,7 @@ datasets <https://huggingface.co/datasets/sst2>`__ using
`Optimum-Intel <https://github.com/huggingface/optimum-intel>`__. It
demonstrates the inference performance advantage on 4th Gen Intel® Xeon®
Scalable Processors by running it with `Sparse Weight
Decompression <https://docs.openvino.ai/latest/openvino_docs_OV_UG_supported_plugins_CPU.html#sparse-weights-decompression>`__,
Decompression <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_supported_plugins_CPU.html#sparse-weights-decompression>`__,
a runtime option that seizes model sparsity for efficiency. The notebook
consists of the following steps:
@@ -26,15 +26,9 @@ Prerequisites
.. code:: ipython3
!pip install -q 'openvino-dev>=2023.0.0'
!pip install -q "git+https://github.com/huggingface/optimum-intel.git" datasets onnx onnxruntime
.. parsed-literal::
ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
tensorflow 2.12.0 requires protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3, but you have protobuf 3.20.2 which is incompatible.
Imports
-------
@@ -50,12 +44,10 @@ Imports
.. parsed-literal::
2023-05-29 23:05:52.938123: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-05-29 23:05:52.973374: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2023-06-21 22:54:03.983988: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-06-21 22:54:04.017706: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-05-29 23:05:53.524698: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/offline_transformations/__init__.py:10: FutureWarning: The module is private and following namespace `offline_transformations` will be removed in the future, use `openvino.runtime.passes` instead!
warnings.warn(
2023-06-21 22:54:04.587651: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
.. parsed-literal::
@@ -99,14 +91,14 @@ model card on Hugging Face.
.. parsed-literal::
Compiling the model and creating the inference request ...
Xformers is not installed correctly. If you want to use memorry_efficient_attention to accelerate training use the following command to install Xformers
Compiling the model...
Xformers is not installed correctly. If you want to use memory_efficient_attention to accelerate training use the following command to install Xformers
pip install xformers.
.. parsed-literal::
[{'label': 'negative', 'score': 0.9981877207756042}]
[{'label': 'negative', 'score': 0.9982142448425293}]
For benchmarking, we will use OpenVINO’s benchmark application and put
@@ -171,18 +163,18 @@ as an example. It is recommended to tune based on your applications.
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to THROUGHPUT.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 192.47 ms
[ INFO ] Read model took 67.89 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] input_ids (node: input_ids) : i64 / [...] / [?,?]
@@ -193,7 +185,7 @@ as an example. It is recommended to tune based on your applications.
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[ INFO ] Reshaping model: 'input_ids': [1,64], 'attention_mask': [1,64], 'token_type_ids': [1,64]
[ INFO ] Reshape model took 33.52 ms
[ INFO ] Reshape model took 25.89 ms
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] input_ids (node: input_ids) : i64 / [...] / [1,64]
@@ -202,7 +194,7 @@ as an example. It is recommended to tune based on your applications.
[ INFO ] Model outputs:
[ INFO ] logits (node: logits) : f32 / [...] / [1,2]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 1506.31 ms
[ INFO ] Compile model took 1229.91 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
@@ -213,7 +205,12 @@ as an example. It is recommended to tune based on your applications.
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'input_ids'!. This input will be filled with random values!
[ WARNING ] No input files were given for input 'attention_mask'!. This input will be filled with random values!
@@ -223,16 +220,17 @@ as an example. It is recommended to tune based on your applications.
[ INFO ] Fill input 'token_type_ids' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 4 inference requests, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 33.84 ms
[ INFO ] First inference took 29.99 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 8796 iterations
[ INFO ] Duration: 60041.91 ms
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 8908 iterations
[ INFO ] Duration: 60048.39 ms
[ INFO ] Latency:
[ INFO ] Median: 27.14 ms
[ INFO ] Average: 27.19 ms
[ INFO ] Min: 26.12 ms
[ INFO ] Max: 40.25 ms
[ INFO ] Throughput: 146.50 FPS
[ INFO ] Median: 26.80 ms
[ INFO ] Average: 26.84 ms
[ INFO ] Min: 25.30 ms
[ INFO ] Max: 40.79 ms
[ INFO ] Throughput: 148.35 FPS
Benchmark quantized sparse inference performance
@@ -272,18 +270,18 @@ for which a layer will be enabled.
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to THROUGHPUT.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 194.14 ms
[ INFO ] Read model took 72.33 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] input_ids (node: input_ids) : i64 / [...] / [?,?]
@@ -294,7 +292,7 @@ for which a layer will be enabled.
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[ INFO ] Reshaping model: 'input_ids': [1,64], 'attention_mask': [1,64], 'token_type_ids': [1,64]
[ INFO ] Reshape model took 33.32 ms
[ INFO ] Reshape model took 26.21 ms
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] input_ids (node: input_ids) : i64 / [...] / [1,64]
@@ -303,7 +301,7 @@ for which a layer will be enabled.
[ INFO ] Model outputs:
[ INFO ] logits (node: logits) : f32 / [...] / [1,2]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 1517.33 ms
[ INFO ] Compile model took 1287.76 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
@@ -314,7 +312,12 @@ for which a layer will be enabled.
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'input_ids'!. This input will be filled with random values!
[ WARNING ] No input files were given for input 'attention_mask'!. This input will be filled with random values!
@@ -324,16 +327,17 @@ for which a layer will be enabled.
[ INFO ] Fill input 'token_type_ids' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 4 inference requests, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 29.59 ms
[ INFO ] First inference took 31.87 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 8796 iterations
[ INFO ] Duration: 60033.65 ms
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 8896 iterations
[ INFO ] Duration: 60049.93 ms
[ INFO ] Latency:
[ INFO ] Median: 27.01 ms
[ INFO ] Average: 27.06 ms
[ INFO ] Min: 24.65 ms
[ INFO ] Max: 40.45 ms
[ INFO ] Throughput: 146.52 FPS
[ INFO ] Median: 26.76 ms
[ INFO ] Average: 26.80 ms
[ INFO ] Min: 24.55 ms
[ INFO ] Max: 42.25 ms
[ INFO ] Throughput: 148.14 FPS
When this might be helpful
@@ -346,6 +350,6 @@ small sequence length, for example, 32 and lower.
For more details about asynchronous inference with OpenVINO, refer to
the following documentation: - `Deployment Optimization
Guide <https://docs.openvino.ai/latest/openvino_docs_deployment_optimization_guide_common.html#doxid-openvino-docs-deployment-optimization-guide-common-1async-api>`__
Guide <https://docs.openvino.ai/2023.0/openvino_docs_deployment_optimization_guide_common.html#doxid-openvino-docs-deployment-optimization-guide-common-1async-api>`__
- `Inference Request
API <https://docs.openvino.ai/latest/openvino_docs_OV_UG_Infer_request.html#doxid-openvino-docs-o-v-u-g-infer-request-1in-out-tensors>`__
API <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Infer_request.html#doxid-openvino-docs-o-v-u-g-infer-request-1in-out-tensors>`__
docs/notebooks/117-model-server-with-output.rst
+17 -8
View File
@@ -145,19 +145,28 @@ structure and according to the following rules:
.. code:: ipython3
import os
import shutil
import sys
sys.path.append("../utils")
from notebook_utils import download_file
dedicated_dir = "models"
model_name = "detection"
model_version = "1"
MODEL_DIR = f"{dedicated_dir}/{model_name}/{model_version}"
XML_PATH = "../004-hello-detection/model/horizontal-text-detection-0001.xml"
BIN_PATH = "../004-hello-detection/model/horizontal-text-detection-0001.bin"
XML_PATH = "horizontal-text-detection-0001.xml"
BIN_PATH = "horizontal-text-detection-0001.bin"
os.makedirs(MODEL_DIR, exist_ok=True)
shutil.copy(XML_PATH, MODEL_DIR)
shutil.copy(BIN_PATH, MODEL_DIR)
print(f"Model Copied to \"./{MODEL_DIR}\".")
model_xml_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.xml"
model_bin_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.bin"
download_file(model_xml_url, XML_PATH, MODEL_DIR)
download_file(model_bin_url, BIN_PATH_name, MODEL_DIR)model_xml_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.xml"
model_bin_url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.3/models_bin/1/horizontal-text-detection-0001/FP32/horizontal-text-detection-0001.bin"
download_file(model_xml_url, model_xml_name, base_model_dir)
download_file(model_bin_url, model_bin_name, base_model_dir)
.. parsed-literal::
@@ -194,7 +203,7 @@ Check whether the OVMS container is running normally:
The required Model Server parameters are listed below. For additional
configuration options, see the `Model Server Parameters
section <https://docs.openvino.ai/latest/ovms_docs_parameters.html#doxid-ovms-docs-parameters>`__.
section <https://docs.openvino.ai/2023.0/ovms_docs_parameters.html#doxid-ovms-docs-parameters>`__.
.. raw:: html
@@ -845,5 +854,5 @@ References
----------
1. `OpenVINO™ Model
Server <https://docs.openvino.ai/latest/ovms_what_is_openvino_model_server.html>`__
Server <https://docs.openvino.ai/2023.0/ovms_what_is_openvino_model_server.html>`__
2. `openvinotoolkit/model_server <https://github.com/openvinotoolkit/model_server/>`__
docs/notebooks/117-model-server-with-output_files/index.html
+4 -4
View File
@@ -1,8 +1,8 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/117-model-server-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/117-model-server-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/117-model-server-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="117-model-server-with-output_20_1.png">117-model-server-with-output_20_1.png</a> 30-May-2023 00:08 112408
<a href="117-model-server-with-output_25_1.png">117-model-server-with-output_25_1.png</a> 30-May-2023 00:08 232667
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/117-model-server-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="117-model-server-with-output_20_1.png">117-model-server-with-output_20_1.png</a> 22-Jun-2023 00:06 112408
<a href="117-model-server-with-output_25_1.png">117-model-server-with-output_25_1.png</a> 22-Jun-2023 00:06 232667
</pre><hr></body>
</html>
docs/notebooks/118-optimize-preprocessing-with-output.rst
+26 -44
View File
@@ -9,9 +9,9 @@ instrument, that enables integration of preprocessing steps into an
execution graph and performing it on a selected device, which can
improve device utilization. For more information about Preprocessing
API, see this
`overview <https://docs.openvino.ai/latest/openvino_docs_OV_UG_Preprocessing_Overview.html#>`__
`overview <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Preprocessing_Overview.html#>`__
and
`details <https://docs.openvino.ai/latest/openvino_docs_OV_UG_Preprocessing_Details.html>`__
`details <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Preprocessing_Details.html>`__
This tutorial include following steps: - Downloading the model. - Setup
preprocessing with ModelOptimizer, loading the model and inference with
@@ -41,10 +41,10 @@ Imports
.. parsed-literal::
2023-05-29 23:08:10.880625: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-05-29 23:08:10.915307: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
2023-06-21 22:56:19.365436: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-06-21 22:56:19.399309: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-05-29 23:08:11.460713: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
2023-06-21 22:56:19.937411: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
Setup image and device
@@ -88,7 +88,7 @@ and save it to the disk.
.. parsed-literal::
2023-05-29 23:08:12.767813: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
2023-06-21 22:56:21.198595: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
@@ -146,7 +146,7 @@ Convert model to OpenVINO IR and setup preprocessing steps with Model Optimizer
Use Model Optimizer to convert a TensorFlow model to OpenVINO IR.
``mo.convert_model`` python function will be used for converting model
using `OpenVINO Model
Optimizer <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Python_API.html>`__.
Optimizer <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Python_API.html>`__.
The function returns instance of OpenVINO Model class, which is ready to
use in Python interface but can also be serialized to OpenVINO IR format
for future execution using ``openvino.runtime.serialize``. The models
@@ -166,7 +166,7 @@ process the image into the desired format.
Also converting of layout could be specified with ``layout`` option.
More information and parameters described in the `Embedding
Preprocessing Computation
article <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Additional_Optimization_Use_Cases.html#embedding-preprocessing-computation>`__.
article <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Additional_Optimization_Use_Cases.html#embedding-preprocessing-computation>`__.
.. code:: ipython3
@@ -186,15 +186,6 @@ article <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Additional_Optimiza
input_shape=[1,299,299,3])
serialize(ov_model_mo_preprocess, str(ir_path_mo_preprocess))
.. parsed-literal::
2023-05-29 23:08:40.329004: I tensorflow/core/grappler/devices.cc:66] Number of eligible GPUs (core count >= 8, compute capability >= 0.0): 2
2023-05-29 23:08:40.329124: I tensorflow/core/grappler/clusters/single_machine.cc:358] Starting new session
2023-05-29 23:08:40.467499: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
Prepare image
~~~~~~~~~~~~~
@@ -256,7 +247,7 @@ Graph modifications of a model shall be performed after the model is
read from a drive and before it is loaded on the actual device.
Pre-processing support following operations (please, see more details
`here <https://docs.openvino.ai/latest/classov_1_1preprocess_1_1PreProcessSteps.html#doxid-classov-1-1preprocess-1-1-pre-process-steps-1aeacaf406d72a238e31a359798ebdb3b7>`__)
`here <https://docs.openvino.ai/2023.0/classov_1_1preprocess_1_1PreProcessSteps.html#doxid-classov-1-1preprocess-1-1-pre-process-steps-1aeacaf406d72a238e31a359798ebdb3b7>`__)
- Mean/Scale Normalization - Converting Precision - Converting layout
(transposing) - Resizing Image - Color Conversion - Custom Operations
@@ -279,20 +270,11 @@ The options for preprocessing are not required.
input_shape=[1,299,299,3])
serialize(ppp_model, str(ir_path))
.. parsed-literal::
2023-05-29 23:09:07.322003: I tensorflow/core/grappler/devices.cc:66] Number of eligible GPUs (core count >= 8, compute capability >= 0.0): 2
2023-05-29 23:09:07.322139: I tensorflow/core/grappler/clusters/single_machine.cc:358] Starting new session
2023-05-29 23:09:07.323643: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
Create PrePostProcessor Object
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The
`PrePostProcessor() <https://docs.openvino.ai/latest/classov_1_1preprocess_1_1PrePostProcessor.html#doxid-classov-1-1preprocess-1-1-pre-post-processor>`__
`PrePostProcessor() <https://docs.openvino.ai/2023.0/classov_1_1preprocess_1_1PrePostProcessor.html#doxid-classov-1-1preprocess-1-1-pre-post-processor>`__
class enables specifying the preprocessing and postprocessing steps for
a model.
@@ -315,7 +297,7 @@ about user’s input tensor will be initialized to same data
(type/shape/etc) as model’s input parameter. User application can
override particular parameters according to application’s data. Refer to
the following
`page <https://docs.openvino.ai/latest/classov_1_1preprocess_1_1InputTensorInfo.html#doxid-classov-1-1preprocess-1-1-input-tensor-info-1a98fb73ff9178c8c71d809ddf8927faf5>`__
`page <https://docs.openvino.ai/2023.0/classov_1_1preprocess_1_1InputTensorInfo.html#doxid-classov-1-1preprocess-1-1-input-tensor-info-1a98fb73ff9178c8c71d809ddf8927faf5>`__
for more information about parameters for overriding.
Below is all the specified input information: - Precision is ``U8``
@@ -340,7 +322,7 @@ for mean/scale normalization.
.. parsed-literal::
<openvino._pyopenvino.preprocess.InputTensorInfo at 0x7f1eb04c0630>
<openvino._pyopenvino.preprocess.InputTensorInfo at 0x7f97f45dc730>
@@ -350,7 +332,7 @@ Declaring Model Layout
Model input already has information about precision and shape.
Preprocessing API is not intended to modify this. The only thing that
may be specified is input data
`layout <https://docs.openvino.ai/latest/openvino_docs_OV_UG_Layout_Overview.html#doxid-openvino-docs-o-v-u-g-layout-overview>`__.
`layout <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_Layout_Overview.html#doxid-openvino-docs-o-v-u-g-layout-overview>`__.
.. code:: ipython3
@@ -369,7 +351,7 @@ may be specified is input data
.. parsed-literal::
<openvino._pyopenvino.preprocess.InputModelInfo at 0x7f1ff40f9e70>
<openvino._pyopenvino.preprocess.InputModelInfo at 0x7f97f462f770>
@@ -378,14 +360,14 @@ Preprocessing Steps
Now, the sequence of preprocessing steps can be defined. For more
information about preprocessing steps, see
`here <https://docs.openvino.ai/latest/api/ie_python_api/_autosummary/openvino.preprocess.PreProcessSteps.html>`__.
`here <https://docs.openvino.ai/2023.0/api/ie_python_api/_autosummary/openvino.preprocess.PreProcessSteps.html>`__.
Perform the following: - Convert ``U8`` to ``FP32`` precision. - Resize
to height/width of a model. Be aware that if a model accepts dynamic
size, for example, ``{?, 3, ?, ?}`` resize will not know how to resize
the picture. Therefore, in this case, target height/ width should be
specified. For more details, see also the
`PreProcessSteps.resize() <https://docs.openvino.ai/latest/classov_1_1preprocess_1_1PreProcessSteps.html#doxid-classov-1-1preprocess-1-1-pre-process-steps-1a40dab78be1222fee505ed6a13400efe6>`__.
`PreProcessSteps.resize() <https://docs.openvino.ai/2023.0/classov_1_1preprocess_1_1PreProcessSteps.html#doxid-classov-1-1preprocess-1-1-pre-process-steps-1a40dab78be1222fee505ed6a13400efe6>`__.
- Subtract mean from each channel. - Divide each pixel data to
appropriate scale value.
@@ -406,7 +388,7 @@ then such conversion will be added explicitly.
.. parsed-literal::
<openvino._pyopenvino.preprocess.PreProcessSteps at 0x7f1ea3068cb0>
<openvino._pyopenvino.preprocess.PreProcessSteps at 0x7f97b8346af0>
@@ -425,7 +407,7 @@ for debugging purposes.
.. parsed-literal::
Dump preprocessor: Input "Func/StatefulPartitionedCall/input/_0:0":
Dump preprocessor: Input "input_1":
User's input tensor: [1,?,?,3], [N,H,W,C], u8
Model's expected tensor: [1,299,299,3], [N,H,W,C], f32
Pre-processing steps (4):
@@ -554,11 +536,11 @@ Compare results on one image
.. parsed-literal::
Result of inference for preprocessing with ModelOptimizer:
n02099601 golden retriever, 0.78978
n02098413 Lhasa, Lhasa apso, 0.11520
n02108915 French bulldog, 0.01851
n02111129 Leonberg, 0.00819
n02097047 miniature schnauzer, 0.00293
n02099601 golden retriever, 0.56439
n02098413 Lhasa, Lhasa apso, 0.35731
n02108915 French bulldog, 0.00730
n02111129 Leonberg, 0.00687
n04404412 television, television system, 0.00317
Result of inference with Preprocessing API:
@@ -618,7 +600,7 @@ Compare performance
.. parsed-literal::
IR model in OpenVINO Runtime/CPU with preprocessing API: 0.0202 seconds per image, FPS: 49.59
IR model in OpenVINO Runtime/CPU with preprocessing API: 0.0157 seconds per image, FPS: 63.85
IR model in OpenVINO Runtime/CPU with preprocessing API: 0.0193 seconds per image, FPS: 51.92
IR model in OpenVINO Runtime/CPU with preprocessing API: 0.0189 seconds per image, FPS: 52.77
IR model in OpenVINO Runtime/CPU with preprocessing API: 0.0153 seconds per image, FPS: 65.26
IR model in OpenVINO Runtime/CPU with preprocessing API: 0.0175 seconds per image, FPS: 57.09
docs/notebooks/118-optimize-preprocessing-with-output_files/index.html
+3 -3
View File
@@ -1,7 +1,7 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/118-optimize-preprocessing-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/118-optimize-preprocessing-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/118-optimize-preprocessing-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="118-optimize-preprocessing-with-output_11_1.png">118-optimize-preprocessing-with-output_11_1.png</a> 30-May-2023 00:08 387941
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/118-optimize-preprocessing-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="118-optimize-preprocessing-with-output_11_1.png">118-optimize-preprocessing-with-output_11_1.png</a> 22-Jun-2023 00:06 387941
</pre><hr></body>
</html>
docs/notebooks/119-tflite-to-openvino-with-output.rst
+227
View File
@@ -0,0 +1,227 @@
Convert a Tensorflow Lite Model to OpenVINO™
============================================
`TensorFlow Lite <https://www.tensorflow.org/lite/guide>`__, often
referred to as TFLite, is an open source library developed for deploying
machine learning models to edge devices.
This short tutorial shows how to convert a TensorFlow Lite
`efficientnet-lite-b0 <https://tfhub.dev/tensorflow/lite-model/efficientnet/lite0/fp32/2>`__
image classification model to OpenVINO `Intermediate
Representation <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_IR_and_opsets.html>`__
(OpenVINO IR) format, using `Model
Optimizer <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
After creating the OpenVINO IR, load the model in `OpenVINO
Runtime <https://docs.openvino.ai/nightly/openvino_docs_OV_UG_OV_Runtime_User_Guide.html>`__
and do inference with a sample image.
.. code:: ipython3
import sys
from pathlib import Path
import numpy as np
from PIL import Image
from openvino.runtime import Core, serialize
from openvino.tools import mo
sys.path.append("../utils")
from notebook_utils import download_file
Download TFLite model
---------------------
.. code:: ipython3
model_dir = Path("model")
tflite_model_path = model_dir / "efficientnet_lite0_fp32_2.tflite"
ov_model_path = tflite_model_path.with_suffix(".xml")
model_url = "https://tfhub.dev/tensorflow/lite-model/efficientnet/lite0/fp32/2?lite-format=tflite"
download_file(model_url, tflite_model_path.name, model_dir)
.. parsed-literal::
model/efficientnet_lite0_fp32_2.tflite: 0%| | 0.00/17.7M [00:00<?, ?B/s]
.. parsed-literal::
PosixPath('/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/119-tflite-to-openvino/model/efficientnet_lite0_fp32_2.tflite')
Convert a Model to OpenVINO IR Format
-------------------------------------
To convert the TFLite model to OpenVINO IR, OpenVINO Model Optimizer
Python API can be used. ``mo.convert_model`` function accept path to
TFLite model and returns OpenVINO Model class instance which represents
this model. Obtained model is ready to use and loading on device using
``compile_model`` or can be saved on disk using ``serialize`` function
reducing loading time for next running. Optionally, we can apply
compression to FP16 model weigths using ``compress_to_fp16=True`` option
and integrate preprocessing using this approach. See the `Model
Optimizer Developer
Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__
for more information about Model Optimizer and TensorFlow Lite `models
suport <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow_Lite.html>`__.
.. code:: ipython3
ov_model = mo.convert_model(tflite_model_path, compress_to_fp16=True)
serialize(ov_model, ov_model_path)
print(f"Model {tflite_model_path} successfully converted and saved to {ov_model_path}")
.. parsed-literal::
Model model/efficientnet_lite0_fp32_2.tflite successfully converted and saved to model/efficientnet_lite0_fp32_2.xml
Load model using OpenVINO TensorFlow Lite Frontend
--------------------------------------------------
TensorFlow Lite models are supported via FrontEnd API. You may skip
conversion to IR and read models directly by OpenVINO runtime API. For
more examples supported formats reading via Frontend API, please look
this `tutorial <../002-openvino-api>`__.
.. code:: ipython3
core = Core()
ov_model = core.read_model(tflite_model_path)
Run OpenVINO model inference
----------------------------
We can find information about model input preprocessing in its
`description <https://tfhub.dev/tensorflow/lite-model/efficientnet/lite0/fp32/2>`__
on `TensorFlow Hub <https://tfhub.dev/>`__.
.. code:: ipython3
image = Image.open("../data/image/coco_bricks.png")
resized_image = image.resize((224, 224))
input_tensor = np.expand_dims((np.array(resized_image).astype(np.float32) - 127) / 128, 0)
.. code:: ipython3
compiled_model = core.compile_model(ov_model)
predicted_scores = compiled_model(input_tensor)[0]
.. code:: ipython3
imagenet_classes = open("../data/datasets/imagenet/imagenet_2012.txt").read().splitlines()
top1_predicted_cls_id = np.argmax(predicted_scores)
top1_predicted_score = predicted_scores[0][top1_predicted_cls_id]
predicted_label = imagenet_classes[top1_predicted_cls_id]
display(image.resize((640, 512)))
print(f"Predicted label: {predicted_label} with probability {top1_predicted_score :2f}")
.. image:: 119-tflite-to-openvino-with-output_files/119-tflite-to-openvino-with-output_11_0.png
.. parsed-literal::
Predicted label: n02109047 Great Dane with probability 0.715318
Estimate Model Performance
--------------------------
`Benchmark
Tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_tool_README.html>`__
is used to measure the inference performance of the model on CPU and
GPU.
**NOTE**: For more accurate performance, it is recommended to run
``benchmark_app`` in a terminal/command prompt after closing other
applications. Run ``benchmark_app -m model.xml -d CPU`` to benchmark
async inference on CPU for one minute. Change ``CPU`` to ``GPU`` to
benchmark on GPU. Run ``benchmark_app --help`` to see an overview of
all command-line options.
.. code:: ipython3
print("Benchmark model inference on CPU")
!benchmark_app -m $ov_model_path -d CPU -t 15
if "GPU" in core.available_devices:
print("\n\nBenchmark model inference on GPU")
!benchmark_app -m $ov_model_path -d GPU -t 15
.. parsed-literal::
Benchmark model inference on CPU
[Step 1/11] Parsing and validating input arguments
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 9.15 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] images (node: images) : f32 / [...] / [1,224,224,3]
[ INFO ] Model outputs:
[ INFO ] Softmax (node: 61) : f32 / [...] / [1,1000]
[Step 5/11] Resizing model to match image sizes and given batch
[ INFO ] Model batch size: 1
[Step 6/11] Configuring input of the model
[ INFO ] Model inputs:
[ INFO ] images (node: images) : u8 / [N,H,W,C] / [1,224,224,3]
[ INFO ] Model outputs:
[ INFO ] Softmax (node: 61) : f32 / [...] / [1,1000]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 149.12 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: TensorFlow_Lite_Frontend_IR
[ INFO ] OPTIMAL_NUMBER_OF_INFER_REQUESTS: 6
[ INFO ] NUM_STREAMS: 6
[ INFO ] AFFINITY: Affinity.CORE
[ INFO ] INFERENCE_NUM_THREADS: 24
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'images'!. This input will be filled with random values!
[ INFO ] Fill input 'images' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 7.30 ms
[Step 11/11] Dumping statistics report
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 17514 iterations
[ INFO ] Duration: 15008.85 ms
[ INFO ] Latency:
[ INFO ] Median: 4.99 ms
[ INFO ] Average: 5.00 ms
[ INFO ] Min: 3.71 ms
[ INFO ] Max: 15.12 ms
[ INFO ] Throughput: 1166.91 FPS
docs/notebooks/119-tflite-to-openvino-with-output_files/119-tflite-to-openvino-with-output_11_0.png
+3
View File
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3b2bcf5770d51d67341faebf7ff45bc93104f1296fb5d7b2594d87299f1ce153
size 612081
docs/notebooks/119-tflite-to-openvino-with-output_files/index.html
+7
View File
@@ -0,0 +1,7 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/119-tflite-to-openvino-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/119-tflite-to-openvino-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="119-tflite-to-openvino-with-output_11_0.png">119-tflite-to-openvino-with-output_11_0.png</a> 22-Jun-2023 00:06 612081
</pre><hr></body>
</html>
docs/notebooks/201-vision-monodepth-with-output.rst
+30 -9
View File
@@ -3,7 +3,7 @@ Monodepth Estimation with OpenVINO
This tutorial demonstrates Monocular Depth Estimation with MidasNet in
OpenVINO. Model information can be found
`here <https://docs.openvino.ai/latest/omz_models_model_midasnet.html>`__.
`here <https://docs.openvino.ai/2023.0/omz_models_model_midasnet.html>`__.
.. figure:: https://user-images.githubusercontent.com/36741649/127173017-a0bbcf75-db24-4d2c-81b9-616e04ab7cd9.gif
:alt: monodepth
@@ -56,17 +56,36 @@ Imports
from openvino.runtime import Core
sys.path.append("../utils")
from notebook_utils import load_image
from notebook_utils import download_file, load_image
Settings
~~~~~~~~
Download the model
~~~~~~~~~~~~~~~~~~
.. code:: ipython3
DEVICE = "CPU"
MODEL_FILE = "model/MiDaS_small.xml"
model_folder = Path('model')
model_xml_path = Path(MODEL_FILE)
ir_model_url = 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/depth-estimation-midas/FP32/'
ir_model_name_xml = 'MiDaS_small.xml'
ir_model_name_bin = 'MiDaS_small.bin'
download_file(ir_model_url + ir_model_name_xml, filename=ir_model_name_xml, directory=model_folder)
download_file(ir_model_url + ir_model_name_bin, filename=ir_model_name_bin, directory=model_folder)
model_xml_path = model_folder / ir_model_name_xml
.. parsed-literal::
model/MiDaS_small.xml: 0%| | 0.00/268k [00:00<?, ?B/s]
.. parsed-literal::
model/MiDaS_small.bin: 0%| | 0.00/31.6M [00:00<?, ?B/s]
Functions
---------
@@ -110,6 +129,8 @@ keys and the expected input shape for the model.
.. code:: ipython3
DEVICE = "CPU"
ie = Core()
ie.set_property({'CACHE_DIR': '../cache'})
model = ie.read_model(model_xml_path)
@@ -348,7 +369,7 @@ Do Inference on a Video and Create Monodepth Video
.. parsed-literal::
Processed 60 frames in 9.17 seconds. Total FPS (including video processing): 6.55.Inference FPS: 56.57
Processed 60 frames in 9.22 seconds. Total FPS (including video processing): 6.51.Inference FPS: 54.82
Monodepth Video saved to 'output/Coco Walking in Berkeley_monodepth.mp4'.
@@ -376,7 +397,7 @@ Display Monodepth Video
.. parsed-literal::
Showing monodepth video saved at
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/201-vision-monodepth/output/Coco Walking in Berkeley_monodepth.mp4
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/201-vision-monodepth/output/Coco Walking in Berkeley_monodepth.mp4
If you cannot see the video in your browser, please click on the following link to download the video
docs/notebooks/201-vision-monodepth-with-output_files/index.html
+3 -3
View File
@@ -1,7 +1,7 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/201-vision-monodepth-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/201-vision-monodepth-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/201-vision-monodepth-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="201-vision-monodepth-with-output_14_0.png">201-vision-monodepth-with-output_14_0.png</a> 30-May-2023 00:09 959858
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/201-vision-monodepth-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="201-vision-monodepth-with-output_14_0.png">201-vision-monodepth-with-output_14_0.png</a> 22-Jun-2023 00:06 959858
</pre><hr></body>
</html>
docs/notebooks/202-vision-superresolution-image-with-output.rst
+41 -7
View File
@@ -5,7 +5,7 @@ Super Resolution is the process of enhancing the quality of an image by
increasing the pixel count using deep learning. This notebook shows the
Single Image Super Resolution (SISR) which takes just one low resolution
image. A model called
`single-image-super-resolution-1032 <https://docs.openvino.ai/latest/omz_models_model_single_image_super_resolution_1032.html>`__,
`single-image-super-resolution-1032 <https://docs.openvino.ai/2023.0/omz_models_model_single_image_super_resolution_1032.html>`__,
which is available in Open Model Zoo, is used in this tutorial. It is
based on the research paper cited below.
@@ -26,6 +26,7 @@ Imports
import time
import requests
from pathlib import Path
import sys
import cv2
import matplotlib.pyplot as plt
@@ -35,6 +36,9 @@ Imports
from IPython.display import Pretty, ProgressBar, clear_output, display
from PIL import Image
from openvino.runtime import Core
sys.path.append("../utils")
from notebook_utils import download_file
Settings
~~~~~~~~
@@ -42,11 +46,41 @@ Settings
.. code:: ipython3
# Device to use for inference. For example, "CPU", or "GPU".
DEVICE = "CPU"
DEVICE = 'CPU'
# 1032: 4x superresolution, 1033: 3x superresolution
MODEL_FILE = "model/single-image-super-resolution-1032.xml"
model_name = os.path.basename(MODEL_FILE)
model_xml_path = Path(MODEL_FILE)
model_name = 'single-image-super-resolution-1032'
base_model_dir = Path("./model").expanduser()
model_xml_name = f'{model_name}.xml'
model_bin_name = f'{model_name}.bin'
model_xml_path = base_model_dir / model_xml_name
model_bin_path = base_model_dir / model_bin_name
if not model_xml_path.exists():
base_url = f'https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/{model_name}/FP16/'
model_xml_url = base_url + model_xml_name
model_bin_url = base_url + model_bin_name
download_file(model_xml_url, model_xml_name, base_model_dir)
download_file(model_bin_url, model_bin_name, base_model_dir)
else:
print(f'{model_name} already downloaded to {base_model_dir}')
.. parsed-literal::
model/single-image-super-resolution-1032.xml: 0%| | 0.00/89.3k [00:00<?, ?B/s]
.. parsed-literal::
model/single-image-super-resolution-1032.bin: 0%| | 0.00/58.4k [00:00<?, ?B/s]
Functions
~~~~~~~~~
@@ -636,8 +670,8 @@ as total time to process each patch.
.. parsed-literal::
Processed 42 patches in 4.63 seconds. Total patches per second (including processing): 9.08.
Inference patches per second: 20.79
Processed 42 patches in 5.06 seconds. Total patches per second (including processing): 8.29.
Inference patches per second: 16.98
Save superresolution image and the bicubic image
docs/notebooks/202-vision-superresolution-image-with-output_files/202-vision-superresolution-image-with-output_18_1.png
+1 -1
View File
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:463999d9642365bff6e17a7bea49339b4cd2cd629a2782cf9cf4c298ae9db2b2
oid sha256:bac1d1f354545d966428c4785215fc43a33134c80a8108fb3694e7efa4df5d16
size 2896276
docs/notebooks/202-vision-superresolution-image-with-output_files/index.html
+6 -6
View File
@@ -1,10 +1,10 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/202-vision-superresolution-image-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/202-vision-superresolution-image-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/202-vision-superresolution-image-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="202-vision-superresolution-image-with-output_10_1.png">202-vision-superresolution-image-with-output_10..&gt;</a> 30-May-2023 00:09 272963
<a href="202-vision-superresolution-image-with-output_12_1.png">202-vision-superresolution-image-with-output_12..&gt;</a> 30-May-2023 00:08 356735
<a href="202-vision-superresolution-image-with-output_18_1.png">202-vision-superresolution-image-with-output_18..&gt;</a> 30-May-2023 00:09 2896276
<a href="202-vision-superresolution-image-with-output_22_1.png">202-vision-superresolution-image-with-output_22..&gt;</a> 30-May-2023 00:09 3207711
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/202-vision-superresolution-image-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="202-vision-superresolution-image-with-output_10_1.png">202-vision-superresolution-image-with-output_10..&gt;</a> 22-Jun-2023 00:06 272963
<a href="202-vision-superresolution-image-with-output_12_1.png">202-vision-superresolution-image-with-output_12..&gt;</a> 22-Jun-2023 00:06 356735
<a href="202-vision-superresolution-image-with-output_18_1.png">202-vision-superresolution-image-with-output_18..&gt;</a> 22-Jun-2023 00:06 2896276
<a href="202-vision-superresolution-image-with-output_22_1.png">202-vision-superresolution-image-with-output_22..&gt;</a> 22-Jun-2023 00:06 3207711
</pre><hr></body>
</html>
docs/notebooks/202-vision-superresolution-video-with-output.rst
+36 -9
View File
@@ -5,7 +5,7 @@ Super Resolution is the process of enhancing the quality of an image by
increasing the pixel count using deep learning. This notebook applies
Single Image Super Resolution (SISR) to frames in a 360p (480×360) video
in 360p resolution. A model called
`single-image-super-resolution-1032 <https://docs.openvino.ai/latest/omz_models_model_single_image_super_resolution_1032.html>`__,
`single-image-super-resolution-1032 <https://docs.openvino.ai/2023.0/omz_models_model_single_image_super_resolution_1032.html>`__,
which is available in Open Model Zoo, is used in this tutorial. It is
based on the research paper cited below.
@@ -33,6 +33,7 @@ Imports
import time
import requests
from pathlib import Path
import sys
import cv2
import numpy as np
@@ -47,18 +48,44 @@ Imports
)
from openvino.runtime import Core
from pytube import YouTube
sys.path.append("../utils")
from notebook_utils import download_file
Settings
~~~~~~~~
.. code:: ipython3
# A device to use for inference. For example, "CPU", or "GPU".
DEVICE = "CPU"
# Device to use for inference. For example, "CPU", or "GPU".
DEVICE = 'CPU'
# 1032: 4x superresolution, 1033: 3x superresolution
MODEL_FILE = "model/single-image-super-resolution-1032.xml"
model_name = Path(MODEL_FILE).name
model_xml_path = Path(MODEL_FILE)
model_name = 'single-image-super-resolution-1032'
base_model_dir = Path('./model').expanduser()
model_xml_name = f'{model_name}.xml'
model_bin_name = f'{model_name}.bin'
model_xml_path = base_model_dir / model_xml_name
model_bin_path = base_model_dir / model_bin_name
if not model_xml_path.exists():
base_url = f'https://storage.openvinotoolkit.org/repositories/open_model_zoo/2023.0/models_bin/1/{model_name}/FP16/'
model_xml_url = base_url + model_xml_name
model_bin_url = base_url + model_bin_name
download_file(model_xml_url, model_xml_name, base_model_dir)
download_file(model_bin_url, model_bin_name, base_model_dir)
else:
print(f'{model_name} already downloaded to {base_model_dir}')
.. parsed-literal::
single-image-super-resolution-1032 already downloaded to model
Functions
~~~~~~~~~
@@ -422,13 +449,13 @@ video.
.. parsed-literal::
Processed frame 100. Inference time: 0.05 seconds (19.89 FPS)
Processed frame 100. Inference time: 0.05 seconds (20.14 FPS)
.. parsed-literal::
Video's saved to output directory.
Processed 100 frames in 242.26 seconds. Total FPS (including video processing): 0.41. Inference FPS: 19.29.
Processed 100 frames in 243.18 seconds. Total FPS (including video processing): 0.41. Inference FPS: 17.41.
Show Side-by-Side Video of Bicubic and Superresolution Version
@@ -461,7 +488,7 @@ Show Side-by-Side Video of Bicubic and Superresolution Version
.. raw:: html
<video controls width="800" >
<source src="data:video/mp4;base64,AAAAHGZ0eXBpc29tAAACAGlzb21pc28ybXA0MQAAAAhmcmVlAG4OD21kYXSCSYNCAO/wQ3YYOCQcGGQcAup/yfo+X/mfs+i+B/13ofovTfG/N8T8fofQdG9N+L4fhOFemD9G9R6bpt2NSXiMuC/S+N/pfV/D6p73p3ruU6XXVP6U8nP2/pHpfH7del/5z13runbui167+mvXeTnpenRm9YtGDfvTvb9L+Zwnp3t3TFR9cbf+munKRcBXB3hnq+her/n/Np+96X73+lPV+x9L6vlv3L+Z7H/PPdeu/qb2n7f7H2PovY+x5eOQP7f9p7r1Hq/Xbx5kNy8uvsf8M9p+l+r9d5Ffv25a6eu/ZvS+o/k90dL/fGk/x/l9J8P8v4/hfG/P8T8e6/978n6fu4zb8T9Pxs1/735MDf7w7P96D6z+z8n5/uW93vcb4fwf834P935v0/b3/Vfi/yv+6/L+D8v4/g/6Xt3+9/y34v8X/qPyfH/P8X8vVn+Z/135vvHf9T90n5fQ9YH/B/8F/mf2fk/0Pg/5/uX7SS8fb6I1G+iuhVOqJ39cb3+mvadYXbv6g9SAAAAOT3kj//+3YjcSuS8qCGZhEcOlvvyUXCQPWfx42oXUsvzguo1kIpwMbMLie2o4sHLIIlnc57wA2r9xh1i0w1N0BvcWro/mdEm+TpdTgwcP+UX36ZkxOxY1ThUFD6Y4MFF1jN0LoLibXwRbxw1Bq7AAdcVK71rIq3xtKK+hTVeOZoq3NPT/89Pi5xFpX+90PekJiNLl1FklXgTTvPAFMXpdLAvrNoF/BBmCBy86nHw3q5vY3QNBYFX8YqN+8wwwJzMoMcWF4w6APjObL1AwY4Q/Zy778ltUkK9mZB4bCOANTtqsXEGE5QW4WkI3yAHZucI3z79/LQxH6xT6yKXDgD8SusdnmZASKgVBomPWYDNRAS/TqPzRT0ay9tggxRAr3/FThLEakYqSc0XqDcMw/O+TeDYtrAfRlhk+NiSTzWYQG0uA0s0s0dBP8mgfpfZ+gP3SX89TGxcw1Z8lzenqNC5pm7lEOSigfMUK5I3Ce+m8in5hFvq/z3U4hgsgGwRvm+oeAAA1sWrV8mwDBWGiom71gu4YO61tDxffl0T1HNpCNiR8gsPw8wd2GPAY3QCyQU67w1bYk/77HjFXqZoCDbhYmH7fYw+iRpkb8A8V05CKlPU8Gj+TOnio51BLFXkHbu4P23YIDVM02ZeNWQCympurXWlBvlLx++MGtHGowDJQR7RIjIWXCeznDU2VDXpbq3JhXop/fC4G0frkvGIAcjIyx47mgCKUdRGm3lcOCF5oMy3X2kqTOtb1mb7dqyznezjBT42xLIUayEiO4ZhI3EjZjCd1AgVdmQ5JR7WzfBLxy2pteVQxDSsG0Ykvul60l3o5hNCuCGg8q+auNcvfFkAfr3jGI2IiXgHRYUB8dJTmisXfc0g+YChkAYYZ3591yZhqdmwfanPA5kF1A4qIxou05XBSgtGIgwvE1c65tTWlBvIV+UNXf7Au6y7mSB5afNwasPf2f1e8D9vUtOl7xkRPD8kOkGAtGH+gUgmWGRb44JZuSpTtodhmyQKeEbbQfncmswE1kXb/bTKn+W0XIRWop1KBJBrA4NXbBioIWj5CKoHoe0mjB1IGXBCQDHxWIFKbmBQyjIw2jrNRaOleeddAwM1oBA3dpz6UTD3cA5mv6ldyRbmIOgIeZvOHxnWh7XsHunr0IMWABIXndRGjXVXeAfDdW2i3aTPUWF3t2Rt2Khly8CDMHgJcp1QfRzA8WR1DkWmEbkIxTOy9CYC7kIXaCTrTMQu1nvs5y4OFLiy6jKjL1Ow263Cxd0lnfG2joA7RNeFjsKjgac1ZMSo5y2juNdKO8hMWgXdplxyPO3IlXNDE5lQ+AK59SQeV0d3tGNXbQLfFZRp1pTuRUWZuMFqvaJG+4nPHM4gjW6YFdu4id8LyCD0LIVYIfJR9jx8/dg84ljjSOUy8YPE+y4FIp/dtPxPxssHP+t8iDDHX7KIoJgcKbiVmuxah4MQSTtOmWdE4xJFD1PNYCyn3owG7CMWqrqjME5ObljaJqWjdMP4LK+Iw7Ie5JrnbFRCgcGFV05fq3a5Tx5jWEfz/Y4II3s0AwBrfwqq8vNVFXSkuN58AfbGrRIBmIhWA/dm1mQlb8TAuG3E36GhX16JZHVIC9DNnU34BR2xCvTjlDRyRv4HkCBe60E0cbj/bhiTO//4Mio4C/LmEyrGEESXnvRnYxKaPdM/IJWGS6m1U7clmSa4Yebtv/PvmkYXORRRU+a8UI43LV55Y5IPdur8UK6hJE2V02bZClXXRZknZsWPDiBUyw5rW/08OkX7WOET2oVA1+rE1x/aRNwer8jL7Wu9cGMe7Z/pRAv8lO55lieNMkbSBSCe2htG+5FUlkW3hU7dLYsNEGyiauC6Qdsf6PVyQggpCsopFmDpfzhEvs96bWF6UoHtEpHAvPiTxarDBKABQM/OymxjfwdLaoFlf6kUdQ6tWvGySgYyPOZwThMahIhWadx3AyH0GXfO+bsZ9RhQpAn0nLIV7ZMxIoUwSbaBiphptreFFKTw7BUmbu17eI+KIqvVTmGqkKTjxaTQT/6kV0rdW8w8t8bci0Fre7c2bdq8CqgAeYKpmC3FMHYgv7gqsU0v189tPiHdm5cmH4u9DZufsjqUeE9NSr3S4L8ikgYmIZjTvL7kADHAyE0IdZEoex6h4Z9njoxHtlVQ5h+Gt/olA5G00Lgn9LWcFUUTXZHuBTBy+RxcFJjFBWHADC2hjFZDmPJIo4AAFERxpMcpyqWvDonnYwoZ3k1kKFsaC1nTrhnR8vobbDhTSKINa1aFrNthgf04AIrXhcFHxTDVdxfGs0jztjufMDsTKSvAymsrnCqyRNv4+MW8YYvUXGoHuhOQCLXX3vEoOwHeA/0B+vND/qndMeDpXMhKFRLswO03HpZah28IKAqj5zgaTTGj2kORIgit7CskWofvbhe+XBxsyiAKRvOhEJ6sWi2QfzT+Y3sTF6oUGPwHefHapMEleXRke/J+fNQY9tqbdy+8Rpk0h4OXMGbOMldILFJsMtXr5RwlPvGlqgE7NAmOmkx3otpzc05dsqwNoNCu8V/uKXzpHmCE5PQBxXaLErmeBA87DZhSo0MbaxqnShFje/zU10eR+BLs64B1HorKYHgmSOQp+AL0RuxiY00J4TfbZu7AlYRbqyGg6EUci5NzKEH5BR2jRG+iJmVuETsWewwnZKrGaiaQJUJWzdhXeVCzMkUJyD/xfBXiYzC+ttb0/XYw1CvMADTElogFFh2Q0Mj/6dIQJn+qOkcZJbRe6aflwAd9t2k0GRIGeBAwMU0ynd7teoSYSHbVIB70inqHDRTcBwev0cNeTiXKPNk1y/uqUhyPQJPtGinLSfG3KwAAiVrmy5c64GIhqRrV5BUOuAVjR6VC5FaBXKwDMspx/gUrd1KbskLwUQTptsz+QKNxIr6OaoTv91ihF6ybo6t7dzCDeLeVoVPGeha54tdnGbJYw+YpmVjBI8F1brv4xKPextICjeu5nZ/MQNn8d+9LZSGpVhokGU+RIMHB/le5GAljCjviMoPSb42EOiFdVGTsF9C5oQ+Xu8bEva9LwXkPWtD8eAneUqSBlcUML8ms3imRJ8Ih6WhAnZ+V3KKlLMSWEGtm+g1OnckCa9zwH62/+A28W3OgXEooL9LAYc5fgZ1IfpbAQq8InvSsI7zHxuHwjx8uwtKzCQxYXpm2gV0n4C88wISmF4FhPQUtpQhdCtQ/qmqJYRu+cTLhSUj5NQGPebmW9Na6ndL2G+dGUFLTSImEAucLhnHW+BGF66CX9aIbNZtnD8FuJwPyc7adv870bMToun2POzgBVGPRKyvKioArYbTruoe4taP75ybBW+guEXrTEJF6aX29LlFGcrl3lYMKa4yY58arXZjP7sT/BuZwo+aaNvti4Nv/KMccdlAuX6XWU0UEVvx/qM5BrV49Sp7dWJxdW41R/RMMnIjWhqkwdTIui/XfF2ps4Nw4gHqrBBbza2xtYY60gmpdpOMQdwJRlqxOV7V11rlkuEDwXkvauUWMT7q2ATem5wAFAc92oCTUND6kBIkXxGL0eT1kUFOzDNSbbmy0MH48faSAAAXN+7w4lH1mem9lCEjrcqeKpf5gO8EO9FUWyow6i//oAW6k3zIrzw84ihQCGwj3eisv+UBpHy/OKnGSnNQKZ1qrxZTgC6TpIvgpdBc8sg+ANoL4ABqYKPIw2DBW4tTAwOHfzbh1HGsaA4F/qis76yAqQSwKpLfgkVKBWK+iJTq3sK2Z/HRqAFvgmWUE/6D5PL5TcgJZ/HuPhv3DhgG0NDwifLX1HHoa5dPEne3Gtn5zexxOlixAq1VM9TJhF+m/0KsDwAGuhanrEAY4Jzv0hS0nhBsH7iVQbWq33gctQaJsKRVji4GFVPOqHmzrm048LpDo4DzzdTbv8FY5pKxCCIPkoZHbcsJeuxeIvf7JF6VdEuz/UXXTZWdK2VI/T04jCMdddca6094WDmSr/5uJKAxk5tqW0Cy1d8gyMvVzNiwNnFzTK2semmH1KlCGWCZlbEXIoOvYtWHmqY7hVspxbDnkXym8drbwsIhw3lD/mHUwuS6DqjqoDzwhKXrPnBVd+iknCa81sfhPw5zwdHTD2tDmDuF+/fDTinP57PYCHh0tXDqdDyu83P/vVtPai8a7zSRW9TUgZS43qkmj2gAvJ2dKNLmsAGdP8APgyYWM6sDXOoz+6d9s64CVsZ5vLgOUDioxOLbsb4xsdRHh+UC6AfqhdqgDvhVYB0WkGHsnL3DkXLH8c7UaxDsNUwu7SsmdAd2gtdfJvBkVMfPBw/31RecVkAKaWdknmzgsUc9vnwEZ+3N0IfiZAt7R0zKcBapQZffsZninsYrs2vpQ4dDE8OH1eIXUECjRLine truncated
<source src="data:video/mp4;base64,AAAAHGZ0eXBpc29tAAACAGlzb21pc28ybXA0MQAAAAhmcmVlAG4N4W1kYXSCSYNCAO/wQ3YYOCQcGGQcAup/yfo+X/mfs+i+B/13ofovTfG/N8T8fofQdG9N+L4fhOFemD9G9R6bpt2NSXiMuC/S+N/pfV/D6p73p3ruU6XXVP6U8nP2/pHpfH7del/5z13runbui167+mvXeTnpenRm9YtGDfvTvb9L+Zwnp3t3TFR9cbf+munKRcBXB3hnq+her/n/Np+96X73+lPV+x9L6vlv3L+Z7H/PPdeu/qb2n7f7H2PovY+x5eOQP7f9p7r1Hq/Xbx5kNy8uvsf8M9p+l+r9d5Ffv25a6eu/ZvS+o/k90dL/fGk/x/l9J8P8v4/hfG/P8T8e6/978n6fu4zb8T9Pxs1/735MDf7w7P96D6z+z8n5/uW93vcb4fwf834P935v0/b3/Vfi/yv+6/L+D8v4/g/6Xt3+9/y34v8X/qPyfH/P8X8vVn+Z/135vvHf9T90n5fQ9YH/B/8F/mf2fk/0Pg/5/uX7SS8fb6I1G+iuhVOqJ39cb3+mvadYXbv6g9SAAAAOT3kj//+3YjcSuS8qCGZhEcOlvvyUXCQPWfx42oXUsvzguo1kIpwMbMLie2o4sHLIIlnc57wA2r9xh1i0w1N0BvcWro/mdEm+TpdTgwcP+UX36ZkxOxY1ThUFD6Y4MFF1jN0LoLibXwRbxw1Bq7AAdcVK71rIq3xtKK+hTVeOZoq3NPT/89Pi5xFpX+90PekJiNLl1FklXgTTvPAFMXpdLAvrNoF/BBmCBy86nHw3q5vY3QNBYFX8YqN+8wwwJzMoMcWF4w6APjObL1AwY4Q/Zy778ltUkK9mZB4bCOANTtqsXEGE5QW4WkI3yAHZucI3z79/LQxH6xT6yKXDgD8SusdnmZASKgVBomPWYDNRAS/TqPzRT0ay9tggxRAr3/FThLEakYqSc0XqDcMw/O+TeDYtrAfRlhk+NiSTzWYQG0uA0s0s0dBP8mgfpfZ+gP3SX89TGxcw1Z8lzenqNC5pm7lEOSigfMUK5I3Ce+m8in5hFvq/z3U4hgsgGwRvm+oeAAA1sWrV8mwDBWGiom71gu4YO61tDxffl0T1HNpCNiR8gsPw8wd2GPAY3QCyQU67w1bYk/77HjFXqZoCDbhYmH7fYw+iRpkb8A8V05CKlPU8Gj+TOnio51BLFXkHbu4P23YIDVM02ZeNWQCympurXWlBvlLx++MGtHGowDJQR7RIjIWXCeznDU2VDXpbq3JhXop/fC4G0frkvGIAcjIyx47mgCKUdRGm3lcOCF5oMy3X2kqTOtb1mb7dqyznezjBT42xLIUayEiO4ZhI3EjZjCd1AgVdmQ5JR7WzfBLxy2pteVQxDSsG0Ykvul60l3o5hNCuCGg8q+auNcvfFkAfr3jGI2IiXgHRYUB8dJTmisXfc0g+YChkAYYZ3591yZhqdmwfanPA5kF1A4qIxou05XBSgtGIgwvE1c65tTWlBvIV+UNXf7Au6y7mSB5afNwasPf2f1e8D9vUtOl7xkRPD8kOkGAtGH+gUgmWGRb44JZuSpTtodhmyQKeEbbQfncmswE1kXb/bTKn+W0XIRWop1KBJBrA4NXbBioIWj5CKoHoe0mjB1IGXBCQDHxWIFKbmBQyjIw2jrNRaOleeddAwM1oBA3dpz6UTD3cA5mv6ldyRbmIOgIeZvOHxnWh7XsHunr0IMWABIXndRGjXVXeAfDdW2i3aTPUWF3t2Rt2Khly8CDMHgJcp1QfRzA8WR1DkWmEbkIxTOy9CYC7kIXaCTrTMQu1nvs5y4OFLiy6jKjL1Ow263Cxd0lnfG2joA7RNeFjsKjgac1ZMSo5y2juNdKO8hMWgXdplxyPO3IlXNDE5lQ+AK59SQeV0d3tGNXbQLfFZRp1pTuRUWZuMFqvaJG+4nPHM4gjW6YFdu4id8LyCD0LIVYIfJR9jx8/dg84ljjSOUy8YPE+y4FIp/dtPxPxssHP+t8iDDHX7KIoJgcKbiVmuxah4MQSTtOmWdE4xJFD1PNYCyn3owG7CMWqrqjME5ObljaJqWjdMP4LK+Iw7Ie5JrnbFRCgcGFV05fq3a5Tx5jWEfz/Y4II3s0AwBrfwqq8vNVFXSkuN58AfbGrRIBmIhWA/dm1mQlb8TAuG3E36GhX16JZHVIC9DNnU34BR2xCvTjlDRyRv4HkCBe60E0cbj/bhiTO//4Mio4C/LmEyrGEESXnvRnYxKaPdM/IJWGS6m1U7clmSa4Yebtv/PvmkYXORRRU+a8UI43LV55Y5IPdur8UK6hJE2V02bZClXXRZknZsWPDiBUyw5rW/08OkX7WOET2oVA1+rE1x/aRNwer8jL7Wu9cGMe7Z/pRAv8lO55lieNMkbSBSCe2htG+5FUlkW3hU7dLYsNEGyiauC6Qdsf6PVyQggpCsopFmDpfzhEvs96bWF6UoHtEpHAvPiTxarDBKABQM/OymxjfwdLaoFlf6kUdQ6tWvGySgYyPOZwThMahIhWadx3AyH0GXfO+bsZ9RhQpAn0nLIV7ZMxIoUwSbaBiphptreFFKTw7BUmbu17eI+KIqvVTmGqkKTjxaTQT/6kV0rdW8w8t8bci0Fre7c2bdq8CqgAeYKpmC3FMHYgv7gqsU0v189tPiHdm5cmH4u9DZufsjqUeE9NSr3S4L8ikgYmIZjTvL7kADHAyE0IdZEoex6h4Z9njoxHtlVQ5h+Gt/olA5G00Lgn9LWcFUUTXZHuBTBy+RxcFJjFBWHADC2hjFZDmPJIo4AAFERxpMcpyqWvDonnYwoZ3k1kKFsaC1nTrhnR8vobbDhTSKINa1aFrNthgf04AIrXhcFHxTDVdxfGs0jztjufMDsTKSvAymsrnCqyRNv4+MW8YYvUXGoHuhOQCLXX3vEoOwHeA/0B+vND/qndMeDpXMhKFRLswO03HpZah28IKAqj5zgaTTGj2kORIgit7CskWofvbhe+XBxsyiAKRvOhEJ6sWi2QfzT+Y3sTF6oUGPwHefHapMEleXRke/J+fNQY9tqbdy+8Rpk0h4OXMGbOMldILFJsMtXr5RwlPvGlqgE7NAmOmkx3otpzc05dsqwNoNCu8V/uKXzpHmCE5PQBxXaLErmeBA87DZhSo0MbaxqnShFje/zU10eR+BLs64B1HorKYHgmSOQp+AL0RuxiY00J4TfbZu7AlYRbqyGg6EUci5NzKEH5BR2jRG+iJmVuETsWewwnZKrGaiaQJUJWzdhXeVCzMkUJyD/xfBXiYzC+ttb0/XYw1CvMADTElogFFh2Q0Mj/6dIQJn+qOkcZJbRe6aflwAd9t2k0GRIGeBAwMU0ynd7teoSYSHbVIB70inqHDRTcBwev0cNeTiXKPNk1y/uqUhyPQJPtGinLSfG3KwAAiVrmy5c64GIhqRrV5BUOuAVjR6VC5FaBXKwDMspx/gUrd1KbskLwUQTptsz+QKNxIr6OaoTv91ihF6ybo6t7dzCDeLeVoVPGeha54tdnGbJYw+YpmVjBI8F1brv4xKPextICjeu5nZ/MQNn8d+9LZSGpVhokGU+RIMHB/le5GAljCjviMoPSb42EOiFdVGTsF9C5oQ+Xu8bEva9LwXkPWtD8eAneUqSBlcUML8ms3imRJ8Ih6WhAnZ+V3KKlLMSWEGtm+g1OnckCa9zwH62/+A28W3OgXEooL9LAYc5fgZ1IfpbAQq8InvSsI7zHxuHwjx8uwtKzCQxYXpm2gV0n4C88wISmF4FhPQUtpQhdCtQ/qmqJYRu+cTLhSUj5NQGPebmW9Na6ndL2G+dGUFLTSImEAucLhnHW+BGF66CX9aIbNZtnD8FuJwPyc7adv870bMToun2POzgBVGPRKyvKioArYbTruoe4taP75ybBW+guEXrTEJF6aX29LlFGcrl3lYMKa4yY58arXZjP7sT/BuZwo+aaNvti4Nv/KMccdlAuX6XWU0UEVvx/qM5BrV49Sp7dWJxdW41R/RMMnIjWhqkwdTIui/XfF2ps4Nw4gHqrBBbza2xtYY60gmpdpOMQdwJRlqxOV7V11rlkuEDwXkvauUWMT7q2ATem5wAFAc92oCTUND6kBIkXxGL0eT1kUFOzDNSbbmy0MH48faSAAAXN+7w4lH1mem9lCEjrcqeKpf5gO8EO9FUWyow6i//oAW6k3zIrzw84ihQCGwj3eisv+UBpHy/OKnGSnNQKZ1qrxZTgC6TpIvgpdBc8sg+ANoL4ABqYKPIw2DBW4tTAwOHfzbh1HGsaA4F/qis76yAqQSwKpLfgkVKBWK+iJTq3sK2Z/HRqAFvgmWUE/6D5PL5TcgJZ/HuPhv3DhgG0NDwifLX1HHoa5dPEne3Gtn5zexxOlixAq1VM9TJhF+m/0KsDwAGuhanrEAY4Jzv0hS0nhBsH7iVQbWq33gctQaJsKRVji4GFVPOqHmzrm048LpDo4DzzdTbv8FY5pKxCCIPkoZHbcsJeuxeIvf7JF6VdEuz/UXXTZWdK2VI/T04jCMdddca6094WDmSr/5uJKAxk5tqW0Cy1d8gyMvVzNiwNnFzTK2semmH1KlCGWCZlbEXIoOvYtWHmqY7hVspxbDnkXym8drbwsIhw3lD/mHUwuS6DqjqoDzwhKXrPnBVd+iknCa81sfhPw5zwdHTD2tDmDuF+/fDTinP57PYCHh0tXDqdDyu83P/vVtPai8a7zSRW9TUgZS43qkmj2gAvJ2dKNLmsAGdP8APgyYWM6sDXOoz+6d9s64CVsZ5vLgOUDioxOLbsb4xsdRHh+UC6AfqhdqgDvhVYB0WkGHsnL3DkXLH8c7UaxDsNUwu7SsmdAd2gtdfJvBkVMfPBw/31RecVkAKaWdknmzgsUc9vnwEZ+3N0IfiZAt7R0zKcBapQZffsZninsYrs2vpQ4dDE8OH1eIXUECjRLine truncated
Your browser does not support the video tag.
</video>
docs/notebooks/203-meter-reader-with-output.rst
+2 -2
View File
@@ -524,7 +524,7 @@ Initialize the model and parameters.
The number of detected meter from detection network can be arbitrary in
some scenarios, which means the batch size of segmentation network input
is a `dynamic
dimension <https://docs.openvino.ai/latest/openvino_docs_OV_UG_DynamicShapes.html>`__,
dimension <https://docs.openvino.ai/2023.0/openvino_docs_OV_UG_DynamicShapes.html>`__,
and it should be specified as ``-1`` or the ``ov::Dimension()`` instead
of a positive number used for static dimensions. In this case, for
memory consumption optimization, we can specify the lower and/or upper
@@ -557,7 +557,7 @@ bounds of input batch size.
.. parsed-literal::
<matplotlib.image.AxesImage at 0x7f33fc644490>
<matplotlib.image.AxesImage at 0x7fbebc7c0250>
docs/notebooks/203-meter-reader-with-output_files/index.html
+7 -7
View File
@@ -1,11 +1,11 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/203-meter-reader-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/203-meter-reader-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/203-meter-reader-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="203-meter-reader-with-output_12_1.png">203-meter-reader-with-output_12_1.png</a> 30-May-2023 00:08 170121
<a href="203-meter-reader-with-output_14_1.png">203-meter-reader-with-output_14_1.png</a> 30-May-2023 00:08 190271
<a href="203-meter-reader-with-output_16_1.png">203-meter-reader-with-output_16_1.png</a> 30-May-2023 00:08 26914
<a href="203-meter-reader-with-output_18_1.png">203-meter-reader-with-output_18_1.png</a> 30-May-2023 00:08 8966
<a href="203-meter-reader-with-output_20_1.png">203-meter-reader-with-output_20_1.png</a> 30-May-2023 00:08 170338
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/203-meter-reader-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="203-meter-reader-with-output_12_1.png">203-meter-reader-with-output_12_1.png</a> 22-Jun-2023 00:06 170121
<a href="203-meter-reader-with-output_14_1.png">203-meter-reader-with-output_14_1.png</a> 22-Jun-2023 00:06 190271
<a href="203-meter-reader-with-output_16_1.png">203-meter-reader-with-output_16_1.png</a> 22-Jun-2023 00:06 26914
<a href="203-meter-reader-with-output_18_1.png">203-meter-reader-with-output_18_1.png</a> 22-Jun-2023 00:06 8966
<a href="203-meter-reader-with-output_20_1.png">203-meter-reader-with-output_20_1.png</a> 22-Jun-2023 00:06 170338
</pre><hr></body>
</html>
docs/notebooks/204-segmenter-semantic-segmentation-with-output.rst
+31 -25
View File
@@ -102,7 +102,7 @@ config for our model.
Cloning into 'segmenter'...
remote: Enumerating objects: 268, done.
remote: Total 268 (delta 0), reused 0 (delta 0), pack-reused 268
Receiving objects: 100% (268/268), 15.34 MiB | 3.86 MiB/s, done.
Receiving objects: 100% (268/268), 15.34 MiB | 3.67 MiB/s, done.
Resolving deltas: 100% (117/117), done.
@@ -180,7 +180,7 @@ Load normalization settings from config file.
.. parsed-literal::
No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda'
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/mmcv/__init__.py:20: UserWarning: On January 1, 2023, MMCV will release v2.0.0, in which it will remove components related to the training process and add a data transformation module. In addition, it will rename the package names mmcv to mmcv-lite and mmcv-full to mmcv. See https://github.com/open-mmlab/mmcv/blob/master/docs/en/compatibility.md for more details.
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/mmcv/__init__.py:20: UserWarning: On January 1, 2023, MMCV will release v2.0.0, in which it will remove components related to the training process and add a data transformation module. In addition, it will rename the package names mmcv to mmcv-lite and mmcv-full to mmcv. See https://github.com/open-mmlab/mmcv/blob/master/docs/en/compatibility.md for more details.
warnings.warn(
@@ -368,23 +368,23 @@ problem.
.. parsed-literal::
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:69: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:69: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
if H % patch_size > 0:
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:71: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:71: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
if W % patch_size > 0:
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/vit.py:122: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/vit.py:122: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
if x.shape[1] != pos_embed.shape[1]:
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/decoder.py:100: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/decoder.py:100: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
masks = rearrange(masks, "b (h w) n -> b n h w", h=int(GS))
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:85: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
if extra_h > 0:
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:87: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/notebooks/204-segmenter-semantic-segmentation/./segmenter/segm/model/utils.py:87: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
if extra_w > 0:
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/_internal/jit_utils.py:258: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/_internal/jit_utils.py:258: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
_C._jit_pass_onnx_node_shape_type_inference(node, params_dict, opset_version)
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:687: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:687: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
_C._jit_pass_onnx_graph_shape_type_inference(
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:1178: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:1178: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
_C._jit_pass_onnx_graph_shape_type_inference(
@@ -515,7 +515,7 @@ Benchmarking performance of converted model
-------------------------------------------
Finally, use the OpenVINO `Benchmark
Tool <https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_tool_README.html>`__
Tool <https://docs.openvino.ai/2023.0/openvino_inference_engine_tools_benchmark_tool_README.html>`__
to measure the inference performance of the model.
NOTE: For more accurate performance, it is recommended to run
@@ -543,18 +543,18 @@ to measure the inference performance of the model.
[ INFO ] Parsing input parameters
[Step 2/11] Loading OpenVINO Runtime
[ INFO ] OpenVINO:
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ] Device info:
[ INFO ] CPU
[ INFO ] Build ................................. 2022.3.0-9052-9752fafe8eb-releases/2022/3
[ INFO ] Build ................................. 2023.0.0-10926-b4452d56304-releases/2023/0
[ INFO ]
[ INFO ]
[Step 3/11] Setting device configuration
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to THROUGHPUT.
[ WARNING ] Performance hint was not explicitly specified in command line. Device(CPU) performance hint will be set to PerformanceMode.THROUGHPUT.
[Step 4/11] Reading model files
[ INFO ] Loading model files
[ INFO ] Read model took 61.33 ms
[ INFO ] Read model took 32.06 ms
[ INFO ] Original model I/O parameters:
[ INFO ] Model inputs:
[ INFO ] input (node: input) : f32 / [...] / [2,3,512,512]
@@ -568,7 +568,7 @@ to measure the inference performance of the model.
[ INFO ] Model outputs:
[ INFO ] output (node: output) : f32 / [...] / [2,150,512,512]
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 290.08 ms
[ INFO ] Compile model took 279.07 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] Model:
[ INFO ] NETWORK_NAME: torch_jit
@@ -579,20 +579,26 @@ to measure the inference performance of the model.
[ INFO ] PERF_COUNT: False
[ INFO ] INFERENCE_PRECISION_HINT: <Type: 'float32'>
[ INFO ] PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
[ INFO ] EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS: 0
[ INFO ] ENABLE_CPU_PINNING: True
[ INFO ] SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
[ INFO ] ENABLE_HYPER_THREADING: True
[ INFO ] EXECUTION_DEVICES: ['CPU']
[Step 9/11] Creating infer requests and preparing input tensors
[ WARNING ] No input files were given for input 'input'!. This input will be filled with random values!
[ INFO ] Fill input 'input' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 225.05 ms
[ INFO ] First inference took 230.53 ms
[Step 11/11] Dumping statistics report
[ INFO ] Count: 684 iterations
[ INFO ] Duration: 60377.87 ms
[ INFO ] Execution Devices:['CPU']
[ INFO ] Count: 690 iterations
[ INFO ] Duration: 60351.52 ms
[ INFO ] Latency:
[ INFO ] Median: 527.02 ms
[ INFO ] Average: 528.54 ms
[ INFO ] Min: 262.07 ms
[ INFO ] Max: 612.30 ms
[ INFO ] Throughput: 22.66 FPS
[ INFO ] Median: 521.86 ms
[ INFO ] Average: 523.54 ms
[ INFO ] Min: 265.46 ms
[ INFO ] Max: 793.17 ms
[ INFO ] Throughput: 22.87 FPS
docs/notebooks/204-segmenter-semantic-segmentation-with-output_files/204-segmenter-semantic-segmentation-with-output_32_0.png
+2 -2
View File
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:447ac8ce1c4a870e1adce81b4c3ad5500ce0bec86bcdf206c5fe1bf65a331c25
size 835089
oid sha256:56a9ea2deb460047985e546855e2d014eebe229eeb03252f477b992c4331895e
size 835091
docs/notebooks/204-segmenter-semantic-segmentation-with-output_files/index.html
+4 -4
View File
@@ -1,8 +1,8 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/204-segmenter-semantic-segmentation-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/204-segmenter-semantic-segmentation-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/204-segmenter-semantic-segmentation-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="204-segmenter-semantic-segmentation-with-output_21_0.png">204-segmenter-semantic-segmentation-with-output..&gt;</a> 30-May-2023 00:08 835089
<a href="204-segmenter-semantic-segmentation-with-output_32_0.png">204-segmenter-semantic-segmentation-with-output..&gt;</a> 30-May-2023 00:08 835089
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/204-segmenter-semantic-segmentation-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="204-segmenter-semantic-segmentation-with-output_21_0.png">204-segmenter-semantic-segmentation-with-output..&gt;</a> 22-Jun-2023 00:06 835089
<a href="204-segmenter-semantic-segmentation-with-output_32_0.png">204-segmenter-semantic-segmentation-with-output..&gt;</a> 22-Jun-2023 00:06 835091
</pre><hr></body>
</html>
docs/notebooks/205-vision-background-removal-with-output.rst
+26 -8
View File
@@ -99,6 +99,24 @@ next cell loads the model and the pre-trained weights.
gdown.download(url=u2net_model.url, output=model_file)
print(f"Model weights have been downloaded to {model_path}")
.. parsed-literal::
Start downloading model weights file...
.. parsed-literal::
Downloading...
From: https://drive.google.com/uc?id=1rbSTGKAE-MTxBYHd-51l2hMOQPT_7EPy
To: <_io.BufferedWriter name='model/u2net_lite/u2net_lite.pth'>
100%|██████████| 4.68M/4.68M [00:01<00:00, 4.43MB/s]
.. parsed-literal::
Model weights have been downloaded to model/u2net_lite/u2net_lite.pth
.. code:: ipython3
# Load the model.
@@ -119,15 +137,15 @@ next cell loads the model and the pre-trained weights.
.. parsed-literal::
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/functional.py:3734: UserWarning: nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/functional.py:3734: UserWarning: nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.
warnings.warn("nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.")
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/functional.py:1967: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/nn/functional.py:1967: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/_internal/jit_utils.py:258: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/_internal/jit_utils.py:258: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
_C._jit_pass_onnx_node_shape_type_inference(node, params_dict, opset_version)
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:687: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:687: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
_C._jit_pass_onnx_graph_shape_type_inference(
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:1178: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/onnx/utils.py:1178: UserWarning: The shape inference of prim::Constant type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (Triggered internally at ../torch/csrc/jit/passes/onnx/shape_type_inference.cpp:1884.)
_C._jit_pass_onnx_graph_shape_type_inference(
@@ -149,7 +167,7 @@ repository <https://github.com/xuebinqin/U-2-Net/>`__ and multiplied by
255 to support images with pixel values from 0-255.
For more information, refer to the `Model Optimizer Developer
Guide <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
Executing the following command may take a while.
@@ -209,7 +227,7 @@ Load the OpenVINO IR model to OpenVINO Runtime and do inference.
.. parsed-literal::
Inference finished. Inference time: 0.123 seconds, FPS: 8.10.
Inference finished. Inference time: 0.122 seconds, FPS: 8.20.
Visualize Results
@@ -314,7 +332,7 @@ References
- `PIP install
openvino-dev <https://github.com/openvinotoolkit/openvino/blob/releases/2021/3/docs/install_guides/pypi-openvino-dev.md>`__
- `Model Optimizer
Documentation <https://docs.openvino.ai/latest/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html>`__
Documentation <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html>`__
- `U^2-Net <https://github.com/xuebinqin/U-2-Net>`__
- U^2-Net research paper: `U^2-Net: Going Deeper with Nested
U-Structure for Salient Object
docs/notebooks/205-vision-background-removal-with-output_files/index.html
+4 -4
View File
@@ -1,8 +1,8 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/205-vision-background-removal-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/205-vision-background-removal-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/205-vision-background-removal-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="205-vision-background-removal-with-output_18_0.png">205-vision-background-removal-with-output_18_0.png</a> 30-May-2023 00:09 279567
<a href="205-vision-background-removal-with-output_20_0.png">205-vision-background-removal-with-output_20_0.png</a> 30-May-2023 00:09 927148
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/205-vision-background-removal-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="205-vision-background-removal-with-output_18_0.png">205-vision-background-removal-with-output_18_0.png</a> 22-Jun-2023 00:06 279567
<a href="205-vision-background-removal-with-output_20_0.png">205-vision-background-removal-with-output_20_0.png</a> 22-Jun-2023 00:06 927148
</pre><hr></body>
</html>
docs/notebooks/206-vision-paddlegan-anime-with-output.rst
+10 -9
View File
@@ -32,6 +32,7 @@ Imports
ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
paddleclas 2.5.1 requires faiss-cpu==1.7.1.post2, but you have faiss-cpu 1.7.4 which is incompatible.
paddleclas 2.5.1 requires gast==0.3.3, but you have gast 0.4.0 which is incompatible.
scikit-image 0.21.0 requires imageio>=2.27, but you have imageio 2.9.0 which is incompatible.
.. code:: ipython3
@@ -122,7 +123,7 @@ source of the function.
.. parsed-literal::
[05/29 23:19:20] ppgan INFO: Found /opt/home/k8sworker/.cache/ppgan/animeganv2_hayao.pdparams
[06/21 23:07:12] ppgan INFO: Found /opt/home/k8sworker/.cache/ppgan/animeganv2_hayao.pdparams
.. code:: ipython3
@@ -251,19 +252,19 @@ succeeds, the output of the next cell will include
.. parsed-literal::
2023-05-29 23:19:28 [INFO] Static PaddlePaddle model saved in model/paddle_model_static_onnx_temp_dir.
2023-06-21 23:07:20 [INFO] Static PaddlePaddle model saved in model/paddle_model_static_onnx_temp_dir.
[Paddle2ONNX] Start to parse PaddlePaddle model...
[Paddle2ONNX] Model file path: model/paddle_model_static_onnx_temp_dir/model.pdmodel
[Paddle2ONNX] Paramters file path: model/paddle_model_static_onnx_temp_dir/model.pdiparams
[Paddle2ONNX] Start to parsing Paddle model...
[Paddle2ONNX] Use opset_version = 11 for ONNX export.
[Paddle2ONNX] PaddlePaddle model is exported as ONNX format now.
2023-05-29 23:19:28 [INFO] ONNX model saved in model/paddlegan_anime.onnx.
2023-06-21 23:07:20 [INFO] ONNX model saved in model/paddlegan_anime.onnx.
.. parsed-literal::
I0529 23:19:28.491920 2574087 interpretercore.cc:267] New Executor is Running.
I0621 23:07:20.536170 1218719 interpretercore.cc:267] New Executor is Running.
Convert to OpenVINO IR
@@ -310,11 +311,11 @@ inputs are known, you can use Model Optimizer and convert the model to
OpenVINO IR with these values. Use ``FP16`` precision and set log level
to ``CRITICAL`` to ignore warnings that are irrelevant for this demo.
For information about setting the parameters, see the `Model Optimizer
Documentation <https://docs.openvino.ai/latest/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html>`__
Documentation <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html>`__
.
**Convert ONNX Model to OpenVINO IR with**\ `Model Optimizer Python
API <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Python_API.html>`__
API <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Python_API.html>`__
.. code:: ipython3
@@ -527,8 +528,8 @@ measure inference on one image. For more accurate benchmarking, use
.. parsed-literal::
OpenVINO IR model in OpenVINO Runtime/CPU: 0.475 seconds per image, FPS: 2.10
PaddleGAN model on CPU: 6.630 seconds per image, FPS: 0.15
OpenVINO IR model in OpenVINO Runtime/CPU: 0.472 seconds per image, FPS: 2.12
PaddleGAN model on CPU: 6.423 seconds per image, FPS: 0.16
References
@@ -539,7 +540,7 @@ References
- `OpenVINO ONNX
support <https://docs.openvino.ai/2021.4/openvino_docs_IE_DG_ONNX_Support.html>`__
- `OpenVINO Model Optimizer
Documentation <https://docs.openvino.ai/latest/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html>`__
Documentation <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html>`__
The PaddleGAN code that is shown in this notebook is written by
PaddlePaddle Authors and licensed under the Apache 2.0 license. The
docs/notebooks/206-vision-paddlegan-anime-with-output_files/206-vision-paddlegan-anime-with-output_36_0.png
+2 -2
View File
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:fe13a31873633d1a801332541f9cb4f789b812aec2c0a9df15792047712d0a93
size 1931528
oid sha256:da8386a7ba2d7aec1fb16cf5ad9bb0665a372c044b8b0950c7c7e2fcbe450e67
size 1931653
docs/notebooks/206-vision-paddlegan-anime-with-output_files/index.html
+4 -4
View File
@@ -1,8 +1,8 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/206-vision-paddlegan-anime-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/206-vision-paddlegan-anime-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/206-vision-paddlegan-anime-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="206-vision-paddlegan-anime-with-output_15_0.png">206-vision-paddlegan-anime-with-output_15_0.png</a> 30-May-2023 00:09 1810982
<a href="206-vision-paddlegan-anime-with-output_36_0.png">206-vision-paddlegan-anime-with-output_36_0.png</a> 30-May-2023 00:09 1931528
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/206-vision-paddlegan-anime-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="206-vision-paddlegan-anime-with-output_15_0.png">206-vision-paddlegan-anime-with-output_15_0.png</a> 22-Jun-2023 00:06 1810982
<a href="206-vision-paddlegan-anime-with-output_36_0.png">206-vision-paddlegan-anime-with-output_36_0.png</a> 22-Jun-2023 00:06 1931653
</pre><hr></body>
</html>
docs/notebooks/207-vision-paddlegan-superresolution-with-output.rst
+8 -8
View File
@@ -86,7 +86,7 @@ source code.
.. parsed-literal::
[05/29 23:19:56] ppgan INFO: Found /opt/home/k8sworker/.cache/ppgan/DF2K_JPEG.pdparams
[06/21 23:07:46] ppgan INFO: Found /opt/home/k8sworker/.cache/ppgan/DF2K_JPEG.pdparams
.. code:: ipython3
@@ -194,12 +194,12 @@ Convert PaddlePaddle Model to ONNX
.. parsed-literal::
2023-05-29 23:20:02 [INFO] Static PaddlePaddle model saved in model/paddle_model_static_onnx_temp_dir.
2023-06-21 23:07:52 [INFO] Static PaddlePaddle model saved in model/paddle_model_static_onnx_temp_dir.
.. parsed-literal::
I0529 23:20:02.063105 2574402 interpretercore.cc:267] New Executor is Running.
I0621 23:07:52.629609 1219036 interpretercore.cc:267] New Executor is Running.
.. parsed-literal::
@@ -210,10 +210,10 @@ Convert PaddlePaddle Model to ONNX
[Paddle2ONNX] Start to parsing Paddle model...
[Paddle2ONNX] Use opset_version = 13 for ONNX export.
[Paddle2ONNX] PaddlePaddle model is exported as ONNX format now.
2023-05-29 23:20:05 [INFO] ONNX model saved in model/paddlegan_sr.onnx.
2023-06-21 23:07:56 [INFO] ONNX model saved in model/paddlegan_sr.onnx.
Convert ONNX Model to OpenVINO IR with `Model Optimizer Python API <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Python_API.html>`__
Convert ONNX Model to OpenVINO IR with `Model Optimizer Python API <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Python_API.html>`__
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. code:: ipython3
@@ -273,7 +273,7 @@ Do Inference on OpenVINO IR Model
.. parsed-literal::
<matplotlib.image.AxesImage at 0x7f2169733e50>
<matplotlib.image.AxesImage at 0x7f2bec0cf4c0>
@@ -302,7 +302,7 @@ Do Inference on OpenVINO IR Model
.. parsed-literal::
Inference duration: 3.34 seconds
Inference duration: 3.24 seconds
.. code:: ipython3
@@ -325,7 +325,7 @@ Do Inference on OpenVINO IR Model
.. parsed-literal::
<matplotlib.image.AxesImage at 0x7f2130210790>
<matplotlib.image.AxesImage at 0x7f2bd96ef520>
docs/notebooks/207-vision-paddlegan-superresolution-with-output_files/207-vision-paddlegan-superresolution-with-output_29_1.png
+2 -2
View File
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:cb63abe889abe594a5c5cc8694b247f868bfd06e21d86bfbbbca5f0f9ab9fe2d
size 476187
oid sha256:f48ad1228e9709e05ef8ecea327b59a021d0a9e94eed951aa1fe481188b9a2ae
size 476190
docs/notebooks/207-vision-paddlegan-superresolution-with-output_files/207-vision-paddlegan-superresolution-with-output_31_0.png
+1 -1
View File
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:ba72b7520b2c698ec70fe5c41b408ac52af2ea8530fa264815624a09a0893b02
oid sha256:e07b52d5b664f1e00daebc13d9d7bbbecc479445b43fd04b5f5488cdfa8ab739
size 2835354
docs/notebooks/207-vision-paddlegan-superresolution-with-output_files/index.html
+5 -5
View File
@@ -1,9 +1,9 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/207-vision-paddlegan-superresolution-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/207-vision-paddlegan-superresolution-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/207-vision-paddlegan-superresolution-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="207-vision-paddlegan-superresolution-with-output_25_1.png">207-vision-paddlegan-superresolution-with-outpu..&gt;</a> 30-May-2023 00:08 436999
<a href="207-vision-paddlegan-superresolution-with-output_29_1.png">207-vision-paddlegan-superresolution-with-outpu..&gt;</a> 30-May-2023 00:08 476187
<a href="207-vision-paddlegan-superresolution-with-output_31_0.png">207-vision-paddlegan-superresolution-with-outpu..&gt;</a> 30-May-2023 00:08 2835354
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/207-vision-paddlegan-superresolution-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="207-vision-paddlegan-superresolution-with-output_25_1.png">207-vision-paddlegan-superresolution-with-outpu..&gt;</a> 22-Jun-2023 00:06 436999
<a href="207-vision-paddlegan-superresolution-with-output_29_1.png">207-vision-paddlegan-superresolution-with-outpu..&gt;</a> 22-Jun-2023 00:06 476190
<a href="207-vision-paddlegan-superresolution-with-output_31_0.png">207-vision-paddlegan-superresolution-with-outpu..&gt;</a> 22-Jun-2023 00:06 2835354
</pre><hr></body>
</html>
docs/notebooks/208-optical-character-recognition-with-output.rst
+9 -9
View File
@@ -7,9 +7,9 @@ This tutorial demonstrates how to perform optical character recognition
tutorial, which shows only text detection.
The
`horizontal-text-detection-0001 <https://docs.openvino.ai/latest/omz_models_model_horizontal_text_detection_0001.html>`__
`horizontal-text-detection-0001 <https://docs.openvino.ai/2023.0/omz_models_model_horizontal_text_detection_0001.html>`__
and
`text-recognition-resnet <https://docs.openvino.ai/latest/omz_models_model_text_recognition_resnet_fc.html>`__
`text-recognition-resnet <https://docs.openvino.ai/2023.0/omz_models_model_text_recognition_resnet_fc.html>`__
models are used together for text detection and then text recognition.
In this tutorial, Open Model Zoo tools including Model Downloader, Model
@@ -300,22 +300,22 @@ Converting text-recognition-resnet-fc…
.. parsed-literal::
========== Converting text-recognition-resnet-fc to ONNX
Conversion to ONNX command: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/internal_scripts/pytorch_to_onnx.py --model-path=/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/models/public/text-recognition-resnet-fc --model-path=model/public/text-recognition-resnet-fc --model-name=get_model --import-module=model '--model-param=file_config=r"model/public/text-recognition-resnet-fc/vedastr/configs/resnet_fc.py"' '--model-param=weights=r"model/public/text-recognition-resnet-fc/vedastr/ckpt/resnet_fc.pth"' --input-shape=1,1,32,100 --input-names=input --output-names=output --output-file=model/public/text-recognition-resnet-fc/resnet_fc.onnx
Conversion to ONNX command: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/internal_scripts/pytorch_to_onnx.py --model-path=/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/model_zoo/models/public/text-recognition-resnet-fc --model-path=model/public/text-recognition-resnet-fc --model-name=get_model --import-module=model '--model-param=file_config=r"model/public/text-recognition-resnet-fc/vedastr/configs/resnet_fc.py"' '--model-param=weights=r"model/public/text-recognition-resnet-fc/vedastr/ckpt/resnet_fc.pth"' --input-shape=1,1,32,100 --input-names=input --output-names=output --output-file=model/public/text-recognition-resnet-fc/resnet_fc.onnx
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchvision/models/_utils.py:252: UserWarning: Accessing the model URLs via the internal dictionary of the module is deprecated since 0.13 and may be removed in the future. Please access them via the appropriate Weights Enum instead.
/opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchvision/models/_utils.py:252: UserWarning: Accessing the model URLs via the internal dictionary of the module is deprecated since 0.13 and may be removed in the future. Please access them via the appropriate Weights Enum instead.
warnings.warn(
ONNX check passed successfully.
========== Converting text-recognition-resnet-fc to IR (FP16)
Conversion command: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/.venv/bin/mo --framework=onnx --data_type=FP16 --output_dir=model/public/text-recognition-resnet-fc/FP16 --model_name=text-recognition-resnet-fc --input=input '--mean_values=input[127.5]' '--scale_values=input[127.5]' --output=output --input_model=model/public/text-recognition-resnet-fc/resnet_fc.onnx '--layout=input(NCHW)' '--input_shape=[1, 1, 32, 100]'
Conversion command: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-433/.workspace/scm/ov-notebook/.venv/bin/mo --framework=onnx --output_dir=/tmp/tmpcw9uu3f_ --model_name=text-recognition-resnet-fc --input=input '--mean_values=input[127.5]' '--scale_values=input[127.5]' --output=output --input_model=model/public/text-recognition-resnet-fc/resnet_fc.onnx '--layout=input(NCHW)' '--input_shape=[1, 1, 32, 100]' --compress_to_fp16=True
[ WARNING ] Use of deprecated cli option --data_type detected. Option use in the following releases will be fatal.
Check for a new version of Intel(R) Distribution of OpenVINO(TM) toolkit here https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit/download.html?cid=other&source=prod&campid=ww_2023_bu_IOTG_OpenVINO-2022-3&content=upg_all&medium=organic or on https://github.com/openvinotoolkit/openvino
[ INFO ] Generated IR will be compressed to FP16. If you get lower accuracy, please consider disabling compression by removing argument --compress_to_fp16 or set it to false --compress_to_fp16=False.
Find more information about compression to FP16 at https://docs.openvino.ai/latest/openvino_docs_MO_DG_FP16_Compression.html
[ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11.
Find more information about API v2.0 and IR v11 at https://docs.openvino.ai/latest/openvino_2_0_transition_guide.html
[ SUCCESS ] Generated IR version 11 model.
[ SUCCESS ] XML file: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/208-optical-character-recognition/model/public/text-recognition-resnet-fc/FP16/text-recognition-resnet-fc.xml
[ SUCCESS ] BIN file: /opt/home/k8sworker/cibuilds/ov-notebook/OVNotebookOps-416/.workspace/scm/ov-notebook/notebooks/208-optical-character-recognition/model/public/text-recognition-resnet-fc/FP16/text-recognition-resnet-fc.bin
[ SUCCESS ] XML file: /tmp/tmpcw9uu3f_/text-recognition-resnet-fc.xml
[ SUCCESS ] BIN file: /tmp/tmpcw9uu3f_/text-recognition-resnet-fc.bin
docs/notebooks/208-optical-character-recognition-with-output_files/index.html
+10 -10
View File
@@ -1,14 +1,14 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/208-optical-character-recognition-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/208-optical-character-recognition-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/208-optical-character-recognition-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="208-optical-character-recognition-with-output_13_0.png">208-optical-character-recognition-with-output_1..&gt;</a> 30-May-2023 00:09 305482
<a href="208-optical-character-recognition-with-output_23_0.png">208-optical-character-recognition-with-output_2..&gt;</a> 30-May-2023 00:09 923631
<a href="208-optical-character-recognition-with-output_25_0.png">208-optical-character-recognition-with-output_2..&gt;</a> 30-May-2023 00:09 10619
<a href="208-optical-character-recognition-with-output_25_10.png">208-optical-character-recognition-with-output_2..&gt;</a> 30-May-2023 00:09 10441
<a href="208-optical-character-recognition-with-output_25_2.png">208-optical-character-recognition-with-output_2..&gt;</a> 30-May-2023 00:09 7881
<a href="208-optical-character-recognition-with-output_25_4.png">208-optical-character-recognition-with-output_2..&gt;</a> 30-May-2023 00:09 2300
<a href="208-optical-character-recognition-with-output_25_6.png">208-optical-character-recognition-with-output_2..&gt;</a> 30-May-2023 00:09 1532
<a href="208-optical-character-recognition-with-output_25_8.png">208-optical-character-recognition-with-output_2..&gt;</a> 30-May-2023 00:09 1573
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/208-optical-character-recognition-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="208-optical-character-recognition-with-output_13_0.png">208-optical-character-recognition-with-output_1..&gt;</a> 22-Jun-2023 00:06 305482
<a href="208-optical-character-recognition-with-output_23_0.png">208-optical-character-recognition-with-output_2..&gt;</a> 22-Jun-2023 00:06 923631
<a href="208-optical-character-recognition-with-output_25_0.png">208-optical-character-recognition-with-output_2..&gt;</a> 22-Jun-2023 00:06 10619
<a href="208-optical-character-recognition-with-output_25_10.png">208-optical-character-recognition-with-output_2..&gt;</a> 22-Jun-2023 00:06 10441
<a href="208-optical-character-recognition-with-output_25_2.png">208-optical-character-recognition-with-output_2..&gt;</a> 22-Jun-2023 00:06 7881
<a href="208-optical-character-recognition-with-output_25_4.png">208-optical-character-recognition-with-output_2..&gt;</a> 22-Jun-2023 00:06 2300
<a href="208-optical-character-recognition-with-output_25_6.png">208-optical-character-recognition-with-output_2..&gt;</a> 22-Jun-2023 00:06 1532
<a href="208-optical-character-recognition-with-output_25_8.png">208-optical-character-recognition-with-output_2..&gt;</a> 22-Jun-2023 00:06 1573
</pre><hr></body>
</html>
docs/notebooks/209-handwritten-ocr-with-output.rst
+2 -2
View File
@@ -8,9 +8,9 @@ Latin alphabet is available in `notebook
This model is capable of processing only one line of symbols at a time.
The models used in this notebook are
`handwritten-japanese-recognition <https://docs.openvino.ai/latest/omz_models_model_handwritten_japanese_recognition_0001.html>`__
`handwritten-japanese-recognition <https://docs.openvino.ai/2023.0/omz_models_model_handwritten_japanese_recognition_0001.html>`__
and
`handwritten-simplified-chinese <https://docs.openvino.ai/latest/omz_models_model_handwritten_simplified_chinese_recognition_0001.html>`__.
`handwritten-simplified-chinese <https://docs.openvino.ai/2023.0/omz_models_model_handwritten_simplified_chinese_recognition_0001.html>`__.
To decode model outputs as readable text
`kondate_nakayosi <https://github.com/openvinotoolkit/open_model_zoo/blob/master/data/dataset_classes/kondate_nakayosi.txt>`__
and
docs/notebooks/209-handwritten-ocr-with-output_files/index.html
+4 -4
View File
@@ -1,8 +1,8 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/209-handwritten-ocr-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/209-handwritten-ocr-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/209-handwritten-ocr-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="209-handwritten-ocr-with-output_20_0.png">209-handwritten-ocr-with-output_20_0.png</a> 30-May-2023 00:08 53571
<a href="209-handwritten-ocr-with-output_29_1.png">209-handwritten-ocr-with-output_29_1.png</a> 30-May-2023 00:08 53571
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/209-handwritten-ocr-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="209-handwritten-ocr-with-output_20_0.png">209-handwritten-ocr-with-output_20_0.png</a> 22-Jun-2023 00:06 53571
<a href="209-handwritten-ocr-with-output_29_1.png">209-handwritten-ocr-with-output_29_1.png</a> 22-Jun-2023 00:06 53571
</pre><hr></body>
</html>
docs/notebooks/211-speech-to-text-with-output.rst
+1 -1
View File
@@ -268,7 +268,7 @@ spectrogram.
.. parsed-literal::
/tmp/ipykernel_3057245/2518307745.py:2: FutureWarning: waveshow() keyword argument 'x_axis' has been renamed to 'axis' in version 0.10.0.
/tmp/ipykernel_2115037/2518307745.py:2: FutureWarning: waveshow() keyword argument 'x_axis' has been renamed to 'axis' in version 0.10.0.
This alias will be removed in version 1.0.
librosa.display.waveshow(y=audio, sr=sampling_rate, max_points=50000, x_axis='time', offset=0.0);
docs/notebooks/211-speech-to-text-with-output_files/index.html
+6 -6
View File
@@ -1,10 +1,10 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/211-speech-to-text-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/211-speech-to-text-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/211-speech-to-text-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="211-speech-to-text-with-output_21_1.png">211-speech-to-text-with-output_21_1.png</a> 30-May-2023 00:08 21971
<a href="211-speech-to-text-with-output_21_3.png">211-speech-to-text-with-output_21_3.png</a> 30-May-2023 00:08 87067
<a href="211-speech-to-text-with-output_29_0.png">211-speech-to-text-with-output_29_0.png</a> 30-May-2023 00:08 50625
<a href="211-speech-to-text-with-output_29_1.png">211-speech-to-text-with-output_29_1.png</a> 30-May-2023 00:08 10083
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/211-speech-to-text-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="211-speech-to-text-with-output_21_1.png">211-speech-to-text-with-output_21_1.png</a> 22-Jun-2023 00:06 21971
<a href="211-speech-to-text-with-output_21_3.png">211-speech-to-text-with-output_21_3.png</a> 22-Jun-2023 00:06 87067
<a href="211-speech-to-text-with-output_29_0.png">211-speech-to-text-with-output_29_0.png</a> 22-Jun-2023 00:06 50625
<a href="211-speech-to-text-with-output_29_1.png">211-speech-to-text-with-output_29_1.png</a> 22-Jun-2023 00:06 10083
</pre><hr></body>
</html>
docs/notebooks/212-pyannote-speaker-diarization-with-output.rst
+2 -2
View File
@@ -203,7 +203,7 @@ pipeline
.. parsed-literal::
Diarization pipeline took 15.66 s
Diarization pipeline took 15.13 s
The result of running the pipeline can be represented as a diagram
@@ -330,7 +330,7 @@ Run speaker diarization with OpenVINO
.. parsed-literal::
Diarization pipeline took 15.14 s
Diarization pipeline took 14.63 s
.. code:: ipython3
docs/notebooks/212-pyannote-speaker-diarization-with-output_files/index.html
+5 -5
View File
@@ -1,9 +1,9 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/212-pyannote-speaker-diarization-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/212-pyannote-speaker-diarization-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/212-pyannote-speaker-diarization-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="212-pyannote-speaker-diarization-with-output_14_0.png">212-pyannote-speaker-diarization-with-output_14..&gt;</a> 30-May-2023 00:09 7969
<a href="212-pyannote-speaker-diarization-with-output_25_0.png">212-pyannote-speaker-diarization-with-output_25..&gt;</a> 30-May-2023 00:09 7969
<a href="212-pyannote-speaker-diarization-with-output_9_1.png">212-pyannote-speaker-diarization-with-output_9_..&gt;</a> 30-May-2023 00:09 43095
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/212-pyannote-speaker-diarization-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="212-pyannote-speaker-diarization-with-output_14_0.png">212-pyannote-speaker-diarization-with-output_14..&gt;</a> 22-Jun-2023 00:06 7969
<a href="212-pyannote-speaker-diarization-with-output_25_0.png">212-pyannote-speaker-diarization-with-output_25..&gt;</a> 22-Jun-2023 00:06 7969
<a href="212-pyannote-speaker-diarization-with-output_9_1.png">212-pyannote-speaker-diarization-with-output_9_..&gt;</a> 22-Jun-2023 00:06 43095
</pre><hr></body>
</html>
docs/notebooks/213-question-answering-with-output.rst
+2 -2
View File
@@ -430,7 +430,7 @@ questions in the box.**
Context: ['Computational complexity theory is a branch of the theory of computation in theoretical computer science that focuses on classifying computational problems according to their inherent difficulty, and relating those classes to each other. A computational problem is understood to be a task that is in principle amenable to being solved by a computer, which is equivalent to stating that the problem may be solved by mechanical application of mathematical steps, such as an algorithm.']
Question: What is the term for a task that generally lends itself to being solved by a computer?
Answer: A computational problem
Score: 0.52
Score: 0.51
Time: 0.03s
@@ -468,5 +468,5 @@ questions in the box.**
Question: What does OpenVINO mean?
Answer: Open Visual Inference and Neural network Optimization
Score: 0.92
Time: 0.05s
Time: 0.06s
docs/notebooks/215-image-inpainting-with-output.rst
+8 -18
View File
@@ -20,8 +20,8 @@ original image. The Following pipeline will be used in this notebook.
import matplotlib.pyplot as plt
import numpy as np
from zipfile import ZipFile
from openvino.runtime import Core, Tensor
from IPython.display import Markdown, display
from openvino.tools import mo
from openvino.runtime import Core, Tensor, serialize
sys.path.append("../utils")
import notebook_utils as utils
@@ -33,7 +33,7 @@ Download ``gmcnn-places2-tf``\ model (this step will be skipped if the
model is already downloaded) and then unzip it. Downloaded model stored
in TensorFlow frozen graph format. The steps how this frozen graph can
be obtained from original model checkpoint can be found in this
`instruction <https://docs.openvino.ai/latest/omz_models_model_gmcnn_places2_tf.html#steps-to-reproduce-conversion-to-frozen-graph>`__
`instruction <https://docs.openvino.ai/2023.0/omz_models_model_gmcnn_places2_tf.html#steps-to-reproduce-conversion-to-frozen-graph>`__
.. code:: ipython3
@@ -65,7 +65,7 @@ The pre-trained model is in TensorFlow format. To use it with OpenVINO,
convert it to OpenVINO IR format. To do this Model Optimizer is used.
For more information about Model Optimizer, see the `Model Optimizer
Developer
Guide <https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
This step is also skipped if the model is already converted.
.. code:: ipython3
@@ -75,10 +75,8 @@ This step is also skipped if the model is already converted.
# Run Model Optimizer to convert model to OpenVINO IR FP32 format, if the IR file does not exist.
if not ir_path.exists():
mo_command = f'mo --input_model {model_path} --input Placeholder,Placeholder_1 --input_shape [1,512,680,3],[1,512,680,1] --output=Minimum --output_dir {model_dir}'
display(Markdown(f"Model Optimizer command to convert the Tensorflow model to IR: `{mo_command}`"))
display(Markdown("_Converting model to IR. This may take a few minutes..._"))
! $mo_command
ov_model = mo.convert_model(model_path, input_shape=[[1,512,680,3],[1,512,680,1]])
serialize(ov_model, str(ir_path))
else:
print(f"{ir_path} already exists.")
@@ -238,19 +236,11 @@ restored image.
result = compiled_model([Tensor(masked_image.astype(np.float32)), Tensor(mask.astype(np.float32))])[output_layer]
result = result.squeeze().astype(np.uint8)
plt.figure(figsize=(16, 12))
plt.imshow(cv2.cvtColor(result, cv2.COLOR_BGR2RGB))
plt.imshow(cv2.cvtColor(result, cv2.COLOR_BGR2RGB));
.. parsed-literal::
<matplotlib.image.AxesImage at 0x7fade00d60a0>
.. image:: 215-image-inpainting-with-output_files/215-image-inpainting-with-output_20_1.png
.. image:: 215-image-inpainting-with-output_files/215-image-inpainting-with-output_20_0.png
Save the Restored Image
docs/notebooks/215-image-inpainting-with-output_files/215-image-inpainting-with-output_12_0.png
+2 -2
View File
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:584fe2799c6deea5bd6623e1e3736e96ff06f7cbe45c949ca032caf6eea47f2a
size 16144
oid sha256:a7a3868211d34f6eb06edce30cb80a949216610c58cb5399e9c17691a8453a5a
size 16222
docs/notebooks/215-image-inpainting-with-output_files/215-image-inpainting-with-output_16_0.png
+2 -2
View File
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:01612126eb0642391529e3d89a550bf1c67a39a55a1e7e65b82a33901e892b7d
size 498003
oid sha256:cda42e8dbb2992ec7644d01f1ec1d8614e4747d85102ca3e294d7edba21ccd54
size 508083
docs/notebooks/215-image-inpainting-with-output_files/215-image-inpainting-with-output_20_0.png
+3
View File
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:4eaeede639723ce8a680a57a25aef08db871d8c572497b8a78665f68ecca5648
size 592799
docs/notebooks/215-image-inpainting-with-output_files/215-image-inpainting-with-output_20_1.png
-3
View File
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a5f3f6c4d313e1894c9130828e1a787d60fa16e1f03b7170bcc9fc972ad103bc
size 591226
docs/notebooks/215-image-inpainting-with-output_files/index.html
+6 -6
View File
@@ -1,10 +1,10 @@
<html>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/215-image-inpainting-with-output_files/</title></head>
<head><title>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/215-image-inpainting-with-output_files/</title></head>
<body bgcolor="white">
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230529220816/dist/rst_files/215-image-inpainting-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="215-image-inpainting-with-output_12_0.png">215-image-inpainting-with-output_12_0.png</a> 30-May-2023 00:08 16144
<a href="215-image-inpainting-with-output_14_0.png">215-image-inpainting-with-output_14_0.png</a> 30-May-2023 00:08 544222
<a href="215-image-inpainting-with-output_16_0.png">215-image-inpainting-with-output_16_0.png</a> 30-May-2023 00:08 498003
<a href="215-image-inpainting-with-output_20_1.png">215-image-inpainting-with-output_20_1.png</a> 30-May-2023 00:08 591226
<h1>Index of /projects/ov-notebook/0.1.0-latest/20230621220808/dist/rst_files/215-image-inpainting-with-output_files/</h1><hr><pre><a href="../">../</a>
<a href="215-image-inpainting-with-output_12_0.png">215-image-inpainting-with-output_12_0.png</a> 22-Jun-2023 00:06 16222
<a href="215-image-inpainting-with-output_14_0.png">215-image-inpainting-with-output_14_0.png</a> 22-Jun-2023 00:06 544222
<a href="215-image-inpainting-with-output_16_0.png">215-image-inpainting-with-output_16_0.png</a> 22-Jun-2023 00:06 508083
<a href="215-image-inpainting-with-output_20_0.png">215-image-inpainting-with-output_20_0.png</a> 22-Jun-2023 00:06 592799
</pre><hr></body>
</html>
docs/notebooks/216-attention-center-with-output.rst
+259
View File
@@ -0,0 +1,259 @@
The attention center model with OpenVINO™
=========================================
This notebook demonstrates how to use the `attention center
model <https://github.com/google/attention-center/tree/main>`__ with
OpenVINO. This model is in the `TensorFlow Lite
format <https://www.tensorflow.org/lite>`__, which is supported in
OpenVINO now by TFlite frontend.
Eye tracking is commonly used in visual neuroscience and cognitive
science to answer related questions such as visual attention and
decision making. Computational models that predict where to look have
direct applications to a variety of computer vision tasks. The attention
center model takes an RGB image as input and return a 2D point as
output. This 2D point is the predicted center of human attention on the
image i.e. the most salient part of images, on which people pay
attention fist to. This allows find the most visually salient regions
and handle it as early as possible. For example, it could be used for
the latest generatipon image format(such as `JPEG
XL <https://github.com/libjxl/libjxl>`__), which supports encoding the
parts that you pay attention to fist. It can help to improve user
experience, image will appear to load faster.
Attention center model architecture is: > The attention center model is
a deep neural net, which takes an image as input, and uses a pre-trained
classification network, e.g, ResNet, MobileNet, etc., as the backbone.
Several intermediate layers that output from the backbone network are
used as input for the attention center prediction module. These
different intermediate layers contain different information e.g.,
shallow layers often contain low level information like
intensity/color/texture, while deeper layers usually contain higher and
more semantic information like shape/object. All are useful for the
attention prediction. The attention center prediction applies
convolution, deconvolution and/or resizing operator together with
aggregation and sigmoid function to generate a weighting map for the
attention center. And then an operator (the Einstein summation operator
in our case) can be applied to compute the (gravity) center from the
weighting map. An L2 norm between the predicted attention center and the
ground-truth attention center can be computed as the training loss.
Source: `google AI
blogpost <https://opensource.googleblog.com/2022/12/open-sourcing-attention-center-model.html>`__.
The attention center model has been trained with images from the `COCO
dataset <https://cocodataset.org/#home>`__ annotated with saliency from
the `salicon dataset <http://salicon.net/>`__.
The tutorial consists of the following steps: \* Downloading the model
\* Loading the model and make inference with OpenVINO API \* Run Live
Attention Center Detection
Imports
-------
.. code:: ipython3
import cv2
import numpy as np
import tensorflow as tf
from pathlib import Path
import matplotlib.pyplot as plt
from openvino.tools import mo
from openvino.runtime import serialize, Core
.. parsed-literal::
2023-06-21 23:13:35.137764: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-06-21 23:13:35.171412: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-06-21 23:13:35.709438: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
Download the attention-center model
-----------------------------------
Download the model as part of `attention-center
repo <https://github.com/google/attention-center/tree/main>`__. The repo
include model in folder ``./model``.
.. code:: ipython3
if not Path('./attention-center').exists():
! git clone https://github.com/google/attention-center
.. parsed-literal::
Cloning into 'attention-center'...
remote: Enumerating objects: 168, done.
remote: Counting objects: 100% (168/168), done.
remote: Compressing objects: 100% (132/132), done.
remote: Total 168 (delta 73), reused 114 (delta 28), pack-reused 0
Receiving objects: 100% (168/168), 26.22 MiB | 4.68 MiB/s, done.
Resolving deltas: 100% (73/73), done.
Convert Tensorflow Lite model to OpenVINO IR format
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The attention-center model is pre-trained model in TensorFlow Lite
format. In this Notebook the model will be converted to OpenVINO IR
format with Model Optimizer. This step will be skipped if the model have
already been converted. For more information about Model Optimizer,
please, see the `Model Optimizer Developer
Guide <https://docs.openvino.ai/2023.0/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html>`__.
Also TFLite models format is supported in OpenVINO by TFlite frontend,
so the model can be passed directly to ``core.read_model()``. You can
find example in
`002-openvino-api <https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/002-openvino-api>`__.
.. code:: ipython3
tflite_model_path = Path("./attention-center/model/center.tflite")
ir_model_path = Path("./model/ir_center_model.xml")
core = Core()
if not ir_model_path.exists():
model = mo.convert_model(tflite_model_path)
serialize(model, ir_model_path.as_posix())
print("IR model saved to {}".format(ir_model_path))
else:
print("Read IR model from {}".format(ir_model_path))
model = core.read_model(ir_model_path)
device = "CPU"
compiled_model = core.compile_model(model=model, device_name=device)
.. parsed-literal::
IR model saved to model/ir_center_model.xml
Prepare image to use with attention-center model
------------------------------------------------
The attention-center model takes an RGB image with shape (480, 640) as
input.
.. code:: ipython3
class Image():
def __init__(self, model_input_image_shape, image_path=None, image=None):
self.model_input_image_shape = model_input_image_shape
self.image = None
self.real_input_image_shape = None
if image_path is not None:
self.image = cv2.imread(str(image_path))
self.real_input_image_shape = self.image.shape
elif image is not None:
self.image = image
self.real_input_image_shape = self.image.shape
else:
raise Exception("Sorry, image can't be found, please, specify image_path or image")
def prepare_image_tensor(self):
rgb_image = cv2.cvtColor(self.image, cv2.COLOR_BGR2RGB)
resized_image = cv2.resize(rgb_image, (self.model_input_image_shape[1], self.model_input_image_shape[0]))
image_tensor = tf.constant(np.expand_dims(resized_image, axis=0),
dtype=tf.float32)
return image_tensor
def scalt_center_to_real_image_shape(self, predicted_center):
new_center_y = round(predicted_center[0] * self.real_input_image_shape[1] / self.model_input_image_shape[1])
new_center_x = round(predicted_center[1] * self.real_input_image_shape[0] / self.model_input_image_shape[0])
return (int(new_center_y), int(new_center_x))
def draw_attention_center_point(self, predicted_center):
image_with_circle = cv2.circle(self.image,
predicted_center,
radius=10,
color=(3, 3, 255),
thickness=-1)
return image_with_circle
def print_image(self, predicted_center=None):
image_to_print = self.image
if predicted_center is not None:
image_to_print = self.draw_attention_center_point(predicted_center)
plt.imshow(cv2.cvtColor(image_to_print, cv2.COLOR_BGR2RGB))
Load input image
----------------
Upload input image using file loading button
.. code:: ipython3
import ipywidgets as widgets
load_file_widget = widgets.FileUpload(
accept="image/*", multiple=False, description="Image file",
)
load_file_widget
.. parsed-literal::
FileUpload(value=(), accept='image/*', description='Image file')
.. code:: ipython3
import io
import PIL
# read uploaded image
image = PIL.Image.open(io.BytesIO(load_file_widget.value[-1]['content'])) if load_file_widget.value else PIL.Image.open("../data/image/coco.jpg")
image.convert("RGB")
input_image = Image((480, 640), image=(np.ascontiguousarray(image)[:, :, ::-1]).astype(np.uint8))
image_tensor = input_image.prepare_image_tensor()
input_image.print_image()
.. parsed-literal::
2023-06-21 23:13:46.531940: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
.. image:: 216-attention-center-with-output_files/216-attention-center-with-output_11_1.png
Get result with OpenVINO IR model
---------------------------------
.. code:: ipython3
output_layer = compiled_model.output(0)
# make inference, get result in input image resolution
res = compiled_model([image_tensor])[output_layer]
# scale point to original image resulution
predicted_center = input_image.scalt_center_to_real_image_shape(res[0])
print(f'Prediction attention center point {predicted_center}')
input_image.print_image(predicted_center)
.. parsed-literal::
Prediction attention center point (292, 277)
.. image:: 216-attention-center-with-output_files/216-attention-center-with-output_13_1.png
docs/notebooks/106-auto-device-with-output_files/106-auto-device-with-output_14_0.png → docs/notebooks/216-attention-center-with-output_files/216-attention-center-with-output_11_1.png
View File
File renamed without changes.
Loaded 100 of 221 files, more files were not shown because too many files have changed in this diff. Show more