From 2bf8d910f641f912ed883ad2cee99f59316e9bc0 Mon Sep 17 00:00:00 2001 From: Anastasia Kuporosova Date: Wed, 13 Sep 2023 21:05:24 +0200 Subject: [PATCH] [Docs][PyOV] update python snippets (#19367) * [Docs][PyOV] update python snippets * first snippet * Fix samples debug * Fix linter * part1 * Fix speech sample * update model state snippet * add serialize * add temp dir * CPU snippets update (#134) * snippets CPU 1/6 * snippets CPU 2/6 * snippets CPU 3/6 * snippets CPU 4/6 * snippets CPU 5/6 * snippets CPU 6/6 * make module TODO: REMEMBER ABOUT EXPORTING PYTONPATH ON CIs ETC * Add static model creation in snippets for CPU * export_comp_model done * leftovers * apply comments * apply comments -- properties * small fixes * rempve debug info * return IENetwork instead of Function * apply comments * revert precision change in common snippets * update opset * [PyOV] Edit docs for the rest of plugins (#136) * modify main.py * GNA snippets * GPU snippets * AUTO snippets * MULTI snippets * HETERO snippets * Added properties * update gna * more samples * Update docs/OV_Runtime_UG/model_state_intro.md * Update docs/OV_Runtime_UG/model_state_intro.md * attempt1 fix ci * new approach to test * temporary remove some files from run * revert cmake changes * fix ci * fix snippet * fix py_exclusive snippet * fix preprocessing snippet * clean-up main * remove numpy installation in gha * check for GPU * add logger * iexclude main * main update * temp * Temp2 * Temp2 * temp * Revert temp * add property execution devices * hide output from samples --------- Co-authored-by: p-wysocki Co-authored-by: Jan Iwaszkiewicz Co-authored-by: Karol Blaszczak --- .github/workflows/linux.yml | 8 + .../shellcheck/shellcheck.cmake | 1 - .../migration_ov_2_0/graph_construction.md | 2 +- docs/OV_Runtime_UG/model_state_intro.md | 36 +++- .../109-latency-tricks-with-output.rst | 2 +- .../109-throughput-tricks-with-output.rst | 2 +- docs/notebooks/115-async-api-with-output.rst | 2 +- .../203-meter-reader-with-output.rst | 2 +- .../nncf/ptq/code/ptq_aa_openvino.py | 6 +- .../nncf/ptq/code/ptq_onnx.py | 2 +- .../nncf/ptq/code/ptq_openvino.py | 2 +- .../nncf/ptq/code/ptq_tensorflow.py | 2 +- .../nncf/ptq/code/ptq_torch.py | 2 +- docs/snippets/ShapeInference.py | 16 +- docs/snippets/__init__.py | 7 + docs/snippets/cpu/Bfloat16Inference.py | 20 +- docs/snippets/cpu/__init__.py | 2 + docs/snippets/cpu/compile_model.py | 30 +-- docs/snippets/cpu/dynamic_shape.py | 10 +- docs/snippets/cpu/multi_threading.py | 41 ++-- docs/snippets/cpu/ov_execution_mode.py | 16 +- .../cpu/ov_sparse_weights_decompression.py | 10 +- docs/snippets/export_compiled_model.py | 18 +- docs/snippets/gna/__init__.py | 2 + docs/snippets/gna/configure.py | 17 +- docs/snippets/gna/import_export.py | 15 +- docs/snippets/gna/set_batch.py | 15 +- docs/snippets/gpu/__init__.py | 2 + docs/snippets/gpu/compile_model.py | 71 ++++--- docs/snippets/gpu/custom_kernels_api.py | 17 +- docs/snippets/gpu/dynamic_batch.py | 43 +++-- .../gpu/preprocessing_nv12_two_planes.py | 39 ++-- docs/snippets/ie_common.py | 18 +- docs/snippets/main.py | 44 +++++ .../snippets/{ngraph.py => ngraph_snippet.py} | 0 docs/snippets/ov_auto.py | 181 ++++++++++++------ docs/snippets/ov_auto_batching.py | 84 ++++---- docs/snippets/ov_caching.py | 26 +-- docs/snippets/ov_common.py | 19 +- docs/snippets/ov_denormals.py | 7 +- docs/snippets/ov_dynamic_shapes.py | 44 +++-- docs/snippets/ov_extensions.py | 10 +- docs/snippets/ov_graph.py | 11 +- docs/snippets/ov_hetero.py | 75 ++++---- docs/snippets/ov_infer_request.py | 36 ++-- docs/snippets/ov_layout.py | 17 +- docs/snippets/ov_model_snippets.py | 45 +++-- docs/snippets/ov_model_with_state_infer.py | 84 ++++++++ docs/snippets/ov_multi.py | 111 ++++++----- docs/snippets/ov_network_state_intro.py | 172 ----------------- docs/snippets/ov_preprocessing.py | 99 +++++----- docs/snippets/ov_preprocessing_migration.py | 55 +++--- docs/snippets/ov_properties_api.py | 97 +++++----- docs/snippets/ov_properties_migration.py | 123 ++++++------ docs/snippets/ov_python_exclusives.py | 52 ++--- docs/snippets/ov_python_inference.py | 13 +- docs/snippets/src/main.py | 2 +- docs/snippets/utils.py | 86 +++++++++ .../bert_benchmark/bert_benchmark.py | 9 +- .../sync_benchmark/sync_benchmark.py | 5 +- .../throughput_benchmark.py | 7 +- .../classification_sample_async.py | 19 +- .../hello_classification.py | 17 +- .../hello_query_device/hello_query_device.py | 4 +- .../hello_reshape_ssd/hello_reshape_ssd.py | 17 +- .../model_creation_sample.py | 45 +++-- samples/python/speech_sample/speech_sample.py | 19 +- src/bindings/python/docs/test_examples.md | 2 +- 68 files changed, 1223 insertions(+), 892 deletions(-) create mode 100644 docs/snippets/__init__.py create mode 100644 docs/snippets/cpu/__init__.py create mode 100644 docs/snippets/gna/__init__.py create mode 100644 docs/snippets/gpu/__init__.py create mode 100644 docs/snippets/main.py rename docs/snippets/{ngraph.py => ngraph_snippet.py} (100%) create mode 100644 docs/snippets/ov_model_with_state_infer.py delete mode 100644 docs/snippets/ov_network_state_intro.py create mode 100644 docs/snippets/utils.py diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index ff109f5d5bc..22d90379886 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -599,6 +599,14 @@ jobs: --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_utils/test_utils.py \ --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_onnx/test_zoo_models.py \ --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_onnx/test_backend.py + + - name: Python API snippets + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}:${{ github.workspace }}/openvino/docs/:$PYTHONPATH + export LD_LIBRARY_PATH=${{ env.INSTALL_TEST_DIR }}:$LD_LIBRARY_PATH + + python3 ${{ github.workspace }}/openvino/docs/snippets/main.py - name: Model Optimizer UT run: | diff --git a/cmake/developer_package/shellcheck/shellcheck.cmake b/cmake/developer_package/shellcheck/shellcheck.cmake index b792a8e6c1a..3b18ff02abf 100644 --- a/cmake/developer_package/shellcheck/shellcheck.cmake +++ b/cmake/developer_package/shellcheck/shellcheck.cmake @@ -39,7 +39,6 @@ function(ie_shellcheck_process) continue() endif() - get_filename_component(dir_name "${script}" DIRECTORY) string(REPLACE "${IE_SHELLCHECK_DIRECTORY}" "${CMAKE_BINARY_DIR}/shellcheck" output_file ${script}) set(output_file "${output_file}.txt") get_filename_component(script_name "${script}" NAME) diff --git a/docs/OV_Runtime_UG/migration_ov_2_0/graph_construction.md b/docs/OV_Runtime_UG/migration_ov_2_0/graph_construction.md index 17ceb9a55ec..9dc0a937b07 100644 --- a/docs/OV_Runtime_UG/migration_ov_2_0/graph_construction.md +++ b/docs/OV_Runtime_UG/migration_ov_2_0/graph_construction.md @@ -20,7 +20,7 @@ nGraph API .. tab-item:: Python :sync: py - .. doxygensnippet:: docs/snippets/ngraph.py + .. doxygensnippet:: docs/snippets/ngraph_snippet.py :language: Python :fragment: ngraph:graph diff --git a/docs/OV_Runtime_UG/model_state_intro.md b/docs/OV_Runtime_UG/model_state_intro.md index eca22cc6eb9..60acfaa79e7 100644 --- a/docs/OV_Runtime_UG/model_state_intro.md +++ b/docs/OV_Runtime_UG/model_state_intro.md @@ -163,9 +163,21 @@ Example of Creating Model OpenVINO API In the following example, the ``SinkVector`` is used to create the ``ov::Model``. For a model with states, except inputs and outputs, the ``Assign`` nodes should also point to the ``Model`` to avoid deleting it during graph transformations. Use the constructor to do it, as shown in the example, or with the special ``add_sinks(const SinkVector& sinks)`` method. After deleting the node from the graph with the ``delete_sink()`` method, a sink can be deleted from ``ov::Model``. -.. doxygensnippet:: docs/snippets/ov_model_with_state_infer.cpp - :language: cpp - :fragment: [model_create] +.. tab-set:: + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/snippets/ov_model_with_state_infer.cpp + :language: cpp + :fragment: [model_create] + + .. tab-item:: Python + :sync: py + + .. doxygensnippet:: docs/snippets/ov_model_with_state_infer.py + :language: python + :fragment: ov:model_create .. _openvino-state-api: @@ -189,10 +201,22 @@ Based on the IR from the previous section, the example below demonstrates infere One infer request and one thread will be used in this example. Using several threads is possible if there are several independent sequences. Then, each sequence can be processed in its own infer request. Inference of one sequence in several infer requests is not recommended. In one infer request, a state will be saved automatically between inferences, but if the first step is done in one infer request and the second in another, a state should be set in a new infer request manually (using the ``ov::IVariableState::set_state`` method). +.. tab-set:: + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/snippets/ov_model_with_state_infer.cpp + :language: cpp + :fragment: [part1] + + .. tab-item:: Python + :sync: py + + .. doxygensnippet:: docs/snippets/ov_model_with_state_infer.py + :language: python + :fragment: ov:part1 -.. doxygensnippet:: docs/snippets/ov_model_with_state_infer.cpp - :language: cpp - :fragment: [part1] For more elaborate examples demonstrating how to work with models with states, diff --git a/docs/notebooks/109-latency-tricks-with-output.rst b/docs/notebooks/109-latency-tricks-with-output.rst index 4ce64f56c5a..af706167522 100644 --- a/docs/notebooks/109-latency-tricks-with-output.rst +++ b/docs/notebooks/109-latency-tricks-with-output.rst @@ -206,7 +206,7 @@ benchmarking process. .. code:: ipython3 - import openvino.runtime as ov + import openvino as ov # initialize OpenVINO core = ov.Core() diff --git a/docs/notebooks/109-throughput-tricks-with-output.rst b/docs/notebooks/109-throughput-tricks-with-output.rst index c9259e4b395..523bb307ed2 100644 --- a/docs/notebooks/109-throughput-tricks-with-output.rst +++ b/docs/notebooks/109-throughput-tricks-with-output.rst @@ -193,7 +193,7 @@ benchmarking process. .. code:: ipython3 - import openvino.runtime as ov + import openvino as ov # initialize OpenVINO core = ov.Core() diff --git a/docs/notebooks/115-async-api-with-output.rst b/docs/notebooks/115-async-api-with-output.rst index 06a43e3aeb2..f8b9eb906a5 100644 --- a/docs/notebooks/115-async-api-with-output.rst +++ b/docs/notebooks/115-async-api-with-output.rst @@ -54,7 +54,7 @@ Imports `⇑ <#top>`__ import time import numpy as np from openvino.runtime import Core, AsyncInferQueue - import openvino.runtime as ov + import openvino as ov from IPython import display import matplotlib.pyplot as plt diff --git a/docs/notebooks/203-meter-reader-with-output.rst b/docs/notebooks/203-meter-reader-with-output.rst index e426fec36bd..0efdff356ed 100644 --- a/docs/notebooks/203-meter-reader-with-output.rst +++ b/docs/notebooks/203-meter-reader-with-output.rst @@ -54,7 +54,7 @@ Import `⇑ <#top>`__ import cv2 import tarfile import matplotlib.pyplot as plt - import openvino.runtime as ov + import openvino as ov sys.path.append("../utils") from notebook_utils import download_file, segmentation_map_to_image diff --git a/docs/optimization_guide/nncf/ptq/code/ptq_aa_openvino.py b/docs/optimization_guide/nncf/ptq/code/ptq_aa_openvino.py index d759695ea8a..7800d47b89a 100644 --- a/docs/optimization_guide/nncf/ptq/code/ptq_aa_openvino.py +++ b/docs/optimization_guide/nncf/ptq/code/ptq_aa_openvino.py @@ -21,7 +21,7 @@ import torch import openvino from sklearn.metrics import accuracy_score -def validate(model: openvino.runtime.CompiledModel, +def validate(model: openvino.CompiledModel, validation_loader: torch.utils.data.DataLoader) -> float: predictions = [] references = [] @@ -39,7 +39,7 @@ def validate(model: openvino.runtime.CompiledModel, #! [validation] #! [quantization] -model = ... # openvino.runtime.Model object +model = ... # openvino.Model object quantized_model = nncf.quantize_with_accuracy_control(model, calibration_dataset=calibration_dataset, @@ -50,7 +50,7 @@ quantized_model = nncf.quantize_with_accuracy_control(model, #! [quantization] #! [inference] -import openvino.runtime as ov +import openvino as ov # compile the model to transform quantized operations to int8 model_int8 = ov.compile_model(quantized_model) diff --git a/docs/optimization_guide/nncf/ptq/code/ptq_onnx.py b/docs/optimization_guide/nncf/ptq/code/ptq_onnx.py index b8104d449c5..9588886bd09 100644 --- a/docs/optimization_guide/nncf/ptq/code/ptq_onnx.py +++ b/docs/optimization_guide/nncf/ptq/code/ptq_onnx.py @@ -23,7 +23,7 @@ quantized_model = nncf.quantize(model, calibration_dataset) #! [quantization] #! [inference] -import openvino.runtime as ov +import openvino as ov from openvino.tools.mo import convert_model # convert ONNX model to OpenVINO model diff --git a/docs/optimization_guide/nncf/ptq/code/ptq_openvino.py b/docs/optimization_guide/nncf/ptq/code/ptq_openvino.py index 81e19fd99f5..1a3b10df219 100644 --- a/docs/optimization_guide/nncf/ptq/code/ptq_openvino.py +++ b/docs/optimization_guide/nncf/ptq/code/ptq_openvino.py @@ -15,7 +15,7 @@ calibration_dataset = nncf.Dataset(calibration_loader, transform_fn) #! [dataset] #! [quantization] -import openvino.runtime as ov +import openvino as ov model = ov.Core().read_model("model_path") quantized_model = nncf.quantize(model, calibration_dataset) diff --git a/docs/optimization_guide/nncf/ptq/code/ptq_tensorflow.py b/docs/optimization_guide/nncf/ptq/code/ptq_tensorflow.py index e4fc817ddb3..f605a5d8737 100644 --- a/docs/optimization_guide/nncf/ptq/code/ptq_tensorflow.py +++ b/docs/optimization_guide/nncf/ptq/code/ptq_tensorflow.py @@ -22,7 +22,7 @@ quantized_model = nncf.quantize(model, calibration_dataset) #! [quantization] #! [inference] -import openvino.runtime as ov +import openvino as ov from openvino.tools.mo import convert_model # convert TensorFlow model to OpenVINO model diff --git a/docs/optimization_guide/nncf/ptq/code/ptq_torch.py b/docs/optimization_guide/nncf/ptq/code/ptq_torch.py index fbdf103b4f2..b0147631c66 100644 --- a/docs/optimization_guide/nncf/ptq/code/ptq_torch.py +++ b/docs/optimization_guide/nncf/ptq/code/ptq_torch.py @@ -22,7 +22,7 @@ quantized_model = nncf.quantize(model, calibration_dataset) #! [quantization] #! [inference] -import openvino.runtime as ov +import openvino as ov from openvino.tools.mo import convert_model input_fp32 = ... # FP32 model input diff --git a/docs/snippets/ShapeInference.py b/docs/snippets/ShapeInference.py index f680c8053b2..e74565696db 100644 --- a/docs/snippets/ShapeInference.py +++ b/docs/snippets/ShapeInference.py @@ -1,23 +1,23 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -from openvino.runtime import Core, Layout, set_batch -ov = Core() -model = ov.read_model("path/to/model") +import openvino as ov +from utils import get_model, get_image + +model = get_model() #! [picture_snippet] model.reshape([8, 3, 448, 448]) #! [picture_snippet] #! [set_batch] -model.get_parameters()[0].set_layout(Layout("N...")) -set_batch(model, 5) +model.get_parameters()[0].set_layout(ov.Layout("N...")) +ov.set_batch(model, 5) #! [set_batch] #! [simple_spatials_change] -from cv2 import imread -image = imread("path/to/image") -model.reshape({1, 3, image.shape[0], image.shape[1]}) +image = get_image() +model.reshape([1, 3, image.shape[0], image.shape[1]]) #! [simple_spatials_change] #! [obj_to_shape] diff --git a/docs/snippets/__init__.py b/docs/snippets/__init__.py new file mode 100644 index 00000000000..b1b02362f64 --- /dev/null +++ b/docs/snippets/__init__.py @@ -0,0 +1,7 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from .utils import get_image +from .utils import get_model +from .utils import get_ngraph_model +from .utils import get_path_to_model diff --git a/docs/snippets/cpu/Bfloat16Inference.py b/docs/snippets/cpu/Bfloat16Inference.py index c1d38c284f5..2d3fabfe6c4 100644 --- a/docs/snippets/cpu/Bfloat16Inference.py +++ b/docs/snippets/cpu/Bfloat16Inference.py @@ -2,22 +2,24 @@ # SPDX-License-Identifier: Apache-2.0 -from openvino.runtime import Core +import openvino as ov + +from snippets import get_model + +model = get_model() #! [part0] -core = Core() -cpu_optimization_capabilities = core.get_property("CPU", "OPTIMIZATION_CAPABILITIES") +core = ov.Core() +cpu_optimization_capabilities = core.get_property("CPU", ov.properties.device.capabilities()) #! [part0] -# TODO: enable part1 when property api will be supported in python #! [part1] -core = Core() -model = core.read_model("model.xml") +core = ov.Core() compiled_model = core.compile_model(model, "CPU") -inference_precision = core.get_property("CPU", "INFERENCE_PRECISION_HINT") +inference_precision = core.get_property("CPU", ov.properties.hint.inference_precision()) #! [part1] #! [part2] -core = Core() -core.set_property("CPU", {"INFERENCE_PRECISION_HINT": "f32"}) +core = ov.Core() +core.set_property("CPU", {ov.properties.hint.inference_precision(): ov.Type.f32}) #! [part2] diff --git a/docs/snippets/cpu/__init__.py b/docs/snippets/cpu/__init__.py new file mode 100644 index 00000000000..cddd115d397 --- /dev/null +++ b/docs/snippets/cpu/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/docs/snippets/cpu/compile_model.py b/docs/snippets/cpu/compile_model.py index 5cb06ae694d..b0dd9d7cefd 100644 --- a/docs/snippets/cpu/compile_model.py +++ b/docs/snippets/cpu/compile_model.py @@ -1,17 +1,23 @@ -# Copyright (C) 2022 Intel Corporation +# Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +from snippets import get_model -#! [compile_model_default] -from openvino.runtime import Core -core = Core() -model = core.read_model("model.xml") -compiled_model = core.compile_model(model, "CPU") -#! [compile_model_default] +def main(): + model = get_model() -#! [compile_model_multi] -core = Core() -model = core.read_model("model.xml") -compiled_model = core.compile_model(model, "MULTI:CPU,GPU.0") -#! [compile_model_multi] + #! [compile_model_default] + import openvino as ov + + core = ov.Core() + compiled_model = core.compile_model(model, "CPU") + #! [compile_model_default] + + if "GPU" not in core.available_devices: + return 0 + + #! [compile_model_multi] + core = ov.Core() + compiled_model = core.compile_model(model, "MULTI:CPU,GPU.0") + #! [compile_model_multi] diff --git a/docs/snippets/cpu/dynamic_shape.py b/docs/snippets/cpu/dynamic_shape.py index 8bfbf5d99ee..1b531d0a8ee 100644 --- a/docs/snippets/cpu/dynamic_shape.py +++ b/docs/snippets/cpu/dynamic_shape.py @@ -1,11 +1,13 @@ -# Copyright (C) 2022 Intel Corporation +# Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +from snippets import get_model -from openvino.runtime import Core +model = get_model() #! [static_shape] -core = Core() -model = core.read_model("model.xml") +import openvino as ov + +core = ov.Core() model.reshape([10, 20, 30, 40]) #! [static_shape] diff --git a/docs/snippets/cpu/multi_threading.py b/docs/snippets/cpu/multi_threading.py index a8113390e9a..9a5baa1e757 100644 --- a/docs/snippets/cpu/multi_threading.py +++ b/docs/snippets/cpu/multi_threading.py @@ -2,28 +2,47 @@ # SPDX-License-Identifier: Apache-2.0 # -import openvino.runtime as ov -from openvino.runtime import Core, Type, OVAny, properties +from openvino import Core, properties +from snippets import get_model + +model = get_model() + +device_name = "CPU" +core = Core() +core.set_property("CPU", properties.intel_cpu.sparse_weights_decompression_rate(0.8)) -device_name = 'CPU' -xml_path = 'model.xml' -core = ov.Core() -core.set_property("CPU", ov.properties.intel_cpu.sparse_weights_decompression_rate(0.8)) -model = core.read_model(model=xml_path) # ! [ov:intel_cpu:multi_threading:part0] # Use one logical processor for inference -compiled_model_1 = core.compile_model(model=model, device_name=device_name, config={properties.inference_num_threads(1)}) +compiled_model_1 = core.compile_model( + model=model, + device_name=device_name, + config={properties.inference_num_threads(): 1}, +) # Use logical processors of Efficient-cores for inference on hybrid platform -compiled_model_2 = core.compile_model(model=model, device_name=device_name, config={properties.hint.scheduling_core_type(properties.hint.SchedulingCoreType.ECORE_ONLY)}) +compiled_model_2 = core.compile_model( + model=model, + device_name=device_name, + config={ + properties.hint.scheduling_core_type(): properties.hint.SchedulingCoreType.ECORE_ONLY, + }, +) # Use one logical processor per CPU core for inference when hyper threading is on -compiled_model_3 = core.compile_model(model=model, device_name=device_name, config={properties.hint.enable_hyper_threading(False)}) +compiled_model_3 = core.compile_model( + model=model, + device_name=device_name, + config={properties.hint.enable_hyper_threading(): False}, +) # ! [ov:intel_cpu:multi_threading:part0] # ! [ov:intel_cpu:multi_threading:part1] # Disable CPU threads pinning for inference when system supoprt it -compiled_model_4 = core.compile_model(model=model, device_name=device_name, config={properties.hint.enable_cpu_pinning(False)}) +compiled_model_4 = core.compile_model( + model=model, + device_name=device_name, + config={properties.hint.enable_cpu_pinning(): False}, +) # ! [ov:intel_cpu:multi_threading:part1] assert compiled_model_1 assert compiled_model_2 diff --git a/docs/snippets/cpu/ov_execution_mode.py b/docs/snippets/cpu/ov_execution_mode.py index 4abd0ccf375..2feff2777f7 100644 --- a/docs/snippets/cpu/ov_execution_mode.py +++ b/docs/snippets/cpu/ov_execution_mode.py @@ -1,12 +1,18 @@ # Copyright (C) 2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -from openvino.runtime import Core +#! [ov:execution_mode:part0] +import openvino as ov -#! [ov:execution_mode:part0] -core = Core() +core = ov.Core() # in case of Accuracy -core.set_property("CPU", {"EXECUTION_MODE_HINT": "ACCURACY"}) +core.set_property( + "CPU", + {ov.properties.hint.execution_mode(): ov.properties.hint.ExecutionMode.ACCURACY}, +) # in case of Performance -core.set_property("CPU", {"EXECUTION_MODE_HINT": "PERFORMANCE"}) +core.set_property( + "CPU", + {ov.properties.hint.execution_mode(): ov.properties.hint.ExecutionMode.PERFORMANCE}, +) #! [ov:execution_mode:part0] diff --git a/docs/snippets/cpu/ov_sparse_weights_decompression.py b/docs/snippets/cpu/ov_sparse_weights_decompression.py index f7ea0772bf7..76de90311d2 100644 --- a/docs/snippets/cpu/ov_sparse_weights_decompression.py +++ b/docs/snippets/cpu/ov_sparse_weights_decompression.py @@ -2,14 +2,16 @@ # SPDX-License-Identifier: Apache-2.0 # -import openvino.runtime as ov +import openvino as ov +from snippets import get_model -device_name = 'CPU' -xml_path = 'model.xml' +model = get_model() + +device_name = "CPU" +xml_path = "model.xml" # ! [ov:intel_cpu:sparse_weights_decompression:part0] core = ov.Core() core.set_property("CPU", ov.properties.intel_cpu.sparse_weights_decompression_rate(0.8)) -model = core.read_model(model=xml_path) compiled_model = core.compile_model(model=model, device_name=device_name) # ! [ov:intel_cpu:sparse_weights_decompression:part0] assert compiled_model diff --git a/docs/snippets/export_compiled_model.py b/docs/snippets/export_compiled_model.py index 39bebffe1b8..b89b9a1479b 100644 --- a/docs/snippets/export_compiled_model.py +++ b/docs/snippets/export_compiled_model.py @@ -1,7 +1,19 @@ -#! [export_compiled_model] +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# -from openvino.runtime import Core +from utils import get_path_to_model -ov.Core().compile_model(device, modelPath, properties).export_model(compiled_blob) +device = "CPU" +model_path = get_path_to_model() +properties = {} #! [export_compiled_model] + +import openvino as ov + +core = ov.Core() + +compiled_model = core.compile_model(model_path, device, properties) +output_stream = compiled_model.export_model() +#! [export_compiled_model] diff --git a/docs/snippets/gna/__init__.py b/docs/snippets/gna/__init__.py new file mode 100644 index 00000000000..6a16273c024 --- /dev/null +++ b/docs/snippets/gna/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/docs/snippets/gna/configure.py b/docs/snippets/gna/configure.py index 6f0ba675665..ec94a54ae41 100644 --- a/docs/snippets/gna/configure.py +++ b/docs/snippets/gna/configure.py @@ -2,14 +2,17 @@ # SPDX-License-Identifier: Apache-2.0 #! [import] -from openvino.runtime import Core +import openvino as ov #! [import] -model_path = "model.xml" +from snippets import get_model +model = get_model() + +# TODO: no GNA properties to replace strings +#! [ov_gna_exec_mode_hw_with_sw_fback] +core = ov.Core() +compiled_model = core.compile_model( + model, device_name="GNA", config={"GNA_DEVICE_MODE": "GNA_HW_WITH_SW_FBACK"} +) #! [ov_gna_exec_mode_hw_with_sw_fback] -core = Core() -model = core.read_model(model=model_path) -compiled_model = core.compile_model(model, device_name="GNA", - config={ 'GNA_DEVICE_MODE' : 'GNA_HW_WITH_SW_FBACK'}) -#! [ov_gna_exec_mode_hw_with_sw_fback] \ No newline at end of file diff --git a/docs/snippets/gna/import_export.py b/docs/snippets/gna/import_export.py index 8ecead1520d..8aa6cd11267 100644 --- a/docs/snippets/gna/import_export.py +++ b/docs/snippets/gna/import_export.py @@ -2,25 +2,26 @@ # SPDX-License-Identifier: Apache-2.0 #! [import] -from openvino.runtime import Core +import openvino as ov from io import BytesIO #! [import] -model_path = "model.xml" +from snippets import get_model + +model = get_model() blob_path = "compiled_model.blob" -core = Core() -model = core.read_model(model=model_path) +core = ov.Core() compiled_model = core.compile_model(model, device_name="GNA") #! [ov_gna_export] user_stream = compiled_model.export_model() -with open(blob_path, 'wb') as f: +with open(blob_path, "wb") as f: f.write(user_stream) #! [ov_gna_export] # [ov_gna_import] -with open(blob_path, 'rb') as f: +with open(blob_path, "rb") as f: buf = BytesIO(f.read()) compiled_model = core.import_model(buf, device_name="GNA") -# [ov_gna_import] \ No newline at end of file +# [ov_gna_import] diff --git a/docs/snippets/gna/set_batch.py b/docs/snippets/gna/set_batch.py index 6ae1cd2ab15..fac783923ca 100644 --- a/docs/snippets/gna/set_batch.py +++ b/docs/snippets/gna/set_batch.py @@ -2,20 +2,21 @@ # SPDX-License-Identifier: Apache-2.0 #! [import] -from openvino.runtime import Core, set_batch -from openvino.preprocess import PrePostProcessor +import openvino as ov #! [import] -model_path = "model.xml" +from snippets import get_path_to_model + batch_size = 8 +model_path = get_path_to_model([1, 32]) #! [ov_gna_read_model] -core = Core() +core = ov.Core() model = core.read_model(model=model_path) #! [ov_gna_read_model] #! [ov_gna_set_nc_layout] -ppp = PrePostProcessor(model) +ppp = ov.preprocess.PrePostProcessor(model) for i in range(len(model.inputs)): input_name = model.input(i).get_any_name() ppp.input(i).model().set_layout("N?") @@ -23,5 +24,5 @@ model = ppp.build() #! [ov_gna_set_nc_layout] #! [ov_gna_set_batch_size] -set_batch(model, batch_size) -#! [ov_gna_set_batch_size] \ No newline at end of file +ov.set_batch(model, batch_size) +#! [ov_gna_set_batch_size] diff --git a/docs/snippets/gpu/__init__.py b/docs/snippets/gpu/__init__.py new file mode 100644 index 00000000000..6a16273c024 --- /dev/null +++ b/docs/snippets/gpu/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/docs/snippets/gpu/compile_model.py b/docs/snippets/gpu/compile_model.py index f1cede42ce8..15e67d41ab0 100644 --- a/docs/snippets/gpu/compile_model.py +++ b/docs/snippets/gpu/compile_model.py @@ -2,40 +2,49 @@ # SPDX-License-Identifier: Apache-2.0 -from openvino.runtime import Core +import openvino as ov +from snippets import get_model -#! [compile_model_default_gpu] -core = Core() -model = core.read_model("model.xml") -compiled_model = core.compile_model(model, "GPU") -#! [compile_model_default_gpu] -#! [compile_model_gpu_with_id] -core = Core() -model = core.read_model("model.xml") -compiled_model = core.compile_model(model, "GPU.1") -#! [compile_model_gpu_with_id] +def main(): + model = get_model() -#! [compile_model_gpu_with_id_and_tile] -core = Core() -model = core.read_model("model.xml") -compiled_model = core.compile_model(model, "GPU.1.0") -#! [compile_model_gpu_with_id_and_tile] + core = ov.Core() + if "GPU" not in core.available_devices: + return 0 -#! [compile_model_multi] -core = Core() -model = core.read_model("model.xml") -compiled_model = core.compile_model(model, "MULTI:GPU.1,GPU.0") -#! [compile_model_multi] + #! [compile_model_default_gpu] + core = ov.Core() + compiled_model = core.compile_model(model, "GPU") + #! [compile_model_default_gpu] -#! [compile_model_batch_plugin] -core = Core() -model = core.read_model("model.xml") -compiled_model = core.compile_model(model, "BATCH:GPU") -#! [compile_model_batch_plugin] + #! [compile_model_gpu_with_id] + core = ov.Core() + compiled_model = core.compile_model(model, "GPU.1") + #! [compile_model_gpu_with_id] -#! [compile_model_auto_batch] -core = Core() -model = core.read_model("model.xml") -compiled_model = core.compile_model(model, "GPU", {"PERFORMANCE_HINT": "THROUGHPUT"}) -#! [compile_model_auto_batch] + #! [compile_model_gpu_with_id_and_tile] + core = ov.Core() + compiled_model = core.compile_model(model, "GPU.1.0") + #! [compile_model_gpu_with_id_and_tile] + + #! [compile_model_multi] + core = ov.Core() + compiled_model = core.compile_model(model, "MULTI:GPU.1,GPU.0") + #! [compile_model_multi] + + #! [compile_model_batch_plugin] + core = ov.Core() + compiled_model = core.compile_model(model, "BATCH:GPU") + #! [compile_model_batch_plugin] + + #! [compile_model_auto_batch] + core = ov.Core() + compiled_model = core.compile_model( + model, + "GPU", + { + ov.properties.hint.performance_mode(): ov.properties.hint.PerformanceMode.THROUGHPUT, + }, + ) + #! [compile_model_auto_batch] diff --git a/docs/snippets/gpu/custom_kernels_api.py b/docs/snippets/gpu/custom_kernels_api.py index f5564f7ba3c..2eaeb610b3f 100644 --- a/docs/snippets/gpu/custom_kernels_api.py +++ b/docs/snippets/gpu/custom_kernels_api.py @@ -1,10 +1,17 @@ # Copyright (C) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +import openvino as ov -from openvino.runtime import Core -#! [part0] -core = Core() -core.set_property("GPU", {"CONFIG_FILE": ""}) -#! [part0] +def main(): + core = ov.Core() + + if "GPU" not in core.available_devices: + return 0 + + # TODO: missing property to replace string + #! [part0] + core = ov.Core() + core.set_property("GPU", {"CONFIG_FILE": ""}) + #! [part0] diff --git a/docs/snippets/gpu/dynamic_batch.py b/docs/snippets/gpu/dynamic_batch.py index 119244efa12..be4096febf5 100644 --- a/docs/snippets/gpu/dynamic_batch.py +++ b/docs/snippets/gpu/dynamic_batch.py @@ -2,27 +2,38 @@ # SPDX-License-Identifier: Apache-2.0 -import openvino.runtime as ov +import openvino as ov +from snippets import get_model -#! [dynamic_batch] -core = ov.Core() -C = 3 -H = 224 -W = 224 +def main(): + model = get_model() + + core = ov.Core() + if "GPU" not in core.available_devices: + return 0 -model = core.read_model("model.xml") -model.reshape([(1, 10), C, H, W]) + #! [dynamic_batch] + core = ov.Core() -# compile model and create infer request -compiled_model = core.compile_model(model, "GPU") -infer_request = compiled_model.create_infer_request() + C = 3 + H = 224 + W = 224 -# create input tensor with specific batch size -input_tensor = ov.Tensor(model.input().element_type, [2, C, H, W]) + model.reshape([(1, 10), C, H, W]) -# ... + # compile model and create infer request + compiled_model = core.compile_model(model, "GPU") + infer_request = compiled_model.create_infer_request() -infer_request.infer([input_tensor]) + # create input tensor with specific batch size + input_tensor = ov.Tensor(model.input().element_type, [2, C, H, W]) -#! [dynamic_batch] + # ... + + results = infer_request.infer([input_tensor]) + + #! [dynamic_batch] + + assert list(results.keys())[0].partial_shape == ov.PartialShape([(1, 10), 3, 224, 224]) + assert list(results.values())[0].shape == tuple(ov.Shape([2, 3, 224, 224])) diff --git a/docs/snippets/gpu/preprocessing_nv12_two_planes.py b/docs/snippets/gpu/preprocessing_nv12_two_planes.py index 7afa5b8eeca..fc4a03f964c 100644 --- a/docs/snippets/gpu/preprocessing_nv12_two_planes.py +++ b/docs/snippets/gpu/preprocessing_nv12_two_planes.py @@ -1,18 +1,31 @@ # Copyright (C) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -#! [init_preproc] -from openvino.runtime import Core, Type, Layout -from openvino.preprocess import PrePostProcessor, ColorFormat +from snippets import get_model +import openvino as ov -core = Core() -model = core.read_model("model.xml") -p = PrePostProcessor(model) -p.input().tensor().set_element_type(Type.u8) \ - .set_color_format(ColorFormat.NV12_TWO_PLANES, ["y", "uv"]) \ - .set_memory_type("GPU_SURFACE") -p.input().preprocess().convert_color(ColorFormat.BGR) -p.input().model().set_layout(Layout("NCHW")) -model_with_preproc = p.build() -#! [init_preproc] +def init_preproc(): + model = get_model() + #! [init_preproc] + import openvino as ov + from openvino.preprocess import PrePostProcessor, ColorFormat + + core = ov.Core() + + p = PrePostProcessor(model) + p.input().tensor().set_element_type(ov.Type.u8).set_color_format( + ColorFormat.NV12_TWO_PLANES, ["y", "uv"] + ).set_memory_type("GPU_SURFACE") + p.input().preprocess().convert_color(ColorFormat.BGR) + p.input().model().set_layout(ov.Layout("NCHW")) + model_with_preproc = p.build() + #! [init_preproc] + + +def main(): + core = ov.Core() + if "GPU" not in core.available_devices: + return 0 + init_preproc() + diff --git a/docs/snippets/ie_common.py b/docs/snippets/ie_common.py index 74c3831ffd9..ff9f97df53f 100644 --- a/docs/snippets/ie_common.py +++ b/docs/snippets/ie_common.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # +from utils import get_path_to_model, get_image, get_path_to_extension_library #! [ie:create_core] import numpy as np @@ -9,8 +10,10 @@ import openvino.inference_engine as ie core = ie.IECore() #! [ie:create_core] +model_path = get_path_to_model(True) + #! [ie:read_model] -network = core.read_network("model.xml") +network = core.read_network(model_path) #! [ie:read_model] #! [ie:compile_model] @@ -26,7 +29,7 @@ exec_network = core.load_network(network, "CPU", num_requests=4) infer_request = exec_network.requests[0] # Get input blobs mapped to input layers names input_blobs = infer_request.input_blobs -data = input_blobs["data1"].buffer +data = input_blobs["data"].buffer # Original I64 precision was converted to I32 assert data.dtype == np.int32 # Fill the first blob ... @@ -36,7 +39,7 @@ assert data.dtype == np.int32 results = infer_request.infer() #! [ie:inference] -input_data = iter(list()) +input_data = get_image() def process_results(results, frame_id): pass @@ -67,11 +70,11 @@ for infer_request in exec_network.requests: total_frames = 100 for _ in range(total_frames): # Wait for at least one free request - exec_network.wait(num_request=1) + exec_network.wait(num_requests=1) # Get idle id idle_id = exec_network.get_idle_request_id() # Start asynchronous inference on idle request - exec_network.start_async(request_id=idle_id, inputs=next(input_data)) + exec_network.start_async(request_id=idle_id, inputs={"data": input_data}) # Wait for all requests to complete exec_network.wait() #! [ie:start_async_and_wait] @@ -79,12 +82,13 @@ exec_network.wait() #! [ie:get_output_tensor] # Get output blobs mapped to output layers names output_blobs = infer_request.output_blobs -data = output_blobs["out1"].buffer +data = output_blobs["relu"].buffer # Original I64 precision was converted to I32 assert data.dtype == np.int32 # Process output data #! [ie:get_output_tensor] +path_to_extension_library = get_path_to_extension_library() #! [ie:load_old_extension] -core.add_extension("path_to_extension_library.so", "CPU") +core.add_extension(path_to_extension_library, "CPU") #! [ie:load_old_extension] diff --git a/docs/snippets/main.py b/docs/snippets/main.py new file mode 100644 index 00000000000..1c9278548e3 --- /dev/null +++ b/docs/snippets/main.py @@ -0,0 +1,44 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import sys +import os +import io +import importlib +from contextlib import redirect_stdout, redirect_stderr + + +skip_snippets = ["main.py", "__init__.py", "utils.py", "ie_common.py", "ov_common.py"] + +def import_python_modules(directory, subdirectory=""): + for item in os.listdir(directory): + if item.endswith('.py') and item not in skip_snippets: + imported_item = item[:-3] + if subdirectory != "": + imported_item=subdirectory + "." + imported_item + print(f"Snippet {item} is executing...") + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + mod = importlib.import_module(imported_item) + + try: + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + mod.main() + except AttributeError as e: + pass + + print(f"Snippet {item} succesfully executed.") + + + if os.path.isdir(os.path.join(directory, item)): + dir_path = os.path.join(directory, item) + import_python_modules(dir_path, item) + + +def main(): + dir_path = os.path.dirname(os.path.realpath(__file__)) + import_python_modules(dir_path) + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/docs/snippets/ngraph.py b/docs/snippets/ngraph_snippet.py similarity index 100% rename from docs/snippets/ngraph.py rename to docs/snippets/ngraph_snippet.py diff --git a/docs/snippets/ov_auto.py b/docs/snippets/ov_auto.py index 9ad3163c4eb..e0dc58587a4 100644 --- a/docs/snippets/ov_auto.py +++ b/docs/snippets/ov_auto.py @@ -1,15 +1,15 @@ import sys -from openvino.runtime import Core +import openvino as ov from openvino.inference_engine import IECore -model_path = "/openvino_CI_CD/result/install_pkg/tests/test_model_zoo/core/models/ir/add_abc.xml" -path_to_model = "/openvino_CI_CD/result/install_pkg/tests/test_model_zoo/core/models/ir/add_abc.xml" +from utils import get_model, get_ngraph_model + +model = get_model() +net = get_ngraph_model() + def part0(): -#! [part0] - core = Core() - - # Read a network in IR, PaddlePaddle, or ONNX format: - model = core.read_model(model_path) + #! [part0] + core = ov.Core() # compile a model on AUTO using the default list of device candidates. # The following lines are equivalent: @@ -20,21 +20,25 @@ def part0(): # You can also specify the devices to be used by AUTO. # The following lines are equivalent: compiled_model = core.compile_model(model=model, device_name="AUTO:GPU,CPU") - compiled_model = core.compile_model(model=model, device_name="AUTO", config={"MULTI_DEVICE_PRIORITIES": "GPU,CPU"}) + compiled_model = core.compile_model( + model=model, + device_name="AUTO", + config={ov.properties.device.priorities(): "GPU,CPU"}, + ) # Optional # the AUTO plugin is pre-configured (globally) with the explicit option: - core.set_property(device_name="AUTO", properties={"MULTI_DEVICE_PRIORITIES":"GPU,CPU"}) -#! [part0] + core.set_property( + device_name="AUTO", properties={ov.properties.device.priorities(): "GPU,CPU"} + ) + #! [part0] + def part1(): -#! [part1] + #! [part1] ### IE API ### ie = IECore() - # Read a network in IR, PaddlePaddle, or ONNX format: - net = ie.read_network(model=path_to_model) - # Load a network to AUTO using the default list of device candidates. # The following lines are equivalent: exec_net = ie.load_network(network=net) @@ -45,87 +49,148 @@ def part1(): # You can also specify the devices to be used by AUTO in its selection process. # The following lines are equivalent: exec_net = ie.load_network(network=net, device_name="AUTO:GPU,CPU") - exec_net = ie.load_network(network=net, device_name="AUTO", config={"MULTI_DEVICE_PRIORITIES": "GPU,CPU"}) + exec_net = ie.load_network( + network=net, + device_name="AUTO", + config={ov.properties.device.priorities(): "GPU,CPU"}, + ) # Optional # the AUTO plugin is pre-configured (globally) with the explicit option: - ie.set_config(config={"MULTI_DEVICE_PRIORITIES":"GPU,CPU"}, device_name="AUTO") -#! [part1] + ie.set_config( + config={ov.properties.device.priorities(): "GPU,CPU"}, device_name="AUTO" + ) + #! [part1] + def part3(): -#! [part3] - core = Core() - # Read a network in IR, PaddlePaddle, or ONNX format: - model = core.read_model(model_path) + #! [part3] + core = ov.Core() + # Compile a model on AUTO with Performance Hints enabled: # To use the “THROUGHPUT” mode: - compiled_model = core.compile_model(model=model, device_name="AUTO", config={"PERFORMANCE_HINT":"THROUGHPUT"}) + compiled_model = core.compile_model( + model=model, + device_name="AUTO", + config={ + ov.properties.hint.performance_mode(): ov.properties.hint.PerformanceMode.THROUGHPUT + }, + ) # To use the “LATENCY” mode: - compiled_model = core.compile_model(model=model, device_name="AUTO", config={"PERFORMANCE_HINT":"LATENCY"}) + compiled_model = core.compile_model( + model=model, + device_name="AUTO", + config={ + ov.properties.hint.performance_mode(): ov.properties.hint.PerformanceMode.LATENCY + }, + ) # To use the “CUMULATIVE_THROUGHPUT” mode: - compiled_model = core.compile_model(model=model, device_name="AUTO", config={"PERFORMANCE_HINT":"CUMULATIVE_THROUGHPUT"}) -#! [part3] + compiled_model = core.compile_model( + model=model, + device_name="AUTO", + config={ + ov.properties.hint.performance_mode(): ov.properties.hint.PerformanceMode.CUMULATIVE_THROUGHPUT + }, + ) + #! [part3] + def part4(): -#! [part4] - core = Core() - model = core.read_model(model_path) + #! [part4] + core = ov.Core() # Example 1 - compiled_model0 = core.compile_model(model=model, device_name="AUTO", config={"MODEL_PRIORITY":"HIGH"}) - compiled_model1 = core.compile_model(model=model, device_name="AUTO", config={"MODEL_PRIORITY":"MEDIUM"}) - compiled_model2 = core.compile_model(model=model, device_name="AUTO", config={"MODEL_PRIORITY":"LOW"}) + compiled_model0 = core.compile_model( + model=model, + device_name="AUTO", + config={ov.properties.hint.model_priority(): ov.properties.hint.Priority.HIGH}, + ) + compiled_model1 = core.compile_model( + model=model, + device_name="AUTO", + config={ + ov.properties.hint.model_priority(): ov.properties.hint.Priority.MEDIUM + }, + ) + compiled_model2 = core.compile_model( + model=model, + device_name="AUTO", + config={ov.properties.hint.model_priority(): ov.properties.hint.Priority.LOW}, + ) # Assume that all the devices (CPU and GPUs) can support all the networks. # Result: compiled_model0 will use GPU.1, compiled_model1 will use GPU.0, compiled_model2 will use CPU. # Example 2 - compiled_model3 = core.compile_model(model=model, device_name="AUTO", config={"MODEL_PRIORITY":"HIGH"}) - compiled_model4 = core.compile_model(model=model, device_name="AUTO", config={"MODEL_PRIORITY":"MEDIUM"}) - compiled_model5 = core.compile_model(model=model, device_name="AUTO", config={"MODEL_PRIORITY":"LOW"}) + compiled_model3 = core.compile_model( + model=model, + device_name="AUTO", + config={ov.properties.hint.model_priority(): ov.properties.hint.Priority.HIGH}, + ) + compiled_model4 = core.compile_model( + model=model, + device_name="AUTO", + config={ + ov.properties.hint.model_priority(): ov.properties.hint.Priority.MEDIUM + }, + ) + compiled_model5 = core.compile_model( + model=model, + device_name="AUTO", + config={ov.properties.hint.model_priority(): ov.properties.hint.Priority.LOW}, + ) # Assume that all the devices (CPU ang GPUs) can support all the networks. # Result: compiled_model3 will use GPU.1, compiled_model4 will use GPU.1, compiled_model5 will use GPU.0. -#! [part4] + #! [part4] + def part5(): -#! [part5] - core = Core() - model = core.read_model(model_path) + #! [part5] + core = ov.Core() + # gpu_config and cpu_config will load during compile_model() compiled_model = core.compile_model(model=model) compiled_model = core.compile_model(model=model, device_name="AUTO") -#! [part5] + #! [part5] + def part6(): -#! [part6] - core = Core() - # read a network in IR, PaddlePaddle, or ONNX format - model = core.read_model(model_path) + #! [part6] + core = ov.Core() + # compile a model on AUTO and set log level to debug - compiled_model = core.compile_model(model=model, device_name="AUTO", config={"LOG_LEVEL":"LOG_DEBUG"}) + compiled_model = core.compile_model( + model=model, + device_name="AUTO", + config={ov.properties.log.level(): ov.properties.log.Level.DEBUG}, + ) # set log level with set_property and compile model - core.set_property(device_name="AUTO", properties={"LOG_LEVEL":"LOG_DEBUG"}) + core.set_property( + device_name="AUTO", + properties={ov.properties.log.level(): ov.properties.log.Level.DEBUG}, + ) compiled_model = core.compile_model(model=model, device_name="AUTO") -#! [part6] + #! [part6] + def part7(): -#! [part7] - core = Core() - # read a network in IR, PaddlePaddle, or ONNX format - model = core.read_model(model_path) + #! [part7] + core = ov.Core() + # compile a model on AUTO and set log level to debug compiled_model = core.compile_model(model=model, device_name="AUTO") # query the runtime target devices on which the inferences are being executed - execution_devices = compiled_model.get_property("EXECUTION_DEVICES") -#! [part7] + execution_devices = compiled_model.get_property(ov.properties.execution_devices()) + #! [part7] + def main(): - part0() - part1() part3() part4() part5() part6() part7() - -if __name__ == '__main__': - sys.exit(main()) + core = ov.Core() + if "GPU" not in core.available_devices: + return 0 + part0() + part1() diff --git a/docs/snippets/ov_auto_batching.py b/docs/snippets/ov_auto_batching.py index cecf2f846a7..5e4f3032f7d 100644 --- a/docs/snippets/ov_auto_batching.py +++ b/docs/snippets/ov_auto_batching.py @@ -1,41 +1,55 @@ -from openvino.runtime import Core +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# -core = Core() -model = core.read_model(model="sample.xml") +import openvino as ov +from utils import get_model -# [compile_model] -config = {"PERFORMANCE_HINT": "THROUGHPUT"} -compiled_model = core.compile_model(model, "GPU", config) -# [compile_model] -# [compile_model_no_auto_batching] -# disabling the automatic batching -# leaving intact other configurations options that the device selects for the 'throughput' hint -config = {"PERFORMANCE_HINT": "THROUGHPUT", - "ALLOW_AUTO_BATCHING": False} -compiled_model = core.compile_model(model, "GPU", config) -# [compile_model_no_auto_batching] +def main(): -# [query_optimal_num_requests] -# when the batch size is automatically selected by the implementation -# it is important to query/create and run the sufficient requests -config = {"PERFORMANCE_HINT": "THROUGHPUT"} -compiled_model = core.compile_model(model, "GPU", config) -num_requests = compiled_model.get_property("OPTIMAL_NUMBER_OF_INFER_REQUESTS") -# [query_optimal_num_requests] + core = ov.Core() + model = get_model() -# [hint_num_requests] -config = {"PERFORMANCE_HINT": "THROUGHPUT", - "PERFORMANCE_HINT_NUM_REQUESTS": "4"} -# limiting the available parallel slack for the 'throughput' -# so that certain parameters (like selected batch size) are automatically accommodated accordingly -compiled_model = core.compile_model(model, "GPU", config) -# [hint_num_requests] + if "GPU" not in core.available_devices: + return 0 -# [hint_plus_low_level] -config = {"PERFORMANCE_HINT": "THROUGHPUT", - "INFERENCE_NUM_THREADS": "4"} -# limiting the available parallel slack for the 'throughput' -# so that certain parameters (like selected batch size) are automatically accommodated accordingly -compiled_model = core.compile_model(model, "CPU", config) -# [hint_plus_low_level]] \ No newline at end of file + # [compile_model] + import openvino.runtime.properties as props + import openvino.runtime.properties.hint as hints + + config = {hints.performance_mode(): hints.PerformanceMode.THROUGHPUT} + compiled_model = core.compile_model(model, "GPU", config) + # [compile_model] + + # [compile_model_no_auto_batching] + # disabling the automatic batching + # leaving intact other configurations options that the device selects for the 'throughput' hint + config = {hints.performance_mode(): hints.PerformanceMode.THROUGHPUT, + hints.allow_auto_batching(): False} + compiled_model = core.compile_model(model, "GPU", config) + # [compile_model_no_auto_batching] + + # [query_optimal_num_requests] + # when the batch size is automatically selected by the implementation + # it is important to query/create and run the sufficient requests + config = {hints.performance_mode(): hints.PerformanceMode.THROUGHPUT} + compiled_model = core.compile_model(model, "GPU", config) + num_requests = compiled_model.get_property(props.optimal_number_of_infer_requests()) + # [query_optimal_num_requests] + + # [hint_num_requests] + config = {hints.performance_mode(): hints.PerformanceMode.THROUGHPUT, + hints.num_requests(): "4"} + # limiting the available parallel slack for the 'throughput' + # so that certain parameters (like selected batch size) are automatically accommodated accordingly + compiled_model = core.compile_model(model, "GPU", config) + # [hint_num_requests] + + # [hint_plus_low_level] + config = {hints.performance_mode(): hints.PerformanceMode.THROUGHPUT, + props.inference_num_threads(): "4"} + # limiting the available parallel slack for the 'throughput' + # so that certain parameters (like selected batch size) are automatically accommodated accordingly + compiled_model = core.compile_model(model, "CPU", config) + # [hint_plus_low_level]] diff --git a/docs/snippets/ov_caching.py b/docs/snippets/ov_caching.py index 51014707769..ef3fd0b4861 100644 --- a/docs/snippets/ov_caching.py +++ b/docs/snippets/ov_caching.py @@ -2,35 +2,39 @@ # SPDX-License-Identifier: Apache-2.0 # -from openvino.runtime import Core +from utils import get_path_to_model, get_temp_dir +import openvino as ov + +import openvino.runtime.properties as props device_name = 'GNA' -xml_path = '/tmp/myModel.xml' +model_path = get_path_to_model() +path_to_cache_dir = get_temp_dir() # ! [ov:caching:part0] -core = Core() -core.set_property({'CACHE_DIR': '/path/to/cache/dir'}) -model = core.read_model(model=xml_path) +core = ov.Core() +core.set_property({props.cache_dir(): path_to_cache_dir}) +model = core.read_model(model=model_path) compiled_model = core.compile_model(model=model, device_name=device_name) # ! [ov:caching:part0] assert compiled_model # ! [ov:caching:part1] -core = Core() -compiled_model = core.compile_model(model_path=xml_path, device_name=device_name) +core = ov.Core() +compiled_model = core.compile_model(model=model_path, device_name=device_name) # ! [ov:caching:part1] assert compiled_model # ! [ov:caching:part2] -core = Core() -core.set_property({'CACHE_DIR': '/path/to/cache/dir'}) -compiled_model = core.compile_model(model_path=xml_path, device_name=device_name) +core = ov.Core() +core.set_property({props.cache_dir(): path_to_cache_dir}) +compiled_model = core.compile_model(model=model_path, device_name=device_name) # ! [ov:caching:part2] assert compiled_model # ! [ov:caching:part3] # Find 'EXPORT_IMPORT' capability in supported capabilities -caching_supported = 'EXPORT_IMPORT' in core.get_property(device_name, 'OPTIMIZATION_CAPABILITIES') +caching_supported = 'EXPORT_IMPORT' in core.get_property(device_name, props.device.capabilities()) # ! [ov:caching:part3] diff --git a/docs/snippets/ov_common.py b/docs/snippets/ov_common.py index c6ea0a8f97d..b3d07524391 100644 --- a/docs/snippets/ov_common.py +++ b/docs/snippets/ov_common.py @@ -4,14 +4,17 @@ import numpy as np +from utils import get_image, get_path_to_extension_library, get_path_to_model #! [ov_api_2_0:create_core] -import openvino.runtime as ov +import openvino as ov core = ov.Core() #! [ov_api_2_0:create_core] +model_path = get_path_to_model() + #! [ov_api_2_0:read_model] -model = core.read_model("model.xml") +model = core.read_model(model_path) #! [ov_api_2_0:read_model] #! [ov_api_2_0:compile_model] @@ -30,7 +33,7 @@ assert input_tensor1.data.dtype == np.int64 # Fill the first data ... # Get input tensor by tensor name -input_tensor2 = infer_request.get_tensor("data2_t") +input_tensor2 = infer_request.get_tensor("input") assert input_tensor2.data.dtype == np.int64 # Fill the second data ... #! [ov_api_2_0:get_input_tensor_aligned] @@ -43,7 +46,7 @@ assert input_tensor1.data.dtype == np.int32 # Fill the first data ... # Get input tensor by tensor name -input_tensor2 = infer_request.get_tensor("data2_t") +input_tensor2 = infer_request.get_tensor("input") # IR v10 works with converted precisions (i64 -> i32) assert input_tensor2.data.dtype == np.int32 # Fill the second data .. @@ -53,7 +56,7 @@ assert input_tensor2.data.dtype == np.int32 results = infer_request.infer() #! [ov_api_2_0:inference] -input_data = iter(list()) +input_data = get_image() def process_results(results, frame_id): pass @@ -83,7 +86,7 @@ infer_queue.set_callback(callback) total_frames = 100 for i in range(total_frames): # Wait for at least one available infer request and start asynchronous inference - infer_queue.start_async(next(input_data), userdata=i) + infer_queue.start_async(input_data, userdata=i) # Wait for all requests to complete infer_queue.wait_all() #! [ov_api_2_0:start_async_and_wait] @@ -104,6 +107,8 @@ assert output_tensor.data.dtype == np.int32 # process output data ... #! [ov_api_2_0:get_output_tensor_v10] +path_to_extension_library = get_path_to_extension_library() + #! [ov_api_2_0:load_old_extension] -core.add_extension("path_to_extension_library.so") +core.add_extension(path_to_extension_library) #! [ov_api_2_0:load_old_extension] diff --git a/docs/snippets/ov_denormals.py b/docs/snippets/ov_denormals.py index 20f54e7eced..45b77483fcc 100644 --- a/docs/snippets/ov_denormals.py +++ b/docs/snippets/ov_denormals.py @@ -2,14 +2,15 @@ # SPDX-License-Identifier: Apache-2.0 # -import openvino.runtime as ov +import openvino as ov +from utils import get_model device_name = 'CPU' -xml_path = 'modelWithDenormals.xml' +model = get_model() + # ! [ov:intel_cpu:denormals_optimization:part0] core = ov.Core() core.set_property("CPU", ov.properties.intel_cpu.denormals_optimization(True)) -model = core.read_model(model=xml_path) compiled_model = core.compile_model(model=model, device_name=device_name) # ! [ov:intel_cpu:denormals_optimization:part0] assert compiled_model diff --git a/docs/snippets/ov_dynamic_shapes.py b/docs/snippets/ov_dynamic_shapes.py index 7d90076a189..ce9474b6750 100644 --- a/docs/snippets/ov_dynamic_shapes.py +++ b/docs/snippets/ov_dynamic_shapes.py @@ -2,13 +2,16 @@ # SPDX-License-Identifier: Apache-2.0 import numpy as np +from utils import get_dynamic_model + #! [import] -import openvino.runtime as ov +import openvino as ov #! [import] +model = get_dynamic_model() + #! [reshape_undefined] -Core = ov.Core() -model = core.read_model(“model.xml”) +core = ov.Core() # Set first dimension to be dynamic while keeping others static model.reshape([-1, 3, 224, 224]) @@ -25,7 +28,7 @@ model.reshape([-1, 3, (112, 448), (112, 448)]) model.reshape([(1, 8), 3, (112, 448), (112, 448)]) #! [reshape_bounds] -model = core.read_model("model.xml") +model = get_dynamic_model() #! [print_dynamic] # Print output partial shape @@ -35,41 +38,43 @@ print(model.output().partial_shape) print(model.input().partial_shape) #! [print_dynamic] -#! [detect_dynamic] -model = core.read_model("model.xml") +model = get_dynamic_model() -if model.input(0).partial_shape.is_dynamic(): +#! [detect_dynamic] + +if model.input(0).partial_shape.is_dynamic: # input is dynamic pass -if model.output(0).partial_shape.is_dynamic(): +if model.output(0).partial_shape.is_dynamic: # output is dynamic pass -if model.output(0).partial_shape[1].is_dynamic(): +if model.output(0).partial_shape[1].is_dynamic: # 1-st dimension of output is dynamic pass #! [detect_dynamic] executable = core.compile_model(model) infer_request = executable.create_infer_request() +input_tensor_name = "input" #! [set_input_tensor] # For first inference call, prepare an input tensor with 1x128 shape and run inference request -Input_data1 = np.ones(shape=[1,128]) -infer_request.infer([input_data1]) +input_data1 = np.ones(shape=[1,128]) +infer_request.infer({input_tensor_name: input_data1}) # Get resulting outputs -Output_tensor1 = infer_request.get_output_tensor() -Output_data1 = output_tensor.data[:] +output_tensor1 = infer_request.get_output_tensor() +output_data1 = output_tensor1.data[:] # For second inference call, prepare a 1x200 input tensor and run inference request -Input_data2 = np.ones(shape=[1,200]) -infer_request.infer([input_data2]) +input_data2 = np.ones(shape=[1,200]) +infer_request.infer({input_tensor_name: input_data2}) # Get resulting outputs -Output_tensor2 = infer_request.get_output_tensor() -Output_data2 = output_tensor.data[:] +output_tensor2 = infer_request.get_output_tensor() +output_data2 = output_tensor2.data[:] #! [set_input_tensor] infer_request = executable.create_infer_request() @@ -96,10 +101,9 @@ infer_request.infer() data2 = output_tensor.data[:] #! [get_input_tensor] -#! [check_inputs] -core = ov.Core() -model = core.read_model("model.xml") +model = get_dynamic_model() +#! [check_inputs] # Print model input layer info for input_layer in model.inputs: print(input_layer.names, input_layer.partial_shape) diff --git a/docs/snippets/ov_extensions.py b/docs/snippets/ov_extensions.py index 3b0d320eba3..99e1412e00e 100644 --- a/docs/snippets/ov_extensions.py +++ b/docs/snippets/ov_extensions.py @@ -2,10 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 # -import openvino.runtime as ov +import openvino as ov #! [py_frontend_extension_ThresholdedReLU_header] -import openvino.runtime.opset8 as ops +import openvino.runtime.opset12 as ops from openvino.frontend import ConversionExtension #! [py_frontend_extension_ThresholdedReLU_header] @@ -17,10 +17,14 @@ from openvino.frontend import ConversionExtension # Not implemented #! [add_frontend_extension] +from utils import get_path_to_extension_library + +path_to_extension_lib = get_path_to_extension_library() + #! [add_extension_lib] core = ov.Core() # Load extensions library to ov.Core -core.add_extension("libopenvino_template_extension.so") +core.add_extension(path_to_extension_lib) #! [add_extension_lib] #! [py_frontend_extension_MyRelu] diff --git a/docs/snippets/ov_graph.py b/docs/snippets/ov_graph.py index e3eb6e9db45..f0d86336391 100644 --- a/docs/snippets/ov_graph.py +++ b/docs/snippets/ov_graph.py @@ -25,21 +25,22 @@ # | result | # |_____________| -import openvino.runtime as ov +import openvino as ov +import openvino.runtime.opset12 as ops -data1 = ov.opset8.parameter([1, 3, 2, 2], ov.Type.i64) +data1 = ops.parameter([1, 3, 2, 2], ov.Type.i64) data1.friendly_name = "data1" # operation name data1.output(0).name = "data1_t" # tensor name -data2 = ov.opset8.parameter([1, 2, 2, 2], ov.Type.i64) +data2 = ops.parameter([1, 2, 2, 2], ov.Type.i64) data2.friendly_name = "data2" # operation name data2.output(0).name = "data2_t" # tensor name -concat = ov.opset8.concat([data1, data2], 1) +concat = ops.concat([data1, data2], 1) concat.friendly_name = "concat" # operation name concat.output(0).name = "concat_t" # tensor name -result = ov.opset8.result(concat) +result = ops.result(concat) result.friendly_name = "result" # operation name model = ov.Model(result, [data1, data2], "model_name") diff --git a/docs/snippets/ov_hetero.py b/docs/snippets/ov_hetero.py index 2d5bbd2d454..95623a1f74a 100644 --- a/docs/snippets/ov_hetero.py +++ b/docs/snippets/ov_hetero.py @@ -1,43 +1,50 @@ -import openvino.runtime as ov +import openvino as ov +from utils import get_model -core = ov.Core() -model = core.read_model("sample.xml") +def main(): + model = get_model() + core = ov.Core() -#! [set_manual_affinities] -for op in model.get_ops(): - rt_info = op.get_rt_info() - rt_info["affinity"] = "CPU" -#! [set_manual_affinities] + #! [set_manual_affinities] + for op in model.get_ops(): + rt_info = op.get_rt_info() + rt_info["affinity"] = "CPU" + #! [set_manual_affinities] -#! [fix_automatic_affinities] -# This example demonstrates how to perform default affinity initialization and then -# correct affinity manually for some layers -device = "HETERO:GPU,CPU" + if "GPU" not in core.available_devices: + return 0 -# query_model result contains mapping of supported operations to devices -supported_ops = core.query_model(model, device) + #! [fix_automatic_affinities] + # This example demonstrates how to perform default affinity initialization and then + # correct affinity manually for some layers + device = "HETERO:GPU,CPU" -# update default affinities manually for specific operations -supported_ops["operation_name"] = "CPU" + # query_model result contains mapping of supported operations to devices + supported_ops = core.query_model(model, device) -# set affinities to a model -for node in model.get_ops(): - affinity = supported_ops[node.get_friendly_name()] - node.get_rt_info()["affinity"] = "CPU" + # update default affinities manually for specific operations + supported_ops["operation_name"] = "CPU" -# load model with manually set affinities -compiled_model = core.compile_model(model, device) -#! [fix_automatic_affinities] + # set affinities to a model + for node in model.get_ops(): + affinity = supported_ops[node.get_friendly_name()] + node.get_rt_info()["affinity"] = "CPU" -#! [compile_model] -compiled_model = core.compile_model(model, device_name="HETERO:GPU,CPU") -# device priorities via configuration property -compiled_model = core.compile_model(model, device_name="HETERO", config={"MULTI_DEVICE_PRIORITIES": "GPU,CPU"}) -#! [compile_model] + # load model with manually set affinities + compiled_model = core.compile_model(model, device) + #! [fix_automatic_affinities] -#! [configure_fallback_devices] -core.set_property("HETERO", {"MULTI_DEVICE_PRIORITIES": "GPU,CPU"}) -core.set_property("GPU", {"PERF_COUNT": "YES"}) -core.set_property("CPU", {"INFERENCE_PRECISION_HINT": "f32"}) -compiled_model = core.compile_model(model=model, device_name="HETERO") -#! [configure_fallback_devices] + #! [compile_model] + compiled_model = core.compile_model(model, device_name="HETERO:GPU,CPU") + # device priorities via configuration property + compiled_model = core.compile_model( + model, device_name="HETERO", config={ov.properties.device.priorities(): "GPU,CPU"} + ) + #! [compile_model] + + #! [configure_fallback_devices] + core.set_property("HETERO", {ov.properties.device.priorities(): "GPU,CPU"}) + core.set_property("GPU", {ov.properties.enable_profiling(): True}) + core.set_property("CPU", {ov.properties.hint.inference_precision(): ov.Type.f32}) + compiled_model = core.compile_model(model=model, device_name="HETERO") + #! [configure_fallback_devices] diff --git a/docs/snippets/ov_infer_request.py b/docs/snippets/ov_infer_request.py index c770684e2da..f2effba5fd4 100644 --- a/docs/snippets/ov_infer_request.py +++ b/docs/snippets/ov_infer_request.py @@ -1,13 +1,14 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -import numpy as np +from utils import get_model + #! [import] -import openvino.runtime as ov +import openvino as ov #! [import] core = ov.Core() -model = core.read_model("model.xml") +model = get_model() compiled_model = core.compile_model(model, "AUTO") #! [create_infer_request] @@ -31,10 +32,12 @@ infer_request.wait_for(10) #! [wait_for] #! [set_callback] -def callback(request, userdata): +def callback(request, _): request.start_async() -infer_request.set_callback(callback) +callbacks_info = {} +callbacks_info["finished"] = 0 +infer_request.set_callback(callback, callbacks_info) #! [set_callback] #! [cancel] @@ -48,24 +51,21 @@ output_tensor = infer_request.get_output_tensor() #! [get_set_index_tensor] input_tensor = infer_request.get_input_tensor(0) -output_tensor = infer_request.get_output_tensor(1) +output_tensor = infer_request.get_output_tensor(0) #! [get_set_index_tensor] -#! [get_set_name_tensor] -input_tensor = infer_request.get_tensor("input_name") -output_tensor = infer_request.get_tensor("output_name") -#! [get_set_name_tensor] +input_tensor_name = "input" #! [get_set_tensor] -tensor1 = infer_request.get_tensor("tensor_name1") -tensor2 = ov.Tensor() -infer_request.set_tensor("tensor_name2", tensor2) +tensor1 = infer_request.get_tensor("result") +tensor2 = ov.Tensor(ov.Type.f32, [1, 3, 32, 32]) +infer_request.set_tensor(input_tensor_name, tensor2) #! [get_set_tensor] #! [get_set_tensor_by_port] input_port = model.input(0) -output_port = model.input("tensor_name") -input_tensor = ov.Tensor() +output_port = model.input(input_tensor_name) +input_tensor = ov.Tensor(ov.Type.f32, [1, 3, 32, 32]) infer_request.set_tensor(input_port, input_tensor) output_tensor = infer_request.get_tensor(output_port) #! [get_set_tensor_by_port] @@ -81,15 +81,15 @@ infer_request2.set_input_tensor(0, output) #! [roi_tensor] # input_tensor points to input of a previous network and # cropROI contains coordinates of output bounding box **/ -input_tensor = ov.Tensor(type=ov.Type.f32, shape=ov.Shape([1, 3, 20, 20])) +input_tensor = ov.Tensor(type=ov.Type.f32, shape=ov.Shape([1, 3, 100, 100])) begin = [0, 0, 0, 0] -end = [1, 2, 3, 3] +end = [1, 3, 32, 32] # ... # roi_tensor uses shared memory of input_tensor and describes cropROI # according to its coordinates **/ roi_tensor = ov.Tensor(input_tensor, begin, end) -infer_request2.set_tensor("input_name", roi_tensor) +infer_request2.set_tensor(input_tensor_name, roi_tensor) #! [roi_tensor] #! [remote_tensor] diff --git a/docs/snippets/ov_layout.py b/docs/snippets/ov_layout.py index 5ce91e74a50..79547045ebd 100644 --- a/docs/snippets/ov_layout.py +++ b/docs/snippets/ov_layout.py @@ -1,10 +1,11 @@ # Copyright (C) 2018-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # -import openvino.runtime as ov +import openvino as ov +import openvino.runtime.opset12 as ops # ! [ov:layout:simple] -from openvino.runtime import Layout +from openvino import Layout layout = Layout('NCHW') # ! [ov:layout:simple] # ! [ov:layout:complex] @@ -61,12 +62,12 @@ def create_simple_model(): # Parameter--->Multiply--->Add--->Result # Constant---' / # Constant---' - data = ov.opset8.parameter([3, 1, 2], ov.Type.f32) - mul_constant = ov.opset8.constant([1.5], ov.Type.f32) - mul = ov.opset8.multiply(data, mul_constant) - add_constant = ov.opset8.constant([0.5], ov.Type.f32) - add = ov.opset8.add(mul, add_constant) - res = ov.opset8.result(add) + data = ops.parameter([3, 1, 2], ov.Type.f32, name="input_tensor_name") + mul_constant = ops.constant([1.5], ov.Type.f32) + mul = ops.multiply(data, mul_constant) + add_constant = ops.constant([0.5], ov.Type.f32) + add = ops.add(mul, add_constant) + res = ops.result(add) return ov.Model([res], [data], "model") model = create_simple_model() diff --git a/docs/snippets/ov_model_snippets.py b/docs/snippets/ov_model_snippets.py index 08da43fc2ab..0bddd24f142 100644 --- a/docs/snippets/ov_model_snippets.py +++ b/docs/snippets/ov_model_snippets.py @@ -3,10 +3,13 @@ import numpy as np #! [import] -import openvino.runtime as ov +import openvino as ov +#! [import] +import openvino.runtime.opset12 as ops #! [import] import openvino.runtime.passes as passes + # ! [ov:create_simple_model] def create_simple_model(): # This example shows how to create ov::Function @@ -14,12 +17,12 @@ def create_simple_model(): # Parameter--->Multiply--->Add--->Result # Constant---' / # Constant---' - data = ov.opset8.parameter([3, 1, 2], ov.Type.f32) - mul_constant = ov.opset8.constant([1.5], ov.Type.f32) - mul = ov.opset8.multiply(data, mul_constant) - add_constant = ov.opset8.constant([0.5], ov.Type.f32) - add = ov.opset8.add(mul, add_constant) - res = ov.opset8.result(add) + data = ops.parameter([3, 1, 2], ov.Type.f32) + mul_constant = ops.constant([1.5], ov.Type.f32) + mul = ops.multiply(data, mul_constant) + add_constant = ops.constant([0.5], ov.Type.f32) + add = ops.add(mul, add_constant) + res = ops.result(add) return ov.Model([res], [data], "model") # ! [ov:create_simple_model] @@ -30,24 +33,24 @@ def create_advanced_model(): # Parameter->Split---0-->Result # | `--1-->Relu-->Result # `----2-->Result - data = ov.opset8.parameter(ov.Shape([1, 3, 64, 64]), ov.Type.f32) + data = ops.parameter(ov.Shape([1, 3, 64, 64]), ov.Type.f32) # Create Constant for axis value - axis_const = ov.opset8.constant(ov.Type.i64, ov.Shape({}), [1]) + axis_const = ops.constant(1, dtype=ov.Type.i64) - # Create opset8::Split operation that splits input to three slices across 1st dimension - split = ov.opset8.split(data, axis_const, 3) + # Create opset12::Split operation that splits input to three slices across 1st dimension + split = ops.split(data, axis_const, 3) - # Create opset8::Relu operation that takes 1st Split output as input - relu = ov.opset8.relu(split.output(1)) + # Create opset12::Relu operation that takes 1st Split output as input + relu = ops.relu(split.output(1)) # Results operations will be created automatically based on provided OutputVector - return ov.Model([split.output(0), relu, split.output[2]], [data], "model") + return ov.Model([split.output(0), relu.output(0), split.output(2)], [data], "model") # ! [ov:create_advanced_model] def ov_api_examples(): # Doesn't work # node = ov.opset8.parameter(ov.PartialShape([ov.Dimension.dynamic(), 3, 64, 64]), np.float32) - node = ov.opset8.parameter(ov.PartialShape([ov.Dimension.dynamic(), ov.Dimension(3), ov.Dimension(64), ov.Dimension(64)]), np.float32) + node = ops.parameter(ov.PartialShape([ov.Dimension.dynamic(), ov.Dimension(3), ov.Dimension(64), ov.Dimension(64)]), np.float32) # it doesn't work: # static_shape = ov.Shape() @@ -59,8 +62,7 @@ def ov_api_examples(): # ! [ov:serialize] def serialize_example(m : ov.Model): - from openvino.runtime import serialize - serialize(m, xml_path='model.xml', bin_path='model.bin') + ov.serialize(m, xml_path='model.xml', bin_path='model.bin') # ! [ov:serialize] # ! [ov:visualize] @@ -68,7 +70,7 @@ def visualize_example(m : ov.Model): # Need import: # * import openvino.runtime.passes as passes pass_manager = passes.Manager() - pass_manager.register_pass(pass_name="VisualTree", file_name='image.svg') + pass_manager.register_pass(passes.VisualizeTree(file_name='image.svg')) pass_manager.run_passes(m) # ! [ov:visualize] @@ -79,7 +81,10 @@ def model_inputs_outputs(model : ov.Model): #! [all_inputs_ouputs] -if __name__ == '__main__': +def main(): ov_api_examples() create_simple_model() - create_advanced_model() + model = create_advanced_model() + serialize_example(model) + visualize_example(model) + model_inputs_outputs(model) diff --git a/docs/snippets/ov_model_with_state_infer.py b/docs/snippets/ov_model_with_state_infer.py new file mode 100644 index 00000000000..13122c492f2 --- /dev/null +++ b/docs/snippets/ov_model_with_state_infer.py @@ -0,0 +1,84 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import logging as log +import numpy as np +import sys + +import openvino as ov +from openvino.runtime import opset12 as ops + + +def main(): + # 1. Load inference engine + log.info("Loading OpenVINO") + core = ov.Core() + + #! [ov:model_create] + # 2. Creating ov.Model + input = ops.parameter([1, 1], dtype=np.float32, name="data") + init_const = ops.constant([0], dtype=np.float32) + read = ops.read_value(init_const, "variable0") + add = ops.add(input, read) + assign = ops.assign(add, "variable0") + add2 = ops.add(add, read) + result = ops.result(add2) + model = ov.Model(results=[result], sinks=[assign], parameters=[input], name="model") + #! [ov:model_create] + + log.info("Loading network files") + + # 3. Load network to CPU + compiled_model = core.compile_model(model, "CPU") + # 4. Create Infer Request + infer_request = compiled_model.create_infer_request() + + # 5. Prepare inputs + input_tensors = [] + for input in compiled_model.inputs: + input_tensors.append(infer_request.get_tensor(input)) + + # 6. Prepare outputs + output_tensors = [] + for output in compiled_model.outputs: + output_tensors.append(infer_request.get_tensor(output)) + + # 7. Initialize memory state before starting + for state in infer_request.query_state(): + state.reset() + + #! [ov:part1] + # input data + input_data = np.arange(start=1, stop=7, dtype=np.float32) + log.info("Infer the first utterance") + for next_input in range(int(len(input_data)/2)): + infer_request.infer({"data" : np.asarray([input_data[next_input]]).reshape([1,1])}) + # check states + states = infer_request.query_state() + if len(states) == 0: + log.error("Queried states are empty") + return -1 + mstate = states[0].state + if not mstate: + log.error("Can't cast state to MemoryBlob") + return -1 + state_buf = mstate.data + log.info(state_buf[0]) + + log.info("\nReset state between utterances...\n") + for state in infer_request.query_state(): + state.reset() + + log.info("Infer the second utterance") + for next_input in range(int(len(input_data)/2), len(input_data)): + infer_request.infer({0 : np.asarray([input_data[next_input]]).reshape([1,1])}) + # check states + states = infer_request.query_state() + mstate = states[0].state + state_buf = mstate.data + log.info(state_buf[0]) + #! [ov:part1] + + log.info("Execution successful") + + return 0 diff --git a/docs/snippets/ov_multi.py b/docs/snippets/ov_multi.py index f4e538c1b84..19ec07d8f9a 100644 --- a/docs/snippets/ov_multi.py +++ b/docs/snippets/ov_multi.py @@ -1,108 +1,121 @@ import sys -from openvino.runtime import Core -model_path = "/openvino_CI_CD/result/install_pkg/tests/test_model_zoo/core/models/ir/add_abc.xml" -path_to_model = "/openvino_CI_CD/result/install_pkg/tests/test_model_zoo/core/models/ir/add_abc.xml" -def MULTI_0(): -#! [MULTI_0] - core = Core() +import openvino as ov +from utils import get_model + +model = get_model() + + +def MULTI_0(): + #! [MULTI_0] + core = ov.Core() - # Read a network in IR, PaddlePaddle, or ONNX format: - model = core.read_model(model_path) - # Option 1 # Pre-configure MULTI globally with explicitly defined devices, # and compile the model on MULTI using the newly specified default device list. - core.set_property(device_name="MULTI", properties={"MULTI_DEVICE_PRIORITIES":"GPU,CPU"}) + core.set_property( + device_name="MULTI", properties={ov.properties.device.priorities(): "GPU,CPU"} + ) compiled_model = core.compile_model(model=model, device_name="MULTI") # Option 2 # Specify the devices to be used by MULTI explicitly at compilation. # The following lines are equivalent: compiled_model = core.compile_model(model=model, device_name="MULTI:GPU,CPU") - compiled_model = core.compile_model(model=model, device_name="MULTI", config={"MULTI_DEVICE_PRIORITIES": "GPU,CPU"}) + compiled_model = core.compile_model( + model=model, + device_name="MULTI", + config={ov.properties.device.priorities(): "GPU,CPU"}, + ) + #! [MULTI_0] -#! [MULTI_0] def MULTI_1(): -#! [MULTI_1] - core = Core() - model = core.read_model(model_path) - core.set_property(device_name="MULTI", properties={"MULTI_DEVICE_PRIORITIES":"CPU,GPU"}) + #! [MULTI_1] + core = ov.Core() + + core.set_property( + device_name="MULTI", properties={ov.properties.device.priorities(): "CPU,GPU"} + ) # Once the priority list is set, you can alter it on the fly: # reverse the order of priorities - core.set_property(device_name="MULTI", properties={"MULTI_DEVICE_PRIORITIES":"GPU,CPU"}) - + core.set_property( + device_name="MULTI", properties={ov.properties.device.priorities(): "GPU,CPU"} + ) + # exclude some devices (in this case, CPU) - core.set_property(device_name="MULTI", properties={"MULTI_DEVICE_PRIORITIES":"GPU"}) - + core.set_property( + device_name="MULTI", properties={ov.properties.device.priorities(): "GPU"} + ) + # bring back the excluded devices - core.set_property(device_name="MULTI", properties={"MULTI_DEVICE_PRIORITIES":"GPU,CPU"}) + core.set_property( + device_name="MULTI", properties={ov.properties.device.priorities(): "GPU,CPU"} + ) # You cannot add new devices on the fly! # Attempting to do so will trigger the following exception: # [ ERROR ] [NOT_FOUND] You can only change device # priorities but not add new devices with the model's # ov::device::priorities. CPU device was not in the original device list! - -#! [MULTI_1] + #! [MULTI_1] # the following two pieces of code appear not to be used anywhere # they should be considered for removal + def available_devices_1(): -#! [available_devices_1] + #! [available_devices_1] all_devices = "MULTI:" - core = Core() - model = core.read_model(model_path) + core = ov.Core() + all_devices += ",".join(core.available_devices) compiled_model = core.compile_model(model=model, device_name=all_devices) -#! [available_devices_1] + #! [available_devices_1] + def available_devices_2(): -#! [available_devices_2] + #! [available_devices_2] match_list = [] all_devices = "MULTI:" dev_match_str = "GPU" - core = Core() - model = core.read_model(model_path) + core = ov.Core() + for d in core.available_devices: if dev_match_str in d: match_list.append(d) all_devices += ",".join(match_list) compiled_model = core.compile_model(model=model, device_name=all_devices) -#! [available_devices_2] - - - - - - + #! [available_devices_2] def MULTI_4(): -#! [MULTI_4] - core = Core() + #! [MULTI_4] + core = ov.Core() cpu_config = {} gpu_config = {} - # Read a network in IR, PaddlePaddle, or ONNX format: - model = core.read_model(model_path) - - # When compiling the model on MULTI, configure CPU and GPU + # When compiling the model on MULTI, configure CPU and GPU # (devices, priorities, and device configurations; gpu_config and cpu_config will load during compile_model() ): - compiled_model = core.compile_model(model=model, device_name="MULTI:GPU,CPU", config={"CPU":"NUM_STREAMS 4", "GPU":"NUM_STREAMS 8"}) + compiled_model = core.compile_model( + model=model, + device_name="MULTI:GPU,CPU", + config={"CPU": "NUM_STREAMS 4", "GPU": "NUM_STREAMS 8"}, + ) # Optionally, query the optimal number of requests: - nireq = compiled_model.get_property("OPTIMAL_NUMBER_OF_INFER_REQUESTS") -#! [MULTI_4] + nireq = compiled_model.get_property( + ov.properties.optimal_number_of_infer_requests() + ) + #! [MULTI_4] + def main(): + core = ov.Core() + if "GPU" not in core.available_devices: + return 0 MULTI_0() MULTI_1() available_devices_1() available_devices_2() MULTI_4() - -if __name__ == '__main__': - sys.exit(main()) diff --git a/docs/snippets/ov_network_state_intro.py b/docs/snippets/ov_network_state_intro.py deleted file mode 100644 index 1ec8c31433d..00000000000 --- a/docs/snippets/ov_network_state_intro.py +++ /dev/null @@ -1,172 +0,0 @@ -# Copyright (C) 2018-2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -import logging as log -import numpy as np -import sys - -from openvino.runtime import opset10 as ops -from openvino.runtime import Core, Model, PartialShape, Tensor, Type -from openvino.runtime.passes import LowLatency2, MakeStateful, Manager - - -def state_network_example(): - #! [ov:state_network] - input = ops.parameter([1, 1], dtype=np.float32) - read = ops.read_value(input, "variable0") - add = ops.add(read, input) - save = ops.assign(add, "variable0") - result = ops.result(add) - model = Model(results=[result], sinks=[save], parameters=[input]) - #! [ov:state_network] - - -def low_latency_2_example(): - #! [ov:low_latency_2] - # Precondition for Model. - # TensorIterator and Parameter are created in body of TensorIterator with names - tensor_iterator_name = "TI_name" - body_parameter_name = "body_parameter_name" - idx = "0" # this is a first variable in the network - - # The State will be named "TI_name/param_name/variable_0" - state_name = tensor_iterator_name + "//" + body_parameter_name + "//" + "variable_" + idx # todo - - #! [ov:get_ov_model] - core = Core() - ov_model = core.read_model("path_to_the_model") - #! [ov:get_ov_model] - - # reshape input if needed - - #! [ov:reshape_ov_model] - ov_model.reshape({"X": PartialShape([1, 1, 16])}) - #! [ov:reshape_ov_model] - - #! [ov:apply_low_latency_2] - manager = Manager() - manager.register_pass(LowLatency2()) - manager.run_passes(ov_model) - #! [ov:apply_low_latency_2] - hd_specific_model = core.compile_model(ov_model) - # Try to find the Variable by name - infer_request = hd_specific_model.create_infer_request() - states = infer_request.query_state() - for state in states: - name = state.get_name() - if (name == state_name): - # some actions - #! [ov:low_latency_2] - - #! [ov:low_latency_2_use_parameters] - manager.register_pass(LowLatency2(False)) - #! [ov:low_latency_2_use_parameters] - - -def apply_make_stateful_tensor_names(): - #! [ov:make_stateful_tensor_names] - core = Core() - ov_model = core.read_model("path_to_the_model") - tensor_names = {"tensor_name_1": "tensor_name_4", - "tensor_name_3": "tensor_name_6"} - manager = Manager() - manager.register_pass(MakeStateful(tensor_names)) - manager.run_passes(ov_model) - #! [ov:make_stateful_tensor_names] - - -def apply_make_stateful_ov_nodes(): - #! [ov:make_stateful_ov_nodes] - core = Core() - ov_model = core.read_model("path_to_the_model") - # Parameter_1, Result_1, Parameter_3, Result_3 are - # ops.parameter/ops.result in the ov_model - pairs = ["""(Parameter_1, Result_1), (Parameter_3, Result_3)"""] - manager = Manager() - manager.register_pass(MakeStateful(pairs)) - manager.run_passes(ov_model) - #! [ov:make_stateful_ov_nodes] - - -def main(): - #! [ov:state_api_usage] - # 1. Load inference engine - log.info("Loading Inference Engine") - core = Core() - - # 2. Read a model - log.info("Loading network files") - model = core.read_model("path_to_the_model") - - - # 3. Load network to CPU - hw_specific_model = core.compile_model(model, "CPU") - - # 4. Create Infer Request - infer_request = hw_specific_model.create_infer_request() - - # 5. Reset memory states before starting - states = infer_request.query_state() - if (states.size() != 1): - log.error(f"Invalid queried state number. Expected 1, but got {str(states.size())}") - return -1 - - for state in states: - state.reset() - - # 6. Inference - input_data = np.arange(start=1, stop=12, dtype=np.float32) - - # This example demonstrates how to work with OpenVINO State API. - # Input_data: some array with 12 float numbers - - # Part1: read the first four elements of the input_data array sequentially. - # Expected output for the first utterance: - # sum of the previously processed elements [ 1, 3, 6, 10] - - # Part2: reset state value (set to 0) and read the next four elements. - # Expected output for the second utterance: - # sum of the previously processed elements [ 5, 11, 18, 26] - - # Part3: set state value to 5 and read the next four elements. - # Expected output for the third utterance: - # sum of the previously processed elements + 5 [ 14, 24, 35, 47] - target_state = states[0] - - # Part 1 - log.info("Infer the first utterance") - for next_input in range(len(input_data)/3): - infer_request.infer({0 : input_data[next_input]}) - state_buf = target_state.state.data - log.info(state_buf[0]) - - # Part 2 - log.info("\nReset state between utterances...\n") - target_state.reset() - - log.info("Infer the second utterance") - for next_input in range(len(input_data)/3, (len(input_data)/3 * 2)): - infer_request.infer({0 : input_data[next_input]}) - state_buf = target_state.state.data - log.info(state_buf[0]) - - # Part 3 - log.info("\nSet state value between utterances to 5...\n") - v = np.asarray([5], dtype=np.float32) - tensor = Tensor(v, shared_memory=True) - target_state.state = tensor - - log.info("Infer the third utterance") - for next_input in range((input_data.size()/3 * 2), input_data.size()): - infer_request.infer({0 : input_data[next_input]}) - - state_buf = target_state.state.data - log.info(state_buf[0]) - - log.info("Execution successful") - #! [ov:state_api_usage] - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/docs/snippets/ov_preprocessing.py b/docs/snippets/ov_preprocessing.py index 45229d37a34..23cd3054811 100644 --- a/docs/snippets/ov_preprocessing.py +++ b/docs/snippets/ov_preprocessing.py @@ -2,24 +2,26 @@ # SPDX-License-Identifier: Apache-2.0 # +import openvino.runtime.properties as props from openvino.preprocess import ResizeAlgorithm, ColorFormat -from openvino.runtime import Layout, Type, serialize +from openvino import Layout, Type, serialize +import openvino as ov +import openvino.runtime.opset12 as ops +from utils import get_model, get_temp_dir, get_path_to_model, get_advanced_model -xml_path = '' -input_name = '' +input_name = "input" +model = get_advanced_model() + # ! [ov:preprocess:create] from openvino.preprocess import PrePostProcessor -from openvino.runtime import Core -core = Core() -model = core.read_model(model=xml_path) ppp = PrePostProcessor(model) # ! [ov:preprocess:create] # ! [ov:preprocess:tensor] from openvino.preprocess import ColorFormat -from openvino.runtime import Layout, Type +from openvino import Layout, Type ppp.input(input_name).tensor() \ .set_element_type(Type.u8) \ .set_shape([1, 480, 640, 3]) \ @@ -52,10 +54,13 @@ ppp.output(2) # Gets output with index=2 (3rd one) in a model # ! [ov:preprocess:input_name] -ppp.input('image') +ppp.input(input_name) ppp.output('result') # ! [ov:preprocess:input_name] +model = get_model() +ppp = PrePostProcessor(model) + # ! [ov:preprocess:input_1] # no index/name is needed if model has one input ppp.input().preprocess().scale(50.) @@ -65,75 +70,75 @@ ppp.output() \ .postprocess().convert_element_type(Type.u8) # ! [ov:preprocess:input_1] # ! [ov:preprocess:mean_scale] -ppp.input('input').preprocess().mean(128).scale(127) +ppp.input(input_name).preprocess().mean(128).scale(127) # ! [ov:preprocess:mean_scale] # ! [ov:preprocess:mean_scale_array] # Suppose model's shape is {1, 3, 224, 224} # N=1, C=3, H=224, W=224 -ppp.input('input').model().set_layout(Layout('NCHW')) +ppp.input(input_name).model().set_layout(Layout('NCHW')) # Mean/Scale has 3 values which matches with C=3 -ppp.input('input').preprocess() \ +ppp.input(input_name).preprocess() \ .mean([103.94, 116.78, 123.68]).scale([57.21, 57.45, 57.73]) # ! [ov:preprocess:mean_scale_array] # ! [ov:preprocess:convert_element_type] # First define data type for your tensor -ppp.input('input').tensor().set_element_type(Type.u8) +ppp.input(input_name).tensor().set_element_type(Type.u8) # Then define preprocessing step -ppp.input('input').preprocess().convert_element_type(Type.f32) +ppp.input(input_name).preprocess().convert_element_type(Type.f32) # If conversion is needed to `model's` element type, 'f32' can be omitted -ppp.input('input').preprocess().convert_element_type() +ppp.input(input_name).preprocess().convert_element_type() # ! [ov:preprocess:convert_element_type] # ! [ov:preprocess:convert_layout] # First define layout for your tensor -ppp.input('input').tensor().set_layout(Layout('NHWC')) +ppp.input(input_name).tensor().set_layout(Layout('NHWC')) # Then define layout of model -ppp.input('input').model().set_layout(Layout('NCHW')) +ppp.input(input_name).model().set_layout(Layout('NCHW')) print(ppp) # Will print 'implicit layout conversion step' # ! [ov:preprocess:convert_layout] # ! [ov:preprocess:convert_layout_2] -ppp.input('input').tensor().set_shape([1, 480, 640, 3]) +ppp.input(input_name).tensor().set_shape([1, 480, 640, 3]) # Model expects shape {1, 3, 480, 640} -ppp.input('input').preprocess()\ +ppp.input(input_name).preprocess()\ .convert_layout([0, 3, 1, 2]) # 0 -> 0; 3 -> 1; 1 -> 2; 2 -> 3 # ! [ov:preprocess:convert_layout_2] # ! [ov:preprocess:resize_1] -ppp.input('input').tensor().set_shape([1, 3, 960, 1280]) -ppp.input('input').model().set_layout(Layout('??HW')) -ppp.input('input').preprocess()\ +ppp.input(input_name).tensor().set_shape([1, 3, 960, 1280]) +ppp.input(input_name).model().set_layout(Layout('??HW')) +ppp.input(input_name).preprocess()\ .resize(ResizeAlgorithm.RESIZE_LINEAR, 480, 640) # ! [ov:preprocess:resize_1] # ! [ov:preprocess:resize_2] -ppp.input('input').tensor().set_shape([1, 3, 960, 1280]) +ppp.input(input_name).tensor().set_shape([1, 3, 960, 1280]) # Model accepts {1, 3, 480, 640} shape, thus last dimensions are 'H' and 'W' -ppp.input('input').model().set_layout(Layout('??HW')) +ppp.input(input_name).model().set_layout(Layout('??HW')) # Resize to model's dimension -ppp.input('input').preprocess().resize(ResizeAlgorithm.RESIZE_LINEAR) +ppp.input(input_name).preprocess().resize(ResizeAlgorithm.RESIZE_LINEAR) # ! [ov:preprocess:resize_2] # ! [ov:preprocess:convert_color_1] -ppp.input('input').tensor().set_color_format(ColorFormat.BGR) +ppp.input(input_name).tensor().set_color_format(ColorFormat.BGR) -ppp.input('input').preprocess().convert_color(ColorFormat.RGB) +ppp.input(input_name).preprocess().convert_color(ColorFormat.RGB) # ! [ov:preprocess:convert_color_1] # ! [ov:preprocess:convert_color_2] # This will split original `input` to 2 separate inputs: `input/y' and 'input/uv' -ppp.input('input').tensor()\ +ppp.input(input_name).tensor()\ .set_color_format(ColorFormat.NV12_TWO_PLANES) -ppp.input('input').preprocess()\ +ppp.input(input_name).preprocess()\ .convert_color(ColorFormat.RGB) print(ppp) # Dump preprocessing steps to see what will happen # ! [ov:preprocess:convert_color_2] # ! [ov:preprocess:custom] # It is possible to insert some custom operations -import openvino.runtime.opset8 as ops +import openvino.runtime.opset12 as ops from openvino.runtime import Output from openvino.runtime.utils.decorators import custom_preprocess_function @@ -142,22 +147,22 @@ def custom_abs(output: Output): # Custom nodes can be inserted as Preprocessing steps return ops.abs(output) -ppp.input("input_image").preprocess() \ +ppp.input("input").preprocess() \ .custom(custom_abs) # ! [ov:preprocess:custom] # ! [ov:preprocess:postprocess] # Model's output has 'NCHW' layout -ppp.output('result_image').model().set_layout(Layout('NCHW')) +ppp.output('result').model().set_layout(Layout('NCHW')) # Set target user's tensor to U8 type + 'NHWC' layout # Precision & layout conversions will be done implicitly -ppp.output('result_image').tensor()\ +ppp.output('result').tensor()\ .set_layout(Layout("NHWC"))\ .set_element_type(Type.u8) # Also it is possible to insert some custom operations -import openvino.runtime.opset8 as ops +import openvino.runtime.opset12 as ops from openvino.runtime import Output from openvino.runtime.utils.decorators import custom_preprocess_function @@ -166,23 +171,23 @@ def custom_abs(output: Output): # Custom nodes can be inserted as Post-processing steps return ops.abs(output) -ppp.output("result_image").postprocess()\ +ppp.output("result").postprocess()\ .custom(custom_abs) # ! [ov:preprocess:postprocess] # ! [ov:preprocess:save_headers] from openvino.preprocess import PrePostProcessor, ColorFormat, ResizeAlgorithm -from openvino.runtime import Core, Layout, Type, set_batch -# First method - imports -from openvino.runtime import serialize -# Second method - imports -from openvino.runtime.passes import Manager, Serialize +from openvino import Core, Layout, Type, set_batch +from openvino import serialize # ! [ov:preprocess:save_headers] +model_path = get_path_to_model() +serialized_model_path = get_path_to_model() + # ! [ov:preprocess:save] # ======== Step 0: read original model ========= core = Core() -model = core.read_model(model='/path/to/some_model.onnx') +model = core.read_model(model=model_path) # ======== Step 1: Preprocessing ================ ppp = PrePostProcessor(model) @@ -213,20 +218,16 @@ model = ppp.build() set_batch(model, 2) # ======== Step 3: Save the model ================ -# First method - using serialize runtime wrapper -serialize(model, '/path/to/some_model_saved.xml', '/path/to/some_model_saved.bin') - -# Second method - using Manager and Serialize pass -manager = Manager() -manager.register_pass(Serialize('/path/to/some_model_saved.xml', '/path/to/some_model_saved.bin')) -manager.run_passes(model) +serialize(model, serialized_model_path) # ! [ov:preprocess:save] +path_to_cache_dir = get_temp_dir() + # ! [ov:preprocess:save_load] core = Core() -core.set_property({'CACHE_DIR': '/path/to/cache/dir'}) +core.set_property({props.cache_dir(): path_to_cache_dir}) # In case that no preprocessing is needed anymore, we can load model on target device directly # With cached model available, it will also save some time on reading original model -compiled_model = core.compile_model('/path/to/some_model_saved.xml', 'CPU') +compiled_model = core.compile_model(serialized_model_path, 'CPU') # ! [ov:preprocess:save_load] diff --git a/docs/snippets/ov_preprocessing_migration.py b/docs/snippets/ov_preprocessing_migration.py index 6426cbed8c8..86d28bad6a3 100644 --- a/docs/snippets/ov_preprocessing_migration.py +++ b/docs/snippets/ov_preprocessing_migration.py @@ -2,8 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 # +from utils import get_model, get_ngraph_model + #! [ov_imports] -from openvino.runtime import Core, Layout, Type +from openvino import Core, Layout, Type from openvino.preprocess import ColorFormat, PrePostProcessor, ResizeAlgorithm #! [ov_imports] @@ -13,11 +15,9 @@ import openvino.inference_engine as ie #include "inference_engine.hpp" -model_path = '' -tensor_name = '' - +tensor_name="input" core = Core() -model = core.read_model(model=model_path) +model = get_model([1,32,32,3]) #! [ov_mean_scale] ppp = PrePostProcessor(model) @@ -30,6 +30,8 @@ input.preprocess().mean([116.78]).scale([57.21, 57.45, 57.73]) model = ppp.build() #! [ov_mean_scale] +model = get_model() + #! [ov_conversions] ppp = PrePostProcessor(model) input = ppp.input(tensor_name) @@ -50,9 +52,11 @@ input.preprocess().convert_color(ColorFormat.BGR) model = ppp.build() #! [ov_color_space] +model = get_model([1, 3, 448, 448]) + #! [ov_image_scale] ppp = PrePostProcessor(model) -input = ppp.input(tensor_name) +input = ppp.input("input") # need to specify H and W dimensions in model, others are not important input.model().set_layout(Layout('??HW')) # scale to model shape @@ -61,39 +65,38 @@ input.preprocess().resize(ResizeAlgorithm.RESIZE_LINEAR, 448, 448) model = ppp.build() #! [ov_image_scale] +import openvino.inference_engine as ie +import ngraph as ng +operation_name = "data" -model_path = '' -operation_name = '' - -core = Core() -network = core.ReadNetwork(model_path) +core = ie.IECore() +network = get_ngraph_model() #! [mean_scale] -preProcess = network.getInputsInfo()[operation_name].getPreProcess() -preProcess.init(3) -preProcess[0].meanValue = 116.78 -preProcess[1].meanValue = 116.78 -preProcess[2].meanValue = 116.78 -preProcess[0].stdScale = 57.21 -preProcess[1].stdScale = 57.45 -preProcess[2].stdScale = 57.73 -preProcess.setVariant(ie.MEAN_VALUE) +preprocess_info = network.input_info[operation_name].preprocess_info +preprocess_info.init(3) +preprocess_info[0].mean_value = 116.78 +preprocess_info[1].mean_value = 116.78 +preprocess_info[2].mean_value = 116.78 +preprocess_info[0].std_scale = 57.21 +preprocess_info[1].std_scale = 57.45 +preprocess_info[2].std_scale = 57.73 +preprocess_info.mean_variant = ie.MeanVariant.MEAN_VALUE #! [mean_scale] #! [conversions] -inputInfo = network.getInputsInfo()[operation_name] -inputInfo.setPrecision(ie.Precision.U8) -inputInfo.setLayout(ie.Layout.NHWC) +input_info = network.input_info[operation_name] +input_info.precision = "U8" +input_info.layout = "NHWC" # model input layout is always NCHW in Inference Engine # for shapes with 4 dimensions #! [conversions] #! [image_scale] -preProcess = network.getInputsInfo()[operation_name].getPreProcess() +preprocess_info = network.input_info[operation_name].preprocess_info # Inference Engine supposes input for resize is always in NCHW layout # while for OpenVINO Runtime API 2.0 `H` and `W` dimensions must be specified # Also, current code snippet supposed resize from dynamic shapes -preProcess.setResizeAlgorithm(ie.ResizeAlgorithm.RESIZE_BILINEAR) -#! [image_scale] +preprocess_info.resize_algorithm = ie.ResizeAlgorithm.RESIZE_BILINEAR diff --git a/docs/snippets/ov_properties_api.py b/docs/snippets/ov_properties_api.py index f9c4b891329..cc9645a7682 100644 --- a/docs/snippets/ov_properties_api.py +++ b/docs/snippets/ov_properties_api.py @@ -2,53 +2,62 @@ # SPDX-License-Identifier: Apache-2.0 # -from openvino.runtime import Core +import openvino as ov +import openvino.runtime.properties as props +import openvino.runtime.properties.hint as hints -# [get_available_devices] -core = Core() -available_devices = core.available_devices -# [get_available_devices] - -# [hetero_priorities] -device_priorites = core.get_property("HETERO", "MULTI_DEVICE_PRIORITIES") -# [hetero_priorities] - -# [cpu_device_name] -cpu_device_name = core.get_property("CPU", "FULL_DEVICE_NAME") -# [cpu_device_name] - -model = core.read_model(model="sample.xml") -# [compile_model_with_property] -config = {"PERFORMANCE_HINT": "THROUGHPUT", - "INFERENCE_PRECISION_HINT": "f32"} -compiled_model = core.compile_model(model, "CPU", config) -# [compile_model_with_property] - -# [optimal_number_of_infer_requests] -compiled_model = core.compile_model(model, "CPU") -nireq = compiled_model.get_property("OPTIMAL_NUMBER_OF_INFER_REQUESTS") -# [optimal_number_of_infer_requests] +from utils import get_model -# [core_set_property_then_compile] -# latency hint is a default for CPU -core.set_property("CPU", {"PERFORMANCE_HINT": "LATENCY"}) -# compiled with latency configuration hint -compiled_model_latency = core.compile_model(model, "CPU") -# compiled with overriden performance hint value -config = {"PERFORMANCE_HINT": "THROUGHPUT"} -compiled_model_thrp = core.compile_model(model, "CPU", config) -# [core_set_property_then_compile] +def main(): + # [get_available_devices] + core = ov.Core() + available_devices = core.available_devices + # [get_available_devices] + + # [hetero_priorities] + device_priorites = core.get_property("HETERO", props.device.priorities()) + # [hetero_priorities] + + # [cpu_device_name] + cpu_device_name = core.get_property("CPU", props.device.full_name()) + # [cpu_device_name] + + model = get_model() + # [compile_model_with_property] + config = {hints.performance_mode(): hints.PerformanceMode.THROUGHPUT, + hints.inference_precision(): ov.Type.f32} + compiled_model = core.compile_model(model, "CPU", config) + # [compile_model_with_property] + + # [optimal_number_of_infer_requests] + compiled_model = core.compile_model(model, "CPU") + nireq = compiled_model.get_property(props.optimal_number_of_infer_requests()) + # [optimal_number_of_infer_requests] -# [inference_num_threads] -compiled_model = core.compile_model(model, "CPU") -nthreads = compiled_model.get_property("INFERENCE_NUM_THREADS") -# [inference_num_threads] + # [core_set_property_then_compile] + # latency hint is a default for CPU + core.set_property("CPU", {hints.performance_mode(): hints.PerformanceMode.LATENCY}) + # compiled with latency configuration hint + compiled_model_latency = core.compile_model(model, "CPU") + # compiled with overriden performance hint value + config = {hints.performance_mode(): hints.PerformanceMode.THROUGHPUT} + compiled_model_thrp = core.compile_model(model, "CPU", config) + # [core_set_property_then_compile] -# [multi_device] -config = {"MULTI_DEVICE_PRIORITIES": "CPU,GPU"} -compiled_model = core.compile_model(model, "MULTI", config) -# change the order of priorities -compiled_model.set_property({"MULTI_DEVICE_PRIORITIES": "GPU,CPU"}) -# [multi_device] + + # [inference_num_threads] + compiled_model = core.compile_model(model, "CPU") + nthreads = compiled_model.get_property(props.inference_num_threads()) + # [inference_num_threads] + + if "GPU" not in available_devices: + return 0 + + # [multi_device] + config = {props.device.priorities(): "CPU,GPU"} + compiled_model = core.compile_model(model, "MULTI", config) + # change the order of priorities + compiled_model.set_property({props.device.priorities(): "GPU,CPU"}) + # [multi_device] diff --git a/docs/snippets/ov_properties_migration.py b/docs/snippets/ov_properties_migration.py index 82f6a37b9e1..a590e3053e8 100644 --- a/docs/snippets/ov_properties_migration.py +++ b/docs/snippets/ov_properties_migration.py @@ -2,81 +2,88 @@ # SPDX-License-Identifier: Apache-2.0 # -from openvino.runtime import Core +import openvino as ov +import openvino.runtime.properties as props +import openvino.runtime.properties.hint as hints +from utils import get_model -core = Core() +def main(): + core = ov.Core() -# ! [core_set_property] -core.set_property(device_name="CPU", properties={"PERF_COUNT": "YES"}) -# ! [core_set_property] + # ! [core_set_property] + core.set_property(device_name="CPU", properties={props.enable_profiling(): True}) + # ! [core_set_property] -model = core.read_model("sample.xml") + model = get_model() -# ! [core_compile_model] -compiled_model = core.compile_model(model=model, device_name="MULTI", config= - { - "MULTI_DEVICE_PRIORITIES": "GPU,CPU", - "PERFORMANCE_HINT": "THROUGHPUT", - "INFERENCE_PRECISION_HINT": "f32" - }) -# ! [core_compile_model] + if "GPU" not in core.available_devices: + return 0 -# ! [compiled_model_set_property] -# turn CPU off for multi-device execution -compiled_model.set_property(properties={"MULTI_DEVICE_PRIORITIES": "GPU"}) -# ! [compiled_model_set_property] + # ! [core_compile_model] + compiled_model = core.compile_model(model=model, device_name="MULTI", config= + { + props.device.priorities(): "GPU,CPU", + hints.performance_mode(): hints.PerformanceMode.THROUGHPUT, + hints.inference_precision(): ov.Type.f32 + }) + # ! [core_compile_model] -# ! [core_get_rw_property] -num_streams = core.get_property("CPU", "NUM_STREAMS") -# ! [core_get_rw_property] + # ! [compiled_model_set_property] + # turn CPU off for multi-device execution + compiled_model.set_property(properties={props.device.priorities(): "GPU"}) + # ! [compiled_model_set_property] -# ! [core_get_ro_property] -full_device_name = core.get_property("CPU", "FULL_DEVICE_NAME") -# ! [core_get_ro_property] + # ! [core_get_rw_property] + num_streams = core.get_property("CPU", props.streams.num()) + # ! [core_get_rw_property] -# ! [compiled_model_get_rw_property] -perf_mode = compiled_model.get_property("PERFORMANCE_HINT") -# ! [compiled_model_get_rw_property] + # ! [core_get_ro_property] + full_device_name = core.get_property("CPU", props.device.full_name()) + # ! [core_get_ro_property] -# ! [compiled_model_get_ro_property] -nireq = compiled_model.get_property("OPTIMAL_NUMBER_OF_INFER_REQUESTS") -# ! [compiled_model_get_ro_property] + # ! [compiled_model_get_rw_property] + perf_mode = compiled_model.get_property(hints.performance_mode()) + # ! [compiled_model_get_rw_property] + # ! [compiled_model_get_ro_property] + nireq = compiled_model.get_property(props.optimal_number_of_infer_requests()) + # ! [compiled_model_get_ro_property] -from openvino.inference_engine import IECore + import ngraph as ng + import openvino.inference_engine as ie + from utils import get_ngraph_model + core = ie.IECore() + #! [core_get_metric] + full_device_name = core.get_metric("CPU", "FULL_DEVICE_NAME") + #! [core_get_metric] -core = IECore() -#! [core_get_metric] -full_device_name = core.get_metric("CPU", "FULL_DEVICE_NAME") -#! [core_get_metric] + #! [core_get_config] + num_streams = core.get_config("CPU", "CPU_THROUGHPUT_STREAMS") + #! [core_get_config] -#! [core_get_config] -num_streams = core.get_config("CPU", "CPU_THROUGHPUT_STREAMS") -#! [core_get_config] + #! [core_set_config] + core.set_config({"PERF_COUNT": "YES"}, "CPU") + #! [core_set_config] -#! [core_set_config] -core.set_config({"PERF_COUNT": "YES"}, "CPU") -#! [core_set_config] + net = get_ngraph_model() -net = core.read_network("sample.xml") + #! [core_load_network] + exec_network = core.load_network(net, "MULTI", {"DEVICE_PRIORITIES": "CPU, GPU", + "PERFORMANCE_HINT": "THROUGHPUT", + "ENFORCE_BF16": "NO"}) + #! [core_load_network] -#! [core_load_network] -exec_network = core.load_network(net, "MULTI", {"DEVICE_PRIORITIES": "CPU, GPU", - "PERFORMANCE_HINT": "THROUGHPUT", - "ENFORCE_BF16": "NO"}) -#! [core_load_network] + #! [executable_network_set_config] + # turn CPU off for multi-device execution + exec_network.set_config({"DEVICE_PRIORITIES": "GPU"}) + #! [executable_network_set_config] -#! [executable_network_set_config] -# turn CPU off for multi-device execution -exec_network.set_config({"DEVICE_PRIORITIES": "GPU"}) -#! [executable_network_set_config] + #! [executable_network_get_metric] + nireq = exec_network.get_metric("OPTIMAL_NUMBER_OF_INFER_REQUESTS") + #! [executable_network_get_metric] -#! [executable_network_get_metric] -nireq = exec_network.get_metric("OPTIMAL_NUMBER_OF_INFER_REQUESTS") -#! [executable_network_get_metric] - -#! [executable_network_get_config] -perf_hint = exec_network.get_config("PERFORMANCE_HINT") -#! [executable_network_get_config] + #! [executable_network_get_config] + perf_hint = exec_network.get_config("PERFORMANCE_HINT") + #! [executable_network_get_config] diff --git a/docs/snippets/ov_python_exclusives.py b/docs/snippets/ov_python_exclusives.py index a8c35285969..556735215dc 100644 --- a/docs/snippets/ov_python_exclusives.py +++ b/docs/snippets/ov_python_exclusives.py @@ -2,18 +2,17 @@ # SPDX-License-Identifier: Apache-2.0 import numpy as np +import openvino as ov -#! [auto_compilation] -import openvino.runtime as ov - -compiled_model = ov.compile_model("model.xml") -#! [auto_compilation] +from utils import get_path_to_model, get_image #! [properties_example] +import openvino.runtime.opset12 as ops + core = ov.Core() -input_a = ov.opset11.parameter([8], name="input_a") -res = ov.opset11.absolute(input_a) +input_a = ops.parameter([8], name="input_a") +res = ops.absolute(input_a) model = ov.Model(res, [input_a]) compiled = core.compile_model(model, "CPU") model.outputs[0].tensor.set_names({"result_0"}) # Add name for Output @@ -25,6 +24,10 @@ print(compiled.inputs) print(compiled.outputs) #! [properties_example] +#! [auto_compilation] +compiled_model = ov.compile_model(model) +#! [auto_compilation] + #! [tensor_basics] data_float64 = np.ones(shape=(2,8)) @@ -37,6 +40,7 @@ tensor = ov.Tensor(data_int32) assert tensor.element_type == ov.Type.i32 #! [tensor_basics] + #! [tensor_shared_mode] data_to_share = np.ones(shape=(2,8)) @@ -98,9 +102,9 @@ _ = next(iter(results.values())) core = ov.Core() # Simple model that adds two inputs together -input_a = ov.opset8.parameter([8]) -input_b = ov.opset8.parameter([8]) -res = ov.opset8.add(input_a, input_b) +input_a = ops.parameter([8]) +input_b = ops.parameter([8]) +res = ops.add(input_a, input_b) model = ov.Model(res, [input_a, input_b]) compiled = core.compile_model(model, "CPU") @@ -137,14 +141,14 @@ infer_queue.wait_all() assert all(data_done) #! [asyncinferqueue_set_callback] -unt8_data = np.ones([100]) +unt8_data = np.ones([100], dtype=np.uint8) #! [packing_data] from openvino.helpers import pack_data packed_buffer = pack_data(unt8_data, ov.Type.u4) # Create tensor with shape in element types -t = ov.Tensor(packed_buffer, [1, 128], ov.Type.u4) +t = ov.Tensor(packed_buffer, [100], ov.Type.u4) #! [packing_data] #! [unpacking] @@ -154,9 +158,12 @@ unpacked_data = unpack_data(t.data, t.element_type, t.shape) assert np.array_equal(unpacked_data , unt8_data) #! [unpacking] +model_path = get_path_to_model() +image = get_image() + + #! [releasing_gil] -import openvino.runtime as ov -import cv2 as cv +import openvino as ov from threading import Thread input_data = [] @@ -164,23 +171,20 @@ input_data = [] # Processing input data will be done in a separate thread # while compilation of the model and creation of the infer request # is going to be executed in the main thread. -def prepare_data(input, image_path): - image = cv.imread(image_path) - h, w = list(input.shape)[-2:] - image = cv.resize(image, (h, w)) - image = image.transpose((2, 0, 1)) - image = np.expand_dims(image, 0) - input_data.append(image) +def prepare_data(input, image): + shape = list(input.shape) + resized_img = np.resize(image, shape) + input_data.append(resized_img) core = ov.Core() -model = core.read_model("model.xml") +model = core.read_model(model_path) # Create thread with prepare_data function as target and start it -thread = Thread(target=prepare_data, args=[model.input(), "path/to/image"]) +thread = Thread(target=prepare_data, args=[model.input(), image]) thread.start() # The GIL will be released in compile_model. # It allows a thread above to start the job, # while main thread is running in the background. -compiled = core.compile_model(model, "GPU") +compiled = core.compile_model(model, "CPU") # After returning from compile_model, the main thread acquires the GIL # and starts create_infer_request which releases it once again. request = compiled.create_infer_request() diff --git a/docs/snippets/ov_python_inference.py b/docs/snippets/ov_python_inference.py index d55afeb5bf1..9d2fb0fb139 100644 --- a/docs/snippets/ov_python_inference.py +++ b/docs/snippets/ov_python_inference.py @@ -2,14 +2,15 @@ # SPDX-License-Identifier: Apache-2.0 import numpy as np -import openvino.runtime as ov +import openvino as ov +import openvino.runtime.opset12 as ops INPUT_SIZE = 1_000_000 # Use bigger values if necessary, i.e.: 300_000_000 -input_0 = ov.opset11.parameter([INPUT_SIZE], name="input_0") -input_1 = ov.opset11.parameter([INPUT_SIZE], name="input_1") -add_inputs = ov.opset11.add(input_0, input_1) -res = ov.opset11.reduce_sum(add_inputs, reduction_axes=0, name="reduced") +input_0 = ops.parameter([INPUT_SIZE], name="input_0") +input_1 = ops.parameter([INPUT_SIZE], name="input_1") +add_inputs = ops.add(input_0, input_1) +res = ops.reduce_sum(add_inputs, reduction_axes=0, name="reduced") model = ov.Model(res, [input_0, input_1], name="my_model") model.outputs[0].tensor.set_names({"reduced_result"}) # Add name for Output @@ -56,7 +57,7 @@ run(time_in_sec) # Hiding latency request.start_async({"input_0": data_0, "input_1": data_1}) -run() +run(time_in_sec) request.wait() results = request.get_output_tensor(0).data # Gather data from InferRequest #! [hiding_latency] diff --git a/docs/snippets/src/main.py b/docs/snippets/src/main.py index fe14021bbab..bb9dd4b1185 100644 --- a/docs/snippets/src/main.py +++ b/docs/snippets/src/main.py @@ -3,7 +3,7 @@ import numpy as np #! [import] -import openvino.runtime as ov +import openvino as ov #! [import] #! [part1] diff --git a/docs/snippets/utils.py b/docs/snippets/utils.py new file mode 100644 index 00000000000..de96f631ed8 --- /dev/null +++ b/docs/snippets/utils.py @@ -0,0 +1,86 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import tempfile +import numpy as np +from sys import platform + +import openvino as ov +import openvino.runtime.opset12 as ops + +import ngraph as ng +from ngraph.impl import Function +import openvino.inference_engine as ie + + +def get_dynamic_model(): + param = ops.parameter(ov.PartialShape([-1, -1]), name="input") + return ov.Model(ops.relu(param), [param]) + + +def get_model(input_shape = None, input_dtype=np.float32) -> ov.Model: + if input_shape is None: + input_shape = [1, 3, 32, 32] + param = ops.parameter(input_shape, input_dtype, name="input") + relu = ops.relu(param, name="relu") + relu.output(0).get_tensor().set_names({"result"}) + model = ov.Model([relu], [param], "test_model") + + assert model is not None + return model + + +def get_advanced_model(): + data = ops.parameter(ov.Shape([1, 3, 64, 64]), ov.Type.f32, name="input") + data1 = ops.parameter(ov.Shape([1, 3, 64, 64]), ov.Type.f32, name="input2") + axis_const = ops.constant(1, dtype=ov.Type.i64) + split = ops.split(data, axis_const, 3) + relu = ops.relu(split.output(1)) + relu.output(0).get_tensor().set_names({"result"}) + return ov.Model([split.output(0), relu.output(0), split.output(2)], [data, data1], "model") + + +def get_ngraph_model(input_shape = None, input_dtype=np.float32): + if input_shape is None: + input_shape = [1, 3, 32, 32] + param = ng.opset11.parameter(input_shape, input_dtype, name="data") + relu = ng.opset11.relu(param, name="relu") + func = Function([relu], [param], "test_model") + caps = ng.Function.to_capsule(func) + net = ie.IENetwork(caps) + + assert net is not None + return net + + +def get_image(shape = (1, 3, 32, 32), dtype = "float32"): + np.random.seed(42) + return np.random.rand(*shape).astype(dtype) + + +def get_path_to_extension_library(): + library_path="" + if platform == "win32": + library_path="openvino_template_extension.dll" + else: + library_path="libopenvino_template_extension.so" + return library_path + + +def get_path_to_model(input_shape = None, is_old_api=False): + if input_shape is None: + input_shape = [1, 3, 32, 32] + path_to_xml = tempfile.NamedTemporaryFile(suffix="_model.xml").name + if is_old_api: + net = get_ngraph_model(input_shape, input_dtype=np.int64) + net.serialize(path_to_xml) + else: + model = get_model(input_shape) + ov.serialize(model, path_to_xml) + return path_to_xml + + +def get_temp_dir(): + temp_dir = tempfile.TemporaryDirectory() + return temp_dir diff --git a/samples/python/benchmark/bert_benchmark/bert_benchmark.py b/samples/python/benchmark/bert_benchmark/bert_benchmark.py index eb50fb7bb52..b15bbd48b34 100755 --- a/samples/python/benchmark/bert_benchmark/bert_benchmark.py +++ b/samples/python/benchmark/bert_benchmark/bert_benchmark.py @@ -10,7 +10,8 @@ import tempfile from time import perf_counter import datasets -from openvino.runtime import Core, get_version, AsyncInferQueue, PartialShape +import openvino as ov +from openvino.runtime import get_version from transformers import AutoTokenizer from transformers.onnx import export from transformers.onnx.features import FeaturesManager @@ -28,7 +29,7 @@ def main(): # Download the tokenizer tokenizer = AutoTokenizer.from_pretrained(model_name) - core = Core() + core = ov.Core() with tempfile.TemporaryDirectory() as tmp: onnx_path = Path(tmp) / f'{model_name}.onnx' @@ -39,7 +40,7 @@ def main(): # Enforce dynamic input shape try: - model.reshape({model_input.any_name: PartialShape([1, '?']) for model_input in model.inputs}) + model.reshape({model_input.any_name: ov.PartialShape([1, '?']) for model_input in model.inputs}) except RuntimeError: log.error("Can't set dynamic shape") raise @@ -50,7 +51,7 @@ def main(): # It is possible to set CUMULATIVE_THROUGHPUT as PERFORMANCE_HINT for AUTO device compiled_model = core.compile_model(model, 'CPU', tput) # AsyncInferQueue creates optimal number of InferRequest instances - ireqs = AsyncInferQueue(compiled_model) + ireqs = ov.AsyncInferQueue(compiled_model) sst2 = datasets.load_dataset('glue', 'sst2') sst2_sentences = sst2['validation']['sentence'] diff --git a/samples/python/benchmark/sync_benchmark/sync_benchmark.py b/samples/python/benchmark/sync_benchmark/sync_benchmark.py index 367776356d3..e270d25a64f 100755 --- a/samples/python/benchmark/sync_benchmark/sync_benchmark.py +++ b/samples/python/benchmark/sync_benchmark/sync_benchmark.py @@ -9,7 +9,8 @@ import sys from time import perf_counter import numpy as np -from openvino.runtime import Core, get_version +import openvino as ov +from openvino.runtime import get_version from openvino.runtime.utils.types import get_dtype @@ -40,7 +41,7 @@ def main(): # Pick a device by replacing CPU, for example AUTO:GPU,CPU. # Using MULTI device is pointless in sync scenario # because only one instance of openvino.runtime.InferRequest is used - core = Core() + core = ov.Core() compiled_model = core.compile_model(sys.argv[1], 'CPU', latency) ireq = compiled_model.create_infer_request() # Fill input data for the ireq diff --git a/samples/python/benchmark/throughput_benchmark/throughput_benchmark.py b/samples/python/benchmark/throughput_benchmark/throughput_benchmark.py index 3867e785b31..c934a765017 100755 --- a/samples/python/benchmark/throughput_benchmark/throughput_benchmark.py +++ b/samples/python/benchmark/throughput_benchmark/throughput_benchmark.py @@ -9,7 +9,8 @@ import statistics from time import perf_counter import numpy as np -from openvino.runtime import Core, get_version, AsyncInferQueue +import openvino as ov +from openvino.runtime import get_version from openvino.runtime.utils.types import get_dtype @@ -39,10 +40,10 @@ def main(): # Create Core and use it to compile a model. # Pick a device by replacing CPU, for example MULTI:CPU(4),GPU(8). # It is possible to set CUMULATIVE_THROUGHPUT as PERFORMANCE_HINT for AUTO device - core = Core() + core = ov.Core() compiled_model = core.compile_model(sys.argv[1], 'CPU', tput) # AsyncInferQueue creates optimal number of InferRequest instances - ireqs = AsyncInferQueue(compiled_model) + ireqs = ov.AsyncInferQueue(compiled_model) # Fill input data for ireqs for ireq in ireqs: for model_input in compiled_model.inputs: diff --git a/samples/python/classification_sample_async/classification_sample_async.py b/samples/python/classification_sample_async/classification_sample_async.py index 8e0b8d244d3..92180f0534e 100755 --- a/samples/python/classification_sample_async/classification_sample_async.py +++ b/samples/python/classification_sample_async/classification_sample_async.py @@ -9,8 +9,7 @@ import sys import cv2 import numpy as np -from openvino.preprocess import PrePostProcessor -from openvino.runtime import AsyncInferQueue, Core, InferRequest, Layout, Type +import openvino as ov def parse_args() -> argparse.Namespace: @@ -32,7 +31,7 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def completion_callback(infer_request: InferRequest, image_path: str) -> None: +def completion_callback(infer_request: ov.InferRequest, image_path: str) -> None: predictions = next(iter(infer_request.results.values())) # Change a shape of a numpy.ndarray with results to get another one with one dimension @@ -61,7 +60,7 @@ def main() -> int: # --------------------------- Step 1. Initialize OpenVINO Runtime Core ------------------------------------------------ log.info('Creating OpenVINO Runtime Core') - core = Core() + core = ov.Core() # --------------------------- Step 2. Read a model -------------------------------------------------------------------- log.info(f'Reading the model: {args.model}') @@ -88,22 +87,22 @@ def main() -> int: input_tensors = [np.expand_dims(image, 0) for image in resized_images] # --------------------------- Step 4. Apply preprocessing ------------------------------------------------------------- - ppp = PrePostProcessor(model) + ppp = ov.preprocess.PrePostProcessor(model) # 1) Set input tensor information: # - input() provides information about a single model input # - precision of tensor is supposed to be 'u8' # - layout of data is 'NHWC' ppp.input().tensor() \ - .set_element_type(Type.u8) \ - .set_layout(Layout('NHWC')) # noqa: N400 + .set_element_type(ov.Type.u8) \ + .set_layout(ov.Layout('NHWC')) # noqa: N400 # 2) Here we suppose model has 'NCHW' layout for input - ppp.input().model().set_layout(Layout('NCHW')) + ppp.input().model().set_layout(ov.Layout('NCHW')) # 3) Set output tensor information: # - precision of tensor is supposed to be 'f32' - ppp.output().tensor().set_element_type(Type.f32) + ppp.output().tensor().set_element_type(ov.Type.f32) # 4) Apply preprocessing modifing the original 'model' model = ppp.build() @@ -115,7 +114,7 @@ def main() -> int: # --------------------------- Step 6. Create infer request queue ------------------------------------------------------ log.info('Starting inference in asynchronous mode') # create async queue with optimal number of infer requests - infer_queue = AsyncInferQueue(compiled_model) + infer_queue = ov.AsyncInferQueue(compiled_model) infer_queue.set_callback(completion_callback) # --------------------------- Step 7. Do inference -------------------------------------------------------------------- diff --git a/samples/python/hello_classification/hello_classification.py b/samples/python/hello_classification/hello_classification.py index 135422ef96c..9e391c32219 100755 --- a/samples/python/hello_classification/hello_classification.py +++ b/samples/python/hello_classification/hello_classification.py @@ -8,8 +8,7 @@ import sys import cv2 import numpy as np -from openvino.preprocess import PrePostProcessor, ResizeAlgorithm -from openvino.runtime import Core, Layout, Type +import openvino as ov def main(): @@ -26,7 +25,7 @@ def main(): # --------------------------- Step 1. Initialize OpenVINO Runtime Core ------------------------------------------------ log.info('Creating OpenVINO Runtime Core') - core = Core() + core = ov.Core() # --------------------------- Step 2. Read a model -------------------------------------------------------------------- log.info(f'Reading the model: {model_path}') @@ -48,7 +47,7 @@ def main(): input_tensor = np.expand_dims(image, 0) # --------------------------- Step 4. Apply preprocessing ------------------------------------------------------------- - ppp = PrePostProcessor(model) + ppp = ov.preprocess.PrePostProcessor(model) _, h, w, _ = input_tensor.shape @@ -58,19 +57,19 @@ def main(): # - layout of data is 'NHWC' ppp.input().tensor() \ .set_shape(input_tensor.shape) \ - .set_element_type(Type.u8) \ - .set_layout(Layout('NHWC')) # noqa: ECE001, N400 + .set_element_type(ov.Type.u8) \ + .set_layout(ov.Layout('NHWC')) # noqa: ECE001, N400 # 2) Adding explicit preprocessing steps: # - apply linear resize from tensor spatial dims to model spatial dims - ppp.input().preprocess().resize(ResizeAlgorithm.RESIZE_LINEAR) + ppp.input().preprocess().resize(ov.preprocess.ResizeAlgorithm.RESIZE_LINEAR) # 3) Here we suppose model has 'NCHW' layout for input - ppp.input().model().set_layout(Layout('NCHW')) + ppp.input().model().set_layout(ov.Layout('NCHW')) # 4) Set output tensor information: # - precision of tensor is supposed to be 'f32' - ppp.output().tensor().set_element_type(Type.f32) + ppp.output().tensor().set_element_type(ov.Type.f32) # 5) Apply preprocessing modifying the original 'model' model = ppp.build() diff --git a/samples/python/hello_query_device/hello_query_device.py b/samples/python/hello_query_device/hello_query_device.py index 5da48752427..8efadc1466e 100755 --- a/samples/python/hello_query_device/hello_query_device.py +++ b/samples/python/hello_query_device/hello_query_device.py @@ -5,7 +5,7 @@ import logging as log import sys -from openvino.runtime import Core +import openvino as ov def param_to_string(parameters) -> str: @@ -20,7 +20,7 @@ def main(): log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.INFO, stream=sys.stdout) # --------------------------- Step 1. Initialize OpenVINO Runtime Core -------------------------------------------- - core = Core() + core = ov.Core() # --------------------------- Step 2. Get metrics of available devices -------------------------------------------- log.info('Available devices:') diff --git a/samples/python/hello_reshape_ssd/hello_reshape_ssd.py b/samples/python/hello_reshape_ssd/hello_reshape_ssd.py index e2e81d67bcc..40d4afdbcf0 100755 --- a/samples/python/hello_reshape_ssd/hello_reshape_ssd.py +++ b/samples/python/hello_reshape_ssd/hello_reshape_ssd.py @@ -9,8 +9,7 @@ import sys import cv2 import numpy as np -from openvino.preprocess import PrePostProcessor -from openvino.runtime import Core, Layout, PartialShape, Type +import openvino as ov def main(): @@ -27,7 +26,7 @@ def main(): # --------------------------- Step 1. Initialize OpenVINO Runtime Core ------------------------------------------------ log.info('Creating OpenVINO Runtime Core') - core = Core() + core = ov.Core() # --------------------------- Step 2. Read a model -------------------------------------------------------------------- log.info(f'Reading the model: {model_path}') @@ -50,25 +49,25 @@ def main(): log.info('Reshaping the model to the height and width of the input image') n, h, w, c = input_tensor.shape - model.reshape({model.input().get_any_name(): PartialShape((n, c, h, w))}) + model.reshape({model.input().get_any_name(): ov.PartialShape((n, c, h, w))}) # --------------------------- Step 4. Apply preprocessing ------------------------------------------------------------- - ppp = PrePostProcessor(model) + ppp = ov.preprocess.PrePostProcessor(model) # 1) Set input tensor information: # - input() provides information about a single model input # - precision of tensor is supposed to be 'u8' # - layout of data is 'NHWC' ppp.input().tensor() \ - .set_element_type(Type.u8) \ - .set_layout(Layout('NHWC')) # noqa: N400 + .set_element_type(ov.Type.u8) \ + .set_layout(ov.Layout('NHWC')) # noqa: N400 # 2) Here we suppose model has 'NCHW' layout for input - ppp.input().model().set_layout(Layout('NCHW')) + ppp.input().model().set_layout(ov.Layout('NCHW')) # 3) Set output tensor information: # - precision of tensor is supposed to be 'f32' - ppp.output().tensor().set_element_type(Type.f32) + ppp.output().tensor().set_element_type(ov.Type.f32) # 4) Apply preprocessing modifing the original 'model' model = ppp.build() diff --git a/samples/python/model_creation_sample/model_creation_sample.py b/samples/python/model_creation_sample/model_creation_sample.py index 27792fef5c0..d76d222c5f8 100755 --- a/samples/python/model_creation_sample/model_creation_sample.py +++ b/samples/python/model_creation_sample/model_creation_sample.py @@ -8,14 +8,13 @@ import typing from functools import reduce import numpy as np -from openvino.preprocess import PrePostProcessor -from openvino.runtime import (Core, Layout, Model, Shape, Type, op, opset1, - opset8, set_batch) +import openvino as ov +from openvino.runtime import op, opset1, opset8 from data import digits -def create_ngraph_function(model_path: str) -> Model: +def create_ngraph_function(model_path: str) -> ov.Model: """Create a model on the fly from the source code using ngraph.""" def shape_and_length(shape: list) -> typing.Tuple[list, int]: @@ -28,17 +27,17 @@ def create_ngraph_function(model_path: str) -> Model: # input input_shape = [64, 1, 28, 28] - param_node = op.Parameter(Type.f32, Shape(input_shape)) + param_node = op.Parameter(ov.Type.f32, ov.Shape(input_shape)) # convolution 1 conv_1_kernel_shape, conv_1_kernel_length = shape_and_length([20, 1, 5, 5]) - conv_1_kernel = op.Constant(Type.f32, Shape(conv_1_kernel_shape), weights[0:conv_1_kernel_length].tolist()) + conv_1_kernel = op.Constant(ov.Type.f32, ov.Shape(conv_1_kernel_shape), weights[0:conv_1_kernel_length].tolist()) weights_offset += conv_1_kernel_length conv_1_node = opset8.convolution(param_node, conv_1_kernel, [1, 1], padding_begin, padding_end, [1, 1]) # add 1 add_1_kernel_shape, add_1_kernel_length = shape_and_length([1, 20, 1, 1]) - add_1_kernel = op.Constant(Type.f32, Shape(add_1_kernel_shape), + add_1_kernel = op.Constant(ov.Type.f32, ov.Shape(add_1_kernel_shape), weights[weights_offset : weights_offset + add_1_kernel_length]) weights_offset += add_1_kernel_length add_1_node = opset8.add(conv_1_node, add_1_kernel) @@ -48,7 +47,7 @@ def create_ngraph_function(model_path: str) -> Model: # convolution 2 conv_2_kernel_shape, conv_2_kernel_length = shape_and_length([50, 20, 5, 5]) - conv_2_kernel = op.Constant(Type.f32, Shape(conv_2_kernel_shape), + conv_2_kernel = op.Constant(ov.Type.f32, ov.Shape(conv_2_kernel_shape), weights[weights_offset : weights_offset + conv_2_kernel_length], ) weights_offset += conv_2_kernel_length @@ -56,7 +55,7 @@ def create_ngraph_function(model_path: str) -> Model: # add 2 add_2_kernel_shape, add_2_kernel_length = shape_and_length([1, 50, 1, 1]) - add_2_kernel = op.Constant(Type.f32, Shape(add_2_kernel_shape), + add_2_kernel = op.Constant(ov.Type.f32, ov.Shape(add_2_kernel_shape), weights[weights_offset : weights_offset + add_2_kernel_length], ) weights_offset += add_2_kernel_length @@ -72,13 +71,13 @@ def create_ngraph_function(model_path: str) -> Model: weights[weights_offset : weights_offset + 2 * reshape_1_length], dtype=np.int64, ) - reshape_1_kernel = op.Constant(Type.i64, Shape(list(dtype_weights.shape)), dtype_weights) + reshape_1_kernel = op.Constant(ov.Type.i64, ov.Shape(list(dtype_weights.shape)), dtype_weights) weights_offset += 2 * reshape_1_length reshape_1_node = opset8.reshape(maxpool_2_node, reshape_1_kernel, True) # matmul 1 matmul_1_kernel_shape, matmul_1_kernel_length = shape_and_length([500, 800]) - matmul_1_kernel = op.Constant(Type.f32, Shape(matmul_1_kernel_shape), + matmul_1_kernel = op.Constant(ov.Type.f32, ov.Shape(matmul_1_kernel_shape), weights[weights_offset : weights_offset + matmul_1_kernel_length], ) weights_offset += matmul_1_kernel_length @@ -86,7 +85,7 @@ def create_ngraph_function(model_path: str) -> Model: # add 3 add_3_kernel_shape, add_3_kernel_length = shape_and_length([1, 500]) - add_3_kernel = op.Constant(Type.f32, Shape(add_3_kernel_shape), + add_3_kernel = op.Constant(ov.Type.f32, ov.Shape(add_3_kernel_shape), weights[weights_offset : weights_offset + add_3_kernel_length], ) weights_offset += add_3_kernel_length @@ -96,12 +95,12 @@ def create_ngraph_function(model_path: str) -> Model: relu_node = opset8.relu(add_3_node) # reshape 2 - reshape_2_kernel = op.Constant(Type.i64, Shape(list(dtype_weights.shape)), dtype_weights) + reshape_2_kernel = op.Constant(ov.Type.i64, ov.Shape(list(dtype_weights.shape)), dtype_weights) reshape_2_node = opset8.reshape(relu_node, reshape_2_kernel, True) # matmul 2 matmul_2_kernel_shape, matmul_2_kernel_length = shape_and_length([10, 500]) - matmul_2_kernel = op.Constant(Type.f32, Shape(matmul_2_kernel_shape), + matmul_2_kernel = op.Constant(ov.Type.f32, ov.Shape(matmul_2_kernel_shape), weights[weights_offset : weights_offset + matmul_2_kernel_length], ) weights_offset += matmul_2_kernel_length @@ -109,7 +108,7 @@ def create_ngraph_function(model_path: str) -> Model: # add 4 add_4_kernel_shape, add_4_kernel_length = shape_and_length([1, 10]) - add_4_kernel = op.Constant(Type.f32, Shape(add_4_kernel_shape), + add_4_kernel = op.Constant(ov.Type.f32, ov.Shape(add_4_kernel_shape), weights[weights_offset : weights_offset + add_4_kernel_length], ) weights_offset += add_4_kernel_length @@ -119,7 +118,7 @@ def create_ngraph_function(model_path: str) -> Model: softmax_axis = 1 softmax_node = opset8.softmax(add_4_node, softmax_axis) - return Model(softmax_node, [param_node], 'lenet') + return ov.Model(softmax_node, [param_node], 'lenet') def main(): @@ -135,35 +134,35 @@ def main(): number_top = 1 # ---------------------------Step 1. Initialize OpenVINO Runtime Core-------------------------------------------------- log.info('Creating OpenVINO Runtime Core') - core = Core() # ---------------------------Step 2. Read a model in OpenVINO Intermediate Representation------------------------------ log.info(f'Loading the model using ngraph function with weights from {model_path}') model = create_ngraph_function(model_path) # ---------------------------Step 3. Apply preprocessing---------------------------------------------------------- # Get names of input and output blobs - ppp = PrePostProcessor(model) + ppp = ov.preprocess.PrePostProcessor(model) # 1) Set input tensor information: # - input() provides information about a single model input # - precision of tensor is supposed to be 'u8' # - layout of data is 'NHWC' ppp.input().tensor() \ - .set_element_type(Type.u8) \ - .set_layout(Layout('NHWC')) # noqa: N400 + .set_element_type(ov.Type.u8) \ + .set_layout(ov.Layout('NHWC')) # noqa: N400 # 2) Here we suppose model has 'NCHW' layout for input - ppp.input().model().set_layout(Layout('NCHW')) + ppp.input().model().set_layout(ov.Layout('NCHW')) # 3) Set output tensor information: # - precision of tensor is supposed to be 'f32' - ppp.output().tensor().set_element_type(Type.f32) + ppp.output().tensor().set_element_type(ov.Type.f32) # 4) Apply preprocessing modifing the original 'model' model = ppp.build() # Set a batch size equal to number of input images - set_batch(model, digits.shape[0]) + ov.set_batch(model, digits.shape[0]) # ---------------------------Step 4. Loading model to the device------------------------------------------------------- log.info('Loading the model to the plugin') + core = ov.Core() compiled_model = core.compile_model(model, device_name) # ---------------------------Step 5. Prepare input--------------------------------------------------------------------- diff --git a/samples/python/speech_sample/speech_sample.py b/samples/python/speech_sample/speech_sample.py index 8c2352defaa..e9ac1a821dc 100755 --- a/samples/python/speech_sample/speech_sample.py +++ b/samples/python/speech_sample/speech_sample.py @@ -9,8 +9,7 @@ from timeit import default_timer from typing import Dict import numpy as np -from openvino.preprocess import PrePostProcessor -from openvino.runtime import Core, InferRequest, Layout, Type, set_batch +import openvino as ov from arg_parser import parse_args from file_options import read_utterance_file, write_utterance_file @@ -20,7 +19,7 @@ from utils import (GNA_ATOM_FREQUENCY, GNA_CORE_FREQUENCY, set_scale_factors) -def do_inference(data: Dict[str, np.ndarray], infer_request: InferRequest, cw_l: int = 0, cw_r: int = 0) -> np.ndarray: +def do_inference(data: Dict[str, np.ndarray], infer_request: ov.InferRequest, cw_l: int = 0, cw_r: int = 0) -> np.ndarray: """Do a synchronous matrix inference.""" frames_to_infer = {} result = {} @@ -69,7 +68,7 @@ def main(): # --------------------------- Step 1. Initialize OpenVINO Runtime Core ------------------------------------------------ log.info('Creating OpenVINO Runtime Core') - core = Core() + core = ov.Core() # --------------------------- Step 2. Read a model -------------------------------------------------------------------- if args.model: @@ -83,19 +82,19 @@ def main(): if args.layout: layouts = get_input_layouts(args.layout, model.inputs) - ppp = PrePostProcessor(model) + ppp = ov.preprocess.PrePostProcessor(model) for i in range(len(model.inputs)): - ppp.input(i).tensor().set_element_type(Type.f32) + ppp.input(i).tensor().set_element_type(ov.Type.f32) input_name = model.input(i).get_any_name() if args.layout and input_name in layouts.keys(): - ppp.input(i).tensor().set_layout(Layout(layouts[input_name])) - ppp.input(i).model().set_layout(Layout(layouts[input_name])) + ppp.input(i).tensor().set_layout(ov.Layout(layouts[input_name])) + ppp.input(i).model().set_layout(ov.Layout(layouts[input_name])) for i in range(len(model.outputs)): - ppp.output(i).tensor().set_element_type(Type.f32) + ppp.output(i).tensor().set_element_type(ov.Type.f32) model = ppp.build() @@ -103,7 +102,7 @@ def main(): batch_size = args.batch_size if args.context_window_left == args.context_window_right == 0 else 1 if any((not _input.node.layout.empty for _input in model.inputs)): - set_batch(model, batch_size) + ov.set_batch(model, batch_size) else: log.warning('Layout is not set for any input, so custom batch size is not set') diff --git a/src/bindings/python/docs/test_examples.md b/src/bindings/python/docs/test_examples.md index 5855f110fa2..80808fdbfd5 100644 --- a/src/bindings/python/docs/test_examples.md +++ b/src/bindings/python/docs/test_examples.md @@ -107,7 +107,7 @@ Let's add a test case for new class. Start with imports and simple test of the c ```python import pytest import numpy as np -import openvino.runtime as ov +import openvino as ov def test_mytensor_creation(): tensor = ov.MyTensor([1, 2, 3])