IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into itikhono/ts/slice

Browse Source
This commit is contained in:
Ivan
2023-03-14 16:20:34 +04:00
parent 54bf0444e4 3a96e06d4c
commit 6e6361f83a
390 changed files with 9956 additions and 5373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.ci/azure/android_arm64.yml
View File

@@ -158,6 +158,7 @@ jobs:
-DENABLE_TESTS=ON
-DBUILD_java_api=ON
-DBUILD_nvidia_plugin=OFF
-DBUILD_custom_operations=OFF
-DENABLE_INTEL_GPU=ON
-DOPENVINO_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)/modules
-DCMAKE_CXX_LINKER_LAUNCHER=ccache

14
.ci/azure/linux.yml
View File

@@ -163,7 +163,7 @@ jobs:
set -e
sudo -E $(REPO_DIR)/install_build_dependencies.sh
# Move jdk into contrib
# 'clang' compiler is to check that samples can be built using it
# 'clang' compiler is used as a default compiler
sudo apt --assume-yes install openjdk-11-jdk libbz2-dev clang
# For Python API
python3 -m pip install --upgrade pip
@@ -236,10 +236,13 @@ jobs:
-DENABLE_FASTER_BUILD=ON
-DENABLE_STRICT_DEPENDENCIES=OFF
-DOPENVINO_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)/modules
-DCUSTOM_OPERATIONS="calculate_grid;complex_mul;fft;grid_sample;sparse_conv;sparse_conv_transpose"
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-DCMAKE_C_COMPILER_LAUNCHER=ccache
-DCMAKE_CXX_LINKER_LAUNCHER=ccache
-DCMAKE_C_LINKER_LAUNCHER=ccache
-DCMAKE_CXX_COMPILER=clang++
-DCMAKE_C_COMPILER=clang
-DCPACK_GENERATOR=$(CMAKE_CPACK_GENERATOR)
-DBUILD_nvidia_plugin=OFF
-S $(REPO_DIR)
@@ -300,7 +303,7 @@ jobs:
# Skip test_onnx/test_zoo_models and test_onnx/test_backend due to long execution time
- script: |
export LD_LIBRARY_PATH=$(REPO_DIR)/temp/gna_03.00.00.1910/linux/x64:$(LD_LIBRARY_PATH)
export LD_LIBRARY_PATH=$(REPO_DIR)/temp/gna_03.05.00.1906/linux/x64:$(LD_LIBRARY_PATH)
python3 -m pytest -s $(INSTALL_TEST_DIR)/pyngraph $(PYTHON_STATIC_ARGS) \
--junitxml=$(INSTALL_TEST_DIR)/TEST-Pyngraph.xml \
--ignore=$(INSTALL_TEST_DIR)/pyngraph/tests/test_onnx/test_zoo_models.py \
@@ -310,7 +313,7 @@ jobs:
# Skip test_onnx/test_zoo_models and test_onnx/test_backend due to long execution time
- script: |
# For python imports to import pybind_mock_frontend
export LD_LIBRARY_PATH=$(REPO_DIR)/temp/gna_03.00.00.1910/linux/x64:$(LD_LIBRARY_PATH)
export LD_LIBRARY_PATH=$(REPO_DIR)/temp/gna_03.05.00.1906/linux/x64:$(LD_LIBRARY_PATH)
export PYTHONPATH=$(INSTALL_TEST_DIR):$(INSTALL_DIR)/python/python3.8:$PYTHONPATH
python3 -m pytest -sv $(INSTALL_TEST_DIR)/pyopenvino $(PYTHON_STATIC_ARGS) \
--junitxml=$(INSTALL_TEST_DIR)/TEST-Pyngraph.xml \
@@ -320,7 +323,7 @@ jobs:
displayName: 'Python API 2.0 Tests'
- script: |
export LD_LIBRARY_PATH=$(REPO_DIR)/temp/gna_03.00.00.1910/linux/x64:$(LD_LIBRARY_PATH)
export LD_LIBRARY_PATH=$(REPO_DIR)/temp/gna_03.05.00.1906/linux/x64:$(LD_LIBRARY_PATH)
python3 -m pytest -s $(INSTALL_TEST_DIR)/mo/unit_tests --junitxml=$(INSTALL_TEST_DIR)/TEST-ModelOptimizer.xml
displayName: 'Model Optimizer UT'
@@ -358,7 +361,7 @@ jobs:
displayName: 'List install files'
- script: $(SAMPLES_INSTALL_DIR)/cpp/build_samples.sh -i $(INSTALL_DIR) -b $(BUILD_DIR)/cpp_samples
displayName: 'Build cpp samples'
displayName: 'Build cpp samples - gcc'
- script: $(SAMPLES_INSTALL_DIR)/cpp/build_samples.sh -b $(BUILD_DIR)/cpp_samples_clang
env:
@@ -427,6 +430,7 @@ jobs:
- script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_gna_unit_tests --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-ov_gna_unit_tests.xml
displayName: 'GNA UT'
enabled: 'false' # TODO: fix
- script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ieMultiPluginUnitTests --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-ieMultiPluginUnitTests.xml
displayName: 'MULTI UT'

10
.ci/azure/linux_debian.yml
View File

@@ -163,8 +163,6 @@ jobs:
-DCMAKE_C_COMPILER_LAUNCHER=ccache
-DCMAKE_CXX_LINKER_LAUNCHER=ccache
-DCMAKE_C_LINKER_LAUNCHER=ccache
-DCMAKE_CXX_COMPILER=clang++
-DCMAKE_C_COMPILER=clang
-DCPACK_GENERATOR=DEB
-S $(REPO_DIR)
-B $(BUILD_DIR)
@@ -283,7 +281,13 @@ jobs:
displayName: 'Clean build dir'
- script: $(SAMPLES_INSTALL_DIR)/cpp/build_samples.sh -i $(INSTALL_DIR)
displayName: 'Build cpp samples'
displayName: 'Build cpp samples - gcc'
- script: $(SAMPLES_INSTALL_DIR)/cpp/build_samples.sh -i $(INSTALL_DIR)
displayName: 'Build cpp samples - clang'
env:
CC: clang
CXX: clang++
- script: $(SAMPLES_INSTALL_DIR)/c/build_samples.sh -i $(INSTALL_DIR)
displayName: 'Build c samples'

1
.ci/azure/windows.yml
View File

@@ -176,6 +176,7 @@ jobs:
-DENABLE_STRICT_DEPENDENCIES=OFF ^
-DENABLE_PYTHON=ON ^
-DBUILD_nvidia_plugin=OFF ^
-DCUSTOM_OPERATIONS="calculate_grid;complex_mul;fft;grid_sample;sparse_conv;sparse_conv_transpose" ^
-DPYTHON_EXECUTABLE="C:\hostedtoolcache\windows\Python\3.10.7\x64\python.exe" ^
-DPYTHON_INCLUDE_DIR="C:\hostedtoolcache\windows\Python\3.10.7\x64\include" ^
-DPYTHON_LIBRARY="C:\hostedtoolcache\windows\Python\3.10.7\x64\libs\python310.lib" ^

2
README.md
View File

@@ -163,7 +163,7 @@ The system requirements vary depending on platform and are available on dedicate
## How to build
See the [OpenVINO Wiki](https://github.com/openvinotoolkit/openvino/wiki#how-to-build) to get more information about the OpenVINO build process.
See [How to build OpenVINO](./docs/dev/build.md) to get more information about the OpenVINO build process.
## How to contribute

4
cmake/dependencies.cmake
View File

@@ -298,8 +298,8 @@ if(ENABLE_INTEL_GNA)
GNA_LIB_DIR
libGNA_INCLUDE_DIRS
libGNA_LIBRARIES_BASE_PATH)
set(GNA_VERSION "03.00.00.1910")
set(GNA_HASH "894ddbc0ae3459f04513b853b0cabc32890dd4ea37228a022b6a32101bdbb7f8")
set(GNA_VERSION "03.05.00.1906")
set(GNA_HASH "4a5be86d9c026b0e10afac2a57fc7c99d762b30e3d506abb3a3380fbcfe2726e")
set(FILES_TO_EXTRACT_LIST gna_${GNA_VERSION}/include)
if(WIN32)

10
cmake/developer_package/clang_format/clang_format.cmake
View File

@@ -3,23 +3,23 @@
#
if(ENABLE_CLANG_FORMAT)
set(clang_format_required_version 9)
set(CLANG_FORMAT_FILENAME clang-format-${clang_format_required_version} clang-format)
set(CLANG_FORMAT_REQUIRED_VERSION 9 CACHE STRING "Clang-format version to use")
set(CLANG_FORMAT_FILENAME clang-format-${CLANG_FORMAT_REQUIRED_VERSION} clang-format)
find_host_program(CLANG_FORMAT NAMES ${CLANG_FORMAT_FILENAME} PATHS ENV PATH)
if(CLANG_FORMAT)
execute_process(COMMAND ${CLANG_FORMAT} ${CMAKE_CURRENT_SOURCE_DIR} ARGS --version OUTPUT_VARIABLE CLANG_VERSION)
if(NOT CLANG_VERSION)
message(WARNING "Supported clang-format version is ${clang_format_required_version}!")
message(WARNING "Supported clang-format version is ${CLANG_FORMAT_REQUIRED_VERSION}!")
set(ENABLE_CLANG_FORMAT OFF)
else()
string(REGEX REPLACE "[^0-9]+([0-9]+)\\..*" "\\1" CLANG_FORMAT_MAJOR_VERSION ${CLANG_VERSION})
if(NOT CLANG_FORMAT_MAJOR_VERSION EQUAL clang_format_required_version)
if(NOT CLANG_FORMAT_MAJOR_VERSION EQUAL CLANG_FORMAT_REQUIRED_VERSION)
message(WARNING "Supported clang-format version is 9! Provided version ${CLANG_FORMAT_MAJOR_VERSION}")
set(ENABLE_CLANG_FORMAT OFF)
endif()
endif()
else()
message(WARNING "Supported clang-format-${clang_format_required_version} is not found!")
message(WARNING "Supported clang-format-${CLANG_FORMAT_REQUIRED_VERSION} is not found!")
set(ENABLE_CLANG_FORMAT OFF)
endif()
endif()

2
cmake/developer_package/compile_flags/sanitizer.cmake
View File

@@ -33,7 +33,7 @@ if (ENABLE_UB_SANITIZER)
# https://github.com/KhronosGroup/OpenCL-CLHPP/issues/17
# Mute -fsanitize=function Indirect call of a function through a function pointer of the wrong type.
# Sample cases:
# call to function GetAPIVersion through pointer to incorrect function type 'void *(*)()'
# call to function get_api_version through pointer to incorrect function type 'void *(*)()'
# Mute -fsanitize=alignment Use of a misaligned pointer or creation of a misaligned reference. Also sanitizes assume_aligned-like attributes.
# Sample cases:
# VPU_FixedMaxHeapTest.DefaultConstructor test case load of misaligned address 0x62000000187f for type 'const DataType', which requires 4 byte alignment

4
cmake/developer_package/frontends/create_frontends_hpp.cmake
View File

@@ -15,8 +15,8 @@ set(OV_FRONTEND_MAP_DEFINITION " FrontendsStaticRegistry registry = {")
foreach(frontend IN LISTS FRONTEND_NAMES)
# common
set(_OV_FRONTEND_DATA_FUNC "GetFrontEndData${frontend}")
set(_OV_VERSION_FUNC "GetAPIVersion${frontend}")
set(_OV_FRONTEND_DATA_FUNC "get_front_end_data_${frontend}")
set(_OV_VERSION_FUNC "get_api_version_${frontend}")
# declarations
set(OV_FRONTEND_DECLARATIONS "${OV_FRONTEND_DECLARATIONS}

32
cmake/developer_package/frontends/frontends.cmake
View File

@@ -190,21 +190,8 @@ macro(ov_add_frontend)
if(NOT BUILD_SHARED_LIBS)
# override default function names
target_compile_definitions(${TARGET_NAME} PRIVATE
"-DGetFrontEndData=GetFrontEndData${OV_FRONTEND_NAME}"
"-DGetAPIVersion=GetAPIVersion${OV_FRONTEND_NAME}")
endif()
# enable LTO
set_target_properties(${TARGET_NAME} PROPERTIES
INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO})
if(OV_FRONTEND_SKIP_NCC_STYLE)
# frontend's CMakeLists.txt must define its own custom 'ov_ncc_naming_style' step
else()
ov_ncc_naming_style(FOR_TARGET ${TARGET_NAME}
SOURCE_DIRECTORY "${frontend_root_dir}/include"
ADDITIONAL_INCLUDE_DIRECTORIES
$<TARGET_PROPERTY:frontend_common::static,INTERFACE_INCLUDE_DIRECTORIES>)
"-Dget_front_end_data=get_front_end_data_${OV_FRONTEND_NAME}"
"-Dget_api_version=get_api_version_${OV_FRONTEND_NAME}")
endif()
target_include_directories(${TARGET_NAME}
@@ -255,6 +242,21 @@ macro(ov_add_frontend)
add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME}
EXCLUDE_PATTERNS ${PROTO_SRCS} ${PROTO_HDRS} ${proto_files} ${flatbuffers_schema_files})
# enable LTO
set_target_properties(${TARGET_NAME} PROPERTIES
INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO})
if(OV_FRONTEND_SKIP_NCC_STYLE)
# frontend's CMakeLists.txt must define its own custom 'ov_ncc_naming_style' step
else()
ov_ncc_naming_style(FOR_TARGET ${TARGET_NAME}
SOURCE_DIRECTORIES "${frontend_root_dir}/include"
"${frontend_root_dir}/src"
ADDITIONAL_INCLUDE_DIRECTORIES
$<TARGET_PROPERTY:${TARGET_NAME},INTERFACE_INCLUDE_DIRECTORIES>
$<TARGET_PROPERTY:${TARGET_NAME},INCLUDE_DIRECTORIES>)
endif()
add_dependencies(ov_frontends ${TARGET_NAME})
# must be called after all target_link_libraries

8
cmake/developer_package/frontends/ov_frontends.hpp.in
View File

@@ -10,12 +10,12 @@
namespace {
using GetFrontEndDataFunc = void*();
using GetAPIVersionFunc = ov::frontend::FrontEndVersion();
using get_front_end_data_func = void*();
using get_api_version_func = ov::frontend::FrontEndVersion();
struct Value {
GetFrontEndDataFunc* m_dataFunc;
GetAPIVersionFunc* m_versionFunc;
get_front_end_data_func* m_dataFunc;
get_api_version_func* m_versionFunc;
};
using FrontendsStaticRegistry = std::vector<Value>;

36
cmake/developer_package/ncc_naming_style/ncc_naming_style.cmake
View File

@@ -112,13 +112,13 @@ endif()
#
# ov_ncc_naming_style(FOR_TARGET target_name
# SOURCE_DIRECTORY dir
# [SOURCE_DIRECTORIES dir1 dir2 ...]
# [STYLE_FILE style_file.style]
# [ADDITIONAL_INCLUDE_DIRECTORIES dir1 dir2 ..]
# [DEFINITIONS def1 def2 ..])
#
# FOR_TARGET - name of the target
# SOURCE_DIRECTORY - directory to check sources from
# SOURCE_DIRECTORIES - directory to check sources from
# STYLE_FILE - path to the specific style file
# ADDITIONAL_INCLUDE_DIRECTORIES - additional include directories used in checked headers
# DEFINITIONS - additional definitions passed to preprocessor stage
@@ -129,9 +129,9 @@ function(ov_ncc_naming_style)
endif()
cmake_parse_arguments(NCC_STYLE "FAIL"
"FOR_TARGET;SOURCE_DIRECTORY;STYLE_FILE" "ADDITIONAL_INCLUDE_DIRECTORIES;DEFINITIONS" ${ARGN})
"FOR_TARGET;STYLE_FILE" "SOURCE_DIRECTORIES;ADDITIONAL_INCLUDE_DIRECTORIES;DEFINITIONS" ${ARGN})
foreach(var FOR_TARGET SOURCE_DIRECTORY)
foreach(var FOR_TARGET SOURCE_DIRECTORIES)
if(NOT DEFINED NCC_STYLE_${var})
message(FATAL_ERROR "${var} is not defined in ov_ncc_naming_style function")
endif()
@@ -141,18 +141,18 @@ function(ov_ncc_naming_style)
set(NCC_STYLE_STYLE_FILE ${ncc_style_dir}/openvino.style)
endif()
file(GLOB_RECURSE sources
RELATIVE "${NCC_STYLE_SOURCE_DIRECTORY}"
"${NCC_STYLE_SOURCE_DIRECTORY}/*.hpp"
"${NCC_STYLE_SOURCE_DIRECTORY}/*.cpp")
foreach(source_dir IN LISTS NCC_STYLE_SOURCE_DIRECTORIES)
file(GLOB_RECURSE local_sources "${source_dir}/*.hpp" "${source_dir}/*.cpp")
list(APPEND sources ${local_sources})
endforeach()
list(APPEND NCC_STYLE_ADDITIONAL_INCLUDE_DIRECTORIES "${NCC_STYLE_SOURCE_DIRECTORY}")
# without it sources with same name from different directories will map to same .ncc_style target
file(RELATIVE_PATH source_dir_rel ${CMAKE_SOURCE_DIR} ${NCC_STYLE_SOURCE_DIRECTORY})
list(APPEND NCC_STYLE_ADDITIONAL_INCLUDE_DIRECTORIES ${NCC_STYLE_SOURCE_DIRECTORIES})
foreach(source IN LISTS sources)
set(output_file "${ncc_style_bin_dir}/${source_dir_rel}/${source}.ncc_style")
set(full_source_path "${NCC_STYLE_SOURCE_DIRECTORY}/${source}")
foreach(source_file IN LISTS sources)
get_filename_component(source_dir "${source_file}" DIRECTORY)
file(RELATIVE_PATH source_dir_rel "${CMAKE_SOURCE_DIR}" "${source_dir}")
get_filename_component(source_name "${source_file}" NAME)
set(output_file "${ncc_style_bin_dir}/${source_dir_rel}/${source_name}.ncc_style")
add_custom_command(
OUTPUT
@@ -161,7 +161,7 @@ function(ov_ncc_naming_style)
"${CMAKE_COMMAND}"
-D "PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}"
-D "NCC_PY_SCRIPT=${ncc_script_py}"
-D "INPUT_FILE=${full_source_path}"
-D "INPUT_FILE=${source_file}"
-D "OUTPUT_FILE=${output_file}"
-D "DEFINITIONS=${NCC_STYLE_DEFINITIONS}"
-D "CLANG_LIB_PATH=${libclang_location}"
@@ -170,12 +170,12 @@ function(ov_ncc_naming_style)
-D "EXPECTED_FAIL=${NCC_STYLE_FAIL}"
-P "${ncc_style_dir}/ncc_run.cmake"
DEPENDS
"${full_source_path}"
"${source_file}"
"${ncc_style_dir}/openvino.style"
"${ncc_script_py}"
"${ncc_style_dir}/ncc_run.cmake"
COMMENT
"[ncc naming style] ${source}"
"[ncc naming style] ${source_dir_rel}/${source_name}"
VERBATIM)
list(APPEND output_files ${output_file})
endforeach()
@@ -191,6 +191,6 @@ endfunction()
if(TARGET ncc_all)
ov_ncc_naming_style(FOR_TARGET ncc_all
SOURCE_DIRECTORY "${ncc_style_dir}/self_check"
SOURCE_DIRECTORIES "${ncc_style_dir}/self_check"
FAIL)
endif()

4
cmake/developer_package/ncc_naming_style/openvino.style
View File

@@ -1,7 +1,7 @@
# custom OpenVINO values
CppMethod: '^(operator\W+|[a-z_\d]+|signaling_NaN|quiet_NaN)$'
ClassName: '^([A-Z][\w]+|b?float16|numeric_limits|ngraph_error|stopwatch|unsupported_op)$'
StructName: '^([A-Z][\w]+|element_type_traits|hash|oi_pair)$'
StructName: '^([A-Z][\w]+|element_type_traits|hash|oi_pair|stat)$'
FunctionName: '^(operator\W+|[a-z_\d]+)|PrintTo$'
Namespace: '^([a-z\d_]*|InferenceEngine)$'
NamespaceAlias: '^([a-z\d_]+|InferenceEngine)$'
@@ -27,7 +27,7 @@ CxxDynamicCastExpression: '^.*$'
# not needed values
ClassTemplatePartialSpecialization: '^.*$'
ConversionFunction: '^.*$'
UsingDirective: 'XXXX'
UsingDirective: '^.*$'
ClassAccessSpecifier: '^.*$' # looks like can be fixed
TypeReference: '^.*$' # looks like can be fixed
CxxBaseSpecifier: '^.*$' # looks like can be fixed

51
docs/Documentation/openvino_ecosystem_ovtf.md
View File

@@ -1,42 +1,55 @@
# OpenVINO™ integration with TensorFlow {#ovtf_integration}
@sphinxdirective
**OpenVINO™ integration with TensorFlow** is a solution for TensorFlow developers who want to get started with OpenVINO™ in their inferencing applications. By adding just two lines of code you can now take advantage of OpenVINO™ toolkit optimizations with TensorFlow inference applications across a range of Intel® computation devices.
This is all you need:
```bash
import openvino_tensorflow
openvino_tensorflow.set_backend('<backend_name>')
```
.. code-block:: bash
import openvino_tensorflow
openvino_tensorflow.set_backend('<backend_name>')
**OpenVINO™ integration with TensorFlow** accelerates inference across many AI models on a variety of Intel® technologies, such as:
- Intel® CPUs
- Intel® integrated GPUs
> **NOTE**: For maximum performance, efficiency, tooling customization, and hardware control, we recommend developers to adopt native OpenVINO™ solutions.
To find out more about the product itself, as well as learn how to use it in your project, check its dedicated [GitHub repository](https://github.com/openvinotoolkit/openvino_tensorflow/tree/master/docs).
* Intel® CPUs
* Intel® integrated GPUs
.. note::
For maximum performance, efficiency, tooling customization, and hardware control, we recommend developers to adopt native OpenVINO™ solutions.
To find out more about the product itself, as well as learn how to use it in your project, check its dedicated `GitHub repository <https://github.com/openvinotoolkit/openvino_tensorflow/tree/master/docs>`__.
To see what you can do with **OpenVINO™ integration with TensorFlow**, explore the demos located in the [examples folder](https://github.com/openvinotoolkit/openvino_tensorflow/tree/master/examples) in our GitHub repository.
To see what you can do with **OpenVINO™ integration with TensorFlow**, explore the demos located in the `examples folder <https://github.com/openvinotoolkit/openvino_tensorflow/tree/master/examples>`__ in our GitHub repository.
Sample tutorials are also hosted on [Intel® DevCloud](https://www.intel.com/content/www/us/en/developer/tools/devcloud/edge/build/ovtfoverview.html). The demo applications are implemented using Jupyter Notebooks. You can interactively execute them on Intel® DevCloud nodes, compare the results of **OpenVINO™ integration with TensorFlow**, native TensorFlow, and OpenVINO™.
Sample tutorials are also hosted on `Intel® DevCloud <https://www.intel.com/content/www/us/en/developer/tools/devcloud/edge/build/ovtfoverview.html>`__. The demo applications are implemented using Jupyter Notebooks. You can interactively execute them on Intel® DevCloud nodes, compare the results of **OpenVINO™ integration with TensorFlow**, native TensorFlow, and OpenVINO™.
## License
**OpenVINO™ integration with TensorFlow** is licensed under [Apache License Version 2.0](https://github.com/openvinotoolkit/openvino_tensorflow/blob/master/LICENSE).
License
#######
**OpenVINO™ integration with TensorFlow** is licensed under `Apache License Version 2.0 <https://github.com/openvinotoolkit/openvino_tensorflow/blob/master/LICENSE>`__.
By contributing to the project, you agree to the license and copyright terms therein
and release your contribution under these terms.
## Support
Support
#######
Submit your questions, feature requests and bug reports via [GitHub issues](https://github.com/openvinotoolkit/openvino_tensorflow/issues).
Submit your questions, feature requests and bug reports via `GitHub issues <https://github.com/openvinotoolkit/openvino_tensorflow/issues>`__.
## How to Contribute
How to Contribute
#################
We welcome community contributions to **OpenVINO™ integration with TensorFlow**. If you have an idea for improvement:
* Share your proposal via [GitHub issues](https://github.com/openvinotoolkit/openvino_tensorflow/issues).
* Submit a [pull request](https://github.com/openvinotoolkit/openvino_tensorflow/pulls).
* Share your proposal via `GitHub issues <https://github.com/openvinotoolkit/openvino_tensorflow/issues>`__.
* Submit a `pull request <https://github.com/openvinotoolkit/openvino_tensorflow/pulls>`__.
We will review your contribution as soon as possible. If any additional fixes or modifications are necessary, we will guide you and provide feedback. Before you make your contribution, make sure you can build **OpenVINO™ integration with TensorFlow** and run all the examples with your fix/patch. If you want to introduce a large feature, create test cases for your feature. Upon our verification of your pull request, we will merge it to the repository provided that the pull request has met the above mentioned requirements and proved acceptable.
---
\* Other names and brands may be claimed as the property of others.
\* Other names and brands may be claimed as the property of others.
@endsphinxdirective

22
docs/IE_PLUGIN_DG/Intro.md
View File

@@ -1,4 +1,4 @@
# Overview of Inference Engine Plugin Library {#openvino_docs_ie_plugin_dg_overview}
# Overview of OpenVINO Plugin Library {#openvino_docs_ie_plugin_dg_overview}
@sphinxdirective
@@ -18,23 +18,23 @@
@endsphinxdirective
The plugin architecture of the Inference Engine allows to develop and plug independent inference
The plugin architecture of the OpenVINO allows to develop and plug independent inference
solutions dedicated to different devices. Physically, a plugin is represented as a dynamic library
exporting the single `CreatePluginEngine` function that allows to create a new plugin instance.
Inference Engine Plugin Library
OpenVINO Plugin Library
-----------------------
Inference Engine plugin dynamic library consists of several main components:
OpenVINO plugin dynamic library consists of several main components:
1. [Plugin class](@ref openvino_docs_ie_plugin_dg_plugin):
- Provides information about devices of a specific type.
- Can create an [executable network](@ref openvino_docs_ie_plugin_dg_executable_network) instance which represents a Neural
Network backend specific graph structure for a particular device in opposite to the InferenceEngine::ICNNNetwork
interface which is backend-independent.
- Can create an [compiled model](@ref openvino_docs_ie_plugin_dg_executable_network) instance which represents a Neural
Network backend specific graph structure for a particular device in opposite to the ov::Model
which is backend-independent.
- Can import an already compiled graph structure from an input stream to an
[executable network](@ref openvino_docs_ie_plugin_dg_executable_network) object.
2. [Executable Network class](@ref openvino_docs_ie_plugin_dg_executable_network):
[compiled model](@ref openvino_docs_ie_plugin_dg_executable_network) object.
2. [Compiled Modek class](@ref openvino_docs_ie_plugin_dg_executable_network):
- Is an execution configuration compiled for a particular device and takes into account its capabilities.
- Holds a reference to a particular device and a task executor for this device.
- Can create several instances of [Inference Request](@ref openvino_docs_ie_plugin_dg_infer_request).
@@ -64,5 +64,5 @@ Detailed guides
API References
-----------------------
* [Inference Engine Plugin API](@ref ie_dev_api)
* [Inference Engine Transformation API](@ref ie_transformation_api)
* [OpenVINO Plugin API](@ref ov_dev_api)
* [OpenVINO Transformation API](@ref ie_transformation_api)

175
docs/IE_PLUGIN_DG/Plugin.md
View File

@@ -1,48 +1,49 @@
# Plugin {#openvino_docs_ie_plugin_dg_plugin}
Inference Engine Plugin usually represents a wrapper around a backend. Backends can be:
OpenVINO Plugin usually represents a wrapper around a backend. Backends can be:
- OpenCL-like backend (e.g. clDNN library) for GPU devices.
- oneDNN backend for Intel CPU devices.
- NVIDIA cuDNN for NVIDIA GPUs.
The responsibility of Inference Engine Plugin:
The responsibility of OpenVINO Plugin:
- Initializes a backend and throw exception in `Engine` constructor if backend cannot be initialized.
- Provides information about devices enabled by a particular backend, e.g. how many devices, their properties and so on.
- Loads or imports [executable network](@ref openvino_docs_ie_plugin_dg_executable_network) objects.
- Loads or imports [compiled model](@ref openvino_docs_ie_plugin_dg_executable_network) objects.
In addition to the Inference Engine Public API, the Inference Engine provides the Plugin API, which is a set of functions and helper classes that simplify new plugin development:
In addition to the OpenVINO Public API, the OpenVINO provides the Plugin API, which is a set of functions and helper classes that simplify new plugin development:
- header files in the `inference_engine/src/plugin_api` directory
- implementations in the `inference_engine/src/inference_engine` directory
- symbols in the Inference Engine Core shared library
- header files in the `src/inference/dev_api/openvino` directory
- implementations in the `src/inference/src/dev/` directory
- symbols in the OpenVINO shared library
To build an Inference Engine plugin with the Plugin API, see the [Inference Engine Plugin Building](@ref openvino_docs_ie_plugin_dg_plugin_build) guide.
To build an OpenVINO plugin with the Plugin API, see the [OpenVINO Plugin Building](@ref openvino_docs_ie_plugin_dg_plugin_build) guide.
Plugin Class
------------------------
Inference Engine Plugin API provides the helper InferenceEngine::IInferencePlugin class recommended to use as a base class for a plugin.
OpenVINO Plugin API provides the helper ov::IPlugin class recommended to use as a base class for a plugin.
Based on that, declaration of a plugin class can look as follows:
@snippet template/src/plugin.hpp plugin:header
#### Class Fields
### Class Fields
The provided plugin class also has several fields:
* `_backend` - a backend engine that is used to perform actual computations for network inference. For `Template` plugin `ngraph::runtime::Backend` is used which performs computations using OpenVINO™ reference implementations.
* `_waitExecutor` - a task executor that waits for a response from a device about device tasks completion.
* `_cfg` of type `Configuration`:
* `m_backend` - a backend engine that is used to perform actual computations for model inference. For `Template` plugin `ov::runtime::Backend` is used which performs computations using OpenVINO™ reference implementations.
* `m_waitExecutor` - a task executor that waits for a response from a device about device tasks completion.
* `m_cfg` of type `Configuration`:
@snippet template/src/config.hpp configuration:header
As an example, a plugin configuration has three value parameters:
- `deviceId` - particular device ID to work with. Applicable if a plugin supports more than one `Template` device. In this case, some plugin methods, like `SetConfig`, `QueryNetwork`, and `LoadNetwork`, must support the CONFIG_KEY(KEY_DEVICE_ID) parameter.
- `perfCounts` - boolean value to identify whether to collect performance counters during [Inference Request](@ref openvino_docs_ie_plugin_dg_infer_request) execution.
- `_streamsExecutorConfig` - configuration of `InferenceEngine::IStreamsExecutor` to handle settings of multi-threaded context.
- `device_id` - particular device ID to work with. Applicable if a plugin supports more than one `Template` device. In this case, some plugin methods, like `set_property`, `query_model`, and `compile_model`, must support the ov::device::id property.
- `perf_counts` - boolean value to identify whether to collect performance counters during [Inference Request](@ref openvino_docs_ie_plugin_dg_infer_request) execution.
- `streams_executor_config` - configuration of `ov::threading::IStreamsExecutor` to handle settings of multi-threaded context.
- `performance_mode` - configuration of `ov::hint::PerformanceMode` to set the performance mode.
### Engine Constructor
### Plugin Constructor
A plugin constructor must contain code that checks the ability to work with a device of the `Template`
type. For example, if some drivers are required, the code must check
@@ -50,132 +51,120 @@ driver availability. If a driver is not available (for example, OpenCL runtime i
case of a GPU device or there is an improper version of a driver is on a host machine), an exception
must be thrown from a plugin constructor.
A plugin must define a device name enabled via the `_pluginName` field of a base class:
A plugin must define a device name enabled via the `set_device_name()` method of a base class:
@snippet template/src/plugin.cpp plugin:ctor
### `LoadExeNetworkImpl()`
### Plugin Destructor
**Implementation details:** The base InferenceEngine::IInferencePlugin class provides a common implementation
of the public InferenceEngine::IInferencePlugin::LoadNetwork method that calls plugin-specific `LoadExeNetworkImpl`, which is defined in a derived class.
A plugin destructor must stop all plugins activities, and clean all allocated resources.
This is the most important function of the `Plugin` class and creates an instance of compiled `ExecutableNetwork`,
which holds a backend-dependent compiled graph in an internal representation:
@snippet template/src/plugin.cpp plugin:dtor
@snippet template/src/plugin.cpp plugin:load_exe_network_impl
### compile_model()
Before a creation of an `ExecutableNetwork` instance via a constructor, a plugin may check if a provided
InferenceEngine::ICNNNetwork object is supported by a device. In the example above, the plugin checks precision information.
The plugin should implement two `compile_model()` methods: the first one compiles model without remote context, the second one with remote context if plugin supports.
The very important part before creation of `ExecutableNetwork` instance is to call `TransformNetwork` method which applies OpenVINO™ transformation passes.
This is the most important function of the `Plugin` class is to create an instance of compiled `CompiledModel`,
which holds a backend-dependent compiled model in an internal representation:
Actual graph compilation is done in the `ExecutableNetwork` constructor. Refer to the [ExecutableNetwork Implementation Guide](@ref openvino_docs_ie_plugin_dg_executable_network) for details.
@snippet template/src/plugin.cpp plugin:compile_model
> **NOTE**: Actual configuration map used in `ExecutableNetwork` is constructed as a base plugin
> configuration set via `Plugin::SetConfig`, where some values are overwritten with `config` passed to `Plugin::LoadExeNetworkImpl`.
> Therefore, the config of `Plugin::LoadExeNetworkImpl` has a higher priority.
@snippet template/src/plugin.cpp plugin:compile_model_with_remote
### `TransformNetwork()`
Before a creation of an `CompiledModel` instance via a constructor, a plugin may check if a provided
ov::Model object is supported by a device if it is needed.
The function accepts a const shared pointer to `ov::Model` object and performs the following steps:
Actual model compilation is done in the `CompiledModel` constructor. Refer to the [CompiledModel Implementation Guide](@ref openvino_docs_ie_plugin_dg_executable_network) for details.
1. Deep copies a const object to a local object, which can later be modified.
2. Applies common and plugin-specific transformations on a copied graph to make the graph more friendly to hardware operations. For details how to write custom plugin-specific transformation, please, refer to [Writing OpenVINO™ transformations](@ref openvino_docs_transformations) guide. See detailed topics about network representation:
> **NOTE**: Actual configuration map used in `CompiledModel` is constructed as a base plugin
> configuration set via `Plugin::set_property`, where some values are overwritten with `config` passed to `Plugin::compile_model`.
> Therefore, the config of `Plugin::compile_model` has a higher priority.
### transform_model()
The function accepts a const shared pointer to `ov::Model` object and applies common and device-specific transformations on a copied model to make it more friendly to hardware operations. For details how to write custom device-specific transformation, please, refer to [Writing OpenVINO™ transformations](@ref openvino_docs_transformations) guide. See detailed topics about model representation:
* [Intermediate Representation and Operation Sets](@ref openvino_docs_MO_DG_IR_and_opsets)
* [Quantized networks](@ref openvino_docs_ie_plugin_dg_quantized_networks).
* [Quantized models](@ref openvino_docs_ie_plugin_dg_quantized_networks).
@snippet template/src/plugin.cpp plugin:transform_network
@snippet template/src/plugin.cpp plugin:transform_model
> **NOTE**: After all these transformations, a `ov::Model` object contains operations which can be perfectly mapped to backend kernels. E.g. if backend has kernel computing `A + B` operations at once, the `TransformNetwork` function should contain a pass which fuses operations `A` and `B` into a single custom operation `A + B` which fits backend kernels set.
> **NOTE**: After all these transformations, an `ov::Model` object contains operations which can be perfectly mapped to backend kernels. E.g. if backend has kernel computing `A + B` operations at once, the `transform_model` function should contain a pass which fuses operations `A` and `B` into a single custom operation `A + B` which fits backend kernels set.
### `QueryNetwork()`
### query_model()
Use the method with the `HETERO` mode, which allows to distribute network execution between different
Use the method with the `HETERO` mode, which allows to distribute model execution between different
devices based on the `ov::Node::get_rt_info()` map, which can contain the `"affinity"` key.
The `QueryNetwork` method analyzes operations of provided `network` and returns a list of supported
operations via the InferenceEngine::QueryNetworkResult structure. The `QueryNetwork` firstly applies `TransformNetwork` passes to input `ov::Model` argument. After this, the transformed network in ideal case contains only operations are 1:1 mapped to kernels in computational backend. In this case, it's very easy to analyze which operations is supposed (`_backend` has a kernel for such operation or extensions for the operation is provided) and not supported (kernel is missed in `_backend`):
The `query_model` method analyzes operations of provided `model` and returns a list of supported
operations via the ov::SupportedOpsMap structure. The `query_model` firstly applies `transform_model` passes to input `ov::Model` argument. After this, the transformed model in ideal case contains only operations are 1:1 mapped to kernels in computational backend. In this case, it's very easy to analyze which operations is supposed (`m_backend` has a kernel for such operation or extensions for the operation is provided) and not supported (kernel is missed in `m_backend`):
1. Store original names of all operations in input `ov::Model`
2. Apply `TransformNetwork` passes. Note, the names of operations in a transformed network can be different and we need to restore the mapping in the steps below.
3. Construct `supported` and `unsupported` maps which contains names of original operations. Note, that since the inference is performed using OpenVINO™ reference backend, the decision whether the operation is supported or not depends on whether the latest OpenVINO opset contains such operation.
4. `QueryNetworkResult.supportedLayersMap` contains only operations which are fully supported by `_backend`.
2. Apply `transform_model` passes. Note, the names of operations in a transformed model can be different and we need to restore the mapping in the steps below.
3. Construct `supported` map which contains names of original operations. Note, that since the inference is performed using OpenVINO™ reference backend, the decision whether the operation is supported or not depends on whether the latest OpenVINO opset contains such operation.
4. `ov.SupportedOpsMap` contains only operations which are fully supported by `m_backend`.
@snippet template/src/plugin.cpp plugin:query_network
@snippet template/src/plugin.cpp plugin:query_model
### `SetConfig()`
### set_property()
Sets new values for plugin configuration keys:
Sets new values for plugin property keys:
@snippet template/src/plugin.cpp plugin:set_config
@snippet template/src/plugin.cpp plugin:set_property
In the snippet above, the `Configuration` class overrides previous configuration values with the new
ones. All these values are used during backend specific graph compilation and execution of inference requests.
ones. All these values are used during backend specific model compilation and execution of inference requests.
> **NOTE**: The function must throw an exception if it receives an unsupported configuration key.
### `GetConfig()`
### get_property()
Returns a current value for a specified configuration key:
Returns a current value for a specified property key:
@snippet template/src/plugin.cpp plugin:get_config
@snippet template/src/plugin.cpp plugin:get_property
The function is implemented with the `Configuration::Get` method, which wraps an actual configuration
key value to the InferenceEngine::Parameter and returns it.
key value to the ov::Any and returns it.
> **NOTE**: The function must throw an exception if it receives an unsupported configuration key.
### `GetMetric()`
### import_model()
Returns a metric value for a metric with the name `name`. A device metric is a static type of information
from a plugin about its devices or device capabilities.
Examples of metrics:
- METRIC_KEY(AVAILABLE_DEVICES) - list of available devices that are required to implement. In this case, you can use
all devices of the same `Template` type with automatic logic of the `MULTI` device plugin.
- METRIC_KEY(FULL_DEVICE_NAME) - full device name. In this case, a particular device ID is specified
in the `option` parameter as `{ CONFIG_KEY(KEY_DEVICE_ID), "deviceID" }`.
- METRIC_KEY(SUPPORTED_METRICS) - list of metrics supported by a plugin
- METRIC_KEY(SUPPORTED_CONFIG_KEYS) - list of configuration keys supported by a plugin that
affects their behavior during a backend specific graph compilation or an inference requests execution
- METRIC_KEY(OPTIMIZATION_CAPABILITIES) - list of optimization capabilities of a device.
For example, supported data types and special optimizations for them.
- Any other device-specific metrics. In this case, place metrics declaration and possible values to
a plugin-specific public header file, for example, `template/config.hpp`. The example below
demonstrates the definition of a new optimization capability value specific for a device:
@snippet template/config.hpp public_header:properties
The snippet below provides an example of the implementation for `GetMetric`:
> **NOTE**: If an unsupported metric key is passed to the function, it must throw an exception.
### `ImportNetwork()`
The importing network mechanism allows to import a previously exported backend specific graph and wrap it
using an [ExecutableNetwork](@ref openvino_docs_ie_plugin_dg_executable_network) object. This functionality is useful if
backend specific graph compilation takes significant time and/or cannot be done on a target host
The importing of compiled model mechanism allows to import a previously exported backend specific model and wrap it
using an [CompiledModel](@ref openvino_docs_ie_plugin_dg_executable_network) object. This functionality is useful if
backend specific model compilation takes significant time and/or cannot be done on a target host
device due to other reasons.
During export of backend specific graph using `ExecutableNetwork::Export`, a plugin may export any
type of information it needs to import a compiled graph properly and check its correctness.
During export of backend specific model using `CompiledModel::export_model`, a plugin may export any
type of information it needs to import a compiled model properly and check its correctness.
For example, the export information may include:
- Compilation options (state of `Plugin::_cfg` structure)
- Compilation options (state of `Plugin::m_cfg` structure)
- Information about a plugin and a device type to check this information later during the import and
throw an exception if the `model` stream contains wrong data. For example, if devices have different
capabilities and a graph compiled for a particular device cannot be used for another, such type of
capabilities and a model compiled for a particular device cannot be used for another, such type of
information must be stored and checked during the import.
- Compiled backend specific graph itself
- Information about precisions and shapes set by the user
- Compiled backend specific model itself
@snippet template/src/plugin.cpp plugin:import_network
@snippet template/src/plugin.cpp plugin:import_model
@snippet template/src/plugin.cpp plugin:import_model_with_remote
### create_context()
The Plugin should implement `Plugin::create_context()` method which returns `ov::RemoteContext` in case if plugin supports remote context, in other case the plugin can throw an exception that this method is not implemented.
@snippet template/src/plugin.cpp plugin:create_context
### get_default_context()
`Plugin::get_default_context()` also needed in case if plugin supports remote context, if the plugin doesn't support it, this method can throw an exception that functionality is not implemented.
@snippet template/src/plugin.cpp plugin:get_default_context
Create Instance of Plugin Class
------------------------
Inference Engine plugin library must export only one function creating a plugin instance using IE_DEFINE_PLUGIN_CREATE_FUNCTION macro:
OpenVINO plugin library must export only one function creating a plugin instance using OV_DEFINE_PLUGIN_CREATE_FUNCTION macro:
@snippet template/src/plugin.cpp plugin:create_plugin_engine
Next step in a plugin library implementation is the [ExecutableNetwork](@ref openvino_docs_ie_plugin_dg_executable_network) class.
Next step in a plugin library implementation is the [CompiledModel](@ref openvino_docs_ie_plugin_dg_executable_network) class.

4
docs/IE_PLUGIN_DG/dev_api_references.md
View File

@@ -6,12 +6,12 @@
:maxdepth: 1
:hidden:
../groupie_dev_api
../groupov_dev_api
../groupie_transformation_api
@endsphinxdirective
The guides below provides extra API references needed for OpenVINO plugin development:
* [OpenVINO Plugin API](@ref ie_dev_api)
* [OpenVINO Plugin API](@ref ov_dev_api)
* [OpenVINO Transformation API](@ref ie_transformation_api)

4
docs/OV_Runtime_UG/Samples_Overview.md
View File

@@ -102,7 +102,7 @@ The officially supported Linux build environment is the following:
* GCC 7.5.0 (for Ubuntu 18.04) or GCC 9.3.0 (for Ubuntu 20.04)
* CMake version 3.10 or higher
> **NOTE**: For building samples from the open-source version of OpenVINO toolkit, see the [build instructions on GitHub](https://github.com/openvinotoolkit/openvino/wiki/BuildingCode).
> **NOTE**: For building samples from the open-source version of OpenVINO toolkit, see the [build instructions on GitHub](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build.md).
To build the C or C++ sample applications for Linux, go to the `<INSTALL_DIR>/samples/c` or `<INSTALL_DIR>/samples/cpp` directory, respectively, and run the `build_samples.sh` script:
```sh
@@ -179,7 +179,7 @@ The officially supported macOS build environment is the following:
* Clang compiler from Xcode 10.1 or higher
* CMake version 3.13 or higher
> **NOTE**: For building samples from the open-source version of OpenVINO toolkit, see the [build instructions on GitHub](https://github.com/openvinotoolkit/openvino/wiki/BuildingCode).
> **NOTE**: For building samples from the open-source version of OpenVINO toolkit, see the [build instructions on GitHub](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build.md).
To build the C or C++ sample applications for macOS, go to the `<INSTALL_DIR>/samples/c` or `<INSTALL_DIR>/samples/cpp` directory, respectively, and run the `build_samples.sh` script:
```sh

4
docs/OV_Runtime_UG/deployment/deployment_intro.md
View File

@@ -26,7 +26,7 @@ Furthermore, to customize your OpenVINO Docker image, use the [Docker CI Framewo
- Grab a necessary functionality of OpenVINO together with your application, also called "local distribution":
- using [OpenVINO Deployment Manager](deployment-manager-tool.md) - providing a convenient way for creating a distribution package;
- using the advanced [local distribution](local-distribution.md) approach;
- using [a static version of OpenVINO Runtime linked to the final app](https://github.com/openvinotoolkit/openvino/wiki/StaticLibraries).
- using [a static version of OpenVINO Runtime linked to the final app](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/static_libaries.md).
The table below shows which distribution type can be used for what target operating system:
@@ -38,7 +38,7 @@ The table below shows which distribution type can be used for what target operat
| PyPI (PIP package manager) | See [https://pypi.org/project/openvino/](https://pypi.org/project/openvino/) |
| [OpenVINO Deployment Manager](deployment-manager-tool.md) | All operating systems |
| [Local distribution](local-distribution.md) | All operating systems |
| [Build OpenVINO statically and link to the final app](https://github.com/openvinotoolkit/openvino/wiki/StaticLibraries) | All operating systems |
| [Build OpenVINO statically and link to the final app](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/static_libaries.md) | All operating systems |
## Granularity of Major Distribution Types

2
docs/OV_Runtime_UG/deployment/local-distribution.md
View File

@@ -3,7 +3,7 @@
With a local distribution, each C or C++ application/installer will have its own copies of OpenVINO Runtime binaries. However, OpenVINO has a scalable plugin-based architecture, which means that some components can be loaded in runtime only when they are really needed. Therefore, it is important to understand which minimal set of libraries is really needed to deploy the application. This guide helps you to achieve that goal.
Local dsitribution is also appropriate for OpenVINO binaries built from sources using [Build instructions](https://github.com/openvinotoolkit/openvino/wiki#how-to-build), but the guide below supposes OpenVINO Runtime is built dynamically. For case of [Static OpenVINO Runtime](https://github.com/openvinotoolkit/openvino/wiki/StaticLibraries) select the required OpenVINO capabilities on CMake configuration stage using [CMake Options for Custom Compilation](https://github.com/openvinotoolkit/openvino/wiki/CMakeOptionsForCustomCompilation), the build and link the OpenVINO components into the final application.
Local dsitribution is also appropriate for OpenVINO binaries built from sources using [Build instructions](https://github.com/openvinotoolkit/openvino/wiki#how-to-build), but the guide below supposes OpenVINO Runtime is built dynamically. For case of [Static OpenVINO Runtime](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/static_libaries.md) select the required OpenVINO capabilities on CMake configuration stage using [CMake Options for Custom Compilation](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/cmake_options_for_custom_comiplation.md), the build and link the OpenVINO components into the final application.
> **NOTE**: The steps below are operating system independent and refer to a library file name without any prefixes (like `lib` on Unix systems) or suffixes (like `.dll` on Windows OS). Do not put `.lib` files on Windows OS to the distribution, because such files are needed only on a linker stage.

242
docs/OV_Runtime_UG/layout_overview.md
View File

@@ -1,172 +1,204 @@
# Layout API Overview {#openvino_docs_OV_UG_Layout_Overview}
@sphinxdirective
The concept of layout helps you (and your application) to understand what each particular dimension of input/output tensor means. For example, if your input has the ``{1, 3, 720, 1280}`` shape and the ``NCHW`` layout, it is clear that ``N(batch) = 1``, ``C(channels) = 3``, ``H(height) = 720``, and ``W(width) = 1280``. Without the layout information, the ``{1, 3, 720, 1280}`` tuple does not give any idea to your application on what these numbers mean and how to resize the input image to fit the expectations of the model.
The concept of layout helps you (and your application) to understand what each particular dimension of input/output tensor means. For example, if your input has the `{1, 3, 720, 1280}` shape and the `NCHW` layout, it is clear that `N(batch) = 1`, `C(channels) = 3`, `H(height) = 720`, and `W(width) = 1280`. Without the layout information, the `{1, 3, 720, 1280}` tuple does not give any idea to your application on what these numbers mean and how to resize the input image to fit the expectations of the model.
With the `NCHW` layout, it is easier to understand what the `{8, 3, 224, 224}` model shape means. Without the layout, it is just a 4-dimensional tensor.
With the ``NCHW`` layout, it is easier to understand what the ``{8, 3, 224, 224}`` model shape means. Without the layout, it is just a 4-dimensional tensor.
Below is a list of cases where input/output layout is important:
- Performing model modification:
- Applying the [preprocessing](./preprocessing_overview.md) steps, such as subtracting means, dividing by scales, resizing an image, and converting `RGB`<->`BGR`.
- Setting/getting a batch for a model.
- Doing the same operations as used during the model conversion phase. For more information, refer to the [Model Optimizer Embedding Preprocessing Computation](../MO_DG/prepare_model/Additional_Optimizations.md) guide.
- Improving the readability of a model input and output.
## Syntax of Layout
* Performing model modification:
* Applying the :doc:`preprocessing <openvino_docs_OV_UG_Preprocessing_Overview>` steps, such as subtracting means, dividing by scales, resizing an image, and converting ``RGB`` <-> ``BGR``.
* Setting/getting a batch for a model.
* Doing the same operations as used during the model conversion phase. For more information, refer to the :doc:`Model Optimizer Embedding Preprocessing Computation <openvino_docs_MO_DG_Additional_Optimization_Use_Cases>` guide.
* Improving the readability of a model input and output.
Syntax of Layout
####################
Short Syntax
++++++++++++++++++++
### Short Syntax
The easiest way is to fully specify each dimension with one alphabet letter.
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_layout.cpp ov:layout:simple
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_layout.cpp
:language: cpp
:fragment: ov:layout:simple
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_layout.py ov:layout:simple
@endsphinxtab
@endsphinxtabset
This assigns `N` to the first dimension, `C` to the second, `H` to the third, and `W` to the fourth.
### Advanced Syntax
The advanced syntax allows assigning a word to a dimension. To do this, wrap a layout with square brackets `[]` and specify each name separated by a comma `,`.
@sphinxtabset
@sphinxtab{C++}
@snippet docs/snippets/ov_layout.cpp ov:layout:complex
@endsphinxtab
@sphinxtab{Python}
@snippet docs/snippets/ov_layout.py ov:layout:complex
@endsphinxtab
@endsphinxtabset
.. doxygensnippet:: docs/snippets/ov_layout.py
:language: python
:fragment: ov:layout:simple
### Partially Defined Layout
If a certain dimension is not important, its name can be set to `?`.
This assigns ``N`` to the first dimension, ``C`` to the second, ``H`` to the third, and ``W`` to the fourth.
@sphinxtabset
Advanced Syntax
++++++++++++++++++++
@sphinxtab{C++}
@snippet docs/snippets/ov_layout.cpp ov:layout:partially_defined
@endsphinxtab
@sphinxtab{Python}
@snippet docs/snippets/ov_layout.py ov:layout:partially_defined
@endsphinxtab
@endsphinxtabset
The advanced syntax allows assigning a word to a dimension. To do this, wrap a layout with square brackets ``[]`` and specify each name separated by a comma ``,``.
### Dynamic Layout
If several dimensions are not important, an ellipsis `...` can be used to specify those dimensions.
.. tab-set::
@sphinxtabset
.. tab-item:: C++
:sync: cpp
@sphinxtab{C++}
.. doxygensnippet:: docs/snippets/ov_layout.cpp
:language: cpp
:fragment: ov:layout:complex
@snippet docs/snippets/ov_layout.cpp ov:layout:dynamic
.. tab-item:: Python
:sync: py
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_layout.py
:language: python
:fragment: ov:layout:complex
@sphinxtab{Python}
@snippet docs/snippets/ov_layout.py ov:layout:dynamic
Partially Defined Layout
++++++++++++++++++++++++
@endsphinxtab
If a certain dimension is not important, its name can be set to ``?``.
@endsphinxtabset
### Predefined Names
.. tab-set::
.. tab-item:: C++
:sync: cpp
.. doxygensnippet:: docs/snippets/ov_layout.cpp
:language: cpp
:fragment: ov:layout:partially_defined
.. tab-item:: Python
:sync: py
.. doxygensnippet:: docs/snippets/ov_layout.py
:language: python
:fragment: ov:layout:partially_defined
Dynamic Layout
++++++++++++++++++++
If several dimensions are not important, an ellipsis ``...`` can be used to specify those dimensions.
.. tab-set::
.. tab-item:: C++
:sync: cpp
.. doxygensnippet:: docs/snippets/ov_layout.cpp
:language: cpp
:fragment: ov:layout:dynamic
.. tab-item:: Python
:sync: py
.. doxygensnippet:: docs/snippets/ov_layout.py
:language: python
:fragment: ov:layout:dynamic
Predefined Names
++++++++++++++++++++
A layout has some pre-defined dimension names, widely used in computer vision:
- `N`/`Batch` - batch size
- `C`/`Channels` - channels
- `D`/`Depth` - depth
- `H`/`Height` - height
- `W`/`Width` - width
These names are used in [PreProcessing API](./preprocessing_overview.md). There is a set of helper functions to get appropriate dimension index from a layout.
* ``N``/``Batch`` - batch size
* ``C``/``Channels`` - channels
* ``D``/``Depth`` - depth
* ``H``/``Height`` - height
* ``W``/``Width`` - width
@sphinxtabset
These names are used in :doc:`PreProcessing API <openvino_docs_OV_UG_Preprocessing_Overview>`. There is a set of helper functions to get appropriate dimension index from a layout.
@sphinxtab{C++}
@snippet docs/snippets/ov_layout.cpp ov:layout:predefined
.. tab-set::
@endsphinxtab
.. tab-item:: C++
:sync: cpp
@sphinxtab{Python}
.. doxygensnippet:: docs/snippets/ov_layout.cpp
:language: cpp
:fragment: ov:layout:predefined
@snippet docs/snippets/ov_layout.py ov:layout:predefined
.. tab-item:: Python
:sync: py
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_layout.py
:language: python
:fragment: ov:layout:predefined
@endsphinxtabset
### Equality
Equality
++++++++++++++++++++
Layout names are case-insensitive, which means that `Layout("NCHW")` = `Layout("nChW") = `Layout("[N,c,H,w]")`.
Layout names are case-insensitive, which means that ``Layout("NCHW")`` = ``Layout("nChW")`` = ``Layout("[N,c,H,w]")``.
### Dump Layout
Dump Layout
++++++++++++++++++++
A layout can be converted to a string in the advanced syntax format. It can be useful for debugging and serialization purposes.
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_layout.cpp ov:layout:dump
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_layout.cpp
:language: cpp
:fragment: ov:layout:dump
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_layout.py ov:layout:dump
.. doxygensnippet:: docs/snippets/ov_layout.py
:language: python
:fragment: ov:layout:dump
@endsphinxtab
@endsphinxtabset
### Get layout from Model Input/Output
Get layout from Model Input/Output
++++++++++++++++++++++++++++++++++
OpenVINO provides helpers which provide a simple interface to get layout from Model input or output.
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_layout.cpp ov:layout:get_from_model
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_layout.cpp
:language: cpp
:fragment: ov:layout:get_from_model
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_layout.py ov:layout:get_from_model
.. doxygensnippet:: docs/snippets/ov_layout.py
:language: python
:fragment: ov:layout:get_from_model
@endsphinxtab
@endsphinxtabset
See also
####################
## See also
* API Reference: `ov::Layout <classov_1_1Layout.html#doxid-classov-1-1-layout>`__ C++ class
* API Reference: <code>ov::Layout</code> C++ class
@endsphinxdirective

1051
docs/OV_Runtime_UG/migration_ov_2_0/common_inference_pipeline.md
View File

File diff suppressed because it is too large Load Diff

8
docs/OV_Runtime_UG/migration_ov_2_0/intro.md
View File

@@ -50,7 +50,13 @@ However, Post-Training Optimization Tool of OpenVINO 2022.1 does not support Ope
> **NOTE**: To quantize your OpenVINO IR v10 models to run with OpenVINO 2022.1, download and use Post-Training Optimization Tool of OpenVINO 2021.4.
@anchor differences_api20_ie
@sphinxdirective
.. _differences_api20_ie:
@endsphinxdirective
### Differences in API 2.0 and Inference Engine API Behaviors
Inference Engine and nGraph APIs do not become deprecated with the introduction of the new API, and they can still be used in applications. However, it is highly recommended to migrate to API 2.0, as it offers more features (further extended in future releases), such as:

301
docs/OV_Runtime_UG/migration_ov_2_0/preprocessing.md
View File

@@ -1,244 +1,291 @@
# Preprocessing {#openvino_2_0_preprocessing}
@sphinxdirective
This guide introduces how preprocessing works in API 2.0 by a comparison with preprocessing in the previous Inference Engine API. It also demonstrates how to migrate preprocessing scenarios from Inference Engine to API 2.0 via code samples.
## How Preprocessing Works in API 2.0
How Preprocessing Works in API 2.0
##################################
Inference Engine API contains preprocessing capabilities in the `InferenceEngine::CNNNetwork` class. Such preprocessing information is not a part of the main inference graph executed by [OpenVINO devices](../supported_plugins/Device_Plugins.md). Therefore, it is stored and executed separately before the inference stage:
- Preprocessing operations are executed on the CPU for most OpenVINO inference plugins. Thus, instead of occupying accelerators, they keep the CPU busy with computational tasks.
- Preprocessing information stored in `InferenceEngine::CNNNetwork` is lost when saving back to the OpenVINO IR file format.
Inference Engine API contains preprocessing capabilities in the `InferenceEngine::CNNNetwork <classInferenceEngine_1_1CNNNetwork.html#doxid-class-inference-engine-1-1-c-n-n-network>`__ class. Such preprocessing information is not a part of the main inference graph executed by :doc:`OpenVINO devices <openvino_docs_OV_UG_Working_with_devices>`. Therefore, it is stored and executed separately before the inference stage:
API 2.0 introduces a [new way of adding preprocessing operations to the model](../preprocessing_overview.md) - each preprocessing or post-processing operation is integrated directly into the model and compiled together with the inference graph:
- API 2.0 first adds preprocessing operations by using `ov::preprocess::PrePostProcessor`,
- and then compiles the model on the target by using `ov::Core::compile_model`.
* Preprocessing operations are executed on the CPU for most OpenVINO inference plugins. Thus, instead of occupying accelerators, they keep the CPU busy with computational tasks.
* Preprocessing information stored in `InferenceEngine::CNNNetwork <classInferenceEngine_1_1CNNNetwork.html#doxid-class-inference-engine-1-1-c-n-n-network>`__ is lost when saving back to the OpenVINO IR file format.
API 2.0 introduces a :doc:`new way of adding preprocessing operations to the model <openvino_docs_OV_UG_Preprocessing_Overview>` - each preprocessing or post-processing operation is integrated directly into the model and compiled together with the inference graph:
* API 2.0 first adds preprocessing operations by using `ov::preprocess::PrePostProcessor <classov_1_1preprocess_1_1PrePostProcessor.html#doxid-classov-1-1preprocess-1-1-pre-post-processor>`__,
* and then compiles the model on the target by using `ov::Core::compile_model <classov_1_1Core.html#doxid-classov-1-1-core-1a46555f0803e8c29524626be08e7f5c5a>`__.
Having preprocessing operations as a part of an OpenVINO opset makes it possible to read and serialize a preprocessed model as the OpenVINO™ IR file format.
More importantly, API 2.0 does not assume any default layouts as Inference Engine did. For example, both `{ 1, 224, 224, 3 }` and `{ 1, 3, 224, 224 }` shapes are supposed to be in the `NCHW` layout, while only the latter is. Therefore, some preprocessing capabilities in the API require layouts to be set explicitly. To learn how to do it, refer to the [Layout overview](../layout_overview.md). For example, to perform image scaling by partial dimensions `H` and `W`, preprocessing needs to know what dimensions `H` and `W` are.
More importantly, API 2.0 does not assume any default layouts as Inference Engine did. For example, both ``{ 1, 224, 224, 3 }`` and ``{ 1, 3, 224, 224 }`` shapes are supposed to be in the `NCHW` layout, while only the latter is. Therefore, some preprocessing capabilities in the API require layouts to be set explicitly. To learn how to do it, refer to the :doc:`Layout overview <openvino_docs_OV_UG_Layout_Overview>`. For example, to perform image scaling by partial dimensions ``H`` and ``W``, preprocessing needs to know what dimensions ``H`` and ``W`` are.
> **NOTE**: Use Model Optimizer preprocessing capabilities to insert preprocessing operations in your model for optimization. Thus, the application does not need to read the model and set preprocessing repeatedly. You can use the [model caching feature](../Model_caching_overview.md) to improve the time-to-inference.
.. note::
Use Model Optimizer preprocessing capabilities to insert preprocessing operations in your model for optimization. Thus, the application does not need to read the model and set preprocessing repeatedly. You can use the :doc:`model caching feature <openvino_docs_OV_UG_Model_caching_overview>` to improve the time-to-inference.
The following sections demonstrate how to migrate preprocessing scenarios from Inference Engine API to API 2.0.
The snippets assume that you need to preprocess a model input with the `tensor_name` in Inference Engine API, using `operation_name` to address the data.
The snippets assume that you need to preprocess a model input with the ``tensor_name`` in Inference Engine API, using ``operation_name`` to address the data.
## Preparation: Import Preprocessing in Python
Preparation: Import Preprocessing in Python
###########################################
In order to utilize preprocessing, the following imports must be added.
**Inference Engine API**
@snippet docs/snippets/ov_preprocessing_migration.py imports
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.py
:language: python
:fragment: imports
**API 2.0**
@snippet docs/snippets/ov_preprocessing_migration.py ov_imports
There are two different namespaces:
- `runtime`, which contains API 2.0 classes;
- and `preprocess`, which provides Preprocessing API.
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.py
:language: python
:fragment: ov_imports
## Using Mean and Scale Values
| There are two different namespaces:
| * ``runtime``, which contains API 2.0 classes;
| * and ``preprocess``, which provides Preprocessing API.
Using Mean and Scale Values
###########################
**Inference Engine API**
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_preprocessing_migration.cpp mean_scale
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.cpp
:language: cpp
:fragment: mean_scale
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_preprocessing_migration.py mean_scale
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.py
:language: python
:fragment: mean_scale
@endsphinxtab
.. tab-item:: C
:sync: c
@sphinxtab{C}
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.c
:language: c
:fragment: c_api_ppp
@snippet docs/snippets/ov_preprocessing_migration.c c_api_ppp
@endsphinxtab
@endsphinxtabset
**API 2.0**
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_preprocessing_migration.cpp ov_mean_scale
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.cpp
:language: cpp
:fragment: ov_mean_scale
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_preprocessing_migration.py ov_mean_scale
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.py
:language: python
:fragment: ov_mean_scale
@endsphinxtab
.. tab-item:: C
:sync: c
@sphinxtab{C}
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.c
:language: c
:fragment: ov_mean_scale
@snippet docs/snippets/ov_preprocessing_migration.c ov_mean_scale
@endsphinxtab
@endsphinxtabset
## Converting Precision and Layout
Converting Precision and Layout
###############################
**Inference Engine API**
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_preprocessing_migration.cpp conversions
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.cpp
:language: cpp
:fragment: conversions
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_preprocessing_migration.py conversions
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.py
:language: python
:fragment: conversions
@endsphinxtab
.. tab-item:: C
:sync: c
@sphinxtab{C}
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.c
:language: c
:fragment: c_api_ppp
@snippet docs/snippets/ov_preprocessing_migration.c c_api_ppp
@endsphinxtab
@endsphinxtabset
**API 2.0**
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_preprocessing_migration.cpp ov_conversions
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.cpp
:language: cpp
:fragment: ov_conversions
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_preprocessing_migration.py ov_conversions
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.py
:language: python
:fragment: ov_conversions
@endsphinxtab
.. tab-item:: C
:sync: c
@sphinxtab{C}
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.c
:language: c
:fragment: ov_conversions
@snippet docs/snippets/ov_preprocessing_migration.c ov_conversions
@endsphinxtab
@endsphinxtabset
## Using Image Scaling
Using Image Scaling
####################
**Inference Engine API**
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_preprocessing_migration.cpp image_scale
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.cpp
:language: cpp
:fragment: image_scale
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_preprocessing_migration.py image_scale
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.py
:language: python
:fragment: image_scale
@endsphinxtab
.. tab-item:: C
:sync: c
@sphinxtab{C}
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.c
:language: c
:fragment: c_api_ppp
@snippet docs/snippets/ov_preprocessing_migration.c c_api_ppp
@endsphinxtab
@endsphinxtabset
**API 2.0**
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_preprocessing_migration.cpp ov_image_scale
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.cpp
:language: cpp
:fragment: ov_image_scale
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_preprocessing_migration.py ov_image_scale
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.py
:language: python
:fragment: ov_image_scale
@endsphinxtab
.. tab-item:: C
:sync: c
@sphinxtab{C}
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.c
:language: c
:fragment: ov_image_scale
@snippet docs/snippets/ov_preprocessing_migration.c ov_image_scale
@endsphinxtab
@endsphinxtabset
### Converting Color Space
Converting Color Space
++++++++++++++++++++++
**Inference Engine API**
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_preprocessing_migration.cpp color_space
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.cpp
:language: cpp
:fragment: color_space
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_preprocessing_migration.py color_space
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.py
:language: python
:fragment: color_space
@endsphinxtab
.. tab-item:: C
:sync: c
@sphinxtab{C}
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.c
:language: c
:fragment: c_api_ppp
@snippet docs/snippets/ov_preprocessing_migration.c c_api_ppp
@endsphinxtab
@endsphinxtabset
**API 2.0**
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_preprocessing_migration.cpp ov_color_space
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.cpp
:language: cpp
:fragment: ov_color_space
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_preprocessing_migration.py ov_color_space
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.py
:language: python
:fragment: ov_color_space
@endsphinxtab
.. tab-item:: C
:sync: c
@sphinxtab{C}
@snippet docs/snippets/ov_preprocessing_migration.c ov_color_space
@endsphinxtab
@endsphinxtabset
.. doxygensnippet:: docs/snippets/ov_preprocessing_migration.c
:language: c
:fragment: ov_color_space
## Additional Resources
Additional Resources
####################
- [Preprocessing details](../preprocessing_details.md)
- [NV12 classification sample](../../../samples/cpp/hello_nv12_input_classification/README.md)
- :doc:`Preprocessing details <openvino_docs_OV_UG_Preprocessing_Details>`
- :doc:`NV12 classification sample <openvino_inference_engine_samples_hello_nv12_input_classification_README>`
@endsphinxdirective

263
docs/OV_Runtime_UG/ov_dynamic_shapes.md
View File

@@ -8,116 +8,128 @@
openvino_docs_OV_UG_NoDynamicShapes
@endsphinxdirective
As it was demonstrated in the [Changing Input Shapes](ShapeInference.md) article, there are models that support changing input shapes before model compilation in `Core::compile_model`.
As it was demonstrated in the :doc:`Changing Input Shapes <openvino_docs_OV_UG_ShapeInference>` article, there are models that support changing input shapes before model compilation in ``Core::compile_model``.
Reshaping models provides an ability to customize the model input shape for the exact size required in the end application.
This article explains how the ability of model to reshape can further be leveraged in more dynamic scenarios.
## Applying Dynamic Shapes
Applying Dynamic Shapes
#######################
Conventional "static" model reshaping works well when it can be done once per many model inference calls with the same shape.
However, this approach does not perform efficiently if the input tensor shape is changed on every inference call. Calling the `reshape()` and `compile_model()` methods each time a new size comes is extremely time-consuming.
However, this approach does not perform efficiently if the input tensor shape is changed on every inference call. Calling the ``reshape()`` and ``compile_model()`` methods each time a new size comes is extremely time-consuming.
A popular example would be inference of natural language processing models (like BERT) with arbitrarily-sized user input sequences.
In this case, the sequence length cannot be predicted and may change every time inference is called.
Dimensions that can be frequently changed are called *dynamic dimensions*.
Dynamic shapes should be considered, when a real shape of input is not known at the time of the `compile_model()` method call.
Dynamic shapes should be considered, when a real shape of input is not known at the time of the ``compile_model()`` method call.
Below are several examples of dimensions that can be naturally dynamic:
- Sequence length dimension for various sequence processing models, like BERT
- Spatial dimensions in segmentation and style transfer models
- Batch dimension
- Arbitrary number of detections in object detection models output
* Sequence length dimension for various sequence processing models, like BERT
* Spatial dimensions in segmentation and style transfer models
* Batch dimension
* Arbitrary number of detections in object detection models output
There are various methods to address input dynamic dimensions through combining multiple pre-reshaped models and input data padding.
The methods are sensitive to model internals, do not always give optimal performance and are cumbersome.
For a short overview of the methods, refer to the [When Dynamic Shapes API is Not Applicable](ov_without_dynamic_shapes.md) page.
For a short overview of the methods, refer to the :doc:`When Dynamic Shapes API is Not Applicable <openvino_docs_OV_UG_NoDynamicShapes>` page.
Apply those methods only if native dynamic shape API described in the following sections does not work or does not perform as expected.
The decision about using dynamic shapes should be based on proper benchmarking of a real application with real data.
Unlike statically shaped models, dynamically shaped ones require different inference time, depending on input data shape or input tensor content.
Furthermore, using the dynamic shapes can bring more overheads in memory and running time of each inference call depending on hardware plugin and model used.
## Handling Dynamic Shapes Natively
Handling Dynamic Shapes Natively
################################
This section describes how to handle dynamically shaped models natively with OpenVINO Runtime API version 2022.1 and higher.
There are three main parts in the flow that differ from static shapes:
- Configure the model.
- Prepare data for inference.
- Read resulting data after inference.
### Configuring the Model
* Configure the model.
* Prepare data for inference.
* Read resulting data after inference.
Configuring the Model
+++++++++++++++++++++
To avoid the methods mentioned in the previous section, there is a way to specify one or multiple dimensions to be dynamic, directly in the model inputs.
This is achieved with the same reshape method that is used for alternating static shape of inputs.
Dynamic dimensions are specified as `-1` or the `ov::Dimension()` instead of a positive number used for static dimensions:
Dynamic dimensions are specified as ``-1`` or the ``ov::Dimension()`` instead of a positive number used for static dimensions:
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_dynamic_shapes.cpp ov_dynamic_shapes:reshape_undefined
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_dynamic_shapes.cpp
:language: cpp
:fragment: ov_dynamic_shapes:reshape_undefined
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_dynamic_shapes.py reshape_undefined
.. doxygensnippet:: docs/snippets/ov_dynamic_shapes.py
:language: python
:fragment: reshape_undefined
@endsphinxtab
.. tab-item:: C
:sync: c
@sphinxtab{C}
.. doxygensnippet:: docs/snippets/ov_dynamic_shapes.c
:language: cpp
:fragment: ov_dynamic_shapes:reshape_undefined
@snippet docs/snippets/ov_dynamic_shapes.c ov_dynamic_shapes:reshape_undefined
@endsphinxtab
@endsphinxtabset
To simplify the code, the examples assume that the model has a single input and single output.
However, there are no limitations on the number of inputs and outputs to apply dynamic shapes.
### Undefined Dimensions "Out Of the Box"
Undefined Dimensions "Out Of the Box"
+++++++++++++++++++++++++++++++++++++
Dynamic dimensions may appear in the input model without calling the `reshape` method.
Dynamic dimensions may appear in the input model without calling the ``reshape`` method.
Many DL frameworks support undefined dimensions.
If such a model is converted with Model Optimizer or read directly by the `Core::read_model`, undefined dimensions are preserved.
If such a model is converted with Model Optimizer or read directly by the ``Core::read_model``, undefined dimensions are preserved.
Such dimensions are automatically treated as dynamic ones.
Therefore, there is no need to call the `reshape` method, if undefined dimensions are already configured in the original or the IR model.
Therefore, there is no need to call the ``reshape`` method, if undefined dimensions are already configured in the original or the IR model.
If the input model has undefined dimensions that will not change during inference. It is recommended to set them to static values, using the same `reshape` method of the model.
If the input model has undefined dimensions that will not change during inference. It is recommended to set them to static values, using the same ``reshape`` method of the model.
From the API perspective, any combination of dynamic and static dimensions can be configured.
Model Optimizer provides identical capability to reshape the model during the conversion, including specifying dynamic dimensions.
Use this capability to save time on calling `reshape` method in the end application.
To get information about setting input shapes using Model Optimizer, refer to [Setting Input Shapes](../MO_DG/prepare_model/convert_model/Converting_Model.md).
Use this capability to save time on calling ``reshape`` method in the end application.
To get information about setting input shapes using Model Optimizer, refer to :doc:`Setting Input Shapes <openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model>`.
### Dimension Bounds
Dimension Bounds
++++++++++++++++++++
Apart from a dynamic dimension, the lower and/or upper bounds can also be specified. They define a range of allowed values for the dimension.
The bounds are coded as arguments for the `ov::Dimension`:
The bounds are coded as arguments for the ``ov::Dimension``:
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_dynamic_shapes.cpp ov_dynamic_shapes:reshape_bounds
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_dynamic_shapes.cpp
:language: cpp
:fragment: ov_dynamic_shapes:reshape_bounds
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_dynamic_shapes.py reshape_bounds
.. doxygensnippet:: docs/snippets/ov_dynamic_shapes.py
:language: python
:fragment: reshape_bounds
@endsphinxtab
.. tab-item:: C
:sync: c
@sphinxtab{C}
.. doxygensnippet:: docs/snippets/ov_dynamic_shapes.c
:language: cpp
:fragment: ov_dynamic_shapes:reshape_bounds
@snippet docs/snippets/ov_dynamic_shapes.c ov_dynamic_shapes:reshape_bounds
@endsphinxtab
@endsphinxtabset
Information about bounds gives an opportunity for the inference plugin to apply additional optimizations.
Using dynamic shapes assumes the plugins apply more flexible optimization approach during model compilation.
@@ -127,70 +139,78 @@ For the same reason, it is not recommended to leave dimensions as undefined, wit
When specifying bounds, the lower bound is not as important as the upper one. The upper bound allows inference devices to allocate memory for intermediate tensors more precisely. It also allows using a fewer number of tuned kernels for different sizes.
More precisely, benefits of specifying the lower or upper bound is device dependent.
Depending on the plugin, specifying the upper bounds can be required. For information about dynamic shapes support on different devices, refer to the [Features Support Matrix](@ref features_support_matrix).
Depending on the plugin, specifying the upper bounds can be required. For information about dynamic shapes support on different devices, refer to the :ref:`feature support table <devicesupport-feature-support-matrix>`.
If the lower and upper bounds for a dimension are known, it is recommended to specify them, even if a plugin can execute a model without the bounds.
### Setting Input Tensors
Setting Input Tensors
+++++++++++++++++++++
Preparing a model with the `reshape` method is the first step.
Preparing a model with the ``reshape`` method is the first step.
The second step is passing a tensor with an appropriate shape to infer request.
This is similar to the [regular steps](integrate_with_your_application.md). However, tensors can now be passed with different shapes for the same executable model and even for the same inference request:
This is similar to the :doc:`regular steps <openvino_docs_OV_UG_Integrate_OV_with_your_application>`. However, tensors can now be passed with different shapes for the same executable model and even for the same inference request:
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_dynamic_shapes.cpp ov_dynamic_shapes:set_input_tensor
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_dynamic_shapes.cpp
:language: cpp
:fragment: ov_dynamic_shapes:set_input_tensor
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_dynamic_shapes.py set_input_tensor
.. doxygensnippet:: docs/snippets/ov_dynamic_shapes.py
:language: python
:fragment: set_input_tensor
@endsphinxtab
.. tab-item:: C
:sync: c
@sphinxtab{C}
.. doxygensnippet:: docs/snippets/ov_dynamic_shapes.c
:language: cpp
:fragment: ov_dynamic_shapes:set_input_tensor
@snippet docs/snippets/ov_dynamic_shapes.c ov_dynamic_shapes:set_input_tensor
@endsphinxtab
@endsphinxtabset
In the example above, the `set_input_tensor` is used to specify input tensors.
In the example above, the ``set_input_tensor`` is used to specify input tensors.
The real dimension of the tensor is always static, because it is a particular tensor and it does not have any dimension variations in contrast to model inputs.
Similar to static shapes, `get_input_tensor` can be used instead of `set_input_tensor`.
In contrast to static input shapes, when using `get_input_tensor` for dynamic inputs, the `set_shape` method for the returned tensor should be called to define the shape and allocate memory.
Without doing so, the tensor returned by `get_input_tensor` is an empty tensor. The shape of the tensor is not initialized and memory is not allocated, because infer request does not have information about the real shape that will be provided.
Similar to static shapes, ``get_input_tensor`` can be used instead of ``set_input_tensor``.
In contrast to static input shapes, when using ``get_input_tensor`` for dynamic inputs, the ``set_shape`` method for the returned tensor should be called to define the shape and allocate memory.
Without doing so, the tensor returned by ``get_input_tensor`` is an empty tensor. The shape of the tensor is not initialized and memory is not allocated, because infer request does not have information about the real shape that will be provided.
Setting shape for an input tensor is required when the corresponding input has at least one dynamic dimension, regardless of the bounds.
Contrary to previous example, the following one shows the same sequence of two infer requests, using `get_input_tensor` instead of `set_input_tensor`:
Contrary to previous example, the following one shows the same sequence of two infer requests, using ``get_input_tensor`` instead of ``set_input_tensor``:
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_dynamic_shapes.cpp ov_dynamic_shapes:get_input_tensor
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_dynamic_shapes.cpp
:language: cpp
:fragment: ov_dynamic_shapes:get_input_tensor
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_dynamic_shapes.py get_input_tensor
.. doxygensnippet:: docs/snippets/ov_dynamic_shapes.py
:language: python
:fragment: get_input_tensor
@endsphinxtab
.. tab-item:: C
:sync: c
@sphinxtab{C}
.. doxygensnippet:: docs/snippets/ov_dynamic_shapes.c
:language: cpp
:fragment: ov_dynamic_shapes:get_input_tensor
@snippet docs/snippets/ov_dynamic_shapes.c ov_dynamic_shapes:get_input_tensor
@endsphinxtab
@endsphinxtabset
### Dynamic Shapes in Outputs
Dynamic Shapes in Outputs
+++++++++++++++++++++++++
Examples above are valid approaches when dynamic dimensions in output may be implied by propagation of dynamic dimension from the inputs.
For example, batch dimension in an input shape is usually propagated through the whole model and appears in the output shape.
@@ -199,56 +219,63 @@ It also applies to other dimensions, like sequence length for NLP models or spat
Whether the output has dynamic dimensions or not can be verified by querying the output partial shape after the model is read or reshaped.
The same applies to inputs. For example:
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_dynamic_shapes.cpp ov_dynamic_shapes:print_dynamic
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_dynamic_shapes.cpp
:language: cpp
:fragment: ov_dynamic_shapes:print_dynamic
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_dynamic_shapes.py print_dynamic
.. doxygensnippet:: docs/snippets/ov_dynamic_shapes.py
:language: python
:fragment: print_dynamic
@endsphinxtab
.. tab-item:: C
:sync: c
@sphinxtab{C}
.. doxygensnippet:: docs/snippets/ov_dynamic_shapes.c
:language: cpp
:fragment: ov_dynamic_shapes:print_dynamic
@snippet docs/snippets/ov_dynamic_shapes.c ov_dynamic_shapes:print_dynamic
@endsphinxtab
@endsphinxtabset
When there are dynamic dimensions in corresponding inputs or outputs, the `?` or ranges like `1..10` appear.
When there are dynamic dimensions in corresponding inputs or outputs, the ``?`` or ranges like ``1..10`` appear.
It can also be verified in a more programmatic way:
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_dynamic_shapes.cpp ov_dynamic_shapes:detect_dynamic
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_dynamic_shapes.cpp
:language: cpp
:fragment: ov_dynamic_shapes:detect_dynamic
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_dynamic_shapes.py detect_dynamic
.. doxygensnippet:: docs/snippets/ov_dynamic_shapes.py
:language: python
:fragment: detect_dynamic
@endsphinxtab
.. tab-item:: C
:sync: c
@sphinxtab{C}
@snippet docs/snippets/ov_dynamic_shapes.c ov_dynamic_shapes:detect_dynamic
@endsphinxtab
@endsphinxtabset
.. doxygensnippet:: docs/snippets/ov_dynamic_shapes.c
:language: cpp
:fragment: ov_dynamic_shapes:detect_dynamic
If at least one dynamic dimension exists in an output of a model, a shape of the corresponding output tensor will be set as the result of inference call.
Before the first inference, memory for such a tensor is not allocated and has the `[0]` shape.
If the `set_output_tensor` method is called with a pre-allocated tensor, the inference will call the `set_shape` internally, and the initial shape is replaced by the calculated shape.
Therefore, setting a shape for output tensors in this case is useful only when pre-allocating enough memory for output tensor. Normally, the `set_shape` method of a `Tensor` re-allocates memory only if a new shape requires more storage.
Before the first inference, memory for such a tensor is not allocated and has the ``[0]`` shape.
If the ``set_output_tensor`` method is called with a pre-allocated tensor, the inference will call the ``set_shape`` internally, and the initial shape is replaced by the calculated shape.
Therefore, setting a shape for output tensors in this case is useful only when pre-allocating enough memory for output tensor. Normally, the ``set_shape`` method of a ``Tensor`` re-allocates memory only if a new shape requires more storage.
@endsphinxdirective

444
docs/OV_Runtime_UG/ov_infer_request.md
View File

@@ -1,289 +1,335 @@
# OpenVINO™ Inference Request {#openvino_docs_OV_UG_Infer_request}
@sphinxdirective
OpenVINO™ Runtime uses Infer Request mechanism which allows running models on different devices in asynchronous or synchronous manners.
The `ov::InferRequest` class is used for this purpose inside the OpenVINO™ Runtime.
The ``ov::InferRequest`` class is used for this purpose inside the OpenVINO™ Runtime.
This class allows you to set and get data for model inputs, outputs and run inference for the model.
## Creating Infer Request
Creating Infer Request
######################
The `ov::InferRequest` can be created from the `ov::CompiledModel`:
The ``ov::InferRequest`` can be created from the ``ov::CompiledModel``:
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_infer_request.cpp create_infer_request
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_infer_request.cpp
:language: cpp
:fragment: create_infer_request
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_infer_request.py create_infer_request
.. doxygensnippet:: docs/snippets/ov_infer_request.py
:language: python
:fragment: create_infer_request
@endsphinxtab
@endsphinxtabset
Run Inference
####################
## Run Inference
The ``ov::InferRequest`` supports synchronous and asynchronous modes for inference.
The `ov::InferRequest` supports synchronous and asynchronous modes for inference.
Synchronous Mode
++++++++++++++++++++
### Synchronous Mode
You can use ``ov::InferRequest::infer``, which blocks the application execution, to infer a model in the synchronous mode:
You can use `ov::InferRequest::infer`, which blocks the application execution, to infer a model in the synchronous mode:
@sphinxtabset
.. tab-set::
@sphinxtab{C++}
.. tab-item:: C++
:sync: cpp
@snippet docs/snippets/ov_infer_request.cpp sync_infer
.. doxygensnippet:: docs/snippets/ov_infer_request.cpp
:language: cpp
:fragment: sync_infer
@endsphinxtab
.. tab-item:: Python
:sync: py
@sphinxtab{Python}
.. doxygensnippet:: docs/snippets/ov_infer_request.py
:language: python
:fragment: sync_infer
@snippet docs/snippets/ov_infer_request.py sync_infer
@endsphinxtab
Asynchronous Mode
++++++++++++++++++++
@endsphinxtabset
The asynchronous mode can improve application's overall frame-rate, by making it work on the host while the accelerator is busy, instead of waiting for inference to complete. To infer a model in the asynchronous mode, use ``ov::InferRequest::start_async``:
### Asynchronous Mode
The asynchronous mode can improve application's overall frame-rate, by making it work on the host while the accelerator is busy, instead of waiting for inference to complete. To infer a model in the asynchronous mode, use `ov::InferRequest::start_async`:
.. tab-set::
@sphinxtabset
.. tab-item:: C++
:sync: cpp
@sphinxtab{C++}
.. doxygensnippet:: docs/snippets/ov_infer_request.cpp
:language: cpp
:fragment: async_infer
@snippet docs/snippets/ov_infer_request.cpp async_infer
.. tab-item:: Python
:sync: py
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_infer_request.py
:language: python
:fragment: async_infer
@sphinxtab{Python}
@snippet docs/snippets/ov_infer_request.py async_infer
@endsphinxtab
@endsphinxtabset
Asynchronous mode supports two ways the application waits for inference results:
* `ov::InferRequest::wait_for` - specifies the maximum duration in milliseconds to block the method. The method is blocked until the specified time has passed, or the result becomes available, whichever comes first.
@sphinxtabset
@sphinxtab{C++}
* ``ov::InferRequest::wait_for`` - specifies the maximum duration in milliseconds to block the method. The method is blocked until the specified time has passed, or the result becomes available, whichever comes first.
@snippet docs/snippets/ov_infer_request.cpp wait_for
.. tab-set::
@endsphinxtab
.. tab-item:: C++
:sync: cpp
@sphinxtab{Python}
.. doxygensnippet:: docs/snippets/ov_infer_request.cpp
:language: cpp
:fragment: wait_for
@snippet docs/snippets/ov_infer_request.py wait_for
.. tab-item:: Python
:sync: py
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_infer_request.py
:language: python
:fragment: wait_for
@endsphinxtabset
* `ov::InferRequest::wait` - waits until inference result becomes available
@sphinxtabset
* ``ov::InferRequest::wait`` - waits until inference result becomes available
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_infer_request.cpp wait
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_infer_request.cpp
:language: cpp
:fragment: wait
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_infer_request.py wait
.. doxygensnippet:: docs/snippets/ov_infer_request.py
:language: python
:fragment: wait
@endsphinxtab
@endsphinxtabset
Both methods are thread-safe.
When you are running several inference requests in parallel, a device can process them simultaneously, with no guarantees on the completion order. This may complicate a possible logic based on the `ov::InferRequest::wait` (unless your code needs to wait for the _all_ requests). For multi-request scenarios, consider using the `ov::InferRequest::set_callback` method to set a callback which is called upon completion of the request:
@sphinxtabset
@sphinxtab{C++}
@snippet docs/snippets/ov_infer_request.cpp set_callback
@endsphinxtab
@sphinxtab{Python}
@snippet docs/snippets/ov_infer_request.py set_callback
@endsphinxtab
@endsphinxtabset
When you are running several inference requests in parallel, a device can process them simultaneously, with no guarantees on the completion order. This may complicate a possible logic based on the ``ov::InferRequest::wait`` (unless your code needs to wait for the *all* requests). For multi-request scenarios, consider using the ``ov::InferRequest::set_callback`` method to set a callback which is called upon completion of the request:
> **NOTE**: Use weak reference of infer_request (`ov::InferRequest*`, `ov::InferRequest&`, `std::weal_ptr<ov::InferRequest>`, etc.) in the callback. It is necessary to avoid cyclic references.
For more details, see the [Classification Async Sample](../../samples/cpp/classification_sample_async/README.md).
.. tab-set::
You can use the `ov::InferRequest::cancel` method if you want to abort execution of the current inference request:
.. tab-item:: C++
:sync: cpp
@sphinxtabset
.. doxygensnippet:: docs/snippets/ov_infer_request.cpp
:language: cpp
:fragment: set_callback
@sphinxtab{C++}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_infer_request.cpp cancel
@endsphinxtab
@sphinxtab{Python}
@snippet docs/snippets/ov_infer_request.py cancel
@endsphinxtab
@endsphinxtabset
@anchor in_out_tensors
## Working with Input and Output tensors
`ov::InferRequest` allows you to get input/output tensors by tensor name, index, port, and without any arguments, if a model has only one input or output.
* `ov::InferRequest::get_input_tensor`, `ov::InferRequest::set_input_tensor`, `ov::InferRequest::get_output_tensor`, `ov::InferRequest::set_output_tensor` methods without arguments can be used to get or set input/output tensor for a model with only one input/output:
@sphinxtabset
@sphinxtab{C++}
@snippet docs/snippets/ov_infer_request.cpp get_set_one_tensor
@endsphinxtab
@sphinxtab{Python}
@snippet docs/snippets/ov_infer_request.py get_set_one_tensor
@endsphinxtab
@endsphinxtabset
* `ov::InferRequest::get_input_tensor`, `ov::InferRequest::set_input_tensor`, `ov::InferRequest::get_output_tensor`, `ov::InferRequest::set_output_tensor` methods with argument can be used to get or set input/output tensor by input/output index:
@sphinxtabset
@sphinxtab{C++}
@snippet docs/snippets/ov_infer_request.cpp get_set_index_tensor
@endsphinxtab
@sphinxtab{Python}
@snippet docs/snippets/ov_infer_request.py get_set_index_tensor
@endsphinxtab
@endsphinxtabset
* `ov::InferRequest::get_tensor`, `ov::InferRequest::set_tensor` methods can be used to get or set input/output tensor by tensor name:
@sphinxtabset
@sphinxtab{C++}
@snippet docs/snippets/ov_infer_request.cpp get_set_tensor
@endsphinxtab
@sphinxtab{Python}
@snippet docs/snippets/ov_infer_request.py get_set_tensor
@endsphinxtab
@endsphinxtabset
* `ov::InferRequest::get_tensor`, `ov::InferRequest::set_tensor` methods can be used to get or set input/output tensor by port:
@sphinxtabset
@sphinxtab{C++}
@snippet docs/snippets/ov_infer_request.cpp get_set_tensor_by_port
@endsphinxtab
@sphinxtab{Python}
@snippet docs/snippets/ov_infer_request.py get_set_tensor_by_port
@endsphinxtab
@endsphinxtabset
.. doxygensnippet:: docs/snippets/ov_infer_request.py
:language: python
:fragment: set_callback
## Examples of Infer Request Usages
.. note::
Use weak reference of infer_request (``ov::InferRequest*``, ``ov::InferRequest&``, ``std::weal_ptr<ov::InferRequest>``, etc.) in the callback. It is necessary to avoid cyclic references.
For more details, see the `Classification Async Sample <../../samples/cpp/classification_sample_async/README.md>`_.
You can use the ``ov::InferRequest::cancel`` method if you want to abort execution of the current inference request:
.. tab-set::
.. tab-item:: C++
:sync: cpp
.. doxygensnippet:: docs/snippets/ov_infer_request.cpp
:language: cpp
:fragment: cancel
.. tab-item:: Python
:sync: py
.. doxygensnippet:: docs/snippets/ov_infer_request.py
:language: python
:fragment: cancel
.. _in_out_tensors:
Working with Input and Output tensors
#####################################
``ov::InferRequest`` allows you to get input/output tensors by tensor name, index, port, and without any arguments, if a model has only one input or output.
* ``ov::InferRequest::get_input_tensor``, ``ov::InferRequest::set_input_tensor``, ``ov::InferRequest::get_output_tensor``, ``ov::InferRequest::set_output_tensor`` methods without arguments can be used to get or set input/output tensor for a model with only one input/output:
.. tab-set::
.. tab-item:: C++
:sync: cpp
.. doxygensnippet:: docs/snippets/ov_infer_request.cpp
:language: cpp
:fragment: get_set_one_tensor
.. tab-item:: Python
:sync: py
.. doxygensnippet:: docs/snippets/ov_infer_request.py
:language: python
:fragment: get_set_one_tensor
* ``ov::InferRequest::get_input_tensor``, ``ov::InferRequest::set_input_tensor``, ``ov::InferRequest::get_output_tensor``, ``ov::InferRequest::set_output_tensor`` methods with argument can be used to get or set input/output tensor by input/output index:
.. tab-set::
.. tab-item:: C++
:sync: cpp
.. doxygensnippet:: docs/snippets/ov_infer_request.cpp
:language: cpp
:fragment: get_set_index_tensor
.. tab-item:: Python
:sync: py
.. doxygensnippet:: docs/snippets/ov_infer_request.py
:language: python
:fragment: get_set_index_tensor
* ``ov::InferRequest::get_tensor``, ``ov::InferRequest::set_tensor`` methods can be used to get or set input/output tensor by tensor name:
.. tab-set::
.. tab-item:: C++
:sync: cpp
.. doxygensnippet:: docs/snippets/ov_infer_request.cpp
:language: cpp
:fragment: get_set_tensor
.. tab-item:: Python
:sync: py
.. doxygensnippet:: docs/snippets/ov_infer_request.py
:language: python
:fragment: get_set_tensor
* ``ov::InferRequest::get_tensor``, ``ov::InferRequest::set_tensor`` methods can be used to get or set input/output tensor by port:
.. tab-set::
.. tab-item:: C++
:sync: cpp
.. doxygensnippet:: docs/snippets/ov_infer_request.cpp
:language: cpp
:fragment: get_set_tensor_by_port
.. tab-item:: Python
:sync: py
.. doxygensnippet:: docs/snippets/ov_infer_request.py
:language: python
:fragment: get_set_tensor_by_port
Examples of Infer Request Usages
################################
Presented below are examples of what the Infer Request can be used for.
### Cascade of Models
Cascade of Models
++++++++++++++++++++
`ov::InferRequest` can be used to organize a cascade of models. Infer Requests are required for each model.
In this case, you can get the output tensor from the first request, using `ov::InferRequest::get_tensor` and set it as input for the second request, using `ov::InferRequest::set_tensor`. Keep in mind that tensors shared across compiled models can be rewritten by the first model if the first infer request is run once again, while the second model has not started yet.
``ov::InferRequest`` can be used to organize a cascade of models. Infer Requests are required for each model.
In this case, you can get the output tensor from the first request, using ``ov::InferRequest::get_tensor`` and set it as input for the second request, using ``ov::InferRequest::set_tensor``. Keep in mind that tensors shared across compiled models can be rewritten by the first model if the first infer request is run once again, while the second model has not started yet.
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_infer_request.cpp cascade_models
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_infer_request.cpp
:language: cpp
:fragment: cascade_models
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_infer_request.py cascade_models
.. doxygensnippet:: docs/snippets/ov_infer_request.py
:language: python
:fragment: cascade_models
@endsphinxtab
@endsphinxtabset
Using of ROI Tensors
++++++++++++++++++++
### Using of ROI Tensors
It is possible to re-use shared input in several models. You do not need to allocate a separate input tensor for a model if it processes a ROI object located inside of an already allocated input of a previous model. For instance, when the first model detects objects in a video frame (stored as an input tensor) and the second model accepts detected bounding boxes (ROI inside of the frame) as input. In this case, it is allowed to re-use a pre-allocated input tensor (used by the first model) by the second model and just crop ROI without allocation of new memory, using ``ov::Tensor`` with passing ``ov::Tensor`` and ``ov::Coordinate`` as parameters.
It is possible to re-use shared input in several models. You do not need to allocate a separate input tensor for a model if it processes a ROI object located inside of an already allocated input of a previous model. For instance, when the first model detects objects in a video frame (stored as an input tensor) and the second model accepts detected bounding boxes (ROI inside of the frame) as input. In this case, it is allowed to re-use a pre-allocated input tensor (used by the first model) by the second model and just crop ROI without allocation of new memory, using `ov::Tensor` with passing `ov::Tensor` and `ov::Coordinate` as parameters.
@sphinxtabset
.. tab-set::
@sphinxtab{C++}
.. tab-item:: C++
:sync: cpp
@snippet docs/snippets/ov_infer_request.cpp roi_tensor
.. doxygensnippet:: docs/snippets/ov_infer_request.cpp
:language: cpp
:fragment: roi_tensor
@endsphinxtab
.. tab-item:: Python
:sync: py
@sphinxtab{Python}
.. doxygensnippet:: docs/snippets/ov_infer_request.py
:language: python
:fragment: roi_tensor
@snippet docs/snippets/ov_infer_request.py roi_tensor
@endsphinxtab
Using Remote Tensors
++++++++++++++++++++
@endsphinxtabset
By using ``ov::RemoteContext`` you can create a remote tensor to work with remote device memory.
### Using Remote Tensors
By using `ov::RemoteContext` you can create a remote tensor to work with remote device memory.
.. tab-set::
@sphinxtabset
.. tab-item:: C++
:sync: cpp
@sphinxtab{C++}
.. doxygensnippet:: docs/snippets/ov_infer_request.cpp
:language: cpp
:fragment: remote_tensor
@snippet docs/snippets/ov_infer_request.cpp remote_tensor
.. tab-item:: Python
:sync: py
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_infer_request.py
:language: python
:fragment: remote_tensor
@sphinxtab{Python}
@snippet docs/snippets/ov_infer_request.py remote_tensor
@endsphinxtab
@endsphinxtabset
@endsphinxdirective

410
docs/OV_Runtime_UG/preprocessing_details.md
View File

@@ -1,320 +1,365 @@
# Preprocessing API - details {#openvino_docs_OV_UG_Preprocessing_Details}
@sphinxdirective
The purpose of this article is to present details on preprocessing API, such as its capabilities and post-processing.
## Pre-processing Capabilities
Pre-processing Capabilities
###########################
Below is a full list of pre-processing API capabilities:
### Addressing Particular Input/Output
Addressing Particular Input/Output
++++++++++++++++++++++++++++++++++
If the model has only one input, then simple `ov::preprocess::PrePostProcessor::input()` will get a reference to pre-processing builder for this input (a tensor, the steps, a model):
If the model has only one input, then simple `ov::preprocess::PrePostProcessor::input() <classov_1_1preprocess_1_1PrePostProcessor.html#doxid-classov-1-1preprocess-1-1-pre-post-processor-1a611b930e59cd16176f380d21e755cda1>`__ will get a reference to pre-processing builder for this input (a tensor, the steps, a model):
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_preprocessing.cpp ov:preprocess:input_1
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_preprocessing.cpp
:language: cpp
:fragment: ov:preprocess:input_1
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_preprocessing.py ov:preprocess:input_1
@endsphinxtab
@endsphinxtabset
.. doxygensnippet:: docs/snippets/ov_preprocessing.py
:language: python
:fragment: ov:preprocess:input_1
In general, when a model has multiple inputs/outputs, each one can be addressed by a tensor name.
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_preprocessing.cpp ov:preprocess:input_name
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_preprocessing.cpp
:language: cpp
:fragment: ov:preprocess:input_name
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_preprocessing.py ov:preprocess:input_name
@endsphinxtab
@endsphinxtabset
.. doxygensnippet:: docs/snippets/ov_preprocessing.py
:language: python
:fragment: ov:preprocess:input_name
Or by it's index.
@sphinxtabset
.. tab-set::
@sphinxtab{C++}
.. tab-item:: C++
:sync: cpp
@snippet docs/snippets/ov_preprocessing.cpp ov:preprocess:input_index
.. doxygensnippet:: docs/snippets/ov_preprocessing.cpp
:language: cpp
:fragment: ov:preprocess:input_index
@endsphinxtab
.. tab-item:: Python
:sync: py
@sphinxtab{Python}
.. doxygensnippet:: docs/snippets/ov_preprocessing.py
:language: python
:fragment: ov:preprocess:input_index
@snippet docs/snippets/ov_preprocessing.py ov:preprocess:input_index
@endsphinxtab
@endsphinxtabset
C++ references:
* `ov::preprocess::InputTensorInfo`
* `ov::preprocess::OutputTensorInfo`
* `ov::preprocess::PrePostProcessor`
* `ov::preprocess::InputTensorInfo <classov_1_1preprocess_1_1InputTensorInfo.html#doxid-classov-1-1preprocess-1-1-input-tensor-info>`__
* `ov::preprocess::OutputTensorInfo <classov_1_1preprocess_1_1OutputTensorInfo.html#doxid-classov-1-1preprocess-1-1-output-tensor-info>`__
* `ov::preprocess::PrePostProcessor <classov_1_1preprocess_1_1PrePostProcessor.html#doxid-classov-1-1preprocess-1-1-pre-post-processor>`__
### Supported Pre-processing Operations
Supported Pre-processing Operations
+++++++++++++++++++++++++++++++++++
C++ references:
* `ov::preprocess::PreProcessSteps`
#### Mean/Scale Normalization
* `ov::preprocess::PreProcessSteps <classov_1_1preprocess_1_1PreProcessSteps.html#doxid-classov-1-1preprocess-1-1-pre-process-steps>`__
Mean/Scale Normalization
------------------------
Typical data normalization includes 2 operations for each data item: subtract mean value and divide to standard deviation. This can be done with the following code:
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_preprocessing.cpp ov:preprocess:mean_scale
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_preprocessing.cpp
:language: cpp
:fragment: ov:preprocess:mean_scale
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_preprocessing.py ov:preprocess:mean_scale
.. doxygensnippet:: docs/snippets/ov_preprocessing.py
:language: python
:fragment: ov:preprocess:mean_scale
@endsphinxtab
@endsphinxtabset
In Computer Vision area normalization is usually done separately for R, G, B values. To do this, :doc:`layout with 'C' dimension <openvino_docs_OV_UG_Layout_Overview>` shall be defined. Example:
In Computer Vision area normalization is usually done separately for R, G, B values. To do this, [layout with 'C' dimension](./layout_overview.md) shall be defined. Example:
@sphinxtabset
.. tab-set::
@sphinxtab{C++}
.. tab-item:: C++
:sync: cpp
@snippet docs/snippets/ov_preprocessing.cpp ov:preprocess:mean_scale_array
.. doxygensnippet:: docs/snippets/ov_preprocessing.cpp
:language: cpp
:fragment: ov:preprocess:mean_scale_array
@endsphinxtab
.. tab-item:: Python
:sync: py
@sphinxtab{Python}
.. doxygensnippet:: docs/snippets/ov_preprocessing.py
:language: python
:fragment: ov:preprocess:mean_scale_array
@snippet docs/snippets/ov_preprocessing.py ov:preprocess:mean_scale_array
@endsphinxtab
@endsphinxtabset
C++ references:
* `ov::preprocess::PreProcessSteps::mean()`
* `ov::preprocess::PreProcessSteps::scale()`
* `ov::preprocess::PreProcessSteps::mean() <classov_1_1preprocess_1_1PreProcessSteps.html#doxid-classov-1-1preprocess-1-1-pre-process-steps-1aef1bb8c1fc5eb0014b07b78749c432dc>`__
* `ov::preprocess::PreProcessSteps::scale() <classov_1_1preprocess_1_1PreProcessSteps.html#doxid-classov-1-1preprocess-1-1-pre-process-steps-1aeacaf406d72a238e31a359798ebdb3b7>`__
#### Converting Precision
Converting Precision
--------------------
In Computer Vision, the image is represented by an array of unsigned 8-bit integer values (for each color), but the model accepts floating point tensors.
To integrate precision conversion into an execution graph as a pre-processing step:
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_preprocessing.cpp ov:preprocess:convert_element_type
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_preprocessing.cpp
:language: cpp
:fragment: ov:preprocess:convert_element_type
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_preprocessing.py ov:preprocess:convert_element_type
.. doxygensnippet:: docs/snippets/ov_preprocessing.py
:language: python
:fragment: ov:preprocess:convert_element_type
@endsphinxtab
@endsphinxtabset
C++ references:
* `ov::preprocess::InputTensorInfo::set_element_type()`
* `ov::preprocess::PreProcessSteps::convert_element_type()`
* `ov::preprocess::InputTensorInfo::set_element_type() <classov_1_1preprocess_1_1InputTensorInfo.html#doxid-classov-1-1preprocess-1-1-input-tensor-info-1a98fb73ff9178c8c71d809ddf8927faf5>`__
* `ov::preprocess::PreProcessSteps::convert_element_type() <classov_1_1preprocess_1_1PreProcessSteps.html#doxid-classov-1-1preprocess-1-1-pre-process-steps-1aac6316155a1690609eb320637c193d50>`__
#### Converting layout (transposing)
Converting layout (transposing)
-------------------------------
Transposing of matrices/tensors is a typical operation in Deep Learning - you may have a BMP image 640x480, which is an array of `{480, 640, 3}` elements, but Deep Learning model can require input with shape `{1, 3, 480, 640}`.
Transposing of matrices/tensors is a typical operation in Deep Learning - you may have a BMP image 640x480, which is an array of ``{480, 640, 3}`` elements, but Deep Learning model can require input with shape ``{1, 3, 480, 640}``.
Conversion can be done implicitly, using the [layout](./layout_overview.md) of a user's tensor and the layout of an original model.
@sphinxtabset
@sphinxtab{C++}
@snippet docs/snippets/ov_preprocessing.cpp ov:preprocess:convert_layout
@endsphinxtab
@sphinxtab{Python}
@snippet docs/snippets/ov_preprocessing.py ov:preprocess:convert_layout
@endsphinxtab
@endsphinxtabset
Conversion can be done implicitly, using the :doc:`layout <openvino_docs_OV_UG_Layout_Overview>` of a user's tensor and the layout of an original model.
For a manual transpose of axes without the use of a [layout](./layout_overview.md) in the code:
.. tab-set::
@sphinxtabset
.. tab-item:: C++
:sync: cpp
@sphinxtab{C++}
.. doxygensnippet:: docs/snippets/ov_preprocessing.cpp
:language: cpp
:fragment: ov:preprocess:convert_layout
@snippet docs/snippets/ov_preprocessing.cpp ov:preprocess:convert_layout_2
.. tab-item:: Python
:sync: py
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_preprocessing.py
:language: python
:fragment: ov:preprocess:convert_layout
@sphinxtab{Python}
@snippet docs/snippets/ov_preprocessing.py ov:preprocess:convert_layout_2
For a manual transpose of axes without the use of a :doc:`layout <openvino_docs_OV_UG_Layout_Overview>` in the code:
@endsphinxtab
@endsphinxtabset
.. tab-set::
.. tab-item:: C++
:sync: cpp
.. doxygensnippet:: docs/snippets/ov_preprocessing.cpp
:language: cpp
:fragment: ov:preprocess:convert_layout_2
.. tab-item:: Python
:sync: py
.. doxygensnippet:: docs/snippets/ov_preprocessing.py
:language: python
:fragment: ov:preprocess:convert_layout_2
It performs the same transpose. However, the approach where source and destination layout are used can be easier to read and understand.
C++ references:
* `ov::preprocess::PreProcessSteps::convert_layout()`
* `ov::preprocess::InputTensorInfo::set_layout()`
* `ov::preprocess::InputModelInfo::set_layout()`
* `ov::Layout`
#### Resizing Image
* `ov::preprocess::PreProcessSteps::convert_layout() <classov_1_1preprocess_1_1PreProcessSteps.html#doxid-classov-1-1preprocess-1-1-pre-process-steps-1a0f65fdadca32e90f5ef3a323b640b978>`__
* `ov::preprocess::InputTensorInfo::set_layout() <classov_1_1preprocess_1_1InputTensorInfo.html#doxid-classov-1-1preprocess-1-1-input-tensor-info-1a6f70eb97d02e90a30cd748573abd7b4b>`__
* `ov::preprocess::InputModelInfo::set_layout() <classov_1_1preprocess_1_1InputModelInfo.html#doxid-classov-1-1preprocess-1-1-input-model-info-1af309bac02af20d048e349a2d421c1169>`__
* `ov::Layout <classov_1_1Layout.html#doxid-classov-1-1-layout>`__
Resizing Image
--------------------
Resizing an image is a typical pre-processing step for computer vision tasks. With pre-processing API, this step can also be integrated into an execution graph and performed on a target device.
To resize the input image, it is needed to define `H` and `W` dimensions of the [layout](./layout_overview.md)
To resize the input image, it is needed to define ``H`` and ``W`` dimensions of the :doc:`layout <openvino_docs_OV_UG_Layout_Overview>`.
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_preprocessing.cpp ov:preprocess:resize_1
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_preprocessing.cpp
:language: cpp
:fragment: ov:preprocess:resize_1
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_preprocessing.py ov:preprocess:resize_1
.. doxygensnippet:: docs/snippets/ov_preprocessing.py
:language: python
:fragment: ov:preprocess:resize_1
@endsphinxtab
@endsphinxtabset
When original model has known spatial dimensions (``width``+``height``), target ``width``/``height`` can be omitted.
When original model has known spatial dimensions (`width`+`height`), target `width`/`height` can be omitted.
@sphinxtabset
.. tab-set::
@sphinxtab{C++}
.. tab-item:: C++
:sync: cpp
@snippet docs/snippets/ov_preprocessing.cpp ov:preprocess:resize_2
.. doxygensnippet:: docs/snippets/ov_preprocessing.cpp
:language: cpp
:fragment: ov:preprocess:resize_2
@endsphinxtab
.. tab-item:: Python
:sync: py
@sphinxtab{Python}
.. doxygensnippet:: docs/snippets/ov_preprocessing.py
:language: python
:fragment: ov:preprocess:resize_2
@snippet docs/snippets/ov_preprocessing.py ov:preprocess:resize_2
@endsphinxtab
@endsphinxtabset
C++ references:
* `ov::preprocess::PreProcessSteps::resize()`
* `ov::preprocess::ResizeAlgorithm`
#### Color Conversion
Color Conversion
--------------------
Typical use case is to reverse color channels from `RGB` to `BGR` and vice versa. To do this, specify source color format in `tensor` section and perform `convert_color` pre-processing operation. In the example below, a `BGR` image needs to be converted to `RGB` as required for the model input.
Typical use case is to reverse color channels from ``RGB`` to ``BGR`` and vice versa. To do this, specify source color format in ``tensor`` section and perform ``convert_color`` pre-processing operation. In the example below, a ``BGR`` image needs to be converted to ``RGB`` as required for the model input.
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_preprocessing.cpp ov:preprocess:convert_color_1
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_preprocessing.cpp
:language: cpp
:fragment: ov:preprocess:convert_color_1
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_preprocessing.py ov:preprocess:convert_color_1
.. doxygensnippet:: docs/snippets/ov_preprocessing.py
:language: python
:fragment: ov:preprocess:convert_color_1
@endsphinxtab
@endsphinxtabset
Color Conversion - NV12/I420
----------------------------
#### Color Conversion - NV12/I420
Pre-processing also supports YUV-family source color formats, i.e. NV12 and I420.
In advanced cases, such YUV images can be split into separate planes, e.g., for NV12 images Y-component may come from one source and UV-component from another one. Concatenating such components in user's application manually is not a perfect solution from performance and device utilization perspectives. However, there is a way to use Pre-processing API. For such cases there are `NV12_TWO_PLANES` and `I420_THREE_PLANES` source color formats, which will split the original `input` into 2 or 3 inputs.
In advanced cases, such YUV images can be split into separate planes, e.g., for NV12 images Y-component may come from one source and UV-component from another one. Concatenating such components in user's application manually is not a perfect solution from performance and device utilization perspectives. However, there is a way to use Pre-processing API. For such cases there are ``NV12_TWO_PLANES`` and ``I420_THREE_PLANES`` source color formats, which will split the original ``input`` into 2 or 3 inputs.
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_preprocessing.cpp ov:preprocess:convert_color_2
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_preprocessing.cpp
:language: cpp
:fragment: ov:preprocess:convert_color_2
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_preprocessing.py ov:preprocess:convert_color_2
.. doxygensnippet:: docs/snippets/ov_preprocessing.py
:language: python
:fragment: ov:preprocess:convert_color_2
@endsphinxtab
@endsphinxtabset
In this example, the original `input` is split to `input/y` and `input/uv` inputs. You can fill `input/y` from one source, and `input/uv` from another source. Color conversion to `RGB` will be performed, using these sources. It is more efficient as there will be no additional copies of NV12 buffers.
In this example, the original ``input`` is split to ``input/y`` and ``input/uv`` inputs. You can fill ``input/y`` from one source, and ``input/uv`` from another source. Color conversion to ``RGB`` will be performed, using these sources. It is more efficient as there will be no additional copies of NV12 buffers.
C++ references:
* `ov::preprocess::ColorFormat`
* `ov::preprocess::PreProcessSteps::convert_color`
* `ov::preprocess::ColorFormat <classov_1_1preprocess_1_1PreProcessSteps.html#doxid-classov-1-1preprocess-1-1-pre-process-steps-1aef1bb8c1fc5eb0014b07b78749c432dc>`__
* `ov::preprocess::PreProcessSteps::convert_color <classov_1_1preprocess_1_1PreProcessSteps.html#doxid-classov-1-1preprocess-1-1-pre-process-steps-1aac6316155a1690609eb320637c193d50>`__
### Custom Operations
Custom Operations
++++++++++++++++++++
Pre-processing API also allows adding `custom` preprocessing steps into an execution graph. The `custom` function accepts the current `input` node, applies the defined preprocessing operations, and returns a new node.
Pre-processing API also allows adding ``custom`` preprocessing steps into an execution graph. The ``custom`` function accepts the current ``input`` node, applies the defined preprocessing operations, and returns a new node.
.. note::
Custom pre-processing function should only insert node(s) after the input. It is done during model compilation. This function will NOT be called during the execution phase. This may appear to be complicated and require knowledge of :doc:`OpenVINO™ operations <openvino_docs_ops_opset>`.
> **NOTE** : Custom pre-processing function should only insert node(s) after the input. It is done during model compilation. This function will NOT be called during the execution phase. This may appear to be complicated and require knowledge of [OpenVINO™ operations](../ops/opset.md).
If there is a need to insert additional operations to the execution graph right after the input, like some specific crops and/or resizes - Pre-processing API can be a good choice to implement this.
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_preprocessing.cpp ov:preprocess:custom
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_preprocessing.cpp
:language: cpp
:fragment: ov:preprocess:custom
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_preprocessing.py ov:preprocess:custom
.. doxygensnippet:: docs/snippets/ov_preprocessing.py
:language: python
:fragment: ov:preprocess:custom
@endsphinxtab
@endsphinxtabset
C++ references:
* `ov::preprocess::PreProcessSteps::custom()`
* [Available Operations Sets](../ops/opset.md)
## Post-processing
* `ov::preprocess::PreProcessSteps::custom() <classov_1_1preprocess_1_1PreProcessSteps.html#doxid-classov-1-1preprocess-1-1-pre-process-steps-1aa88ce522ef69253e4d978f10c3b566f1>`__
* :doc:`Available Operations Sets <openvino_docs_ops_opset>`
Post-processing
####################
Post-processing steps can be added to model outputs. As for pre-processing, these steps will be also integrated into a graph and executed on a selected device.
@@ -323,29 +368,36 @@ Pre-processing uses the following flow: **User tensor** -> **Steps** -> **Model
Post-processing uses the reverse: **Model output** -> **Steps** -> **User tensor**.
Compared to pre-processing, there are not as many operations needed for the post-processing stage. Currently, only the following post-processing operations are supported:
- Convert a [layout](./layout_overview.md).
- Convert an element type.
- Customize operations.
* Convert a :doc:`layout <openvino_docs_OV_UG_Layout_Overview>`.
* Convert an element type.
* Customize operations.
Usage of these operations is similar to pre-processing. See the following example:
@sphinxtabset
@sphinxtab{C++}
.. tab-set::
@snippet docs/snippets/ov_preprocessing.cpp ov:preprocess:postprocess
.. tab-item:: C++
:sync: cpp
@endsphinxtab
.. doxygensnippet:: docs/snippets/ov_preprocessing.cpp
:language: cpp
:fragment: ov:preprocess:postprocess
@sphinxtab{Python}
.. tab-item:: Python
:sync: py
@snippet docs/snippets/ov_preprocessing.py ov:preprocess:postprocess
.. doxygensnippet:: docs/snippets/ov_preprocessing.py
:language: python
:fragment: ov:preprocess:postprocess
@endsphinxtab
@endsphinxtabset
C++ references:
* `ov::preprocess::PostProcessSteps`
* `ov::preprocess::OutputModelInfo`
* `ov::preprocess::OutputTensorInfo`
* `ov::preprocess::PostProcessSteps <classov_1_1preprocess_1_1PostProcessSteps.html#doxid-classov-1-1preprocess-1-1-post-process-steps>`__
* `ov::preprocess::OutputModelInfo <classov_1_1preprocess_1_1OutputModelInfo.html#doxid-classov-1-1preprocess-1-1-output-model-info>`__
* `ov::preprocess::OutputTensorInfo <classov_1_1preprocess_1_1OutputTensorInfo.html#doxid-classov-1-1preprocess-1-1-output-tensor-info>`__
@endsphinxdirective

93
docs/OV_Runtime_UG/supported_plugins/ARM_CPU.md
View File

@@ -1,56 +1,84 @@
# Arm® CPU Device {#openvino_docs_OV_UG_supported_plugins_ARM_CPU}
## Introducing the Arm® CPU Plugin
The Arm® CPU plugin is developed in order to enable deep neural networks inference on Arm® CPU, using [Compute Library](https://github.com/ARM-software/ComputeLibrary) as a backend.
> **NOTE**: This is a community-level add-on to OpenVINO™. Intel® welcomes community participation in the OpenVINO™ ecosystem, technical questions and code contributions on community forums. However, this component has not undergone full release validation or qualification from Intel®, hence no official support is offered.
The set of supported layers and its limitations are defined on the [Op-set specification page](https://github.com/openvinotoolkit/openvino_contrib/wiki/ARM-plugin-operation-set-specification).
@sphinxdirective
## Supported Inference Data Types
Introducing the Arm® CPU Plugin
#######################################
The Arm® CPU plugin is developed in order to enable deep neural networks inference on Arm® CPU, using `Compute Library <https://github.com/ARM-software/ComputeLibrary>`__ as a backend.
.. note::
This is a community-level add-on to OpenVINO™. Intel® welcomes community participation in the OpenVINO™ ecosystem,
as well as technical questions and code contributions on community forums. However, this component has not undergone
full release validation or qualification from Intel®, hence no official support is offered.
The set of supported layers and their limitations is defined in the
`Op-set specification page <https://github.com/openvinotoolkit/openvino_contrib/wiki/ARM-plugin-operation-set-specification>`__.
Supported Inference Data Types
#######################################
The Arm® CPU plugin supports the following data types as inference precision of internal primitives:
- Floating-point data types:
- f32
- f16
- Quantized data types:
- i8 (support is experimental)
[Hello Query Device C++ Sample](../../../samples/cpp/hello_query_device/README.md) can be used to print out supported data types for all detected devices.
:doc:`<Hello Query Device C++ Sample <openvino_inference_engine_samples_hello_query_device_README>` can be used to print out supported data types for all detected devices.
## Supported Features
Supported Features
#######################################
Preprocessing Acceleration
+++++++++++++++++++++++++++++++++++++++
**Preprocessing Acceleration**
The Arm® CPU plugin supports the following accelerated preprocessing operations:
- Precision conversion:
- u8 -> u16, s16, s32
- u16 -> u8, u32
- s16 -> u8, s32
- f16 -> f32
- u8 -> u16, s16, s32
- u16 -> u8, u32
- s16 -> u8, s32
- f16 -> f32
- Transposition of tensors with dims < 5
- Interpolation of 4D tensors with no padding (`pads_begin` and `pads_end` equal 0).
- Interpolation of 4D tensors with no padding (``pads_begin`` and ``pads_end`` equal 0).
The Arm® CPU plugin supports the following preprocessing operations, however they are not accelerated:
- Precision conversion that is not mentioned above
- Color conversion:
- NV12 to RGB
- NV12 to BGR
- i420 to RGB
- i420 to BGR
For more details, see the [preprocessing API guide](../preprocessing_overview.md).
- NV12 to RGB
- NV12 to BGR
- i420 to RGB
- i420 to BGR
## Supported Properties
The plugin supports the properties listed below.
For more details, see the :doc:`preprocessing API guide <openvino_docs_OV_UG_Preprocessing_Overview>`.
**Read-write Properties**
In order to take effect, all parameters must be set before calling `ov::Core::compile_model()` or passed as additional argument to `ov::Core::compile_model()`
Supported Properties
#######################################
Read-write Properties
+++++++++++++++++++++++++++++++++++++++
In order to take effect, all parameters must be set before calling ``ov::Core::compile_model()`` or passed as additional argument to ``ov::Core::compile_model()``
- ov::enable_profiling
**Read-only Properties**
Read-only Properties
+++++++++++++++++++++++++++++++++++++++
- ov::supported_properties
- ov::available_devices
- ov::range_for_async_infer_requests
@@ -59,7 +87,14 @@ In order to take effect, all parameters must be set before calling `ov::Core::co
- ov::device::capabilities
## Additional Resources
* [Arm® plugin developer documentation](https://github.com/openvinotoolkit/openvino_contrib/blob/master/modules/arm_plugin/README.md).
* [How to run YOLOv4 model inference using OpenVINO™ and OpenCV on Arm®](https://opencv.org/how-to-run-yolov4-using-openvino-and-opencv-on-arm/).
* [Face recognition on Android™ using OpenVINO™ toolkit with Arm® plugin](https://opencv.org/face-recognition-on-android-using-openvino-toolkit-with-arm-plugin/).
Additional Resources
#######################################
* `Arm® plugin developer documentation <https://github.com/openvinotoolkit/openvino_contrib/blob/master/modules/arm_plugin/README.md>`__.
* `How to run YOLOv4 model inference using OpenVINO™ and OpenCV on Arm® <https://opencv.org/how-to-run-yolov4-using-openvino-and-opencv-on-arm/>`__.
* `Face recognition on Android™ using OpenVINO™ toolkit with Arm® plugin <https://opencv.org/face-recognition-on-android-using-openvino-toolkit-with-arm-plugin/>`__.
@endsphinxdirective

4
docs/OV_Runtime_UG/supported_plugins/CPU.md
View File

@@ -3,7 +3,7 @@
The CPU plugin is a part of the Intel® Distribution of OpenVINO™ toolkit. It is developed to achieve high performance inference of neural networks on Intel® x86-64 CPUs.
For an in-depth description of CPU plugin, see:
- [CPU plugin developers documentation](https://github.com/openvinotoolkit/openvino/wiki/CPUPluginDevelopersDocs).
- [CPU plugin developers documentation](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/cmake_options_for_custom_comiplation.md).
- [OpenVINO Runtime CPU plugin source files](https://github.com/openvinotoolkit/openvino/tree/master/src/plugins/intel_cpu/).
@@ -320,4 +320,4 @@ Currently, the `sparse weights decompression feature` is supported with the foll
## Additional Resources
* [Supported Devices](Supported_Devices.md)
* [Optimization guide](@ref openvino_docs_deployment_optimization_guide_dldt_optimization_guide)
* [СPU plugin developers documentation](https://github.com/openvinotoolkit/openvino/wiki/CPUPluginDevelopersDocs)
* [СPU plugin developers documentation](https://github.com/openvinotoolkit/openvino/blob/master/src/plugins/intel_cpu/README.md)

104
docs/OV_Runtime_UG/supported_plugins/Device_Plugins.md
View File

@@ -12,78 +12,89 @@
openvino_docs_OV_UG_supported_plugins_GNA
openvino_docs_OV_UG_supported_plugins_ARM_CPU
@endsphinxdirective
OpenVINO™ Runtime can infer deep learning models using the following device types:
* [CPU](CPU.md)
* [GPU](GPU.md)
* [GNA](GNA.md)
* [Arm® CPU](ARM_CPU.md)
* :doc:`CPU <openvino_docs_OV_UG_supported_plugins_CPU>`
* :doc:`GPU <openvino_docs_OV_UG_supported_plugins_GPU>`
* :doc:`GNA <openvino_docs_OV_UG_supported_plugins_GNA>`
* :doc:`Arm® CPU <openvino_docs_OV_UG_supported_plugins_ARM_CPU>`
For a more detailed list of hardware, see [Supported Devices](./Supported_Devices.md)
For a more detailed list of hardware, see :doc:`Supported Devices <openvino_docs_OV_UG_supported_plugins_Supported_Devices>`.
Devices similar to the ones used for benchmarking can be accessed, using [Intel® DevCloud for the Edge](https://devcloud.intel.com/edge/), a remote development environment with access to Intel® hardware and the latest versions of the Intel® Distribution of the OpenVINO™ Toolkit. [Learn more](https://devcloud.intel.com/edge/get_started/devcloud/) or [Register here](https://inteliot.force.com/DevcloudForEdge/s/).
Devices similar to the ones used for benchmarking can be accessed, using `Intel® DevCloud for the Edge <https://devcloud.intel.com/edge/>`__,
a remote development environment with access to Intel® hardware and the latest versions of the Intel® Distribution of the OpenVINO™ Toolkit.
`Learn more <https://devcloud.intel.com/edge/get_started/devcloud/>`__ or `Register here <https://inteliot.force.com/DevcloudForEdge/s/>`__.
@anchor features_support_matrix
## Feature Support Matrix
.. _devicesupport-feature-support-matrix:
Feature Support Matrix
#######################################
The table below demonstrates support of key features by OpenVINO device plugins.
| Capability | [CPU](CPU.md) | [GPU](GPU.md) | [GNA](GNA.md) |[Arm® CPU](ARM_CPU.md) |
| ---------- | --- | --- | --- | --- |
| [Heterogeneous execution](../hetero_execution.md)| Yes | Yes | No | Yes |
| [Multi-device execution](../multi_device.md) | Yes | Yes | Partial | Yes |
| [Automatic batching](../automatic_batching.md) | No | Yes | No | No |
| [Multi-stream execution](../../optimization_guide/dldt_deployment_optimization_tput.md) | Yes | Yes | No | Yes |
| [Models caching](../Model_caching_overview.md) | Yes | Partial | Yes | No |
| [Dynamic shapes](../ov_dynamic_shapes.md) | Yes | Partial | No | No |
| [Import/Export](../../../tools/compile_tool/README.md) | Yes | No | Yes | No |
| [Preprocessing acceleration](../preprocessing_overview.md) | Yes | Yes | No | Partial |
| [Stateful models](../network_state_intro.md) | Yes | No | Yes | No |
| [Extensibility](@ref openvino_docs_Extensibility_UG_Intro) | Yes | Yes | No | No |
========================================================================================= =============== =============== =============== ========================
Capability CPU GPU GNA Arm® CPU
========================================================================================= =============== =============== =============== ========================
:doc:`Heterogeneous execution <openvino_docs_OV_UG_Hetero_execution>` Yes Yes No Yes
:doc:`Multi-device execution <openvino_docs_OV_UG_Running_on_multiple_devices>` Yes Yes Partial Yes
:doc:`Automatic batching <openvino_docs_OV_UG_Automatic_Batching>` No Yes No No
:doc:`Multi-stream execution <openvino_docs_deployment_optimization_guide_tput>` Yes Yes No Yes
:doc:`Models caching <openvino_docs_OV_UG_Model_caching_overview>` Yes Partial Yes No
:doc:`Dynamic shapes <openvino_docs_OV_UG_DynamicShapes>` Yes Partial No No
:doc:`Import/Export <openvino_inference_engine_tools_compile_tool_README>` Yes No Yes No
:doc:`Preprocessing acceleration <openvino_docs_OV_UG_Preprocessing_Overview>` Yes Yes No Partial
:doc:`Stateful models <openvino_docs_OV_UG_network_state_intro>` Yes No Yes No
:doc:`Extensibility <openvino_docs_Extensibility_UG_Intro>` Yes Yes No No
========================================================================================= =============== =============== =============== ========================
For more details on plugin-specific feature limitations, see the corresponding plugin pages.
## Enumerating Available Devices
The OpenVINO Runtime API features dedicated methods of enumerating devices and their capabilities. See the [Hello Query Device C++ Sample](../../../samples/cpp/hello_query_device/README.md). This is an example output from the sample (truncated to device names only):
Enumerating Available Devices
#######################################
The OpenVINO Runtime API features dedicated methods of enumerating devices and their capabilities. See the :doc:`Hello Query Device C++ Sample <openvino_inference_engine_samples_hello_query_device_README>`. This is an example output from the sample (truncated to device names only):
.. code-block:: sh
./hello_query_device
Available devices:
Device: CPU
...
Device: GPU.0
...
Device: GPU.1
...
Device: GNA
```sh
./hello_query_device
Available devices:
Device: CPU
...
Device: GPU.0
...
Device: GPU.1
...
Device: GNA
```
A simple programmatic way to enumerate the devices and use with the multi-device is as follows:
@sphinxdirective
.. tab:: C++
.. doxygensnippet:: docs/snippets/MULTI2.cpp
:language: cpp
:fragment: [part2]
@endsphinxdirective
Beyond the typical "CPU", "GPU", and so on, when multiple instances of a device are available, the names are more qualified. For example, this is how two GPUs can be listed (iGPU is always GPU.0):
```
...
Device: GPU.0
...
Device: GPU.1
```
Beyond the typical "CPU", "GPU", and so on, when multiple instances of a device are available, the names are more qualified.
For example, this is how two GPUs can be listed (iGPU is always GPU.0):
.. code-block:: sh
...
Device: GPU.0
...
Device: GPU.1
So, the explicit configuration to use both would be "MULTI:GPU.1,GPU.0". Accordingly, the code that loops over all available devices of the "GPU" type only is as follows:
@sphinxdirective
.. tab:: C++
@@ -91,7 +102,8 @@ So, the explicit configuration to use both would be "MULTI:GPU.1,GPU.0". Accordi
:language: cpp
:fragment: [part3]
@endsphinxdirective

403
docs/OV_Runtime_UG/supported_plugins/GPU.md
View File

@@ -8,255 +8,318 @@
openvino_docs_OV_UG_supported_plugins_GPU_RemoteTensor_API
@endsphinxdirective
The GPU plugin is an OpenCL based plugin for inference of deep neural networks on Intel GPUs, both integrated and discrete ones.
For an in-depth description of the GPU plugin, see:
- [GPU plugin developers documentation](https://github.com/openvinotoolkit/openvino/wiki/GPUPluginDevelopersDocs)
- [OpenVINO Runtime GPU plugin source files](https://github.com/openvinotoolkit/openvino/tree/master/src/plugins/intel_gpu/)
- [Accelerate Deep Learning Inference with Intel® Processor Graphics](https://software.intel.com/en-us/articles/accelerating-deep-learning-inference-with-intel-processor-graphics).
The GPU plugin is a part of the Intel® Distribution of OpenVINO™ toolkit. For more information on how to configure a system to use it, see the [GPU configuration](@ref openvino_docs_install_guides_configurations_for_intel_gpu).
- `GPU plugin developers documentation <https://github.com/openvinotoolkit/openvino/blob/master/src/plugins/intel_gpu/README.md>`__
- `OpenVINO Runtime GPU plugin source files <https://github.com/openvinotoolkit/openvino/tree/master/src/plugins/intel_gpu/>`__
- `Accelerate Deep Learning Inference with Intel® Processor Graphics <https://software.intel.com/en-us/articles/accelerating-deep-learning-inference-with-intel-processor-graphics>`__
## Device Naming Convention
* Devices are enumerated as `GPU.X`, where `X={0, 1, 2,...}` (only Intel® GPU devices are considered).
* If the system has an integrated GPU, its `id` is always 0 (`GPU.0`).
The GPU plugin is a part of the Intel® Distribution of OpenVINO™ toolkit. For more information on how to configure a system to use it, see the :doc:`GPU configuration <openvino_docs_install_guides_configurations_for_intel_gpu>`.
Device Naming Convention
#######################################
* Devices are enumerated as ``GPU.X``, where ``X={0, 1, 2,...}`` (only Intel® GPU devices are considered).
* If the system has an integrated GPU, its ``id`` is always 0 (``GPU.0``).
* The order of other GPUs is not predefined and depends on the GPU driver.
* The `GPU` is an alias for `GPU.0`.
* The ``GPU`` is an alias for ``GPU.0``.
* If the system does not have an integrated GPU, devices are enumerated, starting from 0.
* For GPUs with multi-tile architecture (multiple sub-devices in OpenCL terms), a specific tile may be addressed as `GPU.X.Y`, where `X,Y={0, 1, 2,...}`, `X` - id of the GPU device, `Y` - id of the tile within device `X`
* For GPUs with multi-tile architecture (multiple sub-devices in OpenCL terms), a specific tile may be addressed as ``GPU.X.Y``, where ``X,Y={0, 1, 2,...}``, ``X`` - id of the GPU device, ``Y`` - id of the tile within device ``X``
For demonstration purposes, see the [Hello Query Device C++ Sample](../../../samples/cpp/hello_query_device/README.md) that can print out the list of available devices with associated indices. Below is an example output (truncated to the device names only):
For demonstration purposes, see the :doc:`Hello Query Device C++ Sample <openvino_inference_engine_samples_hello_query_device_README>` that can print out the list of available devices with associated indices. Below is an example output (truncated to the device names only):
```sh
./hello_query_device
Available devices:
Device: CPU
...
Device: GPU.0
...
Device: GPU.1
...
Device: GNA
```
.. code-block:: sh
./hello_query_device
Available devices:
Device: CPU
...
Device: GPU.0
...
Device: GPU.1
...
Device: GNA
Then, device name can be passed to the `ov::Core::compile_model()` method:
@sphinxtabset
Then, the device name can be passed to the ``ov::Core::compile_model()`` method, running on:
@sphinxtab{Running on default device}
.. tab-set::
.. tab-item:: default device
@sphinxtabset
.. tab-set::
@sphinxtab{C++}
@snippet docs/snippets/gpu/compile_model.cpp compile_model_default_gpu
@endsphinxtab
.. tab-item:: C++
:sync: cpp
@sphinxtab{Python}
@snippet docs/snippets/gpu/compile_model.py compile_model_default_gpu
@endsphinxtab
.. doxygensnippet:: docs/snippets/gpu/compile_model.cpp
:language: cpp
:fragment: compile_model_default_gpu
@endsphinxtabset
.. tab-item:: Python
:sync: py
@endsphinxtab
.. doxygensnippet:: docs/snippets/gpu/compile_model.py
:language: Python
:fragment: compile_model_default_gpu
@sphinxtab{Running on specific GPU}
.. tab-item:: specific GPU
@sphinxtabset
.. tab-set::
@sphinxtab{C++}
@snippet docs/snippets/gpu/compile_model.cpp compile_model_gpu_with_id
@endsphinxtab
.. tab-item:: C++
:sync: cpp
@sphinxtab{Python}
@snippet docs/snippets/gpu/compile_model.py compile_model_gpu_with_id
@endsphinxtab
.. doxygensnippet:: docs/snippets/gpu/compile_model.cpp
:language: cpp
:fragment: compile_model_gpu_with_id
@endsphinxtabset
.. tab-item:: Python
:sync: py
@endsphinxtab
.. doxygensnippet:: docs/snippets/gpu/compile_model.py
:language: Python
:fragment: compile_model_gpu_with_id
@sphinxtab{Running on specific tile}
.. tab-item:: specific tile
@sphinxtabset
.. tab-set::
@sphinxtab{C++}
@snippet docs/snippets/gpu/compile_model.cpp compile_model_gpu_with_id_and_tile
@endsphinxtab
.. tab-item:: C++
:sync: cpp
@sphinxtab{Python}
@snippet docs/snippets/gpu/compile_model.py compile_model_gpu_with_id_and_tile
@endsphinxtab
.. doxygensnippet:: docs/snippets/gpu/compile_model.cpp
:language: cpp
:fragment: compile_model_gpu_with_id_and_tile
@endsphinxtabset
.. tab-item:: Python
:sync: py
@endsphinxtab
.. doxygensnippet:: docs/snippets/gpu/compile_model.py
:language: Python
:fragment: compile_model_gpu_with_id_and_tile
@endsphinxtabset
Supported Inference Data Types
#######################################
## Supported Inference Data Types
The GPU plugin supports the following data types as inference precision of internal primitives:
- Floating-point data types:
- f32
- f16
- Quantized data types:
- u8
- i8
- u1
Selected precision of each primitive depends on the operation precision in IR, quantization primitives, and available hardware capabilities.
The `u1`/`u8`/`i8` data types are used for quantized operations only, which means that they are not selected automatically for non-quantized operations.
For more details on how to get a quantized model, refer to the [Model Optimization guide](@ref openvino_docs_model_optimization_guide).
The ``u1``/``u8``/``i8`` data types are used for quantized operations only, which means that they are not selected automatically for non-quantized operations.
For more details on how to get a quantized model, refer to the :doc:`Model Optimization guide <openvino_docs_model_optimization_guide>`.
Floating-point precision of a GPU primitive is selected based on operation precision in the OpenVINO IR, except for the [compressed f16 OpenVINO IR form](../../MO_DG/prepare_model/FP16_Compression.md), which is executed in the `f16` precision.
Floating-point precision of a GPU primitive is selected based on operation precision in the OpenVINO IR, except for the :doc:`<compressed f16 OpenVINO IR form <openvino_docs_MO_DG_FP16_Compression>`, which is executed in the ``f16`` precision.
> **NOTE**: Hardware acceleration for `i8`/`u8` precision may be unavailable on some platforms. In such cases, a model is executed in the floating-point precision taken from IR. Hardware support of `u8`/`i8` acceleration can be queried via the `ov::device::capabilities` property.
.. note::
[Hello Query Device C++ Sample](../../../samples/cpp/hello_query_device/README.md) can be used to print out the supported data types for all detected devices.
Hardware acceleration for ``i8``/``u8`` precision may be unavailable on some platforms. In such cases, a model is executed in the floating-point precision taken from IR.
Hardware support of ``u8``/``i8`` acceleration can be queried via the `ov::device::capabilities` property.
## Supported Features
:doc:`Hello Query Device C++ Sample<openvino_inference_engine_samples_hello_query_device_README>` can be used to print out the supported data types for all detected devices.
Supported Features
#######################################
The GPU plugin supports the following features:
### Multi-device Execution
Multi-device Execution
+++++++++++++++++++++++++++++++++++++++
If a system has multiple GPUs (for example, an integrated and a discrete Intel GPU), then any supported model can be executed on all GPUs simultaneously.
It is done by specifying `MULTI:GPU.1,GPU.0` as a target device.
It is done by specifying ``MULTI:GPU.1,GPU.0`` as a target device.
@sphinxtabset
.. tab-set::
@sphinxtab{C++}
@snippet docs/snippets/gpu/compile_model.cpp compile_model_multi
@endsphinxtab
.. tab-item:: C++
:sync: cpp
@sphinxtab{Python}
@snippet docs/snippets/gpu/compile_model.py compile_model_multi
@endsphinxtab
.. doxygensnippet:: docs/snippets/gpu/compile_model.cpp
:language: cpp
:fragment: compile_model_multi
@endsphinxtabset
.. tab-item:: Python
:sync: py
For more details, see the [Multi-device execution](../multi_device.md).
.. doxygensnippet:: docs/snippets/gpu/compile_model.py
:language: Python
:fragment: compile_model_multi
### Automatic Batching
The GPU plugin is capable of reporting `ov::max_batch_size` and `ov::optimal_batch_size` metrics with respect to the current hardware
platform and model. Therefore, automatic batching is enabled by default when `ov::optimal_batch_size` is `> 1` and `ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)` is set.
Alternatively, it can be enabled explicitly via the device notion, for example `BATCH:GPU`.
@sphinxtabset
For more details, see the :doc:`Multi-device execution <openvino_docs_OV_UG_Running_on_multiple_devices>`.
@sphinxtab{Batching via BATCH plugin}
Automatic Batching
+++++++++++++++++++++++++++++++++++++++
@sphinxtabset
The GPU plugin is capable of reporting ``ov::max_batch_size`` and ``ov::optimal_batch_size`` metrics with respect to the current hardware
platform and model. Therefore, automatic batching is enabled by default when ``ov::optimal_batch_size`` is ``> 1`` and ``ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)`` is set.
Alternatively, it can be enabled explicitly via the device notion, for example ``BATCH:GPU``.
@sphinxtab{C++}
@snippet docs/snippets/gpu/compile_model.cpp compile_model_batch_plugin
@endsphinxtab
@sphinxtab{Python}
@snippet docs/snippets/gpu/compile_model.py compile_model_batch_plugin
@endsphinxtab
.. tab-set::
.. tab-item:: Batching via BATCH plugin
@endsphinxtabset
.. tab-set::
@endsphinxtab
.. tab-item:: C++
:sync: cpp
@sphinxtab{Batching via throughput hint}
.. doxygensnippet:: docs/snippets/gpu/compile_model.cpp
:language: cpp
:fragment: compile_model_batch_plugin
@sphinxtabset
.. tab-item:: Python
:sync: py
@sphinxtab{C++}
@snippet docs/snippets/gpu/compile_model.cpp compile_model_auto_batch
@endsphinxtab
.. doxygensnippet:: docs/snippets/gpu/compile_model.py
:language: Python
:fragment: compile_model_batch_plugin
@sphinxtab{Python}
@snippet docs/snippets/gpu/compile_model.py compile_model_auto_batch
@endsphinxtab
.. tab-item:: Batching via throughput hint
@endsphinxtabset
.. tab-set::
@endsphinxtab
.. tab-item:: C++
:sync: cpp
@endsphinxtabset
.. doxygensnippet:: docs/snippets/gpu/compile_model.cpp
:language: cpp
:fragment: compile_model_auto_batch
For more details, see the [Automatic batching](../automatic_batching.md).
.. tab-item:: Python
:sync: py
### Multi-stream Execution
If either the `ov::num_streams(n_streams)` with `n_streams > 1` or the `ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)` property is set for the GPU plugin,
.. doxygensnippet:: docs/snippets/gpu/compile_model.py
:language: Python
:fragment: compile_model_auto_batch
For more details, see the :doc:`Automatic batching<openvino_docs_OV_UG_Automatic_Batching>`.
Multi-stream Execution
+++++++++++++++++++++++++++++++++++++++
If either the ``ov::num_streams(n_streams)`` with ``n_streams > 1`` or the ``ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)`` property is set for the GPU plugin,
multiple streams are created for the model. In the case of GPU plugin each stream has its own host thread and an associated OpenCL queue
which means that the incoming infer requests can be processed simultaneously.
> **NOTE**: Simultaneous scheduling of kernels to different queues does not mean that the kernels are actually executed in parallel on the GPU device. The actual behavior depends on the hardware architecture and in some cases the execution may be serialized inside the GPU driver.
.. note::
Simultaneous scheduling of kernels to different queues does not mean that the kernels are actually executed in parallel on the GPU device.
The actual behavior depends on the hardware architecture and in some cases the execution may be serialized inside the GPU driver.
When multiple inferences of the same model need to be executed in parallel, the multi-stream feature is preferred to multiple instances of the model or application.
The reason for this is that the implementation of streams in the GPU plugin supports weight memory sharing across streams, thus, memory consumption may be lower, compared to the other approaches.
For more details, see the [optimization guide](@ref openvino_docs_deployment_optimization_guide_dldt_optimization_guide).
For more details, see the :doc:`optimization guide<openvino_docs_deployment_optimization_guide_dldt_optimization_guide>`.
### Dynamic Shapes
The GPU plugin supports dynamic shapes for batch dimension only (specified as `N` in the [layouts terms](../layout_overview.md)) with a fixed upper bound. Any other dynamic dimensions are unsupported. Internally, GPU plugin creates
`log2(N)` (`N` - is an upper bound for batch dimension here) low-level execution graphs for batch sizes equal to powers of 2 to emulate dynamic behavior, so that incoming infer request with a specific batch size is executed via a minimal combination of internal networks.
For example, batch size 33 may be executed via 2 internal networks with batch size 32 and 1.
Dynamic Shapes
+++++++++++++++++++++++++++++++++++++++
> **NOTE**: Such approach requires much more memory and the overall model compilation time is significantly longer, compared to the static batch scenario.
The GPU plugin supports dynamic shapes for batch dimension only (specified as ``N`` in the :doc:`layouts terms<openvino_docs_OV_UG_Layout_Overview>`) with a fixed upper bound.
Any other dynamic dimensions are unsupported. Internally, GPU plugin creates ``log2(N)`` (``N`` - is an upper bound for batch dimension here)
low-level execution graphs for batch sizes equal to powers of 2 to emulate dynamic behavior, so that incoming infer request
with a specific batch size is executed via a minimal combination of internal networks. For example, batch size 33 may be executed via 2 internal networks with batch size 32 and 1.
.. note::
Such approach requires much more memory and the overall model compilation time is significantly longer, compared to the static batch scenario.
The code snippet below demonstrates how to use dynamic batching in simple scenarios:
@sphinxtabset
.. tab-set::
@sphinxtab{C++}
@snippet docs/snippets/gpu/dynamic_batch.cpp dynamic_batch
@endsphinxtab
.. tab-item:: C++
:sync: cpp
@sphinxtab{Python}
@snippet docs/snippets/gpu/dynamic_batch.py dynamic_batch
@endsphinxtab
.. doxygensnippet:: docs/snippets/gpu/dynamic_batch.cpp
:language: cpp
:fragment: dynamic_batch
@endsphinxtabset
.. tab-item:: Python
:sync: py
For more details, see the [dynamic shapes guide](../ov_dynamic_shapes.md).
.. doxygensnippet:: docs/snippets/gpu/dynamic_batch.py
:language: Python
:fragment: dynamic_batch
For more details, see the :doc:`dynamic shapes guide<openvino_docs_OV_UG_DynamicShapes>`.
Preprocessing Acceleration
+++++++++++++++++++++++++++++++++++++++
### Preprocessing Acceleration
The GPU plugin has the following additional preprocessing options:
- The `ov::intel_gpu::memory_type::surface` and `ov::intel_gpu::memory_type::buffer` values for the `ov::preprocess::InputTensorInfo::set_memory_type()` preprocessing method. These values are intended to be used to provide a hint for the plugin on the type of input Tensors that will be set in runtime to generate proper kernels.
- The ``ov::intel_gpu::memory_type::surface`` and ``ov::intel_gpu::memory_type::buffer`` values for the ``ov::preprocess::InputTensorInfo::set_memory_type()`` preprocessing method. These values are intended to be used to provide a hint for the plugin on the type of input Tensors that will be set in runtime to generate proper kernels.
@sphinxtabset
.. tab-set::
@sphinxtab{C++}
@snippet docs/snippets/gpu/preprocessing_nv12_two_planes.cpp init_preproc
@endsphinxtab
.. tab-item:: C++
:sync: cpp
@sphinxtab{Python}
@snippet docs/snippets/gpu/preprocessing_nv12_two_planes.py init_preproc
@endsphinxtab
.. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_two_planes.cpp
:language: cpp
:fragment: init_preproc
@endsphinxtabset
.. tab-item:: Python
:sync: py
With such preprocessing, GPU plugin will expect `ov::intel_gpu::ocl::ClImage2DTensor` (or derived) to be passed for each NV12 plane via `ov::InferRequest::set_tensor()` or `ov::InferRequest::set_tensors()` methods.
For usage examples, refer to the [RemoteTensor API](./GPU_RemoteTensor_API.md).
For more details, see the [preprocessing API](../preprocessing_overview.md).
### Model Caching
Cache for the GPU plugin may be enabled via the common OpenVINO `ov::cache_dir` property. GPU plugin implementation supports only caching of compiled kernels, so all plugin-specific model transformations are executed on each `ov::Core::compile_model()` call regardless of the `cache_dir` option.
Still, since kernel compilation is a bottleneck in the model loading process, a significant load time reduction can be achieved with the `ov::cache_dir` property enabled.
> **NOTE**: Full model caching support is currently implemented as a preview feature. To activate it, set the OV_GPU_CACHE_MODEL environment variable to 1.
For more details, see the [Model caching overview](../Model_caching_overview.md).
### Extensibility
For information on this subject, see the [GPU Extensibility](@ref openvino_docs_Extensibility_UG_GPU).
### GPU Context and Memory Sharing via RemoteTensor API
For information on this subject, see the [RemoteTensor API of GPU Plugin](GPU_RemoteTensor_API.md).
.. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_two_planes.py
:language: Python
:fragment: init_preproc
## Supported Properties
With such preprocessing, GPU plugin will expect ``ov::intel_gpu::ocl::ClImage2DTensor`` (or derived) to be passed for each NV12 plane via ``ov::InferRequest::set_tensor()`` or ``ov::InferRequest::set_tensors()`` methods.
For usage examples, refer to the :doc:`RemoteTensor API<openvino_docs_OV_UG_supported_plugins_GPU_RemoteTensor_API>`.
For more details, see the :doc:`preprocessing API<openvino_docs_OV_UG_Preprocessing_Overview>`.
Model Caching
+++++++++++++++++++++++++++++++++++++++
Cache for the GPU plugin may be enabled via the common OpenVINO ``ov::cache_dir`` property. GPU plugin implementation supports only caching of compiled kernels, so all plugin-specific model transformations are executed on each ``ov::Core::compile_model()`` call regardless of the ``cache_dir`` option.
Still, since kernel compilation is a bottleneck in the model loading process, a significant load time reduction can be achieved with the ``ov::cache_dir`` property enabled.
.. note::
Full model caching support is currently implemented as a preview feature. To activate it, set the OV_GPU_CACHE_MODEL environment variable to 1.
For more details, see the :doc:`Model caching overview<openvino_docs_OV_UG_Model_caching_overview>`.
Extensibility
+++++++++++++++++++++++++++++++++++++++
For information on this subject, see the :doc:`GPU Extensibility<openvino_docs_Extensibility_UG_GPU>`.
GPU Context and Memory Sharing via RemoteTensor API
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
For information on this subject, see the :doc:`RemoteTensor API of GPU Plugin<openvino_docs_OV_UG_supported_plugins_GPU_RemoteTensor_API>`.
Supported Properties
#######################################
The plugin supports the properties listed below.
### Read-write properties
All parameters must be set before calling `ov::Core::compile_model()` in order to take effect or passed as additional argument to `ov::Core::compile_model()`.
Read-write properties
+++++++++++++++++++++++++++++++++++++++
All parameters must be set before calling ``ov::Core::compile_model()`` in order to take effect or passed as additional argument to ``ov::Core::compile_model()``.
- ov::cache_dir
- ov::enable_profiling
@@ -273,7 +336,9 @@ All parameters must be set before calling `ov::Core::compile_model()` in order t
- ov::intel_gpu::hint::queue_throttle
- ov::intel_gpu::enable_loop_unrolling
### Read-only Properties
Read-only Properties
+++++++++++++++++++++++++++++++++++++++
- ov::supported_properties
- ov::available_devices
- ov::range_for_async_infer_requests
@@ -289,26 +354,38 @@ All parameters must be set before calling `ov::Core::compile_model()` in order t
- ov::intel_gpu::execution_units_count
- ov::intel_gpu::memory_statistics
## Limitations
Limitations
#######################################
In some cases, the GPU plugin may implicitly execute several primitives on CPU using internal implementations, which may lead to an increase in CPU utilization.
Below is a list of such operations:
- Proposal
- NonMaxSuppression
- DetectionOutput
The behavior depends on specific parameters of the operations and hardware configuration.
## GPU Performance Checklist: Summary <a name="gpu-checklist"></a>
GPU Performance Checklist: Summary
#######################################
Since OpenVINO relies on the OpenCL kernels for the GPU implementation, many general OpenCL tips apply:
- Prefer `FP16` inference precision over `FP32`, as Model Optimizer can generate both variants, and the `FP32` is the default. Also, consider using the [Post-training Optimization Tool](https://docs.openvino.ai/latest/pot_introduction.html).
- Try to group individual infer jobs by using [automatic batching](../automatic_batching.md).
- Consider [caching](../Model_caching_overview.md) to minimize model load time.
- If your application performs inference on the CPU alongside the GPU, or otherwise loads the host heavily, make sure that the OpenCL driver threads do not starve. [CPU configuration options](./CPU.md) can be used to limit the number of inference threads for the CPU plugin.
- Even in the GPU-only scenario, a GPU driver might occupy a CPU core with spin-loop polling for completion. If CPU load is a concern, consider the dedicated `queue_throttle` property mentioned previously. Note that this option may increase inference latency, so consider combining it with multiple GPU streams or [throughput performance hints](../performance_hints.md).
- When operating media inputs, consider [remote tensors API of the GPU Plugin](./GPU_RemoteTensor_API.md).
- Prefer ``FP16`` inference precision over ``FP32``, as Model Optimizer can generate both variants, and the ``FP32`` is the default. To learn about optimization options, see :doc:`Optimization Guide<openvino_docs_model_optimization_guide>`.
- Try to group individual infer jobs by using :doc:`automatic batching<openvino_docs_OV_UG_Automatic_Batching>`.
- Consider :doc:`caching<openvino_docs_OV_UG_Model_caching_overview>` to minimize model load time.
- If your application performs inference on the CPU alongside the GPU, or otherwise loads the host heavily, make sure that the OpenCL driver threads do not starve. :doc:`CPU configuration options<openvino_docs_OV_UG_supported_plugins_CPU>` can be used to limit the number of inference threads for the CPU plugin.
- Even in the GPU-only scenario, a GPU driver might occupy a CPU core with spin-loop polling for completion. If CPU load is a concern, consider the dedicated ``queue_throttle`` property mentioned previously. Note that this option may increase inference latency, so consider combining it with multiple GPU streams or :doc:`throughput performance hints<openvino_docs_OV_UG_Performance_Hints>`.
- When operating media inputs, consider :doc:`remote tensors API of the GPU Plugin<openvino_docs_OV_UG_supported_plugins_GPU_RemoteTensor_API>`.
## Additional Resources
* [Supported Devices](Supported_Devices.md)
* [Optimization guide](@ref openvino_docs_deployment_optimization_guide_dldt_optimization_guide)
* [GPU plugin developers documentation](https://github.com/openvinotoolkit/openvino/wiki/GPUPluginDevelopersDocs)
Additional Resources
#######################################
* :doc:`Supported Devices<openvino_docs_OV_UG_supported_plugins_Supported_Devices>`.
* :doc:`Optimization guide<openvino_docs_deployment_optimization_guide_dldt_optimization_guide>`.
* `GPU plugin developers documentation <https://github.com/openvinotoolkit/openvino/blob/master/src/plugins/intel_gpu/README.md>`__
@endsphinxdirective

238
docs/OV_Runtime_UG/supported_plugins/GPU_RemoteTensor_API.md
View File

@@ -1,10 +1,13 @@
# Remote Tensor API of GPU Plugin {#openvino_docs_OV_UG_supported_plugins_GPU_RemoteTensor_API}
The GPU plugin implementation of the `ov::RemoteContext` and `ov::RemoteTensor` interfaces supports GPU
@sphinxdirective
The GPU plugin implementation of the ``ov::RemoteContext`` and ``ov::RemoteTensor`` interfaces supports GPU
pipeline developers who need video memory sharing and interoperability with existing native APIs,
such as OpenCL, Microsoft DirectX, or VAAPI.
The `ov::RemoteContext` and `ov::RemoteTensor` interface implementation targets the need for memory sharing and
The ``ov::RemoteContext`` and ``ov::RemoteTensor`` interface implementation targets the need for memory sharing and
interoperability with existing native APIs, such as OpenCL, Microsoft DirectX, and VAAPI.
They allow you to avoid any memory copy overhead when plugging OpenVINO™ inference
into an existing GPU pipeline. They also enable OpenCL kernels to participate in the pipeline to become
@@ -12,35 +15,35 @@ native buffer consumers or producers of the OpenVINO™ inference.
There are two interoperability scenarios supported by the Remote Tensor API:
* The GPU plugin context and memory objects can be constructed from low-level device, display, or memory
handles and used to create the OpenVINO™ `ov::CompiledModel` or `ov::Tensor` objects.
* The GPU plugin context and memory objects can be constructed from low-level device, display, or memory handles and used to create the OpenVINO™ ``ov::CompiledModel`` or ``ov::Tensor`` objects.
* The OpenCL context or buffer handles can be obtained from existing GPU plugin objects, and used in OpenCL processing on the application side.
Class and function declarations for the API are defined in the following files:
* Windows -- `openvino/runtime/intel_gpu/ocl/ocl.hpp` and `openvino/runtime/intel_gpu/ocl/dx.hpp`
* Linux -- `openvino/runtime/intel_gpu/ocl/ocl.hpp` and `openvino/runtime/intel_gpu/ocl/va.hpp`
* Windows -- ``openvino/runtime/intel_gpu/ocl/ocl.hpp`` and ``openvino/runtime/intel_gpu/ocl/dx.hpp``
* Linux -- ``openvino/runtime/intel_gpu/ocl/ocl.hpp`` and ``openvino/runtime/intel_gpu/ocl/va.hpp``
The most common way to enable the interaction of your application with the Remote Tensor API is to use user-side utility classes
and functions that consume or produce native handles directly.
## Context Sharing Between Application and GPU Plugin
Context Sharing Between Application and GPU Plugin
###########################################################
GPU plugin classes that implement the `ov::RemoteContext` interface are responsible for context sharing.
GPU plugin classes that implement the ``ov::RemoteContext`` interface are responsible for context sharing.
Obtaining a context object is the first step in sharing pipeline objects.
The context object of the GPU plugin directly wraps OpenCL context, setting a scope for sharing the
`ov::CompiledModel` and `ov::RemoteTensor` objects. The `ov::RemoteContext` object can be either created on top of
``ov::CompiledModel`` and ``ov::RemoteTensor`` objects. The ``ov::RemoteContext`` object can be either created on top of
an existing handle from a native API or retrieved from the GPU plugin.
Once you have obtained the context, you can use it to compile a new `ov::CompiledModel` or create `ov::RemoteTensor`
objects.
For network compilation, use a dedicated flavor of `ov::Core::compile_model()`, which accepts the context as an
additional parameter.
Once you have obtained the context, you can use it to compile a new ``ov::CompiledModel`` or create ``ov::RemoteTensor``
objects. For network compilation, use a dedicated flavor of ``ov::Core::compile_model()``, which accepts the context as an additional parameter.
### Creation of RemoteContext from Native Handle
To create the `ov::RemoteContext` object for user context, explicitly provide the context to the plugin using constructor for one
of `ov::RemoteContext` derived classes.
Creation of RemoteContext from Native Handle
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
To create the ``ov::RemoteContext`` object for user context, explicitly provide the context to the plugin using constructor for one
of ``ov::RemoteContext`` derived classes.
@sphinxdirective
.. tab:: Linux
@@ -82,17 +85,17 @@ of `ov::RemoteContext` derived classes.
:language: cpp
:fragment: context_from_d3d_device
@endsphinxdirective
### Getting RemoteContext from the Plugin
Getting RemoteContext from the Plugin
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
If you do not provide any user context, the plugin uses its default internal context.
The plugin attempts to use the same internal context object as long as plugin options are kept the same.
Therefore, all `ov::CompiledModel` objects created during this time share the same context.
Therefore, all ``ov::CompiledModel`` objects created during this time share the same context.
Once the plugin options have been changed, the internal context is replaced by the new one.
To request the current default context of the plugin, use one of the following methods:
@sphinxdirective
.. tab:: Get context from Core
@@ -106,123 +109,121 @@ To request the current default context of the plugin, use one of the following m
:language: cpp
:fragment: default_context_from_model
@endsphinxdirective
## Memory Sharing Between Application and GPU Plugin
Memory Sharing Between Application and GPU Plugin
###########################################################
The classes that implement the `ov::RemoteTensor` interface are the wrappers for native API
The classes that implement the ``ov::RemoteTensor`` interface are the wrappers for native API
memory handles (which can be obtained from them at any time).
To create a shared tensor from a native memory handle, use dedicated `create_tensor`or `create_tensor_nv12` methods
of the `ov::RemoteContext` sub-classes.
`ov::intel_gpu::ocl::ClContext` has multiple overloads of `create_tensor` methods which allow to wrap pre-allocated native handles with the `ov::RemoteTensor`
To create a shared tensor from a native memory handle, use dedicated ``create_tensor``or ``create_tensor_nv12`` methods
of the ``ov::RemoteContext`` sub-classes.
``ov::intel_gpu::ocl::ClContext`` has multiple overloads of ``create_tensor`` methods which allow to wrap pre-allocated native handles with the ``ov::RemoteTensor``
object or request plugin to allocate specific device memory. For more details, see the code snippets below:
@sphinxdirective
.. tab:: Wrap native handles
.. tab-set::
.. tab:: USM pointer
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: wrap_usm_pointer
.. tab-item:: Wrap native handles
.. tab:: cl_mem
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: wrap_cl_mem
.. tab-set::
.. tab:: cl::Buffer
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: wrap_cl_buffer
.. tab-item:: USM pointer
.. tab:: cl::Image2D
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: wrap_cl_image
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: sh
:fragment: wrap_usm_pointer
.. tab:: biplanar NV12 surface
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: wrap_nv12_surface
.. tab-item:: cl_mem
.. tab:: Allocate device memory
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: sh
:fragment: wrap_cl_mem
.. tab:: USM host memory
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: allocate_usm_host
.. tab-item:: cl::Buffer
.. tab:: USM device memory
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: allocate_usm_device
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: sh
:fragment: wrap_cl_buffer
.. tab:: cl::Buffer
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: allocate_cl_buffer
.. tab-item:: cl::Image2D
@endsphinxdirective
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: sh
:fragment: wrap_cl_image
The `ov::intel_gpu::ocl::D3DContext` and `ov::intel_gpu::ocl::VAContext` classes are derived from `ov::intel_gpu::ocl::ClContext`.
.. tab-item:: biplanar NV12 surface
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: sh
:fragment: wrap_nv12_surface
.. tab-item:: Allocate device memory
.. tab-set::
.. tab-item:: USM host memory
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: sh
:fragment: allocate_usm_host
.. tab-item:: USM device memory
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: sh
:fragment: allocate_usm_device
.. tab-item:: cl::Buffer
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: sh
:fragment: allocate_cl_buffer
The ``ov::intel_gpu::ocl::D3DContext`` and ``ov::intel_gpu::ocl::VAContext`` classes are derived from ``ov::intel_gpu::ocl::ClContext``.
Therefore, they provide the functionality described above and extend it
to allow creation of `ov::RemoteTensor` objects from `ID3D11Buffer`, `ID3D11Texture2D` pointers or the `VASurfaceID` handle respectively.
to allow creation of ``ov::RemoteTensor`` objects from ``ID3D11Buffer``, ``ID3D11Texture2D`` pointers or the ``VASurfaceID`` handle respectively.
## Direct NV12 Video Surface Input
Direct NV12 Video Surface Input
###########################################################
To support the direct consumption of a hardware video decoder output, the GPU plugin accepts:
* Two-plane NV12 video surface input - calling the `create_tensor_nv12()` function creates
a pair of `ov::RemoteTensor` objects, representing the Y and UV planes.
* Single-plane NV12 video surface input - calling the `create_tensor()` function creates one
`ov::RemoteTensor` object, representing the Y and UV planes at once (Y elements before UV elements).
* NV12 to Grey video surface input conversion - calling the `create_tensor()` function creates one
`ov::RemoteTensor` object, representing only the Y plane.
* Two-plane NV12 video surface input - calling the ``create_tensor_nv12()`` function creates
a pair of ``ov::RemoteTensor`` objects, representing the Y and UV planes.
* Single-plane NV12 video surface input - calling the ``create_tensor()`` function creates one
``ov::RemoteTensor`` object, representing the Y and UV planes at once (Y elements before UV elements).
* NV12 to Grey video surface input conversion - calling the ``create_tensor()`` function creates one
``ov::RemoteTensor`` object, representing only the Y plane.
To ensure that the plugin generates a correct execution graph, static preprocessing
should be added before model compilation:
@sphinxdirective
.. tab:: two-plane
.. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_two_planes.cpp
:language: cpp
:fragment: [init_preproc]
.. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_two_planes.cpp
:language: cpp
:fragment: [init_preproc]
.. tab:: single-plane
.. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_single_plane.cpp
:language: cpp
:fragment: [init_preproc]
.. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_single_plane.cpp
:language: cpp
:fragment: [init_preproc]
.. tab:: NV12 to Grey
.. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_to_gray.cpp
:language: cpp
:fragment: [init_preproc]
@endsphinxdirective
.. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_to_gray.cpp
:language: cpp
:fragment: [init_preproc]
Since the `ov::intel_gpu::ocl::ClImage2DTensor` and its derived classes do not support batched surfaces,
Since the ``ov::intel_gpu::ocl::ClImage2DTensor`` and its derived classes do not support batched surfaces,
if batching and surface sharing are required at the same time,
inputs need to be set via the `ov::InferRequest::set_tensors` method with vector of shared surfaces for each plane:
inputs need to be set via the ``ov::InferRequest::set_tensors`` method with vector of shared surfaces for each plane:
@sphinxdirective
.. tab:: Single Batch
.. tab:: two-plane
@@ -263,62 +264,62 @@ inputs need to be set via the `ov::InferRequest::set_tensors` method with vector
:language: cpp
:fragment: batched_case
@endsphinxdirective
I420 color format can be processed in a similar way
## Context & Queue Sharing
Context & Queue Sharing
###########################################################
The GPU plugin supports creation of shared context from the `cl_command_queue` handle. In that case,
the `opencl` context handle is extracted from the given queue via OpenCL™ API, and the queue itself is used inside
the plugin for further execution of inference primitives. Sharing the queue changes the behavior of the `ov::InferRequest::start_async()`
The GPU plugin supports creation of shared context from the ``cl_command_queue`` handle. In that case,
the ``opencl`` context handle is extracted from the given queue via OpenCL™ API, and the queue itself is used inside
the plugin for further execution of inference primitives. Sharing the queue changes the behavior of the ``ov::InferRequest::start_async()``
method to guarantee that submission of inference primitives into the given queue is finished before
returning control back to the calling thread.
This sharing mechanism allows performing pipeline synchronization on the app side and avoiding blocking the host thread
on waiting for the completion of inference. The pseudo-code may look as follows:
@sphinxdirective
.. dropdown:: Queue and context sharing example
.. doxygensnippet:: docs/snippets/gpu/queue_sharing.cpp
:language: cpp
:fragment: queue_sharing
@endsphinxdirective
### Limitations
Limitations
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
- Some primitives in the GPU plugin may block the host thread on waiting for the previous primitives before adding its kernels
to the command queue. In such cases, the `ov::InferRequest::start_async()` call takes much more time to return control to the calling thread
to the command queue. In such cases, the ``ov::InferRequest::start_async()`` call takes much more time to return control to the calling thread
as internally it waits for a partial or full network completion.
Examples of operations: Loop, TensorIterator, DetectionOutput, NonMaxSuppression
- Synchronization of pre/post processing jobs and inference pipeline inside a shared queue is user's responsibility.
- Throughput mode is not available when queue sharing is used, i.e., only a single stream can be used for each compiled model.
## Low-Level Methods for RemoteContext and RemoteTensor Creation
Low-Level Methods for RemoteContext and RemoteTensor Creation
#####################################################################
The high-level wrappers mentioned above bring a direct dependency on native APIs to the user program.
If you want to avoid the dependency, you still can directly use the `ov::Core::create_context()`,
`ov::RemoteContext::create_tensor()`, and `ov::RemoteContext::get_params()` methods.
If you want to avoid the dependency, you still can directly use the ``ov::Core::create_context()``,
``ov::RemoteContext::create_tensor()``, and ``ov::RemoteContext::get_params()`` methods.
On this level, native handles are re-interpreted as void pointers and all arguments are passed
using `ov::AnyMap` containers that are filled with `std::string, ov::Any` pairs.
using ``ov::AnyMap`` containers that are filled with ``std::string, ov::Any`` pairs.
Two types of map entries are possible: descriptor and container.
Descriptor sets the expected structure and possible parameter values of the map.
For possible low-level properties and their description, refer to the `openvino/runtime/intel_gpu/remote_properties.hpp` header file .
For possible low-level properties and their description, refer to the ``openvino/runtime/intel_gpu/remote_properties.hpp`` header file .
## Examples
Examples
###########################################################
To see pseudo-code of usage examples, refer to the sections below.
@sphinxdirective
.. NOTE::
For low-level parameter usage examples, see the source code of user-side wrappers from the include files mentioned above.
.. dropdown:: OpenCL Kernel Execution on a Shared Buffer
This example uses the OpenCL context obtained from a compiled model object.
@@ -339,10 +340,11 @@ To see pseudo-code of usage examples, refer to the sections below.
:language: cpp
:fragment: context_sharing_va
@endsphinxdirective
## See Also
See Also
#######################################
* ov::Core
* ov::RemoteTensor
@endsphinxdirective

BIN
docs/_static/download/OV_2023_models_supported.pdf vendored
View File

Binary file not shown.

398
docs/dev/build.md
View File

@@ -1,7 +1,5 @@
# How to build OpenVINO
The guide provides the basic information about the process of building OpenVINO.
```mermaid
gantt
%% Use a hack for centry as a persantage
@@ -14,390 +12,22 @@ gantt
Run tests :active, run, after build, 1716w
```
## Setup environment
The articles below provide the basic informations about the process of building OpenVINO.
This charpter describe how to prepare your machine for OpenVINO development.
* [Windows](build_windows.md)
* [Linux](build_linux.md)
* [Mac (Intel GPU)](build_mac_intel_cpu.md)
* [Mac (ARM)](build_mac_arm.md)
* [Android](build_android.md)
* [Raspbian Stretch](./build_raspbian.md)
* [Web Assembly](./build_webassembly.md)
* [Docker Image](https://github.com/openvinotoolkit/docker_ci/tree/master/dockerfiles/ubuntu18/build_custom)
### Software requirements
> **NOTE**: For the details on how to build static OpenVINO, refer to [Building static OpenVINO libraries](static_libaries.md)
For the OpenVINO build next tools are required:
<details><summary>Windows</summary>
<p>
- [CMake]\* 3.14 or higher
- Microsoft\* Visual Studio 2019, version 16.8 or later
- (Optional) Intel® Graphics Driver for Windows* (30.0) [driver package](https://www.intel.com/content/www/us/en/download/19344/intel-graphics-windows-dch-drivers.html).
- Python 3.7 or higher for OpenVINO Runtime Python API
- [Git for Windows*](https://gitforwindows.org/)
</p>
</details>
<details><summary>Linux</summary>
<p>
## See also
- [CMake] 3.13 or higher
- GCC 7.5 or higher to build OpenVINO Runtime
- Python 3.7 or higher for OpenVINO Runtime Python API
- (Optional) [Install Intel® Graphics Compute Runtime for OpenCL™ Driver package 19.41.14441] to enable inference on Intel integrated GPUs.
<p>
</details>
<details><summary>Mac</summary>
<p>
* [OpenVINO README](../../README.md)
* [OpenVINO Developer Documentation](index.md)
* [OpenVINO Get Started](./get_started.md)
- [brew](https://brew.sh) package manager to install additional dependencies. Use [install brew](https://brew.sh) guide to achieve this.
- [CMake]\* 3.13 or higher
```sh
% brew install cmake
```
- Clang\* compiler, git and other command line tools from Xcode\* 10.1 or higher:
```sh
% xcode-select --install
```
- The step to install python and python libraries is different depending on host architecture:
- **x86_64** Python\* 3.7 or higher for the OpenVINO Runtime Python API, Development tools (Model Optimizer, POT and others):
```sh
% # let's have a look what python versions are available in brew
% brew search python
% # select preferred version of python based on available ones, e.g. 3.10
% brew install python@3.10
```
- **arm64** Select universal2 installer from [Python releases] download page and install `python-3.X.Y-macos11.pkg` image. This allows to have universal python libraries and build x86_64 OpenVINO Python API and Development tools.
- Additional `pip` dependencies to build OpenVINO Runtime Python API, Development tools (Model Optimizer, POT and others):
```sh
% # update pip and setuptools to newer versions
% python3 -m pip install -U pip setuptools
% python3 -m pip install cython
```
In order to build OpenVINO Python API and Development tools as wheel packages, additionally install requirements (after OpenVINO repo clone):
```sh
% python3 -m pip install -r <openvino source tree>/src/bindings/python/wheel/requirements-dev.txt
```
- (Optional; native compilation only) Latest version of TBB library. By default, OpenVINO downloads prebuilt version of TBB 2020.4 library, but if you want to use latest (add `-DENABLE_SYSTEM_TBB=ON` additionally to cmake configuration step):
```sh
% brew install tbb
```
</p>
</details>
### Clone OpenVINO Repository
Clone OpenVINO repository and init submodules:
```sh
git clone https://github.com/openvinotoolkit/openvino.git
cd openvino
git submodule update --init --recursive
```
<details><summary>(Optional) For users in China, clone submodules via gitee mirrors</summary>
<p>
```sh
chmod +x scripts/submodule_update_with_gitee.sh
./scripts/submodule_update_with_gitee.sh
```
<p>
</details>
Congratulate! Now you are ready to build the OpenVINO.
## How to build
<details><summary>Windows</summary>
<p>
> **NOTE**: By default, the build enables the OpenVINO Runtime GPU plugin to infer models on your Intel® Processor Graphics. This requires you to download and install the Intel® Graphics Driver for Windows (26.20) [driver package](https://www.intel.com/content/www/us/en/download/19344/intel-graphics-windows-dch-drivers.html) before running the build. If you don't want to use the GPU plugin, use the `-DENABLE_INTEL_GPU=OFF` CMake build option and skip the installation of the Intel® Graphics Driver.
1. Create build directory:
```sh
mkdir build && cd build
```
2. In the `build` directory, run `cmake` to fetch project dependencies and
generate a Visual Studio solution.
For Microsoft\* Visual Studio 2019 x64 architecture:
```sh
cmake -G "Visual Studio 16 2019" -A x64 -DCMAKE_BUILD_TYPE=Release ..
```
For Microsoft\* Visual Studio 2019 ARM architecture:
```sh
cmake -G "Visual Studio 16 2019" -A ARM -DCMAKE_BUILD_TYPE=Release ..
```
For Microsoft\* Visual Studio 2019 ARM64 architecture:
```sh
cmake -G "Visual Studio 16 2019" -A ARM64 -DCMAKE_BUILD_TYPE=Release ..
```
3. Build generated solution in Visual Studio or run
`cmake --build . --config Release --verbose -j8` to build from the command line. Note that this process may take some time.
4. Before running the samples, add paths to the Threading Building Blocks (TBB) and OpenCV binaries used for
the build to the `%PATH%` environment variable. By default, TBB binaries are
downloaded by the CMake-based script to the `<openvino_repo>/temp/tbb/bin`
folder, OpenCV binaries to the `<openvino_repo>/temp/opencv_4.5.0/opencv/bin`
folder.
### Additional Build Options
- Internal JIT GEMM implementation is used by default.
- Threading Building Blocks (TBB) is used by default. To build Inference
Engine with OpenMP threading, set the `-DTHREADING=OMP` option.
- Required versions of TBB and OpenCV packages are downloaded automatically by
the CMake-based script. If you want to use the automatically-downloaded
packages but you have already installed TBB or OpenCV packages configured in
your environment, you may need to clean the `TBBROOT` and `OpenCV_DIR`
environment variables before running the `cmake` command; otherwise they won't
be downloaded and the build may fail if incompatible versions were installed.
- If the CMake-based build script can not find and download the OpenCV package
that is supported on your platform, or if you want to use a custom build of
the OpenCV library, refer to the
[Use Custom OpenCV Builds](https://github.com/openvinotoolkit/openvino/wiki/CMakeOptionsForCustomCompilation#Building-with-custom-OpenCV)
section for details.
- To switch off/on the CPU and GPU plugins, use the `cmake` options
`-DENABLE_INTEL_CPU=ON/OFF` and `-DENABLE_INTEL_GPU=ON/OFF` respectively.
- To build the OpenVINO Runtime Python API:
1. First, install all additional packages (e.g., cython and opencv) listed in the
`src\bindings\python\src\compatibility\openvino\requirements-dev.txt` file:
```sh
pip install -r requirements-dev.txt
```
2. Second, enable the `-DENABLE_PYTHON=ON` in the CMake (Step #4) option above. To
specify an exact Python version, use the following options:
```sh
-DPYTHON_EXECUTABLE="C:\Program Files\Python37\python.exe" ^
-DPYTHON_LIBRARY="C:\Program Files\Python37\libs\python37.lib" ^
-DPYTHON_INCLUDE_DIR="C:\Program Files\Python37\include"
```
- OpenVINO runtime compilation options:
`-DENABLE_OV_ONNX_FRONTEND=ON` enables the building of the ONNX importer.
### Building OpenVINO with Ninja* Build System
```sh
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\VC\Auxiliary\Build\vcvars64.bat"
cmake -G Ninja -Wno-dev -DCMAKE_BUILD_TYPE=Release ..
cmake --build . --config Release
```
<p>
</details>
<details><summary>Linux</summary>
<p>
1. Install build dependencies using the `install_build_dependencies.sh` script in the
project root folder.
```sh
chmod +x install_build_dependencies.sh
```
```sh
./install_build_dependencies.sh
```
> **NOTE**: By default, the build enables the OpenVINO Runtime GPU plugin to infer models on your Intel® Processor Graphics. This requires you to [Install Intel® Graphics Compute Runtime for OpenCL™ Driver package 19.41.14441] before running the build. If you don't want to use the GPU plugin, use the `-DENABLE_INTEL_GPU=OFF` CMake build option and skip the installation of the Intel® Graphics Compute Runtime for OpenCL™ Driver.
2. Create a build folder:
```sh
mkdir build && cd build
```
3. OpenVINO Runtime uses a CMake-based build system. In the created `build`
directory, run `cmake` to fetch project dependencies and create Unix
makefiles, then run `make` to build the project:
```sh
cmake -DCMAKE_BUILD_TYPE=Release ..
make --jobs=$(nproc --all)
```
The process may take some time to finish.
### Additional Build Options
You can use the following additional build options:
- Threading Building Blocks (TBB) is used by default. To build the OpenVINO
Runtime with OpenMP\* threading set the `-DTHREADING=OMP` option.
- For IA32 operation systems, use [ia32.linux.toolchain.cmake](https://github.com/openvinotoolkit/openvino/blob/master/cmake/toolchains/ia32.linux.toolchain.cmake) CMake toolchain file:
```sh
cmake -DCMAKE_TOOLCHAIN_FILE=<openvino_repo>/cmake/toolchains/ia32.linux.toolchain.cmake ..
```
- Required versions of TBB and OpenCV packages are downloaded automatically by
the CMake-based script. If you want to use the automatically downloaded
packages but you have already installed TBB or OpenCV packages configured in
your environment, you may need to clean the `TBBROOT` and `OpenCV_DIR`
environment variables before running the `cmake` command, otherwise they
will not be downloaded and the build may fail if incompatible versions were
installed.
- If the CMake-based build script can not find and download the OpenCV package
that is supported on your platform, or if you want to use a custom build of
the OpenCV library, see how to [Use Custom OpenCV Builds](https://github.com/openvinotoolkit/openvino/wiki/CMakeOptionsForCustomCompilation#Building-with-custom-OpenCV).
- To build the OpenVINO Runtime Python API:
1. Install all additional packages (e.g., cython and opencv) listed in the
`/src/bindings/python/src/compatibility/openvino/requirements-dev.txt` file:
```sh
pip install -r requirements-dev.txt
```
2. Enable the `-DENABLE_PYTHON=ON` option in the CMake step above (Step 4). To specify an exact Python version, use the following
options:
```
-DPYTHON_EXECUTABLE=`which python3.7` \
-DPYTHON_LIBRARY=/usr/lib/x86_64-linux-gnu/libpython3.7m.so \
-DPYTHON_INCLUDE_DIR=/usr/include/python3.7
```
3. To build a wheel package (.whl), enable the `-DENABLE_WHEEL=ON` option in the CMake step above (Step 4):
4. After the build process finishes, export the newly built Python libraries to the user environment variables:
```
export PYTHONPATH=PYTHONPATH:<openvino_repo>/bin/intel64/Release/python_api/python3.7
export LD_LIBRARY_PATH=LD_LIBRARY_PATH:<openvino_repo>/bin/intel64/Release
```
or install the wheel with pip:
```
pip install <openvino_repo>/build/wheel/openvino-2022.2.0-000-cp37-cp37-manylinux_2_35_x86_64.whl
```
- To switch the CPU and GPU plugins off/on, use the `cmake` options
`-DENABLE_INTEL_CPU=ON/OFF` and `-DENABLE_INTEL_GPU=ON/OFF` respectively.
- OpenVINO runtime compilation options:
`-DENABLE_OV_ONNX_FRONTEND=ON` enables the building of the ONNX importer.
<p>
</details>
<details><summary>Mac</summary>
<p>
This guide shows how to build OpenVINO Runtime for later inference on Apple with:
<details><summary>Intel CPU</summary>
<p>
This can be done using two ways:
- Compile on Intel CPU host using native compilation. Note, that [Build steps](#build-steps) show this scenario.
- Cross-compile on OSX Apple Silicon.
1. Create a build folder:
```sh
mkdir build && cd build
```
2. (CMake configure) OpenVINO project uses a CMake-based build system. In the created `build` directory, run `cmake` to fetch project dependencies and create build rules:
```sh
cmake -DCMAKE_BUILD_TYPE=Release ..
```
> **Note:** By default OpenVINO CMake scripts try to introspect the system and enable all possible functionality based on that. You can look at the CMake output and see warnings, which show that some functionality is turned off and the corresponding reason, guiding what to do to install additionally to enable unavailable functionality. Additionally, you can change CMake options to enable / disable some functionality, add / remove compilation flags, provide custom version of dependencies like TBB, PugiXML, OpenCV, Protobuf. Please, read [CMake Options for Custom Compilation](https://github.com/openvinotoolkit/openvino/wiki/CMakeOptionsForCustomCompilation) for this information.
3. (CMake build) Build OpenVINO project:
```sh
cmake --build . --config Release --jobs=$(nproc --all)
```
All built binaries are located in `<openvino_source_dir>/bin/intel64/Release/` and wheel packages are located in `<openvino_build_dir>/wheels`.
4. (Optional install) Once you have built OpenVINO, you can install artifacts to a preferred location:
```sh
cmake -DCMAKE_INSTALL_PREFIX=<installation location> -P cmake_install.cmake
```
### Cross-compilation
Since OSX version 11.x and Xcode version 12.2, the Apple development tools allows to compile arm64 code on x86 hosts and vice-versa. Based on this, OpenVINO can be compiled even on Apple Silicon machines, then such artifacts can be run on both Intel CPU hosts and Apple Silicon hosts (using [Rosetta]). For this, try to compile OpenVINO using the steps above, but adding `-DCMAKE_OSX_ARCHITECTURES=x86_64` on cmake configure stage. But, **don't enable any system library usage explicitly** via CMake options, because they have `arm64` architecture, e.g.:
```sh
file /opt/homebrew/Cellar/tbb/2021.5.0_2/lib/libtbb.12.5.dylib
/opt/homebrew/Cellar/tbb/2021.5.0_2/lib/libtbb.12.5.dylib: Mach-O 64-bit dynamically linked shared library arm64
```
If you will see the errors like below:
```sh
ld: warning: ignoring file /opt/homebrew/lib/libopencv_imgproc.4.6.0.dylib, building for macOS-x86_64 but attempting to link with file built for macOS-arm64
Undefined symbols for architecture x86_64:
"cv::Mat::Mat(cv::Size_<int>, int, void*, unsigned long)", referenced from:
_image_resize in opencv_c_wrapper.cpp.o
_image_save in opencv_c_wrapper.cpp.o
....
ld: symbol(s) not found for architecture x86_64
clang: error: linker command failed with exit code 1 (use -v to see invocation)
```
Disable its usage in cmake or totally remove such library from the system (e.g. `brew uninstall opencv`), because it's pure arm64 and cannot be used to compile x86_64 binaries.
Or you have to explicitly find / compile x86_64 (or even `universal2`) dependencies by yourself and pass it to OpenVINO cmake scripts. E.g. compile oneTBB using additional option `-DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"`, install and then set `export TBBROOT=<universal oneTBB install root>` which will be used by OpenVINO.
<p>
</details>
<details><summary>ARM</summary>
<p>
There are two options how to use OpenVINO on Apple Silicon:
- (Native) Compile OpenVINO for arm64 architecture with extra module [OpenVINO ARM plugin](https://github.com/openvinotoolkit/openvino_contrib/tree/master/modules/arm_plugin) location in [OpenVINO Contrib](https://github.com/openvinotoolkit/openvino_contrib). Note, build steps will cover this as a default scenario.
- (Rosetta) Compile Intel CPU plugin `x86_64` architecture and run under [Rosetta].
1. Clone contrib repo:
```sh
cd ..
git clone https://github.com/openvinotoolkit/openvino_contrib.git
cd openvino_contrib
git submodule update --init --recursive
cd ../openvino
```
2. Create a build folder:
```sh
mkdir build && cd build
```
3. (CMake configure) OpenVINO project uses a CMake-based build system. In the created `build` directory, run `cmake` to fetch project dependencies and create build rules:
```sh
cmake -DCMAKE_BUILD_TYPE=Release -DOPENVINO_EXTRA_MODULES=../openvino_contrib/modules/arm_plugin ..
```
> **Note:** By default OpenVINO CMake scripts try to introspect the system and enable all possible functionality based on that. You can look at the CMake output and see warnings, which show that some functionality is turned off and the corresponding reason, guiding what to do to install additionally to enable unavailable functionality. Additionally, you can change CMake options to enable / disable some functionality, add / remove compilation flags, provide custom version of dependencies like TBB, PugiXML, OpenCV, Protobuf. Please, read [CMake Options for Custom Compilation](https://github.com/openvinotoolkit/openvino/wiki/CMakeOptionsForCustomCompilation) for this information.
4. (CMake build) Build OpenVINO project:
```sh
cmake --build . --config Release --jobs=$(nproc --all)
```
All built binaries are located in `<openvino_source_dir>/bin/<arm64 | intel64>/Release/` and wheel packages are located in `<openvino_build_dir>/wheels`.
5. (Optional install) Once you have built OpenVINO, you can install artifacts to a preferred location:
```sh
cmake -DCMAKE_INSTALL_PREFIX=<installation location> -P cmake_install.cmake
```
### Building x86_64 binaries
Since OSX version 11.x and Xcode version 12.2, the Apple development tools allows to compile arm64 code on x86 hosts and vice-versa. Based on this, OpenVINO can be compiled as x86_64 binary, then run on Apple Silicon hosts using [Rosetta]. For this, first of all Rosetta must be installed:
```sh
softwareupdate --install-rosetta
```
Then try to compile OpenVINO using the steps above, but adding `-DCMAKE_OSX_ARCHITECTURES=x86_64` on cmake configure stage. But, **don't enable any system library usage explicitly** via CMake options, because they have `arm64` architecture, e.g.:
```sh
file /opt/homebrew/Cellar/tbb/2021.5.0_2/lib/libtbb.12.5.dylib
/opt/homebrew/Cellar/tbb/2021.5.0_2/lib/libtbb.12.5.dylib: Mach-O 64-bit dynamically linked shared library arm64
```
> **Note:** since you are building with `universal2` python libraries, wheel package is created with name `openvino-2022.3.0-000-cp39-cp39-macosx_12_0_universal2.whl` and have proper `universal2` tags, so can *potentially* be used on both Apple Silicon and Intel CPU.
<p>
</details>
<p>
</details>
## Run application
[CMake]:https://cmake.org/download/
[Install Intel® Graphics Compute Runtime for OpenCL™ Driver package 19.41.14441]:https://github.com/intel/compute-runtime/releases/tag/19.41.14441
[Rosetta]:https://support.apple.com/en-us/HT211861

59
docs/dev/build_android.md Normal file
View File

@@ -0,0 +1,59 @@
# Build OpenVINO™ Runtime for Android systems
This article describes how to build Inference Engine for Android operating systems.
## Software requirements
- [CMake](https://cmake.org/download/) 3.13 or higher
- Android NDK (this guide has been validated with r20 release)
## How to build
1. Download and unpack [Android NDK](https://developer.android.com/ndk/downloads). Let's assume that `~/Downloads` is used as a working folder.
```sh
cd ~/Downloads
wget https://dl.google.com/android/repository/android-ndk-r20-linux-x86_64.zip
unzip android-ndk-r20-linux-x86_64.zip
mv android-ndk-r20 android-ndk
```
2. Create a build folder:
```sh
mkdir build
```
3. Change working directory to `build` and run `cmake` to create makefiles. Then run `make`.
```sh
cd build
cmake .. \
-DCMAKE_TOOLCHAIN_FILE=~/Downloads/android-ndk/build/cmake/android.toolchain.cmake \
-DANDROID_ABI=x86_64 \
-DANDROID_PLATFORM=21 \
-DANDROID_STL=c++_shared \
-DENABLE_OPENCV=OFF
make --jobs=$(nproc --all)
```
* `ANDROID_ABI` specifies target architecture:
* `x86_64` for x64 build
* `armeabi-v7a with NEON` for ARM with NEON support
* `arm64-v8a` for ARM 64 bits
* `ANDROID_PLATFORM` - Android API version
* `ANDROID_STL` specifies that shared C++ runtime is used. Copy `~/Downloads/android-ndk/sources/cxx-stl/llvm-libc++/libs/x86_64/libc++_shared.so` from Android NDK along with built binaries
4. To reduce the binaries size, use `strip` tool from NDK:
```bash
~/Downloads/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/x86_64-linux-android/bin/strip openvino/bin/intel64/Release/lib/*.so
```
## See also
* [OpenVINO README](../../README.md)
* [OpenVINO Developer Documentation](index.md)
* [OpenVINO Get Started](./get_started.md)
* [How to build OpenVINO](build.md)

90
docs/dev/build_linux.md Normal file
View File

@@ -0,0 +1,90 @@
# Build OpenVINO™ Runtime for Linux systems
The software was validated on:
- Ubuntu 18.04 (64-bit) with default GCC 7.5.0
- Ubuntu 20.04 (64-bit) with default GCC 9.3.0
- Red Hat Enterprise Linux 8.2 (64-bit) with default GCC 8.5.0
> **NOTE**: To build on CentOS 7 (64-bit), refer to [Building OpenVINO on CentOS 7 Guide](https://github.com/openvinotoolkit/openvino/wiki/Building-OpenVINO-on-CentOS-7-Guide)
## Software requirements
- [CMake](https://cmake.org/download/) 3.13 or higher
- GCC 7.5 or higher to build OpenVINO Runtime
- Python 3.7 or higher for OpenVINO Runtime Python API
- (Optional) [Install Intel® Graphics Compute Runtime for OpenCL™ Driver package 19.41.14441](https://github.com/intel/compute-runtime/releases/tag/19.41.14441) to enable inference on Intel integrated GPUs.
## How to build
1. Clone OpenVINO repository and init submodules:
```sh
git clone https://github.com/openvinotoolkit/openvino.git
cd openvino
git submodule update --init --recursive
```
(Optional) For users in China, clone submodules via gitee mirrors
```sh
chmod +x scripts/submodule_update_with_gitee.sh
./scripts/submodule_update_with_gitee.sh
```
2. Install build dependencies using the `install_build_dependencies.sh` script in the
project root folder.
```sh
chmod +x install_build_dependencies.sh
```
```sh
sudo ./install_build_dependencies.sh
```
> **NOTE**: By default, the build enables the OpenVINO Runtime GPU plugin to infer models on your Intel® Processor Graphics. This requires you to [Install Intel® Graphics Compute Runtime for OpenCL™ Driver package 19.41.14441](https://github.com/intel/compute-runtime/releases/tag/19.41.14441) before running the build. If you don't want to use the GPU plugin, use the `-DENABLE_INTEL_GPU=OFF` CMake build option and skip the installation of the Intel® Graphics Compute Runtime for OpenCL™ Driver.
3. Create a build folder:
```sh
mkdir build && cd build
```
4. OpenVINO Runtime uses a CMake-based build system. In the created `build` directory, run `cmake` to fetch project dependencies and create Unix makefiles, then run `make` to build the project:
```sh
cmake -DCMAKE_BUILD_TYPE=Release ..
make --jobs=$(nproc --all)
```
The process may take some time to finish.
### Additional Build Options
You can use the following additional build options:
- For IA32 operation systems, use [ia32.linux.toolchain.cmake](https://github.com/openvinotoolkit/openvino/blob/master/cmake/toolchains/ia32.linux.toolchain.cmake) CMake toolchain file:
```sh
cmake -DCMAKE_TOOLCHAIN_FILE=<openvino_repo>/cmake/toolchains/ia32.linux.toolchain.cmake ..
```
- To build the OpenVINO Runtime Python API:
1. Install all additional packages (e.g., cython and opencv) listed in the `/src/bindings/python/src/compatibility/openvino/requirements-dev.txt` file:
```sh
pip install -r requirements-dev.txt
```
2. Enable the `-DENABLE_PYTHON=ON` option in the CMake step above (Step 4). To specify an exact Python version, use the following options:
```
-DPYTHON_EXECUTABLE=`which python3.7` \
-DPYTHON_LIBRARY=/usr/lib/x86_64-linux-gnu/libpython3.7m.so \
-DPYTHON_INCLUDE_DIR=/usr/include/python3.7
```
3. To build a wheel package (.whl), enable the `-DENABLE_WHEEL=ON` option in the CMake step above (Step 4):
4. After the build process finishes, export the newly built Python libraries to the user environment variables:
```
export PYTHONPATH=PYTHONPATH:<openvino_repo>/bin/intel64/Release/python_api/python3.7
export LD_LIBRARY_PATH=LD_LIBRARY_PATH:<openvino_repo>/bin/intel64/Release
```
or install the wheel with pip:
```
pip install <openvino_repo>/build/wheel/openvino-2022.2.0-000-cp37-cp37-manylinux_2_35_x86_64.whl
```
## See also
* [OpenVINO README](../../README.md)
* [OpenVINO Developer Documentation](index.md)
* [OpenVINO Get Started](./get_started.md)
* [How to build OpenVINO](build.md)

110
docs/dev/build_mac_arm.md Normal file
View File

@@ -0,0 +1,110 @@
# Build OpenVINO™ Runtime for macOS systems (Apple Silicon)
This guide shows how to build OpenVINO Runtime for later inference on Apple Silicon & Intel MYRIAD devices on OSX.
There are two options how to use OpenVINO on Apple Silicon:
- (Native) Compile OpenVINO for arm64 architecture with extra module [OpenVINO ARM plugin](https://github.com/openvinotoolkit/openvino_contrib/tree/master/modules/arm_plugin) location in [OpenVINO Contrib](https://github.com/openvinotoolkit/openvino_contrib). Note, build steps will cover this as a default scenario.
- (Rosetta) Compile Intel CPU plugin `x86_64` architecture and run under [Rosetta](https://support.apple.com/en-us/HT211861).
The software was validated on:
- macOS 11.x, 12.x, arm64
## Software requirements
- [brew](https://brew.sh) package manager to install additional dependencies. Use [install brew](https://brew.sh) guide to achieve this.
- Installation step for python and python libraries varies depending on the host architecture:
- **arm64** Python 3.7 or higher for the OpenVINO Runtime Python API, Development tools (Model Optimizer, POT and others):
```sh
% # let's have a look what python versions are available in brew
% brew search python
% # select preferred version of python based on available ones, e.g. 3.11
% brew install python@3.11
```
- **x86_64** Select universal2 installer from [Python releases] download page and install `python-3.X.Y-macos11.pkg` image. This allows to have universal python libraries, build x86_64 OpenVINO Python API and Development tools.
- [CMake](https://cmake.org/download/) 3.13 or higher:
```sh
% brew install cmake
```
- Clang compiler, git and other command line tools from Xcode 10.1 or higher:
```sh
% xcode-select --install
% brew install git-lfs
```
- (arm64 only) `scons` to build ARM compute library:
```sh
% python3 -m pip install scons
```
- (arm64 only) TBB library for threading:
```sh
% brew install tbb
```
- Additional `pip` dependencies to build OpenVINO Runtime Python API, Development tools (Model Optimizer, POT and others):
```sh
% # update pip and setuptools to newer versions
% python3 -m pip install -U pip setuptools
% python3 -m pip install cython
```
Additional install requirements (after OpenVINO repo clone) in order to build OpenVINO Python API and Development tools as wheel packages:
```sh
% python3 -m pip install -r <openvino source tree>/src/bindings/python/wheel/requirements-dev.txt
```
## How to build
1. (Get sources) Clone submodules:
```sh
git clone https://github.com/openvinotoolkit/openvino.git
git clone https://github.com/openvinotoolkit/openvino_contrib.git
cd openvino_contrib
git submodule update --init
cd ../openvino
git submodule update --init
```
2. Create a build folder:
```sh
mkdir build && cd build
```
3. (CMake configure) OpenVINO project uses a CMake-based build system. In the created `build` directory, run `cmake` to fetch project dependencies and create build rules:
```sh
cmake -DCMAKE_BUILD_TYPE=Release -DOPENVINO_EXTRA_MODULES=../openvino_contrib/modules/arm_plugin ..
```
> **NOTE**: By default OpenVINO CMake scripts try to introspect the system and enable all possible functionality based on that. You can look at the CMake output and see warnings, which show that some functionality is turned off and the corresponding reason, guiding what to do to install additionally to enable unavailable functionality. Additionally, you can change CMake options to enable / disable some functionality, add / remove compilation flags, provide custom version of dependencies like TBB, PugiXML, OpenCV, Protobuf. For more information, see [CMake Options for Custom Compilation](./cmake_options_for_custom_comiplation.md).
4. (CMake build) Build OpenVINO project:
```sh
cmake --build . --config Release --parallel $(sysctl -n hw.ncpu)
```
All built binaries are located in `<openvino_source_dir>/bin/<arm64 | intel64>/Release/` and wheel packages are located in `<openvino_build_dir>/wheels`.
5. (Optional install) Once you have built OpenVINO, you can install artifacts to a preferred location:
```sh
cmake -DCMAKE_INSTALL_PREFIX=<installation location> -P cmake_install.cmake
```
### Building x86_64 binaries
Since OSX version 11.x and Xcode version 12.2, the Apple development tools allow to compile arm64 code on x86 hosts and vice-versa. Based on this, OpenVINO can be compiled as x86_64 binary, then run on Apple Silicon hosts using [Rosetta](https://support.apple.com/en-us/HT211861). To do this, you must first install Rosetta:
```sh
softwareupdate --install-rosetta
```
Then try to compile OpenVINO using the steps above, but adding `-DCMAKE_OSX_ARCHITECTURES=x86_64` on cmake configure stage. But, **don't enable any system library usage explicitly** via CMake options, because they have `arm64` architecture, e.g.:
```sh
file /opt/homebrew/Cellar/tbb/2021.5.0_2/lib/libtbb.12.5.dylib
/opt/homebrew/Cellar/tbb/2021.5.0_2/lib/libtbb.12.5.dylib: Mach-O 64-bit dynamically linked shared library arm64
```
The same goes for other external dependencies like `libusb`. If you want to enable extra functionality like enable MYRIAD plugin build, you need to provide either x86_64 or universal2 `libusb` library. All other steps are the same as for usual compilation: build, install.
> **NOTE**: When building with `universal2` python libraries, wheel package is created with the `openvino-2022.3.0-000-cp39-cp39-macosx_12_0_universal2.whl` name and have a proper `universal2` tags, so they can *potentially* be used on both Apple Silicon and Intel CPU.
## See also
* [OpenVINO README](../../README.md)
* [OpenVINO Developer Documentation](index.md)
* [OpenVINO Get Started](./get_started.md)
* [How to build OpenVINO](build.md)

100
docs/dev/build_mac_intel_cpu.md Normal file
View File

@@ -0,0 +1,100 @@
# Build OpenVINO™ Runtime for macOS systems (Intel CPU)
This guide shows how to build OpenVINO Runtime for later inference on Intel CPU on macOS with Intel CPU underneath. This can be done using two ways:
- Compile on Intel CPU host using native compilation. Note, that [Build steps](#build-steps) show this scenario.
- Cross-compile on OSX Apple Silicon.
The software was validated on:
- macOS 10.x, 11.x, 12.x x86 64-bit
- macOS 11.x, 12.x, arm64 (cross-compilation)
## Software Requirements
- [brew](https://brew.sh) package manager to install additional dependencies. Use [install brew](https://brew.sh) guide to achieve this.
- [CMake](https://cmake.org/download/) 3.13 or higher
```sh
% brew install cmake
```
- Clang compiler, git and other command line tools from Xcode 10.1 or higher:
```sh
% xcode-select --install
% brew install git-lfs
```
- Installation step for python and python libraries varies depending on the host architecture:
- **x86_64** Python 3.7 or higher for the OpenVINO Runtime Python API, Development tools (Model Optimizer, POT and others):
```sh
% # let's have a look what python versions are available in brew
% brew search python
% # select preferred version of python based on available ones, e.g. 3.11
% brew install python@3.11
```
- **arm64** Select universal2 installer from [Python releases](https://www.python.org/downloads/macos/) download page and install `python-3.X.Y-macos11.pkg` image. This allows to have universal python libraries, build x86_64 OpenVINO Python API and Development tools.
- Additional `pip` dependencies to build OpenVINO Runtime Python API, Development tools (Model Optimizer, POT and others):
```sh
% # update pip and setuptools to newer versions
% python3 -m pip install -U pip setuptools
% python3 -m pip install cython
```
Additional install requirements (after OpenVINO repo clone) in order to build OpenVINO Python API and Development tools as wheel packages:
```sh
% python3 -m pip install -r <openvino source tree>/src/bindings/python/wheel/requirements-dev.txt
```
- (Optional; native compilation only) Latest version of TBB library. By default, OpenVINO downloads prebuilt version of TBB 2020.4 library, if you want to use latest (add `-DENABLE_SYSTEM_TBB=ON` additionally to cmake configuration step):
```sh
% brew install tbb
```
## How to build
1. Create a build folder:
```sh
mkdir build && cd build
```
2. (CMake configure) OpenVINO project uses a CMake-based build system. In the created `build` directory, run `cmake` to fetch project dependencies and create build rules:
```sh
cmake -DCMAKE_BUILD_TYPE=Release ..
```
> **NOTE**: By default OpenVINO CMake scripts try to introspect the system and enable all possible functionality based on that. You can look at the CMake output and see warnings, which show that some functionality is turned off and the corresponding reason, guiding what to do to install additionally to enable unavailable functionality. Additionally, you can change CMake options to enable / disable some functionality, add / remove compilation flags, provide custom version of dependencies like TBB, PugiXML, OpenCV, Protobuf. For more information, see [CMake Options for Custom Compilation](./cmake_options_for_custom_comiplation.md).
3. (CMake build) Build OpenVINO project:
```sh
cmake --build . --config Release --parallel $(sysctl -n hw.ncpu)
```
All built binaries are located in `<openvino_source_dir>/bin/intel64/Release/` and wheel packages are located in `<openvino_build_dir>/wheels`.
4. (Optional install) Once you have built OpenVINO, you can install artifacts to a preferred location:
```sh
cmake -DCMAKE_INSTALL_PREFIX=<installation location> -P cmake_install.cmake
```
### Cross-compilation
Since OSX version 11.x and Xcode version 12.2, the Apple development tools allows to compile arm64 code on x86 hosts and vice-versa. Based on this, OpenVINO can be compiled even on Apple Silicon machines, then such artifacts can be run on both Intel CPU hosts and Apple Silicon hosts (using [Rosetta](https://support.apple.com/en-us/HT211861)). To do this, add `-DCMAKE_OSX_ARCHITECTURES=x86_64 -DENABLE_INTEL_MYRIAD=OFF` in the cmake configuration step when compiling OpenVINO following the steps above. **Don't enable any system library usage explicitly** via CMake options, because they have `arm64` architecture, e.g.:
```sh
file /opt/homebrew/Cellar/tbb/2021.5.0_2/lib/libtbb.12.5.dylib
/opt/homebrew/Cellar/tbb/2021.5.0_2/lib/libtbb.12.5.dylib: Mach-O 64-bit dynamically linked shared library arm64
```
If you will see the errors like the one below:
```sh
ld: warning: ignoring file /opt/homebrew/lib/libopencv_imgproc.4.6.0.dylib, building for macOS-x86_64 but attempting to link with file built for macOS-arm64
Undefined symbols for architecture x86_64:
"cv::Mat::Mat(cv::Size_<int>, int, void*, unsigned long)", referenced from:
_image_resize in opencv_c_wrapper.cpp.o
_image_save in opencv_c_wrapper.cpp.o
....
ld: symbol(s) not found for architecture x86_64
clang: error: linker command failed with exit code 1 (use -v to see invocation)
```
Disable its usage in cmake or completely remove such library from the system (e.g. `brew uninstall opencv`), because it's pure arm64 and cannot be used to compile x86_64 binaries.
> **NOTE**: By using such way, the OpenVINO Intel CPU plugin can be cross-compiled, because MYRIAD plugin cannot be linked against `arm64` version of `libusb`
Or you have to explicitly find / compile x86_64 (or even `universal2`) dependencies by yourself and pass it to OpenVINO cmake scripts. E.g. compile oneTBB using additional option `-DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"`, install and then set `export TBBROOT=<universal oneTBB install root>` which will be used by OpenVINO.
## See also
* [OpenVINO README](../../README.md)
* [OpenVINO Developer Documentation](index.md)
* [OpenVINO Get Started](./get_started.md)
* [How to build OpenVINO](build.md)

68
docs/dev/build_raspbian.md Normal file
View File

@@ -0,0 +1,68 @@
# Build OpenVINO™ Runtime for Raspbian Stretch OS
> **NOTE**: [ARM CPU plugin](https://github.com/openvinotoolkit/openvino_contrib/tree/master/modules/arm_plugin) are supported. The detailed instruction how to build ARM plugin is available in [OpenVINO contrib wiki](https://github.com/openvinotoolkit/openvino_contrib/wiki/How-to-build-ARM-CPU-plugin).
## Hardware Requirements
* Raspberry Pi 2 or 3 with Raspbian Stretch OS (32 or 64-bit).
> **NOTE**: Despite the Raspberry Pi CPU is ARMv8, 32-bit OS detects ARMv7 CPU instruction set. The default `gcc` compiler applies ARMv6 architecture flag for compatibility with lower versions of boards. For more information, run the `gcc -Q --help=target` command and refer to the description of the `-march=` option.
You can compile the OpenVINO Runtime for Raspberry Pi in one of the two ways:
* [Native Compilation](#native-compilation), which is the simplest way, but time-consuming
* [Cross Compilation Using Docker](#cross-compilation-using-docker), which is the recommended way
## Native Compilation
Native compilation of the OpenVINO Runtime is the most straightforward solution. However, it might take at least one hour to complete on Raspberry Pi 3.
1. Install dependencies:
```bash
sudo apt-get update
sudo apt-get install -y git cmake scons build-essential
```
2. Clone the repositories:
```
git clone --recurse-submodules --single-branch --branch=master https://github.com/openvinotoolkit/openvino.git
git clone --recurse-submodules --single-branch --branch=master https://github.com/openvinotoolkit/openvino_contrib.git
```
3. Go to the cloned `openvino` repository:
```bash
cd openvino/
```
4. Create a build folder:
```bash
mkdir build && cd build/
```
5. Build the OpenVINO Runtime:
* for MYRIAD support only:
```bash
cmake -DCMAKE_BUILD_TYPE=Release \
-DOPENVINO_EXTRA_MODULES=<OPENVINO_CONTRIB_PATH>/openvino_contrib/modules/arm_plugin \
-DARM_COMPUTE_SCONS_JOBS=$(nproc --all) \
.. && cmake --build . --parallel
```
## Cross Compilation Using Docker
To cross-compile ARM CPU plugins using pre-configured `Dockerfile` you can use the following instruction: [Build OpenCV, OpenVINO™ and the plugin using pre-configured Dockerfile](https://github.com/openvinotoolkit/openvino_contrib/wiki/How-to-build-ARM-CPU-plugin#approach-1-build-opencv-openvino-and-the-plugin-using-pre-configured-dockerfile-cross-compiling-the-preferred-way).
## Additional Build Options
- To build Python API, install `libpython3-dev:armhf` and `python3-pip`
packages using `apt-get`; then install `numpy` and `cython` python modules
via `pip3`, adding the following options:
```sh
-DENABLE_PYTHON=ON \
-DPYTHON_EXECUTABLE=/usr/bin/python3.7 \
-DPYTHON_LIBRARY=/usr/lib/arm-linux-gnueabihf/libpython3.7m.so \
-DPYTHON_INCLUDE_DIR=/usr/include/python3.7
```
## See also
* [OpenVINO README](../../README.md)
* [OpenVINO Developer Documentation](index.md)
* [OpenVINO Get Started](./get_started.md)
* [How to build OpenVINO](build.md)

44
docs/dev/build_webassembly.md Normal file
View File

@@ -0,0 +1,44 @@
# Build OpenVINO™ Runtime for WebAssembly
This guide shows how to build OpenVINO for WebAssembly using [Emscripten SDK](https://emscripten.org/). Emscripten SDK can be directly downloaded and used, but it is more easier to use the [emscripten/emsdk](https://hub.docker.com/r/emscripten/emsdk) docker image.
The approach is validated on Linux, Windows and macOS operation systems.
## Software Requirements
- [Docker Engine](https://docs.docker.com/engine/install/)
## How to build
1. Clone OpenVINO repository and init submodules:
```sh
git clone https://github.com/openvinotoolkit/openvino.git
cd openvinof
git submodule update --init --recursive
```
2. Run docker image and mount a volume with OpenVINO source code:
```sh
$ docker pull emscripten/emsdk
$ docker run -it --rm -v `pwd`:/openvino emscripten/emsdk
```
3. (CMake configure) Run cmake configure step using helper emscripten command:
```sh
$ mkdir build && cd build
$ emcmake cmake -DCMAKE_BUILD_TYPE=Release /openvino
```
4. (CMake build) Build OpenVINO project:
```sh
$ emmake make -j$(nproc)
```
`openvino.wasm` and `openvino.js` files are located in:
- `<openvino_source_dir>/bin/ia32/Release/` on host machine file system.
- `/openvino/bin/ia32/Release` in docker environment.
These files can be used in browser applications.
## See also
* [OpenVINO README](../../README.md)
* [OpenVINO Developer Documentation](index.md)
* [OpenVINO Get Started](./get_started.md)
* [How to build OpenVINO](build.md)

106
docs/dev/build_windows.md Normal file
View File

@@ -0,0 +1,106 @@
# Build OpenVINO™ Runtime for Windows systems
OpenVINO can be compiled for different architectures on Windows: X64 or ARM64. In order to build for ARM64 architecture, the machine with Windows on ARM is required as only native compilation is supported (see [similar documents](https://www.linaro.org/blog/how-to-set-up-windows-on-arm-for-llvm-development/#:~:text=Install%20the%20Latest%20LLVM%20for,PATH%20yourself%2C%20as%20described%20above.) for details).
Supported configurations:
- Windows 10 x86 64-bit or higher with Visual Studio 2019 or higher build for X64 architecture.
- Windows on ARM (shortly WoA) to build for ARM64 architecture. OpenVINO was validated on [Windows DevKit 2023](https://developer.qualcomm.com/hardware/windows-on-snapdragon/windows-dev-kit-2023)
## Software requirements
- [CMake](https://cmake.org/download/) 3.13 or higher
- Microsoft Visual Studio 2019 or higher, version 16.3 or later
> **NOTE**: Native Microsoft Visual Studio for WoA is available since 2022.
- Python 3.7 or higher for OpenVINO Runtime Python API
> **NOTE**: Python for ARM64 is available since [3.11](https://www.python.org/downloads/windows/) version.
- [Git for Windows*]
- (Windows on ARM only) [LLVM for Windows on ARM (WoA)](https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.6/LLVM-15.0.6-woa64.exe)
> **NOTE**: After installation, make sure `clang-cl` compiler is available from `PATH`.
## How to build
> **NOTE**: By default, the build enables the OpenVINO Runtime GPU plugin to infer models on your Intel® Processor Graphics. This requires you to download and install the Intel® Graphics Driver for Windows (26.20) [driver package](https://www.intel.com/content/www/us/en/download/19344/intel-graphics-windows-dch-drivers.html) before running the build. If you don't want to use the GPU plugin, use the `-DENABLE_INTEL_GPU=OFF` CMake build option and skip the installation of the Intel® Graphics Driver.
1. Clone submodules:
```sh
git clone https://github.com/openvinotoolkit/openvino.git
cd openvino
git submodule update --init --recursive
```
(Extra for WoA) To build on Windows on ARM with ARM plugin:
```sh
git clone https://github.com/openvinotoolkit/openvino_contrib.git
cd openvino_contrib
git submodule update --init --recursive
```
2. Create build directory:
```sh
mkdir build && cd build
```
3. In the `build` directory, run `cmake` to fetch project dependencies and generate a Visual Studio solution.
On Windows x86 64-bits:
```sh
cmake -G "Visual Studio 16 2019" -DCMAKE_BUILD_TYPE=Release <openvino>
```
On Windows on ARM for ARM64 architecture:
```sh
cmake -G "Visual Studio 16 2019" -DOPENVINO_EXTRA_MODULES=<openvino_contrib>/modules/arm_plugin -DCMAKE_BUILD_TYPE=Release <openvino>
```
4. Build generated solution in Visual Studio or run `cmake --build . --config Release --verbose -j8` to build from the command line. Be aware that this process may take some time.
5. Before running the samples, add paths to the Threading Building Blocks (TBB) binaries used for the build to the `%PATH%` environment variable. By default, TBB binaries are downloaded by the CMake-based script to the `<openvino>/temp/tbb/bin` folder.
### Additional Build Options
- Internal JIT GEMM implementation is used by default.
- Threading Building Blocks (TBB) is used by default. To build Inference Engine with OpenMP threading, set the `-DTHREADING=OMP` option.
- Required versions of TBB and OpenCV packages are downloaded automatically by the CMake-based script. If you want to use the automatically-downloaded packages but you have already installed TBB or OpenCV packages configured in your environment, you may need to clean the `TBBROOT` and `OpenCV_DIR` environment variables before running the `cmake` command; otherwise they won'tnbe downloaded and the build may fail if incompatible versions were installed.
- If the CMake-based build script can not find and download the OpenCV package that is supported on your platform, or if you want to use a custom build of the OpenCV library, refer to the [Use Custom OpenCV Builds](./cmake_options_for_custom_comiplation.md#Building-with-custom-OpenCV) section for details.
- To build the OpenVINO Runtime Python API:
1. First, install all additional packages (e.g., cython and opencv) listed in the file:
```sh
pip install -r <openvino>\src\bindings\python\src\compatibility\openvino\requirements-dev.txt
```
2. Second, enable the `-DENABLE_PYTHON=ON` in the CMake (Step #4) option above. To specify an exact Python version, use the following options:
```sh
-DPYTHON_EXECUTABLE="C:\Program Files\Python11\python.exe" ^
-DPYTHON_LIBRARY="C:\Program Files\Python11\libs\python11.lib" ^
-DPYTHON_INCLUDE_DIR="C:\Program Files\Python11\include"
```
3. To build a wheel package (.whl), enable the `-DENABLE_WHEEL=ON` option in the CMake step above (Step 4):
4. After the build process finishes, export the newly built Python libraries to the user environment variables:
```
set PYTHONPATH=<openvino_repo>/bin/<arch>/Release/python_api/python3.11;%PYTHONPATH%
set OPENVINO_LIB_PATH=<openvino_repo>/bin/<arch>/Release;%OPENVINO_LIB_PATH%
```
or install the wheel with pip:
```
pip install build/wheel/openvino-2023.0.0-9612-cp11-cp11-win_arm64.whl
```
- OpenVINO runtime compilation options:
`-DENABLE_OV_ONNX_FRONTEND=ON` enables the building of the ONNX importer.
### Building OpenVINO with Ninja* Build System
```sh
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\VC\Auxiliary\Build\vcvars64.bat"
cmake -G Ninja -Wno-dev -DCMAKE_BUILD_TYPE=Release ..
cmake --build . --config Release
```
## See also
* [OpenVINO README](../../README.md)
* [OpenVINO Developer Documentation](index.md)
* [OpenVINO Get Started](./get_started.md)
* [How to build OpenVINO](build.md)

59
docs/dev/building_documentation.md Normal file
View File

@@ -0,0 +1,59 @@
# How to build documentation
The following procedure was validated on Windows and Ubuntu operation systems.
## Table of content:
* [Installing dependencies](#installing-dependencies)
* [Building documentation](#installing-dependencies)
* [Additional Resources](#additional-resources)
## Installing dependencies
The `doxygen` and `latex` must be installed in addition to usual build dependencies:
### Windows
* [miktex](https://miktex.org/)
* [doxygen](http://doxygen.nl/files/doxygen-1.8.20-setup.exe) (version >= 1.8.20)
* [Graphviz](https://graphviz.org/download/)
### Ubuntu systems
* Latex and graphviz:
```sh
apt-get install texlive-full graphviz
```
* goxygen version >= 1.8.20:
```
$ git clone https://github.com/doxygen/doxygen.git
$ cd doxygen
$ git checkout Release_1_8_20
$ mkdir build
$ cd build
$ cmake ..
$ cmake --build . -j8
$ sudo make install
```
## Building documentation
You should run `cmake` with `-DENABLE_DOCS=ON` and it will find all dependencies automatically on Ubuntu systems, while on Windows we still need to specify paths to the installed dependencies:
```sh
cmake -DLATEX_COMPILER="C:/Program Files/MiKTeX/miktex/bin/x64/latex.exe" \
-DDOXYGEN_DOT_EXECUTABLE="C:/Program Files (x86)/Graphviz2.38/bin/dot.exe" \
-DDOXYGEN_EXECUTABLE="C:/Program Files/doxygen/bin/doxygen.exe" \
-DENABLE_DOCS=ON \
```
Once the dependencies are found, the project must generated using CMake. The target `openvino_docs` must be built to generate doxygen documentation, the generated files can be found at `<binary dir>/docs/html/index.html`
## See also
* [OpenVINO README](../../README.md)
* [OpenVINO Developer Documentation](index.md)
* [How to Build OpenVINO](build.md)

192
docs/dev/cmake_options_for_custom_comiplation.md Normal file
View File

@@ -0,0 +1,192 @@
# CMake Options for Custom Compilation
This document provides description and default values for CMake options that can be used to build a custom OpenVINO runtime using the open source version. For instructions on how to create a custom runtime from the prebuilt OpenVINO release package, refer to the [deployment manager] documentation. To understand all the dependencies when creating a custom runtime from the open source repository, refer to the [OpenVINO Runtime Introduction].
## Table of contents:
* [Disable / enable plugins build and other components](#disable--enable-plugins-build-and-other-components)
* [Options affecting binary size](#options-affecting-binary-size)
* [Building with custom OpenCV](#building-with-custom-opencv)
* [Building with custom TBB](#building-with-custom-tbb)
* [Test capabilities](#test-capabilities)
* [Other options](#other-options)
- [Additional Resources](#additional-resources)
## Disable / enable plugins build and other components
* Inference plugins:
* `ENABLE_INTEL_CPU` enables CPU plugin compilation:
* `ON` is default for x86 platforms; `OFF`, otherwise.
* `ENABLE_INTEL_GPU` enables Intel GPU plugin compilation:
* `ON` is default for x86 platforms; not available, otherwise.
* `ENABLE_INTEL_GNA` enables GNA plugin compilation:
* `ON` is default for x86 platforms; not available, otherwise.
* `ENABLE_HETERO` enables HETERO plugin build:
* `ON` is default.
* `ENABLE_MULTI` enables MULTI plugin build:
* `ON` is default.
* `ENABLE_AUTO` enables AUTO plugin build:
* `ON` is default.
* `ENABLE_TEMPLATE` enables TEMPLATE plugin build:
* `ON` is default.
* `ENABLE_AUTO_BATCH` enables Auto Batch plugin build:
* `ON` is default.
* Frontends to work with models from frameworks:
* `ENABLE_OV_ONNX_FRONTEND` enables [ONNX] frontend plugin for OpenVINO Runtime:
* `ON` is default.
* `ENABLE_OV_PDPD_FRONTEND` enables [PDPD] frontend plugin for OpenVINO Runtime:
* `ON` is default.
* `ENABLE_OV_TF_FRONTEND` enables [TensorFlow] frontend plugin for OpenVINO Runtime:
* `ON` is default.
* `ENABLE_OV_TF_LITE_FRONTEND` enables [TensorFlow Lite] frontend plugin for OpenVINO Runtime:
* `ON` is default.
* `ENABLE_OV_PYTORCH_FRONTEND` enables [PyTorch] frontend plugin for OpenVINO Runtime:
* `ON` is default.
* `ENABLE_OV_IR_FRONTEND` enables OpenVINO Intermediate Representation frontend plugin for OpenVINO Runtime:
* `ON` is default.
* `ENABLE_GAPI_PREPROCESSING` enables G-API preprocessing:
* `ON` is default.
* `OPENVINO_EXTRA_MODULES` specifies path to add extra OpenVINO modules to the build.
* See [OpenVINO Contrib] to add extra modules from.
* `ENABLE_SAMPLES` enables OpenVINO Runtime samples build:
* `ON` is default.
* `ENABLE_PYTHON` enables [Python] API build:
* `ON` if python requirements are satisfied (auto-discovered by CMake).
* `ENABLE_WHEEL` enables [Python] OpenVINO Runtime and Development wheels build:
* `ON` if requirements are satisfied (auto-discovered by CMake).
* `ENABLE_TESTS` enables tests compilation:
* `OFF` is default.
* `ENABLE_IR_V7_READER` enables IR v7 reader:
* `ON` is default.
**Note:** must be turned `OFF` when building OpenVINO runtime as static
* `ENABLE_DOCS` enables building the OpenVINO documentation:
* `OFF` is on Debian (Ubuntu) OSes
* `OFF` is in other cases.
* `ENABLE_SYSTEM_PUGIXML` builds with system version of [pugixml] if it is available on the system.
* `ON` is default.
* [OpenVINO thirdparty pugixml] is used by default.
* `ENABLE_SYSTEM_PROTOBUF` use [protobuf] installed on the system (used by ONNX, PaddlePaddle and TensorFlow frontends):
* `OFF` is default.
* `ENABLE_SYSTEM_FLATBUFFERS` use [FlatBuffers] installed on the system (used by Tensorflow Lite frontend):
* `ON` is default.
* `ENABLE_SYSTEM_TBB` use TBB installed on the system:
* `ON` is on Debian (Ubuntu) OSes.
* `OFF` is in other cases.
* `ENABLE_SYSTEM_OPENCL` use OpenCL installed on the system:
* `ON` is default.
## Options affecting binary size
* `ENABLE_LTO` boolean option to enable [Link Time Optimizations]:
* `OFF` is default, because it takes longer time to link binaries.
* `ON` is enabled for OpenVINO release binaries.
* Available on Unix* compilers only like GCC or CLANG.
* Gives 30% decrease in binary size together with other optimization options used to build OpenVINO.
* `THREADING` points to the OpenVINO threading interface:
* `TBB` is the default option, which enables build with [Intel TBB] and `tbb::static_partitioner`.
* `TBB_AUTO` enables building with [Intel TBB].
* `OMP` enables building with Intel OpenMP.
* `SEQ` disables threading optimizations. Can be used in cases when TBB binaries are absent.
* **Note:** because TBB is a template library, it increases binary size because of multiple instantiations of `tbb::parallel_for`
* `ENABLE_TBBBIND_2_5` enables prebuilt static TBBBind 2.5 usage:
* `ON` is default, because OpenVINO Runtime should be generic out of box.
> **Note:** TBBBind 2.5 is needed when OpenVINO **inference** targets CPUs with:
> * NUMA support (Non-Unified Memory Architecture), e.g. to detect a number of NUMA nodes
> * Hybrid architecture to separate Performance / Efficiency cores and schedule tasks in the optimal way.
> **Note:** if you build OpenVINO runtime with [oneTBB] support where TBBBind 2.5 is automatically loaded by TBB in runtime, then set `ENABLE_TBBBIND_2_5` to `OFF`
* `ENABLE_SSE42` enables SSE4.2 optimizations:
* `ON` is default for x86 platforms; not available for other platforms.
* Affects only OpenVINO Runtime common part and preprocessing plugin, **does not affect the oneDNN library**
* `ENABLE_AVX2` enables AVX2 optimizations:
* `ON` is default for x86 platforms, not available for other platforms.
* Affects only OpenVINO Runtime common part and preprocessing plugin, **does not affect the oneDNN library**
* `ENABLE_AVX512F` enables AVX512 optimizations:
* `ON` is default for x86 platforms, not available for other platforms.
* Affects only OpenVINO Runtime common part and preprocessing plugin, **does not affect the oneDNN library**
* `ENABLE_PROFILING_ITT` enables profiling with [Intel ITT and VTune].
* `OFF` is default, because it increases binary size.
* `SELECTIVE_BUILD` enables [[Conditional compilation|ConditionalCompilation]] feature.
* `OFF` is default.
## Building with custom OpenCV
When OpenVINO CMake scripts are run with enabled OpenCV (`-DENABLE_OPENCV=ON` which is default), CMake OpenVINO scripts automatically download prebuilt OpenCV
from shared drive, such OpenCV is ABI-compatible with the default compiler of your system. If you have a non-default compiler or want to use custom version of OpenCV, you can pass it via CMake option:
```sh
cmake -DOpenCV_DIR=<path to OpenCVConfig.cmake> ...
```
In this case, default OpenCV will not downloaded and provided one will be used.
## Building with custom TBB
When OpenVINO CMake scripts are run with TBB enabled (`-DTHREADING=TBB` which is default, or `-DTHREADING=TBB_AUTO`), CMake OpenVINO scripts automatically download prebuilt version of TBB which is ABI-compatible with the default compiler of your system. If you have a non-default compiler or want to use custom TBB, you can use:
```sh
export TBBROOT=<path to TBB install root>
cmake ...
```
In this case OpenVINO CMake scripts take `TBBROOT` environment variable into account and provided TBB will be used.
**Note:** if you are building TBB from source files, please install TBB after and use `TBBROOT` to point to installation root.
## Test capabilities
* `ENABLE_SANITIZER` builds with clang [address sanitizer] support:
* `OFF` is default.
* `ENABLE_THREAD_SANITIZER` builds with clang [thread-sanitizer] support:
* `OFF` is default.
* `ENABLE_COVERAGE` adds option to enable coverage. See dedicated guide [[how to measure test coverage|InferenceEngineTestsCoverage]]:
* `OFF` is default.
* `ENABLE_FUZZING` enables instrumentation of code for fuzzing:
* `OFF` is default.
## Other options
* `ENABLE_CPPLINT` enables code style check using [cpplint] static code checker:
* `ON` is default.
* `ENABLE_CLANG_FORMAT` enables [Clang format] code style check:
* `ON` is default.
* Used only for ngraph component.
* `TREAT_WARNING_AS_ERROR` treats all warnings as an error:
* `OFF` is default.
* `ENABLE_FASTER_BUILD` enables [precompiled headers] and [unity build] using CMake:
* `OFF` is default.
* `ENABLE_INTEGRITYCHECK` builds DLLs with [/INTEGRITYCHECK] flag:
* `OFF` is default.
* Available on MSVC compiler only.
## Additional Resources
* [OpenVINO README](../../README.md)
* [OpenVINO Developer Documentation](index.md)
* [How to build OpenVINO](build.md)
[Link Time Optimizations]:https://llvm.org/docs/LinkTimeOptimization.html
[thread-sanitizer]:https://clang.llvm.org/docs/ThreadSanitizer.html
[address sanitizer]:https://clang.llvm.org/docs/AddressSanitizer.html
[Intel ITT and VTune]:https://software.intel.com/content/www/us/en/develop/documentation/vtune-help/top/api-support/instrumentation-and-tracing-technology-apis.html
[precompiled headers]:https://cmake.org/cmake/help/git-stage/command/target_precompile_headers.html
[unity build]:https://cmake.org/cmake/help/latest/prop_tgt/UNITY_BUILD.html
[/INTEGRITYCHECK]:https://docs.microsoft.com/en-us/cpp/build/reference/integritycheck-require-signature-check?view=msvc-160
[Intel TBB]:https://software.intel.com/content/www/us/en/develop/tools/threading-building-blocks.html
[Python]:https://www.python.org/
[Java]:https://www.java.com/ru/
[cpplint]:https://github.com/cpplint/cpplint
[Clang format]:http://clang.llvm.org/docs/ClangFormat.html
[OpenVINO Contrib]:https://github.com/openvinotoolkit/openvino_contrib
[OpenVINO thirdparty pugixml]:https://github.com/openvinotoolkit/openvino/tree/master/inference-engine/thirdparty/pugixml
[pugixml]:https://pugixml.org/
[ONNX]:https://onnx.ai/
[protobuf]:https://github.com/protocolbuffers/protobuf
[deployment manager]:https://docs.openvino.ai/latest/openvino_docs_install_guides_deployment_manager_tool.html
[OpenVINO Runtime Introduction]:https://docs.openvino.ai/latest/openvino_docs_OV_UG_Integrate_OV_with_your_application.html
[PDPD]:https://github.com/PaddlePaddle/Paddle
[TensorFlow]:https://www.tensorflow.org/
[TensorFlow Lite]:https://www.tensorflow.org/lite
[PyTorch]:https://www.tensorflow.org/lite
[FlatBuffers]:https://google.github.io/flatbuffers/
[oneTBB]:https://github.com/oneapi-src/oneTBB

10
docs/dev/conditional_compilation.md
View File

@@ -9,13 +9,13 @@
## Introduction
Conditional compilation can significantly reduce OpenVINO™ binaries size by excluding unnecessary components for particular model inference.
The following components can be excluded from the build:
* Operations and graph transformations in OpenVINO and plugins,
Conditional compilation can significantly reduce the size of OpenVINO™ binaries by excluding unnecessary components for inference of particular models. The components are:
* layers and graph transformations in OpenVINO Runtime and plugins,
* OpenVINO Runtime operations,
* jit kernels in a CPU plugin,
* arbitrary code that is not used for particular model inference.
However, the conditional compilation has a significant drawback - **the resulting OpenVINO Runtime will work only with a limited set of models and devices.**
However, conditional compilation has a significant drawback - the resulting OpenVINO runtime will properly infer only using the models and devices for which it was compiled. If just one model is used to collect statistics for compilation, only this particular model is guaranteed to work with the resulting OpenVINO runtime.
There are two conditional compilation build stages: `SELECTIVE_BUILD=COLLECT` and `SELECTIVE_BUILD=ON`.
@@ -66,7 +66,7 @@ To apply conditional compilation, follow the steps below:
1. Run the CMake tool with the following options: `-DSELECTIVE_BUILD=ON -DSELECTIVE_BUILD_STAT=${ABSOLUTE_PATH_TO_STATISTICS_FILES}/*.csv -DENABLE_PROFILING_ITT=OFF`
2. `cmake --build <cmake_build_directory>`
The `-niter 1 -nireq 1` flags are highly recommended for the benchmark_app. Otherwise, the trace files will be very large.
The `-niter 1 -nireq 1` flags are highly recommended for benchmark_app. Otherwise, the trace files will be very large.
If you are using an application other than benchmark_app, remember to limit the number of inference requests and iterations.
## Building for devices with different ISA

2
docs/dev/get_started.md
View File

@@ -17,3 +17,5 @@ Explore other resources to learn more about OpenVINO:
## See also
* [OpenVINO Developer Documentation](./index.md)
* [OpenVINO Samples](../../samples)
* [OpenVINO Building Documentation](./building_documentation.md)
* [CMake Options for Custom Compilation](./cmake_options_for_custom_comiplation.md)

266
docs/dev/installing.md Normal file
View File

@@ -0,0 +1,266 @@
# Installing
Once the project is built you can install OpenVINO™ Runtime into custom location:
```
cmake --install <BUILDDIR> --prefix <INSTALLDIR>
```
## Installation check
<details>
<summary>For versions prior to 2022.1</summary>
<p>
1. Obtaining Open Model Zoo tools and models
To have the ability to run samples and demos, you need to clone the Open Model Zoo repository and copy the folder under `./deployment_tools` to your install directory:
```
git clone https://github.com/openvinotoolkit/open_model_zoo.git
cmake -E copy_directory ./open_model_zoo/ <INSTALLDIR>/deployment_tools/open_model_zoo/
```
2. Adding OpenCV to your environment
Open Model Zoo samples use OpenCV functionality to load images. To use it for demo builds you need to provide the path to your OpenCV custom build by setting `OpenCV_DIR` environment variable and add path OpenCV libraries to the `LD_LIBRARY_PATH (Linux)` or `PATH (Windows)` variable before running demos.
Linux:
```sh
export LD_LIBRARY_PATH=/path/to/opencv_install/lib/:$LD_LIBRARY_PATH
export OpenCV_DIR=/path/to/opencv_install/cmake
```
Windows:
```sh
set PATH=\path\to\opencv_install\bin\;%PATH%
set OpenCV_DIR=\path\to\opencv_install\cmake
```
3. Running demo
To check your installation go to the demo directory and run Classification Demo:
Linux and macOS:
```sh
cd <INSTALLDIR>/deployment_tools/demo
./demo_squeezenet_download_convert_run.sh
```
Windows:
```sh
cd <INSTALLDIR>\deployment_tools\demo
demo_squeezenet_download_convert_run.bat
```
Result:
```
Top 10 results:
Image <INSTALLDIR>/deployment_tools/demo/car.png
classid probability label
------- ----------- -----
817 0.6853030 sports car, sport car
479 0.1835197 car wheel
511 0.0917197 convertible
436 0.0200694 beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon
751 0.0069604 racer, race car, racing car
656 0.0044177 minivan
717 0.0024739 pickup, pickup truck
581 0.0017788 grille, radiator grille
468 0.0013083 cab, hack, taxi, taxicab
661 0.0007443 Model T
[ INFO ] Execution successful
```
</p>
</details>
<details open>
<summary> For 2022.1 and after</summary>
<p>
1. Build samples
To build C++ sample applications, run the following commands:
Linux and macOS:
```sh
cd <INSTALLDIR>/samples/cpp
./build_samples.sh
```
Windows:
```sh
cd <INSTALLDIR>\samples\cpp
build_samples_msvc.bat
```
2. Install OpenVINO Development Tools
> **NOTE**: To build OpenVINO Development Tools (Model Optimizer, Post-Training Optimization Tool, Model Downloader, and Open Model Zoo tools) wheel package locally you are required to use CMake option: `-DENABLE_WHEEL=ON`.
To install OpenVINO Development Tools to work with Caffe models, execute the following commands:
Linux and macOS:
```sh
#setup virtual envrinment
python3 -m venv openvino_env
source openvino_env/bin/activate
pip install pip --upgrade
#install local package from install directory
pip install openvino_dev-<version>-py3-none-any.whl[caffe] --find-links=<INSTALLDIR>/tools
```
Windows:
```bat
rem setup virtual envrinment
python -m venv openvino_env
openvino_env\Scripts\activate.bat
pip install pip --upgrade
rem install local package from install directory
cd <INSTALLDIR>\tools
pip install openvino_dev-<version>-py3-none-any.whl[caffe] --find-links=<INSTALLDIR>\tools
```
3. Download the Models
Download the following model to run the Image Classification Sample:
Linux and macOS:
```sh
omz_downloader --name googlenet-v1 --output_dir ~/models
```
Windows:
```bat
omz_downloader --name googlenet-v1 --output_dir %USERPROFILE%\Documents\models
```
4. Convert the Model with Model Optimizer
Linux and macOS:
```sh
mkdir ~/ir
mo --input_model ~/models/public/googlenet-v1/googlenet-v1.caffemodel --data_type FP16 --output_dir ~/ir
```
Windows:
```bat
mkdir %USERPROFILE%\Documents\ir
mo --input_model %USERPROFILE%\Documents\models\public\googlenet-v1\googlenet-v1.caffemodel --data_type FP16 --output_dir %USERPROFILE%\Documents\ir
```
5. Run Inference on the Sample
Set up the OpenVINO environment variables:
Linux and macOS:
```sh
source <INSTALLDIR>/setupvars.sh
```
Windows:
```bat
<INSTALLDIR>\setupvars.bat
```
The following commands run the Image Classification Code Sample using the [`dog.bmp`](https://storage.openvinotoolkit.org/data/test_data/images/224x224/dog.bmp) file as an input image, the model in IR format from the `ir` directory, and on different hardware devices:
Linux and macOS:
```sh
cd ~/inference_engine_cpp_samples_build/intel64/Release
./classification_sample_async -i ~/Downloads/dog.bmp -m ~/ir/googlenet-v1.xml -d CPU
```
Windows:
```bat
cd %USERPROFILE%\Documents\Intel\OpenVINO\inference_engine_samples_build\intel64\Release
.\classification_sample_async.exe -i %USERPROFILE%\Downloads\dog.bmp -m %USERPROFILE%\Documents\ir\googlenet-v1.xml -d CPU
```
When the sample application is complete, you see the label and confidence data for the top 10 categories on the display:
```
Top 10 results:
Image dog.bmp
classid probability
------- -----------
156 0.6875963
215 0.0868125
218 0.0784114
212 0.0597296
217 0.0212105
219 0.0194193
247 0.0086272
157 0.0058511
216 0.0057589
154 0.0052615
```
</p>
</details>
## Adding OpenVINO Runtime (Inference Engine) to Your Project
<details>
<summary>For versions prior to 2022.1</summary>
<p>
For CMake projects, set the `InferenceEngine_DIR` and when you run CMake tool:
```sh
cmake -DInferenceEngine_DIR=/path/to/openvino/build/ .
```
Then you can find Inference Engine by [`find_package`]:
```cmake
find_package(InferenceEngine REQUIRED)
target_link_libraries(${PROJECT_NAME} PRIVATE ${InferenceEngine_LIBRARIES})
```
</p>
</details>
<details open>
<summary>For 2022.1 and after</summary>
<p>
For CMake projects, set the `OpenVINO_DIR` and when you run CMake tool:
```sh
cmake -DOpenVINO_DIR=<INSTALLDIR>/runtime/cmake .
```
Then you can find OpenVINO Runtime (Inference Engine) by [`find_package`]:
```cmake
find_package(OpenVINO REQUIRED)
add_executable(ov_app main.cpp)
target_link_libraries(ov_app PRIVATE openvino::runtime)
add_executable(ov_c_app main.c)
target_link_libraries(ov_c_app PRIVATE openvino::runtime::c)
```
</p>
</details>
## See also
* [OpenVINO README](../../README.md)
* [OpenVINO Developer Documentation](index.md)
* [OpenVINO How to Build](build.md)

157
docs/dev/static_libaries.md Normal file
View File

@@ -0,0 +1,157 @@
# Building OpenVINO static libraries
## Contents
- [Introduction](#introduction)
- [System requirements](#system-requirements)
- [Configure OpenVINO runtime in CMake stage](#configure-openvino-runtime-in-cmake-stage)
- [Build static OpenVINO libraries](#build-static-openvino-libraries)
- [Link static OpenVINO runtime](#link-static-openvino-runtime)
- [Static OpenVINO libraries + Conditional compilation for particular models](#static-openvino-libraries--conditional-compilation-for-particular-models)
- [Building with static MSVC Runtime](#building-with-static-msvc-runtime)
- [Limitations](#limitations)
- [See also](#see-also)
## Introduction
Building static OpenVINO Runtime libraries allows to additionally reduce the size of a binary when it is used together with conditional compilation.
It is possible because not all interface symbols of OpenVINO Runtime libraries are exported to end users during a static build and can be removed by linker. See [Static OpenVINO libraries + Conditional compilation for particular models](#static-openvino-libraries--conditional-compilation-for-particular-models)
## System requirements
* CMake version 3.17 or higher must be used to build static OpenVINO libraries.
* Supported OSes:
* Windows x64
* Linux x64
* All other OSes may work, but have not been explicitly tested
## Configure OpenVINO Runtime in the CMake stage
The default architecture of OpenVINO Runtime assumes that the following components are subject to dynamic loading during execution:
* (Device) Inference backends (CPU, GPU, MULTI, HETERO, etc.)
* (Model) Frontends (IR, ONNX, PDPD, etc.)
* Preprocessing library (to perform preprocessing, e.g. resize and color space conversions)
* IR v7 reader (used in legacy tests only, if you are not to going to run OpenVINO tests, set `-DENABLE_TESTS=OFF` which disables IR v7 reader)
With the static OpenVINO Runtime, all these modules should be linked into a final user application and **the list of modules/configuration must be known for the CMake configuration stage**. To minimize the total binary size, you can explicitly turn `OFF` unnecessary components. Use [[CMake Options for Custom Compilation|CMakeOptionsForCustomCompilation ]] as a reference for OpenVINO CMake configuration.
For example, to enable only IR v11 reading and CPU inference capabilities, use:
```sh
cmake -DENABLE_INTEL_GPU=OFF \
-DENABLE_INTEL_GNA=OFF \
-DENABLE_TEMPLATE=OFF \
-DENABLE_HETERO=OFF \
-DENABLE_MULTI=OFF \
-DENABLE_AUTO=OFF \
-DENABLE_AUTO_BATCH=OFF \
-DENABLE_OV_ONNX_FRONTEND=OFF \
-DENABLE_OV_PADDLE_FRONTEND=OFF \
-DENABLE_OV_TF_FRONTEND=OFF \
-DENABLE_OV_TF_LITE_FRONTEND=OFF \
-DENABLE_OV_PYTORCH_FRONTEND=OFF \
-DENABLE_IR_V7_READER=OFF \
-DENABLE_GAPI_PREPROCESSING=OFF \
-DENABLE_INTEL_CPU=ON \
-DENABLE_OV_IR_FRONTEND=ON
```
> **NOTE**: Inference backends located in external repositories can also be used in a static build. Use `-DOPENVINO_EXTRA_MODULES=<path to external plugin root>` to enable them. `OpenVINODeveloperPackage.cmake` must not be used to build external plugins, only `OPENVINO_EXTRA_MODULES` is a working solution.
> **NOTE**: The `ENABLE_LTO` CMake option can also be passed to enable link time optimizations to reduce the binary size. But such property should also be enabled on the target which links with static OpenVINO libraries via `set_target_properties(<target_name> PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ON)`
-
## Build static OpenVINO libraries
To build OpenVINO Runtime in a static mode, you need to specify the additional CMake option:
```sh
cmake -DBUILD_SHARED_LIBS=OFF <all other CMake options> <openvino_sources root>
```
Then, use the usual CMake 'build' command:
```sh
cmake --build . --target openvino --config Release -j12
```
Then, the installation step:
```sh
cmake -DCMAKE_INSTALL_PREFIX=<install_root> -P cmake_install.cmake
```
The OpenVINO runtime is located in `<install_root>/runtime/lib`
## Link static OpenVINO Runtime
Once you build static OpenVINO Runtime libraries and install them, you can use one of the two ways to add them to your project:
### CMake interface
Just use CMake's `find_package` as usual and link `openvino::runtime`:
```cmake
find_package(OpenVINO REQUIRED)
target_link_libraries(<application> PRIVATE openvino::runtime)
```
`openvino::runtime` transitively adds all other static OpenVINO libraries to a linker command.
### Pass libraries to linker directly
If you want to configure your project directly, you need to pass all libraries from `<install_root>/runtime/lib` to linker command.
> **NOTE**: Since the proper order of static libraries must be used (dependent library should come **before** dependency in a linker command), consider using the following compiler specific flags to link static OpenVINO libraries:
Microsoft Visual Studio compiler:
```sh
/WHOLEARCHIVE:<ov_library 0> /WHOLEARCHIVE:<ov_library 1> ...
```
GCC like compiler:
```sh
gcc main.cpp -Wl,--whole-archive <all libraries from <root>/runtime/lib> > -Wl,--no-whole-archive -o a.out
```
## Static OpenVINO libraries + Conditional compilation for particular models
OpenVINO Runtime can be compiled for particular models, as shown in the [[Conditional compilation for particular models|ConditionalCompilation]] guide.
The conditional compilation feature can be paired with static OpenVINO libraries to build even smaller end-user applications in terms of binary size. The following procedure can be used, (based on the detailed [[Conditional compilation for particular models|ConditionalCompilation]] guide):
* Build OpenVINO Runtime as usual with the CMake option of `-DSELECTIVE_BUILD=COLLECT`.
* Run target applications on target models and target platforms to collect traces.
* Build the final OpenVINO static Runtime with `-DSELECTIVE_BUILD=ON -DSELECTIVE_BUILD_STAT=/path/*.csv -DBUILD_SHARED_LIBS=OFF`
## Building with static MSVC Runtime
In order to build with static MSVC runtime, use the special [OpenVINO toolchain](https://github.com/openvinotoolkit/openvino/blob/master/cmake/toolchains/mt.runtime.win32.toolchain.cmake) file:
```sh
cmake -DCMAKE_TOOLCHAIN_FILE=<openvino source dir>/cmake/toolchains/mt.runtime.win32.toolchain.cmake <other options>
```
> **NOTE**: all other dependent application and libraries must be built with the same `mt.runtime.win32.toolchain.cmake ` toolchain to have conformed values of the `MSVC_RUNTIME_LIBRARY` target property.
## Limitations
* The enabled and tested capabilities of OpenVINO Runtime in a static build:
* OpenVINO common runtime - work with `ov::Model`, perform model loading on particular device
* CPU and GNA inference plugins (**GPU and MYRIAD are not enabled**)
* MULTI, HETERO, AUTO, and BATCH inference modes
* IR, ONNX, PDPD, and TF frontends to read `ov::Model`
* Static build support for building static libraries only for OpenVINO Runtime libraries. All other third-party prebuilt dependencies remain in the same format:
* `libGNA` is a shared library.
* `TBB` is a shared library; to provide your own TBB build from [[oneTBB source code|https://github.com/oneapi-src/oneTBB]] use `export TBBROOT=<tbb_root>` before OpenVINO CMake scripts are run.
> **NOTE**: The TBB team does not recommend using oneTBB as a static library, see [[Why onetbb does not like a static library?|https://github.com/oneapi-src/oneTBB/issues/646]]
* `TBBBind_2_5` is not available on Windows x64 during a static OpenVINO build (see description for `ENABLE_TBBBIND_2_5` CMake option [[here|CMakeOptionsForCustomCompilation]] to understand what this library is responsible for). So, capabilities enabled by `TBBBind_2_5` are not available. To enable them, build [[oneTBB from source code|https://github.com/oneapi-src/oneTBB]] and provide the path to built oneTBB artifacts via `TBBROOT` environment variable before OpenVINO CMake scripts are run.
* `ov::Op::type_info` static member is deprecated and not available in static build. Don't use `type_info` during implementation of your own custom operations, use `ov::Op::get_type_info_static()` instead.
## See also
* [OpenVINO README](../../README.md)
* [OpenVINO Developer Documentation](index.md)
* [How to Build OpenVINO](build.md)

2
docs/get_started/get_started_demos.md
View File

@@ -20,7 +20,7 @@ To use sample applications, install OpenVINO Runtime via one of the following di
* Archive files (recommended) - [Linux](@ref openvino_docs_install_guides_installing_openvino_from_archive_linux) | [Windows](@ref openvino_docs_install_guides_installing_openvino_from_archive_windows) | [macOS](@ref openvino_docs_install_guides_installing_openvino_from_archive_macos)
* [APT](@ref openvino_docs_install_guides_installing_openvino_apt) or [YUM](@ref openvino_docs_install_guides_installing_openvino_yum) for Linux
* Docker image - [Linux](@ref openvino_docs_install_guides_installing_openvino_docker_linux) | [Windows](@ref openvino_docs_install_guides_installing_openvino_docker_windows)
* [Build from source](https://github.com/openvinotoolkit/openvino/wiki/BuildingCode)
* [Build from source](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build.md)
Make sure that you also [install OpenCV](https://github.com/opencv/opencv/wiki/BuildOpenCV4OpenVINO), as it's required for running sample applications.

59
docs/install_guides/configurations-for-intel-gna.md
View File

@@ -2,34 +2,37 @@
@sphinxdirective
.. note:: On platforms where Intel® GNA is not enabled in the BIOS, the driver cannot be installed, so the GNA plugin uses the software emulation mode only.
.. note::
@endsphinxdirective
On platforms where Intel® GNA is not enabled in the BIOS, the driver cannot be installed, so the GNA plugin uses the software emulation mode only.
## Drivers and Dependencies
Drivers and Dependencies
########################
@sphinxdirective
Intel® GNA hardware requires a driver to be installed on the system.
.. _gna guide:
@endsphinxdirective
Linux
####################
Prerequisites
++++++++++++++++++++
Ensure that make, gcc, and Linux kernel headers are installed. Use the following command to install required software:
.. code-block:: sh
sudo apt-get install gcc make linux-headers-generic
## Linux
Configuration steps
++++++++++++++++++++
### Prerequisites
Ensure that make, gcc, and Linux kernel headers are installed.
### Configuration steps
@sphinxdirective
#. Download `Intel® GNA driver for Ubuntu Linux 18.04.3 LTS (with HWE Kernel version 5.4+) <https://storage.openvinotoolkit.org/drivers/gna/>`__
#. Run the sample_install.sh script provided in the installation package:
1. Download `Intel® GNA driver for Ubuntu Linux 18.04.3 LTS (with HWE Kernel version 5.4+) <https://storage.openvinotoolkit.org/drivers/gna/>`__
2. Run the sample_install.sh script provided in the installation package:
.. code-block:: sh
@@ -54,28 +57,28 @@ To unload the driver:
.. _gna guide windows:
@endsphinxdirective
## Windows
Windows
####################
Intel® GNA driver for Windows is available through Windows Update.
## What’s Next?
@sphinxdirective
What’s Next?
####################
Now you are ready to try out OpenVINO™. You can use the following tutorials to write your applications using Python and C++.
Developing in Python:
* `Start with tensorflow models with OpenVINO™ <https://docs.openvino.ai/nightly/notebooks/101-tensorflow-to-openvino-with-output.html>`_
* `Start with ONNX and PyTorch models with OpenVINO™ <https://docs.openvino.ai/nightly/notebooks/102-pytorch-onnx-to-openvino-with-output.html>`_
* `Start with PaddlePaddle models with OpenVINO™ <https://docs.openvino.ai/nightly/notebooks/103-paddle-onnx-to-openvino-classification-with-output.html>`_
* `Start with tensorflow models with OpenVINO™ <https://docs.openvino.ai/nightly/notebooks/101-tensorflow-to-openvino-with-output.html>`_
* `Start with ONNX and PyTorch models with OpenVINO™ <https://docs.openvino.ai/nightly/notebooks/102-pytorch-onnx-to-openvino-with-output.html>`_
* `Start with PaddlePaddle models with OpenVINO™ <https://docs.openvino.ai/nightly/notebooks/103-paddle-onnx-to-openvino-classification-with-output.html>`_
Developing in C++:
* :doc:`Image Classification Async C++ Sample <openvino_inference_engine_samples_classification_sample_async_README>`
* :doc:`Hello Classification C++ Sample <openvino_inference_engine_samples_hello_classification_README>`
* :doc:`Hello Reshape SSD C++ Sample <openvino_inference_engine_samples_hello_reshape_ssd_README>`
* :doc:`Image Classification Async C++ Sample <openvino_inference_engine_samples_classification_sample_async_README>`
* :doc:`Hello Classification C++ Sample <openvino_inference_engine_samples_hello_classification_README>`
* :doc:`Hello Reshape SSD C++ Sample <openvino_inference_engine_samples_hello_reshape_ssd_README>`
@endsphinxdirective

2
docs/install_guides/installing-openvino-conda.md
View File

@@ -3,7 +3,7 @@
* [Install OpenVINO Runtime from an Archive File](installing-openvino-from-archive-linux.md)
* [Install OpenVINO from PyPI](installing-openvino-pip.md)
* [Install OpenVINO with Docker](installing-openvino-docker-linux.md)
* [Build From Source](https://github.com/openvinotoolkit/openvino/wiki/BuildingCode)
* [Build From Source](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build.md)
The other installation methods are temporarily unavailable.

2
docs/install_guides/installing-openvino-linux.md
View File

@@ -8,7 +8,7 @@ Currently the following ways are provided to install OpenVINO™:
* [Install OpenVINO from PyPI](installing-openvino-pip.md)
* [Install OpenVINO Runtime via HomeBrew](installing-openvino-brew.md)
* [Install OpenVINO with Docker](installing-openvino-docker-linux.md)
* [Build From Source](https://github.com/openvinotoolkit/openvino/wiki/BuildingCode)
* [Build From Source](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build.md)
The other installation methods are temporarily unavailable.

2
docs/install_guides/installing-openvino-macos.md
View File

@@ -5,7 +5,7 @@ Currently only the following ways are provided to install OpenVINO™:
* [Install OpenVINO Runtime from an Archive File](installing-openvino-from-archive-macos.md)
* [Install OpenVINO Runtime via HomeBrew](installing-openvino-brew.md)
* [Install OpenVINO from PyPI](installing-openvino-pip.md)
* [Build From Source](https://github.com/openvinotoolkit/openvino/wiki/BuildingCode)
* [Build From Source](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build.md)
The other installation methods are temporarily unavailable.

6
docs/install_guides/installing-openvino-overview.md
View File

@@ -8,7 +8,7 @@
OpenVINO Runtime <openvino_docs_install_guides_install_runtime>
OpenVINO Development Tools <openvino_docs_install_guides_install_dev_tools>
Build from Source <https://github.com/openvinotoolkit/openvino/wiki/BuildingCode>
Build from Source <https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build.md>
Creating a Yocto Image <openvino_docs_install_guides_installing_openvino_yocto>
@endsphinxdirective
@@ -68,7 +68,7 @@ The following methods are available to install OpenVINO Runtime:
### Option 3. Build OpenVINO from source
Source files are also available in the OpenVINO Toolkit GitHub repository. If you want to build OpenVINO from source for your platform, follow the [OpenVINO Build Instructions](https://github.com/openvinotoolkit/openvino/wiki/BuildingCode).
Source files are also available in the OpenVINO Toolkit GitHub repository. If you want to build OpenVINO from source for your platform, follow the [OpenVINO Build Instructions](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build.md).
## Next Steps
Still unsure if you want to install OpenVINO toolkit? Check out the [OpenVINO tutorials](../tutorials.md) to run example applications directly in your web browser without installing it locally. Here are some exciting demos you can explore:
@@ -79,4 +79,4 @@ Still unsure if you want to install OpenVINO toolkit? Check out the [OpenVINO tu
Follow these links to install OpenVINO:
- [Install OpenVINO Development Tools](installing-model-dev-tools.md)
- [Install OpenVINO Runtime](installing-openvino-runtime.md)
- [Build from Source](https://github.com/openvinotoolkit/openvino/wiki/BuildingCode)
- [Build from Source](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build.md)

157
docs/install_guides/installing-openvino-pip.md
View File

@@ -1,107 +1,142 @@
# Install Intel® Distribution of OpenVINO™ Toolkit from PyPI Repository {#openvino_docs_install_guides_installing_openvino_pip}
@sphinxdirective
You can install both OpenVINO™ Runtime and OpenVINO Development Tools through the PyPI repository. This page provides the main steps for installing OpenVINO Runtime.
> **NOTE**: From the 2022.1 release, the OpenVINO™ Development Tools can only be installed via PyPI. See [Install OpenVINO Development Tools](installing-model-dev-tools.md) for detailed steps.
.. note:
## Installing OpenVINO Runtime
From the 2022.1 release, the OpenVINO™ Development Tools can only be installed via PyPI. See :doc:`Install OpenVINO Development Tools <openvino_docs_install_guides_install_dev_tools>` for detailed steps.
For system requirements and troubleshooting, see <https://pypi.org/project/openvino/>.
### Step 1. Set Up Python Virtual Environment
Installing OpenVINO Runtime
###########################
Use a virtual environment to avoid dependency conflicts.
For system requirements and troubleshooting, see https://pypi.org/project/openvino/
Step 1. Set Up Python Virtual Environment
+++++++++++++++++++++++++++++++++++++++++
Use a virtual environment to avoid dependency conflicts.
To create a virtual environment, use the following command:
@sphinxdirective
.. tab:: Linux and macOS
.. tab-set::
.. code-block:: sh
python3 -m venv openvino_env
.. tab:: Windows
.. tab-item:: Linux and macOS
:sync: linmac
.. code-block:: sh
python -m venv openvino_env
@endsphinxdirective
.. code-block:: sh
### Step 2. Activate Virtual Environment
python3 -m venv openvino_env
@sphinxdirective
.. tab-item:: Windows
:sync: win
.. tab:: Linux and macOS
.. code-block:: sh
.. code-block:: sh
source openvino_env/bin/activate
.. tab:: Windows
python -m venv openvino_env
Step 2. Activate Virtual Environment
++++++++++++++++++++++++++++++++++++
.. tab-set::
.. tab-item:: Linux and macOS
:sync: linmac
.. code-block:: sh
source openvino_env/bin/activate
.. tab-item:: Windows
:sync: win
.. code-block:: sh
openvino_env\Scripts\activate
.. code-block:: sh
openvino_env\Scripts\activate
.. important::
The above command must be re-run every time a new command terminal window is opened.
@endsphinxdirective
### Step 3. Set Up and Update PIP to the Highest Version
Step 3. Set Up and Update PIP to the Highest Version
++++++++++++++++++++++++++++++++++++++++++++++++++++
Use the following command:
```sh
python -m pip install --upgrade pip
```
### Step 4. Install the Package
.. code-block:: sh
python -m pip install --upgrade pip
Step 4. Install the Package
+++++++++++++++++++++++++++
Use the following command:
```
pip install openvino
```
### Step 5. Verify that the Package Is Installed
.. code-block:: sh
pip install openvino
Step 5. Verify that the Package Is Installed
++++++++++++++++++++++++++++++++++++++++++++
Run the command below:
```sh
python -c "from openvino.runtime import Core"
```
.. code-block:: sh
python -c "from openvino.runtime import Core"
If installation was successful, you will not see any error messages (no console output).
Congratulations! You finished installing OpenVINO Runtime. Now you can start exploring OpenVINO's functionality through Jupyter Notebooks and sample applications. See the <a href="#whats-next">What's Next</a> section to learn more!
Congratulations! You finished installing OpenVINO Runtime. Now you can start exploring OpenVINO's functionality through Jupyter Notebooks and sample applications. See the :ref:`What's Next <whats-next>` section to learn more!
## Installing OpenVINO Development Tools
OpenVINO Development Tools adds even more functionality to OpenVINO. It provides tools like Model Optimizer, Benchmark Tool, Post-Training Optimization Tool, and Open Model Zoo Downloader. If you install OpenVINO Development Tools, OpenVINO Runtime will also be installed as a dependency, so you don't need to install OpenVINO Runtime separately.
Installing OpenVINO Development Tools
#####################################
See the [Install OpenVINO Development Tools](installing-model-dev-tools.md) page for step-by-step installation instructions.
OpenVINO Development Tools adds even more functionality to OpenVINO. It provides tools like Model Optimizer, Benchmark Tool, Post-Training Optimization Tool, and Open Model Zoo Downloader. If you install OpenVINO Development Tools, OpenVINO Runtime will also be installed as a dependency, so you don't need to install OpenVINO Runtime separately.
See the :doc:`Install OpenVINO Development Tools <openvino_docs_install_guides_install_dev_tools>` page for step-by-step installation instructions.
.. _whats-next:
What's Next?
####################
<a name="whats-next"></a>
## What's Next?
Now that you've installed OpenVINO Runtime, you're ready to run your own machine learning applications! Learn more about how to integrate a model in OpenVINO applications by trying out the following tutorials.
<img src="https://user-images.githubusercontent.com/15709723/127752390-f6aa371f-31b5-4846-84b9-18dd4f662406.gif" width=400>
.. image:: https://user-images.githubusercontent.com/15709723/127752390-f6aa371f-31b5-4846-84b9-18dd4f662406.gif
:width: 400
Try the [Python Quick Start Example](https://docs.openvino.ai/nightly/notebooks/201-vision-monodepth-with-output.html) to estimate depth in a scene using an OpenVINO monodepth model in a Jupyter Notebook inside your web browser.
Try the `Python Quick Start Example <https://docs.openvino.ai/nightly/notebooks/201-vision-monodepth-with-output.html>`__ to estimate depth in a scene using an OpenVINO monodepth model in a Jupyter Notebook inside your web browser.
### Get started with Python
Visit the [Tutorials](../tutorials.md) page for more Jupyter Notebooks to get you started with OpenVINO, such as:
* [OpenVINO Python API Tutorial](https://docs.openvino.ai/nightly/notebooks/002-openvino-api-with-output.html)
* [Basic image classification program with Hello Image Classification](https://docs.openvino.ai/nightly/notebooks/001-hello-world-with-output.html)
* [Convert a PyTorch model and use it for image background removal](https://docs.openvino.ai/nightly/notebooks/205-vision-background-removal-with-output.html)
Get started with Python
+++++++++++++++++++++++
### Run OpenVINO on accelerated devices
OpenVINO Runtime has a plugin architecture that enables you to run inference on multiple devices without rewriting your code. Supported devices include integrated GPUs, discrete GPUs and GNAs. Visit the [Additional Configurations](configurations-header.md) page for instructions on how to configure your hardware devices to work with OpenVINO.
Visit the :doc:`Tutorials <tutorials>` page for more Jupyter Notebooks to get you started with OpenVINO, such as:
## Additional Resources
* `OpenVINO Python API Tutorial <https://docs.openvino.ai/nightly/notebooks/002-openvino-api-with-output.html>`__
* `Basic image classification program with Hello Image Classification <https://docs.openvino.ai/nightly/notebooks/001-hello-world-with-output.html>`__
* `Convert a PyTorch model and use it for image background removal <https://docs.openvino.ai/nightly/notebooks/205-vision-background-removal-with-output.html>`__
- Intel® Distribution of OpenVINO™ toolkit home page: <https://software.intel.com/en-us/openvino-toolkit>
- For IoT Libraries & Code Samples, see [Intel® IoT Developer Kit](https://github.com/intel-iot-devkit).
- [OpenVINO Installation Selector Tool](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html)
Run OpenVINO on accelerated devices
+++++++++++++++++++++++++++++++++++
OpenVINO Runtime has a plugin architecture that enables you to run inference on multiple devices without rewriting your code. Supported devices include integrated GPUs, discrete GPUs and GNAs. Visit the :doc:`Additional Configurations <openvino_docs_install_guides_configurations_header>` page for instructions on how to configure your hardware devices to work with OpenVINO.
Additional Resources
####################
- Intel® Distribution of OpenVINO™ toolkit home page: https://software.intel.com/en-us/openvino-toolkit
- For IoT Libraries & Code Samples, see `Intel® IoT Developer Kit <https://github.com/intel-iot-devkit>`__.
- `OpenVINO Installation Selector Tool <https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html>`__
@endsphinxdirective

4
docs/install_guides/installing-openvino-raspbian.md
View File

@@ -6,7 +6,7 @@
* These steps apply to Raspbian OS (the official OS for Raspberry Pi boards).
* These steps have been validated with Raspberry Pi 3.
* There is also an open-source version of OpenVINO™ that can be compiled for arch64 (see `build instructions <https://github.com/openvinotoolkit/openvino/wiki/BuildingForRaspbianStretchOS>`_).
* There is also an open-source version of OpenVINO™ that can be compiled for arch64 (see `build instructions <https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build_raspbian.md>`_).
@endsphinxdirective
@@ -152,7 +152,7 @@ If you want to use your model for inference, the model must be converted to the
* Download public and Intel's pre-trained models from the `Open Model Zoo <https://github.com/openvinotoolkit/open_model_zoo>`_ using :doc:`Model Downloader tool <omz_tools_downloader>`. For more information on pre-trained models, see :doc:`Pre-Trained Models Documentation <omz_models_group_intel>`.
* OpenCV is necessary to run demos from Open Model Zoo (OMZ). Some OpenVINO samples can also extend their capabilities when compiled with OpenCV as a dependency. To install OpenCV for OpenVINO, see the `instructions on Github <https://github.com/opencv/opencv/wiki/BuildOpenCV4OpenVINO)`_.
* OpenCV is necessary to run demos from Open Model Zoo (OMZ). Some OpenVINO samples can also extend their capabilities when compiled with OpenCV as a dependency. To install OpenCV for OpenVINO, see the `instructions on Github <https://github.com/opencv/opencv/wiki/BuildOpenCV4OpenVINO)>`_.
* Convert the models using the Model Optimizer. Model Optimizer is provided with OpenVINO Development Tools.

2
docs/install_guides/installing-openvino-windows.md
View File

@@ -5,7 +5,7 @@ Currently only the following ways are provided to install OpenVINO™:
* [Install OpenVINO Runtime from an Archive File](installing-openvino-from-archive-windows.md)
* [Install OpenVINO from PyPI](installing-openvino-pip.md)
* [Install OpenVINO with Docker](installing-openvino-docker-windows.md)
* [Build From Source](https://github.com/openvinotoolkit/openvino/wiki/BuildingCode)
* [Build From Source](https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/build.md)
The other installation methods are temporarily unavailable.

2
docs/nbdoc/consts.py
View File

@@ -8,7 +8,7 @@ repo_owner = "openvinotoolkit"
repo_name = "openvino_notebooks"
artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20230302220806/dist/rst_files/"
artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20230309220806/dist/rst_files/"
blacklisted_extensions = ['.xml', '.bin']

6
docs/optimization_guide/dldt_deployment_optimization_common.md
View File

@@ -57,4 +57,8 @@ To sum it up:
* For example, for the GPU device, the **input/output tensors** are mapped to the host (which is fast) only when the `get_tensor` is used, while for the `set_tensor` a copy into the internal GPU structures may happen.
* In contrast, when the input tensors are already in the **on-device memory** (e.g. as a result of the video-decoding), prefer the `set_tensor` as a zero-copy way to proceed. For more details, see the [GPU device Remote tensors API](../OV_Runtime_UG//supported_plugins/GPU_RemoteTensor_API.md).
Consider the [API examples](@ref in_out_tensors) for the `get_tensor` and `set_tensor`.
@sphinxdirective
Consider the :ref:`API examples <in_out_tensors>` for the `get_tensor` and `set_tensor`.
@endsphinxdirective

13
docs/optimization_guide/dldt_deployment_optimization_guide.md
View File

@@ -23,13 +23,24 @@ Therefore, optimization should start with defining the use case. For example, if
It is also important to understand how the full-stack application would use the inference component "end-to-end." For example, to know what stages need to be orchestrated to save workload devoted to fetching and preparing input data.
For more information on this topic, see the following articles:
* [feature support by device](@ref features_support_matrix)
@sphinxdirective
* :ref:`feature support by device <devicesupport-feature-support-matrix>`
@endsphinxdirective
* [Inputs Pre-processing with the OpenVINO](@ref inputs_pre_processing)
* [Async API](@ref async_api)
* [The 'get_tensor' Idiom](@ref tensor_idiom)
* For variably-sized inputs, consider [dynamic shapes](../OV_Runtime_UG/ov_dynamic_shapes.md)
See the [latency](./dldt_deployment_optimization_latency.md) and [throughput](./dldt_deployment_optimization_tput.md) optimization guides, for **use-case-specific optimizations**
## Writing Performance-Portable Inference Applications

10
docs/optimization_guide/dldt_deployment_optimization_latency.md
View File

@@ -22,7 +22,15 @@ Typically, human expertise is required to get more "throughput" out of the devic
> **NOTE**: [OpenVINO performance hints](../OV_Runtime_UG/performance_hints.md) is a recommended way for performance configuration, which is both device-agnostic and future-proof.
When multiple models are to be used simultaneously, consider running inference on separate devices for each of them. Finally, when multiple models are executed in parallel on a device, using additional `ov::hint::model_priority` may help to define relative priorities of the models. Refer to the documentation on the [matrix features support for OpenVINO devices](@ref features_support_matrix) to check if your device supports the feature.
@sphinxdirective
* feature support by device
When multiple models are to be used simultaneously, consider running inference on separate devices for each of them. Finally, when multiple models are executed in parallel on a device, using additional ``ov::hint::model_priority`` may help to define relative priorities of the models. Refer to the documentation on the :ref:`OpenVINO feature support for devices <devicesupport-feature-support-matrix>` to check if your device supports the feature.
@endsphinxdirective
**First-Inference Latency and Model Load/Compile Time**

1
docs/resources/prerelease_information.md
View File

@@ -24,6 +24,7 @@ a general changelog and the schedule for all versions for the current year.
* Enabled PaddlePaddle Framework 2.4
* Preview of TensorFlow Lite Front End – Load models directly via “read_model” into OpenVINO Runtime and export OpenVINO IR format using Model Optimizer or “convert_model”
* PyTorch Frontend is available as an experimental feature which will allow you to convert PyTorch models, using convert_model Python API directly from your code without the need to export to ONNX. Model coverage is continuously increasing. Feel free to start using the option and give us feedback.
* Model Optimizer now uses the TensorFlow Frontend as the default path for conversion to IR. Known limitations compared to the legacy approach are: TF1 Loop, Complex types, models requiring config files and old python extensions. The solution detects unsupported functionalities and provides fallback. To force using the legacy Frontend "--use_legacy_fronted" can be specified.
* Model Optimizer now supports out-of-the-box conversion of TF2 Object Detection models. At this point, same performance experience is guaranteed only on CPU devices. Feel free to start enjoying TF2 Object Detection models without config files!
* Introduced new option ov::auto::enable_startup_fallback / ENABLE_STARTUP_FALLBACK to control whether to use CPU to accelerate first inference latency for accelerator HW devices like GPU.

2
docs/snippets/CMakeLists.txt
View File

@@ -83,7 +83,7 @@ if(OpenCV_FOUND)
endif()
# ov_ncc_naming_style(FOR_TARGET "${TARGET_NAME}"
# SOURCE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
# SOURCE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}"
# ADDITIONAL_INCLUDE_DIRECTORIES
# $<TARGET_PROPERTY:openvino::runtime,INTERFACE_INCLUDE_DIRECTORIES>)

4
docs/tutorials.md
View File

@@ -225,6 +225,8 @@ Demos that demonstrate inference on a particular model.
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| `233-blip-visual-language-processing <notebooks/233-blip-visual-language-processing-with-output.html>`__ | Visual Question Answering and Image Captioning using BLIP and OpenVINO™ | |n233-img1| |
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| `234-encodec-audio-compression <notebooks/234-encodec-audio-compression-with-output.html>`__ | Audio compression with EnCodec and OpenVINO™ | |n234-img1| |
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
@@ -441,6 +443,8 @@ Made with `contributors-img <https://contrib.rocks>`__.
:target: https://user-images.githubusercontent.com/29454499/218967961-9858efd5-fff2-4eb0-bde9-60852f4b31cb.JPG
.. |n233-img1| image:: https://user-images.githubusercontent.com/29454499/221933762-4ff32ecb-5e5d-4484-80e1-e9396cb3c511.png
:target: https://user-images.githubusercontent.com/29454499/221933762-4ff32ecb-5e5d-4484-80e1-e9396cb3c511.png
.. |n234-img1| image:: https://github.com/facebookresearch/encodec/raw/main/thumbnail.png
:target: https://github.com/facebookresearch/encodec/raw/main/thumbnail.png
.. |n301-img1| image:: https://user-images.githubusercontent.com/15709723/127779607-8fa34947-1c35-4260-8d04-981c41a2a2cc.png
:target: https://user-images.githubusercontent.com/15709723/127779607-8fa34947-1c35-4260-8d04-981c41a2a2cc.png
.. |n401-img1| image:: https://user-images.githubusercontent.com/4547501/141471665-82b28c86-cf64-4bfe-98b3-c314658f2d96.gif

8
samples/cpp/CMakeLists.txt
View File

@@ -88,12 +88,6 @@ if(ARM AND NOT (CMAKE_CROSSCOMPILING OR CMAKE_CXX_COMPILER_ID STREQUAL "MSVC"))
add_compile_options(-march=armv7-a+fp)
endif()
set(CMAKE_POLICY_DEFAULT_CMP0063 NEW)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
set(CMAKE_CXX_VISIBILITY_PRESET hidden)
set(CMAKE_C_VISIBILITY_PRESET hidden)
set(CMAKE_VISIBILITY_INLINES_HIDDEN ON)
####################################
## to use C++11; can overwritten via cmake command line
if(NOT DEFINED CMAKE_CXX_STANDARD)
@@ -242,7 +236,7 @@ macro(ie_add_sample)
endif()
if(COMMAND ov_ncc_naming_style AND NOT c_sample)
ov_ncc_naming_style(FOR_TARGET "${IE_SAMPLE_NAME}"
SOURCE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
SOURCE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}")
endif()
endmacro()

12
samples/cpp/benchmark_app/inputs_filling.cpp
View File

@@ -229,7 +229,7 @@ ov::Tensor create_tensor_from_binary(const std::vector<std::string>& files,
" bytes, but the model expects ",
inputSize);
} else {
throw ov::Exception("Unsupported binary file type: " + extension);
OPENVINO_THROW("Unsupported binary file type: " + extension);
}
if (inputInfo.layout != "CN") {
@@ -352,7 +352,7 @@ ov::Tensor get_image_tensor(const std::vector<std::string>& files,
inputInfo.first,
filenames_used);
} else {
throw ov::Exception("Input type is not supported for " + inputInfo.first);
OPENVINO_THROW("Input type is not supported for " + inputInfo.first);
}
}
@@ -383,7 +383,7 @@ ov::Tensor get_im_info_tensor(const std::pair<size_t, size_t>& image_size,
} else if (type == ov::element::u64) {
return create_tensor_im_info<uint64_t>(image_size, batchSize, inputInfo.second, inputInfo.first);
} else {
throw ov::Exception("Input type is not supported for " + inputInfo.first);
OPENVINO_THROW("Input type is not supported for " + inputInfo.first);
}
}
@@ -471,7 +471,7 @@ ov::Tensor get_numpy_tensor(const std::vector<std::string>& files,
inputInfo.first,
filenames_used);
} else {
throw ov::Exception("Input type is not supported for " + inputInfo.first);
OPENVINO_THROW("Input type is not supported for " + inputInfo.first);
}
}
@@ -559,7 +559,7 @@ ov::Tensor get_binary_tensor(const std::vector<std::string>& files,
inputInfo.first,
filenames_used);
} else {
throw ov::Exception("Input type is not supported for " + inputInfo.first);
OPENVINO_THROW("Input type is not supported for " + inputInfo.first);
}
}
@@ -592,7 +592,7 @@ ov::Tensor get_random_tensor(const std::pair<std::string, benchmark_app::InputIn
} else if (type == ov::element::boolean) {
return create_tensor_random<uint8_t, uint32_t>(inputInfo.second, 0, 1);
} else {
throw ov::Exception("Input type is not supported for " + inputInfo.first);
OPENVINO_THROW("Input type is not supported for " + inputInfo.first);
}
}

2
samples/cpp/benchmark_app/main.cpp
View File

@@ -851,7 +851,7 @@ int main(int argc, char* argv[]) {
if (!modelStream.is_open()) {
throw std::runtime_error("Cannot open model file " + FLAGS_m);
}
compiledModel = core.import_model(modelStream, device_name, {});
compiledModel = core.import_model(modelStream, device_name, device_config);
modelStream.close();
auto duration_ms = get_duration_ms_till_now(startTime);

21
samples/cpp/benchmark_app/remote_tensors_filling.cpp
View File

@@ -61,7 +61,7 @@ void fill_buffer(void* inputBuffer, size_t elementsNum, const ov::element::Type&
} else if (type == ov::element::boolean) {
fill_buffer_random<uint8_t, uint32_t>(inputBuffer, elementsNum, 0, 1);
} else {
throw ov::Exception("Requested type is not supported");
OPENVINO_THROW("Requested type is not supported");
}
}
@@ -124,7 +124,7 @@ std::map<std::string, ov::TensorVector> get_remote_input_tensors(
}
return remoteTensors;
#else
throw ov::Exception("Device memory requested for GPU device, but OpenCL was not linked");
OPENVINO_THROW("Device memory requested for GPU device, but OpenCL was not linked");
#endif
}
@@ -136,19 +136,18 @@ ov::Shape get_static_shape(const ov::Output<const ov::Node>& compiled_output) {
if (compiled_pshape.is_static())
return compiled_pshape.to_shape();
else if (compiled_pshape.rank().is_dynamic())
OPENVINO_UNREACHABLE(
"Benchmark App - NOT IMPLEMENTED - Output of dynamic rank is not supported for remote tensor. ",
"Output: ",
compiled_output);
OPENVINO_THROW("Benchmark App - NOT IMPLEMENTED - Output of dynamic rank is not supported for remote tensor. ",
"Output: ",
compiled_output);
ov::Shape shape;
for (const auto& dimension : compiled_pshape) {
if (dimension.get_interval().has_upper_bound())
shape.push_back(static_cast<ov::Shape::value_type>(dimension.get_max_length()));
else
OPENVINO_UNREACHABLE("Benchmark App - NOT IMPLEMENTED - Fully dynamic output dimensions are not supported "
"for remote tensor. ",
"Output: ",
compiled_output);
OPENVINO_THROW("Benchmark App - NOT IMPLEMENTED - Fully dynamic output dimensions are not supported "
"for remote tensor. ",
"Output: ",
compiled_output);
}
return shape;
}
@@ -184,7 +183,7 @@ std::map<std::string, ov::Tensor> get_remote_output_tensors(const ov::CompiledMo
return outputTensors;
#else
throw ov::Exception("Device memory requested for GPU device, but OpenCL was not linked");
OPENVINO_THROW("Device memory requested for GPU device, but OpenCL was not linked");
#endif
}
} // namespace gpu

4
samples/cpp/common/utils/src/args_helper.cpp
View File

@@ -345,8 +345,8 @@ ov::element::Type getPrecision(std::string value,
ov::element::Type getPrecision2(const std::string& value) {
static const std::unordered_map<std::string, ov::element::Type> supported_precisions = {
{"FP32", ov::element::f32},
{"FP16", ov::element::f16},
{"F32", ov::element::f32},
{"F16", ov::element::f16},
{"BF16", ov::element::bf16},
{"U64", ov::element::u64},
{"I64", ov::element::i64},

4
src/bindings/python/src/pyopenvino/core/async_infer_queue.cpp
View File

@@ -104,7 +104,7 @@ public:
std::rethrow_exception(exception_ptr);
}
} catch (const std::exception& e) {
throw ov::Exception(e.what());
OPENVINO_THROW(e.what());
}
});
}
@@ -145,7 +145,7 @@ public:
std::rethrow_exception(exception_ptr);
}
} catch (const std::exception& e) {
throw ov::Exception(e.what());
OPENVINO_THROW(e.what());
}
});
}

55
src/bindings/python/src/pyopenvino/core/common.cpp
View File

@@ -7,6 +7,7 @@
#include <unordered_map>
#include "Python.h"
#include "openvino/core/except.hpp"
#include "openvino/util/common_util.hpp"
#define C_CONTIGUOUS py::detail::npy_api::constants::NPY_ARRAY_C_CONTIGUOUS_
@@ -105,8 +106,7 @@ py::array as_contiguous(py::array& array, ov::element::Type type) {
case ov::element::bf16:
return array.view("int16").cast<py::array_t<int16_t, py::array::c_style | py::array::forcecast>>();
default:
throw ov::Exception("Tensor cannot be created as contiguous!");
break;
OPENVINO_THROW("Tensor cannot be created as contiguous!");
}
}
@@ -142,8 +142,7 @@ ov::op::v0::Constant create_shared(py::array& array) {
return ov::op::v0::Constant(array_helpers::get_ov_type(array), array_helpers::get_shape(array), memory);
}
// If passed array is not C-style, throw an error.
throw ov::Exception(
"SHARED MEMORY MODE FOR THIS CONSTANT IS NOT APPLICABLE! Passed numpy array must be C contiguous.");
OPENVINO_THROW("SHARED MEMORY MODE FOR THIS CONSTANT IS NOT APPLICABLE! Passed numpy array must be C contiguous.");
}
template <>
@@ -177,8 +176,7 @@ ov::Tensor create_shared(py::array& array) {
array_helpers::get_strides(array));
}
// If passed array is not C-style, throw an error.
throw ov::Exception(
"SHARED MEMORY MODE FOR THIS TENSOR IS NOT APPLICABLE! Passed numpy array must be C contiguous.");
OPENVINO_THROW("SHARED MEMORY MODE FOR THIS TENSOR IS NOT APPLICABLE! Passed numpy array must be C contiguous.");
}
ov::Tensor tensor_from_pointer(py::array& array, const ov::Shape& shape, const ov::element::Type& type) {
@@ -187,8 +185,7 @@ ov::Tensor tensor_from_pointer(py::array& array, const ov::Shape& shape, const o
if (array_helpers::is_contiguous(array)) {
return ov::Tensor(element_type, shape, const_cast<void*>(array.data(0)), {});
}
throw ov::Exception(
"SHARED MEMORY MODE FOR THIS TENSOR IS NOT APPLICABLE! Passed numpy array must be C contiguous.");
OPENVINO_THROW("SHARED MEMORY MODE FOR THIS TENSOR IS NOT APPLICABLE! Passed numpy array must be C contiguous.");
}
ov::PartialShape partial_shape_from_list(const py::list& shape) {
@@ -237,12 +234,9 @@ const Containers::TensorNameMap cast_to_tensor_name_map(const py::dict& inputs)
} else {
throw py::type_error("incompatible function arguments!");
}
if (py::isinstance<ov::Tensor>(input.second)) {
auto tensor = Common::cast_to_tensor(input.second);
result_map[name] = tensor;
} else {
throw ov::Exception("Unable to cast tensor " + name + "!");
}
OPENVINO_ASSERT(py::isinstance<ov::Tensor>(input.second), "Unable to cast tensor ", name, "!");
auto tensor = Common::cast_to_tensor(input.second);
result_map[name] = tensor;
}
return result_map;
}
@@ -256,12 +250,9 @@ const Containers::TensorIndexMap cast_to_tensor_index_map(const py::dict& inputs
} else {
throw py::type_error("incompatible function arguments!");
}
if (py::isinstance<ov::Tensor>(input.second)) {
auto tensor = Common::cast_to_tensor(input.second);
result_map[idx] = tensor;
} else {
throw ov::Exception("Unable to cast tensor " + std::to_string(idx) + "!");
}
OPENVINO_ASSERT(py::isinstance<ov::Tensor>(input.second), "Unable to cast tensor ", idx, "!");
auto tensor = Common::cast_to_tensor(input.second);
result_map[idx] = tensor;
}
return result_map;
}
@@ -288,17 +279,16 @@ void set_request_tensors(ov::InferRequest& request, const py::dict& inputs) {
uint32_t get_optimal_number_of_requests(const ov::CompiledModel& actual) {
try {
auto supported_properties = actual.get_property(ov::supported_properties);
if (std::find(supported_properties.begin(), supported_properties.end(), ov::optimal_number_of_infer_requests) !=
supported_properties.end()) {
return actual.get_property(ov::optimal_number_of_infer_requests);
} else {
throw ov::Exception("Can't load network: " + std::string(ov::optimal_number_of_infer_requests.name()) +
" is not supported!"
" Please specify number of infer requests directly!");
}
OPENVINO_ASSERT(
std::find(supported_properties.begin(), supported_properties.end(), ov::optimal_number_of_infer_requests) !=
supported_properties.end(),
"Can't load network: ",
ov::optimal_number_of_infer_requests.name(),
" is not supported!",
" Please specify number of infer requests directly!");
return actual.get_property(ov::optimal_number_of_infer_requests);
} catch (const std::exception& ex) {
throw ov::Exception("Can't load network: " + std::string(ex.what()) +
" Please specify number of infer requests directly!");
OPENVINO_THROW("Can't load network: ", ex.what(), " Please specify number of infer requests directly!");
}
}
@@ -379,8 +369,9 @@ ov::pass::Serialize::Version convert_to_version(const std::string& version) {
if (version == "IR_V11") {
return Version::IR_V11;
}
throw ov::Exception("Invoked with wrong version argument: '" + version +
"'! The supported versions are: 'UNSPECIFIED'(default), 'IR_V10', 'IR_V11'.");
OPENVINO_THROW("Invoked with wrong version argument: '",
version,
"'! The supported versions are: 'UNSPECIFIED'(default), 'IR_V10', 'IR_V11'.");
}
}; // namespace Common

2
src/bindings/python/src/pyopenvino/core/core.cpp
View File

@@ -362,7 +362,7 @@ void regclass_Core(py::module m) {
std::stringstream str;
str << "Provided python object type " << model_path.get_type().str()
<< " isn't supported as 'model' argument.";
throw ov::Exception(str.str());
OPENVINO_THROW(str.str());
},
py::arg("model"),
py::arg("weights") = py::none(),

2
src/bindings/python/src/pyopenvino/core/infer_request.cpp
View File

@@ -344,7 +344,7 @@ void regclass_InferRequest(py::module m) {
std::rethrow_exception(exception_ptr);
}
} catch (const std::exception& e) {
throw ov::Exception("Caught exception: " + std::string(e.what()));
OPENVINO_THROW("Caught exception: ", e.what());
}
// Acquire GIL, execute Python function
py::gil_scoped_acquire acquire;

32
src/bindings/python/src/pyopenvino/core/infer_request.hpp
View File

@@ -4,12 +4,13 @@
#pragma once
#include <chrono>
#include <pybind11/pybind11.h>
#include <chrono>
#include <openvino/runtime/infer_request.hpp>
#include "openvino/core/except.hpp"
namespace py = pybind11;
typedef std::chrono::high_resolution_clock Time;
@@ -24,20 +25,17 @@ public:
// AsyncInferQueue uses this specifc constructor as setting callback
// for computing a latency will be done there.
InferRequestWrapper(ov::InferRequest&& request)
: InferRequestWrapper(std::move(request), {}, {}, false)
{
}
InferRequestWrapper(
ov::InferRequest&& request,
const std::vector<ov::Output<const ov::Node>>& inputs,
const std::vector<ov::Output<const ov::Node>>& outputs,
bool set_default_callback = true,
py::object userdata = py::none()
) : m_request{std::move(request)}, m_inputs{inputs}, m_outputs{outputs},
m_userdata{userdata} {
InferRequestWrapper(ov::InferRequest&& request) : InferRequestWrapper(std::move(request), {}, {}, false) {}
InferRequestWrapper(ov::InferRequest&& request,
const std::vector<ov::Output<const ov::Node>>& inputs,
const std::vector<ov::Output<const ov::Node>>& outputs,
bool set_default_callback = true,
py::object userdata = py::none())
: m_request{std::move(request)},
m_inputs{inputs},
m_outputs{outputs},
m_userdata{userdata} {
m_start_time = std::make_shared<Time::time_point>(Time::time_point{});
m_end_time = std::make_shared<Time::time_point>(Time::time_point{});
@@ -53,7 +51,7 @@ public:
std::rethrow_exception(exception_ptr);
}
} catch (const std::exception& e) {
throw ov::Exception("Caught exception: " + std::string(e.what()));
OPENVINO_THROW("Caught exception: ", e.what());
}
});
}
@@ -84,7 +82,7 @@ public:
// Data that is passed by user from Python->C++
py::object m_userdata;
// Times of inference's start and finish
std::shared_ptr<Time::time_point> m_start_time; // proposal: change to unique_ptr
std::shared_ptr<Time::time_point> m_start_time; // proposal: change to unique_ptr
std::shared_ptr<Time::time_point> m_end_time;
private:

2
src/bindings/python/src/pyopenvino/core/offline_transformations.cpp
View File

@@ -17,6 +17,7 @@
#include <transformations/common_optimizations/mark_precision_sensitive_shapeof_subgraphs.hpp>
#include <transformations/common_optimizations/moc_legacy_transformations.hpp>
#include <transformations/common_optimizations/moc_transformations.hpp>
#include <transformations/flush_fp32_subnormals_to_zero.hpp>
#include <transformations/op_conversions/convert_sequences_to_tensor_iterator.hpp>
#include <transformations/smart_reshape/smart_reshape.hpp>
@@ -38,6 +39,7 @@ void regmodule_offline_transformations(py::module m) {
if (smart_reshape)
manager.register_pass<ov::pass::SmartReshape>();
manager.register_pass<ov::pass::MOCTransformations>(cf);
manager.register_pass<ov::pass::FlushFP32SubnormalsToZero>();
manager.run_passes(model);
},
py::arg("model"),

4
src/bindings/python/src/pyopenvino/graph/node_factory.cpp
View File

@@ -75,9 +75,7 @@ private:
const auto& s_opsets = ov::get_available_opsets();
auto it = s_opsets.find(opset_ver);
if (it == s_opsets.end()) {
throw ov::Exception("Unsupported opset version requested.");
}
OPENVINO_ASSERT(it != s_opsets.end(), "Unsupported opset version requested.");
return it->second();
}

8
src/bindings/python/src/pyopenvino/graph/passes/pattern_ops.cpp
View File

@@ -36,14 +36,10 @@ static ov::NodeTypeInfo get_type(const std::string& type_name) {
std::string operation_type(it, type_name.end());
const auto& opsets = ov::get_available_opsets();
if (!opsets.count(opset_type)) {
throw ov::Exception("Unsupported opset type: " + opset_type);
}
OPENVINO_ASSERT(opsets.count(opset_type), "Unsupported opset type: ", opset_type);
const ov::OpSet& m_opset = opsets.at(opset_type)();
if (!m_opset.contains_type(operation_type)) {
throw ov::Exception("Unrecognized operation type: " + operation_type);
}
OPENVINO_ASSERT(m_opset.contains_type(operation_type), "Unrecognized operation type: ", operation_type);
return m_opset.create(operation_type)->get_type_info();
}

8
src/bindings/python/src/pyopenvino/utils/utils.cpp
View File

@@ -12,6 +12,7 @@
#include <vector>
#include "Python.h"
#include "openvino/core/except.hpp"
#include "openvino/frontend/decoder.hpp"
using Version = ov::pass::Serialize::Version;
@@ -166,7 +167,7 @@ std::string convert_path_to_string(const py::object& path) {
str << "Path: '" << path << "'"
<< " does not exist. Please provide valid model's path either as a string, bytes or pathlib.Path. "
"Examples:\n(1) '/home/user/models/model.onnx'\n(2) Path('/home/user/models/model/model.onnx')";
throw ov::Exception(str.str());
OPENVINO_THROW(str.str());
}
Version convert_to_version(const std::string& version) {
@@ -176,8 +177,9 @@ Version convert_to_version(const std::string& version) {
return Version::IR_V10;
if (version == "IR_V11")
return Version::IR_V11;
throw ov::Exception("Invoked with wrong version argument: '" + version +
"'! The supported versions are: 'UNSPECIFIED'(default), 'IR_V10', 'IR_V11'.");
OPENVINO_THROW("Invoked with wrong version argument: '",
version,
"'! The supported versions are: 'UNSPECIFIED'(default), 'IR_V10', 'IR_V11'.");
}
void deprecation_warning(const std::string& function_name, const std::string& version, const std::string& message) {

15
src/bindings/python/tests/mock/mock_py_frontend/src/mock_py_frontend.cpp
View File

@@ -366,11 +366,8 @@ InputModel::Ptr FrontEndMockPy::load_impl(const std::vector<ov::Any>& params) co
m_telemetry->send_stack_trace("mock_stack_trace");
}
if (!params.empty()) {
if (params[0].is<std::string>()) {
m_stat.m_load_paths.push_back(params[0].as<std::string>());
} else {
throw ov::Exception("Only path is supported.");
}
OPENVINO_ASSERT(params[0].is<std::string>(), "Only path is supported.");
m_stat.m_load_paths.push_back(params[0].as<std::string>());
}
return std::make_shared<InputModelMockPy>();
@@ -431,14 +428,14 @@ void FrontEndMockPy::clear_stat() {
} // namespace frontend
} // namespace ov
MOCK_C_API ov::frontend::FrontEndVersion GetAPIVersion();
MOCK_C_API void* GetFrontEndData();
MOCK_C_API ov::frontend::FrontEndVersion get_api_version();
MOCK_C_API void* get_front_end_data();
MOCK_C_API ov::frontend::FrontEndVersion GetAPIVersion() {
MOCK_C_API ov::frontend::FrontEndVersion get_api_version() {
return OV_FRONTEND_API_VERSION;
}
MOCK_C_API void* GetFrontEndData() {
MOCK_C_API void* get_front_end_data() {
ov::frontend::FrontEndPluginInfo* res = new ov::frontend::FrontEndPluginInfo();
res->m_name = "mock_py";
res->m_creator = []() {

17
src/bindings/python/tests/test_transformations/test_offline_api.py
View File

@@ -339,3 +339,20 @@ def test_convert_gru_to_tensor_iterator():
# assert that GRU sequence got transformed into TensorIterator
assert "GRUSequence" not in ops_types
assert "TensorIterator" in ops_types
def test_flush_fp32_subnormals_to_zero():
parameter = ov.opset10.parameter([1, 8], name="X")
subnorm_val = -2.0e-45
weights = ov.opset10.constant(np.array([0.0, 1.0, 2.0, 3.0, subnorm_val, subnorm_val, subnorm_val, subnorm_val]),
dtype=np.float32)
add_node = ov.opset10.add(parameter, weights)
result = ov.opset10.result(add_node)
model = Model([result], [parameter])
apply_moc_transformations(model, cf=False, smart_reshape=True) # apply_flush_fp32_subnormals_to_zero is called inside
assert np.all(weights.data[4:8] != subnorm_val)
assert np.all(weights.data[4:8] == 0.0)

32
src/common/itt/include/openvino/itt.hpp
View File

@@ -14,7 +14,7 @@
#include <string>
#include <utility>
/** @ingroup ie_dev_profiling
/** @ingroup ov_dev_profiling
* @brief openvino namespace
*/
namespace openvino
@@ -23,14 +23,14 @@ namespace openvino
{
/**
* @typedef domain_t
* @ingroup ie_dev_profiling
* @ingroup ov_dev_profiling
* @brief A domain type which enables tagging trace data for different modules or libraries in a program.
*/
typedef struct domain_ {} *domain_t;
/**
* @typedef handle_t
* @ingroup ie_dev_profiling
* @ingroup ov_dev_profiling
* @brief Annotation handle for section of code which would be named at runtime.
*/
typedef struct handle_ {} *handle_t;
@@ -52,7 +52,7 @@ namespace openvino
/**
* @fn void threadName(const char* name)
* @ingroup ie_dev_profiling
* @ingroup ov_dev_profiling
* @brief Set thread name using a char string.
* @param name [in] The thread name
*/
@@ -78,7 +78,7 @@ namespace openvino
/**
* @fn handle_t handle(char const *name)
* @ingroup ie_dev_profiling
* @ingroup ov_dev_profiling
* @brief Create annotation handle with a given name.
* @details If template function is instantiated with a tag, the handle is created as a singleton.
* @param name [in] The annotation name
@@ -104,7 +104,7 @@ namespace openvino
/**
* @class ScopedTask
* @ingroup ie_dev_profiling
* @ingroup ov_dev_profiling
* @brief Used to annotate section of code which would be named at runtime
* @tparam The @p domain parameter is domain type which shoud be defined with OV_ITT_DOMAIN() macro.
*/
@@ -130,7 +130,7 @@ namespace openvino
/**
* @class TaskChain
* @ingroup ie_dev_profiling
* @ingroup ov_dev_profiling
* @brief Used to annotate a sequence of sections of code which would be named at runtime
* @tparam The @p domain parameter is domain type which shoud be defined with OV_ITT_DOMAIN() macro.
*/
@@ -211,7 +211,7 @@ namespace openvino
/**
* @def OV_ITT_DOMAIN(domainName)
* @ingroup ie_dev_profiling
* @ingroup ov_dev_profiling
* @brief Declare domain with a given name.
* @param domainName [in] Known at compile time name of module or library (the domain name).
* @param domainDisplayName [in] Domain name used as the ITT counter name and displayed in Intel VTune. Parameter is optional.
@@ -244,7 +244,7 @@ inline openvino::itt::domain_t domainName() noexcept
/**
* @def OV_ITT_SCOPE(domain, handleOrTaskName)
* @ingroup ie_dev_profiling
* @ingroup ov_dev_profiling
* @brief Annotate section of code till scope exit to be profiled using known @p handle or @p taskName as section id.
* @details In case if handle or taskName absent, the current function name is used.
* @param group [in] ITT counter group name used for enabling/disabling at compile time.
@@ -275,7 +275,7 @@ inline openvino::itt::domain_t domainName() noexcept
/**
* @def OV_ITT_SCOPED_TASK(domain, handleOrTaskName)
* @ingroup ie_dev_profiling
* @ingroup ov_dev_profiling
* @brief Annotate section of code till scope exit to be profiled using known @p handle or @p taskName as section id.
* @details In case if handle or taskName absent, the current function name is used.
* @param domainName [in] Known at compile time name of module or library (the domain name).
@@ -285,7 +285,7 @@ inline openvino::itt::domain_t domainName() noexcept
/**
* @def OV_ITT_TASK_CHAIN(chainId, domain, prefix, taskName)
* @ingroup ie_dev_profiling
* @ingroup ov_dev_profiling
* @brief Begins the sequrence of an annotated sections of code using @p prefix and @p taskName as section id.
* @details In case if prefix absent, the current function name is used,
* if taskName absent, the first chain index is used, i.e 1.
@@ -329,7 +329,7 @@ inline openvino::itt::domain_t domainName() noexcept
/**
* @def OV_ITT_SCOPE_NEXT(group, chainId, taskName)
* @ingroup ie_dev_profiling
* @ingroup ov_dev_profiling
* @brief Inserts new annotated section of code to tasks chain using @p taskName as section id.
* @details If taskName is missing, the current chain index is used.
* @param group [in] ITT counter group name used for enabling/disabling at compile time.
@@ -358,7 +358,7 @@ inline openvino::itt::domain_t domainName() noexcept
/**
* @def OV_ITT_SCOPE_SKIP(group, chainId)
* @ingroup ie_dev_profiling
* @ingroup ov_dev_profiling
* @brief Skips the remaining task scope.
* @param group [in] ITT counter group name used for enabling/disabling at compile time.
* @param chainId [in] The tasks chain identifier.
@@ -379,7 +379,7 @@ inline openvino::itt::domain_t domainName() noexcept
/**
* @def OV_ITT_TASK_CHAIN(chainId, domain, prefix, taskName)
* @ingroup ie_dev_profiling
* @ingroup ov_dev_profiling
* @brief Begins the sequrence of an annotated sections of code using @p prefix and @p taskName as section id.
* @details In case if prefix absent, the current function name is used,
* if taskName absent, the first chain index is used, i.e 1.
@@ -392,7 +392,7 @@ inline openvino::itt::domain_t domainName() noexcept
/**
* @def OV_ITT_TASK_NEXT(chainId, taskName)
* @ingroup ie_dev_profiling
* @ingroup ov_dev_profiling
* @brief Inserts new annotated section of code to tasks chain using @p taskName as section id.
* @details If taskName is missing, the current chain index is used.
* @param chainId [in] The tasks chain identifier.
@@ -402,7 +402,7 @@ inline openvino::itt::domain_t domainName() noexcept
/**
* @def OV_ITT_TASK_SKIP(chainId)
* @ingroup ie_dev_profiling
* @ingroup ov_dev_profiling
* @brief Skips the remaining task scope.
* @param chainId [in] The tasks chain identifier.
*/

30
src/common/transformations/include/transformations/flush_fp32_subnormals_to_zero.hpp Normal file
View File

@@ -0,0 +1,30 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <openvino/pass/graph_rewrite.hpp>
#include <string>
#include "openvino/core/model.hpp"
#include "openvino/pass/serialize.hpp"
#include "transformations_visibility.hpp"
namespace ov {
namespace pass {
class TRANSFORMATIONS_API FlushFP32SubnormalsToZero;
} // namespace pass
} // namespace ov
/* @ingroup ie_transformation_common_api
* @brief FlushFP32SubnormalsToZero flushes f32 subnormals to zero.
* This is read/write expensive transformation, therefore should be run at offline phase.
*/
class ov::pass::FlushFP32SubnormalsToZero : public MatcherPass {
public:
OPENVINO_RTTI("FlushFP32SubnormalsToZero", "0");
FlushFP32SubnormalsToZero();
};

8
src/common/transformations/include/transformations/utils/utils.hpp
View File

@@ -188,12 +188,8 @@ template <class T>
Output<Node> eltwise_fold(const Output<Node>& input0, const Output<Node>& input1) {
auto eltwise = std::make_shared<T>(input0, input1);
OutputVector output(eltwise->get_output_size());
if (!eltwise->constant_fold(output, {input0, input1})) {
throw ov::Exception("Can not constant fold eltwise node");
}
if (output.size() != 1) {
throw ov::Exception("Eltwise constant fold has unexpected number of outputs: " + std::to_string(output.size()));
}
OPENVINO_ASSERT(eltwise->constant_fold(output, {input0, input1}), "Can not constant fold eltwise node");
OPENVINO_ASSERT(output.size() == 1, "Eltwise constant fold has unexpected number of outputs: ", output.size());
return output[0];
}

2
src/common/transformations/src/transformations/common_optimizations/simplify_shape_of_sub_graph.cpp
View File

@@ -108,7 +108,7 @@ ov::pass::GroupedGatherElimination::GroupedGatherElimination() {
joint_indices->output(0),
opset1::Constant::create(element::i64, {}, {0})->output(0));
} else {
OPENVINO_UNREACHABLE("Unexpected Gather version");
OPENVINO_THROW("Unexpected Gather version");
}
new_ops.push_back(joint_indices);
new_ops.push_back(new_gather);

51
src/common/transformations/src/transformations/flush_fp32_subnormals_to_zero.cpp Normal file
View File

@@ -0,0 +1,51 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "transformations/flush_fp32_subnormals_to_zero.hpp"
#include <cmath>
#include <memory>
#include <vector>
#include "itt.hpp"
#include "openvino/opsets/opset10.hpp"
#include "openvino/pass/pattern/op/wrap_type.hpp"
#include "transformations/utils/utils.hpp"
using namespace std;
using namespace ov;
using namespace pass;
ov::pass::FlushFP32SubnormalsToZero::FlushFP32SubnormalsToZero() {
MATCHER_SCOPE(FlushFP32SubnormalsToZero);
auto node_pattern = pattern::wrap_type<opset10::Constant>();
matcher_pass_callback callback = [=](pattern::Matcher& m) {
auto node = dynamic_pointer_cast<ov::opset10::Constant>(m.get_match_root());
if (!node)
return false;
if (node->get_output_element_type(0) != element::f32)
return false;
auto* data = const_cast<float*>(node->get_data_ptr<float>());
const auto size = ov::shape_size(node->get_shape());
bool has_subnormals = false;
for (size_t i = 0; i < size; ++i) {
if (fpclassify(std::abs(data[i])) == FP_SUBNORMAL) {
data[i] = 0.0f;
has_subnormals = true;
}
}
if (has_subnormals)
return true;
return false;
};
auto m = make_shared<pattern::Matcher>(node_pattern, matcher_name);
register_matcher(m, callback);
}

2
src/common/transformations/src/transformations/op_conversions/convert_ti_to_sequences.cpp
View File

@@ -165,7 +165,7 @@ bool convertTensorIteratorToSequence(const std::shared_ptr<ov::opset5::TensorIte
gru_cell->get_clip(),
gru_cell->get_linear_before_reset());
} else {
throw ov::Exception("Unsupported sequence type");
OPENVINO_THROW("Unsupported sequence type");
}
ngraph::Output<ngraph::Node> out = sequence->output(0);

8
src/common/transformations/src/transformations/smart_reshape/lstm_states_broadcast.cpp
View File

@@ -22,10 +22,10 @@ ov::Input<ov::Node> get_outer_input_of_ti_by_parameter(const shared_ptr<Paramete
for (const auto& input_descriptor : ti->get_input_descriptions())
if (static_cast<int64_t>(input_descriptor->m_body_parameter_index) == parameter_index)
return ti->input(input_descriptor->m_input_index);
OPENVINO_UNREACHABLE("LSTMStatesBroadcast failed to get outer input of TI by its inner Parameter. TI ",
ti,
" Parameter ",
parameter);
OPENVINO_THROW("LSTMStatesBroadcast failed to get outer input of TI by its inner Parameter. TI ",
ti,
" Parameter ",
parameter);
}
shared_ptr<ov::Node> deduce_outer_source_of_batch_for_inner_lstm_cell(const shared_ptr<TensorIterator>& ti,

4
src/common/transformations/src/transformations/utils/utils.cpp
View File

@@ -47,7 +47,7 @@ bool get_single_value(const std::shared_ptr<op::v0::Constant>& const_node, float
case element::Type_t::u64:
return util::normalize_single_value(const_node->get_vector<uint64_t>(), value);
default:
throw ov::Exception("Unsupported precision for const operation: " + const_node->get_friendly_name());
OPENVINO_THROW("Unsupported precision for const operation: ", const_node->get_friendly_name());
}
}
@@ -125,7 +125,7 @@ std::shared_ptr<ngraph::Node> activation(const std::string& activation_name,
} else if (activation_name == "tanh") {
return std::make_shared<opset4::Tanh>(apply_to);
} else {
throw ov::Exception("Unsupported activation function");
OPENVINO_THROW("Unsupported activation function");
}
}

376
src/common/transformations/tests/common_optimizations/flush_fp32_subnormals_to_zero_test.cpp Normal file
View File

@@ -0,0 +1,376 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <gtest/gtest.h>
#include <openvino/core/model.hpp>
#include <openvino/opsets/opset10.hpp>
#include <openvino/pass/manager.hpp>
#include <transformations/flush_fp32_subnormals_to_zero.hpp>
#include <transformations/init_node_info.hpp>
#include "common_test_utils/ngraph_test_utils.hpp"
using namespace testing;
using namespace ov;
using namespace ov::opset10;
using namespace std;
namespace {
union FloatIntUnion {
uint32_t u;
float f;
};
FloatIntUnion maximum_subnorm_val = {0x007fffff}; // = 2^−126 * (1 - 2^−23) ~= 1.1754942107e-38f
FloatIntUnion minimum_subnorm_val = {0x00000001}; // = 2^−149 ~= 1.4012984643e-45f
FloatIntUnion minimum_norm_val = {0x00800000}; // = 2^−126 ~= 1.1754943508-38f
} // namespace
TEST_F(TransformationTestsF, test_flush_fp32_subnorm_to_zero_max_subnorm) {
float subnormal_val = maximum_subnorm_val.f;
{
auto input = std::make_shared<Parameter>(element::f32, Shape{1, 3, 12, 12});
auto const_weights = Constant::create(element::f32,
Shape{1, 3, 4, 1},
{0.0f,
1.0f,
2.0f,
3.0f,
4.0f,
5.0f,
subnormal_val,
subnormal_val,
subnormal_val,
subnormal_val,
subnormal_val,
subnormal_val});
auto conv = std::make_shared<Convolution>(input,
const_weights,
Strides{1, 1},
CoordinateDiff{0, 0},
CoordinateDiff{0, 0},
Strides{1, 1});
function = std::make_shared<Model>(NodeVector{conv}, ParameterVector{input});
manager.register_pass<pass::FlushFP32SubnormalsToZero>();
}
{
auto input = std::make_shared<Parameter>(element::f32, Shape{1, 3, 12, 12});
auto const_weights = Constant::create(element::f32,
Shape{1, 3, 4, 1},
{0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f});
auto conv = std::make_shared<Convolution>(input,
const_weights,
Strides{1, 1},
CoordinateDiff{0, 0},
CoordinateDiff{0, 0},
Strides{1, 1});
function_ref = std::make_shared<Model>(NodeVector{conv}, ParameterVector{input});
}
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
}
TEST_F(TransformationTestsF, test_flush_fp32_subnorm_to_zero_min_subnorm) {
float subnormal_val = minimum_subnorm_val.f;
{
auto input = std::make_shared<Parameter>(element::f32, Shape{1, 3, 12, 12});
auto const_weights = Constant::create(element::f32,
Shape{1, 3, 4, 1},
{0.0f,
1.0f,
2.0f,
3.0f,
4.0f,
5.0f,
subnormal_val,
subnormal_val,
subnormal_val,
subnormal_val,
subnormal_val,
subnormal_val});
auto conv = std::make_shared<Convolution>(input,
const_weights,
Strides{1, 1},
CoordinateDiff{0, 0},
CoordinateDiff{0, 0},
Strides{1, 1});
function = std::make_shared<Model>(NodeVector{conv}, ParameterVector{input});
manager.register_pass<pass::FlushFP32SubnormalsToZero>();
}
{
auto input = std::make_shared<Parameter>(element::f32, Shape{1, 3, 12, 12});
auto const_weights = Constant::create(element::f32,
Shape{1, 3, 4, 1},
{0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f});
auto conv = std::make_shared<Convolution>(input,
const_weights,
Strides{1, 1},
CoordinateDiff{0, 0},
CoordinateDiff{0, 0},
Strides{1, 1});
function_ref = std::make_shared<Model>(NodeVector{conv}, ParameterVector{input});
}
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
}
TEST_F(TransformationTestsF, test_flush_fp32_subnorm_to_zero_arbitrary_subnorm) {
float subnormal_val = 2.0e-44f;
{
auto input = std::make_shared<Parameter>(element::f32, Shape{1, 3, 12, 12});
auto const_weights = Constant::create(element::f32,
Shape{1, 3, 4, 1},
{0.0f,
1.0f,
2.0f,
3.0f,
4.0f,
5.0f,
subnormal_val,
subnormal_val,
subnormal_val,
subnormal_val,
subnormal_val,
subnormal_val});
auto conv = std::make_shared<Convolution>(input,
const_weights,
Strides{1, 1},
CoordinateDiff{0, 0},
CoordinateDiff{0, 0},
Strides{1, 1});
function = std::make_shared<Model>(NodeVector{conv}, ParameterVector{input});
manager.register_pass<pass::FlushFP32SubnormalsToZero>();
}
{
auto input = std::make_shared<Parameter>(element::f32, Shape{1, 3, 12, 12});
auto const_weights = Constant::create(element::f32,
Shape{1, 3, 4, 1},
{0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f});
auto conv = std::make_shared<Convolution>(input,
const_weights,
Strides{1, 1},
CoordinateDiff{0, 0},
CoordinateDiff{0, 0},
Strides{1, 1});
function_ref = std::make_shared<Model>(NodeVector{conv}, ParameterVector{input});
}
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
}
TEST_F(TransformationTestsF, test_flush_fp32_subnorm_to_zero_max_neg_subnorm) {
float subnormal_val = -maximum_subnorm_val.f;
{
auto input = std::make_shared<Parameter>(element::f32, Shape{1, 3, 12, 12});
auto const_weights = Constant::create(element::f32,
Shape{1, 3, 4, 1},
{0.0f,
1.0f,
2.0f,
3.0f,
4.0f,
5.0f,
subnormal_val,
subnormal_val,
subnormal_val,
subnormal_val,
subnormal_val,
subnormal_val});
auto conv = std::make_shared<Convolution>(input,
const_weights,
Strides{1, 1},
CoordinateDiff{0, 0},
CoordinateDiff{0, 0},
Strides{1, 1});
function = std::make_shared<Model>(NodeVector{conv}, ParameterVector{input});
manager.register_pass<pass::FlushFP32SubnormalsToZero>();
}
{
auto input = std::make_shared<Parameter>(element::f32, Shape{1, 3, 12, 12});
auto const_weights = Constant::create(element::f32,
Shape{1, 3, 4, 1},
{0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f});
auto conv = std::make_shared<Convolution>(input,
const_weights,
Strides{1, 1},
CoordinateDiff{0, 0},
CoordinateDiff{0, 0},
Strides{1, 1});
function_ref = std::make_shared<Model>(NodeVector{conv}, ParameterVector{input});
}
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
}
TEST_F(TransformationTestsF, test_flush_fp32_subnorm_to_zero_min_neg_subnorm) {
float subnormal_val = -minimum_subnorm_val.f;
{
auto input = std::make_shared<Parameter>(element::f32, Shape{1, 3, 12, 12});
auto const_weights = Constant::create(element::f32,
Shape{1, 3, 4, 1},
{0.0f,
1.0f,
2.0f,
3.0f,
4.0f,
5.0f,
subnormal_val,
subnormal_val,
subnormal_val,
subnormal_val,
subnormal_val,
subnormal_val});
auto conv = std::make_shared<Convolution>(input,
const_weights,
Strides{1, 1},
CoordinateDiff{0, 0},
CoordinateDiff{0, 0},
Strides{1, 1});
function = std::make_shared<Model>(NodeVector{conv}, ParameterVector{input});
manager.register_pass<pass::FlushFP32SubnormalsToZero>();
}
{
auto input = std::make_shared<Parameter>(element::f32, Shape{1, 3, 12, 12});
auto const_weights = Constant::create(element::f32,
Shape{1, 3, 4, 1},
{0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f});
auto conv = std::make_shared<Convolution>(input,
const_weights,
Strides{1, 1},
CoordinateDiff{0, 0},
CoordinateDiff{0, 0},
Strides{1, 1});
function_ref = std::make_shared<Model>(NodeVector{conv}, ParameterVector{input});
}
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
}
TEST_F(TransformationTestsF, test_flush_fp32_subnorm_to_zero_arbitrary_neg_subnorm) {
float subnormal_val = -2.0e-45f;
{
auto input = std::make_shared<Parameter>(element::f32, Shape{1, 3, 12, 12});
auto const_weights = Constant::create(element::f32,
Shape{1, 3, 4, 1},
{0.0f,
1.0f,
2.0f,
3.0f,
4.0f,
5.0f,
subnormal_val,
subnormal_val,
subnormal_val,
subnormal_val,
subnormal_val,
subnormal_val});
auto conv = std::make_shared<Convolution>(input,
const_weights,
Strides{1, 1},
CoordinateDiff{0, 0},
CoordinateDiff{0, 0},
Strides{1, 1});
function = std::make_shared<Model>(NodeVector{conv}, ParameterVector{input});
manager.register_pass<pass::FlushFP32SubnormalsToZero>();
}
{
auto input = std::make_shared<Parameter>(element::f32, Shape{1, 3, 12, 12});
auto const_weights = Constant::create(element::f32,
Shape{1, 3, 4, 1},
{0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f});
auto conv = std::make_shared<Convolution>(input,
const_weights,
Strides{1, 1},
CoordinateDiff{0, 0},
CoordinateDiff{0, 0},
Strides{1, 1});
function_ref = std::make_shared<Model>(NodeVector{conv}, ParameterVector{input});
}
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
}
TEST_F(TransformationTestsF, test_flush_fp32_subnorm_to_zero_arbitrary_norm) {
// minimum normalized val should not be flushed to zero
float normal_val = minimum_norm_val.f;
{
auto input = std::make_shared<Parameter>(element::f32, Shape{1, 3, 12, 12});
auto const_weights = Constant::create(element::f32,
Shape{1, 3, 4, 1},
{0.0f,
1.0f,
2.0f,
3.0f,
4.0f,
5.0f,
normal_val,
normal_val,
normal_val,
normal_val,
normal_val,
normal_val});
auto conv = std::make_shared<Convolution>(input,
const_weights,
Strides{1, 1},
CoordinateDiff{0, 0},
CoordinateDiff{0, 0},
Strides{1, 1});
function = std::make_shared<Model>(NodeVector{conv}, ParameterVector{input});
manager.register_pass<pass::FlushFP32SubnormalsToZero>();
}
{
auto input = std::make_shared<Parameter>(element::f32, Shape{1, 3, 12, 12});
auto const_weights = Constant::create(element::f32,
Shape{1, 3, 4, 1},
{0.0f,
1.0f,
2.0f,
3.0f,
4.0f,
5.0f,
normal_val,
normal_val,
normal_val,
normal_val,
normal_val,
normal_val});
auto conv = std::make_shared<Convolution>(input,
const_weights,
Strides{1, 1},
CoordinateDiff{0, 0},
CoordinateDiff{0, 0},
Strides{1, 1});
function_ref = std::make_shared<Model>(NodeVector{conv}, ParameterVector{input});
}
comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES);
}

2
src/common/transformations/tests/op_conversions/eye_decomposition_test.cpp
View File

@@ -100,7 +100,7 @@ public:
} else if (new_args.size() == 4) {
return std::make_shared<FakeEye>(new_args[0], new_args[1], new_args[2], new_args[3], ov::element::f32);
} else {
throw ov::Exception("FakeEye has incorrect input number: " + std::to_string(new_args.size()));
OPENVINO_THROW("FakeEye has incorrect input number: ", new_args.size());
}
}
};

2
src/common/util/CMakeLists.txt
View File

@@ -43,6 +43,6 @@ ov_install_static_lib(${TARGET_NAME} ${OV_CPACK_COMP_CORE})
add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME})
ov_ncc_naming_style(FOR_TARGET ${TARGET_NAME}
SOURCE_DIRECTORY ${UTIL_INCLUDE_DIR})
SOURCE_DIRECTORIES ${UTIL_INCLUDE_DIR})
openvino_developer_export_targets(COMPONENT core TARGETS ${TARGET_NAME})

2
src/common/util/include/openvino/util/common_util.hpp
View File

@@ -70,7 +70,7 @@ inline std::string rtrim(const std::string& s) {
/**
* @brief Trims std::string from both ends (in place)
* @ingroup ie_dev_api_error_debug
* @ingroup ov_dev_api_error_debug
* @param s A reference to a std::tring to trim
* @return A reference to a trimmed std::string
*/

2
src/core/CMakeLists.txt
View File

@@ -87,7 +87,7 @@ target_link_libraries(ngraph_obj PRIVATE ngraph::builder ngraph::reference openv
ie_mark_target_as_cc(ngraph_obj)
ov_ncc_naming_style(FOR_TARGET ngraph_obj
SOURCE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/include")
SOURCE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/include")
add_clang_format_target(ngraph_clang FOR_TARGETS ngraph_obj)

46
src/core/include/openvino/core/any.hpp
View File

@@ -185,9 +185,9 @@ template <typename T>
auto from_string(const std::string& val) ->
typename std::enable_if<!Readable<T>::value && !Istreamable<T>::value && !std::is_same<T, std::string>::value,
T>::type {
OPENVINO_UNREACHABLE("Could read type without std::istream& operator>>(std::istream&, T)",
" defined or ov::util::Read<T> class specialization, T: ",
typeid(T).name());
OPENVINO_THROW("Could read type without std::istream& operator>>(std::istream&, T)",
" defined or ov::util::Read<T> class specialization, T: ",
typeid(T).name());
}
template <typename T, typename A>
@@ -294,9 +294,9 @@ template <typename T>
auto to_string(const T&) ->
typename std::enable_if<!Writable<T>::value && !Ostreamable<T>::value && !std::is_same<T, std::string>::value,
std::string>::type {
OPENVINO_UNREACHABLE("Could convert to string from type without std::ostream& operator>>(std::ostream&, const T&)",
" defined or ov::util::Write<T> class specialization, T: ",
typeid(T).name());
OPENVINO_THROW("Could convert to string from type without std::ostream& operator>>(std::ostream&, const T&)",
" defined or ov::util::Write<T> class specialization, T: ",
typeid(T).name());
}
template <typename T, typename A>
@@ -390,7 +390,7 @@ class OPENVINO_API Any {
template <class U>
[[noreturn]] static typename std::enable_if<!EqualityComparable<U>::value, bool>::type equal_impl(const U&,
const U&) {
OPENVINO_UNREACHABLE("Could not compare types without equality operator");
OPENVINO_THROW("Could not compare types without equality operator");
}
template <typename T>
@@ -526,7 +526,7 @@ class OPENVINO_API Any {
}
void read(std::istream&) override {
OPENVINO_UNREACHABLE("Pointer to runtime attribute is not readable from std::istream");
OPENVINO_THROW("Pointer to runtime attribute is not readable from std::istream");
}
T runtime_attribute;
@@ -613,9 +613,9 @@ class OPENVINO_API Any {
static typename std::enable_if<!util::Readable<U>::value && !util::Istreamable<U>::value>::type read_impl(
std::istream&,
U&) {
OPENVINO_UNREACHABLE("Could read type without std::istream& operator>>(std::istream&, T)",
" defined or ov::util::Read<T> class specialization, T: ",
typeid(T).name());
OPENVINO_THROW("Could read type without std::istream& operator>>(std::istream&, T)",
" defined or ov::util::Read<T> class specialization, T: ",
typeid(T).name());
}
void read(std::istream& is) override {
@@ -755,20 +755,20 @@ public:
} else {
auto runtime_attribute = _impl->as_runtime_attribute();
if (runtime_attribute == nullptr) {
OPENVINO_UNREACHABLE("Any does not contains pointer to runtime_attribute. It contains ",
_impl->type_info().name());
OPENVINO_THROW("Any does not contains pointer to runtime_attribute. It contains ",
_impl->type_info().name());
}
auto vptr = std::dynamic_pointer_cast<typename T::element_type>(runtime_attribute);
if (vptr == nullptr && T::element_type::get_type_info_static() != runtime_attribute->get_type_info() &&
T::element_type::get_type_info_static() != RuntimeAttribute::get_type_info_static()) {
OPENVINO_UNREACHABLE("Could not cast Any runtime_attribute to ",
typeid(T).name(),
" from ",
_impl->type_info().name(),
"; from ",
static_cast<std::string>(runtime_attribute->get_type_info()),
" to ",
static_cast<std::string>(T::element_type::get_type_info_static()));
OPENVINO_THROW("Could not cast Any runtime_attribute to ",
typeid(T).name(),
" from ",
_impl->type_info().name(),
"; from ",
static_cast<std::string>(runtime_attribute->get_type_info()),
" to ",
static_cast<std::string>(T::element_type::get_type_info_static()));
}
_temp = std::make_shared<Impl<decay_t<T>>>(
std::static_pointer_cast<typename T::element_type>(runtime_attribute));
@@ -801,7 +801,7 @@ public:
return *static_cast<decay_t<T>*>(_impl->addressof());
}
}
OPENVINO_UNREACHABLE("Bad cast from: ", _impl->type_info().name(), " to: ", typeid(T).name());
OPENVINO_THROW("Bad cast from: ", _impl->type_info().name(), " to: ", typeid(T).name());
}
/**
@@ -824,7 +824,7 @@ public:
return *static_cast<decay_t<T>*>(_impl->addressof());
}
}
OPENVINO_UNREACHABLE("Bad cast from: ", _impl->type_info().name(), " to: ", typeid(T).name());
OPENVINO_THROW("Bad cast from: ", _impl->type_info().name(), " to: ", typeid(T).name());
}
/**

10
src/core/include/openvino/core/attribute_adapter.hpp
View File

@@ -36,7 +36,7 @@ public:
virtual const DiscreteTypeInfo& get_type_info() const = 0;
virtual ~ValueAccessor() = default;
virtual void set_as_any(const ov::Any& x) {
throw ov::Exception("set_as_any is not implemented");
OPENVINO_NOT_IMPLEMENTED;
}
};
@@ -62,7 +62,7 @@ public:
if (x.is<VAT>()) {
set(*static_cast<const VAT*>(data));
} else {
OPENVINO_UNREACHABLE("Bad cast from: ", x.type_info().name(), " to: ", typeid(VAT).name());
OPENVINO_THROW("Bad cast from: ", x.type_info().name(), " to: ", typeid(VAT).name());
}
}
};
@@ -119,7 +119,7 @@ public:
m_ref = *static_cast<const AT*>(data);
m_buffer_valid = false;
} else {
OPENVINO_UNREACHABLE("Bad cast from: ", x.type_info().name(), " to: ", typeid(AT).name());
OPENVINO_THROW("Bad cast from: ", x.type_info().name(), " to: ", typeid(AT).name());
}
}
@@ -168,7 +168,7 @@ public:
m_ref = *static_cast<const AT*>(data);
m_buffer_valid = false;
} else {
OPENVINO_UNREACHABLE("Bad cast from: ", x.type_info().name(), " to: ", typeid(AT).name());
OPENVINO_THROW("Bad cast from: ", x.type_info().name(), " to: ", typeid(AT).name());
}
}
operator AT&() {
@@ -214,7 +214,7 @@ public:
// instead reimplement logic from set.
m_ref = *static_cast<const AT*>(data);
} else {
OPENVINO_UNREACHABLE("Bad cast from: ", x.type_info().name(), " to: ", typeid(AT).name());
OPENVINO_THROW("Bad cast from: ", x.type_info().name(), " to: ", typeid(AT).name());
}
}

3
src/core/include/openvino/core/except.hpp
View File

@@ -147,7 +147,8 @@ public:
/// implemented with OPENVINO_ASSERT macro.
/// \param ... Additional error message that should describe why that execution path is unreachable.
/// \throws ::ov::AssertFailure if the macro is executed.
#define OPENVINO_UNREACHABLE(...) OPENVINO_ASSERT(false, "Unreachable: ", __VA_ARGS__)
// TODO: throw ov::Exception after migration to functions
#define OPENVINO_THROW(...) OPENVINO_ASSERT(false, __VA_ARGS__)
#define OPENVINO_ASSERT_HELPER(exc_class, ctx, ...) CALL_OVERLOAD(OPENVINO_ASSERT_HELPER, exc_class, ctx, __VA_ARGS__)
#define OPENVINO_NOT_IMPLEMENTED OPENVINO_ASSERT_HELPER(::ov::NotImplemented, "", false, "Not Implemented", "")

4
src/core/include/openvino/core/model.hpp
View File

@@ -481,8 +481,8 @@ private:
std::is_same<T, char*>::value,
bool>::type = true>
const ov::Any& get_rt_arg(const ov::AnyMap& rt_info, const T& name) const {
if (rt_info.find(name) == rt_info.end())
throw ov::Exception("Cannot get runtime attribute. Path to runtime attribute is incorrect.");
OPENVINO_ASSERT(rt_info.find(name) != rt_info.end(),
"Cannot get runtime attribute. Path to runtime attribute is incorrect.");
return get_attr(rt_info.at(name));
}

2
src/core/include/openvino/core/type/bfloat16.hpp
View File

@@ -18,7 +18,7 @@
namespace ov {
class OPENVINO_API bfloat16 {
public:
constexpr bfloat16() : m_value{0} {}
bfloat16() = default;
bfloat16(float value) : m_value {
#if defined ROUND_MODE_TO_NEAREST
round_to_nearest(value)

2
src/core/include/openvino/core/type/float16.hpp
View File

@@ -18,7 +18,7 @@
namespace ov {
class OPENVINO_API float16 {
public:
constexpr float16() : m_value{0} {}
float16() = default;
static uint32_t constexpr frac_size = 10;
static uint32_t constexpr exp_size = 5;

1
src/core/include/openvino/op/adaptive_max_pool.hpp
View File

@@ -42,6 +42,7 @@ public:
element::Type get_index_element_type() const {
return m_index_element_type;
}
void set_index_element_type(const element::Type& type);
protected:
ov::element::Type m_index_element_type = ov::element::i64;

4
src/core/include/openvino/op/constant.hpp
View File

@@ -352,9 +352,7 @@ public:
}
template <typename T>
const T* get_data_ptr() const {
if (sizeof(T) > m_element_type.size() && shape_size(m_shape) > 0) {
throw ov::Exception("Buffer over-read");
}
OPENVINO_ASSERT(sizeof(T) <= m_element_type.size() || shape_size(m_shape) <= 0, "Buffer over-read");
return static_cast<const T*>(get_data_ptr());
}

Some files were not shown because too many files have changed in this diff Show More