IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into itikhono/ts/fix_performance_issues

Browse Source
This commit is contained in:
Tikhonov Ivan 2023-03-07 07:33:09 +00:00
commit 3c5f62c013
961 changed files with 20823 additions and 12168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.ci/azure/android_arm64.yml
View File

@ -118,13 +118,11 @@ jobs:
- checkout: self
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino
- checkout: openvino_contrib
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino_contrib
@ -160,6 +158,7 @@ jobs:
-DENABLE_TESTS=ON
-DBUILD_java_api=ON
-DBUILD_nvidia_plugin=OFF
-DENABLE_INTEL_GPU=ON
-DOPENVINO_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)/modules
-DCMAKE_CXX_LINKER_LAUNCHER=ccache
-DCMAKE_C_LINKER_LAUNCHER=ccache

23
.ci/azure/linux.yml
View File

@ -151,13 +151,11 @@ jobs:
- checkout: self
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino
- checkout: openvino_contrib
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino_contrib
@ -219,7 +217,6 @@ jobs:
# Should be after 'Install dependencies' because Git lfs is not installed
- checkout: testdata
clean: 'true'
fetchDepth: '1'
lfs: 'true'
path: testdata
@ -434,6 +431,9 @@ jobs:
- script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ieMultiPluginUnitTests --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-ieMultiPluginUnitTests.xml
displayName: 'MULTI UT'
- script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_auto_batch_unit_tests --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-ov_auto_batch_unit_tests.xml
displayName: 'AutoBatch UT'
- script: $(RUN_PREFIX) $(INSTALL_TEST_DIR)/ov_template_func_tests --gtest_filter=*smoke* --gtest_output=xml:$(INSTALL_TEST_DIR)/TEST-templateFuncTests.xml
displayName: 'TEMPLATE FuncTests'
@ -527,22 +527,9 @@ jobs:
python3 -m pip install -r $(LAYER_TESTS_DIR)/requirements.txt
export PYTHONPATH=$(LAYER_TESTS_DIR):$PYTHONPATH
export TEST_DEVICE=CPU
$(RUN_PREFIX) python3 -m pytest $(LAYER_TESTS_DIR)/mo_python_api_tests/test_mo_convert_complex_params.py --ir_version=11 --junitxml=./TEST-test_mo_convert_complex_params.xmlTEST
displayName: 'MO Python API Tests - Complex Python params'
$(RUN_PREFIX) python3 -m pytest $(LAYER_TESTS_DIR)/mo_python_api_tests/ --junitxml=./TEST-test_mo_convert.xmlTEST
displayName: 'MO Python API Tests'
- script: |
python3 -m pip install -r $(LAYER_TESTS_DIR)/requirements.txt
export PYTHONPATH=$(LAYER_TESTS_DIR):$PYTHONPATH
export TEST_DEVICE=CPU
$(RUN_PREFIX) python3 -m pytest $(LAYER_TESTS_DIR)/mo_python_api_tests/test_mo_convert_tf.py --ir_version=11 --junitxml=./TEST-test_mo_convert_tf.xmlTEST
displayName: 'MO Python API Tests - Import TF model from memory'
- script: |
python3 -m pip install -r $(LAYER_TESTS_DIR)/requirements.txt
export PYTHONPATH=$(LAYER_TESTS_DIR):$PYTHONPATH
export TEST_DEVICE=CPU
$(RUN_PREFIX) python3 -m pytest $(LAYER_TESTS_DIR)/mo_python_api_tests/test_mo_convert_pytorch.py --ir_version=11 --junitxml=./TEST-test_mo_convert_pytorch.xmlTEST
displayName: 'MO Python API Tests - Import PyTorch model from memory'
- script: |
python3 -m pip install -r $(LAYER_TESTS_DIR)/requirements.txt

2
.ci/azure/linux_arm64.yml
View File

@ -134,13 +134,11 @@ jobs:
- checkout: self
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino
- checkout: openvino_contrib
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino_contrib

2
.ci/azure/linux_conditional_compilation.yml
View File

@ -102,7 +102,6 @@ jobs:
- checkout: self
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino
@ -118,7 +117,6 @@ jobs:
- checkout: testdata
clean: 'true'
fetchDepth: '1'
lfs: 'true'
path: testdata

2
.ci/azure/linux_coverity.yml
View File

@ -82,13 +82,11 @@ jobs:
- checkout: self
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino
- checkout: openvino_contrib
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino_contrib

2
.ci/azure/linux_cuda.yml
View File

@ -100,13 +100,11 @@ jobs:
- checkout: self
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino
- checkout: openvino_contrib
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino_contrib

2
.ci/azure/linux_debian.yml
View File

@ -102,7 +102,6 @@ jobs:
- checkout: self
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino
@ -143,7 +142,6 @@ jobs:
# Should be after 'Install dependencies' because Git lfs is not installed
- checkout: testdata
clean: 'true'
fetchDepth: '1'
lfs: 'true'
path: testdata

2
.ci/azure/linux_lohika.yml
View File

@ -30,7 +30,6 @@ jobs:
# - checkout: self
# clean: 'true'
# fetchDepth: '1'
# submodules: 'true'
# path: openvino
@ -42,7 +41,6 @@ jobs:
# Should be after 'Install dependencies' because Git lfs is not installed
# - checkout: testdata
# clean: 'true'
# fetchDepth: '1'
# submodules: 'true'
# lfs: 'true'
# path: testdata

1
.ci/azure/linux_ngraph_onnx.yml
View File

@ -91,7 +91,6 @@ jobs:
- checkout: self
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino

1
.ci/azure/linux_onnxruntime.yml
View File

@ -101,7 +101,6 @@ jobs:
- checkout: self
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino

3
.ci/azure/mac.yml
View File

@ -100,19 +100,16 @@ jobs:
- checkout: self
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino
- checkout: openvino_contrib
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino_contrib
- checkout: testdata
clean: 'true'
fetchDepth: '1'
lfs: 'true'
path: testdata

6
.ci/azure/windows.yml
View File

@ -122,19 +122,16 @@ jobs:
- checkout: self
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino
- checkout: openvino_contrib
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino_contrib
- checkout: testdata
clean: 'true'
fetchDepth: '1'
lfs: 'true'
path: testdata
@ -305,6 +302,9 @@ jobs:
- script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ieMultiPluginUnitTests --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-ieMultiPluginUnitTests.xml
displayName: 'MULTI UT'
- script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_auto_batch_unit_tests --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-ov_auto_batch_unit_tests.xml
displayName: 'AutoBatch UT'
- script: call $(SETUPVARS) && $(INSTALL_TEST_DIR)\ov_template_func_tests --gtest_output=xml:$(INSTALL_TEST_DIR)\TEST-templateFuncTests.xml
displayName: 'TEMPLATE FuncTests'

2
.ci/azure/windows_conditional_compilation.yml
View File

@ -93,7 +93,6 @@ jobs:
- checkout: self
clean: 'true'
fetchDepth: '1'
submodules: 'true'
path: openvino
@ -107,7 +106,6 @@ jobs:
- checkout: testdata
clean: 'true'
lfs: 'true'
fetchDepth: '1'
path: testdata
- script: |

2
.github/workflows/build_doc.yml vendored
View File

@ -25,7 +25,7 @@ jobs:
lfs: true
- name: Install apt-get dependencies
uses: awalsh128/cache-apt-pkgs-action@v1.1.3
uses: awalsh128/cache-apt-pkgs-action@v1.2.4
with:
packages: graphviz texlive liblua5.2-0 libclang1-9 libclang-cpp9
version: 3.0

7
.github/workflows/code_snippets.yml vendored
View File

@ -30,6 +30,13 @@ jobs:
submodules: recursive
lfs: true
- name: Install OpenCL
uses: awalsh128/cache-apt-pkgs-action@v1.2.4
if: runner.os == 'Linux'
with:
packages: ocl-icd-opencl-dev opencl-headers
version: 3.0
- name: CMake configure
run: cmake -DCMAKE_BUILD_TYPE=Release -B build

2
.github/workflows/mo.yml vendored
View File

@ -30,7 +30,7 @@ jobs:
python-version: '3.10'
- name: Cache pip
uses: actions/cache@v1
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('tools/mo/requirements*.txt') }}

31
CMakeLists.txt
View File

@ -19,10 +19,6 @@ endif()
project(OpenVINO DESCRIPTION "OpenVINO toolkit")
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "CMake build type" FORCE)
endif()
find_package(IEDevScripts REQUIRED
PATHS "${OpenVINO_SOURCE_DIR}/cmake/developer_package"
NO_CMAKE_FIND_ROOT_PATH
@ -39,7 +35,6 @@ if(ENABLE_COVERAGE)
endif()
# resolving dependencies for the project
message (STATUS "PROJECT ............................... " ${PROJECT_NAME})
message (STATUS "CMAKE_VERSION ......................... " ${CMAKE_VERSION})
message (STATUS "CMAKE_BINARY_DIR ...................... " ${CMAKE_BINARY_DIR})
message (STATUS "CMAKE_SOURCE_DIR ...................... " ${CMAKE_SOURCE_DIR})
@ -48,10 +43,28 @@ message (STATUS "OpenVINO_BINARY_DIR ................... " ${OpenVINO_BINARY_DIR
message (STATUS "CMAKE_GENERATOR ....................... " ${CMAKE_GENERATOR})
message (STATUS "CMAKE_C_COMPILER_ID ................... " ${CMAKE_C_COMPILER_ID})
message (STATUS "CMAKE_CXX_COMPILER_ID ................. " ${CMAKE_CXX_COMPILER_ID})
message (STATUS "CMAKE_BUILD_TYPE ...................... " ${CMAKE_BUILD_TYPE})
message (STATUS "CMAKE_TOOLCHAIN_FILE .................. " ${CMAKE_TOOLCHAIN_FILE})
message (STATUS "GLIBC_VERSION.......................... " ${OV_GLIBC_VERSION})
if(OV_GENERATOR_MULTI_CONFIG)
string(REPLACE ";" " " config_types "${CMAKE_CONFIGURATION_TYPES}")
message (STATUS "CMAKE_CONFIGURATION_TYPES ............. " ${config_types})
unset(config_types)
if(CMAKE_GENERATOR MATCHES "^Ninja Multi-Config$")
message (STATUS "CMAKE_DEFAULT_BUILD_TYPE .............. " ${CMAKE_DEFAULT_BUILD_TYPE})
endif()
else()
message (STATUS "CMAKE_BUILD_TYPE ...................... " ${CMAKE_BUILD_TYPE})
endif()
if(CMAKE_GENERATOR_PLATFORM)
message (STATUS "CMAKE_GENERATOR_PLATFORM .............. " ${CMAKE_GENERATOR_PLATFORM})
endif()
if(CMAKE_GENERATOR_TOOLSET)
message (STATUS "CMAKE_GENERATOR_TOOLSET ............... " ${CMAKE_GENERATOR_TOOLSET})
endif()
if(CMAKE_TOOLCHAIN_FILE)
message (STATUS "CMAKE_TOOLCHAIN_FILE .................. " ${CMAKE_TOOLCHAIN_FILE})
endif()
if(OV_GLIBC_VERSION)
message (STATUS "GLIBC_VERSION ......................... " ${OV_GLIBC_VERSION})
endif()
# remove file with exported developer targets to force its regeneration
file(REMOVE "${CMAKE_BINARY_DIR}/ngraphTargets.cmake")

14
cmake/developer_package/IEDevScriptsConfig.cmake
View File

@ -24,7 +24,6 @@ function(set_ci_build_number)
endfunction()
include(features)
include(message)
set_ci_build_number()
@ -112,10 +111,13 @@ else()
set(BIN_FOLDER "bin/${ARCH_FOLDER}")
endif()
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "CMake build type")
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Release;Debug;RelWithDebInfo;MinSizeRel")
if(CMAKE_GENERATOR MATCHES "^Ninja Multi-Config$")
# Ninja-Multi specific, see:
# https://cmake.org/cmake/help/latest/variable/CMAKE_DEFAULT_BUILD_TYPE.html
set(CMAKE_DEFAULT_BUILD_TYPE "Release" CACHE STRING "CMake default build type")
elseif(NOT OV_GENERATOR_MULTI_CONFIG)
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "CMake build type")
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Release;Debug;RelWithDebInfo;MinSizeRel")
endif()
if(USE_BUILD_TYPE_SUBFOLDER)
@ -153,10 +155,10 @@ set(CMAKE_DEBUG_POSTFIX ${IE_DEBUG_POSTFIX})
set(CMAKE_RELEASE_POSTFIX ${IE_RELEASE_POSTFIX})
# Support CMake multi-configuration for Visual Studio / Ninja or Xcode build
if (OV_GENERATOR_MULTI_CONFIG)
if(OV_GENERATOR_MULTI_CONFIG)
set(IE_BUILD_POSTFIX $<$<CONFIG:Debug>:${IE_DEBUG_POSTFIX}>$<$<CONFIG:Release>:${IE_RELEASE_POSTFIX}>)
else ()
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
else()
if(CMAKE_BUILD_TYPE STREQUAL "Debug")
set(IE_BUILD_POSTFIX ${IE_DEBUG_POSTFIX})
else()
set(IE_BUILD_POSTFIX ${IE_RELEASE_POSTFIX})

188
cmake/developer_package/api_validator/api_validator.cmake
View File

@ -5,60 +5,99 @@
if(WIN32)
set(PROGRAMFILES_ENV "ProgramFiles(X86)")
file(TO_CMAKE_PATH $ENV{${PROGRAMFILES_ENV}} PROGRAMFILES)
set(UWP_SDK_PATH "${PROGRAMFILES}/Windows Kits/10/bin/${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION}/x64")
message(STATUS "Trying to find apivalidator in: ${UWP_SDK_PATH}")
find_host_program(UWP_API_VALIDATOR
set(WDK_PATHS "${PROGRAMFILES}/Windows Kits/10/bin/${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION}/x64"
"${PROGRAMFILES}/Windows Kits/10/bin/x64")
message(STATUS "Trying to find apivalidator in: ")
foreach(wdk_path IN LISTS WDK_PATHS)
message(" * ${wdk_path}")
endforeach()
find_host_program(ONECORE_API_VALIDATOR
NAMES apivalidator
PATHS "${UWP_SDK_PATH}"
DOC "ApiValidator for UWP compliance")
PATHS ${WDK_PATHS}
DOC "ApiValidator for OneCore compliance")
if(UWP_API_VALIDATOR)
message(STATUS "Found apivalidator: ${UWP_API_VALIDATOR}")
if(ONECORE_API_VALIDATOR)
message(STATUS "Found apivalidator: ${ONECORE_API_VALIDATOR}")
endif()
endif()
function(_ie_add_api_validator_post_build_step_recursive)
cmake_parse_arguments(API_VALIDATOR "" "TARGET" "" ${ARGN})
list(APPEND API_VALIDATOR_TARGETS ${API_VALIDATOR_TARGET})
set(API_VALIDATOR_TARGETS ${API_VALIDATOR_TARGETS} PARENT_SCOPE)
get_target_property(IS_IMPORTED ${API_VALIDATOR_TARGET} IMPORTED)
if(IS_IMPORTED)
return()
endif()
get_target_property(LIBRARY_TYPE ${API_VALIDATOR_TARGET} TYPE)
if(LIBRARY_TYPE STREQUAL "EXECUTABLE" OR LIBRARY_TYPE STREQUAL "SHARED_LIBRARY")
get_target_property(LINKED_LIBRARIES ${API_VALIDATOR_TARGET} LINK_LIBRARIES)
if(LINKED_LIBRARIES)
foreach(ITEM IN LISTS LINKED_LIBRARIES)
if(NOT TARGET ${ITEM})
continue()
endif()
get_target_property(LIBRARY_TYPE_DEPENDENCY ${ITEM} TYPE)
if(LIBRARY_TYPE_DEPENDENCY STREQUAL "SHARED_LIBRARY")
_ie_add_api_validator_post_build_step_recursive(TARGET ${ITEM})
endif()
endforeach()
endif()
if(LIBRARY_TYPE MATCHES "^(SHARED_LIBRARY|MODULE_LIBRARY|EXECUTABLE)$" AND
NOT ${API_VALIDATOR_TARGET} IN_LIST API_VALIDATOR_TARGETS)
list(APPEND API_VALIDATOR_TARGETS ${API_VALIDATOR_TARGET})
endif()
# keep checks target list to track cyclic dependencies, leading to infinite recursion
list(APPEND checked_targets ${API_VALIDATOR_TARGET})
if(NOT LIBRARY_TYPE STREQUAL "INTERFACE_LIBRARY")
get_target_property(LINKED_LIBRARIES ${API_VALIDATOR_TARGET} LINK_LIBRARIES)
else()
set(LINKED_LIBRARIES)
endif()
get_target_property(INTERFACE_LINKED_LIBRARIES ${API_VALIDATOR_TARGET} INTERFACE_LINK_LIBRARIES)
foreach(library IN LISTS LINKED_LIBRARIES INTERFACE_LINKED_LIBRARIES)
if(TARGET "${library}")
get_target_property(orig_library ${library} ALIASED_TARGET)
if(orig_library IN_LIST checked_targets OR library IN_LIST checked_targets)
# in case of cyclic dependencies, we need to skip current target
continue()
endif()
if(TARGET "${orig_library}")
_ie_add_api_validator_post_build_step_recursive(TARGET ${orig_library})
else()
_ie_add_api_validator_post_build_step_recursive(TARGET ${library})
endif()
endif()
endforeach()
set(API_VALIDATOR_TARGETS ${API_VALIDATOR_TARGETS} PARENT_SCOPE)
endfunction()
set(VALIDATED_LIBRARIES "" CACHE INTERNAL "")
set(VALIDATED_TARGETS "" CACHE INTERNAL "")
function(_ov_add_api_validator_post_build_step)
set(UWP_API_VALIDATOR_APIS "${PROGRAMFILES}/Windows Kits/10/build/universalDDIs/x64/UniversalDDIs.xml")
set(UWP_API_VALIDATOR_EXCLUSION "${UWP_SDK_PATH}/BinaryExclusionlist.xml")
if((NOT UWP_API_VALIDATOR) OR (WINDOWS_STORE OR WINDOWS_PHONE))
if((NOT ONECORE_API_VALIDATOR) OR (WINDOWS_STORE OR WINDOWS_PHONE))
return()
endif()
cmake_parse_arguments(API_VALIDATOR "" "TARGET" "" ${ARGN})
# see https://learn.microsoft.com/en-us/windows-hardware/drivers/develop/validating-windows-drivers#known-apivalidator-issues
# ApiValidator does not run on Arm64 because AitStatic does not work on Arm64
if(HOST_AARCH64)
return()
endif()
if(X86_64)
set(wdk_platform "x64")
elseif(X86)
set(wdk_platform "x86")
elseif(ARM)
set(wdk_platform "arm")
elseif(AARCH64)
set(wdk_platform "arm64")
else()
message(FATAL_ERROR "Unknown configuration: ${CMAKE_HOST_SYSTEM_PROCESSOR}")
endif()
find_file(ONECORE_API_VALIDATOR_APIS NAMES UniversalDDIs.xml
PATHS "${PROGRAMFILES}/Windows Kits/10/build/${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION}/universalDDIs/${wdk_platform}"
"${PROGRAMFILES}/Windows Kits/10/build/universalDDIs/${wdk_platform}"
DOC "Path to UniversalDDIs.xml file")
find_file(ONECORE_API_VALIDATOR_EXCLUSION NAMES BinaryExclusionlist.xml
PATHS ${WDK_PATHS}
DOC "Path to BinaryExclusionlist.xml file")
if(NOT ONECORE_API_VALIDATOR_APIS)
message(FATAL_ERROR "Internal error: apiValidator is found (${ONECORE_API_VALIDATOR}), but UniversalDDIs.xml file has not been found for ${wdk_platform} platform")
endif()
cmake_parse_arguments(API_VALIDATOR "" "TARGET" "EXTRA" "" ${ARGN})
if(NOT API_VALIDATOR_TARGET)
message(FATAL_ERROR "RunApiValidator requires TARGET to validate!")
@ -69,74 +108,81 @@ function(_ov_add_api_validator_post_build_step)
endif()
# collect targets
_ie_add_api_validator_post_build_step_recursive(TARGET ${API_VALIDATOR_TARGET})
if (API_VALIDATOR_EXTRA)
foreach(target IN LISTS API_VALIDATOR_EXTRA)
_ie_add_api_validator_post_build_step_recursive(TARGET ${target})
endforeach()
endif()
# remove targets which were tested before
foreach(target IN LISTS API_VALIDATOR_TARGETS)
list(FIND VALIDATED_LIBRARIES ${target} index)
if (NOT index EQUAL -1)
list(APPEND VALIDATED_TARGETS ${target})
endif()
if(TARGET "${target}")
get_target_property(orig_target ${target} ALIASED_TARGET)
list(FIND VALIDATED_LIBRARIES ${orig_target} index)
if (NOT index EQUAL -1)
list(APPEND VALIDATED_TARGETS ${target})
endif()
endif()
endforeach()
foreach(item IN LISTS VALIDATED_TARGETS)
list(REMOVE_ITEM API_VALIDATOR_TARGETS ${item})
endforeach()
list(REMOVE_DUPLICATES API_VALIDATOR_TARGETS)
if(NOT API_VALIDATOR_TARGETS)
return()
endif()
# apply check
macro(api_validator_get_target_name)
get_target_property(IS_IMPORTED ${target} IMPORTED)
get_target_property(is_imported ${target} IMPORTED)
get_target_property(orig_target ${target} ALIASED_TARGET)
if(IS_IMPORTED)
get_target_property(target_location ${target} LOCATION)
get_filename_component(target_name "${target_location}" NAME_WE)
if(is_imported)
get_target_property(imported_configs ${target} IMPORTED_CONFIGURATIONS)
foreach(imported_config RELEASE RELWITHDEBINFO DEBUG)
if(imported_config IN_LIST imported_configs)
get_target_property(target_location ${target} IMPORTED_LOCATION_${imported_config})
get_filename_component(target_name "${target_location}" NAME_WE)
break()
endif()
endforeach()
unset(imported_configs)
elseif(TARGET "${orig_target}")
set(target_name ${orig_target})
set(target_location $<TARGET_FILE:${orig_target}>)
else()
set(target_name ${target})
set(target_location $<TARGET_FILE:${target}>)
endif()
unset(orig_target)
unset(is_imported)
endmacro()
foreach(target IN LISTS API_VALIDATOR_TARGETS)
api_validator_get_target_name()
if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.21 AND OV_GENERATOR_MULTI_CONFIG)
set(output_file "${CMAKE_BINARY_DIR}/api_validator/$<CONFIG>/${target_name}.txt")
if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.20 AND OV_GENERATOR_MULTI_CONFIG)
set(output_file "${OpenVINO_BINARY_DIR}/api_validator/$<CONFIG>/${target_name}.txt")
else()
set(output_file "${CMAKE_BINARY_DIR}/api_validator/${target_name}.txt")
set(output_file "${OpenVINO_BINARY_DIR}/api_validator/${target_name}.txt")
endif()
add_custom_command(TARGET ${API_VALIDATOR_TARGET} POST_BUILD
COMMAND ${CMAKE_COMMAND} --config $<CONFIG>
-D UWP_API_VALIDATOR=${UWP_API_VALIDATOR}
-D UWP_API_VALIDATOR_TARGET=$<TARGET_FILE:${target}>
-D UWP_API_VALIDATOR_APIS=${UWP_API_VALIDATOR_APIS}
-D UWP_API_VALIDATOR_EXCLUSION=${UWP_API_VALIDATOR_EXCLUSION}
-D UWP_API_VALIDATOR_OUTPUT=${output_file}
list(APPEND post_build_commands
${CMAKE_COMMAND} --config $<CONFIG>
-D ONECORE_API_VALIDATOR=${ONECORE_API_VALIDATOR}
-D ONECORE_API_VALIDATOR_TARGET=${target_location}
-D ONECORE_API_VALIDATOR_APIS=${ONECORE_API_VALIDATOR_APIS}
-D ONECORE_API_VALIDATOR_EXCLUSION=${ONECORE_API_VALIDATOR_EXCLUSION}
-D ONECORE_API_VALIDATOR_OUTPUT=${output_file}
-D CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}
-P "${IEDevScripts_DIR}/api_validator/api_validator_run.cmake"
BYPRODUCTS ${output_file}
COMMENT "[apiValidator] Check ${target_name} for OneCore compliance"
VERBATIM)
-P "${IEDevScripts_DIR}/api_validator/api_validator_run.cmake")
list(APPEND byproducts_files ${output_file})
unset(target_name)
unset(target_location)
endforeach()
add_custom_command(TARGET ${API_VALIDATOR_TARGET} POST_BUILD
COMMAND ${post_build_commands}
BYPRODUCTS ${byproducts_files}
COMMENT "[apiValidator] Check ${API_VALIDATOR_TARGET} and dependencies for OneCore compliance"
VERBATIM)
# update list of validated libraries
list(APPEND VALIDATED_LIBRARIES ${API_VALIDATOR_TARGETS})
set(VALIDATED_LIBRARIES "${VALIDATED_LIBRARIES}" CACHE INTERNAL "" FORCE)
list(APPEND VALIDATED_TARGETS ${API_VALIDATOR_TARGETS})
set(VALIDATED_TARGETS "${VALIDATED_TARGETS}" CACHE INTERNAL "" FORCE)
endfunction()
#

30
cmake/developer_package/api_validator/api_validator_run.cmake
View File

@ -4,9 +4,9 @@
cmake_policy(SET CMP0012 NEW)
foreach(var UWP_API_VALIDATOR UWP_API_VALIDATOR_TARGET
UWP_API_VALIDATOR_APIS UWP_API_VALIDATOR_EXCLUSION
UWP_API_VALIDATOR_OUTPUT CMAKE_TOOLCHAIN_FILE)
foreach(var ONECORE_API_VALIDATOR ONECORE_API_VALIDATOR_TARGET
ONECORE_API_VALIDATOR_APIS ONECORE_API_VALIDATOR_EXCLUSION
ONECORE_API_VALIDATOR_OUTPUT CMAKE_TOOLCHAIN_FILE)
if(NOT DEFINED ${var})
message(FATAL_ERROR "Variable ${var} is not defined")
endif()
@ -14,18 +14,18 @@ endforeach()
# create command
if(NOT EXISTS "${UWP_API_VALIDATOR_APIS}")
message(FATAL_ERROR "${UWP_API_VALIDATOR_APIS} does not exist")
if(NOT EXISTS "${ONECORE_API_VALIDATOR_APIS}")
message(FATAL_ERROR "${ONECORE_API_VALIDATOR_APIS} does not exist")
endif()
set(command "${UWP_API_VALIDATOR}"
-SupportedApiXmlFiles:${UWP_API_VALIDATOR_APIS}
-DriverPackagePath:${UWP_API_VALIDATOR_TARGET})
if(EXISTS "${UWP_API_VALIDATOR_EXCLUSION}")
set(command "${ONECORE_API_VALIDATOR}"
-SupportedApiXmlFiles:${ONECORE_API_VALIDATOR_APIS}
-DriverPackagePath:${ONECORE_API_VALIDATOR_TARGET})
if(EXISTS "${ONECORE_API_VALIDATOR_EXCLUSION}")
list(APPEND command
-BinaryExclusionListXmlFile:${UWP_API_VALIDATOR_EXCLUSION}
-BinaryExclusionListXmlFile:${ONECORE_API_VALIDATOR_EXCLUSION}
-StrictCompliance:TRUE)
set(UWP_HAS_BINARY_EXCLUSION ON)
set(ONECORE_HAS_BINARY_EXCLUSION ON)
endif()
# execute
@ -36,13 +36,13 @@ execute_process(COMMAND ${command}
RESULT_VARIABLE exit_code
OUTPUT_STRIP_TRAILING_WHITESPACE)
file(WRITE "${UWP_API_VALIDATOR_OUTPUT}" "${output_message}\n\n\n${error_message}")
file(WRITE "${ONECORE_API_VALIDATOR_OUTPUT}" "CMAKE COMMAND: ${command}\n\n\n${output_message}\n\n\n${error_message}")
# post-process output
get_filename_component(name "${UWP_API_VALIDATOR_TARGET}" NAME)
get_filename_component(name "${ONECORE_API_VALIDATOR_TARGET}" NAME)
if(NOT UWP_HAS_BINARY_EXCLUSION)
if(NOT ONECORE_HAS_BINARY_EXCLUSION)
if(CMAKE_TOOLCHAIN_FILE MATCHES "onecoreuap.toolchain.cmake$")
# empty since we compile with static MSVC runtime
else()
@ -66,7 +66,7 @@ endif()
# write output
if(UWP_HAS_BINARY_EXCLUSION AND NOT exit_code EQUAL 0)
if(ONECORE_HAS_BINARY_EXCLUSION AND NOT exit_code EQUAL 0)
message(FATAL_ERROR "${error_message}")
endif()

4
cmake/developer_package/clang_format/clang_format.cmake
View File

@ -70,6 +70,10 @@ function(add_clang_format_target TARGET_NAME)
continue()
endif()
if(IS_DIRECTORY "${source_file}")
message(FATAL_ERROR "Directory ${source_file} cannot be passed to clang-format")
endif()
file(RELATIVE_PATH source_file_relative "${CMAKE_CURRENT_SOURCE_DIR}" "${source_file}")
set(output_file "${CMAKE_CURRENT_BINARY_DIR}/clang_format/${source_file_relative}.clang")
string(REPLACE ".." "__" output_file "${output_file}")

11
cmake/developer_package/frontends/frontends.cmake
View File

@ -182,7 +182,7 @@ macro(ov_add_frontend)
add_library(openvino::frontend::${OV_FRONTEND_NAME} ALIAS ${TARGET_NAME})
endif()
# Shutdown protobuf when unloading the front dynamic library
# Shutdown protobuf when unloading the frontend dynamic library
if(proto_files AND BUILD_SHARED_LIBS)
target_link_libraries(${TARGET_NAME} PRIVATE ov_protobuf_shutdown)
endif()
@ -217,8 +217,6 @@ macro(ov_add_frontend)
ie_add_vs_version_file(NAME ${TARGET_NAME}
FILEDESCRIPTION ${OV_FRONTEND_FILEDESCRIPTION})
ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME})
target_link_libraries(${TARGET_NAME} PUBLIC openvino::runtime)
target_link_libraries(${TARGET_NAME} PRIVATE ${OV_FRONTEND_LINK_LIBRARIES})
ov_add_library_version(${TARGET_NAME})
@ -255,10 +253,15 @@ macro(ov_add_frontend)
endif()
add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME}
EXCLUDE_PATTERNS ${PROTO_SRCS} ${PROTO_HDRS} ${flatbuffers_schema_files})
EXCLUDE_PATTERNS ${PROTO_SRCS} ${PROTO_HDRS} ${proto_files} ${flatbuffers_schema_files})
add_dependencies(ov_frontends ${TARGET_NAME})
# must be called after all target_link_libraries
ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME})
# installation
if(NOT OV_FRONTEND_SKIP_INSTALL)
if(BUILD_SHARED_LIBS)
# Note:

27
cmake/developer_package/message.cmake
View File

@ -1,27 +0,0 @@
# Copyright (C) 2018-2023 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
if(UNIX AND ENABLE_ERROR_HIGHLIGHT)
function(message)
string(ASCII 27 ESC)
set(RESET "${ESC}[m")
set(RED "${ESC}[31;1m")
set(YELLOW "${ESC}[33;1m")
list(GET ARGV 0 MessageType)
list(REMOVE_AT ARGV 0)
foreach(arg IN LISTS ARGV)
set(_msg "${_msg}${arg}")
endforeach()
if(MessageType STREQUAL FATAL_ERROR OR MessageType STREQUAL SEND_ERROR)
_message(${MessageType} "${RED}${_msg}${RESET}")
elseif(MessageType STREQUAL WARNING)
_message(${MessageType} "${YELLOW}${_msg}${RESET}")
else()
_message(${MessageType} "${_msg}")
endif()
endfunction()
endif()

10
cmake/developer_package/native_compile.cmake
View File

@ -63,6 +63,15 @@ function(ov_native_compile_external_project)
set(ARG_NATIVE_SOURCE_SUBDIR SOURCE_SUBDIR ${ARG_NATIVE_SOURCE_SUBDIR})
endif()
if(OV_GENERATOR_MULTI_CONFIG)
if(CMAKE_GENERATOR MATCHES "^Ninja Multi-Config$")
list(APPEND ARG_CMAKE_ARGS "-DCMAKE_CONFIGURATION_TYPES=${CMAKE_DEFAULT_BUILD_TYPE}")
list(APPEND ARG_CMAKE_ARGS "-DCMAKE_DEFAULT_BUILD_TYPE=${CMAKE_DEFAULT_BUILD_TYPE}")
endif()
else()
list(APPEND ARG_CMAKE_ARGS "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}")
endif()
ExternalProject_Add(${ARG_TARGET_NAME}
# Directory Options
SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}"
@ -81,7 +90,6 @@ function(ov_native_compile_external_project)
"-DCMAKE_C_FLAGS=${compile_flags}"
"-DCMAKE_POLICY_DEFAULT_CMP0069=NEW"
"-DCMAKE_INSTALL_PREFIX=${ARG_NATIVE_INSTALL_DIR}"
"-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
${ARG_CMAKE_ARGS}
CMAKE_GENERATOR "${CMAKE_GENERATOR}"
${ARG_NATIVE_SOURCE_SUBDIR}

2
cmake/developer_package/ncc_naming_style/openvino.style
View File

@ -12,7 +12,7 @@ TemplateNonTypeParameter: '^\w*$'
ClassTemplate: '^([A-Z][\w]+|element_type_traits)$'
TemplateTypeParameter: '^\w*$'
ParameterName: '^\w*$'
FunctionTemplate: '^(operator.+|[\w]+|Impl<.*>)$'
FunctionTemplate: '^(operator.+|[\w]+|SoPtr.+|Impl<.*>)$'
TypeAliasName: '^\w+$'
VariableReference: '^\w+$'

3
cmake/developer_package/packaging/packaging.cmake
View File

@ -194,7 +194,7 @@ macro(ie_cpack)
set(CPACK_STRIP_FILES ON)
endif()
# TODO: replace with openvino
# TODO: replace with openvino and handle multi-config generators case
if(WIN32)
set(CPACK_PACKAGE_NAME inference-engine_${CMAKE_BUILD_TYPE})
else()
@ -202,6 +202,7 @@ macro(ie_cpack)
endif()
set(CPACK_PACKAGE_VERSION "${OpenVINO_VERSION}")
# build version can be empty in case we are running cmake out of git repository
if(NOT OpenVINO_VERSION_BUILD STREQUAL "000")
set(CPACK_PACKAGE_VERSION "${CPACK_PACKAGE_VERSION}.${OpenVINO_VERSION_BUILD}")
endif()

6
cmake/developer_package/target_flags.cmake
View File

@ -20,7 +20,7 @@ if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*")
set(arch_flag X86_64)
elseif(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "i686.*|i386.*|x86.*|amd64.*|AMD64.*")
set(arch_flag X86)
elseif(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(arm64.*|aarch64.*|AARCH64.*)")
elseif(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(arm64.*|aarch64.*|AARCH64.*|ARM64.*)")
set(arch_flag AARCH64)
elseif(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(arm.*|ARM.*)")
set(arch_flag ARM)
@ -31,8 +31,8 @@ endif()
set(HOST_${arch_flag} ON)
macro(_ie_process_msvc_generator_platform arch_flag)
# if cmake -A <ARM|ARM64> is passed
if(CMAKE_GENERATOR_PLATFORM STREQUAL "ARM64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "ARM64")
# if cmake -A <ARM|ARM64|x64|Win32> is passed
if(CMAKE_GENERATOR_PLATFORM STREQUAL "ARM64")
set(AARCH64 ON)
elseif(CMAKE_GENERATOR_PLATFORM STREQUAL "ARM")
set(ARM ON)

10
cmake/features.cmake
View File

@ -14,7 +14,13 @@ ie_option (ENABLE_COMPILE_TOOL "Enables compile_tool" ON)
ie_option (ENABLE_STRICT_DEPENDENCIES "Skip configuring \"convinient\" dependencies for efficient parallel builds" ON)
ie_dependent_option (ENABLE_INTEL_GPU "GPU OpenCL-based plugin for OpenVINO Runtime" ON "X86_64;NOT APPLE;NOT MINGW;NOT WINDOWS_STORE;NOT WINDOWS_PHONE" OFF)
if(X86_64)
set(ENABLE_INTEL_GPU_DEFAULT ON)
else()
set(ENABLE_INTEL_GPU_DEFAULT OFF)
endif()
ie_dependent_option (ENABLE_INTEL_GPU "GPU OpenCL-based plugin for OpenVINO Runtime" ${ENABLE_INTEL_GPU_DEFAULT} "X86_64 OR AARCH64;NOT APPLE;NOT MINGW;NOT WINDOWS_STORE;NOT WINDOWS_PHONE" OFF)
if (ANDROID OR (CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0))
# oneDNN doesn't support old compilers and android builds for now, so we'll
@ -41,8 +47,6 @@ In case SELECTIVE_BUILD is enabled, the SELECTIVE_BUILD_STAT variable should con
Usage: -DSELECTIVE_BUILD=ON -DSELECTIVE_BUILD_STAT=/path/*.csv" OFF
ALLOWED_VALUES ON OFF COLLECT)
ie_option(ENABLE_ERROR_HIGHLIGHT "Highlight errors and warnings during compile time" ON)
ie_option (ENABLE_DOCS "Build docs using Doxygen" OFF)
find_package(PkgConfig QUIET)

2
cmake/packaging/debian.cmake
View File

@ -52,6 +52,8 @@ macro(ov_cpack_settings)
NOT item STREQUAL OV_CPACK_COMP_PYTHON_WHEELS AND
# see ticket # 82605
NOT item STREQUAL "gna" AND
# don't install Intel OpenMP during debian
NOT item STREQUAL "omp" AND
# even for case of system TBB we have installation rules for wheels packages
# so, need to skip this explicitly
NOT item MATCHES "^tbb(_dev)?$" AND

2
cmake/packaging/rpm.cmake
View File

@ -38,6 +38,8 @@ macro(ov_cpack_settings)
NOT item STREQUAL OV_CPACK_COMP_PYTHON_WHEELS AND
# see ticket # 82605
NOT item STREQUAL "gna" AND
# don't install Intel OpenMP during rpm
NOT item STREQUAL "omp" AND
# even for case of system TBB we have installation rules for wheels packages
# so, need to skip this explicitly
NOT item MATCHES "^tbb(_dev)?$" AND

3
cmake/templates/InferenceEngineDeveloperPackageConfig.cmake.in
View File

@ -16,7 +16,8 @@ set(ie_options "@IE_OPTIONS@")
list(APPEND ie_options CMAKE_CXX_COMPILER_LAUNCHER CMAKE_C_COMPILER_LAUNCHER
CMAKE_CXX_LINKER_LAUNCHER CMAKE_C_LINKER_LAUNCHER
CMAKE_BUILD_TYPE CMAKE_SKIP_RPATH CMAKE_INSTALL_PREFIX
CMAKE_OSX_ARCHITECTURES CMAKE_OSX_DEPLOYMENT_TARGET)
CMAKE_OSX_ARCHITECTURES CMAKE_OSX_DEPLOYMENT_TARGET
CMAKE_CONFIGURATION_TYPES CMAKE_DEFAULT_BUILD_TYPE)
file(TO_CMAKE_PATH "${CMAKE_CURRENT_LIST_DIR}" cache_path)
message(STATUS "The following CMake options are exported from Inference Engine Developer package")

3
cmake/templates/OpenVINODeveloperPackageConfig.cmake.in
View File

@ -14,7 +14,8 @@ set(ov_options "@IE_OPTIONS@")
list(APPEND ov_options CMAKE_CXX_COMPILER_LAUNCHER CMAKE_C_COMPILER_LAUNCHER
CMAKE_CXX_LINKER_LAUNCHER CMAKE_C_LINKER_LAUNCHER
CMAKE_BUILD_TYPE CMAKE_SKIP_RPATH CMAKE_INSTALL_PREFIX
CMAKE_OSX_ARCHITECTURES CMAKE_OSX_DEPLOYMENT_TARGET)
CMAKE_OSX_ARCHITECTURES CMAKE_OSX_DEPLOYMENT_TARGET
CMAKE_CONFIGURATION_TYPES CMAKE_DEFAULT_BUILD_TYPE)
file(TO_CMAKE_PATH "${CMAKE_CURRENT_LIST_DIR}" cache_path)
message(STATUS "The following CMake options are exported from OpenVINO Developer package")

4
docs/Documentation/model_introduction.md
View File

@ -12,7 +12,7 @@
@endsphinxdirective
Every deep learning workflow begins with obtaining a model. You can choose to prepare a custom one, use a ready-made solution and adjust it to your needs, or even download and run a pre-trained network from an online database, such as OpenVINO's [Open Model Zoo](../model_zoo.md).
Every deep learning workflow begins with obtaining a model. You can choose to prepare a custom one, use a ready-made solution and adjust it to your needs, or even download and run a pre-trained network from an online database, such as OpenVINO's [Open Model Zoo](../model_zoo.md).
[OpenVINO™ supports several model formats](../MO_DG/prepare_model/convert_model/supported_model_formats.md) and allows to convert them to it's own, OpenVINO IR, providing a tool dedicated to this task.
@ -20,7 +20,7 @@ Every deep learning workflow begins with obtaining a model. You can choose to pr
The approach to fully convert a model is considered the default choice, as it allows the full extent of OpenVINO features. The OpenVINO IR model format is used by other conversion and preparation tools, such as the Post-Training Optimization Tool, for further optimization of the converted model.
Conversion is not required for ONNX and PaddlePaddle models, as OpenVINO provides C++ and Python APIs for importing them to OpenVINO Runtime directly. It provides a convenient way to quickly switch from framework-based code to OpenVINO-based code in your inference application.
Conversion is not required for ONNX, PaddlePaddle, and TensorFlow models (check [TensorFlow Frontend Capabilities and Limitations](../resources/tensorflow_frontend.md)), as OpenVINO provides C++ and Python APIs for importing them to OpenVINO Runtime directly. It provides a convenient way to quickly switch from framework-based code to OpenVINO-based code in your inference application.
This section describes how to obtain and prepare your model for work with OpenVINO to get the best inference results:
* [See the supported formats and how to use them in your project](../MO_DG/prepare_model/convert_model/supported_model_formats.md)

18
docs/Documentation/openvino_ecosystem.md
View File

@ -6,11 +6,11 @@
:maxdepth: 1
:hidden:
ote_documentation
ovtf_integration
ovsa_get_started
openvino_inference_engine_tools_compile_tool_README
openvino_docs_tuning_utilities
workbench_docs_Workbench_DG_Introduction
@endsphinxdirective
@ -25,6 +25,15 @@ More resources:
* [GitHub](https://github.com/openvinotoolkit/nncf)
* [PyPI](https://pypi.org/project/nncf/)
### OpenVINO™ Training Extensions
A convenient environment to train Deep Learning models and convert them using the OpenVINO™ toolkit for optimized inference.
More resources:
* [Overview](@ref ote_documentation)
* [GitHub](https://github.com/openvinotoolkit/training_extensions)
* [Documentation](https://openvinotoolkit.github.io/training_extensions/stable/guide/get_started/introduction.html)
### OpenVINO™ Security Add-on
A solution for Model Developers and Independent Software Vendors to use secure packaging and secure model execution.
@ -48,6 +57,7 @@ More resources:
* [documentation on GitHub](https://dlstreamer.github.io/index.html)
* [installation Guide on GitHub](https://github.com/openvinotoolkit/dlstreamer_gst/wiki/Install-Guide)
### DL Workbench
A web-based tool for deploying deep learning models. Built on the core of OpenVINO and equipped with a graphics user interface, DL Workbench is a great way to explore the possibilities of the OpenVINO workflow, import, analyze, optimize, and build your pre-trained models. You can do all that by visiting [Intel® Developer Cloud](https://software.intel.com/content/www/us/en/develop/tools/devcloud.html) and launching DL Workbench online.
@ -56,12 +66,6 @@ More resources:
* [Docker Hub](https://hub.docker.com/r/openvino/workbench)
* [PyPI](https://pypi.org/project/openvino-workbench/)
### OpenVINO™ Training Extensions (OTX)
A convenient environment to train Deep Learning models and convert them using the OpenVINO™ toolkit for optimized inference.
More resources:
* [GitHub](https://github.com/openvinotoolkit/training_extensions)
### Computer Vision Annotation Tool (CVAT)
An online, interactive video and image annotation tool for computer vision purposes.

40
docs/Documentation/openvino_training_extensions.md Normal file
View File

@ -0,0 +1,40 @@
# OpenVINO™ Training Extensions {#ote_documentation}
@sphinxdirective
OpenVINO™ Training Extensions provide a suite of advanced algorithms to train
Deep Learning models and convert them using the `OpenVINO™
toolkit <https://software.intel.com/en-us/openvino-toolkit>`__ for optimized
inference. It allows you to export and convert the models to the needed format. OpenVINO Training Extensions independently create and train the model. It is open-sourced and available on `GitHub <https://github.com/openvinotoolkit/training_extensions>`__. Read the OpenVINO Training Extensions `documentation <https://openvinotoolkit.github.io/training_extensions/stable/guide/get_started/introduction.html>`__ to learn more.
Detailed Workflow
#################
.. image:: ./_static/images/training_extensions_framework.png
1. To start working with OpenVINO Training Extensions, prepare and annotate your dataset. For example, on CVAT.
2. OpenVINO Training Extensions train the model, using training interface, and evaluate the model quality on your dataset, using evaluation and inference interfaces.
.. note::
Prepare a separate dataset or split the dataset you have for more accurate quality evaluation.
3. Having successful evaluation results received, you have an opportunity to deploy your model or continue optimizing it, using NNCF and POT. For more information about these frameworks, go to :doc:`Optimization Guide <openvino_docs_model_optimization_guide>`.
If the results are unsatisfactory, add datasets and perform the same steps, starting with dataset annotation.
OpenVINO Training Extensions Components
#######################################
- `OpenVINO Training Extensions SDK <https://github.com/openvinotoolkit/training_extensions/tree/master/ote_sdk>`__
- `OpenVINO Training Extensions CLI <https://github.com/openvinotoolkit/training_extensions/tree/master/ote_cli>`__
- `OpenVINO Training Extensions Algorithms <https://github.com/openvinotoolkit/training_extensions/tree/master/external>`__
Tutorials
#########
`Object Detection <https://github.com/openvinotoolkit/training_extensions/blob/master/ote_cli/notebooks/train.ipynb>`__
@endsphinxdirective

8
docs/Extensibility_UG/Intro.md
View File

@ -53,13 +53,13 @@ You might prefer implementing a custom operation class if you already have a gen
Mapping of custom operation is implemented differently, depending on model format used for import. You may choose one of the following:
1. If a model is represented in the ONNX (including models exported from Pytorch in ONNX) or PaddlePaddle formats, then one of the classes from [Frontend Extension API](frontend_extensions.md) should be used. It consists of several classes available in C++ which can be used with the `--extensions` option in Model Optimizer or when a model is imported directly to OpenVINO runtime using the `read_model` method. Python API is also available for runtime model import.
1. If a model is represented in the ONNX (including models exported from Pytorch in ONNX), PaddlePaddle or TensorFlow formats, then one of the classes from [Frontend Extension API](frontend_extensions.md) should be used. It consists of several classes available in C++ which can be used with the `--extensions` option in Model Optimizer or when a model is imported directly to OpenVINO runtime using the `read_model` method. Python API is also available for runtime model import.
2. If a model is represented in the TensorFlow, Caffe, Kaldi or MXNet formats, then [Model Optimizer Extensions](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md) should be used. This approach is available for model conversion in Model Optimizer only.
2. If a model is represented in the Caffe, Kaldi or MXNet formats, then [Model Optimizer Extensions](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md) should be used. This approach is available for model conversion in Model Optimizer only.
Existing of two approaches simultaneously is explained by two different types of frontends used for model conversion in OpenVINO: new frontends (ONNX, PaddlePaddle) and legacy frontends (TensorFlow, Caffe, Kaldi and Apache MXNet). Model Optimizer can use both front-ends in contrast to the direct import of model with `read_model` method which can use new frontends only. Follow one of the appropriate guides referenced above to implement mappings depending on framework frontend.
Existing of two approaches simultaneously is explained by two different types of frontends used for model conversion in OpenVINO: new frontends (ONNX, PaddlePaddle and TensorFlow) and legacy frontends (Caffe, Kaldi and Apache MXNet). Model Optimizer can use both front-ends in contrast to the direct import of model with `read_model` method which can use new frontends only. Follow one of the appropriate guides referenced above to implement mappings depending on framework frontend.
If you are implementing extensions for new ONNX or PaddlePaddle frontends and plan to use the `--extensions` option in Model Optimizer for model conversion, then the extensions should be:
If you are implementing extensions for new ONNX, PaddlePaddle or TensorFlow frontends and plan to use the `--extensions` option in Model Optimizer for model conversion, then the extensions should be:
1. Implemented in C++ only.

166
docs/Extensibility_UG/frontend_extensions.md
View File

@ -1,14 +1,18 @@
# Frontend Extensions {#openvino_docs_Extensibility_UG_Frontend_Extensions}
The goal of this chapter is to explain how to use Frontend extension classes to facilitate mapping of custom operations from framework model representation to OpenVINO representation. Refer to [Introduction to OpenVINO Extension](Intro.md) to understand entire flow.
@sphinxdirective
This API is applicable for new frontends only, which exist for ONNX and PaddlePaddle. If a different model format is used, follow legacy [Model Optimizer Extensions](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md) guide.
The goal of this chapter is to explain how to use Frontend extension classes to facilitate mapping of custom operations from framework model representation to OpenVINO representation. Refer to :doc:`Introduction to OpenVINO Extension <openvino_docs_Extensibility_UG_Intro>` to understand entire flow.
> **NOTE**: This documentation is written based on the [Template extension](https://github.com/openvinotoolkit/openvino/tree/master/src/core/template_extension/new), which demonstrates extension development details based on minimalistic `Identity` operation that is a placeholder for your real custom operation. You can review the complete code, which is fully compliable, to see how it works.
This API is applicable for new frontends only, which exist for ONNX, PaddlePaddle and TensorFlow. If a different model format is used, follow legacy :doc:`Model Optimizer Extensions <openvino_docs_MO_DG_prepare_model_customize_model_optimizer_Customize_Model_Optimizer>` guide.
## Single Operation Mapping with OpExtension
.. note::
This documentation is written based on the `Template extension <https://github.com/openvinotoolkit/openvino/tree/master/src/core/template_extension/new>`__, which demonstrates extension development details based on minimalistic ``Identity`` operation that is a placeholder for your real custom operation. You can review the complete code, which is fully compliable, to see how it works.
This section covers the case when a single operation in framework representation is mapped to a single operation in OpenVINO representation. This is called *one-to-one mapping*. There is `OpExtension` class that works well if all the following conditions are satisfied:
Single Operation Mapping with OpExtension
#########################################
This section covers the case when a single operation in framework representation is mapped to a single operation in OpenVINO representation. This is called *one-to-one mapping*. There is ``OpExtension`` class that works well if all the following conditions are satisfied:
1. Number of inputs to operation in the Framework representation is the same as in the OpenVINO representation.
@ -20,63 +24,87 @@ This section covers the case when a single operation in framework representation
5. Each attribute in OpenVINO operation can be initialized from one of the attributes of original operation or by some predefined constant value. Value of copied attributes cannot contain expressions, value is accepted as-is, so type of a value should be compatible.
> **NOTE**: `OpExtension` class is currently available for ONNX frontend only. PaddlePaddle frontend has named inputs and outputs for operation (not indexed) therefore OpExtension mapping is not applicable for this case.
.. note::
``OpExtension`` class is currently available for ONNX and TensorFlow frontends. PaddlePaddle frontend has named inputs and outputs for operation (not indexed) therefore OpExtension mapping is not applicable for this case.
The next example maps ONNX operation with type [“Identity”]( https://github.com/onnx/onnx/blob/main/docs/Operators.md#Identity) to OpenVINO template extension `Identity` class.
The next example maps ONNX operation with type `Identity <https://github.com/onnx/onnx/blob/main/docs/Operators.md#Identity>`__ to OpenVINO template extension ``Identity`` class.
@snippet ov_extensions.cpp frontend_extension_Identity_header
@snippet ov_extensions.cpp frontend_extension_Identity
.. doxygensnippet:: docs/snippets/ov_extensions.cpp
:language: cpp
:fragment: [frontend_extension_Identity_header]
.. doxygensnippet:: docs/snippets/ov_extensions.cpp
:language: cpp
:fragment: [frontend_extension_Identity]
The mapping doesn’t involve any attributes, as operation Identity doesn’t have them.
Extension objects, like just constructed `extension` can be used to add to the OpenVINO runtime just before the loading a model that contains custom operations:
Extension objects, like just constructed ``extension`` can be used to add to the OpenVINO runtime just before the loading a model that contains custom operations:
@snippet ov_extensions.cpp frontend_extension_read_model
.. doxygensnippet:: docs/snippets/ov_extensions.cpp
:language: cpp
:fragment: [frontend_extension_read_model]
Or extensions can be constructed in a separately compiled shared library. Separately compiled library can be used in Model Optimizer or `benchmark_app`. Read about how to build and load such library in chapter “Create library with extensions” in [Introduction to OpenVINO Extension](Intro.md).
Or extensions can be constructed in a separately compiled shared library. Separately compiled library can be used in Model Optimizer or ``benchmark_app``. Read about how to build and load such library in chapter “Create library with extensions” in :doc:`Introduction to OpenVINO Extension <openvino_docs_Extensibility_UG_Intro>`.
If operation have multiple inputs and/or outputs they will be mapped in order. The type of elements in input/output tensors should match expected types in the surrounding operations. For example, if custom operation produces `f32` data type then operation that consumes this output should also support `f32`. Otherwise, model conversion fails with an error, there are no automatic type conversion happens.
If operation have multiple inputs and/or outputs they will be mapped in order. The type of elements in input/output tensors should match expected types in the surrounding operations. For example, if custom operation produces ``f32`` data type then operation that consumes this output should also support ``f32``. Otherwise, model conversion fails with an error, there are no automatic type conversion happens.
### Converting to Standard OpenVINO Operation
Converting to Standard OpenVINO Operation
+++++++++++++++++++++++++++++++++++++++++
`OpExtension` class can be used when mapping to one of the operations from standard OpenVINO operation set is what you need and there is no class like `TemplateExtension::Identity` implemented.
``OpExtension`` class can be used when mapping to one of the operations from standard OpenVINO operation set is what you need and there is no class like ``TemplateExtension::Identity`` implemented.
Here is an example for a custom framework operation “MyRelu”. Suppose it is mathematically equivalent to standard `Relu` that exists in OpenVINO operation set, but for some reason has type name “MyRelu”. In this case you can directly say that “MyRelu” -> `Relu` mapping should be used:
Here is an example for a custom framework operation “MyRelu”. Suppose it is mathematically equivalent to standard `Relu` that exists in OpenVINO operation set, but for some reason has type name “MyRelu”. In this case you can directly say that “MyRelu” -> ``Relu`` mapping should be used:
@sphinxtabset
.. tab-set::
@sphinxtab{C++}
@snippet ov_extensions.cpp frontend_extension_MyRelu
@endsphinxtab
@sphinxtab{Python}
@snippet ov_extensions.py py_frontend_extension_MyRelu
@endsphinxtab
.. tab-item:: C++
:sync: cpp
@endsphinxtabset
.. doxygensnippet:: docs/snippets/ov_extensions.cpp
:language: cpp
:fragment: [frontend_extension_MyRelu]
In the resulting converted OpenVINO model, “MyRelu” operation will be replaced by the standard operation `Relu` from the latest available OpenVINO operation set. Notice that when standard operation is used, it can be specified using just a type string (“Relu”) instead of using a `ov::opset8::Relu` class name as a template parameter for `OpExtension`. This method is available for operations from the standard operation set only. For a user custom OpenVINO operation the corresponding class should be always specified as a template parameter as it was demonstrated with `TemplateExtension::Identity`.
.. tab-item:: Python
:sync: python
.. doxygensnippet:: docs/snippets/ov_extensions.py
:language: python
:fragment: [py_frontend_extension_MyRelu]
### Attributes Mapping
As described above, `OpExtension` is useful when attributes can be mapped one by one or initialized by a constant. If the set of attributes in framework representation and OpenVINO representation completely match by their names and types, nothing should be specified in OpExtension constructor parameters. The attributes are discovered and mapped automatically based on `visit_attributes` method that should be defined for any OpenVINO operation.
In the resulting converted OpenVINO model, “MyRelu” operation will be replaced by the standard operation ``Relu`` from the latest available OpenVINO operation set. Notice that when standard operation is used, it can be specified using just a type string (“Relu”) instead of using a ``ov::opset8::Relu`` class name as a template parameter for ``OpExtension``. This method is available for operations from the standard operation set only. For a user custom OpenVINO operation the corresponding class should be always specified as a template parameter as it was demonstrated with ``TemplateExtension::Identity``.
Imagine you have CustomOperation class implementation that has two attributes with names `attr1` and `attr2`:
Attributes Mapping
++++++++++++++++++
@snippet ov_extensions.cpp frontend_extension_CustomOperation
As described above, ``OpExtension`` is useful when attributes can be mapped one by one or initialized by a constant. If the set of attributes in framework representation and OpenVINO representation completely match by their names and types, nothing should be specified in OpExtension constructor parameters. The attributes are discovered and mapped automatically based on ``visit_attributes`` method that should be defined for any OpenVINO operation.
And original model in framework representation also has operation with name “CustomOperatoin” with the same `attr1` and `attr2` attributes. Then with the following code:
Imagine you have CustomOperation class implementation that has two attributes with names ``attr1`` and ``attr2``:
@snippet ov_extensions.cpp frontend_extension_CustomOperation_as_is
.. doxygensnippet:: docs/snippets/ov_extensions.cpp
:language: cpp
:fragment: [frontend_extension_CustomOperation]
both `attr1` and `attr2` are copied from framework representation to OpenVINO representation automatically. If for some reason names of attributes are different but values still can be copied “as-is” you can pass attribute names mapping in `OpExtension` constructor:
And original model in framework representation also has operation with name “CustomOperatoin” with the same ``attr1`` and ``attr2`` attributes. Then with the following code:
@snippet ov_extensions.cpp frontend_extension_CustomOperation_rename
.. doxygensnippet:: docs/snippets/ov_extensions.cpp
:language: cpp
:fragment: [frontend_extension_CustomOperation_as_is]
Where `fw_attr1` and `fw_attr2` are names for corresponding attributes in framework operation representation.
both ``attr1`` and ``attr2`` are copied from framework representation to OpenVINO representation automatically. If for some reason names of attributes are different but values still can be copied “as-is” you can pass attribute names mapping in ``OpExtension`` constructor:
If copying of an attribute is not what you need, `OpExtension` also can set attribute to predefined constant value. For the same `CustomOperation`, imagine you want to set `attr2` to value 5 instead of copying from `fw_attr2`, to achieve that do the following:
.. doxygensnippet:: docs/snippets/ov_extensions.cpp
:language: cpp
:fragment: [frontend_extension_CustomOperation_rename]
@snippet ov_extensions.cpp frontend_extension_CustomOperation_rename_set
Where ``fw_attr1`` and ``fw_attr2`` are names for corresponding attributes in framework operation representation.
If copying of an attribute is not what you need, ``OpExtension`` also can set attribute to predefined constant value. For the same ``CustomOperation``, imagine you want to set ``attr2`` to value 5 instead of copying from ``fw_attr2``, to achieve that do the following:
.. doxygensnippet:: docs/snippets/ov_extensions.cpp
:language: cpp
:fragment: [frontend_extension_CustomOperation_rename_set]
So the conclusion is that each attribute of target OpenVINO operation should be initialized either by
@ -88,46 +116,62 @@ So the conclusion is that each attribute of target OpenVINO operation should be
This is achieved by specifying maps as arguments for `OpExtension` constructor.
Mapping to Multiple Operations with ConversionExtension
#######################################################
## Mapping to Multiple Operations with ConversionExtension
Previous sections cover the case when a single operation is mapped to a single operation with optional adjustment in names and attribute values. That is likely enough for your own custom operation with existing C++ kernel implementation. In this case your framework representation and OpenVINO representation for the operation are under your control and inputs/outpus/attributes can be aligned to make ``OpExtension`` usable.
Previous sections cover the case when a single operation is mapped to a single operation with optional adjustment in names and attribute values. That is likely enough for your own custom operation with existing C++ kernel implementation. In this case your framework representation and OpenVINO representation for the operation are under your control and inputs/outpus/attributes can be aligned to make `OpExtension` usable.
In case if one-to-one mapping is not possible, *decomposition to multiple operations* should be considered. It is achieved by using more verbose and less automated ``ConversionExtension`` class. It enables writing arbitrary code to replace a single framework operation by multiple connected OpenVINO operations constructing dependency graph of any complexity.
In case if one-to-one mapping is not possible, *decomposition to multiple operations* should be considered. It is achieved by using more verbose and less automated `ConversionExtension` class. It enables writing arbitrary code to replace a single framework operation by multiple connected OpenVINO operations constructing dependency graph of any complexity.
``ConversionExtension`` maps a single operation to a function which builds a graph using OpenVINO operation classes. Follow chapter :ref:`Build a Model in OpenVINO Runtime <ov_ug_build_model>` to learn how to use OpenVINO operation classes to build a fragment of model for replacement.
`ConversionExtension` maps a single operation to a function which builds a graph using OpenVINO operation classes. Follow chapter [Build a Model in OpenVINO Runtime](@ref ov_ug_build_model) to learn how to use OpenVINO operation classes to build a fragment of model for replacement.
The next example illustrates using ``ConversionExtension`` for conversion of “ThresholdedRelu” from ONNX according to the formula: ``ThresholdedRelu(x, alpha) -> Multiply(x, Convert(Greater(x, alpha), type=float))``.
The next example illustrates using `ConversionExtension` for conversion of “ThresholdedRelu” from ONNX according to the formula: `ThresholdedRelu(x, alpha) -> Multiply(x, Convert(Greater(x, alpha), type=float))`.
.. note::
``ThresholdedRelu`` is one of the standard ONNX operators which is supported by ONNX frontend natively out-of-the-box. Here we are re-implementing it to illustrate how you can add a similar support for your custom operation instead of ``ThresholdedRelu``.
> **NOTE**: `ThresholdedRelu` is one of the standard ONNX operators which is supported by ONNX frontend natively out-of-the-box. Here we are re-implementing it to illustrate how you can add a similar support for your custom operation instead of `ThresholdedRelu`.
.. tab-set::
@sphinxtabset
.. tab-item:: C++
:sync: cpp
.. doxygensnippet:: docs/snippets/ov_extensions.cpp
:language: cpp
:fragment: [frontend_extension_ThresholdedReLU_header]
@sphinxtab{C++}
@snippet ov_extensions.cpp frontend_extension_ThresholdedReLU_header
@endsphinxtab
@sphinxtab{Python}
@snippet ov_extensions.py py_frontend_extension_ThresholdedReLU_header
@endsphinxtab
.. tab-item:: Python
:sync: python
@endsphinxtabset
.. doxygensnippet:: docs/snippets/ov_extensions.py
:language: python
:fragment: [py_frontend_extension_ThresholdedReLU_header]
@sphinxtabset
@sphinxtab{C++}
@snippet ov_extensions.cpp frontend_extension_ThresholdedReLU
@endsphinxtab
@sphinxtab{Python}
@snippet ov_extensions.py py_frontend_extension_ThresholdedReLU
@endsphinxtab
.. tab-set::
@endsphinxtabset
.. tab-item:: C++
:sync: cpp
.. doxygensnippet:: docs/snippets/ov_extensions.cpp
:language: cpp
:fragment: [frontend_extension_ThresholdedReLU]
To access original framework operation attribute value and connect to inputs, `node` object of type `NodeContext` is used. It has two main methods:
.. tab-item:: Python
:sync: python
.. doxygensnippet:: docs/snippets/ov_extensions.py
:language: python
:fragment: [py_frontend_extension_ThresholdedReLU]
* `NodeContext::get_input` to get input with a given index,
* `NodeContext::get_attribute` to get attribute value with a given name.
To access original framework operation attribute value and connect to inputs, ``node`` object of type ``NodeContext`` is used. It has two main methods:
* ``NodeContext::get_input`` to get input with a given index,
* ``NodeContext::get_attribute`` to get attribute value with a given name.
The conversion function should return a vector of node outputs that are mapped to corresponding outputs of the original framework operation in the same order.
@endsphinxdirective

16
docs/IE_PLUGIN_DG/InferRequest.md
View File

@ -12,7 +12,7 @@ Inference Engine Plugin API provides the helper InferenceEngine::IInferRequestIn
to use as a base class for a synchronous inference request implementation. Based of that, a declaration
of a synchronous request class can look as follows:
@snippet src/infer_request.hpp infer_request:header
@snippet src/sync_infer_request.hpp infer_request:header
#### Class Fields
@ -34,7 +34,7 @@ The example class has several fields:
The constructor initializes helper fields and calls methods which allocate blobs:
@snippet src/infer_request.cpp infer_request:ctor
@snippet src/sync_infer_request.cpp infer_request:ctor
> **NOTE**: Call InferenceEngine::CNNNetwork::getInputsInfo and InferenceEngine::CNNNetwork::getOutputsInfo to specify both layout and precision of blobs, which you can set with InferenceEngine::InferRequest::SetBlob and get with InferenceEngine::InferRequest::GetBlob. A plugin uses these hints to determine its internal layouts and precisions for input and output blobs if needed.
@ -42,7 +42,7 @@ The constructor initializes helper fields and calls methods which allocate blobs
Decrements a number of created inference requests:
@snippet src/infer_request.cpp infer_request:dtor
@snippet src/sync_infer_request.cpp infer_request:dtor
### `InferImpl()`
@ -50,13 +50,13 @@ Decrements a number of created inference requests:
- Checks blobs set by users
- Calls the `InferImpl` method defined in a derived class to call actual pipeline stages synchronously
@snippet src/infer_request.cpp infer_request:infer_impl
@snippet src/sync_infer_request.cpp infer_request:infer_impl
#### 1. `inferPreprocess`
Below is the code of the `inferPreprocess` method to demonstrate Inference Engine common preprocessing step handling:
@snippet src/infer_request.cpp infer_request:infer_preprocess
@snippet src/sync_infer_request.cpp infer_request:infer_preprocess
**Details:**
* `InferImpl` must call the InferenceEngine::IInferRequestInternal::execDataPreprocessing function, which executes common Inference Engine preprocessing step (for example, applies resize or color conversion operations) if it is set by the user. The output dimensions, layout and precision matches the input information set via InferenceEngine::CNNNetwork::getInputsInfo.
@ -66,18 +66,18 @@ Below is the code of the `inferPreprocess` method to demonstrate Inference Engin
Executes a pipeline synchronously using `_executable` object:
@snippet src/infer_request.cpp infer_request:start_pipeline
@snippet src/sync_infer_request.cpp infer_request:start_pipeline
#### 3. `inferPostprocess`
Converts output blobs if precisions of backend output blobs and blobs passed by user are different:
@snippet src/infer_request.cpp infer_request:infer_postprocess
@snippet src/sync_infer_request.cpp infer_request:infer_postprocess
### `GetPerformanceCounts()`
The method sets performance counters which were measured during pipeline stages execution:
@snippet src/infer_request.cpp infer_request:get_performance_counts
@snippet src/sync_infer_request.cpp infer_request:get_performance_counts
The next step in the plugin library implementation is the [Asynchronous Inference Request](@ref openvino_docs_ie_plugin_dg_async_infer_request) class.

5
docs/MO_DG/prepare_model/convert_model/supported_model_formats.md
View File

@ -20,16 +20,15 @@
**OpenVINO IR (Intermediate Representation)** - the proprietary format of OpenVINO™, benefiting from the full extent of its features.
**ONNX, PaddlePaddle** - formats supported directly, which means they can be used with OpenVINO Runtime without any prior conversion. For a guide on how to run inference on ONNX and PaddlePaddle, see how to [Integrate OpenVINO™ with Your Application](../../../OV_Runtime_UG/integrate_with_your_application.md).
**ONNX, PaddlePaddle, TensorFlow** - formats supported directly, which means they can be used with OpenVINO Runtime without any prior conversion. For a guide on how to run inference on ONNX, PaddlePaddle, or TensorFlow, see how to [Integrate OpenVINO™ with Your Application](../../../OV_Runtime_UG/integrate_with_your_application.md).
**TensorFlow, PyTorch, MXNet, Caffe, Kaldi** - formats supported indirectly, which means they need to be converted to OpenVINO IR before running inference. The conversion is done with Model Optimizer and in some cases may involve intermediate steps.
**MXNet, Caffe, Kaldi** - formats supported indirectly, which means they need to be converted to OpenVINO IR before running inference. The conversion is done with Model Optimizer and in some cases may involve intermediate steps.
Refer to the following articles for details on conversion for different formats and models:
* [How to convert ONNX](./Convert_Model_From_ONNX.md)
* [How to convert PaddlePaddle](./Convert_Model_From_Paddle.md)
* [How to convert TensorFlow](./Convert_Model_From_TensorFlow.md)
* [How to convert PyTorch](./Convert_Model_From_PyTorch.md)
* [How to convert MXNet](./Convert_Model_From_MxNet.md)
* [How to convert Caffe](./Convert_Model_From_Caffe.md)
* [How to convert Kaldi](./Convert_Model_From_Kaldi.md)

5
docs/MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md
View File

@ -10,6 +10,11 @@
@endsphinxdirective
This article describes Model Optimizer internals. Altering them may result in application instability, and in case of future changes to the API, lack of backward compatibility.
> **NOTE**: If you want to add support for ONNX, PaddlePaddle or Tensorflow operations, or you are not familiar with other extension alternatives in OpenVINO, read [this guide](../../../Extensibility_UG/Intro.md) instead.
<a name="model-optimizer-extensibility"></a>Model Optimizer extensibility mechanism enables support of new operations and custom transformations to generate the optimized intermediate representation (IR) as described in the
[Deep Learning Network Intermediate Representation and Operation Sets in OpenVINO™](../../IR_and_opsets.md). This
mechanism is a core part of Model Optimizer, as a huge set of examples showing how to add custom logic to support your model.

1
docs/OV_Runtime_UG/Operations_specifications.md
View File

@ -202,6 +202,7 @@
Tile-1 <openvino_docs_ops_movement_Tile_1>
TopK-1 <openvino_docs_ops_sort_TopK_1>
TopK-3 <openvino_docs_ops_sort_TopK_3>
TopK-11 <openvino_docs_ops_sort_TopK_11>
Transpose-1 <openvino_docs_ops_movement_Transpose_1>
Unique-10 <openvino_docs_ops_movement_Unique_10>
Unsqueeze-1 <openvino_docs_ops_shape_Unsqueeze_1>

409
docs/OV_Runtime_UG/auto_device_selection.md
View File

@ -8,11 +8,15 @@
Debugging Auto-Device Plugin <openvino_docs_OV_UG_supported_plugins_AUTO_debugging>
@endsphinxdirective
This article introduces how Automatic Device Selection works and how to use it for inference.
## <a name="how-auto-works"></a> How AUTO Works
.. _how-auto-works:
How AUTO Works
####################
The Automatic Device Selection mode, or AUTO for short, uses a "virtual" or a "proxy" device,
which does not bind to a specific type of hardware, but rather selects the processing unit for inference automatically.
@ -21,13 +25,14 @@ This way, you can write the application once and deploy it anywhere.
The selection also depends on your performance requirements, defined by the “hints” configuration API, as well as device priority list limitations, if you choose to exclude some hardware from the process.
The logic behind the choice is as follows:
1. Check what supported devices are available.
2. Check precisions of the input model (for detailed information on precisions read more on the `ov::device::capabilities`)
3. Select the highest-priority device capable of supporting the given model, as listed in the table below.
4. If model’s precision is FP32 but there is no device capable of supporting it, offload the model to a device supporting FP16.
The logic behind the choice is as follows:
1. Check what supported devices are available.
2. Check precisions of the input model (for detailed information on precisions read more on the ``ov::device::capabilities``).
3. Select the highest-priority device capable of supporting the given model, as listed in the table below.
4. If model’s precision is FP32 but there is no device capable of supporting it, offload the model to a device supporting FP16.
@sphinxdirective
+----------+------------------------------------------------------+-------------------------------------+
| Device || Supported || Supported |
| Priority || Device || model precision |
@ -41,135 +46,140 @@ The logic behind the choice is as follows:
| 3 || Intel® CPU | FP32, FP16, INT8, BIN |
| || (e.g. Intel® Core™ i7-1165G7) | |
+----------+------------------------------------------------------+-------------------------------------+
@endsphinxdirective
To put it simply, when loading the model to the first device on the list fails, AUTO will try to load it to the next device in line, until one of them succeeds.
What is important, **AUTO starts inference with the CPU of the system by default**, as it provides very low latency and can start inference with no additional delays.
To put it simply, when loading the model to the first device on the list fails, AUTO will try to load it to the next device in line, until one of them succeeds.
What is important, **AUTO starts inference with the CPU of the system by default**, as it provides very low latency and can start inference with no additional delays.
While the CPU is performing inference, AUTO continues to load the model to the device best suited for the purpose and transfers the task to it when ready.
This way, the devices which are much slower in compiling models, GPU being the best example, do not impede inference at its initial stages.
For example, if you use a CPU and a GPU, the first-inference latency of AUTO will be better than that of using GPU alone.
Note that if you choose to exclude CPU from the priority list or disable the initial CPU acceleration feature via `ov::intel_auto::enable_startup_fallback`, it will be unable to support the initial model compilation stage.
![](../img/autoplugin_accelerate.svg)
This mechanism can be easily observed in the [Using AUTO with Benchmark app sample](#using-auto-with-openvino-samples-and-benchmark-app) section, showing how the first-inference latency (the time it takes to compile the model and perform the first inference) is reduced when using AUTO. For example:
```sh
benchmark_app -m ../public/alexnet/FP32/alexnet.xml -d GPU -niter 128
```
```sh
benchmark_app -m ../public/alexnet/FP32/alexnet.xml -d AUTO -niter 128
```
Note that if you choose to exclude CPU from the priority list or disable the initial CPU acceleration feature via ``ov::intel_auto::enable_startup_fallback``, it will be unable to support the initial model compilation stage.
.. image:: _static/images/autoplugin_accelerate.svg
This mechanism can be easily observed in the :ref:`Using AUTO with Benchmark app sample <using-auto-with-openvino-samples-and-benchmark-app>` section, showing how the first-inference latency (the time it takes to compile the model and perform the first inference) is reduced when using AUTO. For example:
.. code-block: sh
benchmark_app -m ../public/alexnet/FP32/alexnet.xml -d GPU -niter 128
.. code-block: sh
benchmark_app -m ../public/alexnet/FP32/alexnet.xml -d AUTO -niter 128
@sphinxdirective
.. note::
The longer the process runs, the closer realtime performance will be to that of the best-suited device.
@endsphinxdirective
## Using AUTO
Following the OpenVINO™ naming convention, the Automatic Device Selection mode is assigned the label of “AUTO.” It may be defined with no additional parameters, resulting in defaults being used, or configured further with the following setup options:
Using AUTO
####################
@sphinxdirective
Following the OpenVINO™ naming convention, the Automatic Device Selection mode is assigned the label of "AUTO". It may be defined with no additional parameters, resulting in defaults being used, or configured further with the following setup options:
+---------------------------------------------+----------------------------------------------------------------------+
| | Property | | Values and Description |
+=============================================+======================================================================+
| | <device candidate list> | | **Values**: |
| | | | empty |
| | | | `AUTO` |
| | | | `AUTO: <device names>` (comma-separated, no spaces) |
| | | | |
| | | | Lists the devices available for selection. |
| | | | The device sequence will be taken as priority from high to low. |
| | | | If not specified, `AUTO` will be used as default, |
| | | | and all devices will be "viewed" as candidates. |
+---------------------------------------------+----------------------------------------------------------------------+
| | `ov::device::priorities` | | **Values**: |
| | | | `<device names>` (comma-separated, no spaces) |
| | | | |
| | | | Specifies the devices for AUTO to select. |
| | | | The device sequence will be taken as priority from high to low. |
| | | | This configuration is optional. |
+---------------------------------------------+----------------------------------------------------------------------+
| | `ov::hint::performance_mode` | | **Values**: |
| | | | `ov::hint::PerformanceMode::LATENCY` |
| | | | `ov::hint::PerformanceMode::THROUGHPUT` |
| | | | `ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT` |
| | | | |
| | | | Specifies the performance option preferred by the application. |
+---------------------------------------------+----------------------------------------------------------------------+
| | `ov::hint::model_priority` | | **Values**: |
| | | | `ov::hint::Priority::HIGH` |
| | | | `ov::hint::Priority::MEDIUM` |
| | | | `ov::hint::Priority::LOW` |
| | | | |
| | | | Indicates the priority for a model. |
| | | | IMPORTANT: This property is not fully supported yet. |
+---------------------------------------------+----------------------------------------------------------------------+
| | `ov::execution_devices` | | Lists the runtime target devices on which the inferences are being |
| | | | executed. |
| | | | Examples of returning results could be `(CPU)`(`(CPU)` is a |
| | | | temporary device, indicating that CPU is used for acceleration at |
| | | | the model compilation stage), `CPU`, `GPU`, `CPU GPU`, `GPU.0`, |
| | | | etc. |
+---------------------------------------------+----------------------------------------------------------------------+
| | `ov::intel_auto::enable_startup_fallback` | | **Values**: |
| | | | `true` |
| | | | `false` |
| | | | |
| | | | Enables/disables CPU as acceleration (or the helper device) in the |
| | | | beginning. The default value is `true`, indicating that CPU is used|
| | | | as acceleration by default. |
+---------------------------------------------+----------------------------------------------------------------------+
@endsphinxdirective
+-----------------------------------------------+----------------------------------------------------------------------+
| | Property | | Values and Description |
+===============================================+======================================================================+
| | <device candidate list> | | **Values**: |
| | | | empty |
| | | | ``AUTO`` |
| | | | ``AUTO: <device names>`` (comma-separated, no spaces) |
| | | | |
| | | | Lists the devices available for selection. |
| | | | The device sequence will be taken as priority from high to low. |
| | | | If not specified, ``AUTO`` will be used as default, |
| | | | and all devices will be "viewed" as candidates. |
+-----------------------------------------------+----------------------------------------------------------------------+
| | ``ov::device::priorities`` | | **Values**: |
| | | | ``<device names>`` (comma-separated, no spaces) |
| | | | |
| | | | Specifies the devices for AUTO to select. |
| | | | The device sequence will be taken as priority from high to low. |
| | | | This configuration is optional. |
+-----------------------------------------------+----------------------------------------------------------------------+
| | ``ov::hint::performance_mode`` | | **Values**: |
| | | | ``ov::hint::PerformanceMode::LATENCY`` |
| | | | ``ov::hint::PerformanceMode::THROUGHPUT`` |
| | | | ``ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT`` |
| | | | |
| | | | Specifies the performance option preferred by the application. |
+-----------------------------------------------+----------------------------------------------------------------------+
| | ``ov::hint::model_priority`` | | **Values**: |
| | | | ``ov::hint::Priority::HIGH`` |
| | | | ``ov::hint::Priority::MEDIUM`` |
| | | | ``ov::hint::Priority::LOW`` |
| | | | |
| | | | Indicates the priority for a model. |
| | | | IMPORTANT: This property is not fully supported yet. |
+-----------------------------------------------+----------------------------------------------------------------------+
| | ``ov::execution_devices`` | | Lists the runtime target devices on which the inferences are being |
| | | | executed. |
| | | | Examples of returning results could be ``(CPU)``(``CPU`` is a |
| | | | temporary device, indicating that CPU is used for acceleration at |
| | | | the model compilation stage), ``CPU``, ``GPU``, ``CPU GPU``, |
| | | | ``GPU.0``, etc. |
+-----------------------------------------------+----------------------------------------------------------------------+
| | ``ov::intel_auto::enable_startup_fallback`` | | **Values**: |
| | | | ``true`` |
| | | | ``false`` |
| | | | |
| | | | Enables/disables CPU as acceleration (or the helper device) in the |
| | | | beginning. The default value is ``true``, indicating that CPU is |
| | | | used as acceleration by default. |
+-----------------------------------------------+----------------------------------------------------------------------+
Inference with AUTO is configured similarly to when device plugins are used:
you compile the model on the plugin with configuration and execute inference.
### Device Candidates and Priority
The device candidate list enables you to customize the priority and limit the choice of devices available to AUTO.
- If <device candidate list> is not specified, AUTO assumes all the devices present in the system can be used.
- If `AUTO` without any device names is specified, AUTO assumes all the devices present in the system can be used, and will load the network to all devices and run inference based on their default priorities, from high to low.
To specify the priority of devices, enter the device names in the priority order (from high to low) in `AUTO: <device names>`, or use the `ov::device::priorities` property.
Device Candidates and Priority
++++++++++++++++++++++++++++++
See the following code for using AUTO and specifying devices:
@sphinxdirective
The device candidate list enables you to customize the priority and limit the choice of devices available to AUTO.
* If <device candidate list> is not specified, AUTO assumes all the devices present in the system can be used.
* If ``AUTO`` without any device names is specified, AUTO assumes all the devices present in the system can be used, and will load the network to all devices and run inference based on their default priorities, from high to low.
To specify the priority of devices, enter the device names in the priority order (from high to low) in ``AUTO: <device names>``, or use the ``ov::device::priorities`` property.
See the following code for using AUTO and specifying devices:
.. tab:: C++
.. doxygensnippet:: docs/snippets/AUTO0.cpp
:language: cpp
:fragment: [part0]
.. doxygensnippet:: docs/snippets/AUTO0.cpp
:language: cpp
:fragment: [part0]
.. tab:: Python
.. doxygensnippet:: docs/snippets/ov_auto.py
:language: python
:fragment: [part0]
.. doxygensnippet:: docs/snippets/ov_auto.py
:language: python
:fragment: [part0]
@endsphinxdirective
Note that OpenVINO Runtime lets you use “GPU” as an alias for “GPU.0” in function calls. More details on enumerating devices can be found in [Working with devices](supported_plugins/Device_Plugins.md).
Note that OpenVINO Runtime lets you use "GPU" as an alias for "GPU.0" in function calls. More details on enumerating devices can be found in :doc:`Working with devices <openvino_docs_OV_UG_Working_with_devices>`.
#### Checking Available Devices
Checking Available Devices
--------------------------
To check what devices are present in the system, you can use Device API, as listed below. For information on how to use it, see [Query device properties and configuration](supported_plugins/config_properties.md).
To check what devices are present in the system, you can use Device API, as listed below. For information on how to use it, see :doc:`Query device properties and configuration <openvino_docs_OV_UG_query_api>`.
@sphinxdirective
.. tab:: C++
.. tab:: C++
.. code-block:: sh
ov::runtime::Core::get_available_devices()
ov::runtime::Core::get_available_devices()
See the Hello Query Device C++ Sample for reference.
@ -181,19 +191,18 @@ To check what devices are present in the system, you can use Device API, as list
See the Hello Query Device Python Sample for reference.
@endsphinxdirective
#### Excluding Devices from Device Candidate List
Excluding Devices from Device Candidate List
--------------------------------------------
You can also exclude hardware devices from AUTO, for example, to reserve CPU for other jobs. AUTO will not use the device for inference then. To do that, add a minus sign (-) before CPU in `AUTO: <device names>`, as in the following example:
You can also exclude hardware devices from AUTO, for example, to reserve CPU for other jobs. AUTO will not use the device for inference then. To do that, add a minus sign ``(-)`` before CPU in ``AUTO: <device names>``, as in the following example:
@sphinxdirective
.. tab:: C++
.. code-block:: sh
ov::CompiledModel compiled_model = core.compile_model(model, "AUTO:-CPU");
ov::CompiledModel compiled_model = core.compile_model(model, "AUTO:-CPU");
.. tab:: Python
@ -201,144 +210,156 @@ You can also exclude hardware devices from AUTO, for example, to reserve CPU for
compiled_model = core.compile_model(model=model, device_name="AUTO:-CPU")
@endsphinxdirective
AUTO will then query all available devices and remove CPU from the candidate list.
AUTO will then query all available devices and remove CPU from the candidate list.
Note that if you choose to exclude CPU from device candidate list, CPU will not be able to support the initial model compilation stage. See more information in [How AUTO Works](#how-auto-works).
Note that if you choose to exclude CPU from device candidate list, CPU will not be able to support the initial model compilation stage. See more information in :ref:`How AUTO Works <how-auto-works>`.
### Checking Target Runtime Devices
To query the runtime target devices on which the inferences are being executed using AUTO, you can use the `ov::execution_devices` property. It must be used with `get_property`, for example:
Performance Hints for AUTO
++++++++++++++++++++++++++
@sphinxdirective
The ``ov::hint::performance_mode`` property enables you to specify a performance option for AUTO to be more efficient for particular use cases. The default hint for AUTO is ``LATENCY``.
.. tab:: C++
.. doxygensnippet:: docs/snippets/AUTO7.cpp
:language: cpp
:fragment: [part7]
LATENCY
--------------------
.. tab:: Python
.. doxygensnippet:: docs/snippets/ov_auto.py
:language: python
:fragment: [part7]
@endsphinxdirective
### Performance Hints for AUTO
The `ov::hint::performance_mode` property enables you to specify a performance option for AUTO to be more efficient for particular use cases. The default hint for AUTO is `LATENCY`.
#### LATENCY
This option prioritizes low latency, providing short response time for each inference job. It performs best for tasks where inference is required for a single input image, e.g. a medical analysis of an ultrasound scan image. It also fits the tasks of real-time or nearly real-time applications, such as an industrial robot's response to actions in its environment or obstacle avoidance for autonomous vehicles.
> **NOTE**: If no performance hint is set explicitly, AUTO will set LATENCY for devices that have not set `ov::device::properties`, for example, `ov::device::properties(<DEVICE_NAME>, ov::hint::performance_mode(ov::hint::LATENCY))`.
.. note::
If no performance hint is set explicitly, AUTO will set LATENCY for devices that have not set ``ov::device::properties``, for example, ``ov::device::properties(<DEVICE_NAME>, ov::hint::performance_mode(ov::hint::LATENCY))``.
@sphinxdirective
.. _cumulative throughput:
@endsphinxdirective
#### THROUGHPUT
THROUGHPUT
--------------------
This option prioritizes high throughput, balancing between latency and power. It is best suited for tasks involving multiple jobs, such as inference of video feeds or large numbers of images.
#### CUMULATIVE_THROUGHPUT
While `LATENCY` and `THROUGHPUT` can select one target device with your preferred performance option, the `CUMULATIVE_THROUGHPUT` option enables running inference on multiple devices for higher throughput. With `CUMULATIVE_THROUGHPUT`, AUTO loads the network model to all available devices in the candidate list, and then runs inference on them based on the default or specified priority.
CUMULATIVE_THROUGHPUT has similar behavior as [the Multi-Device execution mode (MULTI)](./multi_device.md). The only difference is that CUMULATIVE_THROUGHPUT uses the devices specified by AUTO, which means that it's not mandatory to add devices manually, while with MULTI, you need to specify the devices before inference.
CUMULATIVE_THROUGHPUT
---------------------
While ``LATENCY`` and ``THROUGHPUT`` can select one target device with your preferred performance option, the ``CUMULATIVE_THROUGHPUT`` option enables running inference on multiple devices for higher throughput. With ``CUMULATIVE_THROUGHPUT``, AUTO loads the network model to all available devices in the candidate list, and then runs inference on them based on the default or specified priority.
CUMULATIVE_THROUGHPUT has similar behavior as :doc:`the Multi-Device execution mode (MULTI) <openvino_docs_OV_UG_Running_on_multiple_devices>`. The only difference is that CUMULATIVE_THROUGHPUT uses the devices specified by AUTO, which means that it's not mandatory to add devices manually, while with MULTI, you need to specify the devices before inference.
With the CUMULATIVE_THROUGHPUT option:
- If `AUTO` without any device names is specified, and the system has more than two GPU devices, AUTO will remove CPU from the device candidate list to keep GPU running at full capacity.
- If device priority is specified, AUTO will run inference requests on devices based on the priority. In the following example, AUTO will always try to use GPU first, and then use CPU if GPU is busy:
```sh
ov::CompiledModel compiled_model = core.compile_model(model, "AUTO:GPU,CPU", ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT));
```
#### Code Examples
* If ``AUTO`` without any device names is specified, and the system has more than two GPU devices, AUTO will remove CPU from the device candidate list to keep GPU running at full capacity.
* If device priority is specified, AUTO will run inference requests on devices based on the priority. In the following example, AUTO will always try to use GPU first, and then use CPU if GPU is busy:
.. code-block: sh
ov::CompiledModel compiled_model = core.compile_model(model, "AUTO:GPU,CPU", ov::hint::performance_mode(ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT));
Code Examples
--------------------
To enable performance hints for your application, use the following code:
To enable performance hints for your application, use the following code:
@sphinxdirective
.. tab:: C++
.. doxygensnippet:: docs/snippets/AUTO3.cpp
:language: cpp
:fragment: [part3]
.. doxygensnippet:: docs/snippets/AUTO3.cpp
:language: cpp
:fragment: [part3]
.. tab:: Python
.. doxygensnippet:: docs/snippets/ov_auto.py
:language: python
:fragment: [part3]
.. doxygensnippet:: docs/snippets/ov_auto.py
:language: python
:fragment: [part3]
@endsphinxdirective
#### Disabling Auto-Batching for THROUGHPUT and CUMULATIVE_THROUGHPUT
Disabling Auto-Batching for THROUGHPUT and CUMULATIVE_THROUGHPUT
----------------------------------------------------------------
The `ov::hint::PerformanceMode::THROUGHPUT` mode and the `ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT` mode will trigger Auto-Batching (for example, for the GPU device) by default. You can disable it by setting `ov::hint::allow_auto_batching(false)`, or change the default timeout value to a large number, e.g. `ov::auto_batch_timeout(1000)`. See [Automatic Batching](./automatic_batching.md) for more details.
The ``ov::hint::PerformanceMode::THROUGHPUT`` mode and the ``ov::hint::PerformanceMode::CUMULATIVE_THROUGHPUT`` mode will trigger Auto-Batching (for example, for the GPU device) by default. You can disable it by setting ``ov::hint::allow_auto_batching(false)``, or change the default timeout value to a large number, e.g. ``ov::auto_batch_timeout(1000)``. See :doc:`Automatic Batching <openvino_docs_OV_UG_Automatic_Batching>` for more details.
### Configuring Model Priority
The `ov::hint::model_priority` property enables you to control the priorities of models in the Auto-Device plugin. A high-priority model will be loaded to a supported high-priority device. A lower-priority model will not be loaded to a device that is occupied by a higher-priority model.
Configuring Model Priority
++++++++++++++++++++++++++
The ``ov::hint::model_priority`` property enables you to control the priorities of models in the Auto-Device plugin. A high-priority model will be loaded to a supported high-priority device. A lower-priority model will not be loaded to a device that is occupied by a higher-priority model.
@sphinxdirective
.. tab:: C++
.. doxygensnippet:: docs/snippets/AUTO4.cpp
:language: cpp
:fragment: [part4]
.. doxygensnippet:: docs/snippets/AUTO4.cpp
:language: cpp
:fragment: [part4]
.. tab:: Python
.. doxygensnippet:: docs/snippets/ov_auto.py
:language: python
:fragment: [part4]
@endsphinxdirective
## Configuring Individual Devices and Creating the Auto-Device plugin on Top
Although the methods described above are currently the preferred way to execute inference with AUTO, the following steps can be also used as an alternative. It is currently available as a legacy feature and used if AUTO is uncapable of utilizing the Performance Hints option.
.. doxygensnippet:: docs/snippets/ov_auto.py
:language: python
:fragment: [part4]
@sphinxdirective
Checking Target Runtime Devices
+++++++++++++++++++++++++++++++
To query the runtime target devices on which the inferences are being executed using AUTO, you can use the ``ov::execution_devices`` property. It must be used with ``get_property``, for example:
.. tab:: C++
.. doxygensnippet:: docs/snippets/AUTO5.cpp
:language: cpp
:fragment: [part5]
.. doxygensnippet:: docs/snippets/AUTO7.cpp
:language: cpp
:fragment: [part7]
.. tab:: Python
.. doxygensnippet:: docs/snippets/ov_auto.py
:language: python
:fragment: [part5]
.. doxygensnippet:: docs/snippets/ov_auto.py
:language: python
:fragment: [part7]
@endsphinxdirective
## <a name="using-auto-with-openvino-samples-and-benchmark-app"></a> Using AUTO with OpenVINO Samples and Benchmark app
Configuring Individual Devices and Creating the Auto-Device plugin on Top
#########################################################################
Although the methods described above are currently the preferred way to execute inference with AUTO, the following steps can be also used as an alternative. It is currently available as a legacy feature and used if AUTO is incapable of utilizing the Performance Hints option.
.. tab:: C++
.. doxygensnippet:: docs/snippets/AUTO5.cpp
:language: cpp
:fragment: [part5]
.. tab:: Python
.. doxygensnippet:: docs/snippets/ov_auto.py
:language: python
:fragment: [part5]
.. _using-auto-with-openvino-samples-and-benchmark-app:
Using AUTO with OpenVINO Samples and Benchmark app
##################################################
To see how the Auto-Device plugin is used in practice and test its performance, take a look at OpenVINO™ samples. All samples supporting the "-d" command-line option (which stands for "device") will accept the plugin out-of-the-box. The Benchmark Application will be a perfect place to start – it presents the optimal performance of the plugin without the need for additional settings, like the number of requests or CPU threads. To evaluate the AUTO performance, you can use the following commands:
For unlimited device choice:
```sh
benchmark_app –d AUTO –m <model> -i <input> -niter 1000
```
.. code-block:sh
benchmark_app –d AUTO –m <model> -i <input> -niter 1000
For limited device choice:
```sh
benchmark_app –d AUTO:CPU,GPU,GNA –m <model> -i <input> -niter 1000
```
.. code-block:sh
For more information, refer to the [C++](../../samples/cpp/benchmark_app/README.md) or [Python](../../tools/benchmark_tool/README.md) version instructions.
benchmark_app –d AUTO:CPU,GPU,GNA –m <model> -i <input> -niter 1000
For more information, refer to the :doc:`C++ <openvino_inference_engine_samples_benchmark_app_README>` or :doc:`Python <openvino_inference_engine_tools_benchmark_tool_README>` version instructions.
@sphinxdirective
.. note::
The default CPU stream is 1 if using “-d AUTO”.
@ -346,11 +367,13 @@ For more information, refer to the [C++](../../samples/cpp/benchmark_app/README.
You can use the FP16 IR to work with auto-device.
No demos are yet fully optimized for AUTO, by means of selecting the most suitable device, using the GPU streams/throttling, and so on.
Additional Resources
####################
- :doc:`Debugging AUTO <openvino_docs_OV_UG_supported_plugins_AUTO_debugging>`
- :doc:`Running on Multiple Devices Simultaneously <openvino_docs_OV_UG_Running_on_multiple_devices>`
- :doc:`Supported Devices <openvino_docs_OV_UG_supported_plugins_Supported_Devices>`
@endsphinxdirective
## Additional Resources
- [Debugging AUTO](AutoPlugin_Debugging.md)
- [Running on Multiple Devices Simultaneously](./multi_device.md)
- [Supported Devices](supported_plugins/Supported_Devices.md)

3
docs/OV_Runtime_UG/deployment/deployment_intro.md
View File

@ -48,8 +48,7 @@ The granularity of OpenVINO packages may vary for different distribution types.
- The main library `openvino` is used by users' C++ applications to link against with. The library provides all OpenVINO Runtime public APIs, including both API 2.0 and the previous Inference Engine and nGraph APIs. For C language applications, `openvino_c` is additionally required for distribution.
- The "optional" plugin libraries like `openvino_intel_cpu_plugin` (matching the `openvino_.+_plugin` pattern) are used to provide inference capabilities on specific devices or additional capabilities like [Hetero Execution](../hetero_execution.md) and [Multi-Device Execution](../multi_device.md).
- The "optional" plugin libraries like `openvino_ir_frontend` (matching `openvino_.+_frontend`) are used to provide capabilities to read models of different file formats such as OpenVINO IR,
TensorFlow (check [TensorFlow Frontend Capabilities and Limitations](../../resources/tensorflow_frontend.md)), ONNX, and PaddlePaddle.
- The "optional" plugin libraries like `openvino_ir_frontend` (matching `openvino_.+_frontend`) are used to provide capabilities to read models of different file formats such as OpenVINO IR, TensorFlow, ONNX, and PaddlePaddle.
Here the term "optional" means that if the application does not use the capability enabled by the plugin, the plugin library or a package with the plugin is not needed in the final distribution.

2
docs/OV_Runtime_UG/deployment/local-distribution.md
View File

@ -113,7 +113,7 @@ The `HETERO`, `MULTI`, `BATCH` and `AUTO` execution modes can also be used expli
OpenVINO Runtime uses frontend libraries dynamically to read models in different formats:
- `openvino_ir_frontend` is used to read OpenVINO IR.
- `openvino_tensorflow_frontend` is used to read TensorFlow file format. Check [TensorFlow Frontend Capabilities and Limitations](../../resources/tensorflow_frontend.md).
- `openvino_tensorflow_frontend` is used to read TensorFlow file format.
- `openvino_onnx_frontend` is used to read ONNX file format.
- `openvino_paddle_frontend` is used to read Paddle file format.

4
docs/OV_Runtime_UG/img/BASIC_FLOW_IE_C.svg
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b08a4dce7e3e9e38107540e78ba0b584b27c65ed5dba89d37ee365210222d710
size 54897
oid sha256:ccc7704d2a27f7491729767443f3d2bdd0ccc930f16fde631a7f9c67d158297a
size 71369

129
docs/OV_Runtime_UG/integrate_with_your_application.md
View File

@ -75,105 +75,106 @@ Use the following code to create OpenVINO™ Core to manage available devices an
Compile the model for a specific device using `ov::Core::compile_model()`:
@sphinxtabset
@sphinxdirective
@sphinxtab{C++}
.. tab:: C++
@sphinxtabset
.. tab:: IR
@sphinxtab{IR}
.. doxygensnippet:: docs/snippets/src/main.cpp
:language: cpp
:fragment: [part2_1]
@snippet docs/snippets/src/main.cpp part2_1
.. tab:: ONNX
@endsphinxtab
.. doxygensnippet:: docs/snippets/src/main.cpp
:language: cpp
:fragment: [part2_2]
@sphinxtab{ONNX}
.. tab:: PaddlePaddle
@snippet docs/snippets/src/main.cpp part2_2
.. doxygensnippet:: docs/snippets/src/main.cpp
:language: cpp
:fragment: [part2_3]
@endsphinxtab
.. tab:: TensorFlow
@sphinxtab{PaddlePaddle}
.. doxygensnippet:: docs/snippets/src/main.cpp
:language: cpp
:fragment: [part2_4]
@snippet docs/snippets/src/main.cpp part2_3
.. tab:: ov::Model
@endsphinxtab
.. doxygensnippet:: docs/snippets/src/main.cpp
:language: cpp
:fragment: [part2_5]
@sphinxtab{ov::Model}
.. tab:: Python
@snippet docs/snippets/src/main.cpp part2_4
.. tab:: IR
@endsphinxtab
.. doxygensnippet:: docs/snippets/src/main.py
:language: python
:fragment: [part2_1]
@endsphinxtabset
.. tab:: ONNX
@endsphinxtab
.. doxygensnippet:: docs/snippets/src/main.py
:language: python
:fragment: [part2_2]
@sphinxtab{Python}
.. tab:: PaddlePaddle
@sphinxtabset
.. doxygensnippet:: docs/snippets/src/main.py
:language: python
:fragment: [part2_3]
@sphinxtab{IR}
.. tab:: TensorFlow
@snippet docs/snippets/src/main.py part2_1
.. doxygensnippet:: docs/snippets/src/main.py
:language: python
:fragment: [part2_4]
@endsphinxtab
.. tab:: ov::Model
@sphinxtab{ONNX}
.. doxygensnippet:: docs/snippets/src/main.py
:language: python
:fragment: [part2_5]
@snippet docs/snippets/src/main.py part2_2
.. tab:: C
@endsphinxtab
.. tab:: IR
@sphinxtab{PaddlePaddle}
.. doxygensnippet:: docs/snippets/src/main.c
:language: cpp
:fragment: [part2_1]
@snippet docs/snippets/src/main.py part2_3
.. tab:: ONNX
@endsphinxtab
.. doxygensnippet:: docs/snippets/src/main.c
:language: cpp
:fragment: [part2_2]
@sphinxtab{ov::Model}
.. tab:: PaddlePaddle
@snippet docs/snippets/src/main.py part2_4
.. doxygensnippet:: docs/snippets/src/main.c
:language: cpp
:fragment: [part2_3]
@endsphinxtab
.. tab:: TensorFlow
@endsphinxtabset
.. doxygensnippet:: docs/snippets/src/main.c
:language: cpp
:fragment: [part2_4]
@endsphinxtab
.. tab:: ov::Model
@sphinxtab{C}
.. doxygensnippet:: docs/snippets/src/main.c
:language: cpp
:fragment: [part2_5]
@sphinxtabset
@endsphinxdirective
@sphinxtab{IR}
@snippet docs/snippets/src/main.c part2_1
@endsphinxtab
@sphinxtab{ONNX}
@snippet docs/snippets/src/main.c part2_2
@endsphinxtab
@sphinxtab{PaddlePaddle}
@snippet docs/snippets/src/main.c part2_3
@endsphinxtab
@sphinxtab{ov::Model}
@snippet docs/snippets/src/main.c part2_4
@endsphinxtab
@endsphinxtabset
@endsphinxtab
@endsphinxtabset
The `ov::Model` object represents any models inside the OpenVINO™ Runtime.
For more details please read article about [OpenVINO™ Model representation](model_representation.md).

2
docs/OV_Runtime_UG/migration_ov_2_0/common_inference_pipeline.md
View File

@ -592,7 +592,7 @@ The Inference Engine API processes outputs as they are of the `I32` precision (*
API 2.0 processes outputs as they are of:
- the `I32` precision (**not** aligned with the original model) for OpenVINO IR v10 models, to match the [old behavior](@ref differences_api20_ie).
- the `I64` precision (aligned with the original model) for OpenVINO IR v11, ONNX, ov::Model and PaddlePaddle models, to match the [new behavior](@ref differences_api20_ie).
- the `I64` precision (aligned with the original model) for OpenVINO IR v11, ONNX, ov::Model, PaddlePaddle and TensorFlow models, to match the [new behavior](@ref differences_api20_ie).
@sphinxtabset

14
docs/OV_Runtime_UG/model_representation.md
View File

@ -33,7 +33,11 @@ OpenVINO™ Runtime enables you to use different approaches to work with model i
@endsphinxtabset
For details on how to build a model in OpenVINO™ Runtime, see the [Build a Model in OpenVINO™ Runtime](@ref ov_ug_build_model) section.
@sphinxdirective
For details on how to build a model in OpenVINO™ Runtime, see the :ref:`Build a Model in OpenVINO Runtime <ov_ug_build_model>` section.
@endsphinxdirective
OpenVINO™ Runtime model representation uses special classes to work with model data types and shapes. The `ov::element::Type` is used for data types. See the section below for representation of shapes.
@ -78,7 +82,13 @@ Each OpenVINO™ Release introduces new operations and adds them to new operatio
For a complete list of operation sets supported in OpenVINO™ toolkit, see the [Available Operations Sets](../ops/opset.md).
To add the support for custom operations, see [OpenVINO Extensibility Mechanism](../Extensibility_UG/Intro.md).
## Building a Model in OpenVINO™ Runtime {#ov_ug_build_model}
@sphinxdirective
.. _ov_ug_build_model:
@endsphinxdirective
## Building a Model in OpenVINO™ Runtime
You can create a model from source. This section illustrates how to construct a model composed of operations from an available operation set.

3
docs/OV_Runtime_UG/openvino_intro.md
View File

@ -17,8 +17,7 @@
@endsphinxdirective
OpenVINO Runtime is a set of C++ libraries with C and Python bindings providing a common API to deliver inference solutions on the platform of your choice. Use the OpenVINO Runtime API to read an Intermediate Representation (IR),
TensorFlow (check [TensorFlow Frontend Capabilities and Limitations](../resources/tensorflow_frontend.md)), ONNX, or PaddlePaddle model and execute it on preferred devices.
OpenVINO Runtime is a set of C++ libraries with C and Python bindings providing a common API to deliver inference solutions on the platform of your choice. Use the OpenVINO Runtime API to read an Intermediate Representation (IR), TensorFlow, ONNX, or PaddlePaddle model and execute it on preferred devices.
OpenVINO Runtime uses a plugin architecture. Its plugins are software components that contain complete implementation for inference on a particular Intel® hardware device: CPU, GPU, GNA, etc. Each plugin implements the unified API and provides additional hardware-specific APIs for configuring devices or API interoperability between OpenVINO Runtime and underlying plugin backend.

2
docs/OV_Runtime_UG/preprocessing_usecase_save.md
View File

@ -10,7 +10,7 @@ Most available preprocessing steps can also be performed via command-line option
## Code example - Saving Model with Preprocessing to OpenVINO IR
When some preprocessing steps cannot be integrated into the execution graph using Model Optimizer command-line options (for example, `YUV`->`RGB` color space conversion, `Resize`, etc.), it is possible to write a simple code which:
- Reads the original model (OpenVINO IR, TensorFlow (check [TensorFlow Frontend Capabilities and Limitations](../resources/tensorflow_frontend.md)), ONNX, PaddlePaddle).
- Reads the original model (OpenVINO IR, TensorFlow, ONNX, PaddlePaddle).
- Adds the preprocessing/postprocessing steps.
- Saves resulting model as IR (`.xml` and `.bin`).

5
docs/OV_Runtime_UG/protecting_model_guide.md
View File

@ -16,9 +16,8 @@ This guide presents how to use OpenVINO securely with protected models.
After a model is optimized by the OpenVINO Model Optimizer, it's deployed
to target devices in the OpenVINO Intermediate Representation (OpenVINO IR) format. An optimized
model is stored on edge device and is executed by the OpenVINO Runtime.
TensorFlow (check [TensorFlow Frontend Capabilities and Limitations](../resources/tensorflow_frontend.md)), ONNX
and PaddlePaddle models can be read natively by OpenVINO Runtime as well.
model is stored on edge device and is executed by the OpenVINO Runtime.
TensorFlow, ONNX and PaddlePaddle models can be read natively by OpenVINO Runtime as well.
Encrypting and optimizing model before deploying it to the edge device can be
used to protect deep-learning models. The edge device should keep the stored model

4
docs/OV_Runtime_UG/supported_plugins/GPU.md
View File

@ -222,11 +222,11 @@ The GPU plugin has the following additional preprocessing options:
@sphinxtabset
@sphinxtab{C++}
@snippet docs/snippets/gpu/preprocessing.cpp init_preproc
@snippet docs/snippets/gpu/preprocessing_nv12_two_planes.cpp init_preproc
@endsphinxtab
@sphinxtab{Python}
@snippet docs/snippets/gpu/preprocessing.py init_preproc
@snippet docs/snippets/gpu/preprocessing_nv12_two_planes.py init_preproc
@endsphinxtab
@endsphinxtabset

371
docs/OV_Runtime_UG/supported_plugins/GPU_RemoteTensor_API.md
View File

@ -3,8 +3,11 @@
The GPU plugin implementation of the `ov::RemoteContext` and `ov::RemoteTensor` interfaces supports GPU
pipeline developers who need video memory sharing and interoperability with existing native APIs,
such as OpenCL, Microsoft DirectX, or VAAPI.
Using these interfaces allows you to avoid any memory copy overhead when plugging OpenVINO™ inference
into an existing GPU pipeline. It also enables OpenCL kernels to participate in the pipeline to become
The `ov::RemoteContext` and `ov::RemoteTensor` interface implementation targets the need for memory sharing and
interoperability with existing native APIs, such as OpenCL, Microsoft DirectX, and VAAPI.
They allow you to avoid any memory copy overhead when plugging OpenVINO™ inference
into an existing GPU pipeline. They also enable OpenCL kernels to participate in the pipeline to become
native buffer consumers or producers of the OpenVINO™ inference.
There are two interoperability scenarios supported by the Remote Tensor API:
@ -23,7 +26,7 @@ and functions that consume or produce native handles directly.
## Context Sharing Between Application and GPU Plugin
GPU plugin classes that implement the `ov::RemoteContext` interface are responsible for context sharing.
Obtaining a context object is the first step of sharing pipeline objects.
Obtaining a context object is the first step in sharing pipeline objects.
The context object of the GPU plugin directly wraps OpenCL context, setting a scope for sharing the
`ov::CompiledModel` and `ov::RemoteTensor` objects. The `ov::RemoteContext` object can be either created on top of
an existing handle from a native API or retrieved from the GPU plugin.
@ -37,60 +40,49 @@ additional parameter.
To create the `ov::RemoteContext` object for user context, explicitly provide the context to the plugin using constructor for one
of `ov::RemoteContext` derived classes.
@sphinxtabset
@sphinxdirective
@sphinxtab{Linux}
.. tab:: Linux
@sphinxtabset
.. tab:: Create from cl_context
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: context_from_cl_context
@sphinxtab{Create from cl_context}
.. tab:: Create from cl_queue
@snippet docs/snippets/gpu/remote_objects_creation.cpp context_from_cl_context
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: context_from_cl_queue
@endsphinxtab
.. tab:: Create from VADisplay
@sphinxtab{Create from cl_queue}
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: context_from_va_display
@snippet docs/snippets/gpu/remote_objects_creation.cpp context_from_cl_queue
.. tab:: Windows
@endsphinxtab
.. tab:: Create from cl_context
@sphinxtab{Create from VADisplay}
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: context_from_cl_context
@snippet docs/snippets/gpu/remote_objects_creation.cpp context_from_va_display
.. tab:: Create from cl_queue
@endsphinxtab
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: context_from_cl_queue
@endsphinxtabset
@endsphinxtab
@sphinxtab{Windows}
@sphinxtabset
@sphinxtab{Create from cl_context}
@snippet docs/snippets/gpu/remote_objects_creation.cpp context_from_cl_context
@endsphinxtab
@sphinxtab{Create from cl_queue}
@snippet docs/snippets/gpu/remote_objects_creation.cpp context_from_cl_queue
@endsphinxtab
@sphinxtab{Create from ID3D11Device}
@snippet docs/snippets/gpu/remote_objects_creation.cpp context_from_d3d_device
@endsphinxtab
@endsphinxtabset
@endsphinxtabset
.. tab:: Create from ID3D11Device
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: context_from_d3d_device
@endsphinxdirective
### Getting RemoteContext from the Plugin
If you do not provide any user context, the plugin uses its default internal context.
@ -100,21 +92,21 @@ Once the plugin options have been changed, the internal context is replaced by t
To request the current default context of the plugin, use one of the following methods:
@sphinxtabset
@sphinxdirective
@sphinxtab{Get context from Core}
.. tab:: Get context from Core
@snippet docs/snippets/gpu/remote_objects_creation.cpp default_context_from_core
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: default_context_from_core
@endsphinxtab
.. tab:: Get context from compiled model
@sphinxtab{Batching via throughput hint}
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: default_context_from_model
@snippet docs/snippets/gpu/remote_objects_creation.cpp default_context_from_model
@endsphinxtab
@endsphinxtabset
@endsphinxdirective
## Memory Sharing Between Application and GPU Plugin
@ -126,108 +118,153 @@ of the `ov::RemoteContext` sub-classes.
`ov::intel_gpu::ocl::ClContext` has multiple overloads of `create_tensor` methods which allow to wrap pre-allocated native handles with the `ov::RemoteTensor`
object or request plugin to allocate specific device memory. For more details, see the code snippets below:
@sphinxtabset
@sphinxdirective
@sphinxtab{Wrap native handles}
.. tab:: Wrap native handles
@sphinxtabset
.. tab:: USM pointer
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: wrap_usm_pointer
@sphinxtab{USM pointer}
.. tab:: cl_mem
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: wrap_cl_mem
@snippet docs/snippets/gpu/remote_objects_creation.cpp wrap_usm_pointer
.. tab:: cl::Buffer
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: wrap_cl_buffer
@endsphinxtab
.. tab:: cl::Image2D
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: wrap_cl_image
@sphinxtab{cl_mem}
.. tab:: biplanar NV12 surface
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: wrap_nv12_surface
@snippet docs/snippets/gpu/remote_objects_creation.cpp wrap_cl_mem
.. tab:: Allocate device memory
@endsphinxtab
.. tab:: USM host memory
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: allocate_usm_host
@sphinxtab{cl::Buffer}
.. tab:: USM device memory
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: allocate_usm_device
@snippet docs/snippets/gpu/remote_objects_creation.cpp wrap_cl_buffer
.. tab:: cl::Buffer
.. doxygensnippet:: docs/snippets/gpu/remote_objects_creation.cpp
:language: cpp
:fragment: allocate_cl_buffer
@endsphinxtab
@sphinxtab{cl::Image2D}
@snippet docs/snippets/gpu/remote_objects_creation.cpp wrap_cl_image
@endsphinxtab
@sphinxtab{biplanar NV12 surface}
@snippet docs/snippets/gpu/remote_objects_creation.cpp wrap_nv12_surface
@endsphinxtab
@endsphinxtabset
@endsphinxtab
@sphinxtab{Allocate device memory}
@sphinxtabset
@sphinxtab{USM host memory}
@snippet docs/snippets/gpu/remote_objects_creation.cpp allocate_usm_host
@endsphinxtab
@sphinxtab{USM device memory}
@snippet docs/snippets/gpu/remote_objects_creation.cpp allocate_usm_device
@endsphinxtab
@sphinxtab{cl::Buffer}
@snippet docs/snippets/gpu/remote_objects_creation.cpp allocate_cl_buffer
@endsphinxtab
@endsphinxtabset
@endsphinxtab
@endsphinxtabset
@endsphinxdirective
The `ov::intel_gpu::ocl::D3DContext` and `ov::intel_gpu::ocl::VAContext` classes are derived from `ov::intel_gpu::ocl::ClContext`.
Therefore, they provide the functionality described above and extend it
to allow creation of `ov::RemoteTensor` objects from `ID3D11Buffer`, `ID3D11Texture2D` pointers or the `VASurfaceID` handle respectively.
## Direct NV12 Video Surface Input
To support the direct consumption of a hardware video decoder output, the plugin accepts two-plane video
surfaces as arguments for the `create_tensor_nv12()` function, which creates a pair of `ov::RemoteTensor`
objects which represent the Y and UV planes.
To support the direct consumption of a hardware video decoder output, the GPU plugin accepts:
To ensure that the plugin generates the correct execution graph for the NV12 dual-plane input, static preprocessing
* Two-plane NV12 video surface input - calling the `create_tensor_nv12()` function creates
a pair of `ov::RemoteTensor` objects, representing the Y and UV planes.
* Single-plane NV12 video surface input - calling the `create_tensor()` function creates one
`ov::RemoteTensor` object, representing the Y and UV planes at once (Y elements before UV elements).
* NV12 to Grey video surface input conversion - calling the `create_tensor()` function creates one
`ov::RemoteTensor` object, representing only the Y plane.
To ensure that the plugin generates a correct execution graph, static preprocessing
should be added before model compilation:
@snippet snippets/gpu/preprocessing.cpp init_preproc
@sphinxdirective
Since the `ov::intel_gpu::ocl::ClImage2DTensor` and its derived classes do not support batched surfaces, if batching and surface sharing are required
at the same time, inputs need to be set via the `ov::InferRequest::set_tensors` method with vector of shared surfaces for each plane:
.. tab:: two-plane
@sphinxtabset
.. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_two_planes.cpp
:language: cpp
:fragment: [init_preproc]
@sphinxtab{Single batch}
.. tab:: single-plane
@snippet docs/snippets/gpu/preprocessing.cpp single_batch
.. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_single_plane.cpp
:language: cpp
:fragment: [init_preproc]
@endsphinxtab
.. tab:: NV12 to Grey
@sphinxtab{Multiple batches}
.. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_to_gray.cpp
:language: cpp
:fragment: [init_preproc]
@snippet docs/snippets/gpu/preprocessing.cpp batched_case
@endsphinxdirective
@endsphinxtab
@endsphinxtabset
Since the `ov::intel_gpu::ocl::ClImage2DTensor` and its derived classes do not support batched surfaces,
if batching and surface sharing are required at the same time,
inputs need to be set via the `ov::InferRequest::set_tensors` method with vector of shared surfaces for each plane:
@sphinxdirective
.. tab:: Single Batch
.. tab:: two-plane
.. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_two_planes.cpp
:language: cpp
:fragment: single_batch
.. tab:: single-plane
.. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_single_plane.cpp
:language: cpp
:fragment: single_batch
.. tab:: NV12 to Grey
.. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_to_gray.cpp
:language: cpp
:fragment: single_batch
.. tab:: Multiple Batches
.. tab:: two-plane
.. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_two_planes.cpp
:language: cpp
:fragment: batched_case
.. tab:: single-plane
.. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_single_plane.cpp
:language: cpp
:fragment: batched_case
.. tab:: NV12 to Grey
.. doxygensnippet:: docs/snippets/gpu/preprocessing_nv12_to_gray.cpp
:language: cpp
:fragment: batched_case
@endsphinxdirective
I420 color format can be processed in a similar way
## Context & Queue Sharing
@ -242,18 +279,12 @@ This sharing mechanism allows performing pipeline synchronization on the app sid
on waiting for the completion of inference. The pseudo-code may look as follows:
@sphinxdirective
.. raw:: html
<div class="collapsible-section" data-title="Queue and context sharing example">
.. dropdown:: Queue and context sharing example
@endsphinxdirective
@snippet snippets/gpu/queue_sharing.cpp queue_sharing
@sphinxdirective
.. raw:: html
</div>
.. doxygensnippet:: docs/snippets/gpu/queue_sharing.cpp
:language: cpp
:fragment: queue_sharing
@endsphinxdirective
@ -282,60 +313,34 @@ For possible low-level properties and their description, refer to the `openvino/
To see pseudo-code of usage examples, refer to the sections below.
> **NOTE**: For low-level parameter usage examples, see the source code of user-side wrappers from the include files mentioned above.
@sphinxdirective
.. raw:: html
<div class="collapsible-section" data-title="OpenCL Kernel Execution on a Shared Buffer">
.. NOTE::
For low-level parameter usage examples, see the source code of user-side wrappers from the include files mentioned above.
.. dropdown:: OpenCL Kernel Execution on a Shared Buffer
This example uses the OpenCL context obtained from a compiled model object.
.. doxygensnippet:: docs/snippets/gpu/context_sharing.cpp
:language: cpp
:fragment: context_sharing_get_from_ov
.. dropdown:: Running GPU Plugin Inference within User-Supplied Shared Context
.. doxygensnippet:: docs/snippets/gpu/context_sharing.cpp
:language: cpp
:fragment: context_sharing_user_handle
.. dropdown:: Direct Consuming of the NV12 VAAPI Video Decoder Surface on Linux
.. doxygensnippet:: docs/snippets/gpu/context_sharing_va.cpp
:language: cpp
:fragment: context_sharing_va
@endsphinxdirective
This example uses the OpenCL context obtained from a compiled model object.
@snippet snippets/gpu/context_sharing.cpp context_sharing_get_from_ov
@sphinxdirective
.. raw:: html
</div>
@endsphinxdirective
@sphinxdirective
.. raw:: html
<div class="collapsible-section" data-title="Running GPU Plugin Inference within User-Supplied Shared Context">
@endsphinxdirective
@snippet snippets/gpu/context_sharing.cpp context_sharing_user_handle
@sphinxdirective
.. raw:: html
</div>
@endsphinxdirective
@sphinxdirective
.. raw:: html
<div class="collapsible-section" data-title="Direct Consuming of the NV12 VAAPI Video Decoder Surface on Linux">
@endsphinxdirective
@snippet snippets/gpu/context_sharing_va.cpp context_sharing_va
@sphinxdirective
.. raw:: html
</div>
@endsphinxdirective
## See Also

319
docs/_static/css/custom.css vendored
View File

@ -1,38 +1,46 @@
/* misc */
/* Misc */
/* =================================================== */
.switcher-set {
margin-bottom:1rem;
}
main img {
cursor: pointer;
}
.doxyrest-title-code-block {
margin-bottom: 0;
}
main .searchForm {
margin-bottom: 2rem;
margin-top: 2rem;
}
pre {
white-space: pre-wrap;
word-wrap: break-word;
}
/* navigation panels override */
/* Navigation panels override */
/* =================================================== */
/* hide home item in the top bar */
/* Hide home item in the top bar */
ul#navbar-main-elements li:first-of-type {
display: none;
}
/* items on hover */
#bd-docs-nav div ul a:hover {
text-decoration: underline;
}
ul#navbar-main-elements > li:hover {
text-decoration: underline;
}
/* first-level items in the side menu */
/* First-level items in the side menu */
#bd-docs-nav > div > ul > li {
padding-bottom: 15px;
}
@ -40,69 +48,82 @@ ul#navbar-main-elements > li:hover {
color: #000000;
font-weight: bold;
}
/* second level items */
/* Second level items */
#bd-docs-nav > div > ul > li > ul {
padding-left: 0.3rem;
}
/* overwrite menu chevron directions for open and closed states */
/* Overwrite menu chevron directions for open and closed states */
.toctree-checkbox~label i {
transform: rotate(270deg);
}
.toctree-checkbox:checked~label i {
transform: rotate(0deg);
}
/* footer links */
/* Footer links */
/* =================================================== */
footer div.container div.footer-item p a {
float: left;
margin-right: 30px;
}
footer div.container div.footer-item p a:nth-child(1) {
margin-right: 50px;
}
footer div.container div.footer-item p:nth-child(2) {
clear: both;
}
/* doc version dropdown formatting override */
/* Doc version dropdown formatting override */
/* =================================================== */
[aria-labelledby="version-selector"] {
min-width: 125px!important;
overflow-x: hidden!important;
}
.sst-dropdown #version-selector {
min-width: 125px!important;
}
[aria-labelledby="version-selector"] .dropdown-item {
padding: 0.25rem 0.5rem!important;
}
/* Content in two columns */
/* =================================================== */
.row-two-col-content {
display: flex;
}
.column-two-col-content {
flex: 50%;
padding-right: 10px!important;
}
/* code reference text formatting override */
/* Code reference text formatting override */
/* =================================================== */
code {
color: black !important;
font-weight: bold;
}
/* Table Sort Button */
/* =================================================== */
.sort-header {
cursor: pointer;
}
.sort-btn {
content: "";
background-image:url('media/arrow-small-opposite-v.svg');
@ -115,23 +136,29 @@ code {
position:relative;
top:0.5rem;
}
.sort-btn.sort-active.ascending,
.sort-btn.sort-active {
background-size: 100% 70%;
}
.sort-btn.sort-active.ascending {
background-image: url('media/union-down.svg');
}
.sort-btn.sort-active {
background-image: url('media/union-up.svg');
}
div.highlight {
margin-bottom: 1.15rem;
}
.highlight .err {
border:none;
color:inherit;
}
.opt-notice-wrapper {
position: fixed;
bottom:0;
@ -141,6 +168,7 @@ div.highlight {
padding: 1rem;
z-index: 1000;
}
.opt-notice {
margin-bottom: 0;
position: absolute;
@ -151,6 +179,7 @@ div.highlight {
color: #fff;
}
/* Transition banner */
/* =================================================== */
.transition-banner {
@ -190,11 +219,13 @@ div.highlight {
text-shadow: 0 1px 0 #fff;
opacity: .5;
}
.hidden-banner {
display: none!important;
}
/* responsiveness */
/* Responsiveness */
/* =================================================== */
@media (max-width: 720px) {
.transition-banner {
@ -217,19 +248,29 @@ div.highlight {
/* =================================================== */
.configure-graphs-header {
padding-left: 16px;
display: flex;
justify-content: space-between;
}
.configure-graphs-header h3 {
float: left;
}
.configure-graphs-content {
overflow: auto;
}
.header-inactive {
color: lightgray;
}
.configure-graphs-btn {
padding: 4px 20px;
background-color: #0054AE;
border-color: #0054AE;
color: #fefefe;
}
.graph-chart-title-header {
font-size: 1.4rem;
line-height: 2rem;
@ -237,6 +278,7 @@ div.highlight {
padding: 12px 0;
margin: 0;
}
.empty-chart-container {
height: 80px;
line-height: 80px;
@ -245,41 +287,50 @@ div.highlight {
background-color: #f3f3f3;
border-radius: 5px;
}
.graph-chart-title {
vertical-align: middle;
padding: 12px 0;
}
.chart-column-header-container {
padding-top: 8px;
display: flex;
flex-direction: row;
width: 100%;
}
.chart-column-title {
min-width: 20%;
flex-grow: 0 1;
}
.chart-column-title .icon {
margin-top: 6px;
margin-right: 8px;
flex-grow: 0;
float: left;
}
.chart-column-title .chart-header {
flex-grow: 1;
float: left;
}
.chart-column-title .title {
font-size: 1rem;
font-weight: 400;
}
.chart-column-title .subtitle {
font-size: .8rem;
color: gray;
}
.chart-labels-container {
width: 18%;
}
.chart-labels-container .title {
text-align: right;
text-overflow: ellipsis;
@ -290,18 +341,19 @@ div.highlight {
line-height: 3.42rem;
color: gray;
}
.chevron-right-btn {
content: url('media/chevron-right.svg');
vertical-align: middle;
padding-left: 8px;
}
.chevron-down-btn {
content: url('media/chevron-down.svg');
vertical-align: middle;
padding-left: 8px;
}
.chart {
height: 500px;
padding:0;
@ -320,7 +372,7 @@ div.highlight {
.build-benchmark-section .title {
flex-grow: 1;
}
}
.build-benchmark-section h3 {
margin-top: 1rem;
@ -357,63 +409,21 @@ div.highlight {
.efficiency-icon {
content: url('media/icon-efficiency.svg');
}
.latency-icon {
content: url('media/icon-latency.svg');
}
.throughput-icon {
content: url('media/icon-throughput.svg');
}
.value-icon {
content: url('media/icon-value.svg');
}
/* Modal */
/* The Close Button */
.modal-close {
color: #aaaaaa;
float: right;
font-size: 28px;
line-height: 24px;
padding-right: 4px;
}
.modal-close:hover,
.modal-close:focus {
color: #000;
text-decoration: none;
cursor: pointer;
}
.clear-all-btn {
float: right;
cursor: pointer;
line-height: 4rem;
}
.clear-all-btn-content {
border: 1.5px solid black;
padding: 6px 10px;
}
.edit-settings-btn {
float: right;
color: #0054AE;
font-size: 1.05rem;
cursor: pointer;
line-height: 4rem;
display: none;
}
.edit-settings-text {
vertical-align: middle;
}
.edit-settings-icon {
vertical-align: middle;
content: url('media/edit-settings.svg');
}
.modal {
display: block;
position: fixed;
@ -426,10 +436,6 @@ div.highlight {
background-color: rgba(0, 0, 0, 0.4);
}
.modal .models-column-one label {
word-break: break-word;
}
.modal-content {
overflow: auto;
background-color: #fefefe;
@ -437,7 +443,8 @@ div.highlight {
padding: 36px;
border: 1px solid #888;
width: 95%;
max-height: 90%;
max-width: 1140px;
max-height: 85%;
}
.modal-content h2 {
@ -454,21 +461,32 @@ div.highlight {
padding-bottom: 1px;
}
.modal-header-grid-container {
display: grid;
padding: 12px 64px 2px 16px;
grid-template-columns: 40% 20% 20% 10% 10%;
column-gap: 16px;
.modal-header {
display: flex;
justify-content: space-between;
border: 0;
padding: 0;
}
.modal-configure-graphs,
.modal-display-graphs {
display: flex;
flex-direction: column;
min-height: 0;
}
.modal-content-grid-container {
display: grid;
padding-left: 24px;
padding-right: 64px;
padding-top: 8px;
grid-template-columns: 20% 20% 20% 20% 10% 10%;
column-gap: 12px;
font-size: .78rem;
padding: 0.75rem 4rem 0.125rem 1rem;
grid-template-columns: repeat(3, 1fr);
grid-template-rows: auto;
gap: 2rem 1rem;
}
.modal-content-grid {
display: grid;
padding-top: .5rem;
grid-template-columns: 1fr;
}
.modal-content-grid-container .column {
@ -488,29 +506,11 @@ div.highlight {
margin-left: -14px;
}
.modal-header-grid-item h5 {
.modal-content-grid-item h5 {
font-weight: 530;
margin: 0;
}
.modal-grid-item h5 {
margin: 0;
}
.modal-build-graphs-btn {
margin-bottom: 10px;
margin-right: 3px;
padding: 4px 16px;
float: right;
border-color: #0054AE;
background-color: #0054AE;
color: #fff;
}
.modal-build-graphs-btn:disabled {
border-color: #8C8C8C;
background-color: lightgray;
}
.modal-footer {
display: none;
padding: 0;
@ -521,7 +521,25 @@ div.highlight {
left: 0;
}
.modal-footer-content {
display: flex;
justify-content: space-between;
}
.modal-disclaimer-box {
padding-right: 0.5rem;
}
.modal-disclaimer-box p {
color: #00000098;
font-size: 0.8rem;
margin-bottom: 0rem;
}
.benchmark-graph-results-header {
display: flex;
justify-content: space-between;
align-items: center;
padding-left: 16px;
}
@ -539,16 +557,118 @@ div.highlight {
width: 20%;
}
@media screen and (max-width:768px) {
.modal-content-grid-container {
grid-template-columns: repeat(2, 1fr);
grid-template-rows: auto;
padding-right: 1rem;
}
}
@media screen and (max-width: 530px) {
.buttons-nav {
margin-top: 0.125rem;
margin-bottom: 0.125rem;
flex-direction: column;
gap: .5rem;
}
.clear-all-btn {
padding: 0;
}
.modal-content-grid-container {
grid-template-columns: 1fr;
grid-template-rows: auto;
padding-right: 1rem;
}
}
@media screen and (min-width: 530px) {
.modal-content-grid--cols-2 {
display: grid;
padding-top: .5rem;
grid-template-columns: 1fr 1fr;
column-gap: 1rem;
}
.span-element-big {
grid-column: 1 / span 2;
}
}
/* Modal buttons */
.modal-close {
color: #aaaaaa;
float: right;
font-size: 28px;
line-height: 24px;
padding-right: 4px;
}
.modal-close:hover,
.modal-close:focus {
color: #000;
text-decoration: none;
cursor: pointer;
}
.buttons-nav {
display: flex;
justify-content: center;
align-items: center;
gap: 1rem;
}
.build-graphs-btn {
border-color: #0054AE;
background-color: #0054AE;
color: #fff;
}
.build-graphs-btn:disabled {
border-color: #8C8C8C;
background-color: lightgray;
}
.clear-all-btn {
cursor: pointer;
}
.clear-all-btn-content {
border: 1.5px solid black;
padding: 6px 10px;
}
.edit-settings-btn {
color: #0054AE;
font-size: 1.05rem;
cursor: pointer;
line-height: 4rem;
}
.edit-settings-text {
vertical-align: middle;
}
.edit-settings-icon {
vertical-align: middle;
content: url('media/edit-settings.svg');
}
.close-btn {
padding: 4px 16px;
border-color: #0054AE;
background-color: #0054AE;
color: #fefefe;
float: right;
align-self: flex-start;
}
/* content formatting for the benchmark pages */
/* Content formatting for the benchmark pages */
.picker-options {
margin: 15px 0;
}
@ -639,7 +759,7 @@ div.highlight {
/* Create a custom checkbox */
.checkmark {
position: absolute;
top: 2px;
top: 5px;
left: 0;
height: 15px;
width: 15px;
@ -660,6 +780,11 @@ div.highlight {
background-color: #0054AE;
}
.checkmark-container input:disabled ~ .checkmark {
background: #d3d3d3;
border: 2px solid #8C8C8C;
}
/* Create the checkmark/indicator (hidden when not checked) */
.checkmark:after {
content: "";

BIN
docs/_static/download/OV_2023_models_supported.pdf vendored Normal file
View File

Binary file not shown.

116
docs/_static/html/modal.html vendored
View File

@ -1,64 +1,78 @@
<div>
<div style="width: 100%">
<span style="float: left"><h2>Benchmark Graph Builder</h2></span>
<div class="modal-header">
<h2>Benchmark Graph Builder</h2>
<span class="modal-close">&times;</span>
</div>
<div class="modal-line-divider"></div>
<div id="configure-graphs-header" class="configure-graphs-header">
<div class="edit-settings-btn">
<div class="edit-settings-text">Edit Settings
<span class="edit-settings-icon"></span>
<section id="modal-configure-graphs" class="modal-configure-graphs">
<div id="configure-graphs-header" class="configure-graphs-header">
<h3>Configure Graphs</h3>
<div class="buttons-nav">
<div>
<button id="build-graphs-btn" disabled="disabled" class="build-graphs-btn">Build Graphs</button>
</div>
<div class="clear-all-btn">
<span class="clear-all-btn-content">Clear All</span>
</div>
</div>
</div>
<span class="clear-all-btn">
<span class="clear-all-btn-content">Clear All</span>
</span>
<h3>STEP 1: Configure Graphs</h3>
</div>
<div class="configure-graphs-content">
<div class="modal-header-grid-container">
<div class="modal-header-grid-item">
<h5>Models</h5>
<div class="modal-line-divider"></div>
</div>
<div class="modal-header-grid-item">
<h5>Platform Type</h5>
<div class="modal-line-divider"></div>
</div>
<div class="modal-header-grid-item">
<h5>Platforms</h5>
<div class="modal-line-divider"></div>
</div>
<div class="modal-header-grid-item">
<h5>Parameters</h5>
<div class="modal-line-divider"></div>
</div>
<div class="modal-header-grid-item">
<h5>Precision</h5>
<div class="modal-line-divider"></div>
<div class="configure-graphs-content">
<div class="modal-content-grid-container">
<div class="modal-content-grid-item span-element-big">
<h5>Models</h5>
<div class="modal-line-divider"></div>
<div class="modal-content-grid modal-content-grid--cols-2">
<div class="models-column-one column"></div>
<div class="models-column-two column"></div>
</div>
</div>
<div class="modal-content-grid-item">
<h5>Platform Type</h5>
<div class="modal-line-divider"></div>
<div class="modal-content-grid">
<div class="ietype-column column"></div>
</div>
</div>
<div class="modal-content-grid-item">
<h5>Platforms</h5>
<div class="modal-line-divider"></div>
<div class="modal-content-grid">
<div class="client-platform-column column"></div>
</div>
</div>
<div class="modal-content-grid-item">
<h5>Parameters</h5>
<div class="modal-line-divider"></div>
<div class="modal-content-grid">
<div class="kpi-column column"></div>
</div>
</div>
<div class="modal-content-grid-item">
<h5>Precision</h5>
<div class="modal-line-divider"></div>
<div class="modal-content-grid">
<div class="precisions-column column"></div>
</div>
</div>
</div>
</div>
<div class="modal-content-grid-container">
<div class="models-column-one column"></div>
<div class="models-column-two column"></div>
<div class="ietype-column column"></div>
<div class="client-platform-column column">
</div>
<div class="kpi-column column"></div>
<div class="precisions-column column">
</section>
<section id="modal-display-graphs" class="modal-display-graphs">
<div class="benchmark-graph-results-header">
<h3>Graph Results</h3>
<div class="edit-settings-btn">
<div class="edit-settings-text">Edit Settings
<span class="edit-settings-icon"></span>
</div>
</div>
</div>
<div class="modal-content-footer">
<button id="modal-build-graphs-btn" disabled="disabled" class="modal-build-graphs-btn">Build Graphs</button>
</div>
</div>
<div class="modal-line-divider"></div>
<div class="benchmark-graph-results-header">
<h3 class="header-inactive">STEP 2: Benchmark Graph Results</h3>
</div>
<div class="chart-placeholder"></div>
<div class="modal-line-divider"></div>
<div class="chart-placeholder"></div>
</section>
<div class="modal-footer">
<button class="close-btn">Close</button>
<div class="modal-line-divider"></div>
<div class="modal-footer-content">
<div class="modal-disclaimer-box"></div>
<button class="close-btn">Close</button>
</div>
</div>
</div>

0
docs/img/autoplugin_accelerate.svg → docs/_static/images/autoplugin_accelerate.svg vendored
View File

3
docs/_static/images/sample-graph-image.png vendored
View File

@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:64e64059e7416353cfd2ad836a36c12071804addf4fb165f0cf5150aa7658fa4
size 123996

3
docs/_static/images/training_extensions_framework.png vendored Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:2b3932d0cf0071c629e1013f3e17a9f8abda800eb01c50b3e826a42127e42da7
size 48770

228
docs/_static/js/graphs.js vendored
View File

@ -1,3 +1,33 @@
// general output config
const chartDisclaimers = {
Value: 'Value: Performance/(No_of_sockets * Price_of_CPU_dGPU), where prices are in USD as of December 2022.',
Efficiency: 'Efficiency: Performance/(No_of_sockets * TDP_of_CPU_dGPU), where total power dissipation (TDP) is in Watt as of December 2022.'
}
const defaultSelections = {
platforms: {name: 'platform',
data: [
'Intel® Core™ i9-12900K CPU-only',
'Intel® Core™ i5-10500TE CPU-only',
'Intel® Core™ i5-8500 CPU-only',
'Intel® Core™ i7-8700T CPU-only',
'Intel® Core™ i9-10900TE CPU-only',
'Intel® Core™ i7-1165G7 CPU-only'
]
},
platformFilters: {name: 'coretype', data: ['CPU']},
models: {name: 'networkmodel',
data: [
'bert-large-uncased-whole-word-masking-squad-0001 ',
'mobilenet-ssd ',
'resnet-50',
'yolo_v3_tiny'
]
},
parameters: {name: 'kpi', data: ['Throughput']},
pracision: {name: 'precision', data: ['INT8', 'FP32']}
}
class Filter {
// param: GraphData[], networkModels[]
@ -284,9 +314,9 @@ class Graph {
case 'int8':
return { data: null, color: '#00C7FD', label: 'FPS (INT8)' };
case 'fp16':
return { data: null, color: '#0068B5', label: 'FPS (FP16)' };
return { data: null, color: '#009fca', label: 'FPS (FP16)' };
case 'fp32':
return { data: null, color: '#00C7FD', label: 'FPS (FP32)' };
return { data: null, color: '#007797', label: 'FPS (FP32)' };
default:
return {};
}
@ -310,24 +340,17 @@ class Graph {
$(document).ready(function () {
$('#build-graphs-btn').on('click', showModal);
$('.ov-toolkit-benchmark-results').on('click', showModal);
function clickBuildGraphs(graph, networkModels, ietype, platforms, kpis, precisions) {
renderData(graph, networkModels, ietype, platforms, kpis, precisions);
$('.edit-settings-btn').show();
$('.clear-all-btn').hide();
$('.modal-footer').show();
$('.configure-graphs-header h3').addClass('header-inactive');
$('.benchmark-graph-results-header h3').removeClass('header-inactive');
$('#modal-display-graphs').show();
$('.edit-settings-btn').on('click', (event) => {
$('.configure-graphs-content').show();
$('.edit-settings-btn').hide();
$('.clear-all-btn').show();
$('#modal-configure-graphs').show();
$('#modal-display-graphs').hide();
$('.modal-footer').hide();
$('.configure-graphs-header h3').removeClass('header-inactive');
$('.benchmark-graph-results-header h3').addClass('header-inactive');
$('.chart-placeholder').empty();
});
@ -367,7 +390,7 @@ $(document).ready(function () {
}
function getSelectedNetworkModels() {
return $('.models-column-one input:checked, .models-column-two input:checked').map(function () {
return $('.models-column-one input:checked, .models-column-two input:checked').not('[data-networkmodel="Select All"]').map(function () {
return $(this).data('networkmodel');
}).get();
}
@ -392,7 +415,7 @@ $(document).ready(function () {
}).get();
}
function getSelectedPrecisions() {
return $('.precisions-column .selected').map(function () {
return $('.precisions-column input:checked').map(function () {
return $(this).data('precision');
}).get();
}
@ -404,16 +427,16 @@ $(document).ready(function () {
&& getSelectedKpis().length > 0) {
if (getSelectedKpis().includes('Throughput')) {
if (getSelectedPrecisions().length > 0) {
$('#modal-build-graphs-btn').prop('disabled', false);
$('#build-graphs-btn').prop('disabled', false);
return;
}
$('#modal-build-graphs-btn').prop('disabled', true);
$('#build-graphs-btn').prop('disabled', true);
return;
}
$('#modal-build-graphs-btn').prop('disabled', false);
$('#build-graphs-btn').prop('disabled', false);
return;
}
$('#modal-build-graphs-btn').prop('disabled', true);
$('#build-graphs-btn').prop('disabled', true);
}
function renderModal(result) {
@ -436,13 +459,16 @@ $(document).ready(function () {
modalContent.addClass('modal-content');
modal.append(modalContent);
// hide edit settings button
$('.edit-settings-btn').hide();
const models = networkModels.map((networkModel) => createCheckMark(networkModel, 'networkmodel'));
modal.find('.models-column-one').append(models.slice(0, models.length / 2));
const selectAllModelsButton = createCheckMark('Select All', 'networkmodel')
modal.find('.models-column-one').append(selectAllModelsButton).append(models.slice(0, models.length / 2));
modal.find('.models-column-two').append(models.slice(models.length / 2));
const precisions = Modal.getPrecisionsLabels().map((precision) => createCheckMark(precision, 'precision'));
modal.find('.precisions-column').append(precisions);
selectAllCheckboxes(precisions);
disableAllCheckboxes(precisions);
const types = ieTypes.map((ieType) => {
var labelText = Modal.getIeTypeLabel(ieType);
if (labelText) {
@ -456,6 +482,8 @@ $(document).ready(function () {
return item;
}
});
modal.find('#modal-display-graphs').hide();
modal.find('.ietype-column').append(types);
modal.find('.ietype-column input').first().prop('checked', true);
@ -464,44 +492,64 @@ $(document).ready(function () {
$('body').prepend(modal);
var fPlatforms = filterClientPlatforms(graph.data, getSelectedNetworkModels(), getSelectedIeType(), Modal.getCoreTypes(getSelectedCoreTypes()));
renderClientPlatforms(modal, Graph.getPlatformNames(fPlatforms));
renderClientPlatforms(graph.data, modal, true);
preselectDefaultSettings(graph.data, modal);
$('.clear-all-btn').on('click', () => {
$('.modal-content-grid-container input:checkbox').each((index, object) => $(object).prop('checked', false));
$('.precisions-column').empty();
modal.find('.ietype-column input').first().prop('checked', true);
validateSelections();
});
$('#modal-build-graphs-btn').on('click', () => {
$('.configure-graphs-content').hide();
$('.clear-all-btn').on('click', clearAll);
$('#build-graphs-btn').on('click', () => {
$('#modal-configure-graphs').hide();
clickBuildGraphs(graph, getSelectedNetworkModels(), getSelectedIeType(), getSelectedClientPlatforms(), getSelectedKpis(), Modal.getPrecisions(getSelectedPrecisions()));
});
$('.modal-close').on('click', hideModal);
$('.close-btn').on('click', hideModal);
modal.find('.ietype-column input').on('click', function (event) {
if (getSelectedIeType() === 'core') {
showCoreSelectorTypes(Modal.getCoreTypesLabels(), graph.data, modal);
}
else {
hideCoreSelectorTypes();
}
var fPlatforms = filterClientPlatforms(graph.data, getSelectedNetworkModels(), getSelectedIeType(), Modal.getCoreTypes(getSelectedCoreTypes()));
renderClientPlatforms(modal, Graph.getPlatformNames(fPlatforms));
});
modal.find('.kpi-column input').on('click', function (event) {
if (getSelectedKpis().includes('Throughput')) {
showPrecisionSelectorTypes(Modal.getPrecisionsLabels());
}
else {
hidePrecisionSelectorTypes();
}
modal.find('.models-column-one input[data-networkmodel="Select All"]').on('click', function() {
if ($(this).prop('checked'))
selectAllCheckboxes(models);
else deSelectAllCheckboxes(models);
});
modal.find('.ietype-column input').on('click', () => renderClientPlatforms(graph.data, modal, true));
modal.find('.kpi-column input').on('click', validateThroughputSelection);
modal.find('input').on('click', validateSelections);
});
}
function validateThroughputSelection() {
const precisions = $('.precisions-column').find('input')
if (getSelectedKpis().includes('Throughput')) {
precisions.prop('disabled', false);
}
else {
precisions.prop('disabled', true);
}
}
function clearAll() {
$('.modal-content-grid-container input:checkbox').each((index, object) => $(object).prop('checked', false));
// Uncomment if you want the Clear All button to reset the Platform Type column as well
// modal.find('.ietype-column input').first().prop('checked', true);
validateThroughputSelection();
validateSelections();
}
function preselectDefaultSettings(data, modal) {
if (defaultSelections.platformFilters) {
const filters = modal.find('.selectable-box-container').children('.selectable-box');
filters.removeClass('selected');
defaultSelections.platformFilters.data.forEach(selection => {
filters.filter(`[data-${defaultSelections.platformFilters.name}="${selection}"]`).addClass('selected');
});
renderClientPlatforms(data, modal);
}
clearAll();
for (setting in defaultSelections) {
let name = defaultSelections[setting].name;
defaultSelections[setting].data.forEach(selection => {
$(`input[data-${name}="${selection}"]`).prop('checked', true);
});
}
validateThroughputSelection();
validateSelections();
}
function showCoreSelectorTypes(coreTypes, graphDataArr, modal) {
if ($('.client-platform-column').find('.selectable-box-container').length) {
@ -524,7 +572,7 @@ $(document).ready(function () {
$(this).addClass('selected');
}
var fPlatforms = filterClientPlatforms(graphDataArr, getSelectedNetworkModels(), getSelectedIeType(), Modal.getCoreTypes(getSelectedCoreTypes()));
renderClientPlatforms(modal, Graph.getPlatformNames(fPlatforms));
renderClientPlatformsItems(modal, Graph.getPlatformNames(fPlatforms), true);
validateSelections();
});
}
@ -533,36 +581,6 @@ $(document).ready(function () {
$('.client-platform-column').find('.selectable-box-container').hide();
}
function showPrecisionSelectorTypes(precisions) {
if ($('.precisions-column').find('.selectable-box-container').length) {
$('.precisions-column').find('.selectable-box-container').show();
return;
}
var container = $('<div>');
container.addClass('selectable-box-container');
precisions.forEach((prec) => {
var box = $('<div>' + prec + '</div>');
box.attr('data-precision', prec);
box.addClass('selectable-box');
container.append(box);
});
$('.precisions-column').prepend(container);
$('.precisions-column .selectable-box').on('click', function () {
if ($(this).hasClass('selected')) {
$(this).removeClass('selected');
} else {
$(this).addClass('selected');
}
validateSelections();
});
}
function hidePrecisionSelectorTypes() {
$('.precisions-column').find('.selectable-box-container').hide();
}
function filterClientPlatforms(data, networkModels, ietype, coreTypes) {
// No longer filtering on the network type, if at some point we want the network type as a filter, uncomment this
// var first = Filter.FilterByNetworkModel(data, networkModels);
@ -575,10 +593,22 @@ $(document).ready(function () {
return Array.from(optionMap.values());
}
function renderClientPlatforms(modal, platformNames) {
function renderClientPlatforms(data, modal, preselectEveryItem) {
if (getSelectedIeType() === 'core') {
showCoreSelectorTypes(Modal.getCoreTypesLabels(), data, modal);
}
else {
hideCoreSelectorTypes();
}
var fPlatforms = filterClientPlatforms(data, getSelectedNetworkModels(), getSelectedIeType(), Modal.getCoreTypes(getSelectedCoreTypes()));
renderClientPlatformsItems(modal, Graph.getPlatformNames(fPlatforms), preselectEveryItem);
}
function renderClientPlatformsItems(modal, platformNames, preselectEveryItem) {
$('.client-platform-column .checkmark-container').remove();
const clientPlatforms = platformNames.map((platform) => createCheckMark(platform, 'platform'));
selectAllCheckboxes(clientPlatforms);
if (preselectEveryItem)
selectAllCheckboxes(clientPlatforms);
modal.find('.client-platform-column').append(clientPlatforms);
modal.find('.client-platform-column input').on('click', validateSelections);
}
@ -597,7 +627,25 @@ $(document).ready(function () {
// receives a jquery list of items and selects all input checkboxes
function selectAllCheckboxes(items) {
items.forEach((item) => {
item.find(':input').attr('checked', true);
item.find(':input').prop('checked', true);
});
}
function enableAllCheckboxes(items) {
items.forEach((item) => {
item.find(':input').prop('disabled', false);
})
}
function disableAllCheckboxes(items) {
items.forEach((item) => {
item.find(':input').prop('disabled', true);
})
}
function deSelectAllCheckboxes(items) {
items.forEach((item) => {
item.find(':input').prop('checked', false);
});
}
@ -659,8 +707,8 @@ $(document).ready(function () {
function renderData(graph, networkModels, ietype, platforms, kpis, precisions) {
$('.chart-placeholder').empty();
$('.modal-disclaimer-box').empty();
networkModels.forEach((networkModel) => {
// graph title
var chartName = networkModel;
var chartSlug = chartName.replace(')', '').replace(' (', '-');
var chartContainer = $('<div>');
@ -686,8 +734,12 @@ $(document).ready(function () {
} else {
createEmptyChartContainer(chartContainer);
}
})
for (let kpi of kpis) {
if (chartDisclaimers[kpi])
$('.modal-disclaimer-box').append($('<p>').text(chartDisclaimers[kpi]))
}
};
function createEmptyChartContainer(chartContainer) {

1
docs/benchmarks/performance_benchmarks.md
View File

@ -21,7 +21,6 @@ Benchmarks are available for:
* [Intel® Distribution of OpenVINO™ toolkit](performance_benchmarks_openvino.md).
You can also test performance for your system yourself, following the guide on [getting performance numbers](../MO_DG/prepare_model/Getting_performance_numbers.md).
Performance of a particular application can also be evaluated virtually using [Intel® DevCloud for the Edge](https://devcloud.intel.com/edge/). It is a remote development environment with access to Intel® hardware and the latest versions of the Intel® Distribution of the OpenVINO™ Toolkit. To learn more about it, visit [the website](https://www.intel.com/content/www/us/en/developer/tools/devcloud/edge/overview.html) or [create an account](https://www.intel.com/content/www/us/en/forms/idz/devcloud-registration.html?tgt=https://www.intel.com/content/www/us/en/secure/forms/devcloud-enrollment/account-provisioning.html).

103
docs/benchmarks/performance_benchmarks_openvino.md
View File

@ -9,89 +9,76 @@
openvino_docs_performance_int8_vs_fp32
Performance Data Spreadsheet (download xlsx) <https://docs.openvino.ai/2022.3/_static/benchmarks_files/OV-2022.3-Performance-Data.xlsx>
@endsphinxdirective
Click the "Benchmark Graphs" button to see the OpenVINO™ benchmark graphs. Select the models, the hardware platforms (CPU SKUs),
precision and performance index from the lists and click the “Build Graphs” button.
@sphinxdirective
.. button-link:: #
:class: ov-toolkit-benchmark-results
:color: primary
:outline:
:material-regular:`bar_chart;1.4em` Benchmark Graphs
.. raw:: html
<section class="build-benchmark-section">
<div class="title">
<h3>Build benchmark graphs to your specifications</h3>
</div>
<div class="btn-container">
<button id="build-graphs-btn" class="configure-graphs-btn">Configure Graphs</button>
</div>
<img src="_static/images/sample-graph-image.png" class="sample-graph-image">
</section>
@endsphinxdirective
Measuring inference performance involves many variables and is extremely use-case and application dependent.
Below are four parameters for measurements, which are key elements to consider for a successful deep learning inference application:
@sphinxdirective
.. raw:: html
.. tab:: :material-regular:`keyboard_double_arrow_right;1.4em` Throughput
<div class="picker-options">
<span class="selectable option throughput selected" data-option="throughput">
Throughput
</span>
<span class="selectable option value" data-option="value">
Value
</span>
<span class="selectable option efficiency" data-option="efficiency">
Efficiency
</span>
<span class="selectable option latency" data-option="latency">
Latency
</span>
<p class="selectable throughput selected">
Measures the number of inferences delivered within a latency threshold. (for example, number of Frames Per Second - FPS). When deploying a system with deep learning inference, select the throughput that delivers the best trade-off between latency and power for the price and performance that meets your requirements.
</p>
<p class="selectable value">
While throughput is important, what is more critical in edge AI deployments is the performance efficiency or performance-per-cost. Application performance in throughput per dollar of system cost is the best measure of value. The value KPI is calculated as “Throughput measured as inferences per second / price of inference engine”. This means for a 2 socket system 2x the price of a CPU is used. Prices are as per date of benchmarking and sources can be found as links in the Hardware Platforms (PDF) description below.
<p class="selectable efficiency">
System power is a key consideration from the edge to the data center. When selecting deep learning solutions, power efficiency (throughput/watt) is a critical factor to consider. Intel designs provide excellent power efficiency for running deep learning workloads. The efficiency KPI is calculated as “Throughput measured as inferences per second / TDP of inference engine”. This means for a 2 socket system 2x the power dissipation (TDP) of a CPU is used. TDP-values are as per date of benchmarking and sources can be found as links in the Hardware Platforms (PDF) description below.
<p class="selectable latency">
This measures the synchronous execution of inference requests and is reported in milliseconds. Each inference request (for example: preprocess, infer, postprocess) is allowed to complete before the next is started. This performance metric is relevant in usage scenarios where a single image input needs to be acted upon as soon as possible. An example would be the healthcare sector where medical personnel only request analysis of a single ultra sound scanning image or in real-time or near real-time applications for example an industrial robot's response to actions in its environment or obstacle avoidance for autonomous vehicles.
</p>
</div>
Measures the number of inferences delivered within a latency threshold (for example, number of Frames Per Second - FPS). When deploying a system with deep learning inference, select the throughput that delivers the best trade-off between latency and power for the price and performance that meets your requirements.
<h3>Platform & Configurations </h3>
<p>For a listing of all platforms and configurations used for testing, refer to the following:</p>
<container class="platform-configurations">
<div>
<a href="https://docs.openvino.ai/2022.3/_static/benchmarks_files/platform_list_22.3.pdf" target="_blank" class="pdf"><img src="_static/css/media/pdf-icon.svg"/>Hardware Platforms (PDF)</a>
</div>
<div>
<a href="https://docs.openvino.ai/2022.3/_static/benchmarks_files/OV-2022.3-system-info-detailed.xlsx" class="xls"><img src="_static/css/media/xls-icon.svg"/>Configuration Details (XLSX)</a>
</div>
</container>
.. tab:: :material-regular:`attach_money;1.4em` Value
While throughput is important, what is more critical in edge AI deployments is the performance efficiency or performance-per-cost. Application performance in throughput per dollar of system cost is the best measure of value. The value KPI is calculated as “Throughput measured as inferences per second / price of inference engine”. This means for a 2 socket system 2x the price of a CPU is used. Prices are as per date of benchmarking and sources can be found as links in the Hardware Platforms (PDF) description below.
.. tab:: :material-regular:`flash_on;1.4em` Efficiency
System power is a key consideration from the edge to the data center. When selecting deep learning solutions, power efficiency (throughput/watt) is a critical factor to consider. Intel designs provide excellent power efficiency for running deep learning workloads. The efficiency KPI is calculated as “Throughput measured as inferences per second / TDP of inference engine”. This means for a 2 socket system 2x the power dissipation (TDP) of a CPU is used. TDP-values are as per date of benchmarking and sources can be found as links in the Hardware Platforms (PDF) description below.
.. tab:: :material-regular:`hourglass_empty;1.4em` Latency
This measures the synchronous execution of inference requests and is reported in milliseconds. Each inference request (for example: preprocess, infer, postprocess) is allowed to complete before the next is started. This performance metric is relevant in usage scenarios where a single image input needs to be acted upon as soon as possible. An example would be the healthcare sector where medical personnel only request analysis of a single ultra sound scanning image or in real-time or near real-time applications for example an industrial robot's response to actions in its environment or obstacle avoidance for autonomous vehicles.
Platform & Configurations
####################################
For a listing of all platforms and configurations used for testing, refer to the following:
.. button-link:: _static/benchmarks_files/platform_list_22.3.pdf
:color: primary
:outline:
:material-regular:`download;1.5em` Click for Hardware Platforms [PDF]
.. button-link:: _static/benchmarks_files/OV-2022.3-system-info-detailed.xlsx
:color: primary
:outline:
:material-regular:`download;1.5em` Click for Configuration Details [XLSX]
@endsphinxdirective
This benchmark setup includes a single machine on which both the benchmark application and the OpenVINO™ installation reside. The presented performance benchmark numbers are based on the release 2022.3 of the Intel® Distribution of OpenVINO™ toolkit.
The benchmark application loads the OpenVINO™ Runtime and executes inferences on the specified hardware (CPU, GPU or GNA).
It measures the time spent on actual inferencing (excluding any pre or post processing) and then reports on the inferences per second (or Frames Per Second).
It measures the time spent on actual inference (excluding any pre or post processing) and then reports on the inferences per second (or Frames Per Second).
## Disclaimers
Disclaimers
####################################
Intel® Distribution of OpenVINO™ toolkit performance benchmark numbers are based on release 2022.3.
Intel technologies’ features and benefits depend on system configuration and may require enabled hardware, software or service activation. Learn more at intel.com, or from the OEM or retailer. Performance results are based on testing as of December 13, 2022 and may not reflect all publicly available updates. See configuration disclosure for details. No product can be absolutely secure.
Performance varies by use, configuration and other factors. Learn more at [www.intel.com/PerformanceIndex](https://www.intel.com/PerformanceIndex).
Performance varies by use, configuration and other factors. Learn more at :ref:`www.intel.com/PerformanceIndex<https://www.intel.com/PerformanceIndex>`.
Your costs and results may vary.
Intel optimizations, for Intel compilers or other products, may not optimize to the same degree for non-Intel products.
© Intel Corporation. Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries. Other names and brands may be claimed as the property of others.
© Intel Corporation. Intel, the Intel logo, and other Intel marks are trademarks of Intel Corporation or its subsidiaries. Other names and brands may be claimed as the property of others.
@endsphinxdirective

2
docs/benchmarks/performance_int8_vs_fp32.md
View File

@ -1,4 +1,4 @@
# Model Accuracy and Performance for INT8 and FP32 {#openvino_docs_performance_int8_vs_fp32}
# Model Accuracy {#openvino_docs_performance_int8_vs_fp32}
The following table presents the absolute accuracy drop calculated as the accuracy difference between FP32 and INT8 representations of a model on two platforms

2
docs/conf.py
View File

@ -79,7 +79,7 @@ html_theme = "openvino_sphinx_theme"
html_theme_path = ['_themes']
html_theme_options = {
"navigation_depth": 6,
"navigation_depth": 8,
"show_nav_level": 2,
"use_edit_page_button": True,
"github_url": "https://github.com/openvinotoolkit/openvino",

4
docs/glossary.md
View File

@ -73,13 +73,13 @@ Glossary of terms used in OpenVINO™
| OpenVINO™ Core is a software component that manages inference on certain Intel(R) hardware devices: CPU, GPU, GNA, etc.
| OpenVINO™ API
| The basic default API for all supported devices, which allows you to load a model from Intermediate Representation or convert from ONNX, PaddlePaddle file formars, set input and output formats and execute the model on various devices.
| The basic default API for all supported devices, which allows you to load a model from Intermediate Representation or convert from ONNX, PaddlePaddle, TensorFlow file formats, set input and output formats and execute the model on various devices.
| OpenVINO™ Runtime
| A C++ library with a set of classes that you can use in your application to infer input tensors and get the results.
| <code>ov::Model</code>
| A class of the Model that OpenVINO™ Runtime reads from IR or converts from ONNX, PaddlePaddle formats. Consists of model structure, weights and biases.
| A class of the Model that OpenVINO™ Runtime reads from IR or converts from ONNX, PaddlePaddle, TensorFlow formats. Consists of model structure, weights and biases.
| <code>ov::CompiledModel</code>
| An instance of the compiled model which allows the OpenVINO™ Runtime to request (several) infer requests and perform inference synchronously or asynchronously.

27
docs/install_guides/uninstalling-openvino.md
View File

@ -1,12 +1,17 @@
# Uninstalling the Intel® Distribution of OpenVINO™ Toolkit {#openvino_docs_install_guides_uninstalling_openvino}
> **NOTE**: Uninstallation procedures remove all Intel® Distribution of OpenVINO™ Toolkit component files but don't affect user files in the installation directory.
@sphinxdirective
## Uninstall Using the Original Installation Package
.. note::
Uninstallation procedures remove all Intel® Distribution of OpenVINO™ Toolkit component files but don't affect user files in the installation directory.
Uninstall Using the Original Installation Package
#################################################
If you have installed OpenVINO Runtime from archive files, you can uninstall it by deleting the archive files and the extracted folders.
@sphinxdirective
.. tab:: Windows
If you have created the symbolic link, remove the link first.
@ -15,25 +20,27 @@ If you have installed OpenVINO Runtime from archive files, you can uninstall it
* Use Windows Explorer to remove the files.
* Open a Command Prompt and run:
.. code-block:: sh
rmdir /s <extracted_folder>
del <path_to_archive>
.. tab:: Linux & macOS
If you have created the symbolic link, remove the link first:
.. code-block:: sh
rm /home/<USER>/intel/openvino_2022
rm /opt/intel/openvino_2022
To delete the files:
.. code-block:: sh
rm -r <extracted_folder> && rm <path_to_archive>
@endsphinxdirective

2
docs/nbdoc/consts.py
View File

@ -8,7 +8,7 @@ repo_owner = "openvinotoolkit"
repo_name = "openvino_notebooks"
artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20221115220807/dist/rst_files/"
artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20230302220806/dist/rst_files/"
blacklisted_extensions = ['.xml', '.bin']

2
docs/ops/opset.md
View File

@ -6,6 +6,7 @@
:maxdepth: 1
:hidden:
openvino_docs_ops_opset11
openvino_docs_ops_opset10
openvino_docs_ops_opset9
openvino_docs_ops_opset8
@ -25,6 +26,7 @@ This topic provides a complete list of available sets of operations supported in
| OpenVINO™ Version | Actual Operations Set |
| :---------------- | :------------------------------- |
| 2023.0 | [opset11](opset11.md) |
| 2022.3 | [opset10](opset10.md) |
| 2022.2 | [opset9](opset9.md) |
| 2022.1 | [opset8](opset8.md) |

187
docs/ops/opset11.md Normal file
View File

@ -0,0 +1,187 @@
# opset11 {#openvino_docs_ops_opset11}
This specification document describes the `opset11` operation set supported in OpenVINO™.
Support for each particular operation from the list below depends on the capabilities of an inference plugin
and may vary among different hardware platforms and devices. Examples of operation instances are provided as IR V10 xml
snippets. Such IR is generated by the Model Optimizer. The semantics match corresponding nGraph operation classes
declared in `namespace opset11`.
## Table of Contents
* [Abs](arithmetic/Abs_1.md)
* [Acos](arithmetic/Acos_1.md)
* [Acosh](arithmetic/Acosh_3.md)
* [AdaptiveAvgPool](pooling/AdaptiveAvgPool_8.md)
* [AdaptiveMaxPool](pooling/AdaptiveMaxPool_8.md)
* [Add](arithmetic/Add_1.md)
* [Asin](arithmetic/Asin_1.md)
* [Asinh](arithmetic/Asinh_3.md)
* [Assign](infrastructure/Assign_3.md)
* [Atan](arithmetic/Atan_1.md)
* [Atanh](arithmetic/Atanh_3.md)
* [AvgPool](pooling/AvgPool_1.md)
* [BatchNormInference](normalization/BatchNormInference_5.md)
* [BatchToSpace](movement/BatchToSpace_2.md)
* [BinaryConvolution](convolution/BinaryConvolution_1.md)
* [Broadcast](movement/Broadcast_3.md)
* [Bucketize](condition/Bucketize_3.md)
* [CTCGreedyDecoder](sequence/CTCGreedyDecoder_1.md)
* [CTCGreedyDecoderSeqLen](sequence/CTCGreedyDecoderSeqLen_6.md)
* [CTCLoss](sequence/CTCLoss_4.md)
* [Ceiling](arithmetic/Ceiling_1.md)
* [Clamp](activation/Clamp_1.md)
* [Concat](movement/Concat_1.md)
* [Constant](infrastructure/Constant_1.md)
* [Convert](type/Convert_1.md)
* [ConvertLike](type/ConvertLike_1.md)
* [Convolution](convolution/Convolution_1.md)
* [ConvolutionBackpropData](convolution/ConvolutionBackpropData_1.md)
* [Cos](arithmetic/Cos_1.md)
* [Cosh](arithmetic/Cosh_1.md)
* [CumSum](arithmetic/CumSum_3.md)
* [DeformableConvolution](convolution/DeformableConvolution_8.md)
* [DeformablePSROIPooling](detection/DeformablePSROIPooling_1.md)
* [DepthToSpace](movement/DepthToSpace_1.md)
* [DetectionOutput](detection/DetectionOutput_8.md)
* [DFT](signals/DFT_7.md)
* [Divide](arithmetic/Divide_1.md)
* [Einsum](matrix/Einsum_7.md)
* [Elu](activation/Elu_1.md)
* [EmbeddingBagOffsetsSum](sparse/EmbeddingBagOffsetsSum_3.md)
* [EmbeddingBagPackedSum](sparse/EmbeddingBagPackedSum_3.md)
* [EmbeddingSegmentsSum](sparse/EmbeddingSegmentsSum_3.md)
* [Equal](comparison/Equal_1.md)
* [Erf](arithmetic/Erf_1.md)
* [Exp](activation/Exp_1.md)
* [ExperimentalDetectronDetectionOutput_6](detection/ExperimentalDetectronDetectionOutput_6.md)
* [ExperimentalDetectronGenerateProposalsSingleImage_6](detection/ExperimentalDetectronGenerateProposalsSingleImage_6.md)
* [ExperimentalDetectronPriorGridGenerator_6](detection/ExperimentalDetectronPriorGridGenerator_6.md)
* [ExperimentalDetectronROIFeatureExtractor_6](detection/ExperimentalDetectronROIFeatureExtractor_6.md)
* [ExperimentalDetectronTopKROIs_6](sort/ExperimentalDetectronTopKROIs_6.md)
* [ExtractImagePatches](movement/ExtractImagePatches_3.md)
* [Eye](generation/Eye_9.md)
* [FakeQuantize](quantization/FakeQuantize_1.md)
* [Floor](arithmetic/Floor_1.md)
* [FloorMod](arithmetic/FloorMod_1.md)
* [Gather](movement/Gather_8.md)
* [GatherElements](movement/GatherElements_6.md)
* [GatherND](movement/GatherND_8.md)
* [GatherTree](movement/GatherTree_1.md)
* [Gelu](activation/GELU_7.md)
* [GenerateProposals](detection/GenerateProposals_9.md)
* [Greater](comparison/Greater_1.md)
* [GreaterEqual](comparison/GreaterEqual_1.md)
* [GridSample](image/GridSample_9.md)
* [GRN](normalization/GRN_1.md)
* [GroupConvolution](convolution/GroupConvolution_1.md)
* [GroupConvolutionBackpropData](convolution/GroupConvolutionBackpropData_1.md)
* [GRUCell](sequence/GRUCell_3.md)
* [GRUSequence](sequence/GRUSequence_5.md)
* [HardSigmoid](activation/HardSigmoid_1.md)
* [HSigmoid](activation/HSigmoid_5.md)
* [HSwish](activation/HSwish_4.md)
* [IDFT](signals/IDFT_7.md)
* [I420toBGR](image/I420toBGR_8.md)
* [I420toRGB](image/I420toRGB_8.md)
* [If](condition/If_8.md)
* [Interpolate](image/Interpolate_4.md)
* [IRDFT](signals/IRDFT_9.md)
* [IsInf](comparison/IsInf_10.md)
* [IsNaN](comparison/IsNaN_10.md)
* [Less](comparison/Less_1.md)
* [LessEqual](comparison/LessEqual_1.md)
* [Log](arithmetic/Log_1.md)
* [LogicalAnd](logical/LogicalAnd_1.md)
* [LogicalNot](logical/LogicalNot_1.md)
* [LogicalOr](logical/LogicalOr_1.md)
* [LogicalXor](logical/LogicalXor_1.md)
* [LogSoftmax](activation/LogSoftmax_5.md)
* [Loop](infrastructure/Loop_5.md)
* [LRN](normalization/LRN_1.md)
* [LSTMCell](sequence/LSTMCell_1.md)
* [LSTMSequence](sequence/LSTMSequence_1.md)
* [MatMul](matrix/MatMul_1.md)
* [MatrixNMS](sort/MatrixNMS_8.md)
* [MaxPool](pooling/MaxPool_8.md)
* [Maximum](arithmetic/Maximum_1.md)
* [Minimum](arithmetic/Minimum_1.md)
* [Mish](activation/Mish_4.md)
* [Mod](arithmetic/Mod_1.md)
* [MVN](normalization/MVN_6.md)
* [MulticlassNMS](sort/MulticlassNonMaxSuppression_9.md)
* [Multiply](arithmetic/Multiply_1.md)
* [Negative](arithmetic/Negative_1.md)
* [NonMaxSuppression](sort/NonMaxSuppression_5.md)
* [NonZero](condition/NonZero_3.md)
* [NormalizeL2](normalization/NormalizeL2_1.md)
* [NotEqual](comparison/NotEqual_1.md)
* [NV12toBGR](image/NV12toBGR_8.md)
* [NV12toRGB](image/NV12toRGB_8.md)
* [OneHot](sequence/OneHot_1.md)
* [Pad](movement/Pad_1.md)
* [Parameter](infrastructure/Parameter_1.md)
* [Power](arithmetic/Power_1.md)
* [PReLU](activation/PReLU_1.md)
* [PriorBoxClustered](detection/PriorBoxClustered_1.md)
* [PriorBox](detection/PriorBox_8.md)
* [Proposal](detection/Proposal_4.md)
* [PSROIPooling](detection/PSROIPooling_1.md)
* [RandomUniform](generation/RandomUniform_8.md)
* [Range](generation/Range_4.md)
* [RDFT](signals/RDFT_9.md)
* [ReLU](activation/ReLU_1.md)
* [ReadValue](infrastructure/ReadValue_3.md)
* [ReduceL1](reduction/ReduceL1_4.md)
* [ReduceL2](reduction/ReduceL2_4.md)
* [ReduceLogicalAnd](reduction/ReduceLogicalAnd_1.md)
* [ReduceLogicalOr](reduction/ReduceLogicalOr_1.md)
* [ReduceMax](reduction/ReduceMax_1.md)
* [ReduceMean](reduction/ReduceMean_1.md)
* [ReduceMin](reduction/ReduceMin_1.md)
* [ReduceProd](reduction/ReduceProd_1.md)
* [ReduceSum](reduction/ReduceSum_1.md)
* [RegionYolo](detection/RegionYolo_1.md)
* [ReorgYolo](detection/ReorgYolo_1.md)
* [Reshape](shape/Reshape_1.md)
* [Result](infrastructure/Result_1.md)
* [ReverseSequence](movement/ReverseSequence_1.md)
* [RNNCell](sequence/RNNCell_3.md)
* [RNNSequence](sequence/RNNSequence_5.md)
* [ROIAlign](detection/ROIAlign_9.md)
* [ROIPooling](detection/ROIPooling_1.md)
* [Roll](movement/Roll_7.md)
* [Round](arithmetic/Round_5.md)
* [ScatterElementsUpdate](movement/ScatterElementsUpdate_3.md)
* [ScatterNDUpdate](movement/ScatterNDUpdate_3.md)
* [ScatterUpdate](movement/ScatterUpdate_3.md)
* [Select](condition/Select_1.md)
* [Selu](activation/Selu_1.md)
* [ShapeOf](shape/ShapeOf_3.md)
* [ShuffleChannels](movement/ShuffleChannels_1.md)
* [Sigmoid](activation/Sigmoid_1.md)
* [Sign](arithmetic/Sign_1.md)
* [Sin](arithmetic/Sin_1.md)
* [Sinh](arithmetic/Sinh_1.md)
* [Slice](movement/Slice_8.md)
* [SoftMax](activation/SoftMax_8.md)
* [SoftPlus](activation/SoftPlus_4.md)
* [SoftSign](activation/SoftSign_9.md)
* [SpaceToBatch](movement/SpaceToBatch_2.md)
* [SpaceToDepth](movement/SpaceToDepth_1.md)
* [Split](movement/Split_1.md)
* [Sqrt](arithmetic/Sqrt_1.md)
* [SquaredDifference](arithmetic/SquaredDifference_1.md)
* [Squeeze](shape/Squeeze_1.md)
* [StridedSlice](movement/StridedSlice_1.md)
* [Subtract](arithmetic/Subtract_1.md)
* [Swish](activation/Swish_4.md)
* [Tan](arithmetic/Tan_1.md)
* [Tanh](arithmetic/Tanh_1.md)
* [TensorIterator](infrastructure/TensorIterator_1.md)
* [Tile](movement/Tile_1.md)
* [TopK](sort/TopK_11.md)
* [Transpose](movement/Transpose_1.md)
* [Unique](movement/Unique_10.md)
* [Unsqueeze](shape/Unsqueeze_1.md)
* [VariadicSplit](movement/VariadicSplit_1.md)

4
docs/ops/sort/TopK_1.md
View File

@ -51,7 +51,7 @@
**Detailed Description**
Output tensor is populated by values computes in the following way:
The output tensor is populated by values computed in the following way:
output[i1, ..., i(axis-1), j, i(axis+1) ..., iN] = top_k(input[i1, ...., i(axis-1), :, i(axis+1), ..., iN]), k, sort, mode)
@ -59,7 +59,7 @@ So for each slice `input[i1, ...., i(axis-1), :, i(axis+1), ..., iN]` which repr
Sorting and minimum/maximum are controlled by `sort` and `mode` attributes:
* *mode*=`max`, *sort*=`value` - descending by value
* *mode*=`max`, *sort*=`index` - ascending by index
* *mode*=`max`, *sort*=`index` - descending by index
* *mode*=`max`, *sort*=`none` - undefined
* *mode*=`min`, *sort*=`value` - ascending by value
* *mode*=`min`, *sort*=`index` - ascending by index

118
docs/ops/sort/TopK_11.md Normal file
View File

@ -0,0 +1,118 @@
# TopK {#openvino_docs_ops_sort_TopK_11}
**Versioned name**: *TopK-11*
**Category**: *Sorting and maximization*
**Short description**: *TopK* computes indices and values of the *k* maximum/minimum values for each slice along a specified axis.
**Attributes**
* *axis*
* **Description**: Specifies the axis along which the values are retrieved.
* **Range of values**: An integer. Negative values means counting dimension from the back.
* **Type**: `int`
* **Required**: *yes*
* *mode*
* **Description**: Specifies whether *TopK* selects the largest or the smallest elements from each slice.
* **Range of values**: "min", "max"
* **Type**: `string`
* **Required**: *yes*
* *sort*
* **Description**: Specifies the order of corresponding elements of the output tensor.
* **Range of values**: `value`, `index`, `none`
* **Type**: `string`
* **Required**: *yes*
* *stable*
* **Description**: Specifies whether the equivalent elements should maintain their relative order from the input tensor. Takes effect only if the `sort` attribute is set to `value`.
* **Range of values**: `true` of `false`
* **Type**: `boolean`
* **Default value**: `false`
* **Required**: *no*
* *index_element_type*
* **Description**: the type of output tensor with indices
* **Range of values**: "i64" or "i32"
* **Type**: string
* **Default value**: "i32"
* **Required**: *no*
**Inputs**:
* **1**: tensor with arbitrary rank and type *T*. **Required.**
* **2**: The value of *K* - a scalar of any integer type that specifies how many elements from the input tensor should be selected. The accepted range of values of *K* is `<1;input1.shape[axis]>`. The behavior of this operator is undefined if the value of *K* does not meet those requirements. **Required.**
**Outputs**:
* **1**: Output tensor of type *T* with *k* values from the input tensor along a specified *axis*. The shape of the tensor is `[input1.shape[0], ..., input1.shape[axis-1], 1..k, input1.shape[axis+1], ..., input1.shape[input1.rank - 1]]`.
* **2**: Output tensor containing indices of the corresponding elements(values) from the first output tensor. The indices point to the location of selected values in the original input tensor. The shape of this output tensor is the same as the shape of the first output, that is `[input1.shape[0], ..., input1.shape[axis-1], 1..k, input1.shape[axis+1], ..., input1.shape[input1.rank - 1]]`. The type of this tensor *T_IND* is controlled by the `index_element_type` attribute.
**Types**
* *T*: any numeric type.
* *T_IND*: `int64` or `int32`.
**Detailed Description**
The output tensor is populated by values computed in the following way:
output[i1, ..., i(axis-1), j, i(axis+1) ..., iN] = top_k(input[i1, ...., i(axis-1), :, i(axis+1), ..., iN]), k, sort, mode)
meaning that for each slice `input[i1, ...., i(axis-1), :, i(axis+1), ..., iN]` the *TopK* values are computed individually.
Sorting and minimum/maximum are controlled by `sort` and `mode` attributes with additional configurability provided by `stable`:
* *sort*=`value`, *mode*=`max`, *stable*=`false` - descending by value, relative order of equal elements not guaranteed to be maintained
* *sort*=`value`, *mode*=`max`, *stable*=`true` - descending by value, relative order of equal elements guaranteed to be maintained
* *sort*=`value`, *mode*=`min`, *stable*=`false` - ascending by value, relative order of equal elements not guaranteed to be maintained
* *sort*=`value`, *mode*=`min`, *stable*=`true` - ascending by value, relative order of equal elements guaranteed to be maintained
* *sort*=`index`, *mode*=`max` - descending by index
* *sort*=`index`, *mode*=`min` - ascending by index
* *sort*=`none` , *mode*=`max` - undefined
* *sort*=`none` , *mode*=`min` - undefined
The relative order of equivalent elements is only preserved if the *stable* attribute is set to `true`. This makes the implementation use stable sorting algorithm during the computation of TopK elements. Otherwise the output order is undefined.
**Example**
This example assumes that `K` is equal to 10:
```xml
<layer ... type="TopK" ... >
<data axis="3" mode="max" sort="value" stable="true" index_element_type="i64"/>
<input>
<port id="0">
<dim>1</dim>
<dim>3</dim>
<dim>224</dim>
<dim>224</dim>
</port>
<port id="1">
</port>
<output>
<port id="2">
<dim>1</dim>
<dim>3</dim>
<dim>224</dim>
<dim>10</dim>
</port>
<port id="3">
<dim>1</dim>
<dim>3</dim>
<dim>224</dim>
<dim>10</dim>
</port>
</output>
</layer>
```

4
docs/ops/sort/TopK_3.md
View File

@ -58,7 +58,7 @@
**Detailed Description**
Output tensor is populated by values computes in the following way:
The output tensor is populated by values computed in the following way:
output[i1, ..., i(axis-1), j, i(axis+1) ..., iN] = top_k(input[i1, ...., i(axis-1), :, i(axis+1), ..., iN]), k, sort, mode)
@ -66,7 +66,7 @@ So for each slice `input[i1, ...., i(axis-1), :, i(axis+1), ..., iN]` which repr
Sorting and minimum/maximum are controlled by `sort` and `mode` attributes:
* *mode*=`max`, *sort*=`value` - descending by value
* *mode*=`max`, *sort*=`index` - ascending by index
* *mode*=`max`, *sort*=`index` - descending by index
* *mode*=`max`, *sort*=`none` - undefined
* *mode*=`min`, *sort*=`value` - ascending by value
* *mode*=`min`, *sort*=`index` - ascending by index

246
docs/optimization_guide/nncf/filter_pruning.md
View File

@ -1,183 +1,227 @@
# Filter Pruning of Convolutional Models {#filter_pruning}
## Introduction
Filter pruning is an advanced optimization method which allows reducing computational complexity of the model by removing redundant or unimportant filters from convolutional operations of the model. This removal is done in two steps:
@sphinxdirective
Introduction
####################
Filter pruning is an advanced optimization method which allows reducing computational complexity of the model by removing
redundant or unimportant filters from convolutional operations of the model. This removal is done in two steps:
1. Unimportant filters are zeroed out by the NNCF optimization with fine-tuning.
2. Zero filters are removed from the model during the export to OpenVINO&trade; Intermediate Representation (IR).
Filter Pruning method from the NNCF can be used stand-alone but we usually recommend to stack it with 8-bit quantization for two reasons. First, 8-bit quantization is the best method in terms of achieving the highest accuracy-performance trade-offs so stacking it with filter pruning can give even better performance results. Second, applying quantization along with filter pruning does not hurt accuracy a lot since filter pruning removes noisy filters from the model which narrows down values ranges of weights and activations and helps to reduce overall quantization error.
2. Zero filters are removed from the model during the export to OpenVINO Intermediate Representation (IR).
Filter Pruning method from the NNCF can be used stand-alone but we usually recommend to stack it with 8-bit quantization for
two reasons. First, 8-bit quantization is the best method in terms of achieving the highest accuracy-performance trade-offs so
stacking it with filter pruning can give even better performance results. Second, applying quantization along with filter
pruning does not hurt accuracy a lot since filter pruning removes noisy filters from the model which narrows down values
ranges of weights and activations and helps to reduce overall quantization error.
.. note::
Filter Pruning usually requires a long fine-tuning or retraining of the model which can be comparable to training the
model from scratch. Otherwise, a large accuracy degradation can be caused. Therefore, the training schedule should be
adjusted accordingly when applying this method.
> **NOTE**: Filter Pruning usually requires a long fine-tuning or retraining of the model which can be comparable to training the model from scratch. Otherwise, a large accuracy degradation can be caused. Therefore, the training schedule should be adjusted accordingly when applying this method.
Below, we provide the steps that are required to apply Filter Pruning + QAT to the model:
## Applying Filter Pruning with fine-tuning
Applying Filter Pruning with fine-tuning
########################################
Here, we show the basic steps to modify the training script for the model and use it to zero out unimportant filters:
### 1. Import NNCF API
1. Import NNCF API
++++++++++++++++++
In this step, NNCF-related imports are added in the beginning of the training script:
@sphinxtabset
.. tab:: PyTorch
@sphinxtab{PyTorch}
.. doxygensnippet:: docs/optimization_guide/nncf/code/pruning_torch.py
:language: python
:fragment: [imports]
@snippet docs/optimization_guide/nncf/code/pruning_torch.py imports
.. tab:: TensorFlow 2
@endsphinxtab
.. doxygensnippet:: docs/optimization_guide/nncf/code/pruning_tf.py
:language: python
:fragment: [imports]
@sphinxtab{TensorFlow 2}
2. Create NNCF configuration
++++++++++++++++++++++++++++
@snippet docs/optimization_guide/nncf/code/pruning_tf.py imports
Here, you should define NNCF configuration which consists of model-related parameters (`"input_info"` section) and parameters
of optimization methods (`"compression"` section).
@endsphinxtab
.. tab:: PyTorch
@endsphinxtabset
.. doxygensnippet:: docs/optimization_guide/nncf/code/pruning_torch.py
:language: python
:fragment: [nncf_congig]
### 2. Create NNCF configuration
Here, you should define NNCF configuration which consists of model-related parameters (`"input_info"` section) and parameters of optimization methods (`"compression"` section).
.. tab:: TensorFlow 2
@sphinxtabset
.. doxygensnippet:: docs/optimization_guide/nncf/code/pruning_tf.py
:language: python
:fragment: [nncf_congig]
@sphinxtab{PyTorch}
Here is a brief description of the required parameters of the Filter Pruning method. For full description refer to the
`GitHub <https://github.com/openvinotoolkit/nncf/blob/develop/docs/compression_algorithms/Pruning.md>`__ page.
@snippet docs/optimization_guide/nncf/code/pruning_torch.py nncf_congig
* ``pruning_init`` - initial pruning rate target. For example, value ``0.1`` means that at the begging of training, convolutions that can be pruned will have 10% of their filters set to zero.
@endsphinxtab
* ``pruning_target`` - pruning rate target at the end of the schedule. For example, the value ``0.5`` means that at the epoch with the number of ``num_init_steps + pruning_steps``, convolutions that can be pruned will have 50% of their filters set to zero.
@sphinxtab{TensorFlow 2}
* ``pruning_steps` - the number of epochs during which the pruning rate target is increased from ``pruning_init` to ``pruning_target`` value. We recommend to keep the highest learning rate during this period.
@snippet docs/optimization_guide/nncf/code/pruning_tf.py nncf_congig
@endsphinxtab
3. Apply optimization methods
+++++++++++++++++++++++++++++
@endsphinxtabset
In the next step, the original model is wrapped by the NNCF object using the ``create_compressed_model()`` API using the
configuration defined in the previous step. This method returns a so-called compression controller and the wrapped model
that can be used the same way as the original model. It is worth noting that optimization methods are applied at this step
so that the model undergoes a set of corresponding transformations and can contain additional operations required for the
optimization.
Here is a brief description of the required parameters of the Filter Pruning method. For full description refer to the [GitHub](https://github.com/openvinotoolkit/nncf/blob/develop/docs/compression_algorithms/Pruning.md) page.
- `pruning_init` - initial pruning rate target. For example, value `0.1` means that at the begging of training, convolutions that can be pruned will have 10% of their filters set to zero.
- `pruning_target` - pruning rate target at the end of the schedule. For example, the value `0.5` means that at the epoch with the number of `num_init_steps + pruning_steps`, convolutions that can be pruned will have 50% of their filters set to zero.
- `pruning_steps` - the number of epochs during which the pruning rate target is increased from `pruning_init` to `pruning_target` value. We recommend to keep the highest learning rate during this period.
### 3. Apply optimization methods
In the next step, the original model is wrapped by the NNCF object using the `create_compressed_model()` API using the configuration defined in the previous step. This method returns a so-called compression controller and the wrapped model that can be used the same way as the original model. It is worth noting that optimization methods are applied at this step so that the model undergoes a set of corresponding transformations and can contain additional operations required for the optimization.
@sphinxtabset
.. tab:: PyTorch
@sphinxtab{PyTorch}
.. doxygensnippet:: docs/optimization_guide/nncf/code/pruning_torch.py
:language: python
:fragment: [wrap_model]
@snippet docs/optimization_guide/nncf/code/pruning_torch.py wrap_model
.. tab:: TensorFlow 2
@endsphinxtab
.. doxygensnippet:: docs/optimization_guide/nncf/code/pruning_tf.py
:language: python
:fragment: [wrap_model]
@sphinxtab{TensorFlow 2}
@snippet docs/optimization_guide/nncf/code/pruning_tf.py wrap_model
4. Fine-tune the model
++++++++++++++++++++++
@endsphinxtab
This step assumes that you will apply fine-tuning to the model the same way as it is done for the baseline model. In the case
of Filter Pruning method we recommend using the training schedule and learning rate similar to what was used for the training
of original model.
@endsphinxtabset
### 4. Fine-tune the model
This step assumes that you will apply fine-tuning to the model the same way as it is done for the baseline model. In the case of Filter Pruning method we recommend using the training schedule and learning rate similar to what was used for the training of original model.
.. tab:: PyTorch
@sphinxtabset
.. doxygensnippet:: docs/optimization_guide/nncf/code/pruning_torch.py
:language: python
:fragment: [tune_model]
@sphinxtab{PyTorch}
.. tab:: TensorFlow 2
@snippet docs/optimization_guide/nncf/code/pruning_torch.py tune_model
.. doxygensnippet:: docs/optimization_guide/nncf/code/pruning_tf.py
:language: python
:fragment: [tune_model]
@endsphinxtab
@sphinxtab{TensorFlow 2}
5. Multi-GPU distributed training
+++++++++++++++++++++++++++++++++
@snippet docs/optimization_guide/nncf/code/pruning_tf.py tune_model
In the case of distributed multi-GPU training (not DataParallel), you should call ``compression_ctrl.distributed()`` before the
fine-tuning that will inform optimization methods to do some adjustments to function in the distributed mode.
@endsphinxtab
@endsphinxtabset
.. tab:: PyTorch
### 5. Multi-GPU distributed training
In the case of distributed multi-GPU training (not DataParallel), you should call `compression_ctrl.distributed()` before the fine-tuning that will inform optimization methods to do some adjustments to function in the distributed mode.
@sphinxtabset
.. doxygensnippet:: docs/optimization_guide/nncf/code/pruning_torch.py
:language: python
:fragment: [distributed]
@sphinxtab{PyTorch}
.. tab:: TensorFlow 2
@snippet docs/optimization_guide/nncf/code/qat_torch.py distributed
.. doxygensnippet:: docs/optimization_guide/nncf/code/pruning_tf.py
:language: python
:fragment: [distributed]
@endsphinxtab
@sphinxtab{TensorFlow 2}
6. Export quantized model
+++++++++++++++++++++++++
@snippet docs/optimization_guide/nncf/code/qat_tf.py distributed
When fine-tuning finishes, the quantized model can be exported to the corresponding format for further inference: ONNX in
the case of PyTorch and frozen graph - for TensorFlow 2.
@endsphinxtab
@endsphinxtabset
.. tab:: PyTorch
### 6. Export quantized model
When fine-tuning finishes, the quantized model can be exported to the corresponding format for further inference: ONNX in the case of PyTorch and frozen graph - for TensorFlow 2.
.. doxygensnippet:: docs/optimization_guide/nncf/code/pruning_torch.py
:language: python
:fragment: [export]
@sphinxtabset
.. tab:: TensorFlow 2
@sphinxtab{PyTorch}
.. doxygensnippet:: docs/optimization_guide/nncf/code/pruning_tf.py
:language: python
:fragment: [export]
@snippet docs/optimization_guide/nncf/code/qat_torch.py export
@endsphinxtab
These were the basic steps to applying the QAT method from the NNCF. However, it is required in some cases to save/load model
checkpoints during the training. Since NNCF wraps the original model with its own object it provides an API for these needs.
@sphinxtab{TensorFlow 2}
@snippet docs/optimization_guide/nncf/code/qat_tf.py export
7. (Optional) Save checkpoint
+++++++++++++++++++++++++++++
@endsphinxtab
@endsphinxtabset
These were the basic steps to applying the QAT method from the NNCF. However, it is required in some cases to save/load model checkpoints during the training. Since NNCF wraps the original model with its own object it provides an API for these needs.
### 7. (Optional) Save checkpoint
To save model checkpoint use the following API:
@sphinxtabset
@sphinxtab{PyTorch}
.. tab:: PyTorch
@snippet docs/optimization_guide/nncf/code/qat_torch.py save_checkpoint
.. doxygensnippet:: docs/optimization_guide/nncf/code/pruning_torch.py
:language: python
:fragment: [save_checkpoint]
@endsphinxtab
.. tab:: TensorFlow 2
@sphinxtab{TensorFlow 2}
.. doxygensnippet:: docs/optimization_guide/nncf/code/pruning_tf.py
:language: python
:fragment: [save_checkpoint]
@snippet docs/optimization_guide/nncf/code/qat_tf.py save_checkpoint
@endsphinxtab
8. (Optional) Restore from checkpoint
+++++++++++++++++++++++++++++++++++++
@endsphinxtabset
### 8. (Optional) Restore from checkpoint
To restore the model from checkpoint you should use the following API:
@sphinxtabset
.. tab:: PyTorch
@sphinxtab{PyTorch}
.. doxygensnippet:: docs/optimization_guide/nncf/code/pruning_torch.py
:language: python
:fragment: [load_checkpoint]
@snippet docs/optimization_guide/nncf/code/qat_torch.py load_checkpoint
.. tab:: TensorFlow 2
@endsphinxtab
.. doxygensnippet:: docs/optimization_guide/nncf/code/pruning_tf.py
:language: python
:fragment: [load_checkpoint]
@sphinxtab{TensorFlow 2}
@snippet docs/optimization_guide/nncf/code/qat_tf.py load_checkpoint
For more details on saving/loading checkpoints in the NNCF, see the following
`documentation <https://github.com/openvinotoolkit/nncf/blob/develop/docs/Usage.md#saving-and-loading-compressed-models>`__.
@endsphinxtab
Deploying pruned model
######################
@endsphinxtabset
The pruned model requres an extra step that should be done to get performance improvement. This step involves removal of the
zero filters from the model. This is done at the model conversion step using :doc:`Model Optimizer <openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide>` tool when model is converted from the framework representation (ONNX, TensorFlow, etc.) to OpenVINO Intermediate Representation.
For more details on saving/loading checkpoints in the NNCF, see the following [documentation](https://github.com/openvinotoolkit/nncf/blob/develop/docs/Usage.md#saving-and-loading-compressed-models).
## Deploying pruned model
The pruned model requres an extra step that should be done to get performance improvement. This step involves removal of the zero filters from the model. This is done at the model convertion step using [Model Optimizer](@ref openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide) tool when model is converted from the framework representation (ONNX, TensorFlow, etc.) to OpenVINO Intermediate Representation.
- To remove zero filters from the pruned model add the following parameter to the model convertion command: `--transform=Pruning`
* To remove zero filters from the pruned model add the following parameter to the model convertion command: ``--transform=Pruning``
After that the model can be deployed with OpenVINO in the same way as the baseline model.
For more details about model deployment with OpenVINO, see the corresponding [documentation](../../OV_Runtime_UG/openvino_intro.md).
For more details about model deployment with OpenVINO, see the corresponding :doc:`documentation <openvino_docs_OV_UG_OV_Runtime_User_Guide>`.
## Examples
- [PyTorch Image Classiication example](https://github.com/openvinotoolkit/nncf/blob/develop/examples/torch/classification)
- [TensorFlow Image Classification example](https://github.com/openvinotoolkit/nncf/tree/develop/examples/tensorflow/classification)
Examples
####################
* `PyTorch Image Classiication example <https://github.com/openvinotoolkit/nncf/blob/develop/examples/torch/classification>`__
* `TensorFlow Image Classification example <https://github.com/openvinotoolkit/nncf/tree/develop/examples/tensorflow/classification>`__
@endsphinxdirective

200
docs/optimization_guide/nncf/ptq/basic_quantization_flow.md
View File

@ -1,135 +1,161 @@
# Basic Quantization Flow {#basic_qauntization_flow}
## Introduction
@sphinxdirective
Introduction
####################
The basic quantization flow is the simplest way to apply 8-bit quantization to the model. It is available for models in the following frameworks: PyTorch, TensorFlow 2.x, ONNX, and OpenVINO. The basic quantization flow is based on the following steps:
* Set up an environment and install dependencies.
* Prepare the **calibration dataset** that is used to estimate quantization parameters of the activations within the model.
* Call the quantization API to apply 8-bit quantization to the model.
## Set up an Environment
Set up an Environment
#####################
It is recommended to set up a separate Python environment for quantization with NNCF. To do this, run the following command:
```bash
python3 -m venv nncf_ptq_env
```
.. code-block:: sh
python3 -m venv nncf_ptq_env
Install all the packages required to instantiate the model object, for example, DL framework. After that, install NNCF on top of the environment:
```bash
pip install nncf
```
## Prepare a Calibration Dataset
.. code-block:: sh
At this step, create an instance of the `nncf.Dataset` class that represents the calibration dataset. The `nncf.Dataset` class can be a wrapper over the framework dataset object that is used for model training or validation. The class constructor receives the dataset object and the transformation function. For example, if you use PyTorch, you can pass an instance of the `torch.utils.data.DataLoader` object.
pip install nncf
The transformation function is a function that takes a sample from the dataset and returns data that can be passed to the model for inference. For example, this function can take a tuple of a data tensor and labels tensor, and return the former while ignoring the latter. The transformation function is used to avoid modifying the dataset code to make it compatible with the quantization API. The function is applied to each sample from the dataset before passing it to the model for inference. The following code snippet shows how to create an instance of the `nncf.Dataset` class:
Prepare a Calibration Dataset
#############################
@sphinxtabset
At this step, create an instance of the ``nncf.Dataset`` class that represents the calibration dataset. The ``nncf.Dataset`` class can be a wrapper over the framework dataset object that is used for model training or validation. The class constructor receives the dataset object and the transformation function. For example, if you use PyTorch, you can pass an instance of the ``torch.utils.data.DataLoader`` object.
@sphinxtab{PyTorch}
The transformation function is a function that takes a sample from the dataset and returns data that can be passed to the model for inference. For example, this function can take a tuple of a data tensor and labels tensor, and return the former while ignoring the latter. The transformation function is used to avoid modifying the dataset code to make it compatible with the quantization API. The function is applied to each sample from the dataset before passing it to the model for inference. The following code snippet shows how to create an instance of the ``nncf.Dataset`` class:
@snippet docs/optimization_guide/nncf/ptq/code/ptq_torch.py dataset
.. tab:: PyTorch
@endsphinxtab
.. doxygensnippet:: docs/optimization_guide/nncf/ptq/code/ptq_torch.py
:language: python
:fragment: [dataset]
@sphinxtab{ONNX}
.. tab:: ONNX
@snippet docs/optimization_guide/nncf/ptq/code/ptq_onnx.py dataset
.. doxygensnippet:: docs/optimization_guide/nncf/ptq/code/ptq_onnx.py
:language: python
:fragment: [dataset]
@endsphinxtab
.. tab:: OpenVINO
@sphinxtab{OpenVINO}
.. doxygensnippet:: docs/optimization_guide/nncf/ptq/code/ptq_openvino.py
:language: python
:fragment: [dataset]
@snippet docs/optimization_guide/nncf/ptq/code/ptq_openvino.py dataset
.. tab:: TensorFlow
@endsphinxtab
@sphinxtab{TensorFlow}
@snippet docs/optimization_guide/nncf/ptq/code/ptq_tensorflow.py dataset
@endsphinxtab
@endsphinxtabset
If there is no framework dataset object, you can create your own entity that implements the `Iterable` interface in Python and returns data samples feasible for inference. In this case, a transformation function is not required.
.. doxygensnippet:: docs/optimization_guide/nncf/ptq/code/ptq_tensorflow.py
:language: python
:fragment: [dataset]
## Run a Quantized Model
If there is no framework dataset object, you can create your own entity that implements the ``Iterable`` interface in Python and returns data samples feasible for inference. In this case, a transformation function is not required.
Run a Quantized Model
#####################
Once the dataset is ready and the model object is instantiated, you can apply 8-bit quantization to it:
@sphinxtabset
@sphinxtab{PyTorch}
.. tab:: PyTorch
@snippet docs/optimization_guide/nncf/ptq/code/ptq_torch.py quantization
.. doxygensnippet:: docs/optimization_guide/nncf/ptq/code/ptq_torch.py
:language: python
:fragment: [quantization]
@endsphinxtab
.. tab:: ONNX
@sphinxtab{ONNX}
.. doxygensnippet:: docs/optimization_guide/nncf/ptq/code/ptq_onnx.py
:language: python
:fragment: [quantization]
@snippet docs/optimization_guide/nncf/ptq/code/ptq_torch.py quantization
.. tab:: OpenVINO
@endsphinxtab
.. doxygensnippet:: docs/optimization_guide/nncf/ptq/code/ptq_openvino.py
:language: python
:fragment: [quantization]
@sphinxtab{OpenVINO}
.. tab:: TensorFlow
@snippet docs/optimization_guide/nncf/ptq/code/ptq_torch.py quantization
.. doxygensnippet:: docs/optimization_guide/nncf/ptq/code/ptq_tensorflow.py
:language: python
:fragment: [quantization]
@endsphinxtab
@sphinxtab{TensorFlow}
@snippet docs/optimization_guide/nncf/ptq/code/ptq_tensorflow.py quantization
@endsphinxtab
@endsphinxtabset
> **NOTE**: The `model` is an instance of the `torch.nn.Module` class for PyTorch, `onnx.ModelProto` for ONNX, and `openvino.runtime.Model` for OpenVINO.
.. note:: The ``model`` is an instance of the ``torch.nn.Module`` class for PyTorch, ``onnx.ModelProto`` for ONNX, and ``openvino.runtime.Model`` for OpenVINO.
After that the model can be exported into th OpenVINO Intermediate Representation if needed and run faster with OpenVINO.
## Tune quantization parameters
Tune quantization parameters
############################
``nncf.quantize()`` function has several parameters that allow to tune quantization process to get more accurate model. Below is the list of parameters and their description:
* ``model_type`` - used to specify quantization scheme required for specific type of the model. For example, **Transformer** models (BERT, distillBERT, etc.) require a special quantization scheme to preserve accuracy after quantization.
.. code-block:: sh
nncf.quantize(model, dataset, model_type=nncf.ModelType.Transformer)
* ``preset`` - defines quantization scheme for the model. Two types of presets are available:
* ``PERFORMANCE`` (default) - defines symmetric quantization of weights and activations
* ``MIXED`` - weights are quantized with symmetric quantization and the activations are quantized with asymmetric quantization. This preset is recommended for models with non-ReLU and asymmetric activation functions, e.g. ELU, PReLU, GELU, etc.
.. code-block:: sh
nncf.quantize(model, dataset, preset=nncf.Preset.MIXED)
* ``fast_bias_correction`` - enables more accurate bias (error) correction algorithm that can be used to improve accuracy of the model. This parameter is available only for OpenVINO representation. ``True`` is used by default.
.. code-block:: sh
nncf.quantize(model, dataset, fast_bias_correction=False)
* ``subset_size`` - defines the number of samples from the calibration dataset that will be used to estimate quantization parameters of activations. The default value is 300.
.. code-block:: sh
nncf.quantize(model, dataset, subset_size=1000)
* ``ignored_scope`` - this parameter can be used to exclude some layers from quantization process. For example, if you want to exclude the last layer of the model from quantization. Below are some examples of how to use this parameter:
`nncf.quantize()` function has several parameters that allow to tune quantization process to get more accurate model. Below is the list of parameters and their description:
* `model_type` - used to specify quantization scheme required for specific type of the model. For example, **Transformer** models (BERT, distillBERT, etc.) require a special quantization scheme to preserve accuracy after quantization.
```python
nncf.quantize(model, dataset, model_type=nncf.ModelType.Transformer)
```
* `preset` - defines quantization scheme for the model. Two types of presets are available:
* `PERFORMANCE` (default) - defines symmetric quantization of weigths and activations
* `MIXED` - weights are quantized with symmetric quantization and the activations are quantized with asymmetric quantization. This preset is recommended for models with non-ReLU and asymmetric activation funstions, e.g. ELU, PReLU, GELU, etc.
```python
nncf.quantize(model, dataset, preset=nncf.Preset.MIXED)
```
* `fast_bias_correction` - enables more accurate bias (error) correction algorithm that can be used to improve accuracy of the model. This parameter is available only for OpenVINO representation. `True` is used by default.
```python
nncf.quantize(model, dataset, fast_bias_correction=False)
```
* `subset_size` - defines the number of samples from the calibration dataset that will be used to estimate quantization parameters of activations. The default value is 300.
```python
nncf.quantize(model, dataset, subset_size=1000)
```
* `ignored_scope` - this parameter can be used to exclude some layers from quantization process. For example, if you want to exclude the last layer of the model from quantization. Below are some examples of how to use this parameter:
* Exclude by layer name:
```python
names = ['layer_1', 'layer_2', 'layer_3']
nncf.quantize(model, dataset, ignored_scope=nncf.IgnoredScope(names=names))
```
.. code-block:: sh
names = ['layer_1', 'layer_2', 'layer_3']
nncf.quantize(model, dataset, ignored_scope=nncf.IgnoredScope(names=names))
* Exclude by layer type:
```python
types = ['Conv2d', 'Linear']
nncf.quantize(model, dataset, ignored_scope=nncf.IgnoredScope(types=types))
```
.. code-block:: sh
types = ['Conv2d', 'Linear']
nncf.quantize(model, dataset, ignored_scope=nncf.IgnoredScope(types=types))
* Exclude by regular expression:
```python
regex = '.*layer_.*'
nncf.quantize(model, dataset, ignored_scope=nncf.IgnoredScope(patterns=regex))
```
If the accuracy of the quantized model is not satisfactory, you can try to use the [Quantization with accuracy control](@ref quantization_w_accuracy_control) flow.
.. code-block:: sh
## See also
regex = '.*layer_.*'
nncf.quantize(model, dataset, ignored_scope=nncf.IgnoredScope(patterns=regex))
* [Example of basic quantization flow in PyTorch](https://github.com/openvinotoolkit/nncf/tree/develop/examples/post_training_quantization/torch/mobilenet_v2)
If the accuracy of the quantized model is not satisfactory, you can try to use the :doc:`Quantization with accuracy control <quantization_w_accuracy_control>` flow.
See also
####################
* `Example of basic quantization flow in PyTorch <https://github.com/openvinotoolkit/nncf/tree/develop/examples/post_training_quantization/torch/mobilenet_v2>`__
@endsphinxdirective

104
docs/optimization_guide/nncf/ptq/quantization_w_accuracy_control.md
View File

@ -1,66 +1,64 @@
# Quantizing with accuracy control {#quantization_w_accuracy_control}
## Introduction
@sphinxdirective
This is the advanced quantization flow that allows to apply 8-bit quantization to the model with control of accuracy metric. This is achieved by keeping the most impactful operations within the model in the original precision. The flow is based on the [Basic 8-bit quantization](@ref basic_qauntization_flow) and has the following differences:
* Besided the calibration dataset, a **validation dataset** is required to compute accuracy metric. They can refer to the same data in the simplest case.
Introduction
####################
This is the advanced quantization flow that allows to apply 8-bit quantization to the model with control of accuracy metric. This is achieved by keeping the most impactful operations within the model in the original precision. The flow is based on the :doc:`Basic 8-bit quantization <basic_qauntization_flow>` and has the following differences:
* Beside the calibration dataset, a **validation dataset** is required to compute accuracy metric. They can refer to the same data in the simplest case.
* **Validation function**, used to compute accuracy metric is required. It can be a function that is already available in the source framework or a custom function.
* Since accuracy validation is run several times during the quantization process, quantization with accuracy control can take more time than the [Basic 8-bit quantization](@ref basic_qauntization_flow) flow.
* The resulted model can provide smaller performance improvement than the [Basic 8-bit quantization](@ref basic_qauntization_flow) flow because some of the operations are kept in the original precision.
* The resulted model can provide smaller performance improvement than the :doc:`Basic 8-bit quantization <basic_qauntization_flow>` flow because some of the operations are kept in the original precision.
> **NOTE**: Currently, this flow is available only for models in OpenVINO representation.
.. note:: Currently, this flow is available only for models in OpenVINO representation.
The steps for the quantizatation with accuracy control are described below.
The steps for the quantization with accuracy control are described below.
## Prepare datasets
Prepare datasets
####################
This step is similar to the [Basic 8-bit quantization](@ref basic_qauntization_flow) flow. The only difference is that two datasets, calibration and validation, are required.
This step is similar to the :doc:`Basic 8-bit quantization <basic_qauntization_flow>` flow. The only difference is that two datasets, calibration and validation, are required.
@sphinxtabset
.. tab:: OpenVINO
@sphinxtab{OpenVINO}
@snippet docs/optimization_guide/nncf/ptq/code/ptq_aa_openvino.py dataset
@endsphinxtab
@endsphinxtabset
## Prepare validation function
Validation funtion receives `openvino.runtime.CompiledModel` object and
validation dataset and returns accuracy metric value. The following code snippet shows an example of validation function for OpenVINO model:
@sphinxtabset
@sphinxtab{OpenVINO}
@snippet docs/optimization_guide/nncf/ptq/code/ptq_aa_openvino.py validation
@endsphinxtab
@endsphinxtabset
## Run quantization with accuracy control
Now, you can run quantization with accuracy control. The following code snippet shows an example of quantization with accuracy control for OpenVINO model:
@sphinxtabset
@sphinxtab{OpenVINO}
@snippet docs/optimization_guide/nncf/ptq/code/ptq_aa_openvino.py quantization
@endsphinxtab
@endsphinxtabset
`max_drop` defines the accuracy drop threshold. The quantization process stops when the degradation of accuracy metric on the validation dataset is less than the `max_drop`.
`nncf.quantize_with_accuracy_control()` API supports all the parameters of `nncf.quantize()` API. For example, you can use `nncf.quantize_with_accuracy_control()` to quantize a model with a custom configuration.
## See also
* [Optimizing Models at Training Time](@ref tmo_introduction)
.. doxygensnippet:: docs/optimization_guide/nncf/ptq/code/ptq_aa_openvino.py
:language: python
:fragment: [dataset]
Prepare validation function
###########################
Validation funtion receives ``openvino.runtime.CompiledModel`` object and validation dataset and returns accuracy metric value. The following code snippet shows an example of validation function for OpenVINO model:
.. tab:: OpenVINO
.. doxygensnippet:: docs/optimization_guide/nncf/ptq/code/ptq_aa_openvino.py
:language: python
:fragment: [validation]
Run quantization with accuracy control
Now, you can run quantization with accuracy control. The following code snippet shows an example of quantization with accuracy control for OpenVINO model:
.. tab:: OpenVINO
.. doxygensnippet:: docs/optimization_guide/nncf/ptq/code/ptq_aa_openvino.py
:language: python
:fragment: [quantization]
``max_drop`` defines the accuracy drop threshold. The quantization process stops when the degradation of accuracy metric on the validation dataset is less than the ``max_drop``.
``nncf.quantize_with_accuracy_control()`` API supports all the parameters of ``nncf.quantize()`` API. For example, you can use ``nncf.quantize_with_accuracy_control()`` to quantize a model with a custom configuration.
See also
####################
* :doc:`Optimizing Models at Training Time <tmo_introduction>`
@endsphinxdirective

233
docs/optimization_guide/nncf/qat.md
View File

@ -1,172 +1,201 @@
# Quantization-aware Training (QAT) {#qat_introduction}
## Introduction
Quantization-aware Training is a popular method that allows quantizing a model and applying fine-tuning to restore accuracy degradation caused by quantization. In fact, this is the most accurate quantization method. This document describes how to apply QAT from the Neural Network Compression Framework (NNCF) to get 8-bit quantized models. This assumes that you are knowledgeable in Python* programming and familiar with the training code for the model in the source DL framework.
@sphinxdirective
## Using NNCF QAT
Here, we provide the steps that are required to integrate QAT from NNCF into the training script written with PyTorch or TensorFlow 2:
Introduction
####################
> **NOTE**: Currently, NNCF for TensorFlow 2 supports optimization of the models created using Keras [Sequesntial API](https://www.tensorflow.org/guide/keras/sequential_model) or [Functional API](https://www.tensorflow.org/guide/keras/functional).
Quantization-aware Training is a popular method that allows quantizing a model and applying fine-tuning to restore accuracy
degradation caused by quantization. In fact, this is the most accurate quantization method. This document describes how to
apply QAT from the Neural Network Compression Framework (NNCF) to get 8-bit quantized models. This assumes that you are
knowledgeable in Python programming and familiar with the training code for the model in the source DL framework.
Using NNCF QAT
####################
Here, we provide the steps that are required to integrate QAT from NNCF into the training script written with
PyTorch or TensorFlow 2:
.. note::
Currently, NNCF for TensorFlow 2 supports optimization of the models created using Keras
`Sequential API <https://www.tensorflow.org/guide/keras/sequential_model>`__ or
`Functional API <https://www.tensorflow.org/guide/keras/functional>`__.
1. Import NNCF API
++++++++++++++++++++
### 1. Import NNCF API
In this step, you add NNCF-related imports in the beginning of the training script:
@sphinxtabset
.. tab:: PyTorch
@sphinxtab{PyTorch}
.. doxygensnippet:: docs/optimization_guide/nncf/code/qat_torch.py
:language: python
:fragment: [imports]
@snippet docs/optimization_guide/nncf/code/qat_torch.py imports
.. tab:: TensorFlow 2
@endsphinxtab
.. doxygensnippet:: docs/optimization_guide/nncf/code/qat_tf.py
:language: python
:fragment: [imports]
@sphinxtab{TensorFlow 2}
@snippet docs/optimization_guide/nncf/code/qat_tf.py imports
2. Create NNCF configuration
++++++++++++++++++++++++++++
@endsphinxtab
Here, you should define NNCF configuration which consists of model-related parameters (``"input_info"`` section) and parameters
of optimization methods (``"compression"`` section). For faster convergence, it is also recommended to register a dataset object
specific to the DL framework. It will be used at the model creation step to initialize quantization parameters.
@endsphinxtabset
.. tab:: PyTorch
### 2. Create NNCF configuration
Here, you should define NNCF configuration which consists of model-related parameters (`"input_info"` section) and parameters of optimization methods (`"compression"` section). For faster convergence, it is also recommended to register a dataset object specific to the DL framework. It will be used at the model creation step to initialize quantization parameters.
.. doxygensnippet:: docs/optimization_guide/nncf/code/qat_torch.py
:language: python
:fragment: [nncf_congig]
@sphinxtabset
.. tab:: TensorFlow 2
@sphinxtab{PyTorch}
.. doxygensnippet:: docs/optimization_guide/nncf/code/qat_tf.py
:language: python
:fragment: [nncf_congig]
@snippet docs/optimization_guide/nncf/code/qat_torch.py nncf_congig
@endsphinxtab
3. Apply optimization methods
+++++++++++++++++++++++++++++
@sphinxtab{TensorFlow 2}
In the next step, you need to wrap the original model object with the ``create_compressed_model()`` API using the configuration
defined in the previous step. This method returns a so-called compression controller and a wrapped model that can be used the
same way as the original model. It is worth noting that optimization methods are applied at this step so that the model
undergoes a set of corresponding transformations and can contain additional operations required for the optimization. In
the case of QAT, the compression controller object is used for model export and, optionally, in distributed training as it
will be shown below.
@snippet docs/optimization_guide/nncf/code/qat_tf.py nncf_congig
.. tab:: PyTorch
@endsphinxtab
.. doxygensnippet:: docs/optimization_guide/nncf/code/qat_torch.py
:language: python
:fragment: [wrap_model]
@endsphinxtabset
.. tab:: TensorFlow 2
### 3. Apply optimization methods
In the next step, you need to wrap the original model object with the `create_compressed_model()` API using the configuration defined in the previous step. This method returns a so-called compression controller and a wrapped model that can be used the same way as the original model. It is worth noting that optimization methods are applied at this step so that the model undergoes a set of corresponding transformations and can contain additional operations required for the optimization. In the case of QAT, the compression controller object is used for model export and, optionally, in distributed training as it will be shown below.
.. doxygensnippet:: docs/optimization_guide/nncf/code/qat_tf.py
:language: python
:fragment: [wrap_model]
@sphinxtabset
@sphinxtab{PyTorch}
4. Fine-tune the model
++++++++++++++++++++++
@snippet docs/optimization_guide/nncf/code/qat_torch.py wrap_model
This step assumes that you will apply fine-tuning to the model the same way as it is done for the baseline model. In the
case of QAT, it is required to train the model for a few epochs with a small learning rate, for example, 10e-5. In principle,
you can skip this step which means that the post-training optimization will be applied to the model.
@endsphinxtab
.. tab:: PyTorch
@sphinxtab{TensorFlow 2}
.. doxygensnippet:: docs/optimization_guide/nncf/code/qat_torch.py
:language: python
:fragment: [tune_model]
@snippet docs/optimization_guide/nncf/code/qat_tf.py wrap_model
.. tab:: TensorFlow 2
@endsphinxtab
.. doxygensnippet:: docs/optimization_guide/nncf/code/qat_tf.py
:language: python
:fragment: [tune_model]
@endsphinxtabset
### 4. Fine-tune the model
This step assumes that you will apply fine-tuning to the model the same way as it is done for the baseline model. In the case of QAT, it is required to train the model for a few epochs with a small learning rate, for example, 10e-5. In principle, you can skip this step which means that the post-training optimization will be applied to the model.
5. Multi-GPU distributed training
+++++++++++++++++++++++++++++++++
@sphinxtabset
In the case of distributed multi-GPU training (not DataParallel), you should call ``compression_ctrl.distributed()`` before
the fine-tuning that will inform optimization methods to do some adjustments to function in the distributed mode.
@sphinxtab{PyTorch}
.. tab:: PyTorch
@snippet docs/optimization_guide/nncf/code/qat_torch.py tune_model
.. doxygensnippet:: docs/optimization_guide/nncf/code/qat_torch.py
:language: python
:fragment: [distributed]
@endsphinxtab
.. tab:: TensorFlow 2
@sphinxtab{TensorFlow 2}
.. doxygensnippet:: docs/optimization_guide/nncf/code/qat_tf.py
:language: python
:fragment: [distributed]
@snippet docs/optimization_guide/nncf/code/qat_tf.py tune_model
6. Export quantized model
+++++++++++++++++++++++++
@endsphinxtab
When fine-tuning finishes, the quantized model can be exported to the corresponding format for further inference: ONNX in
the case of PyTorch and frozen graph - for TensorFlow 2.
@endsphinxtabset
.. tab:: PyTorch
### 5. Multi-GPU distributed training
In the case of distributed multi-GPU training (not DataParallel), you should call `compression_ctrl.distributed()` before the fine-tuning that will inform optimization methods to do some adjustments to function in the distributed mode.
@sphinxtabset
.. doxygensnippet:: docs/optimization_guide/nncf/code/qat_torch.py
:language: python
:fragment: [export]
@sphinxtab{PyTorch}
.. tab:: TensorFlow 2
@snippet docs/optimization_guide/nncf/code/qat_torch.py distributed
.. doxygensnippet:: docs/optimization_guide/nncf/code/qat_tf.py
:language: python
:fragment: [export]
@endsphinxtab
@sphinxtab{TensorFlow 2}
.. note::
The precision of weigths gets INT8 only after the step of model conversion to OpenVINO Intermediate Representation.
You can expect the model footprint reduction only for that format.
@snippet docs/optimization_guide/nncf/code/qat_tf.py distributed
@endsphinxtab
These were the basic steps to applying the QAT method from the NNCF. However, it is required in some cases to save/load model
checkpoints during the training. Since NNCF wraps the original model with its own object it provides an API for these needs.
@endsphinxtabset
7. (Optional) Save checkpoint
+++++++++++++++++++++++++++++
### 6. Export quantized model
When fine-tuning finishes, the quantized model can be exported to the corresponding format for further inference: ONNX in the case of PyTorch and frozen graph - for TensorFlow 2.
@sphinxtabset
@sphinxtab{PyTorch}
@snippet docs/optimization_guide/nncf/code/qat_torch.py export
@endsphinxtab
@sphinxtab{TensorFlow 2}
@snippet docs/optimization_guide/nncf/code/qat_tf.py export
@endsphinxtab
@endsphinxtabset
> **NOTE**: The precision of weigths gets INT8 only after the step of model conversion to OpenVINO Intermediate Representation. You can expect the model footprint reduction only for that format.
These were the basic steps to applying the QAT method from the NNCF. However, it is required in some cases to save/load model checkpoints during the training. Since NNCF wraps the original model with its own object it provides an API for these needs.
### 7. (Optional) Save checkpoint
To save model checkpoint use the following API:
@sphinxtabset
.. tab:: PyTorch
@sphinxtab{PyTorch}
.. doxygensnippet:: docs/optimization_guide/nncf/code/qat_torch.py
:language: python
:fragment: [save_checkpoint]
@snippet docs/optimization_guide/nncf/code/qat_torch.py save_checkpoint
.. tab:: TensorFlow 2
@endsphinxtab
.. doxygensnippet:: docs/optimization_guide/nncf/code/qat_tf.py
:language: python
:fragment: [save_checkpoint]
@sphinxtab{TensorFlow 2}
@snippet docs/optimization_guide/nncf/code/qat_tf.py save_checkpoint
8. (Optional) Restore from checkpoint
+++++++++++++++++++++++++++++++++++++
@endsphinxtab
@endsphinxtabset
### 8. (Optional) Restore from checkpoint
To restore the model from checkpoint you should use the following API:
@sphinxtabset
.. tab:: PyTorch
@sphinxtab{PyTorch}
.. doxygensnippet:: docs/optimization_guide/nncf/code/qat_torch.py
:language: python
:fragment: [load_checkpoint]
@snippet docs/optimization_guide/nncf/code/qat_torch.py load_checkpoint
.. tab:: TensorFlow 2
@endsphinxtab
.. doxygensnippet:: docs/optimization_guide/nncf/code/qat_tf.py
:language: python
:fragment: [load_checkpoint]
@sphinxtab{TensorFlow 2}
@snippet docs/optimization_guide/nncf/code/qat_tf.py load_checkpoint
For more details on saving/loading checkpoints in the NNCF, see the following `documentation <https://github.com/openvinotoolkit/nncf/blob/develop/docs/Usage.md#saving-and-loading-compressed-models>`__.
@endsphinxtab
Deploying quantized model
#########################
@endsphinxtabset
The quantized model can be deployed with OpenVINO in the same way as the baseline model. No extra steps or options are
required in this case. For more details, see the corresponding :doc:`documentation <openvino_docs_OV_UG_OV_Runtime_User_Guide>`.
For more details on saving/loading checkpoints in the NNCF, see the following [documentation](https://github.com/openvinotoolkit/nncf/blob/develop/docs/Usage.md#saving-and-loading-compressed-models).
Examples
####################
## Deploying quantized model
The quantized model can be deployed with OpenVINO in the same way as the baseline model. No extra steps or options are required in this case. For more details, see the corresponding [documentation](../../OV_Runtime_UG/openvino_intro.md).
* `Quantizing PyTorch model with NNCF <https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/302-pytorch-quantization-aware-training>`__
## Examples
- [Quantizing PyTorch model with NNCF](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/302-pytorch-quantization-aware-training)
- [Quantizing TensorFlow model with NNCF](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/305-tensorflow-quantization-aware-training)
* `Quantizing TensorFlow model with NNCF <https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/305-tensorflow-quantization-aware-training>`__
@endsphinxdirective

33
docs/resources/prerelease_information.md Normal file
View File

@ -0,0 +1,33 @@
# Pre-release Information {#prerelease_information}
@sphinxdirective
To ensure you do not have to wait long to test OpenVINO's upcomming features,
OpenVINO developers continue to roll out prerelease versions. In this page you can find
a general changelog and the schedule for all versions for the current year.
.. note::
These versions are pre-release software and have not undergone full validation or qualification. OpenVINO™ toolkit pre-release is:
* NOT to be incorporated into production software/solutions.
* NOT subject to official support.
* Subject to change in the future.
* Introduced to allow early testing and get early feedback from the community.
.. dropdown:: OpenVINO Toolkit 2023.0.0.dev20230217
:open:
:animate: fade-in-slide-down
:color: primary
OpenVINO™ repository tag: `2023.0.0.dev20230217 <https://github.com/openvinotoolkit/openvino/releases/tag/2023.0.0.dev20230217>`__
* Enabled PaddlePaddle Framework 2.4
* Preview of TensorFlow Lite Front End – Load models directly via “read_model” into OpenVINO Runtime and export OpenVINO IR format using Model Optimizer or “convert_model”
* Model Optimizer now uses the TensorFlow Frontend as the default path for conversion to IR. Known limitations compared to the legacy approach are: TF1 Loop, Complex types, models requiring config files and old python extensions. The solution detects unsupported functionalities and provides fallback. To force using the legacy Frontend "--use_legacy_fronted" can be specified.
* Model Optimizer now supports out-of-the-box conversion of TF2 Object Detection models. At this point, same performance experience is guaranteed only on CPU devices. Feel free to start enjoying TF2 Object Detection models without config files!
* Introduced new option ov::auto::enable_startup_fallback / ENABLE_STARTUP_FALLBACK to control whether to use CPU to accelerate first inference latency for accelerator HW devices like GPU.
* New FrontEndManager register_front_end(name, lib_path) interface added, to remove “OV_FRONTEND_PATH” env var (a way to load non-default frontends).
@endsphinxdirective

1
docs/resources/resources.md
View File

@ -9,6 +9,7 @@
openvino_docs_performance_benchmarks
openvino_ir
prerelease_information
.. toctree::
:maxdepth: 1

37
docs/resources/supported_models.md
View File

@ -1,13 +1,12 @@
# Supported Models {#openvino_supported_models}
@sphinxdirective
The OpenVINO team continues the effort to support as many models out-of-the-box as possible.
Based on our research and user feedback, we prioritize the most common models and test them
before every release. These models are considered officially supported.
@sphinxdirective
.. button-link:: _static/download/OV_2022_models_supported.pdf
.. button-link:: _static/download/OV_2023_models_supported.pdf
:color: primary
:outline:
@ -18,36 +17,33 @@ before every release. These models are considered officially supported.
| If your model is not included but is similar to those that are, it is still very likely to work.
If your model fails to execute properly there are a few options available:
@endsphinxdirective
* If the model originates from a framework like TensorFlow or PyTorch, OpenVINO™ offers a hybrid solution. The original model can be run without explicit conversion into the OpenVINO format. For more information, see [OpenVINO TensorFlow Integration](https://docs.openvino.ai/latest/ovtf_integration.html).
* If the model originates from a framework like TensorFlow or PyTorch, OpenVINO™ offers a hybrid solution. The original model can be run without explicit conversion into the OpenVINO format. For more information, see :ref:`OpenVINO TensorFlow Integration <https://docs.openvino.ai/latest/ovtf_integration.html>`.
* You can create a GitHub request for the operation(s) that are missing. These requests are reviewed regularly. You will be informed if and how the request will be accommodated. Additionally, your request may trigger a reply from someone in the community who can help.
* As OpenVINO™ is open source you can enhance it with your own contribution to the GitHub repository. To learn more, see the articles on [OpenVINO Extensibility](https://docs.openvino.ai/latest/openvino_docs_Extensibility_UG_Intro.html).
* As OpenVINO™ is open source you can enhance it with your own contribution to the GitHub repository. To learn more, see the articles on :ref:`OpenVINO Extensibility<https://docs.openvino.ai/latest/openvino_docs_Extensibility_UG_Intro.html>`.
The following table summarizes the number of models supported by OpenVINO™ in different categories:
@sphinxdirective
+--------------------------------------------+-------------------+
| Model Categories: | Number of Models: |
+============================================+===================+
| Object Detection | 149 |
| Object Detection | 149 |
+--------------------------------------------+-------------------+
| Instance Segmentation | 3 |
+--------------------------------------------+-------------------+
| Semantic Segmentation | 19 |
+--------------------------------------------+-------------------+
| Image Processing, Enhancement | 16 |
| Image Processing, Enhancement | 16 |
+--------------------------------------------+-------------------+
| Monodepth | 2 |
| Monodepth | 2 |
+--------------------------------------------+-------------------+
| Colorization | 2 |
| Colorization | 2 |
+--------------------------------------------+-------------------+
| Behavior / Decision Prediction | 1 |
| Behavior / Decision Prediction | 1 |
+--------------------------------------------+-------------------+
| Action Recognition | 2 |
| Action Recognition | 2 |
+--------------------------------------------+-------------------+
| Time Series Forecasting | 1 |
| Time Series Forecasting | 1 |
+--------------------------------------------+-------------------+
| Image Classification | 68 |
+--------------------------------------------+-------------------+
@ -55,14 +51,15 @@ The following table summarizes the number of models supported by OpenVINO™ in
+--------------------------------------------+-------------------+
| Image Classification, Emotion | 1 |
+--------------------------------------------+-------------------+
| Image Translation | 1 |
| Image Translation | 1 |
+--------------------------------------------+-------------------+
| Natural language Processing | 35 |
| Natural language Processing | 35 |
+--------------------------------------------+-------------------+
| Text Detection | 18 |
| Text Detection | 18 |
+--------------------------------------------+-------------------+
| Audio Enhancement | 3 |
| Audio Enhancement | 3 |
+--------------------------------------------+-------------------+
| Sound Classification | 2 |
| Sound Classification | 2 |
+--------------------------------------------+-------------------+
@endsphinxdirective

18
docs/snippets/CMakeLists.txt
View File

@ -17,16 +17,12 @@ endif()
file(GLOB SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/src/*.c"
"${CMAKE_CURRENT_SOURCE_DIR}/gpu/*.cpp")
"${CMAKE_CURRENT_SOURCE_DIR}/src/*.c")
file(GLOB GPU_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/gpu/*.cpp")
# remove GPU remote snippets if OpenCL hasn't been found
if (NOT TARGET OpenCL::OpenCL)
list(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/gpu/context_sharing_va.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/gpu/context_sharing.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/gpu/preprocessing.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/gpu/queue_sharing.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/gpu/remote_objects_creation.cpp")
# add GPU snippets if OpenCL has been found
if(TARGET OpenCL::OpenCL)
list(APPEND SOURCES ${GPU_SOURCES})
endif()
# try to find VA libraries
@ -51,10 +47,6 @@ list(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/dldt_optimization_guide2.c
"${CMAKE_CURRENT_SOURCE_DIR}/dldt_optimization_guide3.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/dldt_optimization_guide4.cpp")
# build separatelly as ov_integration_snippet and ov_integration_snippet_c
list(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/main.c"
"${CMAKE_CURRENT_SOURCE_DIR}/src/main.cpp")
# create a static library
add_library(${TARGET_NAME} STATIC ${SOURCES})

47
docs/snippets/gpu/preprocessing_nv12_single_plane.cpp Normal file
View File

@ -0,0 +1,47 @@
#include <openvino/runtime/core.hpp>
#include <openvino/runtime/intel_gpu/ocl/ocl.hpp>
#include <openvino/runtime/intel_gpu/properties.hpp>
#include <openvino/core/preprocess/pre_post_process.hpp>
ov::intel_gpu::ocl::ClImage2DTensor get_yuv_tensor();
int main() {
ov::Core core;
auto model = core.read_model("model.xml");
//! [init_preproc]
using namespace ov::preprocess;
auto p = PrePostProcessor(model);
p.input().tensor().set_element_type(ov::element::u8)
.set_color_format(ColorFormat::NV12_SINGLE_PLANE)
.set_memory_type(ov::intel_gpu::memory_type::surface);
p.input().preprocess().convert_color(ov::preprocess::ColorFormat::BGR);
p.input().model().set_layout("NCHW");
auto model_with_preproc = p.build();
//! [init_preproc]
auto compiled_model = core.compile_model(model_with_preproc, "GPU");
auto context = compiled_model.get_context().as<ov::intel_gpu::ocl::ClContext>();
auto infer_request = compiled_model.create_infer_request();
{
//! [single_batch]
auto input_yuv = model_with_preproc->input(0);
ov::intel_gpu::ocl::ClImage2DTensor yuv_tensor = get_yuv_tensor();
infer_request.set_tensor(input_yuv.get_any_name(), yuv_tensor);
infer_request.infer();
//! [single_batch]
}
{
auto yuv_tensor_0 = get_yuv_tensor();
auto yuv_tensor_1 = get_yuv_tensor();
//! [batched_case]
auto input_yuv = model_with_preproc->input(0);
std::vector<ov::Tensor> yuv_tensors = {yuv_tensor_0, yuv_tensor_1};
infer_request.set_tensors(input_yuv.get_any_name(), yuv_tensors);
infer_request.infer();
//! [batched_case]
}
return 0;
}

49
docs/snippets/gpu/preprocessing_nv12_to_gray.cpp Normal file
View File

@ -0,0 +1,49 @@
#include <openvino/runtime/intel_gpu/ocl/ocl.hpp>
#include <openvino/runtime/intel_gpu/properties.hpp>
#include <openvino/core/preprocess/pre_post_process.hpp>
ov::intel_gpu::ocl::ClImage2DTensor get_y_tensor();
ov::intel_gpu::ocl::ClImage2DTensor get_uv_tensor();
int main() {
ov::Core core;
auto model = core.read_model("model.xml");
//! [init_preproc]
using namespace ov::preprocess;
auto p = PrePostProcessor(model);
p.input().tensor().set_element_type(ov::element::u8)
.set_layout("NHWC")
.set_memory_type(ov::intel_gpu::memory_type::surface);
p.input().model().set_layout("NCHW");
auto model_with_preproc = p.build();
//! [init_preproc]
auto compiled_model = core.compile_model(model_with_preproc, "GPU");
auto remote_context = compiled_model.get_context().as<ov::intel_gpu::ocl::ClContext>();
auto input = model->input(0);
auto infer_request = compiled_model.create_infer_request();
{
//! [single_batch]
cl::Image2D img_y_plane;
auto input_y = model_with_preproc->input(0);
auto remote_y_tensor = remote_context.create_tensor(input_y.get_element_type(), input.get_shape(), img_y_plane);
infer_request.set_tensor(input_y.get_any_name(), remote_y_tensor);
infer_request.infer();
//! [single_batch]
}
{
//! [batched_case]
cl::Image2D img_y_plane_0, img_y_plane_l;
auto input_y = model_with_preproc->input(0);
auto remote_y_tensor_0 = remote_context.create_tensor(input_y.get_element_type(), input.get_shape(), img_y_plane_0);
auto remote_y_tensor_1 = remote_context.create_tensor(input_y.get_element_type(), input.get_shape(), img_y_plane_l);
std::vector<ov::Tensor> y_tensors = {remote_y_tensor_0, remote_y_tensor_1};
infer_request.set_tensors(input_y.get_any_name(), y_tensors);
infer_request.infer();
//! [batched_case]
}
return 0;
}

0
docs/snippets/gpu/preprocessing.cpp → docs/snippets/gpu/preprocessing_nv12_two_planes.cpp
View File

0
docs/snippets/gpu/preprocessing.py → docs/snippets/gpu/preprocessing_nv12_two_planes.py
View File

172
docs/snippets/ov_network_state_intro.py Normal file
View File

@ -0,0 +1,172 @@
# Copyright (C) 2018-2023 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import logging as log
import numpy as np
import sys
from openvino.runtime import opset10 as ops
from openvino.runtime import Core, Model, PartialShape, Tensor, Type
from openvino.runtime.passes import LowLatency2, MakeStateful, Manager
def state_network_example():
#! [ov:state_network]
input = ops.parameter([1, 1], dtype=np.float32)
read = ops.read_value(input, "variable0")
add = ops.add(read, input)
save = ops.assign(add, "variable0")
result = ops.result(add)
model = Model(results=[result], sinks=[save], parameters=[input])
#! [ov:state_network]
def low_latency_2_example():
#! [ov:low_latency_2]
# Precondition for Model.
# TensorIterator and Parameter are created in body of TensorIterator with names
tensor_iterator_name = "TI_name"
body_parameter_name = "body_parameter_name"
idx = "0" # this is a first variable in the network
# The State will be named "TI_name/param_name/variable_0"
state_name = tensor_iterator_name + "//" + body_parameter_name + "//" + "variable_" + idx # todo
#! [ov:get_ov_model]
core = Core()
ov_model = core.read_model("path_to_the_model")
#! [ov:get_ov_model]
# reshape input if needed
#! [ov:reshape_ov_model]
ov_model.reshape({"X": PartialShape([1, 1, 16])})
#! [ov:reshape_ov_model]
#! [ov:apply_low_latency_2]
manager = Manager()
manager.register_pass(LowLatency2())
manager.run_passes(ov_model)
#! [ov:apply_low_latency_2]
hd_specific_model = core.compile_model(ov_model)
# Try to find the Variable by name
infer_request = hd_specific_model.create_infer_request()
states = infer_request.query_state()
for state in states:
name = state.get_name()
if (name == state_name):
# some actions
#! [ov:low_latency_2]
#! [ov:low_latency_2_use_parameters]
manager.register_pass(LowLatency2(False))
#! [ov:low_latency_2_use_parameters]
def apply_make_stateful_tensor_names():
#! [ov:make_stateful_tensor_names]
core = Core()
ov_model = core.read_model("path_to_the_model")
tensor_names = {"tensor_name_1": "tensor_name_4",
"tensor_name_3": "tensor_name_6"}
manager = Manager()
manager.register_pass(MakeStateful(tensor_names))
manager.run_passes(ov_model)
#! [ov:make_stateful_tensor_names]
def apply_make_stateful_ov_nodes():
#! [ov:make_stateful_ov_nodes]
core = Core()
ov_model = core.read_model("path_to_the_model")
# Parameter_1, Result_1, Parameter_3, Result_3 are
# ops.parameter/ops.result in the ov_model
pairs = ["""(Parameter_1, Result_1), (Parameter_3, Result_3)"""]
manager = Manager()
manager.register_pass(MakeStateful(pairs))
manager.run_passes(ov_model)
#! [ov:make_stateful_ov_nodes]
def main():
#! [ov:state_api_usage]
# 1. Load inference engine
log.info("Loading Inference Engine")
core = Core()
# 2. Read a model
log.info("Loading network files")
model = core.read_model("path_to_the_model")
# 3. Load network to CPU
hw_specific_model = core.compile_model(model, "CPU")
# 4. Create Infer Request
infer_request = hw_specific_model.create_infer_request()
# 5. Reset memory states before starting
states = infer_request.query_state()
if (states.size() != 1):
log.error(f"Invalid queried state number. Expected 1, but got {str(states.size())}")
return -1
for state in states:
state.reset()
# 6. Inference
input_data = np.arange(start=1, stop=12, dtype=np.float32)
# This example demonstrates how to work with OpenVINO State API.
# Input_data: some array with 12 float numbers
# Part1: read the first four elements of the input_data array sequentially.
# Expected output for the first utterance:
# sum of the previously processed elements [ 1, 3, 6, 10]
# Part2: reset state value (set to 0) and read the next four elements.
# Expected output for the second utterance:
# sum of the previously processed elements [ 5, 11, 18, 26]
# Part3: set state value to 5 and read the next four elements.
# Expected output for the third utterance:
# sum of the previously processed elements + 5 [ 14, 24, 35, 47]
target_state = states[0]
# Part 1
log.info("Infer the first utterance")
for next_input in range(len(input_data)/3):
infer_request.infer({0 : input_data[next_input]})
state_buf = target_state.state.data
log.info(state_buf[0])
# Part 2
log.info("\nReset state between utterances...\n")
target_state.reset()
log.info("Infer the second utterance")
for next_input in range(len(input_data)/3, (len(input_data)/3 * 2)):
infer_request.infer({0 : input_data[next_input]})
state_buf = target_state.state.data
log.info(state_buf[0])
# Part 3
log.info("\nSet state value between utterances to 5...\n")
v = np.asarray([5], dtype=np.float32)
tensor = Tensor(v, shared_memory=True)
target_state.state = tensor
log.info("Infer the third utterance")
for next_input in range((input_data.size()/3 * 2), input_data.size()):
infer_request.infer({0 : input_data[next_input]})
state_buf = target_state.state.data
log.info(state_buf[0])
log.info("Execution successful")
#! [ov:state_api_usage]
return 0
if __name__ == '__main__':
sys.exit(main())

10
docs/snippets/src/main.c
View File

@ -30,14 +30,20 @@ ov_compiled_model_t* compiled_model = NULL;
ov_core_compile_model_from_file(core, "model.pdmodel", "AUTO", 0, &compiled_model);
//! [part2_3]
}
{
//! [part2_4]
ov_compiled_model_t* compiled_model = NULL;
ov_core_compile_model_from_file(core, "model.pb", "AUTO", 0, &compiled_model);
//! [part2_4]
}
//! [part2_5]
// Construct a model
ov_model_t* model = NULL;
ov_core_read_model(core, "model.xml", NULL, &model);
ov_compiled_model_t* compiled_model = NULL;
ov_core_compile_model(core, model, "AUTO", 0, &compiled_model);
//! [part2_4]
//! [part2_5]
//! [part3]

7
docs/snippets/src/main.cpp
View File

@ -29,6 +29,11 @@ ov::CompiledModel compiled_model = core.compile_model("model.pdmodel", "AUTO");
}
{
//! [part2_4]
ov::CompiledModel compiled_model = core.compile_model("model.pb", "AUTO");
//! [part2_4]
}
{
//! [part2_5]
auto create_model = []() {
std::shared_ptr<ov::Model> model;
// To construct a model, please follow
@ -37,7 +42,7 @@ auto create_model = []() {
};
std::shared_ptr<ov::Model> model = create_model();
compiled_model = core.compile_model(model, "AUTO");
//! [part2_4]
//! [part2_5]
}
//! [part3]

5
docs/snippets/src/main.py
View File

@ -20,6 +20,9 @@ compiled_model = core.compile_model("model.onnx", "AUTO")
compiled_model = core.compile_model("model.pdmodel", "AUTO")
#! [part2_3]
#! [part2_4]
compiled_model = core.compile_model("model.pb", "AUTO")
#! [part2_4]
#! [part2_5]
def create_model():
# This example shows how to create ov::Function
#
@ -31,7 +34,7 @@ def create_model():
model = create_model()
compiled_model = core.compile_model(model, "AUTO")
#! [part2_4]
#! [part2_5]
#! [part3]
infer_request = compiled_model.create_infer_request()

71
docs/tutorials.md
View File

@ -133,6 +133,11 @@ Tutorials that explain how to optimize and quantize models with OpenVINO tools.
+------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+
| `115-async-api <notebooks/115-async-api-with-output.html>`__ | Use Asynchronous Execution to Improve Data Pipelining |
+------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+
| `116-sparsity-optimization <notebooks/116-sparsity-optimization-with-output.html>`__ | Improve performance of sparse Transformer models |
+------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+
| `118-optimize-preprocessing <notebooks/118-optimize-preprocessing-with-output.html>`__ | Improve performance of image preprocessing step |
+------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------+
.. raw:: html
@ -200,6 +205,31 @@ Demos that demonstrate inference on a particular model.
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| `223-gpt2-text-prediction <notebooks/223-gpt2-text-prediction-with-output.html>`__ | Use GPT-2 to perform text prediction on an input sequence | |n223-img1| |
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| `224-3D-segmentation-point-clouds <notebooks/224-3D-segmentation-point-clouds-with-output.html>`__ | Process point cloud data and run 3D Part Segmentation with OpenVINO | |n224-img1| |
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| `225-stable-diffusion-text-to-image <notebooks/225-stable-diffusion-text-to-image-with-output.html>`__ | Text-to-image generation with Stable Diffusion method | |n225-img1| |
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| `226-yolov7-optimization <notebooks/226-yolov7-optimization-with-output.html>`__ | Optimize YOLOv7 using NNCF PTQ API | |n226-img1| |
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| `227-whisper-subtitles-generation <notebooks/227-whisper-subtitles-generation-with-output.html>`__ | Generate subtitles for video with OpenAI Whisper and OpenVINO | |n227-img1| |
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| `228-clip-zero-shot-image-classification <notebooks/228-clip-zero-shot-image-classification-with-output.html>`__ | Perform Zero-shot Image Classification with CLIP and OpenVINO | |n228-img1| |
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| `229-distilbert-sequence-classification <notebooks/229-distilbert-sequence-classification-with-output.html>`__ | Sequence Classification with OpenVINO | |n229-img1| |
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| `230-yolov8-optimization <notebooks/230-yolov8-optimization-with-output.html>`__ | Optimize YOLOv8 using NNCF PTQ API | |n230-img1| |
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| `231-instruct-pix2pix-image-editing <notebooks/231-instruct-pix2pix-image-editing-with-output.html>`__ | Image editing with InstructPix2Pix | |n231-img1| |
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| `232-clip-language-saliency-map <notebooks/232-clip-language-saliency-map-with-output.html>`__ | Language-Visual Saliency with CLIP and OpenVINO™ | |n232-img1| |
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| `233-blip-visual-language-processing <notebooks/233-blip-visual-language-processing-with-output.html>`__ | Visual Question Answering and Image Captioning using BLIP and OpenVINO™ | |n233-img1| |
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
.. raw:: html
@ -243,8 +273,14 @@ Live inference demos that run on a webcam or video files.
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| `403-action-recognition-webcam <notebooks/403-action-recognition-webcam-with-output.html>`__ |br| |n403| | Human action recognition with a webcam or video file. | |n403-img1| |
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| `404-style-transfer-webcam <notebooks/404-style-transfer-with-output.html>`__ |br| |n404| | Style Transfer with a webcam or video file | |n404-img1| |
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| `405-paddle-ocr-webcam <notebooks/405-paddle-ocr-webcam-with-output.html>`__ |br| |n405| | OCR with a webcam or video file | |n405-img1| |
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| `406-3D-pose-estimation-webcam <notebooks/406-3D-pose-estimation-with-output.html>`__ |br| |n406| | 3D display of human pose estimation with a webcam or video file | |n406-img1| |
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
| `407-person-tracking-webcam <notebooks/407-person-tracking-with-output.html>`__ |br| |n407| | Person tracking with a webcam or video file | |n407-img1| |
+-------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------+
.. raw:: html
@ -385,6 +421,26 @@ Made with `contributors-img <https://contrib.rocks>`__.
:target: https://user-images.githubusercontent.com/18904157/166343139-c6568e50-b856-4066-baef-5cdbd4e8bc18.png
.. |n223-img1| image:: https://user-images.githubusercontent.com/91228207/185105225-0f996b0b-0a3b-4486-872d-364ac6fab68b.png
:target: https://user-images.githubusercontent.com/91228207/185105225-0f996b0b-0a3b-4486-872d-364ac6fab68b.png
.. |n224-img1| image:: https://user-images.githubusercontent.com/91237924/185752178-3882902c-907b-4614-b0e6-ea1de08bf3ef.png
:target: https://user-images.githubusercontent.com/91237924/185752178-3882902c-907b-4614-b0e6-ea1de08bf3ef.png
.. |n225-img1| image:: https://user-images.githubusercontent.com/15709723/200945747-1c584e5c-b3f2-4e43-b1c1-e35fd6edc2c3.png
:target: https://user-images.githubusercontent.com/15709723/200945747-1c584e5c-b3f2-4e43-b1c1-e35fd6edc2c3.png
.. |n226-img1| image:: https://raw.githubusercontent.com/WongKinYiu/yolov7/main/figure/horses_prediction.jpg
:target: https://raw.githubusercontent.com/WongKinYiu/yolov7/main/figure/horses_prediction.jpg
.. |n227-img1| image:: https://user-images.githubusercontent.com/29454499/204548693-1304ef33-c790-490d-8a8b-d5766acb6254.png
:target: https://user-images.githubusercontent.com/29454499/204548693-1304ef33-c790-490d-8a8b-d5766acb6254.png
.. |n228-img1| image:: https://user-images.githubusercontent.com/29454499/207795060-437b42f9-e801-4332-a91f-cc26471e5ba2.png
:target: https://user-images.githubusercontent.com/29454499/207795060-437b42f9-e801-4332-a91f-cc26471e5ba2.png
.. |n229-img1| image:: https://user-images.githubusercontent.com/95271966/206130638-d9847414-357a-4c79-9ca7-76f4ae5a6d7f.png
:target: https://user-images.githubusercontent.com/95271966/206130638-d9847414-357a-4c79-9ca7-76f4ae5a6d7f.png
.. |n230-img1| image:: https://user-images.githubusercontent.com/29454499/212105105-f61c8aab-c1ff-40af-a33f-d0ed1fccc72e.png
:target: https://user-images.githubusercontent.com/29454499/212105105-f61c8aab-c1ff-40af-a33f-d0ed1fccc72e.png
.. |n231-img1| image:: https://user-images.githubusercontent.com/29454499/219943222-d46a2e2d-d348-4259-8431-37cf14727eda.png
:target: https://user-images.githubusercontent.com/29454499/219943222-d46a2e2d-d348-4259-8431-37cf14727eda.png
.. |n232-img1| image:: https://user-images.githubusercontent.com/29454499/218967961-9858efd5-fff2-4eb0-bde9-60852f4b31cb.JPG
:target: https://user-images.githubusercontent.com/29454499/218967961-9858efd5-fff2-4eb0-bde9-60852f4b31cb.JPG
.. |n233-img1| image:: https://user-images.githubusercontent.com/29454499/221933762-4ff32ecb-5e5d-4484-80e1-e9396cb3c511.png
:target: https://user-images.githubusercontent.com/29454499/221933762-4ff32ecb-5e5d-4484-80e1-e9396cb3c511.png
.. |n301-img1| image:: https://user-images.githubusercontent.com/15709723/127779607-8fa34947-1c35-4260-8d04-981c41a2a2cc.png
:target: https://user-images.githubusercontent.com/15709723/127779607-8fa34947-1c35-4260-8d04-981c41a2a2cc.png
.. |n401-img1| image:: https://user-images.githubusercontent.com/4547501/141471665-82b28c86-cf64-4bfe-98b3-c314658f2d96.gif
@ -393,8 +449,14 @@ Made with `contributors-img <https://contrib.rocks>`__.
:target: https://user-images.githubusercontent.com/4547501/138267961-41d754e7-59db-49f6-b700-63c3a636fad7.gif
.. |n403-img1| image:: https://user-images.githubusercontent.com/10940214/151552326-642d6e49-f5a0-4fc1-bf14-ae3f457e1fec.gif
:target: https://user-images.githubusercontent.com/10940214/151552326-642d6e49-f5a0-4fc1-bf14-ae3f457e1fec.gif
.. |n404-img1| image:: https://user-images.githubusercontent.com/109281183/203772234-f17a0875-b068-43ef-9e77-403462fde1f5.gif
:target: https://user-images.githubusercontent.com/109281183/203772234-f17a0875-b068-43ef-9e77-403462fde1f5.gif
.. |n405-img1| image:: https://raw.githubusercontent.com/yoyowz/classification/master/images/paddleocr.gif
:target: https://raw.githubusercontent.com/yoyowz/classification/master/images/paddleocr.gif
.. |n406-img1| image:: https://user-images.githubusercontent.com/42672437/183292131-576cc05a-a724-472c-8dc9-f6bc092190bf.gif
:target: https://user-images.githubusercontent.com/42672437/183292131-576cc05a-a724-472c-8dc9-f6bc092190bf.gif
.. |n407-img1| image:: https://user-images.githubusercontent.com/91237924/210479548-b70dbbaa-5948-4e49-b48e-6cb6613226da.gif
:target: https://user-images.githubusercontent.com/91237924/210479548-b70dbbaa-5948-4e49-b48e-6cb6613226da.gif
.. |launch-jupyter| image:: https://user-images.githubusercontent.com/15709723/120527271-006fd200-c38f-11eb-9935-2d36d50bab9f.gif
:target: https://user-images.githubusercontent.com/15709723/120527271-006fd200-c38f-11eb-9935-2d36d50bab9f.gif
@ -457,11 +519,18 @@ Made with `contributors-img <https://contrib.rocks>`__.
:target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F402-pose-estimation-webcam%2F402-pose-estimation.ipynb
.. |n403| image:: https://mybinder.org/badge_logo.svg
:target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F403-action-recognition-webcam%2F403-action-recognition-webcam.ipynb
.. |n404| image:: https://mybinder.org/badge_logo.svg
:target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F404-style-transfer-webcam%2F404-style-transfer.ipynb
.. |n405| image:: https://mybinder.org/badge_logo.svg
:target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F405-paddle-ocr-webcam%2F405-paddle-ocr-webcam.ipynb
.. |n406| image:: https://mybinder.org/badge_logo.svg
:target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks.git/main?labpath=notebooks%2F406-3D-pose-estimation-webcam%2F406-3D-pose-estimation.ipynb
.. |n407| image:: https://mybinder.org/badge_logo.svg
:target: https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=notebooks%2F407-person-tracking-webcam%2F407-person-tracking.ipynb
.. |binder logo| image:: https://mybinder.org/badge_logo.svg
:alt: Binder button
@endsphinxdirective
@endsphinxdirective

17
samples/cpp/CMakeLists.txt
View File

@ -5,12 +5,21 @@
cmake_minimum_required (VERSION 3.10)
# Enable CMAKE_<LANG>_COMPILER_ID AppleClang
cmake_policy(SET CMP0025 NEW)
if(POLICY CMP0025)
cmake_policy(SET CMP0025 NEW)
endif()
project(Samples)
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "CMake build type")
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Release" "Debug" "RelWithDebInfo" "MinSizeRel")
get_property(OV_GENERATOR_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
if(CMAKE_GENERATOR MATCHES "^Ninja Multi-Config$")
# Ninja-Multi specific, see:
# https://cmake.org/cmake/help/latest/variable/CMAKE_DEFAULT_BUILD_TYPE.html
set(CMAKE_DEFAULT_BUILD_TYPE "Release" CACHE STRING "CMake default build type")
elseif(NOT OV_GENERATOR_MULTI_CONFIG)
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "CMake build type")
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Release;Debug;RelWithDebInfo;MinSizeRel")
endif()
set_property(GLOBAL PROPERTY USE_FOLDERS ON)
@ -69,7 +78,7 @@ if(APPLE)
set(CMAKE_MACOSX_RPATH ON)
endif()
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64.*|aarch64.*|AARCH64.*)")
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64.*|aarch64.*|AARCH64.*|ARM64.*)")
set(AARCH64 ON)
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm.*|ARM.*)")
set(ARM ON)

1
samples/cpp/benchmark_app/CMakeLists.txt
View File

@ -26,6 +26,7 @@ if(NOT TARGET nlohmann_json::nlohmann_json)
if(TARGET nlohmann_json)
# Ubuntu 18.04 case where target 'nlohmann_json' is here, but nlohmann_json_FOUND is OFF
if(NOT TARGET nlohmann_json::nlohmann_json)
set_target_properties(nlohmann_json PROPERTIES IMPORTED_GLOBAL ON)
add_library(nlohmann_json::nlohmann_json ALIAS nlohmann_json)
endif()
set(nlohmann_json_FOUND ON)

7
samples/cpp/benchmark_app/utils.cpp
View File

@ -121,7 +121,12 @@ std::vector<std::string> parse_devices(const std::string& device_string) {
}
auto devices = split(comma_separated_devices, ',');
result.insert(result.end(), devices.begin(), devices.end());
for (auto&& device : devices) {
// e.g. in AUTO:-CPU,-GPU
if (device.front() == '-')
device.erase(device.begin());
result.push_back(device);
}
return result;
}

4
samples/cpp/speech_sample/CMakeLists.txt
View File

@ -15,8 +15,8 @@ endif()
if(NOT TARGET zlib::zlib)
if(PkgConfig_FOUND)
pkg_search_module(zlib QUIET
IMPORTED_TARGET GLOBAL
zlib)
IMPORTED_TARGET GLOBAL
zlib)
if(zlib_FOUND)
add_library(zlib::zlib ALIAS PkgConfig::zlib)
endif()

18
scripts/CMakeLists.txt
View File

@ -27,16 +27,16 @@ ie_shellcheck_process(DIRECTORY "${OpenVINO_SOURCE_DIR}"
ie_cpack_add_component(${OV_CPACK_COMP_SETUPVARS} HIDDEN)
if(UNIX)
set(_setupvars_file setupvars/setupvars.sh)
set(_setupvars_file "${CMAKE_CURRENT_SOURCE_DIR}/setupvars/setupvars.sh")
elseif(WIN32)
set(_setupvars_file setupvars/setupvars.bat)
if (USE_BUILD_TYPE_SUBFOLDER AND CMAKE_BUILD_TYPE AND NOT CMAKE_BUILD_TYPE STREQUAL "Debug" AND NOT CMAKE_BUILD_TYPE STREQUAL "Release")
set(_setupvars_file "${CMAKE_CURRENT_SOURCE_DIR}/setupvars/setupvars.bat")
if(USE_BUILD_TYPE_SUBFOLDER AND CMAKE_BUILD_TYPE AND NOT CMAKE_BUILD_TYPE MATCHES "^(Debug|Release)$")
# Patch primary configuration in setupvars.bat which is "Release" by default.
# Note setupvars secondary configuration is always "Debug".
message(STATUS "Patching content of ${_setupvars_file} for CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}")
file(READ "${_setupvars_file}" _setupvars_content)
string(REPLACE "Release" ${CMAKE_BUILD_TYPE} _setupvars_content "${_setupvars_content}")
set(_setupvars_file "${CMAKE_BINARY_DIR}/${_setupvars_file}")
set(_setupvars_file "${OpenVINO_BINARY_DIR}/${_setupvars_file}")
message(STATUS "Writing patched content to ${_setupvars_file}")
file(WRITE "${_setupvars_file}" "${_setupvars_content}")
endif()
@ -51,11 +51,7 @@ install(PROGRAMS "${_setupvars_file}"
if(LINUX)
ie_cpack_add_component(${OV_CPACK_COMP_INSTALL_DEPENDENCIES} HIDDEN)
set(install_dependencies_files install_openvino_dependencies.sh)
foreach(install_dependencies_file IN LISTS install_dependencies_files)
install(PROGRAMS "${CMAKE_CURRENT_SOURCE_DIR}/install_dependencies/${install_dependencies_file}"
DESTINATION install_dependencies/
COMPONENT ${OV_CPACK_COMP_INSTALL_DEPENDENCIES})
endforeach()
install(PROGRAMS "${CMAKE_CURRENT_SOURCE_DIR}/install_dependencies/install_openvino_dependencies.sh"
DESTINATION install_dependencies/
COMPONENT ${OV_CPACK_COMP_INSTALL_DEPENDENCIES})
endif()

6
scripts/install_dependencies/install_openvino_dependencies.sh
View File

@ -133,24 +133,20 @@ elif [ "$os" == "ubuntu20.04" ] || [ "$os" == "debian10" ] || [ "$os" == "raspbi
[ "$os" == "ubuntu21.10" ] || [ "$os" == "ubuntu22.04" ] || [ "$os" == "debian11" ] || [ "$os" == "raspbian11" ] ||
[ "$os" == "ubuntu22.10" ] || [ "$os" == "debian12" ] || [ "$os" == "raspbian12" ]; then
pkgs_core=(libpugixml1v5)
pkgs_core=(libpugixml1v5 libtbb2)
pkgs_gpu=()
pkgs_python=(python3 python3-venv python3-pip)
pkgs_dev=(cmake pkg-config g++ gcc libc6-dev libgflags-dev zlib1g-dev nlohmann-json3-dev make curl sudo)
if [ "$os" == "debian10" ] || [ "$os" == "raspbian10" ] ; then
pkgs_core=("${pkgs_core[@]}" libtbb2)
pkgs_python=("${pkgs_python[@]}" libpython3.7)
elif [ "$os" == "ubuntu20.04" ] || [ "$os" == "ubuntu20.10" ] || [ "$os" == "ubuntu21.04" ] ; then
pkgs_core=("${pkgs_core[@]}" libtbb2)
pkgs_python=("${pkgs_python[@]}" libpython3.8)
elif [ "$os" == "ubuntu21.10" ] ||
[ "$os" == "debian11" ] || [ "$os" == "raspbian11" ] ; then
pkgs_core=("${pkgs_core[@]}" libtbb2)
pkgs_python=("${pkgs_python[@]}" libpython3.9)
elif [ "$os" == "ubuntu22.04" ] || [ "$os" == "ubuntu22.10" ] ||
[ "$os" == "debian12" ] || [ "$os" == "raspbian12" ] ; then
pkgs_core=("${pkgs_core[@]}" libtbb12)
pkgs_python=("${pkgs_python[@]}" libpython3.10)
fi

2
src/bindings/c/include/openvino/c/openvino.h
View File

@ -27,6 +27,6 @@
#include "openvino/c/ov_prepostprocess.h"
#include "openvino/c/ov_property.h"
#include "openvino/c/ov_rank.h"
#include "openvino/c/ov_remote_context.h"
#include "openvino/c/ov_shape.h"
#include "openvino/c/ov_tensor.h"
#include "openvino/c/ov_remote_context.h"

64
src/bindings/c/include/openvino/c/ov_common.h
View File

@ -64,7 +64,7 @@
/**
* @defgroup ov_c_api OpenVINO Runtime C API
* OpenVINO Runtime C API
*
*
* @defgroup ov_base_c_api Basics
* @ingroup ov_c_api
* @brief The basic definitions & interfaces of OpenVINO C API to work with other components
@ -72,55 +72,55 @@
* @defgroup ov_compiled_model_c_api Compiled Model
* @ingroup ov_c_api
* @brief The operations about compiled model
*
*
* @defgroup ov_core_c_api Core
* @ingroup ov_c_api
* @brief The definitions & operations about core
*
*
* @defgroup ov_dimension_c_api Dimension
* @ingroup ov_c_api
* @brief The definitions & operations about dimension
*
*
* @defgroup ov_infer_request_c_api Infer Request
* @ingroup ov_c_api
* @brief The definitions & operations about infer request
*
*
* @defgroup ov_layout_c_api Layout
* @ingroup ov_c_api
* @brief The definitions & operations about layout
*
*
* @defgroup ov_model_c_api Model
* @ingroup ov_c_api
* @brief The definitions & operations about model
*
*
* @defgroup ov_node_c_api Node
* @ingroup ov_c_api
* @brief The definitions & operations about node
*
*
* @defgroup ov_partial_shape_c_api Partial Shape
* @ingroup ov_c_api
* @brief The definitions & operations about partial shape
*
*
* @defgroup ov_prepostprocess_c_api Pre Post Process
* @ingroup ov_c_api
* @brief The definitions & operations about prepostprocess
*
*
* @defgroup ov_property_c_api Property
* @ingroup ov_c_api
* @brief The definitions & operations about property
*
*
* @defgroup ov_rank_c_api Rank
* @ingroup ov_c_api
* @brief The definitions & operations about rank
*
*
* @defgroup ov_shape_c_api Shape
* @ingroup ov_c_api
* @brief The definitions & operations about shape
*
*
* @defgroup ov_tensor_c_api Tensor
* @ingroup ov_c_api
* @brief The definitions & operations about tensor
*
*
* @defgroup ov_remote_context_c_api ov_remote_context
* @ingroup ov_c_api
* @brief Set of functions representing of RemoteContext
@ -132,33 +132,33 @@
* @brief This enum contains codes for all possible return values of the interface functions
*/
typedef enum {
OK = 0, //!< SUCCESS
OK = 0, //!< SUCCESS
/*
* @brief map exception to C++ interface
*/
GENERAL_ERROR = -1, //!< GENERAL_ERROR
NOT_IMPLEMENTED = -2, //!< NOT_IMPLEMENTED
NETWORK_NOT_LOADED = -3, //!< NETWORK_NOT_LOADED
PARAMETER_MISMATCH = -4, //!< PARAMETER_MISMATCH
NOT_FOUND = -5, //!< NOT_FOUND
OUT_OF_BOUNDS = -6, //!< OUT_OF_BOUNDS
GENERAL_ERROR = -1, //!< GENERAL_ERROR
NOT_IMPLEMENTED = -2, //!< NOT_IMPLEMENTED
NETWORK_NOT_LOADED = -3, //!< NETWORK_NOT_LOADED
PARAMETER_MISMATCH = -4, //!< PARAMETER_MISMATCH
NOT_FOUND = -5, //!< NOT_FOUND
OUT_OF_BOUNDS = -6, //!< OUT_OF_BOUNDS
/*
* @brief exception not of std::exception derived type was thrown
*/
UNEXPECTED = -7, //!< UNEXPECTED
REQUEST_BUSY = -8, //!< REQUEST_BUSY
RESULT_NOT_READY = -9, //!< RESULT_NOT_READY
NOT_ALLOCATED = -10, //!< NOT_ALLOCATED
INFER_NOT_STARTED = -11, //!< INFER_NOT_STARTED
NETWORK_NOT_READ = -12, //!< NETWORK_NOT_READ
INFER_CANCELLED = -13, //!< INFER_CANCELLED
UNEXPECTED = -7, //!< UNEXPECTED
REQUEST_BUSY = -8, //!< REQUEST_BUSY
RESULT_NOT_READY = -9, //!< RESULT_NOT_READY
NOT_ALLOCATED = -10, //!< NOT_ALLOCATED
INFER_NOT_STARTED = -11, //!< INFER_NOT_STARTED
NETWORK_NOT_READ = -12, //!< NETWORK_NOT_READ
INFER_CANCELLED = -13, //!< INFER_CANCELLED
/*
* @brief exception in C wrapper
*/
INVALID_C_PARAM = -14, //!< INVALID_C_PARAM
UNKNOWN_C_ERROR = -15, //!< UNKNOWN_C_ERROR
NOT_IMPLEMENT_C_METHOD = -16, //!< NOT_IMPLEMENT_C_METHOD
UNKNOW_EXCEPTION = -17, //!< UNKNOW_EXCEPTION
INVALID_C_PARAM = -14, //!< INVALID_C_PARAM
UNKNOWN_C_ERROR = -15, //!< UNKNOWN_C_ERROR
NOT_IMPLEMENT_C_METHOD = -16, //!< NOT_IMPLEMENT_C_METHOD
UNKNOW_EXCEPTION = -17, //!< UNKNOW_EXCEPTION
} ov_status_e;
/**

15
src/bindings/c/include/openvino/c/ov_compiled_model.h
View File

@ -79,8 +79,7 @@ ov_compiled_model_input_by_name(const ov_compiled_model_t* compiled_model,
* @return Status code of the operation: OK(0) for success.
*/
OPENVINO_C_API(ov_status_e)
ov_compiled_model_outputs_size(const ov_compiled_model_t* compiled_model,
size_t* size);
ov_compiled_model_outputs_size(const ov_compiled_model_t* compiled_model, size_t* size);
/**
* @brief Get the single const output port of ov_compiled_model_t, which only support single output model.
@ -90,8 +89,7 @@ ov_compiled_model_outputs_size(const ov_compiled_model_t* compiled_model,
* @return Status code of the operation: OK(0) for success.
*/
OPENVINO_C_API(ov_status_e)
ov_compiled_model_output(const ov_compiled_model_t* compiled_model,
ov_output_const_port_t** output_port);
ov_compiled_model_output(const ov_compiled_model_t* compiled_model, ov_output_const_port_t** output_port);
/**
* @brief Get a const output port of ov_compiled_model_t by port index.
@ -127,8 +125,7 @@ ov_compiled_model_output_by_name(const ov_compiled_model_t* compiled_model,
* @return Status code of the operation: OK(0) for success.
*/
OPENVINO_C_API(ov_status_e)
ov_compiled_model_get_runtime_model(const ov_compiled_model_t* compiled_model,
ov_model_t** model);
ov_compiled_model_get_runtime_model(const ov_compiled_model_t* compiled_model, ov_model_t** model);
/**
* @brief Creates an inference request object used to infer the compiled model.
@ -138,8 +135,7 @@ ov_compiled_model_get_runtime_model(const ov_compiled_model_t* compiled_model,
* @return Status code of the operation: OK(0) for success.
*/
OPENVINO_C_API(ov_status_e)
ov_compiled_model_create_infer_request(const ov_compiled_model_t* compiled_model,
ov_infer_request_t** infer_request);
ov_compiled_model_create_infer_request(const ov_compiled_model_t* compiled_model, ov_infer_request_t** infer_request);
/**
* @brief Sets properties for a device, acceptable keys can be found in ov_property_key_xxx.
@ -174,8 +170,7 @@ ov_compiled_model_get_property(const ov_compiled_model_t* compiled_model,
* @return Status code of the operation: OK(0) for success.
*/
OPENVINO_C_API(ov_status_e)
ov_compiled_model_export_model(const ov_compiled_model_t* compiled_model,
const char* export_model_path);
ov_compiled_model_export_model(const ov_compiled_model_t* compiled_model, const char* export_model_path);
/**
* @brief Release the memory allocated by ov_compiled_model_t.

50
src/bindings/c/include/openvino/c/ov_core.h
View File

@ -61,8 +61,8 @@ typedef struct {
* @brief Represent all available devices.
*/
typedef struct {
char** devices; //!< devices' name
size_t size; //!< devices' number
char** devices; //!< devices' name
size_t size; //!< devices' number
} ov_available_devices_t;
/**
@ -126,41 +126,40 @@ OPENVINO_C_API(void)
ov_core_free(ov_core_t* core);
/**
* @brief Reads models from IR/ONNX/PDPD formats.
* @brief Reads models from IR / ONNX / PDPD / TF / TFLite formats.
* @ingroup ov_core_c_api
* @param core A pointer to the ie_core_t instance.
* @param model_path Path to a model.
* @param bin_path Path to a data file.
* For IR format (*.bin):
* * if path is empty, will try to read a bin file with the same name as xml and
* * if `bin_path` is empty, will try to read a bin file with the same name as xml and
* * if the bin file with the same name is not found, will load IR without weights.
* For ONNX format (*.onnx):
* * the bin_path parameter is not used.
* For PDPD format (*.pdmodel)
* * the bin_path parameter is not used.
* For the following file formats the `bin_path` parameter is not used:
* * ONNX format (*.onnx)
* * PDPD (*.pdmodel)
* * TF (*.pb)
* * TFLite (*.tflite)
* @param model A pointer to the newly created model.
* @return Status code of the operation: OK(0) for success.
*/
OPENVINO_C_API(ov_status_e)
ov_core_read_model(const ov_core_t* core,
const char* model_path,
const char* bin_path,
ov_model_t** model);
ov_core_read_model(const ov_core_t* core, const char* model_path, const char* bin_path, ov_model_t** model);
#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT
/**
* @brief Reads models from IR/ONNX/PDPD formats, path is unicode.
* @brief Reads models from IR / ONNX / PDPD / TF / TFLite formats, path is unicode.
* @ingroup ov_core_c_api
* @param core A pointer to the ie_core_t instance.
* @param model_path Path to a model.
* @param bin_path Path to a data file.
* For IR format (*.bin):
* * if path is empty, will try to read a bin file with the same name as xml and
* * if `bin_path` is empty, will try to read a bin file with the same name as xml and
* * if the bin file with the same name is not found, will load IR without weights.
* For ONNX format (*.onnx):
* * the bin_path parameter is not used.
* For PDPD format (*.pdmodel)
* * the bin_path parameter is not used.
* For the following file formats the `bin_path` parameter is not used:
* * ONNX format (*.onnx)
* * PDPD (*.pdmodel)
* * TF (*.pb)
* * TFLite (*.tflite)
* @param model A pointer to the newly created model.
* @return Status code of the operation: OK(0) for success.
*/
@ -172,13 +171,13 @@ ov_core_read_model_unicode(const ov_core_t* core,
#endif
/**
* @brief Reads models from IR/ONNX/PDPD formats.
* @brief Reads models from IR / ONNX / PDPD / TF / TFLite formats.
* @ingroup ov_core_c_api
* @param core A pointer to the ie_core_t instance.
* @param model_str String with a model in IR/ONNX/PDPD format.
* @param model_str String with a model in IR / ONNX / PDPD / TF / TFLite format.
* @param weights Shared pointer to a constant tensor with weights.
* @param model A pointer to the newly created model.
* Reading ONNX/PDPD models does not support loading weights from the @p weights tensors.
* Reading ONNX / PDPD / TF / TFLite models does not support loading weights from the @p weights tensors.
* @note Created model object shares the weights with the @p weights object.
* Thus, do not create @p weights on temporary data that can be freed later, since the model
* constant data will point to an invalid memory.
@ -282,10 +281,7 @@ ov_core_set_property(const ov_core_t* core, const char* device_name, ...);
* @return Status code of the operation: OK(0) for success.
*/
OPENVINO_C_API(ov_status_e)
ov_core_get_property(const ov_core_t* core,
const char* device_name,
const char* property_key,
char** property_value);
ov_core_get_property(const ov_core_t* core, const char* device_name, const char* property_key, char** property_value);
/**
* @brief Returns devices available for inference.
@ -335,9 +331,7 @@ ov_core_import_model(const ov_core_t* core,
* @return Status code of the operation: OK(0) for success.
*/
OPENVINO_C_API(ov_status_e)
ov_core_get_versions_by_device_name(const ov_core_t* core,
const char* device_name,
ov_core_version_list_t* versions);
ov_core_get_versions_by_device_name(const ov_core_t* core, const char* device_name, ov_core_version_list_t* versions);
/**
* @brief Releases memory occupied by ov_core_version_list_t.

32
src/bindings/c/include/openvino/c/ov_infer_request.h
View File

@ -27,8 +27,8 @@ typedef struct ov_infer_request ov_infer_request_t;
* @brief Completion callback definition about the function and args
*/
typedef struct {
void(OPENVINO_C_API_CALLBACK* callback_func)(void* args); //!< The callback func
void* args; //!< The args of callback func
void(OPENVINO_C_API_CALLBACK* callback_func)(void* args); //!< The callback func
void* args; //!< The args of callback func
} ov_callback_t;
/**
@ -37,11 +37,11 @@ typedef struct {
* @brief Store profiling info data
*/
typedef struct {
enum Status { //!< Defines the general status of a node.
NOT_RUN, //!< A node is not executed.
OPTIMIZED_OUT, //!< A node is optimized out during graph optimization phase.
EXECUTED //!< A node is executed.
} status; //!< status
enum Status { //!< Defines the general status of a node.
NOT_RUN, //!< A node is not executed.
OPTIMIZED_OUT, //!< A node is optimized out during graph optimization phase.
EXECUTED //!< A node is executed.
} status; //!< status
int64_t real_time; //!< The absolute time, in microseconds, that the node ran (in total).
int64_t cpu_time; //!< The net host CPU time that the node ran.
const char* node_name; //!< Name of a node.
@ -55,8 +55,8 @@ typedef struct {
* @brief A list of profiling info data
*/
typedef struct {
ov_profiling_info_t* profiling_infos; //!< The list of ov_profilling_info_t
size_t size; //!< The list size
ov_profiling_info_t* profiling_infos; //!< The list of ov_profilling_info_t
size_t size; //!< The list size
} ov_profiling_info_list_t;
/**
@ -68,9 +68,7 @@ typedef struct {
* @return Status code of the operation: OK(0) for success.
*/
OPENVINO_C_API(ov_status_e)
ov_infer_request_set_tensor(ov_infer_request_t* infer_request,
const char* tensor_name,
const ov_tensor_t* tensor);
ov_infer_request_set_tensor(ov_infer_request_t* infer_request, const char* tensor_name, const ov_tensor_t* tensor);
/**
* @brief Set an input/output tensor to infer request for the port.
@ -157,9 +155,7 @@ ov_infer_request_set_output_tensor(ov_infer_request_t* infer_request, const ov_t
* @return Status code of the operation: OK(0) for success.
*/
OPENVINO_C_API(ov_status_e)
ov_infer_request_get_tensor(const ov_infer_request_t* infer_request,
const char* tensor_name,
ov_tensor_t** tensor);
ov_infer_request_get_tensor(const ov_infer_request_t* infer_request, const char* tensor_name, ov_tensor_t** tensor);
/**
* @brief Get an input/output tensor by const port.
@ -209,8 +205,7 @@ ov_infer_request_get_input_tensor_by_index(const ov_infer_request_t* infer_reque
* @return Status code of the operation: OK(0) for success.
*/
OPENVINO_C_API(ov_status_e)
ov_infer_request_get_input_tensor(const ov_infer_request_t* infer_request,
ov_tensor_t** tensor);
ov_infer_request_get_input_tensor(const ov_infer_request_t* infer_request, ov_tensor_t** tensor);
/**
* @brief Get an output tensor by the index of output tensor.
@ -235,8 +230,7 @@ ov_infer_request_get_output_tensor_by_index(const ov_infer_request_t* infer_requ
* @return Status code of the operation: OK(0) for success.
*/
OPENVINO_C_API(ov_status_e)
ov_infer_request_get_output_tensor(const ov_infer_request_t* infer_request,
ov_tensor_t** tensor);
ov_infer_request_get_output_tensor(const ov_infer_request_t* infer_request, ov_tensor_t** tensor);
/**
* @brief Infer specified input(s) in synchronous mode.

Some files were not shown because too many files have changed in this diff Show More