IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into debian-packages

Browse Source
This commit is contained in:
Ilya Lavrenov 2022-03-18 18:08:10 +03:00
commit cebdf3f92a
2321 changed files with 75872 additions and 34963 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

200
.ci/azure/linux_arm64.yml
View File

@ -26,16 +26,28 @@ jobs:
system.debug: true
VSTS_HTTP_RETRY: 5
VSTS_HTTP_TIMEOUT: 200
PYTHON_ARM_VERSION: "3.8.12"
PYTHON_EXEC: "python3.8"
OPENVINO_ARCH: 'aarch64'
NUM_PROC: 1
BUILD_TYPE: Release
OPENVINO_REPO_DIR: $(Build.Repository.LocalPath)
OPENVINO_CONTRIB_REPO_DIR: $(OPENVINO_REPO_DIR)/../openvino_contrib
OPENCV_REPO_DIR: $(OPENVINO_REPO_DIR)/../opencv
BUILD_PYTHON: $(WORK_DIR)/build_python
BUILD_OPENCV: $(WORK_DIR)/build_opencv
BUILD_OPENVINO: $(WORK_DIR)/build
BUILD_OPENVINO_PYTHON: $(WORK_DIR)/build_python
BUILD_OPEN_MODEL_ZOO: $(WORK_DIR)/build_open_model_zoo
INSTALL_OPENVINO: $(WORK_DIR)/install_openvino
INSTALL_PYTHON: $(INSTALL_OPENVINO)/extras/python
INSTALL_OPENCV: $(INSTALL_OPENVINO)/extras/opencv
INSTALL_OPEN_MODEL_ZOO: $(INSTALL_OPENVINO)/extras/open_model_zoo
WORK_DIR: $(Pipeline.Workspace)/_w
BUILD_DIR: $(WORK_DIR)/build
BUILD_DIR_OPENCV: $(WORK_DIR)/build_opencv
TMP_DIR: /mnt/tmp
SHARE_DIR: /mount/cinfsshare/onnxtestdata
CCACHE_DIR: $(SHARE_DIR)/ccache/master/linux_arm64
TMP_DIR: /mnt/tmp
OPENVINO_CCACHE_DIR: $(SHARE_DIR)/ccache/master/linux_arm64
OPENCV_CCACHE_DIR: $(SHARE_DIR)/ccache/master/linux_arm64_opencv
steps:
- script: |
@ -56,17 +68,21 @@ jobs:
df
lsblk -o NAME,HCTL,SIZE,MOUNTPOINT | grep -i "sd"
free -h
echo "##vso[task.setvariable variable=NUM_PROC]$(nproc --all)"
echo "NUM_PROC=$(NUM_PROC)"
displayName: 'System information'
- script: |
rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR)
rm -rf $(BUILD_DIR) ; mkdir $(BUILD_DIR)
mkdir -p $(BUILD_OPENCV) $(BUILD_OPENVINO) $(BUILD_OPENVINO_PYTHON) $(BUILD_PYTHON) $(BUILD_OPEN_MODEL_ZOO)
mkdir -p $(INSTALL_OPENVINO) $(INSTALL_PYTHON) $(INSTALL_OPENCV) $(INSTALL_OPEN_MODEL_ZOO)
sudo rm -rf $(TMP_DIR) ; sudo mkdir $(TMP_DIR) ; sudo chmod 777 -R $(TMP_DIR)
sudo mkdir -p $(SHARE_DIR)
sudo apt --assume-yes update && sudo apt --assume-yes install nfs-common
sudo mount -vvv -t nfs cinfsshare.file.core.windows.net:/cinfsshare/onnxtestdata $(SHARE_DIR) -o vers=4,minorversion=1,sec=sys
mkdir -p $(CCACHE_DIR)
displayName: 'Make directory'
mkdir -p $(OPENVINO_CCACHE_DIR)
mkdir -p $(OPENCV_CCACHE_DIR)
displayName: 'Make directories'
- checkout: self
clean: true
@ -83,16 +99,25 @@ jobs:
- script: |
set -e
$(OPENVINO_REPO_DIR)/install_build_dependencies.sh
# Move into contrib install_build_dependencies.sh
sudo apt --assume-yes install scons crossbuild-essential-arm64 libprotoc-dev protobuf-compiler
# OpenCV should provide install_build_dependencies.sh as well
# Move into resources
git clone https://github.com/opencv/opencv.git --depth 1 $(OPENCV_REPO_DIR)
# Speed up build
wget https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-linux.zip
unzip ninja-linux.zip
sudo cp -v ninja /usr/local/bin/
workingDirectory: $(WORK_DIR)
export CCACHE_DIR=$(OPENCV_CCACHE_DIR)
export CCACHE_TEMPDIR=$(TMP_DIR)/ccache
export CCACHE_BASEDIR=$(Pipeline.Workspace)
export CCACHE_MAXSIZE=50G
export USE_CCACHE=1
export PYTHON_ARM_VERSION=$(PYTHON_ARM_VERSION)
export NUM_PROC=$(NUM_PROC)
export BUILD_PYTHON=$(BUILD_PYTHON)
export WORK_DIR=$(WORK_DIR)
export INSTALL_PYTHON=$(INSTALL_PYTHON)
export BUILD_TYPE=$(BUILD_TYPE)
export OPENVINO_REPO_DIR=$(OPENVINO_REPO_DIR)
export INSTALL_OPENCV=$(INSTALL_OPENCV)
export PYTHON_EXEC=$(PYTHON_EXEC)
export OPENCV_REPO_DIR=$(OPENCV_REPO_DIR)
export BUILD_OPENCV=$(BUILD_OPENCV)
export INSTALL_OPENVINO=$(INSTALL_OPENVINO)
$(OPENVINO_CONTRIB_REPO_DIR)/modules/arm_plugin/scripts/install_build_dependencies.sh
workingDirectory: $(BUILD_OPENVINO)
displayName: 'Install dependencies'
- task: CMake@1
@ -100,30 +125,21 @@ jobs:
cmakeArgs: >
-GNinja
-DVERBOSE_BUILD=ON
-DOpenCV_DIR=$(INSTALL_OPENCV)/cmake
-DENABLE_OPENCV=OFF
-DPYTHON_INCLUDE_DIRS=$(INSTALL_PYTHON)/include/python3.8
-DPYTHON_LIBRARY=$(INSTALL_PYTHON)/lib/libpython3.8.so
-DENABLE_PYTHON=ON
-DPYTHON_MODULE_EXTENSION=".so"
-DENABLE_TESTS=ON
-DENABLE_FUNCTIONAL_TESTS=ON
-DENABLE_GAPI_TESTS=OFF
-DENABLE_GAPI_PREPROCESSING=OFF
-DENABLE_DATA=OFF
-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath-link,$(INSTALL_OPENCV)/lib
-DTHREADING=SEQ -DENABLE_LTO=ON
-DCMAKE_TOOLCHAIN_FILE=$(OPENVINO_REPO_DIR)/cmake/arm64.toolchain.cmake
-DCMAKE_BUILD_TYPE=$(BUILD_TYPE)
-DBUILD_LIST=imgcodecs,videoio,highgui
-DCMAKE_TOOLCHAIN_FILE=$(OPENCV_REPO_DIR)/platforms/linux/aarch64-gnu.toolchain.cmake
$(OPENCV_REPO_DIR)
workingDirectory: $(BUILD_DIR_OPENCV)
- script: ninja
workingDirectory: $(BUILD_DIR_OPENCV)
displayName: 'Build OpenCV Linux ARM64'
- script: ninja install
workingDirectory: $(BUILD_DIR_OPENCV)
displayName: 'Install OpenCV Linux ARM64'
- task: CMake@1
inputs:
cmakeArgs: >
-GNinja
-DVERBOSE_BUILD=ON
-DCMAKE_BUILD_TYPE=$(BUILD_TYPE)
-DCMAKE_TOOLCHAIN_FILE=$(OPENVINO_REPO_DIR)/cmake/arm64.toolchain.cmake
-DOpenCV_DIR=$(BUILD_DIR_OPENCV)/install/lib/cmake/opencv4
-DENABLE_OPENCV=OFF
-DENABLE_TESTS=ON
-DENABLE_SAMPLES=ON
-DBUILD_java_api=OFF
-DENABLE_INTEL_MYRIAD=OFF
@ -131,26 +147,102 @@ jobs:
-DIE_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)/modules
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-DCMAKE_C_COMPILER_LAUNCHER=ccache
-DARM_COMPUTE_SCONS_JOBS=$(NUM_PROC)
-DOUTPUT_ROOT=$(INSTALL_OPENVINO)
-DCMAKE_INSTALL_PREFIX=$(INSTALL_OPENVINO)
$(OPENVINO_REPO_DIR)
workingDirectory: $(BUILD_DIR)
- script: ls -alR $(OPENVINO_REPO_DIR)/temp/
displayName: 'List temp SDKs'
- script: ccache --zero-stats --max-size=50G --show-config
displayName: 'Clean ccache stats'
workingDirectory: $(BUILD_OPENVINO)
displayName: 'CMake OpenVINO ARM plugin'
- script: |
export CCACHE_DIR=$(CCACHE_DIR)
export CCACHE_DIR=$(OPENVINO_CCACHE_DIR)
export CCACHE_TEMPDIR=$(TMP_DIR)/ccache
export CCACHE_BASEDIR=$(Pipeline.Workspace)
export CCACHE_MAXSIZE=50G
export USE_CCACHE=1
ninja
workingDirectory: $(BUILD_DIR)
displayName: 'Build Linux ARM64'
workingDirectory: $(BUILD_OPENVINO)
displayName: 'Build OpenVINO ARM plugin'
- script: ccache --show-stats
displayName: 'Show ccache stats'
- script: ninja install
workingDirectory: $(BUILD_OPENVINO)
displayName: 'Install OpenVINO ARM plugin'
- script: ls -alR $(OPENVINO_REPO_DIR)/bin/
displayName: 'List binary files'
- task: CMake@1
inputs:
cmakeArgs: >
-GNinja
-DInferenceEngineDeveloperPackage_DIR=$(BUILD_OPENVINO)
-DENABLE_PYTHON=ON
-DPYTHON_EXECUTABLE=$(INSTALL_PYTHON)/bin/python3.8
-DPYTHON_INCLUDE_DIRS=$(INSTALL_PYTHON)/include/python3.8
-DPYTHON_LIBRARIES=$(INSTALL_PYTHON)/lib
-DPYTHON3_NUMPY_INCLUDE_DIRS=/usr/local/lib/python3.8/site-packages/numpy/core/include
-DPYTHON_MODULE_EXTENSION=".so"
-DPYBIND11_FINDPYTHON=OFF
-DPYBIND11_NOPYTHON=OFF
-DPYTHONLIBS_FOUND=TRUE
-DCMAKE_BUILD_TYPE=$(BUILD_TYPE)
-DENABLE_DATA=OFF
-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath-link,$(INSTALL_OPENCV)/lib
-DCMAKE_TOOLCHAIN_FILE=$(OPENVINO_REPO_DIR)/cmake/arm64.toolchain.cmake
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-DCMAKE_C_COMPILER_LAUNCHER=ccache
-DCMAKE_INSTALL_PREFIX=$(INSTALL_OPENVINO)
$(OPENVINO_REPO_DIR)/src/bindings/python
workingDirectory: $(BUILD_OPENVINO_PYTHON)
displayName: 'CMake OpenVINO python binding'
- script: |
export CCACHE_DIR=$(OPENVINO_CCACHE_DIR)
export CCACHE_TEMPDIR=$(TMP_DIR)/ccache
export CCACHE_BASEDIR=$(Pipeline.Workspace)
export CCACHE_MAXSIZE=50G
export USE_CCACHE=1
ninja
workingDirectory: $(BUILD_OPENVINO_PYTHON)
displayName: 'Build OpenVINO python binding'
- script: ninja install
workingDirectory: $(BUILD_OPENVINO_PYTHON)
displayName: 'Install OpenVINO python binding'
- task: CMake@1
inputs:
cmakeArgs: >
-GNinja
-DCMAKE_BUILD_TYPE=$(BUILD_TYPE)
-DENABLE_PYTHON=ON
-DPYTHON_EXECUTABLE=/usr/local/bin/python3.8
-DPYTHON_INCLUDE_DIR=$(INSTALL_PYTHON)/include/python3.8
-DPYTHON_LIBRARY=$(INSTALL_PYTHON)/lib
-DCMAKE_TOOLCHAIN_FILE=$(OPENVINO_REPO_DIR)/cmake/arm64.toolchain.cmake
-DOpenVINO_DIR=$(BUILD_OPENVINO)
-DInferenceEngine_DIR=$(BUILD_OPENVINO)
-DOpenCV_DIR=$(INSTALL_OPENCV)/cmake
-Dngraph_DIR=$(BUILD_OPENVINO)
-DIE_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)/modules
-DCMAKE_INSTALL_PREFIX=$(INSTALL_OPEN_MODEL_ZOO)
$(OPENVINO_REPO_DIR)/thirdparty/open_model_zoo/demos
workingDirectory: $(BUILD_OPEN_MODEL_ZOO)
displayName: 'CMake Open Model Zoo demos'
- script: ninja
workingDirectory: $(BUILD_OPEN_MODEL_ZOO)
displayName: 'Build Open Model Zoo demos'
- script: ninja install
workingDirectory: $(BUILD_OPEN_MODEL_ZOO)
displayName: 'Install Open Model Zoo demos'
- script: |
cp -r $(BUILD_OPEN_MODEL_ZOO)/$(OPENVINO_ARCH)/$(BUILD_TYPE)/* $(INSTALL_OPEN_MODEL_ZOO)/
zip -9 -r $(Build.ArtifactStagingDirectory)/openvino_$(OPENVINO_ARCH)_linux.zip ./*
workingDirectory: $(INSTALL_OPENVINO)
displayName: 'Create OpenVINO ARM64 linux package'
- task: PublishBuildArtifacts@1
inputs:
pathToPublish: $(Build.ArtifactStagingDirectory)
artifactName: 'openvino_aarch64_linux'
displayName: 'Publish OpenVINO AArch64 linux package'

19
.ci/azure/windows.yml
View File

@ -30,7 +30,7 @@ jobs:
maxParallel: 2
# About 150% of total time
timeoutInMinutes: 150
timeoutInMinutes: 180
pool:
name: WIN_VMSS_VENV_D8S_WU2
@ -133,7 +133,7 @@ jobs:
- script: |
set PATH=$(WORK_DIR)\ninja-win;%PATH%
call "$(MSVS_VARS_PATH)" && $(CMAKE_CMD) -G "Ninja Multi-Config" -DENABLE_WHEEL=ON -DENABLE_ONEDNN_FOR_GPU=$(CMAKE_BUILD_SHARED_LIBS) -DENABLE_GAPI_PREPROCESSING=$(CMAKE_BUILD_SHARED_LIBS) -DBUILD_SHARED_LIBS=$(CMAKE_BUILD_SHARED_LIBS) -DENABLE_REQUIREMENTS_INSTALL=OFF -DENABLE_FASTER_BUILD=ON -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DENABLE_TESTS=ON -DENABLE_STRICT_DEPENDENCIES=OFF -DENABLE_PYTHON=ON -DPYTHON_EXECUTABLE="C:\hostedtoolcache\windows\Python\3.7.6\x64\python.exe" -DPYTHON_INCLUDE_DIR="C:\hostedtoolcache\windows\Python\3.7.6\x64\include" -DPYTHON_LIBRARY="C:\hostedtoolcache\windows\Python\3.7.6\x64\libs\python37.lib" -DIE_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)\modules -DCMAKE_C_COMPILER:PATH="$(MSVC_COMPILER_PATH)" -DCMAKE_CXX_COMPILER:PATH="$(MSVC_COMPILER_PATH)" $(REPO_DIR)
call "$(MSVS_VARS_PATH)" && $(CMAKE_CMD) -G "Ninja Multi-Config" -DENABLE_WHEEL=ON -DENABLE_ONEDNN_FOR_GPU=$(CMAKE_BUILD_SHARED_LIBS) -DBUILD_SHARED_LIBS=$(CMAKE_BUILD_SHARED_LIBS) -DENABLE_REQUIREMENTS_INSTALL=OFF -DENABLE_FASTER_BUILD=ON -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DENABLE_TESTS=ON -DENABLE_STRICT_DEPENDENCIES=OFF -DENABLE_PYTHON=ON -DPYTHON_EXECUTABLE="C:\hostedtoolcache\windows\Python\3.7.6\x64\python.exe" -DPYTHON_INCLUDE_DIR="C:\hostedtoolcache\windows\Python\3.7.6\x64\include" -DPYTHON_LIBRARY="C:\hostedtoolcache\windows\Python\3.7.6\x64\libs\python37.lib" -DIE_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)\modules -DCMAKE_C_COMPILER:PATH="$(MSVC_COMPILER_PATH)" -DCMAKE_CXX_COMPILER:PATH="$(MSVC_COMPILER_PATH)" $(REPO_DIR)
workingDirectory: $(BUILD_DIR)
displayName: 'CMake'
@ -167,13 +167,6 @@ jobs:
workingDirectory: $(BUILD_SAMPLES_TESTS_DIR)
displayName: 'Install Samples Tests'
- script: $(CMAKE_CMD) -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) -DCOMPONENT=tests -P cmake_install.cmake && xcopy $(REPO_DIR)\temp\opencv_4.5.2\opencv\* $(INSTALL_DIR)\opencv\ /e /h /y
workingDirectory: $(BUILD_DIR)
displayName: 'Install tests'
- script: dir $(INSTALL_DIR) /s
displayName: 'List install files'
- script: $(INSTALL_DIR)\samples\cpp\build_samples_msvc.bat -i $(INSTALL_DIR)
workingDirectory: $(BUILD_SAMPLES_DIR)
displayName: 'Build cpp samples'
@ -198,9 +191,15 @@ jobs:
python -m pytest $(INSTALL_DIR)\tests\smoke_tests\ --env_conf $(INSTALL_DIR)\tests\smoke_tests\env_config.yml -s --junitxml=TEST-SamplesSmokeTests.xml
workingDirectory: $(INSTALL_DIR)
displayName: 'Samples Smoke Tests'
condition: eq(variables['CMAKE_BUILD_SHARED_LIBS'], 'ON')
continueOnError: false
- script: $(CMAKE_CMD) -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) -DCOMPONENT=tests -P cmake_install.cmake && xcopy $(REPO_DIR)\temp\opencv_4.5.2\opencv\* $(INSTALL_DIR)\opencv\ /e /h /y
workingDirectory: $(BUILD_DIR)
displayName: 'Install tests'
- script: dir $(INSTALL_DIR) /s
displayName: 'List install files'
- script: rd /Q /S $(BUILD_DIR)
displayName: 'Clean build dir'
continueOnError: false

9
.github/github_org_control/config.json vendored
View File

@ -5,11 +5,10 @@
"IGNORE_LOGINS": [
"openvino-ci",
"openvino-pushbot",
"lab-nerval",
"lab-nerval-onnx-ci",
"onnx-watchdog-agent",
"workbench-ci-bot",
"openvino-pot-ci"
"openvino-pot-ci",
"sysicvvpux",
"ote-ci-bot"
],
"MAX_MEMBERS_TO_REMOVE": 15,
"EMAILS_FILE_PATH": "dev_emails-test.txt",
@ -28,7 +27,7 @@
"openvino-ie-gna-maintainers": "category: GNA",
"openvino-ie-gpu-maintainers": "category: GPU",
"openvino-ie-lpt-maintainers": "category: LP transformations",
"openvino-ie-multi-maintainers": "category: MULTI",
"openvino-ie-auto-multi-maintainers": "category: MULTI",
"openvino-ie-python-api-maintainers": "category: python api",
"openvino-ie-template-maintainers": "category: TEMPLATE",
"openvino-ie-tests-maintainers": "category: IE Tests",

2
.github/github_org_control/github_api.py vendored
View File

@ -157,7 +157,7 @@ class GithubOrgApi:
self.github_users_by_email[email] = org_member
if not is_valid_name(org_member.name):
self.members_to_fix_name.add(org_member)
elif not is_user_ignored(org_member):
else:
self.members_to_remove.add(org_member)
print("\nOrg members - no Intel emails:")

8
CMakeLists.txt
View File

@ -13,6 +13,10 @@ endif()
project(OpenVINO DESCRIPTION "OpenVINO toolkit")
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "CMake build type" FORCE)
endif()
set(IE_MAIN_SOURCE_DIR ${OpenVINO_SOURCE_DIR}/inference-engine)
find_package(IEDevScripts REQUIRED
@ -34,7 +38,9 @@ endif()
message (STATUS "PROJECT ............................... " ${PROJECT_NAME})
message (STATUS "CMAKE_VERSION ......................... " ${CMAKE_VERSION})
message (STATUS "CMAKE_BINARY_DIR ...................... " ${CMAKE_BINARY_DIR})
message (STATUS "CMAKE_SOURCE_DIR ...................... " ${CMAKE_SOURCE_DIR})
message (STATUS "OpenVINO_SOURCE_DIR ................... " ${OpenVINO_SOURCE_DIR})
message (STATUS "OpenVINO_BINARY_DIR ................... " ${OpenVINO_BINARY_DIR})
message (STATUS "CMAKE_GENERATOR ....................... " ${CMAKE_GENERATOR})
message (STATUS "CMAKE_C_COMPILER_ID ................... " ${CMAKE_C_COMPILER_ID})
message (STATUS "CMAKE_CXX_COMPILER_ID ................. " ${CMAKE_CXX_COMPILER_ID})
@ -42,7 +48,7 @@ message (STATUS "CMAKE_BUILD_TYPE ...................... " ${CMAKE_BUILD_TYPE})
message (STATUS "CMAKE_TOOLCHAIN_FILE .................. " ${CMAKE_TOOLCHAIN_FILE})
# remove file with exported developer targets to force its regeneration
file(REMOVE "${CMAKE_BINARY_DIR}/ngraph/ngraphTargets.cmake")
file(REMOVE "${CMAKE_BINARY_DIR}/ngraphTargets.cmake")
file(REMOVE "${CMAKE_BINARY_DIR}/InferenceEngineTargets.cmake")
file(REMOVE "${CMAKE_BINARY_DIR}/OpenVINOTargets.cmake")
foreach(component IN LISTS openvino_export_components)

20
CODEOWNERS
View File

@ -39,14 +39,19 @@ Jenkinsfile @openvinotoolkit/openvino-admins
# IE CPU:
/src/plugins/intel_cpu/ @openvinotoolkit/openvino-ie-cpu-maintainers @openvinotoolkit/openvino-ie-cpu-developers
/src/common/low_precision_transformations/ @openvinotoolkit/openvino-ie-cpu-maintainers @openvinotoolkit/openvino-ie-cpu-developers
/src/plugins/intel_cpu/thirdparty/mkl-dnn/ @openvinotoolkit/openvino-ie-cpu-maintainers @openvinotoolkit/openvino-ie-cpu-developers
#IE LPT
/src/common/low_precision_transformations/ @openvinotoolkit/openvino-ie-lpt-maintainers
# IE GPU:
/src/inference/include/ie/gpu/ @openvinotoolkit/openvino-ie-gpu-maintainers @openvinotoolkit/openvino-ie-gpu-developers
/src/inference/include/ie/cldnn/ @openvinotoolkit/openvino-ie-gpu-maintainers @openvinotoolkit/openvino-ie-gpu-developers
/src/inference/include/openvino/runtime/intel_gpu/ @openvinotoolkit/openvino-ie-gpu-maintainers @openvinotoolkit/openvino-ie-gpu-developers
/src/plugins/intel_gpu/ @openvinotoolkit/openvino-ie-gpu-maintainers @openvinotoolkit/openvino-ie-gpu-developers
/docs/snippets/gpu/ @openvinotoolkit/openvino-ie-gpu-maintainers @openvinotoolkit/openvino-ie-gpu-developers
/docs/OV_Runtime_UG/supported_plugins/GPU.md @openvinotoolkit/openvino-ie-gpu-maintainers @openvinotoolkit/openvino-ie-gpu-developers
/docs/OV_Runtime_UG/supported_plugins/GPU_RemoteTensor_API.md @openvinotoolkit/openvino-ie-gpu-maintainers @openvinotoolkit/openvino-ie-gpu-developers
# IE VPU:
/src/plugins/intel_myriad @openvinotoolkit/openvino-ie-vpu-maintainers
@ -63,6 +68,9 @@ Jenkinsfile @openvinotoolkit/openvino-admins
/src/plugins/intel_gna/ @openvinotoolkit/openvino-ie-gna-maintainers
/src/inference/include/ie/gna/ @openvinotoolkit/openvino-ie-gna-maintainers
# IE ARM CPU:
/docs/OV_Runtime_UG/supported_plugins/ARM_CPU.md @openvinotoolkit/openvino_contrib-arm_plugin-maintainers
# IE Auto (MULTI) plugin:
/src/plugins/auto/ @openvinotoolkit/openvino-ie-auto-multi-maintainers
/src/inference/include/ie/multi-device/ @openvinotoolkit/openvino-ie-auto-multi-maintainers
@ -71,8 +79,8 @@ Jenkinsfile @openvinotoolkit/openvino-admins
/src/frontends/paddle/ @openvinotoolkit/openvino-ie-paddle-maintainers
# IE Tests:
/src/tests/ @openvinotoolkit/openvino-ie-tests-maintainers
/src/tests_deprecated/ @openvinotoolkit/openvino-ie-tests-maintainers
/src/tests/ @openvinotoolkit/openvino-ie-tests-maintainers @openvinotoolkit/openvino-ie-test-developers
/src/tests_deprecated/ @openvinotoolkit/openvino-ie-tests-maintainers @openvinotoolkit/openvino-ie-test-developers
/src/tests/functional/inference_engine/ngraph_reader/ @openvinotoolkit/openvino-ie-tests-maintainers @openvinotoolkit/openvino-ngraph-maintainers
/src/tests/functional/inference_engine/transformations/ @openvinotoolkit/openvino-ie-tests-maintainers @openvinotoolkit/openvino-ngraph-maintainers
@ -82,6 +90,6 @@ Jenkinsfile @openvinotoolkit/openvino-admins
*.md @openvinotoolkit/openvino-docs-maintainers
# Control 3d party dependencies
**/*requirements*.* @openvino-configuration-mgmt
**/setup.py @openvino-configuration-mgmt
/scripts/install_dependencies/ @openvino-configuration-mgmt
**/*requirements*.* @openvinotoolkit/openvino-configuration-mgmt
**/setup.py @openvinotoolkit/openvino-configuration-mgmt
/scripts/install_dependencies/ @openvinotoolkit/openvino-configuration-mgmt

68
CONTRIBUTING.md Normal file
View File

@ -0,0 +1,68 @@
# How to contribute to the OpenVINO repository
We suppose that you are an enthusiastic coder, want to contribute some code. For that purpose OpenVINO project now has a repository on the GitHub, to simplify everybody's life! All the bug fixes, new functionality, new tutorials etc. should be submitted via the GitHub's mechanism of pull requests.
If you are not familiar with the mechanism - do not worry, it's very simple. Keep reading.
## Before you start contributing you should
- Make sure you agree to contribute your code under [OpenVINO (Apache 2.0)](https://github.com/openvinotoolkit/openvino/blob/master/LICENSE) license.
- If you are submitting a new module, you should go into [openvino_contrib](https://github.com/openvinotoolkit/openvino_contrib) repository by default.
- If you are going to fix a bug, check that it's still exists. This can be done by building the latest [releases/2020/3](https://github.com/openvinotoolkit/openvino/tree/releases/2020/3) branch (LTS release) or the latest master branch, and make sure that the error is still reproducible there. We do not fix bugs that only affect older non-LTS releases like 2020.2 for example (more details about [branching strategy](https://github.com/openvinotoolkit/openvino/wiki/Branches))
- Make sure that nobody beat you into fixing or reporting the issue by doing a search on the [Github OpenVINO issues](https://github.com/openvinotoolkit/openvino/issues) page, and making sure that there isn't someone working on it. In the latter case you might provide support or suggestion in the issue or in the linked pull request.
- If you have a question about the software, then this is **NOT** the right place. You should open up a question at the [OpenVINO forum](https://community.intel.com/t5/Intel-Distribution-of-OpenVINO/bd-p/distribution-openvino-toolkit). In order to post a decent question from the start, feel free to read the official forum guidelines.
Before you open up anything on the OpenVINO GitHub page, be sure that you are at the right place with your problem.
## "Fork & Pull Request model" for code contribution
### [](https://github.com/openvinotoolkit/openvino/wiki/Contribute#the-instruction-in-brief)The instruction in brief
- Register at GitHub. Create your fork of OpenVINO repository [https://github.com/openvinotoolkit/openvino](https://github.com/openvinotoolkit/openvino) (see [https://help.github.com/articles/fork-a-repo](https://help.github.com/articles/fork-a-repo) for details).
- Install Git.
- Set your user name and email address in a Git configuration according to GitHub account (see [https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup](https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup) for details).
- Choose a task for yourself. It could be a bugfix or some new code.
- Choose a base branch for your work. More details about branches and policies are here: [Branches](https://github.com/openvinotoolkit/openvino/wiki/Branches)
- Clone your fork to your computer.
- Create a new branch (with a meaningful name) from the base branch you chose.
- Modify / add the code following our [Coding Style Guide](https://github.com/openvinotoolkit/openvino/wiki/CodingStyleGuideLines) and [Documentation guidelines](https://github.com/openvinotoolkit/openvino/wiki/CodingStyleGuideLinesDocumentation).
- If you want to add a new sample, please look at this [Guide for contributing to C++/C/Python IE samples](https://github.com/openvinotoolkit/openvino/wiki/SampleContribute)
- Run testsuite locally:
- execute each test binary from the artifacts directory, e.g. `<source dir>/bin/intel64/Release/ieFuncTests`
- If you contribute to the documentation and want to add a new guide:
- Create a new markdown file in an appropriate folder.
- **REQUIRED:** The document title must contain a document label in a form: `{#openvino_docs_<name>}`. For example: `Deep Learning Network Intermediate Representation and Operation Sets in OpenVINO™ {#openvino_docs_MO_DG_IR_and_opsets}`.
- Add your file to the documentation structure. Open the documentation structure file [`docs/doxygen/ie_docs.xml`](https://github.com/openvinotoolkit/openvino/blob/master/docs/doxygen/ie_docs.xml) and add your file path to the appropriate section.
- When you are done, make sure that your branch is to date with latest state of the branch you want to contribute to (e.g. `git fetch upstream && git merge upstream/master`), push your branch to your GitHub fork; then create a pull request from your branch to the base branch (see [https://help.github.com/articles/using-pull-requests](https://help.github.com/articles/using-pull-requests) for details).
## Making a good pull request
Following these guidelines will increase the likelihood of your pull request being accepted:
- Before pushing your PR to the repository, make sure that it builds perfectly fine on your local system.
- Add enough information, like a meaningful title, the reason why you made the commit and a link to the issue page if you opened one for this PR.
- Scope your PR to one issue. Before submitting, make sure the diff contains no unrelated changes. If you want to cover more than one issue, submit your changes for each as separate pull requests.
- If you have added new functionality, you should update/create the relevant documentation, as well as add tests for it to the testsuite.
- Try not to include "oops" commits - ones that just fix an error in the previous commit. If you have those, then before submitting [squash](https://github.com/openvinotoolkit/openvino/wiki/Contribute#https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History#Squashing-Commits) those fixes directly into the commits where they belong.
- Make sure to choose the right base branch and to follow the [Coding Style Guide](https://github.com/openvinotoolkit/openvino/wiki/CodingStyleGuideLines) for your code or [Documentation guidelines](https://github.com/openvinotoolkit/openvino/wiki/CodingStyleGuideLinesDocumentation) you are changing documentation files.
- Make sure to add test for new functionality or test that reproduces fixed bug with related test data. Please do not add extra images or videos, if some of existing media files are suitable.
## Testing and merging pull requests
- Your pull request will be automatically tested by OpenVINO's precommit (testing status are automatically reported as "green" or "red" circles in precommit steps on PR's page). If any builders have failed, you should fix the issue. To rerun the automatic builds just push changes to your branch on GitHub. No need to close pull request and open a new one!
- Once all the builders are "green", one of OpenVINO developers will review your code. Reviewer could ask you to modify your pull request. Please provide timely response for reviewers (within weeks, not months), otherwise you submission could be postponed or even rejected.
## PR review good practices
- Originator is responsible for driving the review of changes and should ping reviewers periodically.
- Originator should close comments from the Reviewer when it is resolved. The Reviewer may re-open the comment if he does not agree with the resolution.
- Originator should request re-review from the Reviewer when all comments are resolved by pushing the button in the “Reviewers” section.
- If it is still WIP and you want to check CI test results early then use _Draft_ PR.
- Do **NOT** rewrite history (push -f) once you converted draft PR into regular one, add new commits instead. Looking at diffs makes review easier.
- Write meaningful description of commits resulting from review. _"Addressing review comments"_ is **NOT** a good description! Having a quick look at good descriptions can tell you much what is going on in PR without a need to go through all of resolved comments.
## Merging PR
As soon as the reviewer is fine with the pull request and Precommit likes your code and shows "green" status, the "Approved" review status is put, which signals OpenVINO maintainers that they can merge your pull request.
© Copyright 2018-2022, OpenVINO team

8
README.md
View File

@ -16,12 +16,11 @@ source and public models in popular formats such as TensorFlow, ONNX, PaddlePadd
* [OpenVINO™ Runtime]
* [Model Optimizer]
* [Post-Training Optimization Tool]
* [Samples]
## License
Deep Learning Deployment Toolkit is licensed under [Apache License Version 2.0](LICENSE).
By contributing to the project, you agree to the license and copyright terms therein
and release your contribution under these terms.
OpenVINO™ Toolkit is licensed under [Apache License Version 2.0](LICENSE).
By contributing to the project, you agree to the license and copyright terms therein and release your contribution under these terms.
## Resources
* Docs: https://docs.openvino.ai/
@ -46,5 +45,6 @@ Please report questions, issues and suggestions using:
[OpenVINO™ Runtime]:https://docs.openvino.ai/latest/openvino_docs_OV_Runtime_User_Guide.html
[Model Optimizer]:https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html
[Post-Training Optimization Tool]:https://docs.openvino.ai/latest/pot_README.html
[Samples]:https://github.com/openvinotoolkit/openvino/tree/master/samples
[tag on StackOverflow]:https://stackoverflow.com/search?q=%23openvino

24
cmake/developer_package/IEDevScriptsConfig.cmake
View File

@ -158,16 +158,22 @@ else ()
endif()
add_definitions(-DIE_BUILD_POSTFIX=\"${IE_BUILD_POSTFIX}\")
macro(ov_set_if_not_defined var value)
if(NOT DEFINED ${var})
set(${var} ${value})
endif()
endmacro()
if(NOT UNIX)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${OUTPUT_ROOT}/${BIN_FOLDER})
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${OUTPUT_ROOT}/${BIN_FOLDER})
ov_set_if_not_defined(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${OUTPUT_ROOT}/${BIN_FOLDER})
ov_set_if_not_defined(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${OUTPUT_ROOT}/${BIN_FOLDER})
else()
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${OUTPUT_ROOT}/${BIN_FOLDER}/lib)
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${OUTPUT_ROOT}/${BIN_FOLDER}/lib)
ov_set_if_not_defined(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${OUTPUT_ROOT}/${BIN_FOLDER}/lib)
ov_set_if_not_defined(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${OUTPUT_ROOT}/${BIN_FOLDER}/lib)
endif()
set(CMAKE_COMPILE_PDB_OUTPUT_DIRECTORY ${OUTPUT_ROOT}/${BIN_FOLDER})
set(CMAKE_PDB_OUTPUT_DIRECTORY ${OUTPUT_ROOT}/${BIN_FOLDER})
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${OUTPUT_ROOT}/${BIN_FOLDER})
ov_set_if_not_defined(CMAKE_COMPILE_PDB_OUTPUT_DIRECTORY ${OUTPUT_ROOT}/${BIN_FOLDER})
ov_set_if_not_defined(CMAKE_PDB_OUTPUT_DIRECTORY ${OUTPUT_ROOT}/${BIN_FOLDER})
ov_set_if_not_defined(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${OUTPUT_ROOT}/${BIN_FOLDER})
if(APPLE)
set(CMAKE_MACOSX_RPATH ON)
@ -207,6 +213,10 @@ endif()
macro(ov_install_static_lib target comp)
if(NOT BUILD_SHARED_LIBS)
get_target_property(target_type ${target} TYPE)
if(${target_type} STREQUAL "STATIC_LIBRARY")
set_target_properties(${target} PROPERTIES EXCLUDE_FROM_ALL FALSE)
endif()
install(TARGETS ${target} EXPORT OpenVINOTargets
ARCHIVE DESTINATION ${OV_CPACK_ARCHIVEDIR} COMPONENT ${comp} ${ARGN})
endif()

6
cmake/developer_package/api_validator/api_validator.cmake
View File

@ -51,12 +51,6 @@ endfunction()
set(VALIDATED_LIBRARIES "" CACHE INTERNAL "")
function(_ov_add_api_validator_post_build_step)
if(NOT BUILD_SHARED_LIBS)
# since _ov_add_api_validator_post_build_step
# is currently run only on shared libraries, we have nothing to test
return()
endif()
set(UWP_API_VALIDATOR_APIS "${PROGRAMFILES}/Windows Kits/10/build/universalDDIs/x64/UniversalDDIs.xml")
set(UWP_API_VALIDATOR_EXCLUSION "${UWP_SDK_PATH}/BinaryExclusionlist.xml")

2
cmake/developer_package/cpplint/cpplint.py vendored
View File

@ -3592,7 +3592,7 @@ def CheckOperatorSpacing(filename, clean_lines, linenum, error):
elif not Match(r'#.*include', line):
# Look for < that is not surrounded by spaces. This is only
# triggered if both sides are missing spaces, even though
# technically should should flag if at least one side is missing a
# technically should flag if at least one side is missing a
# space. This is done to avoid some false positives with shifts.
match = Match(r'^(.*[^\s<])<[^\s=<,]', line)
if match:

3
cmake/developer_package/download/download_and_extract.cmake
View File

@ -146,8 +146,6 @@ function (DownloadOrExtractInternal URL archive_path unpacked_path folder fattal
endfunction(DownloadOrExtractInternal)
file(REMOVE ${CMAKE_BINARY_DIR}/dependencies_64.txt)
function (CheckOrDownloadAndExtract component RELATIVE_URL archive_name unpacked_path result_path folder fattal resultExt use_alternatives sha256 files_to_extract)
set (archive_path ${TEMP}/download/${archive_name})
set (status "ON")
@ -164,7 +162,6 @@ function (CheckOrDownloadAndExtract component RELATIVE_URL archive_name unpacked
if (${use_alternatives})
set(DEP_INFO "${component}=${URL}")
debug_message (STATUS "DEPENDENCY_URL: ${DEP_INFO}")
file(APPEND ${CMAKE_BINARY_DIR}/dependencies_64.txt "${DEP_INFO}\n")
endif()
debug_message ("checking that unpacked directory exist: ${unpacked_path}")

11
cmake/developer_package/frontends/frontends.cmake
View File

@ -107,6 +107,17 @@ macro(ov_add_frontend)
set(FRONTEND_NAMES "${FRONTEND_NAMES}" CACHE INTERNAL "" FORCE)
file(GLOB_RECURSE LIBRARY_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp)
if (WIN32)
# Remove linux specific files
file(GLOB_RECURSE LIN_FILES ${CMAKE_CURRENT_SOURCE_DIR}/src/os/lin/*.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/os/lin/*.hpp)
list(REMOVE_ITEM LIBRARY_SRC "${LIN_FILES}")
else()
# Remove windows specific files
file(GLOB_RECURSE WIN_FILES ${CMAKE_CURRENT_SOURCE_DIR}/src/os/win/*.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/os/win/*.hpp)
list(REMOVE_ITEM LIBRARY_SRC "${WIN_FILES}")
endif()
file(GLOB_RECURSE LIBRARY_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/*.hpp)
file(GLOB_RECURSE LIBRARY_PUBLIC_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/include/*.hpp)

4
cmake/developer_package/ncc_naming_style/ncc_naming_style.cmake
View File

@ -107,8 +107,10 @@ function(ov_ncc_naming_style)
list(APPEND NCC_STYLE_ADDITIONAL_INCLUDE_DIRECTORIES "${NCC_STYLE_SOURCE_DIRECTORY}")
# without it sources with same name from different directories will map to same .ncc_style target
file(RELATIVE_PATH source_dir_rel ${CMAKE_SOURCE_DIR} ${NCC_STYLE_SOURCE_DIRECTORY})
foreach(source IN LISTS sources)
set(output_file "${ncc_style_bin_dir}/${source}.ncc_style")
set(output_file "${ncc_style_bin_dir}/${source_dir_rel}/${source}.ncc_style")
set(full_source_path "${NCC_STYLE_SOURCE_DIRECTORY}/${source}")
add_custom_command(

6
cmake/developer_package/plugins/plugins.cmake
View File

@ -230,6 +230,10 @@ macro(ie_register_plugins_dynamic)
endif()
list(GET name 0 device_name)
list(GET name 1 name)
# Skip plugins which don't exist in the possible plugins list
if (IE_REGISTER_POSSIBLE_PLUGINS AND NOT name IN_LIST IE_REGISTER_POSSIBLE_PLUGINS)
continue()
endif()
# create plugin file
set(config_file_name "${CMAKE_BINARY_DIR}/plugins/${device_name}.xml")
@ -358,7 +362,7 @@ function(ie_generate_plugins_hpp)
# for some reason dependency on source files does not work
# so, we have to use explicit target and make it dependency for inference_engine
add_custom_target(_ie_plugins_hpp DEPENDS ${ie_plugins_hpp})
add_dependencies(inference_engine _ie_plugins_hpp)
add_dependencies(inference_engine_obj _ie_plugins_hpp)
# add dependency for object files
get_target_property(sources inference_engine_obj SOURCES)

2
cmake/templates/InferenceEngineDeveloperPackageConfig.cmake.in
View File

@ -45,7 +45,7 @@ find_dependency(InferenceEngine
NO_DEFAULT_PATH)
find_dependency(ngraph
PATHS "${CMAKE_CURRENT_LIST_DIR}/src/core"
PATHS "${CMAKE_CURRENT_LIST_DIR}"
NO_CMAKE_FIND_ROOT_PATH
NO_DEFAULT_PATH)

5
cmake/test_model_zoo.cmake
View File

@ -86,11 +86,6 @@ ov_model_convert("${OpenVINO_SOURCE_DIR}/${rel_path}"
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/test_model_zoo/onnx_import"
ie_onnx_import_out_files)
set(rel_path "docs/onnx_custom_op")
ov_model_convert("${OpenVINO_SOURCE_DIR}/${rel_path}"
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/test_model_zoo/docs/models"
docs_onnx_out_files)
if(ENABLE_TESTS)
if(ENABLE_OV_ONNX_FRONTEND AND ENABLE_REQUIREMENTS_INSTALL)
find_package(PythonInterp 3 REQUIRED)

2
cmake/toolchains/mt.runtime.win32.toolchain.cmake
View File

@ -25,7 +25,7 @@ endif()
if(use_static_runtime)
foreach(lang C CXX)
foreach(build_type "" "_DEBUG" "_MINSIZEREL" "_RELEASE" "_RELWITHDEBINFO")
set(flag_var "CMAKE_${lang}_FLAGS${build_type}")
set(flag_var "CMAKE_${lang}_FLAGS${build_type}_INIT")
string(REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
endforeach()
endforeach()

41
cmake/toolchains/oecore.arm64.toolchain.cmake
View File

@ -1,41 +0,0 @@
# Copyright (C) 2018-2022 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
if(DEFINED OECORE_BASE_DIR)
# OECORE_BASE_DIR was passed via CMake command line, nothing to do
elseif(DEFINED ENV{OECORE_BASE_DIR})
# User sets OECORE_BASE_DIR environment variable
set(OECORE_BASE_DIR $ENV{OECORE_BASE_DIR})
elseif(DEFINED ENV{OECORE_NATIVE_SYSROOT})
# OECORE_NATIVE_SYSROOT is a default environment variable for the OECore toolchain
set(OECORE_BASE_DIR "$ENV{OECORE_NATIVE_SYSROOT}/../..")
else()
# Use default value
set(OECORE_BASE_DIR "/usr/local/oecore-x86_64")
endif()
set(OECORE_TARGET_NAME "aarch64-ese-linux")
set(OECORE_TARGET_SYSROOT "${OECORE_BASE_DIR}/sysroots/${OECORE_TARGET_NAME}")
set(OECORE_HOST_SYSROOT "${OECORE_BASE_DIR}/sysroots/x86_64-esesdk-linux")
set(OECORE_HOST_COMPILER_BIN_DIR "${OECORE_HOST_SYSROOT}/usr/bin/${OECORE_TARGET_NAME}")
set(CMAKE_SYSTEM_NAME "Linux")
set(CMAKE_SYSTEM_PROCESSOR "aarch64")
set(CMAKE_SYSROOT "${OECORE_TARGET_SYSROOT}")
set(CMAKE_C_COMPILER "${OECORE_HOST_COMPILER_BIN_DIR}/aarch64-ese-linux-gcc")
set(CMAKE_CXX_COMPILER "${OECORE_HOST_COMPILER_BIN_DIR}/aarch64-ese-linux-g++")
set(CMAKE_C_FLAGS_INIT "-mcpu=cortex-a53 -mtune=cortex-a53 --sysroot=${OECORE_TARGET_SYSROOT}")
set(CMAKE_CXX_FLAGS_INIT "-mcpu=cortex-a53 -mtune=cortex-a53 --sysroot=${OECORE_TARGET_SYSROOT}")
set(CMAKE_EXE_LINKER_FLAGS_INIT "-Wl,-O1 -Wl,--hash-style=gnu -Wl,--as-needed --sysroot=${OECORE_TARGET_SYSROOT}")
set(CMAKE_SHARED_LINKER_FLAGS_INIT "-Wl,-O1 -Wl,--hash-style=gnu -Wl,--as-needed --sysroot=${OECORE_TARGET_SYSROOT}")
set(CMAKE_MODULE_LINKER_FLAGS_INIT "-Wl,-O1 -Wl,--hash-style=gnu -Wl,--as-needed --sysroot=${OECORE_TARGET_SYSROOT}")
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

18
cmake/toolchains/onecoreuap.toolchain.cmake
View File

@ -35,14 +35,14 @@ if(_onecoreuap_arch STREQUAL "x64")
# Forcefull make VS search for C++ libraries in these folders prior to other c++ standard libraries localizations.
add_link_options("/LIBPATH:\"\$\(VC_LibraryPath_VC_x64_OneCore\)\"")
set(CMAKE_C_STANDARD_LIBRARIES "\$\(UCRTContentRoot\)lib/\$\(TargetUniversalCRTVersion\)/um/\$\(Platform\)/OneCoreUap.lib" CACHE STRING "" FORCE)
set(CMAKE_CXX_STANDARD_LIBRARIES "\$\(UCRTContentRoot\)lib/\$\(TargetUniversalCRTVersion\)/um/\$\(Platform\)/OneCoreUap.lib" CACHE STRING "" FORCE)
set(CMAKE_C_STANDARD_LIBRARIES_INIT "\$\(UCRTContentRoot\)lib/\$\(TargetUniversalCRTVersion\)/um/\$\(Platform\)/OneCoreUap.lib" CACHE STRING "" FORCE)
set(CMAKE_CXX_STANDARD_LIBRARIES_INIT "\$\(UCRTContentRoot\)lib/\$\(TargetUniversalCRTVersion\)/um/\$\(Platform\)/OneCoreUap.lib" CACHE STRING "" FORCE)
elseif(_onecoreuap_arch STREQUAL "X86")
add_link_options("/LIBPATH:\"\$\(VCInstallDir\)lib/onecore\"")
add_link_options("/LIBPATH:\"\$\(VC_LibraryPath_VC_x86_OneCore\)\"")
set(CMAKE_C_STANDARD_LIBRARIES "\$\(UCRTContentRoot\)lib/\$\(TargetUniversalCRTVersion\)/um/x86/OneCoreUap.lib" CACHE STRING "" FORCE)
set(CMAKE_CXX_STANDARD_LIBRARIES "\$\(UCRTContentRoot\)lib/\$\(TargetUniversalCRTVersion\)/um/x86/OneCoreUap.lib" CACHE STRING "" FORCE)
set(CMAKE_C_STANDARD_LIBRARIES_INIT "\$\(UCRTContentRoot\)lib/\$\(TargetUniversalCRTVersion\)/um/x86/OneCoreUap.lib" CACHE STRING "" FORCE)
set(CMAKE_CXX_STANDARD_LIBRARIES_INIT "\$\(UCRTContentRoot\)lib/\$\(TargetUniversalCRTVersion\)/um/x86/OneCoreUap.lib" CACHE STRING "" FORCE)
else()
message(FATAL_ERROR "Unsupported architecture ${_onecoreuap_arch}. Only X86 or X86_64 are supported")
endif()
@ -52,8 +52,8 @@ unset(_onecoreuap_arch)
# compile flags
set(includes "/I\"\$\(UniversalCRT_IncludePath\)\"")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${includes}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${includes}")
set(CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS_INIT} ${includes}")
set(CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS_INIT} ${includes}")
unset(includes)
# linker flags
@ -62,9 +62,9 @@ foreach(lib kernel32 user32 advapi32 ole32 mscoree combase)
set(linker_flags "/NODEFAULTLIB:${lib}.lib ${linker_flags}")
endforeach()
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${linker_flags}")
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${linker_flags}")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${linker_flags}")
set(CMAKE_SHARED_LINKER_FLAGS_INIT "${CMAKE_SHARED_LINKER_FLAGS_INIT} ${linker_flags}")
set(CMAKE_MODULE_LINKER_FLAGS_INIT "${CMAKE_MODULE_LINKER_FLAGS_INIT} ${linker_flags}")
set(CMAKE_EXE_LINKER_FLAGS_INIT "${CMAKE_EXE_LINKER_FLAGS_INIT} ${linker_flags}")
unset(linker_flags)
#

17
docs/CMakeLists.txt
View File

@ -7,8 +7,6 @@ if(NOT ENABLE_DOCKER)
ie_add_compiler_flags(-Wall)
endif()
add_subdirectory(snippets)
# Detect OpenVINO
find_package(OpenVINO QUIET
PATHS "${CMAKE_BINARY_DIR}"
@ -17,9 +15,8 @@ if(NOT ENABLE_DOCKER)
set(OpenVINO_DIR ${CMAKE_BINARY_DIR})
endif()
if(ENABLE_OV_ONNX_FRONTEND)
add_subdirectory(onnx_custom_op)
endif()
add_subdirectory(snippets)
add_subdirectory(template_extension)
set(all_docs_targets
@ -55,6 +52,7 @@ endif()
set(LINKCHECKER_PY "" CACHE FILEPATH "Path to linkchecker.py for documentation check dir.")
set(ENABLE_OPENVINO_NOTEBOOKS OFF CACHE BOOL "Build with openvino notebooks")
set(OMZ_DOCS_DIR "" CACHE PATH "Path to open_model_zoo documentation dir.")
set(OTE_DOCS_DIR "" CACHE PATH "Path to training_extensions documentation dir.")
set(WORKBENCH_DOCS_DIR "" CACHE PATH "Path to workbench documentation dir.")
set(OVMS_DOCS_DIR "" CACHE PATH "Path to model server documentation dir.")
set(GRAPH_CSV_DIR "" CACHE PATH "Path to the folder containing csv data for rendering graphs.")
@ -168,6 +166,15 @@ function(build_docs)
--output_dir=${DOCS_BUILD_DIR}/workbench)
endif()
# ote doc files
if(EXISTS "${OTE_DOCS_DIR}")
get_filename_component(WORKBENCH_DOCS_DIR "${OTE_DOCS_DIR}" ABSOLUTE)
list(APPEND commands COMMAND ${PYTHON_EXECUTABLE} ${DOXY_MD_FILTER}
--input_dir=${OTE_DOCS_DIR}
--output_dir=${DOCS_BUILD_DIR}/ote)
endif()
# ovms doc files
if(EXISTS "${OVMS_DOCS_DIR}")
get_filename_component(OVMS_DOCS_DIR "${OVMS_DOCS_DIR}" ABSOLUTE)

18
docs/Doxyfile.config
View File

@ -719,7 +719,7 @@ SHOW_NAMESPACES = YES
# The FILE_VERSION_FILTER tag can be used to specify a program or script that
# doxygen should invoke to get the current version for each file (typically from
# the version control system). Doxygen will invoke the program by executing (via
# popen()) the command command input-file, where command is the value of the
# popen()) the command input-file, where command is the value of the
# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided
# by doxygen. Whatever the program writes to standard output is used as the file
# version. For an example see the documentation.
@ -843,16 +843,6 @@ INPUT = "@MARKDOWN_INPUT@" \
"@OpenVINO_SOURCE_DIR@/src/common/transformations/include/" \
"@OpenVINO_SOURCE_DIR@/src/common/util/include/" \
"@OpenVINO_SOURCE_DIR@/src/core/include/" \
"@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/" \
"@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/descriptor" \
"@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/op/" \
"@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/op/util" \
"@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/opsets/" \
"@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/pass/" \
"@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/pattern/" \
"@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/pattern/op/" \
"@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/runtime/" \
"@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/type/" \
"@OpenVINO_SOURCE_DIR@/src/core/include/openvino/" \
"@OpenVINO_SOURCE_DIR@/src/core/include/openvino/core/" \
"@OpenVINO_SOURCE_DIR@/src/core/include/openvino/core/descriptor/" \
@ -917,7 +907,9 @@ RECURSIVE = YES
# Note that relative paths are relative to the directory from which doxygen is
# run.
EXCLUDE =
EXCLUDE = "@OpenVINO_SOURCE_DIR@/thirdparty" \
"@OpenVINO_SOURCE_DIR@/temp" \
"@OpenVINO_SOURCE_DIR@/bin"
# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
# directories that are symbolic links (a Unix file system feature) are excluded
@ -936,7 +928,6 @@ EXCLUDE_SYMLINKS = NO
EXCLUDE_PATTERNS = */temp/* \
*/bin/* \
*/tests/* \
*/openvx/* \
*/thirdparty/* \
"@DOXYREST_OUT@" \
"@XML_OUTPUT@" \
@ -1045,7 +1036,6 @@ EXCLUDE_SYMBOLS = InferenceEngine::details \
EXAMPLE_PATH = "@OpenVINO_SOURCE_DIR@" \
"@OpenVINO_SOURCE_DIR@/docs/HOWTO/" \
"@OpenVINO_SOURCE_DIR@/docs/" \
"@OpenVINO_SOURCE_DIR@/docs/onnx_custom_op/" \
"@OpenVINO_SOURCE_DIR@/docs/template_extension/" \
"@OpenVINO_SOURCE_DIR@/docs/template_extension/old/" \
"@OpenVINO_SOURCE_DIR@/docs/template_extension/new/" \

44
docs/OV_Runtime_UG/Extensibility_DG/GPU_Kernel.md → docs/Extensibility_UG/GPU_Extensibility.md
View File

@ -1,17 +1,17 @@
# How to Implement Custom GPU Operations {#openvino_docs_IE_DG_Extensibility_DG_GPU_Kernel}
# How to Implement Custom GPU Operations {#openvino_docs_Extensibility_UG_GPU}
To enable operations not supported by OpenVINO™ out of the box, you need a custom extension for Model Optimizer, a custom nGraph operation set, and a custom kernel for the device you will target. This page describes custom kernel support for the GPU device.
To enable operations not supported by OpenVINO out of the box, you may need an extension for OpenVINO operation set, and a custom kernel for the device you will target. This page describes custom kernel support for the GPU device.
The GPU codepath abstracts many details about OpenCL\*. You need to provide the kernel code in OpenCL C and an XML configuration file that connects the kernel and its parameters to the parameters of the operation.
There are two options for using the custom operation configuration file:
* Include a section with your kernels into the global automatically-loaded `cldnn_global_custom_kernels/cldnn_global_custom_kernels.xml` file, which is hosted in the `<INSTALL_DIR>/runtime/bin` folder
* Call the `InferenceEngine::Core::SetConfig()` method from your application with the `InferenceEngine::PluginConfigParams::KEY_CONFIG_FILE` key and the configuration file name as a value before loading the network that uses custom operations to the plugin:
* Include a section with your kernels into the automatically-loaded `<lib_path>/cldnn_global_custom_kernels/cldnn_global_custom_kernels.xml` file.
* Call the `ov::Core::set_property()` method from your application with the `"CONFIG_FILE"` key and the configuration file name as a value before loading the network that uses custom operations to the plugin:
@snippet snippets/GPU_Kernel.cpp part0
@snippet snippets/gpu/custom_kernels_api.cpp part0
All Inference Engine samples, except the trivial `hello_classification`, and most Open Model Zoo demos
All OpenVINO samples, except the trivial `hello_classification`, and most Open Model Zoo demos
feature a dedicated command-line option `-c` to load custom kernels. For example, to load custom operations for the classification sample, run the command below:
```sh
$ ./classification_sample -m <path_to_model>/bvlc_alexnet_fp16.xml -i ./validation_set/daily/227x227/apron.bmp -d GPU
@ -47,8 +47,7 @@ Notation | Description
### Kernel Node and Sub-Node Structure
`Kernel` node contains all kernel source code configuration. No kernel
node structure exists.
`Kernel` node contains all kernel source code configuration.
**Sub-nodes**: `Source` (1+), `Define` (0+)
@ -134,7 +133,7 @@ queuing an OpenCL program for execution.
## Example Configuration File
The following code sample provides an example configuration file in XML
The following code sample provides an example configuration file in XML
format. For information on the configuration file structure, see
[Configuration File Format](#config-file-format).
```xml
@ -155,8 +154,7 @@ format. For information on the configuration file structure, see
## Built-In Definitions for Custom Layers
The following table includes definitions that are attached before
user sources, where `<TENSOR>` is the actual input and output, for
example, `INPUT0` or `OUTPUT0`.
user sources.
For an example, see [Example Kernel](#example-kernel).
@ -170,19 +168,20 @@ For an example, see [Example Kernel](#example-kernel).
| `<TENSOR>_DIMS`| An array of the tensor dimension sizes. Always ordered as `BFYX` |
| `<TENSOR>_DIMS_SIZE`| The size of the `<TENSOR>_DIMS` array.|
| `<TENSOR>_TYPE`| The datatype of the tensor: `float`, `half`, or `char`|
| `<TENSOR>_FORMAT_` | The format of the tensor, BFYX, BYXF, YXFB , FYXB, or ANY. The format is concatenated to the defined name. You can use the tensor format to define codepaths in your code with `#&zwj;ifdef/#&zwj;endif`. |
| `<TENSOR>_FORMAT_<TENSOR_FORMAT>` | The format of the tensor, BFYX, BYXF, YXFB , FYXB, or ANY. The format is concatenated to the defined name. You can use the tensor format to define codepaths in your code with `#&zwj;ifdef/#&zwj;endif`. |
| `<TENSOR>_LOWER_PADDING` | An array of padding elements used for the tensor dimensions before they start. Always ordered as BFYX.|
| `<TENSOR>_ LOWER_PADDING_SIZE` | The size of the `<TENSOR>_LOWER_PADDING` array |
| `<TENSOR>_LOWER_PADDING_SIZE` | The size of the `<TENSOR>_LOWER_PADDING` array |
| `<TENSOR>_UPPER_PADDING` | An array of padding elements used for the tensor dimensions after they end. Always ordered as BFYX. |
| `<TENSOR>_UPPER_PADDING_SIZE` | The size of the `<TENSOR>_UPPER_PADDING` array |
| `<TENSOR>_PITCHES` | The number of elements between adjacent elements in each dimension. Always ordered as BFYX.|
| `<TENSOR>_PITCHES` | The offset (in elements) between adjacent elements in each dimension. Always ordered as BFYX.|
| `<TENSOR>_PITCHES_SIZE`| The size of the `<TENSOR>_PITCHES` array |
| `<TENSOR>_OFFSET`| The number of elements from the start of the tensor to the first valid element, bypassing the lower padding. |
All `<TENSOR>` values are automatically defined for every tensor
bound to this operation, such as `INPUT0`, `INPUT1`, and `OUTPUT0`, as shown
in the following example:
```sh
```c
#define INPUT0_DIMS_SIZE 4
#define INPUT0_DIMS (int []){ 1,96,55,55, }
```
@ -197,28 +196,25 @@ __kernel void example_relu_kernel(
{
const uint idx = get_global_id(0);
const uint idy = get_global_id(1);
const uint idbf = get_global_id(2);//batches*features, as OpenCL supports 3D nd-ranges only
const uint feature = idbf%OUTPUT0_DIMS[1];
const uint batch = idbf/OUTPUT0_DIMS[1];
const uint idbf = get_global_id(2); // batches*features, as OpenCL supports 3D nd-ranges only
const uint feature = idbf % OUTPUT0_DIMS[1];
const uint batch = idbf / OUTPUT0_DIMS[1];
//notice that pitches are in elements, not in bytes!
const uint in_id = batch*INPUT0_PITCHES[0] + feature*INPUT0_PITCHES[1] + idy*INPUT0_PITCHES[2] + idx*INPUT0_PITCHES[3] + INPUT0_OFFSET;
const uint out_id = batch*OUTPUT0_PITCHES[0] + feature*OUTPUT0_PITCHES[1] + idy*OUTPUT0_PITCHES[2] + idx*OUTPUT0_PITCHES[3] + OUTPUT0_OFFSET;
INPUT0_TYPE value = input0[in_id];
//neg_slope (which is non-zero for leaky ReLU) is put automatically as #define, refer to the config xml
// neg_slope (which is non-zero for leaky ReLU) is put automatically as #define, refer to the config xml
output[out_id] = value < 0 ? value * neg_slope : value;
}
```
> **NOTE**: As described in the previous section, all items like
> `INPUT0_TYPE` are actually defined as OpenCL (pre-)compiler inputs by
> the Inference Engine for efficiency reasons. See [Debugging
> the OpenVINO for efficiency reasons. See [Debugging
> Tips](#debugging-tips) for information on debugging the results.
> **NOTE**: Several GPU-targeted kernels are also added to the binaries upon compilation of samples
> so that the sample application can easy load them.
> Refer to the `cldnn_global_custom_kernels` folder in the GPU plugin installation directory.
## Debugging Tips<a name="debugging-tips"></a>
* **Using `printf` in the OpenCL™ Kernels**.

120
docs/Extensibility_UG/Intro.md Normal file
View File

@ -0,0 +1,120 @@
# OpenVINO Extensibility Mechanism {#openvino_docs_Extensibility_UG_Intro}
@sphinxdirective
.. toctree::
:maxdepth: 1
:hidden:
openvino_docs_Extensibility_UG_add_openvino_ops
openvino_docs_Extensibility_UG_GPU
openvino_docs_MO_DG_prepare_model_customize_model_optimizer_Customize_Model_Optimizer
@endsphinxdirective
The Intel® Distribution of OpenVINO™ toolkit supports neural network models trained with various frameworks, including
TensorFlow, PyTorch, ONNX, PaddlePaddle, MXNet, Caffe, and Kaldi. The list of supported operations (layers) is different for
each of the supported frameworks. To see the operations supported by your framework, refer to
[Supported Framework Operations](../MO_DG/prepare_model/Supported_Frameworks_Layers.md).
Custom operations, that is those not included in the list, are not recognized by OpenVINO™ out-of-the-box. Therefore, creating Intermediate Representation (IR) for a model using them requires additional steps. This guide illustrates the workflow for running inference on topologies featuring custom operations, allowing you to plug in your own implementation for existing or completely new operations.
If your model contains operations not normally supported by OpenVINO™, the OpenVINO™ Extensibility API lets you add support for those custom operations and use one implementation for Model Optimizer and OpenVINO™ Runtime.
There are two steps to support inference of a model with custom operation(s):
1. Add support for a [custom operation in the Model Optimizer](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md) so
the Model Optimizer can generate the IR with the operation.
2. Create a custom operation in it as described in the [Custom Operation](add_openvino_ops.md).
## OpenVINO™ Extensions
OpenVINO™ provides extensions for:
* [Custom OpenVINO™ Operation](add_openvino_ops.md):
- Enables the creation of unsupported operations
- Enables the use of `ov::Core::read_model` to read models with unsupported operations
- Provides a shape inference mechanism for custom operations
- Provides an evaluate method that allows you to support the operation on CPU or perform constant folding
* [Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md):
- Enables support of new operations to generate IR
- Enables support of custom transformations to replace sub-graphs for performance optimization
> **NOTE**: This documentation is written based on the [Template extension](https://github.com/openvinotoolkit/openvino/tree/master/docs/template_extension/new), which demonstrates extension development details. You can review the complete code, which is fully compilable and up-to-date, to see how it works.
## Load extensions to OpenVINO™ Runtime
To load the extensions to the `ov::Core` object, use the `ov::Core::add_extension` method, this method allows to load library with extensions or extensions from the code.
### Load extensions to core
Extensions can be loaded from code with `ov::Core::add_extension` method:
@sphinxdirective
.. tab:: C++
.. doxygensnippet:: docs/snippets/ov_extensions.cpp
:language: cpp
:fragment: add_extension
.. tab:: Python
.. doxygensnippet:: docs/snippets/ov_extensions.py
:language: python
:fragment: add_extension
@endsphinxdirective
### Create library with extensions
You need to create extension library in following cases:
- Load extensions to Model Optimizer
- Load extensions to Python application
If you want to create an extension library, for example in order to load these extensions to the Model Optimizer, you need to do next steps:
Create an entry point for extension library. OpenVINO™ provides an `OPENVINO_CREATE_EXTENSIONS()` macro, which allows to define an entry point to a library with OpenVINO™ Extensions.
This macro should have a vector of all OpenVINO™ Extensions as an argument.
Based on that, the declaration of an extension class can look as follows:
@snippet template_extension/new/ov_extension.cpp ov_extension:entry_point
To configure the build of your extension library, use the following CMake script:
@snippet template_extension/new/CMakeLists.txt cmake:extension
This CMake script finds the OpenVINO™ using the `find_package` CMake command.
To build the extension library, run the commands below:
```sh
$ cd docs/template_extension/new
$ mkdir build
$ cd build
$ cmake -DOpenVINO_DIR=<OpenVINO_DIR> ../
$ cmake --build .
```
After the build you can use path to your extension library to load your extensions to OpenVINO™ Runtime:
@sphinxdirective
.. tab:: C++
.. doxygensnippet:: docs/snippets/ov_extensions.cpp
:language: cpp
:fragment: add_extension_lib
.. tab:: Python
.. doxygensnippet:: docs/snippets/ov_extensions.py
:language: python
:fragment: add_extension_lib
@endsphinxdirective
## See Also
* [OpenVINO Transformations](./ov_transformations.md)
* [Using Inference Engine Samples](../OV_Runtime_UG/Samples_Overview.md)
* [Hello Shape Infer SSD sample](../../samples/cpp/hello_reshape_ssd/README.md)

62
docs/Extensibility_UG/add_openvino_ops.md Normal file
View File

@ -0,0 +1,62 @@
# Custom OpenVINO™ Operations {#openvino_docs_Extensibility_UG_add_openvino_ops}
OpenVINO™ Extension API allows you to register custom operations to support models with operations which OpenVINO™ does not support out-of-the-box.
## Operation Class
To add your custom operation, create a new class that extends `ov::Op`, which is in turn derived from `ov::Node`, the base class for all graph operations in OpenVINO™. To add `ov::Op` please include next file:
@snippet template_extension/new/identity.hpp op:common_include
Follow the steps below to add a custom operation:
1. Add the `OPENVINO_OP` macro which defines a `NodeTypeInfo` object that identifies the type of the operation to the graph users and helps with dynamic type resolution. The type info of an operation currently consists of a string operation identifier and a string for operation version.
2. Implement default constructor and constructors that optionally take the operation inputs and attributes as parameters.
3. Override the shape inference method `validate_and_infer_types`. This method is called multiple times during graph manipulations to determine the shapes and element types of the operations outputs. To access the input shapes and input element types, use the `get_input_partial_shape()` and `get_input_element_type()` methods of `ov::Node`. Set the inferred shape and element type of the output using `set_output_type`.
4. Override the `clone_with_new_inputs` method, which enables graph manipulation routines to create copies of this operation and connect it to different nodes during optimization.
5. Override the `visit_attributes` method, which enables serialization and deserialization of operation attributes. An `AttributeVisitor` is passed to the method, and the implementation is expected to walk over all the attributes in the op using the type-aware `on_attribute` helper. Helpers are already implemented for standard C++ types like `int64_t`, `float`, `bool`, `vector`, and for existing OpenVINO defined types.
6. Override `evaluate`, which is an optional method that enables fallback of some devices to this implementation and the application of constant folding if there is a custom operation on the constant branch. If your operation contains `evaluate` method you also need to override the `has_evaluate` method, this method allow to get information about availability of `evaluate` method for the operation.
7. Add the `OPENVINO_FRAMEWORK_MAP` macro if you want to map custom operation to framework operation with the same name. It is an optional macro which can be used for one to one mapping. In order to use this macro please include frontend specific headers:
@snippet template_extension/new/identity.hpp op:frontend_include
Based on that, declaration of an operation class can look as follows:
@snippet template_extension/new/identity.hpp op:header
### Operation Constructors
OpenVINO™ operation contains two constructors:
* Default constructor, which enables you to create an operation without attributes
* Constructor that creates and validates an operation with specified inputs and attributes
@snippet template_extension/new/identity.cpp op:ctor
### `validate_and_infer_types()`
`ov::Node::validate_and_infer_types` method validates operation attributes and calculates output shapes using attributes of the operation.
@snippet template_extension/new/identity.cpp op:validate
### `clone_with_new_inputs()`
`ov::Node::clone_with_new_inputs` method creates a copy of the operation with new inputs.
@snippet template_extension/new/identity.cpp op:copy
### `visit_attributes()`
`ov::Node::visit_attributes` method enables you to visit all operation attributes.
@snippet template_extension/new/identity.cpp op:visit_attributes
### `evaluate()` and `has_evaluate()`
`ov::Node::evaluate` method enables you to apply constant folding to an operation.
@snippet template_extension/new/identity.cpp op:evaluate

28
docs/Extensibility_UG/graph_rewrite_pass.md Normal file
View File

@ -0,0 +1,28 @@
# OpenVINO Graph Rewrite Pass {#openvino_docs_Extensibility_UG_graph_rewrite_pass}
`ov::pass::GraphRewrite` serves for running multiple matcher passes on `ov::Model` in a single graph traversal.
Example:
@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:graph_rewrite
In addition, GraphRewrite handles nodes that were registered by MatcherPasses during their execution. This nodes will be added to the beginning of the sequence with nodes for pattern matching.
> **NOTE**: when using `ov::pass::Manager` temporary GraphRewrite is used to execute single MatcherPass.
GraphRewrite has two algorithms for MatcherPasses execution. First algorithm is straightforward. It applies each MatcherPass in registration order to current node.
![graph_rewrite_execution]
But it is not really efficient when you have a lot of registered passes. So first of all GraphRewrite checks that all MatcherPass patterns has type-based root node (it means that type of this node is not hidden into predicate).
And then creates map from registered MatcherPasses. That helps to avoid additional cost of applying each MatcherPass for each node.
![graph_rewrite_efficient_search]
> **NOTE**: GraphRewrite execution algorithm cannot be set manually and depends only on root nodes registered inside MatcherPasses.
## See Also
* [OpenVINO™ Transformations](./ov_transformations.md)
[graph_rewrite_execution]: ./img/graph_rewrite_execution.png
[graph_rewrite_efficient_search]: ./img/graph_rewrite_efficient_search.png

0
docs/OV_Runtime_UG/img/graph_rewrite_efficient_search.png → docs/Extensibility_UG/img/graph_rewrite_efficient_search.png
View File

0
docs/OV_Runtime_UG/img/graph_rewrite_execution.png → docs/Extensibility_UG/img/graph_rewrite_execution.png
View File

0
docs/OV_Runtime_UG/img/ngraph_insert_node.png → docs/Extensibility_UG/img/ngraph_insert_node.png
View File

0
docs/OV_Runtime_UG/img/ngraph_replace_node.png → docs/Extensibility_UG/img/ngraph_replace_node.png
View File

0
docs/OV_Runtime_UG/img/register_new_node.png → docs/Extensibility_UG/img/register_new_node.png
View File

0
docs/OV_Runtime_UG/img/transformations_structure.png → docs/Extensibility_UG/img/transformations_structure.png
View File

101
docs/Extensibility_UG/matcher_pass.md Normal file
View File

@ -0,0 +1,101 @@
# OpenVINO Matcher Pass {#openvino_docs_Extensibility_UG_matcher_pass}
`ov::pass::MatcherPass` is used for pattern-based transformations.
Template for MatcherPass transformation class
@snippet src/transformations/template_pattern_transformation.hpp graph_rewrite:template_transformation_hpp
@snippet src/transformations/template_pattern_transformation.cpp graph_rewrite:template_transformation_cpp
To use `ov::pass::MatcherPass`, you need to complete these steps:
1. Create a pattern
2. Implement a callback
3. Register the pattern and Matcher
4. Execute MatcherPass
So let's go through each of these steps.
## Create a pattern
Pattern is a single root `ov::Model`. But the only difference is that you do not need to create a model object, you just need to create and connect opset or special pattern operations.
Then you need to take the last created operation and put it as a root of the pattern. This root node will be used as a root node in pattern matching.
> **NOTE**: Any nodes in a pattern that have no consumers and are not registered as root will not be used in pattern matching.
@snippet ov_model_snippets.cpp pattern:simple_example
The `Parameter` operation in the example above has type and shape specified. These attributes are needed only to create Parameter operation class and will not be used in pattern matching.
For more pattern examples, refer to the [pattern matching](#pattern_matching) section.
## Implement callback
Callback is an action applied to every pattern entrance. In general, callback is the lambda function that takes Matcher object with detected subgraph.
@snippet ov_model_snippets.cpp pattern:callback_example
The example above shows the callback structure and how Matcher can be used for accessing nodes detected by pattern.
Callback return value is `true` if root node was replaced and another pattern cannot be applied to the same root node; otherwise, it is `false`.
> **NOTE**: It is not recommended to manipulate with nodes that are under root node. This may affect GraphRewrite execution as it is expected that all nodes that come after root node in topological order are valid and can be used in pattern matching.
MatcherPass also provides functionality that allows reporting of the newly created nodes that can be used in additional pattern matching.
If MatcherPass was registered in `ov::pass::Manager` or `ov::pass::GraphRewrite`, these registered nodes will be added for additional pattern matching.
That means that matcher passes registered in `ov::pass::GraphRewrite` will be applied to these nodes.
The example below shows how single MatcherPass can fuse sequence of operations using the `register_new_node` method.
@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:relu_fusion
> **NOTE**: If you register multiple nodes, please add them in topological order. We do not topologically sort these nodes as it is a time-consuming operation.
## Register pattern and Matcher
The last step is to register Matcher and callback inside the MatcherPass pass. To do this, call the `register_matcher` method.
> **NOTE**: Only one matcher can be registered for a single MatcherPass class.
```cpp
// Register matcher and callback
register_matcher(m, callback);
```
## Execute MatcherPass
MatcherPass has multiple ways to be executed:
* Run on a single node - it can be useful if you want to run MatcherPass inside another transformation.
@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:run_on_node
* Run on `ov::Model` using GraphRewrite - this approach gives ability to run MatcherPass on whole `ov::Model`. Moreover, multiple MatcherPass transformation can be registered in a single GraphRewite to be executed in a single graph traversal.
@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:graph_rewrite
* Run on `ov::Model` using `ov::pass::Manager` - this approach helps you to register MatcherPass for execution on `ov::Model` as another transformation types.
@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:manager
## Pattern Matching <a name="pattern_matching"></a>
Sometimes patterns cannot be expressed via regular operations or it is too complicated.
For example, if you want to detect **Convolution->Add** sub-graph without specifying particular input type for Convolution operation or you want to create a pattern where some of operations can have different types.
And for these cases OpenVINO™ provides additional helpers to construct patterns for GraphRewrite transformations.
There are two main helpers:
1. `ov::pass::pattern::any_input` - helps to express inputs if their types are undefined.
2. `ov::pass::pattern::wrap_type<T>` - helps to express nodes of pattern without specifying node attributes.
Let's go through the example to have better understanding of how it works:
> **NOTE**: Node attributes do not participate in pattern matching and are needed only for operations creation. Only operation types participate in pattern matching.
The example below shows basic usage of `ov::passpattern::any_input`.
Here we construct Multiply pattern with arbitrary first input and Constant as a second input.
Also as Multiply is commutative operation, it does not matter in which order we set inputs (any_input/Constant or Constant/any_input) because both cases will be matched.
@snippet ov_model_snippets.cpp pattern:label_example
This example shows how we can construct a pattern when operation has arbitrary number of inputs.
@snippet ov_model_snippets.cpp pattern:concat_example
This example shows how to use predicate to construct a pattern. Also it shows how to match pattern manually on given node.
@snippet ov_model_snippets.cpp pattern:predicate_example
> **NOTE**: Be careful with manual matching because Matcher object holds matched nodes. To clear a match, use the m->clear_state() method.
## See Also
* [OpenVINO™ Transformations](./ov_transformations.md)

17
docs/Extensibility_UG/model_pass.md Normal file
View File

@ -0,0 +1,17 @@
# OpenVINO Model Pass {#openvino_docs_Extensibility_UG_model_pass}
`ov::pass::ModelPass` is used for transformations that take entire `ov::Model` as an input and process it.
Template for ModelPass transformation class
@snippet src/transformations/template_model_transformation.hpp model_pass:template_transformation_hpp
@snippet src/transformations/template_model_transformation.cpp model_pass:template_transformation_cpp
Using `ov::pass::ModelPass`, you need to override the `run_on_model` method where you will write the transformation code.
Return value is `true` if the original model has changed during transformation (new operation was added, or operations replacement was made, or node attributes were changed); otherwise, it is `false`.
Also `ov::pass::ModelPass` based transformations can be executed via `ov::pass::Manager`.
## See Also
* [OpenVINO™ Transformations](./ov_transformations.md)

173
docs/Extensibility_UG/ov_transformations.md Normal file
View File

@ -0,0 +1,173 @@
# Overview of Transformations API {#openvino_docs_transformations}
@sphinxdirective
.. toctree::
:maxdepth: 1
:hidden:
openvino_docs_Extensibility_UG_model_pass
openvino_docs_Extensibility_UG_matcher_pass
openvino_docs_Extensibility_UG_graph_rewrite_pass
@endsphinxdirective
OpenVINO Transformation mechanism allows to develop transformation passes to modify `ov::Model`. You can use this mechanism to apply additional optimizations to the original Model or transform unsupported subgraphs and operations to new operations which are supported by the plugin.
This guide contains all necessary information that you need to start implementing OpenVINO™ transformations.
## Working with Model
Before the moving to transformation part it is needed to say several words about functions which allow to modify `ov::Model`.
This chapter extends the [model representation guide](../OV_Runtime_UG/model_representation.md) and shows an API that allows us to manipulate with `ov::Model`.
### Working with node input and output ports
First of all let's talk about `ov::Node` input/output ports. Each OpenVINO™ operation has input and output ports except cases when operation has `Parameter` or `Constant` type.
Every port belongs to its node, so using a port we can access parent node, get shape and type for particular input/output, get all consumers in case of output port, and get producer node in case of input port.
With output port we can set inputs for newly created operations.
Lets look at the code example.
@snippet ov_model_snippets.cpp ov:ports_example
### Node replacement
OpenVINO™ provides two ways for node replacement: via OpenVINO™ helper function and directly via port methods. We are going to review both of them.
Let's start with OpenVINO™ helper functions. The most popular function is `ov::replace_node(old_node, new_node)`.
We will review real replacement case where Negative operation is replaced with Multiply.
![ngraph_replace_node]
@snippet ov_model_snippets.cpp ov:replace_node
`ov::replace_node` has a constraint that number of output ports for both of ops must be the same; otherwise, it raises an exception.
The alternative way to do the same replacement is the following:
@snippet ov_model_snippets.cpp ov:manual_replace
Another transformation example is insertion.
![ngraph_insert_node]
@snippet ov_model_snippets.cpp ov:insert_node
The alternative way to the insert operation is to make a node copy and use `ov::replace_node()`:
@snippet ov_model_snippets.cpp ov:insert_node_with_copy
### Node elimination
Another type of node replacement is its elimination.
To eliminate operation, OpenVINO™ has special method that considers all limitations related to OpenVINO™ Runtime.
@snippet ov_model_snippets.cpp ov:eliminate_node
`ov::replace_output_update_name()` in case of successful replacement it automatically preserves friendly name and runtime info.
## Transformations types <a name="transformations_types"></a>
OpenVINO™ Runtime has three main transformation types:
* [Model pass](./model_pass.md) - straightforward way to work with `ov::Model` directly
* [Matcher pass](./matcher_pass.md) - pattern-based transformation approach
* [Graph rewrite pass](./graph_rewrite_pass.md) - container for matcher passes needed for efficient execution
![transformations_structure]
## Transformation conditional compilation
Transformation library has two internal macros to support conditional compilation feature.
* `MATCHER_SCOPE(region)` - allows to disable the MatcherPass if matcher isn't used. The region name should be unique. This macro creates a local variable `matcher_name` which you should use as a matcher name.
* `RUN_ON_MODEL_SCOPE(region)` - allows to disable run_on_model pass if it isn't used. The region name should be unique.
## Transformation writing essentials <a name="transformation_writing_essentials"></a>
When developing a transformation, you need to follow these transformation rules:
###1. Friendly Names
Each `ov::Node` has an unique name and a friendly name. In transformations we care only about friendly name because it represents the name from the model.
To avoid losing friendly name when replacing node with other node or subgraph, set the original friendly name to the latest node in replacing subgraph. See the example below.
@snippet ov_model_snippets.cpp ov:replace_friendly_name
In more advanced cases, when replaced operation has several outputs and we add additional consumers to its outputs, we make a decision how to set friendly name by arrangement.
###2. Runtime Info
Runtime info is a map `std::map<std::string, ov::Any>` located inside `ov::Node` class. It represents additional attributes in `ov::Node`.
These attributes can be set by users or by plugins and when executing transformation that changes `ov::Model` we need to preserve these attributes as they will not be automatically propagated.
In most cases, transformations have the following types: 1:1 (replace node with another node), 1:N (replace node with a sub-graph), N:1 (fuse sub-graph into a single node), N:M (any other transformation).
Currently, there is no mechanism that automatically detects transformation types, so we need to propagate this runtime information manually. See the examples below.
@snippet ov_model_snippets.cpp ov:copy_runtime_info
When transformation has multiple fusions or decompositions, `ov::copy_runtime_info` must be called multiple times for each case.
**Note**: copy_runtime_info removes rt_info from destination nodes. If you want to keep it, you need to specify them in source nodes like this: copy_runtime_info({a, b, c}, {a, b})
###3. Constant Folding
If your transformation inserts constant sub-graphs that need to be folded, do not forget to use `ov::pass::ConstantFolding()` after your transformation or call constant folding directly for operation.
The example below shows how constant subgraph can be constructed.
@snippet ov_model_snippets.cpp ov:constant_subgraph
Manual constant folding is more preferable than `ov::pass::ConstantFolding()` because it is much faster.
Below you can find an example of manual constant folding:
@snippet src/transformations/template_pattern_transformation.cpp manual_constant_folding
## Common mistakes in transformations <a name="common_mistakes"></a>
In transformation development process:
* Do not use deprecated OpenVINO™ API. Deprecated methods has the `OPENVINO_DEPRECATED` macros in its definition.
* Do not pass `shared_ptr<Node>` as an input for other node if type of node is unknown or it has multiple outputs. Use explicit output port.
* If you replace node with another node that produces different shape, remember that new shape will not be propagated until the first `validate_nodes_and_infer_types` call for `ov::Model`. If you are using `ov::pass::Manager`, it will automatically call this method after each transformation execution.
* Do not forget to call the `ov::pass::ConstantFolding` pass if your transformation creates constant subgraphs.
* Use latest OpSet if you are not developing downgrade transformation pass.
* When developing a callback for `ov::pass::MatcherPass`, do not change nodes that come after the root node in topological order.
## Using pass manager <a name="using_pass_manager"></a>
`ov::pass::Manager` is a container class that can store the list of transformations and execute them. The main idea of this class is to have high-level representation for grouped list of transformations.
It can register and apply any [transformation pass](#transformations_types) on model.
In addition, `ov::pass::Manager` has extended debug capabilities (find more information in the [how to debug transformations](#how_to_debug_transformations) section).
The example below shows basic usage of `ov::pass::Manager`
@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:manager3
Another example shows how multiple matcher passes can be united into single GraphRewrite.
@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:manager2
## How to debug transformations <a name="how_to_debug_transformations"></a>
If you are using `ngraph::pass::Manager` to run sequence of transformations, you can get additional debug capabilities by using the following environment variables:
```
OV_PROFILE_PASS_ENABLE=1 - enables performance measurement for each transformation and prints execution status
OV_ENABLE_VISUALIZE_TRACING=1 - enables visualization after each transformation. By default, it saves dot and svg files.
```
> **Note**: Make sure that you have dot installed on your machine; otherwise, it will silently save only dot file without svg file.
## See Also
* [OpenVINO™ Model Representation](../OV_Runtime_UG/model_representation.md)
* [OpenVINO™ Extensions](./Intro.md)
[ngraph_replace_node]: ./img/ngraph_replace_node.png
[ngraph_insert_node]: ./img/ngraph_insert_node.png
[transformations_structure]: ./img/transformations_structure.png
[register_new_node]: ./img/register_new_node.png

349
docs/HOWTO/Custom_Layers_Guide.md
View File

@ -1,349 +0,0 @@
# Custom Operations Guide {#openvino_docs_HOWTO_Custom_Layers_Guide}
The Intel® Distribution of OpenVINO™ toolkit supports neural network models trained with multiple frameworks including
TensorFlow*, Caffe*, MXNet*, Kaldi* and ONNX* file format. The list of supported operations (layers) is different for
each of the supported frameworks. To see the operations supported by your framework, refer to
[Supported Framework Layers](../MO_DG/prepare_model/Supported_Frameworks_Layers.md).
Custom operations, that is those not included in the list, are not recognized by Model Optimizer out-of-the-box. Therefore, creating Intermediate Representation (IR) for a model using them requires additional steps. This guide illustrates the workflow for running inference on topologies featuring custom operations, allowing you to plug in your own implementation for existing or completely new operations.
> **NOTE**: *Layer* is a legacy term for *operation* which came from Caffe\* framework. Currently it is not used.
> Refer to the [Deep Learning Network Intermediate Representation and Operation Sets in OpenVINO™](../MO_DG/IR_and_opsets.md)
> for more information on the topic.
## Terms Used in This Guide
- *Intermediate Representation (IR)* — OpenVINO's Neural Network format used by Inference Engine. It abstracts different frameworks and describs model topology, operations parameters, and weights.
- *Operation* — an abstract concept of a math function selected for a specific purpose. Operations supported by
OpenVINO™ are listed in the supported operation set provided in the [Available Operations Sets](../ops/opset.md).
Examples of the operations are: [ReLU](../ops/activation/ReLU_1.md), [Convolution](../ops/convolution/Convolution_1.md),
[Add](../ops/arithmetic/Add_1.md), etc.
- *Kernel* — The implementation of an operation function in the OpenVINO™ plugin, in this case, the math programmed (in
C++ and OpenCL) to perform the operation for a target hardware (CPU or GPU).
- *Inference Engine Extension* — Device-specific module implementing custom operations (a set of kernels).
## Custom Operation Support Overview
There are three steps to support inference of a model with custom operation(s):
1. Add support for a custom operation in the [Model Optimizer](../MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) so
the Model Optimizer can generate the IR with the operation.
2. Create an operation set and implement a custom nGraph operation in it as described in the
[Custom nGraph Operation](../OV_Runtime_UG/Extensibility_DG/AddingNGraphOps.md).
3. Implement a customer operation in one of the [OpenVINO™ Runtime](../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md)
plugins to support inference of this operation using a particular target hardware (CPU, GPU or VPU).
To see the operations that are supported by each device plugin for the Inference Engine, refer to the
[Supported Devices](../OV_Runtime_UG/supported_plugins/Supported_Devices.md).
> **NOTE**: If a device doesn't support a particular operation, an alternative to creating a new operation is to target
> an additional device using the HETERO plugin. The [Heterogeneous Plugin](../OV_Runtime_UG/supported_plugins/HETERO.md) may be
> used to run an inference model on multiple devices allowing the unsupported operations on one device to "fallback" to
> run on another device (e.g., CPU) that does support those operations.
### Custom Operation Support for the Model Optimizer
Model Optimizer model conversion pipeline is described in detail in "Model Conversion Pipeline" section of [Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md). It is best to read that article first for a better understanding of the following material.
Model Optimizer provides an extensions mechanism to support new operations and implement custom model transformations to generate optimized IR. This mechanism is described in the "Model Optimizer Extensions" section of
[Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md).
Two types of Model Optimizer extensions should be implemented to support custom operations, at a minimum:
1. Operation class for a new operation. This class stores information about the operation, its attributes, shape inference function, attributes to be saved to an IR and some others internally used attributes. Refer to the "Model Optimizer Operation" section of [Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md) for detailed instructions on how to implement it.
2. Operation attributes extractor. The extractor is responsible for parsing framework-specific representation of the
operation and uses corresponding operation class to update graph node attributes with necessary attributes of the
operation. Refer to the "Operation Extractor" section of
[Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md) for detailed instructions on how to implement it.
> **NOTE**: In some cases you may need to implement some transformation to support the operation. This topic is covered in the "Graph Transformation Extensions" section of [Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md).
## Custom Operations Extensions for the Inference Engine
Inference Engine provides an extension mechanism to support new operations. This mechanism is described in [Inference Engine Extensibility Mechanism](../OV_Runtime_UG/Extensibility_DG/Intro.md).
Each device plugin includes a library of optimized implementations to execute known operations which must be extended to execute a custom operation. The custom operation extension is implemented according to the target device:
- Custom Operation CPU Extension
- A compiled shared library (`.so` or `.dll`) needed by the CPU Plugin for executing the custom operation
on a CPU. Refer to the [How to Implement Custom CPU Operations](../OV_Runtime_UG/Extensibility_DG/CPU_Kernel.md) for more
details.
- Custom Operation GPU Extension
- OpenCL source code (.cl) for the custom operation kernel that will be compiled to execute on the GPU along with an operation description file (.xml) needed by the GPU Plugin for the custom operation kernel. Refer to the [How to Implement Custom GPU Operations](../OV_Runtime_UG/Extensibility_DG/GPU_Kernel.md) for more details.
- Custom Operation VPU Extension
- OpenCL source code (.cl) for the custom operation kernel that will be compiled to execute on the VPU along with an operation description file (.xml) needed by the VPU Plugin for the custom operation kernel. Refer to [How to Implement Custom Operations for VPU](../OV_Runtime_UG/Extensibility_DG/VPU_Kernel.md) for more details.
Also, it is necessary to implement nGraph custom operation according to [Custom nGraph Operation](../OV_Runtime_UG/Extensibility_DG/AddingNGraphOps.md) so the Inference Engine can read an IR with this
operation and correctly infer output tensor shape and type.
## Enabling Magnetic Resonance Image Reconstruction Model
This chapter provides step-by-step instructions on how to enable the magnetic resonance image reconstruction model implemented in the [repository](https://github.com/rmsouza01/Hybrid-CS-Model-MRI/) using a custom operation on CPU. The example is prepared for a model generated from the repository with hash `2ede2f96161ce70dcdc922371fe6b6b254aafcc8`.
### Download and Convert the Model to a Frozen TensorFlow\* Model Format
The original pre-trained model is provided in the hdf5 format which is not supported by OpenVINO directly and needs to be converted to TensorFlow\* frozen model format first.
1. Download repository `https://github.com/rmsouza01/Hybrid-CS-Model-MRI`:<br>
```bash
git clone https://github.com/rmsouza01/Hybrid-CS-Model-MRI
git checkout 2ede2f96161ce70dcdc922371fe6b6b254aafcc8
```
2. Convert pre-trained `.hdf5` to a frozen `.pb` graph using the following script (tested with TensorFlow==1.15.0 and
Keras==2.2.4) which should be executed from the root of the cloned repository:<br>
```py
import keras as K
import numpy as np
import Modules.frequency_spatial_network as fsnet
import tensorflow as tf
under_rate = '20'
stats = np.load("Data/stats_fs_unet_norm_" + under_rate + ".npy")
var_sampling_mask = np.load("Data/sampling_mask_" + under_rate + "perc.npy")
model = fsnet.wnet(stats[0], stats[1], stats[2], stats[3], kshape = (5,5), kshape2=(3,3))
model_name = "Models/wnet_" + under_rate + ".hdf5"
model.load_weights(model_name)
inp = np.random.standard_normal([1, 256, 256, 2]).astype(np.float32)
np.save('inp', inp)
sess = K.backend.get_session()
sess.as_default()
graph_def = sess.graph.as_graph_def()
graph_def = tf.graph_util.convert_variables_to_constants(sess, graph_def, ['conv2d_44/BiasAdd'])
with tf.gfile.FastGFile('wnet_20.pb', 'wb') as f:
f.write(graph_def.SerializeToString())
```
As a result the TensorFlow\* frozen model file "wnet_20.pb" is generated.
### Convert the Frozen TensorFlow\* Model to Intermediate Representation
Firstly, open the model in TensorBoard or other TensorFlow* model visualization tool. The model supports dynamic
batch dimension because the value for the batch dimension is not hardcoded in the model. Model Optimizer need to set all
dynamic dimensions to some specific value to create the IR, therefore specify the command line parameter `-b 1` to set
the batch dimension equal to 1. The actual batch size dimension can be changed at runtime using the Inference Engine API
described in the [Using Shape Inference](../OV_Runtime_UG/ShapeInference.md). Also refer to the General Conversion Parameters section in [Converting a Model to Intermediate Representation (IR)](../MO_DG/prepare_model/convert_model/Converting_Model.md) and [Convert Your TensorFlow* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md)
for more details and command line parameters used for the model conversion.
```sh
mo --input_model <PATH_TO_MODEL>/wnet_20.pb -b 1
```
> **NOTE**: This conversion guide is applicable for the 2021.3 release of OpenVINO and that starting from 2021.4
> the OpenVINO supports this model out of the box.
Model Optimizer produces the following error:
```bash
[ ERROR ] List of operations that cannot be converted to Inference Engine IR:
[ ERROR ] Complex (1)
[ ERROR ] lambda_2/Complex
[ ERROR ] IFFT2D (1)
[ ERROR ] lambda_2/IFFT2D
[ ERROR ] ComplexAbs (1)
[ ERROR ] lambda_2/Abs
[ ERROR ] Part of the nodes was not converted to IR. Stopped.
```
The error means that the Model Optimizer doesn't know how to handle 3 types of TensorFlow\* operations: "Complex",
"IFFT2D" and "ComplexAbs". In order to see more details about the conversion process run the model conversion with
additional parameter `--log_level DEBUG`. It is worth to mention the following lines from the detailed output:
```bash
[ INFO ] Called "tf_native_tf_node_infer" for node "lambda_2/Complex"
[ <TIMESTAMP> ] [ DEBUG ] [ tf:228 ] Added placeholder with name 'lambda_2/lambda_3/strided_slice_port_0_ie_placeholder'
[ <TIMESTAMP> ] [ DEBUG ] [ tf:228 ] Added placeholder with name 'lambda_2/lambda_4/strided_slice_port_0_ie_placeholder'
[ <TIMESTAMP> ] [ DEBUG ] [ tf:241 ] update_input_in_pbs: replace input 'lambda_2/lambda_3/strided_slice' with input 'lambda_2/lambda_3/strided_slice_port_0_ie_placeholder'
[ <TIMESTAMP> ] [ DEBUG ] [ tf:249 ] Replacing input '0' of the node 'lambda_2/Complex' with placeholder 'lambda_2/lambda_3/strided_slice_port_0_ie_placeholder'
[ <TIMESTAMP> ] [ DEBUG ] [ tf:241 ] update_input_in_pbs: replace input 'lambda_2/lambda_4/strided_slice' with input 'lambda_2/lambda_4/strided_slice_port_0_ie_placeholder'
[ <TIMESTAMP> ] [ DEBUG ] [ tf:249 ] Replacing input '1' of the node 'lambda_2/Complex' with placeholder 'lambda_2/lambda_4/strided_slice_port_0_ie_placeholder'
[ <TIMESTAMP> ] [ DEBUG ] [ tf:148 ] Inferred shape of the output tensor with index '0' of the node 'lambda_2/Complex': '[ 1 256 256]'
[ <TIMESTAMP> ] [ DEBUG ] [ infer:145 ] Outputs:
[ <TIMESTAMP> ] [ DEBUG ] [ infer:32 ] output[0]: shape = [ 1 256 256], value = <UNKNOWN>
[ <TIMESTAMP> ] [ DEBUG ] [ infer:129 ] --------------------
[ <TIMESTAMP> ] [ DEBUG ] [ infer:130 ] Partial infer for lambda_2/IFFT2D
[ <TIMESTAMP> ] [ DEBUG ] [ infer:131 ] Op: IFFT2D
[ <TIMESTAMP> ] [ DEBUG ] [ infer:132 ] Inputs:
[ <TIMESTAMP> ] [ DEBUG ] [ infer:32 ] input[0]: shape = [ 1 256 256], value = <UNKNOWN>
```
This is a part of the log of the partial inference phase of the model conversion. See the "Partial Inference" section on
the [Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md) for
more information about this phase. Model Optimizer inferred output shape for the unknown operation of type "Complex"
using a "fallback" to TensorFlow\*. However, it is not enough to generate the IR because Model Optimizer doesn't know
which attributes of the operation should be saved to IR. So it is necessary to implement Model Optimizer extensions to
support these operations.
Before going into the extension development it is necessary to understand what these unsupported operations do according
to the TensorFlow\* framework specification.
* "Complex" - returns a tensor of complex type constructed from two real input tensors specifying real and imaginary
part of a complex number.
* "IFFT2D" - returns a tensor with inverse 2-dimensional discrete Fourier transform over the inner-most 2 dimensions of
an input.
* "ComplexAbs" - returns a tensor with absolute values of input tensor with complex numbers.
The part of the model with all three unsupported operations is depicted below:
![Unsupported sub-graph](img/unsupported_subgraph.png)
This model uses complex numbers during the inference but Inference Engine does not support tensors of this data type. So
it is necessary to find a way how to avoid using tensors of such a type in the model. Fortunately, the complex tensor
appear as a result of "Complex" operation, is used as input in the "IFFT2D" operation then is passed to "ComplexAbs"
which produces real value tensor as output. So there are just 3 operations consuming/producing complex tensors in the
model.
Let's design an OpenVINO operation "FFT" which get a single real number tensor describing the complex number and
produces a single real number tensor describing output complex tensor. This way the fact that the model uses complex
numbers is hidden inside the "FFT" operation implementation. The operation gets a tensor of shape `[N, H, W, 2]` and
produces the output tensor with the same shape, where the innermost dimension contains pairs of real numbers describing
the complex number (its real and imaginary part). As we will see further this operation will allow us to support the
model. The implementation of the Model Optimizer operation should be saved to `mo_extensions/ops/FFT.py` file:
@snippet FFT.py fft:operation
The attribute `inverse` is a flag specifying type of the FFT to apply: forward or inverse.
See the "Model Optimizer Operation" section of [Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md) for detailed instructions on how to implement the operation.
Now it is necessary to implement extractor for the "IFFT2D" operation according to the
"Operation Extractor" section of [Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md). The
following snippet provides two extractors: one for "IFFT2D", another one for "FFT2D", however only on of them is used in this example. The implementation should be saved to the file `mo_extensions/front/tf/FFT_ext.py`.
@snippet FFT_ext.py fft_ext:extractor
> **NOTE**: The graph is in inconsistent state after extracting node attributes because according to original operation
> "IFFT2D" semantic it should have an input consuming a tensor of complex numbers, but the extractor instantiated an
> operation "FFT" which expects a real tensor with specific layout. But the inconsistency will be resolved during
> applying front phase transformations discussed below.
The output shape of the operation "AddV2" from the picture above is `[N, H, W, 2]`. Where the innermost dimension
contains pairs of real numbers describing the complex number (its real and imaginary part). The following "StridedSlice"
operations split the input tensor into 2 parts to get a tensor of real and a tensor of imaginary parts which are then
consumed with the "Complex" operation to produce a tensor of complex numbers. These "StridedSlice" and "Complex"
operations can be removed so the "FFT" operation will get a real value tensor encoding complex numbers. To achieve this
we implement the front phase transformation which searches for a pattern of two "StridedSlice" operations with specific
attributes producing data to "Complex" operation and removes it from the graph. Refer to the
"Pattern-Defined Front Phase Transformations" section of [Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md) for more
information on how this type of transformation works. The code snippet should be saved to the file
`mo_extensions/front/tf/Complex.py`.
@snippet Complex.py complex:transformation
> **NOTE**: The graph is in inconsistent state because the "ComplexAbs" operation consumes complex value tensor but
> "FFT" produces real value tensor.
Now lets implement a transformation which replace a "ComplexAbs" operation with a sub-graph of primitive operations
which calculate the result using the following formulae: \f$module(z) = \sqrt{real(z) \cdot real(z) + imag(z) \cdot imag(z)}\f$.
Original "IFFT2D" operation produces tensor of complex values, but the "FFT" operation produces a real value tensor with
the same format and shape as the input for the operation. So the input shape for the "ComplexAbs" will be `[N, H, W, 2]`
with the innermost dimension containing tuple with real and imaginary part of a complex number. In order to calculate
absolute values for the complex tensor we do the following:
1. Raise all elements in the power of 2.
2. Calculate a reduced sum over the innermost dimension.
3. Calculate a square root.
The implementation should be saved to the file `mo_extensions/front/tf/ComplexAbs.py` and provided below:
@snippet ComplexAbs.py complex_abs:transformation
Now it is possible to convert the model using the following command line:
```sh
mo --input_model <PATH_TO_MODEL>/wnet_20.pb -b 1 --extensions mo_extensions/
```
The sub-graph corresponding to the originally non-supported one is depicted in the image below:
![Converted sub-graph](img/converted_subgraph.png)
> **NOTE**: Model Optimizer performed conversion of the model from NHWC to NCHW layout that is why the dimension with
> the value 2 moved to another position.
### Inference Engine Extension Implementation
Now it is necessary to implement the extension for the CPU plugin with operation "FFT" introduced previously. The code
below is based on the template extension described in [Inference Engine Extensibility Mechanism](../OV_Runtime_UG/Extensibility_DG/Intro.md).
#### CMake Build File
The first step is to create a CMake configuration file which builds the extension. The content of the "CMakeLists.txt"
file is the following:
@snippet template_extension/old/CMakeLists.txt cmake:extension
The CPU FFT kernel implementation uses OpenCV to perform the FFT that is why the extension library is linked with
`opencv_core` which comes with the OpenVINO.
#### Custom nGraph Operation "FFT" Implementation
The next step is to create the nGraph operation FFT. The header file "fft_op.hpp" has the following content:
@snippet template_extension/old/fft_op.hpp fft_op:header
The operation has just one boolean attribute `inverse`. Implementation of the necessary nGraph operation functions are
in the `fft_op.cpp` file with the following content:
@snippet template_extension/old/fft_op.cpp fft_op:implementation
Refer to the [Custom nGraph Operation](../OV_Runtime_UG/Extensibility_DG/AddingNGraphOps.md) for more details.
#### CPU FFT Kernel Implementation
The operation implementation for CPU plugin uses OpenCV to perform the FFT. The header file "fft_kernel.hpp" has the
following content:
@snippet template_extension/old/fft_kernel.hpp fft_kernel:header
The "fft_kernel.cpp" with the implementation of the CPU has the following content:
@snippet template_extension/old/fft_kernel.cpp fft_kernel:implementation
Refer to the [How to Implement Custom CPU Operations](../OV_Runtime_UG/Extensibility_DG/CPU_Kernel.md) for more details.
#### Extension Library Implementation
The last step is to create an extension library "extension.cpp" and "extension.hpp" which will include the FFT
operation for the CPU plugin. The code of the library is described in the [Extension Library](../OV_Runtime_UG/Extensibility_DG/Extension.md).
### Building and Running the Custom Extension
To build the extension, run the following:<br>
```bash
mkdir build && cd build
source /opt/intel/openvino_2022/setupvars.sh
cmake .. -DCMAKE_BUILD_TYPE=Release
make --jobs=$(nproc)
```
The result of this command is a compiled shared library (`.so` or `.dll`). It should be loaded in the
application using `Core` class instance method `AddExtension` like this
`core.AddExtension(std::make_shared<Extension>(compiled_library_file_name), "CPU");`.
To test that the extension is implemented correctly we can run the "mri_reconstruction_demo" with the following content:
@snippet mri_reconstruction_demo.py mri_demo:demo
The script can be executed using the following command line:
```bash
python3 mri_reconstruction_demo.py \
-m <PATH_TO_IR>/wnet_20.xml \
-i <PATH_TO_SAMPLE_MRI_IMAGE>.npy \
-p <Hybrid-CS-Model-MRI_repo>/Data/sampling_mask_20perc.npy \
-l <PATH_TO_BUILD_DIR>/libtemplate_extension.so \
-d CPU
```
## Additional Resources
- Intel® Distribution of OpenVINO™ toolkit home page: [https://software.intel.com/en-us/openvino-toolkit](https://software.intel.com/en-us/openvino-toolkit)
- OpenVINO™ toolkit online documentation: [https://docs.openvino.ai](https://docs.openvino.ai)
- [Model Optimizer Developer Guide](../MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md)
- [Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md)
- [Inference Engine Extensibility Mechanism](../OV_Runtime_UG/Extensibility_DG/Intro.md)
- [OpenVINO™ Toolkit Samples Overview](../OV_Runtime_UG/Samples_Overview.md)
- [Overview of OpenVINO™ Toolkit Pre-Trained Models](@ref omz_models_group_intel)
- For IoT Libraries and Code Samples see the [Intel® IoT Developer Kit](https://github.com/intel-iot-devkit).
## Converting Models:
- [Convert Your Caffe* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_Caffe.md)
- [Convert Your TensorFlow* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md)
- [Convert Your MXNet* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_MxNet.md)
- [Convert Your Kaldi* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_Kaldi.md)
- [Convert Your ONNX* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_ONNX.md)

4
docs/IE_PLUGIN_DG/AsyncInferRequest.md
View File

@ -1,7 +1,7 @@
# Asynchronous Inference Request {#openvino_docs_ie_plugin_dg_async_infer_request}
Asynchronous Inference Request runs an inference pipeline asynchronously in one or several task executors depending on a device pipeline structure.
Inference Engine Plugin API provides the base InferenceEngine::AsyncInferRequestThreadSafeDefault class:
OpenVINO Runtime Plugin API provides the base InferenceEngine::AsyncInferRequestThreadSafeDefault class:
- The class has the `_pipeline` field of `std::vector<std::pair<ITaskExecutor::Ptr, Task> >`, which contains pairs of an executor and executed task.
- All executors are passed as arguments to a class constructor and they are in the running state and ready to run tasks.
@ -10,7 +10,7 @@ Inference Engine Plugin API provides the base InferenceEngine::AsyncInferRequest
`AsyncInferRequest` Class
------------------------
Inference Engine Plugin API provides the base InferenceEngine::AsyncInferRequestThreadSafeDefault class for a custom asynchronous inference request implementation:
OpenVINO Runtime Plugin API provides the base InferenceEngine::AsyncInferRequestThreadSafeDefault class for a custom asynchronous inference request implementation:
@snippet src/template_async_infer_request.hpp async_infer_request:header

2
docs/IE_PLUGIN_DG/Doxyfile
View File

@ -675,7 +675,7 @@ SHOW_NAMESPACES = YES
# The FILE_VERSION_FILTER tag can be used to specify a program or script that
# doxygen should invoke to get the current version for each file (typically from
# the version control system). Doxygen will invoke the program by executing (via
# popen()) the command command input-file, where command is the value of the
# popen()) the command input-file, where command is the value of the
# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided
# by doxygen. Whatever the program writes to standard output is used as the file
# version. For an example see the documentation.

2
docs/IE_PLUGIN_DG/ExecutableNetwork.md
View File

@ -38,7 +38,7 @@ The implementation `CompileNetwork` is fully device-specific.
The function accepts a const shared pointer to `ngraph::Function` object and performs the following steps:
1. Applies ngraph passes using `TransformNetwork` function, which defines plugin-specific conversion pipeline. To support low precision inference, the pipeline can include Low Precision Transformations. These transformations are usually hardware specific. You can find how to use and configure Low Precisions Transformations in [Low Precision Transformations](@ref openvino_docs_IE_DG_lpt) guide.
2. Maps the transformed graph to a backend specific graph representation (for example, to MKLDNN graph for Intel CPU).
2. Maps the transformed graph to a backend specific graph representation (for example, to CPU plugin internal graph representation).
3. Allocates and fills memory for graph weights, backend specific memory handles and so on.
@snippet src/template_executable_network.cpp executable_network:map_graph

2
docs/IE_PLUGIN_DG/InferRequest.md
View File

@ -54,7 +54,7 @@ Decrements a number of created inference requests:
#### 1. `inferPreprocess`
Below is the code of the the `inferPreprocess` method to demonstrate Inference Engine common preprocessing step handling:
Below is the code of the `inferPreprocess` method to demonstrate Inference Engine common preprocessing step handling:
@snippet src/template_infer_request.cpp infer_request:infer_preprocess

2
docs/IE_PLUGIN_DG/Intro.md
View File

@ -56,7 +56,7 @@ Detailed guides
* Plugin and its components [testing](@ref openvino_docs_ie_plugin_dg_plugin_testing)
* [Quantized networks](@ref openvino_docs_ie_plugin_dg_quantized_networks)
* [Low precision transformations](@ref openvino_docs_IE_DG_lpt) guide
* [Writing nGraph transformations](@ref ngraph_transformation) guide
* [Writing OpenVINO™ transformations](@ref openvino_docs_transformations) guide
API References
-----------------------

20
docs/IE_PLUGIN_DG/Plugin.md
View File

@ -2,7 +2,7 @@
Inference Engine Plugin usually represents a wrapper around a backend. Backends can be:
- OpenCL-like backend (e.g. clDNN library) for GPU devices.
- MKLDNN backend for Intel CPU devices.
- oneDNN backend for Intel CPU devices.
- NVIDIA cuDNN for NVIDIA GPUs.
The responsibility of Inference Engine Plugin:
@ -30,7 +30,7 @@ Based on that, declaration of a plugin class can look as follows:
The provided plugin class also has several fields:
* `_backend` - a backend engine that is used to perform actual computations for network inference. For `Template` plugin `ngraph::runtime::Backend` is used which performs computations using ngraph reference implementations.
* `_backend` - a backend engine that is used to perform actual computations for network inference. For `Template` plugin `ngraph::runtime::Backend` is used which performs computations using OpenVINO™ reference implementations.
* `_waitExecutor` - a task executor that waits for a response from a device about device tasks completion.
* `_cfg` of type `Configuration`:
@ -67,7 +67,7 @@ which holds a backend-dependent compiled graph in an internal representation:
Before a creation of an `ExecutableNetwork` instance via a constructor, a plugin may check if a provided
InferenceEngine::ICNNNetwork object is supported by a device. In the example above, the plugin checks precision information.
The very important part before creation of `ExecutableNetwork` instance is to call `TransformNetwork` method which applies ngraph transformation passes.
The very important part before creation of `ExecutableNetwork` instance is to call `TransformNetwork` method which applies OpenVINO™ transformation passes.
Actual graph compilation is done in the `ExecutableNetwork` constructor. Refer to the [ExecutableNetwork Implementation Guide](@ref openvino_docs_ie_plugin_dg_executable_network) for details.
@ -77,27 +77,27 @@ Actual graph compilation is done in the `ExecutableNetwork` constructor. Refer t
### `TransformNetwork()`
The function accepts a const shared pointer to `ngraph::Function` object and performs the following steps:
The function accepts a const shared pointer to `ov::Model` object and performs the following steps:
1. Deep copies a const object to a local object, which can later be modified.
2. Applies common and plugin-specific transformations on a copied graph to make the graph more friendly to hardware operations. For details how to write custom plugin-specific transformation, please, refer to [Writing ngraph transformations](@ref ngraph_transformation) guide. See detailed topics about network representation:
2. Applies common and plugin-specific transformations on a copied graph to make the graph more friendly to hardware operations. For details how to write custom plugin-specific transformation, please, refer to [Writing OpenVINO™ transformations](@ref openvino_docs_transformations) guide. See detailed topics about network representation:
* [Intermediate Representation and Operation Sets](../_docs_MO_DG_IR_and_opsets.html)
* [Quantized networks](@ref openvino_docs_ie_plugin_dg_quantized_networks).
@snippet template_plugin/src/template_plugin.cpp plugin:transform_network
> **NOTE**: After all these transformations, a `ngraph::Function` object contains operations which can be perfectly mapped to backend kernels. E.g. if backend has kernel computing `A + B` operations at once, the `TransformNetwork` function should contain a pass which fuses operations `A` and `B` into a single custom operation `A + B` which fits backend kernels set.
> **NOTE**: After all these transformations, a `ov::Model` object contains operations which can be perfectly mapped to backend kernels. E.g. if backend has kernel computing `A + B` operations at once, the `TransformNetwork` function should contain a pass which fuses operations `A` and `B` into a single custom operation `A + B` which fits backend kernels set.
### `QueryNetwork()`
Use the method with the `HETERO` mode, which allows to distribute network execution between different
devices based on the `ngraph::Node::get_rt_info()` map, which can contain the `"affinity"` key.
devices based on the `ov::Node::get_rt_info()` map, which can contain the `"affinity"` key.
The `QueryNetwork` method analyzes operations of provided `network` and returns a list of supported
operations via the InferenceEngine::QueryNetworkResult structure. The `QueryNetwork` firstly applies `TransformNetwork` passes to input `ngraph::Function` argument. After this, the transformed network in ideal case contains only operations are 1:1 mapped to kernels in computational backend. In this case, it's very easy to analyze which operations is supposed (`_backend` has a kernel for such operation or extensions for the operation is provided) and not supported (kernel is missed in `_backend`):
operations via the InferenceEngine::QueryNetworkResult structure. The `QueryNetwork` firstly applies `TransformNetwork` passes to input `ov::Model` argument. After this, the transformed network in ideal case contains only operations are 1:1 mapped to kernels in computational backend. In this case, it's very easy to analyze which operations is supposed (`_backend` has a kernel for such operation or extensions for the operation is provided) and not supported (kernel is missed in `_backend`):
1. Store original names of all operations in input `ngraph::Function`
1. Store original names of all operations in input `ov::Model`
2. Apply `TransformNetwork` passes. Note, the names of operations in a transformed network can be different and we need to restore the mapping in the steps below.
3. Construct `supported` and `unsupported` maps which contains names of original operations. Note, that since the inference is performed using ngraph reference backend, the decision whether the operation is supported or not depends on whether the latest OpenVINO opset contains such operation.
3. Construct `supported` and `unsupported` maps which contains names of original operations. Note, that since the inference is performed using OpenVINO™ reference backend, the decision whether the operation is supported or not depends on whether the latest OpenVINO opset contains such operation.
4. `QueryNetworkResult.supportedLayersMap` contains only operations which are fully supported by `_backend`.
@snippet template_plugin/src/template_plugin.cpp plugin:query_network

10
docs/IE_PLUGIN_DG/PluginTesting.md
View File

@ -26,7 +26,7 @@ Engine concepts: plugin creation, multiple executable networks support, multiple
@snippet single_layer_tests/convolution.cpp test_convolution:instantiate
3. **Sub-graph tests** (`subgraph_tests` sub-folder). This group of tests is designed to tests small patterns or combination of layers. E.g. when a particular topology is being enabled in a plugin e.g. TF ResNet-50, there is no need to add the whole topology to test tests. In opposite way, a particular repetitive subgraph or pattern can be extracted from `ResNet-50` and added to the tests. The instantiation of the sub-graph tests is done in the same way as for single layer tests.
> **Note**, such sub-graphs or patterns for sub-graph tests should be added to `IE::ngraphFunctions` library first (this library is a pre-defined set of small `ngraph::Function`) and re-used in sub-graph tests after.
> **Note**, such sub-graphs or patterns for sub-graph tests should be added to `IE::ngraphFunctions` library first (this library is a pre-defined set of small `ov::Model`) and re-used in sub-graph tests after.
4. **HETERO tests** (`subgraph_tests` sub-folder) contains tests for `HETERO` scenario (manual or automatic affinities settings, tests for `QueryNetwork`).
@ -41,18 +41,14 @@ To use these tests for your own plugin development, link the `IE::funcSharedTest
To build test binaries together with other build artifacts, use the `make all` command. For details, see
[Build Plugin Using CMake*](@ref openvino_docs_ie_plugin_dg_plugin_build).
### Tests for plugin-specific ngraph transformations
Please, refer to [Transformation testing](@ref ngraph_transformation) guide.
### How to Extend Inference Engine Plugin Tests
Inference Engine Plugin tests are open for contribution.
Add common test case definitions applicable for all plugins to the `IE::funcSharedTests` target within the DLDT repository. Then, any other plugin supporting corresponding functionality can instantiate the new test.
All Inference Engine per-layer tests check test layers functionality. They are developed using nGraph functions
All Inference Engine per-layer tests check test layers functionality. They are developed using ov::Model.
as input graphs used by tests. In this case, to test a new layer with layer tests, extend
the `IE::ngraphFunctions` library, which is also included in the Inference Engine Developer package, with a new nGraph function
the `IE::ngraphFunctions` library, which is also included in the Inference Engine Developer package, with a new model.
including the corresponding operation.
> **NOTE**: When implementing a new subgraph test, add new single-layer tests for each operation of the subgraph if such test does not exist.

2
docs/IE_PLUGIN_DG/QuantizedNetworks.md
View File

@ -9,7 +9,7 @@ For more details about low-precision model representation please refer to this [
During the model load each plugin can interpret quantization rules expressed in *FakeQuantize* operations:
- Independently based on the definition of *FakeQuantize* operation.
- Using a special library of low-precision transformations (LPT) which applies common rules for generic operations,
such as Convolution, Fully-Connected, Eltwise, etc., and translates "fake-quantized" models into the models with low-precision operations. For more information about low-precision flow please refer to the following [document](@ref openvino_docs_IE_DG_Int8Inference).
such as Convolution, Fully-Connected, Eltwise, etc., and translates "fake-quantized" models into the models with low-precision operations. For more information about low-precision flow please refer to the following [document](../OV_Runtime_UG/Int8Inference.md).
Here we provide only a high-level overview of the interpretation rules of FakeQuantize.
At runtime each FakeQuantize can be split into two independent operations: **Quantize** and **Dequantize**.

722
docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md
View File

@ -1,4 +1,4 @@
# Model Optimizer Developer Guide {#openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide}
# Model Optimizer User Guide {#openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide}
@sphinxdirective
@ -7,678 +7,128 @@
.. toctree::
:maxdepth: 1
:hidden:
openvino_docs_MO_DG_IR_and_opsets
openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model
openvino_docs_MO_DG_prepare_model_convert_model_Cutting_Model
openvino_docs_MO_DG_Additional_Optimization_Use_Cases
openvino_docs_MO_DG_prepare_model_customize_model_optimizer_Customize_Model_Optimizer
openvino_docs_MO_DG_FP16_Compression
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_ONNX
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_PyTorch
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Paddle
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_MxNet
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Caffe
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Kaldi
openvino_docs_MO_DG_prepare_model_convert_model_tutorials
openvino_docs_MO_DG_prepare_model_Model_Optimizer_FAQ
openvino_docs_MO_DG_Known_Issues_Limitations
openvino_docs_MO_DG_Default_Model_Optimizer_Optimizations
@endsphinxdirective
## Introduction
## Introduction
Model Optimizer is a cross-platform command-line tool that facilitates the transition between the training and deployment environment, performs static model analysis, and adjusts deep learning models for optimal execution on end-point target devices.
Model Optimizer is a cross-platform command-line tool that facilitates the transition between training and deployment environments, performs static model analysis, and adjusts deep learning models for optimal execution on end-point target devices.
Model Optimizer process assumes you have a network model trained using supported deep learning frameworks: Caffe*, TensorFlow*, Kaldi*, MXNet* or converted to the ONNX* format. Model Optimizer produces an Intermediate Representation (IR) of the network, which can be inferred with the [OpenVINO™ Runtime](../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md).
Using Model Optimizer tool assumes you already have a deep learning model trained using one of the supported frameworks: TensorFlow, PyTorch, PaddlePaddle, MXNet, Caffe, Kaldi, or represented in ONNX* format. Model Optimizer produces an Intermediate Representation (IR) of the model, which can be inferred with [OpenVINO™ Runtime](../OV_Runtime_UG/openvino_intro.md).
> **NOTE**: Model Optimizer does not infer models. Model Optimizer is an offline tool that runs before the inference takes place.
> **NOTE**: Model Optimizer does not infer models. Model Optimizer is an offline tool that converts a model into IR and optimizes before the inference takes place.
The scheme below illustrates the typical workflow for deploying a trained deep learning model:
The scheme below illustrates the typical workflow for deploying a trained deep learning model:
![](img/BASIC_FLOW_MO_simplified.svg)
The IR is a pair of files describing the model:
The IR is a pair of files describing the model:
* <code>.xml</code> - Describes the network topology
* <code>.bin</code> - Contains the weights and biases binary data.
> **NOTE**: The generated IR can be additionally optimized for inference by [Post-training Optimization tool](../../tools/pot/README.md)
> that applies post-training quantization methods.
> **TIP**: You also can work with the Model Optimizer inside the OpenVINO™ [Deep Learning Workbench](https://docs.openvino.ai/latest/workbench_docs_Workbench_DG_Introduction.html) (DL Workbench).
> [DL Workbench](https://docs.openvino.ai/latest/workbench_docs_Workbench_DG_Introduction.html) is a web-based graphical environment that enables you to optimize, fine-tune, analyze, visualize, and compare performance of deep learning models.
## Install Model Optimizer Pre-Requisites
Before running the Model Optimizer, you must install the Model Optimizer pre-requisites for the framework that was used to train the model.
@sphinxdirective
.. tab:: Using configuration scripts
.. tab:: Linux
.. tab:: All frameworks
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
./install_prerequisites.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
./install_prerequisites.shs
.. tab:: Caffe
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisitess
install_prerequisites_caffe.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_caffe.shs
.. tab:: Tensorflow 1.x
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_tf.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_tf.sh
.. tab:: Tensorflow 2.x
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_tf2.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_tf2.sh
.. tab:: MXNet
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_mxnet.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_mxnet.sh
.. tab:: ONNX
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_onnx.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_onnx.sh
.. tab:: Kaldi
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_kaldi.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_kaldi.sh
.. tab:: Windows
.. tab:: All frameworks
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites\
install_prerequisites.bat
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
install_prerequisites.bat
.. tab:: Caffe
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites\
install_prerequisites_caffe.bat
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
install_prerequisites_caffe.bat
.. tab:: Tensorflow 1.x
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites\
install_prerequisites_tf.bat
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
install_prerequisites_tf.bat
.. tab:: Tensorflow 2.x
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites\
install_prerequisites_tf2.bat
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
install_prerequisites_tf2.bat
.. tab:: MXNet
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites\
install_prerequisites_mxnet.bat
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
install_prerequisites_mxnet.bat
.. tab:: ONNX
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites\
install_prerequisites_onnx.bat
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
install_prerequisites_onnx.bat
.. tab:: Kaldi
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites\
install_prerequisites_kaldi.bat
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer\install_prerequisites
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
install_prerequisites_kaldi.bat
.. tab:: macOS
.. tab:: All frameworks
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
./install_prerequisites.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
./install_prerequisites.shs
.. tab:: Caffe
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisitess
install_prerequisites_caffe.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_caffe.shs
.. tab:: Tensorflow 1.x
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_tf.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_tf.sh
.. tab:: Tensorflow 2.x
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_tf2.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_tf2.sh
.. tab:: MXNet
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_mxnet.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_mxnet.sh
.. tab:: ONNX
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_onnx.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_onnx.sh
.. tab:: Kaldi
.. tab:: Install globally
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
install_prerequisites_kaldi.sh
.. tab:: Install to virtualenv
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/install_prerequisites
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate # sh, bash, ksh, or zsh
install_prerequisites_kaldi.sh
.. tab:: Using manual configuration process
.. tab:: Linux
.. tab:: All frameworks
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements.txt
.. tab:: Caffe
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_caffe.txt
.. tab:: Tensorflow 1.x
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_tf.txt
.. tab:: Tensorflow 2.x
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_tf2.txt
.. tab:: MXNet
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_mxnet.txt
.. tab:: ONNX
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_onnx.txt
.. tab:: Kaldi
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_kaldi.txt
.. tab:: Windows
.. tab:: All frameworks
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
pip install -r requirements.txt
.. tab:: Caffe
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
pip install -r requirements_caffe.txt
.. tab:: Tensorflow 1.x
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
pip install -r requirements_tf.txt
.. tab:: Tensorflow 2.x
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
pip install -r requirements_tf2.txt
.. tab:: MXNet
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
pip install -r requirements_mxnet.txt
.. tab:: ONNX
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
pip install -r requirements_onnx.txt
.. tab:: Kaldi
.. code-block:: sh
cd <INSTALL_DIR>\deployment_tools\model_optimizer
virtualenv --system-site-packages -p python .\env
env\Scripts\activate.bat
pip install -r requirements_kaldi.txt
.. tab:: macOS
.. tab:: All frameworks
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements.txt
.. tab:: Caffe
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_caffe.txt
.. tab:: Tensorflow 1.x
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_tf.txt
.. tab:: Tensorflow 2.x
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_tf2.txt
.. tab:: MXNet
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_mxnet.txt
.. tab:: ONNX
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_onnx.txt
.. tab:: Kaldi
.. code-block:: sh
cd <INSTALL_DIR>/deployment_tools/model_optimizer/
virtualenv --system-site-packages -p python3 ./venv
source ./venv/bin/activate
pip3 install -r requirements_kaldi.txt
@endsphinxdirective
## Run Model Optimizer
To convert the model to the Intermediate Representation (IR), run Model Optimizer:
To convert the model to IR, run Model Optimizer:
```sh
mo --input_model INPUT_MODEL --output_dir <OUTPUT_MODEL_DIR>
mo --input_model INPUT_MODEL
```
You need to have have write permissions for an output directory.
If out-of-the-box conversion (only the `--input_model` parameter is specified) is not succeed,
try to use parameters for overriding input shapes and cutting the model, mentioned below.
> **NOTE**: Some models require using additional arguments to specify conversion parameters, such as `--input_shape`, `--scale`, `--scale_values`, `--mean_values`, `--mean_file`. To learn about when you need to use these parameters, refer to [Converting a Model to Intermediate Representation (IR)](prepare_model/convert_model/Converting_Model.md).
To override original input shapes for model conversion, Model Optimizer provides two parameters: `--input` and `--input_shape`.
For more information about these parameters, refer to [Setting Input Shapes](prepare_model/convert_model/Converting_Model.md).
To adjust the conversion process, you may use general parameters defined in the [Converting a Model to Intermediate Representation (IR)](prepare_model/convert_model/Converting_Model.md) and
framework-specific parameters for:
* [Caffe](prepare_model/convert_model/Convert_Model_From_Caffe.md)
* [TensorFlow](prepare_model/convert_model/Convert_Model_From_TensorFlow.md)
* [MXNet](prepare_model/convert_model/Convert_Model_From_MxNet.md)
* [ONNX](prepare_model/convert_model/Convert_Model_From_ONNX.md)
* [Kaldi](prepare_model/convert_model/Convert_Model_From_Kaldi.md)
To cut off unwanted parts of a model, such as unsupported operations and training sub-graphs,
the `--input` and `--output` parameters can be used, defining new inputs and outputs of the converted model.
For a more detailed description, refer to [Cutting Off Parts of a Model](prepare_model/convert_model/Cutting_Model.md).
## Videos
Also, you can insert additional input pre-processing sub-graphs into the converted model using
the `--mean_values`, `scales_values`, `--layout`, and other parameters described
in [Embedding Preprocessing Computation](prepare_model/Additional_Optimizations.md).
@sphinxdirective
Model Optimizer's compression parameter `--data_type` allows to generate IR of the `FP16` data type. For more details,
please refer to [Compression of a Model to FP16](prepare_model/FP16_Compression.md).
.. list-table::
To get the full list of conversion parameters available in Model Optimizer, run the following command:
* - .. raw:: html
```sh
mo --help
```
<iframe allowfullscreen mozallowfullscreen msallowfullscreen oallowfullscreen webkitallowfullscreen width="220"
src="https://www.youtube.com/embed/Kl1ptVb7aI8">
</iframe>
- .. raw:: html
## Examples of CLI Commands
<iframe allowfullscreen mozallowfullscreen msallowfullscreen oallowfullscreen webkitallowfullscreen width="220"
src="https://www.youtube.com/embed/BBt1rseDcy0">
</iframe>
Below is a list of separate examples for different frameworks and Model Optimizer parameters.
- .. raw:: html
1. Launch Model Optimizer for a TensorFlow MobileNet model in the binary protobuf format.
```sh
mo --input_model MobileNet.pb
```
Launch Model Optimizer for a TensorFlow BERT model in the SavedModel format, with three inputs. Explicitly specify input shapes
where the batch size and the sequence length equal 2 and 30 respectively.
```sh
mo --saved_model_dir BERT --input mask,word_ids,type_ids --input_shape [2,30],[2,30],[2,30]
```
For more information on TensorFlow model conversion,
refer to [Converting a TensorFlow Model](prepare_model/convert_model/Convert_Model_From_TensorFlow.md).
<iframe allowfullscreen mozallowfullscreen msallowfullscreen oallowfullscreen webkitallowfullscreen width="220"
src="https://www.youtube.com/embed/RF8ypHyiKrY">
</iframe>
2. Launch Model Optimizer for an ONNX OCR model and explicitly specify new output.
```sh
mo --input_model ocr.onnx --output probabilities
```
For more information on ONNX model conversion,
please refer to [Converting an ONNX Model](prepare_model/convert_model/Convert_Model_From_ONNX.md).
Note that PyTorch models must be exported to the ONNX format before its conversion into IR.
More details can be found in [Converting a PyTorch Model](prepare_model/convert_model/Convert_Model_From_PyTorch.md).
* - **Model Optimizer Concept.**
- **Model Optimizer Basic Operation.**
- **Choosing the Right Precision.**
3. Launch Model Optimizer for a PaddlePaddle UNet model and apply mean-scale normalization to the input.
```sh
mo --input_model unet.pdmodel --mean_values [123,117,104] --scale 255
```
For more information on PaddlePaddle model conversion, please refer to
[Converting a PaddlePaddle Model](prepare_model/convert_model/Convert_Model_From_Paddle.md).
* - Duration: 3:56
- Duration: 2:57
- Duration: 4:18
4. Launch Model Optimizer for an MXNet SSD Inception V3 model and specify first-channel layout for the input.
```sh
mo --input_model ssd_inception_v3-0000.params --layout NCHW
```
For more information on MXNet models conversion, please refer to [Converting an MXNet Model](prepare_model/convert_model/Convert_Model_From_MxNet.md).
@endsphinxdirective
5. Launch Model Optimizer for a Caffe AlexNet model with input channels in the RGB format, which needs to be reversed.
```sh
mo --input_model alexnet.caffemodel --reverse_input_channels
```
For more information on Caffe model conversion, please refer to [Converting a Caffe Model](prepare_model/convert_model/Convert_Model_From_Caffe.md).
6. Launch Model Optimizer for a Kaldi LibriSpeech nnet2 model.
```sh
mo --input_model librispeech_nnet2.mdl --input_shape [1,140]
```
For more information on Kaldi model conversion,
refer to [Converting a Kaldi Model](prepare_model/convert_model/Convert_Model_From_Kaldi.md).
To get conversion recipes for specific TensorFlow, ONNX, PyTorch, MXNet, and Kaldi models,
refer to [Model Conversion Tutorials](prepare_model/convert_model/Convert_Model_Tutorials.md).

47
docs/MO_DG/Known_Issues_Limitations.md
View File

@ -1,47 +0,0 @@
# Known Issues and Limitations in the Model Optimizer {#openvino_docs_MO_DG_Known_Issues_Limitations}
## Model Optimizer for TensorFlow* should be run on Intel® hardware that supports the AVX instruction set
TensorFlow* provides only prebuilt binaries with AVX instructions enabled. When you're configuring the Model Optimizer by running the `install_prerequisites` or `install_prerequisites_tf` scripts, they download only those ones, which are not supported on hardware such as Intel® Pentium® processor N4200/5, N3350/5, N3450/5 (formerly known as Apollo Lake).
To run the Model Optimizer on this hardware, you should compile TensorFlow binaries from source as described at the [TensorFlow website](https://www.tensorflow.org/install/source).
Another option is to run the Model Optimizer to generate an IR on hardware that supports AVX to and then perform inference on hardware without AVX.
## Multiple OpenMP Loadings
If the application uses the Inference Engine with third-party components that depend on Intel OpenMP, multiple loadings of the libiomp library may occur and cause OpenMP runtime initialization conflicts. This may happen, for example, if the application uses Intel® Math Kernel Library (Intel® MKL) through the “Single Dynamic Library” (<code>libmkl_rt.so</code>) mechanism and calls Intel MKL after loading the Inference Engine plugin.
The error log looks as follows:
```sh
OMP: Error #15: Initializing libiomp5.so, but found libiomp5.so already initialized.
OMP: Hint: This means that multiple copies of the OpenMP runtime have been linked into the program. That is dangerous, since it can degrade performance or cause incorrect results. The best thing to do is to ensure that only a single OpenMP runtime is linked into the process, e.g. by avoiding static linking of the OpenMP runtime in any library. As an unsafe, unsupported, undocumented workaround you can set the environment variable KMP_DUPLICATE_LIB_OK=TRUE to allow the program to continue to execute, but that may cause crashes or silently produce incorrect results. For more information, please see http://www.intel.com/software/products/support/.
```
Possible workarounds:
* Preload the OpenMP runtime using the <code>LD_PRELOAD</code> variable:
```sh
LD_PRELOAD=<path_to_libiomp5.so> <path_to your_executable> ```
This eliminates multiple loadings of libiomp, and makes all the components use this specific version of OpenMP.
* Alternatively, you can set <code>KMP_DUPLICATE_LIB_OK=TRUE</code>. However, performance degradation or incorrect results may occur in this case.
## Old proto compiler breaks protobuf library
With python protobuf library version 3.5.1 the following incompatibility can happen.
The known case is for Cent OS 7.4
The error log looks as follows:
```sh
File "../lib64/python3.5/site-packages/google/protobuf/descriptor.py", line 829, in _new_
return _message.default_pool.AddSerializedFile(serialized_pb)
TypeError: expected bytes, str found
```
Possible workaround is to upgrade default protobuf compiler (libprotoc 2.5.0) to newer version, for example
libprotoc 2.6.1.
[protobuf_issue]: https://github.com/google/protobuf/issues/4272

204
docs/MO_DG/img/BASIC_FLOW_MO_simplified.svg
View File

File diff suppressed because one or more lines are too long
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 42 KiB

After

Width:  |  Height:  |  Size: 42 KiB

106
docs/MO_DG/prepare_model/Additional_Optimizations.md
View File

@ -1,15 +1,101 @@
# Optimize Preprocessing Computation{#openvino_docs_MO_DG_Additional_Optimization_Use_Cases}
# Embedding Preprocessing Computation {#openvino_docs_MO_DG_Additional_Optimization_Use_Cases}
Model Optimizer performs preprocessing to a model. It is possible to optimize this step and improve first inference time, to do that, follow the tips bellow:
Input data for inference can be different from the training dataset and requires additional preprocessing before inference.
To accelerate the whole pipeline including preprocessing and inference, Model Optimizer provides special parameters such as `--mean_values`,
`--scale_values`, `--reverse_input_channels`, and `--layout`. Based on these parameters, Model Optimizer generates IR with additionally
inserted sub-graph that performs the defined preprocessing. This preprocessing block can perform mean-scale normalization of input data,
reverting data along channel dimension, and changing the data layout. For more details about these parameters, refer to the paragraphs below.
The same functionality is also available in runtime, please refer to [Overview of Preprocessing API](../../OV_Runtime_UG/preprocessing_overview.md)
for more information.
- **Image mean/scale parameters**<br>
Make sure to use the input image mean/scale parameters (`--scale` and `–mean_values`) with the Model Optimizer when you need pre-processing. It allows the tool to bake the pre-processing into the IR to get accelerated by the Inference Engine.
## When to Specify Layout
- **RGB vs. BGR inputs**<br>
If, for example, your network assumes the RGB inputs, the Model Optimizer can swap the channels in the first convolution using the `--reverse_input_channels` command line option, so you do not need to convert your inputs to RGB every time you get the BGR image, for example, from OpenCV*.
You may need to set input layouts, as it is required by some preprocessing, for example, setting a batch,
applying mean or scales, and reversing input channels (BGR<->RGB).
- **Larger batch size**<br>
Notice that the devices like GPU are doing better with larger batch size. While it is possible to set the batch size in the runtime using the Inference Engine [ShapeInference feature](../../OV_Runtime_UG/ShapeInference.md).
Layout defines the meaning of dimensions in shape and can be specified for both inputs and outputs.
For the layout syntax, check the [Layout API overview](../../OV_Runtime_UG/layout_overview.md).
To specify the layout, you can use `--layout` option followed by the layout value.
- **Resulting IR precision**<br>
The resulting IR precision, for instance, `FP16` or `FP32`, directly affects performance. As CPU now supports `FP16` (while internally upscaling to `FP32` anyway) and because this is the best precision for a GPU target, you may want to always convert models to `FP16`. Notice that this is the only precision that Intel&reg; Movidius&trade; Myriad&trade; 2 and Intel&reg; Myriad&trade; X VPUs support.
For example, for Tensorflow\* `nasnet_large` model that was exported to ONNX format and thus has input with `NHWC` layout:
```
mo --input_model tf_nasnet_large.onnx --layout nhwc
```
Additionally, if a model has more than one input or needs both input and output layouts specified,
you need to provide the name of each input or output to which you apply the layout.
For example, for ONNX\* `Yolo v3 Tiny` model that has first input `input_1` in `NCHW` layout and second input `image_shape`
with 2 dimensions: batch and size of the image which can be expressed as `N?` layout:
```
mo --input_model yolov3-tiny.onnx --layout input_1(nchw),image_shape(n?)
```
## How to Change Layout of a Model Inputs and Outputs
Changing the model layout may be necessary if it differs from the one presented by input data.
To change the layout, you can use either `--layout` or `--source_layout` with `--target_layout`.
For example, for the same `nasnet_large` that were mentioned previously we may want to provide data in `NCHW` layout:
```
mo --input_model tf_nasnet_large.onnx --source_layout nhwc --target_layout nchw
mo --input_model tf_nasnet_large.onnx --layout "nhwc->nchw"
```
Again, if a model has more than one input or needs both input and output layouts specified, you need to provide the name of each input or output to which you apply the layout.
For example, to provide data in the `NHWC` layout for the `Yolo v3 Tiny` model mentioned earlier:
```
mo --input_model yolov3-tiny.onnx --source_layout "input_1(nchw),image_shape(n?)" --target_layout "input_1(nhwc)"
mo --input_model yolov3-tiny.onnx --layout "input_1(nchw->nhwc),image_shape(n?)"
```
## When to Specify Mean and Scale Values
Usually neural network models are trained with the normalized input data. This means that the input data values are converted to be in a specific range,
for example, `[0, 1]` or `[-1, 1]`. Sometimes the mean values (mean images) are subtracted from the input data values as part of the pre-processing.
There are two cases of how the input data pre-processing is implemented.
* The input pre-processing operations are a part of a model. In this case, the application does not pre-process the input data as a separate step: everything is embedded into the model itself.
* The input pre-processing operations are not a part of a model and the pre-processing is performed within the application which feeds the model with input data.
In the first case, the Model Optimizer generates the IR with required pre-processing operations and no `mean` and `scale` parameters are required.
In the second case, information about mean/scale values should be provided to the Model Optimizer to embed it to the generated IR.
Model Optimizer provides command-line parameters to specify the values: `--mean_values`, `--scale_values`, `--scale`.
Using these parameters, Model Optimizer embeds the corresponding preprocessing block for mean-value normalization of the input data
and optimizes this block so that the preprocessing takes negligible time for inference.
For example, run the Model Optimizer for the PaddlePaddle* UNet model and apply mean-scale normalization to the input data.
```sh
mo --input_model unet.pdmodel --mean_values [123,117,104] --scale 255
```
## When to Reverse Input Channels <a name="when_to_reverse_input_channels"></a>
Sometimes input images for your application can be of the RGB (BGR) format and the model is trained on images of the BGR (RGB) format,
the opposite color channel order. In this case, it is important to preprocess the input images by reverting the color channels before inference.
To embed this preprocessing step into IR, Model Optimizer provides the `--reverse_input_channels` command-line parameter to shuffle the color channels.
The `--reverse_input_channels` parameter applies to an input of the model in two cases.
* Only one dimension in the input shape has a size equal to 3.
* One dimension has an undefined size and is marked as `C` channel using `layout` parameters.
Using the `--reverse_input_channels` parameter, Model Optimizer embeds the corresponding preprocessing block for reverting
the input data along channel dimension and optimizes this block so that the preprocessing takes negligible time for inference.
For example, launch the Model Optimizer for the TensorFlow* AlexNet model and embed `reverse_input_channel` preprocessing block into IR.
```sh
mo --input_model alexnet.pb --reverse_input_channels
```
> **NOTE**: If both mean and scale values are specified, the mean is subtracted first and then the scale is applied regardless of the order of options
in the command line. Input values are *divided* by the scale value(s). If also `--reverse_input_channels` option is used, the `reverse_input_channels`
will be applied first, then `mean` and after that `scale`. The data flow in the model looks as follows:
`Parameter -> ReverseInputChannels -> Mean apply-> Scale apply -> the original body of the model`.
## See Also
* [Overview of Preprocessing API](../../OV_Runtime_UG/preprocessing_overview.md)

11
docs/MO_DG/prepare_model/Default_Model_Optimizer_Optimizations.md
View File

@ -1,11 +0,0 @@
# Default Model Optimizer Optimizations {#openvino_docs_MO_DG_Default_Model_Optimizer_Optimizations}
Model Optimizer not only converts a model to IR format, but also performs a number of optimizations. For example, certain primitives like linear operations (BatchNorm and ScaleShift), are automatically fused into convolutions. Generally, these layers should not be manifested in the resulting IR:
![](../img/optimizations/resnet_269.png)
The picture above shows Caffe\* Resnet269\* topology. The left model is the original model, and the one on the right (after conversion) is the resulting model that the Model Optimizer produces, with BatchNorm and ScaleShift layers fused into the convolution weights rather than constituting separate layers.
If you still see these operations, inspect the Model Optimizer output carefully while searching for warnings, such as on the tool being unable to fuse. For example, non-linear operations (like activations) in between convolutions and linear operations might prevent the fusing. If performance is of concern, try to change (and potentially re-train) the topology. Refer to the [Model Optimizer Guide](Model_Optimization_Techniques.md) for more optimizations.
Notice that the activation (`_relu`) is not touched by the Model Optimizer, and while it can be merged into convolution as well, this is rather a device-specific optimization, covered by Inference Engine during the model loading time. You are encouraged to inspect performance counters from plugins that should indicate that these particular layers are not executed (“Optimized out”). For more information, refer to <a href="#performance-counters">Internal Inference Performance Counters</a>.

20
docs/MO_DG/prepare_model/FP16_Compression.md Normal file
View File

@ -0,0 +1,20 @@
# Compression of a Model to FP16 {#openvino_docs_MO_DG_FP16_Compression}
Model Optimizer can convert all floating-point weights to `FP16` data type. The resulting IR is called
compressed `FP16` model.
To compress the model, use the `--data_type` option:
```
mo --input_model INPUT_MODEL --data_type FP16
```
> **NOTE**: Using `--data_type FP32` will give no result and will not force `FP32`
> precision in the model. If the model was `FP16` it will have `FP16` precision in IR as well.
The resulting model will occupy about twice as less space in the file system, but it may have some accuracy drop,
although for the majority of models accuracy degradation is negligible. For details on how plugins handle
compressed `FP16` models refer to [Working with devices](../../OV_Runtime_UG/supported_plugins/Device_Plugins.md) page.
> **NOTE**: `FP16` compression is sometimes used as initial step for `INT8` quantization, please refer to
> [Post-Training Optimization tool](../../../tools/pot/README.md) for more information about that.

111
docs/MO_DG/prepare_model/Getting_performance_numbers.md
View File

@ -3,28 +3,25 @@
## Tip 1. Measure the Proper Set of Operations
When evaluating performance of your model with the Inference Engine, you must measure the proper set of operations. To do so, consider the following tips:
When evaluating performance of your model with the OpenVINO Runtime, you must measure the proper set of operations. To do so, consider the following tips:
- Avoid including one-time costs like model loading.
- Track separately the operations that happen outside the Inference Engine, like video decoding.
- Track separately the operations that happen outside the OpenVINO Runtime, like video decoding.
> **NOTE**: Some image pre-processing can be baked into the IR and accelerated. For more information, refer to [Model Optimizer Knobs Related to Performance](Additional_Optimizations.md)
> **NOTE**: Some image pre-processing can be baked into the IR and accelerated accordingly. For more information, refer to [Embedding the Preprocessing](Additional_Optimizations.md). Also consider [_runtime_ preprocessing optimizations](../../optimization_guide/dldt_deployment_optimization_common).
## Tip 2. Getting Credible Performance Numbers
You need to build your performance conclusions on reproducible data. Do the performance measurements with a large number of invocations of the same routine. Since the first iteration is almost always significantly slower than the subsequent ones, you can use an aggregated value for the execution time for final projections:
- If the warm-up run does not help or execution time still varies, you can try running a large number of iterations and then average or find a mean of the results.
- For time values that range too much, use geomean.
- For time values that range too much, consider geomean.
- Beware of the throttling and other power oddities. A device can exist in one of several different power states. When optimizing your model, for better performance data reproducibility consider fixing the device frequency. However the end to end (application) benchmarking should be also performed under real operational conditions.
Refer to the [Inference Engine Samples](../../OV_Runtime_UG/Samples_Overview.md) for code examples for the performance measurements. Almost every sample, except interactive demos, has a `-ni` option to specify the number of iterations.
## Tip 3. Measure Reference Performance Numbers with OpenVINO's benchmark_app
## Getting performance numbers using OpenVINO tool
To get performance numbers use our Benchmark app.
[Benchmark App](../../../samples/cpp/benchmark_app/README.md) sample is the best performance reference.
To get performance numbers, use the dedicated [Benchmark App](../../../samples/cpp/benchmark_app/README.md) sample which is the best way to produce the performance reference.
It has a lot of device-specific knobs, but the primary usage is as simple as:
```bash
$ ./benchmark_app –d GPU –m <model> -i <input>
@ -36,35 +33,25 @@ $ ./benchmark_app –d CPU –m <model> -i <input>
```
to execute on the CPU instead.
For example, for the CPU throughput mode from the previous section, you can play with number of streams (`-nstreams` command-line param).
Try different values of the `-nstreams` argument from `1` to a number of CPU cores and find one that provides the best performance. For example, on a 8-core CPU, compare the `-nstreams 1` (which is a latency-oriented scenario) to the `2`, `4` and `8` streams. Notice that `benchmark_app` automatically queries/creates/runs number of requests required to saturate the given number of streams.
Finally, notice that when you don't specify number of streams with `-nstreams`, "AUTO" value for the streams is used, e.g. for the CPU this is [CPU_THROUGHPUT_AUTO](../../OV_Runtime_UG/supported_plugins/CPU.md). You can spot the actual value behind "AUTO" for your machine in the application output.
Notice that the "AUTO" number is not necessarily most optimal, so it is generally recommended to play either with the benchmark_app's "-nstreams" as described above, or via [new Workbench tool](@ref workbench_docs_Workbench_DG_Introduction).This allows you to simplify the app-logic, as you don't need to combine multiple inputs into a batch to achieve good CPU performance.
Instead, it is possible to keep a separate infer request per camera or another source of input and process the requests in parallel using Async API.
Each of the [OpenVINO supported devices](../../OV_Runtime_UG/supported_plugins/Supported_Devices.md) offers performance settings that have command-line equivalents in the [Benchmark App](../../../samples/cpp/benchmark_app/README.md).
While these settings provide really low-level control and allow to leverage the optimal model performance on the _specific_ device, we suggest always starting the performance evaluation with the [OpenVINO High-Level Performance Hints](../../OV_Runtime_UG/performance_hints.md) first:
- benchmark_app **-hint tput** -d 'device' -m 'path to your model'
- benchmark_app **-hint latency** -d 'device' -m 'path to your model'
## Comparing Performance with Native/Framework Code
When comparing the Inference Engine performance with the framework or another reference code, make sure that both versions are as similar as possible:
When comparing the OpenVINO Runtime performance with the framework or another reference code, make sure that both versions are as similar as possible:
- Wrap exactly the inference execution (refer to the [Inference Engine Samples](../../OV_Runtime_UG/Samples_Overview.md) for examples).
- Wrap exactly the inference execution (refer to the [Benchmark App](../../../samples/cpp/benchmark_app/README.md) for examples).
- Do not include model loading time.
- Ensure the inputs are identical for the Inference Engine and the framework. For example, Caffe\* allows to auto-populate the input with random values. Notice that it might give different performance than on real images.
- Similarly, for correct performance comparison, make sure the access pattern, for example, input layouts, is optimal for Inference Engine (currently, it is NCHW).
- Any user-side pre-processing should be tracked separately.
- Make sure to try the same environment settings that the framework developers recommend, for example, for TensorFlow*. In many cases, things that are more machine friendly, like respecting NUMA (see <a href="#cpu-checklist">CPU Checklist</a>), might work well for the Inference Engine as well.
- If applicable, use batching with the Inference Engine.
- If possible, demand the same accuracy. For example, TensorFlow allows `FP16` support, so when comparing to that, make sure to test the Inference Engine with the `FP16` as well.
- Ensure the inputs are identical for the OpenVINO Runtime and the framework. For example, beware of random values that can be used to populate the inputs.
- Consider [Image Pre-processing and Conversion](../../OV_Runtime_UG/preprocessing_overview.md), while any user-side pre-processing should be tracked separately.
- When applicable, leverage the [Dynamic Shapes support](../../OV_Runtime_UG/ov_dynamic_shapes.md)
- If possible, demand the same accuracy. For example, TensorFlow allows `FP16` execution, so when comparing to that, make sure to test the OpenVINO Runtime with the `FP16` as well.
## Using Tools <a name="using-tools"></a>
Whether you are tuning for the first time or doing advanced performance optimization, you need a a tool that provides accurate insights. Intel&reg; VTune&trade; Amplifier gives you the tool to mine it and interpret the profiling data.
Alternatively, you can gather the raw profiling data that samples report, the second chapter provides example of how to interpret these.
### Internal Inference Performance Counters <a name="performance-counters"></a>
Almost every sample (inspect command-line options for a specific sample with `-h`) supports a `-pc` command that outputs internal execution breakdown. Refer to the [samples code](../../OV_Runtime_UG/Samples_Overview.md) for the actual Inference Engine API behind that.
## Internal Inference Performance Counters and Execution Graphs <a name="performance-counters"></a>
Further, finer-grained insights into inference performance breakdown can be achieved with device-specific performance counters and/or execution graphs.
Both [C++](../../../samples/cpp/benchmark_app/README.md) and [Python](../../../tools/benchmark_tool/README.md) versions of the `benchmark_app` supports a `-pc` command-line parameter that outputs internal execution breakdown.
Below is example of CPU plugin output for a network (since the device is CPU, the layers wall clock `realTime` and the `cpu` time are the same):
@ -76,58 +63,12 @@ fc6_nChw8c_nchw EXECUTED layerType: Reorder realTime: 20
out_fc6 EXECUTED layerType: Output realTime: 3 cpu: 3 execType: unknown
relu5_9_x2 OPTIMIZED_OUT layerType: ReLU realTime: 0 cpu: 0 execType: undef
```
This contains layers name (as seen in IR), layers type and execution statistics. Notice the `OPTIMIZED_OUT`, which indicates that the particular activation was fused into adjacent convolution.
Both benchmark_app versions also support "exec_graph_path" command-line option governing the OpenVINO to output the same per-layer execution statistics, but in the form of the plugin-specific [Netron-viewable](https://netron.app/) graph to the specified file.
This contains layers name (as seen in IR), layers type and execution statistics. Notice the `OPTIMIZED_OUT`, which indicates that the particular activation was fused into adjacent convolution. Also, the `unknown` stays for the Inference Engine specific CPU (helper) primitives that are not part of the Intel MKL-DNN.
Notice that on some devices, the execution graphs/counters may be pretty intrusive overhead-wise.
Also, especially when performance-debugging the [latency case](../../optimization_guide/dldt_deployment_optimization_latency.md) notice that the counters do not reflect the time spent in the plugin/device/driver/etc queues. If the sum of the counters is too different from the latency of an inference request, consider testing with less inference requests. For example running single [OpenVINO stream](../../optimization_guide/dldt_deployment_optimization_tput.md) with multiple requests would produce nearly identical counters as running single inference request, yet the actual latency can be quite different.
Notice that there are some helper layers in the CPU execution breakdown, which were not presented in the original topology. These are automatically added by the plugin. For example, the `Reorder` re-packs the Intel MKL-DNN internal (blocked) layout to the regular plain NCHW (that the user expects as the output). As explained in the <a href="#device-specific-tips">Few Device-Specific Tips</a>, if your custom kernels introduces a lot of outstanding/expensive Reorders, consider blocked implementation for the kernels.
Finally, the performance statistics with both performance counters and execution graphs is averaged, so such a data for the [dynamically-shaped inputs](../../OV_Runtime_UG/ov_dynamic_shapes.md) should be measured carefully (ideally by isolating the specific shape and executing multiple times in a loop, to gather the reliable data).
Notice that in the heterogeneous cases, there will be additional information on which subgraph the statistics is about (the first subgraph is GPU, so its `cpu`/host time is really small compared to the actual `realTime`):
```
subgraph1: squeeze1x1 EXECUTED layerType: Convolution realTime: 227 cpu:3 execType: GPU
…
subgraph2: detection_out EXECUTED layerType: DetectionOutput realTime: 121 cpu:121 execType: unknown
…
```
As mentioned earlier, `unknown` here means CPU kernel with unknown (for example, not AVX2 or AVX512) acceleration path.
Since FPGA execution does not separate individual kernels, only bulk execution/data transfer statistics is available:
```
subgraph1: 1. input preprocessing (mean data/FPGA):EXECUTED layerType: preprocessing realTime: 129 cpu: 129
subgraph1: 2. input transfer to DDR:EXECUTED layerType: realTime: 201 cpu: 0
subgraph1: 3. FPGA execute time:EXECUTED layerType: realTime: 3808 cpu: 0 subgraph1: 4. output transfer from DDR:EXECUTED layerType: realTime: 55 cpu: 0
subgraph1: 5. FPGA output postprocessing:EXECUTED layerType: realTime: 7 cpu: 7
subgraph1: 6. softmax/copy: EXECUTED layerType: realTime: 2 cpu: 2
subgraph2: out_prob: NOT_RUN layerType: Output realTime: 0 cpu: 0
subgraph2: prob: EXECUTED layerType: SoftMax realTime: 10 cpu: 10
Total time: 4212 microseconds
```
The `softmax/copy` is a glue layer that connects the FPGA subgraph to the CPU subgraph (and copies the data).
### Intel&reg; VTune&trade; Examples <a name="vtune-examples"></a>
All major performance calls of the Inference Engine are instrumented with Instrumentation and Tracing Technology APIs. This allows viewing the Inference Engine calls on the Intel&reg; VTune&trade; timelines and aggregations plus correlating them to the underlying APIs, like OpenCL. In turn, this enables careful per-layer execution breakdown.
When choosing the Analysis type in Intel&reg; VTune&trade; Amplifier, make sure to select the **Analyze user tasks, events, and counters** option:
![](vtune_option.png)
See the [corresponding section in the Intel® VTune™ Amplifier User's Guide](https://software.intel.com/en-us/vtune-amplifier-help-task-analysis) for details.
Example of Inference Engine calls:
- On the Intel VTune Amplifier timeline.
Notice that `Task_runNOThrow` is an Async API wrapper and it is executed in a different thread and triggers the Intel MKL-DNN execution:
![](vtune_timeline.png)
- In the Intel VTune Amplifier **Top-down view**, grouped by the **Task Domain**.
Notice the `Task_runNoThrow` and `MKLDNN _INFER` that are bracketing the actual Intel MKL-DNN kernels execution:
![](vtune_topdown_view.jpg)
Similarly, you can use any GPU analysis in the Intel VTune Amplifier and get general correlation with Inference Engine API as well as the execution breakdown for OpenCL kernels.
Just like with regular native application, further drill down in the counters is possible, however, this is mostly useful for <a href="#optimizing-custom-kernels">optimizing custom kernels</a>. Finally, with the Intel VTune Amplifier, the profiling is not limited to your user-level code (see the [corresponding section in the Intel&reg; VTune&trade; Amplifier User's Guide](https://software.intel.com/en-us/vtune-amplifier-help-analyze-performance)).
OpenVINO in general and individual plugins are heavily instrumented with Intel® instrumentation and tracing technology (ITT), so another option is to compile the OpenVINO from the source code with the ITT enabled and using tools like [Intel® VTune™ Profiler](https://software.intel.com/en-us/vtune) to get detailed inference performance breakdown and additional insights in the application-level performance on the timeline view.

65
docs/MO_DG/prepare_model/Model_Optimization_Techniques.md
View File

@ -1,65 +0,0 @@
# Model Optimization Techniques {#openvino_docs_MO_DG_prepare_model_Model_Optimization_Techniques}
Optimization offers methods to accelerate inference with the convolution neural networks (CNN) that do not require model retraining.
* * *
## Linear Operations Fusing
Many convolution neural networks includes `BatchNormalization` and `ScaleShift` layers (for example, Resnet\*, Inception\*) that can be presented as a sequence of linear operations: additions and multiplications. For example ScaleShift layer can be presented as Mul → Add sequence. These layers can be fused into previous `Convolution` or `FullyConnected` layers, except when Convolution comes after an Add operation (due to Convolution paddings).
### Usage
In the Model Optimizer, this optimization is turned on by default. To disable it, you can pass `--disable_fusing` parameter to the Model Optimizer.
### Optimization Description
This optimization method consists of three stages:
1. <strong>`BatchNormalization` and `ScaleShift` decomposition</strong>: in this stage, `BatchNormalization` layer is decomposed to `Mul → Add → Mul → Add` sequence, and `ScaleShift` layer is decomposed to `Mul → Add` layers sequence.
2. **Linear operations merge**: in this stage, the `Mul` and `Add` operations are merged into a single `Mul → Add` instance.
For example, if there is a `BatchNormalization → ScaleShift` sequence in the topology, it is replaced with `Mul → Add` in the first stage. In the next stage, the latter is replaced with a `ScaleShift` layer if there is no available `Convolution` or `FullyConnected` layer to fuse into next.
3. **Linear operations fusion**: in this stage, the tool fuses `Mul` and `Add` operations to `Convolution` or `FullyConnected` layers. Notice that it searches for `Convolution` and `FullyConnected` layers both backward and forward in the graph (except for `Add` operation that cannot be fused to `Convolution` layer in forward direction).
### Usage Examples
The picture below shows the depicted part of Caffe\* Resnet269 topology where `BatchNorm` and `ScaleShift` layers will be fused to `Convolution` layers.
![Caffe ResNet269 block before and after optimization generated with Netscope*](../img/optimizations/resnet_269.png)
* * *
## ResNet optimization (stride optimization)
ResNet optimization is a specific optimization that applies to Caffe ResNet topologies such as ResNet50, ResNet101, ResNet152 and to ResNet-based topologies. This optimization is turned on by default, and can be disabled with the `--disable_resnet_optimization` key.
### Optimization Description
In the picture below, you can see the original and optimized parts of a Caffe ResNet50 model. The main idea of this optimization is to move the stride that is greater than 1 from Convolution layers with the kernel size = 1 to upper Convolution layers. In addition, the Model Optimizer adds a Pooling layer to align the input shape for a Eltwise layer, if it was changed during the optimization.
![ResNet50 blocks (original and optimized) from Netscope*](../img/optimizations/resnet_optimization.png)
In this example, the stride from the `res3a_branch1` and `res3a_branch2a` Convolution layers moves to the `res2c_branch2b` Convolution layer. In addition, to align the input shape for `res2c` Eltwise, the optimization inserts the Pooling layer with kernel size = 1 and stride = 2.
* * *
## Grouped Convolution Fusing
Grouped convolution fusing is a specific optimization that applies for TensorFlow\* topologies. The main idea of this optimization is to combine convolutions results for the `Split` outputs and then recombine them using `Concat` operation in the same order as they were out from `Split`.
![Split→Convolutions→Concat block from TensorBoard*](../img/optimizations/groups.png)
* * *
## Disable Fusing
Model Optimizer allows to disable optimizations for specified nodes via `--finegrain_fusing <node_name1>,<node_name2>,...` (regex is also supported). Using this key, you mark nodes that will noy be touched by any optimizations.
### Examples of usage
On the picture below you can see two visualized Intermediate Representations (IR) of TensorFlow InceptionV4 topology.
The first one is original IR that will be produced by the Model Optimizer.
The second one will be produced by the Model Optimizer with key `--finegrain_fusing InceptionV4/InceptionV4/Conv2d_1a_3x3/Conv2D`, where you can see that `Convolution` was not fused with `Mul1_3752` and `Mul1_4061/Fused_Mul_5096/FusedScaleShift_5987` operations.
![TF InceptionV4 block without/with key --finegrain_fusing (from IR visualizer)](../img/optimizations/inception_v4.png)

100
docs/MO_DG/prepare_model/Model_Optimizer_FAQ.md
View File

@ -6,7 +6,7 @@ If your question is not covered by the topics below, use the [OpenVINO&trade; Su
Internally, the Model Optimizer uses a protobuf library to parse and load Caffe\* models. This library requires a file grammar and a generated parser. For a Caffe fallback, the Model Optimizer uses a Caffe-generated parser for a Caffe-specific `.proto` file (which is usually located in the `src/caffe/proto` directory). So, if you have Caffe installed on your machine with Python* interface available, make sure that this is exactly the version of Caffe that was used to create the model.
If you just want to experiment with the Model Optimizer and test a Python extension for working with your custom
If you just want to experiment with the Model Optimizer and test a Python extension for working with your custom
layers without building Caffe, add the layer description to the `caffe.proto` file and generate a parser for it.
For example, to add the description of the `CustomReshape` layer, which is an artificial layer not present in any `caffe.proto` files:
@ -25,17 +25,17 @@ For example, to add the description of the `CustomReshape` layer, which is an ar
optional BlobShape shape = 1; // we just use the same parameter type as some other Caffe layers
}
```
2. Generate a new parser:
```shell
cd <SITE_PACKAGES_WITH_INSTALLED_OPENVINO>/openvino/tools/mo/front/caffe/proto
python3 generate_caffe_pb2.py --input_proto <PATH_TO_CUSTOM_CAFFE>/src/caffe/proto/caffe.proto
```
where `PATH_TO_CUSTOM_CAFFE` is the path to the root directory of custom Caffe\*.
3. Now, the Model Optimizer is able to load the model into memory and start working with your extensions if there are any.
However, because your model has custom layers, you must register your custom layers as custom. To learn more about it, refer to the section [Custom Layers in Model Optimizer](customize_model_optimizer/Customize_Model_Optimizer.md).
However, because your model has custom layers, you must register your custom layers as custom. To learn more about it, refer to the section [Custom Layers in Model Optimizer](customize_model_optimizer/Customize_Model_Optimizer.md).
#### 2. How do I create a bare caffemodel, if I have only prototxt? <a name="question-2"></a>
@ -48,8 +48,8 @@ net.save('<PATH_TO_PROTOTXT>/my_net.caffemodel')
```
#### 3. What does the message "[ ERROR ]: Unable to create ports for node with id" mean? <a name="question-3"></a>
Most likely, the Model Optimizer does not know how to infer output shapes of some layers in the given topology.
To lessen the scope, compile the list of layers that are custom for the Model Optimizer: present in the topology,
Most likely, the Model Optimizer does not know how to infer output shapes of some layers in the given topology.
To lessen the scope, compile the list of layers that are custom for the Model Optimizer: present in the topology,
absent in [list of supported layers](Supported_Frameworks_Layers.md) for the target framework. Then refer to available options in the corresponding section in [Custom Layers in Model Optimizer](customize_model_optimizer/Customize_Model_Optimizer.md).
#### 4. What does the message "Input image of shape is larger than mean image from file" mean? <a name="question-4"></a>
@ -100,7 +100,7 @@ message NetParameter {
```
So, the input layer of the provided model must be specified in one of the following styles:
*
*
```sh
input: "data"
input_shape
@ -111,8 +111,8 @@ input_shape
dim: 227
}
```
*
*
```sh
input: "data"
input_shape
@ -129,7 +129,7 @@ input_shape
dim: 3
}
```
*
*
```sh
layer
{
@ -146,7 +146,7 @@ layer
input_param {shape: {dim: 1 dim: 3}}
}
```
*
*
```sh
input: "data"
input_dim: 1
@ -158,7 +158,7 @@ However, if your model contains more than one input, the Model Optimizer is able
#### 9. What does the message "Mean file for topologies with multiple inputs is not supported" mean? <a name="question-9"></a>
Model Optimizer does not support mean file processing for topologies with more than one input. In this case, you need to perform preprocessing of the inputs for a generated Intermediate Representation in the Inference Engine to perform subtraction for every input of your multi-input model.
Model Optimizer does not support mean file processing for topologies with more than one input. In this case, you need to perform preprocessing of the inputs for a generated Intermediate Representation in the OpenVINO Runtime to perform subtraction for every input of your multi-input model, see [Overview of Preprocessing](../../OV_Runtime_UG/preprocessing_overview.md) for details.
#### 10. What does the message "Cannot load or process mean file: value error" mean? <a name="question-10"></a>
@ -214,7 +214,7 @@ One of the layers in the specified topology might not have inputs or values. Ple
#### 24. What does the message "Part of the nodes was not translated to IE. Stopped" mean? <a name="question-24"></a>
Some of the layers are not supported by the Inference Engine and cannot be translated to an Intermediate Representation. You can extend the Model Optimizer by allowing generation of new types of layers and implement these layers in the dedicated Inference Engine plugins. For more information, refer to the [Custom Layers Guide](../../HOWTO/Custom_Layers_Guide.md) and [Inference Engine Extensibility Mechanism](../../OV_Runtime_UG/Extensibility_DG/Intro.md)
Some of the operations are not supported by the OpenVINO Runtime and cannot be translated to an Intermediate Representation. You can extend the Model Optimizer by allowing generation of new types of operations and implement these operations in the dedicated OpenVINO plugins. For more information, refer to the [OpenVINO™ Extensibility Mechanism](../../Extensibility_UG/Intro.md)
#### 25. What does the message "While creating an edge from .. to .. : node name is undefined in the graph. Check correctness of the input model" mean? <a name="question-25"></a>
@ -252,7 +252,7 @@ Looks like you have provided only one shape for the placeholder, however there a
#### 33. What does the message "The amount of input nodes for port is not equal to 1" mean? <a name="question-33"></a>
This error occurs when the `SubgraphMatch.single_input_node` function is used for an input port that supplies more than one node in a sub-graph. The `single_input_node` function can be used only for ports that has a single consumer inside the matching sub-graph. When multiple nodes are connected to the port, use the `input_nodes` function or `node_by_pattern` function instead of `single_input_node`. Please, refer to [Sub-Graph Replacement in the Model Optimizer](customize_model_optimizer/Subgraph_Replacement_Model_Optimizer.md) for more details.
This error occurs when the `SubgraphMatch.single_input_node` function is used for an input port that supplies more than one node in a sub-graph. The `single_input_node` function can be used only for ports that has a single consumer inside the matching sub-graph. When multiple nodes are connected to the port, use the `input_nodes` function or `node_by_pattern` function instead of `single_input_node`. Please, refer to **Graph Transformation Extensions** section in the [Model Optimizer Extensibility](customize_model_optimizer/Customize_Model_Optimizer.md) documentation for more details.
#### 34. What does the message "Output node for port has already been specified" mean? <a name="question-34"></a>
@ -268,7 +268,7 @@ Model Optimizer tried to write an event file in the specified directory but fail
#### 37. What does the message "There is no registered 'infer' function for node with op = .. . Please implement this function in the extensions" mean? <a name="question-37"></a>
Most likely, you tried to extend Model Optimizer with a new primitive, but did not specify an infer function. For more information on extensions, see [Custom Layers Guide](../../HOWTO/Custom_Layers_Guide.md).
Most likely, you tried to extend Model Optimizer with a new primitive, but did not specify an infer function. For more information on extensions, see [OpenVINO™ Extensibility Mechanism](../../Extensibility_UG/Intro.md).
#### 38. What does the message "Stopped shape/value propagation at node" mean? <a name="question-38"></a>
@ -300,7 +300,7 @@ Most likely, there is a problem with the specified file for model. The file exis
#### 45. What does the message "Found custom layer. Model Optimizer does not support this layer. Please, register it in CustomLayersMapping.xml or implement extension" mean? <a name="question-45"></a>
This means that the layer `{layer_name}` is not supported in the Model Optimizer. You can find a list of all unsupported layers in the corresponding section. You should implement the extensions for this layer ([Custom Layers Guide](../../HOWTO/Custom_Layers_Guide.md)).
This means that the layer `{layer_name}` is not supported in the Model Optimizer. You can find a list of all unsupported layers in the corresponding section. You should implement the extensions for this layer ([OpenVINO™ Extensibility Mechanism](../../Extensibility_UG/Intro.md)).
#### 46. What does the message "Custom replacement configuration file does not exist" mean? <a name="question-46"></a>
@ -308,7 +308,7 @@ Path to the custom replacement configuration file was provided with the `--trans
#### 47. What does the message "Extractors collection have case insensitive duplicates" mean? <a name="question-47"></a>
When extending Model Optimizer with new primitives keep in mind that their names are case insensitive. Most likely, another operation with the same name is already defined. For more information, see [Custom Layers Guide](../../HOWTO/Custom_Layers_Guide.md).
When extending Model Optimizer with new primitives keep in mind that their names are case insensitive. Most likely, another operation with the same name is already defined. For more information, see [OpenVINO™ Extensibility Mechanism](../../Extensibility_UG/Intro.md).
#### 48. What does the message "Input model name is not in an expected format, cannot extract iteration number" mean? <a name="question-48"></a>
@ -340,7 +340,7 @@ Please, make sure that inputs are defined and have correct shapes. You can use `
#### 55. What does the message "Attempt to register of custom name for the second time as class. Note that custom names are case-insensitive" mean? <a name="question-55"></a>
When extending Model Optimizer with new primitives keep in mind that their names are case insensitive. Most likely, another operation with the same name is already defined. For more information, see [Custom Layers Guide](../../HOWTO/Custom_Layers_Guide.md).
When extending Model Optimizer with new primitives keep in mind that their names are case insensitive. Most likely, another operation with the same name is already defined. For more information, see [OpenVINO™ Extensibility Mechanism](../../Extensibility_UG/Intro.md).
#### 56. What does the message "Both --input_shape and --batch were provided. Please, provide only one of them" mean? <a name="question-56"></a>
@ -350,7 +350,7 @@ You cannot specify the batch and the input shape at the same time. You should sp
The specified input shape cannot be parsed. Please, define it in one of the following ways:
*
*
```shell
mo --input_model <INPUT_MODEL>.caffemodel --input_shape (1,3,227,227)
```
@ -447,7 +447,7 @@ This message may appear when the `--data_type=FP16` command line option is used.
#### 78. What does the message "The amount of nodes matched pattern ... is not equal to 1" mean? <a name="question-78"></a>
This error occurs when the `SubgraphMatch.node_by_pattern` function is used with a pattern that does not uniquely identify a single node in a sub-graph. Try to extend the pattern string to make unambiguous match to a single sub-graph node. For more details, refer to [Sub-graph Replacement in the Model Optimizer](customize_model_optimizer/Subgraph_Replacement_Model_Optimizer.md).
This error occurs when the `SubgraphMatch.node_by_pattern` function is used with a pattern that does not uniquely identify a single node in a sub-graph. Try to extend the pattern string to make unambiguous match to a single sub-graph node. For more details, refer to **Graph Transformation Extensions** section in the [Model Optimizer Extensibility](customize_model_optimizer/Customize_Model_Optimizer.md) documentation.
#### 79. What does the message "The topology contains no "input" layers" mean? <a name="question-79"></a>
@ -459,18 +459,18 @@ You are using an unsupported Python\* version. Use only versions 3.4 - 3.6 for t
#### 81. What does the message "Arguments --nd_prefix_name, --pretrained_model_name and --input_symbol should be provided. Please provide all or do not use any." mean? <a name="question-81"></a>
This error occurs if you do not provide `--nd_prefix_name`, `--pretrained_model_name` and `--input_symbol` parameters.
Model Optimizer requires both `.params` and `.nd` model files to merge into the result file (`.params`). Topology
This error occurs if you do not provide `--nd_prefix_name`, `--pretrained_model_name` and `--input_symbol` parameters.
Model Optimizer requires both `.params` and `.nd` model files to merge into the result file (`.params`). Topology
description (`.json` file) should be prepared (merged) in advance and provided with `--input_symbol` parameter.
If you add to your model additional layers and weights that are in `.nd` files, the Model Optimizer can build a model
If you add to your model additional layers and weights that are in `.nd` files, the Model Optimizer can build a model
from one `.params` file and two additional `.nd` files (`*_args.nd`, `*_auxs.nd`).
To do that, provide both CLI options or do not pass them if you want to convert an MXNet model without additional weights.
For more information, refer to [Converting a MXNet* Model](convert_model/Convert_Model_From_MxNet.md).
#### 82. What does the message "You should specify input for mean/scale values" mean? <a name="question-82"></a>
In case when the model has multiple inputs and you want to provide mean/scale values, you need to pass those values for each input. More specifically, a number of passed values should be the same as the number of inputs of the model.
In case when the model has multiple inputs and you want to provide mean/scale values, you need to pass those values for each input. More specifically, a number of passed values should be the same as the number of inputs of the model.
For more information, refer to [Converting a Model to Intermediate Representation](convert_model/Converting_Model.md).
#### 83. What does the message "Input with name ... not found!" mean? <a name="question-83"></a>
@ -490,16 +490,16 @@ For more information, refer to [Converting a MXNet* Model](convert_model/Convert
#### 86. What does the message "Operation ... not supported. Please register it as custom op" mean? <a name="question-86"></a>
Model Optimizer tried to load the model that contains some unsupported operations.
Model Optimizer tried to load the model that contains some unsupported operations.
If you want to convert model that contains unsupported operations you need to prepare extension for all such operations.
For more information, refer to [Custom Layers Guide](../../HOWTO/Custom_Layers_Guide.md).
For more information, refer to [OpenVINO™ Extensibility Mechanism](../../Extensibility_UG/Intro.md).
#### 87. What does the message "Can not register Op ... Please, call function 'register_caffe_python_extractor' with parameter 'name'" mean? <a name="question-87"></a>
This error appears if the class of implementation of op for Python Caffe layer could not be used by Model Optimizer. Python layers should be handled differently compared to ordinary Caffe layers.
In particular, you need to call the function `register_caffe_python_extractor` and pass `name` as the second argument of the function.
The name should be the compilation of the layer name and the module name separated by a dot.
The name should be the compilation of the layer name and the module name separated by a dot.
For example, your topology contains this layer with type `Python`:
@ -520,7 +520,7 @@ What you do first is implementing an extension for this layer in the Model Optim
```
class ProposalPythonExampleOp(Op):
op = 'Proposal'
def __init__(self, graph: nx.MultiDiGraph, attrs: dict):
...
```
@ -536,25 +536,25 @@ Op.excluded_classes.append(ProposalPythonExampleOp)
Note that the first call <code>register_caffe_python_extractor(ProposalPythonExampleOp, 'rpn.proposal_layer.ProposalLayer')</code> registers extension of the layer in the Model Optimizer that will be found by the specific name (mandatory to join module name and layer name): <code>rpn.proposal_layer.ProposalLayer</code>.
The second call prevents Model Optimizer from using this extension as if it is an extension for
The second call prevents Model Optimizer from using this extension as if it is an extension for
a layer with type `Proposal`. Otherwise, this layer can be chosen as an implementation of extension that can lead to potential issues.
For more information, refer to the [Custom Layers Guide](../../HOWTO/Custom_Layers_Guide.md).
For more information, refer to the [OpenVINO™ Extensibility Mechanism](../../Extensibility_UG/Intro.md).
#### 88. What does the message "Model Optimizer is unable to calculate output shape of Memory node .." mean? <a name="question-88"></a>
Model Optimizer supports only `Memory` layers, in which `input_memory` goes before `ScaleShift` or `FullyConnected` layer.
Model Optimizer supports only `Memory` layers, in which `input_memory` goes before `ScaleShift` or `FullyConnected` layer.
This error message means that in your model the layer after input memory is not of type `ScaleShift` or `FullyConnected`.
This is a known limitation.
#### 89. What do the messages "File ... does not appear to be a Kaldi file (magic number does not match)", "Kaldi model should start with <Nnet> tag" mean? <a name="question-89"></a>
These error messages mean that the Model Optimizer does not support your Kaldi\* model, because check sum of the model is not
These error messages mean that the Model Optimizer does not support your Kaldi\* model, because check sum of the model is not
16896 (the model should start with this number) or model file does not contain tag `<Net>` as a starting one.
Double check that you provide a path to a true Kaldi model and try again.
#### 90. What do the messages "Expect counts file to be one-line file." or "Expect counts file to contain list of integers" mean? <a name="question-90"></a>
These messages mean that you passed the file counts containing not one line. The count file should start with
These messages mean that you passed the file counts containing not one line. The count file should start with
`[` and end with `]`, and integer values should be separated by space between those signs.
#### 91. What does the message "Model Optimizer is not able to read Kaldi model .." mean? <a name="question-91"></a>
@ -570,10 +570,10 @@ file is not available or does not exist. Also refer to FAQ [#90](#question-90).
#### 93. What does the message "For legacy MXNet models Model Optimizer does not support conversion of old MXNet models (trained with 1.0.0 version of MXNet and lower) with custom layers." mean? <a name="question-93"></a>
This message means that if you have model with custom layers and its json file has been generated with MXNet version
lower than 1.0.0, Model Optimizer does not support such topologies. If you want to convert it you have to rebuild
MXNet with unsupported layers or generate new json with MXNet version 1.0.0 and higher. Also you need to implement
Inference Engine extension for used custom layers.
For more information, refer to the [Custom Layers Guide](../../HOWTO/Custom_Layers_Guide.md).
lower than 1.0.0, Model Optimizer does not support such topologies. If you want to convert it you have to rebuild
MXNet with unsupported layers or generate new json with MXNet version 1.0.0 and higher. Also you need to implement
OpenVINO extension for used custom layers.
For more information, refer to the [OpenVINO™ Extensibility Mechanism](../../Extensibility_UG/Intro.md).
#### 97. What does the message "Graph contains a cycle. Can not proceed .." mean? <a name="question-97"></a>
@ -581,22 +581,22 @@ Model Optimizer supports only straightforward models without cycles.
There are multiple ways to avoid cycles:
For Tensorflow:
For Tensorflow:
* [Convert models, created with TensorFlow Object Detection API](convert_model/tf_specific/Convert_Object_Detection_API_Models.md)
For all frameworks:
1. [Replace cycle containing Sub-graph in Model Optimizer](customize_model_optimizer/Subgraph_Replacement_Model_Optimizer.md)
2. [Custom Layers Guide](../../HOWTO/Custom_Layers_Guide.md)
For all frameworks:
1. [Replace cycle containing Sub-graph in Model Optimizer](customize_model_optimizer/Customize_Model_Optimizer.md)
2. [OpenVINO™ Extensibility Mechanism](../../Extensibility_UG/Intro.md)
or
* Edit network in original framework to exclude cycle.
* Edit model in original framework to exclude cycle.
#### 98. What does the message "Can not transpose attribute '..' with value .. for node '..' .." mean? <a name="question-98"></a>
This message means that model is not supported. It may be caused by using shapes larger than 4-D.
There are two ways to avoid such message:
1. [Cut model part containing such layers in Model Optimizer](convert_model/Cutting_Model.md)
1. [Cutting Off Parts of a Model](convert_model/Cutting_Model.md)
2. Edit network in original framework to exclude such layers.
#### 99. What does the message "Expected token `</ParallelComponent>`, has `...`" mean? <a name="question-99"></a>
@ -611,7 +611,7 @@ But there are exceptions. For example, output value of layer Interp is calculate
#### 101. What does the message "Mean/scale values should ..." mean? <a name="question-101"></a>
It means that your mean/scale values have wrong format. Specify mean/scale values using the form `layer_name(val1,val2,val3)`.
It means that your mean/scale values have wrong format. Specify mean/scale values using the form `layer_name(val1,val2,val3)`.
You need to specify values for each input of the model. For more information, refer to [Converting a Model to Intermediate Representation](convert_model/Converting_Model.md).
#### 102. What does the message "Operation _contrib_box_nms is not supported ..." mean? <a name="question-102"></a>
@ -632,10 +632,10 @@ Note that you might have conflicts between previously installed PyPI dependencie
#### 105. What does the message "The IR preparation was executed by the legacy MO path. ..." mean? <a name="question-105"></a>
For the models in ONNX* format, there are two available paths of IR conversion.
The old one is handled by the old Python* implementation, while the new one uses new C++ frontends.
Starting from the 2022.1 version, the default IR conversion path for ONNX models is processed using the new ONNX frontend.
Certain features, such as `--extensions` and `--transformations_config`, are not yet fully supported on the new frontends.
For `--extensions`, the new frontends support only paths to shared libraries (.dll and .so). For `--transformations_config`, they support JSON configurations with defined library fields.
Inputs freezing (enabled by `--freeze_placeholder_with_value` or `--input` arguments) is not supported on the new frontends.
For the models in ONNX* format, there are two available paths of IR conversion.
The old one is handled by the old Python* implementation, while the new one uses new C++ frontends.
Starting from the 2022.1 version, the default IR conversion path for ONNX models is processed using the new ONNX frontend.
Certain features, such as `--extensions` and `--transformations_config`, are not yet fully supported on the new frontends.
For `--extensions`, the new frontends support only paths to shared libraries (.dll and .so). For `--transformations_config`, they support JSON configurations with defined library fields.
Inputs freezing (enabled by `--freeze_placeholder_with_value` or `--input` arguments) is not supported on the new frontends.
The IR conversion falls back to the old path if a user does not select any expected path of conversion explicitly (by `--use_new_frontend` or `--use_legacy_frontend` MO arguments) and unsupported pre-defined scenario is detected on the new frontend path.

346
docs/MO_DG/prepare_model/Supported_Frameworks_Layers.md
View File

@ -1,9 +1,9 @@
# Supported Framework Layers {#openvino_docs_MO_DG_prepare_model_Supported_Frameworks_Layers}
## Caffe\* Supported Layers
## Caffe Supported Layers
| Layer Name in Caffe\* | Limitations |
| Layer Name in Caffe | Limitations |
|:---------- | :----------|
| Axpy | |
| BN | |
@ -47,10 +47,10 @@
| Tile | |
## MXNet\* Supported Symbols
## MXNet Supported Symbols
| Symbol Name in MXNet\*| Limitations|
| Symbol Name in MXNet| Limitations|
| :----------| :----------|
| _Plus | |
| _contrib_arange_like | |
@ -119,7 +119,7 @@
| Concat | |
| Convolution | |
| Crop | "center_crop" = 1 is not supported |
| Custom | [Custom Layers in the Model Optimizer](customize_model_optimizer/Customize_Model_Optimizer.md) |
| Custom | [Custom Layers in Model Optimizer](customize_model_optimizer/Customize_Model_Optimizer.md) |
| Deconvolution | |
| DeformableConvolution | |
| DeformablePSROIPooling | |
@ -149,12 +149,12 @@
| zeros_like | |
## TensorFlow\* Supported Operations
## TensorFlow Supported Operations
Some TensorFlow\* operations do not match to any Inference Engine layer, but are still supported by the Model Optimizer and can be used on constant propagation path. These layers are labeled 'Constant propagation' in the table.
Some TensorFlow operations do not match to any OpenVINO operation, but are still supported by the Model Optimizer and can be used on constant propagation path. These layers are labeled 'Constant propagation' in the table.
| Operation Name in TensorFlow\* | Limitations|
| Operation Name in TensorFlow | Limitations|
| :----------| :----------|
| Abs | |
| Acosh | |
@ -348,10 +348,10 @@ Some TensorFlow\* operations do not match to any Inference Engine layer, but are
| ZerosLike | |
## TensorFlow 2 Keras\* Supported Operations
## TensorFlow 2 Keras Supported Operations
| Operation Name in TensorFlow 2 Keras\* | Limitations|
| Operation Name in TensorFlow 2 Keras | Limitations|
| :----------| :----------|
| ActivityRegularization | |
| Add | |
@ -431,10 +431,10 @@ Some TensorFlow\* operations do not match to any Inference Engine layer, but are
| ZeroPadding2D | |
| ZeroPadding3D | |
## Kaldi\* Supported Layers
## Kaldi Supported Layers
| Symbol Name in Kaldi\*| Limitations|
| Symbol Name in Kaldi| Limitations|
| :----------| :----------|
| addshift | |
| affinecomponent | |
@ -478,154 +478,194 @@ Some TensorFlow\* operations do not match to any Inference Engine layer, but are
| timeheightconvolutioncomponent | |
## ONNX\* Supported Operators
## ONNX Supported Operators
### Standard ONNX Operators
| Symbol Name in ONNX\*| Limitations|
| :----------| :----------|
| Abs | |
| Acos | |
| Acosh | |
| Add | |
| Affine | |
| And | |
| ArgMax | |
| ArgMin | |
| Asin | |
| Asinh | |
| Atan | |
| Atanh | |
| ATen | Supported only for the 'embedding_bag' operator |
| AveragePool | |
| BatchMatMul | |
| BatchNormalization | |
| Cast | |
| Ceil | |
| Clip | |
| Concat | |
| Constant | |
| ConstantFill | |
| ConstantOfShape | |
| Conv | |
| ConvTranspose | |
| Cos | |
| Cosh | |
| Crop | |
| CumSum | |
| DepthToSpace | |
| DequantizeLinear | |
| DetectionOutput (Intel experimental) | |
| Div | |
| Dropout | Not needed for inference |
| Elu | |
| Equal | |
| Erf | |
| Exp | |
| Expand | |
| ExperimentalDetectronDetectionOutput (Intel experimental) | |
| ExperimentalDetectronGenerateProposalsSingleImage (Intel experimental) | |
| ExperimentalDetectronGroupNorm (Intel experimental) | |
| ExperimentalDetectronPriorGridGenerator (Intel experimental) | |
| ExperimentalDetectronROIFeatureExtractor (Intel experimental) | |
| ExperimentalDetectronTopKROIs (Intel experimental) | |
| FakeQuantize (Intel experimental) | |
| Fill | |
| Flatten | |
| Floor | |
| GRU | |
| Gather | |
| GatherElements | Doesn't work with negative indices |
| GatherND | Doesn't work with negative indices |
| GatherTree | |
| Gemm | |
| GlobalAveragePool | |
| GlobalMaxPool | |
| Greater | |
| GreaterEqual | |
| HardSigmoid | |
| Identity | Not needed for inference |
| ImageScaler | |
| InstanceNormalization | |
| LRN | |
| LSTM | Peepholes are not supported |
| LeakyRelu | |
| Less | |
| LessEqual | |
| Log | |
| LogicalAnd | |
| LogicalOr | |
| LogSoftmax | |
| Loop | |
| LpNormalization | |
| MatMul | |
| Max | |
| MaxPool | |
| MeanVarianceNormalization | Reduction over the batch dimension is not supported, reduction over all dimensions except batch and channel ones is obligatory |
| Min | |
| Mul | |
| Neg | |
| NonMaxSuppression | |
| NonZero | |
| Not | |
| NotEqual | |
| OneHot | |
| Pad | |
| Pow | |
| PriorBox (Intel experimental) | |
| PriorBoxClustered | |
| QuantizeLinear | |
| RNN | |
| ROIAlign | |
| Range | |
| RandomUniform | Operation provides sequence from uniform distribution, but exact values won't match. |
| Reciprocal | |
| ReduceL1 | |
| ReduceL2 | |
| ReduceMax | |
| ReduceMean | |
| ReduceMin | |
| ReduceProd | |
| ReduceSum | |
| Relu | |
| Reshape | |
| Resize | Coordinate transformation mode `tf_crop_and_resize` is not supported, `nearest` mode is not supported for 5D+ inputs. |
| ReverseSequence | |
| Round | |
| Scatter | Supported if fuse-able to ScatterUpdate. MYRIAD only |
| ScatterND | |
| ScatterElements | Supported if fuse-able to ScatterUpdate. MYRIAD only |
| Select | |
| Shape | |
| Sigmoid | |
| Sign | |
| Sin | |
| Size | |
| Slice | |
| Softmax | |
| Softplus | |
| Softsign | |
| SpaceToDepth | |
| Split | |
| Sqrt | |
| Squeeze | The case when squeeze axis is not specified is not supported |
| Sub | |
| Sum | |
| Tan | |
| Tanh | |
| ThresholdedRelu | |
| TopK | |
| Transpose | |
| Unsqueeze | |
| Upsample | |
| Where | |
| Xor | |
| ONNX Operator Name |
| :----------|
| Abs |
| Acos |
| Acosh |
| And |
| ArgMin |
| ArgMax |
| Asin |
| Asinh |
| Atan |
| ATen |
| Atanh |
| AveragePool |
| BatchNormalization |
| BitShift |
| Cast |
| CastLike |
| Ceil |
| Clip |
| Concat |
| Constant |
| ConstantOfShape |
| Conv |
| ConvInteger |
| ConvTranspose |
| Compress |
| Cos |
| Cosh |
| ConstantFill |
| CumSum |
| DepthToSpace |
| DequantizeLinear |
| Div |
| Dropout |
| Einsum |
| Elu |
| Equal |
| Erf |
| Exp |
| Expand |
| EyeLike |
| Flatten |
| Floor |
| Gather |
| GatherElements |
| GatherND |
| Gemm |
| GlobalAveragePool |
| GlobalLpPool |
| GlobalMaxPool |
| Greater |
| GRU |
| Hardmax |
| HardSigmoid |
| HardSwish |
| Identity |
| If |
| ImageScaler |
| InstanceNormalization |
| LeakyRelu |
| Less |
| Log |
| LogSoftmax |
| Loop |
| LpNormalization |
| LRN |
| LSTM |
| MatMulInteger |
| MatMul |
| MaxPool |
| Max |
| Mean |
| MeanVarianceNormalization |
| Min |
| Mod |
| Mul |
| Neg |
| NonMaxSuppression |
| NonZero |
| Not |
| Or |
| OneHot |
| Pad |
| Pow |
| PRelu |
| QLinearConv |
| QLinearMatMul |
| QuantizeLinear |
| Range |
| RandomNormal |
| RandomNormalLike |
| RandomUniform |
| RandomUniformLike |
| Reciprocal |
| ReduceLogSum |
| ReduceLogSumExp |
| ReduceL1 |
| ReduceL2 |
| ReduceMax |
| ReduceMean |
| ReduceMin |
| ReduceProd |
| ReduceSum |
| ReduceSumSquare |
| Relu |
| Reshape |
| Resize |
| ReverseSequence |
| RNN |
| RoiAlign |
| Round |
| ScatterElements |
| ScatterND |
| Selu |
| Shape |
| Shrink |
| Sigmoid |
| Sign |
| Sin |
| Sinh |
| Size |
| Slice |
| Softmax |
| Softplus |
| Softsign |
| SpaceToDepth |
| Split |
| Sqrt |
| Squeeze |
| Sub |
| Sum |
| Tan |
| Tanh |
| ThresholdedRelu |
| Tile |
| TopK |
| Transpose |
| Unsqueeze |
| Where |
| Xor |
### Deprecated ONNX Operators (Supported)
## PaddlePaddle\* Supported Operators
| ONNX Operator Name |
| :----------|
| Affine |
| Crop |
| Scatter |
| Upsample |
### Operators From the org.openvinotoolkit Domain
| Custom ONNX Operator Name |
| :----------|
| DeformableConv2D |
| DetectionOutput |
| ExperimentalDetectronDetectionOutput |
| ExperimentalDetectronGenerateProposalsSingleImage |
| ExperimentalDetectronGroupNorm |
| ExperimentalDetectronPriorGridGenerator |
| ExperimentalDetectronROIFeatureExtractor |
| ExperimentalDetectronTopKROIs |
| FakeQuantize |
| GroupNorm |
| Normalize |
| PriorBox |
| PriorBoxClustered |
| Swish |
### Operators From the com.microsoft Domain
| Custom ONNX Operator Name |
| :----------|
| Attention |
| BiasGelu |
| EmbedLayerNormalization |
| SkipLayerNormalization |
## PaddlePaddle Supported Operators
paddlepaddle>=2.1
| Operator Name in PaddlePaddle\*| Limitations|
| Operator Name in PaddlePaddle| Limitations|
| :----------| :----------|
| adpative_pool2d | 'NHWC' data_layout is not supported |
| arg_max | 'int32' output data_type is not supported |

69
docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Caffe.md
View File

@ -1,60 +1,12 @@
# Converting a Caffe* Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Caffe}
@sphinxdirective
.. _convert model caffe:
@endsphinxdirective
A summary of the steps for optimizing and deploying a model that was trained with Caffe\*:
1. [Configure the Model Optimizer](../../Deep_Learning_Model_Optimizer_DevGuide.md) for Caffe\*.
2. [Convert a Caffe\* Model](#Convert_From_Caffe) to produce an optimized [Intermediate Representation (IR)](../../IR_and_opsets.md) of the model based on the trained network topology, weights, and biases values
3. Test the model in the Intermediate Representation format using the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in the target environment via provided Inference Engine [sample applications](../../../OV_Runtime_UG/Samples_Overview.md)
4. [Integrate](../../../OV_Runtime_UG/Samples_Overview.md) the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in your application to deploy the model in the target environment
## Supported Topologies
* **Classification models:**
* AlexNet
* VGG-16, VGG-19
* SqueezeNet v1.0, SqueezeNet v1.1
* ResNet-50, ResNet-101, Res-Net-152
* Inception v1, Inception v2, Inception v3, Inception v4
* CaffeNet
* MobileNet
* Squeeze-and-Excitation Networks: SE-BN-Inception, SE-Resnet-101, SE-ResNet-152, SE-ResNet-50, SE-ResNeXt-101, SE-ResNeXt-50
* ShuffleNet v2
* **Object detection models:**
* SSD300-VGG16, SSD500-VGG16
* Faster-RCNN
* RefineDet (MYRIAD plugin only)
* **Face detection models:**
* VGG Face
* SSH: Single Stage Headless Face Detector
* **Semantic segmentation models:**
* FCN8
> **NOTE**: It is necessary to specify mean and scale values for most of the Caffe\* models to convert them with the Model Optimizer. The exact values should be determined separately for each model. For example, for Caffe\* models trained on ImageNet, the mean values usually are `123.68`, `116.779`, `103.939` for blue, green and red channels respectively. The scale value is usually `127.5`. Refer to the General Conversion Parameters section in [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) for the information on how to specify mean and scale values.
## Convert a Caffe* Model <a name="Convert_From_Caffe"></a>
To convert a Caffe\* model, run Model Optimizer with the path to the input model `.caffemodel` file and the path to an output directory with write permissions:
To convert a Caffe\* model, run Model Optimizer with the path to the input model `.caffemodel` file:
```sh
mo --input_model <INPUT_MODEL>.caffemodel --output_dir <OUTPUT_MODEL_DIR>
mo --input_model <INPUT_MODEL>.caffemodel
```
Two groups of parameters are available to convert your model:
* Framework-agnostic parameters are used to convert a model trained with any supported framework. For details, see see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page.
* [Caffe-specific parameters](#caffe_specific_conversion_params) are used to convert only Caffe\* models.
### Using Caffe\*-Specific Conversion Parameters <a name="caffe_specific_conversion_params"></a>
The following list provides the Caffe\*-specific parameters.
```
@ -93,16 +45,16 @@ Caffe*-specific parameters:
attributes without flattening nested parameters.
```
#### Command-Line Interface (CLI) Examples Using Caffe\*-Specific Parameters
### Command-Line Interface (CLI) Examples Using Caffe\*-Specific Parameters
* Launching the Model Optimizer for the [bvlc_alexnet.caffemodel](https://github.com/BVLC/caffe/tree/master/models/bvlc_alexnet) with a specified `prototxt` file. This is needed when the name of the Caffe\* model and the `.prototxt` file are different or are placed in different directories. Otherwise, it is enough to provide only the path to the input `model.caffemodel` file. You must have write permissions for the output directory.
```sh
mo --input_model bvlc_alexnet.caffemodel --input_proto bvlc_alexnet.prototxt --output_dir <OUTPUT_MODEL_DIR>
* Launching the Model Optimizer for the [bvlc_alexnet.caffemodel](https://github.com/BVLC/caffe/tree/master/models/bvlc_alexnet) with a specified `prototxt` file. This is needed when the name of the Caffe\* model and the `.prototxt` file are different or are placed in different directories. Otherwise, it is enough to provide only the path to the input `model.caffemodel` file.
```sh
mo --input_model bvlc_alexnet.caffemodel --input_proto bvlc_alexnet.prototxt
```
* Launching the Model Optimizer for the [bvlc_alexnet.caffemodel](https://github.com/BVLC/caffe/tree/master/models/bvlc_alexnet) with a specified `CustomLayersMapping` file. This is the legacy method of quickly enabling model conversion if your model has custom layers. This requires the Caffe\* system on the computer.
Optional parameters without default values and not specified by the user in the `.prototxt` file are removed from the Intermediate Representation, and nested parameters are flattened:
```sh
mo --input_model bvlc_alexnet.caffemodel -k CustomLayersMapping.xml --disable_omitting_optional --enable_flattening_nested_params --output_dir <OUTPUT_MODEL_DIR>
mo --input_model bvlc_alexnet.caffemodel -k CustomLayersMapping.xml --disable_omitting_optional --enable_flattening_nested_params
```
This example shows a multi-input model with input layers: `data`, `rois`
```
@ -124,9 +76,9 @@ Optional parameters without default values and not specified by the user in the
}
```
* Launching the Model Optimizer for a multi-input model with two inputs and providing a new shape for each input in the order they are passed to the Model Optimizer along with a writable output directory. In particular, for data, set the shape to `1,3,227,227`. For rois, set the shape to `1,6,1,1`:
* Launching the Model Optimizer for a multi-input model with two inputs and providing a new shape for each input in the order they are passed to the Model Optimizer. In particular, for data, set the shape to `1,3,227,227`. For rois, set the shape to `1,6,1,1`:
```sh
mo --input_model /path-to/your-model.caffemodel --input data,rois --input_shape (1,3,227,227),[1,6,1,1] --output_dir <OUTPUT_MODEL_DIR>
mo --input_model /path-to/your-model.caffemodel --input data,rois --input_shape (1,3,227,227),[1,6,1,1]
```
## Custom Layer Definition
@ -146,3 +98,6 @@ In this document, you learned:
* Basic information about how the Model Optimizer works with Caffe\* models
* Which Caffe\* models are supported
* How to convert a trained Caffe\* model using the Model Optimizer with both framework-agnostic and Caffe-specific command-line options
## See Also
[Model Conversion Tutorials](Convert_Model_Tutorials.md)

86
docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Kaldi.md
View File

@ -1,58 +1,14 @@
# Converting a Kaldi* Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Kaldi}
@sphinxdirective
.. _convert model kaldi:
.. toctree::
:maxdepth: 1
:hidden:
openvino_docs_MO_DG_prepare_model_convert_model_kaldi_specific_Aspire_Tdnn_Model
@endsphinxdirective
A summary of the steps for optimizing and deploying a model that was trained with Kaldi\*:
1. [Configure the Model Optimizer](../../Deep_Learning_Model_Optimizer_DevGuide.md) for Kaldi\*.
2. [Convert a Kaldi\* Model](#Convert_From_Kaldi) to produce an optimized [Intermediate Representation (IR)](../../IR_and_opsets.md) of the model based on the trained network topology, weights, and biases values.
3. Test the model in the Intermediate Representation format using the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in the target environment via provided Inference Engine [sample applications](../../../OV_Runtime_UG/Samples_Overview.md).
4. [Integrate](../../../OV_Runtime_UG/Samples_Overview.md) the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in your application to deploy the model in the target environment.
> **NOTE**: The Model Optimizer supports the [nnet1](http://kaldi-asr.org/doc/dnn1.html) and [nnet2](http://kaldi-asr.org/doc/dnn2.html) formats of Kaldi models. Support of the [nnet3](http://kaldi-asr.org/doc/dnn3.html) format is limited.
## Supported Topologies
* Convolutional Neural Networks (CNN):
* Wall Street Journal CNN (wsj_cnn4b)
* Resource Management CNN (rm_cnn4a_smbr)
* Long Short Term Memory (LSTM) Networks:
* Resource Management LSTM (rm_lstm4f)
* TED-LIUM LSTM (ted_lstm4f)
* Deep Neural Networks (DNN):
* Wall Street Journal DNN (wsj_dnn5b_smbr);
* TED-LIUM DNN (ted_dnn_smbr)
* Time delay neural network (TDNN)
* [ASpIRE Chain TDNN](kaldi_specific/Aspire_Tdnn_Model.md);
* [Librispeech nnet3](https://github.com/ryanleary/kaldi-test/releases/download/v0.0/LibriSpeech-trained.tgz).
* TDNN-LSTM model
## Convert a Kaldi* Model <a name="Convert_From_Kaldi"></a>
To convert a Kaldi\* model, run Model Optimizer with the path to the input model `.nnet` or `.mdl` file:
To convert a Kaldi\* model, run Model Optimizer with the path to the input model `.nnet` or `.mdl` file and to an output directory where you have write permissions:
```sh
mo --input_model <INPUT_MODEL>.nnet --output_dir <OUTPUT_MODEL_DIR>
mo --input_model <INPUT_MODEL>.nnet
```
Two groups of parameters are available to convert your model:
* Framework-agnostic parameters are used to convert a model trained with any supported framework. For details, see see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page.
* [Kaldi-specific parameters](#kaldi_specific_conversion_params) are used to convert only Kaldi\* models.
### Using Kaldi\*-Specific Conversion Parameters <a name="kaldi_specific_conversion_params"></a>
The following list provides the Kaldi\*-specific parameters.
@ -67,14 +23,14 @@ Kaldi-specific parameters:
### Examples of CLI Commands
* To launch the Model Optimizer for the wsj_dnn5b_smbr model with the specified `.nnet` file and an output directory where you have write permissions:
* To launch the Model Optimizer for the wsj_dnn5b_smbr model with the specified `.nnet` file:
```sh
mo --input_model wsj_dnn5b_smbr.nnet --output_dir <OUTPUT_MODEL_DIR>
mo --input_model wsj_dnn5b_smbr.nnet
```
* To launch the Model Optimizer for the wsj_dnn5b_smbr model with existing file that contains counts for the last layer with biases and a writable output directory:
* To launch the Model Optimizer for the wsj_dnn5b_smbr model with existing file that contains counts for the last layer with biases:
```sh
mo --input_model wsj_dnn5b_smbr.nnet --counts wsj_dnn5b_smbr.counts --output_dir <OUTPUT_MODEL_DIR>
mo --input_model wsj_dnn5b_smbr.nnet --counts wsj_dnn5b_smbr.counts
```
* The Model Optimizer normalizes сounts in the following way:
@ -88,34 +44,36 @@ Kaldi-specific parameters:
\f$|C|\f$ - number of elements in the counts array;
* The normalized counts are subtracted from biases of the last or next to last layer (if last layer is SoftMax).
> **NOTE:** Model Optimizer will show warning if model contains counts values inside model and `--counts` option is not used.
> **NOTE**: Model Optimizer will show warning if model contains counts values inside model and `--counts` option is not used.
* If you want to remove the last SoftMax layer in the topology, launch the Model Optimizer with the
`--remove_output_softmax` flag:
```sh
mo --input_model wsj_dnn5b_smbr.nnet --counts wsj_dnn5b_smbr.counts --remove_output_softmax --output_dir <OUTPUT_MODEL_DIR>_
mo --input_model wsj_dnn5b_smbr.nnet --counts wsj_dnn5b_smbr.counts --remove_output_softmax
```
The Model Optimizer finds the last layer of the topology and removes this layer only if it is a SoftMax layer.
> **NOTE**: Model Optimizer can remove SoftMax layer only if the topology has one output.
> **NOTE**: For sample inference of Kaldi models, you can use the Inference Engine Speech Recognition sample application. The sample supports models with one output. If your model has several outputs, specify the desired one with the `--output` option.
If you want to convert a model for inference on Intel® Movidius™ Myriad™, use the `--remove_memory` option.
It removes Memory layers from the IR. Instead of it, additional inputs and outputs appear in the IR.
> **NOTE**: For sample inference of Kaldi models, you can use the OpenVINO Speech Recognition sample application. The sample supports models with one output. If your model has several outputs, specify the desired one with the `--output` option.
If you want to convert a model for inference on Intel® Movidius™ Myriad™, use the `--remove_memory` option.
It removes Memory layers from the IR. Instead of it, additional inputs and outputs appear in the IR.
The Model Optimizer outputs the mapping between inputs and outputs. For example:
```sh
[ WARNING ] Add input/output mapped Parameter_0_for_Offset_fastlstm2.r_trunc__2Offset_fastlstm2.r_trunc__2_out -> Result_for_Offset_fastlstm2.r_trunc__2Offset_fastlstm2.r_trunc__2_out
[ WARNING ] Add input/output mapped Parameter_1_for_Offset_fastlstm2.r_trunc__2Offset_fastlstm2.r_trunc__2_out -> Result_for_Offset_fastlstm2.r_trunc__2Offset_fastlstm2.r_trunc__2_out
[ WARNING ] Add input/output mapped Parameter_0_for_iteration_Offset_fastlstm3.c_trunc__3390 -> Result_for_iteration_Offset_fastlstm3.c_trunc__3390
[ WARNING ] Add input/output mapped Parameter_0_for_Offset_fastlstm2.r_trunc__2Offset_fastlstm2.r_trunc__2_out -> Result_for_Offset_fastlstm2.r_trunc__2Offset_fastlstm2.r_trunc__2_out
[ WARNING ] Add input/output mapped Parameter_1_for_Offset_fastlstm2.r_trunc__2Offset_fastlstm2.r_trunc__2_out -> Result_for_Offset_fastlstm2.r_trunc__2Offset_fastlstm2.r_trunc__2_out
[ WARNING ] Add input/output mapped Parameter_0_for_iteration_Offset_fastlstm3.c_trunc__3390 -> Result_for_iteration_Offset_fastlstm3.c_trunc__3390
```
Based on this mapping, link inputs and outputs in your application manually as follows:
1. Initialize inputs from the mapping as zeros in the first frame of an utterance.
2. Copy output blobs from the mapping to the corresponding inputs. For example, data from `Result_for_Offset_fastlstm2.r_trunc__2Offset_fastlstm2.r_trunc__2_out`
must be copied to `Parameter_0_for_Offset_fastlstm2.r_trunc__2Offset_fastlstm2.r_trunc__2_out`.
1. Initialize inputs from the mapping as zeros in the first frame of an utterance.
2. Copy output blobs from the mapping to the corresponding inputs. For example, data from `Result_for_Offset_fastlstm2.r_trunc__2Offset_fastlstm2.r_trunc__2_out`
must be copied to `Parameter_0_for_Offset_fastlstm2.r_trunc__2Offset_fastlstm2.r_trunc__2_out`.
## Supported Kaldi\* Layers
Refer to [Supported Framework Layers ](../Supported_Frameworks_Layers.md) for the list of supported standard layers.
## See Also
[Model Conversion Tutorials](Convert_Model_Tutorials.md)

70
docs/MO_DG/prepare_model/convert_model/Convert_Model_From_MxNet.md
View File

@ -1,73 +1,12 @@
# Converting an MXNet* Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_MxNet}
@sphinxdirective
.. _convert model mxnet:
.. toctree::
:maxdepth: 1
:hidden:
openvino_docs_MO_DG_prepare_model_convert_model_mxnet_specific_Convert_Style_Transfer_From_MXNet
openvino_docs_MO_DG_prepare_model_convert_model_mxnet_specific_Convert_GluonCV_Models
@endsphinxdirective
A summary of the steps for optimizing and deploying a model that was trained with the MXNet\* framework:
1. [Configure the Model Optimizer](../../Deep_Learning_Model_Optimizer_DevGuide.md) for MXNet* (MXNet was used to train your model)
2. [Convert a MXNet model](#ConvertMxNet) to produce an optimized [Intermediate Representation (IR)](../../IR_and_opsets.md) of the model based on the trained network topology, weights, and biases values
3. Test the model in the Intermediate Representation format using the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in the target environment via provided Inference Engine [sample applications](../../../OV_Runtime_UG/Samples_Overview.md)
4. [Integrate](../../../OV_Runtime_UG/Samples_Overview.md) the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in your application to deploy the model in the target environment
## Supported Topologies
> **NOTE**: SSD models from the table require converting to the deploy mode. For details, see the [Conversion Instructions](https://github.com/zhreshold/mxnet-ssd/#convert-model-to-deploy-mode) in the GitHub MXNet-SSD repository.
| Model Name| Model File |
| ------------- |:-------------:|
|VGG-16| [Repo](https://github.com/dmlc/mxnet-model-gallery/tree/master), [Symbol](http://data.mxnet.io/models/imagenet/vgg/vgg16-symbol.json), [Params](http://data.mxnet.io/models/imagenet/vgg/vgg16-0000.params)|
|VGG-19| [Repo](https://github.com/dmlc/mxnet-model-gallery/tree/master), [Symbol](http://data.mxnet.io/models/imagenet/vgg/vgg19-symbol.json), [Params](http://data.mxnet.io/models/imagenet/vgg/vgg19-0000.params)|
|ResNet-152 v1| [Repo](https://github.com/dmlc/mxnet-model-gallery/tree/master), [Symbol](http://data.mxnet.io/models/imagenet/resnet/152-layers/resnet-152-symbol.json), [Params](http://data.mxnet.io/models/imagenet/resnet/152-layers/resnet-152-0000.params)|
|SqueezeNet_v1.1| [Repo](https://github.com/dmlc/mxnet-model-gallery/tree/master), [Symbol](http://data.mxnet.io/models/imagenet/squeezenet/squeezenet_v1.1-symbol.json), [Params](http://data.mxnet.io/models/imagenet/squeezenet/squeezenet_v1.1-0000.params)|
|Inception BN| [Repo](https://github.com/dmlc/mxnet-model-gallery/tree/master), [Symbol](http://data.mxnet.io/models/imagenet/inception-bn/Inception-BN-symbol.json), [Params](http://data.mxnet.io/models/imagenet/inception-bn/Inception-BN-0126.params)|
|CaffeNet| [Repo](https://github.com/dmlc/mxnet-model-gallery/tree/master), [Symbol](http://data.mxnet.io/mxnet/models/imagenet/caffenet/caffenet-symbol.json), [Params](http://data.mxnet.io/models/imagenet/caffenet/caffenet-0000.params)|
|DenseNet-121| [Repo](https://github.com/miraclewkf/DenseNet), [Symbol](https://raw.githubusercontent.com/miraclewkf/DenseNet/master/model/densenet-121-symbol.json), [Params](https://drive.google.com/file/d/0ByXcv9gLjrVcb3NGb1JPa3ZFQUk/view?usp=drive_web)|
|DenseNet-161| [Repo](https://github.com/miraclewkf/DenseNet), [Symbol](https://raw.githubusercontent.com/miraclewkf/DenseNet/master/model/densenet-161-symbol.json), [Params](https://drive.google.com/file/d/0ByXcv9gLjrVcS0FwZ082SEtiUjQ/view)|
|DenseNet-169| [Repo](https://github.com/miraclewkf/DenseNet), [Symbol](https://raw.githubusercontent.com/miraclewkf/DenseNet/master/model/densenet-169-symbol.json), [Params](https://drive.google.com/file/d/0ByXcv9gLjrVcOWZJejlMOWZvZmc/view)|
|DenseNet-201| [Repo](https://github.com/miraclewkf/DenseNet), [Symbol](https://raw.githubusercontent.com/miraclewkf/DenseNet/master/model/densenet-201-symbol.json), [Params](https://drive.google.com/file/d/0ByXcv9gLjrVcUjF4MDBwZ3FQbkU/view)|
|MobileNet| [Repo](https://github.com/KeyKy/mobilenet-mxnet), [Symbol](https://github.com/KeyKy/mobilenet-mxnet/blob/master/mobilenet.py), [Params](https://github.com/KeyKy/mobilenet-mxnet/blob/master/mobilenet-0000.params)|
|SSD-ResNet-50| [Repo](https://github.com/zhreshold/mxnet-ssd), [Symbol + Params](https://github.com/zhreshold/mxnet-ssd/releases/download/v0.6/resnet50_ssd_512_voc0712_trainval.zip)|
|SSD-VGG-16-300| [Repo](https://github.com/zhreshold/mxnet-ssd), [Symbol + Params](https://github.com/zhreshold/mxnet-ssd/releases/download/v0.5-beta/vgg16_ssd_300_voc0712_trainval.zip)|
|SSD-Inception v3| [Repo](https://github.com/zhreshold/mxnet-ssd), [Symbol + Params](https://github.com/zhreshold/mxnet-ssd/releases/download/v0.7-alpha/ssd_inceptionv3_512_voc0712trainval.zip)|
|FCN8 (Semantic Segmentation)| [Repo](https://github.com/apache/incubator-mxnet/tree/master/example/fcn-xs), [Symbol](https://www.dropbox.com/sh/578n5cxej7ofd6m/AAA9SFCBN8R_uL2CnAd3WQ5ia/FCN8s_VGG16-symbol.json?dl=0), [Params](https://www.dropbox.com/sh/578n5cxej7ofd6m/AABHWZHCtA2P6iR6LUflkxb_a/FCN8s_VGG16-0019-cpu.params?dl=0)|
|MTCNN part 1 (Face Detection)| [Repo](https://github.com/pangyupo/mxnet_mtcnn_face_detection), [Symbol](https://github.com/pangyupo/mxnet_mtcnn_face_detection/blob/master/model/det1-symbol.json), [Params](https://github.com/pangyupo/mxnet_mtcnn_face_detection/blob/master/model/det1-0001.params)|
|MTCNN part 2 (Face Detection)| [Repo](https://github.com/pangyupo/mxnet_mtcnn_face_detection), [Symbol](https://github.com/pangyupo/mxnet_mtcnn_face_detection/blob/master/model/det2-symbol.json), [Params](https://github.com/pangyupo/mxnet_mtcnn_face_detection/blob/master/model/det2-0001.params)|
|MTCNN part 3 (Face Detection)| [Repo](https://github.com/pangyupo/mxnet_mtcnn_face_detection), [Symbol](https://github.com/pangyupo/mxnet_mtcnn_face_detection/blob/master/model/det3-symbol.json), [Params](https://github.com/pangyupo/mxnet_mtcnn_face_detection/blob/master/model/det3-0001.params)|
|MTCNN part 4 (Face Detection)| [Repo](https://github.com/pangyupo/mxnet_mtcnn_face_detection), [Symbol](https://github.com/pangyupo/mxnet_mtcnn_face_detection/blob/master/model/det4-symbol.json), [Params](https://github.com/pangyupo/mxnet_mtcnn_face_detection/blob/master/model/det4-0001.params)|
|Lightened_moon| [Repo](https://github.com/tornadomeet/mxnet-face/tree/master/model/lightened_moon), [Symbol](https://github.com/tornadomeet/mxnet-face/blob/master/model/lightened_moon/lightened_moon_fuse-symbol.json), [Params](https://github.com/tornadomeet/mxnet-face/blob/master/model/lightened_moon/lightened_moon_fuse-0082.params)|
|RNN-Transducer| [Repo](https://github.com/HawkAaron/mxnet-transducer) |
|word_lm| [Repo](https://github.com/apache/incubator-mxnet/tree/master/example/rnn/word_lm) |
**Other supported topologies**
* [GluonCV SSD and YOLO-v3 models](https://gluon-cv.mxnet.io/model_zoo/detection.html) can be converted using the following [instructions](mxnet_specific/Convert_GluonCV_Models.md).
* [Style transfer model](https://github.com/zhaw/neural_style) can be converted using the following [instructions](mxnet_specific/Convert_Style_Transfer_From_MXNet.md).
## Convert an MXNet* Model <a name="ConvertMxNet"></a>
To convert an MXNet\* model, run Model Optimizer with a path to the input model `.params` file and to an output directory where you have write permissions:
To convert an MXNet\* model, run Model Optimizer with a path to the input model `.params` file:
```sh
mo --input_model model-file-0000.params --output_dir <OUTPUT_MODEL_DIR>
mo --input_model model-file-0000.params
```
Two groups of parameters are available to convert your model:
* Framework-agnostic parameters are used to convert a model trained with any supported framework. For details, see see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page.
* [MXNet-specific parameters](#mxnet_specific_conversion_params) are used to convert only MXNet models.
### Using MXNet\*-Specific Conversion Parameters <a name="mxnet_specific_conversion_params"></a>
The following list provides the MXNet\*-specific parameters.
@ -101,7 +40,7 @@ MXNet-specific parameters:
Internally, when you run the Model Optimizer, it loads the model, goes through the topology, and tries to find each layer type in a list of known layers. Custom layers are layers that are not included in the list of known layers. If your topology contains any layers that are not in this list of known layers, the Model Optimizer classifies them as custom.
## Supported MXNet\* Layers
Refer to [Supported Framework Layers ](../Supported_Frameworks_Layers.md) for the list of supported standard layers.
Refer to [Supported Framework Layers](../Supported_Frameworks_Layers.md) for the list of supported standard layers.
## Frequently Asked Questions (FAQ)
@ -114,3 +53,6 @@ In this document, you learned:
* Basic information about how the Model Optimizer works with MXNet\* models
* Which MXNet\* models are supported
* How to convert a trained MXNet\* model using the Model Optimizer with both framework-agnostic and MXNet-specific command-line options
## See Also
[Model Conversion Tutorials](Convert_Model_Tutorials.md)

78
docs/MO_DG/prepare_model/convert_model/Convert_Model_From_ONNX.md
View File

@ -1,83 +1,21 @@
# Converting a ONNX* Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_ONNX}
@sphinxdirective
.. _convert model onnx:
.. toctree::
:maxdepth: 1
:hidden:
openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_Faster_RCNN
openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_Mask_RCNN
openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_GPT2
openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_DLRM
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_PyTorch
@endsphinxdirective
## Introduction to ONNX
[ONNX*](https://github.com/onnx/onnx) is a representation format for deep learning models. ONNX allows AI developers easily transfer models between different frameworks that helps to choose the best combination for them. Today, PyTorch\*, Caffe2\*, Apache MXNet\*, Microsoft Cognitive Toolkit\* and other tools are developing ONNX support.
## Supported Public ONNX Topologies
| Model Name | Path to <a href="https://github.com/onnx/models">Public Models</a> master branch|
|:----|:----|
| bert_large | [model archive](https://github.com/mlperf/inference/tree/master/v0.7/language/bert) |
| bvlc_alexnet | [model archive](https://s3.amazonaws.com/download.onnx/models/opset_8/bvlc_alexnet.tar.gz) |
| bvlc_googlenet | [model archive](https://s3.amazonaws.com/download.onnx/models/opset_8/bvlc_googlenet.tar.gz) |
| bvlc_reference_caffenet | [model archive](https://s3.amazonaws.com/download.onnx/models/opset_8/bvlc_reference_caffenet.tar.gz) |
| bvlc_reference_rcnn_ilsvrc13 | [model archive](https://s3.amazonaws.com/download.onnx/models/opset_8/bvlc_reference_rcnn_ilsvrc13.tar.gz) |
| inception_v1 | [model archive](https://s3.amazonaws.com/download.onnx/models/opset_8/inception_v1.tar.gz) |
| inception_v2 | [model archive](https://s3.amazonaws.com/download.onnx/models/opset_8/inception_v2.tar.gz) |
| resnet50 | [model archive](https://s3.amazonaws.com/download.onnx/models/opset_8/resnet50.tar.gz) |
| squeezenet | [model archive](https://s3.amazonaws.com/download.onnx/models/opset_8/squeezenet.tar.gz) |
| densenet121 | [model archive](https://s3.amazonaws.com/download.onnx/models/opset_8/densenet121.tar.gz) |
| emotion_ferplus | [model archive](https://www.cntk.ai/OnnxModels/emotion_ferplus/opset_2/emotion_ferplus.tar.gz) |
| mnist | [model archive](https://www.cntk.ai/OnnxModels/mnist/opset_1/mnist.tar.gz) |
| shufflenet | [model archive](https://s3.amazonaws.com/download.onnx/models/opset_8/shufflenet.tar.gz) |
| VGG19 | [model archive](https://s3.amazonaws.com/download.onnx/models/opset_8/vgg19.tar.gz) |
| zfnet512 | [model archive](https://s3.amazonaws.com/download.onnx/models/opset_8/zfnet512.tar.gz) |
| GPT-2 | [model archive](https://github.com/onnx/models/blob/master/text/machine_comprehension/gpt-2/model/gpt2-10.tar.gz) |
| YOLOv3 | [model archive](https://github.com/onnx/models/blob/master/vision/object_detection_segmentation/yolov3/model/yolov3-10.tar.gz) |
Listed models are built with the operation set version 8 except the GPT-2 model (which uses version 10). Models that are upgraded to higher operation set versions may not be supported.
## Supported PaddlePaddle* Models via ONNX Conversion
Starting from the R5 release, the OpenVINO™ toolkit officially supports public PaddlePaddle* models via ONNX conversion.
The list of supported topologies downloadable from PaddleHub is presented below:
| Model Name | Command to download the model from PaddleHub |
|:----|:----|
| [MobileNetV2](https://www.paddlepaddle.org.cn/hubdetail?name=mobilenet_v2_imagenet) | `hub install mobilenet_v2_imagenet==1.0.1` |
| [ResNet18](https://www.paddlepaddle.org.cn/hubdetail?name=resnet_v2_18_imagenet) | `hub install resnet_v2_18_imagenet==1.0.0` |
| [ResNet34](https://www.paddlepaddle.org.cn/hubdetail?name=resnet_v2_34_imagenet) | `hub install resnet_v2_34_imagenet==1.0.0` |
| [ResNet50](https://www.paddlepaddle.org.cn/hubdetail?name=resnet_v2_50_imagenet) | `hub install resnet_v2_50_imagenet==1.0.1` |
| [ResNet101](https://www.paddlepaddle.org.cn/hubdetail?name=resnet_v2_101_imagenet) | `hub install resnet_v2_101_imagenet==1.0.1` |
| [ResNet152](https://www.paddlepaddle.org.cn/hubdetail?name=resnet_v2_152_imagenet) | `hub install resnet_v2_152_imagenet==1.0.1` |
> **NOTE**: To convert a model downloaded from PaddleHub use [paddle2onnx](https://github.com/PaddlePaddle/paddle2onnx) converter.
The list of supported topologies from the [models v1.5](https://github.com/PaddlePaddle/models/tree/release/1.5) package:
* [MobileNetV1](https://github.com/PaddlePaddle/models/blob/release/1.5/PaddleCV/image_classification/models/mobilenet.py)
* [MobileNetV2](https://github.com/PaddlePaddle/models/blob/release/1.5/PaddleCV/image_classification/models/mobilenet_v2.py)
* [ResNet](https://github.com/PaddlePaddle/models/blob/release/1.5/PaddleCV/image_classification/models/resnet.py)
* [ResNet_vc](https://github.com/PaddlePaddle/models/blob/release/1.5/PaddleCV/image_classification/models/resnet_vc.py)
* [ResNet_vd](https://github.com/PaddlePaddle/models/blob/release/1.5/PaddleCV/image_classification/models/resnet_vd.py)
* [ResNeXt](https://github.com/PaddlePaddle/models/blob/release/1.5/PaddleCV/image_classification/models/resnext.py)
* [ResNeXt_vd](https://github.com/PaddlePaddle/models/blob/release/1.5/PaddleCV/image_classification/models/resnext_vd.py)
> **NOTE**: To convert these topologies one should first serialize the model by calling `paddle.fluid.io.save_inference_model`
([description](https://www.paddlepaddle.org.cn/documentation/docs/en/1.3/api/io.html#save-inference-model)) command and
after that use [paddle2onnx](https://github.com/PaddlePaddle/paddle2onnx) converter.
## Convert an ONNX* Model <a name="Convert_From_ONNX"></a>
The Model Optimizer process assumes you have an ONNX model that was directly downloaded from a public repository or converted from any framework that supports exporting to the ONNX format.
To convert an ONNX\* model, run Model Optimizer with the path to the input model `.nnet` file and an output directory where you have write permissions:
To convert an ONNX\* model, run Model Optimizer with the path to the input model `.onnx` file:
```sh
mo --input_model <INPUT_MODEL>.onnx --output_dir <OUTPUT_MODEL_DIR>
mo --input_model <INPUT_MODEL>.onnx
```
There are no ONNX\* specific parameters, so only framework-agnostic parameters are available to convert your model. For details, see see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page.
There are no ONNX\* specific parameters, so only framework-agnostic parameters are available to convert your model. For details, see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page.
## Supported ONNX\* Layers
Refer to [Supported Framework Layers](../Supported_Frameworks_Layers.md) for the list of supported standard layers.
## See Also
[Model Conversion Tutorials](Convert_Model_Tutorials.md)

55
docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Paddle.md
View File

@ -1,58 +1,25 @@
# Converting a Paddle* Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Paddle}
# Converting a PaddlePaddle* Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Paddle}
A summary of the steps for optimizing and deploying a model trained with Paddle\*:
## Convert a PaddlePaddle Model <a name="Convert_From_Paddle"></a>
To convert a PaddlePaddle model, use the `mo` script and specify the path to the input model `.pdmodel` file:
1. [Configure the Model Optimizer](../../Deep_Learning_Model_Optimizer_DevGuide.md) for Paddle\*.
2. [Convert a Paddle\* Model](#Convert_From_Paddle) to produce an optimized [Intermediate Representation (IR)](../../IR_and_opsets.md) of the model based on the trained network topology, weights, and biases.
3. Test the model in the Intermediate Representation format using the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in the target environment via provided Inference Engine [sample applications](../../../OV_Runtime_UG/Samples_Overview.md).
4. [Integrate](../../../OV_Runtime_UG/Samples_Overview.md) the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in your application to deploy the model in the target environment.
## Supported Topologies
| Model Name| Model Type| Description|
| ------------- | ------------ | ------------- |
|ppocr-det| optical character recognition| Models are exported from [PaddleOCR](https://github.com/PaddlePaddle/PaddleOCR/tree/release/2.1/). Refer to [READ.md](https://github.com/PaddlePaddle/PaddleOCR/tree/release/2.1/#pp-ocr-20-series-model-listupdate-on-dec-15).|
|ppocr-rec| optical character recognition| Models are exported from [PaddleOCR](https://github.com/PaddlePaddle/PaddleOCR/tree/release/2.1/). Refer to [READ.md](https://github.com/PaddlePaddle/PaddleOCR/tree/release/2.1/#pp-ocr-20-series-model-listupdate-on-dec-15).|
|ResNet-50| classification| Models are exported from [PaddleClas](https://github.com/PaddlePaddle/PaddleClas/tree/release/2.1/). Refer to [getting_started_en.md](https://github.com/PaddlePaddle/PaddleClas/blob/release/2.1/docs/en/tutorials/getting_started_en.md#4-use-the-inference-model-to-predict)|
|MobileNet v2| classification| Models are exported from [PaddleClas](https://github.com/PaddlePaddle/PaddleClas/tree/release/2.1/). Refer to [getting_started_en.md](https://github.com/PaddlePaddle/PaddleClas/blob/release/2.1/docs/en/tutorials/getting_started_en.md#4-use-the-inference-model-to-predict)|
|MobileNet v3| classification| Models are exported from [PaddleClas](https://github.com/PaddlePaddle/PaddleClas/tree/release/2.1/). Refer to [getting_started_en.md](https://github.com/PaddlePaddle/PaddleClas/blob/release/2.1/docs/en/tutorials/getting_started_en.md#4-use-the-inference-model-to-predict)|
|BiSeNet v2| semantic segmentation| Models are exported from [PaddleSeg](https://github.com/PaddlePaddle/PaddleSeg/tree/release/2.1). Refer to [model_export.md](https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.1/docs/model_export.md#)|
|DeepLab v3 plus| semantic segmentation| Models are exported from [PaddleSeg](https://github.com/PaddlePaddle/PaddleSeg/tree/release/2.1). Refer to [model_export.md](https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.1/docs/model_export.md#)|
|Fast-SCNN| semantic segmentation| Models are exported from [PaddleSeg](https://github.com/PaddlePaddle/PaddleSeg/tree/release/2.1). Refer to [model_export.md](https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.1/docs/model_export.md#)|
|OCRNET| semantic segmentation| Models are exported from [PaddleSeg](https://github.com/PaddlePaddle/PaddleSeg/tree/release/2.1). Refer to [model_export.md](https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.1/docs/model_export.md#)|
|Yolo v3| detection| Models are exported from [PaddleDetection](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.1). Refer to [EXPORT_MODEL.md](https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.1/deploy/EXPORT_MODEL.md#).|
|ppyolo| detection| Models are exported from [PaddleDetection](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.1). Refer to [EXPORT_MODEL.md](https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.1/deploy/EXPORT_MODEL.md#).|
|MobileNetv3-SSD| detection| Models are exported from [PaddleDetection](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.2). Refer to [EXPORT_MODEL.md](https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.2/deploy/EXPORT_MODEL.md#).|
|U-Net| semantic segmentation| Models are exported from [PaddleSeg](https://github.com/PaddlePaddle/PaddleSeg/tree/release/2.3). Refer to [model_export.md](https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.3/docs/model_export.md#)|
|BERT| language representation| Models are exported from [PaddleNLP](https://github.com/PaddlePaddle/PaddleNLP/tree/v2.1.1). Refer to [README.md](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/language_model/bert#readme)|
|ernie| language representation| Models are exported from [PaddleNLP](https://github.com/PaddlePaddle/PaddleNLP/tree/v2.1.1). Refer to [README.md](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/language_model/bert#readme)|
> **NOTE:** The verified models are exported from the repository of branch release/2.1.
## Convert a Paddle* Model <a name="Convert_From_Paddle"></a>
To convert a Paddle\* model:
1. Activate environment with installed OpenVINO if needed
2. Use the `mo` script to simply convert a model, specifying the framework, the path to the input model `.pdmodel` file and the path to an output directory with write permissions:
```sh
mo --input_model <INPUT_MODEL>.pdmodel --output_dir <OUTPUT_MODEL_DIR> --framework=paddle
mo --input_model <INPUT_MODEL>.pdmodel
```
Parameters to convert your model:
### Example of Converting a PaddlePaddle Model
Below is the example command to convert yolo v3 PaddlePaddle network to OpenVINO IR network with Model Optimizer.
* [Framework-agnostic parameters](Converting_Model.md): These parameters are used to convert a model trained with any supported framework.
> **NOTE:** `--scale`, `--scale_values`, `--mean_values` are not supported in the current version of mo_paddle.
### Example of Converting a Paddle* Model
Below is the example command to convert yolo v3 Paddle\* network to OpenVINO IR network with Model Optimizer.
```sh
mo --model_name yolov3_darknet53_270e_coco --output_dir <OUTPUT_MODEL_DIR> --framework=paddle --data_type=FP32 --reverse_input_channels --input_shape=[1,3,608,608],[1,2],[1,2] --input=image,im_shape,scale_factor --output=save_infer_model/scale_0.tmp_1,save_infer_model/scale_1.tmp_1 --input_model=yolov3.pdmodel
mo --input_model=yolov3.pdmodel --input=image,im_shape,scale_factor --input_shape=[1,3,608,608],[1,2],[1,2] --reverse_input_channels --output=save_infer_model/scale_0.tmp_1,save_infer_model/scale_1.tmp_1
```
## Supported Paddle\* Layers
## Supported PaddlePaddle Layers
Refer to [Supported Framework Layers](../Supported_Frameworks_Layers.md) for the list of supported standard layers.
## Frequently Asked Questions (FAQ)
When Model Optimizer is unable to run to completion due to issues like typographical errors, incorrectly used options, etc., it provides explanatory messages. They describe the potential cause of the problem and give a link to the [Model Optimizer FAQ](../Model_Optimizer_FAQ.md), which provides instructions on how to resolve most issues. The FAQ also includes links to relevant sections in the Model Optimizer Developer Guide to help you understand what went wrong.
## See Also
[Model Conversion Tutorials](Convert_Model_Tutorials.md)

60
docs/MO_DG/prepare_model/convert_model/Convert_Model_From_PyTorch.md
View File

@ -1,64 +1,19 @@
# Converting a PyTorch* Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_PyTorch}
@sphinxdirective
.. toctree::
:maxdepth: 1
:hidden:
openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_F3Net
openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_QuartzNet
openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_RNNT
openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_YOLACT
openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_Bert_ner
openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_RCAN
@endsphinxdirective
## Supported Topologies
Here is the list of models that are tested and guaranteed to be supported. However, you can also use these instructions to convert PyTorch\* models that are not presented in the list.
* [Torchvision Models](https://pytorch.org/docs/stable/torchvision/index.html): alexnet, densenet121, densenet161,
densenet169, densenet201, resnet101, resnet152, resnet18, resnet34, resnet50, vgg11, vgg13, vgg16, vgg19.
The models can be converted using [regular instructions](#typical-pytorch).
* [Cadene Pretrained Models](https://github.com/Cadene/pretrained-models.pytorch): alexnet, fbresnet152, resnet101,
resnet152, resnet18, resnet34, resnet152, resnet18, resnet34, resnet50, resnext101_32x4d, resnext101_64x4d, vgg11.
The models can be converted using [regular instructions](#typical-pytorch).
* [ESPNet Models](https://github.com/sacmehta/ESPNet/tree/master/pretrained) can be converted using [regular instructions](#typical-pytorch).
* [MobileNetV3](https://github.com/d-li14/mobilenetv3.pytorch) can be converted using [regular instructions](#typical-pytorch).
* [iSeeBetter](https://github.com/amanchadha/iSeeBetter) can be converted using [regular instructions](#typical-pytorch).
Please refer to [`iSeeBetterTest.py`](https://github.com/amanchadha/iSeeBetter/blob/master/iSeeBetterTest.py) script for code to initialize the model.
* F3Net topology can be converted using steps described in [Convert PyTorch\* F3Net to the IR](pytorch_specific/Convert_F3Net.md)
instruction which is used instead of steps 2 and 3 of [regular instructions](#typical-pytorch).
* QuartzNet topologies from [NeMo project](https://github.com/NVIDIA/NeMo) can be converted using steps described in
[Convert PyTorch\* QuartzNet to the IR](pytorch_specific/Convert_QuartzNet.md) instruction which is used instead of
steps 2 and 3 of [regular instructions](#typical-pytorch).
* YOLACT topology can be converted using steps described in [Convert PyTorch\* YOLACT to the IR](pytorch_specific/Convert_YOLACT.md)
instruction which is used instead of steps 2 and 3 of [regular instructions](#typical-pytorch).
* [RCAN](https://github.com/yulunzhang/RCAN) topology can be converted using steps described in [Convert PyTorch\* RCAN to the IR](pytorch_specific/Convert_RCAN.md)
instruction which is used instead of steps 2 and 3 of [regular instructions](#typical-pytorch).
* [BERT_NER](https://github.com/kamalkraj/BERT-NER) topology can be converted using steps described in [Convert PyTorch* BERT-NER to the IR](pytorch_specific/Convert_Bert_ner.md)
instruction which is used instead of steps 2 and 3 of [regular instructions](#typical-pytorch).
## Typical steps to convert PyTorch\* model <a name="typical-pytorch"></a>
## Typical Steps to Convert PyTorch Model <a name="typical-pytorch"></a>
PyTorch* framework is supported through export to ONNX\* format. A summary of the steps for optimizing and deploying a model that was trained with the PyTorch\* framework:
1. [Configure the Model Optimizer](../../Deep_Learning_Model_Optimizer_DevGuide.md) for ONNX\*.
2. [Export PyTorch model to ONNX\*](#export-to-onnx).
3. [Convert an ONNX\* model](Convert_Model_From_ONNX.md) to produce an optimized [Intermediate Representation (IR)](../../IR_and_opsets.md) of the model based on the trained network topology, weights, and biases values.
4. Test the model in the Intermediate Representation format using the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in the target environment via provided [sample applications](../../../OV_Runtime_UG/Samples_Overview.md).
5. [Integrate](../../../OV_Runtime_UG/Samples_Overview.md) the Inference Engine in your application to deploy the model in the target environment.
1. [Export PyTorch model to ONNX\*](#export-to-onnx).
2. [Convert an ONNX\* model](Convert_Model_From_ONNX.md) to produce an optimized [Intermediate Representation (IR)](../../IR_and_opsets.md) of the model based on the trained network topology, weights, and biases values.
## Export PyTorch\* Model to ONNX\* Format <a name="export-to-onnx"></a>
PyTorch models are defined in a Python\* code, to export such models use `torch.onnx.export()` method. Usually code to
evaluate or test the model is provided with the model code and can be used to initialize and export model.
Only the basics will be covered here, the step to export to ONNX\* is crucial but it is covered by PyTorch\* framework.
For more information, please refer to [PyTorch\* documentation](https://pytorch.org/docs/stable/onnx.html).
For more information, please refer to [Exporting PyTorch models to ONNX format](https://pytorch.org/docs/stable/onnx.html).
To export a PyTorch\* model you need to obtain the model as an instance of `torch.nn.Module` class and call the `export` function.
```python
import torch
@ -66,7 +21,7 @@ import torch
model = SomeModel()
# Evaluate the model to switch some operations from training mode to inference.
model.eval()
# Create dummy input for the model. It will be used to run the model inside export function.
# Create dummy input for the model. It will be used to run the model inside export function.
dummy_input = torch.randn(1, 3, 224, 224)
# Call the export function
torch.onnx.export(model, (dummy_input, ), 'model.onnx')
@ -77,3 +32,6 @@ torch.onnx.export(model, (dummy_input, ), 'model.onnx')
* Not all PyTorch\* operations can be exported to ONNX\* opset 9 which is used by default, as of version 1.8.1.
It is recommended to export models to opset 11 or higher when export to default opset 9 is not working. In that case, use `opset_version`
option of the `torch.onnx.export`. For more information about ONNX* opset, refer to the [Operator Schemas](https://github.com/onnx/onnx/blob/master/docs/Operators.md).
## See Also
[Model Conversion Tutorials](Convert_Model_Tutorials.md)

415
docs/MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md
View File

@ -1,247 +1,40 @@
# Converting a TensorFlow* Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow}
@sphinxdirective
## Convert TensorFlow 1 Models <a name="Convert_From_TF2X"></a>
.. _convert model tf:
### Convert Frozen Model Format <a name="Convert_From_TF"></a>
To convert a TensorFlow model, use the `mo` script to simply convert a model with the path to the input model `.pb` file:
.. toctree::
:maxdepth: 1
:hidden:
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_RetinaNet_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_AttentionOCR_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_YOLO_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_FaceNet_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_NCF_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_DeepSpeech_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_lm_1b_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Object_Detection_API_Models
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Slim_Library_Models
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_CRNN_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_GNMT_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_BERT_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_XLNet_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_WideAndDeep_Family_Models
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_EfficientDet_Models
@endsphinxdirective
A summary of the steps for optimizing and deploying a model that was trained with the TensorFlow\* framework:
1. [Configure the Model Optimizer](../../Deep_Learning_Model_Optimizer_DevGuide.md) for TensorFlow\* (TensorFlow was used to train your model).
2. [Freeze the TensorFlow model](#freeze-the-tensorflow-model) if your model is not already frozen or skip this step and use the [instruction](#loading-nonfrozen-models) to a convert a non-frozen model.
3. [Convert a TensorFlow\* model](#Convert_From_TF) to produce an optimized [Intermediate Representation (IR)](../../IR_and_opsets.md) of the model based on the trained network topology, weights, and biases values.
4. Test the model in the Intermediate Representation format using the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in the target environment via provided [sample applications](../../../OV_Runtime_UG/Samples_Overview.md).
5. [Integrate](../../../OV_Runtime_UG/Samples_Overview.md) the Inference Engine in your application to deploy the model in the target environment.
## Supported Topologies
**Supported Non-Frozen Topologies with Links to the Associated Slim Model Classification Download Files**
Detailed information on how to convert models from the <a href="https://github.com/tensorflow/models/tree/master/research/slim/README.md">TensorFlow\*-Slim Image Classification Model Library</a> is available in the [Converting TensorFlow*-Slim Image Classification Model Library Models](tf_specific/Convert_Slim_Library_Models.md) chapter. The table below contains list of supported TensorFlow\*-Slim Image Classification Model Library models and required mean/scale values. The mean values are specified as if the input image is read in BGR channels order layout like Inference Engine classification sample does.
| Model Name| Slim Model Checkpoint File| \-\-mean_values | \-\-scale|
| ------------- | ------------ | ------------- | -----:|
|Inception v1| [inception_v1_2016_08_28.tar.gz](http://download.tensorflow.org/models/inception_v1_2016_08_28.tar.gz)| [127.5,127.5,127.5]| 127.5|
|Inception v2| [inception_v1_2016_08_28.tar.gz](http://download.tensorflow.org/models/inception_v1_2016_08_28.tar.gz)| [127.5,127.5,127.5]| 127.5|
|Inception v3| [inception_v3_2016_08_28.tar.gz](http://download.tensorflow.org/models/inception_v3_2016_08_28.tar.gz)| [127.5,127.5,127.5]| 127.5|
|Inception V4| [inception_v4_2016_09_09.tar.gz](http://download.tensorflow.org/models/inception_v4_2016_09_09.tar.gz)| [127.5,127.5,127.5]| 127.5|
|Inception ResNet v2| [inception_resnet_v2_2016_08_30.tar.gz](http://download.tensorflow.org/models/inception_resnet_v2_2016_08_30.tar.gz)| [127.5,127.5,127.5]| 127.5|
|MobileNet v1 128| [mobilenet_v1_0.25_128.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_0.25_128.tgz)| [127.5,127.5,127.5]| 127.5|
|MobileNet v1 160| [mobilenet_v1_0.5_160.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_0.5_160.tgz)| [127.5,127.5,127.5]| 127.5|
|MobileNet v1 224| [mobilenet_v1_1.0_224.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_1.0_224.tgz)| [127.5,127.5,127.5]| 127.5|
|NasNet Large| [nasnet-a_large_04_10_2017.tar.gz](https://storage.googleapis.com/download.tensorflow.org/models/nasnet-a_large_04_10_2017.tar.gz)| [127.5,127.5,127.5]| 127.5|
|NasNet Mobile| [nasnet-a_mobile_04_10_2017.tar.gz](https://storage.googleapis.com/download.tensorflow.org/models/nasnet-a_mobile_04_10_2017.tar.gz)| [127.5,127.5,127.5]| 127.5|
|ResidualNet-50 v1| [resnet_v1_50_2016_08_28.tar.gz](http://download.tensorflow.org/models/resnet_v1_50_2016_08_28.tar.gz)| [103.94,116.78,123.68] | 1 |
|ResidualNet-50 v2| [resnet_v2_50_2017_04_14.tar.gz](http://download.tensorflow.org/models/resnet_v2_50_2017_04_14.tar.gz)| [103.94,116.78,123.68] | 1 |
|ResidualNet-101 v1| [resnet_v1_101_2016_08_28.tar.gz](http://download.tensorflow.org/models/resnet_v1_101_2016_08_28.tar.gz)| [103.94,116.78,123.68] | 1 |
|ResidualNet-101 v2| [resnet_v2_101_2017_04_14.tar.gz](http://download.tensorflow.org/models/resnet_v2_101_2017_04_14.tar.gz)| [103.94,116.78,123.68] | 1 |
|ResidualNet-152 v1| [resnet_v1_152_2016_08_28.tar.gz](http://download.tensorflow.org/models/resnet_v1_152_2016_08_28.tar.gz)| [103.94,116.78,123.68] | 1 |
|ResidualNet-152 v2| [resnet_v2_152_2017_04_14.tar.gz](http://download.tensorflow.org/models/resnet_v2_152_2017_04_14.tar.gz)| [103.94,116.78,123.68] | 1 |
|VGG-16| [vgg_16_2016_08_28.tar.gz](http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz)| [103.94,116.78,123.68] | 1 |
|VGG-19| [vgg_19_2016_08_28.tar.gz](http://download.tensorflow.org/models/vgg_19_2016_08_28.tar.gz)| [103.94,116.78,123.68] | 1 |
**Supported Pre-Trained Topologies from TensorFlow 1 Detection Model Zoo**
Detailed information on how to convert models from the <a href="https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf1_detection_zoo.md">TensorFlow 1 Detection Model Zoo</a> is available in the [Converting TensorFlow Object Detection API Models](tf_specific/Convert_Object_Detection_API_Models.md) chapter. The table below contains models from the Object Detection Models zoo that are supported.
| Model Name| TensorFlow 1 Object Detection API Models|
| :------------- | -----:|
|SSD MobileNet V1 COCO\*| [ssd_mobilenet_v1_coco_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/ssd_mobilenet_v1_coco_2018_01_28.tar.gz)|
|SSD MobileNet V1 0.75 Depth COCO| [ssd_mobilenet_v1_0.75_depth_300x300_coco14_sync_2018_07_03.tar.gz](http://download.tensorflow.org/models/object_detection/ssd_mobilenet_v1_0.75_depth_300x300_coco14_sync_2018_07_03.tar.gz)|
|SSD MobileNet V1 PPN COCO| [ssd_mobilenet_v1_ppn_shared_box_predictor_300x300_coco14_sync_2018_07_03.tar.gz](http://download.tensorflow.org/models/object_detection/ssd_mobilenet_v1_ppn_shared_box_predictor_300x300_coco14_sync_2018_07_03.tar.gz)|
|SSD MobileNet V1 FPN COCO| [ssd_mobilenet_v1_fpn_shared_box_predictor_640x640_coco14_sync_2018_07_03.tar.gz](http://download.tensorflow.org/models/object_detection/ssd_mobilenet_v1_fpn_shared_box_predictor_640x640_coco14_sync_2018_07_03.tar.gz)|
|SSD ResNet50 FPN COCO| [ssd_resnet50_v1_fpn_shared_box_predictor_640x640_coco14_sync_2018_07_03.tar.gz](http://download.tensorflow.org/models/object_detection/ssd_resnet50_v1_fpn_shared_box_predictor_640x640_coco14_sync_2018_07_03.tar.gz)|
|SSD MobileNet V2 COCO| [ssd_mobilenet_v2_coco_2018_03_29.tar.gz](http://download.tensorflow.org/models/object_detection/ssd_mobilenet_v2_coco_2018_03_29.tar.gz)|
|SSD Lite MobileNet V2 COCO| [ssdlite_mobilenet_v2_coco_2018_05_09.tar.gz](http://download.tensorflow.org/models/object_detection/ssdlite_mobilenet_v2_coco_2018_05_09.tar.gz)|
|SSD Inception V2 COCO| [ssd_inception_v2_coco_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/ssd_inception_v2_coco_2018_01_28.tar.gz)|
|RFCN ResNet 101 COCO| [rfcn_resnet101_coco_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/rfcn_resnet101_coco_2018_01_28.tar.gz)|
|Faster R-CNN Inception V2 COCO| [faster_rcnn_inception_v2_coco_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_v2_coco_2018_01_28.tar.gz)|
|Faster R-CNN ResNet 50 COCO| [faster_rcnn_resnet50_coco_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/faster_rcnn_resnet50_coco_2018_01_28.tar.gz)|
|Faster R-CNN ResNet 50 Low Proposals COCO| [faster_rcnn_resnet50_lowproposals_coco_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/faster_rcnn_resnet50_lowproposals_coco_2018_01_28.tar.gz)|
|Faster R-CNN ResNet 101 COCO| [faster_rcnn_resnet101_coco_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/faster_rcnn_resnet101_coco_2018_01_28.tar.gz)|
|Faster R-CNN ResNet 101 Low Proposals COCO| [faster_rcnn_resnet101_lowproposals_coco_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/faster_rcnn_resnet101_lowproposals_coco_2018_01_28.tar.gz)|
|Faster R-CNN Inception ResNet V2 COCO| [faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz)|
|Faster R-CNN Inception ResNet V2 Low Proposals COCO| [faster_rcnn_inception_resnet_v2_atrous_lowproposals_coco_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_lowproposals_coco_2018_01_28.tar.gz)|
|Faster R-CNN NasNet COCO| [faster_rcnn_nas_coco_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/faster_rcnn_nas_coco_2018_01_28.tar.gz)|
|Faster R-CNN NasNet Low Proposals COCO| [faster_rcnn_nas_lowproposals_coco_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/faster_rcnn_nas_lowproposals_coco_2018_01_28.tar.gz)|
|Mask R-CNN Inception ResNet V2 COCO| [mask_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/mask_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz)|
|Mask R-CNN Inception V2 COCO| [mask_rcnn_inception_v2_coco_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/mask_rcnn_inception_v2_coco_2018_01_28.tar.gz)|
|Mask R-CNN ResNet 101 COCO| [mask_rcnn_resnet101_atrous_coco_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/mask_rcnn_resnet101_atrous_coco_2018_01_28.tar.gz)|
|Mask R-CNN ResNet 50 COCO| [mask_rcnn_resnet50_atrous_coco_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/mask_rcnn_resnet50_atrous_coco_2018_01_28.tar.gz)|
|Faster R-CNN ResNet 101 Kitti\*| [faster_rcnn_resnet101_kitti_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/faster_rcnn_resnet101_kitti_2018_01_28.tar.gz)|
|Faster R-CNN Inception ResNet V2 Open Images\*| [faster_rcnn_inception_resnet_v2_atrous_oid_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_oid_2018_01_28.tar.gz)|
|Faster R-CNN Inception ResNet V2 Low Proposals Open Images\*| [faster_rcnn_inception_resnet_v2_atrous_lowproposals_oid_2018_01_28.tar.gz](http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_lowproposals_oid_2018_01_28.tar.gz)|
|Faster R-CNN ResNet 101 AVA v2.1\*| [faster_rcnn_resnet101_ava_v2.1_2018_04_30.tar.gz](http://download.tensorflow.org/models/object_detection/faster_rcnn_resnet101_ava_v2.1_2018_04_30.tar.gz)|
**Supported Pre-Trained Topologies from TensorFlow 2 Detection Model Zoo**
Detailed information on how to convert models from the <a href="https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md">TensorFlow 2 Detection Model Zoo</a> is available in the [Converting TensorFlow Object Detection API Models](tf_specific/Convert_Object_Detection_API_Models.md) chapter. The table below contains models from the Object Detection Models zoo that are supported.
| Model Name| TensorFlow 2 Object Detection API Models|
| :------------- | -----:|
| EfficientDet D0 512x512 | [efficientdet_d0_coco17_tpu-32.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/efficientdet_d0_coco17_tpu-32.tar.gz)|
| EfficientDet D1 640x640 | [efficientdet_d1_coco17_tpu-32.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/efficientdet_d1_coco17_tpu-32.tar.gz)|
| EfficientDet D2 768x768 | [efficientdet_d2_coco17_tpu-32.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/efficientdet_d2_coco17_tpu-32.tar.gz)|
| EfficientDet D3 896x896 | [efficientdet_d3_coco17_tpu-32.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/efficientdet_d3_coco17_tpu-32.tar.gz)|
| EfficientDet D4 1024x1024 | [efficientdet_d4_coco17_tpu-32.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/efficientdet_d4_coco17_tpu-32.tar.gz)|
| EfficientDet D5 1280x1280 | [efficientdet_d5_coco17_tpu-32.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/efficientdet_d5_coco17_tpu-32.tar.gz)|
| EfficientDet D6 1280x1280 | [efficientdet_d6_coco17_tpu-32.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/efficientdet_d6_coco17_tpu-32.tar.gz)|
| EfficientDet D7 1536x1536 | [efficientdet_d7_coco17_tpu-32.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/efficientdet_d7_coco17_tpu-32.tar.gz)|
| SSD MobileNet v2 320x320 | [ssd_mobilenet_v2_320x320_coco17_tpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/ssd_mobilenet_v2_320x320_coco17_tpu-8.tar.gz)|
| SSD MobileNet V1 FPN 640x640 | [ssd_mobilenet_v1_fpn_640x640_coco17_tpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/ssd_mobilenet_v1_fpn_640x640_coco17_tpu-8.tar.gz)|
| SSD MobileNet V2 FPNLite 320x320 | [ssd_mobilenet_v2_fpnlite_320x320_coco17_tpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/ssd_mobilenet_v2_fpnlite_320x320_coco17_tpu-8.tar.gz)|
| SSD MobileNet V2 FPNLite 640x640 | [ssd_mobilenet_v2_fpnlite_640x640_coco17_tpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/ssd_mobilenet_v2_fpnlite_640x640_coco17_tpu-8.tar.gz)|
| SSD ResNet50 V1 FPN 640x640 (RetinaNet50) | [ssd_resnet50_v1_fpn_640x640_coco17_tpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/ssd_resnet50_v1_fpn_640x640_coco17_tpu-8.tar.gz)|
| SSD ResNet50 V1 FPN 1024x1024 (RetinaNet50) | [ssd_resnet50_v1_fpn_1024x1024_coco17_tpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/ssd_resnet50_v1_fpn_1024x1024_coco17_tpu-8.tar.gz)|
| SSD ResNet101 V1 FPN 640x640 (RetinaNet101) | [ssd_resnet101_v1_fpn_640x640_coco17_tpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/ssd_resnet101_v1_fpn_640x640_coco17_tpu-8.tar.gz)|
| SSD ResNet101 V1 FPN 1024x1024 (RetinaNet101) | [ssd_resnet101_v1_fpn_1024x1024_coco17_tpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/ssd_resnet101_v1_fpn_1024x1024_coco17_tpu-8.tar.gz)|
| SSD ResNet152 V1 FPN 640x640 (RetinaNet152) | [ssd_resnet152_v1_fpn_640x640_coco17_tpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/ssd_resnet152_v1_fpn_640x640_coco17_tpu-8.tar.gz)|
| SSD ResNet152 V1 FPN 1024x1024 (RetinaNet152) | [ssd_resnet152_v1_fpn_1024x1024_coco17_tpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/ssd_resnet152_v1_fpn_1024x1024_coco17_tpu-8.tar.gz)|
| Faster R-CNN ResNet50 V1 640x640 | [faster_rcnn_resnet50_v1_640x640_coco17_tpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/faster_rcnn_resnet50_v1_640x640_coco17_tpu-8.tar.gz)|
| Faster R-CNN ResNet50 V1 1024x1024 | [faster_rcnn_resnet50_v1_1024x1024_coco17_tpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/faster_rcnn_resnet50_v1_1024x1024_coco17_tpu-8.tar.gz)|
| Faster R-CNN ResNet50 V1 800x1333 | [faster_rcnn_resnet50_v1_800x1333_coco17_gpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/faster_rcnn_resnet50_v1_800x1333_coco17_gpu-8.tar.gz)|
| Faster R-CNN ResNet101 V1 640x640 | [faster_rcnn_resnet101_v1_640x640_coco17_tpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/faster_rcnn_resnet101_v1_640x640_coco17_tpu-8.tar.gz)|
| Faster R-CNN ResNet101 V1 1024x1024 | [faster_rcnn_resnet101_v1_1024x1024_coco17_tpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/faster_rcnn_resnet101_v1_1024x1024_coco17_tpu-8.tar.gz)|
| Faster R-CNN ResNet101 V1 800x1333 | [faster_rcnn_resnet101_v1_800x1333_coco17_gpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/faster_rcnn_resnet101_v1_800x1333_coco17_gpu-8.tar.gz)|
| Faster R-CNN ResNet152 V1 640x640 | [faster_rcnn_resnet152_v1_640x640_coco17_tpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/faster_rcnn_resnet152_v1_640x640_coco17_tpu-8.tar.gz)|
| Faster R-CNN ResNet152 V1 1024x1024 | [faster_rcnn_resnet152_v1_1024x1024_coco17_tpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/faster_rcnn_resnet152_v1_1024x1024_coco17_tpu-8.tar.gz)|
| Faster R-CNN ResNet152 V1 800x1333 | [faster_rcnn_resnet152_v1_800x1333_coco17_gpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/faster_rcnn_resnet152_v1_800x1333_coco17_gpu-8.tar.gz)|
| Faster R-CNN Inception ResNet V2 640x640 | [faster_rcnn_inception_resnet_v2_640x640_coco17_tpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/faster_rcnn_inception_resnet_v2_640x640_coco17_tpu-8.tar.gz)|
| Faster R-CNN Inception ResNet V2 1024x1024 | [faster_rcnn_inception_resnet_v2_1024x1024_coco17_tpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/faster_rcnn_inception_resnet_v2_1024x1024_coco17_tpu-8.tar.gz)|
| Mask R-CNN Inception ResNet V2 1024x1024 | [mask_rcnn_inception_resnet_v2_1024x1024_coco17_gpu-8.tar.gz](http://download.tensorflow.org/models/object_detection/tf2/20200711/mask_rcnn_inception_resnet_v2_1024x1024_coco17_gpu-8.tar.gz)|
**Supported Frozen Quantized Topologies**
The topologies hosted on the TensorFlow\* Lite [site](https://www.tensorflow.org/lite/guide/hosted_models). The frozen model file (`.pb` file) should be fed to the Model Optimizer.
| Model Name | Frozen Model File |
|:----------------------|---------------------------------------------------------------------------------------------------------------------------------:|
| Mobilenet V1 0.25 128 | [mobilenet_v1_0.25_128_quant.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.25_128_quant.tgz) |
| Mobilenet V1 0.25 160 | [mobilenet_v1_0.25_160_quant.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.25_160_quant.tgz) |
| Mobilenet V1 0.25 192 | [mobilenet_v1_0.25_192_quant.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.25_192_quant.tgz) |
| Mobilenet V1 0.25 224 | [mobilenet_v1_0.25_224_quant.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.25_224_quant.tgz) |
| Mobilenet V1 0.50 128 | [mobilenet_v1_0.5_128_quant.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.5_128_quant.tgz) |
| Mobilenet V1 0.50 160 | [mobilenet_v1_0.5_160_quant.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.5_160_quant.tgz) |
| Mobilenet V1 0.50 192 | [mobilenet_v1_0.5_192_quant.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.5_192_quant.tgz) |
| Mobilenet V1 0.50 224 | [mobilenet_v1_0.5_224_quant.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.5_224_quant.tgz) |
| Mobilenet V1 0.75 128 | [mobilenet_v1_0.75_128_quant.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.75_128_quant.tgz) |
| Mobilenet V1 0.75 160 | [mobilenet_v1_0.75_160_quant.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.75_160_quant.tgz) |
| Mobilenet V1 0.75 192 | [mobilenet_v1_0.75_192_quant.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.75_192_quant.tgz) |
| Mobilenet V1 0.75 224 | [mobilenet_v1_0.75_224_quant.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_0.75_224_quant.tgz) |
| Mobilenet V1 1.0 128 | [mobilenet_v1_1.0_128_quant.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_128_quant.tgz) |
| Mobilenet V1 1.0 160 | [mobilenet_v1_1.0_160_quant.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_160_quant.tgz) |
| Mobilenet V1 1.0 192 | [mobilenet_v1_1.0_192_quant.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_192_quant.tgz) |
| Mobilenet V1 1.0 224 | [mobilenet_v1_1.0_224_quant.tgz](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224_quant.tgz) |
| Mobilenet V2 1.0 224 | [mobilenet_v2_1.0_224_quant.tgz](http://download.tensorflow.org/models/tflite_11_05_08/mobilenet_v2_1.0_224_quant.tgz) |
| Inception V1 | [inception_v1_224_quant_20181026.tgz](http://download.tensorflow.org/models/inception_v1_224_quant_20181026.tgz) |
| Inception V2 | [inception_v2_224_quant_20181026.tgz](http://download.tensorflow.org/models/inception_v2_224_quant_20181026.tgz) |
| Inception V3 | [inception_v3_quant.tgz](http://download.tensorflow.org/models/tflite_11_05_08/inception_v3_quant.tgz) |
| Inception V4 | [inception_v4_299_quant_20181026.tgz](http://download.tensorflow.org/models/inception_v4_299_quant_20181026.tgz) |
It is necessary to specify the following command line parameters for the Model Optimizer to convert some of the models from the list above: `--input input --input_shape [1,HEIGHT,WIDTH,3]`.
Where `HEIGHT` and `WIDTH` are the input images height and width for which the model was trained.
**Other supported topologies**
| Model Name| Repository |
| :------------- | -----:|
| ResNext | [Repo](https://github.com/taki0112/ResNeXt-Tensorflow)|
| DenseNet | [Repo](https://github.com/taki0112/Densenet-Tensorflow)|
| CRNN | [Repo](https://github.com/MaybeShewill-CV/CRNN_Tensorflow) |
| NCF | [Repo](https://github.com/tensorflow/models/tree/master/official/recommendation) |
| lm_1b | [Repo](https://github.com/tensorflow/models/tree/master/research/lm_1b) |
| DeepSpeech | [Repo](https://github.com/mozilla/DeepSpeech) |
| A3C | [Repo](https://github.com/miyosuda/async_deep_reinforce) |
| VDCNN | [Repo](https://github.com/WenchenLi/VDCNN) |
| Unet | [Repo](https://github.com/kkweon/UNet-in-Tensorflow) |
| Keras-TCN | [Repo](https://github.com/philipperemy/keras-tcn) |
| PRNet | [Repo](https://github.com/YadiraF/PRNet) |
| YOLOv4 | [Repo](https://github.com/Ma-Dan/keras-yolo4) |
| STN | [Repo](https://github.com/oarriaga/STN.keras) |
* YOLO topologies from DarkNet* can be converted using [these instructions](tf_specific/Convert_YOLO_From_Tensorflow.md).
* FaceNet topologies can be converted using [these instructions](tf_specific/Convert_FaceNet_From_Tensorflow.md).
* CRNN topologies can be converted using [these instructions](tf_specific/Convert_CRNN_From_Tensorflow.md).
* NCF topologies can be converted using [these instructions](tf_specific/Convert_NCF_From_Tensorflow.md).
* [GNMT](https://github.com/tensorflow/nmt) topology can be converted using [these instructions](tf_specific/Convert_GNMT_From_Tensorflow.md).
* [BERT](https://github.com/google-research/bert) topology can be converted using [these instructions](tf_specific/Convert_BERT_From_Tensorflow.md).
* [XLNet](https://github.com/zihangdai/xlnet) topology can be converted using [these instructions](tf_specific/Convert_XLNet_From_Tensorflow.md).
* [Attention OCR](https://github.com/emedvedev/attention-ocr) topology can be converted using [these instructions](tf_specific/Convert_AttentionOCR_From_Tensorflow.md).
## Loading Non-Frozen Models to the Model Optimizer <a name="loading-nonfrozen-models"></a>
There are three ways to store non-frozen TensorFlow models and load them to the Model Optimizer:
1. Checkpoint:
In this case, a model consists of two files:
- `inference_graph.pb` or `inference_graph.pbtxt`
- `checkpoint_file.ckpt`
If you do not have an inference graph file, refer to [Freezing Custom Models in Python](#freeze-the-tensorflow-model).
To convert such a TensorFlow model:
1. Go to the `<INSTALL_DIR>/tools/model_optimizer` directory
2. Run the `mo` script with the path to the checkpoint file to convert a model and an output directory where you have write permissions:
* If input model is in `.pb` format:<br>
```sh
mo --input_model <INFERENCE_GRAPH>.pb --input_checkpoint <INPUT_CHECKPOINT> --output_dir <OUTPUT_MODEL_DIR>
```
* If input model is in `.pbtxt` format:<br>
```sh
mo --input_model <INFERENCE_GRAPH>.pbtxt --input_checkpoint <INPUT_CHECKPOINT> --input_model_is_text --output_dir <OUTPUT_MODEL_DIR>
mo --input_model <INPUT_MODEL>.pb
```
2. MetaGraph:
### Convert Non-Frozen Model Formats <a name="loading-nonfrozen-models"></a>
There are three ways to store non-frozen TensorFlow models and convert them by Model Optimizer:
In this case, a model consists of three or four files stored in the same directory:
- `model_name.meta`
- `model_name.index`
- `model_name.data-00000-of-00001` (digit part may vary)
- `checkpoint` (optional)
To convert such TensorFlow model:
1. Go to the `<INSTALL_DIR>/tools/model_optimizer` directory
2. Run the `mo` script with a path to the MetaGraph `.meta` file and a writable output directory to convert a model:<br>
1. **Checkpoint**. In this case, a model consists of two files: `inference_graph.pb` (or `inference_graph.pbtxt`) and `checkpoint_file.ckpt`.
If you do not have an inference graph file, refer to [Freezing Custom Models in Python](#freeze-the-tensorflow-model).
To convert the model with the inference graph in `.pb` format, run the `mo` script with the path to the checkpoint file to convert a model:
```sh
mo --input_meta_graph <INPUT_META_GRAPH>.meta --output_dir <OUTPUT_MODEL_DIR>
mo --input_model <INFERENCE_GRAPH>.pb --input_checkpoint <INPUT_CHECKPOINT>
```
To convert the model with the inference graph in `.pbtxt` format, run the `mo` script with the path to the checkpoint file to convert a model:
```sh
mo --input_model <INFERENCE_GRAPH>.pbtxt --input_checkpoint <INPUT_CHECKPOINT> --input_model_is_text
```
3. SavedModel format of TensorFlow 1.x and 2.x versions:
In this case, a model consists of a special directory with a `.pb` file and several subfolders: `variables`, `assets`, and `assets.extra`. For more information about the SavedModel directory, refer to the [README](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/python/saved_model#components) file in the TensorFlow repository.
To convert such TensorFlow model:
1. Go to the `<INSTALL_DIR>/tools/model_optimizer` directory
2. Run the `mo` script with a path to the SavedModel directory and a writable output directory to convert a model:<br>
2. **MetaGraph**. In this case, a model consists of three or four files stored in the same directory: `model_name.meta`, `model_name.index`,
`model_name.data-00000-of-00001` (digit part may vary), and `checkpoint` (optional).
To convert such TensorFlow model, run the `mo` script with a path to the MetaGraph `.meta` file:
```sh
mo --saved_model_dir <SAVED_MODEL_DIRECTORY> --output_dir <OUTPUT_MODEL_DIR>
mo --input_meta_graph <INPUT_META_GRAPH>.meta
```
3. **SavedModel format**. In this case, a model consists of a special directory with a `.pb` file
and several subfolders: `variables`, `assets`, and `assets.extra`. For more information about the SavedModel directory, refer to the [README](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/python/saved_model#components) file in the TensorFlow repository.
To convert such TensorFlow model, run the `mo` script with a path to the SavedModel directory:
```sh
mo --saved_model_dir <SAVED_MODEL_DIRECTORY>
```
You can convert TensorFlow 1.x SavedModel format in the environment that has a 1.x or 2.x version of TensorFlow. However, TensorFlow 2.x SavedModel format strictly requires the 2.x version of TensorFlow.
@ -249,12 +42,12 @@ If a model contains operations currently unsupported by OpenVINO, prune these op
To determine custom input nodes, display a graph of the model in TensorBoard. To generate TensorBoard logs of the graph, use the `--tensorboard_logs` option.
TensorFlow 2.x SavedModel format has a specific graph due to eager execution. In case of pruning, find custom input nodes in the `StatefulPartitionedCall/*` subgraph of TensorFlow 2.x SavedModel format.
## Freezing Custom Models in Python\* <a name="freeze-the-tensorflow-model"></a>
### Freezing Custom Models in Python\* <a name="freeze-the-tensorflow-model"></a>
When a network is defined in Python\* code, you have to create an inference graph file. Usually graphs are built in a form
that allows model training. That means that all trainable parameters are represented as variables in the graph.
To be able to use such graph with Model Optimizer such graph should be frozen.
The graph is frozen and dumped to a file with the following code:
```python
import tensorflow as tf
from tensorflow.python.framework import graph_io
@ -273,126 +66,36 @@ Where:
* `inference_graph.pb` is the name of the generated inference graph file.
* `as_text` specifies whether the generated file should be in human readable text format or binary.
## Convert a TensorFlow* Model <a name="Convert_From_TF"></a>
To convert a TensorFlow model:
1. Go to the `<INSTALL_DIR>/tools/model_optimizer` directory
2. Use the `mo` script to simply convert a model with the path to the input model `.pb` file and a writable output directory:
```sh
mo --input_model <INPUT_MODEL>.pb --output_dir <OUTPUT_MODEL_DIR>
```
Two groups of parameters are available to convert your model:
* Framework-agnostic parameters are used to convert a model trained with any supported framework. For details, see see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page.
* [TensorFlow-specific parameters](#tensorflow_specific_conversion_params): Parameters used to convert only TensorFlow models.
> **NOTE**: The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the `RGB<->BGR` conversion specifying the command-line parameter: `--reverse_input_channels`. Otherwise, inference results may be incorrect. For more information about the parameter, refer to **When to Reverse Input Channels** section of [Converting a Model to Intermediate Representation (IR)](Converting_Model.md).
### Using TensorFlow\*-Specific Conversion Parameters <a name="tensorflow_specific_conversion_params"></a>
The following list provides the TensorFlow\*-specific parameters.
```
TensorFlow*-specific parameters:
--input_model_is_text
TensorFlow*: treat the input model file as a text
protobuf format. If not specified, the Model Optimizer
treats it as a binary file by default.
--input_checkpoint INPUT_CHECKPOINT
TensorFlow*: variables file to load.
--input_meta_graph INPUT_META_GRAPH
Tensorflow*: a file with a meta-graph of the model
before freezing
--saved_model_dir SAVED_MODEL_DIR
TensorFlow*: directory with a model in SavedModel format
of TensorFlow 1.x or 2.x version
--saved_model_tags SAVED_MODEL_TAGS
Group of tag(s) of the MetaGraphDef to load, in string
format, separated by ','. For tag-set contains
multiple tags, all tags must be passed in.
--tensorflow_custom_operations_config_update TENSORFLOW_CUSTOM_OPERATIONS_CONFIG_UPDATE
TensorFlow*: update the configuration file with node
name patterns with input/output nodes information.
--tensorflow_object_detection_api_pipeline_config TENSORFLOW_OBJECT_DETECTION_API_PIPELINE_CONFIG
TensorFlow*: path to the pipeline configuration file
used to generate model created with help of Object
Detection API.
--tensorboard_logdir TENSORBOARD_LOGDIR
TensorFlow*: dump the input graph to a given directory
that should be used with TensorBoard.
--tensorflow_custom_layer_libraries TENSORFLOW_CUSTOM_LAYER_LIBRARIES
TensorFlow*: comma separated list of shared libraries
with TensorFlow* custom operations implementation.
--disable_nhwc_to_nchw
[DEPRECATED] Disables default translation from NHWC to NCHW. Since 2022.1
this option is deprecated and used only to maintain backward compatibility
with previous releases.
```
> **NOTE**: Models produces with TensorFlow\* usually have not fully defined shapes (contain `-1` in some dimensions). It is necessary to pass explicit shape for the input using command line parameter `--input_shape` or `-b` to override just batch dimension. If the shape is fully defined, then there is no need to specify either `-b` or `--input_shape` options.
#### Command-Line Interface (CLI) Examples Using TensorFlow\*-Specific Parameters
* Launching the Model Optimizer for Inception V1 frozen model when model file is a plain text protobuf, specifying a writable output directory:
```sh
mo --input_model inception_v1.pbtxt --input_model_is_text -b 1 --output_dir <OUTPUT_MODEL_DIR>
```
* Launching the Model Optimizer for Inception V1 frozen model and update custom sub-graph replacement file `transform.json` with information about input and output nodes of the matched sub-graph, specifying a writable output directory. For more information about this feature, refer to [Sub-Graph Replacement in the Model Optimizer](../customize_model_optimizer/Subgraph_Replacement_Model_Optimizer.md).
```sh
mo --input_model inception_v1.pb -b 1 --tensorflow_custom_operations_config_update transform.json --output_dir <OUTPUT_MODEL_DIR>
```
* Launching the Model Optimizer for Inception V1 frozen model and use custom sub-graph replacement file `transform.json` for model conversion. For more information about this feature, refer to [Sub-Graph Replacement in the Model Optimizer](../customize_model_optimizer/Subgraph_Replacement_Model_Optimizer.md).
```sh
mo --input_model inception_v1.pb -b 1 --transformations_config transform.json --output_dir <OUTPUT_MODEL_DIR>
```
* Launching the Model Optimizer for Inception V1 frozen model and dump information about the graph to TensorBoard log dir `/tmp/log_dir`
```sh
mo --input_model inception_v1.pb -b 1 --tensorboard_logdir /tmp/log_dir --output_dir <OUTPUT_MODEL_DIR>
```
* Launching the Model Optimizer for a model with custom TensorFlow operations (refer to the [TensorFlow* documentation](https://www.tensorflow.org/extend/adding_an_op)) implemented in C++ and compiled into the shared library `my_custom_op.so`. Model Optimizer falls back to TensorFlow to infer output shape of operations implemented in the library if a custom TensorFlow operation library is provided. If it is not provided, a custom operation with an inference function is needed. For more information about custom operations, refer to the [Custom Layers Guide](../../../HOWTO/Custom_Layers_Guide.md).
```sh
mo --input_model custom_model.pb --tensorflow_custom_layer_libraries ./my_custom_op.so --output_dir <OUTPUT_MODEL_DIR>
```
## Convert TensorFlow* 2 Models <a name="Convert_From_TF2X"></a>
In order to convert TensorFlow* 2 models, installation of dependencies from `requirements_tf.txt` is required.
TensorFlow* 2.X officially supports two model formats: SavedModel and Keras H5 (or HDF5).
## Convert TensorFlow 2 Models <a name="Convert_From_TF2X"></a>
To convert TensorFlow* 2 models, ensure that `openvino-dev[tensorflow2]` is installed via `pip`.
TensorFlow* 2.X officially supports two model formats: SavedModel and Keras H5 (or HDF5).
Below are the instructions on how to convert each of them.
### SavedModel Format
### SavedModel Format
A model in the SavedModel format consists of a directory with a `saved_model.pb` file and two subfolders: `variables` and `assets`.
To convert such a model, run the `mo` script with a path to the SavedModel directory:
A model in the SavedModel format consists of a directory with a `saved_model.pb` file and two subfolders: `variables` and `assets`.
To convert such a model:
1. Go to the `<INSTALL_DIR>/tools/model_optimizer` directory.
2. Run the `mo` script with a path to the SavedModel directory and a writable output directory:
```sh
mo --saved_model_dir <SAVED_MODEL_DIRECTORY> --output_dir <OUTPUT_MODEL_DIR>
mo --saved_model_dir <SAVED_MODEL_DIRECTORY>
```
TensorFlow* 2 SavedModel format strictly requires the 2.x version of TensorFlow installed in the
environment for conversion to the Intermediate Representation (IR).
environment for conversion to the Intermediate Representation (IR).
If a model contains operations currently unsupported by OpenVINO™,
prune these operations by explicit specification of input nodes using the `--input` or `--output`
options. To determine custom input nodes, visualize a model graph in the TensorBoard.
options. To determine custom input nodes, visualize a model graph in the TensorBoard.
To generate TensorBoard logs of the graph, use the Model Optimizer `--tensorboard_logs` command-line
option.
option.
TensorFlow* 2 SavedModel format has a specific graph structure due to eager execution. In case of
pruning, find custom input nodes in the `StatefulPartitionedCall/*` subgraph.
### Keras H5
### Keras H5
If you have a model in the HDF5 format, load the model using TensorFlow* 2 and serialize it in the
SavedModel format. Here is an example of how to do it:
```python
import tensorflow as tf
model = tf.keras.models.load_model('model.h5')
@ -401,6 +104,7 @@ tf.saved_model.save(model,'model')
The Keras H5 model with a custom layer has specifics to be converted into SavedModel format.
For example, the model with a custom layer `CustomLayer` from `custom_layer.py` is converted as follows:
```python
import tensorflow as tf
from custom_layer import CustomLayer
@ -412,42 +116,39 @@ Then follow the above instructions for the SavedModel format.
> **NOTE**: Do not use other hacks to resave TensorFlow* 2 models into TensorFlow* 1 formats.
## Command-Line Interface (CLI) Examples Using TensorFlow\*-Specific Parameters
* Launching the Model Optimizer for Inception V1 frozen model when model file is a plain text protobuf:
## Custom Layer Definition
```sh
mo --input_model inception_v1.pbtxt --input_model_is_text -b 1
```
Internally, when you run the Model Optimizer, it loads the model, goes through the topology, and tries to find each layer type in a list of known layers. Custom layers are layers that are not included in the list of known layers. If your topology contains any layers that are not in this list of known layers, the Model Optimizer classifies them as custom.
* Launching the Model Optimizer for Inception V1 frozen model and dump information about the graph to TensorBoard log dir `/tmp/log_dir`
See [Custom Layers in the Model Optimizer](../customize_model_optimizer/Customize_Model_Optimizer.md) for information about:
```sh
mo --input_model inception_v1.pb -b 1 --tensorboard_logdir /tmp/log_dir
```
* Model Optimizer internal procedure for working with custom layers
* How to convert a TensorFlow model that has custom layers
* Custom layer implementation details
* Launching the Model Optimizer for BERT model in the SavedModel format, with three inputs. Explicitly specify input shapes
where the batch size and the sequence length equal 2 and 30 respectively.
```sh
mo --saved_model_dir BERT --input mask,word_ids,type_ids --input_shape [2,30],[2,30],[2,30]
```
## Supported TensorFlow\* and TensorFlow 2 Keras\* Layers
Refer to [Supported Framework Layers ](../Supported_Frameworks_Layers.md) for the list of supported standard layers.
## Frequently Asked Questions (FAQ)
The Model Optimizer provides explanatory messages if it is unable to run to completion due to issues like typographical errors, incorrectly used options, or other issues. The message describes the potential cause of the problem and gives a link to the [Model Optimizer FAQ](../Model_Optimizer_FAQ.md). The FAQ has instructions on how to resolve most issues. The FAQ also includes links to relevant sections in the Model Optimizer Developer Guide to help you understand what went wrong.
## Video: Converting a TensorFlow Model
@sphinxdirective
.. raw:: html
<iframe allowfullscreen mozallowfullscreen msallowfullscreen oallowfullscreen webkitallowfullscreen width="560" height="315"
src="https://www.youtube.com/embed/QW6532LtiTc">
</iframe>
@endsphinxdirective
## Summary
In this document, you learned:
* Basic information about how the Model Optimizer works with TensorFlow\* models
* Basic information about how the Model Optimizer works with TensorFlow models
* Which TensorFlow models are supported
* How to freeze a TensorFlow model
* How to convert a trained TensorFlow model using the Model Optimizer with both framework-agnostic and TensorFlow-specific command-line options
## See Also
[Model Conversion Tutorials](Convert_Model_Tutorials.md)

44
docs/MO_DG/prepare_model/convert_model/Convert_Model_Tutorials.md Normal file
View File

@ -0,0 +1,44 @@
# Model Conversion Tutorials {#openvino_docs_MO_DG_prepare_model_convert_model_tutorials}
@sphinxdirective
.. toctree::
:maxdepth: 1
:hidden:
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_AttentionOCR_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_BERT_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_CRNN_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_DeepSpeech_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_EfficientDet_Models
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_FaceNet_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_GNMT_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_lm_1b_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_NCF_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Object_Detection_API_Models
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_RetinaNet_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Slim_Library_Models
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_WideAndDeep_Family_Models
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_XLNet_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_YOLO_From_Tensorflow
openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_Faster_RCNN
openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_GPT2
openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_Mask_RCNN
openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_Bert_ner
openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_Cascade_RCNN_res101
openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_F3Net
openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_QuartzNet
openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_RCAN
openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_RNNT
openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_YOLACT
openvino_docs_MO_DG_prepare_model_convert_model_mxnet_specific_Convert_GluonCV_Models
openvino_docs_MO_DG_prepare_model_convert_model_mxnet_specific_Convert_Style_Transfer_From_MXNet
openvino_docs_MO_DG_prepare_model_convert_model_kaldi_specific_Aspire_Tdnn_Model
@endsphinxdirective
This section provides you with a set of tutorials that demonstrate conversion steps for specific TensorFlow, ONNX, PyTorch, MXNet, and Kaldi models.
It contains conversion recipes for concrete models, that unnecessarily cover your case.
Try to convert the model out-of-the-box, meaning only the `--input_model` parameter is specified in the command line, before studying the tutorials.
You can also find a collection of [Python tutorials](../../../tutorials.md) written for running on Jupyter* notebooks that provide an introduction to the OpenVINO™ toolkit and explain how to use the Python API and tools for optimized deep learning inference.

314
docs/MO_DG/prepare_model/convert_model/Converting_Model.md
View File

@ -1,297 +1,75 @@
# Converting a Model to Intermediate Representation (IR) {#openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model}
# Setting Input Shapes {#openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model}
@sphinxdirective
Model Optimizer provides the option of making models more efficient by providing additional shape definition.
It is achieved with two parameters: `--input_shape` and `--static_shape`, used under certain conditions.
.. toctree::
:maxdepth: 1
:hidden:
## When to Specify --input_shape Command-line Parameter <a name="when_to_specify_input_shapes"></a>
Model Optimizer supports conversion of models with dynamic input shapes that contain undefined dimensions.
However, if the shape of data is not going to change from one inference to another,
it is recommended to set up static shapes (when all dimensions are fully defined) for the inputs.
It can be beneficial from a performance perspective and memory consumption.
To set up static shapes, Model Optimizer provides the `--input_shape` parameter.
The same functionality is also available in runtime via `reshape` method, please refer to [Changing input shapes](../../../OV_Runtime_UG/ShapeInference.md).
For more information about dynamic shapes in runtime, refer to [Dynamic Shapes](../../../OV_Runtime_UG/ov_dynamic_shapes.md)
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Caffe
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_MxNet
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Kaldi
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_ONNX
openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Paddle
openvino_docs_MO_DG_prepare_model_Model_Optimization_Techniques
openvino_docs_MO_DG_prepare_model_convert_model_Cutting_Model
openvino_docs_MO_DG_prepare_model_Supported_Frameworks_Layers
openvino_docs_MO_DG_prepare_model_convert_model_IR_suitable_for_INT8_inference
openvino_docs_MO_DG_prepare_model_customize_model_optimizer_Subgraph_Replacement_Model_Optimizer
openvino_docs_MO_DG_prepare_model_convert_model_Legacy_IR_Layers_Catalog_Spec
OpenVINO Runtime API can have limitations to infer models with undefined dimensions on some hardware.
In this case, the `--input_shape` parameter and the `reshape` method can help to resolve undefined dimensions.
@endsphinxdirective
Sometimes Model Optimizer is unable to convert models out-of-the-box (only the `--input_model` parameter is specified).
Such problem can relate to models with inputs of undefined ranks and a case of cutting off parts of a model.
In this case, user has to specify input shapes explicitly using `--input_shape` parameter.
To convert the model to the Intermediate Representation (IR), run Model Optimizer using the following command:
For example, run the Model Optimizer for the TensorFlow* MobileNet model with the single input
and specify input shape `[2,300,300,3]`.
```sh
mo --input_model INPUT_MODEL --output_dir <OUTPUT_MODEL_DIR>
mo --input_model MobileNet.pb --input_shape [2,300,300,3]
```
The output directory must have write permissions, so you can run Model Optimizer from the output directory or specify an output path with the `--output_dir` option.
> **NOTE**: The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the `RGB<->BGR` conversion specifying the command-line parameter: `--reverse_input_channels`. Otherwise, inference results may be incorrect. For details, refer to [When to Reverse Input Channels](#when_to_reverse_input_channels).
To adjust the conversion process, you may use general parameters defined in the [General Conversion Parameters](#general_conversion_parameters) and
Framework-specific parameters for:
* [Caffe](Convert_Model_From_Caffe.md)
* [TensorFlow](Convert_Model_From_TensorFlow.md)
* [MXNet](Convert_Model_From_MxNet.md)
* [ONNX](Convert_Model_From_ONNX.md)
* [Kaldi](Convert_Model_From_Kaldi.md)
## General Conversion Parameters
To adjust the conversion process, you can also use the general (framework-agnostic) parameters:
If a model has multiple inputs, `--input_shape` must be used in conjunction with `--input` parameter.
The parameter `--input` contains a list of input names for which shapes in the same order are defined via `--input_shape`.
For example, launch the Model Optimizer for the ONNX* OCR model with a pair of inputs `data` and `seq_len`
and specify shapes `[3,150,200,1]` and `[3]` for them.
```sh
optional arguments:
-h, --help show this help message and exit
--framework {tf,caffe,mxnet,kaldi,onnx}
Name of the framework used to train the input model.
Framework-agnostic parameters:
--input_model INPUT_MODEL, -w INPUT_MODEL, -m INPUT_MODEL
Tensorflow*: a file with a pre-trained model (binary
or text .pb file after freezing). Caffe*: a model
proto file with model weights
--model_name MODEL_NAME, -n MODEL_NAME
Model_name parameter passed to the final create_ir
transform. This parameter is used to name a network in
a generated IR and output .xml/.bin files.
--output_dir OUTPUT_DIR, -o OUTPUT_DIR
Directory that stores the generated IR. By default, it
is the directory from where the Model Optimizer is
launched.
--input_shape INPUT_SHAPE
Input shape(s) that should be fed to an input node(s)
of the model. Shape is defined as a comma-separated
list of integer numbers enclosed in parentheses or
square brackets, for example [1,3,227,227] or
(1,227,227,3), where the order of dimensions depends
on the framework input layout of the model. For
example, [N,C,H,W] is used for Caffe* models and
[N,H,W,C] for TensorFlow* models. Model Optimizer
performs necessary transformations to convert the
shape to the layout required by Inference Engine
(N,C,H,W). The shape should not contain undefined
dimensions (? or -1) and should fit the dimensions
defined in the input operation of the graph. Boundaries
of undefined dimension can be specified with ellipsis,
for example [1,1..10,128,128]. One boundary can be undefined,
for example [1,..100] or [1,3,1..,1..]. If there
are multiple inputs in the model, --input_shape should
contain definition of shape for each input separated
by a comma, for example: [1,3,227,227],[2,4] for a
model with two inputs with 4D and 2D shapes.
Alternatively, specify shapes with the --input
option.
--scale SCALE, -s SCALE
All input values coming from original network inputs
will be divided by this value. When a list of inputs
is overridden by the --input parameter, this scale is
not applied for any input that does not match with the
original input of the model.
If both --mean and --scale are specified,
the mean is subtracted first and then scale is applied
regardless of the order of options in command line.
--reverse_input_channels
Switch the input channels order from RGB to BGR (or
vice versa). Applied to original inputs of the model
if and only if a number of channels equals 3.
When --mean_values/--scale_values are also specified,
reversing of channels will be applied to user's input
data first, so that numbers in --mean_values and
--scale_values go in the order of channels used in
the original model. In other words, if both options are
specified then the data flow in the model looks as following:
Parameter -> ReverseInputChannels -> Mean/Scale apply -> the original body of the model.
--log_level {CRITICAL,ERROR,WARN,WARNING,INFO,DEBUG,NOTSET}
Logger level
--input INPUT Quoted list of comma-separated input nodes names with shapes,
data types, and values for freezing. The order of inputs in converted
model is the same as order of specified operation names. The shape and value are
specified as space-separated lists. The data type of input
node is specified in braces and can have one of the values:
f64 (float64), f32 (float32), f16 (float16), i64 (int64),
i32 (int32), u8 (uint8), boolean (bool). Data type is optional.
If it's not specified explicitly then there are two options:
if input node is a parameter, data type is taken from the
original node dtype, if input node is not a parameter, data type
is set to f32. Example, to set `input_1` with shape [1 100],
and Parameter node `sequence_len` with scalar input with value `150`,
and boolean input `is_training` with `False` value use the
following format: "input_1[1 10],sequence_len->150,is_training->False".
Another example, use the following format to set input port 0
of the node `node_name1` with the shape [3 4] as an input node
and freeze output port 1 of the node `node_name2` with the
value [20 15] of the int32 type and shape [2]:
"0:node_name1[3 4],node_name2:1[2]{i32}->[20 15]".
--output OUTPUT The name of the output operation of the model. For
TensorFlow*, do not add :0 to this name.
The order of outputs in converted model is the same as order of
specified operation names.
--mean_values MEAN_VALUES, -ms MEAN_VALUES
Mean values to be used for the input image per
channel. Values to be provided in the (R,G,B) or
[R,G,B] format. Can be defined for desired input of
the model, for example: "--mean_values
data[255,255,255],info[255,255,255]". The exact
meaning and order of channels depend on how the
original model was trained.
--scale_values SCALE_VALUES
Scale values to be used for the input image per
channel. Values are provided in the (R,G,B) or [R,G,B]
format. Can be defined for desired input of the model,
for example: "--scale_values
data[255,255,255],info[255,255,255]". The exact
meaning and order of channels depend on how the
original model was trained.
If both --mean_values and --scale_values are specified,
the mean is subtracted first and then scale is applied
regardless of the order of options in command line.
--data_type {FP16,FP32,half,float}
Data type for all intermediate tensors and weights. If
original model is in FP32 and --data_type=FP16 is
specified, all model weights and biases are compressed
to FP16.
--disable_fusing Turn off fusing of linear operations to Convolution
--disable_resnet_optimization
Turn off resnet optimization
--finegrain_fusing FINEGRAIN_FUSING
Regex for layers/operations that won't be fused.
Example: --finegrain_fusing Convolution1,.*Scale.*
--disable_gfusing Turn off fusing of grouped convolutions
--enable_concat_optimization
Turn on Concat optimization.
--extensions EXTENSIONS
Directory or a comma separated list of directories
with extensions. To disable all extensions including
those that are placed at the default location, pass an
empty string.
--batch BATCH, -b BATCH
Input batch size
--version Version of Model Optimizer
--silent Prevent any output messages except those that
correspond to log level equals ERROR, that can be set
with the following option: --log_level. By default,
log level is already ERROR.
--freeze_placeholder_with_value FREEZE_PLACEHOLDER_WITH_VALUE
Replaces input layer with constant node with provided
value, for example: "node_name->True". It will be
DEPRECATED in future releases. Use --input option to
specify a value for freezing.
--static_shape Enables IR generation for fixed input shape (folding
`ShapeOf` operations and shape-calculating sub-graphs
to `Constant`). Changing model input shape using
the Inference Engine API in runtime may fail for such an IR.
--disable_weights_compression
Disable compression and store weights with original
precision.
--progress Enable model conversion progress display.
--stream_output Switch model conversion progress display to a
multiline mode.
--transformations_config TRANSFORMATIONS_CONFIG
Use the configuration file with transformations
description.
--use_new_frontend Force the usage of new frontend API for model processing.
--use_legacy_frontend Force the usage of legacy API for model processing.
mo --input_model ocr.onnx --input data,seq_len --input_shape [3,150,200,1],[3]
```
The sections below provide details on using particular parameters and examples of CLI commands.
The alternative way to specify input shapes is to use the `--input` parameter as follows:
## When to Specify Mean and Scale Values
Usually neural network models are trained with the normalized input data. This means that the input data values are converted to be in a specific range, for example, `[0, 1]` or `[-1, 1]`. Sometimes the mean values (mean images) are subtracted from the input data values as part of the pre-processing. There are two cases how the input data pre-processing is implemented.
* The input pre-processing operations are a part of a topology. In this case, the application that uses the framework to infer the topology does not pre-process the input.
* The input pre-processing operations are not a part of a topology and the pre-processing is performed within the application which feeds the model with an input data.
In the first case, the Model Optimizer generates the IR with required pre-processing layers and Inference Engine samples may be used to infer the model.
In the second case, information about mean/scale values should be provided to the Model Optimizer to embed it to the generated IR. Model Optimizer provides a number of command line parameters to specify them: `--mean`, `--scale`, `--scale_values`, `--mean_values`.
> **NOTE:** If both mean and scale values are specified, the mean is subtracted first and then scale is applied regardless of the order of options in command line. Input values are *divided* by the scale value(s). If also `--reverse_input_channels` option is used, the reverse_input_channels will be applied first, then mean and after that scale.
There is no a universal recipe for determining the mean/scale values for a particular model. The steps below could help to determine them:
* Read the model documentation. Usually the documentation describes mean/scale value if the pre-processing is required.
* Open the example script/application executing the model and track how the input data is read and passed to the framework.
* Open the model in a visualization tool and check for layers performing subtraction or multiplication (like `Sub`, `Mul`, `ScaleShift`, `Eltwise` etc) of the input data. If such layers exist, pre-processing is probably part of the model.
## When to Specify Input Shapes <a name="when_to_specify_input_shapes"></a>
There are situations when the input data shape for the model is not fixed, like for the fully-convolutional neural networks. In this case, for example, TensorFlow\* models contain `-1` values in the `shape` attribute of the `Placeholder` operation. Inference Engine does not support input layers with undefined size, so if the input shapes are not defined in the model, the Model Optimizer fails to convert the model. The solution is to provide the input shape(s) using the `--input` or `--input_shape` command line parameter for all input(s) of the model or provide the batch size using the `-b` command line parameter if the model contains just one input with undefined batch size only. In the latter case, the `Placeholder` shape for the TensorFlow\* model looks like this `[-1, 224, 224, 3]`.
## When to Reverse Input Channels <a name="when_to_reverse_input_channels"></a>
Input data for your application can be of RGB or BRG color input order. For example, Inference Engine samples load input images in the BGR channels order. However, the model may be trained on images loaded with the opposite order (for example, most TensorFlow\* models are trained with images in RGB order). In this case, inference results using the Inference Engine samples may be incorrect. The solution is to provide `--reverse_input_channels` command line parameter. Taking this parameter, the Model Optimizer performs first convolution or other channel dependent operation weights modification so these operations output will be like the image is passed with RGB channels order.
## When to Specify `--static_shape` Command Line Parameter
If the `--static_shape` command line parameter is specified the Model Optimizer evaluates shapes of all operations in the model (shape propagation) for a fixed input(s) shape(s). During the shape propagation the Model Optimizer evaluates operations *Shape* and removes them from the computation graph. With that approach, the initial model which can consume inputs of different shapes may be converted to IR working with the input of one fixed shape only. For example, consider the case when some blob is reshaped from 4D of a shape *[N, C, H, W]* to a shape *[N, C, H \* W]*. During the model conversion the Model Optimize calculates output shape as a constant 1D blob with values *[N, C, H \* W]*. So if the input shape changes to some other value *[N,C,H1,W1]* (it is possible scenario for a fully convolutional model) then the reshape layer becomes invalid.
Resulting Intermediate Representation will not be resizable with the help of Inference Engine.
## Examples of CLI Commands
Launch the Model Optimizer for the Caffe bvlc_alexnet model with debug log level:
```sh
mo --input_model bvlc_alexnet.caffemodel --log_level DEBUG --output_dir <OUTPUT_MODEL_DIR>
mo --input_model ocr.onnx --input data[3 150 200 1],seq_len[3]
```
Launch the Model Optimizer for the Caffe bvlc_alexnet model with the output IR called `result.*` in the specified `output_dir`:
The parameter `--input_shape` allows overriding original input shapes to the shapes compatible with a given model.
Dynamic shapes, i.e. with dynamic dimensions, in the original model can be replaced with static shapes for the converted model, and vice versa.
The dynamic dimension can be marked in Model Optimizer command-line as `-1` or `?`.
For example, launch the Model Optimizer for the ONNX* OCR model and specify dynamic batch dimension for inputs.
```sh
mo --input_model bvlc_alexnet.caffemodel --model_name result --output_dir /../../models/
mo --input_model ocr.onnx --input data,seq_len --input_shape [-1,150,200,1],[-1]
```
Launch the Model Optimizer for the Caffe bvlc_alexnet model with one input with scale values:
To optimize memory consumption for models with undefined dimensions in run-time, Model Optimizer provides the capability to define boundaries of dimensions.
The boundaries of undefined dimension can be specified with ellipsis.
For example, launch the Model Optimizer for the ONNX* OCR model and specify a boundary for the batch dimension.
```sh
mo --input_model bvlc_alexnet.caffemodel --scale_values [59,59,59] --output_dir <OUTPUT_MODEL_DIR>
mo --input_model ocr.onnx --input data,seq_len --input_shape [1..3,150,200,1],[1..3]
```
Launch the Model Optimizer for the Caffe bvlc_alexnet model with multiple inputs with scale values:
```sh
mo --input_model bvlc_alexnet.caffemodel --input data,rois --scale_values [59,59,59],[5,5,5] --output_dir <OUTPUT_MODEL_DIR>
```
## When to Specify --static_shape Command-line Parameter
Model Optimizer provides the `--static_shape` parameter that allows evaluating shapes of all operations in the model for fixed input shapes
and to fold shape computing sub-graphs into constants. The resulting IR can be more compact in size and the loading time for such IR can be decreased.
However, the resulting IR will not be reshape-able with the help of the `reshape` method from OpenVINO Runtime API.
It is worth noting that the `--input_shape` parameter does not affect reshape-ability of the model.
Launch the Model Optimizer for the Caffe bvlc_alexnet model with multiple inputs with scale and mean values specified for the particular nodes:
```sh
mo --input_model bvlc_alexnet.caffemodel --input data,rois --mean_values data[59,59,59] --scale_values rois[5,5,5] --output_dir <OUTPUT_MODEL_DIR>
```
For example, launch the Model Optimizer for the ONNX* OCR model using `--static_shape`.
Launch the Model Optimizer for the Caffe bvlc_alexnet model with specified input layer, overridden input shape, scale 5, batch 8 and specified name of an output operation:
```sh
mo --input_model bvlc_alexnet.caffemodel --input "data[1 3 224 224]" --output pool5 -s 5 -b 8 --output_dir <OUTPUT_MODEL_DIR>
mo --input_model ocr.onnx --input data[3 150 200 1],seq_len[3] --static_shape
```
Launch the Model Optimizer for the Caffe bvlc_alexnet model with disabled fusing for linear operations to Convolution and grouped convolutions:
```sh
mo --input_model bvlc_alexnet.caffemodel --disable_fusing --disable_gfusing --output_dir <OUTPUT_MODEL_DIR>
```
Launch the Model Optimizer for the Caffe bvlc_alexnet model with reversed input channels order between RGB and BGR, specified mean values to be used for the input image per channel and specified data type for input tensor values:
```sh
mo --input_model bvlc_alexnet.caffemodel --reverse_input_channels --mean_values [255,255,255] --data_type FP16 --output_dir <OUTPUT_MODEL_DIR>
```
Launch the Model Optimizer for the Caffe bvlc_alexnet model with extensions listed in specified directories, specified mean_images binaryproto
file. For more information about extensions, please refer to the [Custom Layers Guide](../../../HOWTO/Custom_Layers_Guide.md).
```sh
mo --input_model bvlc_alexnet.caffemodel --extensions /home/,/some/other/path/ --mean_file /path/to/binaryproto --output_dir <OUTPUT_MODEL_DIR>
```
Launch the Model Optimizer for TensorFlow* FaceNet* model with a placeholder freezing value.
It replaces the placeholder with a constant layer that contains the passed value.
For more information about FaceNet conversion, please refer to [this](tf_specific/Convert_FaceNet_From_Tensorflow.md) page.
```sh
mo --input_model FaceNet.pb --input "phase_train->False" --output_dir <OUTPUT_MODEL_DIR>
```
Launch the Model Optimizer for any model with a placeholder freezing tensor of values.
It replaces the placeholder with a constant layer that contains the passed values.
Tensor here is represented in square brackets with each value separated from another by a whitespace.
If data type is set in the model, this tensor will be reshaped to a placeholder shape and casted to placeholder data type.
Otherwise, it will be casted to data type passed to `--data_type` parameter (by default, it is FP32).
```sh
mo --input_model FaceNet.pb --input "placeholder_layer_name->[0.1 1.2 2.3]" --output_dir <OUTPUT_MODEL_DIR>
```
## See Also
* [Configuring the Model Optimizer](../../Deep_Learning_Model_Optimizer_DevGuide.md)
* [IR Notation Reference](../../IR_and_opsets.md)
* [Model Optimizer Extensibility](../customize_model_optimizer/Customize_Model_Optimizer.md)
* [Model Cutting](Cutting_Model.md)
* [Introduction](../../Deep_Learning_Model_Optimizer_DevGuide.md)
* [Cutting Off Parts of a Model](Cutting_Model.md)

4
docs/MO_DG/prepare_model/convert_model/Cutting_Model.md
View File

@ -6,10 +6,10 @@ Sometimes some parts of a model must be removed while the Model Optimizer is con
The following examples are the situations when model cutting is useful or even required:
* model has pre- or post-processing parts that cannot be translated to existing Inference Engine layers.
* model has pre- or post-processing parts that cannot be translated to existing OpenVINO operations.
* model has a training part that is convenient to be kept in the model, but not used during inference.
* model is too complex (contains lots of unsupported operations that cannot be easily implemented as custom layers), so the complete model cannot be converted in one shot.
* problem with model conversion in the Model Optimizer or inference in the Inference Engine occurred. To localize the issue, limit the scope for conversion by iteratively searching for problematic places in the model.
* problem with model conversion in the Model Optimizer or inference in the OpenVINO Runtime occurred. To localize the issue, limit the scope for conversion by iteratively searching for problematic places in the model.
* single custom layer or a combination of custom layers is isolated for debugging purposes.
## Command-Line Options

2
docs/MO_DG/prepare_model/convert_model/IR_suitable_for_INT8_inference.md
View File

@ -2,7 +2,7 @@
## Introduction
Inference Engine CPU and GPU plugin can infer models in the low precision.
OpenVINO Runtime CPU and GPU devices can infer models in the low precision.
For details, refer to [Low Precision Inference on the CPU](../../../OV_Runtime_UG/Int8Inference.md).
Intermediate Representation (IR) should be specifically formed to be suitable for low precision inference.

5319
docs/MO_DG/prepare_model/convert_model/Legacy_IR_Layers_Catalog_Spec.md
View File

File diff suppressed because it is too large Load Diff

22
docs/MO_DG/prepare_model/convert_model/kaldi_specific/Aspire_Tdnn_Model.md
View File

@ -1,6 +1,6 @@
# Convert Kaldi* ASpIRE Chain Time Delay Neural Network (TDNN) Model to the Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_kaldi_specific_Aspire_Tdnn_Model}
# Convert Kaldi* ASpIRE Chain Time Delay Neural Network (TDNN) Model {#openvino_docs_MO_DG_prepare_model_convert_model_kaldi_specific_Aspire_Tdnn_Model}
You can [download a pre-trained model](https://kaldi-asr.org/models/1/0001_aspire_chain_model.tar.gz)
You can [download a pre-trained model](https://kaldi-asr.org/models/1/0001_aspire_chain_model.tar.gz)
for the ASpIRE Chain Time Delay Neural Network (TDNN) from the Kaldi* project official website.
## Convert ASpIRE Chain TDNN Model to IR
@ -10,15 +10,15 @@ To generate the Intermediate Representation (IR) of the model, run the Model Opt
mo --input_model exp/chain/tdnn_7b/final.mdl --output output
```
The IR will have two inputs: `input` for data and `ivector` for ivectors.
The IR will have two inputs: `input` for data and `ivector` for ivectors.
## Example: Run ASpIRE Chain TDNN Model with the Speech Recognition Sample
These instructions show how to run the converted model with the [Speech Recognition sample](../../../../../samples/cpp/speech_sample/README.md).
In this example, the input data contains one utterance from one speaker.
In this example, the input data contains one utterance from one speaker.
To follow the steps described below, you must first do the following:
1. Download a [Kaldi repository](https://github.com/kaldi-asr/kaldi).
To follow the steps described below, you must first do the following:
1. Download a [Kaldi repository](https://github.com/kaldi-asr/kaldi).
2. Build it using instructions in `README.md` in the repository.
3. Download the [model archive](https://kaldi-asr.org/models/1/0001_aspire_chain_model.tar.gz) from Kaldi website.
4. Extract the downloaded model archive to the `egs/aspire/s5` folder of the Kaldi repository.
@ -49,10 +49,10 @@ cd <path_to_kaldi_repo>/egs/aspire/s5/
2. Extract ivectors from the data:
```sh
./steps/online/nnet2/extract_ivectors_online.sh --nj 1 --ivector_period <max_frame_count_in_utterance> <data folder> exp/tdnn_7b_chain_online/ivector_extractor <ivector folder>
./steps/online/nnet2/extract_ivectors_online.sh --nj 1 --ivector_period <max_frame_count_in_utterance> <data folder> exp/tdnn_7b_chain_online/ivector_extractor <ivector folder>
```
To simplify the preparation of ivectors for the Speech Recognition sample,
specify the maximum number of frames in utterances as a parameter for `--ivector_period`
To simplify the preparation of ivectors for the Speech Recognition sample,
specify the maximum number of frames in utterances as a parameter for `--ivector_period`
to get only one ivector per utterance.
To get the maximum number of frames in utterances, you can use the following command line:
@ -71,7 +71,7 @@ cd <ivector folder>
<path_to_kaldi_repo>/src/featbin/copy-feats --binary=False ark:ivector_online.1.ark ark,t:ivector_online.1.ark.txt
```
5. For the Speech Recognition sample, the `.ark` file must contain an ivector
5. For the Speech Recognition sample, the `.ark` file must contain an ivector
for each frame. You must copy the ivector `frame_count` times.
To do this, you can run the following script in the Python* command prompt:
```python
@ -108,5 +108,5 @@ Run the Speech Recognition sample with the created ivector `.ark` file as follow
speech_sample -i feats.ark,ivector_online_ie.ark -m final.xml -d CPU -o prediction.ark -cw_l 17 -cw_r 12
```
Results can be decoded as described in "Use of Sample in Kaldi* Speech Recognition Pipeline" chapter
Results can be decoded as described in "Use of Sample in Kaldi* Speech Recognition Pipeline" chapter
in [the Speech Recognition Sample description](../../../../../samples/cpp/speech_sample/README.md).

6
docs/MO_DG/prepare_model/convert_model/mxnet_specific/Convert_GluonCV_Models.md
View File

@ -1,15 +1,15 @@
# Converting GluonCV* Models {#openvino_docs_MO_DG_prepare_model_convert_model_mxnet_specific_Convert_GluonCV_Models}
# Convert MXNet GluonCV* Models {#openvino_docs_MO_DG_prepare_model_convert_model_mxnet_specific_Convert_GluonCV_Models}
This document provides the instructions and examples on how to use Model Optimizer to convert [GluonCV SSD and YOLO-v3 models](https://gluon-cv.mxnet.io/model_zoo/detection.html) to IR.
1. Choose the topology available from the [GluonCV Model Zoo](https://gluon-cv.mxnet.io/model_zoo/detection.html) and export to the MXNet format using the GluonCV API. For example, for the `ssd_512_mobilenet1.0` topology:
1. Choose the topology available from the [GluonCV Model Zoo](https://gluon-cv.mxnet.io/model_zoo/detection.html) and export to the MXNet format using the GluonCV API. For example, for the `ssd_512_mobilenet1.0` topology:
```python
from gluoncv import model_zoo, data, utils
from gluoncv.utils import export_block
net = model_zoo.get_model('ssd_512_mobilenet1.0_voc', pretrained=True)
export_block('ssd_512_mobilenet1.0_voc', net, preprocess=True, layout='HWC')
```
As a result, you will get an MXNet model representation in `ssd_512_mobilenet1.0.params` and `ssd_512_mobilenet1.0.json` files generated in the current directory.
As a result, you will get an MXNet model representation in `ssd_512_mobilenet1.0.params` and `ssd_512_mobilenet1.0.json` files generated in the current directory.
2. Run the Model Optimizer tool specifying the `--enable_ssd_gluoncv` option. Make sure the `--input_shape` parameter is set to the input shape layout of your model (NHWC or NCHW). The examples below illustrates running the Model Optimizer for the SSD and YOLO-v3 models trained with the NHWC layout and located in the `<model_directory>`:
* **For GluonCV SSD topologies:**
```sh

8
docs/MO_DG/prepare_model/convert_model/mxnet_specific/Convert_Style_Transfer_From_MXNet.md
View File

@ -1,4 +1,4 @@
# Converting a Style Transfer Model from MXNet* {#openvino_docs_MO_DG_prepare_model_convert_model_mxnet_specific_Convert_Style_Transfer_From_MXNet}
# Convert MXNet Style Transfer Model {#openvino_docs_MO_DG_prepare_model_convert_model_mxnet_specific_Convert_Style_Transfer_From_MXNet}
The tutorial explains how to generate a model for style transfer using the public MXNet\* neural style transfer sample.
To use the style transfer sample from OpenVINO&trade;, follow the steps below as no public pre-trained style transfer model is provided with the OpenVINO toolkit.
@ -86,8 +86,8 @@ import make_image
maker = make_image.Maker('models/13', (1024, 768))
maker.generate('output.jpg', '../images/tubingen.jpg')
```
Where the `models/13` string is composed of the following substrings:
* `models/`: path to the folder that contains .nd files with pre-trained styles weights
Where the `models/13` string is composed of the following substrings:
* `models/`: path to the folder that contains .nd files with pre-trained styles weights
* `13`: prefix pointing to 13_decoder, which is the default decoder for the repository.
> **NOTE**: If you get an error saying "No module named 'cPickle'", try running the script from this step in Python 2. Then return to Python 3 for the remaining steps.
@ -114,4 +114,4 @@ cp models/13_decoder_auxs.nd nst_model
```sh
mo --input_symbol <path/to/nst_model>/nst_vgg19-symbol.json --framework mxnet --output_dir <path/to/output_dir> --input_shape [1,3,224,224] --nd_prefix_name 13_decoder --pretrained_model <path/to/nst_model>/vgg19-0000.params
```
4. The IR is generated (`.bin`, `.xml` and `.mapping` files) in the specified output directory and ready to be consumed by the Inference Engine.
4. The IR is generated (`.bin`, `.xml` and `.mapping` files) in the specified output directory and ready to be consumed by the OpenVINO Runtime.

32
docs/MO_DG/prepare_model/convert_model/onnx_specific/Convert_DLRM.md
View File

@ -1,32 +0,0 @@
[DEPRECATED] Convert ONNX* DLRM to the Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_DLRM}
===============================
> **NOTE**: These instructions are currently deprecated. Since OpenVINO™ 2020.4 version, no specific steps are needed to convert ONNX\* DLRM models. For general instructions on converting ONNX models, please refer to [Converting a ONNX* Model](../Convert_Model_From_ONNX.md) topic.
These instructions are applicable only to the DLRM converted to the ONNX* file format from the [facebookresearch/dlrm model](https://github.com/facebookresearch/dlrm).
**Step 1**. Save trained Pytorch* model to ONNX* format or download pretrained ONNX* from
[MLCommons/inference/recommendation/dlrm](https://github.com/mlcommons/inference/tree/r1.0/recommendation/dlrm/pytorch#supported-models) repository.
If you train the model using the [script provided in model repository](https://github.com/facebookresearch/dlrm/blob/master/dlrm_s_pytorch.py), just add the `--save-onnx` flag to the command line parameters and you'll get the `dlrm_s_pytorch.onnx` file containing the model serialized in ONNX* format.
**Step 2**. To generate the Intermediate Representation (IR) of the model, change your current working directory to the Model Optimizer installation directory and run the Model Optimizer with the following parameters:
```sh
mo --input_model dlrm_s_pytorch.onnx
```
Note that Pytorch model uses operation `torch.nn.EmbeddingBag`. This operation converts to onnx as custom `ATen` layer and not directly supported by OpenVINO*, but it is possible to convert this operation to:
* `Gather` if each "bag" consists of exactly one index. In this case `offsets` input becomes obsolete and not needed. They will be removed during conversion.
* `ExperimentalSparseWeightedSum` if "bags" contain not just one index. In this case Model Optimizer will print warning that pre-process of offsets is needed, because `ExperimentalSparseWeightedSum` and `torch.nn.EmbeddingBag` have different format of inputs.
For example if you have `indices` input of shape [indices_shape] and `offsets` input of shape [num_bags] you need to get offsets of shape [indices_shape, 2]. To do that you may use the following code snippet:
```python
import numpy as np
new_offsets = np.zeros((indices.shape[-1], 2), dtype=np.int32)
new_offsets[:, 1] = np.arange(indices.shape[-1])
bag_index = 0
for i in range(offsets.shape[-1] - 1):
new_offsets[offsets[i]:offsets[i + 1], 0] = bag_index
bag_index += 1
new_offsets[offsets[-1]:, 0] = bag_index
```
If you have more than one `torch.nn.EmbeddingBag` operation you'll need to do that for every offset input. If your offsets have same shape they will be merged into one input of shape [num_embedding_bags, indices_shape, 2].

4
docs/MO_DG/prepare_model/convert_model/onnx_specific/Convert_Faster_RCNN.md
View File

@ -1,4 +1,4 @@
# Convert ONNX* Faster R-CNN Model to the Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_Faster_RCNN}
# Convert ONNX* Faster R-CNN Model {#openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_Faster_RCNN}
These instructions are applicable only to the Faster R-CNN model converted to the ONNX* file format from the [facebookresearch/maskrcnn-benchmark model](https://github.com/facebookresearch/maskrcnn-benchmark).
@ -11,7 +11,7 @@ These instructions are applicable only to the Faster R-CNN model converted to th
--input_shape [1,3,800,800] \
--input 0:2 \
--mean_values [102.9801,115.9465,122.7717] \
--transformations_config front/onnx/faster_rcnn.json
--transformations_config front/onnx/faster_rcnn.json
```
Note that the height and width specified with the `input_shape` command line parameter could be different. Refer to the [documentation](https://github.com/onnx/models/tree/master/vision/object_detection_segmentation/faster-rcnn) for more information about supported input image dimensions and required pre- and post-processing steps.

2
docs/MO_DG/prepare_model/convert_model/onnx_specific/Convert_GPT2.md
View File

@ -1,4 +1,4 @@
# Convert ONNX* GPT-2 Model to the Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_GPT2}
# Convert ONNX* GPT-2 Model {#openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_GPT2}
[Public pre-trained GPT-2 model](https://github.com/onnx/models/tree/master/text/machine_comprehension/gpt-2) is a large
transformer-based language model with a simple objective: predict the next word, given all of the previous words within some text.

4
docs/MO_DG/prepare_model/convert_model/onnx_specific/Convert_Mask_RCNN.md
View File

@ -1,4 +1,4 @@
# Convert ONNX* Mask R-CNN Model to the Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_Mask_RCNN}
# Convert ONNX* Mask R-CNN Model {#openvino_docs_MO_DG_prepare_model_convert_model_onnx_specific_Convert_Mask_RCNN}
These instructions are applicable only to the Mask R-CNN model converted to the ONNX* file format from the [facebookresearch/maskrcnn-benchmark model](https://github.com/facebookresearch/maskrcnn-benchmark).
@ -11,7 +11,7 @@ These instructions are applicable only to the Mask R-CNN model converted to the
--input "0:2" \
--input_shape [1,3,800,800] \
--mean_values [102.9801,115.9465,122.7717] \
--transformations_config front/onnx/mask_rcnn.json
--transformations_config front/onnx/mask_rcnn.json
```
Note that the height and width specified with the `input_shape` command line parameter could be different. Refer to the [documentation](https://github.com/onnx/models/tree/master/vision/object_detection_segmentation/mask-rcnn) for more information about supported input image dimensions and required pre- and post-processing steps.

2
docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_Bert_ner.md
View File

@ -1,4 +1,4 @@
# Convert PyTorch* BERT-NER to the Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_Bert_ner}
# Convert PyTorch* BERT-NER Model {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_Bert_ner}
## Download and Convert the Model to ONNX*

28
docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_Cascade_RCNN_res101.md Normal file
View File

@ -0,0 +1,28 @@
# Convert PyTorch Cascade RCNN R-101 Model {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_Cascade_RCNN_res101}
## Download and Convert Model to ONNX
* Clone the [repository](https://github.com/open-mmlab/mmdetection):
```bash
git clone https://github.com/open-mmlab/mmdetection
cd mmdetection
```
> **NOTE**: To set up an environment, refer to this [instruction](https://github.com/open-mmlab/mmdetection/blob/master/docs/en/get_started.md#installation).
* Download the pre-trained [model](https://download.openmmlab.com/mmdetection/v2.0/cascade_rcnn/cascade_rcnn_r101_fpn_1x_coco/cascade_rcnn_r101_fpn_1x_coco_20200317-0b6a2fbf.pth). You can also find the link to the model [here](https://github.com/open-mmlab/mmdetection/blob/master/configs/cascade_rcnn/README.md).
* To convert the model to ONNX format, use this [script](https://github.com/open-mmlab/mmdetection/blob/master/tools/deployment/pytorch2onnx.py).
```bash
python3 tools/deployment/pytorch2onnx.py configs/cascade_rcnn/cascade_rcnn_r101_fpn_1x_coco.py cascade_rcnn_r101_fpn_1x_coco_20200317-0b6a2fbf.pth --output-file cascade_rcnn_r101_fpn_1x_coco.onnx
```
The script generates ONNX model file `cascade_rcnn_r101_fpn_1x_coco.onnx` in the directory `tools/deployment/`. If required, you can specify the model name or output directory using `--output-file <path-to-dir>/<model-name>.onnx`
## Convert ONNX Cascade RCNN R-101 Model to IR
```bash
mo --input_model cascade_rcnn_r101_fpn_1x_coco.onnx --mean_values [123.675,116.28,103.53] --scale_values [58.395,57.12,57.375]
```

6
docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_F3Net.md
View File

@ -1,4 +1,4 @@
# Convert PyTorch* F3Net to the Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_F3Net}
# Convert PyTorch* F3Net Model {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_F3Net}
[F3Net](https://github.com/weijun88/F3Net): Fusion, Feedback and Focus for Salient Object Detection
@ -7,12 +7,12 @@
To clone the repository, run the following command:
```sh
git clone http://github.com/weijun88/F3Net.git
git clone http://github.com/weijun88/F3Net.git
```
## Download and Convert the Model to ONNX*
To download the pre-trained model or train the model yourself, refer to the
To download the pre-trained model or train the model yourself, refer to the
[instruction](https://github.com/weijun88/F3Net/blob/master/README.md) in the F3Net model repository. First, convert the model to ONNX\* format. Create and run the following Python script in the `src` directory of the model repository:
```python
import torch

2
docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_QuartzNet.md
View File

@ -1,4 +1,4 @@
# Convert PyTorch* QuartzNet to the Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_QuartzNet}
# Convert PyTorch* QuartzNet Model {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_QuartzNet}
[NeMo project](https://github.com/NVIDIA/NeMo) provides the QuartzNet model.

2
docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_RCAN.md
View File

@ -1,4 +1,4 @@
# Convert PyTorch* RCAN to the Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_RCAN}
# Convert PyTorch* RCAN Model {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_RCAN}
[RCAN](https://github.com/yulunzhang/RCAN): Image Super-Resolution Using Very Deep Residual Channel Attention Networks

24
docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_RNNT.md
View File

@ -1,22 +1,22 @@
# Convert PyTorch\* RNN-T Model to the Intermediate Representation (IR) {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_RNNT}
# Convert PyTorch* RNN-T Model {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_RNNT}
This instruction covers conversion of RNN-T model from [MLCommons](https://github.com/mlcommons) repository. Follow
This instruction covers conversion of RNN-T model from [MLCommons](https://github.com/mlcommons) repository. Follow
the steps below to export a PyTorch* model into ONNX* before converting it to IR:
**Step 1**. Clone RNN-T PyTorch implementation from MLCommons repository (revision r1.0). Make a shallow clone to pull
**Step 1**. Clone RNN-T PyTorch implementation from MLCommons repository (revision r1.0). Make a shallow clone to pull
only RNN-T model without full repository. If you already have a full repository, skip this and go to **Step 2**:
```bash
git clone -b r1.0 -n https://github.com/mlcommons/inference rnnt_for_openvino --depth 1
cd rnnt_for_openvino
git checkout HEAD speech_recognition/rnnt
git checkout HEAD speech_recognition/rnnt
```
**Step 2**. If you already have a full clone of MLCommons inference repository, create a folder for
pretrained PyTorch model, where conversion into IR will take place. You will also need to specify the path to
**Step 2**. If you already have a full clone of MLCommons inference repository, create a folder for
pretrained PyTorch model, where conversion into IR will take place. You will also need to specify the path to
your full clone at **Step 5**. Skip this step if you have a shallow clone.
```bash
mkdir rnnt_for_openvino
mkdir rnnt_for_openvino
cd rnnt_for_openvino
```
@ -25,7 +25,7 @@ For UNIX*-like systems you can use `wget`:
```bash
wget https://zenodo.org/record/3662521/files/DistributedDataParallel_1576581068.9962234-epoch-100.pt
```
The link was taken from `setup.sh` in the `speech_recoginitin/rnnt` subfolder. You will get exactly the same weights as
The link was taken from `setup.sh` in the `speech_recoginitin/rnnt` subfolder. You will get exactly the same weights as
if you were following the steps from [https://github.com/mlcommons/inference/tree/master/speech_recognition/rnnt](https://github.com/mlcommons/inference/tree/master/speech_recognition/rnnt).
**Step 4**. Install required Python packages:
@ -33,7 +33,7 @@ if you were following the steps from [https://github.com/mlcommons/inference/tre
pip3 install torch toml
```
**Step 5**. Export RNN-T model into ONNX with the script below. Copy the code below into a file named
**Step 5**. Export RNN-T model into ONNX with the script below. Copy the code below into a file named
`export_rnnt_to_onnx.py` and run it in the current directory `rnnt_for_openvino`:
> **NOTE**: If you already have a full clone of MLCommons inference repository, you need to
@ -94,7 +94,7 @@ torch.onnx.export(model.joint, (f, g), "rnnt_joint.onnx", opset_version=12,
python3 export_rnnt_to_onnx.py
```
After completing this step, the files `rnnt_encoder.onnx`, `rnnt_prediction.onnx`, and `rnnt_joint.onnx` will be saved in the current directory.
After completing this step, the files `rnnt_encoder.onnx`, `rnnt_prediction.onnx`, and `rnnt_joint.onnx` will be saved in the current directory.
**Step 6**. Run the conversion commands:
@ -103,6 +103,6 @@ mo --input_model rnnt_encoder.onnx --input "input[157 1 240],feature_length->157
mo --input_model rnnt_prediction.onnx --input "symbol[1 1],hidden_in_1[2 1 320],hidden_in_2[2 1 320]"
mo --input_model rnnt_joint.onnx --input "0[1 1 1024],1[1 1 320]"
```
Please note that hardcoded value for sequence length = 157 was taken from the MLCommons but conversion to IR preserves
network [reshapeability](../../../../OV_Runtime_UG/ShapeInference.md), this means you can change input shapes manually to any value either during conversion or
Please note that hardcoded value for sequence length = 157 was taken from the MLCommons but conversion to IR preserves
network [reshapeability](../../../../OV_Runtime_UG/ShapeInference.md), this means you can change input shapes manually to any value either during conversion or
inference.

40
docs/MO_DG/prepare_model/convert_model/pytorch_specific/Convert_YOLACT.md
View File

@ -1,4 +1,4 @@
# Convert PyTorch* YOLACT to the Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_YOLACT}
# Convert PyTorch* YOLACT Model {#openvino_docs_MO_DG_prepare_model_convert_model_pytorch_specific_Convert_YOLACT}
You Only Look At CoefficienTs (YOLACT) is a simple, fully convolutional model for real-time instance segmentation.
The PyTorch\* implementation is publicly available in [this GitHub* repository](https://github.com/dbolya/yolact).
@ -29,7 +29,7 @@ index 547bc0a..bde0680 100644
+++ b/eval.py
@@ -593,9 +593,12 @@ def badhash(x):
return x
def evalimage(net:Yolact, path:str, save_path:str=None):
- frame = torch.from_numpy(cv2.imread(path)).cuda().float()
+ frame = torch.from_numpy(cv2.imread(path)).float()
@ -38,9 +38,9 @@ index 547bc0a..bde0680 100644
batch = FastBaseTransform()(frame.unsqueeze(0))
preds = net(batch)
+ torch.onnx.export(net, batch, "yolact.onnx", opset_version=11)
img_numpy = prep_display(preds, frame, None, None, undo_transform=False)
diff --git a/utils/augmentations.py b/utils/augmentations.py
index cc7a73a..2420603 100644
--- a/utils/augmentations.py
@ -48,7 +48,7 @@ index cc7a73a..2420603 100644
@@ -623,8 +623,11 @@ class FastBaseTransform(torch.nn.Module):
def __init__(self):
super().__init__()
- self.mean = torch.Tensor(MEANS).float().cuda()[None, :, None, None]
- self.std = torch.Tensor( STD ).float().cuda()[None, :, None, None]
+ self.mean = torch.Tensor(MEANS).float()[None, :, None, None]
@ -57,7 +57,7 @@ index cc7a73a..2420603 100644
+ self.mean.cuda()
+ self.std.cuda()
self.transform = cfg.backbone.transform
def forward(self, img):
diff --git a/yolact.py b/yolact.py
index d83703b..f8c787c 100644
@ -66,7 +66,7 @@ index d83703b..f8c787c 100644
@@ -17,19 +17,22 @@ import torch.backends.cudnn as cudnn
from utils import timer
from utils.functions import MovingAverage, make_net
-# This is required for Pytorch 1.0.1 on Windows to initialize Cuda on some driver versions.
-# See the bug report here: https://github.com/pytorch/pytorch/issues/17108
-torch.cuda.current_device()
@ -76,26 +76,26 @@ index d83703b..f8c787c 100644
-if not use_jit:
- print('Multiple GPUs detected! Turning off JIT.')
+use_jit = False
ScriptModuleWrapper = torch.jit.ScriptModule if use_jit else nn.Module
script_method_wrapper = torch.jit.script_method if use_jit else lambda fn, _rcn=None: fn
+def decode(loc, priors):
+ variances = [0.1, 0.2]
+ boxes = torch.cat((priors[:, :2] + loc[:, :, :2] * variances[0] * priors[:, 2:], priors[:, 2:] * torch.exp(loc[:, :, 2:] * variances[1])), 2)
+
+ boxes_result1 = boxes[:, :, :2] - boxes[:, :, 2:] / 2
+ boxes_result2 = boxes[:, :, 2:] + boxes[:, :, :2]
+ boxes_result2 = boxes[:, :, 2:] + boxes_result1
+ boxes_result = torch.cat((boxes_result1, boxes_result2), 2)
+
+ return boxes_result
+
class Concat(nn.Module):
def __init__(self, nets, extra_params):
@@ -476,7 +479,10 @@ class Yolact(nn.Module):
def load_weights(self, path):
""" Loads weights from a compressed save file. """
- state_dict = torch.load(path)
@ -103,23 +103,23 @@ index d83703b..f8c787c 100644
+ state_dict = torch.load(path)
+ else:
+ state_dict = torch.load(path, map_location=torch.device('cpu'))
# For backward compatability, remove these (the new variable is called layers)
for key in list(state_dict.keys()):
@@ -673,8 +679,11 @@ class Yolact(nn.Module):
else:
pred_outs['conf'] = F.softmax(pred_outs['conf'], -1)
- return self.detect(pred_outs, self)
+ pred_outs['boxes'] = decode(pred_outs['loc'], pred_outs['priors']) # decode output boxes
+ pred_outs.pop('priors') # remove unused in postprocessing layers
+ pred_outs.pop('loc') # remove unused in postprocessing layers
+ return pred_outs
--
--
```
3. Save and close the file.

4
docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_AttentionOCR_From_Tensorflow.md
View File

@ -1,4 +1,4 @@
# Convert TensorFlow* Attention OCR Model to Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_AttentionOCR_From_Tensorflow}
# Convert TensorFlow Attention OCR Model {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_AttentionOCR_From_Tensorflow}
This tutorial explains how to convert the Attention OCR (AOCR) model from the [TensorFlow* Attention OCR repository](https://github.com/emedvedev/attention-ocr) to the Intermediate Representation (IR).
@ -20,7 +20,7 @@ The original AOCR model contains data preprocessing which consists of the follow
* Decoding input data to binary format where input data is an image represented as a string.
* Resizing binary image to working resolution.
After that, the resized image is sent to the convolution neural network (CNN). The Model Optimizer does not support image decoding so you should cut of preprocessing part of the model using '--input' command line parameter.
After that, the resized image is sent to the convolution neural network (CNN). The Model Optimizer does not support image decoding so you should cut of preprocessing part of the model using '--input' command line parameter.
```sh
mo \
--input_model=model/path/frozen_graph.pb \

4
docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_BERT_From_Tensorflow.md
View File

@ -1,4 +1,4 @@
# Convert TensorFlow* BERT Model to the Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_BERT_From_Tensorflow}
# Convert TensorFlow BERT Model {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_BERT_From_Tensorflow}
Pre-trained models for BERT (Bidirectional Encoder Representations from Transformers) are
[publicly available](https://github.com/google-research/bert).
@ -112,7 +112,7 @@ Run the Model Optimizer with the following command line parameters to generate r
```sh
mo \
--input_model inference_graph.pb \
--input "IteratorGetNext:0{i32}[1 128],IteratorGetNext:1{i32}[1 128],IteratorGetNext:4{i32}[1 128]"
--input "IteratorGetNext:0{i32}[1 128],IteratorGetNext:1{i32}[1 128],IteratorGetNext:4{i32}[1 128]"
```
For other applicable parameters, refer to [Convert Model from TensorFlow](../Convert_Model_From_TensorFlow.md).

2
docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_CRNN_From_Tensorflow.md
View File

@ -1,4 +1,4 @@
# Convert CRNN* Models to the Intermediate Representation (IR) {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_CRNN_From_Tensorflow}
# Convert TensorFlow CRNN Model {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_CRNN_From_Tensorflow}
This tutorial explains how to convert a CRNN model to Intermediate Representation (IR).

30
docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_DeepSpeech_From_Tensorflow.md
View File

@ -1,4 +1,4 @@
# Convert TensorFlow* DeepSpeech Model to the Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_DeepSpeech_From_Tensorflow}
# Convert TensorFlow DeepSpeech Model {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_DeepSpeech_From_Tensorflow}
[DeepSpeech project](https://github.com/mozilla/DeepSpeech) provides an engine to train speech-to-text models.
@ -9,7 +9,7 @@ Create a directory where model and metagraph with pretrained weights will be sto
mkdir deepspeech
cd deepspeech
```
[Pretrained English speech-to-text model](https://github.com/mozilla/DeepSpeech/releases/tag/v0.8.2) is publicly available.
[Pretrained English speech-to-text model](https://github.com/mozilla/DeepSpeech/releases/tag/v0.8.2) is publicly available.
To download the model, follow the instruction below:
* For UNIX*-like systems, run the following command:
@ -24,7 +24,7 @@ wget -O - https://github.com/mozilla/DeepSpeech/releases/download/v0.8.2/deepspe
## Freeze the Model into a *.pb File
After unpacking the archives above, you have to freeze the model. Note that this requires
After unpacking the archives above, you have to freeze the model. Note that this requires
TensorFlow* version 1 which is not available under Python 3.8, so you need Python 3.7 or lower.
Before freezing, deploy a virtual environment and install the required packages:
```
@ -37,29 +37,29 @@ Freeze the model with the following command:
```
python3 DeepSpeech.py --checkpoint_dir ../deepspeech-0.8.2-checkpoint --export_dir ../
```
After that, you will get the pretrained frozen model file `output_graph.pb` in the directory `deepspeech` created at
the beginning. The model contains the preprocessing and main parts. The first preprocessing part performs conversion of input
spectrogram into a form useful for speech recognition (mel). This part of the model is not convertible into
After that, you will get the pretrained frozen model file `output_graph.pb` in the directory `deepspeech` created at
the beginning. The model contains the preprocessing and main parts. The first preprocessing part performs conversion of input
spectrogram into a form useful for speech recognition (mel). This part of the model is not convertible into
IR because it contains unsupported operations `AudioSpectrogram` and `Mfcc`.
The main and most computationally expensive part of the model converts the preprocessed audio into text.
There are two specificities with the supported part of the model.
The main and most computationally expensive part of the model converts the preprocessed audio into text.
There are two specificities with the supported part of the model.
The first is that the model contains an input with sequence length. So the model can be converted with
a fixed input length shape, thus the model is not reshapeable.
The first is that the model contains an input with sequence length. So the model can be converted with
a fixed input length shape, thus the model is not reshapeable.
Refer to the [Using Shape Inference](../../../../OV_Runtime_UG/ShapeInference.md).
The second is that the frozen model still has two variables: `previous_state_c` and `previous_state_h`, figure
with the frozen *.pb model is below. It means that the model keeps training these variables at each inference.
The second is that the frozen model still has two variables: `previous_state_c` and `previous_state_h`, figure
with the frozen *.pb model is below. It means that the model keeps training these variables at each inference.
![DeepSpeech model view](../../../img/DeepSpeech-0.8.2.png)
At the first inference the variables are initialized with zero tensors. After executing, the results of the `BlockLSTM`
At the first inference the variables are initialized with zero tensors. After executing, the results of the `BlockLSTM`
are assigned to cell state and hidden state, which are these two variables.
## Convert the Main Part of DeepSpeech Model into IR
Model Optimizer assumes that the output model is for inference only. That is why you should cut `previous_state_c`
Model Optimizer assumes that the output model is for inference only. That is why you should cut `previous_state_c`
and `previous_state_h` variables off and resolve keeping cell and hidden states on the application level.
There are certain limitations for the model conversion:
@ -75,7 +75,7 @@ mo \
```
Where:
* `input_lengths->[16]` Replaces the input node with name "input_lengths" with a constant tensor of shape [1] with a
* `input_lengths->[16]` Replaces the input node with name "input_lengths" with a constant tensor of shape [1] with a
single integer value 16. This means that the model now can consume input sequences of length 16 only.
* `input_node[1 16 19 26],previous_state_h[1 2048],previous_state_c[1 2048]` replaces the variables with a placeholder.
* `--output ".../GatherNd_1,.../GatherNd,logits" ` output node names.

21
docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_EfficientDet_Models.md
View File

@ -1,11 +1,11 @@
# Converting EfficientDet Models from TensorFlow {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_EfficientDet_Models}
# Convert TensorFlow EfficientDet Models {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_EfficientDet_Models}
This tutorial explains how to convert EfficientDet\* public object detection models to the Intermediate Representation (IR).
This tutorial explains how to convert EfficientDet\* public object detection models to the Intermediate Representation (IR).
## <a name="efficientdet-to-ir"></a>Convert EfficientDet Model to IR
On GitHub*, you can find several public versions of EfficientDet model implementation. This tutorial explains how to
convert models from the [https://github.com/google/automl/tree/master/efficientdet](https://github.com/google/automl/tree/master/efficientdet)
On GitHub*, you can find several public versions of EfficientDet model implementation. This tutorial explains how to
convert models from the [https://github.com/google/automl/tree/master/efficientdet](https://github.com/google/automl/tree/master/efficientdet)
repository (commit 96e1fee) to IR.
### Get Frozen TensorFlow\* Model
@ -60,15 +60,15 @@ dictionary in the [hparams_config.py](https://github.com/google/automl/blob/96e1
The attribute `image_size` specifies the shape to be specified for the model conversion.
The `transformations_config` command line parameter specifies the configuration json file containing hints
to the Model Optimizer on how to convert the model and trigger transformations implemented in the
to the Model Optimizer on how to convert the model and trigger transformations implemented in the
`<PYTHON_SITE_PACKAGES>/openvino/tools/mo/front/tf/AutomlEfficientDet.py`. The json file contains some parameters which must be changed if you
train the model yourself and modified the `hparams_config` file or the parameters are different from the ones used for EfficientDet-D4.
The attribute names are self-explanatory or match the name in the `hparams_config` file.
> **NOTE**: The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the `RGB<->BGR` conversion specifying the command-line parameter: `--reverse_input_channels`. Otherwise, inference results may be incorrect. For more information about the parameter, refer to **When to Reverse Input Channels** section of [Converting a Model to Intermediate Representation (IR)](../Converting_Model.md).
OpenVINO&trade; toolkit provides samples that can be used to infer EfficientDet model. For more information, refer to
[Open Model Zoo Demos](@ref omz_demos) and
OpenVINO&trade; toolkit provides samples that can be used to infer EfficientDet model. For more information, refer to
[Open Model Zoo Demos](@ref omz_demos) and
## <a name="efficientdet-ir-results-interpretation"></a>Interpreting Results of the TensorFlow Model and the IR
@ -90,9 +90,4 @@ The output of the IR is a list of 7-element tuples: `[image_id, class_id, confid
* `x_max` -- normalized `x` coordinate of the upper right corner of the detected object.
* `y_max` -- normalized `y` coordinate of the upper right corner of the detected object.
The first element with `image_id = -1` means end of data.
---
## See Also
* [Sub-Graph Replacement in Model Optimizer](../../customize_model_optimizer/Subgraph_Replacement_Model_Optimizer.md)
The first element with `image_id = -1` means end of data.

2
docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_FaceNet_From_Tensorflow.md
View File

@ -1,4 +1,4 @@
# Convert TensorFlow* FaceNet Models to Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_FaceNet_From_Tensorflow}
# Convert TensorFlow FaceNet Models {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_FaceNet_From_Tensorflow}
[Public pre-trained FaceNet models](https://github.com/davidsandberg/facenet#pre-trained-models) contain both training
and inference part of graph. Switch between this two states is manageable with placeholder value.

36
docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_GNMT_From_Tensorflow.md
View File

@ -1,4 +1,4 @@
# Convert GNMT* Model to the Intermediate Representation (IR) {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_GNMT_From_Tensorflow}
# Convert TensorFlow GNMT Model {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_GNMT_From_Tensorflow}
This tutorial explains how to convert Google\* Neural Machine Translation (GNMT) model to the Intermediate Representation (IR).
@ -17,20 +17,20 @@ index 2cbef07..e185490 100644
+++ b/nmt/inference.py
@@ -17,9 +17,11 @@
from __future__ import print_function
import codecs
+import os
import time
import tensorflow as tf
+from tensorflow.python.framework import graph_io
from . import attention_model
from . import gnmt_model
@@ -105,6 +107,29 @@ def start_sess_and_load_model(infer_model, ckpt_path):
return sess, loaded_infer_model
+def inference_dump_graph(ckpt_path, path_to_dump, hparams, scope=None):
+ model_creator = get_model_creator(hparams)
+ infer_model = model_helper.create_infer_model(model_creator, hparams, scope)
@ -64,7 +64,7 @@ index f5823d8..a733748 100644
@@ -310,6 +310,13 @@ def add_arguments(parser):
parser.add_argument("--num_intra_threads", type=int, default=0,
help="number of intra_op_parallelism_threads")
+ # Special argument for inference model dumping without inference
+ parser.add_argument("--dump_inference_model", type="bool", nargs="?",
+ const=True, default=False,
@ -72,7 +72,7 @@ index f5823d8..a733748 100644
+
+ parser.add_argument("--path_to_dump", type=str, default="",
+ help="Path to dump inference graph.")
def create_hparams(flags):
"""Create training hparams."""
@@ -396,6 +403,9 @@ def create_hparams(flags):
@ -83,12 +83,12 @@ index f5823d8..a733748 100644
+ dump_inference_model=flags.dump_inference_model,
+ path_to_dump=flags.path_to_dump,
)
@@ -613,7 +623,7 @@ def create_or_load_hparams(
return hparams
-def run_main(flags, default_hparams, train_fn, inference_fn, target_session=""):
+def run_main(flags, default_hparams, train_fn, inference_fn, inference_dump, target_session=""):
"""Run main."""
@ -97,7 +97,7 @@ index f5823d8..a733748 100644
@@ -653,8 +663,26 @@ def run_main(flags, default_hparams, train_fn, inference_fn, target_session=""):
out_dir, default_hparams, flags.hparams_path,
save_hparams=(jobid == 0))
- ## Train / Decode
- if flags.inference_input_file:
+ # Dumping inference model
@ -130,8 +130,8 @@ index f5823d8..a733748 100644
- run_main(FLAGS, default_hparams, train_fn, inference_fn)
+ inference_dump = inference.inference_dump_graph
+ run_main(FLAGS, default_hparams, train_fn, inference_fn, inference_dump)
if __name__ == "__main__":
```
@ -224,7 +224,7 @@ For more information about model cutting, refer to [Cutting Off Parts of a Model
Inputs of the model:
* `IteratorGetNext/placeholder_out_port_0` input with shape `[batch_size, max_sequence_length]` contains `batch_size` decoded input sentences.
Every sentence is decoded the same way as indices of sentence elements in vocabulary and padded with index of `eos` (end of sentence symbol). If the length of the sentence is less than `max_sequence_length`, remaining elements are filled with index of `eos` token.
* `IteratorGetNext/placeholder_out_port_1` input with shape `[batch_size]` contains sequence lengths for every sentence from the first input. \
For example, if `max_sequence_length = 50`, `batch_size = 1` and the sentence has only 30 elements, then the input tensor for `IteratorGetNext/placeholder_out_port_1` should be `[30]`.
@ -244,7 +244,7 @@ python3 benchmark_app.py -m <path to the generated GNMT IR> -d CPU
```
2. With Inference Engine Python API:
2. With OpenVINO Runtime Python API:
> **NOTE**: Before running the example, insert a path to your GNMT `.xml` and `.bin` files into `MODEL_PATH` and `WEIGHTS_PATH`, and fill `input_data_tensor` and `seq_lengths` tensors according to your input data.
@ -274,4 +274,4 @@ exec_net = ie.load_network(network=net, device_name="CPU")
result_ie = exec_net.infer(input_data)
```
For more information about Python API, refer to [Inference Engine Python API](ie_python_api/api.html).
For more information about Python API, refer to [OpenVINO Runtime Python API](ie_python_api/api.html).

14
docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_NCF_From_Tensorflow.md
View File

@ -1,12 +1,12 @@
# Convert Neural Collaborative Filtering Model from TensorFlow* to the Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_NCF_From_Tensorflow}
# Convert TensorFlow Neural Collaborative Filtering Model {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_NCF_From_Tensorflow}
This tutorial explains how to convert Neural Collaborative Filtering (NCF) model to Intermediate Representation (IR).
[Public TensorFlow NCF model](https://github.com/tensorflow/models/tree/master/official/recommendation) does not contain pre-trained weights. To convert this model to the IR:
1. Use [the instructions](https://github.com/tensorflow/models/tree/master/official/recommendation#train-and-evaluate-model) from this repository to train the model.
2. Freeze the inference graph you get on previous step in `model_dir` following
the instructions from the Freezing Custom Models in Python* section of
[Converting a TensorFlow* Model](../Convert_Model_From_TensorFlow.md).
1. Use [the instructions](https://github.com/tensorflow/models/tree/master/official/recommendation#train-and-evaluate-model) from this repository to train the model.
2. Freeze the inference graph you get on previous step in `model_dir` following
the instructions from the Freezing Custom Models in Python* section of
[Converting a TensorFlow* Model](../Convert_Model_From_TensorFlow.md).
Run the following commands:
```python
import tensorflow as tf
@ -22,12 +22,12 @@ graph_io.write_graph(frozen, './', 'inference_graph.pb', as_text=False)
```
where `rating/BiasAdd` is an output node.
3. Convert the model to the IR.If you look at your frozen model, you can see that
3. Convert the model to the IR.If you look at your frozen model, you can see that
it has one input that is split into four `ResourceGather` layers. (Click image to zoom in.)
![NCF model beginning](../../../img/NCF_start.png)
But as the Model Optimizer does not support such data feeding, you should skip it. Cut
But as the Model Optimizer does not support such data feeding, you should skip it. Cut
the edges incoming in `ResourceGather`s port 1:
```sh
mo --input_model inference_graph.pb \

20
docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_Object_Detection_API_Models.md
View File

@ -1,9 +1,9 @@
# Converting TensorFlow* Object Detection API Models {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Object_Detection_API_Models}
# Convert TensorFlow Object Detection API Models {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Object_Detection_API_Models}
> **NOTES**:
> * Starting with the 2022.1 release, the Model Optimizer can convert the TensorFlow\* Object Detection API Faster and Mask RCNNs topologies differently. By default, the Model Optimizer adds operation "Proposal" to the generated IR. This operation needs an additional input to the model with name "image_info" which should be fed with several values describing the pre-processing applied to the input image (refer to the [Proposal](../../../../ops/detection/Proposal_4.md) operation specification for more information). However, this input is redundant for the models trained and inferred with equal size images. Model Optimizer can generate IR for such models and insert operation [DetectionOutput](../../../../ops/detection/DetectionOutput_1.md) instead of `Proposal`. The `DetectionOutput` operation does not require additional model input "image_info" and moreover, for some models the produced inference results are closer to the original TensorFlow\* model. In order to trigger new behaviour the attribute "operation_to_add" in the corresponding JSON transformation configuration file should be set to value "DetectionOutput" instead of default one "Proposal".
> * Starting with the 2021.1 release, the Model Optimizer converts the TensorFlow\* Object Detection API SSDs, Faster and Mask RCNNs topologies keeping shape-calculating sub-graphs by default, so topologies can be re-shaped in the Inference Engine using dedicated reshape API. Refer to [Using Shape Inference](../../../../OV_Runtime_UG/ShapeInference.md) for more information on how to use this feature. It is possible to change the both spatial dimensions of the input image and batch size.
> * To generate IRs for TF 1 SSD topologies, the Model Optimizer creates a number of `PriorBoxClustered` operations instead of a constant node with prior boxes calculated for the particular input image size. This change allows you to reshape the topology in the Inference Engine using dedicated Inference Engine API. The reshaping is supported for all SSD topologies except FPNs which contain hardcoded shapes for some operations preventing from changing topology input shape.
> * Starting with the 2021.1 release, the Model Optimizer converts the TensorFlow\* Object Detection API SSDs, Faster and Mask RCNNs topologies keeping shape-calculating sub-graphs by default, so topologies can be re-shaped in the OpenVINO Runtime using dedicated reshape API. Refer to [Using Shape Inference](../../../../OV_Runtime_UG/ShapeInference.md) for more information on how to use this feature. It is possible to change the both spatial dimensions of the input image and batch size.
> * To generate IRs for TF 1 SSD topologies, the Model Optimizer creates a number of `PriorBoxClustered` operations instead of a constant node with prior boxes calculated for the particular input image size. This change allows you to reshape the topology in the OpenVINO Runtime using dedicated API. The reshaping is supported for all SSD topologies except FPNs which contain hardcoded shapes for some operations preventing from changing topology input shape.
## How to Convert a Model
@ -45,7 +45,7 @@ To convert a TensorFlow\* Object Detection API model, go to the `<INSTALL_DIR>/t
* `--tensorflow_object_detection_api_pipeline_config <path_to_pipeline.config>` --- A special configuration file that describes the topology hyper-parameters and structure of the TensorFlow Object Detection API model. For the models downloaded from the TensorFlow\* Object Detection API zoo, the configuration file is named `pipeline.config`. If you plan to train a model yourself, you can find templates for these files in the [models repository](https://github.com/tensorflow/models/tree/master/research/object_detection/samples/configs).
* `--input_shape` (optional) --- A custom input image shape. Refer to [Custom Input Shape](#tf_od_custom_input_shape) for more information how the `--input_shape` parameter is handled for the TensorFlow* Object Detection API models.
> **NOTE**: The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the `RGB<->BGR` conversion specifying the command-line parameter: `--reverse_input_channels`. Otherwise, inference results may be incorrect. If you convert a TensorFlow\* Object Detection API model to use with the Inference Engine sample applications, you must specify the `--reverse_input_channels` parameter. For more information about the parameter, refer to **When to Reverse Input Channels** section of [Converting a Model to Intermediate Representation (IR)](../Converting_Model.md).
> **NOTE**: The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the `RGB<->BGR` conversion specifying the command-line parameter: `--reverse_input_channels`. Otherwise, inference results may be incorrect. If you convert a TensorFlow\* Object Detection API model to use with the OpenVINO sample applications, you must specify the `--reverse_input_channels` parameter. For more information about the parameter, refer to **When to Reverse Input Channels** section of [Converting a Model to Intermediate Representation (IR)](../Converting_Model.md).
Additionally to the mandatory parameters listed above you can use optional conversion parameters if needed. A full list of parameters is available in the [Converting a TensorFlow* Model](../Convert_Model_From_TensorFlow.md) topic.
@ -57,24 +57,24 @@ mo --input_model=/tmp/ssd_inception_v2_coco_2018_01_28/frozen_inference_graph.pb
## OpenVINO&; Toolkit Samples and Open Model Zoo Demos
Inference Engine comes with a number of samples to demonstrate use of OpenVINO API, additionally,
OpenVINO comes with a number of samples to demonstrate use of OpenVINO Runtime API, additionally,
Open Model Zoo provides set of demo applications to show implementation of close to real life applications
based on deep learning in various tasks, including Image Classifiacton, Visual Object Detection, Text Recognition,
Speech Recognition, Natural Language Processing and others. Refer to the links below for more details.
* [Inference Engine Samples](../../../../OV_Runtime_UG/Samples_Overview.md)
* [OpenVINO Samples](../../../../OV_Runtime_UG/Samples_Overview.md)
* [Open Model Zoo Demos](@ref omz_demos)
## Important Notes About Feeding Input Images to the Samples
There are several important notes about feeding input images to the samples:
1. Inference Engine samples stretch input image to the size of the input operation without preserving aspect ratio. This behavior is usually correct for most topologies (including SSDs), but incorrect for other models like Faster R-CNN, Mask R-CNN and R-FCN. These models usually use keeps aspect ratio resizer. The type of pre-processing is defined in the pipeline configuration file in the section `image_resizer`. If keeping aspect ratio is used, then it is necessary to resize image before passing it to the sample and optionally pad the resized image with 0s (if the attribute "pad_to_max_dimension" in the pipeline.config is equal to "true").
1. OpenVINO samples stretch input image to the size of the input operation without preserving aspect ratio. This behavior is usually correct for most topologies (including SSDs), but incorrect for other models like Faster R-CNN, Mask R-CNN and R-FCN. These models usually use keeps aspect ratio resizer. The type of pre-processing is defined in the pipeline configuration file in the section `image_resizer`. If keeping aspect ratio is used, then it is necessary to resize image before passing it to the sample and optionally pad the resized image with 0s (if the attribute "pad_to_max_dimension" in the pipeline.config is equal to "true").
2. TensorFlow\* implementation of image resize may be different from the one implemented in the sample. Even reading input image from compressed format (like `.jpg`) could give different results in the sample and TensorFlow\*. So, if it is necessary to compare accuracy between the TensorFlow\* and the Inference Engine it is recommended to pass pre-resized input image in a non-compressed format (like `.bmp`).
2. TensorFlow\* implementation of image resize may be different from the one implemented in the sample. Even reading input image from compressed format (like `.jpg`) could give different results in the sample and TensorFlow\*. So, if it is necessary to compare accuracy between the TensorFlow\* and the OpenVINO it is recommended to pass pre-resized input image in a non-compressed format (like `.bmp`).
3. If you want to infer the model with the Inference Engine samples, convert the model specifying the `--reverse_input_channels` command line parameter. The samples load images in BGR channels order, while TensorFlow* models were trained with images in RGB order. When the `--reverse_input_channels` command line parameter is specified, the Model Optimizer performs first convolution or other channel dependent operation weights modification so the output will be like the image is passed with RGB channels order.
3. If you want to infer the model with the OpenVINO samples, convert the model specifying the `--reverse_input_channels` command line parameter. The samples load images in BGR channels order, while TensorFlow* models were trained with images in RGB order. When the `--reverse_input_channels` command line parameter is specified, the Model Optimizer performs first convolution or other channel dependent operation weights modification so the output will be like the image is passed with RGB channels order.
4. Read carefully messaged printed by the Model Optimizer during a model conversion. They contain important instructions on how to prepare input data before running the inference and how to interpret the output.
@ -128,7 +128,7 @@ Models with `keep_aspect_ratio_resizer` were trained to recognize object in real
## Detailed Explanations of Model Conversion Process
This section is intended for users who want to understand how the Model Optimizer performs Object Detection API models conversion in details. The knowledge given in this section is also useful for users having complex models that are not converted with the Model Optimizer out of the box. It is highly recommended to read [Sub-Graph Replacement in Model Optimizer](../../customize_model_optimizer/Subgraph_Replacement_Model_Optimizer.md) chapter first to understand sub-graph replacement concepts which are used here.
This section is intended for users who want to understand how the Model Optimizer performs Object Detection API models conversion in details. The knowledge given in this section is also useful for users having complex models that are not converted with the Model Optimizer out of the box. It is highly recommended to read the **Graph Transformation Extensions** section in the [Model Optimizer Extensibility](../../customize_model_optimizer/Customize_Model_Optimizer.md) documentation first to understand sub-graph replacement concepts which are used here.
It is also important to open the model in the [TensorBoard](https://www.tensorflow.org/guide/summaries_and_tensorboard) to see the topology structure. Model Optimizer can create an event file that can be then fed to the TensorBoard* tool. Run the Model Optimizer with providing two command line parameters:
* `--input_model <path_to_frozen.pb>` --- Path to the frozen model

4
docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_RetinaNet_From_Tensorflow.md
View File

@ -1,8 +1,8 @@
# Converting RetinaNet Model from TensorFlow* to the Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_RetinaNet_From_Tensorflow}
# Converting TensorFlow RetinaNet Model {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_RetinaNet_From_Tensorflow}
This tutorial explains how to convert RetinaNet model to the Intermediate Representation (IR).
[Public RetinaNet model](https://github.com/fizyr/keras-retinanet) does not contain pretrained TensorFlow\* weights.
[Public RetinaNet model](https://github.com/fizyr/keras-retinanet) does not contain pretrained TensorFlow\* weights.
To convert this model to the TensorFlow\* format, you can use [Reproduce Keras* to TensorFlow* Conversion tutorial](https://docs.openvino.ai/latest/omz_models_model_retinanet_tf.html).
After you convert the model to TensorFlow* format, run the Model Optimizer command below:

14
docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_Slim_Library_Models.md
View File

@ -1,13 +1,13 @@
# Converting TensorFlow*-Slim Image Classification Model Library Models {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Slim_Library_Models}
# Convert TensorFlow Slim Image Classification Model Library Models {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Slim_Library_Models}
<a href="https://github.com/tensorflow/models/tree/master/research/slim/README.md">TensorFlow\*-Slim Image Classification Model Library</a> is a library to define, train and evaluate classification models in TensorFlow\*. The library contains Python scripts defining the classification topologies together with checkpoint files for several pre-trained classification topologies. To convert a TensorFlow\*-Slim library model, complete the following steps:
<a href="https://github.com/tensorflow/models/tree/master/research/slim/README.md">TensorFlow\*-Slim Image Classification Model Library</a> is a library to define, train and evaluate classification models in TensorFlow\*. The library contains Python scripts defining the classification topologies together with checkpoint files for several pre-trained classification topologies. To convert a TensorFlow\*-Slim library model, complete the following steps:
1. Download the TensorFlow\*-Slim models [git repository](https://github.com/tensorflow/models).
2. Download the pre-trained model [checkpoint](https://github.com/tensorflow/models/tree/master/research/slim#pre-trained-models).
3. Export the inference graph.
4. Convert the model using the Model Optimizer.
The [Example of an Inception V1 Model Conversion](#example_of_an_inception_v1_model_conversion) section below illustrates the process of converting an Inception V1 Model.
The [Example of an Inception V1 Model Conversion](#example_of_an_inception_v1_model_conversion) section below illustrates the process of converting an Inception V1 Model.
## Example of an Inception V1 Model Conversion <a name="example_of_an_inception_v1_model_conversion"></a>
This example demonstrates how to convert the model on Linux\* OSes, but it could be easily adopted for the Windows\* OSes.
@ -39,7 +39,7 @@ python3 tf_models/research/slim/export_inference_graph.py \
```
Model Optimizer comes with the summarize graph utility, which identifies graph input and output nodes. Run the utility to determine input/output nodes of the Inception V1 model:
```sh
python3 <PYTHON_SITE_PACKAGES>/openvino/tools/mo/utils/summarize_graph.py --input_model ./inception_v1_inference_graph.pb
```
@ -64,9 +64,9 @@ The `-b` command line parameter is required because the Model Optimizer cannot c
Refer to the [Mean and Scale Values for TensorFlow\*-Slim Models](#tf_slim_mean_scale_values) for the information why `--mean_values` and `--scale` command line parameters are used.
## Mean and Scale Values for TensorFlow\*-Slim Models <a name="tf_slim_mean_scale_values"></a>
The TensorFlow\*-Slim Models were trained with normalized input data. There are several different normalization algorithms used in the Slim library. Inference Engine classification sample does not perform image pre-processing except resizing to the input layer size. It is necessary to pass mean and scale values to the Model Optimizer so they are embedded into the generated IR in order to get correct classification results.
The TensorFlow\*-Slim Models were trained with normalized input data. There are several different normalization algorithms used in the Slim library. OpenVINO classification sample does not perform image pre-processing except resizing to the input layer size. It is necessary to pass mean and scale values to the Model Optimizer so they are embedded into the generated IR in order to get correct classification results.
The file [preprocessing_factory.py](https://github.com/tensorflow/models/blob/master/research/slim/preprocessing/preprocessing_factory.py) contains a dictionary variable `preprocessing_fn_map` defining mapping between the model type and pre-processing function to be used. The function code should be analyzed to figure out the mean/scale values.
The file [preprocessing_factory.py](https://github.com/tensorflow/models/blob/master/research/slim/preprocessing/preprocessing_factory.py) contains a dictionary variable `preprocessing_fn_map` defining mapping between the model type and pre-processing function to be used. The function code should be analyzed to figure out the mean/scale values.
The [inception_preprocessing.py](https://github.com/tensorflow/models/blob/master/research/slim/preprocessing/inception_preprocessing.py) file defines the pre-processing function for the Inception models. The `preprocess_for_eval` function contains the following code:
@ -83,7 +83,7 @@ The [inception_preprocessing.py](https://github.com/tensorflow/models/blob/maste
Firstly, the `image` is converted to data type `tf.float32` and the values in the tensor are scaled to the `[0, 1]` range using the [tf.image.convert_image_dtype](https://www.tensorflow.org/api_docs/python/tf/image/convert_image_dtype) function. Then the `0.5` is subtracted from the image values and values multiplied by `2.0`. The final image range of values is `[-1, 1]`.
Inference Engine classification sample reads an input image as a three-dimensional array of integer values from the range `[0, 255]`. In order to scale them to `[-1, 1]` range, the mean value `127.5` for each image channel should be specified as well as scale factor `127.5`.
OpenVINO classification sample reads an input image as a three-dimensional array of integer values from the range `[0, 255]`. In order to scale them to `[-1, 1]` range, the mean value `127.5` for each image channel should be specified as well as scale factor `127.5`.
Similarly, the mean/scale values can be determined for other Slim models.

10
docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_WideAndDeep_Family_Models.md
View File

@ -1,4 +1,4 @@
# Converting TensorFlow* Wide and Deep Family Models to the Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_WideAndDeep_Family_Models}
# Convert TensorFlow Wide and Deep Family Models {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_WideAndDeep_Family_Models}
The Wide and Deep models is a combination of wide and deep parts for memorization and generalization of object features respectively.
These models can contain different types of object features such as numerical, categorical, sparse and sequential features. These feature types are specified
@ -24,7 +24,7 @@ The Wide and Deep model is no longer in the master branch of the repository but
**Step 2**. Train the model
As the OpenVINO&trade; toolkit does not support the categorical with hash and crossed features, such feature types must be switched off in the model
As the OpenVINO&trade; toolkit does not support the categorical with hash and crossed features, such feature types must be switched off in the model
by changing the `build_model_columns()` function in `census_dataset.py` as follows:
```python
@ -61,7 +61,7 @@ def build_model_columns():
age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65])
# Wide columns and deep columns.
base_columns = [
education, marital_status, relationship, workclass,
education, marital_status, relationship, workclass,
age_buckets,
]
crossed_columns = []
@ -92,7 +92,7 @@ python census_main.py
Use the following command line to convert the saved model file with the checkpoint:
```sh
mo
mo
--input_checkpoint checkpoint --input_meta_graph model.ckpt.meta
--input "IteratorGetNext:0[2],
IteratorGetNext:1[2],
@ -122,7 +122,7 @@ Use the following command line to convert the saved model file with the checkpoi
dnn/input_from_feature_columns/input_layer/relationship_indicator/to_sparse_input/dense_shape:0[2]{i64}->[2 50],
dnn/input_from_feature_columns/input_layer/workclass_indicator/to_sparse_input/indices:0[10 2]{i64},
dnn/input_from_feature_columns/input_layer/workclass_indicator/hash_table_Lookup/LookupTableFindV2:0[10]{i64},
dnn/input_from_feature_columns/input_layer/workclass_indicator/to_sparse_input/dense_shape:0[2]{i64}->[2 50]"
dnn/input_from_feature_columns/input_layer/workclass_indicator/to_sparse_input/dense_shape:0[2]{i64}->[2 50]"
--output head/predictions/probabilities
```

12
docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_XLNet_From_Tensorflow.md
View File

@ -1,4 +1,4 @@
# Convert TensorFlow* XLNet Model to the Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_XLNet_From_Tensorflow}
# Convert TensorFlow XLNet Model {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_XLNet_From_Tensorflow}
Pre-trained models for XLNet (Bidirectional Encoder Representations from Transformers) are
[publicly available](https://github.com/zihangdai/xlnet).
@ -16,8 +16,8 @@ Download and unzip an archive with the [XLNet-Base, Cased](https://storage.googl
After the archive is unzipped, the directory `cased_L-12_H-768_A-12` is created and contains the following files:
* TensorFlow checkpoint (`xlnet_model.ckpt`) containing the pre-trained weights (which is actually 3 files)
* sentence piece model (`spiece.model`) used for (de)tokenization
* config file (`xlnet_config.json`) which specifies the hyperparameters of the model
* sentence piece model (`spiece.model`) used for (de)tokenization
* config file (`xlnet_config.json`) which specifies the hyperparameters of the model
To get pb-file from the archive contents, you need to do the following.
@ -33,7 +33,7 @@ To get pb-file from the archive contents, you need to do the following.
mkdir try_save
```
2. Save and run the following Python script in `~/XLNet-Base/xlnet`:
@ -102,8 +102,8 @@ Download and unzip an archive with the [XLNet-Large, Cased](https://storage.goog
After the archive is unzipped, the directory `cased_L-12_H-1024_A-16` is created and contains the following files:
* TensorFlow checkpoint (`xlnet_model.ckpt`) containing the pre-trained weights (which is actually 3 files)
* sentence piece model (`spiece.model`) used for (de)tokenization
* config file (`xlnet_config.json`) which specifies the hyperparameters of the model
* sentence piece model (`spiece.model`) used for (de)tokenization
* config file (`xlnet_config.json`) which specifies the hyperparameters of the model
To get pb-file from the archive contents, you need to do the following.

18
docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_YOLO_From_Tensorflow.md
View File

@ -1,7 +1,7 @@
# Converting YOLO* Models to the Intermediate Representation (IR) {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_YOLO_From_Tensorflow}
# Convert TensorFlow YOLO Models {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_YOLO_From_Tensorflow}
This document explains how to convert real-time object detection YOLOv1\*, YOLOv2\*, YOLOv3\* and YOLOv4\* public models to the Intermediate Representation (IR). All YOLO\* models are originally implemented in the DarkNet\* framework and consist of two files:
* `.cfg` file with model configurations
* `.cfg` file with model configurations
* `.weights` file with model weights
Depending on a YOLO model version, the Model Optimizer converts it differently:
@ -35,9 +35,9 @@ python keras-YOLOv3-model-set/tools/model_converter/convert.py <path_to_cfg_file
4. Run Model Optimizer to converter the model from the TensorFlow 2 format to an IR:
> **NOTE:** Before you run the conversion, make sure you have installed all the Model Optimizer dependencies for TensorFlow 2.
> **NOTE**: Before you run the conversion, make sure you have installed all the Model Optimizer dependencies for TensorFlow 2.
```sh
mo --saved_model_dir yolov4 --output_dir models/IRs --input_shape [1,608,608,3] --model_name yolov4
mo --saved_model_dir yolov4 --output_dir models/IRs --input_shape [1,608,608,3] --model_name yolov4
```
## <a name="yolov3-to-ir"></a>Convert YOLOv3 Model to IR
@ -115,7 +115,7 @@ It consists of several attributes:<br>
where:
- `id` and `match_kind` are parameters that you cannot change.
- `custom_attributes` is a parameter that stores all the YOLOv3 specific attributes:
- `classes`, `coords`, `num`, and `masks` are attributes that you should copy from the configuration
- `classes`, `coords`, `num`, and `masks` are attributes that you should copy from the configuration
file that was used for model training. If you used DarkNet officially shared weights,
you can use `yolov3.cfg` or `yolov3-tiny.cfg` configuration file from https://github.com/david8862/keras-YOLOv3-model-set/tree/master/cfg. Replace the default values in `custom_attributes` with the parameters that
follow the `[yolo]` titles in the configuration file.
@ -184,8 +184,8 @@ To convert YOLOv1 or YOLOv2 model to TensorFlow, go to the root directory of the
python3 flow --model yolov1.cfg --load yolov1.weights --savepb
```
- For YOLOv2 with VOC dataset `--labels` argument should be specified and additional changes in the original exporting script are required.
In the file [https://github.com/thtrieu/darkflow/blob/b187c65/darkflow/utils/loader.py#L121](https://github.com/thtrieu/darkflow/blob/b187c65630f9aa1bb8b809c33ec67c8cc5d60124/darkflow/utils/loader.py#L121)
- For YOLOv2 with VOC dataset `--labels` argument should be specified and additional changes in the original exporting script are required.
In the file [https://github.com/thtrieu/darkflow/blob/b187c65/darkflow/utils/loader.py#L121](https://github.com/thtrieu/darkflow/blob/b187c65630f9aa1bb8b809c33ec67c8cc5d60124/darkflow/utils/loader.py#L121)
change line 121 from `self.offset = 16` to `self.offset = 20`. Then run:
```sh
python3 flow --model yolov2-voc.cfg --load yolov2-voc.weights --labels voc-labels.txt --savepb
@ -204,7 +204,7 @@ File `<model_name>.pb` is a TensorFlow representation of the YOLO model.
#### <a name="yolov1-v2-to-ir"></a>Convert TensorFlow YOLOv1 or YOLOv2 Model to the IR
Converted TensorFlow YOLO model is missing `Region` layer and its parameters. Original YOLO `Region` layer parameters are stored in the configuration `<path_to_model>/<model_name>.cfg`
file under the `[region]` title.
file under the `[region]` title.
To recreate the original model structure, use the corresponding yolo `.json` configuration file with custom operations and `Region` layer
parameters when converting the model to the IR. This file is located in the `<OPENVINO_INSTALL_DIR>/tools/model_optimizer/extensions/front/tf` directory.
@ -223,7 +223,7 @@ To generate the IR of the YOLOv1 model, provide TensorFlow YOLOv1 or YOLOv2 mode
where:
* `--batch` defines shape of model input. In the example, `--batch` is equal to 1, but you can also specify other integers larger than 1.
* `--scale` specifies scale factor that input values will be divided by.
* `--scale` specifies scale factor that input values will be divided by.
The model was trained with input values in the range `[0,1]`. OpenVINO&trade; toolkit samples read input images as values in `[0,255]` range, so the scale 255 must be applied.
* `--transformations_config` adds missing `Region` layers to the model. In the IR, the `Region` layer has name `RegionYolo`.
For other applicable parameters, refer to [Convert Model from TensorFlow](../Convert_Model_From_TensorFlow.md).

18
docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_lm_1b_From_Tensorflow.md
View File

@ -1,4 +1,4 @@
# Converting TensorFlow* Language Model on One Billion Word Benchmark to the Intermediate Representation {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_lm_1b_From_Tensorflow}
# Convert TensorFlow Language Model on One Billion Word Benchmark {#openvino_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_lm_1b_From_Tensorflow}
## Download the Pre-trained Language Model on One Billion Word Benchmark
@ -51,14 +51,14 @@ lm_1b/
ckpt-char-embedding
ckpt-lstm
ckpt-softmax0
ckpt-softmax1
ckpt-softmax2
ckpt-softmax3
ckpt-softmax4
ckpt-softmax5
ckpt-softmax6
ckpt-softmax7
ckpt-softmax8
ckpt-softmax1
ckpt-softmax2
ckpt-softmax3
ckpt-softmax4
ckpt-softmax5
ckpt-softmax6
ckpt-softmax7
ckpt-softmax8
```

31
docs/MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md
View File

@ -10,7 +10,7 @@
@endsphinxdirective
<a name="model-optimizer-extensibility"></a>Model Optimizer extensibility mechanism enables support of new operations and custom transformations to generate the optimized intermediate representation (IR) as described in the
<a name="model-optimizer-extensibility"></a>Model Optimizer extensibility mechanism enables support of new operations and custom transformations to generate the optimized intermediate representation (IR) as described in the
[Deep Learning Network Intermediate Representation and Operation Sets in OpenVINO™](../../IR_and_opsets.md). This
mechanism is a core part of the Model Optimizer. The Model Optimizer itself uses it under the hood, being a huge set of examples on how to add custom logic to support your model.
@ -144,7 +144,7 @@ OpenVINO&trade; [TopK](../../../ops/sort/TopK_3.md) operation semantic, which re
It is important to mention that sometimes it seems like transformation cannot be implemented during the front phase
because the actual values of inputs or shapes are needed. But in fact shapes or values manipulations can be implemented
using operations that are added to the graph. Consider the
using operations that are added to the graph. Consider the
`extensions/front/onnx/flattenONNX_to_reshape.py` transformation, which replaces an ONNX\* operation
[Flatten](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Flatten) with a sub-graph of operations performing
the following (for the case when `axis` is not equal to 0 and 1):
@ -177,9 +177,9 @@ defined as a mathematical expression using the [ShapeOf](../../../ops/shape/Shap
Model Optimizer calculates output shapes for all operations in a model to write them to Intermediate Representation
files.
> **NOTE**: This is a legacy requirement because starting from IR version 10 Inference Engine needs to know shapes of
> **NOTE**: This is a legacy requirement because starting from IR version 10 OpenVINO Runtime needs to know shapes of
> the [Const](../../../ops/infrastructure/Constant_1.md) and the [Parameter](../../../ops/infrastructure/Parameter_1.md)
> operations only. The nGraph component of the Inference Engine calculates output shapes for all operations in a model
> operations only. The OpenVINO Runtime calculates output shapes for all operations in a model
> using shapes of [Parameter](../../../ops/infrastructure/Parameter_1.md) and
> [Const](../../../ops/infrastructure/Constant_1.md) operations defined with respective operation attributes.
@ -257,11 +257,13 @@ More information on how to develop middle transformations and dedicated API desc
[Middle Phase Transformations](#middle-phase-transformations).
### NHWC to NCHW Layout Change <a name="layout-change"></a>
There are several middle transformations responsible for changing model layout from NHWC to NCHW. These transformations
are triggered by default for TensorFlow\* models only because it is the only framework with Convolution operations in
NHWC layout. This layout change is disabled if the model does not have operations that OpenVINO&trade needs to execute in
NCHW layout, for example, Convolutions in NHWC layout. It is still possible to force Model Optimizer to do layout change
using `--disable_nhwc_to_nchw` command-line parameter.
There are several middle transformations responsible for changing model layout from NHWC to NCHW. These transformations are triggered by default for TensorFlow models as TensorFlow supports Convolution operations in the NHWC layout.
This layout change is disabled automatically if the model does not have operations that OpenVINO&trade needs to execute in the NCHW layout, for example, Convolutions in NHWC layout.
It is still possible to force Model Optimizer to do layout change, using `--disable_nhwc_to_nchw` command-line parameter, although it is not advised.
The layout change is a complex problem and detailed explanation of it is out of this document scope. A very brief
explanation of this process is provided below:
@ -285,7 +287,7 @@ The back phase starts after the layout change to NCHW. This phase contains mostl
1. Transformations that should work with a graph in the NCHW layout and thus cannot be implemented in the middle
phase.
2. Transformations that replace nodes corresponding to internal Model Optimizer operations with nodes corresponding to the
2. Transformations that replace nodes corresponding to internal Model Optimizer operations with nodes corresponding to the
[opset](@ref openvino_docs_ops_opset) operations.
3. Transformations that normalize operations inputs according to the specification.
4. Final optimization transformations.
@ -301,7 +303,7 @@ The last phase of a model conversion is the Intermediate Representation emitting
steps:
1. Iterates over all operation nodes in the graph and checks that all nodes have the `type` attribute set. This attribute
defines the operation type and is used in the Inference Engine to instantiate proper operation from the
defines the operation type and is used in the OpenVINO to instantiate proper operation from the
[opset](@ref openvino_docs_ops_opset) specified in the `version` attribute of the node. If some node does not have
attribute `type` or its values is equal to `None`, the Model Optimizer exits with an error.
2. Performs type inference of graph operations similar to the shape inference. Inferred data types are saved to a port
@ -701,7 +703,7 @@ to enable or disable execution of the transformation during a model conversion.
2. Attribute `id` specifies a unique transformation string identifier. This transformation identifier can be used to
enable (disable) the transformation by setting environment variable `MO_ENABLED_TRANSFORMS` (`MO_DISABLED_TRANSFORMS`)
with a comma separated list of `id`s. The environment variables override the value of the `enabled` attribute of the
transformation. Instead of using `id` attribute value you can add fully defined class name to `MO_ENABLED_TRANSFORMS`
transformation. Instead of using `id` attribute value you can add fully defined class name to `MO_ENABLED_TRANSFORMS`
(`MO_DISABLED_TRANSFORMS`) variable, `extensions.back.NonmalizeToNormalizeL2.NormalizeToNormalizeL2` for example. Optional attribute.
3. Attribute `run_not_recursively` specifies whether the transformation should be executed in the sub-graphs, for
example, body of the [TensorIterator](../../../ops/infrastructure/TensorIterator_1.md) and
@ -741,8 +743,7 @@ sub-graph of the original graph isomorphic to the specified pattern.
2. [Specific Operation Front Phase Transformations](#specific-operation-front-phase-transformations) triggered for the
node with a specific `op` attribute value.
3. [Generic Front Phase Transformations](#generic-front-phase-transformations).
4. Manually enabled transformation defined with a JSON configuration file (for TensorFlow\*, ONNX\* and MXNet\* models
only) specified using the `--transformations_config` command line parameter:
4. Manually enabled transformation defined with a JSON configuration file (for TensorFlow, ONNX, MXNet, and PaddlePaddle models) specified using the `--transformations_config` command line parameter:
1. [Node Name Pattern Front Phase Transformations](#node-name-pattern-front-phase-transformation).
2. [Front Phase Transformations Using Start and End Points](#start-end-points-front-phase-transformations).
3. [Generic Front Phase Transformations Enabled with Transformations Configuration File](#generic-transformations-config-front-phase-transformations).
@ -1260,5 +1261,5 @@ Refer to the `extensions/back/GatherNormalizer.py` for the example of a such typ
* [Deep Learning Network Intermediate Representation and Operation Sets in OpenVINO™](../../IR_and_opsets.md)
* [Converting a Model to Intermediate Representation (IR)](../convert_model/Converting_Model.md)
* [OpenVINO Model Representation](../../../OV_Runtime_UG/model_representation.md)
* [Inference Engine Extensibility Mechanism](../../../OV_Runtime_UG/Extensibility_DG/Intro.md)
* [OpenVINO™ Extensibility Mechanism](../../../Extensibility_UG/Intro.md)
* [Extending the Model Optimizer with Caffe* Python Layers](Extending_Model_Optimizer_with_Caffe_Python_Layers.md)

4
docs/MO_DG/prepare_model/customize_model_optimizer/Subgraph_Replacement_Model_Optimizer.md
View File

@ -1,4 +0,0 @@
# [DEPRECATED] Sub-Graph Replacement in the Model Optimizer {#openvino_docs_MO_DG_prepare_model_customize_model_optimizer_Subgraph_Replacement_Model_Optimizer}
The document has been deprecated. Refer to the [Model Optimizer Extensibility](Customize_Model_Optimizer.md)
for the up-to-date documentation.

10
docs/OV_Runtime_UG/API_Changes.md
View File

@ -1,10 +0,0 @@
# OpenVINO™ Runtime API Changes History {#openvino_docs_OV_Runtime_API_Changes}
The sections below contain detailed list of changes made to the OpenVINO™ Runtime API in recent releases.
## 2022.1
### New API
* The OpenVINO™ 2.0 API was introduced.

214
docs/OV_Runtime_UG/Bfloat16Inference.md
View File

@ -1,214 +0,0 @@
# Bfloat16 Inference {#openvino_docs_IE_DG_Bfloat16Inference}
## Bfloat16 Inference Usage (C++)
@sphinxdirective
.. raw:: html
<div id="switcher-cpp" class="switcher-anchor">C++</div>
@endsphinxdirective
### Disclaimer
Inference Engine with the bfloat16 inference implemented on CPU must support the native *avx512_bf16* instruction and therefore the bfloat16 data format. It is possible to use bfloat16 inference in simulation mode on platforms with Intel® Advanced Vector Extensions 512 (Intel® AVX-512), but it leads to significant performance degradation in comparison with FP32 or native *avx512_bf16* instruction usage.
### Introduction
Bfloat16 computations (referred to as BF16) is the Brain Floating-Point format with 16 bits. This is a truncated 16-bit version of the 32-bit IEEE 754 single-precision floating-point format FP32. BF16 preserves 8 exponent bits as FP32 but reduces precision of the sign and mantissa from 24 bits to 8 bits.
![bf16_format]
Preserving the exponent bits keeps BF16 to the same range as the FP32 (~1e-38 to ~3e38). This simplifies conversion between two data types: you just need to skip or flush to zero 16 low bits. Truncated mantissa leads to occasionally less precision, but according to [investigations](https://cloud.google.com/blog/products/ai-machine-learning/bfloat16-the-secret-to-high-performance-on-cloud-tpus), neural networks are more sensitive to the size of the exponent than the mantissa size. Also, in lots of models, precision is needed close to zero but not so much at the maximum range. Another useful feature of BF16 is possibility to encode INT8 in BF16 without loss of accuracy, because INT8 range completely fits in BF16 mantissa field. It reduces data flow in conversion from INT8 input image data to BF16 directly without intermediate representation in FP32, or in combination of [INT8 inference](Int8Inference.md) and BF16 layers.
See the [BFLOAT16 – Hardware Numerics Definition white paper](https://software.intel.com/content/dam/develop/external/us/en/documents/bf16-hardware-numerics-definition-white-paper.pdf) for more bfloat16 format details.
There are two ways to check if CPU device can support bfloat16 computations for models:
1. Query the instruction set using one of these system commands:
* `lscpu | grep avx512_bf16`
* `cat /proc/cpuinfo | grep avx512_bf16`
2. Use the [Query API](InferenceEngine_QueryAPI.md) with `METRIC_KEY(OPTIMIZATION_CAPABILITIES)`, which should return `BF16` in the list of CPU optimization options:
@snippet snippets/Bfloat16Inference0.cpp part0
The current Inference Engine solution for bfloat16 inference uses the Intel® Math Kernel Library for Deep Neural Networks (Intel® MKL-DNN) and supports inference of the significant number of layers in BF16 computation mode.
### Lowering Inference Precision
Lowering precision to increase performance is [widely used](https://software.intel.com/content/www/us/en/develop/articles/lower-numerical-precision-deep-learning-inference-and-training.html) for optimization of inference. The bfloat16 data type usage on CPU for the first time opens the possibility of default optimization approach. The embodiment of this approach is to use the optimization capabilities of the current platform to achieve maximum performance while maintaining the accuracy of calculations within the acceptable range.
Using Bfloat16 precision provides the following performance benefits:
1. Faster multiplication of two BF16 numbers because of shorter mantissa of bfloat16 data.
2. No need to support denormals and handling exceptions as this is a performance optimization.
3. Fast conversion of float32 to bfloat16 and vice versa.
4. Reduced size of data in memory, as a result, larger models fit in the same memory bounds.
5. Reduced amount of data that must be transferred, as a result, reduced data transition time.
For default optimization on CPU, the source model is converted from FP32 or FP16 to BF16 and executed internally on platforms with native BF16 support. In this case, `KEY_ENFORCE_BF16` is set to `YES` in the `PluginConfigParams` for `GetConfig()`. The code below demonstrates how to check if the key is set:
@snippet snippets/Bfloat16Inference1.cpp part1
To disable BF16 internal transformations in C++ API, set the `KEY_ENFORCE_BF16` to `NO`. In this case, the model infers as is without modifications with precisions that were set on each layer edge.
@snippet snippets/Bfloat16Inference2.cpp part2
To disable BF16 in C API:
```
ie_config_t config = { "ENFORCE_BF16", "NO", NULL};
ie_core_load_network(core, network, device_name, &config, &exe_network);
```
An exception with the message `Platform doesn't support BF16 format` is formed in case of setting `KEY_ENFORCE_BF16` to `YES` on CPU without native BF16 support or BF16 simulation mode.
Low-Precision 8-bit integer models cannot be converted to BF16, even if bfloat16 optimization is set by default.
### Bfloat16 Simulation Mode
Bfloat16 simulation mode is available on CPU and Intel® AVX-512 platforms that do not support the native `avx512_bf16` instruction. The simulator does not guarantee good performance. Note that the CPU must still support the AVX-512 extensions.
To enable the simulation of Bfloat16:
* In the [Benchmark App](../../samples/cpp/benchmark_app/README.md), add the `-enforcebf16=true` option
* In C++ API, set `KEY_ENFORCE_BF16` to `YES`
* In C API:
```
ie_config_t config = { "ENFORCE_BF16", "YES", NULL};
ie_core_load_network(core, network, device_name, &config, &exe_network);
```
### Performance Counters
Information about layer precision is stored in the performance counters that are available from the Inference Engine API. The layers have the following marks:
* Suffix `BF16` for layers that had bfloat16 data type input and were computed in BF16 precision
* Suffix `FP32` for layers computed in 32-bit precision
For example, the performance counters table for the Inception model can look as follows:
```
pool5 EXECUTED layerType: Pooling realTime: 143 cpu: 143 execType: jit_avx512_BF16
fc6 EXECUTED layerType: FullyConnected realTime: 47723 cpu: 47723 execType: jit_gemm_BF16
relu6 NOT_RUN layerType: ReLU realTime: 0 cpu: 0 execType: undef
fc7 EXECUTED layerType: FullyConnected realTime: 7558 cpu: 7558 execType: jit_gemm_BF16
relu7 NOT_RUN layerType: ReLU realTime: 0 cpu: 0 execType: undef
fc8 EXECUTED layerType: FullyConnected realTime: 2193 cpu: 2193 execType: jit_gemm_BF16
prob EXECUTED layerType: SoftMax realTime: 68 cpu: 68 execType: jit_avx512_FP32
```
The **execType** column of the table includes inference primitives with specific suffixes.
## Bfloat16 Inference Usage (Python)
@sphinxdirective
.. raw:: html
<div id="switcher-python" class="switcher-anchor">Python</div>
@endsphinxdirective
### Disclaimer
Inference Engine with the bfloat16 inference implemented on CPU must support the native *avx512_bf16* instruction and therefore the bfloat16 data format. It is possible to use bfloat16 inference in simulation mode on platforms with Intel® Advanced Vector Extensions 512 (Intel® AVX-512), but it leads to significant performance degradation in comparison with FP32 or native *avx512_bf16* instruction usage.
### Introduction
Bfloat16 computations (referred to as BF16) is the Brain Floating-Point format with 16 bits. This is a truncated 16-bit version of the 32-bit IEEE 754 single-precision floating-point format FP32. BF16 preserves 8 exponent bits as FP32 but reduces precision of the sign and mantissa from 24 bits to 8 bits.
![bf16_format]
Preserving the exponent bits keeps BF16 to the same range as the FP32 (~1e-38 to ~3e38). This simplifies conversion between two data types: you just need to skip or flush to zero 16 low bits. Truncated mantissa leads to occasionally less precision, but according to investigations, neural networks are more sensitive to the size of the exponent than the mantissa size. Also, in lots of models, precision is needed close to zero but not so much at the maximum range. Another useful feature of BF16 is possibility to encode INT8 in BF16 without loss of accuracy, because INT8 range completely fits in BF16 mantissa field. It reduces data flow in conversion from INT8 input image data to BF16 directly without intermediate representation in FP32, or in combination of [INT8 inference](Int8Inference.md) and BF16 layers.
See the [BFLOAT16 – Hardware Numerics Definition white paper](https://software.intel.com/content/dam/develop/external/us/en/documents/bf16-hardware-numerics-definition-white-paper.pdf) for more bfloat16 format details.
There are two ways to check if CPU device can support bfloat16 computations for models:
1. Query the instruction set using one of these system commands:
* `lscpu | grep avx512_bf16`
* `cat /proc/cpuinfo | grep avx512_bf16`
2. Use the Query API with METRIC_KEY(OPTIMIZATION_CAPABILITIES), which should return BF16 in the list of CPU optimization options:
```python
from openvino.inference_engine import IECore
ie = IECore()
net = ie.read_network(path_to_xml_file)
cpu_caps = ie.get_metric(metric_name="OPTIMIZATION_CAPABILITIES", device_name="CPU")
```
The current Inference Engine solution for bfloat16 inference uses the Intel® Math Kernel Library for Deep Neural Networks (Intel® MKL-DNN) and supports inference of the significant number of layers in BF16 computation mode.
### Lowering Inference Precision
Lowering precision to increase performance is widely used for optimization of inference. The bfloat16 data type usage on CPU for the first time opens the possibility of default optimization approach. The embodiment of this approach is to use the optimization capabilities of the current platform to achieve maximum performance while maintaining the accuracy of calculations within the acceptable range.
Using Bfloat16 precision provides the following performance benefits:
1. Faster multiplication of two BF16 numbers because of shorter mantissa of bfloat16 data.
2. No need to support denormals and handling exceptions as this is a performance optimization.
3. Fast conversion of float32 to bfloat16 and vice versa.
4. Reduced size of data in memory, as a result, larger models fit in the same memory bounds.
5. Reduced amount of data that must be transferred, as a result, reduced data transition time.
For default optimization on CPU, the source model is converted from FP32 or FP16 to BF16 and executed internally on platforms with native BF16 support. In this case, ENFORCE_BF16 is set to YES. The code below demonstrates how to check if the key is set:
```python
from openvino.inference_engine import IECore
ie = IECore()
net = ie.read_network(path_to_xml_file)
exec_net = ie.load_network(network=net, device_name="CPU")
exec_net.get_config("ENFORCE_BF16")
```
To enable BF16 internal transformations, set the key "ENFORCE_BF16" to "YES" in the ExecutableNetwork configuration.
```python
bf16_config = {"ENFORCE_BF16" : "YES"}
exec_net = ie.load_network(network=net, device_name="CPU", config = bf16_config)
```
To disable BF16 internal transformations, set the key "ENFORCE_BF16" to "NO". In this case, the model infers as is without modifications with precisions that were set on each layer edge.
An exception with the message `Platform doesn't support BF16 format` is formed in case of setting "ENFORCE_BF16" to "YES"on CPU without native BF16 support or BF16 simulation mode.
Low-Precision 8-bit integer models cannot be converted to BF16, even if bfloat16 optimization is set by default.
### Bfloat16 Simulation Mode
Bfloat16 simulation mode is available on CPU and Intel® AVX-512 platforms that do not support the native avx512_bf16 instruction. The simulator does not guarantee good performance. Note that the CPU must still support the AVX-512 extensions.
#### To Enable the simulation of Bfloat16:
* In the Benchmark App, add the -enforcebf16=true option
* In Python, use the following code as an example:
```python
from openvino.inference_engine import IECore
ie = IECore()
net = ie.read_network(path_to_xml_file)
bf16_config = {"ENFORCE_BF16" : "YES"}
exec_net = ie.load_network(network=net, device_name="CPU", config=bf16_config)
```
### Performance Counters
Information about layer precision is stored in the performance counters that are available from the Inference Engine API. The layers have the following marks:
* Suffix *BF16* for layers that had bfloat16 data type input and were computed in BF16 precision
* Suffix *FP32* for layers computed in 32-bit precision
For example, the performance counters table for the Inception model can look as follows:
```
pool5 EXECUTED layerType: Pooling realTime: 143 cpu: 143 execType: jit_avx512_BF16
fc6 EXECUTED layerType: FullyConnected realTime: 47723 cpu: 47723 execType: jit_gemm_BF16
relu6 NOT_RUN layerType: ReLU realTime: 0 cpu: 0 execType: undef
fc7 EXECUTED layerType: FullyConnected realTime: 7558 cpu: 7558 execType: jit_gemm_BF16
relu7 NOT_RUN layerType: ReLU realTime: 0 cpu: 0 execType: undef
fc8 EXECUTED layerType: FullyConnected realTime: 2193 cpu: 2193 execType: jit_gemm_BF16
prob EXECUTED layerType: SoftMax realTime: 68 cpu: 68 execType: jit_avx512_FP32
```
The **execType** column of the table includes inference primitives with specific suffixes.
[bf16_format]: img/bf16_format.png

106
docs/OV_Runtime_UG/DynamicBatching.md
View File

@ -1,106 +0,0 @@
# Using Dynamic Batching {#openvino_docs_IE_DG_DynamicBatching}
## Using Dynamic Batching (C++)
@sphinxdirective
.. raw:: html
<div id="switcher-cpp" class="switcher-anchor">C++</div>
@endsphinxdirective
The Dynamic Batching feature allows you to dynamically change batch size for inference calls
within a preset batch size limit. This feature might be useful when batch size is unknown beforehand and using an extra-large batch size is undesirable or impossible due to resource limitations. For example, applying face detection and then mood labeling to a video, you won't know in advance how many frames will contain a face when you pass inferencing results to a secondary model.
You can activate Dynamic Batching by setting `KEY_DYN_BATCH_ENABLED` flag to `YES` in a configuration map that is
passed to the plugin while loading a network.
This configuration creates an `ExecutableNetwork` object that will allow setting batch size
dynamically in all of its infer requests using `SetBatch()` method.
The batch size that was set in the passed `CNNNetwork` object will be used as a maximum batch size limit.
Here is a code example:
@snippet snippets/DynamicBatching.cpp part0
### Limitations
Currently, there are certain limitations for the use of Dynamic Batching exist:
* Use Dynamic Batching with CPU and GPU plugins only.
* Use Dynamic Batching on topologies that consist of certain layers only:
* Convolution
* Deconvolution
* Activation
* LRN
* Pooling
* FullyConnected
* SoftMax
* Split
* Concatenation
* Power
* Eltwise
* Crop
* BatchNormalization
* Copy
The following types of layers are not supported:
* Layers that might arbitrary change tensor shape (such as Flatten, Permute, Reshape)
* Layers specific to object detection topologies (ROIPooling, ProirBox, DetectionOutput)
* Custom layers
Topology analysis is performed during the process of loading a network into plugin, and if the topology is not supported, an exception is generated.
## Using Dynamic Batching (Python)
@sphinxdirective
.. raw:: html
<div id="switcher-python" class="switcher-anchor">Python</div>
@endsphinxdirective
Dynamic Batching is a feature that allows you to dynamically change batch size for inference calls within a preset batch size limit. This feature might be useful when batch size is unknown beforehand, and using extra large batch size is not desired or impossible due to resource limitations. For example, face detection with person age, gender, or mood recognition is a typical usage scenario.
You can activate Dynamic Batching by setting the "DYN_BATCH_ENABLED" flag to "YES" in a configuration map that is passed to the plugin while loading a network. This configuration creates an `ExecutableNetwork` object that will allow setting batch size dynamically in all of its infer requests using the [ie_api.batch_size](api/ie_python_api/_autosummary/openvino.inference_engine.IENetwork.html#openvino.inference_engine.IENetwork.batch_size) method. The batch size that was set in the passed CNNNetwork object will be used as a maximum batch size limit.
```python
from openvino.inference_engine import IECore
ie = IECore()
dyn_config = {"DYN_BATCH_ENABLED": "YES"}
ie.set_config(config=dyn_config, device_name=device)
# Read a network in IR or ONNX format
net = ie.read_network(path_to_model)
net.batch_size = 32 # set the maximum batch size to 32
exec_net = ie.load_network(network=net, device_name=device)
```
### Limitations
Currently, certain limitations for the use of Dynamic Batching exist:
* Use Dynamic Batching with CPU and GPU plugins only.
* Use Dynamic Batching on topologies that consist of certain layers only:
* Convolution
* Deconvolution
* Activation
* LRN
* Pooling
* FullyConnected
* SoftMax
* Split
* Concatenation
* Power
* Eltwise
* Crop
* BatchNormalization
* Copy
The following types of layers are not supported:
* Layers that might arbitrary change tensor shape (such as Flatten, Permute, Reshape)
* Layers specific to object detection topologies (ROIPooling, ProirBox, DetectionOutput)
* Custom layers
Topology analysis is performed during the process of loading a network into plugin, and if the topology is not supported, an exception is generated.

Some files were not shown because too many files have changed in this diff Show More