diff --git a/.ci/azure/linux_arm64.yml b/.ci/azure/linux_arm64.yml index 6d9b68f1045..8197cf1a8da 100644 --- a/.ci/azure/linux_arm64.yml +++ b/.ci/azure/linux_arm64.yml @@ -26,16 +26,28 @@ jobs: system.debug: true VSTS_HTTP_RETRY: 5 VSTS_HTTP_TIMEOUT: 200 + PYTHON_ARM_VERSION: "3.8.12" + PYTHON_EXEC: "python3.8" + OPENVINO_ARCH: 'aarch64' + NUM_PROC: 1 BUILD_TYPE: Release OPENVINO_REPO_DIR: $(Build.Repository.LocalPath) OPENVINO_CONTRIB_REPO_DIR: $(OPENVINO_REPO_DIR)/../openvino_contrib OPENCV_REPO_DIR: $(OPENVINO_REPO_DIR)/../opencv + BUILD_PYTHON: $(WORK_DIR)/build_python + BUILD_OPENCV: $(WORK_DIR)/build_opencv + BUILD_OPENVINO: $(WORK_DIR)/build + BUILD_OPENVINO_PYTHON: $(WORK_DIR)/build_python + BUILD_OPEN_MODEL_ZOO: $(WORK_DIR)/build_open_model_zoo + INSTALL_OPENVINO: $(WORK_DIR)/install_openvino + INSTALL_PYTHON: $(INSTALL_OPENVINO)/extras/python + INSTALL_OPENCV: $(INSTALL_OPENVINO)/extras/opencv + INSTALL_OPEN_MODEL_ZOO: $(INSTALL_OPENVINO)/extras/open_model_zoo WORK_DIR: $(Pipeline.Workspace)/_w - BUILD_DIR: $(WORK_DIR)/build - BUILD_DIR_OPENCV: $(WORK_DIR)/build_opencv - TMP_DIR: /mnt/tmp SHARE_DIR: /mount/cinfsshare/onnxtestdata - CCACHE_DIR: $(SHARE_DIR)/ccache/master/linux_arm64 + TMP_DIR: /mnt/tmp + OPENVINO_CCACHE_DIR: $(SHARE_DIR)/ccache/master/linux_arm64 + OPENCV_CCACHE_DIR: $(SHARE_DIR)/ccache/master/linux_arm64_opencv steps: - script: | @@ -56,17 +68,21 @@ jobs: df lsblk -o NAME,HCTL,SIZE,MOUNTPOINT | grep -i "sd" free -h + echo "##vso[task.setvariable variable=NUM_PROC]$(nproc --all)" + echo "NUM_PROC=$(NUM_PROC)" displayName: 'System information' - script: | rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR) - rm -rf $(BUILD_DIR) ; mkdir $(BUILD_DIR) + mkdir -p $(BUILD_OPENCV) $(BUILD_OPENVINO) $(BUILD_OPENVINO_PYTHON) $(BUILD_PYTHON) $(BUILD_OPEN_MODEL_ZOO) + mkdir -p $(INSTALL_OPENVINO) $(INSTALL_PYTHON) $(INSTALL_OPENCV) $(INSTALL_OPEN_MODEL_ZOO) sudo rm -rf $(TMP_DIR) ; sudo mkdir $(TMP_DIR) ; sudo chmod 777 -R $(TMP_DIR) sudo mkdir -p $(SHARE_DIR) sudo apt --assume-yes update && sudo apt --assume-yes install nfs-common sudo mount -vvv -t nfs cinfsshare.file.core.windows.net:/cinfsshare/onnxtestdata $(SHARE_DIR) -o vers=4,minorversion=1,sec=sys - mkdir -p $(CCACHE_DIR) - displayName: 'Make directory' + mkdir -p $(OPENVINO_CCACHE_DIR) + mkdir -p $(OPENCV_CCACHE_DIR) + displayName: 'Make directories' - checkout: self clean: true @@ -83,16 +99,25 @@ jobs: - script: | set -e $(OPENVINO_REPO_DIR)/install_build_dependencies.sh - # Move into contrib install_build_dependencies.sh - sudo apt --assume-yes install scons crossbuild-essential-arm64 libprotoc-dev protobuf-compiler - # OpenCV should provide install_build_dependencies.sh as well - # Move into resources - git clone https://github.com/opencv/opencv.git --depth 1 $(OPENCV_REPO_DIR) - # Speed up build - wget https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-linux.zip - unzip ninja-linux.zip - sudo cp -v ninja /usr/local/bin/ - workingDirectory: $(WORK_DIR) + export CCACHE_DIR=$(OPENCV_CCACHE_DIR) + export CCACHE_TEMPDIR=$(TMP_DIR)/ccache + export CCACHE_BASEDIR=$(Pipeline.Workspace) + export CCACHE_MAXSIZE=50G + export USE_CCACHE=1 + export PYTHON_ARM_VERSION=$(PYTHON_ARM_VERSION) + export NUM_PROC=$(NUM_PROC) + export BUILD_PYTHON=$(BUILD_PYTHON) + export WORK_DIR=$(WORK_DIR) + export INSTALL_PYTHON=$(INSTALL_PYTHON) + export BUILD_TYPE=$(BUILD_TYPE) + export OPENVINO_REPO_DIR=$(OPENVINO_REPO_DIR) + export INSTALL_OPENCV=$(INSTALL_OPENCV) + export PYTHON_EXEC=$(PYTHON_EXEC) + export OPENCV_REPO_DIR=$(OPENCV_REPO_DIR) + export BUILD_OPENCV=$(BUILD_OPENCV) + export INSTALL_OPENVINO=$(INSTALL_OPENVINO) + $(OPENVINO_CONTRIB_REPO_DIR)/modules/arm_plugin/scripts/install_build_dependencies.sh + workingDirectory: $(BUILD_OPENVINO) displayName: 'Install dependencies' - task: CMake@1 @@ -100,30 +125,21 @@ jobs: cmakeArgs: > -GNinja -DVERBOSE_BUILD=ON + -DOpenCV_DIR=$(INSTALL_OPENCV)/cmake + -DENABLE_OPENCV=OFF + -DPYTHON_INCLUDE_DIRS=$(INSTALL_PYTHON)/include/python3.8 + -DPYTHON_LIBRARY=$(INSTALL_PYTHON)/lib/libpython3.8.so + -DENABLE_PYTHON=ON + -DPYTHON_MODULE_EXTENSION=".so" + -DENABLE_TESTS=ON + -DENABLE_FUNCTIONAL_TESTS=ON + -DENABLE_GAPI_TESTS=OFF + -DENABLE_GAPI_PREPROCESSING=OFF + -DENABLE_DATA=OFF + -DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath-link,$(INSTALL_OPENCV)/lib + -DTHREADING=SEQ -DENABLE_LTO=ON + -DCMAKE_TOOLCHAIN_FILE=$(OPENVINO_REPO_DIR)/cmake/arm64.toolchain.cmake -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) - -DBUILD_LIST=imgcodecs,videoio,highgui - -DCMAKE_TOOLCHAIN_FILE=$(OPENCV_REPO_DIR)/platforms/linux/aarch64-gnu.toolchain.cmake - $(OPENCV_REPO_DIR) - workingDirectory: $(BUILD_DIR_OPENCV) - - - script: ninja - workingDirectory: $(BUILD_DIR_OPENCV) - displayName: 'Build OpenCV Linux ARM64' - - - script: ninja install - workingDirectory: $(BUILD_DIR_OPENCV) - displayName: 'Install OpenCV Linux ARM64' - - - task: CMake@1 - inputs: - cmakeArgs: > - -GNinja - -DVERBOSE_BUILD=ON - -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) - -DCMAKE_TOOLCHAIN_FILE=$(OPENVINO_REPO_DIR)/cmake/arm64.toolchain.cmake - -DOpenCV_DIR=$(BUILD_DIR_OPENCV)/install/lib/cmake/opencv4 - -DENABLE_OPENCV=OFF - -DENABLE_TESTS=ON -DENABLE_SAMPLES=ON -DBUILD_java_api=OFF -DENABLE_INTEL_MYRIAD=OFF @@ -131,26 +147,102 @@ jobs: -DIE_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)/modules -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache + -DARM_COMPUTE_SCONS_JOBS=$(NUM_PROC) + -DOUTPUT_ROOT=$(INSTALL_OPENVINO) + -DCMAKE_INSTALL_PREFIX=$(INSTALL_OPENVINO) $(OPENVINO_REPO_DIR) - workingDirectory: $(BUILD_DIR) - - - script: ls -alR $(OPENVINO_REPO_DIR)/temp/ - displayName: 'List temp SDKs' - - - script: ccache --zero-stats --max-size=50G --show-config - displayName: 'Clean ccache stats' + workingDirectory: $(BUILD_OPENVINO) + displayName: 'CMake OpenVINO ARM plugin' - script: | - export CCACHE_DIR=$(CCACHE_DIR) + export CCACHE_DIR=$(OPENVINO_CCACHE_DIR) export CCACHE_TEMPDIR=$(TMP_DIR)/ccache export CCACHE_BASEDIR=$(Pipeline.Workspace) export CCACHE_MAXSIZE=50G + export USE_CCACHE=1 ninja - workingDirectory: $(BUILD_DIR) - displayName: 'Build Linux ARM64' + workingDirectory: $(BUILD_OPENVINO) + displayName: 'Build OpenVINO ARM plugin' - - script: ccache --show-stats - displayName: 'Show ccache stats' + - script: ninja install + workingDirectory: $(BUILD_OPENVINO) + displayName: 'Install OpenVINO ARM plugin' - - script: ls -alR $(OPENVINO_REPO_DIR)/bin/ - displayName: 'List binary files' + - task: CMake@1 + inputs: + cmakeArgs: > + -GNinja + -DInferenceEngineDeveloperPackage_DIR=$(BUILD_OPENVINO) + -DENABLE_PYTHON=ON + -DPYTHON_EXECUTABLE=$(INSTALL_PYTHON)/bin/python3.8 + -DPYTHON_INCLUDE_DIRS=$(INSTALL_PYTHON)/include/python3.8 + -DPYTHON_LIBRARIES=$(INSTALL_PYTHON)/lib + -DPYTHON3_NUMPY_INCLUDE_DIRS=/usr/local/lib/python3.8/site-packages/numpy/core/include + -DPYTHON_MODULE_EXTENSION=".so" + -DPYBIND11_FINDPYTHON=OFF + -DPYBIND11_NOPYTHON=OFF + -DPYTHONLIBS_FOUND=TRUE + -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) + -DENABLE_DATA=OFF + -DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath-link,$(INSTALL_OPENCV)/lib + -DCMAKE_TOOLCHAIN_FILE=$(OPENVINO_REPO_DIR)/cmake/arm64.toolchain.cmake + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + -DCMAKE_C_COMPILER_LAUNCHER=ccache + -DCMAKE_INSTALL_PREFIX=$(INSTALL_OPENVINO) + $(OPENVINO_REPO_DIR)/src/bindings/python + workingDirectory: $(BUILD_OPENVINO_PYTHON) + displayName: 'CMake OpenVINO python binding' + + - script: | + export CCACHE_DIR=$(OPENVINO_CCACHE_DIR) + export CCACHE_TEMPDIR=$(TMP_DIR)/ccache + export CCACHE_BASEDIR=$(Pipeline.Workspace) + export CCACHE_MAXSIZE=50G + export USE_CCACHE=1 + ninja + workingDirectory: $(BUILD_OPENVINO_PYTHON) + displayName: 'Build OpenVINO python binding' + + - script: ninja install + workingDirectory: $(BUILD_OPENVINO_PYTHON) + displayName: 'Install OpenVINO python binding' + + - task: CMake@1 + inputs: + cmakeArgs: > + -GNinja + -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) + -DENABLE_PYTHON=ON + -DPYTHON_EXECUTABLE=/usr/local/bin/python3.8 + -DPYTHON_INCLUDE_DIR=$(INSTALL_PYTHON)/include/python3.8 + -DPYTHON_LIBRARY=$(INSTALL_PYTHON)/lib + -DCMAKE_TOOLCHAIN_FILE=$(OPENVINO_REPO_DIR)/cmake/arm64.toolchain.cmake + -DOpenVINO_DIR=$(BUILD_OPENVINO) + -DInferenceEngine_DIR=$(BUILD_OPENVINO) + -DOpenCV_DIR=$(INSTALL_OPENCV)/cmake + -Dngraph_DIR=$(BUILD_OPENVINO) + -DIE_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)/modules + -DCMAKE_INSTALL_PREFIX=$(INSTALL_OPEN_MODEL_ZOO) + $(OPENVINO_REPO_DIR)/thirdparty/open_model_zoo/demos + workingDirectory: $(BUILD_OPEN_MODEL_ZOO) + displayName: 'CMake Open Model Zoo demos' + + - script: ninja + workingDirectory: $(BUILD_OPEN_MODEL_ZOO) + displayName: 'Build Open Model Zoo demos' + + - script: ninja install + workingDirectory: $(BUILD_OPEN_MODEL_ZOO) + displayName: 'Install Open Model Zoo demos' + + - script: | + cp -r $(BUILD_OPEN_MODEL_ZOO)/$(OPENVINO_ARCH)/$(BUILD_TYPE)/* $(INSTALL_OPEN_MODEL_ZOO)/ + zip -9 -r $(Build.ArtifactStagingDirectory)/openvino_$(OPENVINO_ARCH)_linux.zip ./* + workingDirectory: $(INSTALL_OPENVINO) + displayName: 'Create OpenVINO ARM64 linux package' + + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: $(Build.ArtifactStagingDirectory) + artifactName: 'openvino_aarch64_linux' + displayName: 'Publish OpenVINO AArch64 linux package' diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a3ae0134db..fe62c631885 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,7 +34,9 @@ endif() message (STATUS "PROJECT ............................... " ${PROJECT_NAME}) message (STATUS "CMAKE_VERSION ......................... " ${CMAKE_VERSION}) message (STATUS "CMAKE_BINARY_DIR ...................... " ${CMAKE_BINARY_DIR}) +message (STATUS "CMAKE_SOURCE_DIR ...................... " ${CMAKE_SOURCE_DIR}) message (STATUS "OpenVINO_SOURCE_DIR ................... " ${OpenVINO_SOURCE_DIR}) +message (STATUS "OpenVINO_BINARY_DIR ................... " ${OpenVINO_BINARY_DIR}) message (STATUS "CMAKE_GENERATOR ....................... " ${CMAKE_GENERATOR}) message (STATUS "CMAKE_C_COMPILER_ID ................... " ${CMAKE_C_COMPILER_ID}) message (STATUS "CMAKE_CXX_COMPILER_ID ................. " ${CMAKE_CXX_COMPILER_ID}) @@ -42,7 +44,7 @@ message (STATUS "CMAKE_BUILD_TYPE ...................... " ${CMAKE_BUILD_TYPE}) message (STATUS "CMAKE_TOOLCHAIN_FILE .................. " ${CMAKE_TOOLCHAIN_FILE}) # remove file with exported developer targets to force its regeneration -file(REMOVE "${CMAKE_BINARY_DIR}/ngraph/ngraphTargets.cmake") +file(REMOVE "${CMAKE_BINARY_DIR}/ngraphTargets.cmake") file(REMOVE "${CMAKE_BINARY_DIR}/InferenceEngineTargets.cmake") file(REMOVE "${CMAKE_BINARY_DIR}/OpenVINOTargets.cmake") foreach(component IN LISTS openvino_export_components) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000000..45a41c374ee --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,68 @@ +# How to contribute to the OpenVINO repository + +We suppose that you are an enthusiastic coder, want to contribute some code. For that purpose OpenVINO project now has a repository on the GitHub, to simplify everybody's life! All the bug fixes, new functionality, new tutorials etc. should be submitted via the GitHub's mechanism of pull requests. + +If you are not familiar with the mechanism - do not worry, it's very simple. Keep reading. + +## Before you start contributing you should + +- Make sure you agree to contribute your code under [OpenVINO (Apache 2.0)](https://github.com/openvinotoolkit/openvino/blob/master/LICENSE) license. +- If you are submitting a new module, you should go into [openvino_contrib](https://github.com/openvinotoolkit/openvino_contrib) repository by default. +- If you are going to fix a bug, check that it's still exists. This can be done by building the latest [releases/2020/3](https://github.com/openvinotoolkit/openvino/tree/releases/2020/3) branch (LTS release) or the latest master branch, and make sure that the error is still reproducible there. We do not fix bugs that only affect older non-LTS releases like 2020.2 for example (more details about [branching strategy](https://github.com/openvinotoolkit/openvino/wiki/Branches)) +- Make sure that nobody beat you into fixing or reporting the issue by doing a search on the [Github OpenVINO issues](https://github.com/openvinotoolkit/openvino/issues) page, and making sure that there isn't someone working on it. In the latter case you might provide support or suggestion in the issue or in the linked pull request. +- If you have a question about the software, then this is **NOT** the right place. You should open up a question at the [OpenVINO forum](https://community.intel.com/t5/Intel-Distribution-of-OpenVINO/bd-p/distribution-openvino-toolkit). In order to post a decent question from the start, feel free to read the official forum guidelines. + +Before you open up anything on the OpenVINO GitHub page, be sure that you are at the right place with your problem. + +## "Fork & Pull Request model" for code contribution + +### [](https://github.com/openvinotoolkit/openvino/wiki/Contribute#the-instruction-in-brief)The instruction in brief + +- Register at GitHub. Create your fork of OpenVINO repository [https://github.com/openvinotoolkit/openvino](https://github.com/openvinotoolkit/openvino) (see [https://help.github.com/articles/fork-a-repo](https://help.github.com/articles/fork-a-repo) for details). +- Install Git. + - Set your user name and email address in a Git configuration according to GitHub account (see [https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup](https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup) for details). +- Choose a task for yourself. It could be a bugfix or some new code. +- Choose a base branch for your work. More details about branches and policies are here: [Branches](https://github.com/openvinotoolkit/openvino/wiki/Branches) +- Clone your fork to your computer. +- Create a new branch (with a meaningful name) from the base branch you chose. +- Modify / add the code following our [Coding Style Guide](https://github.com/openvinotoolkit/openvino/wiki/CodingStyleGuideLines) and [Documentation guidelines](https://github.com/openvinotoolkit/openvino/wiki/CodingStyleGuideLinesDocumentation). +- If you want to add a new sample, please look at this [Guide for contributing to C++/C/Python IE samples](https://github.com/openvinotoolkit/openvino/wiki/SampleContribute) +- Run testsuite locally: + - execute each test binary from the artifacts directory, e.g. `/bin/intel64/Release/ieFuncTests` +- If you contribute to the documentation and want to add a new guide: + - Create a new markdown file in an appropriate folder. + - **REQUIRED:** The document title must contain a document label in a form: `{#openvino_docs_}`. For example: `Deep Learning Network Intermediate Representation and Operation Sets in OpenVINO™ {#openvino_docs_MO_DG_IR_and_opsets}`. + - Add your file to the documentation structure. Open the documentation structure file [`docs/doxygen/ie_docs.xml`](https://github.com/openvinotoolkit/openvino/blob/master/docs/doxygen/ie_docs.xml) and add your file path to the appropriate section. +- When you are done, make sure that your branch is to date with latest state of the branch you want to contribute to (e.g. `git fetch upstream && git merge upstream/master`), push your branch to your GitHub fork; then create a pull request from your branch to the base branch (see [https://help.github.com/articles/using-pull-requests](https://help.github.com/articles/using-pull-requests) for details). + +## Making a good pull request + +Following these guidelines will increase the likelihood of your pull request being accepted: + +- Before pushing your PR to the repository, make sure that it builds perfectly fine on your local system. +- Add enough information, like a meaningful title, the reason why you made the commit and a link to the issue page if you opened one for this PR. +- Scope your PR to one issue. Before submitting, make sure the diff contains no unrelated changes. If you want to cover more than one issue, submit your changes for each as separate pull requests. +- If you have added new functionality, you should update/create the relevant documentation, as well as add tests for it to the testsuite. +- Try not to include "oops" commits - ones that just fix an error in the previous commit. If you have those, then before submitting [squash](https://github.com/openvinotoolkit/openvino/wiki/Contribute#https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History#Squashing-Commits) those fixes directly into the commits where they belong. +- Make sure to choose the right base branch and to follow the [Coding Style Guide](https://github.com/openvinotoolkit/openvino/wiki/CodingStyleGuideLines) for your code or [Documentation guidelines](https://github.com/openvinotoolkit/openvino/wiki/CodingStyleGuideLinesDocumentation) you are changing documentation files. +- Make sure to add test for new functionality or test that reproduces fixed bug with related test data. Please do not add extra images or videos, if some of existing media files are suitable. + +## Testing and merging pull requests + +- Your pull request will be automatically tested by OpenVINO's precommit (testing status are automatically reported as "green" or "red" circles in precommit steps on PR's page). If any builders have failed, you should fix the issue. To rerun the automatic builds just push changes to your branch on GitHub. No need to close pull request and open a new one! +- Once all the builders are "green", one of OpenVINO developers will review your code. Reviewer could ask you to modify your pull request. Please provide timely response for reviewers (within weeks, not months), otherwise you submission could be postponed or even rejected. + +## PR review good practices + +- Originator is responsible for driving the review of changes and should ping reviewers periodically. +- Originator should close comments from the Reviewer when it is resolved. The Reviewer may re-open the comment if he does not agree with the resolution. +- Originator should request re-review from the Reviewer when all comments are resolved by pushing the button in the “Reviewers” section. +- If it is still WIP and you want to check CI test results early then use _Draft_ PR. +- Do **NOT** rewrite history (push -f) once you converted draft PR into regular one, add new commits instead. Looking at diffs makes review easier. +- Write meaningful description of commits resulting from review. _"Addressing review comments"_ is **NOT** a good description! Having a quick look at good descriptions can tell you much what is going on in PR without a need to go through all of resolved comments. + +## Merging PR + +As soon as the reviewer is fine with the pull request and Precommit likes your code and shows "green" status, the "Approved" review status is put, which signals OpenVINO maintainers that they can merge your pull request. + +© Copyright 2018-2022, OpenVINO team \ No newline at end of file diff --git a/README.md b/README.md index fc47dbe49d4..95431e98dce 100644 --- a/README.md +++ b/README.md @@ -16,12 +16,11 @@ source and public models in popular formats such as TensorFlow, ONNX, PaddlePadd * [OpenVINO™ Runtime] * [Model Optimizer] * [Post-Training Optimization Tool] - +* [Samples] ## License -Deep Learning Deployment Toolkit is licensed under [Apache License Version 2.0](LICENSE). -By contributing to the project, you agree to the license and copyright terms therein -and release your contribution under these terms. +OpenVINO™ Toolkit is licensed under [Apache License Version 2.0](LICENSE). +By contributing to the project, you agree to the license and copyright terms therein and release your contribution under these terms. ## Resources * Docs: https://docs.openvino.ai/ @@ -46,5 +45,6 @@ Please report questions, issues and suggestions using: [OpenVINO™ Runtime]:https://docs.openvino.ai/latest/openvino_docs_OV_Runtime_User_Guide.html [Model Optimizer]:https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html [Post-Training Optimization Tool]:https://docs.openvino.ai/latest/pot_README.html +[Samples]:https://github.com/openvinotoolkit/openvino/tree/master/samples [tag on StackOverflow]:https://stackoverflow.com/search?q=%23openvino diff --git a/cmake/developer_package/IEDevScriptsConfig.cmake b/cmake/developer_package/IEDevScriptsConfig.cmake index e7f342a1b6c..29ec0fbf7af 100644 --- a/cmake/developer_package/IEDevScriptsConfig.cmake +++ b/cmake/developer_package/IEDevScriptsConfig.cmake @@ -212,6 +212,10 @@ endif() macro(ov_install_static_lib target comp) if(NOT BUILD_SHARED_LIBS) + get_target_property(target_type ${target} TYPE) + if(${target_type} STREQUAL "STATIC_LIBRARY") + set_target_properties(${target} PROPERTIES EXCLUDE_FROM_ALL FALSE) + endif() install(TARGETS ${target} EXPORT OpenVINOTargets ARCHIVE DESTINATION ${IE_CPACK_ARCHIVE_PATH} COMPONENT ${comp} ${ARGN}) endif() diff --git a/cmake/developer_package/api_validator/api_validator.cmake b/cmake/developer_package/api_validator/api_validator.cmake index 6e073b53ad8..33a21696a9e 100644 --- a/cmake/developer_package/api_validator/api_validator.cmake +++ b/cmake/developer_package/api_validator/api_validator.cmake @@ -51,12 +51,6 @@ endfunction() set(VALIDATED_LIBRARIES "" CACHE INTERNAL "") function(_ov_add_api_validator_post_build_step) - if(NOT BUILD_SHARED_LIBS) - # since _ov_add_api_validator_post_build_step - # is currently run only on shared libraries, we have nothing to test - return() - endif() - set(UWP_API_VALIDATOR_APIS "${PROGRAMFILES}/Windows Kits/10/build/universalDDIs/x64/UniversalDDIs.xml") set(UWP_API_VALIDATOR_EXCLUSION "${UWP_SDK_PATH}/BinaryExclusionlist.xml") diff --git a/cmake/developer_package/cpplint/cpplint.py b/cmake/developer_package/cpplint/cpplint.py index efc12ba5c6b..2700bae29f8 100644 --- a/cmake/developer_package/cpplint/cpplint.py +++ b/cmake/developer_package/cpplint/cpplint.py @@ -3592,7 +3592,7 @@ def CheckOperatorSpacing(filename, clean_lines, linenum, error): elif not Match(r'#.*include', line): # Look for < that is not surrounded by spaces. This is only # triggered if both sides are missing spaces, even though - # technically should should flag if at least one side is missing a + # technically should flag if at least one side is missing a # space. This is done to avoid some false positives with shifts. match = Match(r'^(.*[^\s<])<[^\s=<,]', line) if match: diff --git a/cmake/developer_package/download/download_and_extract.cmake b/cmake/developer_package/download/download_and_extract.cmake index a9b71ace064..01662589e2c 100644 --- a/cmake/developer_package/download/download_and_extract.cmake +++ b/cmake/developer_package/download/download_and_extract.cmake @@ -146,8 +146,6 @@ function (DownloadOrExtractInternal URL archive_path unpacked_path folder fattal endfunction(DownloadOrExtractInternal) -file(REMOVE ${CMAKE_BINARY_DIR}/dependencies_64.txt) - function (CheckOrDownloadAndExtract component RELATIVE_URL archive_name unpacked_path result_path folder fattal resultExt use_alternatives sha256 files_to_extract) set (archive_path ${TEMP}/download/${archive_name}) set (status "ON") @@ -164,7 +162,6 @@ function (CheckOrDownloadAndExtract component RELATIVE_URL archive_name unpacked if (${use_alternatives}) set(DEP_INFO "${component}=${URL}") debug_message (STATUS "DEPENDENCY_URL: ${DEP_INFO}") - file(APPEND ${CMAKE_BINARY_DIR}/dependencies_64.txt "${DEP_INFO}\n") endif() debug_message ("checking that unpacked directory exist: ${unpacked_path}") diff --git a/cmake/developer_package/packaging.cmake b/cmake/developer_package/packaging.cmake index 0a9b5eb5106..cc287ff1f3b 100644 --- a/cmake/developer_package/packaging.cmake +++ b/cmake/developer_package/packaging.cmake @@ -15,6 +15,10 @@ function(ie_cpack_set_library_dir) set(IE_CPACK_LIBRARY_PATH runtime/lib/${ARCH_FOLDER}/$ PARENT_SCOPE) set(IE_CPACK_RUNTIME_PATH runtime/bin/${ARCH_FOLDER}/$ PARENT_SCOPE) set(IE_CPACK_ARCHIVE_PATH runtime/lib/${ARCH_FOLDER}/$ PARENT_SCOPE) + elseif(APPLE) + set(IE_CPACK_LIBRARY_PATH runtime/lib/${ARCH_FOLDER}/$ PARENT_SCOPE) + set(IE_CPACK_RUNTIME_PATH runtime/lib/${ARCH_FOLDER}/$ PARENT_SCOPE) + set(IE_CPACK_ARCHIVE_PATH runtime/lib/${ARCH_FOLDER}/$ PARENT_SCOPE) else() set(IE_CPACK_LIBRARY_PATH runtime/lib/${ARCH_FOLDER} PARENT_SCOPE) set(IE_CPACK_RUNTIME_PATH runtime/lib/${ARCH_FOLDER} PARENT_SCOPE) diff --git a/cmake/developer_package/plugins/plugins.cmake b/cmake/developer_package/plugins/plugins.cmake index 90a1b96dbbf..a630d6c354e 100644 --- a/cmake/developer_package/plugins/plugins.cmake +++ b/cmake/developer_package/plugins/plugins.cmake @@ -347,7 +347,7 @@ function(ie_generate_plugins_hpp) # for some reason dependency on source files does not work # so, we have to use explicit target and make it dependency for inference_engine add_custom_target(_ie_plugins_hpp DEPENDS ${ie_plugins_hpp}) - add_dependencies(inference_engine _ie_plugins_hpp) + add_dependencies(inference_engine_obj _ie_plugins_hpp) # add dependency for object files get_target_property(sources inference_engine_obj SOURCES) diff --git a/cmake/templates/InferenceEngineDeveloperPackageConfig.cmake.in b/cmake/templates/InferenceEngineDeveloperPackageConfig.cmake.in index 5a895708354..bcec89b34a0 100644 --- a/cmake/templates/InferenceEngineDeveloperPackageConfig.cmake.in +++ b/cmake/templates/InferenceEngineDeveloperPackageConfig.cmake.in @@ -44,7 +44,7 @@ find_dependency(InferenceEngine NO_DEFAULT_PATH) find_dependency(ngraph - PATHS "${CMAKE_CURRENT_LIST_DIR}/src/core" + PATHS "${CMAKE_CURRENT_LIST_DIR}" NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH) diff --git a/cmake/test_model_zoo.cmake b/cmake/test_model_zoo.cmake index 176e1b55220..18442560350 100644 --- a/cmake/test_model_zoo.cmake +++ b/cmake/test_model_zoo.cmake @@ -86,11 +86,6 @@ ov_model_convert("${OpenVINO_SOURCE_DIR}/${rel_path}" "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/test_model_zoo/onnx_import" ie_onnx_import_out_files) -set(rel_path "docs/onnx_custom_op") -ov_model_convert("${OpenVINO_SOURCE_DIR}/${rel_path}" - "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/test_model_zoo/docs/models" - docs_onnx_out_files) - if(ENABLE_TESTS) if(ENABLE_OV_ONNX_FRONTEND AND ENABLE_REQUIREMENTS_INSTALL) find_package(PythonInterp 3 REQUIRED) diff --git a/cmake/toolchains/mt.runtime.win32.toolchain.cmake b/cmake/toolchains/mt.runtime.win32.toolchain.cmake index 0da0ccf1e6e..62619cd5683 100644 --- a/cmake/toolchains/mt.runtime.win32.toolchain.cmake +++ b/cmake/toolchains/mt.runtime.win32.toolchain.cmake @@ -25,7 +25,7 @@ endif() if(use_static_runtime) foreach(lang C CXX) foreach(build_type "" "_DEBUG" "_MINSIZEREL" "_RELEASE" "_RELWITHDEBINFO") - set(flag_var "CMAKE_${lang}_FLAGS${build_type}") + set(flag_var "CMAKE_${lang}_FLAGS${build_type}_INIT") string(REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}") endforeach() endforeach() diff --git a/cmake/toolchains/oecore.arm64.toolchain.cmake b/cmake/toolchains/oecore.arm64.toolchain.cmake deleted file mode 100644 index 0d0b63398b7..00000000000 --- a/cmake/toolchains/oecore.arm64.toolchain.cmake +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (C) 2018-2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -if(DEFINED OECORE_BASE_DIR) - # OECORE_BASE_DIR was passed via CMake command line, nothing to do -elseif(DEFINED ENV{OECORE_BASE_DIR}) - # User sets OECORE_BASE_DIR environment variable - set(OECORE_BASE_DIR $ENV{OECORE_BASE_DIR}) -elseif(DEFINED ENV{OECORE_NATIVE_SYSROOT}) - # OECORE_NATIVE_SYSROOT is a default environment variable for the OECore toolchain - set(OECORE_BASE_DIR "$ENV{OECORE_NATIVE_SYSROOT}/../..") -else() - # Use default value - set(OECORE_BASE_DIR "/usr/local/oecore-x86_64") -endif() - -set(OECORE_TARGET_NAME "aarch64-ese-linux") -set(OECORE_TARGET_SYSROOT "${OECORE_BASE_DIR}/sysroots/${OECORE_TARGET_NAME}") -set(OECORE_HOST_SYSROOT "${OECORE_BASE_DIR}/sysroots/x86_64-esesdk-linux") -set(OECORE_HOST_COMPILER_BIN_DIR "${OECORE_HOST_SYSROOT}/usr/bin/${OECORE_TARGET_NAME}") - -set(CMAKE_SYSTEM_NAME "Linux") -set(CMAKE_SYSTEM_PROCESSOR "aarch64") - -set(CMAKE_SYSROOT "${OECORE_TARGET_SYSROOT}") - -set(CMAKE_C_COMPILER "${OECORE_HOST_COMPILER_BIN_DIR}/aarch64-ese-linux-gcc") -set(CMAKE_CXX_COMPILER "${OECORE_HOST_COMPILER_BIN_DIR}/aarch64-ese-linux-g++") - -set(CMAKE_C_FLAGS_INIT "-mcpu=cortex-a53 -mtune=cortex-a53 --sysroot=${OECORE_TARGET_SYSROOT}") -set(CMAKE_CXX_FLAGS_INIT "-mcpu=cortex-a53 -mtune=cortex-a53 --sysroot=${OECORE_TARGET_SYSROOT}") - -set(CMAKE_EXE_LINKER_FLAGS_INIT "-Wl,-O1 -Wl,--hash-style=gnu -Wl,--as-needed --sysroot=${OECORE_TARGET_SYSROOT}") -set(CMAKE_SHARED_LINKER_FLAGS_INIT "-Wl,-O1 -Wl,--hash-style=gnu -Wl,--as-needed --sysroot=${OECORE_TARGET_SYSROOT}") -set(CMAKE_MODULE_LINKER_FLAGS_INIT "-Wl,-O1 -Wl,--hash-style=gnu -Wl,--as-needed --sysroot=${OECORE_TARGET_SYSROOT}") - -set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) -set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) diff --git a/cmake/toolchains/onecoreuap.toolchain.cmake b/cmake/toolchains/onecoreuap.toolchain.cmake index 0accc4ed7da..c97bca379ed 100644 --- a/cmake/toolchains/onecoreuap.toolchain.cmake +++ b/cmake/toolchains/onecoreuap.toolchain.cmake @@ -35,14 +35,14 @@ if(_onecoreuap_arch STREQUAL "x64") # Forcefull make VS search for C++ libraries in these folders prior to other c++ standard libraries localizations. add_link_options("/LIBPATH:\"\$\(VC_LibraryPath_VC_x64_OneCore\)\"") - set(CMAKE_C_STANDARD_LIBRARIES "\$\(UCRTContentRoot\)lib/\$\(TargetUniversalCRTVersion\)/um/\$\(Platform\)/OneCoreUap.lib" CACHE STRING "" FORCE) - set(CMAKE_CXX_STANDARD_LIBRARIES "\$\(UCRTContentRoot\)lib/\$\(TargetUniversalCRTVersion\)/um/\$\(Platform\)/OneCoreUap.lib" CACHE STRING "" FORCE) + set(CMAKE_C_STANDARD_LIBRARIES_INIT "\$\(UCRTContentRoot\)lib/\$\(TargetUniversalCRTVersion\)/um/\$\(Platform\)/OneCoreUap.lib" CACHE STRING "" FORCE) + set(CMAKE_CXX_STANDARD_LIBRARIES_INIT "\$\(UCRTContentRoot\)lib/\$\(TargetUniversalCRTVersion\)/um/\$\(Platform\)/OneCoreUap.lib" CACHE STRING "" FORCE) elseif(_onecoreuap_arch STREQUAL "X86") add_link_options("/LIBPATH:\"\$\(VCInstallDir\)lib/onecore\"") add_link_options("/LIBPATH:\"\$\(VC_LibraryPath_VC_x86_OneCore\)\"") - set(CMAKE_C_STANDARD_LIBRARIES "\$\(UCRTContentRoot\)lib/\$\(TargetUniversalCRTVersion\)/um/x86/OneCoreUap.lib" CACHE STRING "" FORCE) - set(CMAKE_CXX_STANDARD_LIBRARIES "\$\(UCRTContentRoot\)lib/\$\(TargetUniversalCRTVersion\)/um/x86/OneCoreUap.lib" CACHE STRING "" FORCE) + set(CMAKE_C_STANDARD_LIBRARIES_INIT "\$\(UCRTContentRoot\)lib/\$\(TargetUniversalCRTVersion\)/um/x86/OneCoreUap.lib" CACHE STRING "" FORCE) + set(CMAKE_CXX_STANDARD_LIBRARIES_INIT "\$\(UCRTContentRoot\)lib/\$\(TargetUniversalCRTVersion\)/um/x86/OneCoreUap.lib" CACHE STRING "" FORCE) else() message(FATAL_ERROR "Unsupported architecture ${_onecoreuap_arch}. Only X86 or X86_64 are supported") endif() @@ -52,8 +52,8 @@ unset(_onecoreuap_arch) # compile flags set(includes "/I\"\$\(UniversalCRT_IncludePath\)\"") -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${includes}") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${includes}") +set(CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS_INIT} ${includes}") +set(CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS_INIT} ${includes}") unset(includes) # linker flags @@ -62,9 +62,9 @@ foreach(lib kernel32 user32 advapi32 ole32 mscoree combase) set(linker_flags "/NODEFAULTLIB:${lib}.lib ${linker_flags}") endforeach() -set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${linker_flags}") -set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${linker_flags}") -set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${linker_flags}") +set(CMAKE_SHARED_LINKER_FLAGS_INIT "${CMAKE_SHARED_LINKER_FLAGS_INIT} ${linker_flags}") +set(CMAKE_MODULE_LINKER_FLAGS_INIT "${CMAKE_MODULE_LINKER_FLAGS_INIT} ${linker_flags}") +set(CMAKE_EXE_LINKER_FLAGS_INIT "${CMAKE_EXE_LINKER_FLAGS_INIT} ${linker_flags}") unset(linker_flags) # diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 1e0699b931d..75dbeb180c6 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -7,8 +7,6 @@ if(NOT ENABLE_DOCKER) ie_add_compiler_flags(-Wall) endif() - add_subdirectory(snippets) - # Detect OpenVINO find_package(OpenVINO QUIET PATHS "${CMAKE_BINARY_DIR}" @@ -17,9 +15,8 @@ if(NOT ENABLE_DOCKER) set(OpenVINO_DIR ${CMAKE_BINARY_DIR}) endif() - if(ENABLE_OV_ONNX_FRONTEND) - add_subdirectory(onnx_custom_op) - endif() + add_subdirectory(snippets) + add_subdirectory(template_extension) set(all_docs_targets diff --git a/docs/Doxyfile.config b/docs/Doxyfile.config index 1538db98e38..adffa442688 100644 --- a/docs/Doxyfile.config +++ b/docs/Doxyfile.config @@ -719,7 +719,7 @@ SHOW_NAMESPACES = YES # The FILE_VERSION_FILTER tag can be used to specify a program or script that # doxygen should invoke to get the current version for each file (typically from # the version control system). Doxygen will invoke the program by executing (via -# popen()) the command command input-file, where command is the value of the +# popen()) the command input-file, where command is the value of the # FILE_VERSION_FILTER tag, and input-file is the name of an input file provided # by doxygen. Whatever the program writes to standard output is used as the file # version. For an example see the documentation. @@ -843,16 +843,6 @@ INPUT = "@MARKDOWN_INPUT@" \ "@OpenVINO_SOURCE_DIR@/src/common/transformations/include/" \ "@OpenVINO_SOURCE_DIR@/src/common/util/include/" \ "@OpenVINO_SOURCE_DIR@/src/core/include/" \ - "@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/" \ - "@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/descriptor" \ - "@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/op/" \ - "@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/op/util" \ - "@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/opsets/" \ - "@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/pass/" \ - "@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/pattern/" \ - "@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/pattern/op/" \ - "@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/runtime/" \ - "@OpenVINO_SOURCE_DIR@/src/core/include/ngraph/type/" \ "@OpenVINO_SOURCE_DIR@/src/core/include/openvino/" \ "@OpenVINO_SOURCE_DIR@/src/core/include/openvino/core/" \ "@OpenVINO_SOURCE_DIR@/src/core/include/openvino/core/descriptor/" \ @@ -917,7 +907,9 @@ RECURSIVE = YES # Note that relative paths are relative to the directory from which doxygen is # run. -EXCLUDE = +EXCLUDE = "@OpenVINO_SOURCE_DIR@/thirdparty" \ + "@OpenVINO_SOURCE_DIR@/temp" \ + "@OpenVINO_SOURCE_DIR@/bin" # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded @@ -936,7 +928,6 @@ EXCLUDE_SYMLINKS = NO EXCLUDE_PATTERNS = */temp/* \ */bin/* \ */tests/* \ - */openvx/* \ */thirdparty/* \ "@DOXYREST_OUT@" \ "@XML_OUTPUT@" \ @@ -1045,7 +1036,6 @@ EXCLUDE_SYMBOLS = InferenceEngine::details \ EXAMPLE_PATH = "@OpenVINO_SOURCE_DIR@" \ "@OpenVINO_SOURCE_DIR@/docs/HOWTO/" \ "@OpenVINO_SOURCE_DIR@/docs/" \ - "@OpenVINO_SOURCE_DIR@/docs/onnx_custom_op/" \ "@OpenVINO_SOURCE_DIR@/docs/template_extension/" \ "@OpenVINO_SOURCE_DIR@/docs/template_extension/old/" \ "@OpenVINO_SOURCE_DIR@/docs/template_extension/new/" \ diff --git a/docs/Extensibility_UG/Intro.md b/docs/Extensibility_UG/Intro.md new file mode 100644 index 00000000000..67902dd56e6 --- /dev/null +++ b/docs/Extensibility_UG/Intro.md @@ -0,0 +1,115 @@ +# OpenVINO Extensibility Mechanism {#openvino_docs_Extensibility_UG_Intro} + +@sphinxdirective + +.. toctree:: + :maxdepth: 1 + :hidden: + + openvino_docs_Extensibility_UG_add_openvino_ops + +@endsphinxdirective + +The Intel® Distribution of OpenVINO™ toolkit supports neural network models trained with multiple frameworks including +TensorFlow, Caffe, MXNet, Kaldi, PaddlePaddle, and ONNX. The list of supported operations (layers) is different for +each of the supported frameworks. To see the operations supported by your framework, refer to +[Supported Framework Operations](../MO_DG/prepare_model/Supported_Frameworks_Layers.md). + +Custom operations, that is those not included in the list, are not recognized by OpenVINO™ out-of-the-box. Therefore, creating Intermediate Representation (IR) for a model using them requires additional steps. This guide illustrates the workflow for running inference on topologies featuring custom operations, allowing you to plug in your own implementation for existing or completely new operations. + +If your model contains operations not normally supported by OpenVINO™, the OpenVINO™ Extensibility API lets you add support for those custom operations and use one implementation for Model Optimizer and OpenVINO™ Runtime. + +There are two steps to support inference of a model with custom operation(s): +1. Add support for a [custom operation in the Model Optimizer](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md) so +the Model Optimizer can generate the IR with the operation. +2. Create a custom operation in it as described in the [Custom Operation](add_openvino_ops.md). + +## OpenVINO™ Extensions + +An OpenVINO™ provides extensions for: + + * [Custom OpenVINO™ Operation](add_openvino_ops.md): + - Enables the creation of unsupported operations + - Enables the use of `ov::Core::read_model` to read models with unsupported operations + - Provides a shape inference mechanism for custom operations + - Provides an evaluate method which allow to support the operation on CPU or perform constant folding + +> **NOTE**: This documentation is written based on the [Template extension](https://github.com/openvinotoolkit/openvino/tree/master/docs/template_extension/new), which demonstrates extension development details. You can review the complete code, which is fully compilable and up-to-date, to see how it works. + +## Load extensions to OpenVINO™ Runtime + +To load the extensions to the `ov::Core` object, use the `ov::Core::add_extension` method, this method allows to load library with extensions or extensions from the code. + +### Load extensions to core + +Extensions can be loaded from code with `ov::Core::add_extension` method: + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_extensions.cpp + :language: cpp + :fragment: add_extension + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_extensions.py + :language: python + :fragment: add_extension + +@endsphinxdirective + +### Create library with extensions + +You need to create extension library in following cases: + - Load extensions to Model Optimizer + - Load extensions to Python application + +If you want to create an extension library, for example in order to load these extensions to the Model Optimizer, you need to do next steps: +Create an entry point for extension library. OpenVINO™ provides an `OPENVINO_CREATE_EXTENSIONS()` macro, which allows to define an entry point to a library with OpenVINO™ Extensions. +This macro should have a vector of all OpenVINO™ Extensions as an argument. + +Based on that, the declaration of an extension class can look as follows: + +@snippet template_extension/new/ov_extension.cpp ov_extension:entry_point + +To configure the build of your extension library, use the following CMake script: + +@snippet template_extension/new/CMakeLists.txt cmake:extension + +This CMake script finds the OpenVINO™ using the `find_package` CMake command. + +To build the extension library, run the commands below: + +```sh +$ cd docs/template_extension/new +$ mkdir build +$ cd build +$ cmake -DOpenVINO_DIR= ../ +$ cmake --build . +``` + +After the build you can use path to your extension library to load your extensions to OpenVINO™ Runtime: + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_extensions.cpp + :language: cpp + :fragment: add_extension_lib + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_extensions.py + :language: python + :fragment: add_extension_lib + +@endsphinxdirective + +## See Also + +* [OpenVINO Transformations](./ov_transformations.md) +* [Using Inference Engine Samples](../OV_Runtime_UG/Samples_Overview.md) +* [Hello Shape Infer SSD sample](../../samples/cpp/hello_reshape_ssd/README.md) diff --git a/docs/Extensibility_UG/add_openvino_ops.md b/docs/Extensibility_UG/add_openvino_ops.md new file mode 100644 index 00000000000..7c5ed06f1fd --- /dev/null +++ b/docs/Extensibility_UG/add_openvino_ops.md @@ -0,0 +1,62 @@ +# Custom OpenVINO™ Operations {#openvino_docs_Extensibility_UG_add_openvino_ops} + +OpenVINO™ Extension API allows you to register custom operations to support models with operations which OpenVINO™ does not support out-of-the-box. + +## Operation Class + +To add your custom operation, create a new class that extends `ov::Op`, which is in turn derived from `ov::Node`, the base class for all graph operations in OpenVINO™. To add `ov::Op` please include next file: + +@snippet template_extension/new/identity.hpp op:common_include + +Follow the steps below to add a custom operation: + +1. Add the `OPENVINO_OP` macro which defines a `NodeTypeInfo` object that identifies the type of the operation to the graph users and helps with dynamic type resolution. The type info of an operation currently consists of a string operation identifier and a string for operation version. + +2. Implement default constructor and constructors that optionally take the operation inputs and attributes as parameters. + +3. Override the shape inference method `validate_and_infer_types`. This method is called multiple times during graph manipulations to determine the shapes and element types of the operations outputs. To access the input shapes and input element types, use the `get_input_partial_shape()` and `get_input_element_type()` methods of `ov::Node`. Set the inferred shape and element type of the output using `set_output_type`. + +4. Override the `clone_with_new_inputs` method, which enables graph manipulation routines to create copies of this operation and connect it to different nodes during optimization. + +5. Override the `visit_attributes` method, which enables serialization and deserialization of operation attributes. An `AttributeVisitor` is passed to the method, and the implementation is expected to walk over all the attributes in the op using the type-aware `on_attribute` helper. Helpers are already implemented for standard C++ types like `int64_t`, `float`, `bool`, `vector`, and for existing OpenVINO defined types. + +6. Override `evaluate`, which is an optional method that enables fallback of some devices to this implementation and the application of constant folding if there is a custom operation on the constant branch. If your operation contains `evaluate` method you also need to override the `has_evaluate` method, this method allow to get information about availability of `evaluate` method for the operation. + +7. Add the `OPENVINO_FRAMEWORK_MAP` macro if you want to map custom operation to framework operation with the same name. It is an optional macro which can be used for one to one mapping. In order to use this macro please include frontend specific headers: + @snippet template_extension/new/identity.hpp op:frontend_include + +Based on that, declaration of an operation class can look as follows: + +@snippet template_extension/new/identity.hpp op:header + +### Operation Constructors + +OpenVINO™ operation contains two constructors: +* Default constructor, which enables you to create an operation without attributes +* Constructor that creates and validates an operation with specified inputs and attributes + +@snippet template_extension/new/identity.cpp op:ctor + +### `validate_and_infer_types()` + +`ov::Node::validate_and_infer_types` method validates operation attributes and calculates output shapes using attributes of the operation. + +@snippet template_extension/new/identity.cpp op:validate + +### `clone_with_new_inputs()` + +`ov::Node::clone_with_new_inputs` method creates a copy of the operation with new inputs. + +@snippet template_extension/new/identity.cpp op:copy + +### `visit_attributes()` + +`ov::Node::visit_attributes` method enables you to visit all operation attributes. + +@snippet template_extension/new/identity.cpp op:visit_attributes + +### `evaluate()` and `has_evaluate()` + +`ov::Node::evaluate` method enables you to apply constant folding to an operation. + +@snippet template_extension/new/identity.cpp op:evaluate diff --git a/docs/Extensibility_UG/graph_rewrite_pass.md b/docs/Extensibility_UG/graph_rewrite_pass.md new file mode 100644 index 00000000000..11f178be4e3 --- /dev/null +++ b/docs/Extensibility_UG/graph_rewrite_pass.md @@ -0,0 +1,28 @@ +# OpenVINO Graph Rewrite Pass {#openvino_docs_Extensibility_UG_graph_rewrite_pass} + +`ov::pass::GraphRewrite` serves for running multiple matcher passes on `ov::Model` in a single graph traversal. +Example: + +@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:graph_rewrite + +In addition, GraphRewrite handles nodes that were registered by MatcherPasses during their execution. This nodes will be added to the beginning of the sequence with nodes for pattern matching. + +> **NOTE**: when using `ov::pass::Manager` temporary GraphRewrite is used to execute single MatcherPass. + +GraphRewrite has two algorithms for MatcherPasses execution. First algorithm is straightforward. It applies each MatcherPass in registration order to current node. + +![graph_rewrite_execution] + +But it is not really efficient when you have a lot of registered passes. So first of all GraphRewrite checks that all MatcherPass patterns has type-based root node (it means that type of this node is not hidden into predicate). +And then creates map from registered MatcherPasses. That helps to avoid additional cost of applying each MatcherPass for each node. + +![graph_rewrite_efficient_search] + +> **NOTE**: GraphRewrite execution algorithm cannot be set manually and depends only on root nodes registered inside MatcherPasses. + +## See Also + +* [OpenVINO™ Transformations](./ov_transformations.md) + +[graph_rewrite_execution]: ./img/graph_rewrite_execution.png +[graph_rewrite_efficient_search]: ./img/graph_rewrite_efficient_search.png diff --git a/docs/OV_Runtime_UG/img/graph_rewrite_efficient_search.png b/docs/Extensibility_UG/img/graph_rewrite_efficient_search.png similarity index 100% rename from docs/OV_Runtime_UG/img/graph_rewrite_efficient_search.png rename to docs/Extensibility_UG/img/graph_rewrite_efficient_search.png diff --git a/docs/OV_Runtime_UG/img/graph_rewrite_execution.png b/docs/Extensibility_UG/img/graph_rewrite_execution.png similarity index 100% rename from docs/OV_Runtime_UG/img/graph_rewrite_execution.png rename to docs/Extensibility_UG/img/graph_rewrite_execution.png diff --git a/docs/OV_Runtime_UG/img/ngraph_insert_node.png b/docs/Extensibility_UG/img/ngraph_insert_node.png similarity index 100% rename from docs/OV_Runtime_UG/img/ngraph_insert_node.png rename to docs/Extensibility_UG/img/ngraph_insert_node.png diff --git a/docs/OV_Runtime_UG/img/ngraph_replace_node.png b/docs/Extensibility_UG/img/ngraph_replace_node.png similarity index 100% rename from docs/OV_Runtime_UG/img/ngraph_replace_node.png rename to docs/Extensibility_UG/img/ngraph_replace_node.png diff --git a/docs/OV_Runtime_UG/img/register_new_node.png b/docs/Extensibility_UG/img/register_new_node.png similarity index 100% rename from docs/OV_Runtime_UG/img/register_new_node.png rename to docs/Extensibility_UG/img/register_new_node.png diff --git a/docs/OV_Runtime_UG/img/transformations_structure.png b/docs/Extensibility_UG/img/transformations_structure.png similarity index 100% rename from docs/OV_Runtime_UG/img/transformations_structure.png rename to docs/Extensibility_UG/img/transformations_structure.png diff --git a/docs/Extensibility_UG/matcher_pass.md b/docs/Extensibility_UG/matcher_pass.md new file mode 100644 index 00000000000..f85d0ecaefe --- /dev/null +++ b/docs/Extensibility_UG/matcher_pass.md @@ -0,0 +1,101 @@ +# OpenVINO Matcher Pass {#openvino_docs_Extensibility_UG_matcher_pass} + +`ov::pass::MatcherPass` is used for pattern-based transformations. + +Template for MatcherPass transformation class +@snippet src/transformations/template_pattern_transformation.hpp graph_rewrite:template_transformation_hpp + +@snippet src/transformations/template_pattern_transformation.cpp graph_rewrite:template_transformation_cpp + +To use `ov::pass::MatcherPass`, you need to complete these steps: +1. Create a pattern +2. Implement a callback +3. Register the pattern and Matcher +4. Execute MatcherPass + +So let's go through each of these steps. + +## Create a pattern + +Pattern is a single root `ov::Model`. But the only difference is that you do not need to create a model object, you just need to create and connect opset or special pattern operations. +Then you need to take the last created operation and put it as a root of the pattern. This root node will be used as a root node in pattern matching. +> **NOTE**: Any nodes in a pattern that have no consumers and are not registered as root will not be used in pattern matching. + +@snippet ov_model_snippets.cpp pattern:simple_example + +The `Parameter` operation in the example above has type and shape specified. These attributes are needed only to create Parameter operation class and will not be used in pattern matching. + +For more pattern examples, refer to the [pattern matching](#pattern_matching) section. + +## Implement callback + +Callback is an action applied to every pattern entrance. In general, callback is the lambda function that takes Matcher object with detected subgraph. + +@snippet ov_model_snippets.cpp pattern:callback_example + +The example above shows the callback structure and how Matcher can be used for accessing nodes detected by pattern. +Callback return value is `true` if root node was replaced and another pattern cannot be applied to the same root node; otherwise, it is `false`. +> **NOTE**: It is not recommended to manipulate with nodes that are under root node. This may affect GraphRewrite execution as it is expected that all nodes that come after root node in topological order are valid and can be used in pattern matching. + +MatcherPass also provides functionality that allows reporting of the newly created nodes that can be used in additional pattern matching. +If MatcherPass was registered in `ov::pass::Manager` or `ov::pass::GraphRewrite`, these registered nodes will be added for additional pattern matching. +That means that matcher passes registered in `ov::pass::GraphRewrite` will be applied to these nodes. + +The example below shows how single MatcherPass can fuse sequence of operations using the `register_new_node` method. + +@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:relu_fusion + +> **NOTE**: If you register multiple nodes, please add them in topological order. We do not topologically sort these nodes as it is a time-consuming operation. + +## Register pattern and Matcher + +The last step is to register Matcher and callback inside the MatcherPass pass. To do this, call the `register_matcher` method. +> **NOTE**: Only one matcher can be registered for a single MatcherPass class. + +```cpp +// Register matcher and callback +register_matcher(m, callback); +``` +## Execute MatcherPass + +MatcherPass has multiple ways to be executed: +* Run on a single node - it can be useful if you want to run MatcherPass inside another transformation. +@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:run_on_node +* Run on `ov::Model` using GraphRewrite - this approach gives ability to run MatcherPass on whole `ov::Model`. Moreover, multiple MatcherPass transformation can be registered in a single GraphRewite to be executed in a single graph traversal. +@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:graph_rewrite +* Run on `ov::Model` using `ov::pass::Manager` - this approach helps you to register MatcherPass for execution on `ov::Model` as another transformation types. +@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:manager + +## Pattern Matching + +Sometimes patterns cannot be expressed via regular operations or it is too complicated. +For example, if you want to detect **Convolution->Add** sub-graph without specifying particular input type for Convolution operation or you want to create a pattern where some of operations can have different types. +And for these cases OpenVINO™ provides additional helpers to construct patterns for GraphRewrite transformations. + +There are two main helpers: +1. `ov::pass::pattern::any_input` - helps to express inputs if their types are undefined. +2. `ov::pass::pattern::wrap_type` - helps to express nodes of pattern without specifying node attributes. + +Let's go through the example to have better understanding of how it works: + +> **NOTE**: Node attributes do not participate in pattern matching and are needed only for operations creation. Only operation types participate in pattern matching. + +The example below shows basic usage of `ov::passpattern::any_input`. +Here we construct Multiply pattern with arbitrary first input and Constant as a second input. +Also as Multiply is commutative operation, it does not matter in which order we set inputs (any_input/Constant or Constant/any_input) because both cases will be matched. + +@snippet ov_model_snippets.cpp pattern:label_example + +This example shows how we can construct a pattern when operation has arbitrary number of inputs. + +@snippet ov_model_snippets.cpp pattern:concat_example + +This example shows how to use predicate to construct a pattern. Also it shows how to match pattern manually on given node. + +@snippet ov_model_snippets.cpp pattern:predicate_example + +> **NOTE**: Be careful with manual matching because Matcher object holds matched nodes. To clear a match, use the m->clear_state() method. + +## See Also + +* [OpenVINO™ Transformations](./ov_transformations.md) diff --git a/docs/Extensibility_UG/model_pass.md b/docs/Extensibility_UG/model_pass.md new file mode 100644 index 00000000000..d2add64d3a5 --- /dev/null +++ b/docs/Extensibility_UG/model_pass.md @@ -0,0 +1,17 @@ +# OpenVINO Model Pass {#openvino_docs_Extensibility_UG_model_pass} + +`ov::pass::ModelPass` is used for transformations that take entire `ov::Model` as an input and process it. + +Template for ModelPass transformation class + +@snippet src/transformations/template_model_transformation.hpp model_pass:template_transformation_hpp + +@snippet src/transformations/template_model_transformation.cpp model_pass:template_transformation_cpp + +Using `ov::pass::ModelPass`, you need to override the `run_on_model` method where you will write the transformation code. +Return value is `true` if the original model has changed during transformation (new operation was added, or operations replacement was made, or node attributes were changed); otherwise, it is `false`. +Also `ov::pass::ModelPass` based transformations can be executed via `ov::pass::Manager`. + +## See Also + +* [OpenVINO™ Transformations](./ov_transformations.md) diff --git a/docs/Extensibility_UG/ov_transformations.md b/docs/Extensibility_UG/ov_transformations.md new file mode 100644 index 00000000000..3a39866eb95 --- /dev/null +++ b/docs/Extensibility_UG/ov_transformations.md @@ -0,0 +1,172 @@ +# Overview of Transformations API {#openvino_docs_transformations} + +@sphinxdirective + +.. toctree:: + :maxdepth: 1 + :hidden: + + openvino_docs_Extensibility_UG_model_pass + openvino_docs_Extensibility_UG_matcher_pass + openvino_docs_Extensibility_UG_graph_rewrite_pass + +@endsphinxdirective + +This guide contains all necessary information that you need to start implementing OpenVINO™ transformations. + +## Working with Model + +Before the moving to transformation part it is needed to say several words about functions which allow to modify `ov::Model`. +This chapter extends the [model representation guide](../OV_Runtime_UG/model_representation.md) and shows an API that allows us to manipulate with `ov::Model`. + +### Working with node input and output ports + +First of all let's talk about `ov::Node` input/output ports. Each OpenVINO™ operation has input and output ports except cases when operation has `Parameter` or `Constant` type. + +Every port belongs to its node, so using a port we can access parent node, get shape and type for particular input/output, get all consumers in case of output port, and get producer node in case of input port. +With output port we can set inputs for newly created operations. + +Lets look at the code example. + +@snippet ov_model_snippets.cpp ov:ports_example + +### Node replacement + +OpenVINO™ provides two ways for node replacement: via OpenVINO™ helper function and directly via port methods. We are going to review both of them. + +Let's start with OpenVINO™ helper functions. The most popular function is `ov::replace_node(old_node, new_node)`. + +We will review real replacement case where Negative operation is replaced with Multiply. + +![ngraph_replace_node] + +@snippet ov_model_snippets.cpp ov:replace_node + +`ov::replace_node` has a constraint that number of output ports for both of ops must be the same; otherwise, it raises an exception. + + +The alternative way to do the same replacement is the following: + +@snippet ov_model_snippets.cpp ov:manual_replace + +Another transformation example is insertion. + +![ngraph_insert_node] + +@snippet ov_model_snippets.cpp ov:insert_node + +The alternative way to the insert operation is to make a node copy and use `ov::replace_node()`: + +@snippet ov_model_snippets.cpp ov:insert_node_with_copy + +### Node elimination + +Another type of node replacement is its elimination. + +To eliminate operation, OpenVINO™ has special method that considers all limitations related to OpenVINO™ Runtime. + +@snippet ov_model_snippets.cpp ov:eliminate_node + +`ov::replace_output_update_name()` in case of successful replacement it automatically preserves friendly name and runtime info. + +## Transformations types + +OpenVINO™ Runtime has three main transformation types: + +* [Model pass](./model_pass.md) - straightforward way to work with `ov::Model` directly +* [Matcher pass](./matcher_pass.md) - pattern-based transformation approach +* [Graph rewrite pass](./graph_rewrite_pass.md) - container for matcher passes needed for efficient execution + +![transformations_structure] + +## Transformation conditional compilation + +Transformation library has two internal macros to support conditional compilation feature. + +* `MATCHER_SCOPE(region)` - allows to disable the MatcherPass if matcher isn't used. The region name should be unique. This macro creates a local variable `matcher_name` which you should use as a matcher name. +* `RUN_ON_MODEL_SCOPE(region)` - allows to disable run_on_model pass if it isn't used. The region name should be unique. + +## Transformation writing essentials + +When developing a transformation, you need to follow these transformation rules: + +###1. Friendly Names + +Each `ov::Node` has an unique name and a friendly name. In transformations we care only about friendly name because it represents the name from the model. +To avoid losing friendly name when replacing node with other node or subgraph, set the original friendly name to the latest node in replacing subgraph. See the example below. + +@snippet ov_model_snippets.cpp ov:replace_friendly_name + +In more advanced cases, when replaced operation has several outputs and we add additional consumers to its outputs, we make a decision how to set friendly name by arrangement. + +###2. Runtime Info + +Runtime info is a map `std::map` located inside `ov::Node` class. It represents additional attributes in `ov::Node`. +These attributes can be set by users or by plugins and when executing transformation that changes `ov::Model` we need to preserve these attributes as they will not be automatically propagated. +In most cases, transformations have the following types: 1:1 (replace node with another node), 1:N (replace node with a sub-graph), N:1 (fuse sub-graph into a single node), N:M (any other transformation). +Currently, there is no mechanism that automatically detects transformation types, so we need to propagate this runtime information manually. See the examples below. + +@snippet ov_model_snippets.cpp ov:copy_runtime_info + +When transformation has multiple fusions or decompositions, `ov::copy_runtime_info` must be called multiple times for each case. + +**Note**: copy_runtime_info removes rt_info from destination nodes. If you want to keep it, you need to specify them in source nodes like this: copy_runtime_info({a, b, c}, {a, b}) + +###3. Constant Folding + +If your transformation inserts constant sub-graphs that need to be folded, do not forget to use `ov::pass::ConstantFolding()` after your transformation or call constant folding directly for operation. +The example below shows how constant subgraph can be constructed. + +@snippet ov_model_snippets.cpp ov:constant_subgraph + +Manual constant folding is more preferable than `ov::pass::ConstantFolding()` because it is much faster. + +Below you can find an example of manual constant folding: + +@snippet src/transformations/template_pattern_transformation.cpp manual_constant_folding + +## Common mistakes in transformations + +In transformation development process: + +* Do not use deprecated OpenVINO™ API. Deprecated methods has the `OPENVINO_DEPRECATED` macros in its definition. +* Do not pass `shared_ptr` as an input for other node if type of node is unknown or it has multiple outputs. Use explicit output port. +* If you replace node with another node that produces different shape, remember that new shape will not be propagated until the first `validate_nodes_and_infer_types` call for `ov::Model`. If you are using `ov::pass::Manager`, it will automatically call this method after each transformation execution. +* Do not forget to call the `ov::pass::ConstantFolding` pass if your transformation creates constant subgraphs. +* Use latest OpSet if you are not developing downgrade transformation pass. +* When developing a callback for `ov::pass::MatcherPass`, do not change nodes that come after the root node in topological order. + +## Using pass manager + +`ov::pass::Manager` is a container class that can store the list of transformations and execute them. The main idea of this class is to have high-level representation for grouped list of transformations. +It can register and apply any [transformation pass](#transformations_types) on model. +In addition, `ov::pass::Manager` has extended debug capabilities (find more information in the [how to debug transformations](#how_to_debug_transformations) section). + +The example below shows basic usage of `ov::pass::Manager` + +@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:manager3 + +Another example shows how multiple matcher passes can be united into single GraphRewrite. + +@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:manager2 + +## How to debug transformations + +If you are using `ngraph::pass::Manager` to run sequence of transformations, you can get additional debug capabilities by using the following environment variables: + +``` +OV_PROFILE_PASS_ENABLE=1 - enables performance measurement for each transformation and prints execution status +OV_ENABLE_VISUALIZE_TRACING=1 - enables visualization after each transformation. By default, it saves dot and svg files. +``` + +> **Note**: Make sure that you have dot installed on your machine; otherwise, it will silently save only dot file without svg file. + +## See Also + +* [OpenVINO™ Model Representation](../OV_Runtime_UG/model_representation.md) +* [OpenVINO™ Extensions](./Intro.md) + +[ngraph_replace_node]: ./img/ngraph_replace_node.png +[ngraph_insert_node]: ./img/ngraph_insert_node.png +[transformations_structure]: ./img/transformations_structure.png +[register_new_node]: ./img/register_new_node.png diff --git a/docs/HOWTO/Custom_Layers_Guide.md b/docs/HOWTO/Custom_Layers_Guide.md deleted file mode 100644 index b3dd18f3b14..00000000000 --- a/docs/HOWTO/Custom_Layers_Guide.md +++ /dev/null @@ -1,349 +0,0 @@ -# Custom Operations Guide {#openvino_docs_HOWTO_Custom_Layers_Guide} - -The Intel® Distribution of OpenVINO™ toolkit supports neural network models trained with multiple frameworks including -TensorFlow*, Caffe*, MXNet*, Kaldi* and ONNX* file format. The list of supported operations (layers) is different for -each of the supported frameworks. To see the operations supported by your framework, refer to -[Supported Framework Layers](../MO_DG/prepare_model/Supported_Frameworks_Layers.md). - -Custom operations, that is those not included in the list, are not recognized by Model Optimizer out-of-the-box. Therefore, creating Intermediate Representation (IR) for a model using them requires additional steps. This guide illustrates the workflow for running inference on topologies featuring custom operations, allowing you to plug in your own implementation for existing or completely new operations. - -> **NOTE**: *Layer* is a legacy term for *operation* which came from Caffe\* framework. Currently it is not used. -> Refer to the [Deep Learning Network Intermediate Representation and Operation Sets in OpenVINO™](../MO_DG/IR_and_opsets.md) -> for more information on the topic. - -## Terms Used in This Guide - -- *Intermediate Representation (IR)* — OpenVINO's Neural Network format used by Inference Engine. It abstracts different frameworks and describs model topology, operations parameters, and weights. - -- *Operation* — an abstract concept of a math function selected for a specific purpose. Operations supported by - OpenVINO™ are listed in the supported operation set provided in the [Available Operations Sets](../ops/opset.md). - Examples of the operations are: [ReLU](../ops/activation/ReLU_1.md), [Convolution](../ops/convolution/Convolution_1.md), - [Add](../ops/arithmetic/Add_1.md), etc. - -- *Kernel* — The implementation of an operation function in the OpenVINO™ plugin, in this case, the math programmed (in - C++ and OpenCL) to perform the operation for a target hardware (CPU or GPU). - -- *Inference Engine Extension* — Device-specific module implementing custom operations (a set of kernels). - -## Custom Operation Support Overview - -There are three steps to support inference of a model with custom operation(s): -1. Add support for a custom operation in the [Model Optimizer](../MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) so -the Model Optimizer can generate the IR with the operation. -2. Create an operation set and implement a custom nGraph operation in it as described in the -[Custom nGraph Operation](../OV_Runtime_UG/Extensibility_DG/AddingNGraphOps.md). -3. Implement a customer operation in one of the [OpenVINO™ Runtime](../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) -plugins to support inference of this operation using a particular target hardware (CPU, GPU or VPU). - -To see the operations that are supported by each device plugin for the Inference Engine, refer to the -[Supported Devices](../OV_Runtime_UG/supported_plugins/Supported_Devices.md). - -> **NOTE**: If a device doesn't support a particular operation, an alternative to creating a new operation is to target -> an additional device using the HETERO plugin. The [Heterogeneous Plugin](../OV_Runtime_UG/supported_plugins/HETERO.md) may be -> used to run an inference model on multiple devices allowing the unsupported operations on one device to "fallback" to -> run on another device (e.g., CPU) that does support those operations. - -### Custom Operation Support for the Model Optimizer - -Model Optimizer model conversion pipeline is described in detail in "Model Conversion Pipeline" section of [Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md). It is best to read that article first for a better understanding of the following material. - -Model Optimizer provides an extensions mechanism to support new operations and implement custom model transformations to generate optimized IR. This mechanism is described in the "Model Optimizer Extensions" section of -[Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md). - -Two types of Model Optimizer extensions should be implemented to support custom operations, at a minimum: -1. Operation class for a new operation. This class stores information about the operation, its attributes, shape inference function, attributes to be saved to an IR and some others internally used attributes. Refer to the "Model Optimizer Operation" section of [Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md) for detailed instructions on how to implement it. -2. Operation attributes extractor. The extractor is responsible for parsing framework-specific representation of the -operation and uses corresponding operation class to update graph node attributes with necessary attributes of the -operation. Refer to the "Operation Extractor" section of -[Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md) for detailed instructions on how to implement it. - -> **NOTE**: In some cases you may need to implement some transformation to support the operation. This topic is covered in the "Graph Transformation Extensions" section of [Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md). - -## Custom Operations Extensions for the Inference Engine - -Inference Engine provides an extension mechanism to support new operations. This mechanism is described in [Inference Engine Extensibility Mechanism](../OV_Runtime_UG/Extensibility_DG/Intro.md). - -Each device plugin includes a library of optimized implementations to execute known operations which must be extended to execute a custom operation. The custom operation extension is implemented according to the target device: - -- Custom Operation CPU Extension - - A compiled shared library (`.so` or `.dll`) needed by the CPU Plugin for executing the custom operation - on a CPU. Refer to the [How to Implement Custom CPU Operations](../OV_Runtime_UG/Extensibility_DG/CPU_Kernel.md) for more - details. -- Custom Operation GPU Extension - - OpenCL source code (.cl) for the custom operation kernel that will be compiled to execute on the GPU along with an operation description file (.xml) needed by the GPU Plugin for the custom operation kernel. Refer to the [How to Implement Custom GPU Operations](../OV_Runtime_UG/Extensibility_DG/GPU_Kernel.md) for more details. -- Custom Operation VPU Extension - - OpenCL source code (.cl) for the custom operation kernel that will be compiled to execute on the VPU along with an operation description file (.xml) needed by the VPU Plugin for the custom operation kernel. Refer to [How to Implement Custom Operations for VPU](../OV_Runtime_UG/Extensibility_DG/VPU_Kernel.md) for more details. - -Also, it is necessary to implement nGraph custom operation according to [Custom nGraph Operation](../OV_Runtime_UG/Extensibility_DG/AddingNGraphOps.md) so the Inference Engine can read an IR with this -operation and correctly infer output tensor shape and type. - -## Enabling Magnetic Resonance Image Reconstruction Model -This chapter provides step-by-step instructions on how to enable the magnetic resonance image reconstruction model implemented in the [repository](https://github.com/rmsouza01/Hybrid-CS-Model-MRI/) using a custom operation on CPU. The example is prepared for a model generated from the repository with hash `2ede2f96161ce70dcdc922371fe6b6b254aafcc8`. - -### Download and Convert the Model to a Frozen TensorFlow\* Model Format -The original pre-trained model is provided in the hdf5 format which is not supported by OpenVINO directly and needs to be converted to TensorFlow\* frozen model format first. - -1. Download repository `https://github.com/rmsouza01/Hybrid-CS-Model-MRI`:
-```bash - git clone https://github.com/rmsouza01/Hybrid-CS-Model-MRI - git checkout 2ede2f96161ce70dcdc922371fe6b6b254aafcc8 -``` - -2. Convert pre-trained `.hdf5` to a frozen `.pb` graph using the following script (tested with TensorFlow==1.15.0 and -Keras==2.2.4) which should be executed from the root of the cloned repository:
-```py - import keras as K - import numpy as np - import Modules.frequency_spatial_network as fsnet - import tensorflow as tf - - under_rate = '20' - - stats = np.load("Data/stats_fs_unet_norm_" + under_rate + ".npy") - var_sampling_mask = np.load("Data/sampling_mask_" + under_rate + "perc.npy") - - model = fsnet.wnet(stats[0], stats[1], stats[2], stats[3], kshape = (5,5), kshape2=(3,3)) - model_name = "Models/wnet_" + under_rate + ".hdf5" - model.load_weights(model_name) - - inp = np.random.standard_normal([1, 256, 256, 2]).astype(np.float32) - np.save('inp', inp) - - sess = K.backend.get_session() - sess.as_default() - graph_def = sess.graph.as_graph_def() - graph_def = tf.graph_util.convert_variables_to_constants(sess, graph_def, ['conv2d_44/BiasAdd']) - with tf.gfile.FastGFile('wnet_20.pb', 'wb') as f: - f.write(graph_def.SerializeToString()) -``` - -As a result the TensorFlow\* frozen model file "wnet_20.pb" is generated. - -### Convert the Frozen TensorFlow\* Model to Intermediate Representation - -Firstly, open the model in TensorBoard or other TensorFlow* model visualization tool. The model supports dynamic -batch dimension because the value for the batch dimension is not hardcoded in the model. Model Optimizer need to set all -dynamic dimensions to some specific value to create the IR, therefore specify the command line parameter `-b 1` to set -the batch dimension equal to 1. The actual batch size dimension can be changed at runtime using the Inference Engine API -described in the [Using Shape Inference](../OV_Runtime_UG/ShapeInference.md). Also refer to the General Conversion Parameters section in [Converting a Model to Intermediate Representation (IR)](../MO_DG/prepare_model/convert_model/Converting_Model.md) and [Convert Your TensorFlow* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md) -for more details and command line parameters used for the model conversion. - -```sh -mo --input_model /wnet_20.pb -b 1 -``` - -> **NOTE**: This conversion guide is applicable for the 2021.3 release of OpenVINO and that starting from 2021.4 -> the OpenVINO supports this model out of the box. - -Model Optimizer produces the following error: -```bash -[ ERROR ] List of operations that cannot be converted to Inference Engine IR: -[ ERROR ] Complex (1) -[ ERROR ] lambda_2/Complex -[ ERROR ] IFFT2D (1) -[ ERROR ] lambda_2/IFFT2D -[ ERROR ] ComplexAbs (1) -[ ERROR ] lambda_2/Abs -[ ERROR ] Part of the nodes was not converted to IR. Stopped. -``` - -The error means that the Model Optimizer doesn't know how to handle 3 types of TensorFlow\* operations: "Complex", -"IFFT2D" and "ComplexAbs". In order to see more details about the conversion process run the model conversion with -additional parameter `--log_level DEBUG`. It is worth to mention the following lines from the detailed output: - -```bash -[ INFO ] Called "tf_native_tf_node_infer" for node "lambda_2/Complex" -[ ] [ DEBUG ] [ tf:228 ] Added placeholder with name 'lambda_2/lambda_3/strided_slice_port_0_ie_placeholder' -[ ] [ DEBUG ] [ tf:228 ] Added placeholder with name 'lambda_2/lambda_4/strided_slice_port_0_ie_placeholder' -[ ] [ DEBUG ] [ tf:241 ] update_input_in_pbs: replace input 'lambda_2/lambda_3/strided_slice' with input 'lambda_2/lambda_3/strided_slice_port_0_ie_placeholder' -[ ] [ DEBUG ] [ tf:249 ] Replacing input '0' of the node 'lambda_2/Complex' with placeholder 'lambda_2/lambda_3/strided_slice_port_0_ie_placeholder' -[ ] [ DEBUG ] [ tf:241 ] update_input_in_pbs: replace input 'lambda_2/lambda_4/strided_slice' with input 'lambda_2/lambda_4/strided_slice_port_0_ie_placeholder' -[ ] [ DEBUG ] [ tf:249 ] Replacing input '1' of the node 'lambda_2/Complex' with placeholder 'lambda_2/lambda_4/strided_slice_port_0_ie_placeholder' -[ ] [ DEBUG ] [ tf:148 ] Inferred shape of the output tensor with index '0' of the node 'lambda_2/Complex': '[ 1 256 256]' -[ ] [ DEBUG ] [ infer:145 ] Outputs: -[ ] [ DEBUG ] [ infer:32 ] output[0]: shape = [ 1 256 256], value = -[ ] [ DEBUG ] [ infer:129 ] -------------------- -[ ] [ DEBUG ] [ infer:130 ] Partial infer for lambda_2/IFFT2D -[ ] [ DEBUG ] [ infer:131 ] Op: IFFT2D -[ ] [ DEBUG ] [ infer:132 ] Inputs: -[ ] [ DEBUG ] [ infer:32 ] input[0]: shape = [ 1 256 256], value = -``` - -This is a part of the log of the partial inference phase of the model conversion. See the "Partial Inference" section on -the [Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md) for -more information about this phase. Model Optimizer inferred output shape for the unknown operation of type "Complex" -using a "fallback" to TensorFlow\*. However, it is not enough to generate the IR because Model Optimizer doesn't know -which attributes of the operation should be saved to IR. So it is necessary to implement Model Optimizer extensions to -support these operations. - -Before going into the extension development it is necessary to understand what these unsupported operations do according -to the TensorFlow\* framework specification. - -* "Complex" - returns a tensor of complex type constructed from two real input tensors specifying real and imaginary -part of a complex number. -* "IFFT2D" - returns a tensor with inverse 2-dimensional discrete Fourier transform over the inner-most 2 dimensions of - an input. -* "ComplexAbs" - returns a tensor with absolute values of input tensor with complex numbers. - -The part of the model with all three unsupported operations is depicted below: - -![Unsupported sub-graph](img/unsupported_subgraph.png) - -This model uses complex numbers during the inference but Inference Engine does not support tensors of this data type. So -it is necessary to find a way how to avoid using tensors of such a type in the model. Fortunately, the complex tensor -appear as a result of "Complex" operation, is used as input in the "IFFT2D" operation then is passed to "ComplexAbs" -which produces real value tensor as output. So there are just 3 operations consuming/producing complex tensors in the -model. - -Let's design an OpenVINO operation "FFT" which get a single real number tensor describing the complex number and -produces a single real number tensor describing output complex tensor. This way the fact that the model uses complex -numbers is hidden inside the "FFT" operation implementation. The operation gets a tensor of shape `[N, H, W, 2]` and -produces the output tensor with the same shape, where the innermost dimension contains pairs of real numbers describing -the complex number (its real and imaginary part). As we will see further this operation will allow us to support the -model. The implementation of the Model Optimizer operation should be saved to `mo_extensions/ops/FFT.py` file: - -@snippet FFT.py fft:operation - -The attribute `inverse` is a flag specifying type of the FFT to apply: forward or inverse. - -See the "Model Optimizer Operation" section of [Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md) for detailed instructions on how to implement the operation. - -Now it is necessary to implement extractor for the "IFFT2D" operation according to the -"Operation Extractor" section of [Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md). The -following snippet provides two extractors: one for "IFFT2D", another one for "FFT2D", however only on of them is used in this example. The implementation should be saved to the file `mo_extensions/front/tf/FFT_ext.py`. - -@snippet FFT_ext.py fft_ext:extractor - -> **NOTE**: The graph is in inconsistent state after extracting node attributes because according to original operation -> "IFFT2D" semantic it should have an input consuming a tensor of complex numbers, but the extractor instantiated an -> operation "FFT" which expects a real tensor with specific layout. But the inconsistency will be resolved during -> applying front phase transformations discussed below. - -The output shape of the operation "AddV2" from the picture above is `[N, H, W, 2]`. Where the innermost dimension -contains pairs of real numbers describing the complex number (its real and imaginary part). The following "StridedSlice" -operations split the input tensor into 2 parts to get a tensor of real and a tensor of imaginary parts which are then -consumed with the "Complex" operation to produce a tensor of complex numbers. These "StridedSlice" and "Complex" -operations can be removed so the "FFT" operation will get a real value tensor encoding complex numbers. To achieve this -we implement the front phase transformation which searches for a pattern of two "StridedSlice" operations with specific -attributes producing data to "Complex" operation and removes it from the graph. Refer to the -"Pattern-Defined Front Phase Transformations" section of [Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md) for more -information on how this type of transformation works. The code snippet should be saved to the file -`mo_extensions/front/tf/Complex.py`. - -@snippet Complex.py complex:transformation - -> **NOTE**: The graph is in inconsistent state because the "ComplexAbs" operation consumes complex value tensor but -> "FFT" produces real value tensor. - -Now lets implement a transformation which replace a "ComplexAbs" operation with a sub-graph of primitive operations -which calculate the result using the following formulae: \f$module(z) = \sqrt{real(z) \cdot real(z) + imag(z) \cdot imag(z)}\f$. -Original "IFFT2D" operation produces tensor of complex values, but the "FFT" operation produces a real value tensor with -the same format and shape as the input for the operation. So the input shape for the "ComplexAbs" will be `[N, H, W, 2]` -with the innermost dimension containing tuple with real and imaginary part of a complex number. In order to calculate -absolute values for the complex tensor we do the following: -1. Raise all elements in the power of 2. -2. Calculate a reduced sum over the innermost dimension. -3. Calculate a square root. - -The implementation should be saved to the file `mo_extensions/front/tf/ComplexAbs.py` and provided below: - -@snippet ComplexAbs.py complex_abs:transformation - -Now it is possible to convert the model using the following command line: -```sh -mo --input_model /wnet_20.pb -b 1 --extensions mo_extensions/ -``` - -The sub-graph corresponding to the originally non-supported one is depicted in the image below: - -![Converted sub-graph](img/converted_subgraph.png) - -> **NOTE**: Model Optimizer performed conversion of the model from NHWC to NCHW layout that is why the dimension with -> the value 2 moved to another position. - -### Inference Engine Extension Implementation -Now it is necessary to implement the extension for the CPU plugin with operation "FFT" introduced previously. The code -below is based on the template extension described in [Inference Engine Extensibility Mechanism](../OV_Runtime_UG/Extensibility_DG/Intro.md). - -#### CMake Build File -The first step is to create a CMake configuration file which builds the extension. The content of the "CMakeLists.txt" -file is the following: - -@snippet template_extension/old/CMakeLists.txt cmake:extension - -The CPU FFT kernel implementation uses OpenCV to perform the FFT that is why the extension library is linked with -`opencv_core` which comes with the OpenVINO. - -#### Custom nGraph Operation "FFT" Implementation -The next step is to create the nGraph operation FFT. The header file "fft_op.hpp" has the following content: - -@snippet template_extension/old/fft_op.hpp fft_op:header - -The operation has just one boolean attribute `inverse`. Implementation of the necessary nGraph operation functions are -in the `fft_op.cpp` file with the following content: - -@snippet template_extension/old/fft_op.cpp fft_op:implementation - -Refer to the [Custom nGraph Operation](../OV_Runtime_UG/Extensibility_DG/AddingNGraphOps.md) for more details. - -#### CPU FFT Kernel Implementation -The operation implementation for CPU plugin uses OpenCV to perform the FFT. The header file "fft_kernel.hpp" has the -following content: - -@snippet template_extension/old/fft_kernel.hpp fft_kernel:header - -The "fft_kernel.cpp" with the implementation of the CPU has the following content: - -@snippet template_extension/old/fft_kernel.cpp fft_kernel:implementation - -Refer to the [How to Implement Custom CPU Operations](../OV_Runtime_UG/Extensibility_DG/CPU_Kernel.md) for more details. - -#### Extension Library Implementation -The last step is to create an extension library "extension.cpp" and "extension.hpp" which will include the FFT -operation for the CPU plugin. The code of the library is described in the [Extension Library](../OV_Runtime_UG/Extensibility_DG/Extension.md). - -### Building and Running the Custom Extension -To build the extension, run the following:
-```bash -mkdir build && cd build -source /opt/intel/openvino_2022/setupvars.sh -cmake .. -DCMAKE_BUILD_TYPE=Release -make --jobs=$(nproc) -``` - -The result of this command is a compiled shared library (`.so` or `.dll`). It should be loaded in the -application using `Core` class instance method `AddExtension` like this -`core.AddExtension(std::make_shared(compiled_library_file_name), "CPU");`. - -To test that the extension is implemented correctly we can run the "mri_reconstruction_demo" with the following content: - -@snippet mri_reconstruction_demo.py mri_demo:demo - -The script can be executed using the following command line: -```bash -python3 mri_reconstruction_demo.py \ - -m /wnet_20.xml \ - -i .npy \ - -p /Data/sampling_mask_20perc.npy \ - -l /libtemplate_extension.so \ - -d CPU -``` - -## Additional Resources - -- Intel® Distribution of OpenVINO™ toolkit home page: [https://software.intel.com/en-us/openvino-toolkit](https://software.intel.com/en-us/openvino-toolkit) -- OpenVINO™ toolkit online documentation: [https://docs.openvino.ai](https://docs.openvino.ai) -- [Model Optimizer Developer Guide](../MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) -- [Model Optimizer Extensibility](../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md) -- [Inference Engine Extensibility Mechanism](../OV_Runtime_UG/Extensibility_DG/Intro.md) -- [OpenVINO™ Toolkit Samples Overview](../OV_Runtime_UG/Samples_Overview.md) -- [Overview of OpenVINO™ Toolkit Pre-Trained Models](@ref omz_models_group_intel) -- For IoT Libraries and Code Samples see the [Intel® IoT Developer Kit](https://github.com/intel-iot-devkit). - -## Converting Models: - -- [Convert Your Caffe* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_Caffe.md) -- [Convert Your TensorFlow* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md) -- [Convert Your MXNet* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_MxNet.md) -- [Convert Your Kaldi* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_Kaldi.md) -- [Convert Your ONNX* Model](../MO_DG/prepare_model/convert_model/Convert_Model_From_ONNX.md) diff --git a/docs/IE_PLUGIN_DG/AsyncInferRequest.md b/docs/IE_PLUGIN_DG/AsyncInferRequest.md index d9aa2a5d3c0..59c5beadd55 100644 --- a/docs/IE_PLUGIN_DG/AsyncInferRequest.md +++ b/docs/IE_PLUGIN_DG/AsyncInferRequest.md @@ -1,7 +1,7 @@ # Asynchronous Inference Request {#openvino_docs_ie_plugin_dg_async_infer_request} Asynchronous Inference Request runs an inference pipeline asynchronously in one or several task executors depending on a device pipeline structure. -Inference Engine Plugin API provides the base InferenceEngine::AsyncInferRequestThreadSafeDefault class: +OpenVINO Runtime Plugin API provides the base InferenceEngine::AsyncInferRequestThreadSafeDefault class: - The class has the `_pipeline` field of `std::vector >`, which contains pairs of an executor and executed task. - All executors are passed as arguments to a class constructor and they are in the running state and ready to run tasks. @@ -10,7 +10,7 @@ Inference Engine Plugin API provides the base InferenceEngine::AsyncInferRequest `AsyncInferRequest` Class ------------------------ -Inference Engine Plugin API provides the base InferenceEngine::AsyncInferRequestThreadSafeDefault class for a custom asynchronous inference request implementation: +OpenVINO Runtime Plugin API provides the base InferenceEngine::AsyncInferRequestThreadSafeDefault class for a custom asynchronous inference request implementation: @snippet src/template_async_infer_request.hpp async_infer_request:header diff --git a/docs/IE_PLUGIN_DG/Doxyfile b/docs/IE_PLUGIN_DG/Doxyfile index 7d7735b1fbc..84416e0483e 100644 --- a/docs/IE_PLUGIN_DG/Doxyfile +++ b/docs/IE_PLUGIN_DG/Doxyfile @@ -675,7 +675,7 @@ SHOW_NAMESPACES = YES # The FILE_VERSION_FILTER tag can be used to specify a program or script that # doxygen should invoke to get the current version for each file (typically from # the version control system). Doxygen will invoke the program by executing (via -# popen()) the command command input-file, where command is the value of the +# popen()) the command input-file, where command is the value of the # FILE_VERSION_FILTER tag, and input-file is the name of an input file provided # by doxygen. Whatever the program writes to standard output is used as the file # version. For an example see the documentation. diff --git a/docs/IE_PLUGIN_DG/InferRequest.md b/docs/IE_PLUGIN_DG/InferRequest.md index 5850c3fc625..5d50c8e36fc 100644 --- a/docs/IE_PLUGIN_DG/InferRequest.md +++ b/docs/IE_PLUGIN_DG/InferRequest.md @@ -54,7 +54,7 @@ Decrements a number of created inference requests: #### 1. `inferPreprocess` -Below is the code of the the `inferPreprocess` method to demonstrate Inference Engine common preprocessing step handling: +Below is the code of the `inferPreprocess` method to demonstrate Inference Engine common preprocessing step handling: @snippet src/template_infer_request.cpp infer_request:infer_preprocess diff --git a/docs/IE_PLUGIN_DG/Intro.md b/docs/IE_PLUGIN_DG/Intro.md index 17228b12115..99cbffbec9c 100644 --- a/docs/IE_PLUGIN_DG/Intro.md +++ b/docs/IE_PLUGIN_DG/Intro.md @@ -56,7 +56,7 @@ Detailed guides * Plugin and its components [testing](@ref openvino_docs_ie_plugin_dg_plugin_testing) * [Quantized networks](@ref openvino_docs_ie_plugin_dg_quantized_networks) * [Low precision transformations](@ref openvino_docs_IE_DG_lpt) guide -* [Writing nGraph transformations](@ref ngraph_transformation) guide +* [Writing OpenVINO™ transformations](@ref openvino_docs_transformations) guide API References ----------------------- diff --git a/docs/IE_PLUGIN_DG/Plugin.md b/docs/IE_PLUGIN_DG/Plugin.md index f92a03021a8..410bed856c9 100644 --- a/docs/IE_PLUGIN_DG/Plugin.md +++ b/docs/IE_PLUGIN_DG/Plugin.md @@ -30,7 +30,7 @@ Based on that, declaration of a plugin class can look as follows: The provided plugin class also has several fields: -* `_backend` - a backend engine that is used to perform actual computations for network inference. For `Template` plugin `ngraph::runtime::Backend` is used which performs computations using ngraph reference implementations. +* `_backend` - a backend engine that is used to perform actual computations for network inference. For `Template` plugin `ngraph::runtime::Backend` is used which performs computations using OpenVINO™ reference implementations. * `_waitExecutor` - a task executor that waits for a response from a device about device tasks completion. * `_cfg` of type `Configuration`: @@ -67,7 +67,7 @@ which holds a backend-dependent compiled graph in an internal representation: Before a creation of an `ExecutableNetwork` instance via a constructor, a plugin may check if a provided InferenceEngine::ICNNNetwork object is supported by a device. In the example above, the plugin checks precision information. -The very important part before creation of `ExecutableNetwork` instance is to call `TransformNetwork` method which applies ngraph transformation passes. +The very important part before creation of `ExecutableNetwork` instance is to call `TransformNetwork` method which applies OpenVINO™ transformation passes. Actual graph compilation is done in the `ExecutableNetwork` constructor. Refer to the [ExecutableNetwork Implementation Guide](@ref openvino_docs_ie_plugin_dg_executable_network) for details. @@ -77,27 +77,27 @@ Actual graph compilation is done in the `ExecutableNetwork` constructor. Refer t ### `TransformNetwork()` -The function accepts a const shared pointer to `ngraph::Function` object and performs the following steps: +The function accepts a const shared pointer to `ov::Model` object and performs the following steps: 1. Deep copies a const object to a local object, which can later be modified. -2. Applies common and plugin-specific transformations on a copied graph to make the graph more friendly to hardware operations. For details how to write custom plugin-specific transformation, please, refer to [Writing ngraph transformations](@ref ngraph_transformation) guide. See detailed topics about network representation: +2. Applies common and plugin-specific transformations on a copied graph to make the graph more friendly to hardware operations. For details how to write custom plugin-specific transformation, please, refer to [Writing OpenVINO™ transformations](@ref openvino_docs_transformations) guide. See detailed topics about network representation: * [Intermediate Representation and Operation Sets](../_docs_MO_DG_IR_and_opsets.html) * [Quantized networks](@ref openvino_docs_ie_plugin_dg_quantized_networks). @snippet template_plugin/src/template_plugin.cpp plugin:transform_network -> **NOTE**: After all these transformations, a `ngraph::Function` object contains operations which can be perfectly mapped to backend kernels. E.g. if backend has kernel computing `A + B` operations at once, the `TransformNetwork` function should contain a pass which fuses operations `A` and `B` into a single custom operation `A + B` which fits backend kernels set. +> **NOTE**: After all these transformations, a `ov::Model` object contains operations which can be perfectly mapped to backend kernels. E.g. if backend has kernel computing `A + B` operations at once, the `TransformNetwork` function should contain a pass which fuses operations `A` and `B` into a single custom operation `A + B` which fits backend kernels set. ### `QueryNetwork()` Use the method with the `HETERO` mode, which allows to distribute network execution between different -devices based on the `ngraph::Node::get_rt_info()` map, which can contain the `"affinity"` key. +devices based on the `ov::Node::get_rt_info()` map, which can contain the `"affinity"` key. The `QueryNetwork` method analyzes operations of provided `network` and returns a list of supported -operations via the InferenceEngine::QueryNetworkResult structure. The `QueryNetwork` firstly applies `TransformNetwork` passes to input `ngraph::Function` argument. After this, the transformed network in ideal case contains only operations are 1:1 mapped to kernels in computational backend. In this case, it's very easy to analyze which operations is supposed (`_backend` has a kernel for such operation or extensions for the operation is provided) and not supported (kernel is missed in `_backend`): +operations via the InferenceEngine::QueryNetworkResult structure. The `QueryNetwork` firstly applies `TransformNetwork` passes to input `ov::Model` argument. After this, the transformed network in ideal case contains only operations are 1:1 mapped to kernels in computational backend. In this case, it's very easy to analyze which operations is supposed (`_backend` has a kernel for such operation or extensions for the operation is provided) and not supported (kernel is missed in `_backend`): -1. Store original names of all operations in input `ngraph::Function` +1. Store original names of all operations in input `ov::Model` 2. Apply `TransformNetwork` passes. Note, the names of operations in a transformed network can be different and we need to restore the mapping in the steps below. -3. Construct `supported` and `unsupported` maps which contains names of original operations. Note, that since the inference is performed using ngraph reference backend, the decision whether the operation is supported or not depends on whether the latest OpenVINO opset contains such operation. +3. Construct `supported` and `unsupported` maps which contains names of original operations. Note, that since the inference is performed using OpenVINO™ reference backend, the decision whether the operation is supported or not depends on whether the latest OpenVINO opset contains such operation. 4. `QueryNetworkResult.supportedLayersMap` contains only operations which are fully supported by `_backend`. @snippet template_plugin/src/template_plugin.cpp plugin:query_network diff --git a/docs/IE_PLUGIN_DG/PluginTesting.md b/docs/IE_PLUGIN_DG/PluginTesting.md index a5d75cd21b8..f985ad57771 100644 --- a/docs/IE_PLUGIN_DG/PluginTesting.md +++ b/docs/IE_PLUGIN_DG/PluginTesting.md @@ -26,7 +26,7 @@ Engine concepts: plugin creation, multiple executable networks support, multiple @snippet single_layer_tests/convolution.cpp test_convolution:instantiate 3. **Sub-graph tests** (`subgraph_tests` sub-folder). This group of tests is designed to tests small patterns or combination of layers. E.g. when a particular topology is being enabled in a plugin e.g. TF ResNet-50, there is no need to add the whole topology to test tests. In opposite way, a particular repetitive subgraph or pattern can be extracted from `ResNet-50` and added to the tests. The instantiation of the sub-graph tests is done in the same way as for single layer tests. -> **Note**, such sub-graphs or patterns for sub-graph tests should be added to `IE::ngraphFunctions` library first (this library is a pre-defined set of small `ngraph::Function`) and re-used in sub-graph tests after. +> **Note**, such sub-graphs or patterns for sub-graph tests should be added to `IE::ngraphFunctions` library first (this library is a pre-defined set of small `ov::Model`) and re-used in sub-graph tests after. 4. **HETERO tests** (`subgraph_tests` sub-folder) contains tests for `HETERO` scenario (manual or automatic affinities settings, tests for `QueryNetwork`). @@ -41,18 +41,14 @@ To use these tests for your own plugin development, link the `IE::funcSharedTest To build test binaries together with other build artifacts, use the `make all` command. For details, see [Build Plugin Using CMake*](@ref openvino_docs_ie_plugin_dg_plugin_build). -### Tests for plugin-specific ngraph transformations - -Please, refer to [Transformation testing](@ref ngraph_transformation) guide. - ### How to Extend Inference Engine Plugin Tests Inference Engine Plugin tests are open for contribution. Add common test case definitions applicable for all plugins to the `IE::funcSharedTests` target within the DLDT repository. Then, any other plugin supporting corresponding functionality can instantiate the new test. -All Inference Engine per-layer tests check test layers functionality. They are developed using nGraph functions +All Inference Engine per-layer tests check test layers functionality. They are developed using ov::Model. as input graphs used by tests. In this case, to test a new layer with layer tests, extend -the `IE::ngraphFunctions` library, which is also included in the Inference Engine Developer package, with a new nGraph function +the `IE::ngraphFunctions` library, which is also included in the Inference Engine Developer package, with a new model. including the corresponding operation. > **NOTE**: When implementing a new subgraph test, add new single-layer tests for each operation of the subgraph if such test does not exist. diff --git a/docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md b/docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md index 731eb073ea7..61b8a623d21 100644 --- a/docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md +++ b/docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md @@ -22,7 +22,7 @@ Model Optimizer is a cross-platform command-line tool that facilitates the transition between the training and deployment environment, performs static model analysis, and adjusts deep learning models for optimal execution on end-point target devices. -Model Optimizer process assumes you have a network model trained using supported deep learning frameworks: Caffe*, TensorFlow*, Kaldi*, MXNet* or converted to the ONNX* format. Model Optimizer produces an Intermediate Representation (IR) of the network, which can be inferred with the [OpenVINO™ Runtime](../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md). +Model Optimizer process assumes you have a network model trained using supported deep learning frameworks: Caffe*, TensorFlow*, Kaldi*, MXNet* or converted to the ONNX* format. Model Optimizer produces an Intermediate Representation (IR) of the network, which can be inferred with the [OpenVINO™ Runtime](../OV_Runtime_UG/openvino_intro.md). > **NOTE**: Model Optimizer does not infer models. Model Optimizer is an offline tool that runs before the inference takes place. @@ -637,7 +637,7 @@ To convert the model to the Intermediate Representation (IR), run Model Optimize mo --input_model INPUT_MODEL --output_dir ``` -You need to have have write permissions for an output directory. +You need to have write permissions for an output directory. > **NOTE**: Some models require using additional arguments to specify conversion parameters, such as `--input_shape`, `--scale`, `--scale_values`, `--mean_values`, `--mean_file`. To learn about when you need to use these parameters, refer to [Converting a Model to Intermediate Representation (IR)](prepare_model/convert_model/Converting_Model.md). diff --git a/docs/MO_DG/Known_Issues_Limitations.md b/docs/MO_DG/Known_Issues_Limitations.md index ec8897d06c6..610bafc579c 100644 --- a/docs/MO_DG/Known_Issues_Limitations.md +++ b/docs/MO_DG/Known_Issues_Limitations.md @@ -7,41 +7,3 @@ TensorFlow* provides only prebuilt binaries with AVX instructions enabled. When To run the Model Optimizer on this hardware, you should compile TensorFlow binaries from source as described at the [TensorFlow website](https://www.tensorflow.org/install/source). Another option is to run the Model Optimizer to generate an IR on hardware that supports AVX to and then perform inference on hardware without AVX. - - -## Multiple OpenMP Loadings - -If the application uses the Inference Engine with third-party components that depend on Intel OpenMP, multiple loadings of the libiomp library may occur and cause OpenMP runtime initialization conflicts. This may happen, for example, if the application uses Intel® Math Kernel Library (Intel® MKL) through the “Single Dynamic Library” (libmkl_rt.so) mechanism and calls Intel MKL after loading the Inference Engine plugin. -The error log looks as follows: -```sh -OMP: Error #15: Initializing libiomp5.so, but found libiomp5.so already initialized. -OMP: Hint: This means that multiple copies of the OpenMP runtime have been linked into the program. That is dangerous, since it can degrade performance or cause incorrect results. The best thing to do is to ensure that only a single OpenMP runtime is linked into the process, e.g. by avoiding static linking of the OpenMP runtime in any library. As an unsafe, unsupported, undocumented workaround you can set the environment variable KMP_DUPLICATE_LIB_OK=TRUE to allow the program to continue to execute, but that may cause crashes or silently produce incorrect results. For more information, please see http://www.intel.com/software/products/support/. -``` - -Possible workarounds: - -* Preload the OpenMP runtime using the LD_PRELOAD variable: - ```sh - LD_PRELOAD= ``` - This eliminates multiple loadings of libiomp, and makes all the components use this specific version of OpenMP. - -* Alternatively, you can set KMP_DUPLICATE_LIB_OK=TRUE. However, performance degradation or incorrect results may occur in this case. - - -## Old proto compiler breaks protobuf library - -With python protobuf library version 3.5.1 the following incompatibility can happen. -The known case is for Cent OS 7.4 - -The error log looks as follows: - -```sh -File "../lib64/python3.5/site-packages/google/protobuf/descriptor.py", line 829, in _new_ -return _message.default_pool.AddSerializedFile(serialized_pb) -TypeError: expected bytes, str found -``` - -Possible workaround is to upgrade default protobuf compiler (libprotoc 2.5.0) to newer version, for example -libprotoc 2.6.1. - -[protobuf_issue]: https://github.com/google/protobuf/issues/4272 diff --git a/docs/MO_DG/prepare_model/Additional_Optimizations.md b/docs/MO_DG/prepare_model/Additional_Optimizations.md index 4a927563b6c..726a2139bfd 100644 --- a/docs/MO_DG/prepare_model/Additional_Optimizations.md +++ b/docs/MO_DG/prepare_model/Additional_Optimizations.md @@ -3,13 +3,13 @@ Model Optimizer performs preprocessing to a model. It is possible to optimize this step and improve first inference time, to do that, follow the tips bellow: - **Image mean/scale parameters**
- Make sure to use the input image mean/scale parameters (`--scale` and `–mean_values`) with the Model Optimizer when you need pre-processing. It allows the tool to bake the pre-processing into the IR to get accelerated by the Inference Engine. + Make sure to use the input image mean/scale parameters (`--scale` and `–mean_values`) with the Model Optimizer when you need pre-processing. It allows the tool to bake the pre-processing into the IR to get accelerated by the OpenVINO Runtime. - **RGB vs. BGR inputs**
If, for example, your network assumes the RGB inputs, the Model Optimizer can swap the channels in the first convolution using the `--reverse_input_channels` command line option, so you do not need to convert your inputs to RGB every time you get the BGR image, for example, from OpenCV*. - **Larger batch size**
- Notice that the devices like GPU are doing better with larger batch size. While it is possible to set the batch size in the runtime using the Inference Engine [ShapeInference feature](../../OV_Runtime_UG/ShapeInference.md). + Notice that the devices like GPU are doing better with larger batch size. While it is possible to set the batch size in the runtime using the OpenVINO Runtime API [ShapeInference feature](../../OV_Runtime_UG/ShapeInference.md). - **Resulting IR precision**
The resulting IR precision, for instance, `FP16` or `FP32`, directly affects performance. As CPU now supports `FP16` (while internally upscaling to `FP32` anyway) and because this is the best precision for a GPU target, you may want to always convert models to `FP16`. Notice that this is the only precision that Intel® Movidius™ Myriad™ 2 and Intel® Myriad™ X VPUs support. diff --git a/docs/MO_DG/prepare_model/Default_Model_Optimizer_Optimizations.md b/docs/MO_DG/prepare_model/Default_Model_Optimizer_Optimizations.md index 6b4f1826384..b40da184e66 100644 --- a/docs/MO_DG/prepare_model/Default_Model_Optimizer_Optimizations.md +++ b/docs/MO_DG/prepare_model/Default_Model_Optimizer_Optimizations.md @@ -8,4 +8,4 @@ The picture above shows Caffe\* Resnet269\* topology. The left model is the orig If you still see these operations, inspect the Model Optimizer output carefully while searching for warnings, such as on the tool being unable to fuse. For example, non-linear operations (like activations) in between convolutions and linear operations might prevent the fusing. If performance is of concern, try to change (and potentially re-train) the topology. Refer to the [Model Optimizer Guide](Model_Optimization_Techniques.md) for more optimizations. -Notice that the activation (`_relu`) is not touched by the Model Optimizer, and while it can be merged into convolution as well, this is rather a device-specific optimization, covered by Inference Engine during the model loading time. You are encouraged to inspect performance counters from plugins that should indicate that these particular layers are not executed (“Optimized out”). For more information, refer to Internal Inference Performance Counters. +Notice that the activation (`_relu`) is not touched by the Model Optimizer, and while it can be merged into convolution as well, this is rather a device-specific optimization, covered by OpenVINO Runtime during the model loading time. You are encouraged to inspect performance counters from plugins that should indicate that these particular layers are not executed (“Optimized out”). For more information, refer to Internal Inference Performance Counters. diff --git a/docs/MO_DG/prepare_model/Getting_performance_numbers.md b/docs/MO_DG/prepare_model/Getting_performance_numbers.md index 1d11be89064..7e734432bd1 100644 --- a/docs/MO_DG/prepare_model/Getting_performance_numbers.md +++ b/docs/MO_DG/prepare_model/Getting_performance_numbers.md @@ -3,11 +3,11 @@ ## Tip 1. Measure the Proper Set of Operations -When evaluating performance of your model with the Inference Engine, you must measure the proper set of operations. To do so, consider the following tips: +When evaluating performance of your model with the OpenVINO Runtime, you must measure the proper set of operations. To do so, consider the following tips: - Avoid including one-time costs like model loading. -- Track separately the operations that happen outside the Inference Engine, like video decoding. +- Track separately the operations that happen outside the OpenVINO Runtime, like video decoding. > **NOTE**: Some image pre-processing can be baked into the IR and accelerated. For more information, refer to [Model Optimizer Knobs Related to Performance](Additional_Optimizations.md) @@ -18,7 +18,7 @@ You need to build your performance conclusions on reproducible data. Do the perf - If the warm-up run does not help or execution time still varies, you can try running a large number of iterations and then average or find a mean of the results. - For time values that range too much, use geomean. -Refer to the [Inference Engine Samples](../../OV_Runtime_UG/Samples_Overview.md) for code examples for the performance measurements. Almost every sample, except interactive demos, has a `-ni` option to specify the number of iterations. +Refer to the [OpenVINO Samples](../../OV_Runtime_UG/Samples_Overview.md) for code examples for the performance measurements. Almost every sample, except interactive demos, has a `-ni` option to specify the number of iterations. ## Getting performance numbers using OpenVINO tool @@ -45,26 +45,26 @@ Instead, it is possible to keep a separate infer request per camera or another s ## Comparing Performance with Native/Framework Code -When comparing the Inference Engine performance with the framework or another reference code, make sure that both versions are as similar as possible: +When comparing the OpenVINO Runtime performance with the framework or another reference code, make sure that both versions are as similar as possible: -- Wrap exactly the inference execution (refer to the [Inference Engine Samples](../../OV_Runtime_UG/Samples_Overview.md) for examples). +- Wrap exactly the inference execution (refer to the [OpenVINO Samples](../../OV_Runtime_UG/Samples_Overview.md) for examples). - Do not include model loading time. -- Ensure the inputs are identical for the Inference Engine and the framework. For example, Caffe\* allows to auto-populate the input with random values. Notice that it might give different performance than on real images. -- Similarly, for correct performance comparison, make sure the access pattern, for example, input layouts, is optimal for Inference Engine (currently, it is NCHW). +- Ensure the inputs are identical for the OpenVINO Runtime and the framework. For example, Caffe\* allows to auto-populate the input with random values. Notice that it might give different performance than on real images. +- Similarly, for correct performance comparison, make sure the access pattern, for example, input layouts, is optimal for OpenVINO Runtime (currently, it is NCHW). - Any user-side pre-processing should be tracked separately. -- Make sure to try the same environment settings that the framework developers recommend, for example, for TensorFlow*. In many cases, things that are more machine friendly, like respecting NUMA (see CPU Checklist), might work well for the Inference Engine as well. -- If applicable, use batching with the Inference Engine. -- If possible, demand the same accuracy. For example, TensorFlow allows `FP16` support, so when comparing to that, make sure to test the Inference Engine with the `FP16` as well. +- Make sure to try the same environment settings that the framework developers recommend, for example, for TensorFlow*. In many cases, things that are more machine friendly, like respecting NUMA (see CPU Checklist), might work well for the OpenVINO Runtime as well. +- If applicable, use batching. +- If possible, demand the same accuracy. For example, TensorFlow allows `FP16` support, so when comparing to that, make sure to test the OpenVINO Runtime with the `FP16` as well. ## Using Tools -Whether you are tuning for the first time or doing advanced performance optimization, you need a a tool that provides accurate insights. Intel® VTune™ Amplifier gives you the tool to mine it and interpret the profiling data. +Whether you are tuning for the first time or doing advanced performance optimization, you need a tool that provides accurate insights. Intel® VTune™ Amplifier gives you the tool to mine it and interpret the profiling data. Alternatively, you can gather the raw profiling data that samples report, the second chapter provides example of how to interpret these. ### Internal Inference Performance Counters -Almost every sample (inspect command-line options for a specific sample with `-h`) supports a `-pc` command that outputs internal execution breakdown. Refer to the [samples code](../../OV_Runtime_UG/Samples_Overview.md) for the actual Inference Engine API behind that. +Almost every sample (inspect command-line options for a specific sample with `-h`) supports a `-pc` command that outputs internal execution breakdown. Refer to the [OpenVINO Samples](../../OV_Runtime_UG/Samples_Overview.md) for the actual OpenVINO Runtime API behind that. Below is example of CPU plugin output for a network (since the device is CPU, the layers wall clock `realTime` and the `cpu` time are the same): diff --git a/docs/MO_DG/prepare_model/Model_Optimizer_FAQ.md b/docs/MO_DG/prepare_model/Model_Optimizer_FAQ.md index 91106d0ee86..58c786e8424 100644 --- a/docs/MO_DG/prepare_model/Model_Optimizer_FAQ.md +++ b/docs/MO_DG/prepare_model/Model_Optimizer_FAQ.md @@ -158,7 +158,7 @@ However, if your model contains more than one input, the Model Optimizer is able #### 9. What does the message "Mean file for topologies with multiple inputs is not supported" mean? -Model Optimizer does not support mean file processing for topologies with more than one input. In this case, you need to perform preprocessing of the inputs for a generated Intermediate Representation in the Inference Engine to perform subtraction for every input of your multi-input model. +Model Optimizer does not support mean file processing for topologies with more than one input. In this case, you need to perform preprocessing of the inputs for a generated Intermediate Representation in the OpenVINO Runtime to perform subtraction for every input of your multi-input model, see [Overview of Preprocessing](../../OV_Runtime_UG/preprocessing_overview.md) for details. #### 10. What does the message "Cannot load or process mean file: value error" mean? @@ -214,7 +214,7 @@ One of the layers in the specified topology might not have inputs or values. Ple #### 24. What does the message "Part of the nodes was not translated to IE. Stopped" mean? -Some of the layers are not supported by the Inference Engine and cannot be translated to an Intermediate Representation. You can extend the Model Optimizer by allowing generation of new types of layers and implement these layers in the dedicated Inference Engine plugins. For more information, refer to the [Custom Layers Guide](../../HOWTO/Custom_Layers_Guide.md) and [Inference Engine Extensibility Mechanism](../../OV_Runtime_UG/Extensibility_DG/Intro.md) +Some of the operations are not supported by the OpenVINO Runtime and cannot be translated to an Intermediate Representation. You can extend the Model Optimizer by allowing generation of new types of operations and implement these operations in the dedicated OpenVINO plugins. For more information, refer to the [OpenVINO™ Extensibility Mechanism](../../Extensibility_UG/Intro.md) #### 25. What does the message "While creating an edge from .. to .. : node name is undefined in the graph. Check correctness of the input model" mean? @@ -268,7 +268,7 @@ Model Optimizer tried to write an event file in the specified directory but fail #### 37. What does the message "There is no registered 'infer' function for node with op = .. . Please implement this function in the extensions" mean? -Most likely, you tried to extend Model Optimizer with a new primitive, but did not specify an infer function. For more information on extensions, see [Custom Layers Guide](../../HOWTO/Custom_Layers_Guide.md). +Most likely, you tried to extend Model Optimizer with a new primitive, but did not specify an infer function. For more information on extensions, see [OpenVINO™ Extensibility Mechanism](../../Extensibility_UG/Intro.md). #### 38. What does the message "Stopped shape/value propagation at node" mean? @@ -300,7 +300,7 @@ Most likely, there is a problem with the specified file for model. The file exis #### 45. What does the message "Found custom layer. Model Optimizer does not support this layer. Please, register it in CustomLayersMapping.xml or implement extension" mean? -This means that the layer `{layer_name}` is not supported in the Model Optimizer. You can find a list of all unsupported layers in the corresponding section. You should implement the extensions for this layer ([Custom Layers Guide](../../HOWTO/Custom_Layers_Guide.md)). +This means that the layer `{layer_name}` is not supported in the Model Optimizer. You can find a list of all unsupported layers in the corresponding section. You should implement the extensions for this layer ([OpenVINO™ Extensibility Mechanism](../../Extensibility_UG/Intro.md)). #### 46. What does the message "Custom replacement configuration file does not exist" mean? @@ -308,7 +308,7 @@ Path to the custom replacement configuration file was provided with the `--trans #### 47. What does the message "Extractors collection have case insensitive duplicates" mean? -When extending Model Optimizer with new primitives keep in mind that their names are case insensitive. Most likely, another operation with the same name is already defined. For more information, see [Custom Layers Guide](../../HOWTO/Custom_Layers_Guide.md). +When extending Model Optimizer with new primitives keep in mind that their names are case insensitive. Most likely, another operation with the same name is already defined. For more information, see [OpenVINO™ Extensibility Mechanism](../../Extensibility_UG/Intro.md). #### 48. What does the message "Input model name is not in an expected format, cannot extract iteration number" mean? @@ -340,7 +340,7 @@ Please, make sure that inputs are defined and have correct shapes. You can use ` #### 55. What does the message "Attempt to register of custom name for the second time as class. Note that custom names are case-insensitive" mean? -When extending Model Optimizer with new primitives keep in mind that their names are case insensitive. Most likely, another operation with the same name is already defined. For more information, see [Custom Layers Guide](../../HOWTO/Custom_Layers_Guide.md). +When extending Model Optimizer with new primitives keep in mind that their names are case insensitive. Most likely, another operation with the same name is already defined. For more information, see [OpenVINO™ Extensibility Mechanism](../../Extensibility_UG/Intro.md). #### 56. What does the message "Both --input_shape and --batch were provided. Please, provide only one of them" mean? @@ -492,7 +492,7 @@ For more information, refer to [Converting a MXNet* Model](convert_model/Convert Model Optimizer tried to load the model that contains some unsupported operations. If you want to convert model that contains unsupported operations you need to prepare extension for all such operations. -For more information, refer to [Custom Layers Guide](../../HOWTO/Custom_Layers_Guide.md). +For more information, refer to [OpenVINO™ Extensibility Mechanism](../../Extensibility_UG/Intro.md). #### 87. What does the message "Can not register Op ... Please, call function 'register_caffe_python_extractor' with parameter 'name'" mean? @@ -538,7 +538,7 @@ Note that the first call register_caffe_python_extractor(ProposalPythonExa The second call prevents Model Optimizer from using this extension as if it is an extension for a layer with type `Proposal`. Otherwise, this layer can be chosen as an implementation of extension that can lead to potential issues. -For more information, refer to the [Custom Layers Guide](../../HOWTO/Custom_Layers_Guide.md). +For more information, refer to the [OpenVINO™ Extensibility Mechanism](../../Extensibility_UG/Intro.md). #### 88. What does the message "Model Optimizer is unable to calculate output shape of Memory node .." mean? @@ -572,8 +572,8 @@ file is not available or does not exist. Also refer to FAQ [#90](#question-90). This message means that if you have model with custom layers and its json file has been generated with MXNet version lower than 1.0.0, Model Optimizer does not support such topologies. If you want to convert it you have to rebuild MXNet with unsupported layers or generate new json with MXNet version 1.0.0 and higher. Also you need to implement -Inference Engine extension for used custom layers. -For more information, refer to the [Custom Layers Guide](../../HOWTO/Custom_Layers_Guide.md). +OpenVINO extension for used custom layers. +For more information, refer to the [OpenVINO™ Extensibility Mechanism](../../Extensibility_UG/Intro.md). #### 97. What does the message "Graph contains a cycle. Can not proceed .." mean? @@ -586,7 +586,7 @@ For Tensorflow: For all frameworks: 1. [Replace cycle containing Sub-graph in Model Optimizer](customize_model_optimizer/Subgraph_Replacement_Model_Optimizer.md) -2. [Custom Layers Guide](../../HOWTO/Custom_Layers_Guide.md) +2. [OpenVINO™ Extensibility Mechanism](../../Extensibility_UG/Intro.md) or * Edit network in original framework to exclude cycle. diff --git a/docs/MO_DG/prepare_model/Supported_Frameworks_Layers.md b/docs/MO_DG/prepare_model/Supported_Frameworks_Layers.md index b43910dd12c..cf336bf6684 100644 --- a/docs/MO_DG/prepare_model/Supported_Frameworks_Layers.md +++ b/docs/MO_DG/prepare_model/Supported_Frameworks_Layers.md @@ -1,9 +1,9 @@ # Supported Framework Layers {#openvino_docs_MO_DG_prepare_model_Supported_Frameworks_Layers} -## Caffe\* Supported Layers +## Caffe Supported Layers -| Layer Name in Caffe\* | Limitations | +| Layer Name in Caffe | Limitations | |:---------- | :----------| | Axpy | | | BN | | @@ -47,10 +47,10 @@ | Tile | | -## MXNet\* Supported Symbols +## MXNet Supported Symbols -| Symbol Name in MXNet\*| Limitations| +| Symbol Name in MXNet| Limitations| | :----------| :----------| | _Plus | | | _contrib_arange_like | | @@ -119,7 +119,7 @@ | Concat | | | Convolution | | | Crop | "center_crop" = 1 is not supported | -| Custom | [Custom Layers in the Model Optimizer](customize_model_optimizer/Customize_Model_Optimizer.md) | +| Custom | [Custom Layers in Model Optimizer](customize_model_optimizer/Customize_Model_Optimizer.md) | | Deconvolution | | | DeformableConvolution | | | DeformablePSROIPooling | | @@ -149,12 +149,12 @@ | zeros_like | | -## TensorFlow\* Supported Operations +## TensorFlow Supported Operations -Some TensorFlow\* operations do not match to any Inference Engine layer, but are still supported by the Model Optimizer and can be used on constant propagation path. These layers are labeled 'Constant propagation' in the table. +Some TensorFlow operations do not match to any OpenVINO operation, but are still supported by the Model Optimizer and can be used on constant propagation path. These layers are labeled 'Constant propagation' in the table. -| Operation Name in TensorFlow\* | Limitations| +| Operation Name in TensorFlow | Limitations| | :----------| :----------| | Abs | | | Acosh | | @@ -348,10 +348,10 @@ Some TensorFlow\* operations do not match to any Inference Engine layer, but are | ZerosLike | | -## TensorFlow 2 Keras\* Supported Operations +## TensorFlow 2 Keras Supported Operations -| Operation Name in TensorFlow 2 Keras\* | Limitations| +| Operation Name in TensorFlow 2 Keras | Limitations| | :----------| :----------| | ActivityRegularization | | | Add | | @@ -431,10 +431,10 @@ Some TensorFlow\* operations do not match to any Inference Engine layer, but are | ZeroPadding2D | | | ZeroPadding3D | | -## Kaldi\* Supported Layers +## Kaldi Supported Layers -| Symbol Name in Kaldi\*| Limitations| +| Symbol Name in Kaldi| Limitations| | :----------| :----------| | addshift | | | affinecomponent | | @@ -478,10 +478,10 @@ Some TensorFlow\* operations do not match to any Inference Engine layer, but are | timeheightconvolutioncomponent | | -## ONNX\* Supported Operators +## ONNX Supported Operators -| Symbol Name in ONNX\*| Limitations| +| Symbol Name in ONNX| Limitations| | :----------| :----------| | Abs | | | Acos | | @@ -621,11 +621,11 @@ Some TensorFlow\* operations do not match to any Inference Engine layer, but are | Xor | | -## PaddlePaddle\* Supported Operators +## PaddlePaddle Supported Operators paddlepaddle>=2.1 -| Operator Name in PaddlePaddle\*| Limitations| +| Operator Name in PaddlePaddle| Limitations| | :----------| :----------| | adpative_pool2d | 'NHWC' data_layout is not supported | | arg_max | 'int32' output data_type is not supported | diff --git a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Caffe.md b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Caffe.md index d8f2f906c40..91d808f4878 100644 --- a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Caffe.md +++ b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Caffe.md @@ -10,8 +10,8 @@ A summary of the steps for optimizing and deploying a model that was trained wit 1. [Configure the Model Optimizer](../../Deep_Learning_Model_Optimizer_DevGuide.md) for Caffe\*. 2. [Convert a Caffe\* Model](#Convert_From_Caffe) to produce an optimized [Intermediate Representation (IR)](../../IR_and_opsets.md) of the model based on the trained network topology, weights, and biases values -3. Test the model in the Intermediate Representation format using the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in the target environment via provided Inference Engine [sample applications](../../../OV_Runtime_UG/Samples_Overview.md) -4. [Integrate](../../../OV_Runtime_UG/Samples_Overview.md) the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in your application to deploy the model in the target environment +3. Test the model in the Intermediate Representation format using the [OpenVINO™ Runtime](../../../OV_Runtime_UG/openvino_intro.md) in the target environment via provided [OpenVINO samples](../../../OV_Runtime_UG/Samples_Overview.md) +4. [Integrate](../../../OV_Runtime_UG/Samples_Overview.md) the [OpenVINO™ Runtime](../../../OV_Runtime_UG/openvino_intro.md) in your application to deploy the model in the target environment ## Supported Topologies @@ -50,7 +50,7 @@ To convert a Caffe\* model, run Model Optimizer with the path to the input model Two groups of parameters are available to convert your model: -* Framework-agnostic parameters are used to convert a model trained with any supported framework. For details, see see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page. +* Framework-agnostic parameters are used to convert a model trained with any supported framework. For details, see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page. * [Caffe-specific parameters](#caffe_specific_conversion_params) are used to convert only Caffe\* models. ### Using Caffe\*-Specific Conversion Parameters diff --git a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Kaldi.md b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Kaldi.md index 511823f047e..79276825513 100644 --- a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Kaldi.md +++ b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Kaldi.md @@ -16,8 +16,8 @@ A summary of the steps for optimizing and deploying a model that was trained wit 1. [Configure the Model Optimizer](../../Deep_Learning_Model_Optimizer_DevGuide.md) for Kaldi\*. 2. [Convert a Kaldi\* Model](#Convert_From_Kaldi) to produce an optimized [Intermediate Representation (IR)](../../IR_and_opsets.md) of the model based on the trained network topology, weights, and biases values. -3. Test the model in the Intermediate Representation format using the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in the target environment via provided Inference Engine [sample applications](../../../OV_Runtime_UG/Samples_Overview.md). -4. [Integrate](../../../OV_Runtime_UG/Samples_Overview.md) the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in your application to deploy the model in the target environment. +3. Test the model in the Intermediate Representation format using the [OpenVINO™ Runtime](../../../OV_Runtime_UG/openvino_intro.md) in the target environment via provided [OpenVINO Samples](../../../OV_Runtime_UG/Samples_Overview.md). +4. [Integrate](../../../OV_Runtime_UG/Samples_Overview.md) the [OpenVINO™ Runtime](../../../OV_Runtime_UG/openvino_intro.md) in your application to deploy the model in the target environment. > **NOTE**: The Model Optimizer supports the [nnet1](http://kaldi-asr.org/doc/dnn1.html) and [nnet2](http://kaldi-asr.org/doc/dnn2.html) formats of Kaldi models. Support of the [nnet3](http://kaldi-asr.org/doc/dnn3.html) format is limited. @@ -50,7 +50,7 @@ mo --input_model .nnet --output_dir Two groups of parameters are available to convert your model: -* Framework-agnostic parameters are used to convert a model trained with any supported framework. For details, see see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page. +* Framework-agnostic parameters are used to convert a model trained with any supported framework. For details, see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page. * [Kaldi-specific parameters](#kaldi_specific_conversion_params) are used to convert only Kaldi\* models. ### Using Kaldi\*-Specific Conversion Parameters @@ -100,7 +100,7 @@ The Model Optimizer finds the last layer of the topology and removes this layer > **NOTE**: Model Optimizer can remove SoftMax layer only if the topology has one output. - > **NOTE**: For sample inference of Kaldi models, you can use the Inference Engine Speech Recognition sample application. The sample supports models with one output. If your model has several outputs, specify the desired one with the `--output` option. + > **NOTE**: For sample inference of Kaldi models, you can use the OpenVINO Speech Recognition sample application. The sample supports models with one output. If your model has several outputs, specify the desired one with the `--output` option. If you want to convert a model for inference on Intel® Movidius™ Myriad™, use the `--remove_memory` option. It removes Memory layers from the IR. Instead of it, additional inputs and outputs appear in the IR. diff --git a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_MxNet.md b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_MxNet.md index 8841fccbba1..575218d5e4b 100644 --- a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_MxNet.md +++ b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_MxNet.md @@ -17,8 +17,8 @@ A summary of the steps for optimizing and deploying a model that was trained wit 1. [Configure the Model Optimizer](../../Deep_Learning_Model_Optimizer_DevGuide.md) for MXNet* (MXNet was used to train your model) 2. [Convert a MXNet model](#ConvertMxNet) to produce an optimized [Intermediate Representation (IR)](../../IR_and_opsets.md) of the model based on the trained network topology, weights, and biases values -3. Test the model in the Intermediate Representation format using the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in the target environment via provided Inference Engine [sample applications](../../../OV_Runtime_UG/Samples_Overview.md) -4. [Integrate](../../../OV_Runtime_UG/Samples_Overview.md) the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in your application to deploy the model in the target environment +3. Test the model in the Intermediate Representation format using the [OpenVINO™ Runtime](../../../OV_Runtime_UG/openvino_intro.md) in the target environment via provided [OpenVINO Samples](../../../OV_Runtime_UG/Samples_Overview.md) +4. [Integrate](../../../OV_Runtime_UG/Samples_Overview.md) the [OpenVINO™ Runtime](../../../OV_Runtime_UG/openvino_intro.md) in your application to deploy the model in the target environment ## Supported Topologies @@ -64,7 +64,7 @@ To convert an MXNet\* model, run Model Optimizer with a path to the input model Two groups of parameters are available to convert your model: -* Framework-agnostic parameters are used to convert a model trained with any supported framework. For details, see see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page. +* Framework-agnostic parameters are used to convert a model trained with any supported framework. For details, see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page. * [MXNet-specific parameters](#mxnet_specific_conversion_params) are used to convert only MXNet models. diff --git a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_ONNX.md b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_ONNX.md index 7734c5902d7..7c4bfbf3f29 100644 --- a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_ONNX.md +++ b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_ONNX.md @@ -77,7 +77,7 @@ To convert an ONNX\* model, run Model Optimizer with the path to the input model ```sh mo --input_model .onnx --output_dir ``` -There are no ONNX\* specific parameters, so only framework-agnostic parameters are available to convert your model. For details, see see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page. +There are no ONNX\* specific parameters, so only framework-agnostic parameters are available to convert your model. For details, see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page. ## Supported ONNX\* Layers Refer to [Supported Framework Layers](../Supported_Frameworks_Layers.md) for the list of supported standard layers. diff --git a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Paddle.md b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Paddle.md index 19c487b21ec..5e9100e4723 100644 --- a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Paddle.md +++ b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_Paddle.md @@ -1,11 +1,11 @@ -# Converting a Paddle* Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Paddle} +# Converting a PaddlePaddle Model {#openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_Paddle} -A summary of the steps for optimizing and deploying a model trained with Paddle\*: +A summary of the steps for optimizing and deploying a model trained with PaddlePaddle: -1. [Configure the Model Optimizer](../../Deep_Learning_Model_Optimizer_DevGuide.md) for Paddle\*. -2. [Convert a Paddle\* Model](#Convert_From_Paddle) to produce an optimized [Intermediate Representation (IR)](../../IR_and_opsets.md) of the model based on the trained network topology, weights, and biases. -3. Test the model in the Intermediate Representation format using the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in the target environment via provided Inference Engine [sample applications](../../../OV_Runtime_UG/Samples_Overview.md). -4. [Integrate](../../../OV_Runtime_UG/Samples_Overview.md) the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in your application to deploy the model in the target environment. +1. [Configure Model Optimizer](../../Deep_Learning_Model_Optimizer_DevGuide.md) for PaddlePaddle. +2. [Convert a PaddlePaddle Model](#Convert_From_Paddle) to produce an optimized [Intermediate Representation (IR)](../../IR_and_opsets.md) of the model based on the trained network topology, weights, and biases. +3. Test the model in the Intermediate Representation format using the [OpenVINO™ Runtime](../../../OV_Runtime_UG/openvino_intro.md) in the target environment via provided [OpenVINO Samples](../../../OV_Runtime_UG/Samples_Overview.md). +4. [Integrate](../../../OV_Runtime_UG/Samples_Overview.md) the [OpenVINO™ Runtime](../../../OV_Runtime_UG/openvino_intro.md) in your application to deploy the model in the target environment. ## Supported Topologies @@ -29,11 +29,11 @@ A summary of the steps for optimizing and deploying a model trained with Paddle\ > **NOTE:** The verified models are exported from the repository of branch release/2.1. -## Convert a Paddle* Model +## Convert a PaddlePaddle Model -To convert a Paddle\* model: +To convert a PaddlePaddle model: -1. Activate environment with installed OpenVINO if needed +1. Activate environment with installed OpenVINO™ if needed 2. Use the `mo` script to simply convert a model, specifying the framework, the path to the input model `.pdmodel` file and the path to an output directory with write permissions: ```sh mo --input_model .pdmodel --output_dir --framework=paddle @@ -44,13 +44,13 @@ Parameters to convert your model: * [Framework-agnostic parameters](Converting_Model.md): These parameters are used to convert a model trained with any supported framework. > **NOTE:** `--scale`, `--scale_values`, `--mean_values` are not supported in the current version of mo_paddle. -### Example of Converting a Paddle* Model -Below is the example command to convert yolo v3 Paddle\* network to OpenVINO IR network with Model Optimizer. +### Example of Converting a PaddlePaddle Model +Below is the example command to convert yolo v3 PaddlePaddle network to OpenVINO IR network with Model Optimizer. ```sh mo --model_name yolov3_darknet53_270e_coco --output_dir --framework=paddle --data_type=FP32 --reverse_input_channels --input_shape=[1,3,608,608],[1,2],[1,2] --input=image,im_shape,scale_factor --output=save_infer_model/scale_0.tmp_1,save_infer_model/scale_1.tmp_1 --input_model=yolov3.pdmodel ``` -## Supported Paddle\* Layers +## Supported PaddlePaddle Layers Refer to [Supported Framework Layers](../Supported_Frameworks_Layers.md) for the list of supported standard layers. ## Frequently Asked Questions (FAQ) diff --git a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_PyTorch.md b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_PyTorch.md index 7ffe0fa11ad..5915feb05c2 100644 --- a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_PyTorch.md +++ b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_PyTorch.md @@ -40,6 +40,8 @@ Here is the list of models that are tested and guaranteed to be supported. Howev instruction which is used instead of steps 2 and 3 of [regular instructions](#typical-pytorch). * [BERT_NER](https://github.com/kamalkraj/BERT-NER) topology can be converted using steps described in [Convert PyTorch* BERT-NER to the IR](pytorch_specific/Convert_Bert_ner.md) instruction which is used instead of steps 2 and 3 of [regular instructions](#typical-pytorch). +* ResNeXt-101 from [facebookresearch/semi-supervised-ImageNet1K-models](https://github.com/facebookresearch/semi-supervised-ImageNet1K-models) + can be converted using [regular instructions](#typical-pytorch). ## Typical steps to convert PyTorch\* model @@ -48,8 +50,8 @@ PyTorch* framework is supported through export to ONNX\* format. A summary of th 1. [Configure the Model Optimizer](../../Deep_Learning_Model_Optimizer_DevGuide.md) for ONNX\*. 2. [Export PyTorch model to ONNX\*](#export-to-onnx). 3. [Convert an ONNX\* model](Convert_Model_From_ONNX.md) to produce an optimized [Intermediate Representation (IR)](../../IR_and_opsets.md) of the model based on the trained network topology, weights, and biases values. -4. Test the model in the Intermediate Representation format using the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in the target environment via provided [sample applications](../../../OV_Runtime_UG/Samples_Overview.md). -5. [Integrate](../../../OV_Runtime_UG/Samples_Overview.md) the Inference Engine in your application to deploy the model in the target environment. +4. Test the model in the Intermediate Representation format using the [OpenVINO™ Runtime](../../../OV_Runtime_UG/openvino_intro.md) in the target environment via provided [sample applications](../../../OV_Runtime_UG/Samples_Overview.md). +5. [Integrate OpenVINO Runtime](../../../OV_Runtime_UG/Samples_Overview.md) in your application to deploy the model in the target environment. ## Export PyTorch\* Model to ONNX\* Format diff --git a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md index 1ba6aa2d4d4..c3fa4528816 100644 --- a/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md +++ b/docs/MO_DG/prepare_model/convert_model/Convert_Model_From_TensorFlow.md @@ -31,14 +31,14 @@ A summary of the steps for optimizing and deploying a model that was trained wit 1. [Configure the Model Optimizer](../../Deep_Learning_Model_Optimizer_DevGuide.md) for TensorFlow\* (TensorFlow was used to train your model). 2. [Freeze the TensorFlow model](#freeze-the-tensorflow-model) if your model is not already frozen or skip this step and use the [instruction](#loading-nonfrozen-models) to a convert a non-frozen model. 3. [Convert a TensorFlow\* model](#Convert_From_TF) to produce an optimized [Intermediate Representation (IR)](../../IR_and_opsets.md) of the model based on the trained network topology, weights, and biases values. -4. Test the model in the Intermediate Representation format using the [OpenVINO™ Runtime](../../../OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) in the target environment via provided [sample applications](../../../OV_Runtime_UG/Samples_Overview.md). -5. [Integrate](../../../OV_Runtime_UG/Samples_Overview.md) the Inference Engine in your application to deploy the model in the target environment. +4. Test the model in the Intermediate Representation format using the [OpenVINO™ Runtime](../../../OV_Runtime_UG/openvino_intro.md) in the target environment via provided [sample applications](../../../OV_Runtime_UG/Samples_Overview.md). +5. [Integrate OpenVINO Runtime](../../../OV_Runtime_UG/Samples_Overview.md) in your application to deploy the model in the target environment. ## Supported Topologies **Supported Non-Frozen Topologies with Links to the Associated Slim Model Classification Download Files** -Detailed information on how to convert models from the TensorFlow\*-Slim Image Classification Model Library is available in the [Converting TensorFlow*-Slim Image Classification Model Library Models](tf_specific/Convert_Slim_Library_Models.md) chapter. The table below contains list of supported TensorFlow\*-Slim Image Classification Model Library models and required mean/scale values. The mean values are specified as if the input image is read in BGR channels order layout like Inference Engine classification sample does. +Detailed information on how to convert models from the TensorFlow\*-Slim Image Classification Model Library is available in the [Converting TensorFlow*-Slim Image Classification Model Library Models](tf_specific/Convert_Slim_Library_Models.md) chapter. The table below contains list of supported TensorFlow\*-Slim Image Classification Model Library models and required mean/scale values. The mean values are specified as if the input image is read in BGR channels order layout like OpenVINO classification sample does. | Model Name| Slim Model Checkpoint File| \-\-mean_values | \-\-scale| | ------------- | ------------ | ------------- | -----:| @@ -285,7 +285,7 @@ To convert a TensorFlow model: Two groups of parameters are available to convert your model: -* Framework-agnostic parameters are used to convert a model trained with any supported framework. For details, see see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page. +* Framework-agnostic parameters are used to convert a model trained with any supported framework. For details, see the General Conversion Parameters section on the [Converting a Model to Intermediate Representation (IR)](Converting_Model.md) page. * [TensorFlow-specific parameters](#tensorflow_specific_conversion_params): Parameters used to convert only TensorFlow models. > **NOTE**: The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the `RGB<->BGR` conversion specifying the command-line parameter: `--reverse_input_channels`. Otherwise, inference results may be incorrect. For more information about the parameter, refer to **When to Reverse Input Channels** section of [Converting a Model to Intermediate Representation (IR)](Converting_Model.md). @@ -354,7 +354,7 @@ TensorFlow*-specific parameters: mo --input_model inception_v1.pb -b 1 --tensorboard_logdir /tmp/log_dir --output_dir ``` -* Launching the Model Optimizer for a model with custom TensorFlow operations (refer to the [TensorFlow* documentation](https://www.tensorflow.org/extend/adding_an_op)) implemented in C++ and compiled into the shared library `my_custom_op.so`. Model Optimizer falls back to TensorFlow to infer output shape of operations implemented in the library if a custom TensorFlow operation library is provided. If it is not provided, a custom operation with an inference function is needed. For more information about custom operations, refer to the [Custom Layers Guide](../../../HOWTO/Custom_Layers_Guide.md). +* Launching the Model Optimizer for a model with custom TensorFlow operations (refer to the [TensorFlow* documentation](https://www.tensorflow.org/extend/adding_an_op)) implemented in C++ and compiled into the shared library `my_custom_op.so`. Model Optimizer falls back to TensorFlow to infer output shape of operations implemented in the library if a custom TensorFlow operation library is provided. If it is not provided, a custom operation with an inference function is needed. For more information about custom operations, refer to the [OpenVINO™ Extensibility Mechanism](../../../Extensibility_UG/Intro.md). ```sh mo --input_model custom_model.pb --tensorflow_custom_layer_libraries ./my_custom_op.so --output_dir ``` diff --git a/docs/MO_DG/prepare_model/convert_model/Converting_Model.md b/docs/MO_DG/prepare_model/convert_model/Converting_Model.md index b0fdd565f19..60d4ad1f934 100644 --- a/docs/MO_DG/prepare_model/convert_model/Converting_Model.md +++ b/docs/MO_DG/prepare_model/convert_model/Converting_Model.md @@ -24,7 +24,7 @@ To convert the model to the Intermediate Representation (IR), run Model Optimizer using the following command: ```sh -mo --input_model INPUT_MODEL --output_dir +mo --input_model INPUT_MODEL ``` The output directory must have write permissions, so you can run Model Optimizer from the output directory or specify an output path with the `--output_dir` option. @@ -37,6 +37,7 @@ Framework-specific parameters for: * [TensorFlow](Convert_Model_From_TensorFlow.md) * [MXNet](Convert_Model_From_MxNet.md) * [ONNX](Convert_Model_From_ONNX.md) +* [PaddlePaddle](Convert_Model_From_Paddle.md) * [Kaldi](Convert_Model_From_Kaldi.md) @@ -70,12 +71,9 @@ Framework-agnostic parameters: square brackets, for example [1,3,227,227] or (1,227,227,3), where the order of dimensions depends on the framework input layout of the model. For - example, [N,C,H,W] is used for Caffe* models and - [N,H,W,C] for TensorFlow* models. Model Optimizer - performs necessary transformations to convert the - shape to the layout required by Inference Engine - (N,C,H,W). The shape should not contain undefined - dimensions (? or -1) and should fit the dimensions + example, [N,C,H,W] is used for ONNX* models and + [N,H,W,C] for TensorFlow* models. The shape can contain + undefined dimensions (? or -1) and should fit the dimensions defined in the input operation of the graph. Boundaries of undefined dimension can be specified with ellipsis, for example [1,1..10,128,128]. One boundary can be undefined, @@ -155,13 +153,12 @@ Framework-agnostic parameters: original model is in FP32 and --data_type=FP16 is specified, all model weights and biases are compressed to FP16. - --disable_fusing Turn off fusing of linear operations to Convolution + --disable_fusing [DEPRECATED] Turn off fusing of linear operations to Convolution. --disable_resnet_optimization - Turn off resnet optimization + [DEPRECATED] Turn off ResNet optimization. --finegrain_fusing FINEGRAIN_FUSING - Regex for layers/operations that won't be fused. + [DEPRECATED] Regex for layers/operations that won't be fused. Example: --finegrain_fusing Convolution1,.*Scale.* - --disable_gfusing Turn off fusing of grouped convolutions --enable_concat_optimization Turn on Concat optimization. --extensions EXTENSIONS @@ -184,9 +181,9 @@ Framework-agnostic parameters: --static_shape Enables IR generation for fixed input shape (folding `ShapeOf` operations and shape-calculating sub-graphs to `Constant`). Changing model input shape using - the Inference Engine API in runtime may fail for such an IR. + the OpenVINO Runtime API in runtime may fail for such an IR. --disable_weights_compression - Disable compression and store weights with original + [DEPRECATED] Disable compression and store weights with original precision. --progress Enable model conversion progress display. --stream_output Switch model conversion progress display to a @@ -194,8 +191,13 @@ Framework-agnostic parameters: --transformations_config TRANSFORMATIONS_CONFIG Use the configuration file with transformations description. - --use_new_frontend Force the usage of new frontend API for model processing. - --use_legacy_frontend Force the usage of legacy API for model processing. + --use_new_frontend Force the usage of new Frontend of Model Optimizer for model conversion into IR. + The new Frontend is C++ based and is available for ONNX* and PaddlePaddle* models. + Model optimizer uses new Frontend for ONNX* and PaddlePaddle* by default that means + `--use_new_frontend` and `--use_legacy_frontend` options are not specified. + --use_legacy_frontend Force the usage of legacy Frontend of Model Optimizer for model conversion into IR. + The legacy Frontend is Python based and is available for TensorFlow*, ONNX*, MXNet*, + Caffe*, and Kaldi* models. ``` The sections below provide details on using particular parameters and examples of CLI commands. @@ -205,7 +207,7 @@ Usually neural network models are trained with the normalized input data. This m * The input pre-processing operations are a part of a topology. In this case, the application that uses the framework to infer the topology does not pre-process the input. * The input pre-processing operations are not a part of a topology and the pre-processing is performed within the application which feeds the model with an input data. -In the first case, the Model Optimizer generates the IR with required pre-processing layers and Inference Engine samples may be used to infer the model. +In the first case, the Model Optimizer generates the IR with required pre-processing operations and OpenVINO Samples may be used to infer the model. In the second case, information about mean/scale values should be provided to the Model Optimizer to embed it to the generated IR. Model Optimizer provides a number of command line parameters to specify them: `--mean`, `--scale`, `--scale_values`, `--mean_values`. @@ -217,67 +219,64 @@ There is no a universal recipe for determining the mean/scale values for a parti * Open the model in a visualization tool and check for layers performing subtraction or multiplication (like `Sub`, `Mul`, `ScaleShift`, `Eltwise` etc) of the input data. If such layers exist, pre-processing is probably part of the model. ## When to Specify Input Shapes -There are situations when the input data shape for the model is not fixed, like for the fully-convolutional neural networks. In this case, for example, TensorFlow\* models contain `-1` values in the `shape` attribute of the `Placeholder` operation. Inference Engine does not support input layers with undefined size, so if the input shapes are not defined in the model, the Model Optimizer fails to convert the model. The solution is to provide the input shape(s) using the `--input` or `--input_shape` command line parameter for all input(s) of the model or provide the batch size using the `-b` command line parameter if the model contains just one input with undefined batch size only. In the latter case, the `Placeholder` shape for the TensorFlow\* model looks like this `[-1, 224, 224, 3]`. +There are situations when Model Optimizer is unable to deduce input shapes of the model, for example, in case of model cutting due to unsupported operations. +The solution is to provide input shapes of a static rank explicitly. ## When to Reverse Input Channels -Input data for your application can be of RGB or BRG color input order. For example, Inference Engine samples load input images in the BGR channels order. However, the model may be trained on images loaded with the opposite order (for example, most TensorFlow\* models are trained with images in RGB order). In this case, inference results using the Inference Engine samples may be incorrect. The solution is to provide `--reverse_input_channels` command line parameter. Taking this parameter, the Model Optimizer performs first convolution or other channel dependent operation weights modification so these operations output will be like the image is passed with RGB channels order. +Input data for your application can be of RGB or BRG color input order. For example, OpenVINO Samples load input images in the BGR channels order. However, the model may be trained on images loaded with the opposite order (for example, most TensorFlow\* models are trained with images in RGB order). In this case, inference results using the OpenVINO samples may be incorrect. The solution is to provide `--reverse_input_channels` command line parameter. Taking this parameter, the Model Optimizer performs first convolution or other channel dependent operation weights modification so these operations output will be like the image is passed with RGB channels order. ## When to Specify `--static_shape` Command Line Parameter If the `--static_shape` command line parameter is specified the Model Optimizer evaluates shapes of all operations in the model (shape propagation) for a fixed input(s) shape(s). During the shape propagation the Model Optimizer evaluates operations *Shape* and removes them from the computation graph. With that approach, the initial model which can consume inputs of different shapes may be converted to IR working with the input of one fixed shape only. For example, consider the case when some blob is reshaped from 4D of a shape *[N, C, H, W]* to a shape *[N, C, H \* W]*. During the model conversion the Model Optimize calculates output shape as a constant 1D blob with values *[N, C, H \* W]*. So if the input shape changes to some other value *[N,C,H1,W1]* (it is possible scenario for a fully convolutional model) then the reshape layer becomes invalid. -Resulting Intermediate Representation will not be resizable with the help of Inference Engine. +Resulting Intermediate Representation will not be resizable with the help of OpenVINO Runtime API. ## Examples of CLI Commands Launch the Model Optimizer for the Caffe bvlc_alexnet model with debug log level: ```sh -mo --input_model bvlc_alexnet.caffemodel --log_level DEBUG --output_dir +mo --input_model bvlc_alexnet.caffemodel --log_level DEBUG ``` Launch the Model Optimizer for the Caffe bvlc_alexnet model with the output IR called `result.*` in the specified `output_dir`: ```sh -mo --input_model bvlc_alexnet.caffemodel --model_name result --output_dir /../../models/ +mo --input_model bvlc_alexnet.caffemodel --model_name result --output_dir ``` Launch the Model Optimizer for the Caffe bvlc_alexnet model with one input with scale values: ```sh -mo --input_model bvlc_alexnet.caffemodel --scale_values [59,59,59] --output_dir +mo --input_model bvlc_alexnet.caffemodel --scale_values [59,59,59] ``` Launch the Model Optimizer for the Caffe bvlc_alexnet model with multiple inputs with scale values: ```sh -mo --input_model bvlc_alexnet.caffemodel --input data,rois --scale_values [59,59,59],[5,5,5] --output_dir +mo --input_model bvlc_alexnet.caffemodel --input data,rois --scale_values [59,59,59],[5,5,5] ``` Launch the Model Optimizer for the Caffe bvlc_alexnet model with multiple inputs with scale and mean values specified for the particular nodes: ```sh -mo --input_model bvlc_alexnet.caffemodel --input data,rois --mean_values data[59,59,59] --scale_values rois[5,5,5] --output_dir +mo --input_model bvlc_alexnet.caffemodel --input data,rois --mean_values data[59,59,59] --scale_values rois[5,5,5] ``` Launch the Model Optimizer for the Caffe bvlc_alexnet model with specified input layer, overridden input shape, scale 5, batch 8 and specified name of an output operation: ```sh -mo --input_model bvlc_alexnet.caffemodel --input "data[1 3 224 224]" --output pool5 -s 5 -b 8 --output_dir -``` -Launch the Model Optimizer for the Caffe bvlc_alexnet model with disabled fusing for linear operations to Convolution and grouped convolutions: -```sh -mo --input_model bvlc_alexnet.caffemodel --disable_fusing --disable_gfusing --output_dir +mo --input_model bvlc_alexnet.caffemodel --input data --output pool5 -s 5 -b 8 ``` Launch the Model Optimizer for the Caffe bvlc_alexnet model with reversed input channels order between RGB and BGR, specified mean values to be used for the input image per channel and specified data type for input tensor values: ```sh -mo --input_model bvlc_alexnet.caffemodel --reverse_input_channels --mean_values [255,255,255] --data_type FP16 --output_dir +mo --input_model bvlc_alexnet.caffemodel --reverse_input_channels --mean_values [255,255,255] --data_type FP16 ``` Launch the Model Optimizer for the Caffe bvlc_alexnet model with extensions listed in specified directories, specified mean_images binaryproto - file. For more information about extensions, please refer to the [Custom Layers Guide](../../../HOWTO/Custom_Layers_Guide.md). + file. For more information about extensions, please refer to the [OpenVINO™ Extensibility Mechanism](../../../Extensibility_UG/Intro.md). ```sh -mo --input_model bvlc_alexnet.caffemodel --extensions /home/,/some/other/path/ --mean_file /path/to/binaryproto --output_dir +mo --input_model bvlc_alexnet.caffemodel --extensions /home/,/some/other/path/ --mean_file /path/to/binaryproto ``` Launch the Model Optimizer for TensorFlow* FaceNet* model with a placeholder freezing value. It replaces the placeholder with a constant layer that contains the passed value. For more information about FaceNet conversion, please refer to [this](tf_specific/Convert_FaceNet_From_Tensorflow.md) page. ```sh -mo --input_model FaceNet.pb --input "phase_train->False" --output_dir +mo --input_model FaceNet.pb --input "phase_train->False" ``` Launch the Model Optimizer for any model with a placeholder freezing tensor of values. It replaces the placeholder with a constant layer that contains the passed values. @@ -286,7 +285,7 @@ Tensor here is represented in square brackets with each value separated from ano If data type is set in the model, this tensor will be reshaped to a placeholder shape and casted to placeholder data type. Otherwise, it will be casted to data type passed to `--data_type` parameter (by default, it is FP32). ```sh -mo --input_model FaceNet.pb --input "placeholder_layer_name->[0.1 1.2 2.3]" --output_dir +mo --input_model FaceNet.pb --input "placeholder_layer_name->[0.1 1.2 2.3]" ``` diff --git a/docs/MO_DG/prepare_model/convert_model/Cutting_Model.md b/docs/MO_DG/prepare_model/convert_model/Cutting_Model.md index aac7d73f006..8b22503439e 100644 --- a/docs/MO_DG/prepare_model/convert_model/Cutting_Model.md +++ b/docs/MO_DG/prepare_model/convert_model/Cutting_Model.md @@ -6,10 +6,10 @@ Sometimes some parts of a model must be removed while the Model Optimizer is con The following examples are the situations when model cutting is useful or even required: -* model has pre- or post-processing parts that cannot be translated to existing Inference Engine layers. +* model has pre- or post-processing parts that cannot be translated to existing OpenVINO operations. * model has a training part that is convenient to be kept in the model, but not used during inference. * model is too complex (contains lots of unsupported operations that cannot be easily implemented as custom layers), so the complete model cannot be converted in one shot. -* problem with model conversion in the Model Optimizer or inference in the Inference Engine occurred. To localize the issue, limit the scope for conversion by iteratively searching for problematic places in the model. +* problem with model conversion in the Model Optimizer or inference in the OpenVINO Runtime occurred. To localize the issue, limit the scope for conversion by iteratively searching for problematic places in the model. * single custom layer or a combination of custom layers is isolated for debugging purposes. ## Command-Line Options diff --git a/docs/MO_DG/prepare_model/convert_model/IR_suitable_for_INT8_inference.md b/docs/MO_DG/prepare_model/convert_model/IR_suitable_for_INT8_inference.md index 34df1408cdb..9f7ac357e30 100644 --- a/docs/MO_DG/prepare_model/convert_model/IR_suitable_for_INT8_inference.md +++ b/docs/MO_DG/prepare_model/convert_model/IR_suitable_for_INT8_inference.md @@ -2,7 +2,7 @@ ## Introduction -Inference Engine CPU and GPU plugin can infer models in the low precision. +OpenVINO Runtime CPU and GPU devices can infer models in the low precision. For details, refer to [Low Precision Inference on the CPU](../../../OV_Runtime_UG/Int8Inference.md). Intermediate Representation (IR) should be specifically formed to be suitable for low precision inference. diff --git a/docs/MO_DG/prepare_model/convert_model/mxnet_specific/Convert_Style_Transfer_From_MXNet.md b/docs/MO_DG/prepare_model/convert_model/mxnet_specific/Convert_Style_Transfer_From_MXNet.md index 86a4990ee58..a3dad226b7e 100644 --- a/docs/MO_DG/prepare_model/convert_model/mxnet_specific/Convert_Style_Transfer_From_MXNet.md +++ b/docs/MO_DG/prepare_model/convert_model/mxnet_specific/Convert_Style_Transfer_From_MXNet.md @@ -114,4 +114,4 @@ cp models/13_decoder_auxs.nd nst_model ```sh mo --input_symbol /nst_vgg19-symbol.json --framework mxnet --output_dir --input_shape [1,3,224,224] --nd_prefix_name 13_decoder --pretrained_model /vgg19-0000.params ``` -4. The IR is generated (`.bin`, `.xml` and `.mapping` files) in the specified output directory and ready to be consumed by the Inference Engine. +4. The IR is generated (`.bin`, `.xml` and `.mapping` files) in the specified output directory and ready to be consumed by the OpenVINO Runtime. diff --git a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_GNMT_From_Tensorflow.md b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_GNMT_From_Tensorflow.md index 236dd4aac90..19fbb757198 100644 --- a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_GNMT_From_Tensorflow.md +++ b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_GNMT_From_Tensorflow.md @@ -244,7 +244,7 @@ python3 benchmark_app.py -m -d CPU ``` -2. With Inference Engine Python API: +2. With OpenVINO Runtime Python API: > **NOTE**: Before running the example, insert a path to your GNMT `.xml` and `.bin` files into `MODEL_PATH` and `WEIGHTS_PATH`, and fill `input_data_tensor` and `seq_lengths` tensors according to your input data. @@ -274,4 +274,4 @@ exec_net = ie.load_network(network=net, device_name="CPU") result_ie = exec_net.infer(input_data) ``` -For more information about Python API, refer to [Inference Engine Python API](ie_python_api/api.html). +For more information about Python API, refer to [OpenVINO Runtime Python API](ie_python_api/api.html). diff --git a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_Object_Detection_API_Models.md b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_Object_Detection_API_Models.md index 56324c1facb..0489ff08768 100644 --- a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_Object_Detection_API_Models.md +++ b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_Object_Detection_API_Models.md @@ -2,8 +2,8 @@ > **NOTES**: > * Starting with the 2022.1 release, the Model Optimizer can convert the TensorFlow\* Object Detection API Faster and Mask RCNNs topologies differently. By default, the Model Optimizer adds operation "Proposal" to the generated IR. This operation needs an additional input to the model with name "image_info" which should be fed with several values describing the pre-processing applied to the input image (refer to the [Proposal](../../../../ops/detection/Proposal_4.md) operation specification for more information). However, this input is redundant for the models trained and inferred with equal size images. Model Optimizer can generate IR for such models and insert operation [DetectionOutput](../../../../ops/detection/DetectionOutput_1.md) instead of `Proposal`. The `DetectionOutput` operation does not require additional model input "image_info" and moreover, for some models the produced inference results are closer to the original TensorFlow\* model. In order to trigger new behaviour the attribute "operation_to_add" in the corresponding JSON transformation configuration file should be set to value "DetectionOutput" instead of default one "Proposal". -> * Starting with the 2021.1 release, the Model Optimizer converts the TensorFlow\* Object Detection API SSDs, Faster and Mask RCNNs topologies keeping shape-calculating sub-graphs by default, so topologies can be re-shaped in the Inference Engine using dedicated reshape API. Refer to [Using Shape Inference](../../../../OV_Runtime_UG/ShapeInference.md) for more information on how to use this feature. It is possible to change the both spatial dimensions of the input image and batch size. -> * To generate IRs for TF 1 SSD topologies, the Model Optimizer creates a number of `PriorBoxClustered` operations instead of a constant node with prior boxes calculated for the particular input image size. This change allows you to reshape the topology in the Inference Engine using dedicated Inference Engine API. The reshaping is supported for all SSD topologies except FPNs which contain hardcoded shapes for some operations preventing from changing topology input shape. +> * Starting with the 2021.1 release, the Model Optimizer converts the TensorFlow\* Object Detection API SSDs, Faster and Mask RCNNs topologies keeping shape-calculating sub-graphs by default, so topologies can be re-shaped in the OpenVINO Runtime using dedicated reshape API. Refer to [Using Shape Inference](../../../../OV_Runtime_UG/ShapeInference.md) for more information on how to use this feature. It is possible to change the both spatial dimensions of the input image and batch size. +> * To generate IRs for TF 1 SSD topologies, the Model Optimizer creates a number of `PriorBoxClustered` operations instead of a constant node with prior boxes calculated for the particular input image size. This change allows you to reshape the topology in the OpenVINO Runtime using dedicated API. The reshaping is supported for all SSD topologies except FPNs which contain hardcoded shapes for some operations preventing from changing topology input shape. ## How to Convert a Model @@ -45,7 +45,7 @@ To convert a TensorFlow\* Object Detection API model, go to the `/t * `--tensorflow_object_detection_api_pipeline_config ` --- A special configuration file that describes the topology hyper-parameters and structure of the TensorFlow Object Detection API model. For the models downloaded from the TensorFlow\* Object Detection API zoo, the configuration file is named `pipeline.config`. If you plan to train a model yourself, you can find templates for these files in the [models repository](https://github.com/tensorflow/models/tree/master/research/object_detection/samples/configs). * `--input_shape` (optional) --- A custom input image shape. Refer to [Custom Input Shape](#tf_od_custom_input_shape) for more information how the `--input_shape` parameter is handled for the TensorFlow* Object Detection API models. -> **NOTE**: The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the `RGB<->BGR` conversion specifying the command-line parameter: `--reverse_input_channels`. Otherwise, inference results may be incorrect. If you convert a TensorFlow\* Object Detection API model to use with the Inference Engine sample applications, you must specify the `--reverse_input_channels` parameter. For more information about the parameter, refer to **When to Reverse Input Channels** section of [Converting a Model to Intermediate Representation (IR)](../Converting_Model.md). +> **NOTE**: The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the `RGB<->BGR` conversion specifying the command-line parameter: `--reverse_input_channels`. Otherwise, inference results may be incorrect. If you convert a TensorFlow\* Object Detection API model to use with the OpenVINO sample applications, you must specify the `--reverse_input_channels` parameter. For more information about the parameter, refer to **When to Reverse Input Channels** section of [Converting a Model to Intermediate Representation (IR)](../Converting_Model.md). Additionally to the mandatory parameters listed above you can use optional conversion parameters if needed. A full list of parameters is available in the [Converting a TensorFlow* Model](../Convert_Model_From_TensorFlow.md) topic. @@ -57,24 +57,24 @@ mo --input_model=/tmp/ssd_inception_v2_coco_2018_01_28/frozen_inference_graph.pb ## OpenVINO&; Toolkit Samples and Open Model Zoo Demos -Inference Engine comes with a number of samples to demonstrate use of OpenVINO API, additionally, +OpenVINO comes with a number of samples to demonstrate use of OpenVINO Runtime API, additionally, Open Model Zoo provides set of demo applications to show implementation of close to real life applications based on deep learning in various tasks, including Image Classifiacton, Visual Object Detection, Text Recognition, Speech Recognition, Natural Language Processing and others. Refer to the links below for more details. -* [Inference Engine Samples](../../../../OV_Runtime_UG/Samples_Overview.md) +* [OpenVINO Samples](../../../../OV_Runtime_UG/Samples_Overview.md) * [Open Model Zoo Demos](@ref omz_demos) ## Important Notes About Feeding Input Images to the Samples There are several important notes about feeding input images to the samples: -1. Inference Engine samples stretch input image to the size of the input operation without preserving aspect ratio. This behavior is usually correct for most topologies (including SSDs), but incorrect for other models like Faster R-CNN, Mask R-CNN and R-FCN. These models usually use keeps aspect ratio resizer. The type of pre-processing is defined in the pipeline configuration file in the section `image_resizer`. If keeping aspect ratio is used, then it is necessary to resize image before passing it to the sample and optionally pad the resized image with 0s (if the attribute "pad_to_max_dimension" in the pipeline.config is equal to "true"). +1. OpenVINO samples stretch input image to the size of the input operation without preserving aspect ratio. This behavior is usually correct for most topologies (including SSDs), but incorrect for other models like Faster R-CNN, Mask R-CNN and R-FCN. These models usually use keeps aspect ratio resizer. The type of pre-processing is defined in the pipeline configuration file in the section `image_resizer`. If keeping aspect ratio is used, then it is necessary to resize image before passing it to the sample and optionally pad the resized image with 0s (if the attribute "pad_to_max_dimension" in the pipeline.config is equal to "true"). -2. TensorFlow\* implementation of image resize may be different from the one implemented in the sample. Even reading input image from compressed format (like `.jpg`) could give different results in the sample and TensorFlow\*. So, if it is necessary to compare accuracy between the TensorFlow\* and the Inference Engine it is recommended to pass pre-resized input image in a non-compressed format (like `.bmp`). +2. TensorFlow\* implementation of image resize may be different from the one implemented in the sample. Even reading input image from compressed format (like `.jpg`) could give different results in the sample and TensorFlow\*. So, if it is necessary to compare accuracy between the TensorFlow\* and the OpenVINO it is recommended to pass pre-resized input image in a non-compressed format (like `.bmp`). -3. If you want to infer the model with the Inference Engine samples, convert the model specifying the `--reverse_input_channels` command line parameter. The samples load images in BGR channels order, while TensorFlow* models were trained with images in RGB order. When the `--reverse_input_channels` command line parameter is specified, the Model Optimizer performs first convolution or other channel dependent operation weights modification so the output will be like the image is passed with RGB channels order. +3. If you want to infer the model with the OpenVINO samples, convert the model specifying the `--reverse_input_channels` command line parameter. The samples load images in BGR channels order, while TensorFlow* models were trained with images in RGB order. When the `--reverse_input_channels` command line parameter is specified, the Model Optimizer performs first convolution or other channel dependent operation weights modification so the output will be like the image is passed with RGB channels order. 4. Read carefully messaged printed by the Model Optimizer during a model conversion. They contain important instructions on how to prepare input data before running the inference and how to interpret the output. diff --git a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_Slim_Library_Models.md b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_Slim_Library_Models.md index 518fe816893..c4b5f75d0b0 100644 --- a/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_Slim_Library_Models.md +++ b/docs/MO_DG/prepare_model/convert_model/tf_specific/Convert_Slim_Library_Models.md @@ -64,7 +64,7 @@ The `-b` command line parameter is required because the Model Optimizer cannot c Refer to the [Mean and Scale Values for TensorFlow\*-Slim Models](#tf_slim_mean_scale_values) for the information why `--mean_values` and `--scale` command line parameters are used. ## Mean and Scale Values for TensorFlow\*-Slim Models -The TensorFlow\*-Slim Models were trained with normalized input data. There are several different normalization algorithms used in the Slim library. Inference Engine classification sample does not perform image pre-processing except resizing to the input layer size. It is necessary to pass mean and scale values to the Model Optimizer so they are embedded into the generated IR in order to get correct classification results. +The TensorFlow\*-Slim Models were trained with normalized input data. There are several different normalization algorithms used in the Slim library. OpenVINO classification sample does not perform image pre-processing except resizing to the input layer size. It is necessary to pass mean and scale values to the Model Optimizer so they are embedded into the generated IR in order to get correct classification results. The file [preprocessing_factory.py](https://github.com/tensorflow/models/blob/master/research/slim/preprocessing/preprocessing_factory.py) contains a dictionary variable `preprocessing_fn_map` defining mapping between the model type and pre-processing function to be used. The function code should be analyzed to figure out the mean/scale values. @@ -83,7 +83,7 @@ The [inception_preprocessing.py](https://github.com/tensorflow/models/blob/maste Firstly, the `image` is converted to data type `tf.float32` and the values in the tensor are scaled to the `[0, 1]` range using the [tf.image.convert_image_dtype](https://www.tensorflow.org/api_docs/python/tf/image/convert_image_dtype) function. Then the `0.5` is subtracted from the image values and values multiplied by `2.0`. The final image range of values is `[-1, 1]`. -Inference Engine classification sample reads an input image as a three-dimensional array of integer values from the range `[0, 255]`. In order to scale them to `[-1, 1]` range, the mean value `127.5` for each image channel should be specified as well as scale factor `127.5`. +OpenVINO classification sample reads an input image as a three-dimensional array of integer values from the range `[0, 255]`. In order to scale them to `[-1, 1]` range, the mean value `127.5` for each image channel should be specified as well as scale factor `127.5`. Similarly, the mean/scale values can be determined for other Slim models. diff --git a/docs/MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md b/docs/MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md index d3252704549..2ff2fc05c05 100644 --- a/docs/MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md +++ b/docs/MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md @@ -177,9 +177,9 @@ defined as a mathematical expression using the [ShapeOf](../../../ops/shape/Shap Model Optimizer calculates output shapes for all operations in a model to write them to Intermediate Representation files. -> **NOTE**: This is a legacy requirement because starting from IR version 10 Inference Engine needs to know shapes of +> **NOTE**: This is a legacy requirement because starting from IR version 10 OpenVINO Runtime needs to know shapes of > the [Const](../../../ops/infrastructure/Constant_1.md) and the [Parameter](../../../ops/infrastructure/Parameter_1.md) -> operations only. The nGraph component of the Inference Engine calculates output shapes for all operations in a model +> operations only. The OpenVINO Runtime calculates output shapes for all operations in a model > using shapes of [Parameter](../../../ops/infrastructure/Parameter_1.md) and > [Const](../../../ops/infrastructure/Constant_1.md) operations defined with respective operation attributes. @@ -257,11 +257,13 @@ More information on how to develop middle transformations and dedicated API desc [Middle Phase Transformations](#middle-phase-transformations). ### NHWC to NCHW Layout Change -There are several middle transformations responsible for changing model layout from NHWC to NCHW. These transformations -are triggered by default for TensorFlow\* models only because it is the only framework with Convolution operations in -NHWC layout. This layout change is disabled if the model does not have operations that OpenVINO&trade needs to execute in -NCHW layout, for example, Convolutions in NHWC layout. It is still possible to force Model Optimizer to do layout change -using `--disable_nhwc_to_nchw` command-line parameter. + +There are several middle transformations responsible for changing model layout from NHWC to NCHW. These transformations are triggered by default for TensorFlow models as TensorFlow supports Convolution operations in the NHWC layout. + +This layout change is disabled automatically if the model does not have operations that OpenVINO&trade needs to execute in the NCHW layout, for example, Convolutions in NHWC layout. + +It is still possible to force Model Optimizer to do layout change, using `--disable_nhwc_to_nchw` command-line parameter, although it is not advised. + The layout change is a complex problem and detailed explanation of it is out of this document scope. A very brief explanation of this process is provided below: @@ -301,7 +303,7 @@ The last phase of a model conversion is the Intermediate Representation emitting steps: 1. Iterates over all operation nodes in the graph and checks that all nodes have the `type` attribute set. This attribute -defines the operation type and is used in the Inference Engine to instantiate proper operation from the +defines the operation type and is used in the OpenVINO to instantiate proper operation from the [opset](@ref openvino_docs_ops_opset) specified in the `version` attribute of the node. If some node does not have attribute `type` or its values is equal to `None`, the Model Optimizer exits with an error. 2. Performs type inference of graph operations similar to the shape inference. Inferred data types are saved to a port @@ -741,8 +743,7 @@ sub-graph of the original graph isomorphic to the specified pattern. 2. [Specific Operation Front Phase Transformations](#specific-operation-front-phase-transformations) triggered for the node with a specific `op` attribute value. 3. [Generic Front Phase Transformations](#generic-front-phase-transformations). -4. Manually enabled transformation defined with a JSON configuration file (for TensorFlow\*, ONNX\* and MXNet\* models -only) specified using the `--transformations_config` command line parameter: +4. Manually enabled transformation defined with a JSON configuration file (for TensorFlow, ONNX, MXNet, and PaddlePaddle models) specified using the `--transformations_config` command line parameter: 1. [Node Name Pattern Front Phase Transformations](#node-name-pattern-front-phase-transformation). 2. [Front Phase Transformations Using Start and End Points](#start-end-points-front-phase-transformations). 3. [Generic Front Phase Transformations Enabled with Transformations Configuration File](#generic-transformations-config-front-phase-transformations). @@ -1260,5 +1261,5 @@ Refer to the `extensions/back/GatherNormalizer.py` for the example of a such typ * [Deep Learning Network Intermediate Representation and Operation Sets in OpenVINO™](../../IR_and_opsets.md) * [Converting a Model to Intermediate Representation (IR)](../convert_model/Converting_Model.md) * [OpenVINO Model Representation](../../../OV_Runtime_UG/model_representation.md) -* [Inference Engine Extensibility Mechanism](../../../OV_Runtime_UG/Extensibility_DG/Intro.md) +* [OpenVINO™ Extensibility Mechanism](../../../Extensibility_UG/Intro.md) * [Extending the Model Optimizer with Caffe* Python Layers](Extending_Model_Optimizer_with_Caffe_Python_Layers.md) diff --git a/docs/OV_Runtime_UG/API_Changes.md b/docs/OV_Runtime_UG/API_Changes.md deleted file mode 100644 index 0490f9228f2..00000000000 --- a/docs/OV_Runtime_UG/API_Changes.md +++ /dev/null @@ -1,10 +0,0 @@ -# OpenVINO™ Runtime API Changes History {#openvino_docs_OV_Runtime_API_Changes} - -The sections below contain detailed list of changes made to the OpenVINO™ Runtime API in recent releases. - -## 2022.1 - -### New API - -* The OpenVINO™ 2.0 API was introduced. - diff --git a/docs/OV_Runtime_UG/Bfloat16Inference.md b/docs/OV_Runtime_UG/Bfloat16Inference.md index 7339e24a042..5091901e986 100644 --- a/docs/OV_Runtime_UG/Bfloat16Inference.md +++ b/docs/OV_Runtime_UG/Bfloat16Inference.md @@ -26,7 +26,7 @@ There are two ways to check if CPU device can support bfloat16 computations for 1. Query the instruction set using one of these system commands: * `lscpu | grep avx512_bf16` * `cat /proc/cpuinfo | grep avx512_bf16` -2. Use the [Properties API](PropertiesAPI.md) with `METRIC_KEY(OPTIMIZATION_CAPABILITIES)`, which should return `BF16` in the list of CPU optimization options: +2. Use the [Configure devices](supported_plugins/config_properties.md) with `METRIC_KEY(OPTIMIZATION_CAPABILITIES)`, which should return `BF16` in the list of CPU optimization options: @snippet snippets/Bfloat16Inference0.cpp part0 diff --git a/docs/OV_Runtime_UG/DynamicBatching.md b/docs/OV_Runtime_UG/DynamicBatching.md deleted file mode 100644 index d962aa2a664..00000000000 --- a/docs/OV_Runtime_UG/DynamicBatching.md +++ /dev/null @@ -1,106 +0,0 @@ -# Using Dynamic Batching {#openvino_docs_IE_DG_DynamicBatching} - -## Using Dynamic Batching (C++) - -@sphinxdirective -.. raw:: html - -
C++
-@endsphinxdirective - -The Dynamic Batching feature allows you to dynamically change batch size for inference calls -within a preset batch size limit. This feature might be useful when batch size is unknown beforehand and using an extra-large batch size is undesirable or impossible due to resource limitations. For example, applying face detection and then mood labeling to a video, you won't know in advance how many frames will contain a face when you pass inferencing results to a secondary model. - - -You can activate Dynamic Batching by setting `KEY_DYN_BATCH_ENABLED` flag to `YES` in a configuration map that is -passed to the plugin while loading a network. -This configuration creates an `ExecutableNetwork` object that will allow setting batch size -dynamically in all of its infer requests using `SetBatch()` method. -The batch size that was set in the passed `CNNNetwork` object will be used as a maximum batch size limit. - -Here is a code example: - -@snippet snippets/DynamicBatching.cpp part0 - - -### Limitations - -Currently, there are certain limitations for the use of Dynamic Batching exist: - -* Use Dynamic Batching with CPU and GPU plugins only. -* Use Dynamic Batching on topologies that consist of certain layers only: - * Convolution - * Deconvolution - * Activation - * LRN - * Pooling - * FullyConnected - * SoftMax - * Split - * Concatenation - * Power - * Eltwise - * Crop - * BatchNormalization - * Copy - -The following types of layers are not supported: - -* Layers that might arbitrary change tensor shape (such as Flatten, Permute, Reshape) -* Layers specific to object detection topologies (ROIPooling, ProirBox, DetectionOutput) -* Custom layers - -Topology analysis is performed during the process of loading a network into plugin, and if the topology is not supported, an exception is generated. - -## Using Dynamic Batching (Python) - -@sphinxdirective -.. raw:: html - -
Python
-@endsphinxdirective - -Dynamic Batching is a feature that allows you to dynamically change batch size for inference calls within a preset batch size limit. This feature might be useful when batch size is unknown beforehand, and using extra large batch size is not desired or impossible due to resource limitations. For example, face detection with person age, gender, or mood recognition is a typical usage scenario. - -You can activate Dynamic Batching by setting the "DYN_BATCH_ENABLED" flag to "YES" in a configuration map that is passed to the plugin while loading a network. This configuration creates an `ExecutableNetwork` object that will allow setting batch size dynamically in all of its infer requests using the [ie_api.batch_size](api/ie_python_api/_autosummary/openvino.inference_engine.IENetwork.html#openvino.inference_engine.IENetwork.batch_size) method. The batch size that was set in the passed CNNNetwork object will be used as a maximum batch size limit. - -```python -from openvino.inference_engine import IECore - -ie = IECore() -dyn_config = {"DYN_BATCH_ENABLED": "YES"} -ie.set_config(config=dyn_config, device_name=device) -# Read a network in IR or ONNX format -net = ie.read_network(path_to_model) -net.batch_size = 32 # set the maximum batch size to 32 -exec_net = ie.load_network(network=net, device_name=device) -``` - -### Limitations - -Currently, certain limitations for the use of Dynamic Batching exist: - -* Use Dynamic Batching with CPU and GPU plugins only. -* Use Dynamic Batching on topologies that consist of certain layers only: - * Convolution - * Deconvolution - * Activation - * LRN - * Pooling - * FullyConnected - * SoftMax - * Split - * Concatenation - * Power - * Eltwise - * Crop - * BatchNormalization - * Copy - -The following types of layers are not supported: - -* Layers that might arbitrary change tensor shape (such as Flatten, Permute, Reshape) -* Layers specific to object detection topologies (ROIPooling, ProirBox, DetectionOutput) -* Custom layers - -Topology analysis is performed during the process of loading a network into plugin, and if the topology is not supported, an exception is generated. \ No newline at end of file diff --git a/docs/OV_Runtime_UG/Extensibility_DG/AddingNGraphOps.md b/docs/OV_Runtime_UG/Extensibility_DG/AddingNGraphOps.md deleted file mode 100644 index 79ab802a5a1..00000000000 --- a/docs/OV_Runtime_UG/Extensibility_DG/AddingNGraphOps.md +++ /dev/null @@ -1,82 +0,0 @@ -# Custom nGraph Operations {#openvino_docs_IE_DG_Extensibility_DG_AddingNGraphOps} - -Inference Engine Extension API allows you to register operation sets (opsets) with custom nGraph operations to support models with operations which OpenVINO™ does not support out-of-the-box. - -Besides creating custom nGraph operations, to [support custom operations](../../HOWTO/Custom_Layers_Guide.md) in your model you must also create a Model Optimizer extension for the custom operations and an Inference Engine device plugin extension for the device you will use for inference. - -## Operation Class - -To add your custom nGraph operation, create a new class that extends `ngraph::Op`, which is in turn derived from `ngraph::Node`, the base class for all graph operations in nGraph. Follow the steps below to add a custom nGraph operation: - -1. Add the `NGRAPH_RTTI_DECLARATION` and `NGRAPH_RTTI_DEFINITION` macros which define a `NodeTypeInfo` object that identifies the type of the operation to the graph users and helps with dynamic type resolution. The type info of an nGraph operation currently consists of a string identifier and a version number, but this may change in the future. - -2. Implement constructors that optionally take the operation inputs and attributes as parameters. - -3. Override the shape inference method `validate_and_infer_types`. This method is called multiple times during graph manipulations to determine the shapes and element types of the operations outputs. To access the input shapes and input element types, use the `get_input_partial_shape()` and `get_input_element_type()` methods of `ngraph::Node`. Set the inferred shape and element type of the output using `set_output_type`. - -4. Override the `clone_with_new_inputs` method, which enables graph manipulation routines to create copies of this operation and connect it to different nodes during optimization. - -5. Override the `visit_attributes` method, which enables serialization and deserialization of operation attributes. An `AttributeVisitor` is passed to the method, and the implementation is expected to walk over all the attributes in the op using the type-aware `on_attribute` helper. Helpers are already implemented for standard C++ types like `int64_t`, `float`, `bool`, `vector`, and for existing nGraph defined types. - -6. Override `evaluate`, which is an optional method that enables the application of constant folding if there is a custom operation on the constant branch. If your operation contains `evaluate` method you also need to override the `has_evaluate` method, this method allow to get information about availability of `evaluate` method for the operation. - -Based on that, declaration of an operation class can look as follows: - -@snippet template_extension/old/op.hpp op:header - -### Class Fields - -The provided implementation has several fields: - - * `add` of type `int64_t` is an attribute of a custom operation - * `type_info` of type `ngraph::NodeTypeInfo` defines type and version of an operation - -### Operation Constructors - -nGraph operation contains two constructors: -* Default constructor, which enables you to create an operation without attributes -* Constructor that creates and validates an operation with specified inputs and attributes - -@snippet template_extension/old/op.cpp op:ctor - -### `validate_and_infer_types()` - -`ngraph::Node::validate_and_infer_types` method validates operation attributes and calculates output shapes using attributes of the operation. - -@snippet template_extension/old/op.cpp op:validate - -### `clone_with_new_inputs()` - -`ngraph::Node::clone_with_new_inputs` method creates a copy of the nGraph operation with new inputs. - -@snippet template_extension/old/op.cpp op:copy - -### `visit_attributes()` - -`ngraph::Node::visit_attributes` method enables you to visit all operation attributes. - -@snippet template_extension/old/op.cpp op:visit_attributes - -### `evaluate()` and `has_evaluate()` - -`ngraph::Node::evaluate` method enables you to apply constant folding to an operation. - -@snippet template_extension/old/op.cpp op:evaluate - -## Register Custom Operations in Extension Class - -To add custom operations to the [Extension](Extension.md) class, create an operation set with custom operations and implement the `InferenceEngine::IExtension::getOpSets` method: - -@snippet template_extension/old/extension.cpp extension:getOpSets - -This method returns a map of opsets that exist in the [extension library](Extension.md). -nGraph provides an opset mechanism to group operations into clusters. Different opsets distinguish between different versions of one operation. - -When specifying opset names, follow the rules below: -* Use unique opset names. -* Do not use the following built-in opset names: `extension`, `experimental`, `opset1`, `opset2`, `opset3`, ... , `opsetN`. -* [Make sure that the Model Optimizer](../../HOWTO/Custom_Layers_Guide.md) and your extension use the same opset names. -* IR v10 operations have the mandatory `version` attribute specifying the opset. -Operations from the default opset cannot be redefined. - -Use a custom opset to create a new operation or extend functionality of an existing operation from another opset. diff --git a/docs/OV_Runtime_UG/Extensibility_DG/Building.md b/docs/OV_Runtime_UG/Extensibility_DG/Building.md deleted file mode 100644 index b4cf5a1f84d..00000000000 --- a/docs/OV_Runtime_UG/Extensibility_DG/Building.md +++ /dev/null @@ -1,19 +0,0 @@ -# Build Extension Library Using CMake* {#openvino_docs_IE_DG_Extensibility_DG_Building} - -Inference Engine build infrastructure provides the Inference Engine Package for application development. - -To configure the build of your extension library, use the following CMake script: - -@snippet template_extension/old/CMakeLists.txt cmake:extension - -This CMake script finds the Inference Engine and nGraph using the `find_package` CMake command. - -To build the extension library, run the commands below: - -```sh -$ cd template_extension/old -$ mkdir build -$ cd build -$ cmake -DOpenVINO_DIR=[OpenVINO_DIR] ../ -$ cmake --build . -``` diff --git a/docs/OV_Runtime_UG/Extensibility_DG/CPU_Kernel.md b/docs/OV_Runtime_UG/Extensibility_DG/CPU_Kernel.md deleted file mode 100644 index 223a1401600..00000000000 --- a/docs/OV_Runtime_UG/Extensibility_DG/CPU_Kernel.md +++ /dev/null @@ -1,71 +0,0 @@ -# CPU Kernel Custom Operations {#openvino_docs_IE_DG_Extensibility_DG_CPU_Kernel} - -To enable operations not supported by OpenVINO™ out of the box, you need a custom extension for Model Optimizer, a custom nGraph operation set, and a custom kernel for the device you will target. This page describes custom kernel support for the CPU device. - -The primary means of the performance of the CPU codepath in the Inference Engine is the Intel® Math Kernel Library for Deep Neural Networks (Intel® MKL-DNN), and new CPU kernels extend the Inference Engine plugin for the Intel MKL-DNN. Implementing the InferenceEngine::ILayerExecImpl API call defines a general CPU-side extension. There are no Intel MKL-DNN specifics in the way you need to implement a kernel. - -## Implementation Class - -All custom kernels for the CPU plugin should be inherited from the InferenceEngine::ILayerExecImpl interface. -Based on that, declaration of a kernel implementation class can look as follows: - -@snippet template_extension/old/cpu_kernel.hpp cpu_implementation:header - -### Class Fields - -The provided implementation has several fields: - - * `add` of the type `int64_t` is an attribute of a custom operation. - * `inShape` of the type `ngraph::Shape` is an input shape. - * `outShape` of the type `ngraph::Shape` is an output shape. - * `error` of the type `std::string` is a field to handle errors from a constructor. - -### Constructor of Implementation - -An implementation constructor checks parameters of an nGraph operation, stores required attributes, and stores an error message in case of an error. - -@snippet template_extension/old/cpu_kernel.cpp cpu_implementation:ctor - -### `getSupportedConfigurations` - -The InferenceEngine::ILayerExecImpl::getSupportedConfigurations method returns all supported configuration formats (input/output tensor layouts) for your implementation. To specify formats of data, use InferenceEngine::TensorDesc. - -@snippet template_extension/old/cpu_kernel.cpp cpu_implementation:getSupportedConfigurations - -### `init` - -The InferenceEngine::ILayerExecImpl::init method gets a runtime-selected configuration from a vector that is populated from the `getSupportedConfigurations` method and checks the parameters: - -@snippet template_extension/old/cpu_kernel.cpp cpu_implementation:init - -### `execute` - -The InferenceEngine::ILayerExecImpl::execute method accepts and processes the actual tensors as input/output blobs: - -@snippet template_extension/old/cpu_kernel.cpp cpu_implementation:execute - -## Register Implementation in `Extension` Class - -To register custom kernel implementation in the [Extension](Extension.md) class, implement the following methods: - -* getImplTypes -* getImplementation - -### getImplTypes - -InferenceEngine::IExtension::getImplTypes returns a vector of implementation types for an operation. - -@snippet template_extension/old/extension.cpp extension:getImplTypes - -### getImplementation - -InferenceEngine::IExtension::getImplementation returns the kernel implementation with a specified type for an operation. - -@snippet template_extension/old/extension.cpp extension:getImplementation - - -## Load Extension with Executable Kernels to Plugin - -Use the `AddExtension` method of the general plugin interface to load your primitives: - -@snippet snippets/CPU_Kernel.cpp part0 diff --git a/docs/OV_Runtime_UG/Extensibility_DG/Custom_ONNX_Ops.md b/docs/OV_Runtime_UG/Extensibility_DG/Custom_ONNX_Ops.md deleted file mode 100644 index 772bfb9da90..00000000000 --- a/docs/OV_Runtime_UG/Extensibility_DG/Custom_ONNX_Ops.md +++ /dev/null @@ -1,78 +0,0 @@ -# Custom ONNX* Operators {#openvino_docs_IE_DG_Extensibility_DG_Custom_ONNX_Ops} - -The ONNX\* importer provides a mechanism to register custom ONNX operators based on predefined or custom nGraph operations. -The function responsible for registering a new operator is called `ngraph::onnx_import::register_operator` and defined in the `onnx_import/onnx_utils.hpp` file. - -## Register Custom ONNX Operator Based on Predefined nGraph Operations - -The steps below explain how to register a custom ONNX operator, for example, CustomRelu, in a domain called `com.example`. -CustomRelu is defined as follows: -``` -x >= 0 => f(x) = x * alpha -x < 0 => f(x) = x * beta -``` -where `alpha` and `beta` are float constants. - -1. Include headers: - -@snippet onnx_custom_op/onnx_custom_op.cpp onnx_custom_op:headers - -2. Register the CustomRelu operator in the ONNX importer: - -@snippet onnx_custom_op/onnx_custom_op.cpp onnx_custom_op:register_operator - -The `register_operator` function takes four arguments: op_type, opset version, domain, and a function object. -The function object is a user-defined function that takes `ngraph::onnx_import::Node` as an input and based on that, returns a graph with nGraph operations. -The `ngraph::onnx_import::Node` class represents a node in an ONNX model. It provides functions to fetch input node(s) using `get_ng_inputs`, attribute value using `get_attribute_value`, and many more. See the `onnx_import/core/node.hpp` file for the full class declaration. - -New operator registration must happen before an ONNX model is read. For example, if an model uses the `CustomRelu` operator, call `register_operator("CustomRelu", ...)` before InferenceEngine::Core::ReadNetwork. -Reregistering ONNX operators within the same process is supported. If you register an existing operator, you get a warning. - -The example below demonstrates an exemplary model that requires a previously created `CustomRelu` operator: -``` -@include onnx_custom_op/custom_relu_model.prototxt -``` - -This model is in text format, so before it can be passed to Inference Engine, it has to be converted to binary using: -```py -from google.protobuf import text_format -import onnx - -with open("custom_relu_model.prototxt") as in_file: - proto = onnx.ModelProto() - text_format.Parse(in_file.read(), proto, allow_field_number=True) - s = onnx._serialize(proto) - onnx._save_bytes(s, "custom_relu_model.onnx") -``` - - -To create a graph with nGraph operations, visit [Custom nGraph Operations](AddingNGraphOps.md). -For a complete list of predefined nGraph operators, visit [Available Operations Sets](../../ops/opset.md). - -If you do not need an operator anymore, unregister it by calling `unregister_operator`. The function takes three arguments: `op_type`, `version`, and `domain`. - -@snippet onnx_custom_op/onnx_custom_op.cpp onnx_custom_op:unregister_operator - -## Register Custom ONNX Operator Based on Custom nGraph Operations - -The same principles apply when registering a custom ONNX operator based on custom nGraph operations. -This example shows how to register a custom ONNX operator based on `Operation` presented in [this tutorial](AddingNGraphOps.md), which is used in [TemplateExtension](Extension.md): - -@snippet template_extension/old/extension.cpp extension:ctor - -Here, the `register_operator` function is called in the constructor of Extension. The constructor makes sure that the function is called before InferenceEngine::Core::ReadNetwork, because InferenceEngine::Core::AddExtension must be called before a model with a custom operator is read. - -The example below demonstrates how to unregister an operator from the destructor of Extension: -@snippet template_extension/old/extension.cpp extension:dtor - -> **REQUIRED**: It is mandatory to unregister a custom ONNX operator if it is defined in a dynamic shared library. - -## Requirements for Building with CMake - -A program that uses the `register_operator` functionality requires `openvino::core` and `openvino::frontend::onnx` libraries in addition to the OpenVINO Inference Runtime. -The `openvino::frontend::onnx` is a component of the `OpenVINO` package , so `find_package(OpenVINO REQUIRED COMPONENTS ONNX)` can find both. -Those libraries need to be passed to the `target_link_libraries` command in the CMakeLists.txt file. - -See CMakeLists.txt below for reference: - -@snippet onnx_custom_op/CMakeLists.txt cmake:onnx_custom_op diff --git a/docs/OV_Runtime_UG/Extensibility_DG/Extension.md b/docs/OV_Runtime_UG/Extensibility_DG/Extension.md deleted file mode 100644 index 1925ce9e25e..00000000000 --- a/docs/OV_Runtime_UG/Extensibility_DG/Extension.md +++ /dev/null @@ -1,29 +0,0 @@ -# Extension Library {#openvino_docs_IE_DG_Extensibility_DG_Extension} - -Inference Engine provides an InferenceEngine::IExtension interface, which defines the interface for Inference Engine Extension libraries. -Inherit all extension libraries from this interface. The example below contains an implementation of two operations: `Template` -used as an example in this document and `FFT` used as a more complex example from the [Custom Operations Guide](../../HOWTO/Custom_Layers_Guide.md). - -> **NOTE**: `FFT` operation is implemented using the OpenCV library functions `cv::dft` and `cv::idft`. - -Based on that, the declaration of an extension class can look as follows: - -@snippet template_extension/old/extension.hpp extension:header - -The extension library should use `IE_DEFINE_EXTENSION_CREATE_FUNCTION` macro to export a function, which creates an `Extension` class: - -@snippet template_extension/old/extension.cpp extension:CreateExtension - -Also, an `Extension` object should implement the following methods: - -* InferenceEngine::IExtension::Release deletes an extension object. - -* InferenceEngine::IExtension::GetVersion returns information about the version of the library. - -@snippet template_extension/old/extension.cpp extension:GetVersion - -Implement the InferenceEngine::IExtension::getOpSets method if the extension contains custom layers. -Read [Custom nGraph Operation](AddingNGraphOps.md) for more information. - -To integrate execution kernels to the extension library, read [How to Implement Custom CPU Operations](CPU_Kernel.md). -To register a custom ONNX\* operator to the extension library, read [Custom ONNX Operators](Custom_ONNX_Ops.md). diff --git a/docs/OV_Runtime_UG/Extensibility_DG/GPU_Kernel.md b/docs/OV_Runtime_UG/Extensibility_DG/GPU_Kernel.md deleted file mode 100644 index 3305c00085b..00000000000 --- a/docs/OV_Runtime_UG/Extensibility_DG/GPU_Kernel.md +++ /dev/null @@ -1,233 +0,0 @@ -# How to Implement Custom GPU Operations {#openvino_docs_IE_DG_Extensibility_DG_GPU_Kernel} - -To enable operations not supported by OpenVINO™ out of the box, you need a custom extension for Model Optimizer, a custom nGraph operation set, and a custom kernel for the device you will target. This page describes custom kernel support for the GPU device. - -The GPU codepath abstracts many details about OpenCL\*. You need to provide the kernel code in OpenCL C and an XML configuration file that connects the kernel and its parameters to the parameters of the operation. - -There are two options for using the custom operation configuration file: - -* Include a section with your kernels into the global automatically-loaded `cldnn_global_custom_kernels/cldnn_global_custom_kernels.xml` file, which is hosted in the `/runtime/bin` folder -* Call the `InferenceEngine::Core::SetConfig()` method from your application with the `InferenceEngine::PluginConfigParams::KEY_CONFIG_FILE` key and the configuration file name as a value before loading the network that uses custom operations to the plugin: - -@snippet snippets/GPU_Kernel.cpp part0 - -All Inference Engine samples, except the trivial `hello_classification`, and most Open Model Zoo demos -feature a dedicated command-line option `-c` to load custom kernels. For example, to load custom operations for the classification sample, run the command below: -```sh -$ ./classification_sample -m /bvlc_alexnet_fp16.xml -i ./validation_set/daily/227x227/apron.bmp -d GPU - -c /custom_layer_example.xml -``` - -## Configuration File Format - -The configuration file is expected to follow the `.xml` file structure -with a node of the type `CustomLayer` for every custom operation you provide. - -The definitions described in the sections below use the following notations: - -Notation | Description ----|--- -(0/1) | Can have zero or one instance of this node or attribute -(1) | Must have only one instance of this node or attribute -(0+) | Can have any number of instances of this node or attribute -(1+) | Can have one or more instances of this node or attribute - -### CustomLayer Node and Sub-Node Structure - -`CustomLayer` node contains the entire configuration for a single custom operation. - -| Attribute Name |\# | Description | -|-----|-----|-----| -| `name` | (1) | The name of the operation type to be used. This name should be identical to the type used in the IR.| -| `type` | (1) | Must be `SimpleGPU`. | -| `version` | (1) | Must be `1`. | - -**Sub-nodes**: `Kernel` (1), `Buffers` (1), `CompilerOptions` (0+), -`WorkSizes` (0/1) - -### Kernel Node and Sub-Node Structure - -`Kernel` node contains all kernel source code configuration. No kernel -node structure exists. - -**Sub-nodes**: `Source` (1+), `Define` (0+) - -### Source Node and Sub-Node Structure - -`Source` node points to a single OpenCL source file. - -| Attribute Name | \# |Description| -|-----|-----|-----| -| `filename` | (1) | Name of the file containing OpenCL source code. Note that the path is relative to your executable. Multiple source nodes will have their sources concatenated in order. | - -**Sub-nodes**: None - -### Define Node and Sub-Node Structure - -`Define` node configures a single `#‍define` instruction to be added to -the sources during compilation (JIT). - -| Attribute Name | \# | Description | -|------|-------|------| -| `name` | (1) | The name of the defined JIT. For static constants, this can include the value as well, which is taken as a string. | -| `param` | (0/1) | This parameter value is used as the value of this JIT definition. | -| `type` | (0/1) | The parameter type. Accepted values: `int`, `float`, and `int[]`, `float[]` for arrays. | -| `default` | (0/1) | The default value to be used if the specified parameters are missing from the operation in the IR. | - -**Sub-nodes:** None - -The resulting JIT has the following form: -`#‍define [name] [type] [value/default]`. - -### Buffers Node and Sub-Node Structure - -`Buffers` node configures all input/output buffers for the OpenCL entry -function. No buffers node structure exists. - -**Sub-nodes:** `Data` (0+), `Tensor` (1+) - -### Data Node and Sub-Node Structure - -`Data` node configures a single input with static data, for example, -weights or biases. - -| Attribute Name | \# | Description | -|----|-----|------| -| `name` | (1) | Name of a blob attached to an operation in the IR | -| `arg-index` | (1) | 0-based index in the entry function arguments to be bound to | - -**Sub-nodes**: None - -### Tensor Node and Sub-Node Structure - -`Tensor` node configures a single input or output tensor. - -| Attribute Name | \# | Description | -|------|-------|-------| -| `arg-index` | (1) | 0-based index in the entry function arguments to be bound to. | -| `type` | (1) | `input` or `output` | -| `port-index` | (1) | 0-based index in the operation input/output ports in the IR | -| `format` | (0/1) | Data layout declaration for the tensor. Accepted values: `BFYX`, `BYXF`, `YXFB`, `FYXB`, and same values in all lowercase. Default value: `BFYX` | - -### CompilerOptions Node and Sub-Node Structure - -`CompilerOptions` node configures the compilation flags for the OpenCL -sources. - -| Attribute Name | \# | Description | -|--------|-----|------| -| `options` | (1) | Options string to be passed to the OpenCL compiler | - -**Sub-nodes**: None - -### WorkSizes Node and Sub-Node Structure - -`WorkSizes` node configures the global/local work sizes to be used when -queuing an OpenCL program for execution. - -| Attribute Name | \# | Description | -|-----|------|-----| -| `global`
`local` | (0/1)
(0/1) | An array of up to three integers or formulas for defining OpenCL work-sizes to be used during execution.
The formulas can use the values of the B,F,Y,X dimensions and contain the operators: +,-,/,\*,%. All operators are evaluated in integer arithmetic.
Default value: `global=”B*F*Y*X” local=””` | -| `dim` | (0/1) | A tensor to take the work-size from. Accepted values: `input N`, `output`, where `N` is an index of input tensor starting with 0. Default value: `output` | - -**Sub-nodes**: None - -## Example Configuration File - -The following code sample provides an example configuration file in XML -format. For information on the configuration file structure, see -[Configuration File Format](#config-file-format). -```xml - - - - - - - - - - - - -``` - -## Built-In Definitions for Custom Layers - -The following table includes definitions that are attached before -user sources, where `` is the actual input and output, for -example, `INPUT0` or `OUTPUT0`. - -For an example, see [Example Kernel](#example-kernel). - -| Name | Value | -|---|---| -| `NUM_INPUTS` | Number of the input tensors bound to this kernel | -| `GLOBAL_WORKSIZE` | An array of global work sizes used to execute this kernel | -| `GLOBAL_WORKSIZE_SIZE` | The size of the `GLOBAL_WORKSIZE` array | -| `LOCAL_WORKSIZE` | An array of local work sizes used to execute this kernel | -| `LOCAL_WORKSIZE_SIZE` | The size of the `LOCAL_WORKSIZE` array | -| `_DIMS`| An array of the tensor dimension sizes. Always ordered as `BFYX` | -| `_DIMS_SIZE`| The size of the `_DIMS` array.| -| `_TYPE`| The datatype of the tensor: `float`, `half`, or `char`| -| `_FORMAT_` | The format of the tensor, BFYX, BYXF, YXFB , FYXB, or ANY. The format is concatenated to the defined name. You can use the tensor format to define codepaths in your code with `#‍ifdef/#‍endif`. | -| `_LOWER_PADDING` | An array of padding elements used for the tensor dimensions before they start. Always ordered as BFYX.| -| `_ LOWER_PADDING_SIZE` | The size of the `_LOWER_PADDING` array | -| `_UPPER_PADDING` | An array of padding elements used for the tensor dimensions after they end. Always ordered as BFYX. | -| `_UPPER_PADDING_SIZE` | The size of the `_UPPER_PADDING` array | -| `_PITCHES` | The number of elements between adjacent elements in each dimension. Always ordered as BFYX.| -| `_PITCHES_SIZE`| The size of the `_PITCHES` array | -| `_OFFSET`| The number of elements from the start of the tensor to the first valid element, bypassing the lower padding. | -All `` values are automatically defined for every tensor -bound to this operation, such as `INPUT0`, `INPUT1`, and `OUTPUT0`, as shown -in the following example: - -```sh -#define INPUT0_DIMS_SIZE 4 -#define INPUT0_DIMS (int []){ 1,96,55,55, } -``` - -## Example Kernel - -```c -#pragma OPENCL EXTENSION cl_khr_fp16 : enable -__kernel void example_relu_kernel( - const __global INPUT0_TYPE* input0, - __global OUTPUT0_TYPE* output) -{ - const uint idx = get_global_id(0); - const uint idy = get_global_id(1); - const uint idbf = get_global_id(2);//batches*features, as OpenCL supports 3D nd-ranges only - const uint feature = idbf%OUTPUT0_DIMS[1]; - const uint batch = idbf/OUTPUT0_DIMS[1]; - //notice that pitches are in elements, not in bytes! - const uint in_id = batch*INPUT0_PITCHES[0] + feature*INPUT0_PITCHES[1] + idy*INPUT0_PITCHES[2] + idx*INPUT0_PITCHES[3] + INPUT0_OFFSET; - const uint out_id = batch*OUTPUT0_PITCHES[0] + feature*OUTPUT0_PITCHES[1] + idy*OUTPUT0_PITCHES[2] + idx*OUTPUT0_PITCHES[3] + OUTPUT0_OFFSET; - - INPUT0_TYPE value = input0[in_id]; - //neg_slope (which is non-zero for leaky ReLU) is put automatically as #define, refer to the config xml - output[out_id] = value < 0 ? value * neg_slope : value; -} -``` - -> **NOTE**: As described in the previous section, all items like -> `INPUT0_TYPE` are actually defined as OpenCL (pre-)compiler inputs by -> the Inference Engine for efficiency reasons. See [Debugging -> Tips](#debugging-tips) for information on debugging the results. - -> **NOTE**: Several GPU-targeted kernels are also added to the binaries upon compilation of samples -> so that the sample application can easy load them. -> Refer to the `cldnn_global_custom_kernels` folder in the GPU plugin installation directory. - -## Debugging Tips - -* **Using `printf` in the OpenCL™ Kernels**. -To debug the specific values, you can use `printf` in your kernels. -However, be careful not to output excessively, which -could generate too much data. The `printf` output is typical, so -your output can be truncated to fit the buffer. Also, because of -buffering, you actually get an entire buffer of output when the -execution ends.
- -For more information, refer to the [printf -Function](https://www.khronos.org/registry/OpenCL/sdk/1.2/docs/man/xhtml/printfFunction.html). diff --git a/docs/OV_Runtime_UG/Extensibility_DG/Intro.md b/docs/OV_Runtime_UG/Extensibility_DG/Intro.md deleted file mode 100644 index ca3217a26ce..00000000000 --- a/docs/OV_Runtime_UG/Extensibility_DG/Intro.md +++ /dev/null @@ -1,60 +0,0 @@ -# Inference Engine Extensibility Mechanism {#openvino_docs_IE_DG_Extensibility_DG_Intro} - -@sphinxdirective - -.. toctree:: - :maxdepth: 1 - :hidden: - - openvino_docs_IE_DG_Extensibility_DG_AddingNGraphOps - openvino_docs_IE_DG_Extensibility_DG_Custom_ONNX_Ops - CPU Kernels Extensibility - GPU Kernels Extensibility - VPU Kernels Extensibility - openvino_docs_IE_DG_Extensibility_DG_Extension - openvino_docs_IE_DG_Extensibility_DG_Building - -@endsphinxdirective - -If your model contains operations not normally supported by OpenVINO, the Inference Engine Extensibility API lets you add support for those custom operations in a library containing custom nGraph operation sets, corresponding extensions to the Model Optimizer, and a device plugin extension. See the overview in the [Custom Operations Guide](../../HOWTO/Custom_Layers_Guide.md) to learn how these work together. - -To load the Extensibility library to the `InferenceEngine::Core` object, use the `InferenceEngine::Core::AddExtension` method. - -## Inference Engine Extension Library - -An Inference Engine Extension dynamic library contains the following components: - - * [Extension Library](Extension.md): - - Contains custom operation sets - - Provides CPU implementations for custom operations - * [Custom nGraph Operation](AddingNGraphOps.md): - - Enables the use of `InferenceEngine::Core::ReadNetwork` to read Intermediate Representation (IR) with unsupported - operations - - Enables the creation of `ngraph::Function` with unsupported operations - - Provides a shape inference mechanism for custom operations - -> **NOTE**: This documentation is written based on the [Template extension](https://github.com/openvinotoolkit/openvino/tree/master/docs/template_extension), which demonstrates extension development details. You can review the complete code, which is fully compilable and up-to-date, to see how it works. - -## Execution Kernels - -The Inference Engine workflow involves the creation of custom kernels and either custom or existing operations. - -An _operation_ is a network building block implemented in the training framework, for example, `Convolution` in Caffe*. -A _kernel_ is defined as the corresponding implementation in the Inference Engine. - -Refer to the [Model Optimizer Extensibility](../../MO_DG/prepare_model/customize_model_optimizer/Customize_Model_Optimizer.md) -for details on how a mapping between framework operations and Inference Engine kernels is registered. - -In short, you can plug your own kernel implementations into the Inference Engine and map them to the operations in the original framework. - -The following pages describe how to integrate custom _kernels_ into the Inference Engine: - - * [Introduction to development of custom CPU kernels](CPU_Kernel.md) - * [Introduction to development of custom GPU kernels](GPU_Kernel.md) - * [Introduction to development of custom VPU kernels](VPU_Kernel.md) - -## See Also - -* [Build an extension library using CMake*](Building.md) -* [Using Inference Engine Samples](../Samples_Overview.md) -* [Hello Shape Infer SSD sample](../../../samples/cpp/hello_reshape_ssd/README.md) diff --git a/docs/OV_Runtime_UG/Extensibility_DG/VPU_Kernel.md b/docs/OV_Runtime_UG/Extensibility_DG/VPU_Kernel.md deleted file mode 100644 index 86b2ad092bc..00000000000 --- a/docs/OV_Runtime_UG/Extensibility_DG/VPU_Kernel.md +++ /dev/null @@ -1,682 +0,0 @@ -# How to Implement Custom Layers for VPU (Intel® Neural Compute Stick 2) {#openvino_docs_IE_DG_Extensibility_DG_VPU_Kernel} - -To enable operations not supported by OpenVINO™ out of the box, you need a custom extension for Model Optimizer, a custom nGraph operation set, and a custom kernel for the device you will target. This page describes custom kernel support for one the VPU, the Intel® Neural Compute Stick 2 device, which uses the MYRIAD device plugin. - -> **NOTES:** -> * OpenCL\* custom layer support is available in the preview mode. -> * This section assumes you are familiar with developing kernels using OpenCL. - -To customize your topology with an OpenCL layer, carry out the tasks described on this page: - -1. Write and compile your OpenCL code with the standalone offline OpenCL compiler (`clc`). -2. Write a configuration file to bind the OpenCL kernel to the topology file (`.xml`) of the model IR. -3. Pass the configuration file to the Inference Engine with the model IR. - -## Compile OpenCL code for VPU (Intel® Neural Compute Stick 2) - -> **NOTE**: OpenCL compiler, targeting Intel® Neural Compute Stick 2 for the SHAVE* processor only, is redistributed with OpenVINO. -OpenCL support is provided by ComputeAorta* and is distributed under a license agreement between Intel® and Codeplay* Software Ltd. - -The OpenCL toolchain for the Intel® Neural Compute Stick 2 supports offline compilation only, so first compile OpenCL C code using the standalone `clc` compiler. You can find the compiler binary at `/tools/cl_compiler`. - -> **NOTE**: By design, custom OpenCL layers support any OpenCL kernels written assuming OpenCL version 1.2. It also supports half float extension and is optimized for this type, because it is a native type for Intel® Movidius™ VPUs. - -1. Prior to running a compilation, make sure that the following variables are set: - * `SHAVE_MA2X8XLIBS_DIR=/tools/cl_compiler/lib/` - * `SHAVE_LDSCRIPT_DIR=/tools/cl_compiler/ldscripts/` - * `SHAVE_MYRIAD_LD_DIR=/tools/cl_compiler/bin/` - * `SHAVE_MOVIASM_DIR=/tools/cl_compiler/bin/` -2. Run the compilation with the command below. You should use `--strip-binary-header` to make an OpenCL runtime-agnostic binary runnable with the Inference Engine. - ```bash - cd /tools/cl_compiler/bin - ./clc --strip-binary-header custom_layer.cl -o custom_layer.bin - ``` - -## Write a Configuration File - -To tie the topology IR for a layer you customize, prepare a configuration file, so that the Inference Engine can find parameters for your kernel and the execution work grid is described. -For example, consider the following OpenCL kernel signature: -```cpp -__kernel void reorg_nhwc(__global const half *src, __global half *out, int w, int h, int c, int stride); -``` -A configuration file for this kernel might be the following: -```xml - - - - - - - - - - - - - - -``` -Each custom layer is described with the `CustomLayer` node. It has the following nodes and attributes: - - Root node `CustomLayer` contains the following attributes: - - `name` – (Required) The name of the Inference Engine layer to bind the kernel with. - - `type` and `version` – (Required) Reserved for future use. Set them to `MVCL` and `1` respectively. - - `max-shaves` – (Optional) The maximum number of SHAVE cores that should be dedicated for the layer. It is useful for debugging concurrency issues or for resource saving that memory bound kernel does not scale well with the number of cores, so more resources can be left for the rest of a topology. - - Sub-node `Kernel` must contain the following attributes: - - `entry` – The name of your kernel function as you defined it in a source file. In the example above, it is `reorg_nhwc`. - - Node `Source` must contain the following attributes: - - `filename` – The path to a compiled binary relative to the XML configuration file. - - Sub-node `Parameters` – Describes parameters bindings. For more information, see the description below. - - Sub-node `WorkSizes` – Describes local and global work group sizes and the source for dimension deduction as a pair `direction,port`. In the example above, the work group is described relatively to the dimension of the input tensor that comes through port 0 in the IR. `global` and `local` work group configurations support any simple math expressions with +,-,\*,/, and () from `B`(batch), `Y`(height), `X`(width) and `F`(channels). - - Sub-node `Where` – Allows to customize bindings with the `key="value"` attribute. For example, to substitute only 3x3 convolutions, write `` in the binding xml. - - Parameter description supports `Tensor` of one of tensor types such as `input`, `output`, `input_buffer`, `output_buffer` or `data`, `Scalar`, or `Data` nodes and has the following format: - - Each `Tensor` node of `input` or `output` type must contain the following attributes: - - `arg-name` – The name of a kernel parameter in the kernel signature. - - `type` – Node type: `input` or `output` as specified in the IR. - - `port-index` – A number of input/output ports as specified in the IR. - - `format` – The channel order in the tensor. Optional conversion layers are generated if the custom layer format is not compatible with formats of neighboring layers. `BFXY`, `BYXF`, and `ANY` formats are supported currently. - - Each `Tensor` node of `input_buffer` or `output_buffer` type must contain the following attributes: - - `arg-name` – The name of a kernel parameter in the kernel signature. - - `type` – Node type: `input_buffer` or `output_buffer`. Use the appropriate type to bind multiple kernels that correspond to different stages of the same layer. - - `port-index` – The unique identifier to bind by. - - `dim` – The dim source with the same `direction,port` format used for `WorkSizes` bindings. - - `size` – Amount of bytes needed. Current expression syntax supports only expression over dimensions of over selected input/output tensor or constants and might be expended in the future. - - Here is an example of multi-stage MVN layer binding: - ```xml - - - - - - - - - - - - - - - - - - - - - - - - - - ``` - - Each `Tensor` node that has the type `data` must contain the following attributes: - - `source` – A name of the blob as it is in the IR. Typical example is `weights` for convolution. - - `format` – Specifies the channel order in the tensor. Optional conversion layers are generated if the custom layer format is not. - ```xml - - - - - - - - - - - - - ``` - - Each `Scalar` node must contain the following attributes: - - `arg-name` – The name of a kernel parameter in the kernel signature. - - `type` – `int` or `float` value. It is used for correct argument extraction from IR parameters. - - `source` – Contains the name of the parameter in the IR file or input/output (`I`/`O`, `In`/`On`, where `n` is a port number) - followed by dimension `B`(batch), `Y`(height), `X`(width), or `F`(channels). - - - Each `Data` node must contain the following attributes: - - `arg-name` – The name of a kernel parameter in the kernel signature. - - `type` – Node type. Currently, `local_data` is the only supported value, which defines buffer allocated in fast local on-chip memory. It is limited to 100KB for all `__local` and - `__private` arrays defined inside the kernel as well as all `__local` parameters passed to the kernel. Note that a manual-DMA extension requires double buffering. - If the custom layer is detected to run out of local memory, the inference fails. - - `dim` – The dim source with the same `direction,port` format used for `WorkSizes` bindings. - - `size` – Amount of bytes needed. The current expression syntax supports only expression over dimensions of over selected input/output tensor or constants and may be extended in the future. - The example binding below illustrates a kernel with two local buffers passed to the kernel. - ```xml - - - - - - - - - - - - - - -``` - -## Pass Configuration File to Inference Runtime - -> **NOTE**: If both native and custom layer implementations are present, the custom kernel has a priority over the native one. - -Before loading the network that features the custom layers, provide a separate configuration file and load it using the InferenceEngine::Core::SetConfig() method with the PluginConfigParams::KEY_CONFIG_FILE key and the configuration file name as a value: -```cpp -InferenceEngine::Core core; -// Load custom layers -core.SetConfig({ { InferenceEngine::PluginConfigParams::KEY_CONFIG_FILE, "" } }, "MYRIAD"); -``` -Optionally, set a path to a custom layers description with a pair of `VPU_CUSTOM_LAYERS` and `/path/to/your/customLayers.xml` -as a network configuration: -```cpp -InferenceEngine::Core core; -std::map networkConfig; -config["VPU_CUSTOM_LAYERS"] = "/path/to/your/customLayers.xml"; -// Load custom layers in network config -auto exeNetwork = core.LoadNetwork(cnnNetwork, "MYRIAD", networkConfig); -``` - -## Optimizing Kernels with OpenCL for VPU (Intel® Neural Compute Stick 2) - -This section provides optimization guidelines on writing custom layers with OpenCL for VPU devices. Knowledge about general OpenCL -programming model and OpenCL kernel language is assumed and not a subject of this section. The OpenCL model mapping to VPU is described in the table below. - -| OpenCL Model | VPU Mapping| -|-----|----| -| Device code | Executed on SHAVE cores | -| Private memory | Mapped to CMX internal memory, limited to 100KB per work group, valid only while the work group is executed | -| Local memory | Mapped to CMX internal memory, limited to 100KB per work group, valid only while the work group is executed | -| Global memory | Mapped to DDR, used to pass execution preserved parameters for inputs, outputs, and blobs | -| Work group | Executed on a single SHAVE core iterating over multiple work items | - -Note that by the OpenCL specification, the work group execution order is not specified. This means that it is your -responsibility to ensure that race conditions among work groups are not introduced. Custom layer runtime spits evenly -work grid among available compute resources and executes them in an arbitrary order. This static scheduling approach works best if the load is evenly spread out across work groups, which is a typical case for Deep Learning kernels. The following guidelines are recommended to use for work group partitioning: - -1. Split work evenly across work groups. -2. Adjust work group granularity to maintain equal workload for all compute codes. -3. Set the maximum number of cores using the `max-shaves` attribute for the `CustomLayer` node. This keeps more resources for the rest of topology. It is also useful if the kernel scalability reached its limits, which may happen while optimizing memory bound kernels or kernels with poor parallelization. -4. Try an alternate data layout (`BFXY`/`BYXF`) for the kernel if it improves work group partitioning or data access patterns. -Consider not just specific layer boost, but full topology performance because data conversion layers would be automatically inserted -as appropriate. - -Offline OpenCL compiler (`clc`) features automatic vectorization over `get_global_id(0)` usage, if uniform access is detected. -For example, the kernel below could be automatically vectorized: -```cpp -__kernel void cvtf32f16(__global float* restrict inImage, __global half* restrict outImage, - float scale, float bais) -{ - int idx = get_global_id(0) + get_global_id(1) * get_global_size(0) + get_global_id(2) * get_global_size(0) * get_global_size(1); - outImage[idx] = convert_half(inImage[idx]*scale+bais); -} -``` -However, this work-group based vectorizer (WGV) conflicts with the default LLVM vectorizer based on superword level parallelism -(SLP) for the current compiler version. Manual vectorization is recommended to provide the best performance for non-uniform code -patterns. WGV works if and only if vector types are not used in the code. - -Here is a short list of optimization tips: - -1. Help auto-vectorizer ensure non-aliasing pointers for kernel parameters by putting `restrict` where possible. - - This can give a performance boost, especially for kernels with unrolling, like `ocl_grn` from the example below. - - Place `restrict` markers for kernels with manually vectorized codes. In the `ocl_grn` kernel below, the unrolled version without `restrict` is up to 20% slower than the most optimal one, which combines unrolling and `restrict`. -2. Put `#‍pragma unroll N` to your loop header. The compiler does not trigger unrolling by default, so it is your responsibility to -annotate the code with pragmas as appropriate. The `ocl_grn` version with `#‍pragma unroll 4` is up to 50% faster, most of which comes from unrolling the first loop, because LLVM, in general, is better in scheduling 3-stage loops (load-compute-store), while the fist loop - `variance += (float)(src_data[c*H*W + y*W + x] * src_data[c*H*W + y*W + x]);` is only 2-stage (load-compute). Pay -attention to unrolling such cases first. Unrolling factor is loop-dependent. Choose the smallest number that -still improves performance as an optimum between the kernel size and execution speed. For this specific kernel, changing the unroll factor from `4` to `6` results in the same performance, so unrolling factor equal to 4 is an optimum. For Intel® Neural Compute Stick 2, unrolling is conjugated with the automatic software pipelining for load, store, and compute stages: -```cpp -__kernel void ocl_grn(__global const half* restrict src_data, __global half* restrict dst_data, int C, float bias) -{ - int x = get_global_id(0); - int W = get_global_size(0); - int y = get_global_id(1); - int H = get_global_size(1); - - float variance = bias + 1e-9f; - - #pragma unroll 4 - for (int c = 0; c < C; c++) - variance += (float)(src_data[c*H*W + y*W + x] * src_data[c*H*W + y*W + x]); - - variance = 1.f / native_sqrt(variance); - - #pragma unroll 4 - for (int c = 0; c < C; c++) - dst_data[c*H*W + y*W + x] = (half)((float)src_data[c*H*W + y*W + x] * variance); -} -``` -To check the efficiency of WGV, you can compare performance of the kernel above with the kernel below, which is manually vectorized over width: -```cpp -__kernel void ocl_grn_line(__global const half* restrict src_data, __global half* restrict dst_data, int C, int W, float bias) -{ - int y = get_global_id(1); - int H = get_global_size(1); - - for (int x = 0; x < W/8; x++) - { - float8 variance = (float8)(bias+1e-9f); - - #pragma unroll 4 - for (int c = 0; c < C; c++) - { - __global const half8* restrict src_line = ((__global const half8 * restrict)(src_data + c*H*W + y*W)); - half8 sh = src_line[x]; - variance += convert_float8(sh*sh); - } - - variance = 1.f/native_sqrt(variance); - - #pragma unroll 4 - for (int c = 0; c < C; c++) - { - __global const half8* restrict src_line = ((__global const half8 * restrict)(src_data + c*H*W + y*W)); - __global half8* restrict dst_line = ((__global half8 * restrict)(dst_data + c*H*W + y*W)); - - dst_line[x] = convert_half8(convert_float8(src_line[x])*variance); - } - } - for (int x = W/8*8; x < W; x++) - { - float variance = bias+1e-9f; - #pragma unroll 4 - for (int c = 0; c < C; c++) - variance += (float)(src_data[c*H*W + y*W + x]*src_data[c*H*W + y*W + x]); - - variance = 1.f/native_sqrt(variance); - - #pragma unroll 4 - for (int c = 0; c < C; c++) - dst_data[c*H*W + y*W + x] = (float)src_data[c*H*W + y*W + x]*variance; - } -} -``` -Both versions perform the same, but the second one has more complex code. - -3. If it is easy to predict the work group size, you can also use the `reqd_work_group_size` kernel attribute to ask the compiler -to unroll the code up to the local size of the work group. Note that if the kernel is actually executed with the -different work group configuration, the result is undefined. - -4. Prefer to use the `half` compute if it keeps reasonable accuracy. 16-bit float is a native type for Intel® Neural Compute Stick 2, most of the functions `half_*` are mapped to a single hardware instruction. -Use the standard `native_*` function for the rest of types. - -5. Prefer to use the `convert_half` function over `vstore_half` if conversion to 32-bit float is required. `convert_half` is mapped to a single hardware instruction. For the `cvtf32f16` kernel above, the line `outImage[idx] = convert_half(inImage[idx]*scale+bais);` is eight times slower than the code with `vstore_half`. - -6. Mind early exits. Early exit can be extremely costly for the current version of the `clc` compiler due to conflicts with the -auto-vectorizer. The generic advice would be to setup local size by `x` dimension equal to inputs or/and outputs width. -If it is impossible to define the work grid that exactly matches inputs or/and outputs to eliminate checks, for example, -`if (get_global_id(0) >= width) return`, use line-wise kernel variant with manual vectorization. -The kernel example below demonstrates the impact of early exits on kernel performance. - ```cpp - // Initial version - __kernel void reorg(const __global half* restrict src, __global half* restrict out, int stride) - { - int w = get_global_id(0); - int W = get_global_size(0); - - int h = get_global_id(1); - int H = get_global_size(1); - - int c = get_global_id(2); - int C = get_global_size(2); - - int C2 = C/(stride*stride); - int offset = c / C2; - int c2 = c - C2 * offset; - - int H2 = H*stride; - int W2 = W*stride; - - int h2 = h*stride + offset / stride; - int w2 = w*stride + offset - stride * (offset / stride); - - out[W*H*c + W*h + w] = src[W2*H2*c2 + W2*h2 + w2]; - } - ``` -This `reorg` kernel is auto-vectorizable, but an input for YOLO v2 topology is `NCHW=<1,64,26,26>` and it is not multiple of vector width, which is `8` for `half` data type. As a result, the Inference Engine does not select the auto-vectorized kernel. -To compare performance of auto-vectorized and scalar version of the kernel, change the input size to`NCHW=<1,64,26,32>`. This enables the auto-vectorized version to be selected by the Inference Engine and can give you about 30% uplift. -Since the auto-vectorized version is faster, it makes sense to enable it for the YOLO v2 topology input size by setting the local size multiple of vector, for example, 32, and adjust global sizes accordingly. As a result, the execution work grid exceeds actual input dimension, so out-of-bound checks should be inserted. See the updated kernel version below: - ```cpp - // Version with out-of-bound checks added - __kernel void reorg(const __global half* restrict src, __global half* restrict out, int W, int stride) - { - int w = get_global_id(0); - w = min(w, W-1); - - int h = get_global_id(1); - int H = get_global_size(1); - - int c = get_global_id(2); - int C = get_global_size(2); - - int C2 = C/(stride*stride); - int offset = c / C2; - int c2 = c - C2 * offset; - - int H2 = H*stride; - int W2 = W*stride; - - int h2 = h*stride + offset / stride; - int w2 = w*stride + offset - stride * (offset / stride); - - out[W*H*c + W*h + w] = src[W2*H2*c2 + W2*h2 + w2]; - } - ``` -This code performs the same as the initial kernel above (scalar) due to branching overhead. If you replace min/max expression `w = min(w, W-1);` with `if (w >= W) return;`, runtime increases up to 2x against to code without branching (initial version).
-If branching is inevitable for your element-based kernel, it is recommended to change the scheme to line-based. See the kernel variant below: -```cpp -// Line-wise version -__kernel void reorg(const __global half* restrict src, __global half* restrict out, int H, int W, int stride) -{ - int h = min((int)get_global_id(0), H-1); - - int c = get_global_id(1); - int C = get_global_size(1); - int C2 = C/(stride*stride); - int offset = c / C2; - int c2 = c - C2 * offset; - - int H2 = H*stride; - int W2 = W*stride; - - for (int w = 0; w < W; ++w) - { - int h2 = h*stride + offset / stride; - int w2 = w*stride + offset - stride * (offset / stride); - - out[W*H*c + W*h + w] = src[W2*H2*c2 + W2*h2 + w2]; - } -} -``` -This decreases the execution time up to 40% against the best performing vectorized kernel without early exits (initial version). -7. Reuse computations among work items by using line-based kernels or sharing values though `__local` memory. -8. Improve data access locality. Most of custom kernels are memory bound while convolution and fully connected layers are hardware-implemented. The code below demonstrates a further optimized version of the `reorg` kernel unrolled by `stride`: - ```cpp - // Unrolled line-wise version - __kernel void reorg_unrolled_by_stride(const __global half* restrict src, __global half* restrict dst, - int H, int W, int stride) - { - int h = min((int)get_global_id(0), H-1); - - int c2 = get_global_id(1); - int C2 = get_global_size(1); - int C = C2*stride*stride; - - int H2 = H*stride; - int W2 = W*stride; - - for (int stride_y = 0; stride_y < stride; stride_y++) - for (int stride_x = 0; stride_x < stride; stride_x++) - for (int w2 = 0, w = 0; w < W; w2 += stride, w++) - dst[W*H*C2*(stride_y*stride+stride_x) + W*H*c2 + W*h + w] = src[W2*H2*c2 + W2*h*stride + W2*stride_y + w2 + stride_x]; - } - ``` -`scr` data in this case loaded only once. As the result, the cycle count drops up to 45% against the line-wise version. - -9. Copy data from `__dlobal` to `__local` or `__private` memory if the data is accessed more than once. Access to -`__dlobal` memory is orders of magnitude slower than access to `__local`/`__private` due to statically scheduled pipeline, which -stalls completely on memory access without any prefetch. The same recommendation is applicable for scalar load/store -from/to a `__blobal` pointer since work-group copying could be done in a vector fashion. - -10. Use a manual DMA extension. Local (on-chip) memory throughput is up to 24x higher than DDR throughput. Starting from OpenVINO™ 2020.1, VPU OpenCL features manual-DMA kernel extension to copy sub-tensor used by work group into local memory and performing compute without DDR evolved. Here is the simple GRN kernel implementation that runs over DDR. Local size is in the form (width of the input tensor, 1, 1) to define a large enough work group to get code automatically vectorized and unrolled, while global size is (width of the input tensor, height of the input tensor, 1): - ```cpp - __kernel void grn_NCHW( - __global const half* restrict src_data, - __global half* restrict dst_data, - int C, - float bias) - { - float variance = bias + 1e-9f; - - #pragma unroll 4 - for (int c = 0; c < C; c++) - { - float val = (float) src_data[c*get_global_size(1)*get_global_size(0) + get_global_id(1)*get_global_size(0) + get_global_id(0)]; - variance += val*val; - } - - half hvariance = (half)(native_rsqrt((half)(variance/16.f))*0.25f); - - #pragma unroll 4 - for (int c = 0; c < C; c++) - { - dst_data[c*get_global_size(1)*get_global_size(0) + get_global_id(1)*get_global_size(0) + get_global_id(0)] - = src_data[c*get_global_size(1)*get_global_size(0) + get_global_id(1)*get_global_size(0) + get_global_id(0)] * hvariance; - } - } - ``` - -This kernel can be rewritten to introduce special data binding `__dma_preload` and `__dma_postwrite intrinsics`. This means that instead of one kernel, a group of three kernels should be implemented: `kernelName`, `__dma_preload_kernelName`, and `__dma_postwrite_kernelName`. `__dma_preload_kernelName` for a particular work group `n` is guaranteed to be executed before the `n`-th work group itself, while `__dma_postwrite_kernelName` is guaranteed to be executed after a corresponding work group. You can define one of those functions that are intended to be used to copy data from-to `__global` and `__local` memory. The syntactics requires exact functional signature match. The example below illustrates how to prepare your kernel for manual-DMA. - - ```cpp - __kernel void __dma_preload_grn_NCHW( - __global const half* restrict src, - __global half* restrict dst, - __local half* restrict local_src, - __local half* restrict local_dst, - int C, - float bias) - { - // ToDO: copy required piece of src tensor into local_src - } - - __kernel void __dma_postwrite_grn_NCHW( - __global const half* restrict src, - __global half* restrict dst, - __local const half* restrict local_src, - __local half* restrict local_dst, - int C, - float bias) - { - // ToDO: copy back computed piece of local_dst into dst - } - - __kernel void grn_NCHW( - __global const half* restrict src_data, - __global half* restrict dst_data, - __local half* restrict src, - __local half* restrict dst, - int C, - float bias) - { - // same as the example above - } - ``` -The GRN kernel operates on channel-major tensors to compute average over full channel range and then normalizes input elements to produce the output. -As a part of the manual DMA extension, a group of work group copy functions are introduced in addition to `async_work_group_copy`, which is also mapped to a DMA call. - -Here is the list of supported functions: -```cpp -// 2D sub-tensor copy -event_t WorkGroupDmaCreateStrideTransaction( - const local T *src, - global T *dst, - size_t src_width, // width of the line of source in bytes - size_t dst_width, // width of the line of destination in bytes - size_t src_stride, // stride between corresponding 2 consecutive lines of source in bytes - size_t dst_stride, // stride between corresponding 2 consecutive lines of destination in bytes - size_t size, // total number of bytes loaded for all lines from source to destination - event_t event) __OVERLOAD; - - -event_t WorkGroupDmaCreateStrideTransaction( - const global T *src, - local T *dst, - size_t src_width, // width of the line of source in bytes - size_t dst_width, // width of the line of destination in bytes - size_t src_stride, // stride between corresponding 2 consecutive lines of source in bytes - size_t dst_stride, // stride between corresponding 2 consecutive lines of destination in bytes - size_t size, // total number of bytes loaded for all lines from source to destination - event_t event) __OVERLOAD; - -// 3D sub-tensor copy -event_t WorkGroupDmaCreate3DTransaction( - const local T *src, - global T *dst, - size_t src_width, // width of the line of source in bytes - size_t dst_width, // width of the line of destination in bytes - size_t src_stride, // stride between corresponding 2 consecutive lines of source in bytes - size_t dst_stride, // stride between corresponding 2 consecutive lines of destination in bytes - size_t num_planes, // number of planes to be copied - size_t src_plane_stride, // stride between corresponding 2 consecutive planes of source in bytes - size_t dst_plane_stride, // stride between corresponding 2 consecutive planes of destination in bytes - size_t size, // size of the loaded plane in bytes, analogues to the size in 2D case - event_t event) __OVERLOAD; - -event_t WorkGroupDmaCreate3DTransaction( - const global T *src, - local T *dst, - size_t src_width, // width of the line of source in bytes - size_t dst_width, // width of the line of destination in bytes - size_t src_stride, // stride between corresponding 2 consecutive lines of source in bytes - size_t dst_stride, // stride between corresponding 2 consecutive lines of destination in bytes - size_t num_planes, // number of planes to be copied - size_t src_plane_stride, // stride between corresponding 2 consecutive planes of source in bytes - size_t dst_plane_stride, // stride between corresponding 2 consecutive planes of destination in bytes - size_t size, // size of the loaded plane in bytes, analogues to the size in 2D case - event_t event) __OVERLOAD; -``` -where `T` can be `uchar`, `char`, `short`, `ushort`, `int`, `uint`, `long`, `ulong`, `half` or `float`. - -Modified version of the GRN kernel could be the following: -```cpp -__kernel void __dma_preload_grn_NCHW( - __global const half* restrict src, - __global half* restrict dst, - __local half* restrict local_src, - __local half* restrict local_dst, - int C, - float bias) -{ - WorkGroupDmaCreate3DTransaction( - src + get_group_id(0)*get_local_size(0) - + get_group_id(1)*get_local_size(1)*get_global_size(0), // src - local_src, // dst - get_local_size(0) * sizeof(half), // src width - get_local_size(0) * sizeof(half), // dst width - get_global_size(0) * sizeof(half), // src stride - get_local_size(0) * sizeof(half), // dst stride - C, // num planes - get_global_size(0) * get_global_size(1) * sizeof(half), // src plane stride - get_local_size(0) * get_local_size(1) * sizeof(half), // dst plane stride - get_local_size(0) * get_local_size(1) * sizeof(half), // plane size - 0); -} - -__kernel void __dma_postwrite_grn_NCHW( - __global const half* restrict src, - __global half* restrict dst, - __local const half* restrict local_src, - __local half* restrict local_dst, - int C, - float bias) -{ - WorkGroupDmaCreate3DTransaction( - local_dst, // src - dst + get_group_id(0)*get_local_size(0) - + get_group_id(1)*get_local_size(1)*get_global_size(0), // dst - get_local_size(0) * sizeof(half), // src width - get_local_size(0) * sizeof(half), // dst width - get_local_size(0) * sizeof(half), // src stride - get_global_size(0) * sizeof(half), // dst stride - C, // num planes - get_local_size(0) * get_local_size(1) * sizeof(half), // src plane stride - get_global_size(0) * get_global_size(1) * sizeof(half), // dst plane stride - get_local_size(0) * get_local_size(1) * sizeof(half), // plane size - 0); -} - -__kernel void grn_NCHW( - __global const half* restrict src_data, - __global half* restrict dst_data, - __local half* restrict src, - __local half* restrict dst, - int C, - float bias) -{ - float variance = bias + 1e-9f; - - #pragma unroll 8 - for (int c = 0; c < C; c++) - { - float val = (float) src[c*get_local_size(1)*get_local_size(0) + get_local_id(1)*get_local_size(0) + get_local_id(0)]; - variance += val*val; - } - - half hvariance = (half)(native_rsqrt((half)(variance/16.f))*0.25f); - - #pragma unroll 8 - for (int c = 0; c < C; c++) - { - dst[c*get_local_size(1)*get_local_size(0) + get_local_id(1)*get_local_size(0) + get_local_id(0)] - = src[c*get_local_size(1)*get_local_size(0) + get_local_id(1)*get_local_size(0) + get_local_id(0)] * hvariance; - } -} -``` - -Note the `get_local_size` and `get_local_id` usage inside the kernel. 21x speedup is expected for a kernel on enet-curbs setup because it was completely limited by memory usage. - -An alternative method to using DMA is to use work item copy extension. Those functions are executed inside a kernel and requires work groups equal to single work item. - -Here is the list of supported work item functions: -```cpp -item_dma_event_t WorkItemDmaCreateTransaction( - const global T *src, - private T *dst, - size_t size, - item_dma_event_t event) __OVERLOAD; - -item_dma_event_t WorkItemDmaCreateTransaction( - const private T *src, - global T *dst, - size_t size, - item_dma_event_t event) __OVERLOAD; - -item_dma_event_t WorkItemDmaCreateStrideTransaction( - const global T *src, - private T *dst, - size_t src_width, - size_t dst_width, - size_t src_stride, - size_t dst_stride, - size_t size, - item_dma_event_t event) __OVERLOAD; - -item_dma_event_t WorkItemDmaCreateStrideTransaction( - const private T *src, - global T *dst, - size_t src_width, - size_t dst_width, - size_t src_stride, - size_t dst_stride, - size_t size, - item_dma_event_t event) __OVERLOAD; - -item_dma_event_t WorkItemDmaCreate3DTransaction( - const global T *src, - private T *dst, - size_t src_width, - size_t dst_width, - size_t src_stride, - size_t dst_stride, - size_t num_planes, - size_t src_plane_stride, - size_t dst_plane_stride, - size_t size, - item_dma_event_t event) __OVERLOAD; - -item_dma_event_t WorkItemDmaCreate3DTransaction( - const private T *src, - global T *dst, - size_t src_width, - size_t dst_width, - size_t src_stride, - size_t dst_stride, - size_t num_planes, - size_t src_plane_stride, - size_t dst_plane_stride, - size_t size, - item_dma_event_t event) __OVERLOAD; -``` -where `T` can be `uchar`, `char`, `short`, `ushort`, `int`, `uint`, `long`, `ulong`, `half` or `float`. diff --git a/docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md b/docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md deleted file mode 100644 index ebbec0e3c6e..00000000000 --- a/docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md +++ /dev/null @@ -1,490 +0,0 @@ -# Integrate Inference Engine {#openvino_docs_IE_DG_Integrate_with_customer_application_new_API} - -## Integrate Inference Engine with Your C++ Application - -@sphinxdirective -.. raw:: html - -
C++
-@endsphinxdirective - -The following diagram illustrates the typical Inference Engine С++ API workflow: - -![ie_api_flow_cpp] - -Read the sections below to learn about each item. - -> **NOTE**: Before start using Inference Engine, make sure you set all environment variables during the installation. If you did not, follow the instructions from the _Set the Environment Variables_ section in the installation guides: -> * [For Windows* 10](../install_guides/installing-openvino-windows.md) -> * [For Linux*](../install_guides/installing-openvino-linux.md) -> * [For macOS*](../install_guides/installing-openvino-macos.md) -> * To build an open source version, use the [Inference Engine Build Instructions](https://github.com/openvinotoolkit/openvino/wiki/BuildingCode). - -### Link with Inference Library - -1. **Create a structure** for the project: - ``` sh - project/ - ├── CMakeLists.txt - CMake file to build - ├── ... - Additional folders like includes/ - └── src/ - source folder - └── main.cpp - build/ - build directory - ... - ``` - -2. **Include Inference Engine, nGraph and OpenCV libraries** in `project/CMakeLists.txt` -[OpenCV](https://docs.opencv.org/master/db/df5/tutorial_linux_gcc_cmake.html) integration is needed mostly for pre-processing input data and model representation in OpenVINO™ Runtime for more complex applications using [OpenVINO Model API](../OV_Runtime_UG/model_representation.md). - ``` cmake - cmake_minimum_required(VERSION 3.0.0) - project(project_name) - find_package(OpenVINO REQUIRED) - add_executable(${PROJECT_NAME} src/main.cpp) - target_link_libraries(${PROJECT_NAME} PRIVATE openvino::runtime) - ``` - -### Use Inference Engine API to Implement Inference Pipeline - -This section provides step-by-step instructions to implement a typical inference pipeline with the Inference Engine C++ API: - -![ie_api_use_cpp] -#### Step 1. Create Inference Engine Core - -Use the following code to create Inference Engine Core to manage available devices and read network objects: - -@snippet snippets/Integrate_with_customer_application_new_API.cpp part0 - -#### Step 2 (Optional). Configure Input and Output of the Model - -@sphinxdirective -.. raw:: html - -
-@endsphinxdirective - - -Optionally, configure input and output of the model using the steps below: - -1. Load a model to a Core object: - @sphinxdirective - - .. tab:: IR - - .. code-block:: c - - auto network = core.ReadNetwork("model.xml"); - - .. tab:: ONNX - - .. code-block:: c - - auto network = core.ReadNetwork("model.onnx"); - - .. tab:: nGraph - - .. code-block:: c - - std::shared_ptr createNetwork() { - // To construct a network, please follow - // https://docs.openvino.ai/latest/openvino_docs_nGraph_DG_build_function.html - } - auto network = CNNNetwork(createNetwork()); - - @endsphinxdirective - -2. Request input and output information using `InferenceEngine::CNNNetwork::getInputsInfo()`, and `InferenceEngine::CNNNetwork::getOutputsInfo()` methods: - ```cpp - /** Take information about all topology inputs **/ - InferenceEngine::InputsDataMap input_info = network.getInputsInfo(); - /** Iterate over all input info**/ - for (auto &item : input_info) { - auto input_data = item.second; - // Add your input configuration steps here - } - - /** Take information about all topology outputs **/ - InferenceEngine::OutputsDataMap output_info = network.getOutputsInfo(); - /** Iterate over all output info**/ - for (auto &item : output_info) { - auto output_data = item.second; - // Add your output configuration steps here - } - ``` - Configuring options: - 1. **Set precision** (number format): FP16, FP32, INT8, etc. Refer to the Supported Configurations section on the [Supported Devices](supported_plugins/Supported_Devices.md) page to choose the relevant configuration.
- For input (*iterate over all input info*): - ```cpp - input_data->setPrecision(InferenceEngine::Precision::U8); - ``` - For output (*iterate over all output info*): - ```cpp - output_data->setPrecision(InferenceEngine::Precision::FP32); - ``` - **By default**, the input and output precision is set to `Precision::FP32`. - - 2. **Set layout** (NCHW, ).
- For input (*iterate over all input info*): - ```cpp - input_data->setLayout(InferenceEngine::Layout::NCHW); - ``` - **By default**, the input layout is set to `Layout::NCHW`.
- For output (*iterate over all output info*): - ```cpp - output_data->setLayout(InferenceEngine::Layout::NC); - ``` - **By default**, the output layout depends on a number of its dimensions:
- |Number of dimensions | 5 | 4 | 3 | 2 | 1 | - |:--------------------|-------|------|-----|----|----| - |Layout | NCDHW | NCHW | CHW | NC | C | - 3. **Set resize algorithm for inputs** (Bilinear). You can allow input of any size. To do this, mark each input as resizable by setting a desired resize algorithm (e.g. `BILINEAR`) inside of the appropriate input info (*Iterate over all input info*): - ```cpp - input_data->getPreProcess().setResizeAlgorithm(InferenceEngine::RESIZE_BILINEAR); - ``` - **By default**, no resize algorithm is set for inputs. - - 4. **Set color format** (BGR, RGB, NV12). Basic color format conversions are supported as well. **By default**, the Inference Engine assumes that the input color format is BGR and color format conversions are disabled. Set `ColorFormat::RAW` input color format if the input does not need color conversions. The Inference Engine supports the following color format conversions: - * RGB->BGR - * RGBX->BGR - * BGRX->BGR - * NV12->BGR - where X is a channel that will be ignored during inference. To enable the conversions, set a desired color format (for example, RGB) for each input inside of the appropriate input info (*iterate over all input info*): - ```cpp - input_data->getPreProcess().setColorFormat(InferenceEngine::ColorFormat::RGB); - ``` - > **NOTE**: NV12 input color format pre-processing differs from other color conversions. In case of NV12, Inference Engine expects two separate image planes (Y and UV). You must use a specific `InferenceEngine::NV12Blob` object instead of default blob object and set this blob to the Inference Engine Infer Request using `InferenceEngine::InferRequest::SetBlob()`. Refer to [Hello NV12 Input Classification C++ Sample](../../samples/cpp/hello_nv12_input_classification/README.md) for more details. - - 5. **Run on multiple images** with setting batch. If you want to run inference for multiple images at once, you can use the built-in batch pre-processing functionality. - - **NOTE** : Batch pre-processing is not supported if input color format is set to `ColorFormat::NV12`. - -@sphinxdirective -.. raw:: html - -
-@endsphinxdirective - -#### Step 3. Load the Model to the Device - -Load the model to the device using `InferenceEngine::Core::LoadNetwork()`: - - -@sphinxdirective - -.. tab:: IR - - .. code-block:: c - - executable_network = core.LoadNetwork("model.xml", "CPU"); - -.. tab:: ONNX - - .. code-block:: c - - executable_network = core.LoadNetwork("model.onnx", "CPU"); - -.. tab:: nGraph - - .. code-block:: c - - std::shared_ptr createNetwork() { - // To construct a network, please follow - // https://docs.openvino.ai/latest/openvino_docs_nGraph_DG_build_function.html - } - auto network = CNNNetwork(createNetwork()); - executable_network = core.LoadNetwork(network, "CPU"); - -.. tab:: Model From Step 2 - - Follow this step only if you went through optional "Step 2 (Optional). Configure Input and Output of the Model", otherwise use another tab for your model type: IR (OpenVINO Intermediate Representation), ONNX or nGraph. - - .. code-block:: c - - executable_network = core.LoadNetwork(network, "CPU"); - -@endsphinxdirective - - -It creates an executable network from a network object. The executable network is associated with single hardware device. -It is possible to create as many networks as needed and to use them simultaneously (up to the limitation of the hardware resources). - -Third parameter is a configuration for plugin. It is map of pairs: (parameter name, parameter value). Choose device from -[Supported devices](supported_plugins/Supported_Devices.md) page for more details about supported configuration parameters. - -@snippet snippets/Integrate_with_customer_application_new_API.cpp part6 - -#### Step 4. Create an Inference Request - -Create an infer request using the following code: - -@snippet snippets/Integrate_with_customer_application_new_API.cpp part7 - -#### Step 5. Prepare Input - -You can use one of the following options to prepare input: - -* **Optimal way for a single network.** Get blobs allocated by an infer request using `InferenceEngine::InferRequest::GetBlob()` and feed an image and the input data to the blobs. In this case, input data must be aligned (resized manually) with a given blob size and have a correct color format. - - @snippet snippets/Integrate_with_customer_application_new_API.cpp part8 - -* **Optimal way for a cascade of networks (output of one network is input for another).** Get output blob from the first request using `InferenceEngine::InferRequest::GetBlob()` and set it as input for the second request using `InferenceEngine::InferRequest::SetBlob()`. - - @snippet snippets/Integrate_with_customer_application_new_API.cpp part9 - -* **Optimal way to handle ROI (a ROI object located inside of input of one network is input for another).** It is possible to re-use shared input by several networks. You do not need to allocate separate input blob for a network if it processes a ROI object located inside of already allocated input of a previous network. For instance, when first network detects objects on a video frame (stored as input blob) and second network accepts detected bounding boxes (ROI inside of the frame) as input. In this case, it is allowed to re-use pre-allocated input blob (used by first network) by second network and just crop ROI without allocation of new memory using `InferenceEngine::make_shared_blob()` with passing of `InferenceEngine::Blob::Ptr` and `InferenceEngine::ROI` as parameters. - - @snippet snippets/Integrate_with_customer_application_new_API.cpp part10 - - Make sure that shared input is kept valid during execution of each network. Otherwise, ROI blob may be corrupted if the original input blob (that ROI is cropped from) has already been rewritten. - -* Allocate input blobs of the appropriate types and sizes, feed an image and the input data to the blobs, and call `InferenceEngine::InferRequest::SetBlob()` to set these blobs for an infer request: - - @snippet snippets/Integrate_with_customer_application_new_API.cpp part11 - -A blob can be filled before and after `SetBlob()`. - -> **NOTE**: -> -> * The `SetBlob()` method compares precision and layout of an input blob with the ones defined in step 3 and -> throws an exception if they do not match. It also compares a size of the input blob with input -> size of the read network. But if input was configured as resizable, you can set an input blob of -> any size (for example, any ROI blob). Input resize will be invoked automatically using resize -> algorithm configured on step 3. Similarly to the resize, color format conversions allow the color -> format of an input blob to differ from the color format of the read network. Color format -> conversion will be invoked automatically using color format configured on step 3. -> -> * `GetBlob()` logic is the same for pre-processable and not pre-processable input. Even if it is -> called with input configured as resizable or as having specific color format, a blob allocated by -> an infer request is returned. Its size and color format are already consistent with the -> corresponding values of the read network. No pre-processing will happen for this blob. If you -> call `GetBlob()` after `SetBlob()`, you will get the blob you set in `SetBlob()`. - -#### Step 6. Start Inference - -Start inference in asynchronous or synchronous mode. Async API usage can improve overall frame-rate of the application, because rather than wait for inference to complete, the app can continue doing things on the host, while accelerator is busy. - -* For synchronous inference request: - ```cpp - infer_request.Infer(); - ``` - -* For asynchronous inference request: - ```cpp - infer_request.StartAsync(); - infer_request.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY); - ``` - `StartAsync` returns immediately and starts inference without blocking main thread, `Infer` blocks main thread and returns when inference is completed. Call `Wait` for waiting result to become available for asynchronous request. - - There are three ways to use it: - * specify maximum duration in milliseconds to block for. The method is blocked until the specified timeout has elapsed, or the result becomes available, whichever comes first. - * `InferenceEngine::InferRequest::WaitMode::RESULT_READY` - waits until inference result becomes available - * `InferenceEngine::InferRequest::WaitMode::STATUS_ONLY` - immediately returns request status.It does not - block or interrupts current thread. - - -Both requests are thread-safe: can be called from different threads without fearing corruption and failures. - -Multiple requests for single `ExecutableNetwork` are executed sequentially one by one in FIFO order. - -While request is ongoing, all its methods except `InferenceEngine::InferRequest::Wait` would throw an -exception. - -#### Step 7. Process the Inference Results - -Go over the output blobs and process the inference results. Note that casting `Blob` to `TBlob` via `std::dynamic_pointer_cast` is not the recommended way. It's better to access data via the `buffer()` and `as()` methods as follows: - -@snippet snippets/Integrate_with_customer_application_new_API.cpp part14 - -### Build Your Application - -For details about building your application, refer to the CMake files for the sample applications. -All samples source code is located in the `/samples` directory, where `INSTALL_DIR` is the OpenVINO™ installation directory. - -To build your project using CMake with the default build tools currently available on your machine, execute the following commands: - -> **NOTE**: Make sure you set environment variables first by running `/setupvars.sh` (or `setupvars.bat` for Windows). Otherwise the `InferenceEngine_DIR` and `OpenCV_DIR` variables won't be configured properly to pass `find_package` calls. - -```sh -cd build/ -cmake ../project -cmake --build . -``` -It's allowed to specify additional build options (e.g. to build CMake project on Windows with a specific build tools). Please refer to the [CMake page](https://cmake.org/cmake/help/latest/manual/cmake.1.html#manual:cmake(1)) for details. - -### Run Your Application - -> **NOTE**: Before running, make sure you completed **Set the Environment Variables** section in [OpenVINO Installation](../../samples/cpp/hello_nv12_input_classification/README.md) document so that the application can find the libraries. - -To run compiled applications on Microsoft* Windows* OS, make sure that Microsoft* Visual C++ 2017 -Redistributable and Intel® C++ Compiler 2017 Redistributable packages are installed and -`/bin/intel64/Release/*.dll` files are placed to the -application folder or accessible via `%PATH%` environment variable. - -## Integrate Inference Engine with Your Python Application - -@sphinxdirective -.. raw:: html - -
Python
-@endsphinxdirective - -This document explains how to integrate and use the Inference Engine API with your Python application. - -The following diagram illustrates the typical Inference Engine Python API workflow: -![ie_api_flow_python] - -Read the sections below to learn about each item. - -### Import Inference Module - -To make use of the Inference Engine functionality, import IECore to your application: - -```py -from openvino.inference_engine import IECore -``` - -### Use Inference Engine API - -This section provides step-by-step instructions to implement a typical inference pipeline with the Inference Engine API: - -![ie_api_use_python] - -#### Step 1. Create Inference Engine Core - -Use the following code to create Inference Engine Core to manage available devices and read network objects: -```py -ie = IECore() -``` -#### Step 2 (Optional). Read model. Configure Input and Output of the Model - -@sphinxdirective -.. raw:: html - -
-@endsphinxdirective - -Optionally, configure input and output of the model using the steps below: - -1. Read model - @sphinxdirective - - .. tab:: IR - - .. code-block:: python - - net = ie.read_network(model="model.xml") - - .. tab:: ONNX - - .. code-block:: python - - net = ie.read_network(model="model.onnx") - - .. tab:: nGraph - - .. code-block:: python - - #Basic example of nGraph model creation - param = Parameter(Type.f32, Shape([1, 3, 22, 22])) - relu = ng.relu(param) - func = Function([relu], [param], 'test') - caps = Function.to_capsule(func) - net = IENetwork(caps) - - @endsphinxdirective - -2. Request input and output information using input_info, outputs - ```py - inputs = net.input_info - input_name = next(iter(net.input_info)) - - outputs = net.outputs - output_name = next(iter(net.outputs)) - ``` - Information for this input layer is stored in input_info. The next cell prints the input layout, precision and shape. - ```py - print("Inputs:") - for name, info in net.input_info.items(): - print("\tname: {}".format(name)) - print("\tshape: {}".format(info.tensor_desc.dims)) - print("\tlayout: {}".format(info.layout)) - print("\tprecision: {}\n".format(info.precision)) - ``` - This cell output tells us that the model expects inputs with a shape of [1,3,224,224], and that this is in NCHW layout. This means that the model expects input data with a batch size (N) of 1, 3 channels (C), and images of a height (H) and width (W) of 224. The input data is expected to be of FP32 (floating point) precision. - - Getting the output layout, precision and shape is similar to getting the input layout, precision and shape. - ```py - print("Outputs:") - for name, info in net.outputs.items(): - print("\tname: {}".format(name)) - print("\tshape: {}".format(info.shape)) - print("\tlayout: {}".format(info.layout)) - print("\tprecision: {}\n".format(info.precision)) - ``` - This cell output shows that the model returns outputs with a shape of [1, 1001], where 1 is the batch size (N) and 1001 the number of classes (C). The output is returned as 32-bit floating point. - -@sphinxdirective -.. raw:: html - -
-@endsphinxdirective - -#### Step 3. Load model to the Device - -Load the model to the device using `load_network()`: - -@sphinxdirective - -.. tab:: IR - - .. code-block:: python - - exec_net = ie.load_network(network= "model.xml", device_name="CPU") -.. tab:: ONNX - - .. code-block:: python - - exec_net = ie.load_network(network= "model.onnx", device_name="CPU") - -.. tab:: Model from step 2 - - .. code-block:: python - - exec_net = ie.load_network(network=net, device_name="CPU") - -@endsphinxdirective - -This example is designed for CPU device, refer to the [Supported Devices](../OV_Runtime_UG/supported_plugins/Supported_Devices.md) page to read about more devices. - -#### Step 4. Prepare input -```py -import cv2 -import numpy as np - -image = cv2.imread("image.png") - -# Resize with OpenCV your image if needed to match with net input shape -# N, C, H, W = net.input_info[input_name].tensor_desc.dims -# image = cv2.resize(src=image, dsize=(W, H)) - -# Converting image to NCHW format with FP32 type -input_data = np.expand_dims(np.transpose(image, (2, 0, 1)), 0).astype(np.float32) -``` - -#### Step 5. Start Inference -```py -result = exec_net.infer({input_name: input_data}) -``` - -#### Step 6. Process the Inference Results -```py -output = result[output_name] -``` - -### Run Your Application - -Congratulations, you have made your first Python application with OpenVINO™ toolkit, now you may run it. - -[ie_api_flow_cpp]: img/BASIC_IE_API_workflow_Cpp.svg -[ie_api_use_cpp]: img/IMPLEMENT_PIPELINE_with_API_C.svg -[ie_api_flow_python]: img/BASIC_IE_API_workflow_Python.svg -[ie_api_use_python]: img/IMPLEMENT_PIPELINE_with_API_Python.svg diff --git a/docs/OV_Runtime_UG/Model_caching_overview.md b/docs/OV_Runtime_UG/Model_caching_overview.md index bf85c4ef680..074f471b4de 100644 --- a/docs/OV_Runtime_UG/Model_caching_overview.md +++ b/docs/OV_Runtime_UG/Model_caching_overview.md @@ -8,9 +8,9 @@
C++
@endsphinxdirective -As described in the [OpenVINO™ Runtime User Guide](OpenVINO_Runtime_User_Guide.md), a common application flow consists of the following steps: +As described in the [OpenVINO™ Runtime User Guide](openvino_intro.md), a common application flow consists of the following steps: -1. **Create an Inference Engine Core object**: First step to manage available devices and read network objects +1. **Create a Core object**: First step to manage available devices and read network objects 2. **Read the Intermediate Representation**: Read an Intermediate Representation file into an object of the `InferenceEngine::CNNNetwork` @@ -72,9 +72,9 @@ To check in advance if a particular device supports model caching, your applicat
Python
@endsphinxdirective -As described in Inference Engine Developer Guide, a common application flow consists of the following steps: +As described in OpenVINO User Guide, a common application flow consists of the following steps: -1. **Create an Inference Engine Core Object** +1. **Create a Core Object** 2. **Read the Intermediate Representation** - Read an Intermediate Representation file into an object of the [ie_api.IENetwork](api/ie_python_api/_autosummary/openvino.inference_engine.IENetwork.html) 3. **Prepare inputs and outputs** 4. **Set configuration** - Pass device-specific loading configurations to the device diff --git a/docs/OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md b/docs/OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md deleted file mode 100644 index deaea820d7b..00000000000 --- a/docs/OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md +++ /dev/null @@ -1,61 +0,0 @@ -# OpenVINO™ Runtime User Guide {#openvino_docs_OV_Runtime_User_Guide} - -@sphinxdirective - -.. _deep learning inference engine: - -.. toctree:: - :maxdepth: 1 - :hidden: - - openvino_docs_IE_DG_Integrate_with_customer_application_new_API - openvino_docs_OV_Runtime_UG_Model_Representation - openvino_docs_OV_Runtime_UG_Preprocessing_Overview - - openvino_docs_IE_DG_ShapeInference - openvino_docs_IE_DG_Device_Plugins - - - openvino_docs_IE_DG_DynamicBatching - - openvino_docs_IE_DG_supported_plugins_AUTO - - openvino_docs_IE_DG_supported_plugins_MULTI - openvino_docs_OV_UG_Hetero_execution - openvino_docs_IE_DG_network_state_intro - openvino_2_0_transition_guide - - openvino_docs_deployment_optimization_guide_dldt_optimization_guide - openvino_docs_IE_DG_Model_caching_overview - openvino_docs_IE_DG_Extensibility_DG_Intro - openvino_docs_IE_DG_Int8Inference - openvino_docs_IE_DG_Bfloat16Inference - ngraph_transformation - openvino_docs_OV_Runtime_API_Changes - -@endsphinxdirective - -## Introduction -Inference Engine is a set of C++ libraries with C and Python bindings providing a common API to deliver inference solutions on the platform of your choice. Use the Inference Engine API to read the Intermediate Representation (IR), ONNX and execute the model on devices. - -Inference Engine uses a plugin architecture. Inference Engine plugin is a software component that contains complete implementation for inference on a certain Intel® hardware device: CPU, GPU, VPU, etc. Each plugin implements the unified API and provides additional hardware-specific APIs. - -The scheme below illustrates the typical workflow for deploying a trained deep learning model: - -![](img/BASIC_FLOW_IE_C.svg) - - -## Video - -@sphinxdirective - -.. list-table:: - - * - .. raw:: html - - - * - **Inference Engine Concept**. Duration: 3:43 - -@endsphinxdirective diff --git a/docs/OV_Runtime_UG/PropertiesAPI.md b/docs/OV_Runtime_UG/PropertiesAPI.md deleted file mode 100644 index 435e0e3cd88..00000000000 --- a/docs/OV_Runtime_UG/PropertiesAPI.md +++ /dev/null @@ -1,243 +0,0 @@ -# Introduction to OpenVINO™ Device Properties API {#openvino_docs_IE_DG_InferenceEngine_QueryAPI} - -## OpenVINO™ Properties API (C++) - -@sphinxdirective -.. raw:: html - -
C++
-@endsphinxdirective - -The OpenVINO™ toolkit supports inferencing with several types of devices (processors or accelerators). -This section provides a high-level description of the process of querying of different device properties and configuration values at runtime. Refer to the [Hello Query Device С++ Sample](../../samples/cpp/hello_query_device/README.md) sources and the [Multi-Device Plugin documentation](supported_plugins/MULTI.md) for examples of using the OpenVINO™ Properties API in user applications. - -### Using the OpenVINO™ Properties API in Your Code - -The `ov::Core` class provides the following API to query device information, set or get different device configuration properties: - -* `ov::Core::get_available_devices` - Provides a list of available devices. If there are more than one instance of a specific device, the devices are enumerated with `.suffix` where `suffix` is a unique string identifier. The device name can be passed to all methods of the `ov::Core` class that work with devices, for example `ov::Core::compile_model`. -* `ov::Core::get_property` - Gets the current value of a specific property. -* `ov::Core::set_property` - Sets a new value for the property. - -The `ov::CompiledModel` class is also extended to support the Properties API: - -* `ov::CompiledModel::get_property` -* `ov::CompiledModel::set)property` - -### Properties API in the Core Class - -#### get_available_devices - -@snippet snippets/ov_properties_api.cpp part0 - -The function returns a list of available devices, for example: - -``` -MYRIAD.1.2-ma2480 -MYRIAD.1.4-ma2480 -CPU -GPU.0 -GPU.1 -``` - -Each device name can then be passed to: - -* `ov::Core::compile_model` to load the network to a specific device. -* `ov::Core::get_property` to get common or device specific properties. -* All other methods of the `ov::Core` class that accept `deviceName`. - -#### ov::Core methods - -`ov::Core` methods like: - -* `ov::Core::compile_model` -* `ov::Core::import_model` -* `ov::Core::query_model` -* `ov::Core::query_model` -* `ov::Core::create_context` - -accept variadic list of properties as last arguments. Each property in such parameters lists should be used as function call to pass property value with specified property type - -@snippet snippets/ov_properties_api.cpp part3 - -#### get_property() - -For documentation about common configuration keys, refer to `openvino/runtime/properties.hpp`. Device specific configuration keys can be found in corresponding plugin folders. - -* The code below demonstrates how to query `HETERO` device priority of devices which will be used to infer the model: - -@snippet snippets/ov_properties_api.cpp part1 - -* To extract device properties such as available device, device name, supported configuration keys, and others, use the `ov::Core::get_property` method: - -@snippet snippets/ov_properties_api.cpp part2 - -A returned value appears as follows: `Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz`. - -> **NOTE**: All properties have a type, which is specified during property declaration. The list of common device-agnostic properties can be found in `openvino/runtime/properties.hpp`. Device specific properties (for example, for HDDL or MYRIAD devices) can be found in corresponding plugin folders. - -### Properties API in the CompiledModel Class - -#### get_property() - -The method is used to get configuration values the compiled model has been created with or compiled model specific property such as `ov::optimal_number_of_infer_requests`: - -@snippet snippets/ov_properties_api.cpp part4 - -Or the current temperature of the `MYRIAD` device: - -@snippet snippets/ov_properties_api.cpp part5 - -Or the number of threads that would be used for inference in `CPU` device: - -@snippet snippets/ov_properties_api.cpp part6 - -#### set_property() - -The only device that supports this method is [Multi-Device](supported_plugins/MULTI.md). - -## OpenVINO™ Properties API (Python) - -@sphinxdirective -.. raw:: html - -
Python
-@endsphinxdirective - -This section provides a high-level description of the process of querying of different device properties and configuration values. Refer to the [Hello Properties Device Python Sample](../../samples/python/hello_query_device/README.md) sources and the [Multi-Device Plugin documentation](supported_plugins/MULTI.md) for examples of using the OpenVINO™ Properties API in user applications. - -### Using the OpenVINO™ Properties API in Your Code - -The OpenVINO™ [Core](api/ie_python_api/_autosummary/openvino.inference_engine.IECore.html#openvino-inference-engine-iecore) class provides the following API to query device information, set or get different device configuration properties: - -* [ie_api.IECore.available_devices](api/ie_python_api/_autosummary/openvino.inference_engine.IECore.html#openvino.inference_engine.IECore.available_devices) - Provides a list of available devices. If there are more than one instance of a specific device, the devices are enumerated with .suffix where suffix is a unique string identifier. The device name can be passed to all methods of the IECore class that work with devices, for example [ie_api.IECore.load_network](api/ie_python_api/_autosummary/openvino.inference_engine.IECore.html#openvino.inference_engine.IECore.load_network). -* [ie_api.ieCore.get_property](api/ie_python_api/_autosummary/openvino.inference_engine.IECore.html#openvino.inference_engine.IECore.get_property) - Provides information about specific device. -* [ie_api.IECore.get_config](api/ie_python_api/_autosummary/openvino.inference_engine.IECore.html#openvino.inference_engine.IECore.get_config) - Gets the current value of a specific configuration key. -* [ie_api.IECore.set_config](api/ie_python_api/_autosummary/openvino.inference_engine.IECore.html#openvino.inference_engine.IECore.set_config) - Sets a new value for the configuration key. - -The [ie_api.CompiledModel](api/ie_python_api/_autosummary/openvino.inference_engine.CompiledModel.html) class is also extended to support the Properties API: -* [ie_api.CompiledModel.get_property](api/ie_python_api/_autosummary/openvino.inference_engine.CompiledModel.html#openvino.inference_engine.CompiledModel.get_property) -* [ie_api.CompiledModel.get_config](latest/api/ie_python_api/_autosummary/openvino.inference_engine.CompiledModel.html#openvino.inference_engine.CompiledModel.get_config) -* There is no method to call for set_config, but the equivalent action is described below. - -### Properties API in the IECore Class - -#### Get Available Devices - -```python -from openvino.inference_engine import IECore - -ie = IECore() -print(ie.available_devices) -``` - -This code prints a list of available devices, for example: - -``` -MYRIAD.1.2-ma2480 -MYRIAD.1.4-ma2480 -FPGA.0 -FPGA.1 -CPU -GPU.0 -GPU.1 -``` - -Each device name can then be passed to: - -* `IECore.load_network` to load the network to a specific device. -* `IECore.get_property` to get common or device specific properties. -* All other methods of the `IECore` class that accept a device name. - -#### Get Metric - -To extract device properties such as available device, device name, supported configuration keys, and others, use the [IECore.get_property](api/ie_python_api/_autosummary/openvino.inference_engine.IECore.html#openvino.inference_engine.IECore.get_property) method: - -```python -from openvino.inference_engine import IECore - -ie = IECore() -ie.get_property(device_name="CPU", property_name="FULL_DEVICE_NAME") -``` - -A returned value appears as follows: `Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz`. - -To list all supported properties for a device: - -```python -from openvino.inference_engine import IECore - -ie = IECore() -ie.get_property(device_name="GPU", property_name="SUPPORTED_METRICS") -``` - -#### Get Configuration - -The code below uses the [IECore.get_config](api/ie_python_api/_autosummary/openvino.inference_engine.IECore.html#openvino.inference_engine.IECore.get_config) method and demonstrates how to understand whether the HETERO device dumps .dot files with split graphs during the split stage: - -```python -from openvino.inference_engine import IECore - -ie = IECore() -ie.get_config(device_name="HETERO", config_name="HETERO_DUMP_GRAPH_DOT") -``` - -To list all supported configuration keys for a device: - -```python -from openvino.inference_engine import IECore - -ie = IECore() -ie.get_property(device_name=device, property_name="SUPPORTED_CONFIG_KEYS") -``` - -For documentation about common configuration keys, refer to `ie_plugin_config.hpp`. Device specific configuration keys can be found in corresponding plugin folders. - - -### Properties API in the CompiledModel Class - -#### Get Metric - -To get the name of the loaded network: - -```python -from openvino.inference_engine import IECore - -ie = IECore() -net = ie.read_network(model=path_to_xml_file) -exec_net = ie.load_network(network=net, device_name=device) -exec_net.get_property("NETWORK_NAME") -``` - -Use `exec_net.get_property("SUPPORTED_METRICS")` to list all supported properties for an CompiledModel instance. - - -#### Get Configuration - -The [IECore.get_config](api/ie_python_api/_autosummary/openvino.inference_engine.IECore.html#openvino.inference_engine.IECore.get_config) method is used to get information about configuration values the compiled model has been created with: - -```python -from openvino.inference_engine import IECore - -ie = IECore() -net = ie.read_network(model=path_to_xml_file) -exec_net = ie.load_network(network=net, device_name="CPU") -exec_net.get_config("CPU_THREADS_NUM") -``` - -Or the current temperature of MYRIAD device: - -```python -from openvino.inference_engine import IECore - -ie = IECore() -net = ie.read_network(model=path_to_xml_file) -exec_net = ie.load_network(network=net, device_name="MYRIAD") -exec_net.get_config("DEVICE_THERMAL") -``` - -Use `exec_net.get_property("SUPPORTED_CONFIG_KEYS")` to list all supported configuration keys. - -#### Set Configuration - -The only device that supports this method in the CompiledModel class is the [Multi-Device](supported_plugins/MULTI.md), where you can change the priorities of the devices for the Multi plugin in real time: `exec_net.set_config({{"MULTI_DEVICE_PRIORITIES", "GPU,CPU"}})`. See the Multi-Device documentation for more details. \ No newline at end of file diff --git a/docs/OV_Runtime_UG/PythonPackage_Overview.md b/docs/OV_Runtime_UG/PythonPackage_Overview.md new file mode 100644 index 00000000000..5e03eb3295c --- /dev/null +++ b/docs/OV_Runtime_UG/PythonPackage_Overview.md @@ -0,0 +1,14 @@ +# OpenVINO™ Python* Package + +OpenVINO™ Python\* package includes types to measure model and calibrate to low precision. + +The OpenVINO™ Python\* package available in the `/python/python3.X` directory. + +The OpenVINO™ Python\* package includes the following sub-packages: + + - [openvino.inference_engine](../../src/bindings/python/docs/api_overview.md) - Python\* wrapper on OpenVINO™ Inference Engine. + - `openvino.tools.accuracy_checker` - Measure accuracy. + - `openvino.tools.benchmark` - Measure latency and throughput. + +## See Also +* [Integrate with Customer Application New API](integrate_with_your_application.md) diff --git a/docs/OV_Runtime_UG/Samples_Overview.md b/docs/OV_Runtime_UG/Samples_Overview.md index 2e43e342a26..ddb7a476aca 100644 --- a/docs/OV_Runtime_UG/Samples_Overview.md +++ b/docs/OV_Runtime_UG/Samples_Overview.md @@ -1,4 +1,4 @@ -# Inference Engine Samples {#openvino_docs_IE_DG_Samples_Overview} +# OpenVINO Samples {#openvino_docs_IE_DG_Samples_Overview} @sphinxdirective @@ -19,8 +19,8 @@ openvino_inference_engine_ie_bridges_c_samples_hello_nv12_input_classification_README openvino_inference_engine_samples_hello_query_device_README openvino_inference_engine_ie_bridges_python_sample_hello_query_device_README - openvino_inference_engine_samples_ngraph_function_creation_sample_README - openvino_inference_engine_ie_bridges_python_sample_ngraph_function_creation_sample_README + openvino_inference_engine_samples_model_creation_sample_README + openvino_inference_engine_ie_bridges_python_sample_model_creation_sample_README openvino_inference_engine_samples_speech_sample_README openvino_inference_engine_ie_bridges_python_sample_speech_sample_README openvino_inference_engine_samples_benchmark_app_README @@ -28,14 +28,14 @@ @endsphinxdirective -The Inference Engine sample applications are simple console applications that show how to utilize specific Inference Engine capabilities within an application, assist developers in executing specific tasks such as loading a model, running inference, querying specific device capabilities and etc. +The OpenVINO sample applications are simple console applications that show how to utilize specific OpenVINO API capabilities within an application, assist developers in executing specific tasks such as loading a model, running inference, querying specific device capabilities and etc. After installation of Intel® Distribution of OpenVINO™ toolkit, С, C++ and Python* sample applications are available in the following directories, respectively: * `/samples/c` * `/samples/cpp` * `/samples/python` -Inference Engine sample applications include the following: +OpenVINO sample applications include the following: - **Speech Sample** - Acoustic model inference based on Kaldi neural networks and speech feature vectors. - [Automatic Speech Recognition C++ Sample](../../samples/cpp/speech_sample/README.md) @@ -50,7 +50,7 @@ Inference Engine sample applications include the following: - **Hello NV12 Input Classification Sample** – Input of any size and layout can be provided to an infer request. The sample transforms the input to the NV12 color format and pre-process it automatically during inference. The sample supports only images as inputs. - [Hello NV12 Input Classification C++ Sample](../../samples/cpp/hello_nv12_input_classification/README.md) - [Hello NV12 Input Classification C Sample](../../samples/c/hello_nv12_input_classification/README.md) -- **Hello Query Device Sample** – Query of available Inference Engine devices and their metrics, configuration values. +- **Hello Query Device Sample** – Query of available OpenVINO devices and their metrics, configuration values. - [Hello Query Device C++ Sample](../../samples/cpp/hello_query_device/README.md) - [Hello Query Device Python* Sample](../../samples/python/hello_query_device/README.md) - **Hello Reshape SSD Sample** – Inference of SSD networks resized by ShapeInfer API according to an input size. @@ -59,10 +59,10 @@ Inference Engine sample applications include the following: - **Image Classification Sample Async** – Inference of image classification networks like AlexNet and GoogLeNet using Asynchronous Inference Request API (the sample supports only images as inputs). - [Image Classification Async C++ Sample](../../samples/cpp/classification_sample_async/README.md) - [Image Classification Async Python* Sample](../../samples/python/classification_sample_async/README.md) -- **nGraph Function Creation Sample** – Construction of the LeNet network using the nGraph function creation sample. - - [nGraph Function Creation C++ Sample](../../samples/cpp/ngraph_function_creation_sample/README.md) - - [nGraph Function Creation Python Sample](../../samples/python/ngraph_function_creation_sample/README.md) - +- **OpenVINO Model Creation Sample** – Construction of the LeNet model using the OpenVINO model creation sample. + - [OpenVINO Model Creation C++ Sample](../../samples/cpp/model_creation_sample/README.md) + - [OpenVINO Model Creation Python Sample](../../samples/python/model_creation_sample/README.md) + > **NOTE**: All C++ samples support input paths containing only ASCII characters, except the Hello Classification Sample, that supports Unicode. ## Media Files Available for Samples @@ -79,8 +79,8 @@ To run the sample, you can use [public](@ref omz_models_group_public) or [Intel' The officially supported Linux* build environment is the following: -* Ubuntu* 18.04 LTS 64-bit or CentOS* 7 64-bit -* GCC* 7.5.0 (for Ubuntu* 18.04) or GCC* 4.8.5 (for CentOS* 7.6) +* Ubuntu* 18.04 LTS 64-bit or Ubuntu* 20.04 LTS 64-bit +* GCC* 7.5.0 (for Ubuntu* 18.04) or GCC* 9.3.0 (for Ubuntu* 20.04) * CMake* version 3.10 or higher > **NOTE**: For building samples from the open-source version of OpenVINO™ toolkit, see the [build instructions on GitHub](https://github.com/openvinotoolkit/openvino/wiki/BuildingCode). @@ -102,7 +102,7 @@ You can also build the sample applications manually: ```sh mkdir build ``` -> **NOTE**: If you ran the Image Classification verification script during the installation, the C++ samples build directory was already created in your home directory: `~/inference_engine_samples_build/` +> **NOTE**: If you ran the Image Classification verification script during the installation, the C++ samples build directory was already created in your home directory: `~/inference_engine_cpp_samples_build/` 2. Go to the created directory: ```sh @@ -130,22 +130,17 @@ for the debug configuration — in `/intel64/Debug/`. The recommended Windows* build environment is the following: * Microsoft Windows* 10 -* Microsoft Visual Studio* 2017, or 2019 +* Microsoft Visual Studio* 2019 * CMake* version 3.10 or higher -> **NOTE**: If you want to use Microsoft Visual Studio 2019, you are required to install CMake 3.14. +> **NOTE**: If you want to use Microsoft Visual Studio 2019, you are required to install CMake 3.14 or higher. To build the C or C++ sample applications on Windows, go to the `\samples\c` or `\samples\cpp` directory, respectively, and run the `build_samples_msvc.bat` batch file: ```sh build_samples_msvc.bat ``` -By default, the script automatically detects the highest Microsoft Visual Studio version installed on the machine and uses it to create and build -a solution for a sample code. Optionally, you can also specify the preferred Microsoft Visual Studio version to be used by the script. Supported -versions are `VS2017` and `VS2019`. For example, to build the C++ samples using the Microsoft Visual Studio 2017, use the following command: -```sh -\samples\cpp\build_samples_msvc.bat VS2017 -``` +By default, the script automatically detects the highest Microsoft Visual Studio version installed on the machine and uses it to create and build a solution for a sample code Once the build is completed, you can find sample binaries in the following folders: * C samples: `C:\Users\\Documents\Intel\OpenVINO\inference_engine_c_samples_build\intel64\Release` @@ -159,7 +154,7 @@ directory. The officially supported macOS* build environment is the following: -* macOS* 10.15 64-bit +* macOS* 10.15 64-bit or higher * Clang* compiler from Xcode* 10.1 or higher * CMake* version 3.13 or higher @@ -180,7 +175,7 @@ You can also build the sample applications manually: > **NOTE**: Before proceeding, make sure you have OpenVINO™ environment set correctly. This can be done manually by ```sh -cd /bin +cd / source setupvars.sh ``` @@ -188,7 +183,7 @@ source setupvars.sh ```sh mkdir build ``` -> **NOTE**: If you ran the Image Classification verification script during the installation, the C++ samples build directory was already created in your home directory: `~/inference_engine_samples_build/` +> **NOTE**: If you ran the Image Classification verification script during the installation, the C++ samples build directory was already created in your home directory: `~/inference_engine_cpp_samples_build/` 2. Go to the created directory: ```sh @@ -217,7 +212,7 @@ for the debug configuration — in `/intel64/Debug/`. ### Get Ready for Running the Sample Applications on Linux* Before running compiled binary files, make sure your application can find the -Inference Engine and OpenCV libraries. +OpenVINO Runtime libraries. Run the `setupvars` script to set all necessary environment variables: ```sh source /setupvars.sh @@ -246,7 +241,7 @@ list above. ### Get Ready for Running the Sample Applications on Windows* Before running compiled binary files, make sure your application can find the -Inference Engine and OpenCV libraries. +OpenVINO Runtime libraries. Use the `setupvars` script, which sets all necessary environment variables: ```sh \setupvars.bat @@ -255,13 +250,13 @@ Use the `setupvars` script, which sets all necessary environment variables: To debug or run the samples on Windows in Microsoft Visual Studio, make sure you have properly configured **Debugging** environment settings for the **Debug** and **Release** configurations. Set correct paths to the OpenCV libraries, and -debug and release versions of the Inference Engine libraries. +debug and release versions of the OpenVINO Runtime libraries. For example, for the **Debug** configuration, go to the project's **Configuration Properties** to the **Debugging** category and set the `PATH` variable in the **Environment** field to the following: ```sh -PATH=\runtime\bin;\opencv\bin;%PATH% +PATH=\runtime\bin;%PATH% ``` where `` is the directory in which the OpenVINO toolkit is installed. @@ -270,4 +265,4 @@ sample, read the sample documentation by clicking the sample name in the samples list above. ## See Also -* [OpenVINO™ Runtime User Guide](OpenVINO_Runtime_User_Guide.md) +* [OpenVINO™ Runtime User Guide](openvino_intro.md) diff --git a/docs/OV_Runtime_UG/ShapeInference.md b/docs/OV_Runtime_UG/ShapeInference.md index 43a48ce5e89..4cb274ad827 100644 --- a/docs/OV_Runtime_UG/ShapeInference.md +++ b/docs/OV_Runtime_UG/ShapeInference.md @@ -1,4 +1,4 @@ -# Using the Reshape Inference Feature {#openvino_docs_IE_DG_ShapeInference} +# Changing input shapes {#openvino_docs_IE_DG_ShapeInference} ## Introduction (C++) @@ -112,7 +112,7 @@ To keep the model valid after the reshape, choose a new input shape that satisfi For details, refer to the Tensorflow Object Detection API models resizing techniques. ### Extensibility -The Inference Engine provides a special mechanism that allows adding support of shape inference for custom operations. This mechanism is described in the [Extensibility documentation](Extensibility_DG/Intro.md) +The Inference Engine provides a special mechanism that allows adding support of shape inference for custom operations. This mechanism is described in the [Extensibility documentation](../Extensibility_UG/Intro.md) ## Introduction (Python) @@ -166,7 +166,7 @@ To feed input data of a shape that is different from the model input shape, resh Once the input shape of IENetwork is set, call the `IECore.load_network` method to get an ExecutableNetwork object for inference with updated shapes. -There are other approaches to reshape the model during the stage of IR generation or [nGraph function](https://docs.openvino.ai/latest/openvino_docs_nGraph_DG_PythonAPI.html#create_an_ngraph_function_from_a_graph) creation. +There are other approaches to reshape the model during the stage of IR generation or [OpenVINO model](https://docs.openvino.ai/latest/openvino_docs_nGraph_DG_PythonAPI.html#create_an_ngraph_function_from_a_graph) creation. Practically, some models are not ready to be reshaped. In this case, a new input shape cannot be set with the Model Optimizer or the `IENetwork.reshape` method. @@ -218,7 +218,7 @@ exec_net = ie.load_network(network=net, device_name="CPU") ``` ### Extensibility -The Inference Engine provides a special mechanism that allows adding support of shape inference for custom operations. This mechanism is described in the [Extensibility documentation](Extensibility_DG/Intro.md) +The Inference Engine provides a special mechanism that allows adding support of shape inference for custom operations. This mechanism is described in the [Extensibility documentation](../Extensibility_UG/Intro.md) ### See Also: diff --git a/docs/OV_Runtime_UG/supported_plugins/AUTO.md b/docs/OV_Runtime_UG/auto_device_selection.md similarity index 95% rename from docs/OV_Runtime_UG/supported_plugins/AUTO.md rename to docs/OV_Runtime_UG/auto_device_selection.md index 4d692abb3ab..e127f7e31b8 100644 --- a/docs/OV_Runtime_UG/supported_plugins/AUTO.md +++ b/docs/OV_Runtime_UG/auto_device_selection.md @@ -1,4 +1,4 @@ -# Auto-Device Plugin {#openvino_docs_IE_DG_supported_plugins_AUTO} +# Automatic device selection {#openvino_docs_IE_DG_supported_plugins_AUTO} ## Auto-Device Plugin Execution (C++) @@ -39,7 +39,7 @@ There are two ways to use Auto-device: Both methods allow limiting the list of device candidates for the AUTO plugin. -> **NOTE**: The Inference Engine lets you use "GPU" as an alias for "GPU.0" in function calls. +> **NOTE**: The OpenVINO Runtime lets you use "GPU" as an alias for "GPU.0" in function calls. The Auto-device plugin supports query device optimization capabilities in metric. @@ -49,8 +49,8 @@ The Auto-device plugin supports query device optimization capabilities in metric ### Enumerating Devices and Selection Logic -The Inference Engine now features a dedicated API to enumerate devices and their capabilities. -See [Hello Query Device C++ Sample](../../../samples/cpp/hello_query_device/README.md). +The OpenVINO Runtime API now features a dedicated methods to enumerate devices and their capabilities. +See [Hello Query Device C++ Sample](../../samples/cpp/hello_query_device/README.md). This is the example output from the sample (truncated to device names only): ```sh @@ -85,7 +85,7 @@ For example, CPU, dGPU and iGPU can support the following precision and optimiza In cases when loading the network to dGPU or iGPU fails, CPU is the fall-back choice. -According to the Auto-device selection logic from the previous section, tell the Inference Engine +According to the Auto-device selection logic from the previous section, tell the OpenVINO Runtime to use the most suitable device from available devices as follows: @snippet snippets/AUTO2.cpp part2 @@ -208,7 +208,7 @@ The Auto-device plugin supports query device optimization capabilities in metric ### Enumerating Devices and Selection Logic -The Inference Engine now features a dedicated API to enumerate devices and their capabilities. See the [Hello Query Device Python Sample](../../../inference_engine/ie_bridges/python/sample_hello_query_device_README.html) for code. +The OpenVINO Runtime API now features a dedicated methods to enumerate devices and their capabilities. See the [Hello Query Device Python Sample](../../samples/python/hello_query_device/README.md) for code. This is the example output from the sample (truncated to device names only): diff --git a/docs/OV_Runtime_UG/automatic_batching.md b/docs/OV_Runtime_UG/automatic_batching.md new file mode 100644 index 00000000000..eb64330198e --- /dev/null +++ b/docs/OV_Runtime_UG/automatic_batching.md @@ -0,0 +1,107 @@ +# Automatic Batching {#openvino_docs_OV_UG_Automatic_Batching} + +## (Automatic) Batching Execution + +The Automatic-Batching is a preview of the new functionality in the OpenVINO™ toolkit. It performs on-the-fly automatic batching (i.e. grouping inference requests together) to improve device utilization, with no programming effort from the user. +Inputs gathering and outputs scattering from the individual inference requests required for the batch happen transparently, without affecting the application code. + +The feature primarily targets existing code written for inferencing many requests (each instance with the batch size 1). To obtain corresponding performance improvements, the application must be *running many inference requests simultaneously*. +As explained below, the auto-batching functionality can be also used via a special *virtual* device. + +Batching is a straightforward way of leveraging the GPU compute power and saving on communication overheads. The automatic batching is _implicitly_ triggered on the GPU when the `ov::hint::PerformanceMode::THROUGHPUT` is specified for the `ov::hint::performance_mode` property for the compile_model or set_property calls. +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_auto_batching.cpp + :language: cpp + :fragment: [compile_model] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_auto_batching.py + :language: python + :fragment: [compile_model] + +@endsphinxdirective +> **NOTE**: You can disable the Auto-Batching (for example, for the GPU device) from being triggered by the `ov::hint::PerformanceMode::THROUGHPUT`. To do that, pass the `ov::hint::allow_auto_batching` set to **false** in addition to the `ov::hint::performance_mode`: +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_auto_batching.cpp + :language: cpp + :fragment: [compile_model_no_auto_batching] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_auto_batching.py + :language: python + :fragment: [compile_model_no_auto_batching] + +@endsphinxdirective + + +Alternatively, to enable the Auto-Batching in the legacy apps not akin to the notion of the performance hints, you may need to use the **explicit** device notion, such as 'BATCH:GPU'. In both cases (the *throughput* hint or explicit BATCH device), the optimal batch size selection happens automatically. The actual value depends on the model and device specifics, for example, on-device memory for the dGPUs. + +This _automatic batch size selection_ assumes that the application queries the `ov::optimal_number_of_infer_requests` to create and run the returned number of requests simultaneously: +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_auto_batching.cpp + :language: cpp + :fragment: [query_optimal_num_requests] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_auto_batching.py + :language: python + :fragment: [query_optimal_num_requests] + +@endsphinxdirective +If not enough inputs were collected, the `timeout` value makes the transparent execution fall back to the execution of individual requests. Configuration-wise, this is the AUTO_BATCH_TIMEOUT property. +The timeout, which adds itself to the execution time of the requests, heavily penalizes the performance. To avoid this, in cases when your parallel slack is bounded, give the OpenVINO an additional hint. + +For example, the application processes only 4 video streams, so there is no need to use a batch larger than 4. The most future-proof way to communicate the limitations on the parallelism is to equip the performance hint with the optional `ov::hint::num_requests` configuration key set to 4. For the GPU this will limit the batch size, for the CPU - the number of inference streams, so each device uses the `ov::hint::num_requests` while converting the hint to the actual device configuration options: +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_auto_batching.cpp + :language: cpp + :fragment: [hint_num_requests] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_auto_batching.py + :language: python + :fragment: hint_num_requests] + +@endsphinxdirective + +For the *explicit* usage, you can limit the batch size using "BATCH:GPU(4)", where 4 is the number of requests running in parallel. + +### Other Performance Considerations + +To achieve the best performance with the Automatic Batching, the application should: + - Operate the number of inference requests that represents the multiple of the batch size. In the above example, for batch size 4, the application should operate 4, 8, 12, 16, etc. requests. + - Use the requests, grouped by the batch size, together. For example, the first 4 requests are inferred, while the second group of the requests is being populated. + +The following are limitations of the current implementations: + - Although less critical for the throughput-oriented scenarios, the load-time with auto-batching increases by almost 2x. + - Certain networks are not reshape-able by the "batching" dimension (specified as 'N' in the layouts terms) or if the dimension is not zero-th, the auto-batching is not triggered. + - Performance improvements happen at the cost of the memory footprint growth, yet the auto-batching queries the available memory (especially for the dGPUs) and limits the selected batch size accordingly. + + + +### Configuring the Automatic Batching +Following the OpenVINO convention for devices names, the *batching* device is named *BATCH*. The configuration options are as follows: + +| Parameter name | Parameter description | Default | Examples | +| :--- | :--- | :--- |:-----------------------------------------------------------------------------| +| "AUTO_BATCH_DEVICE" | Device name to apply the automatic batching and optional batch size in brackets | N/A | BATCH:GPU which triggers the automatic batch size selection or explicit batch size BATCH:GPU(4) | +| "AUTO_BATCH_TIMEOUT" | timeout value, in ms | 1000 | you can reduce the timeout value (to avoid performance penalty when the data arrives too non-evenly) e.g. pass the "100", or in contrast make it large enough e.g. to accommodate inputs preparation (e.g. when it is serial process) | + +### See Also +[Supported Devices](supported_plugins/Supported_Devices.md) \ No newline at end of file diff --git a/docs/OV_Runtime_UG/supported_plugins/HETERO.md b/docs/OV_Runtime_UG/hetero_execution.md similarity index 99% rename from docs/OV_Runtime_UG/supported_plugins/HETERO.md rename to docs/OV_Runtime_UG/hetero_execution.md index 04823782c55..0d90971a37d 100644 --- a/docs/OV_Runtime_UG/supported_plugins/HETERO.md +++ b/docs/OV_Runtime_UG/hetero_execution.md @@ -154,4 +154,4 @@ where: You can point more than two devices: `-d HETERO:MYRIAD,GPU,CPU` ### See Also -[Supported Devices](Supported_Devices.md) +[Supported Devices](supported_plugins/Supported_Devices.md) diff --git a/docs/OV_Runtime_UG/integrate_with_your_application.md b/docs/OV_Runtime_UG/integrate_with_your_application.md new file mode 100644 index 00000000000..6472e9ec8c5 --- /dev/null +++ b/docs/OV_Runtime_UG/integrate_with_your_application.md @@ -0,0 +1,265 @@ +# Integrate OpenVINO™ with Your Application {#openvino_docs_Integrate_OV_with_your_application} + +@sphinxdirective + +.. toctree:: + :maxdepth: 1 + :hidden: + + openvino_docs_OV_Runtime_UG_Model_Representation + openvino_docs_OV_Runtime_UG_Infer_request + +@endsphinxdirective + +> **NOTE**: Before start using OpenVINO™ Runtime, make sure you set all environment variables during the installation. If you did not, follow the instructions from the _Set the Environment Variables_ section in the installation guides: +> * [For Windows* 10](../install_guides/installing-openvino-windows.md) +> * [For Linux*](../install_guides/installing-openvino-linux.md) +> * [For macOS*](../install_guides/installing-openvino-macos.md) +> * To build an open source version, use the [OpenVINO™ Runtime Build Instructions](https://github.com/openvinotoolkit/openvino/wiki/BuildingCode). + +## Use OpenVINO™ Runtime API to Implement Inference Pipeline + +This section provides step-by-step instructions to implement a typical inference pipeline with the OpenVINO™ Runtime C++ API: + +![ie_api_use_cpp] + +### Step 1. Create OpenVINO™ Runtime Core + +Include next files to work with OpenVINO™ Runtime: + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [include] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [import] + +@endsphinxdirective + +Use the following code to create OpenVINO™ Core to manage available devices and read model objects: + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part1] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part1] + +@endsphinxdirective + +### Step 2. Compile the Model + +`ov::CompiledModel` class represents a device specific compiled model. `ov::CompiledModel` allows you to get information inputs or output ports by a tensor name or index. + +Compile the model for a specific device using `ov::Core::compile_model()`: + +@sphinxdirective + +.. tab:: C++ + + .. tab:: IR + + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part2_1] + + .. tab:: ONNX + + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part2_2] + + .. tab:: PaddlePaddle + + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part2_3] + + .. tab:: ov::Model + + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part2_4] + +.. tab:: Python + + .. tab:: IR + + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part2_1] + + .. tab:: ONNX + + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part2_2] + + .. tab:: PaddlePaddle + + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part2_3] + + .. tab:: ov::Model + + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part2_4] + +@endsphinxdirective + +The `ov::Model` object represents any models inside the OpenVINO™ Runtime. +For more details please read article about [OpenVINO™ Model representation](model_representation.md). + +The code above creates a compiled model associated with a single hardware device from the model object. +It is possible to create as many compiled models as needed and use them simultaneously (up to the limitation of the hardware resources). +To learn how to change the device configuration, read the [Query device properties](./supported_plugins/config_properties.md) article. + +### Step 3. Create an Inference Request + +`ov::InferRequest` class provides methods for model inference in the OpenVINO™ Runtime. +This section demonstrates a simple pipeline, to get more information about other use cases, read the [InferRequest documentation](./ov_infer_request.md) dedicated article. +Create an infer request using the following code: + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part3] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part3] + +@endsphinxdirective + +### Step 4. Set Inputs + +You can use external memory to create `ov::Tensor` and use the `ov::InferRequest::set_input_tensor` method to put this tensor on the device: + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part4] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part4] + +@endsphinxdirective + +### Step 5. Start Inference + +OpenVINO™ Runtime supports inference in asynchronous or synchronous mode. Async API usage can improve overall frame-rate of the application, because rather than wait for inference to complete, the app can continue doing things on the host, while the accelerator is busy. You can use `ov::InferRequest::start_async()` to start model inference in the asynchronous mode and call `ov::InferRequest::wait()` to wait for the inference results: + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part5] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part5] + +@endsphinxdirective + +The asynchronous mode supports two methods to get the inference results: + * `ov::InferRequest::wait_for()` - Waits until the specified timeout (in milliseconds) has elapsed or the inference result becomes available, whichever comes first. + * `ov::InferRequest::wait()` - Waits until the inference result becomes available. + +Both requests are thread-safe, which means they can be called from different threads without exposing erroneous behavior or producing unpredictable results. + +While the request is ongoing, all its methods except `ov::InferRequest::cancel`, `ov::InferRequest::wait` or `ov::InferRequest::wait_for` throw +the `ov::Busy` exception indicating the request is busy with computations. + +### Step 6. Process the Inference Results + +Go over the output tensors and process the inference results. + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/src/main.cpp + :language: cpp + :fragment: [part6] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/src/main.py + :language: python + :fragment: [part6] + +@endsphinxdirective + +## Link and Build Your C++ Application with OpenVINO™ Runtime + +The example uses CMake for project configuration. + +1. **Create a structure** for the project: + ``` sh + project/ + ├── CMakeLists.txt - CMake file to build + ├── ... - Additional folders like includes/ + └── src/ - source folder + └── main.cpp + build/ - build directory + ... + ``` + +2. **Include OpenVINO™ Runtime libraries** in `project/CMakeLists.txt` + + @snippet snippets/CMakeLists.txt cmake:integration_example + +To build your project using CMake with the default build tools currently available on your machine, execute the following commands: + +> **NOTE**: Make sure you set environment variables first by running `/setupvars.sh` (or `setupvars.bat` for Windows). Otherwise the `OpenVINO_DIR` variable won't be configured properly to pass `find_package` calls. + +```sh +cd build/ +cmake ../project +cmake --build . +``` +It's allowed to specify additional build options (e.g. to build CMake project on Windows with a specific build tools). Please refer to the [CMake page](https://cmake.org/cmake/help/latest/manual/cmake.1.html#manual:cmake(1)) for details. + +## Run Your Application + +Congratulations, you have made your first application with OpenVINO™ toolkit, now you may run it. + +## See also + + - [OpenVINO™ Runtime Preprocessing](./preprocessing_overview.md) + +[ie_api_flow_cpp]: img/BASIC_IE_API_workflow_Cpp.svg +[ie_api_use_cpp]: img/IMPLEMENT_PIPELINE_with_API_C.svg +[ie_api_flow_python]: img/BASIC_IE_API_workflow_Python.svg +[ie_api_use_python]: img/IMPLEMENT_PIPELINE_with_API_Python.svg diff --git a/docs/OV_Runtime_UG/migration_ov_2_0/common_inference_pipeline.md b/docs/OV_Runtime_UG/migration_ov_2_0/common_inference_pipeline.md new file mode 100644 index 00000000000..72970192bda --- /dev/null +++ b/docs/OV_Runtime_UG/migration_ov_2_0/common_inference_pipeline.md @@ -0,0 +1,200 @@ +# Inference Pipeline {#openvino_2_0_inference_pipeline} + +Usually to inference model with the OpenVINO™ Runtime an user needs to do the following steps in the application pipeline: +- 1. Create Core object +- 2. Read model from the disk + - 2.1. (Optional) Model preprocessing +- 3. Load the model to the device +- 4. Create an inference request +- 5. Fill input tensors with data +- 6. Start inference +- 7. Process the inference results + +Code snippets below cover these steps and show how application code should be changed for migration to OpenVINO™ Runtime 2.0. + +## 1. Create Core + +Inference Engine API: + +@snippet docs/snippets/ie_common.cpp ie:create_core + +OpenVINO™ Runtime API 2.0: + +@snippet docs/snippets/ov_common.cpp ov_api_2_0:create_core + +## 2. Read model from the disk + +Inference Engine API: + +@snippet docs/snippets/ie_common.cpp ie:read_model + +OpenVINO™ Runtime API 2.0: + +@snippet docs/snippets/ov_common.cpp ov_api_2_0:read_model + +Read model has the same structure as in the example from [Model Creation](./graph_construction.md) migration guide. + +Note, you can combine read and compile model stages into a single call `ov::Core::compile_model(filename, devicename)`. + +### 2.1 (Optional) Model preprocessing + +When application's input data doesn't perfectly match with model's input format, preprocessing steps may need to be added. +See detailed guide [how to migrate preprocessing in OpenVINO Runtime API 2.0](./preprocessing.md) + +## 3. Load the Model to the Device + +Inference Engine API: + +@snippet docs/snippets/ie_common.cpp ie:compile_model + +OpenVINO™ Runtime API 2.0: + +@snippet docs/snippets/ov_common.cpp ov_api_2_0:compile_model + +If you need to configure OpenVINO Runtime devices with additional configuration parameters, please, refer to the migration [Configure devices](./configure_devices.md) guide. + +## 4. Create an Inference Request + +Inference Engine API: + +@snippet docs/snippets/ie_common.cpp ie:create_infer_request + +OpenVINO™ Runtime API 2.0: + +@snippet docs/snippets/ov_common.cpp ov_api_2_0:create_infer_request + +## 5. Fill input tensors + +Inference Engine API fills inputs as `I32` precision (**not** aligned with the original model): + +@sphinxdirective + +.. tab:: IR v10 + + .. doxygensnippet:: docs/snippets/ie_common.cpp + :language: cpp + :fragment: [ie:get_input_tensor] + +.. tab:: IR v11 + + .. doxygensnippet:: docs/snippets/ie_common.cpp + :language: cpp + :fragment: [ie:get_input_tensor] + +.. tab:: ONNX + + .. doxygensnippet:: docs/snippets/ie_common.cpp + :language: cpp + :fragment: [ie:get_input_tensor] + +.. tab:: Model created in code + + .. doxygensnippet:: docs/snippets/ie_common.cpp + :language: cpp + :fragment: [ie:get_input_tensor] + +@endsphinxdirective + +OpenVINO™ Runtime API 2.0 fills inputs as `I64` precision (aligned with the original model): + +@sphinxdirective + +.. tab:: IR v10 + + .. doxygensnippet:: docs/snippets/ov_common.cpp + :language: cpp + :fragment: [ov_api_2_0:get_input_tensor_v10] + +.. tab:: IR v11 + + .. doxygensnippet:: docs/snippets/ov_common.cpp + :language: cpp + :fragment: [ov_api_2_0:get_input_tensor_aligned] + +.. tab:: ONNX + + .. doxygensnippet:: docs/snippets/ov_common.cpp + :language: cpp + :fragment: [ov_api_2_0:get_input_tensor_aligned] + +.. tab:: Model created in code + + .. doxygensnippet:: docs/snippets/ov_common.cpp + :language: cpp + :fragment: [ov_api_2_0:get_input_tensor_aligned] + +@endsphinxdirective + +## 6. Start Inference + +Inference Engine API: + +@snippet docs/snippets/ie_common.cpp ie:inference + +OpenVINO™ Runtime API 2.0: + +@snippet docs/snippets/ov_common.cpp ov_api_2_0:inference + +## 7. Process the Inference Results + +Inference Engine API processes outputs as `I32` precision (**not** aligned with the original model): + +@sphinxdirective + +.. tab:: IR v10 + + .. doxygensnippet:: docs/snippets/ie_common.cpp + :language: cpp + :fragment: [ie:get_output_tensor] + +.. tab:: IR v11 + + .. doxygensnippet:: docs/snippets/ie_common.cpp + :language: cpp + :fragment: [ie:get_output_tensor] + +.. tab:: ONNX + + .. doxygensnippet:: docs/snippets/ie_common.cpp + :language: cpp + :fragment: [ie:get_output_tensor] + +.. tab:: Model created in code + + .. doxygensnippet:: docs/snippets/ie_common.cpp + :language: cpp + :fragment: [ie:get_output_tensor] + +@endsphinxdirective + +OpenVINO™ Runtime API 2.0 processes outputs: +- For IR v10 as `I32` precision (**not** aligned with the original model) to match **old** behavior +- For IR v11, ONNX, ov::Model, Paddle as `I64` precision (aligned with the original model) to match **new** behavior + +@sphinxdirective + +.. tab:: IR v10 + + .. doxygensnippet:: docs/snippets/ov_common.cpp + :language: cpp + :fragment: [ov_api_2_0:get_output_tensor_v10] + +.. tab:: IR v11 + + .. doxygensnippet:: docs/snippets/ov_common.cpp + :language: cpp + :fragment: [ov_api_2_0:get_output_tensor_aligned] + +.. tab:: ONNX + + .. doxygensnippet:: docs/snippets/ov_common.cpp + :language: cpp + :fragment: [ov_api_2_0:get_output_tensor_aligned] + +.. tab:: Model created in code + + .. doxygensnippet:: docs/snippets/ov_common.cpp + :language: cpp + :fragment: [ov_api_2_0:get_output_tensor_aligned] + +@endsphinxdirective diff --git a/docs/OV_Runtime_UG/migration_ov_2_0/configure_devices.md b/docs/OV_Runtime_UG/migration_ov_2_0/configure_devices.md new file mode 100644 index 00000000000..2e19b825c4c --- /dev/null +++ b/docs/OV_Runtime_UG/migration_ov_2_0/configure_devices.md @@ -0,0 +1,129 @@ +# Configure devices {#openvino_2_0_configure_devices} + +### Introduction + +Inference Engine API provides an [ability to configure devices](https://docs.openvino.ai/2021.4/openvino_docs_IE_DG_InferenceEngine_QueryAPI.html) via configuration keys and [get device specific metrics](https://docs.openvino.ai/2021.4/openvino_docs_IE_DG_InferenceEngine_QueryAPI.html#getmetric). The values taken from `InferenceEngine::Core::GetConfig` are requested by its string name, while return type is `InferenceEngine::Parameter` and users don't know what is the actual type is stored in this parameter. + +OpenVINO Runtime API 2.0 solves these issues by introducing [properties](../supported_plugins/config_properties.md), which unify metrics and configuration key concepts, but the main advantage of properties - they have C++ type: + +``` +static constexpr Property full_name{"FULL_DEVICE_NAME"}; +``` + +And the property can be requested from an inference device as: + +@snippet ov_properties_migration.cpp core_get_ro_property + +The snippets below show how to migrate from Inference Engine device configuration to OpenVINO Runtime API 2.0 steps. + +### Set configuration values + +Inference Engine API: + +@sphinxdirective + +.. tab:: Devices + + .. doxygensnippet:: docs/snippets/ov_properties_migration.cpp + :language: cpp + :fragment: [core_set_config] + +.. tab:: Model Loading + + .. doxygensnippet:: docs/snippets/ov_properties_migration.cpp + :language: cpp + :fragment: [core_load_network] + +.. tab:: Execution + + .. doxygensnippet:: docs/snippets/ov_properties_migration.cpp + :language: cpp + :fragment: [executable_network_set_config] + +@endsphinxdirective + +OpenVINO Runtime API 2.0: + +@sphinxdirective + +.. tab:: Devices + + .. doxygensnippet:: docs/snippets/ov_properties_migration.cpp + :language: cpp + :fragment: [core_set_property] + +.. tab:: Model Loading + + .. doxygensnippet:: docs/snippets/ov_properties_migration.cpp + :language: cpp + :fragment: [core_compile_model] + +.. tab:: Execution + + .. doxygensnippet:: docs/snippets/ov_properties_migration.cpp + :language: cpp + :fragment: [compiled_model_set_property] + +@endsphinxdirective + +### Get information + +Inference Engine API: + +@sphinxdirective + +.. tab:: Device configuration + + .. doxygensnippet:: docs/snippets/ov_properties_migration.cpp + :language: cpp + :fragment: [core_get_config] + +.. tab:: Device metrics + + .. doxygensnippet:: docs/snippets/ov_properties_migration.cpp + :language: cpp + :fragment: [core_get_metric] + +.. tab:: Execution config + + .. doxygensnippet:: docs/snippets/ov_properties_migration.cpp + :language: cpp + :fragment: [executable_network_get_config] + +.. tab:: Execution metrics + + .. doxygensnippet:: docs/snippets/ov_properties_migration.cpp + :language: cpp + :fragment: [executable_network_get_metric] + +@endsphinxdirective + +OpenVINO Runtime API 2.0: + +@sphinxdirective + +.. tab:: Device configuration + + .. doxygensnippet:: docs/snippets/ov_properties_migration.cpp + :language: cpp + :fragment: [core_get_rw_property] + +.. tab:: Device metrics + + .. doxygensnippet:: docs/snippets/ov_properties_migration.cpp + :language: cpp + :fragment: [core_get_ro_property] + +.. tab:: Execution config + + .. doxygensnippet:: docs/snippets/ov_properties_migration.cpp + :language: cpp + :fragment: [compiled_model_get_rw_property] + +.. tab:: Execution metrics + + .. doxygensnippet:: docs/snippets/ov_properties_migration.cpp + :language: cpp + :fragment: [compiled_model_get_ro_property] + +@endsphinxdirective diff --git a/docs/OV_Runtime_UG/migration_ov_2_0/graph_construction.md b/docs/OV_Runtime_UG/migration_ov_2_0/graph_construction.md new file mode 100644 index 00000000000..0ec5ec644ee --- /dev/null +++ b/docs/OV_Runtime_UG/migration_ov_2_0/graph_construction.md @@ -0,0 +1,16 @@ +# Model creation in runtime {#openvino_2_0_model_creation} + +OpenVINO™ Runtime API 2.0 includes nGraph engine as a common part. The `ngraph` namespace was changed to `ov`, all other ngraph API is preserved as is. +Code snippets below show how application code should be changed for migration to OpenVINO™ Runtime API 2.0. + +### nGraph API + +@snippet snippets/ngraph.cpp ngraph:graph + +### OpenVINO™ Runtime API 2.0: + +@snippet snippets/ov_graph.cpp ov:graph + +**See also:** +- [Hello Model Creation C++ Sample](../../../samples/cpp/model_creation_sample/README.md) +- [Hello Model Creation Python Sample](../../../samples/python/model_creation_sample/README.md) diff --git a/docs/OV_Runtime_UG/migration_ov_2_0/intro.md b/docs/OV_Runtime_UG/migration_ov_2_0/intro.md new file mode 100644 index 00000000000..fa0010fbb51 --- /dev/null +++ b/docs/OV_Runtime_UG/migration_ov_2_0/intro.md @@ -0,0 +1,81 @@ +# OpenVINO™ 2.0 Transition Guide {#openvino_2_0_transition_guide} + +@sphinxdirective + +.. toctree:: + :maxdepth: 1 + :hidden: + + openvino_2_0_inference_pipeline + openvino_2_0_configure_devices + openvino_2_0_preprocessing + openvino_2_0_model_creation + +@endsphinxdirective + +### Introduction + +Older versions of OpenVINO (prior to 2022.1) required to change the logic of applications when an user migrates from the frameworks like TensorFlow, ONNX Runtime, PyTorch, PaddlePaddle, etc. The change of application's logic is connected with: + +- Model Optimizer changed input precisions for some inputs. For example, neural langauge processing models with `I64` input are becoming to have `I32` input element type. +- Model Optimizer changed layouts for TensorFlow models (see [Layouts in OpenVINO](../layout_overview.md)). It leads to unexpected user behavior that a user needs to use a different layout for its input data with compare to the framework: +![tf_openvino] +- Inference Engine API (`InferenceEngine::CNNNetwork`) also applied some conversion rules for input and output precisions because of device plugins limitations. +- Users need to specify input shapes during model conversions in Model Optimizer and work with static shapes in the application. + +OpenVINO Runtime API 2.0 is introduced to align logic of working with model as it is done in the frameworks - no layout and precision changes, operates with tensor names and indeces to address inputs and outputs. OpenVINO Runtime is composed of Inference Engine API used for inference and ngraph API targeted to work with models, operations. The OpenVINO API 2.0 has common structure, naming convention styles, namespaces, removes duplicated structures. See [How to migrate to OpenVINO 2.0 API](./common_inference_pipeline.md) for details. + +> **NOTE**: Most important is that your existing application can continue working with OpenVINO Runtime 2.0 as it used to be, but we recommend migration to new API to unlock additional features like [Preprocessing](../preprocessing_overview.md) and [Dynamic shapes support](../ov_dynamic_shapes.md). + +### Introduce IR v11 + +To support these features, OpenVINO introduced IR v11 which is generated by Model Optimizer by default since 2022.1. The model represented in IR v11 fully matches the original model in a original framework format in terms of inputs and outputs. Also, a user does not have to specify input shapes during the conversion, so the resulting IR v11 contains `-1` to denote undefined dimensions (see [Working with dynamic shapes](../ov_dynamic_shapes.md) to fully utilize this feature; or [Changning input shapes](../ShapeInference.md) to reshape to static shapes in the application). + +What is also important to mention - the IR v11 is fully compatible with old applications written with Inference Engine API from older versions of OpenVINO. This is achieved by adding additional runtime information to the IR v11 which is responsible for backwark compatible behavior. So, once the IR v11 is read by the old Inference Engine based application, it's internally converted to IR v10 to provide backward-compatible behavior. + +The IR v11 is supported by all OpenVINO Development tools including Post Training Optimization tool, Benchmark app, etc. + +### IR v10 compatibility + +OpenVINO Runtime API 2.0 also supports model in IR v10 for backward compatibility. So, if a user has an IR v10, such IR v10 can be fed to OpenVINO Runtime as well (see [migration steps](./common_inference_pipeline.md)). + +Some OpenVINO Development Tools also support both IR v10 and IR v11 as an input: +- Accuracy checker also supports IR v10, but requires an additional option to denote which API is used underneath. +- [Compile tool](../../../tools/compile_tool/README.md) compiles the model to be used in OpenVINO 2.0 API by default. If a user wants to use the resulting compiled blob in Inference Engine API, the additional `ov_api_1_0` option should be passed. + +But the following OpenVINO tools don't support IR v10 as an input, they require to regenerate an IR v11 from the original model with latest Model Optimizer: +- Post Training Optimization tool +- Deep Learning WorkBench + +### Differences between Inference Engine and OpenVINO Runtime 2.0 + +Inference Engine and ngraph APIs are not deprecated, they are fully functional and can be used in applications. But OpenVINO recommends users to migrate to new OpenVINO Runtime API 2.0, because it already has additional features and this list will be extended later. The following list of additional features is supported by new API: +- [Working with dynamic shapes](../ov_dynamic_shapes.md). The feature is quite usefull for best performance for NLP (Neural Language Processing) models, super resolution models and other which accepts dynamic input shapes. +- [Preprocessing of the model](../preprocessing_overview.md) to add preprocessing operations to the inference models and fully ocupay the accelerator and free CPU resources. + +To define a difference on the API level between Inference Engine and OpenVINO RUntime API 2.0, let's define two types of behaviors: +- **Old behavior** of OpenVINO supposes: + - Model Optimizer can change input element types, order of dimensions (layouts) with compare to the model from the original framework. + - Inference Engine can override input and output element types. + - Inference Engine API operates with operation names to address inputs and outputs (e.g. InferenceEngine::InferRequest::GetBlob). + - Does not support compiling of models with dynamic input shapes. +- **New behavior** assumes full model aligment with the framework and is implemented in OpenVINO 2.0: + - Model Optimizer preserves the input element types, order of dimensions (layouts) and stores tensor names from the original models. + - OpenVINO Runtime 2.0 reads models in any formats (IR v10, IR v11, ONNX, PaddlePaddle, etc) as is. + - OpenVINO Runtime API 2.0 operates with tensor names. Note, the difference between tensor names and operations names is that in case if a single operation has several output tensors, such tensors cannot identified in a unique manner, so tensor names are used for addressing as it's usually done in the frameworks. + - OpenVINO Runtime API 2.0 can address input and outputs tensors also by its index. Some model formats like ONNX are sensitive to order of inputs, outputs and its preserved by OpenVINO Runtime 2.0. + +The table below demonstrates which behavior **old** or **new** is used depending on a model source, used APIs. + +| API | IR v10 | IR v11 | ONNX file | Model created in code | +|-------------------------------|---------|---------|-----------|-----------------------| +|Inference Engine / ngraph APIs | Old | Old | Old | Old | +|OpenVINO Runtime API 2.0 | Old | New | New | New | + +Please look at next transition guides to understand how migrate Inference Engine-based application to OpenVINO™ Runtime API 2.0: + - [OpenVINO™ Common Inference pipeline](common_inference_pipeline.md) + - [Preprocess your model](./preprocessing.md) + - [Configure device](./configure_devices.md) + - [OpenVINO™ Model Creation](graph_construction.md) + +[tf_openvino]: ../../img/tf_openvino.png diff --git a/docs/OV_Runtime_UG/migration_ov_2_0/preprocessing.md b/docs/OV_Runtime_UG/migration_ov_2_0/preprocessing.md new file mode 100644 index 00000000000..9288249dbe9 --- /dev/null +++ b/docs/OV_Runtime_UG/migration_ov_2_0/preprocessing.md @@ -0,0 +1,64 @@ +# Preprocessing {#openvino_2_0_preprocessing} + +### Introduction + +Inference Engine API has preprocessing capabilities in `InferenceEngine::CNNNetwork` class. Such preprocessing information is not a part of the main inference graph executed by the [OpenVINO devices](../supported_plugins/Device_Plugins.md), so it is stored and executed separately before an inference stage: +- Preprocessing operations are executed on CPU processor for most of the OpenVINO inference plugins. So, instead of occupying of acceleators, CPU processor is also busy with computational tasks. +- Preprocessing information stored in `InferenceEngine::CNNNetwork` is lost during saving back to IR file format. + +OpenVINO Runtime API 2.0 introduces [new way of adding preprocessing operations to the model](../preprocessing_overview.md) - each preprocessing or postprocessing operation is integrated directly to the model and compiled together with inference graph: +- Add preprocessing operations first using `ov::preprocess::PrePostProcessor` +- Compile model on the target then using `ov::Core::compile_model` + +Having preprocessing operations as a part of OpenVINO opset allows to read and serialize preprocessed model as the IR file format. + +It's also important to mention that since OpenVINO 2.0, the Runtime API does not assume any default layouts like Inference Engine did, for example both `{ 1, 224, 224, 3 }` and `{ 1, 3, 224, 224 }` shapes are supposed to have `NCHW` layout while only the last shape has `NCHW`. So, some preprocessing capabilities in OpenVINO Runtime API 2.0 requires explicitly set layouts, see [Layout overview](../layout_overview.md) how to do it. For example, to perform image scaling by partial dimensions `H` and `W`, preprocessing needs to know what dimensions are `H` and `W`. + +> **NOTE**: Use Model Optimizer preprocessing capabilities to insert and optimize preprocessing operations to the model. In this case you don't need to read model in runtime application and set preprocessing, you can use [model caching feature](../Model_caching_overview.md) to improve time to inference stage. + +The steps below demonstrates how to migrate preprocessing scenarios from Inference Engine API to OpenVINO Runtime API 2.0. +The snippets suppose we need to preprocess a model input with tensor name `tensor_name`, in Inferenece Engine API using operation names to address the data, it's called `operation_name`. + +### Mean and scale values + +Inference Engine API: + +@snippet docs/snippets/ov_preprocessing_migration.cpp mean_scale + +OpenVINO Runtime API 2.0: + +@snippet docs/snippets/ov_preprocessing_migration.cpp ov_mean_scale + +### Precision and layout conversions + +Inference Engine API: + +@snippet docs/snippets/ov_preprocessing_migration.cpp conversions + +OpenVINO Runtime API 2.0: + +@snippet docs/snippets/ov_preprocessing_migration.cpp ov_conversions + +### Image scaling + +Inference Engine API: + +@snippet docs/snippets/ov_preprocessing_migration.cpp image_scale + +OpenVINO Runtime API 2.0: + +@snippet docs/snippets/ov_preprocessing_migration.cpp ov_image_scale + +### Color space conversions + +Inference Engine API: + +@snippet docs/snippets/ov_preprocessing_migration.cpp color_space + +OpenVINO Runtime API 2.0: + +@snippet docs/snippets/ov_preprocessing_migration.cpp ov_color_space + +**See also:** +- [Preprocessing details](../preprocessing_details.md) +- [NV12 classification sample](../../../samples/cpp/hello_nv12_input_classification/README.md) diff --git a/docs/OV_Runtime_UG/model_representation.md b/docs/OV_Runtime_UG/model_representation.md index 91202aeb8c8..0e932278373 100644 --- a/docs/OV_Runtime_UG/model_representation.md +++ b/docs/OV_Runtime_UG/model_representation.md @@ -2,8 +2,6 @@ In OpenVINO™ Runtime a model is represented by the `ov::Model` class. -## Model Representation - The `ov::Model` object stores shared pointers to `ov::op::v0::Parameter`, `ov::op::v0::Result` and `ov::op::Sink` operations that are inputs, outputs and sinks of the graph. Sinks of the graph have no consumers and are not included in the results vector. All other operations hold each other via shared pointers: child operation holds its parent (hard link). If an operation has no consumers and it's not the `Result` or `Sink` operation (shared pointer counter is zero), then it will be destructed and won't be accessible anymore. @@ -12,24 +10,63 @@ Each operation in `ov::Model` has the `std::shared_ptr` type. For details on how to build a model in OpenVINO™ Runtime, see the [Build a Model in OpenVINO™ Runtime](@ref build_model) section. +OpenVINO™ Runtime allows using tensor names or indexes to work wit model inputs/outputs. To get model input/output ports, use the `ov::Model::inputs()` or `ov::Model::outputs()` methods respectively. + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_model_snippets.cpp + :language: cpp + :fragment: [all_inputs_ouputs] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_model_snippets.py + :language: python + :fragment: [all_inputs_ouputs] + +@endsphinxdirective + +OpenVINO™ Runtime model representation uses special classes to work with model data types and shapes. For data types the `ov::element::Type` is used. + +## Shapes Representation + +OpenVINO™ Runtime provides two types for shape representation: + +* `ov::Shape` - Represents static (fully defined) shapes. + +* `ov::PartialShape` - Represents dynamic shapes. That means that the rank or some of dimensions are dynamic (dimension defines an interval or undefined). `ov::PartialShape` can be converted to `ov::Shape` using the `get_shape()` method if all dimensions are static; otherwise the conversion raises an exception. + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_model_snippets.cpp + :language: cpp + :fragment: [ov:partial_shape] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_model_snippets.py + :language: python + :fragment: [ov:partial_shape] + +@endsphinxdirective + + But in most cases before getting static shape using `get_shape()` method, you need to check that shape is static. + ## Operations -The `ov::Op` class represents any abstract operation in the model representation. Use this class to create [custom operations](../OV_Runtime_UG/Extensibility_DG/AddingNGraphOps.md). +The `ov::Op` class represents any abstract operation in the model representation. Use this class to create [custom operations](../Extensibility_UG/add_openvino_ops.md). ## Operation Sets Operation set (opset) is a collection of operations that can be used to construct a model. The `ov::OpSet` class provides a functionality to work with operation sets. -For each operation set, OpenVINO™ Runtime provides a separate namespace, for example `opset8`. +For each operation set, OpenVINO™ Runtime provides a separate namespace, for example `opset8`. Each OpenVINO™ Release release introduces new operations and add these operations to a new operation set. New operation sets help to introduce a new version of operations that change behavior of previous operations. Using operation sets allows you to avoid changes in your application if new operations have been introduced. - -## Static and Partial Shapes - -There are two types used for shape representation: - -* `ov::Shape` - Represents static (fully defined) shapes. - -* `ov::PartialShape` - Represents dynamic shapes. That means that the rank or some of dimensions are dynamic (undefined). `ov::PartialShape` can be converted to `ov::Shape` using the `get_shape()` method if all dimensions are static; otherwise the conversion raises an exception. - +For a complete list of operation sets supported in OpenVINO™ toolkit, see [Available Operations Sets](../ops/opset.md). +To add support of custom operations, see the [Add Custom OpenVINO Operations](../Extensibility_UG/Intro.md) document. ## Build a Model in OpenVINO™ Runtime {#build_model} @@ -37,55 +74,109 @@ You can create a model from source. This section illustrates how to construct a Operation set `opsetX` integrates a list of pre-compiled operations that work for this purpose. In other words, `opsetX` defines a set of operations for building a graph. -For a complete list of operation sets supported in OpenVINO™ toolkit, see [Available Operations Sets](../ops/opset.md). - -To add suport of custom operations, see the [Add Custom OpenVINO Operations](../OV_Runtime_UG/Extensibility_DG/Intro.md) document. - To build an `ov::Model` instance from `opset8` operations, include the following files: - - C++ - @snippet example_ngraph_utils.cpp ov:include +@sphinxdirective - - Python - ```python - import openvino.runtime.opset8 as ov - from openvino.runtime import Model - ``` +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_model_snippets.cpp + :language: cpp + :fragment: [ov:include] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_model_snippets.py + :language: python + :fragment: [import] + +@endsphinxdirective The following code demonstrates how to create a simple model: - - C++ - @snippet example_ngraph_utils.cpp ov:create_simple_model +@sphinxdirective - - Python - ```python - TBD - ``` +.. tab:: C++ + .. doxygensnippet:: docs/snippets/ov_model_snippets.cpp + :language: cpp + :fragment: [ov:create_simple_model] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_model_snippets.py + :language: python + :fragment: [ov:create_simple_model] + +@endsphinxdirective The following code creates a model with several outputs: - - C++ - @snippet example_ngraph_utils.cpp ov:create_advanced_model +@sphinxdirective - - Python - ```python - TBD - ``` +.. tab:: C++ -## FAQ + .. doxygensnippet:: docs/snippets/ov_model_snippets.cpp + :language: cpp + :fragment: [ov:create_advanced_model] -### Does OpenVINO™ Runtime provide any capabilities to debug the model structure and model modification? - - To receive additional messages about applied graph modifications, rebuild the OpenVINO™ Runtime library with the `-DENABLE_OPENVINO_DEBUG=ON` option. - - A model can be visualized to image from the xDot format: - @snippet example_ngraph_utils.cpp ov:visualize - - A model can be serialized to IR: - @snippet example_ngraph_utils.cpp ov:serialize +.. tab:: Python -### How can I develop my own transformation pass? - See the [Transformations Developer Guide](./nGraphTransformation.md). + .. doxygensnippet:: docs/snippets/ov_model_snippets.py + :language: python + :fragment: [ov:create_advanced_model] + +@endsphinxdirective + +## Model debug capabilities + +OpenVINO™ provides several debug capabilities: + - To receive additional messages about applied model modifications, rebuild the OpenVINO™ Runtime library with the `-DENABLE_OPENVINO_DEBUG=ON` option. + - Model can be visualized to image from the xDot format: + @sphinxdirective + + .. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_model_snippets.cpp + :language: cpp + :fragment: [ov:visualize] + + .. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_model_snippets.py + :language: python + :fragment: [ov:visualize] + + @endsphinxdirective + + `ov::pass::VisualizeTree` can be parametrized via environment variables: + + OV_VISUALIZE_TREE_OUTPUT_SHAPES=1 - visualize shapes + OV_VISUALIZE_TREE_OUTPUT_TYPES=1 - visualize types + OV_VISUALIZE_TREE_MIN_MAX_DENORMAL=1 - pretty denormal values + OV_VISUALIZE_TREE_RUNTIME_INFO=1 - print runtime information + OV_VISUALIZE_TREE_IO=1 - print I/O ports + OV_VISUALIZE_TREE_MEMBERS_NAME=1 - print member names + + - Also model can be serialized to IR: + @sphinxdirective + + .. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_model_snippets.cpp + :language: cpp + :fragment: [ov:serialize] + + .. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_model_snippets.py + :language: python + :fragment: [ov:serialize] + + @endsphinxdirective ## See Also * [Available Operation Sets](../ops/opset.md) -* [OpenVINO™ Runtime Extensibility Developer Guide](../OV_Runtime_UG/Extensibility_DG/Intro.md) +* [OpenVINO™ Runtime Extensibility Developer Guide](../Extensibility_UG/Intro.md) +* [Transformations Developer Guide](../Extensibility_UG/ov_transformations.md). diff --git a/docs/OV_Runtime_UG/supported_plugins/MULTI.md b/docs/OV_Runtime_UG/multi_device.md similarity index 89% rename from docs/OV_Runtime_UG/supported_plugins/MULTI.md rename to docs/OV_Runtime_UG/multi_device.md index 64659111a3c..f49a94cc0ca 100644 --- a/docs/OV_Runtime_UG/supported_plugins/MULTI.md +++ b/docs/OV_Runtime_UG/multi_device.md @@ -1,4 +1,4 @@ -# Multi-Device Plugin {#openvino_docs_IE_DG_supported_plugins_MULTI} +# Running on multiple device simultaneously {#openvino_docs_OV_UG_Running_on_multiple_devices} ## Introducing the Multi-Device Plugin (C++) @@ -44,7 +44,7 @@ Notice that the priorities of the devices can be changed in real time for the ex Finally, there is a way to specify number of requests that the Multi-Device will internally keep for each device. Suppose your original app was running 4 cameras with 4 inference requests. You would probably want to share these 4 requests between 2 devices used in MULTI. The easiest way is to specify a number of requests for each device using parentheses: "MULTI:CPU(2),GPU(2)" and use the same 4 requests in your app. However, such an explicit configuration is not performance-portable and hence not recommended. Instead, the better way is to configure the individual devices and query the resulting number of requests to be used at the application level (see [Configuring the Individual Devices and Creating the Multi-Device On Top](#configuring-the-individual-devices-and-creating-the-multi-device-on-top)). ### Enumerating Available Devices -The Inference Engine features a dedicated API to enumerate devices and their capabilities. See the [Hello Query Device C++ Sample](../../../samples/cpp/hello_query_device/README.md). This is example output from the sample (truncated to device names only): +The OpenVINO Runtime API features a dedicated methods to enumerate devices and their capabilities. See the [Hello Query Device C++ Sample](../../samples/cpp/hello_query_device/README.md). This is example output from the sample (truncated to device names only): ```sh ./hello_query_device @@ -86,13 +86,13 @@ Note that while the performance of accelerators combines really well with Multi- See the [Using the Multi-Device with OpenVINO samples and benchmarking the performance](#using-the-multi-device-with-openvino-samples-and-benchmarking-the-performance) section below. ### Querying the Optimal Number of Inference Requests -You can use the [Properties API](../PropertiesAPI.md) API to query the optimal number of requests. Similarly, when using the Multi-Device you don't need to sum over included devices yourself, you can query property directly: +You can use the [configure devices](supported_plugins/config_properties.md) to query the optimal number of requests. Similarly, when using the Multi-Device you don't need to sum over included devices yourself, you can query property directly: @snippet snippets/MULTI5.cpp part5 ### Using the Multi-Device with OpenVINO Samples and Benchmarking the Performance -Every OpenVINO sample that supports the `-d` (which stands for "device") command-line option transparently accepts Multi-Device. The [Benchmark Application](../../../samples/cpp/benchmark_app/README.md) is the best reference for the optimal usage of Multi-Device. As discussed earlier, you do not need to set up the number of requests, CPU streams or threads because the application provides optimal performance out of the box. Below is an example command to evaluate HDDL+GPU performance with that: +Every OpenVINO sample that supports the `-d` (which stands for "device") command-line option transparently accepts Multi-Device. The [Benchmark Application](../../samples/cpp/benchmark_app/README.md) is the best reference for the optimal usage of Multi-Device. As discussed earlier, you do not need to set up the number of requests, CPU streams or threads because the application provides optimal performance out of the box. Below is an example command to evaluate HDDL+GPU performance with that: ```sh ./benchmark_app –d MULTI:HDDL,GPU –m -i -niter 1000 @@ -110,7 +110,7 @@ The Multi-Device plugin supports FP16 IR files. The CPU plugin automatically upc @endsphinxdirective ### See Also -[Supported Devices](Supported_Devices.md) +[Supported Devices](supported_plugins/Supported_Devices.md) ## Introducing the Multi-Device Plugin (Python) @@ -182,7 +182,7 @@ You can set the configuration directly as a string, or use the metric key `MULTI ### Enumerating Available Devices -The Inference Engine features a dedicated API to enumerate devices and their capabilities. See the [Hello Query Device Python Sample](../../../samples/python/hello_query_device/README.md). This is example output from the sample (truncated to device names only): +The OpenVINO Runtime API features a dedicated methods to enumerate devices and their capabilities. See the [Hello Query Device Python Sample](../../samples/python/hello_query_device/README.md). This is example output from the sample (truncated to device names only): ```sh ./hello_query_device @@ -268,7 +268,7 @@ Note that while the performance of accelerators works well with Multi-Device, th ### Using the Multi-Device with OpenVINO Samples and Benchmarking the Performance -Every OpenVINO sample that supports the `-d` (which stands for "device") command-line option transparently accepts Multi-Device. The [Benchmark application](../../../tools/benchmark_tool/README.md) is the best reference for the optimal usage of Multi-Device. As discussed earlier, you do not need to set up the number of requests, CPU streams or threads because the application provides optimal performance out of the box. Below is an example command to evaluate CPU+GPU performance with the Benchmark application: +Every OpenVINO sample that supports the `-d` (which stands for "device") command-line option transparently accepts Multi-Device. The [Benchmark application](../../tools/benchmark_tool/README.md) is the best reference for the optimal usage of Multi-Device. As discussed earlier, you do not need to set up the number of requests, CPU streams or threads because the application provides optimal performance out of the box. Below is an example command to evaluate CPU+GPU performance with the Benchmark application: ```sh ./benchmark_app.py –d MULTI:CPU,GPU –m @@ -289,4 +289,4 @@ The Multi-Device plugin supports FP16 IR files. The CPU plugin automatically upc @endsphinxdirective ### See Also -[Supported Devices](Supported_Devices.md) \ No newline at end of file +[Supported Devices](supported_plugins/Supported_Devices.md) \ No newline at end of file diff --git a/docs/OV_Runtime_UG/nGraphTransformation.md b/docs/OV_Runtime_UG/nGraphTransformation.md deleted file mode 100644 index 39fb3606e5a..00000000000 --- a/docs/OV_Runtime_UG/nGraphTransformation.md +++ /dev/null @@ -1,449 +0,0 @@ -# Overview of Transformations API {#ngraph_transformation} - -This guide contains all necessary information that you need to start implementing nGraph transformations. - -## Prerequisites -Before creating a transformation, do the following: - -* Make sure that there is no transformation with the same functionality in the [Transformation Library](group__ie__transformation__api.html) -* Learn how the [Transformation Library](group__ie__transformation__api.html) is structured and how transformations are organized -* Understand where to put your transformation code - -### Transformation Library Structure -OpenVINO transformations are located in the `src/common/transformations` directory. - -Transformations root directory contains two folders: -* `ngraph_ops` - Contains internal opset operations that are common for plugins. -* `transformations` - Includes all transformations, utils, runtime info attributes, and pass managers. - -All internal operations and transformations located inside the [Transformation Library](group__ie__transformation__api.html) can be used inside plugins. -All legacy operations and transformations were moved to a legacy library and are not recommended to be used. - -### Transformation Flow Layers -Transformation flow in the transformation library has several layers: - -1. Pass managers - Execute any type of transformations and provide additional debug capabilities. -2. Transformations - Perform a particular transformation algorithm on `ngraph::Function`. -3. Low-level functions - Take a set of nodes and perform some transformation action. -They are not mandatory and all transformation code can be located inside the transformation. -But if some transformation parts can potentially be reused in other transformations, we suggest keeping them as separate functions. - -### Location for Your Transformation Code -To decide where to store your transformation code, please follow these rules: - -1. If it is a plugin-specific transformation and cannot be reused by other plugins, keep source code inside plugin. -2. If this transformation relates to opset operation conversion or optimization, keep sources inside the transformation library. - -After you decide where to store your transformation code, you can start developing your own nGraph transformation. - -## ngraph::Function and graph representation - -nGraph function is a very simple thing: it stores shared pointers to `ngraph::op::Parameter`, `ngraph::op::Result` and `ngraph::op::Sink` operations that are inputs, outputs and sinks of the graph. -Sinks of the graph have no consumers and not included into results vector. All other operations hold each other via shared pointers: child operation holds its parent (hard link). If operation has no consumers and it's not Result or Sink operation -(shared pointer counter is zero) then it will be destructed and won't be accessible anymore. Each operation in `ngraph::Function` has a `std::shared_ptr` type. - -For examples of how to build an nGraph function, see the [Build nGraph Function](./model_representation.md) page. - -## Transformations types - -nGraph has three main transformation types: - -* `ngraph::pass::FunctionPass` - straightforward way to work with `ngraph::Function` directly -* `ngraph::pass::MatcherPass` - pattern-based transformation approach -* `ngraph::pass::GraphRewrite` - container for matcher passes needed for efficient execution - -![transformations_structure] - -### ngraph::pass::FunctionPass - -`ngraph::pass::FunctionPass` is used for transformations that take entire `ngraph::Function` as an input and process it. - -Template for FunctionPass transformation class - -@snippet src/transformations/template_function_transformation.hpp function_pass:template_transformation_hpp - -@snippet src/transformations/template_function_transformation.cpp function_pass:template_transformation_cpp - -Using `ngraph::FunctionPass`, you need to override the `run_on_function` method where you will write the transformation code. -Return value is `true` if the original function has changed during transformation (new operation was added, or operations replacement was made, or node attributes were changed); otherwise, it is `false`. -For transformation API, please follow the [working with ngraph::Function](#working_with_ngraph_function) section. -Also `ngraph::FunctionPass` based transformations can be executed via `pass::Manager`. See the examples in the [Using pass manager](#using_pass_manager) section. - -### ngraph::pass::MatcherPass - -`ngraph::pass::MatcherPass` is used for pattern-based transformations. - -Template for MatcherPass transformation class -@snippet src/transformations/template_pattern_transformation.hpp graph_rewrite:template_transformation_hpp - -@snippet src/transformations/template_pattern_transformation.cpp graph_rewrite:template_transformation_cpp - -To use `ngraph::pass::MatcherPass`, you need to complete these steps: -1. Create a pattern -2. Implement a callback -3. Register the pattern and Matcher -4. Execute MatcherPass - -So let's go through each of these steps. - -### Create a pattern -Pattern is a single root `ngraph::Function`. But the only difference is that you do not need to create a function object, you just need to create and connect opset or special pattern operations. -Then you need to take the last created operation and put it as a root of the pattern. This root node will be used as a root node in pattern matching. -> **NOTE**: Any nodes in a pattern that have no consumers and are not registered as root will not be used in pattern matching. - -@snippet example_ngraph_utils.cpp pattern:simple_example - -The `Parameter` operation in the example above has type and shape specified. These attributes are needed only to create Parameter operation class and will not be used in pattern matching. - -For more pattern examples, refer to the [pattern matching](#pattern_matching) section. - -### Implement callback -Callback is an action applied to every pattern entrance. In general, callback is the lambda function that takes Matcher object with detected subgraph. - -@snippet example_ngraph_utils.cpp pattern:callback_example - -The example above shows the callback structure and how Matcher can be used for accessing nodes detected by pattern. -Callback return value is `true` if root node was replaced and another pattern cannot be applied to the same root node; otherwise, it is `false`. -> **NOTE**: It is not recommended to manipulate with nodes that are under root node. This may affect GraphRewrite execution as it is expected that all nodes that come after root node in topological order are valid and can be used in pattern matching. - -MatcherPass also provides functionality that allows reporting of the newly created nodes that can be used in additional pattern matching. -If MatcherPass was registered in `pass::Manager` or `pass::GraphRewrite`, these registered nodes will be added for additional pattern matching. -That means that matcher passes registered in `pass::GraphRewrite` will be applied to these nodes. - -The example below shows how single MatcherPass can fuse sequence of operations using the `register_new_node` method. - -@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:relu_fusion - -> **NOTE**: If you register multiple nodes, please add them in topological order. We do not topologically sort these nodes as it is a time-consuming operation. - -### Register pattern and Matcher -The last step is to register Matcher and callback inside the MatcherPass pass. To do this, call the `register_matcher` method. -> **NOTE**: Only one matcher can be registered for a single MatcherPass class. - -```cpp -// Register matcher and callback -register_matcher(m, callback); -``` -### Execute MatcherPass -MatcherPass has multiple ways to be executed: -* Run on a single node - it can be useful if you want to run MatcherPass inside another transformation. -@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:run_on_node -* Run on `ngraph::Function` using GraphRewrite - this approach gives ability to run MatcherPass on whole `ngraph::Function`. Moreover, multiple MatcherPass transformation can be registered in a single GraphRewite to be executed in a single graph traversal. -@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:graph_rewrite -* Run on `ngraph::Function` using `pass::Manager` - this approach helps you to register MatcherPass for execution on `ngraph::Function` as another transformation types. -@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:manager - - -### ngraph::pass::GraphRewrite - -GraphRewrite pass serves for running multiple matcher passes on `ngraph::Function` in a single graph traversal. -Example: - -@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:graph_rewrite - -In addition, GraphRewrite handles nodes that were registered by MatcherPasses during their execution. This nodes will be added to the beginning of the sequence with nodes for pattern matching. - -> **NOTE**: when using `pass::Manager` temporary GraphRewrite is used to execute single MatcherPass. - -GraphRewrite has two algorithms for MatcherPasses execution. First algorithm is straightforward. It applies each MatcherPass in registration order to current node. - -![graph_rewrite_execution] - -But it is not really efficient when you have a lot of registered passes. So first of all GraphRewrite checks that all MatcherPass patterns has type-based root node (it means that type of this node is not hidden into predicate). -And then creates map from registered MatcherPasses. That helps to avoid additional cost of applying each MatcherPass for each node. - -![graph_rewrite_efficient_search] - -> **NOTE**: GraphRewrite execution algorithm cannot be set manually and depends only on root nodes registered inside MatcherPasses. - -## Pattern Matching - -Sometimes patterns cannot be expressed via regular nGraph operations or it is too complicated. -For example, if you want to detect Convolution->Add sub-graph without specifying particular input type for Convolution operation or you want to create a pattern where some of operations can have different types. -And for these cases nGraph provides additional helpers to construct patterns for GraphRewrite transformations. - -There are two main helpers: -1. `ngraph::pattern::any_input` - helps to express inputs if their types are undefined. -2. `ngraph::pattern::wrap_type` - helps to express nodes of pattern without specifying node attributes. - -Let's go through the example to have better understanding of how it works: - -> **NOTE**: Node attributes do not participate in pattern matching and are needed only for operations creation. Only operation types participate in pattern matching. - -The example below shows basic usage of `pattern::any_input`. -Here we construct Multiply pattern with arbitrary first input and Constant as a second input. -Also as Multiply is commutative operation, it does not matter in which order we set inputs (any_input/Constant or Constant/any_input) because both cases will be matched. - -@snippet example_ngraph_utils.cpp pattern:label_example - -This example shows how we can construct a pattern when operation has arbitrary number of inputs. - -@snippet example_ngraph_utils.cpp pattern:concat_example - -This example shows how to use predicate to construct a pattern. Also it shows how to match pattern manually on given node. - -@snippet example_ngraph_utils.cpp pattern:predicate_example - -> **NOTE**: Be careful with manual matching because Matcher object holds matched nodes. To clear a match, use the m->clear_state() method. - -## Working with ngraph::Function - -In this chapter we will review nGraph API that allows us to manipulate with `ngraph::Function`. - -### ngraph::Node input and output ports - -First of all let's talk about `ngraph::Node` input/output ports. Each nGraph operation has input and output ports except cases when operation has `Result`, `Parameter`, or `Constant` type. - -Every port belongs to its node, so using a port we can access parent node, get shape and type for particular input/output, get all consumers in case of output port, and get producer node in case of input port. -With output port we can set inputs for newly created operations. - -Lets look at the code example. - -@snippet example_ngraph_utils.cpp ngraph:ports_example - -You may notice that we usually construct operations in this way: -```cpp -std::shared_ptr neg_const = opset1::Constant::create(sub->get_input_element_type(1), Shape{1}, {-1})); -Output data = node->input_value(0); -auto neg = std::make_shared(data, neg_const); -``` -In this example, the `opset3::Multiply` operation takes `Output` and `std::shared_ptr` as inputs. But the constructor takes both as `Output`. -In this case, `std::shared_ptr` will be automatically converted to `Output` if node has exactly one output port; otherwise, conversion raises an exception. - -### ngraph::Node replacement - -nGraph provides two ways for node replacement: via nGraph helper function and directly via port methods. We are going to review both of them. - -Let's start with nGraph helper functions. The most popular function is `ngraph::replace_node(old_node, new_node)`. - -We will review real replacement case where Negative operation is replaced with Multiply. - -![ngraph_replace_node] - -@snippet example_ngraph_utils.cpp ngraph:replace_node - -`ngraph::replace_node` has a constraint that number of output ports for both of ops must be the same; otherwise, it raises an exception. - - -The alternative way to do the same replacement is the following: -```cpp -// All neg->output(0) consumers will be moved to mul->output(0) port -neg->output(0).replace(mul->output(0)); -``` - -Another transformation example is insertion. - -![ngraph_insert_node] - -@snippet example_ngraph_utils.cpp ngraph:insert_node - -The alternative way to the insert operation is to make a node copy and use `replace_node`: - -@snippet example_ngraph_utils.cpp ngraph:insert_node_with_copy - -### ngraph::Node elimination - -Another type of node replacement is its elimination. - -To eliminate operation, nGraph has special method that considers all limitations related to InferenceEngine. - -@snippet example_ngraph_utils.cpp ngraph:eliminate_node - -`replace_output_update_name` in case of successful replacement it automatically preserves friendly name and runtime info. - - -## Transformation conditional compilation - -Transformation library has two internal macros to support conditional compilation feature. - -* `MATCHER_SCOPE(region)` - allows to disable the MatcherPass if matcher isn't used. The region name should be unique. This macro creates a local variable `matcher_name` which you should use as a matcher name. -* `RUN_ON_FUNCTION_SCOPE(region)` - allows to disable run_on_function pass if it isn't used. The region name should be unique. - -## Transformation writing essentials - -When developing a transformation, you need to follow these transformation rules: - -###1. Operation Set (OpSet) - -Use the latest version of OpSet in your transformation. An exception is op_conversion transformations, where different opsets can be used. - -@snippet example_ngraph_utils.cpp ov:include - -###2. Dynamic Shape and Rank - -nGraph has two types for shape representation: -`ngraph::Shape` - represents static shape. -`ngraph::PartialShape` - represents dynamic shape. It means that rank or some of dimensions are dynamic (undefined). -`ngraph::PartialShape` can be converted to `ngraph::Shape` using the `get_shape()` method if all dimensions are static; otherwise, conversion raises an exception. - -@snippet example_ngraph_utils.cpp ngraph:shape - -But in most cases before getting static shape using `get_shape()` method, you need to check that shape is static. - -Also if your transformation requires only input shape rank or particular dimension value, please do not use the `get_shape()` method. See the example below demonstrating how to avoid using `get_shape()` - -@snippet example_ngraph_utils.cpp ngraph:shape_check - -Not using `get_shape()` method makes your transformation more flexible and applicable for more cases. - -###3. Friendly Names - -Each `ngraph::Node` has a unique name (used for nGraph internals) and a friendly name. In transformations we care only about friendly name because it represents the name from intermediate representation (IR). -Also friendly name is used as output tensor name (until we do not have other way to represent output tensor name) and user code that requests intermediate outputs based on these names. -To avoid losing friendly name when replacing node with other node or subgraph, set the original friendly name to the latest node in replacing subgraph. See the example below. - -```cpp -// Replace Div operation with Power and Multiply sub-graph and set original friendly name to Multiply operation -auto pow = std::make_shared(div->input(1).get_source_output(), - op::Constant::create(div->get_input_element_type(1), Shape{1}, {-1})); -auto mul = std::make_shared(div->input(0).get_source_output(), pow); -mul->set_friendly_name(div->get_friendly_name()); -ngraph::replace_node(div, mul); -``` - -In more advanced cases, when replaced operation has several outputs and we add additional consumers to its outputs, we make a decision how to set friendly name by arrangement. - -###4. Runtime Info - -Runtime info is a map `std::map` located inside `ngraph::Node` class. It represents additional attributes in `ngraph::Node`. -These attributes can be set by users or by plugins and when executing transformation that changes `ngraph::Function` we need to preserve these attributes as they will not be automatically propagated. -In most cases, transformations have the following types: 1:1 (replace node with another node), 1:N (replace node with a sub-graph), N:1 (fuse sub-graph into a single node), N:M (any other transformation). -Currently, there is no mechanism that automatically detects transformation types, so we need to propagate this runtime information manually. See the examples below. - -```cpp -// Replace Transpose with Reshape operation (1:1) -ngraph::copy_runtime_info(transpose, reshape); -``` - -```cpp -// Replace Div operation with Power and Multiply sub-graph (1:N) -ngraph::copy_runtime_info(div, {pow, mul}); -``` - -```cpp -// Fuse Convolution with Add operation (N:1) -ngraph::copy_runtime_info({conv, bias}, {conv_ie}); -``` - -```cpp -// Any other transformation that replaces one sub-graph with another sub-graph (N:M) -ngraph::copy_runtime_info({a, b, c}, {e, f}); -``` - -When transformation has multiple fusions or decompositions, `ngraph::copy_runtime_info` must be called multiple times for each case. - -> **Note**: copy_runtime_info removes rt_info from destination nodes. If you want to keep it, you need to specify them in source nodes like this: copy_runtime_info({a, b, c}, {a, b}) - -###5. Constant Folding - -If your transformation inserts constant sub-graphs that need to be folded, do not forget to use `ngraph::pass::ConstantFolding()` after your transformation or call constant folding directly for operation. -The example below shows how constant subgraph can be constructed. - -```cpp -// After ConstantFolding pass Power will be replaced with Constant -auto pow = std::make_shared( - opset3::Constant::create(element::f32, Shape{1}, {2}) - opset3::Constant::create(element::f32, Shape{1}, {3})); -auto mul = std::make_shared(input /* not constant input */, pow); -``` - -Manual constant folding is more preferable than `ngraph::pass::ConstantFolding()` because it is much faster. - -Below you can find an example of manual constant folding: - -@snippet src/transformations/template_pattern_transformation.cpp manual_constant_folding - -## Common mistakes in transformations - -In transformation development process: - -* Do not use deprecated nGraph API. Deprecated methods has the `NGRAPH_DEPRECATED` macros in its definition. -* Do not pass `shared_ptr` as an input for other node if type of node is unknown or it has multiple outputs. Use explicit output port. -* If you replace node with another node that produces different shape, remember that new shape will not be propagated until the first `validate_nodes_and_infer_types` call for `ngraph::Function`. If you are using `pass::Manager`, it will automatically call this method after each transformation execution. -* Do not forget to call the `ngraph::ConstantFolding` pass if your transformation creates constant subgraphs. -* Use latest OpSet if you are not developing downgrade transformation pass. -* When developing a callback for `ngraph::pass::MatcherPass`, do not change nodes that come after the root node in topological order. - -## Using pass manager - -`ngraph::pass::Manager` is a container class that can store the list of transformations and execute them. The main idea of this class is to have high-level representation for grouped list of transformations. -It can register and apply any [transformation types](#transformations_types) on function. -In addition, `ngraph::pass::Manager` has extended debug capabilities (find more information in the [how to debug transformations](#how_to_debug_transformations) section). - -The example below shows basic usage of `ngraph::pass::Manager` - -@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:manager3 - -Another example shows how multiple matcher passes can be united into single GraphRewrite. - -@snippet src/transformations/template_pattern_transformation.cpp matcher_pass:manager2 - -> **NOTE**: nGraph used to have the `pass::PassConfig` class for transformation pipeline manipulation. -This mechanism is now obsolete and the `pass::PassConfig` class will be removed in future release. - -## How to debug transformations - -The most popular tool for transformations debugging is the `ngraph::pass::VisualizeTree` transformation, which visualizes ngraph::Function. - -Usage example: - -@snippet example_ngraph_utils.cpp ov:visualize - -`ngraph::pass::VisualizeTree` can be parametrized via environment variables: - -``` -OV_VISUALIZE_TREE_OUTPUT_SHAPES=1 - visualize shapes -OV_VISUALIZE_TREE_OUTPUT_TYPES=1 - visualize types -OV_VISUALIZE_TREE_MIN_MAX_DENORMAL=1 - pretty denormal values -OV_VISUALIZE_TREE_RUNTIME_INFO=1 - print runtime information -OV_VISUALIZE_TREE_IO=1 - print I/O ports -OV_VISUALIZE_TREE_MEMBERS_NAME=1 - print member names -``` - -> **Note**: current VisualTree does not have user-friendly interface and it will be changed in the nearest future. The intention is to move visualization abilities inside transformations. - -If you are using `ngraph::pass::Manager` to run sequence of transformations, you can get additional debug capabilities by using the following environment variables: - -``` -OV_PROFILE_PASS_ENABLE=1 - enables performance measurement for each transformation and prints execution status -OV_ENABLE_VISUALIZE_TRACING=1 - enables visualization after each transformation. By default, it saves dot and svg files. -``` - -> **Note**: Make sure that you have dot installed on your machine; otherwise, it will silently save only dot file without svg file. - -## Disabling/Enabling specific transformations for plugin X - -In transformation library, we provide plugins transformations like CommonOptimizations, which contains predefined sequence of transformations. -We also provide a tool that helps to disable or partially disable particular transformations in a transformation pipeline. -For example, if a plugin uses the CommonOptimization transformation and needs to disable the ConvertGELU transformation, then inside the plugin we have to take the PassConfig instance -from pass::Manger and call disable method. - -@snippet example_ngraph_utils.cpp ngraph:disable_gelu - -In some cases, we need to disable transformation for some condition: - -@snippet example_ngraph_utils.cpp ngraph:disable_callback - -In some cases, pass::Manager pipelines inside transformations may have transformations disabled by default but enabled inside plugins. - -@snippet example_ngraph_utils.cpp ngraph:disabled_by_default - -PassConfig instance taken from pass::Manager is shared across all registered transformations including nested transformations. So it does not matter where we work with this object (before passes registration or after). - -## Transformations testing - -If you are developing new transformation inside plugin, you need to add test into the `template_plugin/tests/functional/transformations` folder. -We have two types of tests: nGraph reader tests located in `src/tests/functional/inference_engine/ngraph_reader` and transformation tests located in `src/tests/functional/inference_engine/transformations` -Reader tests are IR based and test end-to-end conversion from IR to CNNNetwork. Transformation tests test single ngraph transformations or low-level functions that are used inside transformations. - -The basic transformation test looks like this: - -@snippet tests/functional/transformations/template_transformations_test.cpp transformation:test - - -[ngraph_replace_node]: ./img/ngraph_replace_node.png -[ngraph_insert_node]: ./img/ngraph_insert_node.png -[transformations_structure]: ./img/transformations_structure.png -[register_new_node]: ./img/register_new_node.png -[graph_rewrite_execution]: ./img/graph_rewrite_execution.png -[graph_rewrite_efficient_search]: ./img/graph_rewrite_efficient_search.png diff --git a/docs/OV_Runtime_UG/network_state_intro.md b/docs/OV_Runtime_UG/network_state_intro.md index 2a04dd05dc9..1721f6145e6 100644 --- a/docs/OV_Runtime_UG/network_state_intro.md +++ b/docs/OV_Runtime_UG/network_state_intro.md @@ -1,4 +1,4 @@ -Introduction to OpenVINO state API {#openvino_docs_IE_DG_network_state_intro} +Stateful models {#openvino_docs_IE_DG_network_state_intro} ============================== This section describes how to work with stateful networks in OpenVINO toolkit, specifically: @@ -15,7 +15,7 @@ The section additionally provides small examples of stateful network and code to between data portions should be addressed. For that, networks save some data between inferences - state. When one dependent sequence is over, state should be reset to initial value and new sequence can be started. - Several frameworks have special API for states in networks. For example, Keras have special option for RNNs `stateful` that turns on saving state + Several frameworks have special API for states in networks. For example, Keras has special option for RNNs `stateful` that turns on saving state between inferences. Kaldi contains special specifier `Offset` to define time offset in a network. OpenVINO also contains special API to simplify work with networks with states. State is automatically saved between inferences, @@ -196,9 +196,7 @@ sink from `ngraph::Function` after deleting the node from graph with the `delete Let's take an IR from the previous section example. The example below demonstrates inference of two independent sequences of data. State should be reset between these sequences. -One infer request and one thread -will be used in this example. Using several threads is possible if you have several independent sequences. Then each sequence can be processed in its own infer -request. Inference of one sequence in several infer requests is not recommended. In one infer request state will be saved automatically between inferences, but +One infer request and one thread will be used in this example. Using several threads is possible if you have several independent sequences. Then each sequence can be processed in its own infer request. Inference of one sequence in several infer requests is not recommended. In one infer request state will be saved automatically between inferences, but if the first step is done in one infer request and the second in another, state should be set in new infer request manually (using `IVariableState::SetState` method). @snippet openvino/docs/snippets/InferenceEngine_network_with_state_infer.cpp part1 @@ -213,7 +211,7 @@ Decsriptions can be found in [Samples Overview](./Samples_Overview.md) If the original framework does not have a special API for working with states, after importing the model, OpenVINO representation will not contain Assign/ReadValue layers. For example, if the original ONNX model contains RNN operations, IR will contain TensorIterator operations and the values will be obtained only after execution of the whole TensorIterator primitive. Intermediate values from each iteration will not be available. To enable you to work with these intermediate values of each iteration and receive them with a low latency after each infer request, special LowLatency and LowLatency2 transformations were introduced. -### How to get TensorIterator/Loop operaions from different frameworks via ModelOptimizer. +### How to get TensorIterator/Loop operations from different frameworks via ModelOptimizer. **ONNX and frameworks supported via ONNX format:** *LSTM, RNN, GRU* original layers are converted to the TensorIterator operation. TensorIterator body contains LSTM/RNN/GRU Cell. Peepholes, InputForget modifications are not supported, sequence_lengths optional input is supported. *ONNX Loop* layer is converted to the OpenVINO Loop operation. @@ -245,7 +243,7 @@ After applying the transformation, ReadValue operations can receive other operat 1. Get CNNNetwork. Either way is acceptable: - * [from IR or ONNX model](./Integrate_with_customer_application_new_API.md) + * [from IR or ONNX model](./integrate_with_your_application.md) * [from ov::Model](../OV_Runtime_UG/model_representation.md) 2. Change the number of iterations inside TensorIterator/Loop nodes in the network using the [Reshape](ShapeInference.md) feature. @@ -349,7 +347,7 @@ After applying the transformation, ReadValue operations can receive other operat 1. Get CNNNetwork. Either way is acceptable: - * [from IR or ONNX model](./Integrate_with_customer_application_new_API.md) + * [from IR or ONNX model](./integrate_with_your_application.md) * [from ov::Model](../OV_Runtime_UG/model_representation.md) 2. [Reshape](ShapeInference.md) the CNNNetwork network if necessary. **Necessary case:** where the sequence_lengths dimension of input > 1, it means TensorIterator layer will have number_iterations > 1. We should reshape the inputs of the network to set sequence_dimension to exactly 1. diff --git a/docs/OV_Runtime_UG/openvino_intro.md b/docs/OV_Runtime_UG/openvino_intro.md new file mode 100644 index 00000000000..a4a7ae8a716 --- /dev/null +++ b/docs/OV_Runtime_UG/openvino_intro.md @@ -0,0 +1,51 @@ +# OpenVINO™ Runtime User Guide {#openvino_docs_OV_Runtime_User_Guide} + +@sphinxdirective + +.. _deep learning inference engine: + +.. toctree:: + :maxdepth: 1 + :hidden: + + openvino_docs_Integrate_OV_with_your_application + + openvino_docs_IE_DG_ShapeInference + openvino_docs_OV_UG_Working_with_devices + openvino_docs_OV_Runtime_UG_Preprocessing_Overview + openvino_docs_OV_UG_DynamicShapes + openvino_docs_IE_DG_supported_plugins_AUTO + openvino_docs_OV_UG_Running_on_multiple_devices + openvino_docs_OV_UG_Hetero_execution + openvino_docs_OV_UG_Automatic_Batching + openvino_docs_IE_DG_network_state_intro + openvino_2_0_transition_guide + openvino_docs_OV_Should_be_in_performance + +@endsphinxdirective + +## Introduction +OpenVINO Runtime is a set of C++ libraries with C and Python bindings providing a common API to deliver inference solutions on the platform of your choice. Use the OpenVINO Runtime API to read an Intermediate Representation (IR), ONNX, or PaddlePaddle model and execute it on preferred devices. + +OpenVINO Runtime uses a plugin architecture. Its plugins are software components that contain complete implementation for inference on a particular Intel® hardware device: CPU, GPU, VPU, etc. Each plugin implements the unified API and provides additional hardware-specific APIs, for configuring devices, or API interoperability between OpenVINO Runtime and underlying plugin backend. + +The scheme below illustrates the typical workflow for deploying a trained deep learning model: + + +![](img/BASIC_FLOW_IE_C.svg) + + +## Video + +@sphinxdirective + +.. list-table:: + + * - .. raw:: html + + + * - **Inference Engine Concept**. Duration: 3:43 + +@endsphinxdirective diff --git a/docs/OV_Runtime_UG/openvino_temporary.md b/docs/OV_Runtime_UG/openvino_temporary.md new file mode 100644 index 00000000000..203f9dfd1e6 --- /dev/null +++ b/docs/OV_Runtime_UG/openvino_temporary.md @@ -0,0 +1,19 @@ +# Should be moved to performance / extensibility {#openvino_docs_OV_Should_be_in_performance} + +@sphinxdirective + +.. _deep learning inference engine: + +.. toctree:: + :maxdepth: 1 + :hidden: + + openvino_docs_deployment_optimization_guide_dldt_optimization_guide + openvino_docs_IE_DG_Model_caching_overview + openvino_docs_IE_DG_Int8Inference + openvino_docs_IE_DG_Bfloat16Inference + openvino_docs_OV_UG_NoDynamicShapes + +@endsphinxdirective + +## TEMP: should be moved to performance / extensibility guides diff --git a/docs/OV_Runtime_UG/ov_dynamic_shapes.md b/docs/OV_Runtime_UG/ov_dynamic_shapes.md new file mode 100644 index 00000000000..a2c046c961c --- /dev/null +++ b/docs/OV_Runtime_UG/ov_dynamic_shapes.md @@ -0,0 +1,122 @@ +# Dynamic Shapes {#openvino_docs_OV_UG_DynamicShapes} + +As it was demonstrated in the [Changing Input Shapes](ShapeInference.md) article, there are models that support changing of input shapes before model compilation in `Core::compile_model`. +Reshaping models provides an ability to customize the model input shape for exactly that size that is required in the end application. +This article explains how the ability of model to reshape can further be leveraged in more dynamic scenarios. + + +## When to Apply Dynamic Shapes + +Conventional "static" model reshaping works well when it can be done once per many model inference calls with the same shape. +However, this approach doesn't perform efficiently if the input tensor shape is changed on every inference call: calling `reshape()` and `compile_model()` each time when a new size comes is extremely time-consuming. +A popular example would be an inference of natural language processing models (like BERT) with arbitrarily-sized input sequences that come from the user. +In this case, the sequence length cannot be predicted and may change every time you need to call inference. +Below, such dimensions that can be frequently changed are called *dynamic dimensions*. +When real shape of input is not known at `compile_model` time, that's the case when dynamic shapes should be considered. + +Here are several examples of dimensions that can be naturally dynamic: + - Sequence length dimension for various sequence processing models, like BERT + - Spatial dimensions in segmentation and style transfer models + - Batch dimension + - Arbitrary number of detections in object detection models output + +There are various tricks to address input dynamic dimensions through combining multiple pre-reshaped models and input data padding. +The tricks are sensitive to model internals, do not always give optimal performance and cumbersome. +Short overview of the methods you can find [here](ov_without_dynamic_shapes.md). +Apply those methods only if native dynamic shape API described in the following sections doesn't work for you or doesn't give desired performance. + +The decision about using dynamic shapes should be based on proper benchmarking of real application with real data. +That's because unlike statically shaped models, inference of dynamically shaped ones takes different inference time depending on input data shape or input tensor content. + +## Dynamic Shapes without Tricks + +This section describes how to handle dynamically shaped models natively with OpenVINO Runtime API version 2022.1 and higher. +There are three main parts in the flow that differ from static shapes: + - configure the model + - prepare data for inference + - read resulting data after inference + +### Configure the Model + +To avoid the tricks mentioned in the previous section there is a way to directly specify one or multiple dimensions in the model inputs to be dynamic. +This is achieved with the same reshape method that is used for alternating static shape of inputs. +Dynamic dimensions are specified as `-1` or `ov::Dimension()` instead of a positive number used for static dimensions: + +@snippet snippets/ov_dynamic_shapes.cpp ov_dynamic_shapes:reshape_undefined + +To simplify the code, the examples assume that the model has a single input and single output. +However, there are no limitations on the number of inputs and outputs to apply dynamic shapes. + +### Undefined Dimensions "Out Of the Box" + +Dynamic dimensions may appear in the input model without calling reshape. +Many DL frameworks support undefined dimensions. +If such a model is converted with Model Optimizer or read directly by Core::read_model, undefined dimensions are preserved. +Such dimensions automatically treated as dynamic ones. +So you don't need to call reshape if undefined dimensions are already configured in the original model or in the IR file. + +If the input model has undefined dimensions that you are not going to change during the inference, you can set them to static values, using the same `reshape` method of the model. +From the API perspective any combination of dynamic and static dimensions can be configured. + +Model Optimizer provides capability to reshape the model during the conversion, including specifying dynamic dimensions. +Use this capability to save time on calling `reshape` method in the end application. + +### Dimension Bounds + +Besides marking a dimension just dynamic, you can also specify lower and/or upper bounds that define a range of allowed values for the dimension. +Bounds are coded as arguments for `ov::Dimension`: + +@snippet snippets/ov_dynamic_shapes.cpp ov_dynamic_shapes:reshape_bounds + +Information about bounds gives opportunity for the inference plugin to apply additional optimizations. +Using dynamic shapes assumes the plugins apply more loose optimization technique during model compilation +It may require more time/memory for model compilation and inference. +So providing any additional information like bounds can be beneficial. +For the same reason it is not recommended to leave dimensions as undefined without the real need. + +When specifying bounds, the lower bound is not so important as upper bound, because knowing of upper bound allows inference devices to more precisely allocate memory for intermediate tensors for inference and use lesser number of tuned kernels for different sizes. +Precisely speaking benefits of specifying lower or upper bound is device dependent. +Depending on the plugin specifying upper bounds can be required. +. +If users known lower and upper bounds for dimension it is recommended to specify them even when plugin can execute model without the bounds. + +### Setting Input Tensors + +Preparing model with the reshape method was the first step. +The second step is passing a tensor with an appropriate shape to infer request. +This is similar to [regular steps](integrate_with_your_application.md), but now we can pass tensors with different shapes for the same executable model and even for the same inference request: + +@snippet snippets/ov_dynamic_shapes.cpp ov_dynamic_shapes:set_input_tensor + +In the example above `set_input_tensor` is used to specify input tensors. +The real dimensions of the tensor is always static, because it is a concrete tensor and it doesn't have any dimension variations in contrast to model inputs. + +Similar to static shapes, `get_input_tensor` can be used instead of `set_input_tensor`. +In contrast to static input shapes, when using `get_input_tensor` for dynamic inputs, `set_shape` method for the returned tensor should be called to define the shape and allocate memory. +Without doing that, the tensor returned by `get_input_tensor` is an empty tensor, it's shape is not initialized and memory is not allocated, because infer request doesn't have information about real shape you are going to feed. +Setting shape for input tensor is required when the corresponding input has at least one dynamic dimension regardless of bounds information. +The following example makes the same sequence of two infer request as the previous example but using `get_input_tensor` instead of `set_input_tensor`: + +@snippet snippets/ov_dynamic_shapes.cpp ov_dynamic_shapes:get_input_tensor + +### Dynamic Shapes in Outputs + +Examples above handle correctly case when dynamic dimensions in output may be implied by propagating of dynamic dimension from the inputs. +For example, batch dimension in input shape is usually propagated through the whole model and appears in the output shape. +The same is true for other dimensions, like sequence length for NLP models or spatial dimensions for segmentation models, that are propagated through the entire network. + +Whether or not output has dynamic dimensions can be examined by querying output partial shape after model read or reshape. +The same is applicable for inputs. For example: + +@snippet snippets/ov_dynamic_shapes.cpp ov_dynamic_shapes:print_dynamic + +Appearing `?` or ranges like `1..10` means there are dynamic dimensions in corresponding inputs or outputs. + +Or more programmatically: + +@snippet snippets/ov_dynamic_shapes.cpp ov_dynamic_shapes:detect_dynamic + +If at least one dynamic dimension exists in output of the model, shape of the corresponding output tensor will be set as the result of inference call. +Before the first inference, memory for such a tensor is not allocated and has shape `[0]`. +If user call `set_output_tensor` with pre-allocated tensor, the inference will call `set_shape` internally, and the initial shape is replaced by the really calculated shape. +So setting shape for output tensors in this case is useful only if you want to pre-allocate enough memory for output tensor, because `Tensor`'s `set_shape` method will re-allocate memory only if new shape requires more storage. diff --git a/docs/OV_Runtime_UG/ov_infer_request.md b/docs/OV_Runtime_UG/ov_infer_request.md new file mode 100644 index 00000000000..c984a4e6a92 --- /dev/null +++ b/docs/OV_Runtime_UG/ov_infer_request.md @@ -0,0 +1,277 @@ +# OpenVINO™ Inference Request {#openvino_docs_OV_Runtime_UG_Infer_request} + +OpenVINO™ Runtime uses Infer Request mechanism which allows to run models on different devices in asynchronous or synchronous manners. +`ov::InferRequest` class is used for this purpose inside the OpenVINO™ Runtime. +This class allows to set and get data for model inputs, outputs and run inference for the model. + +## Creating Infer Request + +`ov::InferRequest` can be created from the `ov::CompiledModel`: + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_infer_request.cpp + :language: cpp + :fragment: [create_infer_request] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_infer_request.py + :language: python + :fragment: [create_infer_request] + +@endsphinxdirective + +## Run inference + +`ov::InferRequest` supports synchronous and asynchronous modes for inference. + +### Synchronous mode + +You can use `ov::InferRequest::infer()`, which blocks the application execution, to infer model in synchronous mode: + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_infer_request.cpp + :language: cpp + :fragment: [sync_infer] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_infer_request.py + :language: python + :fragment: [sync_infer] + +@endsphinxdirective + +### Asynchronous mode + +Asynchronous mode can improve overall frame-rate of the application, because rather than wait for inference to complete, the app can continue doing things on the host, while accelerator is busy. You can use `ov::InferRequest::start_async()` to infer model in asynchronous mode: + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_infer_request.cpp + :language: cpp + :fragment: [async_infer] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_infer_request.py + :language: python + :fragment: [async_infer] + +@endsphinxdirective + +Asynchronous mode supports two ways to wait inference results: + * `ov::InferRequest::wait_for()` - specify maximum duration in milliseconds to block for. The method is blocked until the specified timeout has elapsed, or the result becomes available, whichever comes first. + @sphinxdirective + + .. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_infer_request.cpp + :language: cpp + :fragment: [wait_for] + + .. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_infer_request.py + :language: python + :fragment: [wait_for] + + @endsphinxdirective + * `ov::InferRequest::wait()` - waits until inference result becomes available + @sphinxdirective + + .. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_infer_request.cpp + :language: cpp + :fragment: [wait] + + .. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_infer_request.py + :language: python + :fragment: [wait] + + @endsphinxdirective + +Both requests are thread-safe: can be called from different threads without fearing corruption and failures. + +Also InferRequest provides an functionality which allows to avoid a call of `ov::InferRequest::wait()`, in order to do it, you can use `ov::InferRequest::set_callback()` method. This method allows to set callback which will be called after completing run of InferRequest, please use weak reference of infer_request (`ov::InferRequest*`, `ov::InferRequest&`, `std::weal_ptr` and etc) in the callback, it is needed to avoid cyclic references. +For more details please take a look too [Classification Sample Async](../../samples/cpp/classification_sample_async/README.md). + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_infer_request.cpp + :language: cpp + :fragment: [set_callback] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_infer_request.py + :language: python + :fragment: [set_callback] + +@endsphinxdirective + +You can use `ov::InferRequest::cancel()` method in case if you want to cancel the current inference request: + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_infer_request.cpp + :language: cpp + :fragment: [cancel] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_infer_request.py + :language: python + :fragment: [cancel] + +@endsphinxdirective + +## Working with Input and Output tensors + +`ov::InferRequest` allows to get input/output tensors by tensor name, index, port and without any arguments in case if model has only one input or output. + + * `ov::InferRequest::get_input_tensor()`, `ov::InferRequest::set_input_tensor()`, `ov::InferRequest::get_output_tensor()`, `ov::InferRequest::set_output_tensor()` methods without arguments can be used to get or set input/output tensor for model with only one input/output: + @sphinxdirective + + .. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_infer_request.cpp + :language: cpp + :fragment: [get_set_one_tensor] + + .. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_infer_request.py + :language: python + :fragment: [get_set_one_tensor] + + @endsphinxdirective + + * `ov::InferRequest::get_input_tensor()`, `ov::InferRequest::set_input_tensor()`, `ov::InferRequest::get_output_tensor()`, `ov::InferRequest::set_output_tensor()` methods with argument can be used to get or set input/output tensor by input/output index: + @sphinxdirective + + .. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_infer_request.cpp + :language: cpp + :fragment: [get_set_index_tensor] + + .. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_infer_request.py + :language: python + :fragment: [get_set_index_tensor] + + @endsphinxdirective + + * `ov::InferRequest::get_tensor()`, `ov::InferRequest::set_tensor()` methods can be used to get or set input/output tensor by tensor name: + @sphinxdirective + + .. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_infer_request.cpp + :language: cpp + :fragment: [get_set_tensor] + + .. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_infer_request.py + :language: python + :fragment: [get_set_tensor] + + @endsphinxdirective + + * `ov::InferRequest::get_tensor()`, `ov::InferRequest::set_tensor()` methods can be used to get or set input/output tensor by port: + @sphinxdirective + + .. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_infer_request.cpp + :language: cpp + :fragment: [get_set_tensor_by_port] + + .. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_infer_request.py + :language: python + :fragment: [get_set_tensor_by_port] + + @endsphinxdirective + +## Examples of InferRequest usages + +### Cascade of models + +`ov::InferRequest` can be used to organize cascade of models. You need to have infer requests for each model. +In this case you can get output tensor from the first request using `ov::InferRequest::get_tensor()` and set it as input for the second request using `ov::InferRequest::set_tensor()`. But be careful, shared tensors across compiled models can be rewritten by the first model if the first infer request is run once again, while the second model has not started yet. + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_infer_request.cpp + :language: cpp + :fragment: [cascade_models] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_infer_request.py + :language: python + :fragment: [cascade_models] + +@endsphinxdirective + +### Using of ROI tensors + +It is possible to re-use shared input by several models. You do not need to allocate separate input tensor for a model if it processes a ROI object located inside of already allocated input of a previous model. For instance, when first model detects objects on a video frame (stored as input tensor) and second model accepts detected bounding boxes (ROI inside of the frame) as input. In this case, it is allowed to re-use pre-allocated input tensor (used by first model) by second model and just crop ROI without allocation of new memory using `ov::Tensor()` with passing of `ov::Tensor` and `ov::Coordinate` as parameters. + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_infer_request.cpp + :language: cpp + :fragment: [roi_tensor] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_infer_request.py + :language: python + :fragment: [roi_tensor] + +@endsphinxdirective + +### Using of remote tensors + +You can create a remote tensor to work with remote device memory. `ov::RemoteContext` allows to create remote tensor. + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_infer_request.cpp + :language: cpp + :fragment: [remote_tensor] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_infer_request.py + :language: python + :fragment: [remote_tensor] + +@endsphinxdirective diff --git a/docs/OV_Runtime_UG/ov_without_dynamic_shapes.md b/docs/OV_Runtime_UG/ov_without_dynamic_shapes.md new file mode 100644 index 00000000000..8e07d1b7821 --- /dev/null +++ b/docs/OV_Runtime_UG/ov_without_dynamic_shapes.md @@ -0,0 +1,44 @@ +# When Dynamic Shapes API is Not Applicable {#openvino_docs_OV_UG_NoDynamicShapes} + +Several approaches to emulate dynamic shapes are considered in this chapter +Apply these methods only if [native dynamic shape API](ov_dynamic_shapes.md) doesn't work for you or doesn't give desired performance. + +## Padding + +The model can be designed in a way that supports partially filled tensors. +For the BERT model you can use a special input to the model to mask unused elements out. +So, the model can be reshaped for some predefined big sequence length once and compiled once, and then the input tensors are used only partially with mask specifying valid tokens. +This approach is called *padding*. + +However, padding is not applicable to every model and every use case. +You should be aware of model internals to apply padding. Otherwise, if the model is not designed to handle dummy element gracefully in padding area, +then the results of inference may be totally scrambled, +or accuracy is significantly affected. +Model can even crash during inference. + +Besides the bad developer experience, +the main disadvantage of padding is a bad performance due to spending time for processing dummy elements in the padding area, +even if the model is properly designed to be used with padding. +It turns out that usually such models are designed in a way where calculations in the padded area still happen not affecting the end result. + +## Multiple Precompiled Models + +Another approach to handle arbitrary sized inputs is to pre-compile several models reshaped for different input shapes. +This method works well if the number of different shapes is small enough to afford increased time for multiple reshapes and compilations +as well as increased amount of consumed memory. +As this method cannot be scaled well it is used in combination with the padding: +model with the most suitable input shape among pre-reshaped models is chosen. +It gives smaller pad area in comparison to a single model. + +## Dimension Partitioning + +Another practical but still a complicated approach is when the input tensor can be divided into multiple chunks along the dynamic dimension. +For example, if we have a batch of independent inputs as a single tensor. +If arbitrary division along batch dimension is possible - and for batch dimension it should be possible by the dimension purpose - +you can run multiple inferences using the approach with several pre-compiled models choosing sized to have the minimal number of inferences +having a particular batch size in the input tensor. + +For example, if there are models pre-compiled for batch sizes 1, 2, 4 and 8, +the input tensor with batch 5 can be processed with two inference calls with batch size 1 and 4. +(Here it's assumed the batch processing is required for performance reasons, otherwise you can just loop over images in a batch, +and process image by image with a single compiled model.) diff --git a/docs/OV_Runtime_UG/protecting_model_guide.md b/docs/OV_Runtime_UG/protecting_model_guide.md index e710eeeb668..222bdb90ffc 100644 --- a/docs/OV_Runtime_UG/protecting_model_guide.md +++ b/docs/OV_Runtime_UG/protecting_model_guide.md @@ -16,22 +16,22 @@ This guide demonstrates how to use OpenVINO securely with protected models. After a model is optimized by the OpenVINO Model Optimizer, it's deployed to target devices in the Intermediate Representation (IR) format. An optimized -model is stored on an edge device and executed by the Inference Engine. -(ONNX and nGraph models can also be read natively by the Inference Engine.) +model is stored on an edge device and executed by the OpenVINO Runtime. +(ONNX, PDPD models can also be read natively by the OpenVINO Runtime.) To protect deep-learning models, you can encrypt an optimized model before deploying it to the edge device. The edge device should keep the stored model protected at all times and have the model decrypted **in runtime only** for use -by the Inference Engine. +by the OpenVINO Runtime. ![deploy_encrypted_model](img/deploy_encrypted_model.png) ## Loading Encrypted Models -The OpenVINO Inference Engine requires model decryption before loading. Allocate +The OpenVINO Runtime requires model decryption before loading. Allocate a temporary memory block for model decryption and use the -`InferenceEngine::Core::ReadNetwork` method to load the model from a memory buffer. -For more information, see the `InferenceEngine::Core` Class Reference Documentation. +`ov::Core::read_model` method to load the model from a memory buffer. +For more information, see the `ov::Core` Class Reference Documentation. @snippet snippets/protecting_model_guide.cpp part0 @@ -40,12 +40,12 @@ Hardware-based protection such as Intel® Software Guard Extensions bind them to a device. For more information, go to [Intel® Software Guard Extensions](https://software.intel.com/en-us/sgx). -Use `InferenceEngine::Core::ReadNetwork()` to set model representations and +Use `ov::Core::read_model` to set model representations and weights respectively. Currently there is no way to read external weights from memory for ONNX models. -The `ReadNetwork(const std::string& model, const Blob::CPtr& weights)` function -should be called with `weights` passed as an empty `Blob`. +The `ov::Core::read_model(const std::string& model, const Tensor& weights)` method +should be called with `weights` passed as an empty `ov::Tensor`. @snippet snippets/protecting_model_guide.cpp part1 @@ -54,7 +54,7 @@ should be called with `weights` passed as an empty `Blob`. - Intel® Distribution of OpenVINO™ toolkit home page: [https://software.intel.com/en-us/openvino-toolkit](https://software.intel.com/en-us/openvino-toolkit) - OpenVINO™ toolkit online documentation: [https://docs.openvino.ai](https://docs.openvino.ai) - Model Optimizer Developer Guide: [Model Optimizer Developer Guide](../MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) -- [OpenVINO™ runTime User Guide](OpenVINO_Runtime_User_Guide.md) -- For more information on Sample Applications, see the [Inference Engine Samples Overview](Samples_Overview.md) +- [OpenVINO™ runTime User Guide](openvino_intro.md) +- For more information on Sample Applications, see the [OpenVINO Samples Overview](Samples_Overview.md) - For information on a set of pre-trained models, see the [Overview of OpenVINO™ Toolkit Pre-Trained Models](@ref omz_models_group_intel) - For IoT Libraries and Code Samples see the [Intel® IoT Developer Kit](https://github.com/intel-iot-devkit). diff --git a/docs/OV_Runtime_UG/supported_plugins/CPU.md b/docs/OV_Runtime_UG/supported_plugins/CPU.md index 08eafe3ac0d..950b46c2dec 100644 --- a/docs/OV_Runtime_UG/supported_plugins/CPU.md +++ b/docs/OV_Runtime_UG/supported_plugins/CPU.md @@ -1,4 +1,4 @@ -# CPU Plugin {#openvino_docs_IE_DG_supported_plugins_CPU} +# CPU device {#openvino_docs_OV_UG_supported_plugins_CPU} ## Introducing the CPU Plugin @@ -6,7 +6,7 @@ The CPU plugin was developed to achieve high performance of neural networks on C Currently, the CPU plugin uses Intel® Threading Building Blocks (Intel® TBB) in order to parallelize calculations. Please refer to the [Optimization Guide](../../optimization_guide/dldt_optimization_guide.md) for associated performance considerations. -The set of supported layers can be expanded with [the Extensibility mechanism](../Extensibility_DG/Intro.md). +The set of supported layers can be expanded with [the Extensibility mechanism](../../Extensibility_UG/Intro.md). ## Supported Platforms diff --git a/docs/OV_Runtime_UG/supported_plugins/Device_Plugins.md b/docs/OV_Runtime_UG/supported_plugins/Device_Plugins.md index ada0e6ce977..30ba9de15ba 100644 --- a/docs/OV_Runtime_UG/supported_plugins/Device_Plugins.md +++ b/docs/OV_Runtime_UG/supported_plugins/Device_Plugins.md @@ -1,4 +1,4 @@ -# Device Plugin Support {#openvino_docs_IE_DG_Device_Plugins} +# Working with devices {#openvino_docs_OV_UG_Working_with_devices} @sphinxdirective @@ -6,27 +6,31 @@ :maxdepth: 1 :hidden: - openvino_docs_IE_DG_InferenceEngine_QueryAPI - openvino_docs_IE_DG_supported_plugins_CPU - openvino_docs_IE_DG_supported_plugins_GPU + openvino_docs_OV_UG_query_api + openvino_docs_OV_UG_supported_plugins_CPU + openvino_docs_OV_UG_supported_plugins_GPU openvino_docs_IE_DG_supported_plugins_VPU - openvino_docs_IE_DG_supported_plugins_GNA + openvino_docs_OV_UG_supported_plugins_GNA @endsphinxdirective -Inference Engine uses a plugin architecture. Inference Engine plugin is a software component that contains complete implementation for inference on a certain Intel® hardware device: CPU, GPU, VPU, GNA, etc. Each plugin implements the unified API and provides additional hardware-specific APIs. - -The Inference Engine provides capabilities to infer deep learning models on the following device types with corresponding plugins: +The OpenVINO Runtime provides capabilities to infer deep learning models on the following device types with corresponding plugins: | Plugin | Device types | |------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| -|[GPU plugin](GPU.md) |Intel® Processor Graphics, including Intel® HD Graphics and Intel® Iris® Graphics | |[CPU plugin](CPU.md) |Intel® Xeon® with Intel® Advanced Vector Extensions 2 (Intel® AVX2), Intel® Advanced Vector Extensions 512 (Intel® AVX-512), and AVX512_BF16, Intel® Core™ Processors with Intel® AVX2, Intel® Atom® Processors with Intel® Streaming SIMD Extensions (Intel® SSE) | +|[GPU plugin](GPU.md) |Intel® Processor Graphics, including Intel® HD Graphics and Intel® Iris® Graphics | |[VPU plugins](VPU.md) (available in the Intel® Distribution of OpenVINO™ toolkit) |Intel® Neural Compute Stick 2 powered by the Intel® Movidius™ Myriad™ X, Intel® Vision Accelerator Design with Intel® Movidius™ VPUs | |[GNA plugin](GNA.md) (available in the Intel® Distribution of OpenVINO™ toolkit) |Intel® Speech Enabling Developer Kit, Amazon Alexa* Premium Far-Field Developer Kit, Intel® Pentium® Silver J5005 Processor, Intel® Pentium® Silver N5000 Processor, Intel® Celeron® J4005 Processor, Intel® Celeron® J4105 Processor, Intel® Celeron® Processor N4100, Intel® Celeron® Processor N4000, Intel® Core™ i3-8121U Processor, Intel® Core™ i7-1065G7 Processor, Intel® Core™ i7-1060G7 Processor, Intel® Core™ i5-1035G4 Processor, Intel® Core™ i5-1035G7 Processor, Intel® Core™ i5-1035G1 Processor, Intel® Core™ i5-1030G7 Processor, Intel® Core™ i5-1030G4 Processor, Intel® Core™ i3-1005G1 Processor, Intel® Core™ i3-1000G1 Processor, Intel® Core™ i3-1000G4 Processor| -|[Multi-Device plugin](MULTI.md) |Multi-Device plugin enables simultaneous inference of the same network on several Intel® devices in parallel | -|[Auto-Device plugin](AUTO.md) |Auto-Device plugin enables selecting Intel® device for inference automatically | -|[Heterogeneous plugin](HETERO.md) |Heterogeneous plugin enables automatic inference splitting between several Intel® devices (for example if a device doesn't [support certain layers](#supported-layers)). | + +OpenVINO runtime also has several execution capabilities which work on top of other devices: + +| Capability | Description | +|------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| +|[Multi-Device execution](../multi_device.md) |Multi-Device enables simultaneous inference of the same model on several devices in parallel | +|[Auto-Device selection](../auto_device_selection.md) |Auto-Device selection enables selecting Intel® device for inference automatically | +|[Heterogeneous execution](../hetero_execution.md) |Heterogeneous execution enables automatic inference splitting between several devices (for example if a device doesn't [support certain operation](#supported-layers))| +|[Automatic Batching](../automatic_batching.md) | Auto-Batching plugin enables the batching (on top of the specified device) that is completely transparent to the application | Devices similar to the ones we have used for benchmarking can be accessed using [Intel® DevCloud for the Edge](https://devcloud.intel.com/edge/), a remote development environment with access to Intel® hardware and the latest versions of the Intel® Distribution of the OpenVINO™ Toolkit. [Learn more](https://devcloud.intel.com/edge/get_started/devcloud/) or [Register here](https://inteliot.force.com/DevcloudForEdge/s/). diff --git a/docs/OV_Runtime_UG/supported_plugins/GNA.md b/docs/OV_Runtime_UG/supported_plugins/GNA.md index 4b177d98c31..d54ff3c5e48 100644 --- a/docs/OV_Runtime_UG/supported_plugins/GNA.md +++ b/docs/OV_Runtime_UG/supported_plugins/GNA.md @@ -1,4 +1,4 @@ -# GNA Plugin {#openvino_docs_IE_DG_supported_plugins_GNA} +# GNA device {#openvino_docs_OV_UG_supported_plugins_GNA} ## Introducing the GNA Plugin The Intel® Gaussian & Neural Accelerator is a low-power neural coprocessor for continuous inference at the edge. @@ -225,10 +225,10 @@ For the list of supported layers, see the **GNA** column of the **Supported Laye Limitations include: -- Only 1D convolutions are natively supported. +- Only 1D convolutions are natively supported on the HW prior to GNA 3.0; 2D convolutions have specific limitations (see the table above). - The number of output channels for convolutions must be a multiple of 4. - The maximum number of filters is 65532 for GNA 2.0 and 8192 for GNA 3.0. -- Permute layer support is limited to the cases where no data reordering is needed or when reordering is happening for two dimensions, at least one of which is not greater than 8. +- Transpose layer support is limited to the cases where no data reordering is needed or when reordering is happening for two dimensions, at least one of which is not greater than 8. - Splits and concatenations are supported for continuous portions of memory (e.g., split of 1,2,3,4 to 1,1,3,4 and 1,1,3,4 or concats of 1,2,3,4 and 1,2,3,5 to 2,2,3,4). - For Multiply, Add and Subtract layers, auto broadcasting is only supported for constant inputs. @@ -236,13 +236,13 @@ Limitations include: The Intel® GNA 1.0 and 2.0 hardware natively supports only 1D convolutions. -However, 2D convolutions can be mapped to 1D when a convolution kernel moves in a single direction. GNA Plugin performs such a transformation for Kaldi `nnet1` convolution. From this perspective, the Intel® GNA hardware convolution operation accepts an `NHWC` input and produces an `NHWC` output. Because OpenVINO™ only supports the `NCHW` layout, you may need to insert `Permute` layers before or after convolutions. +However, 2D convolutions can be mapped to 1D when a convolution kernel moves in a single direction. GNA Plugin performs such a transformation for Kaldi `nnet1` convolution. From this perspective, the Intel® GNA hardware convolution operation accepts an `NHWC` input and produces an `NHWC` output. Because OpenVINO™ only supports the `NCHW` layout, you may need to insert `Transpose` layers before or after convolutions. -For example, the Kaldi model optimizer inserts such a permute after convolution for the [rm_cnn4a network](https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/rm_cnn4a_smbr/). This `Permute` layer is automatically removed by the GNA Plugin, because the Intel® GNA hardware convolution layer already produces the required `NHWC` result. +For example, the Kaldi model optimizer inserts such a transpose after convolution for the [rm_cnn4a network](https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/rm_cnn4a_smbr/). This `Transpose` layer is automatically removed by the GNA Plugin, because the Intel® GNA hardware convolution layer already produces the required `NHWC` result. ## Operation Precision -Intel® GNA essentially operates in the low-precision mode, which represents a mix of 8-bit (`I8`), 16-bit (`I16`), and 32-bit (`I32`) integer computations. Outputs calculated using a reduced integer precision are different from the scores calculated using the floating point format, for example, `FP32` outputs calculated on CPU using the Inference Engine [CPU Plugin](CPU.md). +Intel® GNA essentially operates in the low-precision mode, which represents a mix of 8-bit (`I8`), 16-bit (`I16`), and 32-bit (`I32`) integer computations. Outputs calculated using a reduced integer precision are different from the scores calculated using the floating point format, for example, `FP32` outputs calculated on CPU using the OpenVINO [CPU device](CPU.md). Unlike other plugins supporting low-precision execution, the GNA plugin can calculate quantization factors at the model loading time, so you can run a model without calibration using the [Post-Training Optimization Tool](@ref pot_README). However, this mode may not provide satisfactory accuracy because the internal quantization algorithm is based on heuristics which may or may not be efficient, depending on the model and dynamic range of input data. diff --git a/docs/OV_Runtime_UG/supported_plugins/GPU.md b/docs/OV_Runtime_UG/supported_plugins/GPU.md index 8e55e4f647d..06e110e26f0 100644 --- a/docs/OV_Runtime_UG/supported_plugins/GPU.md +++ b/docs/OV_Runtime_UG/supported_plugins/GPU.md @@ -1,4 +1,4 @@ -# GPU Plugin {#openvino_docs_IE_DG_supported_plugins_GPU} +# GPU device {#openvino_docs_OV_UG_supported_plugins_GPU} @sphinxdirective @@ -6,14 +6,14 @@ :maxdepth: 1 :hidden: - openvino_docs_IE_DG_supported_plugins_GPU_RemoteBlob_API + openvino_docs_OV_UG_supported_plugins_GPU_RemoteBlob_API @endsphinxdirective The GPU plugin uses the Intel® Compute Library for Deep Neural Networks (clDNN) to infer deep neural networks. clDNN is an open source performance library for Deep Learning (DL) applications intended for acceleration of Deep Learning Inference on Intel® Processor Graphics including Intel® HD Graphics, Intel® Iris® Graphics, Intel® Iris® Xe Graphics, and Intel® Iris® Xe MAX graphics. -For an in-depth description of clDNN, see [Inference Engine source files](https://github.com/openvinotoolkit/openvino/tree/master/src/plugins/intel_gpu/) and [Accelerate Deep Learning Inference with Intel® Processor Graphics](https://software.intel.com/en-us/articles/accelerating-deep-learning-inference-with-intel-processor-graphics). +For an in-depth description of clDNN, see [OpenVINO Runtime GPU plugin source files](https://github.com/openvinotoolkit/openvino/tree/master/src/plugins/intel_gpu/) and [Accelerate Deep Learning Inference with Intel® Processor Graphics](https://software.intel.com/en-us/articles/accelerating-deep-learning-inference-with-intel-processor-graphics). ## Device Naming Convention * Devices are enumerated as "GPU.X" where `X={0, 1, 2,...}`. Only Intel® GPU devices are considered. diff --git a/docs/OV_Runtime_UG/supported_plugins/GPU_RemoteBlob_API.md b/docs/OV_Runtime_UG/supported_plugins/GPU_RemoteBlob_API.md index a5232b580d9..ffb287f47d9 100644 --- a/docs/OV_Runtime_UG/supported_plugins/GPU_RemoteBlob_API.md +++ b/docs/OV_Runtime_UG/supported_plugins/GPU_RemoteBlob_API.md @@ -1,4 +1,4 @@ -Remote Blob API of GPU Plugin {#openvino_docs_IE_DG_supported_plugins_GPU_RemoteBlob_API} +Remote Blob API of GPU Plugin {#openvino_docs_OV_UG_supported_plugins_GPU_RemoteBlob_API} ================================ The GPU plugin implementation of the `RemoteContext` and `RemoteBlob` interfaces supports GPU diff --git a/docs/OV_Runtime_UG/supported_plugins/HDDL.md b/docs/OV_Runtime_UG/supported_plugins/HDDL.md index 6d7784aa9ad..bb05719a223 100644 --- a/docs/OV_Runtime_UG/supported_plugins/HDDL.md +++ b/docs/OV_Runtime_UG/supported_plugins/HDDL.md @@ -1,13 +1,17 @@ -# HDDL Plugin {#openvino_docs_IE_DG_supported_plugins_HDDL} +# HDDL device {#openvino_docs_OV_UG_supported_plugins_HDDL} ## Introducing the HDDL Plugin -The Inference Engine HDDL plugin was developed for inference with neural networks on Intel® Vision Accelerator Design with Intel® Movidius™ VPUs. It is designed for use cases that require large throughput for deep learning inference, up to dozens of times more than the MYRIAD Plugin. +The OpenVINO Runtime HDDL plugin was developed for inference with neural networks on Intel® Vision Accelerator Design with Intel® Movidius™ VPUs. It is designed for use cases that require large throughput for deep learning inference, up to dozens of times more than the MYRIAD Plugin. ## Configuring the HDDL Plugin To configure your Intel® Vision Accelerator Design With Intel® Movidius™ on supported operating systems, refer to the Steps for Intel® Vision Accelerator Design with Intel® Movidius™ VPUs section in the installation guides for [Linux](../../install_guides/installing-openvino-linux.md) or [Windows](../../install_guides/installing-openvino-windows.md). +> **NOTE**: The HDDL and Myriad plugins may cause conflicts when used at the same time. +> To ensure proper operation in such a case, the number of booted devices needs to be limited in the 'hddl_autoboot.config' file. +> Otherwise, the HDDL plugin will boot all available Intel® Movidius™ Myriad™ X devices. + ## Supported networks To see the list of supported networks for the HDDL plugin, refer to the list on the [MYRIAD Plugin page](MYRIAD.md). diff --git a/docs/OV_Runtime_UG/supported_plugins/MYRIAD.md b/docs/OV_Runtime_UG/supported_plugins/MYRIAD.md index 53addae9c51..d6ecdf5ce51 100644 --- a/docs/OV_Runtime_UG/supported_plugins/MYRIAD.md +++ b/docs/OV_Runtime_UG/supported_plugins/MYRIAD.md @@ -1,13 +1,17 @@ -# MYRIAD Plugin {#openvino_docs_IE_DG_supported_plugins_MYRIAD} +# MYRIAD device {#openvino_docs_OV_UG_supported_plugins_MYRIAD} ## Introducing MYRIAD Plugin -The Inference Engine MYRIAD plugin has been developed for inference of neural networks on Intel® Neural Compute Stick 2. +The OpenVINO Runtime MYRIAD plugin has been developed for inference of neural networks on Intel® Neural Compute Stick 2. ## Configuring the MYRIAD Plugin To configure your Intel® Vision Accelerator Design With Intel® Movidius™ on supported operating systemss, refer to the Steps for Intel® Vision Accelerator Design with Intel® Movidius™ VPUs section in the installation guides for [Linux](../../install_guides/installing-openvino-linux.md) or [Windows](../../install_guides/installing-openvino-windows.md). + > **NOTE**: The HDDL and MYRIAD plugins may cause conflicts when used at the same time. +> To ensure proper operation in such a case, the number of booted devices needs to be limited in the 'hddl_autoboot.config' file. +> Otherwise, the HDDL plugin will boot all available Intel® Movidius™ Myriad™ X devices. + ## Supported Configuration Parameters See VPU common configuration parameters for the [VPU Plugins](VPU.md). diff --git a/docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md b/docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md index ad13af79d3f..0c200186708 100644 --- a/docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md +++ b/docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md @@ -1,11 +1,11 @@ Supported Devices {#openvino_docs_IE_DG_supported_plugins_Supported_Devices} ================== -The Inference Engine can infer models in different formats with various input and output formats. This section provides supported and optimal configurations per device. In OpenVINO™ documentation, "device" refers to an Intel® processors used for inference, which can be a supported CPU, GPU, VPU (vision processing unit), or GNA (Gaussian neural accelerator coprocessor), or a combination of those devices. +The OpenVINO Runtime can infer models in different formats with various input and output formats. This section provides supported and optimal configurations per device. In OpenVINO™ documentation, "device" refers to an Intel® processors used for inference, which can be a supported CPU, GPU, VPU (vision processing unit), or GNA (Gaussian neural accelerator coprocessor), or a combination of those devices. > **NOTE**: With OpenVINO™ 2020.4 release, Intel® Movidius™ Neural Compute Stick is no longer supported. -The Inference Engine provides unique capabilities to infer deep learning models on the following device types with corresponding plugins: +The OpenVINO Runtime provides unique capabilities to infer deep learning models on the following device types with corresponding plugins: | Plugin | Device types | |------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -13,9 +13,9 @@ The Inference Engine provides unique capabilities to infer deep learning models |[CPU plugin](CPU.md) |Intel® Xeon® with Intel® Advanced Vector Extensions 2 (Intel® AVX2), Intel® Advanced Vector Extensions 512 (Intel® AVX-512), and AVX512_BF16, Intel® Core™ Processors with Intel® AVX2, Intel® Atom® Processors with Intel® Streaming SIMD Extensions (Intel® SSE) | |[VPU plugins](VPU.md) (available in the Intel® Distribution of OpenVINO™ toolkit) |Intel® Neural Compute Stick 2 powered by the Intel® Movidius™ Myriad™ X, Intel® Vision Accelerator Design with Intel® Movidius™ VPUs | |[GNA plugin](GNA.md) (available in the Intel® Distribution of OpenVINO™ toolkit) |Intel® Speech Enabling Developer Kit, Amazon Alexa* Premium Far-Field Developer Kit, Intel® Pentium® Silver J5005 Processor, Intel® Pentium® Silver N5000 Processor, Intel® Celeron® J4005 Processor, Intel® Celeron® J4105 Processor, Intel® Celeron® Processor N4100, Intel® Celeron® Processor N4000, Intel® Core™ i3-8121U Processor, Intel® Core™ i7-1065G7 Processor, Intel® Core™ i7-1060G7 Processor, Intel® Core™ i5-1035G4 Processor, Intel® Core™ i5-1035G7 Processor, Intel® Core™ i5-1035G1 Processor, Intel® Core™ i5-1030G7 Processor, Intel® Core™ i5-1030G4 Processor, Intel® Core™ i3-1005G1 Processor, Intel® Core™ i3-1000G1 Processor, Intel® Core™ i3-1000G4 Processor| -|[Multi-Device plugin](MULTI.md) |Multi-Device plugin enables simultaneous inference of the same network on several Intel® devices in parallel | -|[Auto-Device plugin](AUTO.md) |Auto-Device plugin enables selecting Intel® device for inference automatically | -|[Heterogeneous plugin](HETERO.md) |Heterogeneous plugin enables automatic inference splitting between several Intel® devices (for example if a device doesn't [support certain layers](#supported-layers)). | +|[Multi-Device execution](../multi_device.md) |Multi-Device execution enables simultaneous inference of the same model on several devices in parallel | +|[Auto-Device plugin](../auto_device_selection.md) |Auto-Device plugin enables selecting Intel® device for inference automatically | +|[Heterogeneous plugin](../hetero_execution.md) |Heterogeneous execution enables automatic inference splitting between several devices (for example if a device doesn't [support certain operation](#supported-layers)). | Devices similar to the ones we have used for benchmarking can be accessed using [Intel® DevCloud for the Edge](https://devcloud.intel.com/edge/), a remote development environment with access to Intel® hardware and the latest versions of the Intel® Distribution of the OpenVINO™ Toolkit. [Learn more](https://devcloud.intel.com/edge/get_started/devcloud/) or [Register here](https://inteliot.force.com/DevcloudForEdge/s/). @@ -35,7 +35,7 @@ The table below shows the plugin libraries and additional dependencies for Linux ## Supported Configurations -The Inference Engine can inference models in different formats with various input and output formats. +The OpenVINO Runtime can inference models in different formats with various input and output formats. This page shows supported and optimal configurations for each plugin. ### Terminology @@ -73,7 +73,7 @@ For example, the CHW value at index (c,h,w) is physically located at index (c\*H |VPU plugins |Not supported |Supported |Not supported | |GNA plugin |Supported |Supported |Not supported |
\* - currently, only limited set of topologies might benefit from enabling I8 model on GPU
-For [Multi-Device](MULTI.md) and [Heterogeneous](HETERO.md) execution +For [Multi-Device](../multi_device.md) and [Heterogeneous](../hetero_execution.md) executions the supported models formats depends on the actual underlying devices. _Generally, FP16 is preferable as it is most ubiquitous and performant_. ### Supported Input Precision @@ -86,7 +86,7 @@ the supported models formats depends on the actual underlying devices. _Generall |GNA plugin |Supported |Not supported |Supported |Not supported |Supported |Supported |
\* - Supported via `SetBlob` only, `GetBlob` returns FP32
-For [Multi-Device](MULTI.md) and [Heterogeneous](HETERO.md) execution +For [Multi-Device](../multi_device.md) and [Heterogeneous](../hetero_execution.md) executions the supported input precision depends on the actual underlying devices. _Generally, U8 is preferable as it is most ubiquitous_. ### Supported Output Precision @@ -97,7 +97,7 @@ the supported input precision depends on the actual underlying devices. _Genera |GPU plugin |Supported |Supported | |VPU plugins |Supported |Supported | |GNA plugin |Supported |Not supported | -For [Multi-Device](MULTI.md) and [Heterogeneous](HETERO.md) execution +For [Multi-Device](../multi_device.md) and [Heterogeneous](../hetero_execution.md) executions the supported output precision depends on the actual underlying devices. _Generally, FP32 is preferable as it is most ubiquitous_. ### Supported Input Layout @@ -116,7 +116,7 @@ the supported output precision depends on the actual underlying devices. _Gener |Layout |NCDHW|NCHW |CHW |NC |C | For setting relevant configuration, refer to the -[Integrate with Customer Application New Request API](../Integrate_with_customer_application_new_API.md) topic +[Integrate with Customer Application](../integrate_with_your_application.md) topic (step 3 "Configure input and output"). ### Supported Layers @@ -271,6 +271,6 @@ The following layers are supported by the plugins and by [Shape Inference featur \*- support is limited to the specific parameters. Refer to "Known Layers Limitation" section for the device [from the list of supported](Supported_Devices.md). -\*\*- support is implemented via [Extensibility mechanism](../Extensibility_DG/Intro.md). +\*\*- support is implemented via [Extensibility mechanism](../../Extensibility_UG/Intro.md). \*\*\*- supports NCDHW layout. diff --git a/docs/OV_Runtime_UG/supported_plugins/VPU.md b/docs/OV_Runtime_UG/supported_plugins/VPU.md index 62a390fdbb5..e1b2efcdf91 100644 --- a/docs/OV_Runtime_UG/supported_plugins/VPU.md +++ b/docs/OV_Runtime_UG/supported_plugins/VPU.md @@ -1,4 +1,4 @@ -# VPU Plugins {#openvino_docs_IE_DG_supported_plugins_VPU} +# VPU devices {#openvino_docs_IE_DG_supported_plugins_VPU} @sphinxdirective @@ -6,12 +6,12 @@ :maxdepth: 1 :hidden: - openvino_docs_IE_DG_supported_plugins_MYRIAD - openvino_docs_IE_DG_supported_plugins_HDDL + openvino_docs_OV_UG_supported_plugins_MYRIAD + openvino_docs_OV_UG_supported_plugins_HDDL @endsphinxdirective -This chapter provides information on the Inference Engine plugins that enable inference of deep learning models on the supported VPU devices: +This chapter provides information on the OpenVINO Runtime plugins that enable inference of deep learning models on the supported VPU devices: * Intel® Neural Compute Stick 2 powered by the Intel® Movidius™ Myriad™ X — Supported by the [MYRIAD Plugin](MYRIAD.md) * Intel® Vision Accelerator Design with Intel® Movidius™ VPUs — Supported by the [HDDL Plugin](HDDL.md) @@ -138,7 +138,7 @@ In a perfect pipeline this time should be near zero, which means that the data w **Get the following message when running inference with the VPU plugin: "[VPU] Cannot convert layer due to unsupported layer type "** -This means that your topology has a layer that is unsupported by your target VPU plugin. To resolve this issue, you can implement the custom layer for the target device using the [Inference Engine Extensibility mechanism](../Extensibility_DG/Intro.md). Or, to quickly get a working prototype, you can use the heterogeneous scenario with the default fallback policy (see the [HETERO Plugin](HETERO.md) section). Use the HETERO plugin with a fallback device that supports this layer, for example, CPU: `HETERO:MYRIAD,CPU`. +This means that your topology has a layer that is unsupported by your target VPU plugin. To resolve this issue, you can implement the custom layer for the target device using the [OpenVINO™ Extensibility mechanism](../../Extensibility_UG/Intro.md). Or, to quickly get a working prototype, you can use the heterogeneous scenario with the default fallback policy (see the [Heterogeneous execution](../hetero_execution.md) section). Use the HETERO mode with a fallback device that supports this layer, for example, CPU: `HETERO:MYRIAD,CPU`. For a list of VPU-supported layers, see the Supported Layers section of the [Supported Devices](Supported_Devices.md) page. ## Known Layers Limitations diff --git a/docs/OV_Runtime_UG/supported_plugins/config_properties.md b/docs/OV_Runtime_UG/supported_plugins/config_properties.md new file mode 100644 index 00000000000..5b0fef66c20 --- /dev/null +++ b/docs/OV_Runtime_UG/supported_plugins/config_properties.md @@ -0,0 +1,226 @@ +# Query device properties, configuration {#openvino_docs_OV_UG_query_api} + +## Query device properties and devices configuration + +The OpenVINO™ toolkit supports inferencing with several types of devices (processors or accelerators). +This section provides a high-level description of the process of querying of different device properties and configuration values at runtime. + +OpenVINO runtime has two types of properties: +- Read only properties which provides information about the devices (such device name, termal, execution capabilities, etc) and information about ov::CompiledModel to understand what configuration values were used to compile the model with. +- Mutable properties which are primarily used to configure ov::Core::compile_model process and affect final inference on the specific set of devices. Such properties can be set globally per device via ov::Core::set_property or locally for particular model in ov::Core::compile_model and ov::Core::query_model calls. + +OpenVINO property is represented as a named constexpr variable with a given string name and type (see ). Example: +``` +static constexpr Property, PropertyMutability::RO> available_devices{"AVAILABLE_DEVICES"}; +``` +represents a read-only property with C++ name `ov::available_devices`, string name `AVAILABLE_DEVICES` and type `std::vector`. + +Refer to the [Hello Query Device С++ Sample](../../../samples/cpp/hello_query_device/README.md) sources and the [Multi-Device execution](../multi_device.md) documentation for examples of using setting and getting properties in user applications. + +### Get a set of available devices + +Based on read-only property `ov::available_devices`, OpenVINO Core collects information about currently available devices enabled by OpenVINO plugins and returns information using the `ov::Core::get_available_devices` method: + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_properties_api.cpp + :language: cpp + :fragment: [get_available_devices] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_properties_api.py + :language: python + :fragment: [get_available_devices] + +@endsphinxdirective + +The function returns a list of available devices, for example: + +``` +MYRIAD.1.2-ma2480 +MYRIAD.1.4-ma2480 +CPU +GPU.0 +GPU.1 +``` + +If there are more than one instance of a specific device, the devices are enumerated with `.suffix` where `suffix` is a unique string identifier. Each device name can then be passed to: + +* `ov::Core::compile_model` to load the model to a specific device with specific configuration properties. +* `ov::Core::get_property` to get common or device specific properties. +* All other methods of the `ov::Core` class that accept `deviceName`. + +### Working with properties in Your Code + +The `ov::Core` class provides the following method to query device information, set or get different device configuration properties: + +* `ov::Core::get_property` - Gets the current value of a specific property. +* `ov::Core::set_property` - Sets a new value for the property globally for specified `device_name`. + +The `ov::CompiledModel` class is also extended to support the properties: + +* `ov::CompiledModel::get_property` +* `ov::CompiledModel::set_property` + +For documentation about OpenVINO common device-independent properties, refer to `openvino/runtime/properties.hpp`. Device specific configuration keys can be found in corresponding device folders (for example, `openvino/runtime/intel_gpu/properties.hpp`). + +### Working with properties via Core + +#### Getting device properties + +The code below demonstrates how to query `HETERO` device priority of devices which will be used to infer the model: + +@snippet snippets/ov_properties_api.cpp hetero_priorities + +> **NOTE**: All properties have a type, which is specified during property declaration. Based on this, actual type under `auto` is automatically deduced by C++ compiler. + +To extract device properties such as available devices (`ov::available_devices`), device name (`ov::device::full_name`), supported properties (`ov::supported_properties`), and others, use the `ov::Core::get_property` method: + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_properties_api.cpp + :language: cpp + :fragment: [cpu_device_name] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_properties_api.py + :language: python + :fragment: [cpu_device_name] + +@endsphinxdirective + +A returned value appears as follows: `Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz`. + +> **NOTE**: In order to understand a list of supported properties on `ov::Core` or `ov::CompiledModel` levels, use `ov::supported_properties` which contains a vector of supported property names. Properties which can be changed, has `ov::PropertyName::is_mutable` returning the `true` value. Most of the properites which are changable on ov::Core level, cannot be changed once the model is compiled, so it becomes immutable read-only property. + +#### Configure a work with a model + +`ov::Core` methods like: + +* `ov::Core::compile_model` +* `ov::Core::import_model` +* `ov::Core::query_model` + +accept variadic list of properties as last arguments. Each property in such parameters lists should be used as function call to pass property value with specified property type. + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_properties_api.cpp + :language: cpp + :fragment: [compile_model_with_property] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_properties_api.py + :language: python + :fragment: [compile_model_with_property] + +@endsphinxdirective + +The example below specifies hints that a model should be compiled to be inferenced with multiple inference requests in parallel to achive best throughput while inference should be performed without accuracy loss with FP32 precision. + +#### Setting properties globally + +`ov::Core::set_property` with a given device name should be used to set global configuration properties which are the same accross multiple `ov::Core::compile_model`, `ov::Core::query_model`, etc. calls, while setting property on the specific `ov::Core::compile_model` call applies properties only for current call: + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_properties_api.cpp + :language: cpp + :fragment: [core_set_property_then_compile] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_properties_api.py + :language: python + :fragment: [core_set_property_then_compile] + +@endsphinxdirective + +### Properties on CompiledModel level + +#### Getting property + +The `ov::CompiledModel::get_property` method is used to get property values the compiled model has been created with or a compiled model level property such as `ov::optimal_number_of_infer_requests`: + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_properties_api.cpp + :language: cpp + :fragment: [optimal_number_of_infer_requests] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_properties_api.py + :language: python + :fragment: [optimal_number_of_infer_requests] + +@endsphinxdirective + +Or the current temperature of the `MYRIAD` device: + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_properties_api.cpp + :language: cpp + :fragment: [device_thermal] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_properties_api.py + :language: python + :fragment: [device_thermal] + +@endsphinxdirective + +Or the number of threads that would be used for inference on `CPU` device: + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_properties_api.cpp + :language: cpp + :fragment: [inference_num_threads] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_properties_api.py + :language: python + :fragment: [inference_num_threads] + +@endsphinxdirective + +#### Setting properties for compiled model + +The only mode that supports this method is [Multi-Device execution](../multi_device.md): + +@sphinxdirective + +.. tab:: C++ + + .. doxygensnippet:: docs/snippets/ov_properties_api.cpp + :language: cpp + :fragment: [multi_device] + +.. tab:: Python + + .. doxygensnippet:: docs/snippets/ov_properties_api.py + :language: python + :fragment: [multi_device] + +@endsphinxdirective diff --git a/docs/_static/images/DL_WB_start.png b/docs/_static/images/DL_WB_start.png new file mode 100644 index 00000000000..322c4480219 --- /dev/null +++ b/docs/_static/images/DL_WB_start.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e7dae1b0f7316d11d4d0459fc372d632e1b47edf0addcb90d26be7dc1492e21 +size 109620 diff --git a/docs/_static/images/accuracy_yolov4.png b/docs/_static/images/accuracy_yolov4.png new file mode 100644 index 00000000000..281395e2c0b --- /dev/null +++ b/docs/_static/images/accuracy_yolov4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e44a115c712032a30e705771a0f1a49221090c340901d0497b26d208fed6cc8c +size 110224 diff --git a/docs/_static/images/color_transformations.png b/docs/_static/images/color_transformations.png new file mode 100644 index 00000000000..08efcf7b536 --- /dev/null +++ b/docs/_static/images/color_transformations.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e4d3d29a4cec7dfffa3c743336925ce565722f9950c8cd3f9c6b6bf75c38c53 +size 508773 diff --git a/docs/_static/images/configure_input.png b/docs/_static/images/configure_input.png new file mode 100644 index 00000000000..c6996203bbd --- /dev/null +++ b/docs/_static/images/configure_input.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75196ce75c33936fb8b06bc27e7328b23cf08f066a6f996f6a413fee42aff6d6 +size 28423 diff --git a/docs/_static/images/convert_omz_to_IR.png b/docs/_static/images/convert_omz_to_IR.png new file mode 100644 index 00000000000..963666d2ab6 --- /dev/null +++ b/docs/_static/images/convert_omz_to_IR.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f0cd7ef06dc11c0f25409b2625869f6b725a8a004d9016a858b25c59dd01465 +size 19178 diff --git a/docs/_static/images/create_project.png b/docs/_static/images/create_project.png new file mode 100644 index 00000000000..9abf32f56da --- /dev/null +++ b/docs/_static/images/create_project.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de6f29d1c04a9affba1ca4f065f6d403001033a58b0cb586f1e39dc3701034c0 +size 83105 diff --git a/docs/_static/images/create_project_selected.png b/docs/_static/images/create_project_selected.png new file mode 100644 index 00000000000..ebd1deb909c --- /dev/null +++ b/docs/_static/images/create_project_selected.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6895aa172437708eda816bb50acce4307c58da3b85b095c7f3e3c1fc9f24140a +size 121292 diff --git a/docs/_static/images/custom_dataset.png b/docs/_static/images/custom_dataset.png new file mode 100644 index 00000000000..5a11ef4c81f --- /dev/null +++ b/docs/_static/images/custom_dataset.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8bcf5b310b6d2d6cf4a14635f82491ec4fa8f1efbf6f0485b4968d3f01e36ee +size 540797 diff --git a/docs/_static/images/custom_dataset_imported.png b/docs/_static/images/custom_dataset_imported.png new file mode 100644 index 00000000000..2497a10a381 --- /dev/null +++ b/docs/_static/images/custom_dataset_imported.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aed8f8a3d61e764b12cf8f1c99db3d4c42f17008d4b3367a36aefec761e7840b +size 19079 diff --git a/docs/_static/images/dataset.png b/docs/_static/images/dataset.png new file mode 100644 index 00000000000..960d4d032a7 --- /dev/null +++ b/docs/_static/images/dataset.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caf1538ca8b64cbc243ab8e4a87b38a7eb071c2f19955fe881cd221807f485b7 +size 312545 diff --git a/docs/_static/images/dataset_import.png b/docs/_static/images/dataset_import.png new file mode 100644 index 00000000000..8495b48a5e8 --- /dev/null +++ b/docs/_static/images/dataset_import.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bea2db4a7618e4370398d29358bffaad8bb9fe61316012b38ba54d305615a77 +size 37413 diff --git a/docs/_static/images/dataset_imported.png b/docs/_static/images/dataset_imported.png new file mode 100644 index 00000000000..8187b1a8c1a --- /dev/null +++ b/docs/_static/images/dataset_imported.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:974bdd3a7cfba238ec7647af349fbd5e291fb79f30f3ab337df274c338b6ee6e +size 20895 diff --git a/docs/_static/images/dataset_selection.png b/docs/_static/images/dataset_selection.png new file mode 100644 index 00000000000..64352e54679 --- /dev/null +++ b/docs/_static/images/dataset_selection.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f551cf331e3dd23598f80511b6b4be27829a578eca9aa8a83e63f70c743822bb +size 25656 diff --git a/docs/_static/images/generate_datasets_closeup.png b/docs/_static/images/generate_datasets_closeup.png new file mode 100644 index 00000000000..051a6f3a43d --- /dev/null +++ b/docs/_static/images/generate_datasets_closeup.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bef9577a4ff23b7f102a314c90fffd167fab56b0f6a9492970e02de3a0de998 +size 399477 diff --git a/docs/_static/images/github.png b/docs/_static/images/github.png new file mode 100644 index 00000000000..4bf56a3d3e4 --- /dev/null +++ b/docs/_static/images/github.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b2749d93dff16fc9062a0fa01fd694bf19385a0b4f3d0e409eb56f2648e3cfc +size 11929 diff --git a/docs/_static/images/horizontal_flip.png b/docs/_static/images/horizontal_flip.png new file mode 100644 index 00000000000..ff4dc034864 --- /dev/null +++ b/docs/_static/images/horizontal_flip.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f66d41137ef6cfd0fe4bd8f635814203bc963b546a20da9718b42f58b6fa4bd9 +size 168160 diff --git a/docs/_static/images/horizontal_flip_closeup.png b/docs/_static/images/horizontal_flip_closeup.png new file mode 100644 index 00000000000..1ba92fd8615 --- /dev/null +++ b/docs/_static/images/horizontal_flip_closeup.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f57758f726846dc6f83dc14ed0f96622a7d527287330b6c6f8ac95cecfd30cbf +size 432513 diff --git a/docs/_static/images/import_annotated_dataset.png b/docs/_static/images/import_annotated_dataset.png new file mode 100644 index 00000000000..12da21ab981 --- /dev/null +++ b/docs/_static/images/import_annotated_dataset.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51c33a7b19d2d21bab642372efcbeccdbcb6f8813d98cfe8fb9c03f801287474 +size 38600 diff --git a/docs/_static/images/import_dataset_page.png b/docs/_static/images/import_dataset_page.png new file mode 100644 index 00000000000..18eac942c2f --- /dev/null +++ b/docs/_static/images/import_dataset_page.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3994e201f3077a11da2870ae6d13242d0d0d353d35f837cdd7e01686a445859d +size 266463 diff --git a/docs/_static/images/import_image_dataset.png b/docs/_static/images/import_image_dataset.png new file mode 100644 index 00000000000..144a8dd4de7 --- /dev/null +++ b/docs/_static/images/import_image_dataset.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f055ebe36900ad24d4501d89f52e343f7c2b14a025e020b747c7272325d2c4a2 +size 10521 diff --git a/docs/_static/images/import_model.png b/docs/_static/images/import_model.png new file mode 100644 index 00000000000..34c266a1a69 --- /dev/null +++ b/docs/_static/images/import_model.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e628143bbd02dc43ccff24e7f436f8f22fcffb3a606fdf65e49c18097741949 +size 50825 diff --git a/docs/_static/images/import_tokenizer.png b/docs/_static/images/import_tokenizer.png new file mode 100644 index 00000000000..192c5a0823b --- /dev/null +++ b/docs/_static/images/import_tokenizer.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f9b185567143692721ea68b0afd1c63d000df1d6b216c51567306d98bac5de8 +size 31986 diff --git a/docs/_static/images/import_yolov4.png b/docs/_static/images/import_yolov4.png new file mode 100644 index 00000000000..77c88ad2ac3 --- /dev/null +++ b/docs/_static/images/import_yolov4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94e10d4ec22ca3822faad9d0eea18661b3feec09611ae7dd19b64b8d636487cd +size 47182 diff --git a/docs/_static/images/input_name.png b/docs/_static/images/input_name.png new file mode 100644 index 00000000000..6ed2f94db7e --- /dev/null +++ b/docs/_static/images/input_name.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcdb6eac41fb75ae1248c7103046fb51298f0d0c08b52ff22cc5f8c81b394741 +size 15202 diff --git a/docs/_static/images/input_shape.png b/docs/_static/images/input_shape.png new file mode 100644 index 00000000000..d407c3b17c8 --- /dev/null +++ b/docs/_static/images/input_shape.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af7835134b35136d8b50e2b22944a88f2bcfb3ff114afdfd052d52b4358212b3 +size 22970 diff --git a/docs/_static/images/inputs.png b/docs/_static/images/inputs.png new file mode 100644 index 00000000000..d4518d9de79 --- /dev/null +++ b/docs/_static/images/inputs.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1f8bc12837e03b1a2c1386c2bac512c21b1fb073d990379079c317488e9ce1c +size 39015 diff --git a/docs/_static/images/model_analysis.png b/docs/_static/images/model_analysis.png new file mode 100644 index 00000000000..55f0323e314 --- /dev/null +++ b/docs/_static/images/model_analysis.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9640f8e9260a4ea796b7768de3368762719644319cfdad332fe1a8dc79946a4c +size 49674 diff --git a/docs/_static/images/model_analysis_details.png b/docs/_static/images/model_analysis_details.png new file mode 100644 index 00000000000..975b8714506 --- /dev/null +++ b/docs/_static/images/model_analysis_details.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e02a91f6ae4a94f7a5bc08987b2bd5a239d13dd80ae485189f79a2ee30bc5528 +size 13546 diff --git a/docs/_static/images/noise_injection.png b/docs/_static/images/noise_injection.png new file mode 100644 index 00000000000..3011823cf99 --- /dev/null +++ b/docs/_static/images/noise_injection.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cd558b8355e04507aedc1b18ca8075e74d934d311320ea25942a646d47f9a21 +size 366753 diff --git a/docs/_static/images/omz_import.png b/docs/_static/images/omz_import.png new file mode 100644 index 00000000000..b66de979251 --- /dev/null +++ b/docs/_static/images/omz_import.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:349e96028d651807b887cf73a4be79eb2123e62c6abe2ed3ec4b419e306ce3d5 +size 56444 diff --git a/docs/_static/images/optimization_settings.png b/docs/_static/images/optimization_settings.png new file mode 100644 index 00000000000..91a3314121f --- /dev/null +++ b/docs/_static/images/optimization_settings.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9c013e8d76cf87c461407132ef6eb59e57c14f8c4cf242f71ae617140bf4209 +size 120403 diff --git a/docs/_static/images/optimize_face_detection.png b/docs/_static/images/optimize_face_detection.png new file mode 100644 index 00000000000..d984d5f1e53 --- /dev/null +++ b/docs/_static/images/optimize_face_detection.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:499c44eb2099f8f0de23800b6dd75395c110783eb80f92f1bb91034c7f93724d +size 132642 diff --git a/docs/_static/images/ov_chart.png b/docs/_static/images/ov_chart.png index f91923443c3..fa25daf3601 100644 --- a/docs/_static/images/ov_chart.png +++ b/docs/_static/images/ov_chart.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f9a0d138f7f6d2546f0e48d9240d6a90aec18dc6d8092e1082b2fc3125f1ce3d -size 108434 +oid sha256:83f0013e02ea792b553b5bd0a5630fb456a6fefc8dd701cd4430fc83d75cbff7 +size 78205 diff --git a/docs/_static/images/pack.png b/docs/_static/images/pack.png new file mode 100644 index 00000000000..6e9c713f394 --- /dev/null +++ b/docs/_static/images/pack.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91b43730b70b6d83bffb6cfbe9982cf806b0ac7f8a58001ba74596cface2fe5d +size 92680 diff --git a/docs/_static/images/performance.png b/docs/_static/images/performance.png new file mode 100644 index 00000000000..e874daf3eb5 --- /dev/null +++ b/docs/_static/images/performance.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b64c598656309f5156e291d93a2865537ebe2b87f62688b2d1fd2da35c63a471 +size 110383 diff --git a/docs/_static/images/performance_change.jpeg b/docs/_static/images/performance_change.jpeg new file mode 100644 index 00000000000..71ba4eab00c Binary files /dev/null and b/docs/_static/images/performance_change.jpeg differ diff --git a/docs/_static/images/random_erase.png b/docs/_static/images/random_erase.png new file mode 100644 index 00000000000..b72482898cb --- /dev/null +++ b/docs/_static/images/random_erase.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8dfe286f3aa8eb104b0217fba8148fc1f69e86b4a1557ff36bcbead6dba60e5 +size 298320 diff --git a/docs/_static/images/rgb.png b/docs/_static/images/rgb.png new file mode 100644 index 00000000000..87f8a672d83 --- /dev/null +++ b/docs/_static/images/rgb.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f3dc6397527d62052243539fbcab4dc393a0c434fdddb34ef9f36031abbdf49 +size 22234 diff --git a/docs/_static/images/scales.png b/docs/_static/images/scales.png new file mode 100644 index 00000000000..e2dce47459c --- /dev/null +++ b/docs/_static/images/scales.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddb124e0d67f958042cd04bfcaeb59c5505558651d86dee1d68a41b4f87a4d5e +size 5502 diff --git a/docs/_static/images/select_tokenizer.png b/docs/_static/images/select_tokenizer.png new file mode 100644 index 00000000000..8f0f48eb9ce --- /dev/null +++ b/docs/_static/images/select_tokenizer.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:080563fa8984d82c4d0151e5c42c44c4cf872dffbc0eb5f42fff0ab58adee7f0 +size 18584 diff --git a/docs/_static/images/start_page_dl_wb.png b/docs/_static/images/start_page_dl_wb.png new file mode 100644 index 00000000000..6fb7c80641c --- /dev/null +++ b/docs/_static/images/start_page_dl_wb.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c57c68582ee21c31115c78515c2fd9189fe4010c223f356e734c004f160ad569 +size 129931 diff --git a/docs/_static/images/text_dataset.png b/docs/_static/images/text_dataset.png new file mode 100644 index 00000000000..c9ccb90a2c2 --- /dev/null +++ b/docs/_static/images/text_dataset.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58fdd181204bf35bca5a5db4b4ca0a673c5eb009cb7c40e006713f671ccfc86f +size 84039 diff --git a/docs/_static/images/tokenizer_fill.png b/docs/_static/images/tokenizer_fill.png new file mode 100644 index 00000000000..c0fe2487ac2 --- /dev/null +++ b/docs/_static/images/tokenizer_fill.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93f4045a51868adb17b7ed949f27f430836fdc29fca870f6bc07564bc4b800fd +size 17119 diff --git a/docs/_static/images/vertical_flip.png b/docs/_static/images/vertical_flip.png new file mode 100644 index 00000000000..460a01b0b7d --- /dev/null +++ b/docs/_static/images/vertical_flip.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13cfb4f7a604f3c9693ccd59500e509b5adb5f4e81171feaa38cc6e16b773765 +size 305754 diff --git a/docs/_static/images/yolov4_imported.png b/docs/_static/images/yolov4_imported.png new file mode 100644 index 00000000000..6f9e8db2419 --- /dev/null +++ b/docs/_static/images/yolov4_imported.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01b6dd951d8077c3a2420479c4cae59f5e8a58bac6272afc2c9259901f21f1d9 +size 53491 diff --git a/docs/_static/js/custom.js b/docs/_static/js/custom.js index 8b027164517..bfc54b881be 100644 --- a/docs/_static/js/custom.js +++ b/docs/_static/js/custom.js @@ -27,6 +27,7 @@ var wapSection = 'openvinotoolkit'; $(document).ready(function () { createVersions(); + updateTitleTag(); init_col_sections(); init_switchers(); handleSwitcherParam(); @@ -44,6 +45,13 @@ function getPageUrlWithVersion(version) { return encodeURI(newURL); } +function updateTitleTag() { + var title = $('title'); + var currentVersion = getCurrentVersion(); + var newTitle = (title.text() + ' — Version(' + currentVersion + ')').replace(/\s+/g, ' ').trim(); + title.text(newTitle); +} + function getCurrentVersion() { var protocol = window.location.protocol + "//"; var index = window.location.href.indexOf(protocol); diff --git a/docs/_static/js/graphs_ov_tf.js b/docs/_static/js/graphs_ov_tf.js new file mode 100644 index 00000000000..bf16e9dacc5 --- /dev/null +++ b/docs/_static/js/graphs_ov_tf.js @@ -0,0 +1,109 @@ +$(document).ready(function () { + var chartBlock = $('.chart-block-tf-ov'); + chartBlock.each(function () { + var url = $(this).data('loadcsv'); + Papa.parse(url, { + download: true, + complete: renderData($(this)) + }) + }); + + function getLabels(data) { + return data + .map((item) => item[1]); + } + + function getChartOptions(title, displayLabels) { + return { + responsive: false, + maintainAspectRatio: false, + legend: { display: true, position: 'bottom' }, + title: { + display: true, + text: title + }, + scales: { + xAxes: [{ + ticks: { + beginAtZero: true + } + }], + yAxes: [{ + ticks: { + display: displayLabels, //this will remove only the label + beginAtZero: true + } + }] + }, + plugins: { + datalabels: { + color: "#4A4A4A", + anchor: "end", + align: "end", + clamp: false, + offset: 0, + display: true, + font: { + size: 8, + family: 'Roboto' + } + } + } + } + } + + function getChartData(data) { + function getDataset(data, col, label, color) { + return { + label: label, + data: data.map(function (item) { + return item[col] + }), + backgroundColor: color, + borderColor: 'rgba(170,170,170,0)', + barThickness: 12 + } + } + return { + labels: getLabels(data), + datasets: [getDataset(data, 2, 'openvino', '#00C7FD'), getDataset(data, 3, 'TF', '#8F5DA2')] + }; + } + + function renderData(currentChart) { + return function (result) { + var data = result.data; + // remove col names + data.shift(0); + var chartName = data[1][0]; + var chartSlug = chartName.replace(')', '').replace(' (', '-'); + var graphContainer = $('
'); + var chartContainer = $('
'); + graphContainer.attr('id', 'ov-graph-container-' + chartSlug); + chartContainer.addClass('chart-container'); + chartContainer.addClass('container'); + var chartWrap = $('
'); + chartWrap.addClass('chart-wrap'); + chartWrap.addClass('container'); + chartContainer.append(chartWrap); + var chart = $('
'); + chart.addClass('chart'); + chart.addClass('col-md-12'); + var canvas = $(''); + chart.append(canvas); + var container = $('
'); + container.addClass('row'); + container.append(chart); + var context = canvas.get(0).getContext('2d'); + context.canvas.width = context.canvas.width * 2.5; + var chartTitle = chartName + ', Throughput (FPS) Precision: FP32 (Higher is better)'; + new Chart(context, { + type: 'horizontalBar', + data: getChartData(data), + options: getChartOptions(chartTitle, true) + }); + chartContainer.append(container); + currentChart.append(chartContainer); + } + } +}); diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index f12ea6528e0..de7a83e8c19 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -12,7 +12,5 @@ - - {% endblock %} diff --git a/docs/api/ie_python_api/api.rst b/docs/api/ie_python_api/api.rst index 3528148dd87..a2f69af4790 100644 --- a/docs/api/ie_python_api/api.rst +++ b/docs/api/ie_python_api/api.rst @@ -67,12 +67,6 @@ OpenVINO Python API openvino.runtime.passes -.. autosummary:: - :toctree: _autosummary - :template: custom-module-template.rst - - openvino.runtime.utils - .. autosummary:: :toctree: _autosummary :template: custom-module-template.rst @@ -85,12 +79,6 @@ OpenVINO Python API openvino.frontend -.. autosummary:: - :toctree: _autosummary - :template: custom-module-template.rst - - openvino.offline_transformations - .. toctree:: :maxdepth: 2 :hidden: diff --git a/docs/benchmarks/performance_benchmarks_faq.md b/docs/benchmarks/performance_benchmarks_faq.md index 2010111bf3b..b628b12f116 100644 --- a/docs/benchmarks/performance_benchmarks_faq.md +++ b/docs/benchmarks/performance_benchmarks_faq.md @@ -66,6 +66,6 @@ The CAFFE version of resnet-50, mobilenet-v1-1.0-224 and mobilenet-v2 have been The web site format has changed in order to support the more common search approach of looking for the performance of a given neural network model on different HW-platforms. As opposed to review a given HW-platform's performance on different neural network models. #### 13. How is Latency measured? -Latency is measured by running the OpenVINO™ inference engine in synchronous mode. In synchronous mode each frame or image is processed through the entire set of stages (pre-processing, inference, post-processing) before the next frame or image is processed. This KPI is relevant for applications where the inference on a single image is required, for example the analysis of an ultra sound image in a medical application or the analysis of a seismic image in the oil & gas industry. Other use cases include real-time or near real-time applications like an industrial robot's response to changes in its environment and obstacle avoidance for autonomous vehicles where a quick response to the result of the inference is required. +Latency is measured by running the OpenVINO™ Runtime in synchronous mode. In synchronous mode each frame or image is processed through the entire set of stages (pre-processing, inference, post-processing) before the next frame or image is processed. This KPI is relevant for applications where the inference on a single image is required, for example the analysis of an ultra sound image in a medical application or the analysis of a seismic image in the oil & gas industry. Other use cases include real-time or near real-time applications like an industrial robot's response to changes in its environment and obstacle avoidance for autonomous vehicles where a quick response to the result of the inference is required. For more complete information about performance and benchmark results, visit: [www.intel.com/benchmarks](https://www.intel.com/benchmarks) and [Optimization Notice](https://software.intel.com/articles/optimization-notice). [Legal Information](../Legal_Information.md). diff --git a/docs/benchmarks/performance_benchmarks_openvino.md b/docs/benchmarks/performance_benchmarks_openvino.md index ca55baf8c2c..89b282ebf6a 100644 --- a/docs/benchmarks/performance_benchmarks_openvino.md +++ b/docs/benchmarks/performance_benchmarks_openvino.md @@ -14,7 +14,7 @@ This benchmark setup includes a single machine on which both the benchmark application and the OpenVINO™ installation reside. -The benchmark application loads the Inference Engine (SW) at run time and executes inferences on the specified hardware inference engine, (CPU, GPU or VPU). The benchmark application measures the time spent on actual inferencing (excluding any pre or post processing) and then reports on the inferences per second (or Frames Per Second). For more information on the benchmark application, please also refer to the entry 5 of the [FAQ section](performance_benchmarks_faq.md). +The benchmark application loads the OpenVINO Runtime (SW) at runtime and executes inferences on the specified hardware (CPU, GPU or VPU). The benchmark application measures the time spent on actual inferencing (excluding any pre or post processing) and then reports on the inferences per second (or Frames Per Second). For more information on the benchmark application, please also refer to the entry 5 of the [FAQ section](performance_benchmarks_faq.md). Devices similar to the ones we have used for benchmarking can be accessed using [Intel® DevCloud for the Edge](https://devcloud.intel.com/edge/), a remote development environment with access to Intel® hardware and the latest versions of the Intel® Distribution of the OpenVINO™ Toolkit. [Learn more](https://devcloud.intel.com/edge/get_started/devcloud/) or [Register here](https://inteliot.force.com/DevcloudForEdge/s/). diff --git a/docs/benchmarks/performance_benchmarks_ovms.md b/docs/benchmarks/performance_benchmarks_ovms.md index 57495835d86..d7393aa5047 100644 --- a/docs/benchmarks/performance_benchmarks_ovms.md +++ b/docs/benchmarks/performance_benchmarks_ovms.md @@ -1,6 +1,6 @@ # OpenVINO™ Model Server Benchmark Results {#openvino_docs_performance_benchmarks_ovms} -OpenVINO™ Model Server is an open-source, production-grade inference platform that exposes a set of models via a convenient inference API over gRPC or HTTP/REST. It employs the inference engine libraries for from the Intel® Distribution of OpenVINO™ toolkit to extend workloads across Intel® hardware including CPU, GPU and others. +OpenVINO™ Model Server is an open-source, production-grade inference platform that exposes a set of models via a convenient inference API over gRPC or HTTP/REST. It employs the OpenVINO Runtime libraries for from the Intel® Distribution of OpenVINO™ toolkit to extend workloads across Intel® hardware including CPU, GPU and others. ![OpenVINO™ Model Server](../img/performance_benchmarks_ovms_01.png) diff --git a/docs/conf.py b/docs/conf.py index 49222306ee0..92c9fc66f37 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -63,7 +63,8 @@ templates_path = ['_templates'] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'openvino/inference-engine'] +exclude_patterns = ['_build', 'Thumbs.db', + '.DS_Store', 'openvino/inference-engine'] panels_add_bootstrap_css = False @@ -134,6 +135,7 @@ html_static_path = ['_static'] # monkeypatch sphinx api doc to prevent showing inheritance from object and enum.Enum add_line = ClassDocumenter.add_line + def add_line_no_base_object(self, line, *args, **kwargs): if line.strip() in ['Bases: :class:`object`', 'Bases: :class:`enum.Enum`']: return @@ -143,14 +145,30 @@ def add_line_no_base_object(self, line, *args, **kwargs): ClassDocumenter.add_line = add_line_no_base_object +# OpenVINO Python API Reference Configuration +exclude_pyapi_methods = ('__weakref__', + '__doc__', + '__module__', + '__dict__', + 'add_openvino_libs_to_path' + ) + + +def autodoc_skip_member(app, what, name, obj, skip, options): + return name in exclude_pyapi_methods + def setup(app): logger = logging.getLogger(__name__) - app.add_config_value('doxygen_mapping_file', doxygen_mapping_file, rebuild=True) + app.add_config_value('doxygen_mapping_file', + doxygen_mapping_file, rebuild=True) app.add_config_value('repositories', repositories, rebuild=True) + app.connect('autodoc-skip-member', autodoc_skip_member) app.add_js_file('js/custom.js') app.add_js_file('js/graphs.js') + app.add_js_file('js/graphs_ov_tf.js') try: - shutil.copytree(os.path.join(app.srcdir, 'csv'), os.path.join(app.outdir, 'csv'), dirs_exist_ok=True) + shutil.copytree(os.path.join(app.srcdir, 'csv'), os.path.join( + app.outdir, 'csv'), dirs_exist_ok=True) except FileNotFoundError: logger.warning('csv directory not found.') diff --git a/docs/documentation.md b/docs/documentation.md index 731c7d7aa13..ea26b3f22ff 100644 --- a/docs/documentation.md +++ b/docs/documentation.md @@ -8,7 +8,6 @@ :hidden: openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide - openvino_docs_HOWTO_Custom_Layers_Guide omz_tools_downloader @@ -53,8 +52,6 @@ Intel® Deep Learning Streamer openvino_docs_gapi_gapi_intro - OpenVX Developer Guide - OpenVX API Reference OpenCV* Developer Guide OpenCL™ Developer Guide @@ -68,9 +65,11 @@ .. toctree:: :maxdepth: 1 - :caption: Developing Inference Engine Plugins + :caption: OpenVINO Extensibility :hidden: + openvino_docs_Extensibility_UG_Intro + openvino_docs_transformations Inference Engine Plugin Developer Guide groupie_dev_api Plugin Transformation Pipeline @@ -93,7 +92,7 @@ This section provides reference documents that guide you through developing your With the [Model Downloader](@ref omz_tools_downloader) and [Model Optimizer](MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) guides, you will learn to download pre-trained models and convert them for use with the OpenVINO™ toolkit. You can provide your own model or choose a public or Intel model from a broad selection provided in the [Open Model Zoo](model_zoo.md). ## Deploying Inference -The [OpenVINO™ Runtime User Guide](OV_Runtime_UG/OpenVINO_Runtime_User_Guide.md) explains the process of creating your own application that runs inference with the OpenVINO™ toolkit. The [API Reference](./api_references.html) defines the Inference Engine API for Python, C++, and C and the nGraph API for Python and C++. The Inference Engine API is what you'll use to create an OpenVINO™ application, while the nGraph API is available for using enhanced operations sets and other features. After writing your application, you can use the [Deployment Manager](install_guides/deployment-manager-tool.md) for deploying to target devices. +The [OpenVINO™ Runtime User Guide](OV_Runtime_UG/openvino_intro.md) explains the process of creating your own application that runs inference with the OpenVINO™ toolkit. The [API Reference](./api_references.html) defines the OpenVINO Runtime API for Python, C++, and C. The OpenVINO Runtime API is what you'll use to create an OpenVINO™ inference application, use enhanced operations sets and other features. After writing your application, you can use the [Deployment Manager](install_guides/deployment-manager-tool.md) for deploying to target devices. ## Tuning for Performance The toolkit provides a [Performance Optimization Guide](optimization_guide/dldt_optimization_guide.md) and utilities for squeezing the best performance out of your application, including [Accuracy Checker](@ref omz_tools_accuracy_checker), [Post-Training Optimization Tool](@ref pot_README), and other tools for measuring accuracy, benchmarking performance, and tuning your application. @@ -108,5 +107,5 @@ The OpenVINO™ toolkit also works with the following media processing framework * [Intel® Deep Learning Streamer (Intel® DL Streamer)](@ref openvino_docs_dlstreamer) — A streaming media analytics framework based on GStreamer, for creating complex media analytics pipelines optimized for Intel hardware platforms. Go to the Intel® DL Streamer [documentation](https://dlstreamer.github.io/) website to learn more. * [Intel® oneAPI Video Processing Library (oneVPL)](https://www.intel.com/content/www/us/en/develop/documentation/oneapi-programming-guide/top/api-based-programming/intel-oneapi-video-processing-library-onevpl.html) — A programming interface for video decoding, encoding, and processing to build portable media pipelines on CPUs, GPUs, and other accelerators. -You can also add computer vision capabilities to your application using optimized versions of [OpenCV](https://opencv.org/) and [OpenVX](https://khronos.org/openvx). +You can also add computer vision capabilities to your application using optimized versions of [OpenCV](https://opencv.org/). diff --git a/docs/doxygen-xfail.txt b/docs/doxygen-xfail.txt index d0be8692508..72d9cd0acdc 100644 --- a/docs/doxygen-xfail.txt +++ b/docs/doxygen-xfail.txt @@ -66,3 +66,4 @@ ovms_docs_shape_batch_layout.rst api/api_reference.rst workbench/docs/workbench_dg/key_concepts.md workbench/docs/workbench_dg/run_single_inference.md +omz_tools_downloader.rst diff --git a/docs/gapi/face_beautification.md b/docs/gapi/face_beautification.md index 25619ae8e0b..7026d9b98a0 100644 --- a/docs/gapi/face_beautification.md +++ b/docs/gapi/face_beautification.md @@ -338,7 +338,7 @@ The steps to get the masks are: Once the graph is fully expressed, we can finally compile it and run on real data. G-API graph compilation is the stage where the G-API framework actually understands which kernels and networks to use. This configuration happens via G-API compilation arguments. ### DNN Parameters -This sample is using OpenVINO™ Toolkit Inference Engine backend for DL inference, which is configured the following way: +This sample is using OpenVINO™ Toolkit OpenVINO Runtime backend for DL inference, which is configured the following way: ```cpp auto faceParams = cv::gapi::ie::Params { diff --git a/docs/gapi/gapi_face_analytics_pipeline.md b/docs/gapi/gapi_face_analytics_pipeline.md index 6b544485668..be07aaae573 100644 --- a/docs/gapi/gapi_face_analytics_pipeline.md +++ b/docs/gapi/gapi_face_analytics_pipeline.md @@ -41,7 +41,7 @@ Constructing a G-API graph for a video streaming case does not differ much from ### Declare Deep Learning topologies {#gapi_ifd_declaring_nets} -In contrast with traditional CV functions (see [core](https://docs.opencv.org/4.5.0/df/d1f/group__gapi__core.html) and [imgproc](https://docs.opencv.org/4.5.0/d2/d00/group__gapi__imgproc.html)) where G-API declares distinct operations for every function, inference in G-API is a single generic operation `cv::gapi::infer<>`. As usual, it is just an interface and it can be implemented in a number of ways under the hood. In OpenCV 4.2, only OpenVINO™ Inference Engine-based backend is available, and OpenCV's own DNN module-based backend is to come. +In contrast with traditional CV functions (see [core](https://docs.opencv.org/4.5.0/df/d1f/group__gapi__core.html) and [imgproc](https://docs.opencv.org/4.5.0/d2/d00/group__gapi__imgproc.html)) where G-API declares distinct operations for every function, inference in G-API is a single generic operation `cv::gapi::infer<>`. As usual, it is just an interface and it can be implemented in a number of ways under the hood. In OpenCV 4.2, only OpenVINO™ Runtime-based backend is available, and OpenCV's own DNN module-based backend is to come. `cv::gapi::infer<>` is _parametrized_ by the details of a topology we are going to execute. Like operations, topologies in G-API are strongly typed and are defined with a special macro `G_API_NET()`: @@ -126,7 +126,7 @@ G-API strictly separates construction from configuration -- with the idea to kee Platform-specific details arise when the pipeline is *compiled* -- i.e. is turned from a declarative to an executable form. The way *how* to run stuff is specified via compilation arguments, and new inference/streaming features are no exception from this rule. -G-API is built on backends which implement interfaces (see [Architecture](https://docs.opencv.org/4.5.0/de/d4d/gapi_hld.html) and [Kernels](kernel_api.md) for details) -- thus `cv::gapi::infer<>` is a function which can be implemented by different backends. In OpenCV 4.2, only OpenVINO™ Inference Engine backend for inference is available. Every inference backend in G-API has to provide a special parameterizable structure to express *backend-specific* neural network parameters -- and in this case, it is `cv::gapi::ie::Params`: +G-API is built on backends which implement interfaces (see [Architecture](https://docs.opencv.org/4.5.0/de/d4d/gapi_hld.html) and [Kernels](kernel_api.md) for details) -- thus `cv::gapi::infer<>` is a function which can be implemented by different backends. In OpenCV 4.2, only OpenVINO™ Runtime backend for inference is available. Every inference backend in G-API has to provide a special parameterizable structure to express *backend-specific* neural network parameters -- and in this case, it is `cv::gapi::ie::Params`: ```cpp auto det_net = cv::gapi::ie::Params { @@ -148,7 +148,7 @@ auto emo_net = cv::gapi::ie::Params { Here we define three parameter objects: `det_net`, `age_net`, and `emo_net`. Every object is a `cv::gapi::ie::Params` structure parametrization for each particular network we use. On a compilation stage, G-API automatically matches network parameters with their `cv::gapi::infer<>` calls in graph using this information. -Regardless of the topology, every parameter structure is constructed with three string arguments – specific to the OpenVINO™ Inference Engine: +Regardless of the topology, every parameter structure is constructed with three string arguments – specific to the OpenVINO™ Runtime: * Path to the topology's intermediate representation (.xml file); * Path to the topology's model weights (.bin file); diff --git a/docs/get_started.md b/docs/get_started.md index f576ab82b02..c55b1055421 100644 --- a/docs/get_started.md +++ b/docs/get_started.md @@ -41,7 +41,7 @@ .. toctree:: :maxdepth: 1 :hidden: - :caption: Inference Engine Code Samples + :caption: OpenVINO Code Samples openvino_docs_IE_DG_Samples_Overview @@ -87,7 +87,7 @@

Use a web-based version of OpenVINO with a Graphical User Interface. Installing a DL Workbench container is required.

-

Inference Engine samples

+

OpenVINO samples

See ready-made applications explaining OpenVINO features and various use-cases.

diff --git a/docs/get_started/get_started_demos.md b/docs/get_started/get_started_demos.md index 25eb5786d5e..73e6dc4c21e 100644 --- a/docs/get_started/get_started_demos.md +++ b/docs/get_started/get_started_demos.md @@ -39,15 +39,15 @@ To build OpenVINO samples: @sphinxdirective .. tab:: Linux - Go to the :doc:`Inference Engine Samples page ` and see the "Build the Sample Applications on Linux*" section. + Go to the :doc:`OpenVINO Samples page ` and see the "Build the Sample Applications on Linux*" section. .. tab:: Windows - Go to the :doc:`Inference Engine Samples page ` and see the "Build the Sample Applications on Microsoft Windows* OS" section. + Go to the :doc:`OpenVINO Samples page ` and see the "Build the Sample Applications on Microsoft Windows* OS" section. .. tab:: macOS - Go to the :doc:`Inference Engine Samples page ` and see the "Build the Sample Applications on macOS*" section. + Go to the :doc:`OpenVINO Samples page ` and see the "Build the Sample Applications on macOS*" section. @endsphinxdirective @@ -358,9 +358,9 @@ Your screen looks similar to this after the download: ## Step 2: Convert the Model with Model Optimizer -In this step, your trained models are ready to run through the Model Optimizer to convert them to the IR (Intermediate Representation) format. For most model types, this is required before using the Inference Engine with the model. +In this step, your trained models are ready to run through the Model Optimizer to convert them to the IR (Intermediate Representation) format. For most model types, this is required before using the OpenVINO Runtime with the model. -Models in the IR format always include an `.xml` and `.bin` file and may also include other files such as `.json` or `.mapping`. Make sure you have these files together in a single directory so the Inference Engine can find them. +Models in the IR format always include an `.xml` and `.bin` file and may also include other files such as `.json` or `.mapping`. Make sure you have these files together in a single directory so the OpenVINO Runtime can find them. REQUIRED: `model_name.xml` REQUIRED: `model_name.bin` @@ -393,7 +393,7 @@ Create an `` directory to contain the model's Intermediate Representatio @endsphinxdirective -The Inference Engine can perform inference on different precision formats, such as FP32, FP16, or INT8. To generate an IR with a specific precision, run the Model Optimizer with the appropriate `--data_type` option. +The OpenVINO Runtime can perform inference on different precision formats, such as FP32, FP16, or INT8. To generate an IR with a specific precision, run the Model Optimizer with the appropriate `--data_type` option. Generic Model Optimizer script: diff --git a/docs/glossary.md b/docs/glossary.md index 5e168add190..a0b9d690e5b 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -58,19 +58,19 @@ Glossary of terms used in the OpenVINO™ | Term | Description | | :--- |:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Batch | Number of images to analyze during one call of infer. Maximum batch size is a property of the network and it is set before loading of the network to the plugin. In NHWC, NCHW and NCDHW image data layout representation, the N refers to the number of images in the batch | -| Tensor | Memory container used for storing inputs, outputs of the network, weights and biases of the layers | -| Device (Affinitity) | A preferred Intel(R) hardware device to run the inference (CPU, GPU, etc.) | -| Extensibility mechanism, Custom layers | The mechanism that provides you with capabilities to extend the OpenVINO™ Runtime and Model Optimizer so that they can work with topologies containing layers that are not yet supported | -| ov::Model | A class of the Model that OpenVINO™ Runtime reads from IR. Consists of topology, weights and biases | -| ov::CompiledModel | An instance of the loaded network which allows the OpenVINO™ Runtime to request (several) infer requests and perform inference synchronously or asynchronously | -| InferRequest | A class that represents the end point of inference on the model loaded to the plugin and represented by executable network. Inputs are set here, outputs should be requested from this interface as well | -| ov::ProfileInfo | Represents basic inference profiling information per layer | -| OpenVINO™ Runtime | A C++ library with a set of classes that you can use in your application to infer input data (images) and get the result | -| OpenVINO™ API | The basic default API for all supported devices, which allows you to load a model from Intermediate Representation, set input and output formats and execute the model on various devices | +| Batch | Number of images to analyze during one call of infer. Maximum batch size is a property of the model and it is set before compiling of the model by the device. In NHWC, NCHW and NCDHW image data layout representation, the N refers to the number of images in the batch | +| Tensor | Memory container used for storing inputs, outputs of the model, weights and biases of the operations | +| Device (Affinitity) | A preferred Intel(R) hardware device to run the inference (CPU, GPU, GNA, etc.) | +| Extensibility mechanism, Custom layers | The mechanism that provides you with capabilities to extend the OpenVINO™ Runtime and Model Optimizer so that they can work with models containing operations that are not yet supported | +| ov::Model | A class of the Model that OpenVINO™ Runtime reads from IR or converts from ONNX, PaddlePaddle formats. Consists of model structure, weights and biases | +| ov::CompiledModel | An instance of the compiled model which allows the OpenVINO™ Runtime to request (several) infer requests and perform inference synchronously or asynchronously | +| ov::InferRequest | A class that represents the end point of inference on the model compiled by the device and represented by a compiled model. Inputs are set here, outputs should be requested from this interface as well | +| ov::ProfilingInfo | Represents basic inference profiling information per operation | +| OpenVINO™ Runtime | A C++ library with a set of classes that you can use in your application to infer input tensors and get the results | +| OpenVINO™ API | The basic default API for all supported devices, which allows you to load a model from Intermediate Representation or convert from ONNX, PaddlePaddle file formars, set input and output formats and execute the model on various devices | | OpenVINO™ Core | OpenVINO™ Core is a software component that manages inference on certain Intel(R) hardware devices: CPU, GPU, MYRIAD, GNA, etc. | | ov::Layout | Image data layout refers to the representation of images batch. Layout shows a sequence of 4D or 5D tensor data in memory. A typical NCHW format represents pixel in horizontal direction, rows by vertical dimension, planes by channel and images into batch. See also [Layout API Overview](./OV_Runtime_UG/layout_overview.md) | -| ov::element::Type | Represents data element type. For example, f32 is 32-bit floating point, f16 is 16-bit floating point. Element type can be changed before loading the network to the plugin | +| ov::element::Type | Represents data element type. For example, f32 is 32-bit floating point, f16 is 16-bit floating point. | ## See Also diff --git a/docs/how_tos/how-to-links.md b/docs/how_tos/how-to-links.md index 598c3e6e2c8..e808efa1ef9 100644 --- a/docs/how_tos/how-to-links.md +++ b/docs/how_tos/how-to-links.md @@ -13,7 +13,7 @@ * [Accelerate Vision-based AI with Intel® Distribution of OpenVINO™ Toolkit](https://www.intel.ai/accelerate-vision-based-ai-with-intel-distribution-of-openvino-toolkit/) ## Custom Operations Guide -To learn about what is *custom operation* and how to work with them in the Deep Learning Deployment Toolkit, see the [Custom Operations Guide](../HOWTO/Custom_Layers_Guide.md). +To learn about what is *custom operation* and how to work with them in the Deep Learning Deployment Toolkit, see the [Custom Operations Guide](../Extensibility_UG/Intro.md). ## Introducing OpenVINO™ and Computer Vision | IoT Developer Show Season 2 | Intel Software diff --git a/docs/img/tf_openvino.png b/docs/img/tf_openvino.png new file mode 100644 index 00000000000..21a62a5f22d --- /dev/null +++ b/docs/img/tf_openvino.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21e4d7d0c2de5de20bee14196641e8efacd62ef732a8bfa765e7928d78f611a0 +size 86782 diff --git a/docs/index.rst b/docs/index.rst index 46b71ca485d..bd472b359b7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -73,8 +73,8 @@ OpenVINO™ Documentation

Learn about the alternative, web-based version of OpenVINO. DL Workbench container installation Required.

-

Inference Engine

-

Learn about OpenVINO's inference mechanism which executes the IR and ONNX models on target devices.

+

OpenVINO™ Runtime

+

Learn about OpenVINO's inference mechanism which executes the IR, ONNX, Paddle models on target devices.

Tune & Optimize

diff --git a/docs/install_guides/deployment-manager-tool.md b/docs/install_guides/deployment-manager-tool.md index 6963b7f35c6..23e1ecebbae 100644 --- a/docs/install_guides/deployment-manager-tool.md +++ b/docs/install_guides/deployment-manager-tool.md @@ -18,6 +18,8 @@ The Deployment Manager is a Python* command-line tool that creates a deployment > **IMPORTANT**: The operating system on the target system must be the same as the development system on which you are creating the package. For example, if the target system is Ubuntu 18.04, the deployment package must be created from the OpenVINO™ toolkit installed on Ubuntu 18.04. +> **TIP**: If your application requires additional dependencies, including the Microsoft Visual C++ Redistributable, use the ['--user_data' option](https://docs.openvino.ai/latest/openvino_docs_install_guides_deployment_manager_tool.html#run-standard-cli-mode) to add them to the deployment archive. Install these dependencies on the target host before running inference. + ## Create Deployment Package Using Deployment Manager There are two ways to create a deployment package that includes inference-related components of the OpenVINO™ toolkit: you can run the Deployment Manager tool in either interactive or standard CLI mode. @@ -155,7 +157,7 @@ The script successfully completes, and the deployment package is generated in th After the Deployment Manager has successfully completed, you can find the generated `.tar.gz` (for Linux or macOS) or `.zip` (for Windows) package in the output directory you specified. -To deploy the Inference Engine components from the development machine to the target system, perform the following steps: +To deploy the OpenVINO Runtime components from the development machine to the target system, perform the following steps: 1. Copy the generated archive to the target system using your preferred method. @@ -221,4 +223,4 @@ The package is unpacked to the destination directory and the following files and @endsphinxdirective -You have now finished the deployment of the Inference Engine components to the target system. +You have now finished the deployment of the OpenVINO Runtime components to the target system. diff --git a/docs/install_guides/installing-openvino-apt.md b/docs/install_guides/installing-openvino-apt.md index c2e45d71ad0..396f4c00735 100644 --- a/docs/install_guides/installing-openvino-apt.md +++ b/docs/install_guides/installing-openvino-apt.md @@ -140,5 +140,5 @@ If you are using Intel® Processor Graphics, Intel® Vision Accelerator Design w - OpenVINO™ toolkit online documentation: . - [Model Optimizer Developer Guide](../MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). - [OpenVINO Runtime User Guide](../OV_Runtime_UG/OpenVINO_Runtime_User_Guide). -- For more information on Sample Applications, see the [Inference Engine Samples Overview](../OV_Runtime_UG/Samples_Overview.md). +- For more information on Sample Applications, see the [OpenVINO Samples Overview](../OV_Runtime_UG/Samples_Overview.md). - For IoT Libraries & Code Samples see the [Intel® IoT Developer Kit](https://github.com/intel-iot-devkit). diff --git a/docs/install_guides/installing-openvino-conda.md b/docs/install_guides/installing-openvino-conda.md index a6e7b55e5ef..38a77d7e45d 100644 --- a/docs/install_guides/installing-openvino-conda.md +++ b/docs/install_guides/installing-openvino-conda.md @@ -63,5 +63,5 @@ Now you can start developing your application. - OpenVINO™ toolkit online documentation: . - [Model Optimizer Developer Guide](../MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). - [OpenVINO Runtime User Guide](../OV_Runtime_UG/OpenVINO_Runtime_User_Guide). -- For more information on Sample Applications, see the [Inference Engine Samples Overview](../OV_Runtime_UG/Samples_Overview.md). +- For more information on Sample Applications, see the [OpenVINO Samples Overview](../OV_Runtime_UG/Samples_Overview.md). - Intel® Distribution of OpenVINO™ toolkit Anaconda home page: [https://anaconda.org/intel/openvino-ie4py](https://anaconda.org/intel/openvino-ie4py) diff --git a/docs/install_guides/installing-openvino-docker-linux.md b/docs/install_guides/installing-openvino-docker-linux.md index c40e36b8069..de9f5b02577 100644 --- a/docs/install_guides/installing-openvino-docker-linux.md +++ b/docs/install_guides/installing-openvino-docker-linux.md @@ -50,7 +50,7 @@ You can find prebuilt images on: ## Preparing a Dockerfile -You can use the [available Dockerfiles on GitHub](https://github.com/openvinotoolkit/docker_ci/tree/master/dockerfiles) or generate a Dockerfile with your settings via [DockerHub CI Framework](https://github.com/openvinotoolkit/docker_ci) which can generate a Dockerfile, build, test and deploy an image with the the Intel® Distribution of OpenVINO™ toolkit. +You can use the [available Dockerfiles on GitHub](https://github.com/openvinotoolkit/docker_ci/tree/master/dockerfiles) or generate a Dockerfile with your settings via [DockerHub CI Framework](https://github.com/openvinotoolkit/docker_ci) which can generate a Dockerfile, build, test and deploy an image with the Intel® Distribution of OpenVINO™ toolkit. You can also try our [Tutorials](https://github.com/openvinotoolkit/docker_ci/tree/master/docs/tutorials) which demonstrate the usage of Docker containers with OpenVINO. ## Configuring the Image for Different Devices @@ -59,7 +59,7 @@ If you want to run inferences on a CPU or Intel® Neural Compute Stick 2, no ext ### Configuring Docker Image for GPU -By default, the distributed Docker image for OpenVINO has the the recommended version of Intel® Graphics Compute Runtime for oneAPI Level Zero and OpenCL Driver for the operating system installed inside. If you want to build an image with a custom version of OpenCL Runtime included, you need to modify the Dockerfile using the lines below (the 19.41.14441 version is used as an example) and build the image manually: +By default, the distributed Docker image for OpenVINO has the recommended version of Intel® Graphics Compute Runtime for oneAPI Level Zero and OpenCL Driver for the operating system installed inside. If you want to build an image with a custom version of OpenCL Runtime included, you need to modify the Dockerfile using the lines below (the 19.41.14441 version is used as an example) and build the image manually: **Ubuntu 18.04/20.04**: diff --git a/docs/install_guides/installing-openvino-docker-windows.md b/docs/install_guides/installing-openvino-docker-windows.md index 3910eb09489..49b27081772 100644 --- a/docs/install_guides/installing-openvino-docker-windows.md +++ b/docs/install_guides/installing-openvino-docker-windows.md @@ -52,7 +52,7 @@ You can find prebuilt images on: ## Preparing a Dockerfile -You can use the [available Dockerfiles on GitHub](https://github.com/openvinotoolkit/docker_ci/tree/master/dockerfiles) or generate a Dockerfile with your settings via [DockerHub CI Framework](https://github.com/openvinotoolkit/docker_ci) which can generate a Dockerfile, build, test and deploy an image with the the Intel® Distribution of OpenVINO™ toolkit. +You can use the [available Dockerfiles on GitHub](https://github.com/openvinotoolkit/docker_ci/tree/master/dockerfiles) or generate a Dockerfile with your settings via [DockerHub CI Framework](https://github.com/openvinotoolkit/docker_ci) which can generate a Dockerfile, build, test and deploy an image with the Intel® Distribution of OpenVINO™ toolkit. ## Configuring the Docker Image for Different Devices diff --git a/docs/install_guides/installing-openvino-images.md b/docs/install_guides/installing-openvino-images.md index 82cb0f4cdd5..101e247529b 100644 --- a/docs/install_guides/installing-openvino-images.md +++ b/docs/install_guides/installing-openvino-images.md @@ -10,5 +10,5 @@ You may install Intel® Distribution of OpenVINO™ toolkit from images and repo * [Yocto](installing-openvino-yocto.md) * [PyPI](installing-openvino-pip.md) -The open source version is available in the [OpenVINO™ toolkit GitHub repository](https://github.com/openvinotoolkit/openvino) and you can build it for supported platforms using the Inference Engine Build Instructions. +The open source version is available in the [OpenVINO™ toolkit GitHub repository](https://github.com/openvinotoolkit/openvino) and you can build it for supported platforms using the OpenVINO Build Instructions. diff --git a/docs/install_guides/installing-openvino-linux.md b/docs/install_guides/installing-openvino-linux.md index 4ea4225234e..7221a204ddd 100644 --- a/docs/install_guides/installing-openvino-linux.md +++ b/docs/install_guides/installing-openvino-linux.md @@ -85,14 +85,8 @@ This guide provides step-by-step instructions on how to install the Intel® Dist ```sh ./l_openvino_toolkit_p_.sh -a --cli ``` - - @sphinxdirective + > **NOTE**: To get additional information on all parameters that can be used, check up the help option: `--help`. Among others, you can find there `-s` option which offers silent mode, which together with `--eula approve` allows you to run whole installation with default values without any user inference. - .. note:: - To get additional information on all parameters that can be used, check up the help option: `--help`. Among others, you can find there `-s` option which offers silent mode, which together with `--eula approve` allows you to run whole installation with default values without any user inference. - - @endsphinxdirective - 6. Follow the instructions on your screen. During the installation you will be asked to accept the license agreement. The acceptance is required to continue. Check out the installation process on the image below:
![](../img/openvino-install-linux-run-boostrapper-script.gif) @@ -137,7 +131,7 @@ If you have more than one OpenVINO™ version on your machine, you can easily sw The environment variables are set. Next, you can download some additional tools. -## Step 4 (Optional): Download additional components +## Step 4 (Optional): Download Additional Components > **NOTE**: Since the OpenVINO™ 2022.1 release, the following development tools: Model Optimizer, Post-Training Optimization Tool, Model Downloader and other Open Model Zoo tools, Accuracy Checker, and Annotation Converter are not part of the installer. The OpenVINO™ Development Tools can only be installed via PyPI now. See [Install OpenVINO™ Development Tools](installing-model-dev-tools.md) for detailed steps. @@ -154,7 +148,7 @@ The environment variables are set. Next, you can download some additional tools. @endsphinxdirective -## Step 5 (Optional): Configure Inference on non-CPU Devices +## Step 5 (Optional): Configure Inference on Non-CPU Devices @sphinxdirective .. tab:: GPU @@ -164,7 +158,7 @@ The environment variables are set. Next, you can download some additional tools. .. tab:: NCS 2 Only if you want to perform inference on Intel® Neural Compute Stick 2 powered by the Intel® Movidius™ Myriad™ X VPU, follow the steps on :ref:`NCS2 Setup Guide `. - For more details, see the `Get Started page for Intel® Neural Compute Stick 2 `_. + .. tab:: VPU @@ -176,12 +170,12 @@ The environment variables are set. Next, you can download some additional tools. @endsphinxdirective -## Step 6: What's next? +## Step 6: What's Next? Now you are ready to try out the toolkit. Developing in Python: - * [Start with tensorflow models with OpenVINO™](https://docs.openvino.ai/latest/notebooks/101-tensorflow-to-openvino-with-output.html) + * [Start with TensorFlow models with OpenVINO™](https://docs.openvino.ai/latest/notebooks/101-tensorflow-to-openvino-with-output.html) * [Start with ONNX and PyTorch models with OpenVINO™](https://docs.openvino.ai/latest/notebooks/102-pytorch-onnx-to-openvino-with-output.html) * [Start with PaddlePaddle models with OpenVINO™](https://docs.openvino.ai/latest/notebooks/103-paddle-onnx-to-openvino-classification-with-output.html) diff --git a/docs/install_guides/installing-openvino-overview.md b/docs/install_guides/installing-openvino-overview.md index b3243efaa31..2160bc11d3f 100644 --- a/docs/install_guides/installing-openvino-overview.md +++ b/docs/install_guides/installing-openvino-overview.md @@ -34,4 +34,4 @@ For C++ developers, you may choose one of the following installation options to > **NOTE**: From the 2022.1 release, the OpenVINO Development Tools can **only** be installed via PyPI. See [Install OpenVINO Development Tools](installing-model-dev-tools.md) for detailed steps. -Besides, the open source version is also available in the [OpenVINO™ toolkit GitHub repository](https://github.com/openvinotoolkit/openvino/wiki/BuildingCode). You can build it for supported platforms using the Inference Engine Build Instructions. +Besides, the open source version is also available in the [OpenVINO™ toolkit GitHub repository](https://github.com/openvinotoolkit/openvino/). You can build it for supported platforms using the [OpenVINO Build Instructions](https://github.com/openvinotoolkit/openvino/wiki/BuildingCode). diff --git a/docs/install_guides/installing-openvino-pip.md b/docs/install_guides/installing-openvino-pip.md index 1db00af2ee2..08e60700142 100644 --- a/docs/install_guides/installing-openvino-pip.md +++ b/docs/install_guides/installing-openvino-pip.md @@ -37,5 +37,5 @@ For system requirements and more detailed steps, see Meta-clang Clone these Git repositories to your host machine: ```sh -git clone https://git.yoctoproject.org/git/poky -git clone https://git.yoctoproject.org/git/meta-intel -git clone https://git.openembedded.org/meta-openembedded -git clone https://github.com/kraj/meta-clang.git +git clone https://git.yoctoproject.org/git/poky --branch honister +git clone https://git.yoctoproject.org/git/meta-intel --branch honister +git clone https://git.openembedded.org/meta-openembedded --branch honister +git clone https://github.com/kraj/meta-clang.git --branch honister ``` -### Set up BitBake* Layers +### Set up BitBake Layers ```sh source poky/oe-init-build-env @@ -36,7 +34,7 @@ bitbake-layers add-layer ../meta-clang ### Set up BitBake Configurations -Include extra configuration in conf/local.conf in your build directory as required. +Include extra configuration in `conf/local.conf` in your build directory as required. ```sh # Build with SSE4.2, AVX2 etc. extensions @@ -47,42 +45,42 @@ MACHINE = "intel-skylake-64" # and is not enabled by default. PACKAGECONFIG_append_pn-openvino-inference-engine = " opencl" -# Enable building inference engine python API. +# Enable building OpenVINO Python API. # This requires meta-python layer to be included in bblayers.conf. PACKAGECONFIG_append_pn-openvino-inference-engine = " python3" -# This adds inference engine related libraries in the target image. +# This adds OpenVINO related libraries in the target image. CORE_IMAGE_EXTRA_INSTALL_append = " openvino-inference-engine" -# This adds inference engine samples in the target image. +# This adds OpenVINO samples in the target image. CORE_IMAGE_EXTRA_INSTALL_append = " openvino-inference-engine-samples" -# Include inference engine python API package in the target image. +# Include OpenVINO Python API package in the target image. CORE_IMAGE_EXTRA_INSTALL_append = " openvino-inference-engine-python3" # Enable MYRIAD plugin CORE_IMAGE_EXTRA_INSTALL_append = " openvino-inference-engine-vpu-firmware" -# Include model optimizer in the target image. +# Include Model Optimizer in the target image. CORE_IMAGE_EXTRA_INSTALL_append = " openvino-model-optimizer" ``` -## Build a Yocto Image with OpenVINO Packages +## Step 2: Build a Yocto Image with OpenVINO Packages -Run BitBake to build the minimal image with OpenVINO packages: +Run BitBake to build your image with OpenVINO packages. To build the minimal image, for example, run: ```sh bitbake core-image-minimal ``` -## Verify the Created Yocto Image with OpenVINO Packages +## Step 3: Verify the Yocto Image with OpenVINO Packages Verify that OpenVINO packages were built successfully. -Run 'oe-pkgdata-util list-pkgs | grep openvino' command. +Run the following command: ```sh oe-pkgdata-util list-pkgs | grep openvino ``` -Verify that it returns the list of packages below: +If the image was built successfully, it will return the list of packages as below: ```sh openvino-inference-engine openvino-inference-engine-dbg diff --git a/docs/install_guides/installing-openvino-yum.md b/docs/install_guides/installing-openvino-yum.md index eea119a72b5..89066763acc 100644 --- a/docs/install_guides/installing-openvino-yum.md +++ b/docs/install_guides/installing-openvino-yum.md @@ -129,5 +129,5 @@ If you are using Intel® Processor Graphics, Intel® Vision Accelerator Design w - OpenVINO™ toolkit online documentation: . - [Model Optimizer Developer Guide](../MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). - [OpenVINO Runtime User Guide](../OV_Runtime_UG/OpenVINO_Runtime_User_Guide). -- For more information on Sample Applications, see the [Inference Engine Samples Overview](../OV_Runtime_UG/Samples_Overview.md). +- For more information on Sample Applications, see the [OpenVINO Samples Overview](../OV_Runtime_UG/Samples_Overview.md). - For IoT Libraries & Code Samples see the [Intel® IoT Developer Kit](https://github.com/intel-iot-devkit). diff --git a/docs/install_guides/pypi-openvino-dev.md b/docs/install_guides/pypi-openvino-dev.md index a7d39e61a70..4069e93f3e1 100644 --- a/docs/install_guides/pypi-openvino-dev.md +++ b/docs/install_guides/pypi-openvino-dev.md @@ -8,11 +8,11 @@ OpenVINO™ toolkit is a comprehensive toolkit for quickly developing applicatio | Component | Console Script | Description | |------------------|---------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [Model Optimizer](https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html) | `mo` |**Model Optimizer** imports, converts, and optimizes models that were trained in popular frameworks to a format usable by Intel tools, especially the Inference Engine. 
Supported frameworks include Caffe\*, TensorFlow\*, MXNet\*, and ONNX\*. | +| [Model Optimizer](https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html) | `mo` |**Model Optimizer** imports, converts, and optimizes models that were trained in popular frameworks to a format usable by OpenVINO components. 
Supported frameworks include Caffe\*, TensorFlow\*, MXNet\*, PaddlePaddle\*, and ONNX\*. | | [Benchmark Tool](https://docs.openvino.ai/latest/openvino_inference_engine_tools_benchmark_tool_README.html)| `benchmark_app` | **Benchmark Application** allows you to estimate deep learning inference performance on supported devices for synchronous and asynchronous modes. | | [Accuracy Checker](https://docs.openvino.ai/latest/omz_tools_accuracy_checker.html) and
[Annotation Converter](https://docs.openvino.ai/latest/omz_tools_accuracy_checker_annotation_converters.html) | `accuracy_check`
`convert_annotation` |**Accuracy Checker** is a deep learning accuracy validation tool that allows you to collect accuracy metrics against popular datasets. The main advantages of the tool are the flexibility of configuration and a set of supported datasets, preprocessing, postprocessing, and metrics.
**Annotation Converter** is a utility that prepares datasets for evaluation with Accuracy Checker. | | [Post-Training Optimization Tool](https://docs.openvino.ai/latest/pot_README.html)| `pot` |**Post-Training Optimization Tool** allows you to optimize trained models with advanced capabilities, such as quantization and low-precision optimizations, without the need to retrain or fine-tune models. Optimizations are also available through the [API](https://docs.openvino.ai/latest/pot_compression_api_README.html). | -| [Model Downloader and other Open Model Zoo tools](https://docs.openvino.ai/latest/omz_tools_downloader.html)| `omz_downloader`
`omz_converter`
`omz_quantizer`
`omz_info_dumper`| **Model Downloader** is a tool for getting access to the collection of high-quality and extremely fast pre-trained deep learning [public](https://docs.openvino.ai/latest/omz_models_group_public.html) and [Intel](https://docs.openvino.ai/latest/omz_models_group_intel.html)-trained models. These free pre-trained models can be used to speed up the development and production deployment process without training your own models. The tool downloads model files from online sources and, if necessary, patches them to make them more usable with Model Optimizer. A number of additional tools are also provided to automate the process of working with downloaded models:
**Model Converter** is a tool for converting Open Model Zoo models that are stored in an original deep learning framework format into the Inference Engine Intermediate Representation (IR) using Model Optimizer.
**Model Quantizer** is a tool for automatic quantization of full-precision models in the IR format into low-precision versions using the Post-Training Optimization Tool.
**Model Information Dumper** is a helper utility for dumping information about the models to a stable, machine-readable format. +| [Model Downloader and other Open Model Zoo tools](https://docs.openvino.ai/latest/omz_tools_downloader.html)| `omz_downloader`
`omz_converter`
`omz_quantizer`
`omz_info_dumper`| **Model Downloader** is a tool for getting access to the collection of high-quality and extremely fast pre-trained deep learning [public](https://docs.openvino.ai/latest/omz_models_group_public.html) and [Intel](https://docs.openvino.ai/latest/omz_models_group_intel.html)-trained models. These free pre-trained models can be used to speed up the development and production deployment process without training your own models. The tool downloads model files from online sources and, if necessary, patches them to make them more usable with Model Optimizer. A number of additional tools are also provided to automate the process of working with downloaded models:
**Model Converter** is a tool for converting Open Model Zoo models that are stored in an original deep learning framework format into the OpenVINO Intermediate Representation (IR) using Model Optimizer.
**Model Quantizer** is a tool for automatic quantization of full-precision models in the IR format into low-precision versions using the Post-Training Optimization Tool.
**Model Information Dumper** is a helper utility for dumping information about the models to a stable, machine-readable format. The developer package also installs the OpenVINO™ Runtime package as a dependency. @@ -102,7 +102,7 @@ For example, to install and configure the components for working with TensorFlow ``` You will see the help message for Model Optimizer if installation finished successfully. -- To verify that Inference Engine from the **runtime package** is available, run the command below: +- To verify that OpenVINO Runtime from the **runtime package** is available, run the command below: ```sh python -c "from openvino.runtime import Core" ``` diff --git a/docs/migration_ov_2_0/docs/common_inference_pipeline.md b/docs/migration_ov_2_0/docs/common_inference_pipeline.md deleted file mode 100644 index 7a524ea9008..00000000000 --- a/docs/migration_ov_2_0/docs/common_inference_pipeline.md +++ /dev/null @@ -1,161 +0,0 @@ -# OpenVINO™ Inference Pipeline {#openvino_inference_pipeline} - -Usually to inference network with the OpenVINO™ toolkit users need to do next steps: - 1. Create Core - 2. (Optional) Read model from the disk - 2.1. Configure Input and Output of the Model - 3. Load the Model to the Device - 4. Create an Inference Request - 5. Prepare Input - 6. Start Inference - 7. Process the Inference Results - -Code snippets below cover these steps and show how application code should be changed for migration to OpenVINO™ 2.0. - -## 1. Create Core - -Inference Engine API: - -@snippet snippets/ie_common.cpp ie:create_core - -OpenVINO™ 2.0 API: - -@snippet snippets/ov_common.cpp ov_api_2_0:create_core - -## 2. (Optional) Read model from the disk - -Inference Engine API: - -@snippet snippets/ie_common.cpp ie:read_model - -OpenVINO™ 2.0 API: - -@snippet snippets/ov_common.cpp ov_api_2_0:read_model - -Read model has the same structure as in the example from [OpenVINO™ Graph Construction](@ref openvino_graph_construction) guide. - -### 2.1 Configure Input and Output of the Model - -Inference Engine API: - -@snippet snippets/ie_common.cpp ie:get_inputs_outputs - -OpenVINO™ 2.0 API: - -@snippet snippets/ov_common.cpp ov_api_2_0:get_inputs_outputs - -## 3. Load the Model to the Device - -Inference Engine API: - -@snippet snippets/ie_common.cpp ie:compile_model - -OpenVINO™ 2.0 API: - -@snippet snippets/ov_common.cpp ov_api_2_0:compile_model - -## 4. Create an Inference Request - -Inference Engine API: - -@snippet snippets/ie_common.cpp ie:create_infer_request - -OpenVINO™ 2.0 API: - -@snippet snippets/ov_common.cpp ov_api_2_0:create_infer_request - -## 5. Prepare input - -### IR v10 - -Inference Engine API: - -@snippet snippets/ie_common.cpp ie:get_input_tensor - -OpenVINO™ 2.0 API: - -@snippet snippets/ov_common.cpp ov_api_2_0:get_input_tensor_v10 - -### IR v11 - -Inference Engine API: - -@snippet snippets/ie_common.cpp ie:get_input_tensor - -OpenVINO™ 2.0 API: - -@snippet snippets/ov_common.cpp ov_api_2_0:get_input_tensor_aligned - -### ONNX - -Inference Engine API: - -@snippet snippets/ie_common.cpp ie:get_input_tensor - -OpenVINO™ 2.0 API: - -@snippet snippets/ov_common.cpp ov_api_2_0:get_input_tensor_aligned - -### From Function - -Inference Engine API: - -@snippet snippets/ie_common.cpp ie:get_input_tensor - -OpenVINO™ 2.0 API: - -@snippet snippets/ov_common.cpp ov_api_2_0:get_input_tensor_aligned - -## 6. Start Inference - -Inference Engine API: - -@snippet snippets/ie_common.cpp ie:inference - -OpenVINO™ 2.0 API: - -@snippet snippets/ov_common.cpp ov_api_2_0:inference - - -## 7. Process the Inference Results - -### IR v10 - -Inference Engine API: - -@snippet snippets/ie_common.cpp ie:get_output_tensor - -OpenVINO™ 2.0 API: - -@snippet snippets/ov_common.cpp ov_api_2_0:get_output_tensor_v10 - -### IR v11 - -Inference Engine API: - -@snippet snippets/ie_common.cpp ie:get_output_tensor - -OpenVINO™ 2.0 API: - -@snippet snippets/ov_common.cpp ov_api_2_0:get_output_tensor_aligned - -### ONNX - -Inference Engine API: - -@snippet snippets/ie_common.cpp ie:get_output_tensor - -OpenVINO™ 2.0 API: - -@snippet snippets/ov_common.cpp ov_api_2_0:get_output_tensor_aligned - -### From Function - -Inference Engine API: - -@snippet snippets/ie_common.cpp ie:get_output_tensor - -OpenVINO™ 2.0 API: - -@snippet snippets/ov_common.cpp ov_api_2_0:get_output_tensor_aligned - diff --git a/docs/migration_ov_2_0/docs/graph_construction.md b/docs/migration_ov_2_0/docs/graph_construction.md deleted file mode 100644 index 8c7f22c17df..00000000000 --- a/docs/migration_ov_2_0/docs/graph_construction.md +++ /dev/null @@ -1,12 +0,0 @@ -# OpenVINO™ graph construction {#openvino_graph_construction} - -OpenVINO™ 2.0 includes nGraph engine in a common part. The `ngraph` namespace was changed to `ov`. -Code snippets below show how application code should be changed for migration to OpenVINO™ 2.0. - -nGraph API: - -@snippet snippets/ngraph.cpp ngraph:graph - -OpenVINO™ 2.0 API: - -@snippet snippets/ov_graph.cpp ov:graph diff --git a/docs/migration_ov_2_0/docs/intro.md b/docs/migration_ov_2_0/docs/intro.md deleted file mode 100644 index 90ac48cdff9..00000000000 --- a/docs/migration_ov_2_0/docs/intro.md +++ /dev/null @@ -1,24 +0,0 @@ -# OpenVINO™ API 2.0 Transition Guide {#openvino_2_0_transition_guide} - -@sphinxdirective - -.. toctree:: - :maxdepth: 1 - :hidden: - - openvino_inference_pipeline - openvino_graph_construction - -@endsphinxdirective - -The OpenVINO™ API 2.0 introduced in order to simplify migration from other frameworks and make the OpenVINO™ API more user-friendly. -The list with differences between APIs below: - - - OpenVINO™ API 2.0 uses tensor names or indexes to work with Inputs or Outputs, the old API works with operation names. - - Structures for Shapes, element types were changed. - - Naming style was changed. The old API uses CamelCaseStyle and OpenVINO™ API 2.0 uses snake_case for function names. - - Namespaces were aligned between components. - -Please look at next transition guides to understand how transit own application to OpenVINO™ API 2.0. - - [OpenVINO™ Graph Construction](graph_construction.md) - - [OpenVINO™ Common Inference pipeline](common_inference_pipeline.md) diff --git a/docs/model_zoo.md b/docs/model_zoo.md index 2d94c62ef79..6122a054d9a 100644 --- a/docs/model_zoo.md +++ b/docs/model_zoo.md @@ -20,6 +20,13 @@ omz_demos +.. toctree:: + :maxdepth: 1 + :hidden: + :caption: Model API + + omz_model_api_ovms_adapter + @endsphinxdirective diff --git a/docs/onnx_custom_op/CMakeLists.txt b/docs/onnx_custom_op/CMakeLists.txt deleted file mode 100644 index 2f2a77c58b7..00000000000 --- a/docs/onnx_custom_op/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (C) 2018-2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -# - -# [cmake:onnx_custom_op] -set(CMAKE_CXX_STANDARD 11) - -set(TARGET_NAME "onnx_custom_op") - -find_package(OpenVINO REQUIRED COMPONENTS ONNX) - -add_library(${TARGET_NAME} STATIC onnx_custom_op.cpp onnx_custom_op.hpp) - -target_link_libraries(${TARGET_NAME} PUBLIC openvino::core openvino::frontend::onnx) -# [cmake:onnx_custom_op] - -# Enable code style check -add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME}) diff --git a/docs/onnx_custom_op/custom_relu_model.prototxt b/docs/onnx_custom_op/custom_relu_model.prototxt deleted file mode 100644 index 3845cc00ce2..00000000000 --- a/docs/onnx_custom_op/custom_relu_model.prototxt +++ /dev/null @@ -1,52 +0,0 @@ -ir_version: 3 -producer_name: "nGraph ONNX Importer" -graph { - node { - input: "in" - output: "out" - name: "customrelu" - op_type: "CustomRelu" - domain: "com.example" - attribute { - name: "alpha" - type: FLOAT - f: 2 - } - attribute { - name: "beta" - type: FLOAT - f: 3 - } - } - name: "custom relu graph" - input { - name: "in" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 8 - } - } - } - } - } - output { - name: "out" - type { - tensor_type { - elem_type: 1 - shape { - dim { - dim_value: 8 - } - } - } - } - } -} -opset_import { - domain: "com.example" - version: 1 -} diff --git a/docs/onnx_custom_op/onnx_custom_op.cpp b/docs/onnx_custom_op/onnx_custom_op.cpp deleted file mode 100644 index 399a6d2b4a3..00000000000 --- a/docs/onnx_custom_op/onnx_custom_op.cpp +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (C) 2018-2022 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -//! [onnx_custom_op:headers] -// onnx_import/onnx_utils.hpp provides ngraph::onnx_import::register_operator function, that registers operator in ONNX importer's set. -#include -// ngraph/opsets/opset5.hpp provides the declaration of predefined nGraph operator set -#include -//! [onnx_custom_op:headers] - -void register_custom_relu_operator() { - // CustomRelu is defined as follows: - // x >= 0 => f(x) = x * alpha - // x < 0 => f(x) = x * beta - -//! [onnx_custom_op:register_operator] - ngraph::onnx_import::register_operator( - "CustomRelu", 1, "com.example", [](const ngraph::onnx_import::Node& onnx_node) -> ngraph::OutputVector { - namespace opset = ngraph::opset5; - - ngraph::OutputVector ng_inputs{onnx_node.get_ng_inputs()}; - const ngraph::Output& data = ng_inputs.at(0); - // create constant node with a single element that's equal to zero - std::shared_ptr zero_node = opset::Constant::create(data.get_element_type(), ngraph::Shape{}, {0}); - // create a negative map for 'data' node, 1 for negative values , 0 for positive values or zero - // then convert it from boolean type to `data.get_element_type()` - std::shared_ptr negative_map = std::make_shared( - std::make_shared(data, zero_node), data.get_element_type()); - // create a positive map for 'data' node, 0 for negative values , 1 for positive values or zero - // then convert it from boolean type to `data.get_element_type()` - std::shared_ptr positive_map = std::make_shared( - std::make_shared(data, zero_node), data.get_element_type()); - - // fetch alpha and beta attributes from ONNX node - float alpha = onnx_node.get_attribute_value("alpha", 1); // if 'alpha' attribute is not provided in the model, then the default value is 1 - float beta = onnx_node.get_attribute_value("beta"); - // create constant node with a single element 'alpha' with type f32 - std::shared_ptr alpha_node = opset::Constant::create(ngraph::element::f32, ngraph::Shape{}, {alpha}); - // create constant node with a single element 'beta' with type f32 - std::shared_ptr beta_node = opset::Constant::create(ngraph::element::f32, ngraph::Shape{}, {beta}); - - return { - std::make_shared( - std::make_shared(alpha_node, std::make_shared(data, positive_map)), - std::make_shared(beta_node, std::make_shared(data, negative_map)) - ) - }; - }); -//! [onnx_custom_op:register_operator] -} - -void unregister_custom_relu_operator() { -//! [onnx_custom_op:unregister_operator] - ngraph::onnx_import::unregister_operator("CustomRelu", 1, "com.example"); -//! [onnx_custom_op:unregister_operator] -} diff --git a/docs/onnx_custom_op/onnx_custom_op.hpp b/docs/onnx_custom_op/onnx_custom_op.hpp deleted file mode 100644 index 3554226da6b..00000000000 --- a/docs/onnx_custom_op/onnx_custom_op.hpp +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (C) 2018-2022 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -void register_custom_relu_operator(); -void unregister_custom_relu_operator(); diff --git a/docs/ops/opset1.md b/docs/ops/opset1.md index 3eee791065d..fc4db9ce049 100644 --- a/docs/ops/opset1.md +++ b/docs/ops/opset1.md @@ -3,7 +3,7 @@ This specification document describes `opset1` operation set supported in OpenVINO. Support for each particular operation from the list below depends on the capabilities available in a inference plugin and may vary among different hardware platforms and devices. Examples of operation instances are expressed as IR V10 xml -snippets. Such IR is generated by the Model Optimizer. The semantics match corresponding nGraph operation classes +snippets. Such IR is generated by the Model Optimizer. The semantics match corresponding OpenVINO operation classes declared in `namespace opset1`. diff --git a/docs/ops/opset2.md b/docs/ops/opset2.md index 04f9dfe048f..3ff00c6b762 100644 --- a/docs/ops/opset2.md +++ b/docs/ops/opset2.md @@ -3,7 +3,7 @@ This specification document describes `opset2` operation set supported in OpenVINO. Support for each particular operation from the list below depends on the capabilities available in a inference plugin and may vary among different hardware platforms and devices. Examples of operation instances are expressed as IR V10 xml -snippets. Such IR is generated by the Model Optimizer. The semantics match corresponding nGraph operation classes +snippets. Such IR is generated by the Model Optimizer. The semantics match corresponding OpenVINO operation classes declared in `namespace opset2`. diff --git a/docs/ops/opset3.md b/docs/ops/opset3.md index 525e25d3449..dfdf64710be 100644 --- a/docs/ops/opset3.md +++ b/docs/ops/opset3.md @@ -3,7 +3,7 @@ This specification document describes `opset3` operation set supported in OpenVINO. Support for each particular operation from the list below depends on the capabilities available in a inference plugin and may vary among different hardware platforms and devices. Examples of operation instances are expressed as IR V10 xml -snippets. Such IR is generated by the Model Optimizer. The semantics match corresponding nGraph operation classes +snippets. Such IR is generated by the Model Optimizer. The semantics match corresponding OpenVINO operation classes declared in `namespace opset3`. diff --git a/docs/ops/opset4.md b/docs/ops/opset4.md index b20fe4ac619..96e864bb5ae 100644 --- a/docs/ops/opset4.md +++ b/docs/ops/opset4.md @@ -3,7 +3,7 @@ This specification document describes `opset4` operation set supported in OpenVINO. Support for each particular operation from the list below depends on the capabilities available in a inference plugin and may vary among different hardware platforms and devices. Examples of operation instances are expressed as IR V10 xml -snippets. Such IR is generated by the Model Optimizer. The semantics match corresponding nGraph operation classes +snippets. Such IR is generated by the Model Optimizer. The semantics match corresponding OpenVINO operation classes declared in `namespace opset4`. diff --git a/docs/ops/opset5.md b/docs/ops/opset5.md index f980a96a043..d0c6653a0c6 100644 --- a/docs/ops/opset5.md +++ b/docs/ops/opset5.md @@ -3,7 +3,7 @@ This specification document describes `opset5` operation set supported in OpenVINO. Support for each particular operation from the list below depends on the capabilities available in a inference plugin and may vary among different hardware platforms and devices. Examples of operation instances are expressed as IR V10 xml -snippets. Such IR is generated by the Model Optimizer. The semantics match corresponding nGraph operation classes +snippets. Such IR is generated by the Model Optimizer. The semantics match corresponding OpenVINO operation classes declared in `namespace opset5`. diff --git a/docs/ops/opset6.md b/docs/ops/opset6.md index 3154484d56e..a2f35e51834 100644 --- a/docs/ops/opset6.md +++ b/docs/ops/opset6.md @@ -3,7 +3,7 @@ This specification document describes `opset6` operation set supported in OpenVINO. Support for each particular operation from the list below depends on the capabilities available in a inference plugin and may vary among different hardware platforms and devices. Examples of operation instances are expressed as IR V10 xml -snippets. Such IR is generated by the Model Optimizer. The semantics match corresponding nGraph operation classes +snippets. Such IR is generated by the Model Optimizer. The semantics match corresponding OpenVINO operation classes declared in `namespace opset6`. diff --git a/docs/ops/opset7.md b/docs/ops/opset7.md index 8a786fc2b39..95a0734fa89 100644 --- a/docs/ops/opset7.md +++ b/docs/ops/opset7.md @@ -3,7 +3,7 @@ This specification document describes the `opset7` operation set supported in OpenVINO™. Support for each particular operation from the list below depends on the capabilities available in an inference plugin and may vary among different hardware platforms and devices. Examples of operation instances are provided as IR V10 xml -snippets. Such IR is generated by the Model Optimizer. The semantics match corresponding nGraph operation classes +snippets. Such IR is generated by the Model Optimizer. The semantics match corresponding OpenVINO operation classes declared in `namespace opset7`. diff --git a/docs/ops/opset8.md b/docs/ops/opset8.md index a6274cdf968..70a9e98fecb 100644 --- a/docs/ops/opset8.md +++ b/docs/ops/opset8.md @@ -3,7 +3,7 @@ This specification document describes the `opset8` operation set supported in OpenVINO™. Support for each particular operation from the list below depends on the capabilities of an inference plugin and may vary among different hardware platforms and devices. Examples of operation instances are provided as IR V10 xml -snippets. Such IR is generated by the Model Optimizer. The semantics match corresponding nGraph operation classes +snippets. Such IR is generated by the Model Optimizer. The semantics match corresponding OpenVINO operation classes declared in `namespace opset8`. diff --git a/docs/ops/sequence/RNNCell_3.md b/docs/ops/sequence/RNNCell_3.md index 58f1b9ddd87..70ee0d9c8e0 100644 --- a/docs/ops/sequence/RNNCell_3.md +++ b/docs/ops/sequence/RNNCell_3.md @@ -57,11 +57,11 @@ Formula: * **2**: `H` - 2D tensor of type *T* `[batch_size, hidden_size]`, initial hidden state. **Required.** -* **3**: `W` - 2D tensor tensor of type *T* `[hidden_size, input_size]`, the weights for matrix multiplication. **Required.** +* **3**: `W` - 2D tensor of type *T* `[hidden_size, input_size]`, the weights for matrix multiplication. **Required.** -* **4**: `R` - 2D tensor tensor of type *T* `[hidden_size, hidden_size]`, the recurrence weights for matrix multiplication. **Required.** +* **4**: `R` - 2D tensor of type *T* `[hidden_size, hidden_size]`, the recurrence weights for matrix multiplication. **Required.** -* **5**: `B` 1D tensor tensor of type *T* `[hidden_size]`, the sum of biases (weights and recurrence weights). **Required.** +* **5**: `B` 1D tensor of type *T* `[hidden_size]`, the sum of biases (weights and recurrence weights). **Required.** **Outputs** diff --git a/docs/optimization_guide/dldt_deployment_optimization_guide.md b/docs/optimization_guide/dldt_deployment_optimization_guide.md index 4293b3b4140..f26c0dd558a 100644 --- a/docs/optimization_guide/dldt_deployment_optimization_guide.md +++ b/docs/optimization_guide/dldt_deployment_optimization_guide.md @@ -114,7 +114,7 @@ Inference precision directly affects the performance. Model Optimizer can produce an IR with different precision. For example, an FP16 IR initially targets VPU and GPU devices, while, for example, for the CPU, an FP16 IR is typically up-scaled to the regular FP32 automatically upon loading. But notice that further device-specific inference precision settings are available, for example, [8-bit integer](../OV_Runtime_UG/Int8Inference.md) or [bfloat16](../OV_Runtime_UG/Bfloat16Inference.md), which is specific to the CPU inference, below. -Note that for the [MULTI device](../OV_Runtime_UG/supported_plugins/MULTI.md) plugin that supports automatic inference on multiple devices in parallel, you can use an FP16 IR (no need for FP32). +Note that for the [Multi-Device execution](../OV_Runtime_UG/multi_device.md) that supports automatic inference on multiple devices in parallel, you can use an FP16 IR (no need for FP32). You can find more information, including preferred data types for specific devices, in the [Supported Devices](../OV_Runtime_UG/supported_plugins/Supported_Devices.md) document. @@ -199,7 +199,7 @@ Intel® Vision Accelerator Design with Intel® Movidius™ VPUs requir ## Heterogeneity -Heterogeneous execution (constituted by the dedicated Inference Engine [“Hetero” plugin](../OV_Runtime_UG/supported_plugins/HETERO.md)) enables to schedule a network inference to the multiple devices. +Heterogeneous execution (constituted by the dedicated Inference Engine [“Hetero” device](../OV_Runtime_UG/hetero_execution.md)) enables to schedule a network inference to the multiple devices. ### Typical Heterogeneous Scenarios of Concern @@ -231,7 +231,7 @@ Similarly, if there are too much subgraphs, the synchronization and data transfe The general affinity “rule of thumb” is to keep computationally-intensive kernels on the accelerator, and "glue" or helper kernels on the CPU. Notice that this includes the granularity considerations. For example, running some custom activation (that comes after every accelerator-equipped convolution) on the CPU might result in performance degradation due to too much data type and/or layout conversions, even though the activation itself can be extremely fast. In this case, it might make sense to consider implementing the kernel for the accelerator (see Optimizing Custom Kernels). The conversions typically manifest themselves as outstanding (comparing to CPU-only execution) 'Reorder' entries (see Internal Inference Performance Counters). -For general details on the heterogeneous plugin, refer to the [corresponding section in the Inference Engine Developer Guide](../OV_Runtime_UG/supported_plugins/HETERO.md). +For general details on the heterogeneous mode, refer to the [Heterogeneous execution guide](../OV_Runtime_UG/hetero_execution.md). ### Trying the Heterogeneous Plugin with Inference Engine Samples @@ -270,7 +270,7 @@ The following tips are provided to give general guidance on optimizing execution ### Analyzing Heterogeneous Execution -There is a dedicated configuration option that enables dumping the visualization of the subgraphs created by the heterogeneous plugin, please see code example in the [HETERO plugin documentation](../OV_Runtime_UG/supported_plugins/HETERO.md) +There is a dedicated configuration option that enables dumping the visualization of the subgraphs created by the heterogeneous mode, please see code example in the [Heterogeneous execution guide](../OV_Runtime_UG/hetero_execution.md) After enabling the configuration key, the heterogeneous plugin generates two files: @@ -279,10 +279,10 @@ After enabling the configuration key, the heterogeneous plugin generates two fil You can use GraphViz\* utility or `.dot` converters (for example, to `.png` or `.pdf`), like xdot\*, available on Linux\* OS with `sudo apt-get install xdot`. -You can also use performance data (in the [Benchmark App](../../samples/cpp/benchmark_app/README.md), it is an option `-pc`) to get performance data on each subgraph. Again, refer to the [HETERO plugin documentation](../OV_Runtime_UG/supported_plugins/HETERO.md) and to Internal Inference Performance Counters for a general counters information. +You can also use performance data (in the [Benchmark App](../../samples/cpp/benchmark_app/README.md), it is an option `-pc`) to get performance data on each subgraph. Again, refer to the [Heterogeneous execution guide](../OV_Runtime_UG/hetero_execution.md) and to Internal Inference Performance Counters for a general counters information. ## Multi-Device Execution -OpenVINO™ toolkit supports automatic multi-device execution, please see [MULTI-Device plugin description](../OV_Runtime_UG/supported_plugins/MULTI.md). +OpenVINO™ toolkit supports automatic multi-device execution, please see [Multi-Device execution](../OV_Runtime_UG/multi_device.md) description. In the next chapter you can find the device-specific tips, while this section covers few recommendations for the multi-device execution: - MULTI usually performs best when the fastest device is specified first in the list of the devices. diff --git a/docs/optimization_guide/dldt_deployment_optimization_guide_additional.md b/docs/optimization_guide/dldt_deployment_optimization_guide_additional.md index 9f685e57468..fd70c080d61 100644 --- a/docs/optimization_guide/dldt_deployment_optimization_guide_additional.md +++ b/docs/optimization_guide/dldt_deployment_optimization_guide_additional.md @@ -26,7 +26,7 @@ usually gives the same best latency as a single request on the single socket, bu $ benchmark_app -m -nstreams 2 ``` Number of NUMA nodes on the machine can be queried via 'lscpu'. -Please see more on the NUMA support in the [Optimization Guide](../OV_Runtime_UG/supported_plugins/MULTI.md). +Please see more on the NUMA support in the [Optimization Guide](../OV_Runtime_UG/multi_device.md). ## Threading diff --git a/docs/optimization_guide/dldt_optimization_guide.md b/docs/optimization_guide/dldt_optimization_guide.md index 7381545800e..33b39bc1da8 100644 --- a/docs/optimization_guide/dldt_optimization_guide.md +++ b/docs/optimization_guide/dldt_optimization_guide.md @@ -15,7 +15,7 @@ Latency measures inference time (ms) required to process a single input. When it ## How to Improve Performance -> **NOTE**: Make sure that your model can be successfully inferred with OpenVINO Inference Engine. +> **NOTE**: Make sure that your model can be successfully inferred with OpenVINO Runtime. Inside OpenVINO there are two ways how to get better performance number: during developing and deployment your model. **It is possible to combine both developing and deployment optimizations**. diff --git a/docs/scripts/doxy_md_filter.py b/docs/scripts/doxy_md_filter.py index 59219ed6b2e..4ed1b50196f 100644 --- a/docs/scripts/doxy_md_filter.py +++ b/docs/scripts/doxy_md_filter.py @@ -71,6 +71,13 @@ class DoxyMDFilter: rel_path = os.path.relpath(link_path, self.input_dir).replace('\\', '/') self.content = self.content.replace(link, rel_path) + def remove_comment_block_sphinxdirective(self): + """ + Remove comment blocks from `sphinxdirective` + """ + self.content = re.sub(r'\<\!\-\-\s*?\@sphinxdirective', '@sphinxdirective', self.content) + self.content = re.sub(r'\@endsphinxdirective\s*?\-\-\>', '@endsphinxdirective', self.content) + def copy_images(self): """ Go through image links and copy them into output_folder @@ -97,6 +104,7 @@ class DoxyMDFilter: Do all processing operations on a markdown file """ self.replace_image_links() + self.remove_comment_block_sphinxdirective() self.replace_md_links() self.copy_markdown() self.copy_images() diff --git a/docs/snippets/CMakeLists.txt b/docs/snippets/CMakeLists.txt index 2e4ef05c1a7..889936e8579 100644 --- a/docs/snippets/CMakeLists.txt +++ b/docs/snippets/CMakeLists.txt @@ -35,6 +35,7 @@ list(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/dldt_optimization_guide2.c # create a static library add_library(${TARGET_NAME} STATIC ${SOURCES}) +target_include_directories(${TARGET_NAME} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../template_extension/new/") if(CLDNN__IOCL_ICD_INCDIRS) target_include_directories(${TARGET_NAME} SYSTEM PRIVATE ${CLDNN__IOCL_ICD_INCDIRS}) @@ -57,3 +58,22 @@ if(NOT MSVC) endif() target_link_libraries(${TARGET_NAME} PRIVATE openvino::runtime openvino::runtime::dev) + +set(TARGET_NAME "ov_integration_snippet") +# [cmake:integration_example] +cmake_minimum_required(VERSION 3.10) +set(CMAKE_CXX_STANDARD 11) + + +find_package(OpenVINO REQUIRED) + +add_executable(${TARGET_NAME} src/main.cpp) + +target_link_libraries(${TARGET_NAME} PRIVATE openvino::runtime) +# [cmake:integration_example] +if(NOT MSVC) + target_compile_options(${TARGET_NAME} PRIVATE -Wno-unused-variable) + if(CMAKE_COMPILER_IS_GNUCXX) + target_compile_options(${TARGET_NAME} PRIVATE -Wno-unused-but-set-variable) + endif() +endif() diff --git a/docs/snippets/Integrate_with_customer_application_new_API.cpp b/docs/snippets/Integrate_with_customer_application_new_API.cpp deleted file mode 100644 index 3bee3d4df4c..00000000000 --- a/docs/snippets/Integrate_with_customer_application_new_API.cpp +++ /dev/null @@ -1,133 +0,0 @@ -#include - -int main() { -const std::string output_name = "output_name"; -const std::string input_name = "input_name"; -//! [part0] -InferenceEngine::Core core; -InferenceEngine::CNNNetwork network; -InferenceEngine::ExecutableNetwork executable_network; -//! [part0] - -//! [part1] -network = core.ReadNetwork("Model.xml"); -//! [part1] - -//! [part2] -network = core.ReadNetwork("model.onnx"); -//! [part2] - -//! [part3] -/** Take information about all topology inputs **/ -InferenceEngine::InputsDataMap input_info = network.getInputsInfo(); -/** Take information about all topology outputs **/ -InferenceEngine::OutputsDataMap output_info = network.getOutputsInfo(); -//! [part3] - -//! [part4] -/** Iterate over all input info**/ -for (auto &item : input_info) { - auto input_data = item.second; - input_data->setPrecision(InferenceEngine::Precision::U8); - input_data->setLayout(InferenceEngine::Layout::NCHW); - input_data->getPreProcess().setResizeAlgorithm(InferenceEngine::RESIZE_BILINEAR); - input_data->getPreProcess().setColorFormat(InferenceEngine::ColorFormat::RGB); -} -/** Iterate over all output info**/ -for (auto &item : output_info) { - auto output_data = item.second; - output_data->setPrecision(InferenceEngine::Precision::FP32); - output_data->setLayout(InferenceEngine::Layout::NC); -} -//! [part4] - -//! [part5] -executable_network = core.LoadNetwork(network, "CPU"); -//! [part5] - -//! [part6] -/** Optional config. E.g. this enables profiling of performance counters. **/ -std::map config = {{ InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, InferenceEngine::PluginConfigParams::YES }}; -executable_network = core.LoadNetwork(network, "CPU", config); -//! [part6] - -//! [part7] -auto infer_request = executable_network.CreateInferRequest(); -//! [part7] - -auto infer_request1 = executable_network.CreateInferRequest(); -auto infer_request2 = executable_network.CreateInferRequest(); - -//! [part8] -/** Iterate over all input blobs **/ -for (auto & item : input_info) { - auto input_name = item.first; - /** Get input blob **/ - auto input = infer_request.GetBlob(input_name); - /** Fill input tensor with planes. First b channel, then g and r channels **/ -// ... -} -//! [part8] - -//! [part9] -auto output = infer_request1.GetBlob(output_name); -infer_request2.SetBlob(input_name, output); -//! [part9] - -//! [part10] -/** inputBlob points to input of a previous network and - cropROI contains coordinates of output bounding box **/ -InferenceEngine::Blob::Ptr inputBlob; -InferenceEngine::ROI cropRoi; -//... - -/** roiBlob uses shared memory of inputBlob and describes cropROI - according to its coordinates **/ -auto roiBlob = InferenceEngine::make_shared_blob(inputBlob, cropRoi); -infer_request2.SetBlob(input_name, roiBlob); -//! [part10] - -//! [part11] -/** Iterate over all input blobs **/ -for (auto & item : input_info) { - auto input_data = item.second; - /** Create input blob **/ - InferenceEngine::TBlob::Ptr input; - // assuming input precision was asked to be U8 in prev step - input = InferenceEngine::make_shared_blob( - InferenceEngine::TensorDesc(InferenceEngine::Precision::U8, input_data->getTensorDesc().getDims(), - input_data->getTensorDesc().getLayout())); - input->allocate(); - infer_request.SetBlob(item.first, input); - - /** Fill input tensor with planes. First b channel, then g and r channels **/ -// ... -} -//! [part11] - -//! [part12] -infer_request.StartAsync(); -infer_request.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY); -//! [part12] - -auto sync_infer_request = executable_network.CreateInferRequest(); - -//! [part13] -sync_infer_request.Infer(); -//! [part13] - -//! [part14] - for (auto &item : output_info) { - auto output_name = item.first; - auto output = infer_request.GetBlob(output_name); - { - auto const memLocker = output->cbuffer(); // use const memory locker - // output_buffer is valid as long as the lifetime of memLocker - const float *output_buffer = memLocker.as(); - /** output_buffer[] - accessing output blob data **/ - } - } -//! [part14] - -return 0; -} diff --git a/docs/snippets/example_ngraph_utils.cpp b/docs/snippets/example_ngraph_utils.cpp index c6d32373551..6cf9faf21a4 100644 --- a/docs/snippets/example_ngraph_utils.cpp +++ b/docs/snippets/example_ngraph_utils.cpp @@ -20,115 +20,6 @@ #include // ! [ov:include] - -// ! [ov:create_simple_model] -std::shared_ptr create_simple_function() { - // This example shows how to create ov::Function - // - // Parameter--->Multiply--->Add--->Result - // Constant---' / - // Constant---' - - // Create opset8::Parameter operation with static shape - auto data = std::make_shared(ov::element::f32, ov::Shape{3, 1, 2}); - - auto mul_constant = ov::opset8::Constant::create(ov::element::f32, ov::Shape{1}, {1.5}); - auto mul = std::make_shared(data, mul_constant); - - auto add_constant = ov::opset8::Constant::create(ov::element::f32, ov::Shape{1}, {0.5}); - auto add = std::make_shared(mul, add_constant); - - // Create opset8::Result operation - auto res = std::make_shared(mul); - - // Create nGraph function - return std::make_shared(ov::ResultVector{res}, ov::ParameterVector{data}); -} -// ! [ov:create_simple_model] - -// ! [ov:create_advanced_model] -std::shared_ptr create_advanced_function() { - // Advanced example with multi output operation - // - // Parameter->Split---0-->Result - // | `--1-->Relu-->Result - // `----2-->Result - - auto data = std::make_shared(ov::element::f32, ov::Shape{1, 3, 64, 64}); - - // Create Constant for axis value - auto axis_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{}/*scalar shape*/, {1}); - - // Create opset8::Split operation that splits input to three slices across 1st dimension - auto split = std::make_shared(data, axis_const, 3); - - // Create opset8::Relu operation that takes 1st Split output as input - auto relu = std::make_shared(split->output(1)/*specify explicit output*/); - - // Results operations will be created automatically based on provided OutputVector - return std::make_shared(ov::OutputVector{split->output(0), relu, split->output(2)}, ov::ParameterVector{data}); -} -// ! [ov:create_advanced_model] - -void pattern_matcher_examples(std::shared_ptr node) { -{ -// ! [pattern:simple_example] -// Pattern example -auto input = std::make_shared(ov::element::i64, ov::Shape{1}); -auto shapeof = std::make_shared(input); - -// Create Matcher with Parameter->ShapeOf pattern -auto m = std::make_shared(shapeof, "MyPatternBasedTransformation"); -// ! [pattern:simple_example] - -// ! [pattern:callback_example] -ov::graph_rewrite_callback callback = [](ov::pass::pattern::Matcher& m) { - // Get root node - std::shared_ptr root_node = m.get_match_root(); - - // Get all nodes matched by pattern - ov::NodeVector nodes = m.get_matched_nodes(); - - // Transformation code - return false; -}; -// ! [pattern:callback_example] -} - -{ -// ! [pattern:label_example] -// Detect Multiply with arbitrary first input and second as Constant -// ov::pattern::op::Label - represent arbitrary input -auto input = ov::pass::pattern::any_input(); -auto value = ov::opset8::Constant::create(ov::element::f32, ov::Shape{1}, {0.5}); -auto mul = std::make_shared(input, value); -auto m = std::make_shared(mul, "MultiplyMatcher"); -// ! [pattern:label_example] -} - -{ -// ! [pattern:concat_example] -// Detect Concat operation with arbitrary number of inputs -auto concat = ov::pass::pattern::wrap_type(); -auto m = std::make_shared(concat, "ConcatMatcher"); -// ! [pattern:concat_example] -} - -{ -// ! [pattern:predicate_example] -// Detect Multiply->Add sequence where mul has exactly one consumer -auto mul = ov::pass::pattern::wrap_type(ov::pass::pattern::consumers_count(1)/*сheck consumers count*/); -auto add = ov::pass::pattern::wrap_type({mul, ov::pass::pattern::any_input()}); -auto m = std::make_shared(add, "MultiplyAddMatcher"); -// Matcher can be used to match pattern manually on given node -if (m->match(node->output(0))) { - // Successfully matched -} -// ! [pattern:predicate_example] -} - -} - bool ngraph_api_examples(std::shared_ptr node) { { // ! [ngraph:ports_example] @@ -154,16 +45,6 @@ auto consumers = output.get_target_inputs(); // ! [ngraph:ports_example] } -{ -// ! [ngraph:shape] -auto partial_shape = node->input(0).get_partial_shape(); // get zero input partial shape -if (partial_shape.is_dynamic() /* or !partial_shape.is_static() */) { - return false; -} -auto static_shape = partial_shape.get_shape(); -// ! [ngraph:shape] -} - { // ! [ngraph:shape_check] auto partial_shape = node->input(0).get_partial_shape(); // get zero input partial shape @@ -185,63 +66,6 @@ auto dim = partial_shape[1].get_length(); return true; } -// ! [ngraph:replace_node] -bool ngraph_replace_node(std::shared_ptr node) { - // Step 1. Verify that node has opset8::Negative type - auto neg = std::dynamic_pointer_cast(node); - if (!neg) { - return false; - } - - // Step 2. Create opset8::Multiply operation where the first input is negative operation input and second as Constant with -1 value - auto mul = std::make_shared(neg->input_value(0), - ov::opset8::Constant::create(neg->get_element_type(), ov::Shape{1}, {-1})); - - mul->set_friendly_name(neg->get_friendly_name()); - // TODO: Move to new API - ngraph::copy_runtime_info(neg, mul); - - // Step 3. Replace Negative operation with Multiply operation - ov::replace_node(neg, mul); - return true; - - // Step 4. Negative operation will be removed automatically because all consumers was moved to Multiply operation -} -// ! [ngraph:replace_node] - -// ! [ngraph:insert_node] -// Step 1. Lets suppose that we have a node with single output port and we want to insert additional operation new_node after it -void insert_example(std::shared_ptr node) { - // Get all consumers for node - auto consumers = node->output(0).get_target_inputs(); - - // Step 2. Create new node. Let it be opset1::Relu. - auto new_node = std::make_shared(node); - - // Step 3. Reconnect all consumers to new_node - for (auto input : consumers) { - input.replace_source_output(new_node); - } -} -// ! [ngraph:insert_node] - -// ! [ngraph:insert_node_with_copy] -void insert_example_with_copy(std::shared_ptr node) { - // Make a node copy - auto node_copy = node->clone_with_new_inputs(node->input_values()); - // Create new node - auto new_node = std::make_shared(node_copy); - ov::replace_node(node, new_node); -} -// ! [ngraph:insert_node_with_copy] - -void eliminate_example(std::shared_ptr node) { -// ! [ngraph:eliminate_node] -// Suppose we have a node that we want to remove -bool success = replace_output_update_name(node->output(0), node->input_value(0)); -// ! [ngraph:eliminate_node] -} - // ! [ov:serialize] void serialize_example(std::shared_ptr f) { ov::pass::Manager manager; diff --git a/docs/snippets/ie_common.cpp b/docs/snippets/ie_common.cpp index 25cf78a0c43..7c408b14bf0 100644 --- a/docs/snippets/ie_common.cpp +++ b/docs/snippets/ie_common.cpp @@ -13,10 +13,8 @@ int main() { InferenceEngine::CNNNetwork network = core.ReadNetwork("model.xml"); //! [ie:read_model] - //! [ie:get_inputs_outputs] InferenceEngine::InputsDataMap inputs = network.getInputsInfo(); InferenceEngine::OutputsDataMap outputs = network.getOutputsInfo(); - //! [ie:get_inputs_outputs] //! [ie:compile_model] InferenceEngine::ExecutableNetwork exec_network = core.LoadNetwork(network, "CPU"); @@ -29,7 +27,6 @@ int main() { //! [ie:get_input_tensor] InferenceEngine::Blob::Ptr input_blob1 = infer_request.GetBlob(inputs.begin()->first); // fill first blob - InferenceEngine::SizeVector dims1 = input_blob1->getTensorDesc().getDims(); InferenceEngine::MemoryBlob::Ptr minput1 = InferenceEngine::as(input_blob1); if (minput1) { // locked memory holder should be alive all time while access to its @@ -39,6 +36,7 @@ int main() { auto data = minputHolder.as::value_type*>(); // Fill data ... } + InferenceEngine::Blob::Ptr input_blob2 = infer_request.GetBlob("data2"); // fill first blob InferenceEngine::MemoryBlob::Ptr minput2 = InferenceEngine::as(input_blob2); diff --git a/docs/snippets/ov_auto_batching.cpp b/docs/snippets/ov_auto_batching.cpp new file mode 100644 index 00000000000..1e943b3c8f5 --- /dev/null +++ b/docs/snippets/ov_auto_batching.cpp @@ -0,0 +1,41 @@ +#include + +int main() { + ov::Core core; + auto model = core.read_model("sample.xml"); + +//! [compile_model] +{ + auto compiled_model = core.compile_model(model, "GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)); +} +//! [compile_model] + +//! [compile_model_no_auto_batching] +{ + // disabling the automatic batching + // leaving intact other configurations options that the device selects for the 'throughput' hint + auto compiled_model = core.compile_model(model, "GPU", {ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), + ov::hint::allow_auto_batching(false)}); +} +//! [compile_model_no_auto_batching] + +//! [query_optimal_num_requests] +{ + // when the batch size is automatically selected by the implementation + // it is important to query/create and run the sufficient #requests + auto compiled_model = core.compile_model(model, "GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)); + auto num_requests = compiled_model.get_property(ov::optimal_number_of_infer_requests); +} +//! [query_optimal_num_requests] + +//! [hint_num_requests] +{ + // limiting the available parallel slack for the 'throughput' hint via the ov::hint::num_requests + // so that certain parameters (like selected batch size) are automatically accommodated accordingly + auto compiled_model = core.compile_model(model, "GPU", {ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), + ov::hint::num_requests(4)}); +} +//! [hint_num_requests] + + return 0; +} diff --git a/docs/snippets/ov_auto_batching.py b/docs/snippets/ov_auto_batching.py new file mode 100644 index 00000000000..1e943b3c8f5 --- /dev/null +++ b/docs/snippets/ov_auto_batching.py @@ -0,0 +1,41 @@ +#include + +int main() { + ov::Core core; + auto model = core.read_model("sample.xml"); + +//! [compile_model] +{ + auto compiled_model = core.compile_model(model, "GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)); +} +//! [compile_model] + +//! [compile_model_no_auto_batching] +{ + // disabling the automatic batching + // leaving intact other configurations options that the device selects for the 'throughput' hint + auto compiled_model = core.compile_model(model, "GPU", {ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), + ov::hint::allow_auto_batching(false)}); +} +//! [compile_model_no_auto_batching] + +//! [query_optimal_num_requests] +{ + // when the batch size is automatically selected by the implementation + // it is important to query/create and run the sufficient #requests + auto compiled_model = core.compile_model(model, "GPU", ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)); + auto num_requests = compiled_model.get_property(ov::optimal_number_of_infer_requests); +} +//! [query_optimal_num_requests] + +//! [hint_num_requests] +{ + // limiting the available parallel slack for the 'throughput' hint via the ov::hint::num_requests + // so that certain parameters (like selected batch size) are automatically accommodated accordingly + auto compiled_model = core.compile_model(model, "GPU", {ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), + ov::hint::num_requests(4)}); +} +//! [hint_num_requests] + + return 0; +} diff --git a/docs/snippets/ov_common.cpp b/docs/snippets/ov_common.cpp index ca3c8f83fa1..00b834954b9 100644 --- a/docs/snippets/ov_common.cpp +++ b/docs/snippets/ov_common.cpp @@ -62,16 +62,11 @@ int main() { //! [ov_api_2_0:create_core] //! [ov_api_2_0:read_model] - std::shared_ptr network = core.read_model("model.xml"); + std::shared_ptr model = core.read_model("model.xml"); //! [ov_api_2_0:read_model] - //! [ov_api_2_0:get_inputs_outputs] - std::vector> inputs = network->inputs(); - std::vector> outputs = network->outputs(); - //! [ov_api_2_0:get_inputs_outputs] - //! [ov_api_2_0:compile_model] - ov::CompiledModel compiled_model = core.compile_model(network, "CPU"); + ov::CompiledModel compiled_model = core.compile_model(model, "CPU"); //! [ov_api_2_0:compile_model] //! [ov_api_2_0:create_infer_request] diff --git a/docs/snippets/ov_dynamic_shapes.cpp b/docs/snippets/ov_dynamic_shapes.cpp new file mode 100644 index 00000000000..6c259c972dc --- /dev/null +++ b/docs/snippets/ov_dynamic_shapes.cpp @@ -0,0 +1,157 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include +#include + +void reshape_with_dynamics() { +{ +//! [ov_dynamic_shapes:reshape_undefined] +ov::Core core; +auto model = core.read_model("model.xml"); + +// Set one static dimension (= 1) and another dynamic dimension (= Dimension()) +model->reshape({{1, ov::Dimension()}}); // {1,?} + +// The same as above +model->reshape({{1, -1}}); // {1,?} + +// Or set both dimensions as dynamic if both are going to be changed dynamically +model->reshape({{ov::Dimension(), ov::Dimension()}}); // {?,?} + +// The same as above +model->reshape({{-1, -1}}); // {?,?} +//! [ov_dynamic_shapes:reshape_undefined] +//! [ov_dynamic_shapes:reshape_bounds] +// Both dimensions are dynamic, first may have size within 1..10 and the second is withing 8..512 +model->reshape({{ov::Dimension(1, 10), ov::Dimension(8, 512)}}); // {1..10,8..512} + +// Both dimensions are dynamic, first doesn't have bounds, the second is in 8..512 +model->reshape({{-1, ov::Dimension(8, 512)}}); // {?,8..512} +//! [ov_dynamic_shapes:reshape_bounds] +} +{ +ov::Core core; +auto model = core.read_model("model.xml"); +//! [ov_dynamic_shapes:print_dynamic] +// Print output partial shape +std::cout << model->output().get_partial_shape() << "\n"; + +// Print input partial shape +std::cout << model->input().get_partial_shape() << "\n"; +//! [ov_dynamic_shapes:print_dynamic] +} +{ +ov::Core core; +//! [ov_dynamic_shapes:detect_dynamic] +auto model = core.read_model("model.xml"); + +if (model->input(0).get_partial_shape().is_dynamic()) { + // input is dynamic +} + +if (model->output(0).get_partial_shape().is_dynamic()) { + // output is dynamic +} + +if (model->output(0).get_partial_shape()[1].is_dynamic()) { + // 1-st dimension of output is dynamic +} +//! [ov_dynamic_shapes:detect_dynamic] +} +} + +void set_tensor() { +ov::Core core; +auto model = core.read_model("model.xml"); +auto executable = core.compile_model(model); +auto infer_request = executable.create_infer_request(); +//! [ov_dynamic_shapes:set_input_tensor] +// The first inference call + +// Create tensor compatible to the model input +// Shape {1, 128} is compatible to any reshape statements made in previous examples +auto input_tensor_1 = ov::Tensor(model->input().get_element_type(), {1, 128}); +// ... write values to input_tensor_1 + +// Set the tensor as a model input within infer request +infer_request.set_input_tensor(input_tensor_1); + +// Do the inference +infer_request.infer(); + +// Retrieve a tensor representing the output data +ov::Tensor output_tensor = infer_request.get_output_tensor(); + +// For dynamic models output shape usually depends on input shape, +// that means shape of output tensor is initialized after the first inference only +// and has to be queried after every infer request +auto output_shape_1 = output_tensor.get_shape(); + +// Take a pointer of an appropriate type to tensor data and read elements according to the shape +// Assuming model output is f32 data type +auto data_1 = output_tensor.data(); +// ... read values + +// The second inference call, repeat steps: + +// Create another tensor (if the previous one cannot be utilized) +// Notice, the shape is different from input_tensor_1 +auto input_tensor_2 = ov::Tensor(model->input().get_element_type(), {1, 200}); +// ... write values to input_tensor_2 + +infer_request.set_input_tensor(input_tensor_2); + +infer_request.infer(); + +// No need to call infer_request.get_output_tensor() again +// output_tensor queried after the first inference call above is valid here. +// But it may not be true for the memory underneath as shape changed, so re-take a pointer: +auto data_2 = output_tensor.data(); + +// and new shape as well +auto output_shape_2 = output_tensor.get_shape(); + +// ... read values in data_2 according to the shape output_shape_2 +//! [ov_dynamic_shapes:set_input_tensor] +} + +void get_tensor() { +ov::Core core; +auto model = core.read_model("model.xml"); +auto executable = core.compile_model(model); +auto infer_request = executable.create_infer_request(); +//! [ov_dynamic_shapes:get_input_tensor] +// The first inference call + +// Get the tensor; shape is not initialized +auto input_tensor = infer_request.get_input_tensor(); + +// Set shape is required +input_tensor.set_shape({1, 128}); +// ... write values to input_tensor + +infer_request.infer(); +ov::Tensor output_tensor = infer_request.get_output_tensor(); +auto output_shape_1 = output_tensor.get_shape(); +auto data_1 = output_tensor.data(); +// ... read values + +// The second inference call, repeat steps: + +// Set a new shape, may reallocate tensor memory +input_tensor.set_shape({1, 200}); +// ... write values to input_tensor memory + +infer_request.infer(); +auto data_2 = output_tensor.data(); +auto output_shape_2 = output_tensor.get_shape(); +// ... read values in data_2 according to the shape output_shape_2 +//! [ov_dynamic_shapes:get_input_tensor] +} + +int main() { +reshape_with_dynamics(); +get_tensor(); +set_tensor(); +} diff --git a/docs/snippets/ov_extensions.cpp b/docs/snippets/ov_extensions.cpp new file mode 100644 index 00000000000..0abab9d3bfa --- /dev/null +++ b/docs/snippets/ov_extensions.cpp @@ -0,0 +1,26 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include +#include +#include + +int main() { +{ +//! [add_extension] +ov::Core core; +// Use operation type to add operation extension +core.add_extension(); +// or you can add operation extension to this method +core.add_extension(ov::OpExtension()); +//! [add_extension] +} +{ +//! [add_extension_lib] +ov::Core core; +// Load extensions library to ov::Core +core.add_extension("openvino_template_extension.so"); +//! [add_extension_lib] +} +return 0; +} diff --git a/docs/snippets/ov_extensions.py b/docs/snippets/ov_extensions.py new file mode 100644 index 00000000000..4f53700c746 --- /dev/null +++ b/docs/snippets/ov_extensions.py @@ -0,0 +1,15 @@ +# Copyright (C) 2018-2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import openvino.runtime as ov + +#! [add_extension] +# Not implemented +#! [add_extension] + +#! [add_extension_lib] +core = ov.Core() +# Load extensions library to ov::Core +core.add_extension("openvino_template_extension.so") +#! [add_extension_lib] diff --git a/docs/snippets/ov_hetero.cpp b/docs/snippets/ov_hetero.cpp index 52874aea2bc..c21a70be639 100644 --- a/docs/snippets/ov_hetero.cpp +++ b/docs/snippets/ov_hetero.cpp @@ -43,9 +43,12 @@ auto compiled_model = core.compile_model(model, device); { //! [configure_fallback_devices] auto compiled_model = core.compile_model(model, "HETERO", - ov::device::priorities("GPU", "CPU"), // GPU with fallback to CPU - ov::device::properties("CPU", ov::enable_profiling(true)), // profiling is enabled only for CPU - ov::device::properties("GPU", ov::hint::inference_precision(ov::element::f16)) // FP16 inference precision only for GPU + // GPU with fallback to CPU + ov::device::priorities("GPU", "CPU"), + // profiling is enabled only for CPU + ov::device::properties("CPU", ov::enable_profiling(true)), + // FP16 inference precision only for GPU + ov::device::properties("GPU", ov::hint::inference_precision(ov::element::f16)) ); //! [configure_fallback_devices] } diff --git a/docs/snippets/ov_hetero.py b/docs/snippets/ov_hetero.py index 4f42847efe0..52874aea2bc 100644 --- a/docs/snippets/ov_hetero.py +++ b/docs/snippets/ov_hetero.py @@ -34,12 +34,10 @@ auto compiled_model = core.compile_model(model, device); //! [compile_model] { auto compiled_model = core.compile_model(model, "HETERO:GPU,CPU"); -} -{ - auto compiled_model = core.compile_model(model, "HETERO", ov::device::priorities("GPU", "CPU")); -} -{ - auto compiled_model = core.compile_model(model, "HETERO", ov::device::priorities("GPU,CPU")); + // or with ov::device::priorities with multiple args + compiled_model = core.compile_model(model, "HETERO", ov::device::priorities("GPU", "CPU")); + // or with ov::device::priorities with a single argument + compiled_model = core.compile_model(model, "HETERO", ov::device::priorities("GPU,CPU")); } //! [compile_model] { diff --git a/docs/snippets/ov_infer_request.cpp b/docs/snippets/ov_infer_request.cpp new file mode 100644 index 00000000000..42be537252e --- /dev/null +++ b/docs/snippets/ov_infer_request.cpp @@ -0,0 +1,112 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +// ! [ov:include] +#include +// ! [ov:include] + +int main() { +ov::Core core; +std::shared_ptr model; +ov::CompiledModel compiled_model; + +//! [create_infer_request] +auto infer_request = compiled_model.create_infer_request(); +//! [create_infer_request] + +//! [sync_infer] +infer_request.infer(); +//! [sync_infer] + +//! [async_infer] +infer_request.start_async(); +//! [async_infer] + +//! [wait] +infer_request.wait(); +//! [wait] + +//! [wait_for] +infer_request.wait_for(std::chrono::milliseconds(10)); +//! [wait_for] + +//! [set_callback] +infer_request.set_callback([&](std::exception_ptr ex_ptr) { + if (!ex_ptr) { + // all done. Output data can be processed. + // You can fill the input data and run inference one more time: + infer_request.start_async(); + } else { + // Something wrong, you can analyze exception_ptr + } +}); +//! [set_callback] + +//! [cancel] +infer_request.cancel(); +//! [cancel] + +{ +//! [get_set_one_tensor] +auto input_tensor = infer_request.get_input_tensor(); +auto output_tensor = infer_request.get_output_tensor(); +//! [get_set_one_tensor] +} + +{ +//! [get_set_index_tensor] +auto input_tensor = infer_request.get_input_tensor(0); +auto output_tensor = infer_request.get_output_tensor(1); +//! [get_set_index_tensor] +} + +//! [get_set_tensor] +auto tensor1 = infer_request.get_tensor("tensor_name1"); +ov::Tensor tensor2; +infer_request.set_tensor("tensor_name2", tensor2); +//! [get_set_tensor] + +{ +//! [get_set_tensor_by_port] +auto input_port = model->input(0); +auto output_port = model->output("tensor_name"); +ov::Tensor input_tensor; +infer_request.set_tensor(input_port, input_tensor); +auto output_tensor = infer_request.get_tensor(output_port); +//! [get_set_tensor_by_port] +} + +auto infer_request1 = compiled_model.create_infer_request(); +auto infer_request2 = compiled_model.create_infer_request(); + +//! [cascade_models] +auto output = infer_request1.get_output_tensor(0); +infer_request2.set_input_tensor(0, output); +//! [cascade_models] + +//! [roi_tensor] +/** input_tensor points to input of a previous network and + cropROI contains coordinates of output bounding box **/ +ov::Tensor input_tensor(ov::element::f32, ov::Shape({1, 3, 20, 20})); +ov::Coordinate begin({0, 0, 0, 0}); +ov::Coordinate end({1, 2, 3, 3}); +//... + +/** roi_tensor uses shared memory of input_tensor and describes cropROI + according to its coordinates **/ +ov::Tensor roi_tensor(input_tensor, begin, end); +infer_request2.set_tensor("input_name", roi_tensor); +//! [roi_tensor] + +{ +//! [remote_tensor] +ov::RemoteContext context = core.get_default_context("GPU"); +auto input_port = compiled_model.input("tensor_name"); +ov::RemoteTensor remote_tensor = context.create_tensor(input_port.get_element_type(), input_port.get_shape()); +infer_request.set_tensor(input_port, remote_tensor); +//! [remote_tensor] +} + +return 0; +} diff --git a/docs/snippets/ov_infer_request.py b/docs/snippets/ov_infer_request.py new file mode 100644 index 00000000000..c18ea6316f2 --- /dev/null +++ b/docs/snippets/ov_infer_request.py @@ -0,0 +1,97 @@ +# Copyright (C) 2018-2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import numpy as np +#! [import] +import openvino.runtime as ov +#! [import] + +core = ov.Core() +model = core.read_model("model.xml") +compiled_model = core.compile_model(model, "AUTO") + +#! [create_infer_request] +infer_request = compiled_model.create_infer_request() +#! [create_infer_request] + +#! [sync_infer] +infer_request.infer() +#! [sync_infer] + +#! [async_infer] +infer_request.start_async() +#! [async_infer] + +#! [wait] +infer_request.wait() +#! [wait] + +#! [wait_for] +infer_request.wait_for(10) +#! [wait_for] + +#! [set_callback] +def callback(request, userdata): + request.start_async() + +infer_request.set_callback(callback) +#! [set_callback] + +#! [cancel] +infer_request.cancel() +#! [cancel] + +#! [get_set_one_tensor] +input_tensor = infer_request.get_input_tensor() +output_tensor = infer_request.get_output_tensor() +#! [get_set_one_tensor] + +#! [get_set_index_tensor] +input_tensor = infer_request.get_input_tensor(0) +output_tensor = infer_request.get_output_tensor(1) +#! [get_set_index_tensor] + +#! [get_set_name_tensor] +input_tensor = infer_request.get_tensor("input_name") +output_tensor = infer_request.get_tensor("output_name") +#! [get_set_name_tensor] + +#! [get_set_tensor] +tensor1 = infer_request.get_tensor("tensor_name1") +tensor2 = ov.Tensor() +infer_request.set_tensor("tensor_name2", tensor2) +#! [get_set_tensor] + +#! [get_set_tensor_by_port] +input_port = model.input(0) +output_port = model.input("tensor_name") +input_tensor = ov.Tensor() +infer_request.set_tensor(input_port, input_tensor) +output_tensor = infer_request.get_tensor(output_port) +#! [get_set_tensor_by_port] + +infer_request1 = compiled_model.create_infer_request() +infer_request2 = compiled_model.create_infer_request() + +#! [cascade_models] +output = infer_request1.get_output_tensor(0) +infer_request2.set_input_tensor(0, output) +#! [cascade_models] + +#! [roi_tensor] +# input_tensor points to input of a previous network and +# cropROI contains coordinates of output bounding box **/ +input_tensor = ov.Tensor(type=ov.Type.f32, shape=ov.Shape([1, 3, 20, 20])) +begin = [0, 0, 0, 0] +end = [1, 2, 3, 3] +# ... + +# roi_tensor uses shared memory of input_tensor and describes cropROI +# according to its coordinates **/ +roi_tensor = ov.Tensor(input_tensor, begin, end) +infer_request2.set_tensor("input_name", roi_tensor) +#! [roi_tensor] + +#! [remote_tensor] +# NOT SUPPORTED +#! [remote_tensor] diff --git a/docs/snippets/ov_model_snippets.cpp b/docs/snippets/ov_model_snippets.cpp new file mode 100644 index 00000000000..12a8c732fc3 --- /dev/null +++ b/docs/snippets/ov_model_snippets.cpp @@ -0,0 +1,337 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +// ! [ov:include] +#include +#include +// ! [ov:include] + +#include +#include +#include +#include +#include +#include +#include +#include + + +// ! [ov:create_simple_model] +std::shared_ptr create_simple_model() { + // This example shows how to create ov::Model + // + // Parameter--->Multiply--->Add--->Result + // Constant---' / + // Constant---' + + // Create opset8::Parameter operation with static shape + auto data = std::make_shared(ov::element::f32, ov::Shape{3, 1, 2}); + + auto mul_constant = ov::opset8::Constant::create(ov::element::f32, ov::Shape{1}, {1.5}); + auto mul = std::make_shared(data, mul_constant); + + auto add_constant = ov::opset8::Constant::create(ov::element::f32, ov::Shape{1}, {0.5}); + auto add = std::make_shared(mul, add_constant); + + // Create opset8::Result operation + auto res = std::make_shared(mul); + + // Create nGraph function + return std::make_shared(ov::ResultVector{res}, ov::ParameterVector{data}); +} +// ! [ov:create_simple_model] + +// ! [ov:create_advanced_model] +std::shared_ptr create_advanced_model() { + // Advanced example with multi output operation + // + // Parameter->Split---0-->Result + // | `--1-->Relu-->Result + // `----2-->Result + + auto data = std::make_shared(ov::element::f32, ov::Shape{1, 3, 64, 64}); + + // Create Constant for axis value + auto axis_const = ov::opset8::Constant::create(ov::element::i64, ov::Shape{} /*scalar shape*/, {1}); + + // Create opset8::Split operation that splits input to three slices across 1st dimension + auto split = std::make_shared(data, axis_const, 3); + + // Create opset8::Relu operation that takes 1st Split output as input + auto relu = std::make_shared(split->output(1) /*specify explicit output*/); + + // Results operations will be created automatically based on provided OutputVector + return std::make_shared(ov::OutputVector{split->output(0), relu, split->output(2)}, + ov::ParameterVector{data}); +} +// ! [ov:create_advanced_model] + +void ov_api_examples() { + std::shared_ptr node = std::make_shared(ov::element::f32, ov::PartialShape{ov::Dimension::dynamic(), 3, 64, 64}); + + // ! [ov:partial_shape] + ov::Shape static_shape; + ov::PartialShape partial_shape = node->output(0).get_partial_shape(); // get zero output partial shape + if (!partial_shape.is_dynamic() /* or partial_shape.is_static() */) { + static_shape = partial_shape.get_shape(); + } + // ! [ov:partial_shape] +} + +// ! [ov:serialize] +void serialize_example(const std::shared_ptr& f) { + // Need include: + // * openvino/pass/manager.hpp + // * openvino/pass/serialize.hpp + ov::pass::Manager manager; + + // Serialize ov::Model to IR + manager.register_pass("/path/to/file/model.xml", "/path/to/file/model.bin"); + + manager.run_passes(f); +} +// ! [ov:serialize] + +// ! [ov:visualize] +void visualize_example(const std::shared_ptr& m) { + // Need include: + // * openvino/pass/manager.hpp + // * openvino/pass/visualize_tree.hpp + ov::pass::Manager manager; + + // Serialize ov::Model to before.svg file before transformation + manager.register_pass("image.svg"); + + manager.run_passes(m); +} +// ! [ov:visualize] + +void model_inputs() { +std::shared_ptr model; +//! [all_inputs_ouputs] +/* Take information about all topology inputs */ +auto inputs = model->inputs(); +/* Take information about all topology outputs */ +auto outputs = model->outputs(); +//! [all_inputs_ouputs] +} + +void pattern_matcher_examples(std::shared_ptr node) { +{ +// ! [pattern:simple_example] +// Pattern example +auto input = std::make_shared(ov::element::i64, ov::Shape{1}); +auto shapeof = std::make_shared(input); + +// Create Matcher with Parameter->ShapeOf pattern +auto m = std::make_shared(shapeof, "MyPatternBasedTransformation"); +// ! [pattern:simple_example] + +// ! [pattern:callback_example] +ov::graph_rewrite_callback callback = [](ov::pass::pattern::Matcher& m) { + // Get root node + std::shared_ptr root_node = m.get_match_root(); + + // Get all nodes matched by pattern + ov::NodeVector nodes = m.get_matched_nodes(); + + // Transformation code + return false; +}; +// ! [pattern:callback_example] +} + +{ +// ! [pattern:label_example] +// Detect Multiply with arbitrary first input and second as Constant +// ov::pattern::op::Label - represent arbitrary input +auto input = ov::pass::pattern::any_input(); +auto value = ov::opset8::Constant::create(ov::element::f32, ov::Shape{1}, {0.5}); +auto mul = std::make_shared(input, value); +auto m = std::make_shared(mul, "MultiplyMatcher"); +// ! [pattern:label_example] +} + +{ +// ! [pattern:concat_example] +// Detect Concat operation with arbitrary number of inputs +auto concat = ov::pass::pattern::wrap_type(); +auto m = std::make_shared(concat, "ConcatMatcher"); +// ! [pattern:concat_example] +} + +{ +// ! [pattern:predicate_example] +// Detect Multiply->Add sequence where mul has exactly one consumer +auto mul = ov::pass::pattern::wrap_type(ov::pass::pattern::consumers_count(1)/*сheck consumers count*/); +auto add = ov::pass::pattern::wrap_type({mul, ov::pass::pattern::any_input()}); +auto m = std::make_shared(add, "MultiplyAddMatcher"); +// Matcher can be used to match pattern manually on given node +if (m->match(node->output(0))) { + // Successfully matched +} +// ! [pattern:predicate_example] +} + +} + +bool openvino_api_examples(std::shared_ptr node) { +{ +// ! [ov:ports_example] +// Let's suppose that node is opset8::Convolution operation +// as we know opset8::Convolution has two input ports (data, weights) and one output port +ov::Input data = node->input(0); +ov::Input weights = node->input(1); +ov::Output output = node->output(0); + +// Getting shape and type +auto pshape = data.get_partial_shape(); +auto el_type = data.get_element_type(); + +// Getting parent for input port +ov::Output parent_output; +parent_output = data.get_source_output(); + +// Another short way to get partent for output port +parent_output = node->input_value(0); + +// Getting all consumers for output port +auto consumers = output.get_target_inputs(); +// ! [ov:ports_example] +} + +{ +// ! [ngraph:shape_check] +auto partial_shape = node->input(0).get_partial_shape(); // get zero input partial shape + +// Check that input shape rank is static +if (!partial_shape.rank().is_static()) { + return false; +} +auto rank_size = partial_shape.rank().get_length(); + +// Check that second dimension is not dynamic +if (rank_size < 2 || partial_shape[1].is_dynamic()) { + return false; +} +auto dim = partial_shape[1].get_length(); +// ! [ngraph:shape_check] +} + +return true; +} + +// ! [ov:replace_node] +bool ov_replace_node(std::shared_ptr node) { + // Step 1. Verify that node has opset8::Negative type + auto neg = std::dynamic_pointer_cast(node); + if (!neg) { + return false; + } + + // Step 2. Create opset8::Multiply operation where the first input is negative operation input and second as Constant with -1 value + auto mul = std::make_shared(neg->input_value(0), + ov::opset8::Constant::create(neg->get_element_type(), ov::Shape{1}, {-1})); + + mul->set_friendly_name(neg->get_friendly_name()); + ov::copy_runtime_info(neg, mul); + + // Step 3. Replace Negative operation with Multiply operation + ov::replace_node(neg, mul); + return true; + + // Step 4. Negative operation will be removed automatically because all consumers was moved to Multiply operation +} +// ! [ov:replace_node] + +bool ov_manual_replace_node(std::shared_ptr node) { +auto neg = std::dynamic_pointer_cast(node); +if (!neg) { + return false; +} + +auto mul = std::make_shared(neg->input_value(0), + ov::opset8::Constant::create(neg->get_element_type(), ov::Shape{1}, {-1})); + +mul->set_friendly_name(neg->get_friendly_name()); +ov::copy_runtime_info(neg, mul); + +// ! [ov:manual_replace] +// All neg->output(0) consumers will be moved to mul->output(0) port +neg->output(0).replace(mul->output(0)); +// ! [ov:manual_replace] +return true; +} + +// ! [ov:insert_node] +// Step 1. Lets suppose that we have a node with single output port and we want to insert additional operation new_node after it +void insert_example(std::shared_ptr node) { + // Get all consumers for node + auto consumers = node->output(0).get_target_inputs(); + + // Step 2. Create new node. Let it be opset8::Relu. + auto new_node = std::make_shared(node); + + // Step 3. Reconnect all consumers to new_node + for (auto input : consumers) { + input.replace_source_output(new_node); + } +} +// ! [ov:insert_node] + +// ! [ov:insert_node_with_copy] +void insert_example_with_copy(std::shared_ptr node) { + // Make a node copy + auto node_copy = node->clone_with_new_inputs(node->input_values()); + // Create new node + auto new_node = std::make_shared(node_copy); + ov::replace_node(node, new_node); +} +// ! [ov:insert_node_with_copy] + +void eliminate_example(std::shared_ptr node) { +// ! [ov:eliminate_node] +// Suppose we have a node that we want to remove +bool success = ov::replace_output_update_name(node->output(0), node->input_value(0)); +// ! [ov:eliminate_node] +} + +void replace_friendly_name() { +auto div = std::make_shared(); +// ! [ov:replace_friendly_name] +// Replace Div operation with Power and Multiply sub-graph and set original friendly name to Multiply operation +auto pow = std::make_shared(div->input(1).get_source_output(), + ov::op::v0::Constant::create(div->get_input_element_type(1), ov::Shape{1}, {-1})); +auto mul = std::make_shared(div->input(0).get_source_output(), pow); +mul->set_friendly_name(div->get_friendly_name()); +ngraph::replace_node(div, mul); +// ! [ov:replace_friendly_name] +} + +void constant_subgraph() { +// ! [ov:constant_subgraph] +// After ConstantFolding pass Power will be replaced with Constant +auto input = std::make_shared(ov::element::f32, ov::Shape{1}); +auto pow = std::make_shared(ov::opset8::Constant::create(ov::element::f32, ov::Shape{1}, {2}), + ov::opset8::Constant::create(ov::element::f32, ov::Shape{1}, {3})); +auto mul = std::make_shared(input /* not constant input */, pow); +// ! [ov:constant_subgraph] +} + +void copy_runtime_info_snippet() { +std::shared_ptr transpose, reshape, div, pow, mul, conv, bias, conv_fused, a, b, c, e, f; +// ! [ov:copy_runtime_info] +// Replace Transpose with Reshape operation (1:1) +ov::copy_runtime_info(transpose, reshape); + +// Replace Div operation with Power and Multiply sub-graph (1:N) +ov::copy_runtime_info(div, {pow, mul}); + +// Fuse Convolution with Add operation (N:1) +ov::copy_runtime_info({conv, bias}, {conv_fused}); + +// Any other transformation that replaces one sub-graph with another sub-graph (N:M) +ov::copy_runtime_info({a, b, c}, {e, f}); +// ! [ov:copy_runtime_info] +} diff --git a/docs/snippets/ov_model_snippets.py b/docs/snippets/ov_model_snippets.py new file mode 100644 index 00000000000..575ab07e80e --- /dev/null +++ b/docs/snippets/ov_model_snippets.py @@ -0,0 +1,88 @@ +# Copyright (C) 2018-2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import numpy as np +#! [import] +import openvino.runtime as ov +#! [import] +import openvino.runtime.passes as passes + +# ! [ov:create_simple_model] +def create_simple_model(): + # This example shows how to create ov::Function + # + # Parameter--->Multiply--->Add--->Result + # Constant---' / + # Constant---' + data = ov.opset8.parameter([3, 1, 2], ov.Type.f32) + mul_constant = ov.opset8.constant([1.5], ov.Type.f32) + mul = ov.opset8.multiply(data, mul_constant) + add_constant = ov.opset8.constant([0.5], ov.Type.f32) + add = ov.opset8.add(mul, add_constant) + res = ov.opset8.result(add) + return ov.Model([res], [data], "model") +# ! [ov:create_simple_model] + +# ! [ov:create_advanced_model] +def create_advanced_model(): + # Advanced example with multi output operation + # + # Parameter->Split---0-->Result + # | `--1-->Relu-->Result + # `----2-->Result + data = ov.opset8.parameter(ov.Shape([1, 3, 64, 64]), ov.Type.f32) + # Create Constant for axis value + axis_const = ov.opset8.constant(ov.Type.i64, ov.Shape({}), [1]) + + # Create opset8::Split operation that splits input to three slices across 1st dimension + split = ov.opset8.split(data, axis_const, 3) + + # Create opset8::Relu operation that takes 1st Split output as input + relu = ov.opset8.relu(split.output(1)) + + # Results operations will be created automatically based on provided OutputVector + return ov.Model([split.output(0), relu, split.output[2]], [data], "model") +# ! [ov:create_advanced_model] + +def ov_api_examples(): + # Doesn't work + # node = ov.opset8.parameter(ov.PartialShape([ov.Dimension.dynamic(), 3, 64, 64]), np.float32) + node = ov.opset8.parameter(ov.PartialShape([ov.Dimension.dynamic(), ov.Dimension(3), ov.Dimension(64), ov.Dimension(64)]), np.float32) + + # it doesn't work: + # static_shape = ov.Shape() + # ! [ov:partial_shape] + partial_shape = node.output(0).get_partial_shape() # get zero output partial shape + if not partial_shape.is_dynamic: # or partial_shape.is_static + static_shape = partial_shape.get_shape() + # ! [ov:partial_shape] + +# ! [ov:serialize] +def serialize_example(m : ov.Model): + # Need import: + # * import openvino.runtime.passes as passes + pass_manager = passes.Manager() + pass_manager.register_pass(pass_name="Serialize", xml_path='model.xml', bin_path='model.bin') + pass_manager.run_passes(m) +# ! [ov:serialize] + +# ! [ov:visualize] +def visualize_example(m : ov.Model): + # Need import: + # * import openvino.runtime.passes as passes + pass_manager = passes.Manager() + pass_manager.register_pass(pass_name="VisualTree", file_name='image.svg') + pass_manager.run_passes(m) +# ! [ov:visualize] + +def model_inputs_outputs(model : ov.Model): + #! [all_inputs_ouputs] + inputs = model.inputs + outputs = model.outputs + #! [all_inputs_ouputs] + + +if __name__ == '__main__': + ov_api_examples() + create_simple_model() + create_advanced_model() diff --git a/docs/snippets/ov_preprocessing_migration.cpp b/docs/snippets/ov_preprocessing_migration.cpp new file mode 100644 index 00000000000..032afb9c8b8 --- /dev/null +++ b/docs/snippets/ov_preprocessing_migration.cpp @@ -0,0 +1,124 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include +#include +#include + +#include "inference_engine.hpp" + +int main_new() { + std::string model_path; + std::string tensor_name; + + ov::Core core; + std::shared_ptr model = core.read_model(model_path); + ov::preprocess::PrePostProcessor ppp(model); + + { + //! [ov_mean_scale] +ov::preprocess::PrePostProcessor ppp(model); +ov::preprocess::InputInfo& input = ppp.input(tensor_name); +// we only need to know where is C dimension +input.model().set_layout("...C"); +// specify scale and mean values, order of operations is important +input.preprocess().mean(116.78f).scale({ 57.21f, 57.45f, 57.73f }); +// insert preprocessing operations to the 'model' +model = ppp.build(); + //! [ov_mean_scale] + } + + { + //! [ov_conversions] +ov::preprocess::PrePostProcessor ppp(model); +ov::preprocess::InputInfo& input = ppp.input(tensor_name); +input.tensor().set_layout("NHWC").set_element_type(ov::element::u8); +input.model().set_layout("NCHW"); +// layout and precision conversion is inserted automatically, +// because tensor format != model input format +model = ppp.build(); + //! [ov_conversions] + } + + { + //! [ov_color_space] +ov::preprocess::PrePostProcessor ppp(model); +ov::preprocess::InputInfo& input = ppp.input(tensor_name); +input.tensor().set_color_format(ov::preprocess::ColorFormat::NV12_TWO_PLANES); +// add NV12 to BGR conversion +input.preprocess().convert_color(ov::preprocess::ColorFormat::BGR); +// and insert operations to the model +model = ppp.build(); + //! [ov_color_space] + } + + { + //! [ov_image_scale] +ov::preprocess::PrePostProcessor ppp(model); +ov::preprocess::InputInfo& input = ppp.input(tensor_name); +// scale from the specified tensor size +input.tensor().set_spatial_static_shape(448, 448); +// need to specify H and W dimensions in model, others are not important +input.model().set_layout("??HW"); +// scale to model shape +input.preprocess().resize(ov::preprocess::ResizeAlgorithm::RESIZE_LINEAR); +// and insert operations to the model +model = ppp.build(); + //! [ov_image_scale] + } + +return 0; +} + +int main_old() { + std::string model_path; + std::string operation_name; + + InferenceEngine::Core core; + InferenceEngine::CNNNetwork network = core.ReadNetwork(model_path); + + { + //! [mean_scale] +auto preProcess = network.getInputsInfo()[operation_name]->getPreProcess(); +preProcess.init(3); +preProcess[0]->meanValue = 116.78f; +preProcess[1]->meanValue = 116.78f; +preProcess[2]->meanValue = 116.78f; +preProcess[0]->stdScale = 57.21f; +preProcess[1]->stdScale = 57.45f; +preProcess[2]->stdScale = 57.73f; +preProcess.setVariant(InferenceEngine::MEAN_VALUE); + //! [mean_scale] + } + + { + //! [conversions] +auto inputInfo = network.getInputsInfo()[operation_name]; +inputInfo->setPrecision(InferenceEngine::Precision::U8); +inputInfo->setLayout(InferenceEngine::Layout::NHWC); +// model input layout is always NCHW in Inference Engine +// for shapes with 4 dimensions + //! [conversions] + } + + { + //! [color_space] +auto preProcess = network.getInputsInfo()[operation_name]->getPreProcess(); +// Inference Engine supposes NV12 as two inputs which need to be passed +// as InferenceEngine::NV12Blob composed of two Y and UV planes +preProcess.setColorFormat(InferenceEngine::NV12); + //! [color_space] + } + + { + //! [image_scale] +auto preProcess = network.getInputsInfo()[operation_name]->getPreProcess(); +// Inference Engine supposes input for resize is always in NCHW layout +// while for OpenVINO Runtime API 2.0 `H` and `W` dimensions must be specified +// Also, current code snippet supposed resize from dynamic shapes +preProcess.setResizeAlgorithm(InferenceEngine::ResizeAlgorithm::RESIZE_BILINEAR); + //! [image_scale] + } + + return 0; +} diff --git a/docs/snippets/ov_properties_api.cpp b/docs/snippets/ov_properties_api.cpp index 2e3d761cc2b..1d971f52ced 100644 --- a/docs/snippets/ov_properties_api.cpp +++ b/docs/snippets/ov_properties_api.cpp @@ -1,47 +1,67 @@ #include int main() { -//! [part0] +//! [get_available_devices] ov::Core core; -auto available_devices = core.get_available_devices(); -//! [part0] +std::vector available_devices = core.get_available_devices(); +//! [get_available_devices] -//! [part1] +//! [hetero_priorities] auto device_priorites = core.get_property("HETERO", ov::device::priorities); -//! [part1] +//! [hetero_priorities] -//! [part2] -auto cpu_device_name = core.get_property("GPU", ov::device::full_name); -//! [part2] +//! [cpu_device_name] +auto cpu_device_name = core.get_property("CPU", ov::device::full_name); +//! [cpu_device_name] -//! [part3] auto model = core.read_model("sample.xml"); { - auto compiled_model = core.compile_model(model, "CPU", - ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), - ov::hint::inference_precision(ov::element::f32)); +//! [compile_model_with_property] +auto compiled_model = core.compile_model(model, "CPU", + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), + ov::hint::inference_precision(ov::element::f32)); +//! [compile_model_with_property] } -//! [part3] -//! [part4] { - auto compiled_model = core.compile_model(model, "CPU"); - auto nireq = compiled_model.get_property(ov::optimal_number_of_infer_requests); +//! [optimal_number_of_infer_requests] +auto compiled_model = core.compile_model(model, "CPU"); +auto nireq = compiled_model.get_property(ov::optimal_number_of_infer_requests); +//! [optimal_number_of_infer_requests] +} +{ +//! [core_set_property_then_compile] +// set letency hint is a default for CPU +core.set_property("CPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)); +// compiled with latency configuration hint +auto compiled_model_latency = core.compile_model(model, "CPU"); +// compiled with overriden ov::hint::performance_mode value +auto compiled_model_thrp = core.compile_model(model, "CPU", + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)); +//! [core_set_property_then_compile] } -//! [part4] -//! [part5] { - auto compiled_model = core.compile_model(model, "MYRIAD"); - auto temperature = compiled_model.get_property(ov::device::thermal); +//! [device_thermal] +auto compiled_model = core.compile_model(model, "MYRIAD"); +float temperature = compiled_model.get_property(ov::device::thermal); +//! [device_thermal] } -//! [part5] -//! [part6] { - auto compiled_model = core.compile_model(model, "CPU"); - auto nthreads = compiled_model.get_property(ov::inference_num_threads); +//! [inference_num_threads] +auto compiled_model = core.compile_model(model, "CPU"); +auto nthreads = compiled_model.get_property(ov::inference_num_threads); +//! [inference_num_threads] +} + +{ +//! [multi_device] +auto compiled_model = core.compile_model(model, "MULTI", + ov::device::priorities("CPU", "GPU")); +// change the order of priorities +compiled_model.set_property(ov::device::priorities("GPU", "CPU")); +//! [multi_device] } -//! [part6] return 0; } diff --git a/docs/snippets/ov_properties_api.py b/docs/snippets/ov_properties_api.py new file mode 100644 index 00000000000..1d971f52ced --- /dev/null +++ b/docs/snippets/ov_properties_api.py @@ -0,0 +1,67 @@ +#include + +int main() { +//! [get_available_devices] +ov::Core core; +std::vector available_devices = core.get_available_devices(); +//! [get_available_devices] + +//! [hetero_priorities] +auto device_priorites = core.get_property("HETERO", ov::device::priorities); +//! [hetero_priorities] + +//! [cpu_device_name] +auto cpu_device_name = core.get_property("CPU", ov::device::full_name); +//! [cpu_device_name] + +auto model = core.read_model("sample.xml"); +{ +//! [compile_model_with_property] +auto compiled_model = core.compile_model(model, "CPU", + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), + ov::hint::inference_precision(ov::element::f32)); +//! [compile_model_with_property] +} + +{ +//! [optimal_number_of_infer_requests] +auto compiled_model = core.compile_model(model, "CPU"); +auto nireq = compiled_model.get_property(ov::optimal_number_of_infer_requests); +//! [optimal_number_of_infer_requests] +} +{ +//! [core_set_property_then_compile] +// set letency hint is a default for CPU +core.set_property("CPU", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)); +// compiled with latency configuration hint +auto compiled_model_latency = core.compile_model(model, "CPU"); +// compiled with overriden ov::hint::performance_mode value +auto compiled_model_thrp = core.compile_model(model, "CPU", + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)); +//! [core_set_property_then_compile] +} + +{ +//! [device_thermal] +auto compiled_model = core.compile_model(model, "MYRIAD"); +float temperature = compiled_model.get_property(ov::device::thermal); +//! [device_thermal] +} + +{ +//! [inference_num_threads] +auto compiled_model = core.compile_model(model, "CPU"); +auto nthreads = compiled_model.get_property(ov::inference_num_threads); +//! [inference_num_threads] +} + +{ +//! [multi_device] +auto compiled_model = core.compile_model(model, "MULTI", + ov::device::priorities("CPU", "GPU")); +// change the order of priorities +compiled_model.set_property(ov::device::priorities("GPU", "CPU")); +//! [multi_device] +} +return 0; +} diff --git a/docs/snippets/ov_properties_migration.cpp b/docs/snippets/ov_properties_migration.cpp new file mode 100644 index 00000000000..101200a3f12 --- /dev/null +++ b/docs/snippets/ov_properties_migration.cpp @@ -0,0 +1,95 @@ +#include +#include + +int main_new() { + ov::Core core; + +//! [core_get_ro_property] +// 'auto' is automatically deduced as std::string +// since the type is stored in the property +auto full_device_name = core.get_property("CPU", ov::device::full_name); +//! [core_get_ro_property] + +//! [core_get_rw_property] +// 'auto' is automatically deduced as ov::streams::Num +// since the type is stored in the property +auto num_streams = core.get_property("CPU", ov::streams::num); +//! [core_get_rw_property] + +//! [core_set_property] +core.set_property("CPU", ov::enable_profiling(true)); +//! [core_set_property] + +auto model = core.read_model("sample.xml"); +//! [core_compile_model] +auto compiled_model = core.compile_model(model, "MULTI", + ov::device::priorities("GPU", "CPU"), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT), + ov::hint::inference_precision(ov::element::f32)); +//! [core_compile_model] + +//! [compiled_model_set_property] +// turn CPU off for multi-device execution +compiled_model.set_property(ov::device::priorities("GPU")); +//! [compiled_model_set_property] + +{ +//! [compiled_model_get_ro_property] +// 'auto' is deduced to 'uint32_t' +auto nireq = compiled_model.get_property(ov::optimal_number_of_infer_requests); +//! [compiled_model_get_ro_property] +} + +{ +//! [compiled_model_get_rw_property] +ov::hint::PerformanceMode perf_model = compiled_model.get_property(ov::hint::performance_mode); +//! [compiled_model_get_rw_property] +} + + +return 0; +} + + +int main_old() { + InferenceEngine::Core core; +//! [core_get_metric] +auto full_device_name = core.GetConfig("CPU", METRIC_KEY(FULL_DEVICE_NAME)).as(); +//! [core_get_metric] + +//! [core_get_config] +// a user has to parse std::string after +auto num_streams = core.GetMetric("CPU", CONFIG_KEY(CPU_THROUGHPUT_STREAMS)).as(); +//! [core_get_config] + +//! [core_set_config] +core.SetConfig({ { CONFIG_KEY(PERF_COUNT), CONFIG_VALUE(YES) } }, "CPU"); +//! [core_set_config] + +auto model = core.ReadNetwork("sample.xml"); +//! [core_load_network] +auto exec_network = core.LoadNetwork(model, "MULTI", { + { MULTI_CONFIG_KEY(DEVICE_PRIORITIES), "CPU, GPU" }, + { CONFIG_KEY(PERFORMANCE_HINT), CONFIG_VALUE(THROUGHPUT) }, + { CONFIG_KEY(ENFORCE_BF16), CONFIG_VALUE(NO) } }); +//! [core_load_network] + +//! [executable_network_set_config] +// turn CPU off for multi-device execution +exec_network.SetConfig({ { MULTI_CONFIG_KEY(DEVICE_PRIORITIES), "GPU" } }); +//! [executable_network_set_config] + +{ +//! [executable_network_get_metric] +auto nireq = exec_network.GetMetric(EXEC_NETWORK_METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as(); +//! [executable_network_get_metric] +} + +{ +//! [executable_network_get_config] +std::string perf_model = exec_network.GetConfig(CONFIG_KEY(PERFORMANCE_HINT)).as(); +//! [executable_network_get_config] +} + +return 0; +} diff --git a/docs/snippets/protecting_model_guide.cpp b/docs/snippets/protecting_model_guide.cpp index bcf6856023f..0f148e76167 100644 --- a/docs/snippets/protecting_model_guide.cpp +++ b/docs/snippets/protecting_model_guide.cpp @@ -1,7 +1,8 @@ -#include #include #include +#include "openvino/runtime/core.hpp" + void decrypt_file(std::ifstream & stream, const std::string & pass, std::vector & result) { @@ -9,24 +10,22 @@ void decrypt_file(std::ifstream & stream, int main() { //! [part0] -std::vector model; -std::vector weights; +std::vector model_data, weights_data; std::string password; // taken from an user std::ifstream model_file("model.xml"), weights_file("model.bin"); // Read model files and decrypt them into temporary memory block -decrypt_file(model_file, password, model); -decrypt_file(weights_file, password, weights); +decrypt_file(model_file, password, model_data); +decrypt_file(weights_file, password, weights_data); //! [part0] //! [part1] -InferenceEngine::Core core; +ov::Core core; // Load model from temporary memory block -std::string strModel(model.begin(), model.end()); -InferenceEngine::CNNNetwork network = core.ReadNetwork(strModel, - InferenceEngine::make_shared_blob({InferenceEngine::Precision::U8, - {weights.size()}, InferenceEngine::C}, weights.data())); +std::string str_model(model_data.begin(), model_data.end()); +auto model = core.read_model(str_model, + ov::Tensor(ov::element::u8, {weights_data.size()}, weights_data.data())); //! [part1] return 0; diff --git a/docs/snippets/src/main.cpp b/docs/snippets/src/main.cpp new file mode 100644 index 00000000000..c600e063865 --- /dev/null +++ b/docs/snippets/src/main.cpp @@ -0,0 +1,69 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +//! [include] +#include +//! [include] + +int main() { +//! [part1] +ov::Core core; +//! [part1] + +ov::CompiledModel compiled_model; +{ +//! [part2_1] +ov::CompiledModel compiled_model = core.compile_model("model.xml", "AUTO"); +//! [part2_1] +} +{ +//! [part2_2] +ov::CompiledModel compiled_model = core.compile_model("model.onnx", "AUTO"); +//! [part2_2] +} +{ +//! [part2_3] +ov::CompiledModel compiled_model = core.compile_model("model.pdmodel", "AUTO"); +//! [part2_3] +} +{ +//! [part2_4] +auto create_model = []() { + std::shared_ptr model; + // To construct a model, please follow + // https://docs.openvino.ai/latest/openvino_docs_OV_Runtime_UG_Model_Representation.html + return model; +}; +std::shared_ptr model = create_model(); +compiled_model = core.compile_model(model, "AUTO"); +//! [part2_4] +} + +//! [part3] +ov::InferRequest infer_request = compiled_model.create_infer_request(); +//! [part3] + +void * memory_ptr = nullptr; +//! [part4] +// Get input port for model with one input +auto input_port = compiled_model.input(); +// Create tensor from external memory +ov::Tensor input_tensor(input_port.get_element_type(), input_port.get_shape(), memory_ptr); +// Set input tensor for model with one input +infer_request.set_input_tensor(input_tensor); +//! [part4] + +//! [part5] +infer_request.start_async(); +infer_request.wait(); +//! [part5] + +//! [part6] +// Get output tensor by tensor name +auto output = infer_request.get_tensor("tensor_name"); +const float *output_buffer = output.data(); +/* output_buffer[] - accessing output tensor data */ +//! [part6] +return 0; +} diff --git a/docs/snippets/src/main.py b/docs/snippets/src/main.py new file mode 100644 index 00000000000..a86986d56be --- /dev/null +++ b/docs/snippets/src/main.py @@ -0,0 +1,58 @@ +# Copyright (C) 2018-2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import numpy as np +#! [import] +import openvino.runtime as ov +#! [import] + +#! [part1] +core = ov.Core() +#! [part1] + +#! [part2_1] +compiled_model = core.compile_model("model.xml", "AUTO") +#! [part2_1] +#! [part2_2] +compiled_model = core.compile_model("model.onnx", "AUTO") +#! [part2_2] +#! [part2_3] +compiled_model = core.compile_model("model.pdmodel", "AUTO") +#! [part2_3] +#! [part2_4] +def create_model(): + # This example shows how to create ov::Function + # + # To construct a model, please follow + # https://docs.openvino.ai/latest/openvino_docs_OV_Runtime_UG_Model_Representation.html + data = ov.opset8.parameter([3, 1, 2], ov.Type.f32) + res = ov.opset8.result(data) + return ov.Model([res], [data], "model") + +model = create_model() +compiled_model = core.compile_model(model, "AUTO") +#! [part2_4] + +#! [part3] +infer_request = compiled_model.create_infer_request() +#! [part3] + +memory = np.array([1, 2, 3, 4]) +#! [part4] +# Create tensor from external memory +input_tensor = ov.Tensor(array=memory, shared_memory=True) +# Set input tensor for model with one input +infer_request.set_input_tensor(input_tensor) +#! [part4] + +#! [part5] +infer_request.start_async() +infer_request.wait() +#! [part5] + +#! [part6] +# Get output tensor for model with one output +output = infer_request.get_output_tensor() +output_buffer = output.data +# output_buffer[] - accessing output tensor data +#! [part6] diff --git a/docs/suppress_warnings.txt b/docs/suppress_warnings.txt index 572497bf642..af53128820e 100644 --- a/docs/suppress_warnings.txt +++ b/docs/suppress_warnings.txt @@ -13,7 +13,7 @@ the name \'.+?\' supplied as the argument warning: while setting up extension conf.py: csv directory not found argument \'.+?\' of command no uniquely matching class member found for -example example was already documented\. ignoring documentation found here\. +example was already documented\. ignoring documentation found here\. documentation for unknown define detected potential recursive class relation between class no matching file member found for diff --git a/docs/template_extension/new/CMakeLists.txt b/docs/template_extension/new/CMakeLists.txt index 32bdeda4ea7..20b81dc1adf 100644 --- a/docs/template_extension/new/CMakeLists.txt +++ b/docs/template_extension/new/CMakeLists.txt @@ -14,7 +14,13 @@ set(SRC identity.cpp ov_extension.cpp) add_library(${TARGET_NAME} MODULE ${SRC}) target_compile_definitions(${TARGET_NAME} PRIVATE IMPLEMENT_OPENVINO_EXTENSION_API) -target_link_libraries(${TARGET_NAME} PRIVATE openvino::core) +target_link_libraries(${TARGET_NAME} PRIVATE openvino::runtime) + +# To map custom operation to framework +if(OpenVINO_Frontend_ONNX_FOUND) + target_link_libraries(${TARGET_NAME} PRIVATE openvino::frontend::onnx) + target_compile_definitions(${TARGET_NAME} PRIVATE OPENVINO_ONNX_FRONTEND_ENABLED) +endif() # [cmake:extension] # Enable code style check diff --git a/docs/template_extension/new/identity.hpp b/docs/template_extension/new/identity.hpp index f31ab239c34..b8c5160014d 100644 --- a/docs/template_extension/new/identity.hpp +++ b/docs/template_extension/new/identity.hpp @@ -4,7 +4,14 @@ #pragma once +//! [op:common_include] #include +//! [op:common_include] +//! [op:frontend_include] +#ifdef OPENVINO_ONNX_FRONTEND_ENABLED +# include +#endif +//! [op:frontend_include] //! [op:header] namespace TemplateExtension { @@ -13,6 +20,10 @@ class Identity : public ov::op::Op { public: OPENVINO_OP("Identity"); +#ifdef OPENVINO_ONNX_FRONTEND_ENABLED + OPENVINO_FRAMEWORK_MAP(onnx) +#endif + Identity() = default; Identity(const ov::Output& arg); void validate_and_infer_types() override; diff --git a/docs/template_extension/new/ov_extension.cpp b/docs/template_extension/new/ov_extension.cpp index 79d414a82e4..d2fa1e35361 100644 --- a/docs/template_extension/new/ov_extension.cpp +++ b/docs/template_extension/new/ov_extension.cpp @@ -7,5 +7,11 @@ #include "identity.hpp" +// clang-format off +//! [ov_extension:entry_point] OPENVINO_CREATE_EXTENSIONS( - std::vector({std::make_shared>()})); + std::vector({ + std::make_shared>() + })); +//! [ov_extension:entry_point] +// clang-format on diff --git a/docs/template_plugin/backend/evaluates_map.cpp b/docs/template_plugin/backend/evaluates_map.cpp index af0d8c37dcb..53101ae7d47 100644 --- a/docs/template_plugin/backend/evaluates_map.cpp +++ b/docs/template_plugin/backend/evaluates_map.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -40,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -399,6 +401,42 @@ bool evaluate(const shared_ptr& op, return true; } +template +bool evaluate(const shared_ptr& op, const HostTensorVector& outputs, const HostTensorVector& inputs) { + const auto in0_data_ptr = inputs[0]->get_data_ptr(); + const auto in1_data_ptr = inputs[1]->get_data_ptr(); + const auto out_data_ptr = outputs[0]->get_data_ptr(); + const auto in0_shape = inputs[0]->get_shape(); + const auto in1_shape = inputs[1]->get_shape(); + const auto broadcast_spec = op->get_autob(); + runtime::reference::greater::value_type, + typename element_type_traits::value_type>(in0_data_ptr, + in1_data_ptr, + out_data_ptr, + in0_shape, + in1_shape, + broadcast_spec); + return true; +} + +template +bool evaluate(const shared_ptr& op, const HostTensorVector& outputs, const HostTensorVector& inputs) { + const auto in0_data_ptr = inputs[0]->get_data_ptr(); + const auto in1_data_ptr = inputs[1]->get_data_ptr(); + const auto out_data_ptr = outputs[0]->get_data_ptr(); + const auto in0_shape = inputs[0]->get_shape(); + const auto in1_shape = inputs[1]->get_shape(); + const auto broadcast_spec = op->get_autob(); + runtime::reference::equal::value_type, + typename element_type_traits::value_type>(in0_data_ptr, + in1_data_ptr, + out_data_ptr, + in0_shape, + in1_shape, + broadcast_spec); + return true; +} + namespace cum_sum_v0 { template inline void evaluate(const shared_ptr& op, @@ -428,6 +466,145 @@ bool evaluate(const shared_ptr& op, const HostTensorVector& outp return true; } +namespace if_op { +bool call(const HostTensorVector& func_outputs, + const HostTensorVector& func_inputs, + const std::shared_ptr& function) { + // map function params -> HostTensor + std::unordered_map> tensor_map; + size_t input_count = 0; + for (const auto& param : function->get_parameters()) { + for (size_t i = 0; i < param->get_output_size(); ++i) { + descriptor::Tensor* tensor = ¶m->output(i).get_tensor(); + tensor_map.insert({tensor, func_inputs[input_count++]}); + } + } + + std::unordered_map, size_t> results_map; + // map function outputs -> HostTensor + for (size_t output_count = 0; output_count < function->get_results().size(); ++output_count) { + auto output = function->get_results()[output_count]; + results_map[output] = output_count; + } + + // for each ordered op in the graph + for (const auto& op : function->get_ordered_ops()) { + if (op::is_parameter(op)) { + continue; + } + + // get op inputs from map + std::vector> op_inputs; + for (auto input : op->inputs()) { + descriptor::Tensor* tensor = &input.get_tensor(); + op_inputs.push_back(tensor_map.at(tensor)); + } + + // get op outputs from map or create + std::vector> op_outputs; + for (size_t i = 0; i < op->get_output_size(); ++i) { + descriptor::Tensor* tensor = &op->output(i).get_tensor(); + std::shared_ptr host_tensor; + auto it = tensor_map.find(tensor); + if (op::is_output(op)) { + host_tensor = func_outputs[results_map[op]]; + } else if (it == tensor_map.end()) { + host_tensor = std::make_shared(op->output(i)); + tensor_map.insert({tensor, host_tensor}); + } else { + host_tensor = it->second; + } + op_outputs.push_back(host_tensor); + } + op->validate_and_infer_types(); + OPENVINO_SUPPRESS_DEPRECATED_START + if (!op->evaluate(op_outputs, op_inputs)) { + auto evaluates_map = ngraph::runtime::interpreter::get_evaluators_map(); + auto it = evaluates_map.find(op->get_type_info()); + if (!it->second(op, op_outputs, op_inputs)) { + return false; + } + } + OPENVINO_SUPPRESS_DEPRECATED_END + } + return true; +} + +void function(const std::shared_ptr& function, + const HostTensorVector& inputs, + HostTensorVector& outputs) { + const auto& parameters = function->get_parameters(); + const auto& parametersNumber = parameters.size(); + const auto& inputsNumber = inputs.size(); + NGRAPH_CHECK(parametersNumber == inputsNumber, + "Got function (", + function->get_friendly_name(), + ") with ", + parametersNumber, + " parameters, but ", + inputsNumber, + " input blobs"); + + for (const auto& parameter : parameters) { + const auto& parameterIndex = function->get_parameter_index(parameter); + const auto& parameterShape = parameter->get_shape(); + const auto& parameterType = parameter->get_element_type(); + const auto& parameterSize = shape_size(parameterShape) * parameterType.size(); + + const auto& input = inputs[parameterIndex]; + const auto& inputSize = input->get_size_in_bytes(); + NGRAPH_CHECK(parameterSize == inputSize, + "Got parameter (", + parameter->get_friendly_name(), + ") of size ", + parameterSize, + " bytes, but corresponding input with index ", + parameterIndex, + " has ", + inputSize, + " bytes"); + } + + const auto& results = function->get_results(); + outputs.reserve(results.size()); + for (size_t i = 0; i < results.size(); ++i) { + outputs.push_back(std::make_shared()); + } + call(outputs, inputs, function); +} + +void if_reference(const std::vector>& bodies, + const std::vector& out_descs, + const std::vector& input_descs, + const HostTensorVector& out, + const HostTensorVector& args) { + NGRAPH_CHECK(args.size() > 0, "If operation must have input condition value"); + + auto condition_value = args[0]->get_data_ptr()[0]; + auto branch_index = (condition_value) ? op::v8::If::THEN_BODY_INDEX : op::v8::If::ELSE_BODY_INDEX; + HostTensorVector inputs_to_body; + HostTensorVector outs_from_body; + inputs_to_body.resize(input_descs[branch_index].size()); + auto inputs_size = args.size(); + auto output_size = out.size(); + for (const auto& input_desc : input_descs[branch_index]) { + NGRAPH_CHECK(inputs_size > input_desc->m_input_index, + "Incorrect associating! If has not input with id ", + input_desc->m_input_index); + inputs_to_body[input_desc->m_body_parameter_index] = args[input_desc->m_input_index]; + } + function(bodies[branch_index], inputs_to_body, outs_from_body); + for (const auto& out_descr : out_descs[branch_index]) { + NGRAPH_CHECK(output_size > out_descr->m_output_index, + "Incorrect associating! If has not output with id ", + out_descr->m_output_index); + auto res = outs_from_body[out_descr->m_body_value_index]; + out[out_descr->m_output_index]->set_shape(res->get_shape()); + out[out_descr->m_output_index]->write(res->get_data_ptr(), res->get_size_in_bytes()); + } +} +} // namespace if_op + template bool evaluate(const shared_ptr& op, const HostTensorVector& outputs, const HostTensorVector& inputs) { std::vector> bodies; @@ -442,7 +619,11 @@ bool evaluate(const shared_ptr& op, const HostTensorVector& outputs, for (size_t i = 0; i < op->get_output_descriptions_size(); i++) { out_descs.emplace_back(op->get_output_descriptions(i)); } - runtime::reference::if_reference(bodies, out_descs, in_descs, outputs, inputs); + try { + runtime::reference::if_reference(bodies, out_descs, in_descs, outputs, inputs); + } catch (...) { + if_op::if_reference(bodies, out_descs, in_descs, outputs, inputs); + } return true; } @@ -857,7 +1038,7 @@ bool evaluate(const shared_ptr& op, &valid_outputs, info.sort_result_descending); - auto selected_scores_type = (inputs.size() < 4) ? element::f32 : inputs[3]->get_element_type(); + auto selected_scores_type = (outputs.size() < 3) ? element::f32 : outputs[1]->get_element_type(); runtime::reference::nms5_postprocessing(outputs, info.output_type, @@ -3430,7 +3611,7 @@ bool evaluate(const shared_ptr& op, template bool evaluate_node(std::shared_ptr node, const HostTensorVector& outputs, const HostTensorVector& inputs) { auto element_type = node->get_output_element_type(0); - if (ov::is_type(node)) + if (ov::is_type(node) || ov::is_type(node)) element_type = node->get_input_element_type(1); switch (element_type) { diff --git a/docs/template_plugin/backend/opset_int_tbl.hpp b/docs/template_plugin/backend/opset_int_tbl.hpp index d8edd45d4b9..952488d28d6 100644 --- a/docs/template_plugin/backend/opset_int_tbl.hpp +++ b/docs/template_plugin/backend/opset_int_tbl.hpp @@ -45,6 +45,8 @@ NGRAPH_OP(ConvertLike, op::v1) NGRAPH_OP(Convolution, ngraph::op::v1) NGRAPH_OP(ConvolutionBackpropData, ngraph::op::v1) NGRAPH_OP(DeformablePSROIPooling, ngraph::op::v1) +NGRAPH_OP(Equal, ngraph::op::v1) +NGRAPH_OP(Greater, ngraph::op::v1) NGRAPH_OP(GroupConvolution, ngraph::op::v1) NGRAPH_OP(GroupConvolutionBackpropData, ngraph::op::v1) NGRAPH_OP(DeformableConvolution, ngraph::op::v1) diff --git a/docs/template_plugin/src/template_plugin.cpp b/docs/template_plugin/src/template_plugin.cpp index acd4a809664..76c3dbc2182 100644 --- a/docs/template_plugin/src/template_plugin.cpp +++ b/docs/template_plugin/src/template_plugin.cpp @@ -82,8 +82,8 @@ std::shared_ptr TransformNetwork(const std::shared_ptr(); - passManager.register_pass(); + passManager.register_pass(); + passManager.register_pass(); // Register any other transformations // .. diff --git a/docs/template_plugin/src/transformations/template_function_transformation.hpp b/docs/template_plugin/src/transformations/template_function_transformation.hpp deleted file mode 100644 index 68641470730..00000000000 --- a/docs/template_plugin/src/transformations/template_function_transformation.hpp +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (C) 2018-2022 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include - -namespace ngraph { -namespace pass { - -class MyFunctionTransformation; - -} // namespace pass -} // namespace ngraph - -// ! [function_pass:template_transformation_hpp] -// template_function_transformation.hpp -class ngraph::pass::MyFunctionTransformation : public ngraph::pass::FunctionPass { -public: - NGRAPH_RTTI_DECLARATION; - bool run_on_model(const std::shared_ptr& f) override; -}; -// ! [function_pass:template_transformation_hpp] diff --git a/docs/template_plugin/src/transformations/template_function_transformation.cpp b/docs/template_plugin/src/transformations/template_model_transformation.cpp similarity index 72% rename from docs/template_plugin/src/transformations/template_function_transformation.cpp rename to docs/template_plugin/src/transformations/template_model_transformation.cpp index 669b5c811e2..dd6e56b1c42 100644 --- a/docs/template_plugin/src/transformations/template_function_transformation.cpp +++ b/docs/template_plugin/src/transformations/template_model_transformation.cpp @@ -2,18 +2,15 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "template_function_transformation.hpp" +#include "template_model_transformation.hpp" -#include +#include "openvino/cc/pass/itt.hpp" -using namespace ngraph; - -// ! [function_pass:template_transformation_cpp] +// ! [model_pass:template_transformation_cpp] // template_function_transformation.cpp -NGRAPH_RTTI_DEFINITION(ngraph::pass::MyFunctionTransformation, "MyFunctionTransformation", 0); -bool pass::MyFunctionTransformation::run_on_model(const std::shared_ptr& f) { - RUN_ON_FUNCTION_SCOPE(MyFunctionTransformation); +bool ov::pass::MyModelTransformation::run_on_model(const std::shared_ptr& f) { + RUN_ON_MODEL_SCOPE(MyModelTransformation); // Example transformation code NodeVector nodes; @@ -40,4 +37,4 @@ bool pass::MyFunctionTransformation::run_on_model(const std::shared_ptr& f) override; +}; +// ! [model_pass:template_transformation_hpp] diff --git a/docs/template_plugin/src/transformations/template_pattern_transformation.cpp b/docs/template_plugin/src/transformations/template_pattern_transformation.cpp index 7169d5946fc..7c451be5317 100644 --- a/docs/template_plugin/src/transformations/template_pattern_transformation.cpp +++ b/docs/template_plugin/src/transformations/template_pattern_transformation.cpp @@ -4,154 +4,147 @@ #include "transformations/template_pattern_transformation.hpp" -#include -#include -#include -#include -#include - -#include "transformations/template_function_transformation.hpp" - -using namespace ngraph; +#include "openvino/cc/pass/itt.hpp" +#include "openvino/core/rt_info.hpp" +#include "openvino/opsets/opset3.hpp" +#include "openvino/pass/manager.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "transformations/template_model_transformation.hpp" // ! [graph_rewrite:template_transformation_cpp] // template_pattern_transformation.cpp -NGRAPH_RTTI_DEFINITION(ngraph::pass::DecomposeDivideMatcher, "DecomposeDivideMatcher", 0); - -ngraph::pass::DecomposeDivideMatcher::DecomposeDivideMatcher() { +ov::pass::DecomposeDivideMatcher::DecomposeDivideMatcher() { MATCHER_SCOPE(DecomposeDivideMatcher); // Pattern example auto input0 = pattern::any_input(); auto input1 = pattern::any_input(); - auto div = std::make_shared(input0, input1); + auto div = std::make_shared(input0, input1); - ngraph::matcher_pass_callback callback = [](pattern::Matcher& m) { - auto div = std::dynamic_pointer_cast(m.get_match_root()); + ov::matcher_pass_callback callback = [](pattern::Matcher& m) { + auto div = std::dynamic_pointer_cast(m.get_match_root()); // We can not apply this transformation in case with integer input data type if (!div || div->input(0).get_element_type().is_integral()) { return false; } // Decompose Divide into Multiply with Power operations - auto pow = std::make_shared( + auto pow = std::make_shared( div->input_value(1), opset3::Constant::create(div->get_input_element_type(1), Shape{1}, {-1})); - auto mul = std::make_shared(div->input_value(0), pow); + auto mul = std::make_shared(div->input_value(0), pow); // Save original name to last operation in replacement sub-graph mul->set_friendly_name(div->get_friendly_name()); // Copy runtime info attributes to newly created operation - ngraph::copy_runtime_info(div, {pow, mul}); + ov::copy_runtime_info(div, {pow, mul}); // Replace Divide operation with Multiply - ngraph::replace_node(div, mul); + ov::replace_node(div, mul); // Return true as the root node was changed return true; }; // Register pattern with Divide operation as a pattern root node - auto m = std::make_shared(div, "ConvertDivide"); + auto m = std::make_shared(div, "ConvertDivide"); // Register Matcher register_matcher(m, callback); } // ! [graph_rewrite:template_transformation_cpp] // ! [matcher_pass:relu_fusion] -NGRAPH_RTTI_DEFINITION(ngraph::pass::ReluReluFusionMatcher, "ReluReluFusionMatcher", 0); - -ngraph::pass::ReluReluFusionMatcher::ReluReluFusionMatcher() { +ov::pass::ReluReluFusionMatcher::ReluReluFusionMatcher() { MATCHER_SCOPE(ReluReluFusionMatcher); - auto m_relu1 = ngraph::pattern::wrap_type(pattern::consumers_count(1)); - auto m_relu2 = ngraph::pattern::wrap_type({m_relu1}); + auto m_relu1 = ov::pass::pattern::wrap_type(pattern::consumers_count(1)); + auto m_relu2 = ov::pass::pattern::wrap_type({m_relu1}); - ngraph::matcher_pass_callback callback = [=](pattern::Matcher& m) { + ov::matcher_pass_callback callback = [=](pattern::Matcher& m) { // Map that helps to connect labels with matched outputs auto& node_to_output = m.get_pattern_value_map(); // Create new Relu operation and add register it for additional execution auto new_relu = - register_new_node(node_to_output.at(m_relu1).get_node_shared_ptr()->input_value(0)); + register_new_node(node_to_output.at(m_relu1).get_node_shared_ptr()->input_value(0)); // Copy runtime info attributes to newly created operation - ngraph::copy_runtime_info(m.get_matched_nodes(), new_relu); + ov::copy_runtime_info(m.get_matched_nodes(), new_relu); // Save last Relu name to new Relu operation new_relu->set_friendly_name(m.get_match_root()->get_friendly_name()); // Replace Relu->Relu with Relu - ngraph::replace_node(m.get_match_root(), new_relu); + ov::replace_node(m.get_match_root(), new_relu); // Return true as the root node was changed return true; }; // Register pattern with Relu operation as a pattern root node - auto m = std::make_shared(m_relu2, "ReluReluFusion"); + auto m = std::make_shared(m_relu2, "ReluReluFusion"); // Register Matcher register_matcher(m, callback); } // ! [matcher_pass:relu_fusion] -void run_matcher_on_node(std::shared_ptr node) { +void run_matcher_on_node(std::shared_ptr node) { // ! [matcher_pass:run_on_node] - if (ngraph::pass::DecomposeDivideMatcher().apply(node)) { + if (ov::pass::DecomposeDivideMatcher().apply(node)) { // successful execution (root node was replaced) } // ! [matcher_pass:run_on_node] } -void run_matcher_with_manager(std::shared_ptr f) { +void run_matcher_with_manager(std::shared_ptr f) { // ! [matcher_pass:manager] // Two matchers will run independently (two independent graph traversals) // pass::Manager automatically creates GraphRewrite container for each MatcherPass - pass::Manager manager; - manager.register_pass(); - manager.register_pass(); + ov::pass::Manager manager; + manager.register_pass(); + manager.register_pass(); manager.run_passes(f); // ! [matcher_pass:manager] } -void run_matcher_with_manager2(std::shared_ptr f) { +void run_matcher_with_manager2(std::shared_ptr f) { // ! [matcher_pass:manager2] // Register anchor GraphRewrite pass inside manager that will execute two matchers simultaneously - pass::Manager manager; - auto anchor = manager.register_pass(); - anchor->add_matcher(); - anchor->add_matcher(); + ov::pass::Manager manager; + auto anchor = manager.register_pass(); + anchor->add_matcher(); + anchor->add_matcher(); manager.run_passes(f); // ! [matcher_pass:manager2] } -void run_matcher_with_manager3(std::shared_ptr f) { +void run_matcher_with_manager3(std::shared_ptr f) { // ! [matcher_pass:manager3] - pass::Manager manager; - manager.register_pass(); + ov::pass::Manager manager; + manager.register_pass(); // Two matchers will run independently (two independent graph traversals) // pass::Manager automatically creates GraphRewrite container for each MatcherPass - manager.register_pass(); - manager.register_pass(); + manager.register_pass(); + manager.register_pass(); manager.run_passes(f); // ! [matcher_pass:manager3] } -void run_matcher_with_gr(std::shared_ptr f) { +void run_matcher_with_gr(std::shared_ptr f) { // ! [matcher_pass:graph_rewrite] // Two matcher passes will run simultaneously in a single graph traversal - ngraph::pass::GraphRewrite pass; - pass.add_matcher(); - pass.add_matcher(); + ov::pass::GraphRewrite pass; + pass.add_matcher(); + pass.add_matcher(); pass.run_on_model(f); // ! [matcher_pass:graph_rewrite] } // ! [manual_constant_folding] template -Output eltwise_fold(const Output& input0, const Output& input1) { +ov::Output eltwise_fold(const ov::Output& input0, const ov::Output& input1) { auto eltwise = std::make_shared(input0, input1); - OutputVector output(eltwise->get_output_size()); + ov::OutputVector output(eltwise->get_output_size()); // If constant folding wasn't successful return eltwise output if (!eltwise->constant_fold(output, {input0, input1})) { return eltwise->output(0); diff --git a/docs/template_plugin/src/transformations/template_pattern_transformation.hpp b/docs/template_plugin/src/transformations/template_pattern_transformation.hpp index 9329ed92ead..5a16133b451 100644 --- a/docs/template_plugin/src/transformations/template_pattern_transformation.hpp +++ b/docs/template_plugin/src/transformations/template_pattern_transformation.hpp @@ -4,16 +4,16 @@ #pragma once -#include +#include "openvino/pass/graph_rewrite.hpp" -namespace ngraph { +namespace ov { namespace pass { class DecomposeDivideMatcher; class ReluReluFusionMatcher; } // namespace pass -} // namespace ngraph +} // namespace ov // ! [graph_rewrite:template_transformation_hpp] // transformations/template_pattern_transformation.hpp @@ -21,15 +21,15 @@ class ReluReluFusionMatcher; * @ingroup ie_transformation_common_api * @brief Add transformation description. */ -class ngraph::pass::DecomposeDivideMatcher : public ngraph::pass::MatcherPass { +class ov::pass::DecomposeDivideMatcher : public ov::pass::MatcherPass { public: - NGRAPH_RTTI_DECLARATION; + OPENVINO_RTTI("DecomposeDivideMatcher", "0"); DecomposeDivideMatcher(); }; // ! [graph_rewrite:template_transformation_hpp] -class ngraph::pass::ReluReluFusionMatcher : public ngraph::pass::MatcherPass { +class ov::pass::ReluReluFusionMatcher : public ov::pass::MatcherPass { public: - NGRAPH_RTTI_DECLARATION; + OPENVINO_RTTI("ReluReluFusionMatcher", "0"); ReluReluFusionMatcher(); }; diff --git a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/ov_executable_network/properties.cpp b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/ov_executable_network/properties.cpp index 8a7e41278f8..10ba97613de 100644 --- a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/ov_executable_network/properties.cpp +++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/ov_executable_network/properties.cpp @@ -29,6 +29,8 @@ const std::vector auto_inproperties = { const std::vector auto_batch_inproperties = { {ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")}, + {{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , std::string(CommonTestUtils::DEVICE_TEMPLATE) + "(4)"}, + {ov::auto_batch_timeout(-1)}}, }; INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVCompiledModelPropertiesIncorrectTests, @@ -89,9 +91,9 @@ const std::vector multi_properties = { }; const std::vector auto_batch_properties = { - {{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , CommonTestUtils::DEVICE_TEMPLATE}}, - {{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , CommonTestUtils::DEVICE_TEMPLATE}, - {CONFIG_KEY(AUTO_BATCH_TIMEOUT) , "1"}}, + {{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , std::string(CommonTestUtils::DEVICE_TEMPLATE) + "(4)"}}, + {{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , std::string(CommonTestUtils::DEVICE_TEMPLATE) + "(4)"}, {CONFIG_KEY(AUTO_BATCH_TIMEOUT) , "1"}}, + {{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , std::string(CommonTestUtils::DEVICE_TEMPLATE) + "(4)"}, {ov::auto_batch_timeout(10)}}, }; INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVCompiledModelPropertiesTests, diff --git a/docs/template_plugin/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp new file mode 100644 index 00000000000..d6a87759e01 --- /dev/null +++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp @@ -0,0 +1,120 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/ov_plugin/properties_tests.hpp" +#include "openvino/runtime/properties.hpp" + +using namespace ov::test::behavior; + +namespace { + +const std::vector inproperties = { + {ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")}, +}; + +const std::vector hetero_inproperties = { + {ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")}, +}; + +const std::vector multi_inproperties = { + {ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")}, +}; + + +const std::vector auto_inproperties = { + {ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")}, +}; + + +const std::vector auto_batch_inproperties = { + {ov::device::id("UNSUPPORTED_DEVICE_ID_STRING")}, +}; + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_BehaviorTests, OVPropertiesIncorrectTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE), + ::testing::ValuesIn(inproperties)), + OVPropertiesIncorrectTests::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_Hetero_BehaviorTests, OVPropertiesIncorrectTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_HETERO), + ::testing::ValuesIn(hetero_inproperties)), + OVPropertiesIncorrectTests::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_Multi_BehaviorTests, OVPropertiesIncorrectTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_MULTI), + ::testing::ValuesIn(multi_inproperties)), + OVPropertiesIncorrectTests::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_Auto_BehaviorTests, OVPropertiesIncorrectTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_AUTO), + ::testing::ValuesIn(auto_inproperties)), + OVPropertiesIncorrectTests::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_AutoBatch_BehaviorTests, OVPropertiesIncorrectTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_BATCH), + ::testing::ValuesIn(auto_batch_inproperties)), + OVPropertiesIncorrectTests::getTestCaseName); + +const std::vector default_properties = { + {ov::enable_profiling(true)}, + {ov::device::id(0)}, +}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVPropertiesDefaultTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE), + ::testing::ValuesIn(default_properties)), + OVPropertiesDefaultTests::getTestCaseName); + +const std::vector properties = { + {ov::enable_profiling(true)}, + {ov::device::id(0)}, +}; + +const std::vector hetero_properties = { + {ov::device::priorities(CommonTestUtils::DEVICE_TEMPLATE), ov::enable_profiling(true)}, + {ov::device::priorities(CommonTestUtils::DEVICE_TEMPLATE), ov::device::id(0)}, +}; + + +const std::vector multi_properties = { + {ov::device::priorities(CommonTestUtils::DEVICE_TEMPLATE), ov::enable_profiling(true)}, + {ov::device::priorities(CommonTestUtils::DEVICE_TEMPLATE), ov::device::id(0)}, +}; + +const std::vector auto_batch_properties = { + {{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , CommonTestUtils::DEVICE_TEMPLATE}}, + {{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , CommonTestUtils::DEVICE_TEMPLATE}, + {CONFIG_KEY(AUTO_BATCH_TIMEOUT) , "1"}}, +}; + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVPropertiesTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE), + ::testing::ValuesIn(properties)), + OVPropertiesTests::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_Hetero_BehaviorTests, OVPropertiesTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_HETERO), + ::testing::ValuesIn(hetero_properties)), + OVPropertiesTests::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_Multi_BehaviorTests, OVPropertiesTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_MULTI), + ::testing::ValuesIn(multi_properties)), + OVPropertiesTests::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_AutoBatch_BehaviorTests, OVPropertiesTests, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_BATCH), + ::testing::ValuesIn(auto_batch_properties)), + OVPropertiesTests::getTestCaseName); +} // namespace diff --git a/docs/template_plugin/tests/functional/shared_tests_instances/hetero/synthetic.cpp b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/plugin/synthetic.cpp similarity index 99% rename from docs/template_plugin/tests/functional/shared_tests_instances/hetero/synthetic.cpp rename to docs/template_plugin/tests/functional/shared_tests_instances/behavior/plugin/synthetic.cpp index dfadd8f4a84..e91afe05441 100644 --- a/docs/template_plugin/tests/functional/shared_tests_instances/hetero/synthetic.cpp +++ b/docs/template_plugin/tests/functional/shared_tests_instances/behavior/plugin/synthetic.cpp @@ -4,7 +4,7 @@ #include -#include "hetero/synthetic.hpp" +#include "behavior/plugin/hetero_synthetic.hpp" #include "ngraph_functions/builders.hpp" #include "ngraph_functions/subgraph_builders.hpp" diff --git a/licensing/onednn_third-party-programs.txt b/licensing/onednn_third-party-programs.txt index 0fed7990633..3897a99f34c 100644 --- a/licensing/onednn_third-party-programs.txt +++ b/licensing/onednn_third-party-programs.txt @@ -6,7 +6,7 @@ terms. This third party software, even if included with the distribution of the Intel software, may be governed by separate license terms, including without limitation, third party license terms, other Intel software license terms, and open source software license terms. These separate license terms -govern your use of the third party programs as set forth in in the +govern your use of the third party programs as set forth in the "THIRD-PARTY-PROGRAMS" file. Third party programs and their corresponding required notices and/or license @@ -554,4 +554,4 @@ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/samples/c/common/opencv_c_wrapper/bmp_reader.c b/samples/c/common/opencv_c_wrapper/bmp_reader.c index 848415d204a..40fd5fc4bd4 100644 --- a/samples/c/common/opencv_c_wrapper/bmp_reader.c +++ b/samples/c/common/opencv_c_wrapper/bmp_reader.c @@ -4,11 +4,11 @@ #include #include -#define CLEANUP_AND_RETURN(x) \ - if (x && !image && !image->data) \ - free(image->data); \ - if (input != NULL) \ - fclose(input); \ +#define CLEANUP_AND_RETURN(x) \ + if (0 != x && NULL != image && NULL != image->data) \ + free(image->data); \ + if (input != NULL) \ + fclose(input); \ return x; int readBmpImage(const char* fileName, BitMap* image) { diff --git a/samples/c/hello_classification/README.md b/samples/c/hello_classification/README.md index f9daa132d69..a77c10f5f5f 100644 --- a/samples/c/hello_classification/README.md +++ b/samples/c/hello_classification/README.md @@ -26,7 +26,7 @@ Upon the start-up, the sample application reads command line parameters, loads s Then, the sample creates an synchronous inference request object. When inference is done, the application outputs data to the standard output stream. You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) section of "Integrate the Inference Engine with Your Application" guide. +each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. ## Building @@ -92,8 +92,8 @@ This sample is an API example, for any performance measurements please use the d ## See Also -- [Integrate the Inference Engine with Your Application](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) -- [Using Inference Engine Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) +- [Integrate OpenVINO™ into Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) +- [Using OpenVINO™ Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) - [Model Downloader](@ref omz_tools_downloader) - [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) diff --git a/samples/c/hello_nv12_input_classification/README.md b/samples/c/hello_nv12_input_classification/README.md index a3879a129d0..da71ed74e7f 100644 --- a/samples/c/hello_nv12_input_classification/README.md +++ b/samples/c/hello_nv12_input_classification/README.md @@ -25,7 +25,7 @@ image in the NV12 color format to an Inference Engine plugin. Then, the sample c application outputs data to the standard output stream. You can see the explicit description of -each sample step at [Integration Steps](https://docs.openvino.ai/latest/openvino_docs_IE_DG_Integrate_with_customer_application_new_API.html) section of "Integrate the Inference Engine with Your Application" guide. +each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. ## Building @@ -107,8 +107,8 @@ This sample is an API example, for any performance measurements please use the d ## See Also -- [Integrate the Inference Engine with Your Application](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) -- [Using Inference Engine Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) +- [Integrate the OpenVINO™ into Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) +- [Using OpenVINO™ Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) - [Model Downloader](@ref omz_tools_downloader) - [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) diff --git a/samples/cpp/benchmark_app/README.md b/samples/cpp/benchmark_app/README.md index 08f7491fbb8..11a3239a362 100644 --- a/samples/cpp/benchmark_app/README.md +++ b/samples/cpp/benchmark_app/README.md @@ -56,7 +56,7 @@ Note that the benchmark_app usually produces optimal performance for any device But it is still may be sub-optimal for some cases, especially for very small networks. More details can read in [Performance Optimization Guide](../../../docs/optimization_guide/dldt_optimization_guide.md). -As explained in the [Performance Optimization Guide](../../../docs/optimization_guide/dldt_optimization_guide.md) section, for all devices, including new [MULTI device](../../../docs/OV_Runtime_UG/supported_plugins/MULTI.md) it is preferable to use the FP16 IR for the model. +As explained in the [Performance Optimization Guide](../../../docs/optimization_guide/dldt_optimization_guide.md) section, for all devices, including new [MULTI device](../../../docs/OV_Runtime_UG/multi_device.md) it is preferable to use the FP16 IR for the model. Also if latency of the CPU inference on the multi-socket machines is of concern, please refer to the same [Performance Optimization Guide](../../../docs/optimization_guide/dldt_optimization_guide.md). @@ -166,7 +166,7 @@ This section provides step-by-step instructions on how to run the Benchmark Tool > **NOTE**: The Internet access is required to execute the following steps successfully. If you have access to the Internet through the proxy server only, please make sure that it is configured in your OS environment. -1. Download the model. Go to the the Model Downloader directory and run the `downloader.py` script with specifying the model name and directory to download the model to: +1. Download the model. Go to the Model Downloader directory and run the `downloader.py` script with specifying the model name and directory to download the model to: ```sh cd /extras/open_model_zoo/tools/downloader ``` diff --git a/samples/cpp/benchmark_app/main.cpp b/samples/cpp/benchmark_app/main.cpp index b4613fac299..3518ffc5eee 100644 --- a/samples/cpp/benchmark_app/main.cpp +++ b/samples/cpp/benchmark_app/main.cpp @@ -789,8 +789,11 @@ int main(int argc, char* argv[]) { std::map inputsData; if (isFlagSetInCommandLine("use_device_mem")) { if (device_name.find("GPU") == 0) { - inputsData = - ::gpu::get_remote_input_tensors(inputFiles, app_inputs_info, compiledModel, clInputsBuffer); + inputsData = ::gpu::get_remote_input_tensors(inputFiles, + app_inputs_info, + compiledModel, + clInputsBuffer, + inferRequestsQueue.requests.size()); useGpuMem = true; } else if (device_name.find("CPU") == 0) { if (newInputType) { diff --git a/samples/cpp/benchmark_app/remote_tensors_filling.cpp b/samples/cpp/benchmark_app/remote_tensors_filling.cpp index 0f2065c2979..40bc581d153 100644 --- a/samples/cpp/benchmark_app/remote_tensors_filling.cpp +++ b/samples/cpp/benchmark_app/remote_tensors_filling.cpp @@ -69,7 +69,8 @@ std::map get_remote_input_tensors( const std::map>& inputFiles, const std::vector& app_inputs_info, const ov::CompiledModel& compiledModel, - std::vector& clBuffer) { + std::vector& clBuffer, + size_t num_requests) { #ifdef HAVE_DEVICE_MEM_SUPPORT slog::info << "Device memory will be used for input and output blobs" << slog::endl; if (inputFiles.size()) { @@ -82,43 +83,45 @@ std::map get_remote_input_tensors( auto& oclContext = static_cast(context); auto oclInstance = std::make_shared(oclContext.get()); - for (auto& inputs_info : app_inputs_info) { - for (auto& input : inputs_info) { - // Fill random - slog::info << "Prepare remote blob for input '" << input.first << "' with random values (" - << std::string((input.second.is_image() ? "image" : "some binary data")) << " is expected)" - << slog::endl; + for (int i = 0; i < num_requests; i++) { + for (auto& inputs_info : app_inputs_info) { + for (auto& input : inputs_info) { + // Fill random + slog::info << "Prepare remote blob for input '" << input.first << "' with random values (" + << std::string((input.second.is_image() ? "image" : "some binary data")) << " is expected)" + << slog::endl; - // Creating and filling shared buffers - cl_int err; - auto elementsNum = std::accumulate(begin(input.second.dataShape), - end(input.second.dataShape), - 1, - std::multiplies()); - auto inputSize = elementsNum * input.second.type.bitwidth() / 8; + // Creating and filling shared buffers + cl_int err; + auto elementsNum = std::accumulate(begin(input.second.dataShape), + end(input.second.dataShape), + 1, + std::multiplies()); + auto inputSize = elementsNum * input.second.type.bitwidth() / 8; - clBuffer.push_back( - cl::Buffer(oclInstance->_context, CL_MEM_READ_WRITE, (cl::size_type)inputSize, NULL, &err)); + clBuffer.push_back( + cl::Buffer(oclInstance->_context, CL_MEM_READ_WRITE, (cl::size_type)inputSize, NULL, &err)); - void* mappedPtr = oclInstance->_queue.enqueueMapBuffer(clBuffer.back(), - CL_TRUE, - CL_MEM_READ_WRITE, - 0, - (cl::size_type)inputSize); + void* mappedPtr = oclInstance->_queue.enqueueMapBuffer(clBuffer.back(), + CL_TRUE, + CL_MEM_READ_WRITE, + 0, + (cl::size_type)inputSize); - auto tensor = oclContext.create_tensor(input.second.type, input.second.dataShape, clBuffer.back().get()); - remoteTensors[input.first].push_back(tensor); + auto tensor = + oclContext.create_tensor(input.second.type, input.second.dataShape, clBuffer.back().get()); + remoteTensors[input.first].push_back(tensor); - if (inputFiles.empty()) { - // Filling in random data - fill_buffer(mappedPtr, elementsNum, input.second.type); - } else { - // TODO: add filling with real image data + if (inputFiles.empty()) { + // Filling in random data + fill_buffer(mappedPtr, elementsNum, input.second.type); + } else { + // TODO: add filling with real image data + } + oclInstance->_queue.enqueueUnmapMemObject(clBuffer.back(), mappedPtr); } - oclInstance->_queue.enqueueUnmapMemObject(clBuffer.back(), mappedPtr); } } - return remoteTensors; #else IE_THROW() << "Device memory requested for GPU device, but OpenCL was not linked"; diff --git a/samples/cpp/benchmark_app/remote_tensors_filling.hpp b/samples/cpp/benchmark_app/remote_tensors_filling.hpp index 7cb919f565f..4e8555b844e 100644 --- a/samples/cpp/benchmark_app/remote_tensors_filling.hpp +++ b/samples/cpp/benchmark_app/remote_tensors_filling.hpp @@ -61,7 +61,8 @@ std::map get_remote_input_tensors( const std::map>& inputFiles, const std::vector& app_inputs_info, const ov::CompiledModel& compiledModel, - std::vector& clBuffer); + std::vector& clBuffer, + size_t num_requests); std::map get_remote_output_tensors(const ov::CompiledModel& compiledModel, std::map& clBuffer); diff --git a/samples/cpp/benchmark_app/utils.cpp b/samples/cpp/benchmark_app/utils.cpp index b9384303859..f943d637afa 100644 --- a/samples/cpp/benchmark_app/utils.cpp +++ b/samples/cpp/benchmark_app/utils.cpp @@ -451,6 +451,7 @@ std::vector get_inputs_info(const std::string& shape_ for (size_t i = 0; i < min_size; ++i) { benchmark_app::InputsInfo info_map; + bool is_there_at_least_one_batch_dim = false; for (auto& item : input_info) { benchmark_app::InputInfo info; auto name = item.get_any_name(); @@ -602,6 +603,7 @@ std::vector get_inputs_info(const std::string& shape_ } info.dataShape[batch_index] = batch_size; reshape_required = true; + is_there_at_least_one_batch_dim = true; } } else { slog::warn << "Input '" << item.get_any_name() @@ -612,6 +614,12 @@ std::vector get_inputs_info(const std::string& shape_ info_map[name] = info; } + if (batch_size > 1 && !is_there_at_least_one_batch_dim) { + throw std::runtime_error("-b option is provided in command line, but there's no inputs with batch(B) " + "dimension in input layout, so batch cannot be set. " + "You may specify layout explicitly using -layout option."); + } + // Update scale and mean std::map> scale_map = parse_scale_or_mean(scale_string, info_map); std::map> mean_map = parse_scale_or_mean(mean_string, info_map); diff --git a/samples/cpp/classification_sample_async/README.md b/samples/cpp/classification_sample_async/README.md index c9f601cc4c8..fbddfc4df2b 100644 --- a/samples/cpp/classification_sample_async/README.md +++ b/samples/cpp/classification_sample_async/README.md @@ -36,7 +36,7 @@ After that, the application starts inference for the first infer request and wai When inference is done, the application outputs data to the standard output stream. You can place labels in .labels file near the model to get pretty output. You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) section of "Integrate the OpenVINO™ Runtime with Your Application" guide. +each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. ## Building @@ -172,7 +172,7 @@ classid probability ## See Also -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) +- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) - [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) - [Model Downloader](@ref omz_tools_downloader) - [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) diff --git a/samples/cpp/hello_classification/README.md b/samples/cpp/hello_classification/README.md index 27af315a355..f0ee0f343b0 100644 --- a/samples/cpp/hello_classification/README.md +++ b/samples/cpp/hello_classification/README.md @@ -26,7 +26,7 @@ The following C++ API is used in the application: At startup, the sample application reads command line parameters, prepares input data, loads a specified model and image to the OpenVINO™ Runtime plugin and performs synchronous inference. Then processes output data and write it to a standard output stream. You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) section of "Integrate the OpenVINO™ Runtime with Your Application" guide. +each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. ## Building @@ -116,7 +116,7 @@ classid probability ## See Also -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) +- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) - [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) - [Model Downloader](@ref omz_tools_downloader) - [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) diff --git a/samples/cpp/hello_nv12_input_classification/README.md b/samples/cpp/hello_nv12_input_classification/README.md index 83fd42793a0..c1a78d56893 100644 --- a/samples/cpp/hello_nv12_input_classification/README.md +++ b/samples/cpp/hello_nv12_input_classification/README.md @@ -25,7 +25,7 @@ Basic OpenVINO™ Runtime API is covered by [Hello Classification C++ sample](.. At startup, the sample application reads command line parameters, loads the specified model and an image in the NV12 color format to an OpenVINO™ Runtime plugin. Then, the sample creates an synchronous inference request object. When inference is done, the application outputs data to the standard output stream. You can place labels in .labels file near the model to get pretty output. -You can see the explicit description of each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) section of "Integrate the OpenVINO™ Runtime with Your Application" guide. +You can see the explicit description of each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. ## Building @@ -130,7 +130,7 @@ classid probability ## See Also -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) +- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) - [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) - [Model Downloader](@ref omz_tools_downloader) - [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) diff --git a/samples/cpp/hello_query_device/README.md b/samples/cpp/hello_query_device/README.md index a3cd7ae034f..202f733542b 100644 --- a/samples/cpp/hello_query_device/README.md +++ b/samples/cpp/hello_query_device/README.md @@ -1,6 +1,6 @@ # Hello Query Device C++ Sample {#openvino_inference_engine_samples_hello_query_device_README} -This sample demonstrates how to execute an query OpenVINO™ Runtime devices, prints their metrics and default configuration values, using [Properties API](../../../docs/OV_Runtime_UG/PropertiesAPI.md). +This sample demonstrates how to execute an query OpenVINO™ Runtime devices, prints their metrics and default configuration values, using [Properties API](../../../docs/OV_Runtime_UG/supported_plugins/config_properties.md). The following C++ API is used in the application: @@ -13,7 +13,7 @@ Basic OpenVINO™ Runtime API is covered by [Hello Classification C++ sample](.. | Options | Values | | :--- |:--- | Supported devices | [All](../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [Python](../../../samples/python/hello_query_device/README.md) | +| Other language realization | [Python](../../python/hello_query_device/README.md) | ## How It Works @@ -90,5 +90,5 @@ The application prints all available devices with their supported metrics and de ## See Also -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) +- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) - [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) diff --git a/samples/cpp/hello_reshape_ssd/README.md b/samples/cpp/hello_reshape_ssd/README.md index ef9d21ba169..cdd0dcfe421 100644 --- a/samples/cpp/hello_reshape_ssd/README.md +++ b/samples/cpp/hello_reshape_ssd/README.md @@ -27,7 +27,7 @@ Upon the start-up the sample application reads command line parameters, loads sp Engine plugin. Then, the sample creates an synchronous inference request object. When inference is done, the application creates output image and output data to the standard output stream. You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) section of "Integrate the OpenVINO™ Runtime with Your Application" guide. +each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. ## Building @@ -116,7 +116,7 @@ This sample is an API example, for any performance measurements please use the d ## See Also -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) +- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) - [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) - [Model Downloader](@ref omz_tools_downloader) - [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) diff --git a/samples/cpp/ngraph_function_creation_sample/CMakeLists.txt b/samples/cpp/model_creation_sample/CMakeLists.txt similarity index 52% rename from samples/cpp/ngraph_function_creation_sample/CMakeLists.txt rename to samples/cpp/model_creation_sample/CMakeLists.txt index c96ea7fae49..5c9d64db6aa 100644 --- a/samples/cpp/ngraph_function_creation_sample/CMakeLists.txt +++ b/samples/cpp/model_creation_sample/CMakeLists.txt @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 # -set(TARGET_NAME "ngraph_function_creation_sample") +set(TARGET_NAME "model_creation_sample") -ie_add_sample(NAME ngraph_function_creation_sample +ie_add_sample(NAME model_creation_sample SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/main.cpp" - HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/ngraph_function_creation_sample.hpp" + HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/model_creation_sample.hpp" DEPENDENCIES format_reader ie_samples_utils) diff --git a/samples/cpp/ngraph_function_creation_sample/README.md b/samples/cpp/model_creation_sample/README.md similarity index 75% rename from samples/cpp/ngraph_function_creation_sample/README.md rename to samples/cpp/model_creation_sample/README.md index 390d522a13b..542d6d82ec0 100644 --- a/samples/cpp/ngraph_function_creation_sample/README.md +++ b/samples/cpp/model_creation_sample/README.md @@ -1,8 +1,8 @@ -# nGraph Function Creation C++ Sample {#openvino_inference_engine_samples_ngraph_function_creation_sample_README} +# Model Creation C++ Sample {#openvino_inference_engine_samples_model_creation_sample_README} This sample demonstrates how to execute an synchronous inference using [model](../../../docs/OV_Runtime_UG/model_representation.md) built on the fly which uses weights from LeNet classification model, which is known to work well on digit classification tasks. -You do not need an XML file to create a model. The API of ngraph::Function allows creating a model on the fly from the source code. +You do not need an XML file to create a model. The API of ov::Model allows creating a model on the fly from the source code. The following C++ API is used in the application: @@ -13,7 +13,7 @@ The following C++ API is used in the application: | Tensor Operations | `ov::Tensor::get_byte_size`, `ov::Tensor:data` | Get tensor byte size and its data | | Model Operations | `ov::set_batch` | Operate with model batch size | | Infer Request Operations | `ov::InferRequest::get_input_tensor` | Get a input tensor | -| nGraph Functions | `ov::opset8::Parameter`, `ov::Node::output`, `ov::opset8::Constant`, `ov::opset8::Convolution`, `ov::opset8::Add`, `ov::opset1::MaxPool`, `ov::opset8::Reshape`, `ov::opset8::MatMul`, `ov::opset8::Relu`, `ov::opset8::Softmax`, `ov::descriptor::Tensor::set_names`, `ov::opset8::Result`, `ov::Model`, `ov::ParameterVector::vector` | Used to construct an nGraph function | +| Model creation objects | `ov::opset8::Parameter`, `ov::Node::output`, `ov::opset8::Constant`, `ov::opset8::Convolution`, `ov::opset8::Add`, `ov::opset1::MaxPool`, `ov::opset8::Reshape`, `ov::opset8::MatMul`, `ov::opset8::Relu`, `ov::opset8::Softmax`, `ov::descriptor::Tensor::set_names`, `ov::opset8::Result`, `ov::Model`, `ov::ParameterVector::vector` | Used to construct an OpenVINO model | Basic OpenVINO™ Runtime API is covered by [Hello Classification C++ sample](../hello_classification/README.md). @@ -23,7 +23,7 @@ Basic OpenVINO™ Runtime API is covered by [Hello Classification C++ sample](.. | Model Format | model weights file (\*.bin) | | Validated images | single-channel `MNIST ubyte` images | | Supported devices | [All](../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [Python](../../../samples/python/ngraph_function_creation_sample/README.md) | +| Other language realization | [Python](../../../samples/python/model_creation_sample/README.md) | ## How It Works @@ -33,7 +33,7 @@ At startup, the sample application does the following: - Loads the model and input data to the OpenVINO™ Runtime plugin - Performs synchronous inference and processes output data, logging each step in a standard output stream -You can see the explicit description of each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) section of "Integrate the OpenVINO™ Runtime with Your Application" guide. +You can see the explicit description of each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. ## Building @@ -42,7 +42,7 @@ To build the sample, please use instructions available at [Build the Sample Appl ## Running ``` -ngraph_function_creation_sample +model_creation_sample ``` > **NOTES**: @@ -56,7 +56,7 @@ ngraph_function_creation_sample You can do inference of an image using a pre-trained model on a GPU using the following command: ``` -ngraph_function_creation_sample lenet.bin GPU +model_creation_sample lenet.bin GPU ``` ## Sample Output @@ -176,12 +176,8 @@ classid probability label -*Starting with the OpenVINO™ toolkit 2020.2 release, all of the features previously available through nGraph have been merged into the OpenVINO™ toolkit. As a result, all the features previously available through ONNX RT Execution Provider for nGraph have been merged with ONNX RT Execution Provider for OpenVINO™ toolkit.* - -*Therefore, ONNX RT Execution Provider for nGraph will be deprecated starting June 1, 2020 and will be completely removed on December 1, 2020. Users are recommended to migrate to the ONNX RT Execution Provider for OpenVINO™ toolkit as the unified solution for all AI inferencing on Intel® hardware.* - ## See Also -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) +- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) - [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) - [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) diff --git a/samples/cpp/ngraph_function_creation_sample/lenet.bin b/samples/cpp/model_creation_sample/lenet.bin similarity index 100% rename from samples/cpp/ngraph_function_creation_sample/lenet.bin rename to samples/cpp/model_creation_sample/lenet.bin diff --git a/samples/cpp/ngraph_function_creation_sample/lenet.labels b/samples/cpp/model_creation_sample/lenet.labels similarity index 100% rename from samples/cpp/ngraph_function_creation_sample/lenet.labels rename to samples/cpp/model_creation_sample/lenet.labels diff --git a/samples/cpp/ngraph_function_creation_sample/main.cpp b/samples/cpp/model_creation_sample/main.cpp similarity index 98% rename from samples/cpp/ngraph_function_creation_sample/main.cpp rename to samples/cpp/model_creation_sample/main.cpp index cb552cdf76c..1ae6db0554c 100644 --- a/samples/cpp/ngraph_function_creation_sample/main.cpp +++ b/samples/cpp/model_creation_sample/main.cpp @@ -21,7 +21,7 @@ #include "samples/classification_results.h" #include "samples/slog.hpp" -#include "ngraph_function_creation_sample.hpp" +#include "model_creation_sample.hpp" // clang-format on constexpr auto N_TOP_RESULTS = 1; @@ -214,10 +214,7 @@ std::shared_ptr create_model(const std::string& path_to_weights) { } /** - * @brief The entry point for inference engine automatic ov::Model - * creation sample - * @file ngraph_function_creation_sample/main.cpp - * @example ngraph_function_creation_sample/main.cpp + * @brief The entry point for OpenVINO ov::Model creation sample */ int main(int argc, char* argv[]) { try { diff --git a/samples/cpp/ngraph_function_creation_sample/ngraph_function_creation_sample.hpp b/samples/cpp/model_creation_sample/model_creation_sample.hpp similarity index 100% rename from samples/cpp/ngraph_function_creation_sample/ngraph_function_creation_sample.hpp rename to samples/cpp/model_creation_sample/model_creation_sample.hpp diff --git a/samples/cpp/speech_sample/README.md b/samples/cpp/speech_sample/README.md index 74ce696a106..0f440ba767d 100644 --- a/samples/cpp/speech_sample/README.md +++ b/samples/cpp/speech_sample/README.md @@ -1,6 +1,6 @@ # Automatic Speech Recognition C++ Sample {#openvino_inference_engine_samples_speech_sample_README} -This sample demonstrates how to execute an Asynchronous Inference of acoustic model based on Kaldi\* neural networks and speech feature vectors. +This sample demonstrates how to execute an Asynchronous Inference of acoustic model based on Kaldi\* neural networks and speech feature vectors. The sample works with Kaldi ARK or Numpy* uncompressed NPZ files, so it does not cover an end-to-end speech recognition scenario (speech to text), requiring additional preprocessing (feature extraction) to get a feature vector from a speech signal, as well as postprocessing (decoding) to produce text from scores. @@ -31,7 +31,7 @@ At startup, the sample application reads command-line parameters, loads a specif If the `-r` option is given, error statistics are provided for each speech utterance as shown above. You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) section of "Integrate the OpenVINO™ Runtime with Your Application" guide. +each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. ### GNA-specific details @@ -237,7 +237,7 @@ All of mentioned files can be downloaded from [https://storage.openvinotoolkit.o ## See Also -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) +- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) - [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) - [Model Downloader](@ref omz_tools_downloader) - [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) diff --git a/samples/cpp/speech_sample/main.cpp b/samples/cpp/speech_sample/main.cpp index fd484be240e..f62d9f009f0 100644 --- a/samples/cpp/speech_sample/main.cpp +++ b/samples/cpp/speech_sample/main.cpp @@ -86,10 +86,11 @@ int main(int argc, char* argv[]) { uint32_t batchSize = (FLAGS_cw_r > 0 || FLAGS_cw_l > 0 || !FLAGS_bs) ? 1 : (uint32_t)FLAGS_bs; std::shared_ptr model; std::vector outputs; + std::vector output_names; std::vector ports; // --------------------------- Processing custom outputs --------------------------------------------- if (!FLAGS_oname.empty()) { - std::vector output_names = convert_str_to_vector(FLAGS_oname); + output_names = convert_str_to_vector(FLAGS_oname); for (const auto& output_name : output_names) { auto pos_layer = output_name.rfind(":"); if (pos_layer == std::string::npos) { @@ -178,19 +179,22 @@ int main(int argc, char* argv[]) { } if (FLAGS_q.compare("user") == 0) { if (!FLAGS_rg.empty()) { - slog::warn << "Custom scale factor will be used for imported gna model: " << FLAGS_rg << slog::endl; - } - auto scale_factors_per_input = parse_scale_factors(model->inputs(), FLAGS_sf); - if (numInputFiles != scale_factors_per_input.size()) { - std::string errMessage( - "Incorrect command line for multiple inputs: " + std::to_string(scale_factors_per_input.size()) + - " scale factors provided for " + std::to_string(numInputFiles) + " input files."); + std::string errMessage("Custom scale factor can not be set for imported gna model: " + FLAGS_rg); throw std::logic_error(errMessage); + } else { + auto scale_factors_per_input = parse_scale_factors(model->inputs(), FLAGS_sf); + if (numInputFiles != scale_factors_per_input.size()) { + std::string errMessage("Incorrect command line for multiple inputs: " + + std::to_string(scale_factors_per_input.size()) + + " scale factors provided for " + std::to_string(numInputFiles) + + " input files."); + throw std::logic_error(errMessage); + } + for (auto&& sf : scale_factors_per_input) { + slog::info << "For input " << sf.first << " using scale factor of " << sf.second << slog::endl; + } + gnaPluginConfig[ov::intel_gna::scale_factors_per_input.name()] = scale_factors_per_input; } - for (auto&& sf : scale_factors_per_input) { - slog::info << "For input " << sf.first << " using scale factor of " << sf.second << slog::endl; - } - gnaPluginConfig[ov::intel_gna::scale_factors_per_input.name()] = scale_factors_per_input; } else { // "static" quantization with calculated scale factor if (!FLAGS_rg.empty()) { @@ -245,10 +249,9 @@ int main(int argc, char* argv[]) { auto t0 = Time::now(); ms loadTime = std::chrono::duration_cast(Time::now() - t0); slog::info << "Model loading time " << loadTime.count() << " ms" << slog::endl; - slog::info << "Loading model to the device " << FLAGS_d << slog::endl; ov::CompiledModel executableNet; if (!FLAGS_m.empty()) { - slog::info << "Loading model to the device" << slog::endl; + slog::info << "Loading model to the device " << FLAGS_d << slog::endl; executableNet = core.compile_model(model, deviceStr, genericPluginConfig); } else { slog::info << "Importing model to the device" << slog::endl; @@ -341,157 +344,184 @@ int main(int argc, char* argv[]) { } // ----------------------------------------------------------------------------------------------------- // --------------------------- Step 5. Do inference -------------------------------------------------------- - for (size_t next_output = 0; next_output < count_file; next_output++) { - std::vector> ptrUtterances; - std::vector ptrScores; - std::vector ptrReferenceScores; - ScoreErrorT frameError, totalError; - ptrUtterances.resize(inputFiles.size()); - // initialize memory state before starting - for (auto&& state : inferRequests.begin()->inferRequest.query_state()) { - state.reset(); - } - /** Work with each utterance **/ - for (uint32_t utteranceIndex = 0; utteranceIndex < numUtterances; ++utteranceIndex) { - std::map utterancePerfMap; - uint64_t totalNumberOfRunsOnHw = 0; - std::string uttName; - uint32_t numFrames(0), n(0); - std::vector numFrameElementsInput; - uint32_t numFramesReference(0), numFrameElementsReference(0), numBytesPerElementReference(0), - numBytesReferenceScoreThisUtterance(0); - auto dims = executableNet.outputs()[0].get_shape(); - const auto numScoresPerFrame = - std::accumulate(std::begin(dims), std::end(dims), size_t{1}, std::multiplies()); - slog::info << "Number scores per frame : " << numScoresPerFrame << slog::endl; - /** Get information from input file for current utterance **/ - numFrameElementsInput.resize(numInputFiles); - for (size_t i = 0; i < inputFiles.size(); i++) { - std::vector ptrUtterance; - auto inputFilename = inputFiles[i].c_str(); - uint32_t currentNumFrames(0), currentNumFrameElementsInput(0), currentNumBytesPerElementInput(0); - file->get_file_info(inputFilename, utteranceIndex, &n, &numBytesThisUtterance[i]); - ptrUtterance.resize(numBytesThisUtterance[i]); - file->load_file(inputFilename, - utteranceIndex, - uttName, - ptrUtterance, - ¤tNumFrames, - ¤tNumFrameElementsInput, - ¤tNumBytesPerElementInput); - if (numFrames == 0) { - numFrames = currentNumFrames; - } else if (numFrames != currentNumFrames) { - std::string errMessage("Number of frames in input files is different: " + - std::to_string(numFrames) + " and " + std::to_string(currentNumFrames)); - throw std::logic_error(errMessage); - } - ptrUtterances[i] = ptrUtterance; - numFrameElementsInput[i] = currentNumFrameElementsInput; - } - int i = 0; - for (auto& ptrInputBlob : ptrInputBlobs) { - if (ptrInputBlob.get_size() != numFrameElementsInput[i++] * batchSize) { - throw std::logic_error("network input size(" + std::to_string(ptrInputBlob.get_size()) + - ") mismatch to input file size (" + - std::to_string(numFrameElementsInput[i - 1] * batchSize) + ")"); - } - } - ptrScores.resize(numFrames * numScoresPerFrame * sizeof(float)); - if (!FLAGS_r.empty()) { - /** Read file with reference scores **/ - BaseFile* fileReferenceScores; - auto exReferenceScoresFile = fileExt(FLAGS_r); - if (exReferenceScoresFile == "ark") { - fileReferenceScores = &arkFile; - } else if (exReferenceScoresFile == "npz") { - fileReferenceScores = &numpyFile; - } else { - throw std::logic_error("Invalid Reference Scores file"); - } - std::string refUtteranceName; - fileReferenceScores->get_file_info(reference_name_files[next_output].c_str(), - utteranceIndex, - &n, - &numBytesReferenceScoreThisUtterance); - ptrReferenceScores.resize(numBytesReferenceScoreThisUtterance); - fileReferenceScores->load_file(reference_name_files[next_output].c_str(), - utteranceIndex, - refUtteranceName, - ptrReferenceScores, - &numFramesReference, - &numFrameElementsReference, - &numBytesPerElementReference); - } - double totalTime = 0.0; - std::cout << "Utterance " << utteranceIndex << ": " << std::endl; - clear_score_error(&totalError); - totalError.threshold = frameError.threshold = MAX_SCORE_DIFFERENCE; - auto outputFrame = &ptrScores.front(); - std::vector inputFrame; - for (auto& ut : ptrUtterances) { - inputFrame.push_back(&ut.front()); - } - std::map callPerfMap; - size_t frameIndex = 0; - uint32_t numFramesFile = numFrames; - numFrames += FLAGS_cw_l + FLAGS_cw_r; - uint32_t numFramesThisBatch{batchSize}; - auto t0 = Time::now(); - auto t1 = t0; - while (frameIndex <= numFrames) { - if (frameIndex == numFrames) { - if (std::find_if(inferRequests.begin(), inferRequests.end(), [&](InferRequestStruct x) { - return (x.frameIndex != -1); - }) == inferRequests.end()) { - break; - } - } - bool inferRequestFetched = false; - /** Start inference loop **/ - for (auto& inferRequest : inferRequests) { - if (frameIndex == numFrames) { - numFramesThisBatch = 1; - } else { - numFramesThisBatch = - (numFrames - frameIndex < batchSize) ? (numFrames - frameIndex) : batchSize; - } + std::vector> ptrUtterances; + std::vector> vectorPtrScores((outputs.size() == 0) ? 1 : outputs.size()); + std::vector numScoresPerOutput((outputs.size() == 0) ? 1 : outputs.size()); + std::vector> vectorPtrReferenceScores(reference_name_files.size()); + std::vector vectorFrameError(reference_name_files.size()), + vectorTotalError(reference_name_files.size()); + ptrUtterances.resize(inputFiles.size()); + // initialize memory state before starting + for (auto&& state : inferRequests.begin()->inferRequest.query_state()) { + state.reset(); + } + /** Work with each utterance **/ + for (uint32_t utteranceIndex = 0; utteranceIndex < numUtterances; ++utteranceIndex) { + std::map utterancePerfMap; + uint64_t totalNumberOfRunsOnHw = 0; + std::string uttName; + uint32_t numFrames(0), n(0); + std::vector numFrameElementsInput; + std::vector numFramesReference(reference_name_files.size()), + numFrameElementsReference(reference_name_files.size()), + numBytesPerElementReference(reference_name_files.size()), + numBytesReferenceScoreThisUtterance(reference_name_files.size()); + + /** Get information from input file for current utterance **/ + numFrameElementsInput.resize(numInputFiles); + for (size_t i = 0; i < inputFiles.size(); i++) { + std::vector ptrUtterance; + auto inputFilename = inputFiles[i].c_str(); + uint32_t currentNumFrames(0), currentNumFrameElementsInput(0), currentNumBytesPerElementInput(0); + file->get_file_info(inputFilename, utteranceIndex, &n, &numBytesThisUtterance[i]); + ptrUtterance.resize(numBytesThisUtterance[i]); + file->load_file(inputFilename, + utteranceIndex, + uttName, + ptrUtterance, + ¤tNumFrames, + ¤tNumFrameElementsInput, + ¤tNumBytesPerElementInput); + if (numFrames == 0) { + numFrames = currentNumFrames; + } else if (numFrames != currentNumFrames) { + std::string errMessage("Number of frames in input files is different: " + + std::to_string(numFrames) + " and " + std::to_string(currentNumFrames)); + throw std::logic_error(errMessage); + } + ptrUtterances[i] = ptrUtterance; + numFrameElementsInput[i] = currentNumFrameElementsInput; + } + int i = 0; + for (auto& ptrInputBlob : ptrInputBlobs) { + if (ptrInputBlob.get_size() != numFrameElementsInput[i++] * batchSize) { + throw std::logic_error("network input size(" + std::to_string(ptrInputBlob.get_size()) + + ") mismatch to input file size (" + + std::to_string(numFrameElementsInput[i - 1] * batchSize) + ")"); + } + } + + double totalTime = 0.0; + + for (size_t errorIndex = 0; errorIndex < vectorFrameError.size(); errorIndex++) { + clear_score_error(&vectorTotalError[errorIndex]); + vectorTotalError[errorIndex].threshold = vectorFrameError[errorIndex].threshold = MAX_SCORE_DIFFERENCE; + } + + std::vector inputFrame; + for (auto& ut : ptrUtterances) { + inputFrame.push_back(&ut.front()); + } + std::map callPerfMap; + size_t frameIndex = 0; + uint32_t numFramesFile = numFrames; + numFrames += FLAGS_cw_l + FLAGS_cw_r; + uint32_t numFramesThisBatch{batchSize}; + auto t0 = Time::now(); + auto t1 = t0; + + BaseFile* fileReferenceScores; + std::string refUtteranceName; + + if (!FLAGS_r.empty()) { + /** Read file with reference scores **/ + auto exReferenceScoresFile = fileExt(FLAGS_r); + if (exReferenceScoresFile == "ark") { + fileReferenceScores = &arkFile; + } else if (exReferenceScoresFile == "npz") { + fileReferenceScores = &numpyFile; + } else { + throw std::logic_error("Invalid Reference Scores file"); + } + for (size_t next_output = 0; next_output < count_file; next_output++) { + if (fileReferenceScores != nullptr) { + fileReferenceScores->get_file_info(reference_name_files[next_output].c_str(), + utteranceIndex, + &n, + &numBytesReferenceScoreThisUtterance[next_output]); + vectorPtrReferenceScores[next_output].resize(numBytesReferenceScoreThisUtterance[next_output]); + fileReferenceScores->load_file(reference_name_files[next_output].c_str(), + utteranceIndex, + refUtteranceName, + vectorPtrReferenceScores[next_output], + &numFramesReference[next_output], + &numFrameElementsReference[next_output], + &numBytesPerElementReference[next_output]); + } + } + } + + while (frameIndex <= numFrames) { + if (frameIndex == numFrames) { + if (std::find_if(inferRequests.begin(), inferRequests.end(), [&](InferRequestStruct x) { + return (x.frameIndex != -1); + }) == inferRequests.end()) { + break; + } + } + bool inferRequestFetched = false; + /** Start inference loop **/ + for (auto& inferRequest : inferRequests) { + if (frameIndex == numFrames) { + numFramesThisBatch = 1; + } else { + numFramesThisBatch = + (numFrames - frameIndex < batchSize) ? (numFrames - frameIndex) : batchSize; + } + + /* waits until inference result becomes available */ + if (inferRequest.frameIndex != -1) { + inferRequest.inferRequest.wait(); + if (inferRequest.frameIndex >= 0) + for (size_t next_output = 0; next_output < count_file; next_output++) { + std::string outputName = (outputs.size() == 0) ? executableNet.output(0).get_any_name() + : output_names[next_output]; + auto dims = executableNet.output(outputName).get_shape(); + numScoresPerOutput[next_output] = std::accumulate(std::begin(dims), + std::end(dims), + size_t{1}, + std::multiplies()); + + vectorPtrScores[next_output].resize(numFramesFile * numScoresPerOutput[next_output] * + sizeof(float)); - /* waits until inference result becomes available */ - if (inferRequest.frameIndex != -1) { - inferRequest.inferRequest.wait(); - if (inferRequest.frameIndex >= 0) { if (!FLAGS_o.empty()) { /* Prepare output data for save to file in future */ - outputFrame = &ptrScores.front() + - numScoresPerFrame * sizeof(float) * (inferRequest.frameIndex); + auto outputFrame = + &vectorPtrScores[next_output].front() + + numScoresPerOutput[next_output] * sizeof(float) * (inferRequest.frameIndex); ov::Tensor outputBlob = - inferRequest.inferRequest.get_tensor(executableNet.outputs()[0]); + inferRequest.inferRequest.get_tensor(executableNet.output(outputName)); if (!outputs.empty()) { outputBlob = - inferRequest.inferRequest.get_tensor(executableNet.output(FLAGS_oname)); + inferRequest.inferRequest.get_tensor(executableNet.output(outputName)); } - // locked memory holder should be alive all time while access to its buffer - // happens - auto byteSize = numScoresPerFrame * sizeof(float); + // locked memory holder should be alive all time while access to its buffer happens + auto byteSize = numScoresPerOutput[next_output] * sizeof(float); std::memcpy(outputFrame, outputBlob.data(), byteSize); } if (!FLAGS_r.empty()) { /** Compare output data with reference scores **/ ov::Tensor outputBlob = - inferRequest.inferRequest.get_tensor(executableNet.outputs()[0]); - if (!FLAGS_oname.empty()) - outputBlob = - inferRequest.inferRequest.get_tensor(executableNet.output(FLAGS_oname)); - compare_scores( - outputBlob.data(), - &ptrReferenceScores[inferRequest.frameIndex * numFrameElementsReference * - numBytesPerElementReference], - &frameError, - inferRequest.numFramesThisBatch, - numFrameElementsReference); - update_score_error(&frameError, &totalError); + inferRequest.inferRequest.get_tensor(executableNet.output(outputName)); + + if (numScoresPerOutput[next_output] / numFrameElementsReference[next_output] == + batchSize) { + compare_scores( + outputBlob.data(), + &vectorPtrReferenceScores[next_output] + [inferRequest.frameIndex * + numFrameElementsReference[next_output] * + numBytesPerElementReference[next_output]], + &vectorFrameError[next_output], + inferRequest.numFramesThisBatch, + numFrameElementsReference[next_output]); + update_score_error(&vectorFrameError[next_output], + &vectorTotalError[next_output]); + } else { + throw std::logic_error("Number of output and reference frames does not match."); + } } if (FLAGS_pc) { // retrieve new counters @@ -500,90 +530,110 @@ int main(int argc, char* argv[]) { sum_performance_counters(callPerfMap, utterancePerfMap, totalNumberOfRunsOnHw); } } - // ----------------------------------------------------------------------------------------------------- - } - if (frameIndex == numFrames) { - inferRequest.frameIndex = -1; - continue; - } - ptrInputBlobs.clear(); - if (FLAGS_iname.empty()) { - for (auto& input : cInputInfo) { - ptrInputBlobs.push_back(inferRequest.inferRequest.get_tensor(input)); - } - } else { - std::vector inputNameBlobs = convert_str_to_vector(FLAGS_iname); - for (const auto& input : inputNameBlobs) { - ov::Tensor blob = inferRequests.begin()->inferRequest.get_tensor(input); - if (!blob) { - std::string errMessage("No blob with name : " + input); - throw std::logic_error(errMessage); - } - ptrInputBlobs.push_back(blob); - } - } - - /** Iterate over all the input blobs **/ - for (size_t i = 0; i < numInputFiles; ++i) { - ov::Tensor minput = ptrInputBlobs[i]; - if (!minput) { - std::string errMessage("We expect ptrInputBlobs[" + std::to_string(i) + - "] to be inherited from Tensor, " + - "but in fact we were not able to cast input to Tensor"); - throw std::logic_error(errMessage); - } - memcpy(minput.data(), inputFrame[i], minput.get_byte_size()); - // Used to infer fewer frames than the batch size - if (batchSize != numFramesThisBatch) { - memset(minput.data() + numFramesThisBatch * numFrameElementsInput[i], - 0, - (batchSize - numFramesThisBatch) * numFrameElementsInput[i]); - } - } // ----------------------------------------------------------------------------------------------------- - int index = static_cast(frameIndex) - (FLAGS_cw_l + FLAGS_cw_r); - /* Starting inference in asynchronous mode*/ - inferRequest.inferRequest.start_async(); - inferRequest.frameIndex = index < 0 ? -2 : index; - inferRequest.numFramesThisBatch = numFramesThisBatch; - frameIndex += numFramesThisBatch; - for (size_t j = 0; j < inputFiles.size(); j++) { - if (FLAGS_cw_l > 0 || FLAGS_cw_r > 0) { - int idx = frameIndex - FLAGS_cw_l; - if (idx > 0 && idx < static_cast(numFramesFile)) { - inputFrame[j] += sizeof(float) * numFrameElementsInput[j] * numFramesThisBatch; - } else if (idx >= static_cast(numFramesFile)) { - inputFrame[j] = &ptrUtterances[j].front() + (numFramesFile - 1) * sizeof(float) * - numFrameElementsInput[j] * - numFramesThisBatch; - } else if (idx <= 0) { - inputFrame[j] = &ptrUtterances[j].front(); - } - } else { - inputFrame[j] += sizeof(float) * numFrameElementsInput[j] * numFramesThisBatch; - } - } - inferRequestFetched |= true; } - /** Inference was finished for current frame **/ - if (!inferRequestFetched) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + if (frameIndex == numFrames) { + inferRequest.frameIndex = -1; continue; } - } - t1 = Time::now(); - fsec fs = t1 - t0; - ms d = std::chrono::duration_cast(fs); - totalTime += d.count(); - // resetting state between utterances - for (auto&& state : inferRequests.begin()->inferRequest.query_state()) { - state.reset(); - } - // ----------------------------------------------------------------------------------------------------- + ptrInputBlobs.clear(); + if (FLAGS_iname.empty()) { + for (auto& input : cInputInfo) { + ptrInputBlobs.push_back(inferRequest.inferRequest.get_tensor(input)); + } + } else { + std::vector inputNameBlobs = convert_str_to_vector(FLAGS_iname); + for (const auto& input : inputNameBlobs) { + ov::Tensor blob = inferRequests.begin()->inferRequest.get_tensor(input); + if (!blob) { + std::string errMessage("No blob with name : " + input); + throw std::logic_error(errMessage); + } + ptrInputBlobs.push_back(blob); + } + } - // --------------------------- Step 6. Process output - // ------------------------------------------------------- + /** Iterate over all the input blobs **/ + for (size_t i = 0; i < numInputFiles; ++i) { + ov::Tensor minput = ptrInputBlobs[i]; + if (!minput) { + std::string errMessage("We expect ptrInputBlobs[" + std::to_string(i) + + "] to be inherited from Tensor, " + + "but in fact we were not able to cast input to Tensor"); + throw std::logic_error(errMessage); + } + memcpy(minput.data(), + inputFrame[i], + numFramesThisBatch * numFrameElementsInput[i] * sizeof(float)); + // Used to infer fewer frames than the batch size + if (batchSize != numFramesThisBatch) { + memset(minput.data() + numFramesThisBatch * numFrameElementsInput[i], + 0, + (batchSize - numFramesThisBatch) * numFrameElementsInput[i]); + } + } + // ----------------------------------------------------------------------------------------------------- + int index = static_cast(frameIndex) - (FLAGS_cw_l + FLAGS_cw_r); + /* Starting inference in asynchronous mode*/ + inferRequest.inferRequest.start_async(); + inferRequest.frameIndex = index < 0 ? -2 : index; + inferRequest.numFramesThisBatch = numFramesThisBatch; + frameIndex += numFramesThisBatch; + for (size_t j = 0; j < inputFiles.size(); j++) { + if (FLAGS_cw_l > 0 || FLAGS_cw_r > 0) { + int idx = frameIndex - FLAGS_cw_l; + if (idx > 0 && idx < static_cast(numFramesFile)) { + inputFrame[j] += sizeof(float) * numFrameElementsInput[j] * numFramesThisBatch; + } else if (idx >= static_cast(numFramesFile)) { + inputFrame[j] = &ptrUtterances[j].front() + (numFramesFile - 1) * sizeof(float) * + numFrameElementsInput[j] * + numFramesThisBatch; + } else if (idx <= 0) { + inputFrame[j] = &ptrUtterances[j].front(); + } + } else { + inputFrame[j] += sizeof(float) * numFrameElementsInput[j] * numFramesThisBatch; + } + } + inferRequestFetched |= true; + } + /** Inference was finished for current frame **/ + if (!inferRequestFetched) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + continue; + } + } + t1 = Time::now(); + fsec fs = t1 - t0; + ms d = std::chrono::duration_cast(fs); + totalTime += d.count(); + // resetting state between utterances + for (auto&& state : inferRequests.begin()->inferRequest.query_state()) { + state.reset(); + } + // ----------------------------------------------------------------------------------------------------- + // --------------------------- Step 6. Process output + // ------------------------------------------------------- + + /** Show performance results **/ + std::cout << "Utterance " << utteranceIndex << ": " << std::endl; + std::cout << "Total time in Infer (HW and SW):\t" << totalTime << " ms" << std::endl; + std::cout << "Frames in utterance:\t\t\t" << numFrames << " frames" << std::endl; + std::cout << "Average Infer time per frame:\t\t" << totalTime / static_cast(numFrames) << " ms\n" + << std::endl; + + if (FLAGS_pc) { + // print performance results + print_performance_counters(utterancePerfMap, + frameIndex, + std::cout, + getFullDeviceName(core, FLAGS_d), + totalNumberOfRunsOnHw, + FLAGS_d); + } + + for (size_t next_output = 0; next_output < count_file; next_output++) { if (!FLAGS_o.empty()) { auto exOutputScoresFile = fileExt(FLAGS_o); if (exOutputScoresFile == "ark") { @@ -598,33 +648,21 @@ int main(int argc, char* argv[]) { fileOutput->save_file(output_name_files[next_output].c_str(), shouldAppend, uttName, - &ptrScores.front(), + &vectorPtrScores[next_output].front(), numFramesFile, - numScoresPerFrame); - } - /** Show performance results **/ - std::cout << "Total time in Infer (HW and SW):\t" << totalTime << " ms" << std::endl; - std::cout << "Frames in utterance:\t\t\t" << numFrames << " frames" << std::endl; - std::cout << "Average Infer time per frame:\t\t" << totalTime / static_cast(numFrames) << " ms" - << std::endl; - if (FLAGS_pc) { - // print performance results - print_performance_counters(utterancePerfMap, - frameIndex, - std::cout, - getFullDeviceName(core, FLAGS_d), - totalNumberOfRunsOnHw, - FLAGS_d); + numScoresPerOutput[next_output]); } if (!FLAGS_r.empty()) { // print statistical score error - print_reference_compare_results(totalError, numFrames, std::cout); + std::string outputName = + (outputs.size() == 0) ? executableNet.output(0).get_any_name() : output_names[next_output]; + std::cout << "Output name: " << outputName << std::endl; + std::cout << "Number scores per frame: " << numScoresPerOutput[next_output] / batchSize << std::endl + << std::endl; + print_reference_compare_results(vectorTotalError[next_output], numFrames, std::cout); } - std::cout << "End of Utterance " << utteranceIndex << std::endl << std::endl; - // ----------------------------------------------------------------------------------------------------- } } - // ----------------------------------------------------------------------------------------------------- } catch (const std::exception& error) { slog::err << error.what() << slog::endl; return 1; diff --git a/samples/python/classification_sample_async/README.md b/samples/python/classification_sample_async/README.md index 98c38a47166..24f4095001a 100644 --- a/samples/python/classification_sample_async/README.md +++ b/samples/python/classification_sample_async/README.md @@ -23,7 +23,7 @@ Basic OpenVINO™ Runtime API is covered by [Hello Classification Python* Sample At startup, the sample application reads command-line parameters, prepares input data, loads a specified model and image(s) to the OpenVINO™ Runtime plugin, performs synchronous inference, and processes output data, logging each step in a standard output stream. You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) section of "Integrate the OpenVINO™ Runtime with Your Application" guide. +each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. ## Running @@ -135,7 +135,7 @@ The sample application logs each step in a standard output stream and outputs to ## See Also -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) +- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) - [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) - [Model Downloader](@ref omz_tools_downloader) - [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) diff --git a/samples/python/hello_classification/README.md b/samples/python/hello_classification/README.md index 71f15ba8315..a5dd902fc1b 100644 --- a/samples/python/hello_classification/README.md +++ b/samples/python/hello_classification/README.md @@ -24,7 +24,7 @@ The following Python API is used in the application: At startup, the sample application reads command-line parameters, prepares input data, loads a specified model and image to the OpenVINO™ Runtime plugin, performs synchronous inference, and processes output data, logging each step in a standard output stream. You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) section of "Integrate the OpenVINO™ Runtime with Your Application" guide. +each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. ## Running @@ -98,7 +98,7 @@ The sample application logs each step in a standard output stream and outputs to ## See Also -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) +- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) - [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) - [Model Downloader](@ref omz_tools_downloader) - [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) diff --git a/samples/python/hello_query_device/README.md b/samples/python/hello_query_device/README.md index 67087932a69..aa934529df0 100644 --- a/samples/python/hello_query_device/README.md +++ b/samples/python/hello_query_device/README.md @@ -1,6 +1,6 @@ # Hello Query Device Python* Sample {#openvino_inference_engine_ie_bridges_python_sample_hello_query_device_README} -This sample demonstrates how to show OpenVINO™ Runtime devices and prints their metrics and default configuration values using [Query Device API feature](../../../docs/OV_Runtime_UG/PropertiesAPI.md). +This sample demonstrates how to show OpenVINO™ Runtime devices and prints their metrics and default configuration values using [Query Device API feature](../../../docs/OV_Runtime_UG/supported_plugins/config_properties.md). The following Python API is used in the application: diff --git a/samples/python/hello_reshape_ssd/README.md b/samples/python/hello_reshape_ssd/README.md index ef16766af1f..f7bc8e6aba7 100644 --- a/samples/python/hello_reshape_ssd/README.md +++ b/samples/python/hello_reshape_ssd/README.md @@ -24,7 +24,7 @@ At startup, the sample application reads command-line parameters, prepares input As a result, the program creates an output image, logging each step in a standard output stream. You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) section of "Integrate the OpenVINO™ Runtime with Your Application" guide. +each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. ## Running @@ -86,7 +86,7 @@ The sample application logs each step in a standard output stream and creates an ## See Also -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) +- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) - [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) - [Model Downloader](@ref omz_tools_downloader) - [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) diff --git a/samples/python/ngraph_function_creation_sample/README.md b/samples/python/model_creation_sample/README.md similarity index 88% rename from samples/python/ngraph_function_creation_sample/README.md rename to samples/python/model_creation_sample/README.md index 22979415726..068cb25894c 100644 --- a/samples/python/ngraph_function_creation_sample/README.md +++ b/samples/python/model_creation_sample/README.md @@ -1,13 +1,13 @@ -# nGraph Function Creation Python* Sample {#openvino_inference_engine_ie_bridges_python_sample_ngraph_function_creation_sample_README} +# Model Creation Python* Sample {#openvino_inference_engine_ie_bridges_python_sample_model_creation_sample_README} -This sample demonstrates how to run inference using a [model](../../../docs/OV_Runtime_UG/model_representation.md) built on the fly that uses weights from the LeNet classification model, which is known to work well on digit classification tasks. You do not need an XML file, the model is created from the source code on the fly. +This sample demonstrates how to run inference using a [model](../../../docs/OV_Runtime_UG/model_representation.md) built on the fly that uses weights from the LeNet classification model, which is known to work well on digit classification tasks. You do not need an XML file, the model is created from the source code on the fly. -The following Python API is used in the application: +The following OpenVINO Python API is used in the application: | Feature | API | Description | | :--------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------ | | Model Operations | [openvino.runtime.Model], [openvino.runtime.set_batch], [openvino.runtime.Model.input] | Managing of model | -| nGraph Functions | [openvino.runtime.op.Parameter], [openvino.runtime.op.Constant], [openvino.runtime.opset8.convolution], [openvino.runtime.opset8.add], [openvino.runtime.opset1.max_pool], [openvino.runtime.opset8.reshape], [openvino.runtime.opset8.matmul], [openvino.runtime.opset8.relu], [openvino.runtime.opset8.softmax] | Description of a model topology using nGraph Python API | +| Opset operations | [openvino.runtime.op.Parameter], [openvino.runtime.op.Constant], [openvino.runtime.opset8.convolution], [openvino.runtime.opset8.add], [openvino.runtime.opset1.max_pool], [openvino.runtime.opset8.reshape], [openvino.runtime.opset8.matmul], [openvino.runtime.opset8.relu], [openvino.runtime.opset8.softmax] | Description of a model topology using OpenVINO Python API | Basic OpenVINO™ Runtime API is covered by [Hello Classification Python* Sample](../hello_classification/README.md). @@ -16,7 +16,7 @@ Basic OpenVINO™ Runtime API is covered by [Hello Classification Python* Sample | Validated Models | LeNet | | Model Format | Model weights file (\*.bin) | | Supported devices | [All](../../../docs/OV_Runtime_UG/supported_plugins/Supported_Devices.md) | -| Other language realization | [C++](../../../samples/cpp/ngraph_function_creation_sample/README.md) | +| Other language realization | [C++](../../../samples/cpp/model_creation_sample/README.md) | ## How It Works @@ -28,14 +28,14 @@ At startup, the sample application does the following: You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) section of "Integrate the OpenVINO™ Runtime with Your Application" guide. +each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. ## Running To run the sample, you need to specify model weights and device. ``` -python ngraph_function_creation_sample.py +python model_creation_sample.py ``` > **NOTE**: @@ -49,7 +49,7 @@ python ngraph_function_creation_sample.py For example: ``` -python ngraph_function_creation_sample.py lenet.bin GPU +python model_creation_sample.py lenet.bin GPU ``` ## Sample Output @@ -127,7 +127,7 @@ The sample application logs each step in a standard output stream and outputs 10 ## See Also -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) +- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) - [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) - [Model Downloader](@ref omz_tools_downloader) - [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) diff --git a/samples/python/ngraph_function_creation_sample/data.py b/samples/python/model_creation_sample/data.py similarity index 100% rename from samples/python/ngraph_function_creation_sample/data.py rename to samples/python/model_creation_sample/data.py diff --git a/samples/python/ngraph_function_creation_sample/lenet.bin b/samples/python/model_creation_sample/lenet.bin similarity index 100% rename from samples/python/ngraph_function_creation_sample/lenet.bin rename to samples/python/model_creation_sample/lenet.bin diff --git a/samples/python/ngraph_function_creation_sample/ngraph_function_creation_sample.py b/samples/python/model_creation_sample/model_creation_sample.py similarity index 100% rename from samples/python/ngraph_function_creation_sample/ngraph_function_creation_sample.py rename to samples/python/model_creation_sample/model_creation_sample.py diff --git a/samples/python/speech_sample/README.md b/samples/python/speech_sample/README.md index 1f460379557..3c8dfa99b4c 100644 --- a/samples/python/speech_sample/README.md +++ b/samples/python/speech_sample/README.md @@ -29,7 +29,7 @@ Basic OpenVINO™ Runtime API is covered by [Hello Classification Python* Sample At startup, the sample application reads command-line parameters, loads a specified model and input data to the OpenVINO™ Runtime plugin, performs synchronous inference on all speech utterances stored in the input file, logging each step in a standard output stream. You can see the explicit description of -each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) section of "Integrate the OpenVINO™ Runtime with Your Application" guide. +each sample step at [Integration Steps](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) section of "Integrate OpenVINO™ Runtime with Your Application" guide. ## GNA-specific details @@ -328,7 +328,7 @@ The sample application logs each step in a standard output stream. ## See Also -- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/Integrate_with_customer_application_new_API.md) +- [Integrate the OpenVINO™ Runtime with Your Application](../../../docs/OV_Runtime_UG/integrate_with_your_application.md) - [Using OpenVINO™ Toolkit Samples](../../../docs/OV_Runtime_UG/Samples_Overview.md) - [Model Downloader](@ref omz_tools_downloader) - [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) diff --git a/scripts/setupvars/setupvars.sh b/scripts/setupvars/setupvars.sh index fdfcc0f34f3..ab186881dc6 100755 --- a/scripts/setupvars/setupvars.sh +++ b/scripts/setupvars/setupvars.sh @@ -34,8 +34,8 @@ if [ -e "$INSTALLDIR/runtime" ]; then export HDDL_INSTALL_DIR=$INSTALLDIR/runtime/3rdparty/hddl if [[ "$OSTYPE" == "darwin"* ]]; then - export DYLD_LIBRARY_PATH=${IE_PLUGINS_PATH}${DYLD_LIBRARY_PATH:+:DYLD_LIBRARY_PATH} - export LD_LIBRARY_PATH=${IE_PLUGINS_PATH}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} + export DYLD_LIBRARY_PATH=${IE_PLUGINS_PATH}/Release:${IE_PLUGINS_PATH}/Debug${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH} + export LD_LIBRARY_PATH=${IE_PLUGINS_PATH}/Release:${IE_PLUGINS_PATH}/Debug${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} else export LD_LIBRARY_PATH=$HDDL_INSTALL_DIR/lib:${IE_PLUGINS_PATH}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} fi @@ -49,7 +49,7 @@ fi if [ -e "$INSTALLDIR/runtime/3rdparty/tbb" ]; then if [[ "$OSTYPE" == "darwin"* ]]; then - export DYLD_LIBRARY_PATH=$INSTALLDIR/runtime/3rdparty/tbb/lib:${DYLD_LIBRARY_PATH:+:DYLD_LIBRARY_PATH} + export DYLD_LIBRARY_PATH=$INSTALLDIR/runtime/3rdparty/tbb/lib:${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH} fi export LD_LIBRARY_PATH=$INSTALLDIR/runtime/3rdparty/tbb/lib:${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} export TBB_DIR=$INSTALLDIR/runtime/3rdparty/tbb/cmake diff --git a/src/bindings/python/src/compatibility/ngraph/opset5/__init__.py b/src/bindings/python/src/compatibility/ngraph/opset5/__init__.py index 4605a5faa65..5a60a9ff2aa 100644 --- a/src/bindings/python/src/compatibility/ngraph/opset5/__init__.py +++ b/src/bindings/python/src/compatibility/ngraph/opset5/__init__.py @@ -74,7 +74,7 @@ from ngraph.opset5.ops import log_softmax from ngraph.opset5.ops import loop from ngraph.opset1.ops import lrn from ngraph.opset4.ops import lstm_cell -from ngraph.opset1.ops import lstm_sequence +from ngraph.opset5.ops import lstm_sequence from ngraph.opset1.ops import matmul from ngraph.opset1.ops import max_pool from ngraph.opset1.ops import maximum diff --git a/src/bindings/python/src/compatibility/ngraph/opset6/__init__.py b/src/bindings/python/src/compatibility/ngraph/opset6/__init__.py index f1e175aa6f6..6a15cdfcaa3 100644 --- a/src/bindings/python/src/compatibility/ngraph/opset6/__init__.py +++ b/src/bindings/python/src/compatibility/ngraph/opset6/__init__.py @@ -76,7 +76,7 @@ from ngraph.opset5.ops import log_softmax from ngraph.opset5.ops import loop from ngraph.opset1.ops import lrn from ngraph.opset4.ops import lstm_cell -from ngraph.opset1.ops import lstm_sequence +from ngraph.opset5.ops import lstm_sequence from ngraph.opset1.ops import matmul from ngraph.opset1.ops import max_pool from ngraph.opset1.ops import maximum diff --git a/src/bindings/python/src/compatibility/ngraph/opset7/__init__.py b/src/bindings/python/src/compatibility/ngraph/opset7/__init__.py index 13a39ef7815..ca7be65d95b 100644 --- a/src/bindings/python/src/compatibility/ngraph/opset7/__init__.py +++ b/src/bindings/python/src/compatibility/ngraph/opset7/__init__.py @@ -79,7 +79,7 @@ from ngraph.opset5.ops import log_softmax from ngraph.opset5.ops import loop from ngraph.opset1.ops import lrn from ngraph.opset4.ops import lstm_cell -from ngraph.opset1.ops import lstm_sequence +from ngraph.opset5.ops import lstm_sequence from ngraph.opset1.ops import matmul from ngraph.opset1.ops import max_pool from ngraph.opset1.ops import maximum diff --git a/src/bindings/python/src/compatibility/ngraph/opset8/__init__.py b/src/bindings/python/src/compatibility/ngraph/opset8/__init__.py index fc94b1243b4..d6e225e665f 100644 --- a/src/bindings/python/src/compatibility/ngraph/opset8/__init__.py +++ b/src/bindings/python/src/compatibility/ngraph/opset8/__init__.py @@ -84,7 +84,7 @@ from ngraph.opset5.ops import log_softmax from ngraph.opset5.ops import loop from ngraph.opset1.ops import lrn from ngraph.opset4.ops import lstm_cell -from ngraph.opset1.ops import lstm_sequence +from ngraph.opset5.ops import lstm_sequence from ngraph.opset1.ops import matmul from ngraph.opset8.ops import matrix_nms from ngraph.opset8.ops import max_pool diff --git a/src/bindings/python/src/openvino/runtime/ie_api.py b/src/bindings/python/src/openvino/runtime/ie_api.py index 67bb84ef68a..b43d335463d 100644 --- a/src/bindings/python/src/openvino/runtime/ie_api.py +++ b/src/bindings/python/src/openvino/runtime/ie_api.py @@ -132,7 +132,7 @@ class InferRequest(InferRequestBase): :param inputs: Data to be set on input tensors. :type inputs: Union[Dict[keys, values], List[values]], optional - :param userdata: Any data that will be passed inside callback call. + :param userdata: Any data that will be passed inside the callback. :type userdata: Any """ super().start_async( @@ -164,8 +164,8 @@ class CompiledModel(CompiledModelBase): Blocks all methods of CompiledModel while request is running. Method creates new temporary InferRequest and run inference on it. - It is advised to use dedicated InferRequest class for performance, - optimizing workflows and creating advanced pipelines. + It is advised to use a dedicated InferRequest class for performance, + optimizing workflows, and creating advanced pipelines. The allowed types of keys in the `inputs` dictionary are: @@ -188,7 +188,10 @@ class CompiledModel(CompiledModelBase): ) def __call__(self, inputs: Union[dict, list] = None) -> dict: - """Callable infer wrapper for CompiledModel. Look at `infer_new_request` for reference.""" + """Callable infer wrapper for CompiledModel. + + Take a look at `infer_new_request` for reference. + """ return self.infer_new_request(inputs) @@ -245,7 +248,7 @@ class Core(CoreBase): """Core class represents OpenVINO runtime Core entity. User applications can create several Core class instances, but in this - case the underlying plugins are created multiple times and not shared + case, the underlying plugins are created multiple times and not shared between several Core instances. The recommended way is to have a single Core instance per application. """ diff --git a/src/bindings/python/src/openvino/runtime/opset1/ops.py b/src/bindings/python/src/openvino/runtime/opset1/ops.py index 7507acb2a3f..c05fe5159a2 100644 --- a/src/bindings/python/src/openvino/runtime/opset1/ops.py +++ b/src/bindings/python/src/openvino/runtime/opset1/ops.py @@ -43,7 +43,7 @@ def absolute(node: NodeInput, name: Optional[str] = None) -> Node: :param node: One of: input node, array or scalar. :param name: Optional new name for output node. - returns New node with Abs operation applied on it. + :return: New node with Abs operation applied on it. """ return _get_node_factory_opset1().create("Abs", [node]) @@ -54,7 +54,7 @@ def acos(node: NodeInput, name: Optional[str] = None) -> Node: :param node: One of: input node, array or scalar. :param name: Optional new name for output node. - returns New node with arccos operation applied on it. + :return: New node with arccos operation applied on it. """ return _get_node_factory_opset1().create("Acos", [node]) @@ -78,7 +78,7 @@ def asin(node: NodeInput, name: Optional[str] = None) -> Node: :param node: One of: input node, array or scalar. :param name: Optional new name for output node. - returns New node with arcsin operation applied on it. + :return: New node with arcsin operation applied on it. """ return _get_node_factory_opset1().create("Asin", [node]) @@ -89,7 +89,7 @@ def atan(node: NodeInput, name: Optional[str] = None) -> Node: :param node: One of: input node, array or scalar. :param name: Optional new name for output node. - returns New node with arctan operation applied on it. + :return: New node with arctan operation applied on it. """ return _get_node_factory_opset1().create("Atan", [node]) @@ -120,7 +120,7 @@ def avg_pool( [None, 'same_upper', 'same_lower', 'valid'] :param name: Optional name for the new output node. - returns New node with AvgPool operation applied on its data. + :return: New node with AvgPool operation applied on its data. """ if auto_pad is None: auto_pad = "explicit" @@ -159,7 +159,7 @@ def batch_norm_inference( :param epsilon: The number to be added to the variance to avoid division by zero when normalizing a value. :param name: The optional name of the output node. - returns The new node which performs BatchNormInference. + :return: The new node which performs BatchNormInference. """ inputs = as_nodes(gamma, beta, data, mean, variance) return _get_node_factory_opset1().create("BatchNormInference", inputs, {"epsilon": epsilon}) @@ -190,7 +190,7 @@ def binary_convolution( :param pad_value: Floating-point value used to fill pad area. :param auto_pad: The type of padding. Range of values: explicit, same_upper, same_lower, valid. :param name: The optional new name for output node. - returns New node performing binary convolution operation. + :return: New node performing binary convolution operation. """ return _get_node_factory_opset1().create( "BinaryConvolution", @@ -224,7 +224,7 @@ def broadcast( :param mode: The type of broadcasting that specifies mapping of input tensor axes to output shape axes. Range of values: NUMPY, EXPLICIT. :param name: Optional new name for output node. - returns New node with broadcast shape. + :return: New node with broadcast shape. """ inputs = as_nodes(data, target_shape) if mode.upper() == "EXPLICIT": @@ -247,7 +247,7 @@ def ctc_greedy_decoder( :param sequence_mask: The tensor with sequence masks for each sequence in the batch. :param merge_repeated: The flag for merging repeated labels during the CTC calculation. :param name: Optional name for output node. - returns The new node performing an CTCGreedyDecoder operation on input tensor. + :return: The new node performing an CTCGreedyDecoder operation on input tensor. """ node_inputs = as_nodes(data, sequence_mask) return _get_node_factory_opset1().create( @@ -261,7 +261,7 @@ def ceiling(node: NodeInput, name: Optional[str] = None) -> Node: :param node: The node providing data to ceiling operation. :param name: Optional name for output node. - returns The node performing element-wise ceiling. + :return: The node performing element-wise ceiling. """ return _get_node_factory_opset1().create("Ceiling", [node]) @@ -276,7 +276,7 @@ def clamp( :param min_value: The lower bound of the range. Scalar value. :param max_value: The upper bound of the range. Scalar value. :param name: Optional output node name. - returns The new node performing a clamp operation on its input data element-wise. + :return: The new node performing a clamp operation on its input data element-wise. Performs a clipping operation on an input value between a pair of boundary values. @@ -306,7 +306,7 @@ def concat(nodes: List[NodeInput], axis: int, name: Optional[str] = None) -> Nod :param nodes: The nodes we want concatenate into single new node. :param axis: The axis along which we want to concatenate input nodes. :param name: The optional new name for output node. - returns Return new node that is a concatenation of input nodes. + :return: Return new node that is a concatenation of input nodes. """ return _get_node_factory_opset1().create("Concat", as_nodes(*nodes), {"axis": axis}) @@ -322,7 +322,7 @@ def constant( :param value: One of: array of values or scalar to initialize node with. :param dtype: The data type of provided data. :param name: Optional name for output node. - returns The Constant node initialized with provided data. + :return: The Constant node initialized with provided data. """ return make_constant_node(value, dtype) @@ -336,7 +336,7 @@ def convert( :param data: Node which produces the input tensor. :param destination_type: Provides the target type for the conversion. :param name: Optional name for the output node. - returns New node performing the conversion operation. + :return: New node performing the conversion operation. """ if not isinstance(destination_type, str): destination_type = get_element_type_str(destination_type) @@ -352,7 +352,7 @@ def convert_like(data: NodeInput, like: NodeInput, name: Optional[str] = None) - :param data: Node which produces the input tensor :param like: Node which provides the target type information for the conversion :param name: Optional name for the output node. - returns New node performing the conversion operation. + :return: New node performing the conversion operation. """ return _get_node_factory_opset1().create("ConvertLike", [data, like]) @@ -378,7 +378,7 @@ def convolution( :param dilations: The data batch dilation strides. :param auto_pad: The type of padding. Range of values: explicit, same_upper, same_lower, valid. :param name: The optional new name for output node. - returns New node performing batched convolution operation. + :return: New node performing batched convolution operation. """ return _get_node_factory_opset1().create( "Convolution", @@ -419,7 +419,7 @@ def convolution_backprop_data( in the filter. :param name: The node name. - returns The node object representing ConvolutionBackpropData operation. + :return: The node object representing ConvolutionBackpropData operation. """ spatial_dim_count = len(strides) if pads_begin is None: @@ -456,7 +456,7 @@ def cos(node: NodeInput, name: Optional[str] = None) -> Node: :param node: One of: input node, array or scalar. :param name: Optional new name for output node. - returns New node with cos operation applied on it. + :return: New node with cos operation applied on it. """ return _get_node_factory_opset1().create("Cos", [node]) @@ -467,7 +467,7 @@ def cosh(node: NodeInput, name: Optional[str] = None) -> Node: :param node: One of: input node, array or scalar. :param name: Optional new name for output node. - returns New node with cosh operation applied on it. + :return: New node with cosh operation applied on it. """ return _get_node_factory_opset1().create("Cosh", [node]) @@ -499,7 +499,7 @@ def deformable_convolution( :param deformable_group: The number of groups which deformable values and output should be split into along the channel axis. :param name: The optional new name for output node. - returns New node performing deformable convolution operation. + :return: New node performing deformable convolution operation. """ return _get_node_factory_opset1().create( "DeformableConvolution", @@ -548,7 +548,7 @@ def deformable_psroi_pooling( :param part_size: The number of parts the output tensor spatial dimensions are divided into. :param offsets: Optional node. 4D input blob with transformation values (offsets). :param name: The optional new name for output node. - returns New node performing DeformablePSROIPooling operation. + :return: New node performing DeformablePSROIPooling operation. """ node_inputs = as_nodes(feature_maps, coords) if offsets is not None: @@ -592,7 +592,7 @@ def depth_to_space(node: Node, mode: str, block_size: int = 1, name: str = None) :param block_size: The size of the spatial block of values describing how the tensor's data is to be rearranged. :param name: Optional output node name. - returns The new node performing an DepthToSpace operation on its input tensor. + :return: The new node performing an DepthToSpace operation on its input tensor. """ return _get_node_factory_opset1().create( "DepthToSpace", [node], {"mode": mode, "block_size": block_size}, @@ -618,7 +618,7 @@ def detection_output( :param aux_class_preds: The 2D input tensor with additional class predictions information. :param aux_box_preds: The 2D input tensor with additional box predictions information. :param name: Optional name for the output node. - returns Node representing DetectionOutput operation. + :return: Node representing DetectionOutput operation. Available attributes are: @@ -774,7 +774,7 @@ def divide( :param right_node: The node providing divisor data. :param auto_broadcast: Specifies rules used for auto-broadcasting of input tensors. :param name: Optional name for output node. - returns The node performing element-wise division. + :return: The node performing element-wise division. """ return _get_node_factory_opset1().create( "Divide", [left_node, right_node], {"auto_broadcast": auto_broadcast.upper()} @@ -793,7 +793,7 @@ def elu(data: NodeInput, alpha: NumericType, name: Optional[str] = None) -> Node :param data: Input tensor. One of: input node, array or scalar. :param alpha: Scalar multiplier for negative values. :param name: Optional output node name. - returns The new node performing an ELU operation on its input data element-wise. + :return: The new node performing an ELU operation on its input data element-wise. """ return _get_node_factory_opset1().create("Elu", [as_node(data)], {"alpha": alpha}) @@ -812,7 +812,7 @@ def equal( :param auto_broadcast: The type of broadcasting specifies rules used for auto-broadcasting of input tensors. :param name: The optional name for output new node. - returns The node performing element-wise equality check. + :return: The node performing element-wise equality check. """ return _get_node_factory_opset1().create( "Equal", [left_node, right_node], {"auto_broadcast": auto_broadcast.upper()} @@ -825,7 +825,7 @@ def erf(node: NodeInput, name: Optional[str] = None) -> Node: :param node: The node providing data for operation. :param name: The optional name for new output node. - returns The new node performing element-wise Erf operation. + :return: The new node performing element-wise Erf operation. """ return _get_node_factory_opset1().create("Erf", [node]) @@ -836,7 +836,7 @@ def exp(node: NodeInput, name: Optional[str] = None) -> Node: :param node: The node providing data for operation. :param name: The optional name for new output node. - returns The new node performing natural exponential operation. + :return: The new node performing natural exponential operation. """ return _get_node_factory_opset1().create("Exp", [node]) @@ -862,7 +862,7 @@ def fake_quantize( :param levels: The number of quantization levels. Integer value. :param auto_broadcast: The type of broadcasting specifies rules used for auto-broadcasting of input tensors. - returns New node with quantized value. + :return: New node with quantized value. Input floating point values are quantized into a discrete set of floating point values. @@ -895,7 +895,7 @@ def floor(node: NodeInput, name: Optional[str] = None) -> Node: :param node: The input node providing data. :param name: The optional name for new output node. - returns The node performing element-wise floor operation. + :return: The node performing element-wise floor operation. """ return _get_node_factory_opset1().create("Floor", [node]) @@ -913,7 +913,7 @@ def floor_mod( :param right_node: The second input node for FloorMod operation. :param auto_broadcast: Specifies rules used for auto-broadcasting of input tensors. :param name: Optional name for output node. - returns The node performing element-wise FloorMod operation. + :return: The node performing element-wise FloorMod operation. """ return _get_node_factory_opset1().create( "FloorMod", [left_node, right_node], {"auto_broadcast": auto_broadcast.upper()} @@ -930,7 +930,7 @@ def gather( :param indices: Tensor with indexes to gather. :param axis: The dimension index to gather data from. :param name: Optional name for output node. - returns The new node performing a Gather operation on the data input tensor. + :return: The new node performing a Gather operation on the data input tensor. """ node_inputs = as_nodes(data, indices, axis) return _get_node_factory_opset1().create("Gather", node_inputs) @@ -951,7 +951,7 @@ def gather_tree( :param max_seq_len: The tensor with maximum lengths for each sequence in the batch. :param end_token: The scalar tensor with value of the end marker in a sequence. :param name: Optional name for output node. - returns The new node performing a GatherTree operation. + :return: The new node performing a GatherTree operation. The GatherTree node generates the complete beams from the indices per each step and the parent beam indices. @@ -988,7 +988,7 @@ def greater( :param auto_broadcast: The type of broadcasting specifies rules used for auto-broadcasting of input tensors. :param name: The optional new name for output node. - returns The node performing element-wise check whether left_node is greater than right_node. + :return: The node performing element-wise check whether left_node is greater than right_node. """ return _get_node_factory_opset1().create( "Greater", [left_node, right_node], {"auto_broadcast": auto_broadcast.upper()} @@ -1009,7 +1009,7 @@ def greater_equal( :param auto_broadcast: The type of broadcasting specifies rules used for auto-broadcasting of input tensors. :param name: The optional new name for output node. - returns The node performing element-wise check whether left_node is greater than or equal + :return: The node performing element-wise check whether left_node is greater than or equal right_node. """ return _get_node_factory_opset1().create( @@ -1027,7 +1027,7 @@ def grn(data: Node, bias: float, name: Optional[str] = None) -> Node: :param data: The node with data tensor. :param bias: The bias added to the variance. Scalar value. :param name: Optional output node name. - returns The new node performing a GRN operation on tensor's channels. + :return: The new node performing a GRN operation on tensor's channels. """ return _get_node_factory_opset1().create("GRN", [data], {"bias": bias}) @@ -1062,7 +1062,7 @@ def group_convolution( Ceil(num_dims/2) at the end VALID: No padding :param name: Optional output node name. - returns The new node performing a Group Convolution operation on tensor from input node. + :return: The new node performing a Group Convolution operation on tensor from input node. """ return _get_node_factory_opset1().create( "GroupConvolution", @@ -1113,7 +1113,7 @@ def group_convolution_backprop_data( :param output_padding: The additional amount of paddings added per each spatial axis in the output tensor. :param name: Optional output node name. - returns The new node performing a Group Convolution operation on tensor from input node. + :return: The new node performing a Group Convolution operation on tensor from input node. """ spatial_dim_count = len(strides) if dilations is None: @@ -1150,7 +1150,7 @@ def hard_sigmoid(data: Node, alpha: NodeInput, beta: NodeInput, name: Optional[s :param alpha: A node producing the alpha parameter. :param beta: A node producing the beta parameter :param name: Optional output node name. - returns The new node performing a Hard Sigmoid element-wise on input tensor. + :return: The new node performing a Hard Sigmoid element-wise on input tensor. Hard Sigmoid uses the following logic: @@ -1171,7 +1171,7 @@ def interpolate( :param output_shape: 1D tensor describing output shape for spatial axes. :param attrs: The dictionary containing key, value pairs for attributes. :param name: Optional name for the output node. - returns Node representing interpolation operation. + :return: Node representing interpolation operation. Available attributes are: @@ -1251,7 +1251,7 @@ def less( :param auto_broadcast: The type of broadcasting specifies rules used for auto-broadcasting of input tensors. :param name: The optional new name for output node. - returns The node performing element-wise check whether left_node is less than the right_node. + :return: The node performing element-wise check whether left_node is less than the right_node. """ return _get_node_factory_opset1().create( "Less", [left_node, right_node], {"auto_broadcast": auto_broadcast.upper()} @@ -1272,7 +1272,7 @@ def less_equal( :param auto_broadcast: The type of broadcasting specifies rules used for auto-broadcasting of input tensors. :param name: The optional new name for output node. - returns The node performing element-wise check whether left_node is less than or equal the + :return: The node performing element-wise check whether left_node is less than or equal the right_node. """ return _get_node_factory_opset1().create( @@ -1286,7 +1286,7 @@ def log(node: NodeInput, name: Optional[str] = None) -> Node: :param node: The input node providing data for operation. :param name: The optional new name for output node. - returns The new node performing log operation element-wise. + :return: The new node performing log operation element-wise. """ return _get_node_factory_opset1().create("Log", [node]) @@ -1305,7 +1305,7 @@ def logical_and( :param auto_broadcast: The type of broadcasting that specifies mapping of input tensor axes to output shape axes. Range of values: numpy, explicit. :param name: The optional new name for output node. - returns The node performing logical and operation on input nodes corresponding elements. + :return: The node performing logical and operation on input nodes corresponding elements. """ return _get_node_factory_opset1().create( "LogicalAnd", [left_node, right_node], {"auto_broadcast": auto_broadcast.upper()} @@ -1318,7 +1318,7 @@ def logical_not(node: NodeInput, name: Optional[str] = None) -> Node: :param node: The input node providing data. :param name: The optional new name for output node. - returns The node performing element-wise logical NOT operation with given tensor. + :return: The node performing element-wise logical NOT operation with given tensor. """ return _get_node_factory_opset1().create("LogicalNot", [node]) @@ -1337,7 +1337,7 @@ def logical_or( :param auto_broadcast: The type of broadcasting that specifies mapping of input tensor axes to output shape axes. Range of values: numpy, explicit. :param name: The optional new name for output node. - returns The node performing logical or operation on input nodes corresponding elements. + :return: The node performing logical or operation on input nodes corresponding elements. """ return _get_node_factory_opset1().create( "LogicalOr", [left_node, right_node], {"auto_broadcast": auto_broadcast.upper()} @@ -1358,7 +1358,7 @@ def logical_xor( :param auto_broadcast: The type of broadcasting that specifies mapping of input tensor axes to output shape axes. Range of values: numpy, explicit. :param name: The optional new name for output node. - returns The node performing logical or operation on input nodes corresponding elements. + :return: The node performing logical or operation on input nodes corresponding elements. """ return _get_node_factory_opset1().create( "LogicalXor", [left_node, right_node], {"auto_broadcast": auto_broadcast.upper()} @@ -1383,7 +1383,7 @@ def lrn( :param bias: An offset (usually positive) to avoid dividing by 0. :param size: Width of the 1-D normalization window. :param name: An optional name of the output node. - returns The new node which performs LRN. + :return: The new node which performs LRN. """ attributes = {"alpha": alpha, "beta": beta, "bias": bias, "size": size} return _get_node_factory_opset1().create("LRN", as_nodes(data, axes), attributes) @@ -1419,7 +1419,7 @@ def lstm_cell( :param clip: Specifies bound values [-C, C] for tensor clipping performed before activations. :param name: An optional name of the output node. - returns The new node represents LSTMCell. Node outputs count: 2. + :return: The new node represents LSTMCell. Node outputs count: 2. """ if activations is None: activations = ["sigmoid", "tanh", "tanh"] @@ -1493,7 +1493,7 @@ def lstm_sequence( :param clip: Specifies bound values [-C, C] for tensor clipping performed before activations. :param name: An optional name of the output node. - returns The new node represents LSTMSequence. Node outputs count: 3. + :return: The new node represents LSTMSequence. Node outputs count: 3. """ if activations is None: activations = ["sigmoid", "tanh", "tanh"] @@ -1546,7 +1546,7 @@ def matmul( :param data_b: right-hand side matrix :param transpose_a: should the first matrix be transposed before operation :param transpose_b: should the second matrix be transposed - returns MatMul operation node + :return: MatMul operation node """ return _get_node_factory_opset1().create( "MatMul", as_nodes(data_a, data_b), {"transpose_a": transpose_a, "transpose_b": transpose_b} @@ -1578,7 +1578,7 @@ def max_pool( [None, 'same_upper', 'same_lower', 'valid'] :param name: The optional name for the created output node. - returns The new node performing max pooling operation. + :return: The new node performing max pooling operation. """ if auto_pad is None: auto_pad = "explicit" @@ -1635,7 +1635,7 @@ def mod( :param right_node: The second input node for mod operation. :param auto_broadcast: Specifies rules used for auto-broadcasting of input tensors. :param name: Optional name for output node. - returns The node performing element-wise Mod operation. + :return: The node performing element-wise Mod operation. """ return _get_node_factory_opset1().create( "Mod", [left_node, right_node], {"auto_broadcast": auto_broadcast.upper()} @@ -1683,7 +1683,7 @@ def non_max_suppression( :param box_encoding: Format of boxes data encoding. Range of values: corner or cente. :param sort_result_descending: Flag that specifies whenever it is necessary to sort selected boxes across batches or not. - returns The new node which performs NonMaxSuppression + :return: The new node which performs NonMaxSuppression """ if max_output_boxes_per_class is None: max_output_boxes_per_class = make_constant_node(0, np.int64) @@ -1711,7 +1711,7 @@ def normalize_l2( :param axes: Node indicating axes along which L2 reduction is calculated :param eps: The epsilon added to L2 norm :param eps_mode: how eps is combined with L2 value (`add` or `max`) - returns New node which performs the L2 normalization. + :return: New node which performs the L2 normalization. """ return _get_node_factory_opset1().create( "NormalizeL2", as_nodes(data, axes), {"eps": eps, "mode": eps_mode} @@ -1732,7 +1732,7 @@ def not_equal( :param auto_broadcast: The type of broadcasting specifies rules used for auto-broadcasting of input tensors. :param name: The optional name for output new node. - returns The node performing element-wise inequality check. + :return: The node performing element-wise inequality check. """ return _get_node_factory_opset1().create( "NotEqual", [left_node, right_node], {"auto_broadcast": auto_broadcast.upper()} @@ -1759,7 +1759,7 @@ def one_hot( by indices in input take. :param name: The optional name for new output node. - returns New node performing one-hot operation. + :return: New node performing one-hot operation. """ return _get_node_factory_opset1().create( "OneHot", as_nodes(indices, depth, on_value, off_value), {"axis": axis} @@ -1783,7 +1783,7 @@ def pad( :param pads_end: number of padding elements to be added after the last element. :param pad_mode: "constant", "edge", "reflect" or "symmetric" :param arg_pad_value: value used for padding if pad_mode is "constant" - returns Pad operation node. + :return: Pad operation node. """ input_nodes = as_nodes(arg, pads_begin, pads_end) if arg_pad_value: @@ -1818,7 +1818,7 @@ def power( :param name: The optional name for the new output node. :param auto_broadcast: The type of broadcasting specifies rules used for auto-broadcasting of input tensors. - returns The new node performing element-wise exponentiation operation on input nodes. + :return: The new node performing element-wise exponentiation operation on input nodes. """ return _get_node_factory_opset1().create( "Power", [left_node, right_node], {"auto_broadcast": auto_broadcast.upper()} @@ -1832,7 +1832,7 @@ def prelu(data: NodeInput, slope: NodeInput, name: Optional[str] = None) -> Node :param data: The node with data tensor. :param slope: The node with the multipliers for negative values. :param name: Optional output node name. - returns The new node performing a PRelu operation on tensor's channels. + :return: The new node performing a PRelu operation on tensor's channels. PRelu uses the following logic: @@ -1858,7 +1858,7 @@ def prior_box_clustered( specifies shape of the image for which boxes are generated. :param attrs: The dictionary containing key, value pairs for attributes. :param name: Optional name for the output node. - returns Node representing PriorBoxClustered operation. + :return: Node representing PriorBoxClustered operation. Available attributes are: @@ -1942,7 +1942,7 @@ def prior_box( :param image_shape: Shape of image to which prior boxes are scaled. :param attrs: The dictionary containing key, value pairs for attributes. :param name: Optional name for the output node. - returns Node representing prior box operation. + :return: Node representing prior box operation. Available attributes are: @@ -2062,7 +2062,7 @@ def proposal( :param image_shape: The 1D input tensor with 3 or 4 elements describing image shape. :param attrs: The dictionary containing key, value pairs for attributes. :param name: Optional name for the output node. - returns Node representing Proposal operation. + :return: Node representing Proposal operation. * base_size The size of the anchor to which scale and ratio attributes are applied. Range of values: a positive unsigned integer number @@ -2196,15 +2196,15 @@ def psroi_pooling( ) -> Node: """Return a node which produces a PSROIPooling operation. - :param input: Input feature map {N, C, ...} - :param coords: Coordinates of bounding boxes - :param output_dim: Output channel number - :param group_size: Number of groups to encode position-sensitive scores - :param spatial_scale: Ratio of input feature map over input image size - :param spatial_bins_x: Numbers of bins to divide the input feature maps over - :param spatial_bins_y: Numbers of bins to divide the input feature maps over - :param mode: Mode of pooling - "avg" or "bilinear" - returns PSROIPooling node + :param input: Input feature map `{N, C, ...}`. + :param coords: Coordinates of bounding boxes. + :param output_dim: Output channel number. + :param group_size: Number of groups to encode position-sensitive scores. + :param spatial_scale: Ratio of input feature map over input image size. + :param spatial_bins_x: Numbers of bins to divide the input feature maps over. + :param spatial_bins_y: Numbers of bins to divide the input feature maps over. + :param mode: Mode of pooling - "avg" or "bilinear". + :return: PSROIPooling node """ mode = mode.lower() return _get_node_factory_opset1().create( @@ -2225,11 +2225,11 @@ def psroi_pooling( def range(start: Node, stop: NodeInput, step: NodeInput, name: Optional[str] = None) -> Node: """Return a node which produces the Range operation. - :param start: The start value of the generated range - :param stop: The stop value of the generated range - :param step: The step value for the generated range + :param start: The start value of the generated range. + :param stop: The stop value of the generated range. + :param step: The step value for the generated range. :param name: Optional name for output node. - returns Range node + :return: Range node """ return _get_node_factory_opset1().create("Range", as_nodes(start, stop, step)) @@ -2240,7 +2240,7 @@ def relu(node: NodeInput, name: Optional[str] = None) -> Node: :param node: One of: input node, array or scalar. :param name: The optional output node name. - returns The new node performing relu operation on its input element-wise. + :return: The new node performing relu operation on its input element-wise. """ return _get_node_factory_opset1().create("Relu", [node]) @@ -2253,9 +2253,9 @@ def reduce_logical_and( :param node: The tensor we want to reduce. :param reduction_axes: The axes to eliminate through AND operation. - :param keep_dims: If set to True it holds axes that are used for reduction + :param keep_dims: If set to True it holds axes that are used for reduction. :param name: Optional name for output node. - returns The new node performing reduction operation. + :return: The new node performing reduction operation. """ return _get_node_factory_opset1().create( "ReduceLogicalAnd", as_nodes(node, reduction_axes), {"keep_dims": keep_dims} @@ -2270,9 +2270,9 @@ def reduce_logical_or( :param node: The tensor we want to reduce. :param reduction_axes: The axes to eliminate through OR operation. - :param keep_dims: If set to True it holds axes that are used for reduction + :param keep_dims: If set to True it holds axes that are used for reduction. :param name: Optional name for output node. - returns The new node performing reduction operation. + :return: The new node performing reduction operation. """ return _get_node_factory_opset1().create( "ReduceLogicalOr", as_nodes(node, reduction_axes), {"keep_dims": keep_dims} @@ -2287,7 +2287,7 @@ def reduce_max( :param node: The tensor we want to max-reduce. :param reduction_axes: The axes to eliminate through max operation. - :param keep_dims: If set to True it holds axes that are used for reduction + :param keep_dims: If set to True it holds axes that are used for reduction. :param name: Optional name for output node. """ return _get_node_factory_opset1().create( @@ -2303,9 +2303,9 @@ def reduce_mean( :param node: The tensor we want to mean-reduce. :param reduction_axes: The axes to eliminate through mean operation. - :param keep_dims: If set to True it holds axes that are used for reduction + :param keep_dims: If set to True it holds axes that are used for reduction. :param name: Optional name for output node. - returns The new node performing mean-reduction operation. + :return: The new node performing mean-reduction operation. """ return _get_node_factory_opset1().create( "ReduceMean", as_nodes(node, reduction_axes), {"keep_dims": keep_dims} @@ -2338,7 +2338,7 @@ def reduce_prod( :param reduction_axes: The axes to eliminate through product operation. :param keep_dims: If set to True it holds axes that are used for reduction :param name: Optional name for output node. - returns The new node performing product-reduction operation. + :return: The new node performing product-reduction operation. """ return _get_node_factory_opset1().create( "ReduceProd", as_nodes(node, reduction_axes), {"keep_dims": keep_dims} @@ -2355,7 +2355,7 @@ def reduce_sum( :param reduction_axes: The axes to eliminate through summation. :param keep_dims: If set to True it holds axes that are used for reduction :param name: The optional new name for output node. - returns The new node performing summation along `reduction_axes` element-wise. + :return: The new node performing summation along `reduction_axes` element-wise. """ return _get_node_factory_opset1().create( "ReduceSum", as_nodes(node, reduction_axes), {"keep_dims": keep_dims} @@ -2387,7 +2387,7 @@ def region_yolo( :param end_axis: Axis to end softmax on :param anchors: A flattened list of pairs `[width, height]` that describes prior box sizes :param name: Optional name for output node. - returns RegionYolo node + :return: RegionYolo node """ if anchors is None: anchors = [] @@ -2434,7 +2434,7 @@ def result(data: NodeInput, name: Optional[str] = None) -> Node: """Return a node which represents an output of a graph (Model). :param data: The tensor containing the input data - returns Result node + :return: Result node """ return _get_node_factory_opset1().create("Result", [data]) @@ -2453,7 +2453,7 @@ def reverse_sequence( :param seq_lengths: 1D tensor of integers with sequence lengths in the input tensor. :param batch_axis: index of the batch dimension. :param seq_axis: index of the sequence dimension. - returns ReverseSequence node + :return: ReverseSequence node """ return _get_node_factory_opset1().create( "ReverseSequence", @@ -2479,7 +2479,7 @@ def select( item value is `False`. :param auto_broadcast: Mode specifies rules used for auto-broadcasting of input tensors. :param name: The optional new name for output node. - returns The new node with values selected according to provided arguments. + :return: The new node with values selected according to provided arguments. """ inputs = as_nodes(cond, then_node, else_node) return _get_node_factory_opset1().create( @@ -2499,7 +2499,7 @@ def selu( :param alpha: Alpha coefficient of SELU operation :param lambda_value: Lambda coefficient of SELU operation :param name: The optional output node name. - returns The new node performing relu operation on its input element-wise. + :return: The new node performing relu operation on its input element-wise. """ return _get_node_factory_opset1().create("Selu", as_nodes(data, alpha, lambda_value)) @@ -2509,7 +2509,7 @@ def shape_of(data: NodeInput, name: Optional[str] = None) -> Node: """Return a node which produces a tensor containing the shape of its input data. :param data: The tensor containing the input data. - returns ShapeOf node + :return: ShapeOf node """ return _get_node_factory_opset1().create("ShapeOf", [as_node(data)]) @@ -2519,7 +2519,7 @@ def sigmoid(data: NodeInput, name: Optional[str] = None) -> Node: """Return a node which applies the sigmoid function element-wise. :param data: The tensor containing the input data - returns Sigmoid node + :return: Sigmoid node """ return _get_node_factory_opset1().create("Sigmoid", [data]) @@ -2530,7 +2530,7 @@ def sign(node: NodeInput, name: Optional[str] = None) -> Node: :param node: One of: input node, array or scalar. :param name: The optional new name for output node. - returns The node with mapped elements of the input tensor to -1 (if it is negative), + :return: The node with mapped elements of the input tensor to -1 (if it is negative), 0 (if it is zero), or 1 (if it is positive). """ return _get_node_factory_opset1().create("Sign", [node]) @@ -2542,7 +2542,7 @@ def sin(node: NodeInput, name: Optional[str] = None) -> Node: :param node: One of: input node, array or scalar. :param name: Optional new name for output node. - returns New node with sin operation applied on it. + :return: New node with sin operation applied on it. """ return _get_node_factory_opset1().create("Sin", [node]) @@ -2553,7 +2553,7 @@ def sinh(node: NodeInput, name: Optional[str] = None) -> Node: :param node: One of: input node, array or scalar. :param name: Optional new name for output node. - returns New node with sin operation applied on it. + :return: New node with sin operation applied on it. """ return _get_node_factory_opset1().create("Sinh", [node]) @@ -2564,7 +2564,7 @@ def softmax(data: NodeInput, axis: int, name: Optional[str] = None) -> Node: :param data: The tensor providing input data. :param axis: An axis along which Softmax should be calculated - returns The new node with softmax operation applied on each element. + :return: The new node with softmax operation applied on each element. """ return _get_node_factory_opset1().create("Softmax", [as_node(data)], {"axis": axis}) @@ -2574,7 +2574,7 @@ def space_to_depth(data: Node, mode: str, block_size: int = 1, name: str = None) """Perform SpaceToDepth operation on the input tensor. SpaceToDepth rearranges blocks of spatial data into depth. - The operator returns a copy of the input tensor where values from the height + The operator :return: a copy of the input tensor where values from the height and width dimensions are moved to the depth dimension. :param data: The node with data tensor. @@ -2585,7 +2585,7 @@ def space_to_depth(data: Node, mode: str, block_size: int = 1, name: str = None) :param block_size: The size of the block of values to be moved. Scalar value. :param name: Optional output node name. - returns The new node performing a SpaceToDepth operation on input tensor. + :return: The new node performing a SpaceToDepth operation on input tensor. """ return _get_node_factory_opset1().create( "SpaceToDepth", [data], {"mode": mode, "block_size": block_size}, @@ -2599,7 +2599,7 @@ def split(data: NodeInput, axis: NodeInput, num_splits: int, name: Optional[str] :param data: The input tensor to be split :param axis: Axis along which the input data will be split :param num_splits: Number of the output tensors that should be produced - returns Split node + :return: Split node """ return _get_node_factory_opset1().create( "Split", @@ -2614,7 +2614,7 @@ def sqrt(node: NodeInput, name: Optional[str] = None) -> Node: :param node: One of: input node, array or scalar. :param name: Optional new name for output node. - returns The new node with sqrt operation applied element-wise. + :return: The new node with sqrt operation applied element-wise. """ return _get_node_factory_opset1().create("Sqrt", [node]) @@ -2632,7 +2632,7 @@ def squared_difference( :param auto_broadcast: The type of broadcasting that specifies mapping of input tensor axes to output shape axes. Range of values: numpy, explicit. :param name: Optional new name for output node. - returns The new node performing a squared difference between two tensors. + :return: The new node performing a squared difference between two tensors. """ return _get_node_factory_opset1().create( "SquaredDifference", [x1, x2], {"auto_broadcast": auto_broadcast.upper()} @@ -2647,7 +2647,7 @@ def squeeze(data: NodeInput, axes: NodeInput, name: Optional[str] = None) -> Nod :param axes: List of non-negative integers, indicate the dimensions to squeeze. One of: input node or array. :param name: Optional new name for output node. - returns The new node performing a squeeze operation on input tensor. + :return: The new node performing a squeeze operation on input tensor. Remove single-dimensional entries from the shape of a tensor. Takes a parameter `axes` with a list of axes to squeeze. @@ -2690,7 +2690,7 @@ def strided_slice( :param new_axis_mask: A mask indicating dimensions where '1' should be inserted :param shrink_axis_mask: A mask indicating which dimensions should be deleted :param ellipsis_mask: Indicates positions where missing dimensions should be inserted - returns StridedSlice node + :return: StridedSlice node """ if new_axis_mask is None: new_axis_mask = [] @@ -2725,7 +2725,7 @@ def subtract( :param auto_broadcast: The type of broadcasting that specifies mapping of input tensor axes to output shape axes. Range of values: numpy, explicit. :param name: The optional name for output node. - returns The new output node performing subtraction operation on both tensors element-wise. + :return: The new output node performing subtraction operation on both tensors element-wise. """ return _get_node_factory_opset1().create( "Subtract", [left_node, right_node], {"auto_broadcast": auto_broadcast.upper()} @@ -2738,7 +2738,7 @@ def tan(node: NodeInput, name: Optional[str] = None) -> Node: :param node: One of: input node, array or scalar. :param name: Optional new name for output node. - returns New node with tan operation applied on it. + :return: New node with tan operation applied on it. """ return _get_node_factory_opset1().create("Tan", [node]) diff --git a/src/bindings/python/src/openvino/runtime/opset2/ops.py b/src/bindings/python/src/openvino/runtime/opset2/ops.py index 9d863962c8d..c833e3a31ea 100644 --- a/src/bindings/python/src/openvino/runtime/opset2/ops.py +++ b/src/bindings/python/src/openvino/runtime/opset2/ops.py @@ -54,7 +54,7 @@ def batch_to_space( :param crops_begin: Specifies the amount to crop from the beginning along each axis of `data`. :param crops_end: Specifies the amount to crop from the end along each axis of `data`. :param name: Optional output node name. - returns The new node performing a BatchToSpace operation. + :return: The new node performing a BatchToSpace operation. """ return _get_node_factory_opset2().create( "BatchToSpace", as_nodes(data, block_shape, crops_begin, crops_end) @@ -73,7 +73,7 @@ def gelu(node: NodeInput, name: Optional[str] = None) -> Node: :param node: Input tensor. One of: input node, array or scalar. :param name: Optional output node name. - returns The new node performing a GELU operation on its input data element-wise. + :return: The new node performing a GELU operation on its input data element-wise. """ return _get_node_factory_opset2().create("Gelu", [node]) @@ -96,9 +96,9 @@ def mvn( :param across_channels: Denotes if mean values are shared across channels. :param normalize_variance: Denotes whether to perform variance normalization. :param eps: The number added to the variance to avoid division by zero - when normalizing the value. Scalar value. + when normalizing the value. Scalar value. :param name: Optional output node name. - returns The new node performing a MVN operation on input tensor. + :return: The new node performing a MVN operation on input tensor. """ return _get_node_factory_opset2().create( "MVN", @@ -111,10 +111,10 @@ def mvn( def reorg_yolo(input: Node, stride: List[int], name: Optional[str] = None) -> Node: """Return a node which produces the ReorgYolo operation. - :param input: Input data - :param stride: Stride to reorganize input by + :param input: Input data. + :param stride: Stride to reorganize input by. :param name: Optional name for output node. - returns ReorgYolo node + :return: ReorgYolo node. """ return _get_node_factory_opset2().create("ReorgYolo", [input], {"stride": stride}) @@ -130,12 +130,12 @@ def roi_pooling( ) -> Node: """Return a node which produces an ROIPooling operation. - :param input: Input feature map {N, C, ...} - :param coords: Coordinates of bounding boxes - :param output_size: Height/Width of ROI output features (shape) - :param spatial_scale: Ratio of input feature map over input image size (float) - :param method: Method of pooling - string: "max" or "bilinear" - returns ROIPooling node + :param input: Input feature map `{N, C, ...}`. + :param coords: Coordinates of bounding boxes. + :param output_size: Height/Width of ROI output features (shape). + :param spatial_scale: Ratio of input feature map over input image size (float). + :param method: Method of pooling - string: "max" or "bilinear". + :return: ROIPooling node. """ method = method.lower() return _get_node_factory_opset2().create( @@ -164,7 +164,7 @@ def space_to_batch( :param pads_begin: Specifies the padding for the beginning along each axis of `data`. :param pads_end: Specifies the padding for the ending along each axis of `data`. :param name: Optional output node name. - returns The new node performing a SpaceToBatch operation. + :return: The new node performing a SpaceToBatch operation. """ return _get_node_factory_opset2().create( "SpaceToBatch", as_nodes(data, block_shape, pads_begin, pads_end) diff --git a/src/bindings/python/src/openvino/runtime/opset3/ops.py b/src/bindings/python/src/openvino/runtime/opset3/ops.py index 6afefdf20fb..47f745228ae 100644 --- a/src/bindings/python/src/openvino/runtime/opset3/ops.py +++ b/src/bindings/python/src/openvino/runtime/opset3/ops.py @@ -44,7 +44,7 @@ def assign(new_value: NodeInput, variable_id: str, name: Optional[str] = None) - :param new_value: Node producing a value to be assigned to a variable. :param variable_id: Id of a variable to be updated. :param name: Optional name for output node. - returns Assign node + :return: Assign node """ return _get_node_factory_opset3().create( "Assign", @@ -70,7 +70,7 @@ def broadcast( :param broadcast_spec: The type of broadcasting that specifies mapping of input tensor axes to output shape axes. Range of values: NUMPY, EXPLICIT, BIDIRECTIONAL. :param name: Optional new name for output node. - returns New node with broadcast shape. + :return: New node with broadcast shape. """ inputs = as_nodes(data, target_shape) if broadcast_spec.upper() == "EXPLICIT": @@ -96,7 +96,7 @@ def bucketize( :param with_right_bound: indicates whether bucket includes the right or left edge of interval. default true = includes right edge :param name: Optional name for output node. - returns Bucketize node + :return: Bucketize node """ return _get_node_factory_opset3().create( "Bucketize", @@ -119,7 +119,7 @@ def cum_sum( :param axis: zero dimension tensor specifying axis position along which sum will be performed. :param exclusive: if set to true, the top element is not included :param reverse: if set to true, will perform the sums in reverse direction - returns New node performing the operation + :return: New node performing the operation """ return _get_node_factory_opset3().create( "CumSum", as_nodes(arg, axis), {"exclusive": exclusive, "reverse": reverse} @@ -143,7 +143,7 @@ def embedding_bag_offsets_sum( :param per_sample_weights: Tensor with weights for each sample. :param default_index: Scalar containing default index in embedding table to fill empty bags. :param name: Optional name for output node. - returns The new node which performs EmbeddingBagOffsetsSum + :return: The new node which performs EmbeddingBagOffsetsSum """ inputs = [emb_table, as_node(indices), as_node(offsets)] if per_sample_weights is not None: @@ -171,7 +171,7 @@ def embedding_bag_packed_sum( :param indices: Tensor with indices. :param per_sample_weights: Weights to be multiplied with embedding table. :param name: Optional name for output node. - returns EmbeddingBagPackedSum node + :return: EmbeddingBagPackedSum node """ inputs = [as_node(emb_table), as_node(indices)] if per_sample_weights is not None: @@ -202,7 +202,7 @@ def embedding_segments_sum( :param default_index: Scalar containing default index in embedding table to fill empty bags. :param per_sample_weights: Weights to be multiplied with embedding table. :param name: Optional name for output node. - returns EmbeddingSegmentsSum node + :return: EmbeddingSegmentsSum node """ inputs = [as_node(emb_table), as_node(indices), as_node(segment_ids)] if per_sample_weights is not None: @@ -235,7 +235,7 @@ def extract_image_patches( :param rates: Element seleciton rate for creating a patch. :param auto_pad: Padding type. :param name: Optional name for output node. - returns ExtractImagePatches node + :return: ExtractImagePatches node """ return _get_node_factory_opset3().create( "ExtractImagePatches", @@ -288,7 +288,7 @@ def gru_cell( :param linear_before_reset: Flag denotes if the layer behaves according to the modification of GRUCell described in the formula in the ONNX documentation. :param name: Optional output node name. - returns The new node performing a GRUCell operation on tensor from input node. + :return: The new node performing a GRUCell operation on tensor from input node. """ if activations is None: activations = ["sigmoid", "tanh"] @@ -333,7 +333,7 @@ def non_max_suppression( :param sort_result_descending: Flag that specifies whenever it is necessary to sort selected boxes across batches or not. :param output_type: Output element type. - returns The new node which performs NonMaxSuppression + :return: The new node which performs NonMaxSuppression """ if max_output_boxes_per_class is None: max_output_boxes_per_class = make_constant_node(0, np.int64) @@ -359,7 +359,7 @@ def non_zero(data: NodeInput, output_type: str = "i64", name: Optional[str] = No :param data: Input data. :param output_type: Output tensor type. - returns The new node which performs NonZero + :return: The new node which performs NonZero """ return _get_node_factory_opset3().create( "NonZero", @@ -375,7 +375,7 @@ def read_value(init_value: NodeInput, variable_id: str, name: Optional[str] = No :param init_value: Node producing a value to be returned instead of an unassigned variable. :param variable_id: Id of a variable to be read. :param name: Optional name for output node. - returns ReadValue node + :return: ReadValue node """ return _get_node_factory_opset3().create( "ReadValue", @@ -422,7 +422,7 @@ def rnn_cell( :param clip: The value defining clipping range [-clip, clip] on input of activation functions. :param name: Optional output node name. - returns The new node performing a RNNCell operation on tensor from input node. + :return: The new node performing a RNNCell operation on tensor from input node. """ if activations is None: activations = ["tanh"] @@ -467,7 +467,7 @@ def roi_align( :param spatial_scale: Multiplicative spatial scale factor to translate ROI coordinates. :param mode: Method to perform pooling to produce output feature map elements. - returns The new node which performs ROIAlign + :return: The new node which performs ROIAlign """ inputs = as_nodes(data, rois, batch_indices) attributes = { @@ -494,7 +494,7 @@ def scatter_elements_update( :param indices: The tensor with indexes which will be updated. :param updates: The tensor with update values. :param axis: The axis for scatter. - returns ScatterElementsUpdate node + :return: ScatterElementsUpdate node ScatterElementsUpdate creates a copy of the first input tensor with updated elements specified with second and third input tensors. @@ -523,7 +523,7 @@ def scatter_update( :param indices: The tensor with indexes which will be updated. :param updates: The tensor with update values. :param axis: The axis at which elements will be updated. - returns ScatterUpdate node + :return: ScatterUpdate node """ return _get_node_factory_opset3().create( "ScatterUpdate", @@ -537,7 +537,7 @@ def shape_of(data: NodeInput, output_type: str = "i64", name: Optional[str] = No :param data: The tensor containing the input data. :param output_type: Output element type. - returns ShapeOf node + :return: ShapeOf node """ return _get_node_factory_opset3().create( "ShapeOf", @@ -557,7 +557,7 @@ def shuffle_channels(data: Node, axis: int, group: int, name: Optional[str] = No :param group: The channel dimension specified by the axis parameter should be split into this number of groups. :param name: Optional output node name. - returns The new node performing a permutation on data in the channel dimension + :return: The new node performing a permutation on data in the channel dimension of the input tensor. The operation is the equivalent with the following transformation of the input tensor @@ -617,7 +617,7 @@ def topk( :param mode: Compute TopK largest ('max') or smallest ('min') :param sort: Order of output elements (sort by: 'none', 'index' or 'value') :param index_element_type: Type of output tensor with indices. - returns The new node which performs TopK (both indices and values) + :return: The new node which performs TopK (both indices and values) """ return _get_node_factory_opset3().create( "TopK", diff --git a/src/bindings/python/src/openvino/runtime/opset4/ops.py b/src/bindings/python/src/openvino/runtime/opset4/ops.py index 8ee3ffef925..3c16762e549 100644 --- a/src/bindings/python/src/openvino/runtime/opset4/ops.py +++ b/src/bindings/python/src/openvino/runtime/opset4/ops.py @@ -59,7 +59,7 @@ def ctc_loss( :param preprocess_collapse_repeated: Flag for preprocessing labels before loss calculation. :param ctc_merge_repeated: Flag for merging repeated characters in a potential alignment. :param unique: Flag to find unique elements in a target. - returns The new node which performs CTCLoss + :return: The new node which performs CTCLoss """ if blank_index is not None: inputs = as_nodes(logits, logit_length, labels, label_length, blank_index) @@ -99,7 +99,7 @@ def non_max_suppression( :param sort_result_descending: Flag that specifies whenever it is necessary to sort selected boxes across batches or not. :param output_type: Output element type. - returns The new node which performs NonMaxSuppression + :return: The new node which performs NonMaxSuppression """ if max_output_boxes_per_class is None: max_output_boxes_per_class = make_constant_node(0, np.int64) @@ -123,7 +123,7 @@ def softplus(data: NodeInput, name: Optional[str] = None) -> Node: """Apply SoftPlus operation on each element of input tensor. :param data: The tensor providing input data. - returns The new node with SoftPlus operation applied on each element. + :return: The new node with SoftPlus operation applied on each element. """ return _get_node_factory_opset4().create("SoftPlus", as_nodes(data), {}) @@ -133,7 +133,7 @@ def mish(data: NodeInput, name: Optional[str] = None,) -> Node: """Return a node which performs Mish. :param data: Tensor with input data floating point type. - returns The new node which performs Mish + :return: The new node which performs Mish """ return _get_node_factory_opset4().create("Mish", as_nodes(data), {}) @@ -143,7 +143,7 @@ def hswish(data: NodeInput, name: Optional[str] = None,) -> Node: """Return a node which performs HSwish (hard version of Swish). :param data: Tensor with input data floating point type. - returns The new node which performs HSwish + :return: The new node which performs HSwish """ return _get_node_factory_opset4().create("HSwish", as_nodes(data), {}) @@ -157,7 +157,7 @@ def swish( """Return a node which performing Swish activation function Swish(x, beta=1.0) = x * sigmoid(x * beta)). :param data: Tensor with input data floating point type. - returns The new node which performs Swish + :return: The new node which performs Swish """ if beta is None: beta = make_constant_node(1.0, np.float32) @@ -170,7 +170,7 @@ def acosh(node: NodeInput, name: Optional[str] = None) -> Node: :param node: One of: input node, array or scalar. :param name: Optional new name for output node. - returns New node with arccosh operation applied on it. + :return: New node with arccosh operation applied on it. """ return _get_node_factory_opset4().create("Acosh", [node]) @@ -181,7 +181,7 @@ def asinh(node: NodeInput, name: Optional[str] = None) -> Node: :param node: One of: input node, array or scalar. :param name: Optional new name for output node. - returns New node with arcsinh operation applied on it. + :return: New node with arcsinh operation applied on it. """ return _get_node_factory_opset4().create("Asinh", [node]) @@ -192,7 +192,7 @@ def atanh(node: NodeInput, name: Optional[str] = None) -> Node: :param node: One of: input node, array or scalar. :param name: Optional new name for output node. - returns New node with arctanh operation applied on it. + :return: New node with arctanh operation applied on it. """ return _get_node_factory_opset4().create("Atanh", [node]) @@ -292,7 +292,7 @@ def proposal( } Optional attributes which are absent from dictionary will be set with corresponding default. - returns Node representing Proposal operation. + :return: Node representing Proposal operation. """ requirements = [ ("base_size", True, np.unsignedinteger, is_positive_value), @@ -328,7 +328,7 @@ def reduce_l1( :param reduction_axes: The axes to eliminate through mean operation. :param keep_dims: If set to True it holds axes that are used for reduction :param name: Optional name for output node. - returns The new node performing mean-reduction operation. + :return: The new node performing mean-reduction operation. """ return _get_node_factory_opset4().create( "ReduceL1", as_nodes(node, reduction_axes), {"keep_dims": keep_dims} @@ -345,7 +345,7 @@ def reduce_l2( :param reduction_axes: The axes to eliminate through mean operation. :param keep_dims: If set to True it holds axes that are used for reduction :param name: Optional name for output node. - returns The new node performing mean-reduction operation. + :return: The new node performing mean-reduction operation. """ return _get_node_factory_opset4().create( "ReduceL2", as_nodes(node, reduction_axes), {"keep_dims": keep_dims} @@ -382,7 +382,7 @@ def lstm_cell( :param clip: Specifies bound values [-C, C] for tensor clipping performed before activations. :param name: An optional name of the output node. - returns The new node represents LSTMCell. Node outputs count: 2. + :return: The new node represents LSTMCell. Node outputs count: 2. """ if activations is None: activations = ["sigmoid", "tanh", "tanh"] diff --git a/src/bindings/python/src/openvino/runtime/opset5/__init__.py b/src/bindings/python/src/openvino/runtime/opset5/__init__.py index 74fb610a02b..f18d61e927d 100644 --- a/src/bindings/python/src/openvino/runtime/opset5/__init__.py +++ b/src/bindings/python/src/openvino/runtime/opset5/__init__.py @@ -74,7 +74,7 @@ from openvino.runtime.opset5.ops import log_softmax from openvino.runtime.opset5.ops import loop from openvino.runtime.opset1.ops import lrn from openvino.runtime.opset4.ops import lstm_cell -from openvino.runtime.opset1.ops import lstm_sequence +from openvino.runtime.opset5.ops import lstm_sequence from openvino.runtime.opset1.ops import matmul from openvino.runtime.opset1.ops import max_pool from openvino.runtime.opset1.ops import maximum diff --git a/src/bindings/python/src/openvino/runtime/opset5/ops.py b/src/bindings/python/src/openvino/runtime/opset5/ops.py index b024210f4a8..f02200cf3ae 100644 --- a/src/bindings/python/src/openvino/runtime/opset5/ops.py +++ b/src/bindings/python/src/openvino/runtime/opset5/ops.py @@ -57,7 +57,7 @@ def batch_norm_inference( :param epsilon: The number to be added to the variance to avoid division by zero when normalizing a value. :param name: The optional name of the output node. - @return: The new node which performs BatchNormInference. + :return: The new node which performs BatchNormInference. """ inputs = as_nodes(data, gamma, beta, mean, variance) return _get_node_factory_opset5().create("BatchNormInference", inputs, {"epsilon": epsilon}) @@ -75,7 +75,7 @@ def gather_nd( :param data: N-D tensor with data for gathering :param indices: K-D tensor of tuples with indices by which data is gathered :param batch_dims: Scalar value of batch dimensions - @return: The new node which performs GatherND + :return: The new node which performs GatherND """ inputs = as_nodes(data, indices) @@ -92,7 +92,7 @@ def log_softmax(data: NodeInput, axis: int, name: Optional[str] = None) -> Node: :param data: The tensor providing input data. :param axis: An axis along which LogSoftmax should be calculated - @return: The new node with LogSoftmax operation applied on each element. + :return: The new node with LogSoftmax operation applied on each element. """ return _get_node_factory_opset5().create("LogSoftmax", [as_node(data)], {"axis": axis}) @@ -123,7 +123,7 @@ def non_max_suppression( :param sort_result_descending: Flag that specifies whenever it is necessary to sort selected boxes across batches or not. :param output_type: Output element type. - @return: The new node which performs NonMaxSuppression + :return: The new node which performs NonMaxSuppression """ if max_output_boxes_per_class is None: max_output_boxes_per_class = make_constant_node(0, np.int64) @@ -158,7 +158,7 @@ def round(data: NodeInput, mode: str = "half_to_even", name: Optional[str] = Non integer or rounding in such a way that the result heads away from zero if `mode` attribute is 'half_away_from_zero`. :param name: An optional name of the output node. - @return: The new node with Round operation applied on each element. + :return: The new node with Round operation applied on each element. """ return _get_node_factory_opset5().create("Round", as_nodes(data), {"mode": mode.upper()}) @@ -205,7 +205,7 @@ def lstm_sequence( :param clip: Specifies bound values [-C, C] for tensor clipping performed before activations. :param name: An optional name of the output node. - @return: The new node represents LSTMSequence. Node outputs count: 3. + :return: The new node represents LSTMSequence. Node outputs count: 3. """ if activations is None: activations = ["sigmoid", "tanh", "tanh"] @@ -231,7 +231,7 @@ def hsigmoid(data: NodeInput, name: Optional[str] = None,) -> Node: """Return a node which performs HSigmoid. :param data: Tensor with input data floating point type. - @return: The new node which performs HSigmoid + :return: The new node which performs HSigmoid """ return _get_node_factory_opset5().create("HSigmoid", as_nodes(data), {}) @@ -277,7 +277,7 @@ def gru_sequence( of GRU described in the formula in the ONNX documentation. :param name: An optional name of the output node. - @return: The new node represents GRUSequence. Node outputs count: 2. + :return: The new node represents GRUSequence. Node outputs count: 2. """ if activations is None: activations = ["sigmoid", "tanh"] @@ -337,7 +337,7 @@ def rnn_sequence( :param clip: Specifies bound values [-C, C] for tensor clipping performed before activations. :param name: An optional name of the output node. - @return: The new node represents RNNSequence. Node outputs count: 2. + :return: The new node represents RNNSequence. Node outputs count: 2. """ if activations is None: activations = ["tanh"] diff --git a/src/bindings/python/src/openvino/runtime/opset6/__init__.py b/src/bindings/python/src/openvino/runtime/opset6/__init__.py index 3153669ea09..9721ad311b0 100644 --- a/src/bindings/python/src/openvino/runtime/opset6/__init__.py +++ b/src/bindings/python/src/openvino/runtime/opset6/__init__.py @@ -76,7 +76,7 @@ from openvino.runtime.opset5.ops import log_softmax from openvino.runtime.opset5.ops import loop from openvino.runtime.opset1.ops import lrn from openvino.runtime.opset4.ops import lstm_cell -from openvino.runtime.opset1.ops import lstm_sequence +from openvino.runtime.opset5.ops import lstm_sequence from openvino.runtime.opset1.ops import matmul from openvino.runtime.opset1.ops import max_pool from openvino.runtime.opset1.ops import maximum diff --git a/src/bindings/python/src/openvino/runtime/opset6/ops.py b/src/bindings/python/src/openvino/runtime/opset6/ops.py index 08d62ea1b3b..45b295a216e 100644 --- a/src/bindings/python/src/openvino/runtime/opset6/ops.py +++ b/src/bindings/python/src/openvino/runtime/opset6/ops.py @@ -53,7 +53,7 @@ def ctc_greedy_decoder_seq_len( :param sequence_length: Input 1D tensor with sequence length. Shape: [batch_size] :param blank_index: Scalar or 1D tensor with specifies the class index to use for the blank class. Optional parameter. Default value is num_classes-1. - @return: The new node which performs CTCGreedyDecoderSeqLen. + :return: The new node which performs CTCGreedyDecoderSeqLen. """ if blank_index is not None: inputs = as_nodes(data, sequence_length, blank_index) @@ -81,7 +81,7 @@ def gather_elements( :param data: N-D tensor with data for gathering :param indices: N-D tensor with indices by which data is gathered :param axis: axis along which elements are gathered - @return: The new node which performs GatherElements + :return: The new node which performs GatherElements """ inputs = as_nodes(data, indices) @@ -110,7 +110,7 @@ def mvn( when normalizing the value. Scalar value. :param eps_mode: how eps is applied (`inside_sqrt` or `outside_sqrt`) :param name: Optional output node name. - returns The new node performing a MVN operation on input tensor. + :return: The new node performing a MVN operation on input tensor. """ inputs = as_nodes(data, axes) @@ -130,7 +130,7 @@ def assign(new_value: NodeInput, variable_id: str, name: Optional[str] = None) - :param new_value: Node producing a value to be assigned to a variable. :param variable_id: Id of a variable to be updated. :param name: Optional name for output node. - returns Assign node + :return: Assign node """ return _get_node_factory_opset6().create( "Assign", @@ -146,7 +146,7 @@ def read_value(init_value: NodeInput, variable_id: str, name: Optional[str] = No :param init_value: Node producing a value to be returned instead of an unassigned variable. :param variable_id: Id of a variable to be read. :param name: Optional name for output node. - returns ReadValue node + :return: ReadValue node """ return _get_node_factory_opset6().create( "ReadValue", diff --git a/src/bindings/python/src/openvino/runtime/opset7/__init__.py b/src/bindings/python/src/openvino/runtime/opset7/__init__.py index 5c794e986ef..ec91b1b0f74 100644 --- a/src/bindings/python/src/openvino/runtime/opset7/__init__.py +++ b/src/bindings/python/src/openvino/runtime/opset7/__init__.py @@ -79,7 +79,7 @@ from openvino.runtime.opset5.ops import log_softmax from openvino.runtime.opset5.ops import loop from openvino.runtime.opset1.ops import lrn from openvino.runtime.opset4.ops import lstm_cell -from openvino.runtime.opset1.ops import lstm_sequence +from openvino.runtime.opset5.ops import lstm_sequence from openvino.runtime.opset1.ops import matmul from openvino.runtime.opset1.ops import max_pool from openvino.runtime.opset1.ops import maximum diff --git a/src/bindings/python/src/openvino/runtime/opset7/ops.py b/src/bindings/python/src/openvino/runtime/opset7/ops.py index b07772fb572..f5ee112beab 100644 --- a/src/bindings/python/src/openvino/runtime/opset7/ops.py +++ b/src/bindings/python/src/openvino/runtime/opset7/ops.py @@ -46,7 +46,7 @@ def einsum( :param inputs: The list of input nodes :param equation: Einsum equation - @return: The new node performing Einsum operation on the inputs + :return: The new node performing Einsum operation on the inputs """ attributes = { "equation": equation @@ -66,7 +66,7 @@ def gelu( :param data: The node with data tensor. :param approximation_mode: defines which approximation to use ('tanh' or 'erf') :param name: Optional output node name. - returns The new node performing a Gelu activation with the input tensor. + :return: The new node performing a Gelu activation with the input tensor. """ inputs = as_nodes(data) @@ -88,7 +88,7 @@ def roll( :param data: The node with data tensor. :param shift: The node with the tensor with numbers of places by which elements are shifted. :param axes: The node with the tensor with axes along which elements are shifted. - returns The new node performing a Roll operation on the input tensor. + :return: The new node performing a Roll operation on the input tensor. """ inputs = as_nodes(data, shift, axes) @@ -108,7 +108,7 @@ def gather( :param indices: N-D tensor with indices by which data is gathered :param axis: axis along which elements are gathered :param batch_dims: number of batch dimensions - @return: The new node which performs Gather + :return: The new node which performs Gather """ inputs = as_nodes(data, indices, axis) attributes = { @@ -127,7 +127,7 @@ def dft( :param data: Tensor with transformed data. :param axes: Tensor with axes to transform. :param signal_size: Tensor specifying signal size with respect to axes from the input 'axes'. - @return: The new node which performs DFT operation on the input data tensor. + :return: The new node which performs DFT operation on the input data tensor. """ if signal_size is None: inputs = as_nodes(data, axes) @@ -148,7 +148,7 @@ def idft( :param data: Tensor with transformed data. :param axes: Tensor with axes to transform. :param signal_size: Tensor specifying signal size with respect to axes from the input 'axes'. - @return: The new node which performs IDFT operation on the input data tensor. + :return: The new node which performs IDFT operation on the input data tensor. """ if signal_size is None: inputs = as_nodes(data, axes) diff --git a/src/bindings/python/src/openvino/runtime/opset8/__init__.py b/src/bindings/python/src/openvino/runtime/opset8/__init__.py index cce21756f76..9fe98e1a76a 100644 --- a/src/bindings/python/src/openvino/runtime/opset8/__init__.py +++ b/src/bindings/python/src/openvino/runtime/opset8/__init__.py @@ -84,7 +84,7 @@ from openvino.runtime.opset5.ops import log_softmax from openvino.runtime.opset5.ops import loop from openvino.runtime.opset1.ops import lrn from openvino.runtime.opset4.ops import lstm_cell -from openvino.runtime.opset1.ops import lstm_sequence +from openvino.runtime.opset5.ops import lstm_sequence from openvino.runtime.opset1.ops import matmul from openvino.runtime.opset8.ops import matrix_nms from openvino.runtime.opset8.ops import max_pool diff --git a/src/bindings/python/src/openvino/runtime/opset8/ops.py b/src/bindings/python/src/openvino/runtime/opset8/ops.py index 0a809f00bf5..0aa381592fb 100644 --- a/src/bindings/python/src/openvino/runtime/opset8/ops.py +++ b/src/bindings/python/src/openvino/runtime/opset8/ops.py @@ -62,7 +62,7 @@ def deformable_convolution( :param bilinear_interpolation_pad: The flag that determines the mode of bilinear interpolation execution. :param name: The optional new name for output node. - returns New node performing deformable convolution operation. + :return: New node performing deformable convolution operation. """ if mask is None: inputs = as_nodes(data, offsets, filters) @@ -94,7 +94,7 @@ def adaptive_avg_pool( :param data: The list of input nodes :param output_shape: the shape of spatial dimentions after operation - @return: The new node performing AdaptiveAvgPool operation on the data + :return: The new node performing AdaptiveAvgPool operation on the data """ inputs = as_nodes(data, output_shape) return _get_node_factory_opset8().create("AdaptiveAvgPool", inputs) @@ -111,7 +111,7 @@ def adaptive_max_pool( :param data: The list of input nodes :param output_shape: the shape of spatial dimentions after operation :param index_element_type: Type of indices output. - @return: The new node performing AdaptiveMaxPool operation on the data + :return: The new node performing AdaptiveMaxPool operation on the data """ inputs = as_nodes(data, output_shape) @@ -158,7 +158,7 @@ def multiclass_nms( :param background_class: Specifies the background class id, -1 meaning to keep all classes :param nms_eta: Specifies eta parameter for adpative NMS, in close range [0, 1.0] :param normalized: Specifies whether boxes are normalized or not - @return: The new node which performs MuticlassNms + :return: The new node which performs MuticlassNms """ inputs = as_nodes(boxes, scores) @@ -218,7 +218,7 @@ def matrix_nms( :param post_threshold: Specifies threshold to filter out boxes with low confidence score after decaying :param normalized: Specifies whether boxes are normalized or not - @return: The new node which performs MatrixNms + :return: The new node which performs MatrixNms """ inputs = as_nodes(boxes, scores) @@ -253,7 +253,7 @@ def gather( indicate reverse indexing from the end :param axis: axis along which elements are gathered :param batch_dims: number of batch dimensions - @return: The new node which performs Gather + :return: The new node which performs Gather """ inputs = as_nodes(data, indices, axis) attributes = { @@ -296,7 +296,7 @@ def max_pool( starting at the provided axis. Defaults to 0. :param name: The optional name for the created output node. - returns The new node performing max pooling operation. + :return: The new node performing max pooling operation. """ if auto_pad is None: auto_pad = "explicit" @@ -335,7 +335,7 @@ def random_uniform( 'i64', 'i32', 'f64', 'f32', 'f16', 'bf16'. :param global_seed: Specifies global seed value. Required to be a positive integer or 0. :param op_seed: Specifies operational seed value. Required to be a positive integer or 0. - returns The new node which performs generation of random values from uniform distribution. + :return: The new node which performs generation of random values from uniform distribution. """ inputs = as_nodes(output_shape, min_val, max_val) @@ -370,7 +370,7 @@ def slice( :param step: The node providing step values. :param axes: The optional node providing axes to slice, default [0, 1, ..., len(start)-1]. :param name: The optional name for the created output node. - returns The new node performing Slice operation. + :return: The new node performing Slice operation. """ if axes is None: inputs = as_nodes(data, start, stop, step) @@ -392,7 +392,7 @@ def gather_nd( :param data: N-D tensor with data for gathering :param indices: K-D tensor of tuples with indices by which data is gathered :param batch_dims: Scalar value of batch dimensions - @return: The new node which performs GatherND + :return: The new node which performs GatherND """ inputs = as_nodes(data, indices) @@ -413,7 +413,7 @@ def prior_box( :param image_shape: Shape of image to which prior boxes are scaled. :param attrs: The dictionary containing key, value pairs for attributes. :param name: Optional name for the output node. - returns Node representing prior box operation. + :return: Node representing prior box operation. Available attributes are: * min_size The minimum box size (in pixels). Range of values: positive floating point numbers @@ -524,7 +524,7 @@ def i420_to_bgr( :param arg_u: The node providing U plane data. Required for separate planes. :param arg_v: The node providing V plane data. Required for separate planes. :param name: The optional name for the created output node. - returns The new node performing I420toBGR operation. + :return: The new node performing I420toBGR operation. """ if arg_u is None and arg_v is None: inputs = as_nodes(arg) @@ -551,7 +551,7 @@ def i420_to_rgb( :param arg_u: The node providing U plane data. Required for separate planes. :param arg_v: The node providing V plane data. Required for separate planes. :param name: The optional name for the created output node. - returns The new node performing I420toRGB operation. + :return: The new node performing I420toRGB operation. """ if arg_u is None and arg_v is None: inputs = as_nodes(arg) @@ -576,7 +576,7 @@ def nv12_to_bgr( :param arg: The node providing single or Y plane data. :param arg_uv: The node providing UV plane data. Required for separate planes. :param name: The optional name for the created output node. - returns The new node performing NV12toBGR operation. + :return: The new node performing NV12toBGR operation. """ if arg_uv is None: inputs = as_nodes(arg) @@ -597,7 +597,7 @@ def nv12_to_rgb( :param arg: The node providing single or Y plane data. :param arg_uv: The node providing UV plane data. Required for separate planes. :param name: The optional name for the created output node. - returns The new node performing NV12toRGB operation. + :return: The new node performing NV12toRGB operation. """ if arg_uv is None: inputs = as_nodes(arg) @@ -626,7 +626,7 @@ def detection_output( :param aux_class_preds: The 2D input tensor with additional class predictions information. :param aux_box_preds: The 2D input tensor with additional box predictions information. :param name: Optional name for the output node. - returns Node representing DetectionOutput operation. + :return: Node representing DetectionOutput operation. Available attributes are: * background_label_id The background label id. Range of values: integer value @@ -751,6 +751,6 @@ def softmax(data: NodeInput, axis: int, name: Optional[str] = None) -> Node: :param data: The tensor providing input data. :param axis: An axis along which Softmax should be calculated. Can be positive or negative. :param name: Optional name for the node. - returns The new node with softmax operation applied on each element. + :return: The new node with softmax operation applied on each element. """ return _get_node_factory_opset8().create("Softmax", [as_node(data)], {"axis": axis}) diff --git a/src/bindings/python/src/pyopenvino/CMakeLists.txt b/src/bindings/python/src/pyopenvino/CMakeLists.txt index 7e645628278..c4c14077828 100644 --- a/src/bindings/python/src/pyopenvino/CMakeLists.txt +++ b/src/bindings/python/src/pyopenvino/CMakeLists.txt @@ -78,6 +78,8 @@ endif() target_include_directories(${PROJECT_NAME} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/..") target_link_libraries(${PROJECT_NAME} PRIVATE openvino::runtime ${OFFLINE_TRANSFORMATIONS_LIB}) +addVersionDefines(pyopenvino.cpp CI_BUILD_NUMBER) + # perform copy if(OpenVINO_SOURCE_DIR) add_custom_command(TARGET ${PROJECT_NAME} diff --git a/src/bindings/python/src/pyopenvino/core/async_infer_queue.cpp b/src/bindings/python/src/pyopenvino/core/async_infer_queue.cpp index 6d69aa81c6e..c6438228ea1 100644 --- a/src/bindings/python/src/pyopenvino/core/async_infer_queue.cpp +++ b/src/bindings/python/src/pyopenvino/core/async_infer_queue.cpp @@ -202,7 +202,7 @@ void regclass_AsyncInferQueue(py::module m) { py::arg("inputs"), py::arg("userdata"), R"( - Run asynchronous inference using next available InferRequest. + Run asynchronous inference using the next available InferRequest. This function releases the GIL, so another Python thread can work while this function runs in the background. @@ -262,8 +262,8 @@ void regclass_AsyncInferQueue(py::module m) { }, R"( Sets unified callback on all InferRequests from queue's pool. - Signature of such function should have two arguments, where - first one is InferRequest object and second one is userdata + The signature of such function should have two arguments, where + the first one is InferRequest object and the second one is userdata connected to InferRequest from the AsyncInferQueue's pool. .. code-block:: python diff --git a/src/bindings/python/src/pyopenvino/core/common.cpp b/src/bindings/python/src/pyopenvino/core/common.cpp index 33855935143..3ff7a32d465 100644 --- a/src/bindings/python/src/pyopenvino/core/common.cpp +++ b/src/bindings/python/src/pyopenvino/core/common.cpp @@ -6,6 +6,8 @@ #include +#include "openvino/util/common_util.hpp" + #define C_CONTIGUOUS py::detail::npy_api::constants::NPY_ARRAY_C_CONTIGUOUS_ namespace Common { @@ -88,6 +90,108 @@ ov::Tensor tensor_from_numpy(py::array& array, bool shared_memory) { return tensor; } +ov::PartialShape partial_shape_from_list(const py::list& shape) { + using value_type = ov::Dimension::value_type; + ov::PartialShape pshape; + for (py::handle dim : shape) { + if (py::isinstance(dim)) { + pshape.insert(pshape.end(), ov::Dimension(dim.cast())); + } else if (py::isinstance(dim)) { + pshape.insert(pshape.end(), Common::dimension_from_str(dim.cast())); + } else if (py::isinstance(dim)) { + pshape.insert(pshape.end(), dim.cast()); + } else if (py::isinstance(dim) || py::isinstance(dim)) { + py::list bounded_dim = dim.cast(); + if (bounded_dim.size() != 2) { + throw py::type_error("Two elements are expected in tuple(lower, upper) for dynamic dimension, but " + + std::to_string(bounded_dim.size()) + " elements were given."); + } + if (!(py::isinstance(bounded_dim[0]) && py::isinstance(bounded_dim[1]))) { + throw py::type_error("Incorrect pair of types (" + std::string(bounded_dim[0].get_type().str()) + ", " + + std::string(bounded_dim[1].get_type().str()) + + ") for dynamic dimension, ints are expected."); + } + pshape.insert(pshape.end(), + ov::Dimension(bounded_dim[0].cast(), bounded_dim[1].cast())); + } else { + throw py::type_error("Incorrect type " + std::string(dim.get_type().str()) + + " for dimension. Expected types are: " + "int, str, openvino.runtime.Dimension, list/tuple with lower and upper values for " + "dynamic dimension."); + } + } + return pshape; +} + +bool check_all_digits(const std::string& value) { + auto val = ov::util::trim(value); + for (const auto& c : val) { + if (!std::isdigit(c) || c == '-') { + return false; + } + } + return true; +} + +template +T stringToType(const std::string& valStr) { + T ret{0}; + std::istringstream ss(valStr); + if (!ss.eof()) { + ss >> ret; + } + return ret; +} + +ov::Dimension dimension_from_str(const std::string& value) { + using value_type = ov::Dimension::value_type; + auto val = ov::util::trim(value); + if (val == "?" || val == "-1") { + return {-1}; + } + if (val.find("..") == std::string::npos) { + OPENVINO_ASSERT(Common::check_all_digits(val), "Cannot parse dimension: \"", val, "\""); + return {Common::stringToType(val)}; + } + + std::string min_value_str = val.substr(0, val.find("..")); + OPENVINO_ASSERT(Common::check_all_digits(min_value_str), "Cannot parse min bound: \"", min_value_str, "\""); + + value_type min_value; + if (min_value_str.empty()) { + min_value = 0; + } else { + min_value = Common::stringToType(min_value_str); + } + + std::string max_value_str = val.substr(val.find("..") + 2); + value_type max_value; + if (max_value_str.empty()) { + max_value = -1; + } else { + max_value = Common::stringToType(max_value_str); + } + + OPENVINO_ASSERT(Common::check_all_digits(max_value_str), "Cannot parse max bound: \"", max_value_str, "\""); + + return {min_value, max_value}; +} + +ov::PartialShape partial_shape_from_str(const std::string& value) { + auto val = ov::util::trim(value); + if (val == "...") { + return ov::PartialShape::dynamic(); + } + ov::PartialShape res; + std::stringstream ss(val); + std::string field; + while (getline(ss, field, ',')) { + OPENVINO_ASSERT(!field.empty(), "Cannot get vector of dimensions! \"", val, "\" is incorrect"); + res.insert(res.end(), Common::dimension_from_str(field)); + } + return res; +} + py::array as_contiguous(py::array& array, ov::element::Type type) { switch (type) { // floating diff --git a/src/bindings/python/src/pyopenvino/core/common.hpp b/src/bindings/python/src/pyopenvino/core/common.hpp index 12108096116..8a3199643b2 100644 --- a/src/bindings/python/src/pyopenvino/core/common.hpp +++ b/src/bindings/python/src/pyopenvino/core/common.hpp @@ -33,6 +33,12 @@ ov::Tensor tensor_from_pointer(py::array& array, const ov::Shape& shape); ov::Tensor tensor_from_numpy(py::array& array, bool shared_memory); +ov::PartialShape partial_shape_from_list(const py::list& shape); + +ov::PartialShape partial_shape_from_str(const std::string& value); + +ov::Dimension dimension_from_str(const std::string& value); + py::array as_contiguous(py::array& array, ov::element::Type type); const ov::Tensor& cast_to_tensor(const py::handle& tensor); diff --git a/src/bindings/python/src/pyopenvino/core/compiled_model.cpp b/src/bindings/python/src/pyopenvino/core/compiled_model.cpp index 3203000b0e2..768768a1872 100644 --- a/src/bindings/python/src/pyopenvino/core/compiled_model.cpp +++ b/src/bindings/python/src/pyopenvino/core/compiled_model.cpp @@ -53,11 +53,11 @@ void regclass_CompiledModel(py::module m) { py::arg("inputs"), R"( Infers specified input(s) in synchronous mode. - Blocks all methods of CompiledModel while request is running. + Blocks all methods of CompiledModel while the request is running. Method creates new temporary InferRequest and run inference on it. - It is advised to use dedicated InferRequest class for performance, - optimizing workflows and creating advanced pipelines. + It is advised to use a dedicated InferRequest class for performance, + optimizing workflows, and creating advanced pipelines. :param inputs: Data to set on input tensors. :type inputs: Dict[Union[int, str, openvino.runtime.ConstOutput], openvino.runtime.Tensor] @@ -108,10 +108,10 @@ void regclass_CompiledModel(py::module m) { R"( Exports the compiled model to bytes/output stream. - Advanced version of `export_model`. It utilizes, streams from standard + Advanced version of `export_model`. It utilizes, streams from the standard Python library `io`. - Function performs flushing of the stream, writes to it and then rewinds + Function performs flushing of the stream, writes to it, and then rewinds the stream to the beginning (using seek(0)). :param model_stream: A stream object to which the model will be serialized. @@ -149,15 +149,6 @@ void regclass_CompiledModel(py::module m) { :rtype: None )"); - // todo: remove after Accuracy Checker migration to set/get_property API - cls.def( - "get_config", - [](ov::CompiledModel& self, const std::string& name) -> py::object { - PyErr_WarnEx(PyExc_DeprecationWarning, "get_config() is deprecated, use get_property() instead.", 1); - return Common::from_ov_any(self.get_property(name)).as(); - }, - py::arg("name")); - cls.def( "get_property", [](ov::CompiledModel& self, const std::string& name) -> py::object { @@ -172,26 +163,17 @@ void regclass_CompiledModel(py::module m) { :rtype: Any )"); - // todo: remove after Accuracy Checker migration to set/get_property API - cls.def( - "get_metric", - [](ov::CompiledModel& self, const std::string& name) -> py::object { - PyErr_WarnEx(PyExc_DeprecationWarning, "get_metric() is deprecated, use get_property() instead.", 1); - return Common::from_ov_any(self.get_property(name)).as(); - }, - py::arg("name")); - cls.def("get_runtime_model", &ov::CompiledModel::get_runtime_model, R"( Gets runtime model information from a device. - This object (returned model) represents the internal device specific model - which is optimized for particular accelerator. It contains device specific nodes, - runtime information and can be used only to understand how the source model - is optimized and which kernels, element types and layouts are selected. + This object (returned model) represents the internal device-specific model + which is optimized for the particular accelerator. It contains device-specific nodes, + runtime information, and can be used only to understand how the source model + is optimized and which kernels, element types, and layouts are selected. - :return: Model containing Executable Graph information. + :return: Model, containing Executable Graph information. :rtype: openvino.runtime.Model )"); @@ -219,7 +201,7 @@ void regclass_CompiledModel(py::module m) { py::arg("index"), R"( Gets input of a compiled model identified by an index. - If an input with given index is not found, this method throws an exception. + If the input with given index is not found, this method throws an exception. :param index: An input index. :type index: int @@ -232,9 +214,9 @@ void regclass_CompiledModel(py::module m) { py::arg("tensor_name"), R"( Gets input of a compiled model identified by a tensor_name. - If an input with given tensor name is not found, this method throws an exception. + If the input with given tensor name is not found, this method throws an exception. - :param tensor_name: An input tensor's name. + :param tensor_name: An input tensor name. :type tensor_name: str :return: A compiled model input. :rtype: openvino.runtime.ConstOutput @@ -253,7 +235,7 @@ void regclass_CompiledModel(py::module m) { (ov::Output(ov::CompiledModel::*)() const) & ov::CompiledModel::output, R"( Gets a single output of a compiled model. - If a model has more than one output, this method throws an exception. + If the model has more than one output, this method throws an exception. :return: A compiled model output. :rtype: openvino.runtime.ConstOutput @@ -264,7 +246,7 @@ void regclass_CompiledModel(py::module m) { py::arg("index"), R"( Gets output of a compiled model identified by an index. - If an output with given index is not found, this method throws an exception. + If the output with given index is not found, this method throws an exception. :param index: An output index. :type index: int @@ -277,9 +259,9 @@ void regclass_CompiledModel(py::module m) { py::arg("tensor_name"), R"( Gets output of a compiled model identified by a tensor_name. - If an output with given tensor name is not found, this method throws an exception. + If the output with given tensor name is not found, this method throws an exception. - :param tensor_name: An output tensor's name. + :param tensor_name: An output tensor name. :type tensor_name: str :return: A compiled model output. :rtype: openvino.runtime.ConstOutput diff --git a/src/bindings/python/src/pyopenvino/core/core.cpp b/src/bindings/python/src/pyopenvino/core/core.cpp index 722967f91cc..7163bbe129b 100644 --- a/src/bindings/python/src/pyopenvino/core/core.cpp +++ b/src/bindings/python/src/pyopenvino/core/core.cpp @@ -27,24 +27,11 @@ void regclass_Core(py::module m) { py::class_> cls(m, "Core"); cls.doc() = "openvino.runtime.Core class represents OpenVINO runtime Core entity. User applications can create several " - "Core class instances, but in this case the underlying plugins are created multiple times and not shared " + "Core class instances, but in this case, the underlying plugins are created multiple times and not shared " "between several Core instances. The recommended way is to have a single Core instance per application."; cls.def(py::init(), py::arg("xml_config_file") = ""); - // todo: remove after Accuracy Checker migration to set/get_property API - cls.def( - "set_config", - [](ov::Core& self, const std::map& config, const std::string& device_name) { - PyErr_WarnEx(PyExc_DeprecationWarning, "set_config() is deprecated, use set_property() instead.", 1); - self.set_property(device_name, {config.begin(), config.end()}); - }, - py::arg("device_name") = "", - py::arg("properties"), - R"( - Sets properties for the device. - )"); - cls.def( "set_property", [](ov::Core& self, const std::map& properties) { @@ -95,12 +82,12 @@ void regclass_Core(py::module m) { py::arg("config") = py::dict(), R"( Creates a compiled model from a source model object. - Users can create as many compiled models as they need and use them simultaneously + Users can create as many compiled models as they need, and use them simultaneously (up to the limitation of the hardware resources). :param model: Model acquired from read_model function. :type model: openvino.runtime.Model - :param device_name: Name of the device to load the model to. + :param device_name: Name of the device which will load the model. :type device_name: str :param properties: Optional dict of pairs: (property name, property value) relevant only for this load operation. :type properties: dict @@ -119,7 +106,7 @@ void regclass_Core(py::module m) { py::arg("config") = py::dict(), R"( Creates and loads a compiled model from a source model to the default OpenVINO device - selected by AUTO plugin. Users can create as many compiled models as they need and use + selected by AUTO plugin. Users can create as many compiled models as they need, and use them simultaneously (up to the limitation of the hardware resources). :param model: Model acquired from read_model function. @@ -229,8 +216,8 @@ void regclass_Core(py::module m) { :param model: A path to a model in IR / ONNX / PDPD format. :type model: str :param weights: A path to a data file For IR format (*.bin): if path is empty, - will try to read bin file with the same name as xml and if bin - file with the same name was not found, will load IR without weights. + it tries to read a bin file with the same name as xml and if the bin + file with the same name was not found, loads IR without weights. For ONNX format (*.onnx): weights parameter is not used. For PDPD format (*.pdmodel) weights parameter is not used. :type weights: str @@ -268,9 +255,8 @@ void regclass_Core(py::module m) { :param model: A string with model in IR / ONNX / PDPD format. :type model: str :param weights: A path to a data file For IR format (*.bin): if path is empty, - will try to read bin file with the same name as xml and if bin - file with the same name was not found, will load IR without weights. - For ONNX format (*.onnx): weights parameter is not used. + it tries to read a bin file with the same name as xml and if the bin + file with the same name was not found, loads IR without weights. For ONNX format (*.onnx): weights parameter is not used. For PDPD format (*.pdmodel) weights parameter is not used. :type weights: str :return: A model. @@ -293,10 +279,10 @@ void regclass_Core(py::module m) { R"( Imports a compiled model from a previously exported one. - :param model_stream: Input stream containing a model previously exported using export_model method. + :param model_stream: Input stream, containing a model previously exported, using export_model method. :type model_stream: bytes - :param device_name: Name of device to import compiled model for. - Note, if device_name device was not used to compile the original mode, an exception is thrown. + :param device_name: Name of device to which compiled model is imported. + Note: if device_name is not used to compile the original model, an exception is thrown. :type device_name: str :param properties: Optional map of pairs: (property name, property value) relevant only for this load operation. :type properties: dict, optional @@ -345,10 +331,10 @@ void regclass_Core(py::module m) { Python library `io`. - :param model_stream: Input stream containing a model previously exported using export_model method. + :param model_stream: Input stream, containing a model previously exported, using export_model method. :type model_stream: io.BytesIO - :param device_name: Name of device to import compiled model for. - Note, if device_name device was not used to compile the original mode, an exception is thrown. + :param device_name: Name of device to which compiled model is imported. + Note: if device_name is not used to compile the original model, an exception is thrown. :type device_name: str :param properties: Optional map of pairs: (property name, property value) relevant only for this load operation. :type properties: dict, optional @@ -369,16 +355,6 @@ void regclass_Core(py::module m) { new_compiled = core.import_model(user_stream, "CPU") )"); - // todo: remove after Accuracy Checker migration to set/get_property API - cls.def( - "get_config", - [](ov::Core& self, const std::string& device_name, const std::string& name) -> py::object { - PyErr_WarnEx(PyExc_DeprecationWarning, "get_config() is deprecated, use get_property() instead.", 1); - return Common::from_ov_any(self.get_property(device_name, name)).as(); - }, - py::arg("device_name"), - py::arg("name")); - cls.def( "get_property", [](ov::Core& self, const std::string& device_name, const std::string& name) -> py::object { @@ -397,16 +373,6 @@ void regclass_Core(py::module m) { :rtype: object )"); - // todo: remove after Accuracy Checker migration to set/get_property API - cls.def( - "get_metric", - [](ov::Core& self, const std::string device_name, const std::string name) -> py::object { - PyErr_WarnEx(PyExc_DeprecationWarning, "get_metric() is deprecated, use get_property() instead.", 1); - return Common::from_ov_any(self.get_property(device_name, name)).as(); - }, - py::arg("device_name"), - py::arg("name")); - cls.def("register_plugin", &ov::Core::register_plugin, py::arg("plugin_name"), diff --git a/src/bindings/python/src/pyopenvino/core/infer_request.cpp b/src/bindings/python/src/pyopenvino/core/infer_request.cpp index c9aac19a9e1..8d70c7bd23c 100644 --- a/src/bindings/python/src/pyopenvino/core/infer_request.cpp +++ b/src/bindings/python/src/pyopenvino/core/infer_request.cpp @@ -51,8 +51,8 @@ void regclass_InferRequest(py::module m) { py::arg("tensors"), R"( Sets batch of tensors for input data to infer by tensor name. - Model input shall have batch dimension and number of tensors shall - match with batch size. Current version supports set tensors to model inputs only. + Model input needs to have batch dimension and the number of tensors needs to be + matched with batch size. Current version supports set tensors to model inputs only. In case if `tensor_name` is associated with output (or any other non-input node), an exception will be thrown. @@ -60,7 +60,7 @@ void regclass_InferRequest(py::module m) { :type tensor_name: str :param tensors: Input tensors for batched infer request. The type of each tensor must match the model input element type and shape (except batch dimension). - Total size of tensors shall match with input's size. + Total size of tensors needs to match with input's size. :type tensors: List[openvino.runtime.Tensor] )"); @@ -73,8 +73,8 @@ void regclass_InferRequest(py::module m) { py::arg("tensors"), R"( Sets batch of tensors for input data to infer by tensor name. - Model input shall have batch dimension and number of tensors shall - match with batch size. Current version supports set tensors to model inputs only. + Model input needs to have batch dimension and the number of tensors needs to be + matched with batch size. Current version supports set tensors to model inputs only. In case if `port` is associated with output (or any other non-input node), an exception will be thrown. @@ -83,7 +83,7 @@ void regclass_InferRequest(py::module m) { :type port: openvino.runtime.ConstOutput :param tensors: Input tensors for batched infer request. The type of each tensor must match the model input element type and shape (except batch dimension). - Total size of tensors shall match with input's size. + Total size of tensors needs to match with input's size. :type tensors: List[openvino.runtime.Tensor] :rtype: None )"); @@ -130,12 +130,12 @@ void regclass_InferRequest(py::module m) { py::arg("tensors"), R"( Sets batch of tensors for single input data. - Model input shall have batch dimension and number of `tensors` - shall match with batch size. + Model input needs to have batch dimension and the number of `tensors` + needs to match with batch size. :param tensors: Input tensors for batched infer request. The type of each tensor must match the model input element type and shape (except batch dimension). - Total size of tensors shall match with input's size. + Total size of tensors needs to match with input's size. :type tensors: List[openvino.runtime.Tensor] )"); @@ -148,14 +148,14 @@ void regclass_InferRequest(py::module m) { py::arg("tensors"), R"( Sets batch of tensors for single input data to infer by index. - Model input shall have batch dimension and number of `tensors` - shall match with batch size. + Model input needs to have batch dimension and the number of `tensors` + needs to match with batch size. :param idx: Index of input tensor. :type idx: int :param tensors: Input tensors for batched infer request. The type of each tensor must match the model input element type and shape (except batch dimension). - Total size of tensors shall match with input's size. + Total size of tensors needs to match with input's size. )"); cls.def( @@ -513,8 +513,8 @@ void regclass_InferRequest(py::module m) { return self._request.get_profiling_info(); }, R"( - Queries performance measures per layer to get feedback of what - is the most time consuming operation, not all plugins provide + Queries performance is measured per layer to get feedback on what + is the most time-consuming operation, not all plugins provide meaningful data. :return: List of profiling information for operations in model. @@ -616,7 +616,7 @@ void regclass_InferRequest(py::module m) { return self._request.get_profiling_info(); }, R"( - Performance measures per layer to get feedback of what is the most time consuming operation. + Performance is measured per layer to get feedback on the most time-consuming operation. Not all plugins provide meaningful data! :return: Inference time. diff --git a/src/bindings/python/src/pyopenvino/core/offline_transformations.cpp b/src/bindings/python/src/pyopenvino/core/offline_transformations.cpp index 319f1ad2705..3495033cf26 100644 --- a/src/bindings/python/src/pyopenvino/core/offline_transformations.cpp +++ b/src/bindings/python/src/pyopenvino/core/offline_transformations.cpp @@ -146,7 +146,7 @@ void regmodule_offline_transformations(py::module m) { py::arg("weights_path"), py::arg("version") = "UNSPECIFIED", R"( - Serialize given model into IR. The generated .xml and .bin files will be save + Serialize given model into IR. The generated .xml and .bin files will be saved into provided paths. :param model: model which will be converted to IR representation diff --git a/src/bindings/python/src/pyopenvino/core/tensor.cpp b/src/bindings/python/src/pyopenvino/core/tensor.cpp index bedf9b2186d..7d3d77098f4 100644 --- a/src/bindings/python/src/pyopenvino/core/tensor.cpp +++ b/src/bindings/python/src/pyopenvino/core/tensor.cpp @@ -26,10 +26,10 @@ void regclass_Tensor(py::module m) { :param array: Array to create tensor from. :type array: numpy.array - :param shared_memory: If `True` this Tensor memory is being shared with a host, + :param shared_memory: If `True`, this Tensor memory is being shared with a host, that means the responsibility of keeping host memory is on the side of a user. Any action performed on the host - memory will be reflected on this Tensor's memory! + memory is reflected on this Tensor's memory! If `False`, data is being copied to this Tensor. Requires data to be C_CONTIGUOUS if `True`. :type shared_memory: bool @@ -43,8 +43,8 @@ void regclass_Tensor(py::module m) { R"( Another Tensor's special constructor. - It take an array or slice of it and shape that will be - selected starting from the first element of given array/slice. + It takes an array or slice of it, and shape that will be + selected, starting from the first element of the given array/slice. Please use it only in advanced cases if necessary! :param array: Underlaying methods will retrieve pointer on first element diff --git a/src/bindings/python/src/pyopenvino/graph/descriptors/tensor.cpp b/src/bindings/python/src/pyopenvino/graph/descriptors/tensor.cpp index 7e73074576b..6f28ea5619f 100644 --- a/src/bindings/python/src/pyopenvino/graph/descriptors/tensor.cpp +++ b/src/bindings/python/src/pyopenvino/graph/descriptors/tensor.cpp @@ -101,7 +101,7 @@ void regclass_graph_descriptor_Tensor(py::module m) { Set of names )"); - tensor.def("set_names", + tensor.def("add_names", &ov::descriptor::Tensor::add_names, py::arg("names"), R"( diff --git a/src/bindings/python/src/pyopenvino/graph/dimension.cpp b/src/bindings/python/src/pyopenvino/graph/dimension.cpp index bec1643aee6..4475abdb4a5 100644 --- a/src/bindings/python/src/pyopenvino/graph/dimension.cpp +++ b/src/bindings/python/src/pyopenvino/graph/dimension.cpp @@ -11,6 +11,7 @@ #include #include +#include "pyopenvino/core/common.hpp" #include "pyopenvino/graph/dimension.hpp" namespace py = pybind11; @@ -41,6 +42,10 @@ void regclass_graph_Dimension(py::module m) { :type max_dimension: int )"); + dim.def(py::init([](const std::string& value) { + return Common::dimension_from_str(value); + })); + dim.def_static("dynamic", &ov::Dimension::dynamic); dim.def_property_readonly("is_dynamic", diff --git a/src/bindings/python/src/pyopenvino/graph/model.cpp b/src/bindings/python/src/pyopenvino/graph/model.cpp index 3d4e09b39bb..b2e3af2811a 100644 --- a/src/bindings/python/src/pyopenvino/graph/model.cpp +++ b/src/bindings/python/src/pyopenvino/graph/model.cpp @@ -264,49 +264,114 @@ void regclass_graph_Model(py::module m) { [](ov::Model& self, const ov::PartialShape& partial_shape) { self.reshape(partial_shape); }, - py::arg("partial_shapes"), + py::arg("partial_shape"), R"( - :param partial_shapes: Index of Output. - :type partial_shapes: PartialShape + :param partial_shape: New shape. + :type partial_shape: PartialShape :return : void )"); function.def( "reshape", - [](ov::Model& self, const std::map& partial_shapes) { - self.reshape(partial_shapes); + [](ov::Model& self, const py::list& partial_shape) { + self.reshape(Common::partial_shape_from_list(partial_shape)); }, - py::arg("partial_shapes"), + py::arg("partial_shape"), R"( - - :param partial_shapes: Index of Output. - :type partial_shapes: Dict[int, PartialShape] - :return: void + :param partial_shape: New shape. + :type partial_shape: list + :return : void )"); function.def( "reshape", - [](ov::Model& self, const std::map& partial_shapes) { - self.reshape(partial_shapes); + [](ov::Model& self, const py::tuple& partial_shape) { + self.reshape(Common::partial_shape_from_list(partial_shape.cast())); }, - py::arg("partial_shapes"), + py::arg("partial_shape"), R"( - :param partial_shapes: Index of Output. - :type partial_shapes: Dict[string, PartialShape] - :return: void + :param partial_shape: New shape. + :type partial_shape: tuple + :return : void )"); function.def( "reshape", - [](ov::Model& self, const std::map, ov::PartialShape>& partial_shapes) { - self.reshape(partial_shapes); + [](ov::Model& self, const std::string& partial_shape) { + self.reshape(Common::partial_shape_from_str(partial_shape)); + }, + py::arg("partial_shape"), + R"( + :param partial_shape: New shape. + :type partial_shape: str + :return : void + )"); + + function.def( + "reshape", + [](ov::Model& self, const py::dict& partial_shapes) { + std::map, ov::PartialShape> new_shapes; + for (const auto& item : partial_shapes) { + std::pair, ov::PartialShape> new_shape; + // check keys + if (py::isinstance(item.first)) { + new_shape.first = self.input(item.first.cast()); + } else if (py::isinstance(item.first)) { + new_shape.first = self.input(item.first.cast()); + } else if (py::isinstance>(item.first)) { + new_shape.first = item.first.cast>(); + } else { + throw py::type_error("Incorrect key type " + std::string(item.first.get_type().str()) + + " to reshape a model, expected keys as openvino.runtime.Output, int or str."); + } + // check values + if (py::isinstance(item.second)) { + new_shape.second = item.second.cast(); + } else if (py::isinstance(item.second) || py::isinstance(item.second)) { + new_shape.second = Common::partial_shape_from_list(item.second.cast()); + } else if (py::isinstance(item.second)) { + new_shape.second = Common::partial_shape_from_str(item.second.cast()); + } else { + throw py::type_error( + "Incorrect value type " + std::string(item.second.get_type().str()) + + " to reshape a model, expected values as openvino.runtime.PartialShape, str, list or tuple."); + } + new_shapes.insert(new_shape); + } + self.reshape(new_shapes); }, py::arg("partial_shapes"), - R"( - :param partial_shapes: Index of Output. - :type partial_shapes: Dict[Output, PartialShape] - :return: void - )"); + R"( Reshape model inputs. + + The allowed types of keys in the `partial_shapes` dictionary are: + + (1) `int`, input index + (2) `str`, input tensor name + (3) `openvino.runtime.Output` + + The allowed types of values in the `partial_shapes` are: + + (1) `openvino.runtime.PartialShape` + (2) `list` consisting of dimensions + (3) `tuple` consisting of dimensions + (4) `str`, string representation of `openvino.runtime.PartialShape` + + When list or tuple are used to describe dimensions, each dimension can be written in form: + + (1) non-negative `int` which means static value for the dimension + (2) `[min, max]`, dynamic dimension where `min` specifies lower bound and `max` specifies upper bound; the range includes both `min` and `max`; using `-1` for `min` or `max` means no known bound + (3) `(min, max)`, the same as above + (4) `-1` is a dynamic dimension without known bounds + (4) `openvino.runtime.Dimension` + (5) `str` using next syntax: + '?' - to define fully dinamic dimension + '1' - to define dimension which length is 1 + '1..10' - to define bounded dimension + '..10' or '1..' to define dimension with only lower or only upper limit + + :param partial_shapes: New shapes. + :type partial_shapes: Dict[keys, values] + )"); function.def("get_output_size", &ov::Model::get_output_size, @@ -566,7 +631,7 @@ void regclass_graph_Model(py::module m) { Return -1 if parameter not matched. - :param parameter: Parameter which index is to be found. + :param parameter: Parameter, which index is to be found. :type parameter: op.Parameter :return: Index for parameter :rtype: int diff --git a/src/bindings/python/src/pyopenvino/graph/node_output.hpp b/src/bindings/python/src/pyopenvino/graph/node_output.hpp index 30305555040..b891a828aa1 100644 --- a/src/bindings/python/src/pyopenvino/graph/node_output.hpp +++ b/src/bindings/python/src/pyopenvino/graph/node_output.hpp @@ -101,7 +101,7 @@ void regclass_graph_Output(py::module m, std::string typestring) output.def("get_target_inputs", &ov::Output::get_target_inputs, R"( - A set containing handles for all inputs targeted by the output + A set containing handles for all inputs, targeted by the output, referenced by this output handle. :return: Set of Inputs. diff --git a/src/bindings/python/src/pyopenvino/graph/partial_shape.cpp b/src/bindings/python/src/pyopenvino/graph/partial_shape.cpp index 1f6a1c2499c..dfc5595f7fe 100644 --- a/src/bindings/python/src/pyopenvino/graph/partial_shape.cpp +++ b/src/bindings/python/src/pyopenvino/graph/partial_shape.cpp @@ -13,6 +13,7 @@ #include "openvino/core/dimension.hpp" // ov::Dimension #include "openvino/core/shape.hpp" // ov::Shape +#include "pyopenvino/core/common.hpp" #include "pyopenvino/graph/partial_shape.hpp" namespace py = pybind11; @@ -23,15 +24,17 @@ void regclass_graph_PartialShape(py::module m) { py::class_> shape(m, "PartialShape"); shape.doc() = "openvino.runtime.PartialShape wraps ov::PartialShape"; - shape.def(py::init([](const std::vector& dimensions) { - return ov::PartialShape(std::vector(dimensions.begin(), dimensions.end())); - })); - shape.def(py::init&>()); - shape.def(py::init&>()); - shape.def(py::init&>()); - shape.def(py::init&>()); shape.def(py::init()); shape.def(py::init()); + shape.def(py::init([](py::list& shape) { + return Common::partial_shape_from_list(shape); + })); + shape.def(py::init([](py::tuple& shape) { + return Common::partial_shape_from_list(shape.cast()); + })); + shape.def(py::init([](const std::string& shape) { + return Common::partial_shape_from_str(shape); + })); shape.def_static("dynamic", &ov::PartialShape::dynamic, py::arg("rank") = ov::Dimension()); diff --git a/src/bindings/python/src/pyopenvino/graph/preprocess/pre_post_process.cpp b/src/bindings/python/src/pyopenvino/graph/preprocess/pre_post_process.cpp index 56ef90f5db8..d7ec912524f 100644 --- a/src/bindings/python/src/pyopenvino/graph/preprocess/pre_post_process.cpp +++ b/src/bindings/python/src/pyopenvino/graph/preprocess/pre_post_process.cpp @@ -109,18 +109,18 @@ static void regclass_graph_PreProcessSteps(py::module m) { steps.def( "convert_element_type", - [](ov::preprocess::PreProcessSteps& self, ov::element::Type type) { + [](ov::preprocess::PreProcessSteps& self, ov::element::Type type = {}) { return &self.convert_element_type(type); }, - py::arg("type"), + py::arg_v("type", ov::element::undefined, "openvino.runtime.Type.undefined"), R"( Converts input tensor element type to specified type. - Input tensor must have openvino.Type.f32 data type. + Input tensor must have openvino.Type data type. Parameters ---------- type : openvino.runtime.Type - Destination type. + Destination type. If not specified, type will be taken from model input's element type. Returns ---------- @@ -200,18 +200,18 @@ static void regclass_graph_PostProcessSteps(py::module m) { steps.def( "convert_element_type", - [](ov::preprocess::PostProcessSteps& self, ov::element::Type type) { + [](ov::preprocess::PostProcessSteps& self, ov::element::Type type = {}) { return &self.convert_element_type(type); }, - py::arg("type"), + py::arg_v("type", ov::element::undefined, "openvino.runtime.Type.undefined"), R"( Converts tensor element type to specified type. - Tensor must have openvino.Type.f32 data type. + Tensor must have openvino.Type data type. Parameters ---------- type : Type - Destination type. + Destination type. If not specified, type will be taken from model output's element type. Returns ---------- @@ -322,7 +322,7 @@ static void regclass_graph_InputTensorInfo(py::module m) { return &self.set_color_format(format, sub_names); }, py::arg("format"), - py::arg("sub_names")); + py::arg("sub_names") = std::vector{}); info.def( "set_memory_type", diff --git a/src/bindings/python/src/pyopenvino/graph/shape.cpp b/src/bindings/python/src/pyopenvino/graph/shape.cpp index 975aad6f84c..1f306ac2649 100644 --- a/src/bindings/python/src/pyopenvino/graph/shape.cpp +++ b/src/bindings/python/src/pyopenvino/graph/shape.cpp @@ -19,6 +19,7 @@ namespace py = pybind11; void regclass_graph_Shape(py::module m) { py::class_> shape(m, "Shape"); shape.doc() = "openvino.runtime.Shape wraps ov::Shape"; + shape.def(py::init<>()); shape.def(py::init&>(), py::arg("axis_lengths")); shape.def(py::init&>(), py::arg("axis_lengths")); shape.def(py::init(), py::arg("axis_lengths")); diff --git a/src/bindings/python/src/pyopenvino/graph/types/element_type.cpp b/src/bindings/python/src/pyopenvino/graph/types/element_type.cpp index 204d125a330..b41d5c000d5 100644 --- a/src/bindings/python/src/pyopenvino/graph/types/element_type.cpp +++ b/src/bindings/python/src/pyopenvino/graph/types/element_type.cpp @@ -29,10 +29,13 @@ void regclass_graph_Type(py::module m) { type.attr("u32") = ov::element::u32; type.attr("u64") = ov::element::u64; type.attr("bf16") = ov::element::bf16; + type.attr("undefined") = ov::element::undefined; type.def("__repr__", [](const ov::element::Type& self) { std::string bitwidth = std::to_string(self.bitwidth()); - if (self.is_signed()) { + if (self == ov::element::undefined) { + return ""; + } else if (self.is_signed()) { return ""; } return ""; diff --git a/src/bindings/python/src/pyopenvino/graph/util.cpp b/src/bindings/python/src/pyopenvino/graph/util.cpp index 5bfdf7f6a16..497cbe52fde 100644 --- a/src/bindings/python/src/pyopenvino/graph/util.cpp +++ b/src/bindings/python/src/pyopenvino/graph/util.cpp @@ -27,7 +27,7 @@ void regmodule_graph_util(py::module m) { :param index: Output node. :type index: openvino.runtime.Output :return: If it succeeded to calculate both bounds and - they are the same returns Constant operation + they are the same, returns Constant operation from the resulting bound, otherwise Null. :rtype: openvino.runtime.op.Constant or openvino.runtime.Node )"); diff --git a/src/bindings/python/src/pyopenvino/pyopenvino.cpp b/src/bindings/python/src/pyopenvino/pyopenvino.cpp index 012a5bf6fc5..53ac0003301 100644 --- a/src/bindings/python/src/pyopenvino/pyopenvino.cpp +++ b/src/bindings/python/src/pyopenvino/pyopenvino.cpp @@ -69,6 +69,21 @@ std::string get_version() { PYBIND11_MODULE(pyopenvino, m) { m.doc() = "Package openvino.pyopenvino which wraps openvino C++ APIs"; + std::string pyopenvino_version = CI_BUILD_NUMBER; + std::string runtime_version = get_version(); + bool is_custom_pyopenvino_version = pyopenvino_version.empty() || pyopenvino_version.find("custom_") == 0; + bool is_custom_runtime_version = runtime_version.empty() || runtime_version.find("custom_") == 0; + auto versions_compatible = + is_custom_pyopenvino_version || is_custom_runtime_version || pyopenvino_version == runtime_version; + OPENVINO_ASSERT(versions_compatible, + "OpenVINO Python version (", + pyopenvino_version, + ") mismatches with OpenVINO Runtime library version (", + runtime_version, + "). It can happen if you have 2 or more different versions of OpenVINO installed in system. " + "Please ensure that environment variables (e.g. PATH, PYTHONPATH) are set correctly so that " + "OpenVINO Runtime and Python libraries point to same release."); + m.def("get_version", &get_version); m.def("get_batch", &ov::get_batch); m.def("set_batch", &ov::set_batch); diff --git a/src/bindings/python/tests/test_inference_engine/test_compiled_model.py b/src/bindings/python/tests/test_inference_engine/test_compiled_model.py index 60b5f44e8fc..e0bb872bf4d 100644 --- a/src/bindings/python/tests/test_inference_engine/test_compiled_model.py +++ b/src/bindings/python/tests/test_inference_engine/test_compiled_model.py @@ -29,8 +29,8 @@ def test_get_property(device): pytest.skip("Can't run on ARM plugin due-to CPU dependent test") func = core.read_model(model=test_net_xml, weights=test_net_bin) exec_net = core.compile_model(func, device) - config = exec_net.get_property("PERF_COUNT") - assert config == "NO" + profiling_enabled = exec_net.get_property("PERF_COUNT") + assert not profiling_enabled def test_get_runtime_model(device): diff --git a/src/bindings/python/tests/test_inference_engine/test_function.py b/src/bindings/python/tests/test_inference_engine/test_function.py index adaa2da2302..9863474ad6b 100644 --- a/src/bindings/python/tests/test_inference_engine/test_function.py +++ b/src/bindings/python/tests/test_inference_engine/test_function.py @@ -361,3 +361,76 @@ def test_reshape(device): core = Core() compiled = core.compile_model(model, device) assert compiled.input().partial_shape == ref_shape + + +def test_reshape_with_python_types(device): + model = create_test_model() + + def check_shape(new_shape): + for input in model.inputs: + assert input.partial_shape == new_shape + + shape1 = [1, 4] + new_shapes = {input: shape1 for input in model.inputs} + model.reshape(new_shapes) + check_shape(PartialShape(shape1)) + + shape2 = [1, 6] + new_shapes = {input.any_name: shape2 for input in model.inputs} + model.reshape(new_shapes) + check_shape(PartialShape(shape2)) + + shape3 = [1, 8] + new_shapes = {i: shape3 for i, input in enumerate(model.inputs)} + model.reshape(new_shapes) + check_shape(PartialShape(shape3)) + + shape4 = [1, -1] + new_shapes = {input: shape4 for input in model.inputs} + model.reshape(new_shapes) + check_shape(PartialShape([Dimension(1), Dimension(-1)])) + + shape5 = [1, (1, 10)] + new_shapes = {input: shape5 for input in model.inputs} + model.reshape(new_shapes) + check_shape(PartialShape([Dimension(1), Dimension(1, 10)])) + + shape6 = [Dimension(3), Dimension(3, 10)] + new_shapes = {input: shape6 for input in model.inputs} + model.reshape(new_shapes) + check_shape(PartialShape(shape6)) + + shape7 = "1..10, ?" + new_shapes = {input: shape7 for input in model.inputs} + model.reshape(new_shapes) + check_shape(PartialShape(shape7)) + + # reshape mixed keys + shape8 = [(1, 20), -1] + new_shapes = {"data1": shape8, 1: shape8} + model.reshape(new_shapes) + check_shape(PartialShape([Dimension(1, 20), Dimension(-1)])) + + # reshape with one input + param = ops.parameter([1, 3, 28, 28]) + model = Model(ops.relu(param), [param]) + + shape9 = [-1, 3, (28, 56), (28, 56)] + model.reshape(shape9) + check_shape(PartialShape([Dimension(-1), Dimension(3), Dimension(28, 56), Dimension(28, 56)])) + + shape10 = "?,3,..224,..224" + model.reshape(shape10) + check_shape(PartialShape([Dimension(-1), Dimension(3), Dimension(-1, 224), Dimension(-1, 224)])) + + # check exceptions + shape10 = [1, 1, 1, 1] + with pytest.raises(TypeError) as e: + model.reshape({model.input().node: shape10}) + assert "Incorrect key type to reshape a model, " \ + "expected keys as openvino.runtime.Output, int or str." in str(e.value) + + with pytest.raises(TypeError) as e: + model.reshape({0: range(1, 9)}) + assert "Incorrect value type to reshape a model, " \ + "expected values as openvino.runtime.PartialShape, str, list or tuple." in str(e.value) diff --git a/src/bindings/python/tests/test_ngraph/test_core.py b/src/bindings/python/tests/test_ngraph/test_core.py index a8919e8900e..f505afe756b 100644 --- a/src/bindings/python/tests/test_ngraph/test_core.py +++ b/src/bindings/python/tests/test_ngraph/test_core.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import numpy as np +import pytest import openvino.runtime.opset8 as ov from openvino.runtime import Dimension, Model, PartialShape, Shape @@ -76,6 +77,33 @@ def test_dimension_comparisons(): assert not d2.compatible(d1) assert not d2.same_scheme(d1) + d = Dimension("?") + assert d == Dimension() + + d = Dimension("1") + assert d == Dimension(1) + + d = Dimension("..10") + assert d == Dimension(-1, 10) + + d = Dimension("10..") + assert d == Dimension(10, -1) + + d = Dimension("5..10") + assert d == Dimension(5, 10) + + with pytest.raises(RuntimeError) as e: + d = Dimension("C") + assert 'Cannot parse dimension: "C"' in str(e.value) + + with pytest.raises(RuntimeError) as e: + d = Dimension("?..5") + assert 'Cannot parse min bound: "?"' in str(e.value) + + with pytest.raises(RuntimeError) as e: + d = Dimension("5..?") + assert 'Cannot parse max bound: "?"' in str(e.value) + def test_partial_shape(): ps = PartialShape([1, 2, 3, 4]) @@ -140,6 +168,40 @@ def test_partial_shape(): assert list(ps.get_max_shape())[0] > 1000000000 assert repr(ps) == "" + shape_list = [(1, 10), [2, 5], 4, Dimension(2), "..10"] + ref_ps = PartialShape([Dimension(1, 10), Dimension(2, 5), Dimension(4), Dimension(2), Dimension(-1, 10)]) + assert PartialShape(shape_list) == ref_ps + assert PartialShape(tuple(shape_list)) == ref_ps + + with pytest.raises(TypeError) as e: + PartialShape([(1, 2, 3)]) + assert "Two elements are expected in tuple(lower, upper) " \ + "for dynamic dimension, but 3 elements were given." in str(e.value) + + with pytest.raises(TypeError) as e: + PartialShape([("?", "?")]) + assert "Incorrect pair of types (, ) " \ + "for dynamic dimension, ints are expected." in str(e.value) + + with pytest.raises(TypeError) as e: + PartialShape([range(10)]) + assert "Incorrect type for dimension. Expected types are: " \ + "int, str, openvino.runtime.Dimension, list/tuple with lower " \ + "and upper values for dynamic dimension." in str(e.value) + + ps = PartialShape("...") + assert ps == PartialShape.dynamic() + + ps = PartialShape("?, 3, ..224, 28..224") + assert ps == PartialShape([Dimension(-1), Dimension(3), Dimension(-1, 224), Dimension(28, 224)]) + + with pytest.raises(RuntimeError) as e: + ps = PartialShape("?,,3") + assert 'Cannot get vector of dimensions! "?,,3" is incorrect' in str(e.value) + + shape = Shape() + assert len(shape) == 0 + def test_partial_shape_compatible(): ps1 = PartialShape.dynamic() diff --git a/src/bindings/python/tests/test_ngraph/test_preprocess.py b/src/bindings/python/tests/test_ngraph/test_preprocess.py index bf0741cbfdb..c8636410789 100644 --- a/src/bindings/python/tests/test_ngraph/test_preprocess.py +++ b/src/bindings/python/tests/test_ngraph/test_preprocess.py @@ -90,7 +90,7 @@ def test_ngraph_preprocess_mean_scale_convert(): p = PrePostProcessor(function) inp2 = p.input(1) inp2.tensor().set_element_type(Type.i32) - inp2.preprocess().convert_element_type(Type.f32).mean(1.).scale(2.) + inp2.preprocess().convert_element_type(Type.f32).mean(1.).scale(2.).convert_element_type() inp1 = p.input(0) inp1.preprocess().convert_element_type(Type.f32).mean(1.).custom(custom_preprocess) function = p.build() @@ -159,10 +159,14 @@ def test_ngraph_preprocess_output_postprocess(): inp.tensor().set_layout(layout1) inp.preprocess().convert_element_type(Type.f32).mean([1., 2., 3.]) out = p.output() + out.tensor().set_element_type(Type.f32) out.model().set_layout(layout1) out.postprocess().convert_element_type(Type.f32) \ .convert_layout(layout2) \ - .convert_layout(layout3).custom(custom_postprocess) + .convert_layout(layout3) \ + .custom(custom_postprocess) \ + .convert_element_type(Type.f16) \ + .convert_element_type() function = p.build() input_data = np.array([[-1, -2, -3], [-4, -5, -6]]).astype(np.int32) @@ -185,7 +189,7 @@ def test_ngraph_preprocess_spatial_static_shape(): p = PrePostProcessor(function) inp = p.input() - inp.tensor().set_layout(layout).set_spatial_static_shape(2, 2).set_color_format(color_format, []) + inp.tensor().set_layout(layout).set_spatial_static_shape(2, 2).set_color_format(color_format) inp.preprocess().convert_element_type(Type.f32).mean([1., 2.]) inp.model().set_layout(layout) out = p.output() diff --git a/src/bindings/python/wheel/setup.py b/src/bindings/python/wheel/setup.py index 7e3b41cfdab..bbc9465a415 100644 --- a/src/bindings/python/wheel/setup.py +++ b/src/bindings/python/wheel/setup.py @@ -27,7 +27,7 @@ WHEEL_LIBS_PACKAGE = 'openvino.libs' PYTHON_VERSION = f'python{sys.version_info.major}.{sys.version_info.minor}' LIBS_DIR = 'bin' if platform.system() == 'Windows' else 'lib' -CONFIG = 'Release' if platform.system() == 'Windows' else '' +CONFIG = 'Release' if platform.system() == 'Windows' or platform.system() == 'Darwin' else '' machine = platform.machine() if machine == 'x86_64' or machine == 'AMD64': diff --git a/src/common/conditional_compilation/include/openvino/cc/ngraph/itt.hpp b/src/common/conditional_compilation/include/openvino/cc/ngraph/itt.hpp index 710020e6069..9c60ca4d200 100644 --- a/src/common/conditional_compilation/include/openvino/cc/ngraph/itt.hpp +++ b/src/common/conditional_compilation/include/openvino/cc/ngraph/itt.hpp @@ -4,36 +4,4 @@ #pragma once -#include -#include - -OV_CC_DOMAINS(ngraph_pass); - -/* - * RUN_ON_FUNCTION_SCOPE macro allows to disable the run_on_function pass - * MATCHER_SCOPE macro allows to disable the MatcherPass if matcher isn't applied - */ -#if defined(SELECTIVE_BUILD_ANALYZER) - -#define RUN_ON_FUNCTION_SCOPE(region) OV_SCOPE(ngraph_pass, OV_PP_CAT(region, _run_on_function)) -#define MATCHER_SCOPE(region) const std::string matcher_name(OV_PP_TOSTRING(region)) - -#elif defined(SELECTIVE_BUILD) - -#define MATCHER_SCOPE_(scope, region) \ - if (OV_CC_SCOPE_IS_ENABLED(OV_PP_CAT3(scope, _, region)) == 0) \ - throw ngraph::ngraph_error(std::string(OV_PP_TOSTRING(OV_PP_CAT3(scope, _, region))) + \ - " is disabled!") - -#define MATCHER_SCOPE(region) \ - const std::string matcher_name(OV_PP_TOSTRING(region)); \ - if (OV_CC_SCOPE_IS_ENABLED(OV_PP_CAT3(ngraph_pass, _, region)) == 0) \ - return -#define RUN_ON_FUNCTION_SCOPE(region) \ - MATCHER_SCOPE_(ngraph_pass, OV_PP_CAT(region, _run_on_function)) - -#else - -#define MATCHER_SCOPE(region) const std::string matcher_name(OV_PP_TOSTRING(region)) -#define RUN_ON_FUNCTION_SCOPE(region) -#endif +#include diff --git a/src/common/conditional_compilation/include/openvino/cc/pass/itt.hpp b/src/common/conditional_compilation/include/openvino/cc/pass/itt.hpp new file mode 100644 index 00000000000..dd3c23f9f5b --- /dev/null +++ b/src/common/conditional_compilation/include/openvino/cc/pass/itt.hpp @@ -0,0 +1,46 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +OV_CC_DOMAINS(ov_pass); + +/* + * RUN_ON_MODEL_SCOPE macro allows to disable the run_on_function pass + * RUN_ON_FUNCTION_SCOPE macro allows to disable the run_on_function pass + * MATCHER_SCOPE macro allows to disable the MatcherPass if matcher isn't applied + */ +#if defined(SELECTIVE_BUILD_ANALYZER) + +#define RUN_ON_FUNCTION_SCOPE(region) OV_SCOPE(ov_pass, OV_PP_CAT(region, _run_on_function)) +#define MATCHER_SCOPE(region) const std::string matcher_name(OV_PP_TOSTRING(region)) +#define RUN_ON_MODEL_SCOPE(region) OV_SCOPE(ov_pass, OV_PP_CAT(region, _run_on_model)) + +#elif defined(SELECTIVE_BUILD) + +#define MATCHER_SCOPE_(scope, region) \ + if (OV_CC_SCOPE_IS_ENABLED(OV_PP_CAT3(scope, _, region)) == 0) \ + throw ngraph::ngraph_error(std::string(OV_PP_TOSTRING(OV_PP_CAT3(scope, _, region))) + \ + " is disabled!") + +#define MATCHER_SCOPE(region) \ + const std::string matcher_name(OV_PP_TOSTRING(region)); \ + if (OV_CC_SCOPE_IS_ENABLED(OV_PP_CAT3(ov_pass, _, region)) == 0) \ + return +#define RUN_ON_FUNCTION_SCOPE(region) \ + MATCHER_SCOPE_(ov_pass, OV_PP_CAT(region, _run_on_function)) + +#define RUN_ON_MODEL_SCOPE(region) \ + MATCHER_SCOPE_(ov_pass, OV_PP_CAT(region, _run_on_model)) + +#else + +#define MATCHER_SCOPE(region) const std::string matcher_name(OV_PP_TOSTRING(region)) +#define RUN_ON_FUNCTION_SCOPE(region) +#define RUN_ON_MODEL_SCOPE(region) +#endif + diff --git a/src/common/snippets/include/snippets/emitter.hpp b/src/common/snippets/include/snippets/emitter.hpp index 469c8794cb9..2ba0f85c5de 100644 --- a/src/common/snippets/include/snippets/emitter.hpp +++ b/src/common/snippets/include/snippets/emitter.hpp @@ -48,6 +48,7 @@ public: */ virtual void emit_data() const { } + virtual ~Emitter() = default; }; } // namespace snippets diff --git a/src/common/snippets/include/snippets/generator.hpp b/src/common/snippets/include/snippets/generator.hpp index 0deb9d9eade..e1a1fdf720a 100644 --- a/src/common/snippets/include/snippets/generator.hpp +++ b/src/common/snippets/include/snippets/generator.hpp @@ -60,6 +60,7 @@ public: bool has(const ngraph::DiscreteTypeInfo type) const { return jitters.find(type) != jitters.end(); } + virtual ~TargetMachine() = default; protected: std::map(std::shared_ptr)>> jitters; diff --git a/src/common/snippets/src/op/subgraph.cpp b/src/common/snippets/src/op/subgraph.cpp index 7dff3913085..c5d8ce47d27 100644 --- a/src/common/snippets/src/op/subgraph.cpp +++ b/src/common/snippets/src/op/subgraph.cpp @@ -133,13 +133,7 @@ Shape snippets::op::Subgraph::canonicalize(const BlockedShapeVector& outputShape NODE_VALIDATION_CHECK(this, outputShapes.size() == m_body->get_results().size(), "number of results for snippet doesn't match passed to generate method: ", outputShapes.size(), " vs ", m_body->get_results().size(), "."); - // todo: does it allowed to have outputs with different layouts? I assume no, remove if invalid - const AxisVector outOrder = get<1>(outputShapes[0]); - for (size_t i = 1; i < outputShapes.size(); i++) { - const AxisVector order_i = get<1>(outputShapes[i]); - NODE_VALIDATION_CHECK(this, outOrder.size() == order_i.size() && equal(outOrder.begin(), outOrder.end(), order_i.begin()), - "Snippets output shapes must have the same layout"); - } + auto getMaxRankBlockedShape = [](const BlockedShapeVector& blockedShapes) -> const BlockedShape& { return *std::max_element(blockedShapes.begin(), blockedShapes.end(), [&](const BlockedShape& lhs, const BlockedShape& rhs) { @@ -187,18 +181,33 @@ Shape snippets::op::Subgraph::canonicalize(const BlockedShapeVector& outputShape } m_body->validate_nodes_and_infer_types(); + auto skipStartEndOnes = [](const Shape& shape) { + auto begin = shape.begin(); + auto end = shape.end(); + while (begin != end && *begin == 1) + begin++; + while (begin != end && *(end-1) == 1) + end--; + Shape trimmedShape(end - begin, 1); + std::copy(begin, end, trimmedShape.begin()); + return trimmedShape; + }; // Check that output shapes are broadcastable => can be scheduled const auto& body_results = m_body->get_results(); PartialShape outPShape = body_results[0]->get_shape(); for (size_t i = 0; i < body_results.size(); i++) { auto shape_i = body_results[i]->get_shape(); - PartialShape pShape_i(shape_i); + auto outputShape_i = std::get<0>(outputShapes[i]); // Check that the produced output shape corresponds to the passed shape - bool compatibleWithPassedShape = PartialShape::broadcast_merge_into(pShape_i, std::get<0>(outputShapes[i]), + // Some produced shapes may have been changed to be broadcastable (e.g. blocked + planar outputs), + // so we need to remove leading and trailing "1" before the comparison + PartialShape pShape_i(skipStartEndOnes(shape_i)); + bool compatibleWithPassedShape = PartialShape::broadcast_merge_into(pShape_i, skipStartEndOnes(outputShape_i), ::ngraph::op::AutoBroadcastType::NUMPY); - NODE_VALIDATION_CHECK(this, compatibleWithPassedShape, "Inferred and passed results shapes are difference for snippet : ", - shape_i, " vs ", std::get<0>(outputShapes[i]), "."); + NODE_VALIDATION_CHECK(this, ov::shape_size(shape_i) == ov::shape_size(outputShape_i) && + compatibleWithPassedShape, "Inferred and passed results shapes are incompatible for snippet ", + get_friendly_name(), " : ", shape_i, " vs ", outputShape_i, "."); // Check that output shapes are broadcastable to each other => can be scheduled bool compatibleWithOtherOutputs = PartialShape::broadcast_merge_into(outPShape, shape_i, ::ngraph::op::AutoBroadcastType::NUMPY); diff --git a/src/common/snippets/src/pass/collapse_subgraph.cpp b/src/common/snippets/src/pass/collapse_subgraph.cpp index 1529b58a81d..0bd462f1ee5 100644 --- a/src/common/snippets/src/pass/collapse_subgraph.cpp +++ b/src/common/snippets/src/pass/collapse_subgraph.cpp @@ -372,9 +372,12 @@ TokenizeSnippets::TokenizeSnippets() { auto internal = input_body_parameters[i]; auto internal_consumers = internal->outputs(); - if (auto to_replace_with = ov::as_type_ptr(subgraph->get_input_node_shared_ptr(i))) { - for (auto output : internal_consumers) { + // todo: In principle, we can still attach the node to the subgraph if cyclic dependency is introduced during ternary merge. + // Need to support. + if (cyclicDependencyIsIntoduced(to_replace_with, currentTopoBounds)) + return abort_with_strategy("Attempt to perform recurrent merge for cyclic-dependent subgraphs. Aborting."); + for (const auto& output : internal_consumers) { for (auto consumer : output.get_target_inputs()) { auto other_body = clones[subgraph->get_input_node_shared_ptr(i)]; auto other_body_result = other_body->get_results()[consumer.get_source_output().get_index()]; diff --git a/src/common/transformations/include/transformations/common_optimizations/dimension_tracking.hpp b/src/common/transformations/include/transformations/common_optimizations/dimension_tracking.hpp index 123bf121f8e..a1137a70233 100644 --- a/src/common/transformations/include/transformations/common_optimizations/dimension_tracking.hpp +++ b/src/common/transformations/include/transformations/common_optimizations/dimension_tracking.hpp @@ -17,7 +17,6 @@ namespace ov { namespace pass { class TRANSFORMATIONS_API FindBatch; -class TRANSFORMATIONS_API FindBatchDontTrack; } // namespace pass } // namespace ov @@ -25,15 +24,10 @@ class TRANSFORMATIONS_API FindBatchDontTrack; class ov::pass::FindBatch: public ov::pass::ModelPass { public: OPENVINO_RTTI("FindBatch"); - FindBatch(bool track = true) : track(track) {} + FindBatch(bool detach_detection_output = false, bool track = true) : track(track), detach_do(detach_detection_output) {} bool run_on_model(const std::shared_ptr& m) override; protected: - bool track = true; -}; - -class ov::pass::FindBatchDontTrack: public ov::pass::FindBatch { -public: - FindBatchDontTrack() : FindBatch(false) {} + bool track = true, detach_do = false; }; namespace ov { @@ -48,5 +42,6 @@ namespace batch_util { const std::map, ov::PartialShape>& parameter_to_shape, bool leave_batch_dynamic = true); bool check_batch_tracks_through_all_the_nodes(const std::shared_ptr& m); P2Btype find_batch(const std::shared_ptr &m); + bool detach_detection_output(const std::shared_ptr& f); } // namespace batch_util } // namespace ov diff --git a/src/common/transformations/src/transformations/common_optimizations/dimension_tracking.cpp b/src/common/transformations/src/transformations/common_optimizations/dimension_tracking.cpp index edf96b55df1..547ea7e5b42 100644 --- a/src/common/transformations/src/transformations/common_optimizations/dimension_tracking.cpp +++ b/src/common/transformations/src/transformations/common_optimizations/dimension_tracking.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "dimension_tracker.hpp" @@ -142,7 +143,8 @@ P2Btype ov::batch_util::find_batch(const std::shared_ptr& f) { for (const auto& output : curr_node->outputs()) { // we do not need to walk through shape-of sub-graphs for (const auto& t_input : output.get_target_inputs()) { - if (ov::is_type(t_input.get_node()) || + if (ov::is_type(t_input.get_node()) || + ov::is_type(t_input.get_node()) || ov::is_type(t_input.get_node())) continue; nodes.push_back(t_input.get_node()); @@ -205,7 +207,7 @@ bool ov::batch_util::check_batch_tracks_through_all_the_nodes(const std::shared_ all_outputs_has_batch &= name_stays; // && others_are_static; } if (any_input_has_batch && !all_outputs_has_batch && !ov::is_type(node) && - !ov::is_type(node)) { + !ov::is_type(node) && !ov::is_type(node)) { failed_to_propagate_batch = true; node->validate_and_infer_types(); } @@ -221,16 +223,41 @@ bool ov::batch_util::check_batch_tracks_through_all_the_nodes(const std::shared_ return failed_to_propagate_batch; } +bool ov::batch_util::detach_detection_output(const std::shared_ptr& f) { + ResultVector new_outputs, outputs_to_delete; + for (auto& result_node : f->get_results()) { + auto do_node = result_node->input_value(0).get_node_shared_ptr(); + if (ov::is_type(do_node)) // cases with do->convert->result + do_node = do_node->get_input_node_shared_ptr(0); + if (ov::is_type(do_node) || ov::is_type(do_node)) { + for (auto& new_result_src : do_node->input_values()) { + auto new_result = std::make_shared(new_result_src); + ngraph::copy_runtime_info(result_node, new_result); + new_outputs.push_back(new_result); + } + outputs_to_delete.push_back(result_node); + } + } + for (auto& result : outputs_to_delete) + f->remove_result(result); + f->add_results(new_outputs); + return !new_outputs.empty() || !outputs_to_delete.empty(); +} + bool ov::pass::FindBatch::run_on_model(const std::shared_ptr& m) { auto te = std::make_shared(); ov::DimensionTracker dt(te); + bool model_has_changed = false; + if (detach_do) + model_has_changed |= batch_util::detach_detection_output(m); + const auto& parameters = m->get_parameters(); std::map, PartialShape> parameter_to_shape; for (const auto& parameter : parameters) { auto shape = parameter->get_partial_shape(); if (shape.rank().is_dynamic()) - return false; + return model_has_changed; parameter_to_shape[parameter] = shape; } diff --git a/src/common/transformations/src/transformations/op_conversions/convert_interpolate1_to_interpolate4.cpp b/src/common/transformations/src/transformations/op_conversions/convert_interpolate1_to_interpolate4.cpp index 1b9202fdd3f..de0feeb8dcb 100644 --- a/src/common/transformations/src/transformations/op_conversions/convert_interpolate1_to_interpolate4.cpp +++ b/src/common/transformations/src/transformations/op_conversions/convert_interpolate1_to_interpolate4.cpp @@ -9,40 +9,37 @@ #include #include +#include #include #include #include +#include +#include NGRAPH_RTTI_DEFINITION(ngraph::pass::ConvertInterpolate1ToInterpolate4, "ConvertInterpolate1ToInterpolate4", 0); ngraph::pass::ConvertInterpolate1ToInterpolate4::ConvertInterpolate1ToInterpolate4() { MATCHER_SCOPE(ConvertInterpolate1ToInterpolate4); - auto interpolate1 = ngraph::pattern::wrap_type({pattern::any_input(pattern::has_static_shape()), pattern::any_input()}); + auto interpolate1 = ngraph::pattern::wrap_type({pattern::any_input(pattern::has_static_rank()), pattern::any_input()}); ngraph::matcher_pass_callback callback = [this](pattern::Matcher& m) { auto interpolationV0 = std::dynamic_pointer_cast(m.get_match_root()); if (!interpolationV0) { return false; } - auto& inp_partial_shape = interpolationV0->get_input_partial_shape(0); - auto& out_shape = interpolationV0->get_output_shape(0); auto attrsV0 = interpolationV0->get_attrs(); + std::vector axes{attrsV0.axes.begin(), attrsV0.axes.end()}; + const auto& out_dims = std::make_shared(interpolationV0->input_value(1), element::f32); + const auto& in_dims = std::make_shared(ngraph::op::util::node_to_get_shape_value_of_indices_from_shape_source( + interpolationV0->input_value(0), axes), element::f32); - std::vector scales(attrsV0.axes.size(), 1.0f); - auto inp_shape = inp_partial_shape.to_shape(); - size_t i = 0; - for (std::size_t axis : attrsV0.axes) { - scales[i] = static_cast(out_shape.at(axis))/inp_shape.at(axis); - i++; - } - - auto input_shape_rank = inp_partial_shape.rank().get_length(); - auto scalesConstant = ngraph::op::Constant::create(ngraph::element::f32, {scales.size()}, scales); - auto axisConstant = ngraph::op::Constant::create(ngraph::element::i64, {attrsV0.axes.size()}, - std::vector{attrsV0.axes.begin(), attrsV0.axes.end()}); + std::shared_ptr scales = std::make_shared(out_dims, in_dims); + if (const auto& constant = ov::get_constant_from_source(scales)) + scales = constant; + auto axisConstant = ngraph::op::Constant::create(ngraph::element::i64, {axes.size()}, axes); ngraph::opset4::Interpolate::InterpolateAttrs attrsV4; - + auto input_shape_rank = interpolationV0->get_input_partial_shape(0).rank().get_length(); if (attrsV0.mode == "nearest") { attrsV4.mode = ngraph::opset4::Interpolate::InterpolateMode::NEAREST; } else if (attrsV0.mode == "linear") { @@ -85,7 +82,7 @@ ngraph::pass::ConvertInterpolate1ToInterpolate4::ConvertInterpolate1ToInterpolat } auto interpolateV4 = std::make_shared(interpolationV0->input_value(0), interpolationV0->input_value(1), - scalesConstant, axisConstant, attrsV4); + scales, axisConstant, attrsV4); interpolateV4->set_friendly_name(interpolationV0->get_friendly_name()); ngraph::copy_runtime_info(interpolationV0, interpolateV4); diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 6301fa70523..cc64b94ca32 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -121,14 +121,14 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ PATTERN "*.h") configure_package_config_file(${OpenVINO_SOURCE_DIR}/cmake/templates/ngraphConfig.cmake.in - ${CMAKE_CURRENT_BINARY_DIR}/ngraphConfig.cmake + ${CMAKE_BINARY_DIR}/ngraphConfig.cmake INSTALL_DESTINATION cmake) -write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/ngraphConfigVersion.cmake +write_basic_package_version_file(${CMAKE_BINARY_DIR}/ngraphConfigVersion.cmake VERSION ${IE_VERSION_MAJOR}.${IE_VERSION_MINOR}.${IE_VERSION_PATCH} COMPATIBILITY SameMajorVersion) -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ngraphConfig.cmake - ${CMAKE_CURRENT_BINARY_DIR}/ngraphConfigVersion.cmake +install(FILES ${CMAKE_BINARY_DIR}/ngraphConfig.cmake + ${CMAKE_BINARY_DIR}/ngraphConfigVersion.cmake DESTINATION "runtime/cmake" COMPONENT core_dev) diff --git a/src/core/include/ngraph/rt_info.hpp b/src/core/include/ngraph/rt_info.hpp index 79b3b0d6466..f72389ab56a 100644 --- a/src/core/include/ngraph/rt_info.hpp +++ b/src/core/include/ngraph/rt_info.hpp @@ -9,22 +9,11 @@ #include "ngraph/ngraph_visibility.hpp" #include "ngraph/node.hpp" #include "ngraph/type.hpp" +#include "openvino/core/rt_info.hpp" namespace ngraph { -NGRAPH_API -void copy_runtime_info(std::shared_ptr from, std::shared_ptr to); - -NGRAPH_API -void copy_runtime_info(std::shared_ptr from, ngraph::NodeVector to); - -NGRAPH_API -void copy_runtime_info(const ngraph::NodeVector& from, std::shared_ptr to); - -NGRAPH_API -void copy_runtime_info(const ngraph::NodeVector& from, ngraph::NodeVector to); - -NGRAPH_API -void copy_output_runtime_info(const ngraph::OutputVector& from, ngraph::OutputVector to); +using ov::copy_output_runtime_info; +using ov::copy_runtime_info; } // namespace ngraph using ngraph::copy_runtime_info; diff --git a/src/core/include/openvino/core/any.hpp b/src/core/include/openvino/core/any.hpp index f2c26c0e051..3d98462e8cc 100644 --- a/src/core/include/openvino/core/any.hpp +++ b/src/core/include/openvino/core/any.hpp @@ -541,6 +541,8 @@ class OPENVINO_API Any { template Impl(Args&&... args) : value(std::forward(args)...) {} + virtual ~Impl(){}; + const std::type_info& type_info() const override { return typeid(T); } diff --git a/src/core/include/openvino/core/core.hpp b/src/core/include/openvino/core/core.hpp index c25d5940d87..e0c3e46b5fd 100644 --- a/src/core/include/openvino/core/core.hpp +++ b/src/core/include/openvino/core/core.hpp @@ -28,6 +28,7 @@ #include "openvino/core/node_vector.hpp" #include "openvino/core/partial_shape.hpp" #include "openvino/core/rank.hpp" +#include "openvino/core/rt_info.hpp" #include "openvino/core/rtti.hpp" #include "openvino/core/runtime_attribute.hpp" #include "openvino/core/shape.hpp" diff --git a/src/core/include/openvino/core/descriptor/input.hpp b/src/core/include/openvino/core/descriptor/input.hpp index 8302ed759b6..f357fb9581d 100644 --- a/src/core/include/openvino/core/descriptor/input.hpp +++ b/src/core/include/openvino/core/descriptor/input.hpp @@ -28,12 +28,12 @@ class OPENVINO_API Input { public: /// \param node The node that owns this input - /// \param index The position of this this tensor in all input tensors + /// \param index The position of this tensor in all input tensors /// \param output The output that supplies a value for this input Input(Node* node, size_t index, Output& output); /// \brief Create an Input that is not connected to an output /// \param node The node that owns this input - /// \param index The position of this this tensor in all input tensors + /// \param index The position of this tensor in all input tensors Input(Node* node, size_t index); ~Input(); diff --git a/src/core/include/openvino/core/model.hpp b/src/core/include/openvino/core/model.hpp index f3e6751e850..a5a357ad865 100644 --- a/src/core/include/openvino/core/model.hpp +++ b/src/core/include/openvino/core/model.hpp @@ -398,8 +398,8 @@ OPENVINO_API ov::Dimension get_batch(const std::shared_ptr& f); /// applied. Possible reason could be that layout was not set for some parameters, or batch size can't be applied to /// model at all /// -/// \param f model where to set batch_size value +/// \param model model where to set batch_size value /// \param batch_size Batch size value. For dynamic batch size, Dimension::dynamic() can be passed. -OPENVINO_API void set_batch(const std::shared_ptr& f, ov::Dimension batch_size); +OPENVINO_API void set_batch(const std::shared_ptr& model, ov::Dimension batch_size); } // namespace ov diff --git a/src/core/include/openvino/core/rt_info.hpp b/src/core/include/openvino/core/rt_info.hpp new file mode 100644 index 00000000000..d930d20cc7d --- /dev/null +++ b/src/core/include/openvino/core/rt_info.hpp @@ -0,0 +1,28 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "openvino/core/core_visibility.hpp" +#include "openvino/core/node.hpp" +#include "openvino/core/type.hpp" + +namespace ov { +OPENVINO_API +void copy_runtime_info(const std::shared_ptr& from, const std::shared_ptr& to); + +OPENVINO_API +void copy_runtime_info(const std::shared_ptr& from, ov::NodeVector to); + +OPENVINO_API +void copy_runtime_info(const ov::NodeVector& from, const std::shared_ptr& to); + +OPENVINO_API +void copy_runtime_info(const ov::NodeVector& from, ov::NodeVector to); + +OPENVINO_API +void copy_output_runtime_info(const ov::OutputVector& from, ov::OutputVector to); +} // namespace ov diff --git a/src/core/include/openvino/op/parameter.hpp b/src/core/include/openvino/op/parameter.hpp index 0a5bf1e180b..b6ede9bcd8d 100644 --- a/src/core/include/openvino/op/parameter.hpp +++ b/src/core/include/openvino/op/parameter.hpp @@ -10,11 +10,11 @@ namespace ov { namespace op { namespace v0 { -/// \brief A function parameter. +/// \brief A model parameter. /// /// Parameters are nodes that represent the arguments that will be passed to -/// user-defined functions. Function creation requires a sequence of parameters. -/// Basic graph operations do not need parameters attached to a function. +/// user-defined models. Model creation requires a sequence of parameters. +/// Basic graph operations do not need parameters attached to a model. class OPENVINO_API Parameter : public op::Op { public: OPENVINO_OP("Parameter", "opset1"); diff --git a/src/core/include/openvino/op/softmax.hpp b/src/core/include/openvino/op/softmax.hpp index b24fb890183..e8e617c361f 100644 --- a/src/core/include/openvino/op/softmax.hpp +++ b/src/core/include/openvino/op/softmax.hpp @@ -47,7 +47,7 @@ private: } // namespace v1 namespace v8 { -/// \brief Softmax operation with with negative axis values +/// \brief Softmax operation with negative axis values class OPENVINO_API Softmax : public Op { public: OPENVINO_OP("Softmax", "opset8"); diff --git a/src/core/include/openvino/op/util/variable_extension.hpp b/src/core/include/openvino/op/util/variable_extension.hpp index 25e8179fd36..53dc533652c 100644 --- a/src/core/include/openvino/op/util/variable_extension.hpp +++ b/src/core/include/openvino/op/util/variable_extension.hpp @@ -38,6 +38,9 @@ public: /// \brief Returns the identifier of corresponding variable. virtual std::string get_variable_id() const = 0; +protected: + virtual ~VariableExtension(){}; + protected: std::shared_ptr m_variable; }; diff --git a/src/core/src/rt_info.cpp b/src/core/src/rt_info.cpp index b735340877a..4baa061fbb2 100644 --- a/src/core/src/rt_info.cpp +++ b/src/core/src/rt_info.cpp @@ -2,14 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "ngraph/rt_info.hpp" +#include "openvino/core/rt_info.hpp" -#include "ngraph/node.hpp" #include "ngraph/variant.hpp" namespace { -std::unordered_map> get_copyable_attrs(const ngraph::OutputVector& outputs) { +std::unordered_map> get_copyable_attrs(const ov::OutputVector& outputs) { std::unordered_map> attrs; for (const auto& output : outputs) { for (const auto& item : output.get_rt_info()) { @@ -25,7 +24,7 @@ std::unordered_map> get_copyable_attrs(const n return attrs; } -std::unordered_map> get_copyable_attrs(const ngraph::NodeVector& nodes) { +std::unordered_map> get_copyable_attrs(const ov::NodeVector& nodes) { std::unordered_map> attrs; for (const auto& node : nodes) { for (const auto& item : node->get_rt_info()) { @@ -42,10 +41,10 @@ std::unordered_map> get_copyable_attrs(const n } template -ngraph::Node::RTMap mergeRuntimeInfo(const T& items) { +ov::Node::RTMap mergeRuntimeInfo(const T& items) { std::unordered_map> attrs = get_copyable_attrs(items); - ngraph::Node::RTMap merged_attrs; + ov::Node::RTMap merged_attrs; for (auto& item : attrs) { auto attr = *item.second.begin(); if (item.second.size() == 1) { @@ -63,7 +62,7 @@ ngraph::Node::RTMap mergeRuntimeInfo(const T& items) { return merged_attrs; } -ov::Any get_opset(const ngraph::Node::RTMap& rt_info) { +ov::Any get_opset(const ov::Node::RTMap& rt_info) { auto it = rt_info.find("opset"); if (it != rt_info.end()) { return it->second; @@ -71,7 +70,7 @@ ov::Any get_opset(const ngraph::Node::RTMap& rt_info) { return nullptr; } -void assign_runtime_info(const ngraph::Node::RTMap& from, ngraph::Node::RTMap& to) { +void assign_runtime_info(const ov::Node::RTMap& from, ov::Node::RTMap& to) { auto opset = get_opset(to); for (auto& item : from) { to[item.first] = item.second; @@ -83,7 +82,7 @@ void assign_runtime_info(const ngraph::Node::RTMap& from, ngraph::Node::RTMap& t } // namespace -void ngraph::copy_runtime_info(std::shared_ptr from, std::shared_ptr to) { +void ov::copy_runtime_info(const std::shared_ptr& from, const std::shared_ptr& to) { auto& attrs = to->get_rt_info(); auto opset = get_opset(attrs); @@ -102,18 +101,18 @@ void ngraph::copy_runtime_info(std::shared_ptr from, std::shared_p } } -void ngraph::copy_runtime_info(std::shared_ptr from, ngraph::NodeVector to) { +void ov::copy_runtime_info(const std::shared_ptr& from, ov::NodeVector to) { for (auto& op : to) { copy_runtime_info(from, op); } } -void ngraph::copy_runtime_info(const ngraph::NodeVector& from, std::shared_ptr to) { +void ov::copy_runtime_info(const ov::NodeVector& from, const std::shared_ptr& to) { auto& rtInfoTo = to->get_rt_info(); assign_runtime_info(mergeRuntimeInfo(from), rtInfoTo); } -void ngraph::copy_runtime_info(const ngraph::NodeVector& from, ngraph::NodeVector to) { +void ov::copy_runtime_info(const ov::NodeVector& from, ov::NodeVector to) { auto mergedInfo = mergeRuntimeInfo(from); for (auto& node : to) { auto& rtInfoTo = node->get_rt_info(); @@ -121,7 +120,7 @@ void ngraph::copy_runtime_info(const ngraph::NodeVector& from, ngraph::NodeVecto } } -void ngraph::copy_output_runtime_info(const ngraph::OutputVector& from, ngraph::OutputVector to) { +void ov::copy_output_runtime_info(const ov::OutputVector& from, ov::OutputVector to) { auto mergedInfo = mergeRuntimeInfo(from); for (auto& node : to) { auto& rtInfoTo = node.get_rt_info(); diff --git a/src/core/tests/type_prop/einsum.cpp b/src/core/tests/type_prop/einsum.cpp index 6fdbf854279..ac55f742862 100644 --- a/src/core/tests/type_prop/einsum.cpp +++ b/src/core/tests/type_prop/einsum.cpp @@ -120,7 +120,7 @@ TEST(type_prop, einsum_dynamicshape_dotproduct) { auto I2 = make_shared(element::f32, input2_shape); auto O = make_shared(OutputVector{I1, I2}, equation); ASSERT_EQ(O->get_element_type(), element::f32); - ASSERT_TRUE(O->get_output_partial_shape(0).same_scheme(out_shape)); + ASSERT_EQ(O->get_output_partial_shape(0), out_shape); } TEST(type_prop, einsum_dynamicshape_diagextraction) { @@ -130,13 +130,10 @@ TEST(type_prop, einsum_dynamicshape_diagextraction) { auto I1 = make_shared(element::i32, input1_shape); auto O = make_shared(OutputVector{I1}, equation); ASSERT_EQ(O->get_element_type(), element::i32); - ASSERT_TRUE(O->get_output_partial_shape(0).same_scheme(out_shape)); + ASSERT_EQ(O->get_output_partial_shape(0), out_shape); } -TEST(type_prop, DISABLED_einsum_dynamicshape_ellipsis1) { - // TODO: fix bug #53518 - PartialShape::broadcast_merge_into or Dimension::broadcast_merge - // to support broadcasting between Dimension(3, 5) and Dimension(1, 3) - // for which the result must be Dimension(3, 5) +TEST(type_prop, einsum_dynamicshape_ellipsis1) { std::string equation = "a...b,b...->a..."; const auto input1_shape = PartialShape{11, 1, Dimension(3, 5), 3}; const auto input2_shape = PartialShape{3, 11, 7, Dimension(1, 3)}; @@ -145,7 +142,7 @@ TEST(type_prop, DISABLED_einsum_dynamicshape_ellipsis1) { auto I2 = make_shared(element::f32, input2_shape); auto O = make_shared(OutputVector{I1, I2}, equation); ASSERT_EQ(O->get_element_type(), element::f32); - ASSERT_TRUE(O->get_output_partial_shape(0).same_scheme(out_shape)); + ASSERT_EQ(O->get_output_partial_shape(0), out_shape); } TEST(type_prop, einsum_implicitmode_mixedcaseletters) { @@ -156,7 +153,7 @@ TEST(type_prop, einsum_implicitmode_mixedcaseletters) { const auto out_shape = PartialShape{1, Dimension(4, 5), Dimension(2, 3)}; auto O = make_shared(OutputVector{I1}, equation); ASSERT_EQ(O->get_element_type(), element::f32); - ASSERT_TRUE(O->get_output_partial_shape(0).same_scheme(out_shape)); + ASSERT_EQ(O->get_output_partial_shape(0), out_shape); } TEST(type_prop, einsum_implicitmode_mixedcaseletters2) { @@ -169,7 +166,7 @@ TEST(type_prop, einsum_implicitmode_mixedcaseletters2) { auto I2 = make_shared(element::f32, input2_shape); auto O = make_shared(OutputVector{I1, I2}, equation); ASSERT_EQ(O->get_element_type(), element::f32); - ASSERT_TRUE(O->get_output_partial_shape(0).same_scheme(out_shape)); + ASSERT_EQ(O->get_output_partial_shape(0), out_shape); } TEST(type_prop, einsum_implicitmode_repeatedlabels) { @@ -182,7 +179,7 @@ TEST(type_prop, einsum_implicitmode_repeatedlabels) { auto I2 = make_shared(element::f32, input2_shape); auto O = make_shared(OutputVector{I1, I2}, equation); ASSERT_EQ(O->get_element_type(), element::f32); - ASSERT_TRUE(O->get_output_partial_shape(0).same_scheme(out_shape)); + ASSERT_EQ(O->get_output_partial_shape(0), out_shape); } TEST(type_prop, einsum_implicitmode_innerprod) { @@ -195,7 +192,7 @@ TEST(type_prop, einsum_implicitmode_innerprod) { auto I2 = make_shared(element::f32, input2_shape); auto O = make_shared(OutputVector{I1, I2}, equation); ASSERT_EQ(O->get_element_type(), element::f32); - ASSERT_TRUE(O->get_output_partial_shape(0).same_scheme(out_shape)); + ASSERT_EQ(O->get_output_partial_shape(0), out_shape); } TEST(type_prop, einsum_dynamicrank_multimatmul) { @@ -223,7 +220,7 @@ TEST(type_prop, einsum_dynamicrank_multimatmul2) { auto I3 = make_shared(element::i32, input3_shape); auto O = make_shared(OutputVector{I1, I2, I3}, equation); ASSERT_EQ(O->get_element_type(), element::i32); - ASSERT_TRUE(O->get_output_partial_shape(0).same_scheme(out_shape)); + ASSERT_EQ(O->get_output_partial_shape(0), out_shape); } TEST(type_prop, einsum_incorrectequation_subscriptnumber) { diff --git a/src/frontends/common/include/openvino/frontend/extension/op.hpp b/src/frontends/common/include/openvino/frontend/extension/op.hpp index bfbb8938317..1c6f63eb77b 100644 --- a/src/frontends/common/include/openvino/frontend/extension/op.hpp +++ b/src/frontends/common/include/openvino/frontend/extension/op.hpp @@ -99,7 +99,7 @@ public: const std::string& target_name = p_name != m_attr_names_map.end() ? p_name->second : name; try { adapter.set_as_any(m_context.get_attribute_as_any(target_name)); - } catch (::ov::AssertFailure ex) { + } catch (::ov::AssertFailure& ex) { OPENVINO_ASSERT(false, ex.what(), "\nValue for attribute \"", @@ -259,4 +259,4 @@ using OpExtension = ov::frontend::OpExtensionBase>& m_variables; /// - /// store information about parameters/results order during function creation + /// store information about parameters/results order during a model creation /// it will be used during Inputs/Outputs Description creation in SubGraph processing /// IoMap io_map; diff --git a/src/frontends/tensorflow/src/op/conv_3d_backprop.cpp b/src/frontends/tensorflow/src/op/conv_3d_backprop.cpp new file mode 100644 index 00000000000..c29c881bd9d --- /dev/null +++ b/src/frontends/tensorflow/src/op/conv_3d_backprop.cpp @@ -0,0 +1,104 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "op_table.hpp" +#include "openvino/opsets/opset8.hpp" + +using namespace std; +using namespace ov::opset8; + +namespace ov { +namespace frontend { +namespace tensorflow { +namespace op { + +OutputVector translate_conv_3d_backprop_input_v2_op(const NodeContext& node) { + auto ng_filter = node.get_input(1); + auto ng_out_backprop = node.get_input(2); + + // TODO: refactor me to be less redundant with other convolution ops + auto tf_strides = node.get_attribute>("strides"); + auto tf_dilations = node.get_attribute>("dilations"); + auto tf_padding_type = node.get_attribute("padding"); + auto tf_data_format = node.get_attribute("data_format"); + + TENSORFLOW_OP_VALIDATION(node, + tf_data_format == "NDHWC" || tf_data_format == "NCDHW", + "Conv3DBackpropInputV2 data format is neither NDHWC nor NCDHW. " + "Provided data format: ", + tf_data_format); + + std::vector tf_input_sizes; + get_const_input(node, 0, &tf_input_sizes); + + if (std::any_of(tf_input_sizes.begin(), tf_input_sizes.end(), [](int32_t size) { + return size <= 0; + })) { + FRONT_END_THROW("Conv3DBackpropInputV2 input sizes must be positive integers"); + } + + bool is_ndhwc = (tf_data_format == "NDHWC"); + + ov::Strides ng_strides(3); + ov::Strides ng_dilations(3); + ov::Shape ng_image_shape(3); + ov::Shape ng_kernel_shape(3); + ov::Shape ng_batch_shape(5); + + convert_nhwc_to_hw(is_ndhwc, tf_strides, ng_strides); + convert_nhwc_to_hw(is_ndhwc, tf_dilations, ng_dilations); + convert_nhwc_to_hw(is_ndhwc, tf_input_sizes, ng_image_shape); + convert_nhwc_to_nchw(node.get_name(), is_ndhwc, ng_out_backprop); + if (is_ndhwc) { + ng_batch_shape = {static_cast(tf_input_sizes[0]), + static_cast(tf_input_sizes[4]), + static_cast(tf_input_sizes[1]), + static_cast(tf_input_sizes[2]), + static_cast(tf_input_sizes[3])}; + } else { + ng_batch_shape = {static_cast(tf_input_sizes[0]), + static_cast(tf_input_sizes[1]), + static_cast(tf_input_sizes[2]), + static_cast(tf_input_sizes[3]), + static_cast(tf_input_sizes[4])}; + } + + auto& ng_filter_shape = ng_filter.get_shape(); + ng_kernel_shape[0] = ng_filter_shape[0]; + ng_kernel_shape[1] = ng_filter_shape[1]; + ng_kernel_shape[2] = ng_filter_shape[2]; + transpose_3d<4, 3, 0, 1, 2>(ng_filter); + + ov::CoordinateDiff ng_padding_below; + ov::CoordinateDiff ng_padding_above; + + make_padding(tf_padding_type, + ng_image_shape, + ng_kernel_shape, + ng_strides, + ng_dilations, + ng_padding_below, + ng_padding_above); + + auto ng_output_shape = make_shared(element::i64, + Shape{ng_batch_shape.size() - 2}, + vector(ng_batch_shape.begin() + 2, ng_batch_shape.end())); + + auto res_node = make_shared(ng_out_backprop, + ng_filter, + ng_output_shape, + ng_strides, + ng_padding_below, + ng_padding_above, + ng_dilations); + auto res = res_node->output(0); + + convert_nchw_to_nhwc(node.get_name(), is_ndhwc, res); + set_node_name(node.get_name(), res.get_node_shared_ptr()); + return {res}; +} +} // namespace op +} // namespace tensorflow +} // namespace frontend +} // namespace ov diff --git a/src/frontends/tensorflow/src/op/scatter_nd.cpp b/src/frontends/tensorflow/src/op/scatter_nd.cpp new file mode 100644 index 00000000000..9eb95b7538a --- /dev/null +++ b/src/frontends/tensorflow/src/op/scatter_nd.cpp @@ -0,0 +1,29 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "op_table.hpp" +#include "openvino/opsets/opset8.hpp" + +using namespace std; +using namespace ov::opset8; + +namespace ov { +namespace frontend { +namespace tensorflow { +namespace op { +OutputVector translate_scatter_nd_op(const NodeContext& node) { + auto input_indices = node.get_input(0); + auto updates = node.get_input(1); + auto shape = node.get_input(2); + + auto input_data = make_shared(updates.get_element_type(), Shape{1}, 0); + auto broadcast = make_shared(input_data, shape); + auto res = make_shared(broadcast, input_indices, updates); + set_node_name(node.get_name(), res); + return res->outputs(); +} +} // namespace op +} // namespace tensorflow +} // namespace frontend +} // namespace ov diff --git a/src/frontends/tensorflow/src/op_table.cpp b/src/frontends/tensorflow/src/op_table.cpp index aeeb0983278..12c26217117 100644 --- a/src/frontends/tensorflow/src/op_table.cpp +++ b/src/frontends/tensorflow/src/op_table.cpp @@ -33,6 +33,7 @@ OP_CONVERTER(translate_const_op); OP_CONVERTER(translate_conv_2d_op); OP_CONVERTER(translate_conv_2d_backprop_input_op); OP_CONVERTER(translate_conv_3d_op); +OP_CONVERTER(translate_conv_3d_backprop_input_v2_op); OP_CONVERTER(translate_cumsum_op); OP_CONVERTER(translate_crop_and_resize_op); OP_CONVERTER(translate_depth_to_space_op); @@ -73,6 +74,7 @@ OP_CONVERTER(translate_reverse_op); OP_CONVERTER(translate_roll_op); OP_CONVERTER(translate_round_op); OP_CONVERTER(translate_rsqrt_op); +OP_CONVERTER(translate_scatter_nd_op); OP_CONVERTER(translate_select_op); OP_CONVERTER(translate_shape_op); OP_CONVERTER(translate_size_op); @@ -167,6 +169,7 @@ const std::map get_supported_ops() { {"Conv2D", translate_conv_2d_op}, {"Conv2DBackpropInput", translate_conv_2d_backprop_input_op}, {"Conv3D", translate_conv_3d_op}, + {"Conv3DBackpropInputV2", translate_conv_3d_backprop_input_v2_op}, {"CropAndResize", translate_crop_and_resize_op}, {"Cumsum", translate_cumsum_op}, {"DepthToSpace", translate_depth_to_space_op}, @@ -220,6 +223,7 @@ const std::map get_supported_ops() { {"Roll", translate_roll_op}, {"Round", translate_round_op}, {"Rsqrt", translate_rsqrt_op}, + {"ScatterNd", translate_scatter_nd_op}, {"Select", translate_select_op}, {"SelectV2", translate_select_op}, {"Shape", translate_shape_op}, diff --git a/src/inference/include/ie/gna/gna_config.hpp b/src/inference/include/ie/gna/gna_config.hpp index 8f7a8bff166..1a9ff7af36b 100644 --- a/src/inference/include/ie/gna/gna_config.hpp +++ b/src/inference/include/ie/gna/gna_config.hpp @@ -38,7 +38,7 @@ namespace GNAConfigParams { * @brief Scale factor that is calculated by user, in order to use static quantisation feature * This option should be used with floating point value serialized to string with decimal separator equals to . (dot) * @details For multiple input case, individual scale factors can be passed, using - * KEY_GNA_SCALE_FACTOR[_input_layer_name] where input_layer can be obtained from from CNNNetwork::GetInputsInfo + * KEY_GNA_SCALE_FACTOR[_input_layer_name] where input_layer can be obtained from CNNNetwork::GetInputsInfo */ DECLARE_GNA_CONFIG_KEY(SCALE_FACTOR); diff --git a/src/inference/include/openvino/runtime/core.hpp b/src/inference/include/openvino/runtime/core.hpp index d22e071eb43..e98928f32a3 100644 --- a/src/inference/include/openvino/runtime/core.hpp +++ b/src/inference/include/openvino/runtime/core.hpp @@ -644,23 +644,23 @@ public: * @brief Creates a new remote shared context object on the specified accelerator device * using specified plugin-specific low-level device API parameters (device handle, pointer, context, etc.). * @param device_name Name of a device to create a new shared context on. - * @param properties Map of device-specific shared context properties. + * @param remote_properties Map of device-specific shared context remote properties. * @return Reference to a created remote context. */ - RemoteContext create_context(const std::string& device_name, const AnyMap& properties); + RemoteContext create_context(const std::string& device_name, const AnyMap& remote_properties); /** * @brief Creates a new shared context object on specified accelerator device * using specified plugin-specific low level device API properties (device handle, pointer, etc.) * @tparam Properties Should be the pack of `std::pair` types * @param device_name Name of a device to create new shared context on. - * @param properties Pack of device-specific shared context properties. + * @param remote_properties Pack of device-specific shared context remote properties. * @return A shared pointer to a created remote context. */ template util::EnableIfAllStringAny create_context(const std::string& device_name, - Properties&&... properties) { - return create_context(device_name, AnyMap{std::forward(properties)...}); + Properties&&... remote_properties) { + return create_context(device_name, AnyMap{std::forward(remote_properties)...}); } /** diff --git a/src/inference/include/openvino/runtime/intel_gna/properties.hpp b/src/inference/include/openvino/runtime/intel_gna/properties.hpp index 3e67440064a..fc7d5d3599f 100644 --- a/src/inference/include/openvino/runtime/intel_gna/properties.hpp +++ b/src/inference/include/openvino/runtime/intel_gna/properties.hpp @@ -13,6 +13,10 @@ #include "openvino/runtime/properties.hpp" namespace ov { + +/** + * @brief Namespace with Intel GNA specific properties + */ namespace intel_gna { /** diff --git a/src/inference/include/openvino/runtime/intel_gpu/ocl/ocl.hpp b/src/inference/include/openvino/runtime/intel_gpu/ocl/ocl.hpp index 6faff0955f8..3923fb2d81c 100644 --- a/src/inference/include/openvino/runtime/intel_gpu/ocl/ocl.hpp +++ b/src/inference/include/openvino/runtime/intel_gpu/ocl/ocl.hpp @@ -21,6 +21,10 @@ namespace ov { namespace intel_gpu { + +/** + * @brief Namespace with Intel GPU OpenCL specific remote objects + */ namespace ocl { /** diff --git a/src/inference/include/openvino/runtime/intel_gpu/properties.hpp b/src/inference/include/openvino/runtime/intel_gpu/properties.hpp index 3604f9f3eb5..a52081e1ad6 100644 --- a/src/inference/include/openvino/runtime/intel_gpu/properties.hpp +++ b/src/inference/include/openvino/runtime/intel_gpu/properties.hpp @@ -13,6 +13,10 @@ #include "openvino/runtime/properties.hpp" namespace ov { + +/** + * @brief Namespace with Intel GPU specific properties + */ namespace intel_gpu { /** diff --git a/src/inference/include/openvino/runtime/intel_myriad/hddl_properties.hpp b/src/inference/include/openvino/runtime/intel_myriad/hddl_properties.hpp index 73964c8a120..f24d71a73d3 100644 --- a/src/inference/include/openvino/runtime/intel_myriad/hddl_properties.hpp +++ b/src/inference/include/openvino/runtime/intel_myriad/hddl_properties.hpp @@ -8,8 +8,14 @@ namespace ov { namespace intel_myriad { + +/** + * @brief Namespace with Intel HDDL specific properties + */ namespace hddl { + // RO properties + /** * @brief Property to get a int of the device number */ @@ -169,6 +175,7 @@ static constexpr Property use_sgad{"HDDL_USE_SGAD" * Each device has their own group id. Device in one group shares same group id. */ static constexpr Property group_device{"HDDL_GROUP_DEVICE"}; + } // namespace hddl } // namespace intel_myriad -}; // namespace ov +} // namespace ov diff --git a/src/inference/include/openvino/runtime/intel_myriad/myriad_properties.hpp b/src/inference/include/openvino/runtime/intel_myriad/myriad_properties.hpp index 0d9adc71956..db4b4403256 100644 --- a/src/inference/include/openvino/runtime/intel_myriad/myriad_properties.hpp +++ b/src/inference/include/openvino/runtime/intel_myriad/myriad_properties.hpp @@ -8,7 +8,12 @@ #include namespace ov { + +/** + * @brief Namespace with Intel MYRIAD specific properties + */ namespace intel_myriad { + /** * @brief Turn on HW stages usage (applicable for MyriadX devices only). */ @@ -126,4 +131,4 @@ inline std::istream& operator>>(std::istream& is, DDRType& ddrType) { static constexpr Property ddr_type{"MYRIAD_DDR_TYPE"}; } // namespace intel_myriad -}; // namespace ov +} // namespace ov diff --git a/src/inference/include/openvino/runtime/properties.hpp b/src/inference/include/openvino/runtime/properties.hpp index a4e67f2595a..90a5a8e8e68 100644 --- a/src/inference/include/openvino/runtime/properties.hpp +++ b/src/inference/include/openvino/runtime/properties.hpp @@ -138,7 +138,7 @@ class Property : public util::BaseProperty { template struct Forward { template ::value && + typename std::enable_if::type, std::string>::value && std::is_convertible::value, bool>::type = true> explicit operator U() { @@ -146,15 +146,15 @@ class Property : public util::BaseProperty { } template ::value && + typename std::enable_if::type, std::string>::value && !std::is_convertible::value, bool>::type = true> explicit operator U() { - return Any{value}.as(); + return Any{value}.as(); } template ::value && + typename std::enable_if::type, std::string>::value && std::is_convertible::value, bool>::type = true> explicit operator U() { @@ -162,7 +162,7 @@ class Property : public util::BaseProperty { } template ::value && + typename std::enable_if::type, std::string>::value && !std::is_convertible::value, bool>::type = true> explicit operator U() { @@ -220,6 +220,9 @@ static constexpr Property model_name{"NETWO static constexpr Property optimal_number_of_infer_requests{ "OPTIMAL_NUMBER_OF_INFER_REQUESTS"}; +/** + * @brief Namespace with hint properties + */ namespace hint { /** @@ -345,6 +348,9 @@ static constexpr Property allow_auto_batching{"ALL */ static constexpr Property enable_profiling{"PERF_COUNT"}; +/** + * @brief Namespace with log level property and its possible values + */ namespace log { /** @@ -456,6 +462,12 @@ static constexpr Property optimal_batch_si */ static constexpr Property max_batch_size{"MAX_BATCH_SIZE"}; +/** + * @brief Read-write property to set the timeout used to collect the inputs for the auto-batching + * impact. + */ +static constexpr Property auto_batch_timeout{"AUTO_BATCH_TIMEOUT"}; + /** * @brief Read-only property to provide a hint for a range for number of async infer requests. If device supports * streams, the metric provides range for number of IRs per stream. @@ -468,6 +480,9 @@ static constexpr Property max_batch_size{"MAX_ static constexpr Property, PropertyMutability::RO> range_for_async_infer_requests{"RANGE_FOR_ASYNC_INFER_REQUESTS"}; +/** + * @brief Namespace with device properties + */ namespace device { /** @@ -616,6 +631,10 @@ static constexpr Property thermal{"DEVICE_THERMAL * @brief Read-only property to get a std::vector of capabilities options per device. */ static constexpr Property, PropertyMutability::RO> capabilities{"OPTIMIZATION_CAPABILITIES"}; + +/** + * @brief Namespace with possible values for ov::device::capabilities property + */ namespace capability { constexpr static const auto FP32 = "FP32"; //!< Device supports fp32 inference constexpr static const auto BF16 = "BF16"; //!< Device supports bf16 inference @@ -626,8 +645,12 @@ constexpr static const auto BIN = "BIN"; //!< Device suppor constexpr static const auto WINOGRAD = "WINOGRAD"; //!< Device supports winograd optimization constexpr static const auto EXPORT_IMPORT = "EXPORT_IMPORT"; //!< Device supports compiled model export and import } // namespace capability + } // namespace device +/** + * @brief Namespace for streams in streams executor + */ namespace streams { /** * @brief Class to represent number of streams in streams executor @@ -635,23 +658,11 @@ namespace streams { struct Num { using Base = std::tuple; //!< NumStreams is representable as int32_t - /** - * @brief Special value for ov::execution::num_streams property. - */ - enum Special { - AUTO = -1, //!< Creates bare minimum of streams to improve the performance - NUMA = -2, //!< Creates as many streams as needed to accommodate NUMA and avoid associated penalties - }; - - constexpr Num() : num{AUTO} {}; + constexpr Num() : num{-1} {}; constexpr Num(const int32_t num_) : num{num_} {} - operator int32_t() { - return num; - } - - operator int32_t() const { + constexpr operator int32_t() const { return num; } @@ -663,32 +674,32 @@ struct Num { */ static constexpr Property num{"NUM_STREAMS"}; -static constexpr Num AUTO{Num::AUTO}; //!< Creates bare minimum of streams to improve the performance +static constexpr Num AUTO{-1}; //!< Creates bare minimum of streams to improve the performance static constexpr Num NUMA{ - Num::NUMA}; //!< Creates as many streams as needed to accommodate NUMA and avoid associated penalties + -2}; //!< Creates as many streams as needed to accommodate NUMA and avoid associated penalties /** @cond INTERNAL */ -inline std::ostream& operator<<(std::ostream& os, const Num& num) { - switch (num.num) { - case Num::AUTO: +inline std::ostream& operator<<(std::ostream& os, const Num& num_val) { + switch (num_val) { + case AUTO: return os << "AUTO"; - case Num::NUMA: + case NUMA: return os << "NUMA"; default: - return os << num.num; + return os << num_val.num; } } -inline std::istream& operator>>(std::istream& is, Num& num) { +inline std::istream& operator>>(std::istream& is, Num& num_val) { std::string str; is >> str; if (str == "AUTO") { - num = AUTO; + num_val = AUTO; } else if (str == "NUMA") { - num = NUMA; + num_val = NUMA; } else { try { - num = {std::stoi(str)}; + num_val = {std::stoi(str)}; } catch (const std::exception& e) { throw ov::Exception{std::string{"Could not read number of streams from str: "} + str + "; " + e.what()}; } @@ -698,11 +709,6 @@ inline std::istream& operator>>(std::istream& is, Num& num) { /** @endcond */ } // namespace streams -/** - * @brief Class to represent number of streams in streams executor - */ -using NumStreams = streams::Num; - /** * @brief The number of executor logical partitions */ diff --git a/src/inference/src/check_network_batchable.cpp b/src/inference/src/check_network_batchable.cpp new file mode 100644 index 00000000000..8ce148dd152 --- /dev/null +++ b/src/inference/src/check_network_batchable.cpp @@ -0,0 +1,79 @@ +#include "check_network_batchable.hpp" + +#include "dimension_tracker.hpp" +#include "ie_ngraph_utils.hpp" +#include "ngraph/opsets/opset.hpp" +#include "openvino/op/detection_output.hpp" +#include "openvino/op/ops.hpp" +#include "openvino/pass/manager.hpp" +#include "transformations/common_optimizations/dimension_tracking.hpp" +#include "transformations/init_node_info.hpp" + +namespace InferenceEngine { +namespace details { + +NetworkBatchAbility isNetworkBatchable(const CNNNetwork& orig_network, + const std::string& deviceNameWithoutBatch, + bool strictly_track_dims) { + CNNNetwork clonedNetwork(cloneNetwork(orig_network)); + auto function = clonedNetwork.getFunction(); + // find the batch dim + ov::pass::Manager m; + m.register_pass(); + m.register_pass(true, strictly_track_dims); + m.run_passes(function); + bool any_batched_inputs = false; + // do not reshape/re-batch originally batched networks and when there are no inputs with the N* layouts + // input(s) should have the batch dim as the first dim or none (current limitation of the auto-batching impl) + const auto& params = function->get_parameters(); + for (size_t input_id = 0; input_id < params.size(); input_id++) { + const auto& input = params[input_id]; + const auto& shape = input->get_partial_shape(); + // currently no plugin support batched execution for dynamic networks + if (shape.is_dynamic()) + return NetworkBatchAbility::NO; + // check the batch dim: either 0th (and the original batch size of 1) or none + if (shape.size() && ov::DimensionTracker::get_label(shape[0])) { + const auto& static_shape = input->get_shape(); + if (static_shape[0] != 1) + return NetworkBatchAbility::NO; + else + any_batched_inputs = true; + } else { + // if the 0-th dim is not for the batch, then we support only the case when NONE dimension is batch + for (size_t s = 1; s < shape.size(); s++) + if (ov::DimensionTracker::get_label(shape[s])) + return NetworkBatchAbility::NO; + } + } + if (!any_batched_inputs) + return NetworkBatchAbility::NO; + + for (auto&& node : orig_network.getFunction()->get_ops()) + node->get_rt_info()["affinity"] = "BATCH"; // default affinity (ignored if HETERO is not triggered) + // have to execute the DetectionOutput separately (without batching) + // as this layer does mix-in the values from the different inputs (batch id) + bool bDetectionOutput = false; + for (auto& result_node : orig_network.getFunction()->get_results()) { + auto do_node = result_node->input_value(0).get_node_shared_ptr(); + std::shared_ptr convert_node; + if (ov::is_type(do_node)) { // cases with do->convert->result + convert_node = do_node; + do_node = convert_node->get_input_node_shared_ptr(0); + } + // the code below doesn't need to separate the versions (opsets) of the DetectionOutput + // so base class check is enough + auto detectionOutputBase = std::dynamic_pointer_cast(do_node); + if (detectionOutputBase) { + result_node->get_rt_info()["affinity"] = deviceNameWithoutBatch; + do_node->get_rt_info()["affinity"] = deviceNameWithoutBatch; + if (convert_node) + convert_node->get_rt_info()["affinity"] = deviceNameWithoutBatch; + bDetectionOutput = true; + } + } + return bDetectionOutput ? NetworkBatchAbility::WITH_HETERO : NetworkBatchAbility::AS_IS; +} + +} // namespace details +} // namespace InferenceEngine \ No newline at end of file diff --git a/src/inference/src/check_network_batchable.hpp b/src/inference/src/check_network_batchable.hpp new file mode 100644 index 00000000000..ed0efdff4d3 --- /dev/null +++ b/src/inference/src/check_network_batchable.hpp @@ -0,0 +1,23 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include + +#include "cnn_network_ngraph_impl.hpp" + +namespace InferenceEngine { +namespace details { +/** + * @brief Checks if the input network is batch-able (e.g. no dynamic inputs, inputs has the batch dimension, etc) + * @param function A ngraph function to check for automatic-batching applicability + * @return An enum value indicating whether the network can be safely batched (with HETERO or as is) or not + */ +enum NetworkBatchAbility : uint32_t { NO = 0, AS_IS, WITH_HETERO }; +NetworkBatchAbility isNetworkBatchable(const CNNNetwork& network, + const std::string& deviceNoBatch, + bool strictly_track_dims); + +} // namespace details +} // namespace InferenceEngine diff --git a/src/inference/src/ie_core.cpp b/src/inference/src/ie_core.cpp index fac529fc855..b62a7b176d5 100644 --- a/src/inference/src/ie_core.cpp +++ b/src/inference/src/ie_core.cpp @@ -13,6 +13,7 @@ #include #include "any_copy.hpp" +#include "check_network_batchable.hpp" #include "cnn_network_ngraph_impl.hpp" #include "compilation_context.hpp" #include "cpp/ie_cnn_network.h" @@ -557,6 +558,8 @@ public: std::string& deviceName, std::map& config) { std::string deviceNameWithBatchSize, deviceNameWithoutBatch; + // fully strict dims tracking by default (Auto-Batching is enabled implicitly) + bool strictly_check_dims = true; if (deviceName.find("BATCH") != std::string::npos) { // explicitly enabled Auto-Batching auto pos = deviceName.find_first_of(":"); @@ -564,6 +567,9 @@ public: return; // BATCH device is already configured via the config deviceNameWithBatchSize = deviceName.substr(pos + 1); deviceNameWithoutBatch = DeviceIDParser::getBatchDevice(deviceNameWithBatchSize); + // when user sets the BATCH device explicitly, we may check the dims less strictly + // as the result is being checked by the user + strictly_check_dims = false; } else { // check whether the Auto-Batching is disabled explicitly const auto& batch_mode = config.find(ov::hint::allow_auto_batching.name()); @@ -594,38 +600,18 @@ public: if (bExclReqsEnabled || (!bTputInPlg && !bTputInLoadCfg)) return; } - auto function = network.getFunction(); - // have to execute the DetectionOutput separately (without batching) - // as this layer mix-in the values from the different inputs (batch id) - bool bDetectionOutput = false; - const std::string detectionOutputOpName = ngraph::op::DetectionOutput::get_type_info_static().name; - const std::string resultOpName = ngraph::op::Result::get_type_info_static().name; - for (auto&& node : function->get_ops()) { - auto isDetectionOutputParent = [&detectionOutputOpName](decltype(node)& nd) { - for (size_t n = 0; n < nd->get_input_size(); n++) { - // the code below doesn't need to separate the versions (opsets) of the DetectionOutput - // so type_info name check is enough - // (if in a future there will be a new ver that doesn't mix the batch, this will be new op) - if (detectionOutputOpName == nd->get_input_node_ptr(n)->get_type_info().name) - return true; - } - return false; - }; - - if ((detectionOutputOpName == node->get_type_info().name) || - ((resultOpName == node->get_type_info().name) && isDetectionOutputParent(node))) { - node->get_rt_info()["affinity"] = deviceNameWithoutBatch; - bDetectionOutput = true; - } else { - node->get_rt_info()["affinity"] = "BATCH"; - } - } auto batchConfig = deviceNameWithBatchSize.empty() ? deviceNameWithoutBatch : deviceNameWithBatchSize; - if (bDetectionOutput) { + auto res = InferenceEngine::details::isNetworkBatchable(network, deviceNameWithoutBatch, strictly_check_dims); + switch (res) { + case InferenceEngine::details::NetworkBatchAbility::NO: + return; + case InferenceEngine::details::NetworkBatchAbility::AS_IS: + deviceName = "BATCH:" + batchConfig; + break; + case InferenceEngine::details::NetworkBatchAbility::WITH_HETERO: deviceName = "HETERO:BATCH," + deviceNameWithoutBatch; config[CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG)] = batchConfig; - } else { - deviceName = "BATCH:" + batchConfig; + break; } } diff --git a/src/plugins/auto_batch/auto_batch.cpp b/src/plugins/auto_batch/auto_batch.cpp index b5b2665442b..0c7d15514d8 100644 --- a/src/plugins/auto_batch/auto_batch.cpp +++ b/src/plugins/auto_batch/auto_batch.cpp @@ -30,19 +30,19 @@ using namespace InferenceEngine; std::vector supported_configKeys = {CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG), CONFIG_KEY(AUTO_BATCH_TIMEOUT)}; template -Blob::Ptr create_shared_blob_on_top_of_batched_blob(Blob::Ptr batched_blob, size_t batch_id, size_t batch_num) { +Blob::Ptr create_shared_blob_on_top_of_batched_blob(Blob::Ptr batched_blob, + std::string name, + const std::set& batched_names, + size_t batch_id, + size_t batch_num) { typedef typename PrecisionTrait::value_type TYPE; typedef typename std::add_pointer::type TYPEPTR; auto ptr = batched_blob->buffer().as(); auto sizePerBatch = batched_blob->size() / batch_num; - auto layout = batched_blob->getTensorDesc().getLayout(); SizeVector dims = batched_blob->getTensorDesc().getDims(); // for performance reason (copy avoidance) current impl of the auto-batching supports only batching by 0th dim - if (layout == InferenceEngine::Layout::NC || layout == InferenceEngine::Layout::NCDHW || - layout == InferenceEngine::Layout::NCHW || layout == InferenceEngine::Layout::NHWC || - layout == InferenceEngine::Layout::NDHWC) { + if (batched_names.count(name)) { dims[0] = 1; - assert(batched_blob->getTensorDesc().getPrecision() == precision); return make_shared_blob({precision, dims, batched_blob->getTensorDesc().getLayout()}, ptr + sizePerBatch * batch_id, sizePerBatch); @@ -57,27 +57,32 @@ AutoBatchInferRequest::AutoBatchInferRequest(const std::vector>& outputs, AutoBatchExecutableNetwork::WorkerInferRequest& workerRequest, int batch_id, - int num_batch) + int num_batch, + const std::set& batchedInputs, + const std::set& batchedOutputs) : IInferRequestInternal(inputs, outputs), _myBatchedRequestWrapper(workerRequest), _batchId(batch_id), _batchSize(num_batch) { - ShareBlobsWithBatchRequest(); + ShareBlobsWithBatchRequest(batchedInputs, batchedOutputs); } AutoBatchInferRequest::AutoBatchInferRequest(const InputsDataMap& networkInputs, const OutputsDataMap& networkOutputs, AutoBatchExecutableNetwork::WorkerInferRequest& workerRequest, int batch_id, - int num_batch) + int num_batch, + const std::set& batchedInputs, + const std::set& batchedOutputs) : IInferRequestInternal(networkInputs, networkOutputs), _myBatchedRequestWrapper(workerRequest), _batchId(batch_id), _batchSize(num_batch) { - ShareBlobsWithBatchRequest(); + ShareBlobsWithBatchRequest(batchedInputs, batchedOutputs); } -void AutoBatchInferRequest::ShareBlobsWithBatchRequest() { +void AutoBatchInferRequest::ShareBlobsWithBatchRequest(const std::set& batchedInputs, + const std::set& batchedOutputs) { // Allocate all input blobs for (const auto& it : _networkInputs) { auto blob = _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first); @@ -86,78 +91,104 @@ void AutoBatchInferRequest::ShareBlobsWithBatchRequest() { case InferenceEngine::Precision::FP32: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedInputs, _batchId, _batchSize); break; case InferenceEngine::Precision::I32: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedInputs, _batchId, _batchSize); break; case InferenceEngine::Precision::I8: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedInputs, _batchId, _batchSize); break; case InferenceEngine::Precision::I16: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedInputs, _batchId, _batchSize); break; case InferenceEngine::Precision::U16: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedInputs, _batchId, _batchSize); break; case InferenceEngine::Precision::U32: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedInputs, _batchId, _batchSize); break; case InferenceEngine::Precision::FP64: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedInputs, _batchId, _batchSize); break; case InferenceEngine::Precision::FP16: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedInputs, _batchId, _batchSize); break; case InferenceEngine::Precision::BF16: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedInputs, _batchId, _batchSize); break; case InferenceEngine::Precision::U64: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedInputs, _batchId, _batchSize); break; case InferenceEngine::Precision::I64: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedInputs, _batchId, _batchSize); break; case InferenceEngine::Precision::U8: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedInputs, _batchId, _batchSize); break; case InferenceEngine::Precision::BOOL: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedInputs, _batchId, _batchSize); break; @@ -174,78 +205,104 @@ void AutoBatchInferRequest::ShareBlobsWithBatchRequest() { case InferenceEngine::Precision::FP32: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedOutputs, _batchId, _batchSize); break; case InferenceEngine::Precision::I32: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedOutputs, _batchId, _batchSize); break; case InferenceEngine::Precision::I8: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedOutputs, _batchId, _batchSize); break; case InferenceEngine::Precision::I16: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedOutputs, _batchId, _batchSize); break; case InferenceEngine::Precision::U16: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedOutputs, _batchId, _batchSize); break; case InferenceEngine::Precision::U32: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedOutputs, _batchId, _batchSize); break; case InferenceEngine::Precision::FP64: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedOutputs, _batchId, _batchSize); break; case InferenceEngine::Precision::FP16: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedOutputs, _batchId, _batchSize); break; case InferenceEngine::Precision::BF16: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedOutputs, _batchId, _batchSize); break; case InferenceEngine::Precision::U64: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedOutputs, _batchId, _batchSize); break; case InferenceEngine::Precision::I64: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedOutputs, _batchId, _batchSize); break; case InferenceEngine::Precision::U8: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedOutputs, _batchId, _batchSize); break; case InferenceEngine::Precision::BOOL: res = create_shared_blob_on_top_of_batched_blob( _myBatchedRequestWrapper._inferRequestBatched->GetBlob(it.first), + it.first, + batchedOutputs, _batchId, _batchSize); break; @@ -371,12 +428,16 @@ AutoBatchExecutableNetwork::AutoBatchExecutableNetwork( const InferenceEngine::SoExecutableNetworkInternal& networkWithBatch, const InferenceEngine::SoExecutableNetworkInternal& networkWithoutBatch, const DeviceInformation& networkDevice, - const std::unordered_map& config) + const std::unordered_map& config, + const std::set& batchedInputs, + const std::set& batchedOutputs) : InferenceEngine::ExecutableNetworkThreadSafeDefault(nullptr, std::make_shared()), _network{networkWithBatch}, _networkWithoutBatch{networkWithoutBatch}, - _config{config} { + _config{config}, + _batchedInputs(batchedInputs), + _batchedOutputs(batchedOutputs) { // WA for gcc 4.8 ( fails compilation with member init-list) _device = networkDevice; auto time_out = config.find(CONFIG_KEY(AUTO_BATCH_TIMEOUT)); @@ -411,7 +472,9 @@ InferenceEngine::IInferRequestInternal::Ptr AutoBatchExecutableNetwork::CreateIn networkOutputs, workerRequestPtrAndId.first, workerRequestPtrAndId.second, - _device.batchForDevice); + _device.batchForDevice, + _batchedInputs, + _batchedOutputs); } InferenceEngine::IInferRequestInternal::Ptr AutoBatchExecutableNetwork::CreateInferRequestImpl( @@ -427,7 +490,9 @@ InferenceEngine::IInferRequestInternal::Ptr AutoBatchExecutableNetwork::CreateIn outputs, workerRequestPtrAndId.first, workerRequestPtrAndId.second, - _device.batchForDevice); + _device.batchForDevice, + _batchedInputs, + _batchedOutputs); } std::pair AutoBatchExecutableNetwork::GetWorkerInferRequest() { @@ -761,6 +826,7 @@ InferenceEngine::IExecutableNetworkInternal::Ptr AutoBatchInferencePlugin::LoadN deviceConfigNoAutoBatch[CONFIG_KEY(ALLOW_AUTO_BATCHING)] = CONFIG_VALUE(NO); std::set batched_inputs; + std::set batched_outputs; // check that the auto-batching is applicable in general try { // if applicable, the Auto-Batching is implicitly enabled via the performance hints @@ -768,20 +834,17 @@ InferenceEngine::IExecutableNetworkInternal::Ptr AutoBatchInferencePlugin::LoadN const bool bTputInPlg = core->GetConfig(deviceName, CONFIG_KEY(PERFORMANCE_HINT)).as() == tput; const auto& mode = deviceConfig.find(CONFIG_KEY(PERFORMANCE_HINT)); const bool bTputInLoadCfg = (mode != deviceConfig.end() && mode->second == tput); - // if the auto-batching is enabled implicitly, we shall check the dims carefully, to avoid outstanding failures + // if the auto-batching is enabled implicitly, check the dims carefully, to avoid outstanding failures const bool check_dims = (bTputInPlg || bTputInLoadCfg); CNNNetwork clonedNetwork(InferenceEngine::details::cloneNetwork(network)); auto function = clonedNetwork.getFunction(); // find the batch dim ov::pass::Manager m; m.register_pass(); - if (check_dims) - m.register_pass(); - else - m.register_pass(); + m.register_pass(false, check_dims); m.run_passes(function); // do not reshape/re-batch originally batched networks and when there are no inputs with the N* layouts - // input(s) should have the batch dim as the first dim or none (current limitation of the auto-batching impl) + // input(s) should have the batch dim as the first dim (current limitation of the auto-batching impl) const auto& params = function->get_parameters(); for (size_t input_id = 0; input_id < params.size(); input_id++) { const auto& input = params[input_id]; @@ -804,8 +867,30 @@ InferenceEngine::IExecutableNetworkInternal::Ptr AutoBatchInferencePlugin::LoadN << "Auto-batching operates only networks with inputs/outputs batched by 0th dimension"; } } - if (!batched_inputs.size()) - IE_THROW(NotImplemented) << "Auto-batching supports only networks with inputs featuring batched dim!"; + const auto& results = function->get_results(); + for (size_t output_id = 0; output_id < results.size(); output_id++) { + const auto& output = results[output_id]; + const auto& shape = output->get_output_partial_shape(0); + if (shape.is_dynamic()) + IE_THROW(NotImplemented) << "Auto-batching does not support dynamic networks!"; + // check the batch dim: either 0th (and the original batch size of 1) or none + if (shape.size() && ov::DimensionTracker::get_label(shape[0])) { + if (shape[0] != 1) + IE_THROW(NotImplemented) << "Auto-batching does not reshape/re-batch originally batched networks!"; + const auto& node = output->input_value(0); + batched_outputs.insert(ngraph::op::util::get_ie_output_name( + ov::Output(node.get_node(), node.get_index()))); + } else { + // if the 0-th dim is not for the batch, then we support only the case when NONE dimension is batch + for (size_t s = 1; s < shape.size(); s++) + if (ov::DimensionTracker::get_label(shape[s])) + IE_THROW(NotImplemented) + << "Auto-batching operates only networks with outputs batched by 0th dimension"; + } + } + if (!batched_inputs.size() || !batched_outputs.size()) + IE_THROW(NotImplemented) + << "Auto-batching supports only networks with inputs/outputs featuring batched dim!"; } catch (...) { metaDevice.batchForDevice = 1; } @@ -881,7 +966,9 @@ InferenceEngine::IExecutableNetworkInternal::Ptr AutoBatchInferencePlugin::LoadN return std::make_shared(executableNetworkWithBatch, executableNetworkWithoutBatch, metaDevice, - networkConfig); + networkConfig, + batched_inputs, + batched_outputs); } InferenceEngine::IExecutableNetworkInternal::Ptr AutoBatchInferencePlugin::LoadExeNetworkImpl( diff --git a/src/plugins/auto_batch/auto_batch.hpp b/src/plugins/auto_batch/auto_batch.hpp index e5aff7f3c0f..0994d4377f7 100644 --- a/src/plugins/auto_batch/auto_batch.hpp +++ b/src/plugins/auto_batch/auto_batch.hpp @@ -49,7 +49,9 @@ public: const InferenceEngine::SoExecutableNetworkInternal& networkForDevice, const InferenceEngine::SoExecutableNetworkInternal& networkForDeviceWithoutBatch, const DeviceInformation& networkDevices, - const std::unordered_map& config); + const std::unordered_map& config, + const std::set& batchedIntputs, + const std::set& batchedOutputs); void SetConfig(const std::map& config) override; InferenceEngine::Parameter GetConfig(const std::string& name) const override; @@ -80,6 +82,9 @@ protected: bool _needPerfCounters = false; std::atomic_size_t _numRequestsCreated = {0}; std::atomic_int _timeOut = {0}; // in ms + + const std::set _batchedInputs; + const std::set _batchedOutputs; }; class AutoBatchInferRequest : public InferenceEngine::IInferRequestInternal { @@ -89,12 +94,16 @@ public: const InferenceEngine::OutputsDataMap& networkOutputs, AutoBatchExecutableNetwork::WorkerInferRequest& workerRequestPtr, int batch_id, - int num_batch); + int num_batch, + const std::set& batchedIntputs, + const std::set& batchedOutputs); explicit AutoBatchInferRequest(const std::vector>& inputs, const std::vector>& outputs, AutoBatchExecutableNetwork::WorkerInferRequest& workerRequestPtr, int batch_id, - int num_batch); + int num_batch, + const std::set& batchedIntputs, + const std::set& batchedOutputs); // Batch-Device impl specific: sets the data (blobs from the device request to the batched device request) void SetBlobsToAnotherRequest(InferenceEngine::SoIInferRequestInternal& req); @@ -110,7 +119,8 @@ public: protected: void CopyBlobIfNeeded(InferenceEngine::Blob::CPtr src, InferenceEngine::Blob::Ptr dst, bool bInput); - void ShareBlobsWithBatchRequest(); + void ShareBlobsWithBatchRequest(const std::set& batchedIntputs, + const std::set& batchedOutputs); size_t _batchId; size_t _batchSize; }; diff --git a/src/plugins/intel_cpu/src/emitters/cpu_generator.hpp b/src/plugins/intel_cpu/src/emitters/cpu_generator.hpp index 4e0ad438b5f..7301fcb177b 100644 --- a/src/plugins/intel_cpu/src/emitters/cpu_generator.hpp +++ b/src/plugins/intel_cpu/src/emitters/cpu_generator.hpp @@ -28,7 +28,6 @@ private: class CPUGenerator : public ngraph::snippets::Generator { public: CPUGenerator(dnnl::impl::cpu::x64::cpu_isa_t isa); - ~CPUGenerator() = default; }; } // namespace intel_cpu diff --git a/src/plugins/intel_cpu/src/exec_network.cpp b/src/plugins/intel_cpu/src/exec_network.cpp index b2f20cd9020..5116c57989a 100644 --- a/src/plugins/intel_cpu/src/exec_network.cpp +++ b/src/plugins/intel_cpu/src/exec_network.cpp @@ -26,6 +26,7 @@ #include "cpp_interfaces/interface/ie_iplugin_internal.hpp" #include "ie_icore.hpp" #include "openvino/runtime/properties.hpp" +#include "openvino/util/common_util.hpp" #include #include @@ -310,7 +311,8 @@ InferenceEngine::Parameter MKLDNNExecNetwork::GetMetric(const std::string &name) if (name == ov::model_name) { // @todo Does not seem ok to 'dump()' the whole graph everytime in order to get a name - return graph.dump()->get_friendly_name(); + const std::string modelName = graph.dump()->get_friendly_name(); + return decltype(ov::model_name)::value_type(modelName); } else if (name == ov::optimal_number_of_infer_requests) { const auto streams = config.streamExecutorConfig._streams; return decltype(ov::optimal_number_of_infer_requests)::value_type(streams); // ov::optimal_number_of_infer_requests has no negative values @@ -332,19 +334,20 @@ InferenceEngine::Parameter MKLDNNExecNetwork::GetMetric(const std::string &name) return ov::Affinity::NONE; } else if (name == ov::inference_num_threads) { const auto num_threads = config.streamExecutorConfig._threads; - return num_threads; + return decltype(ov::inference_num_threads)::value_type(num_threads); } else if (name == ov::enable_profiling.name()) { const bool perfCount = config.collectPerfCounters; - return perfCount ? "YES" : "NO"; + return decltype(ov::enable_profiling)::value_type(perfCount); } else if (name == ov::hint::inference_precision) { const auto enforceBF16 = config.enforceBF16; - return enforceBF16 ? ov::element::bf16 : ov::element::f32; + const auto inference_precision = enforceBF16 ? ov::element::bf16 : ov::element::f32; + return decltype(ov::hint::inference_precision)::value_type(inference_precision); } else if (name == ov::hint::performance_mode) { - const auto perfHint = config.perfHintsConfig.ovPerfHint; + const auto perfHint = ov::util::from_string(config.perfHintsConfig.ovPerfHint, ov::hint::performance_mode); return perfHint; } else if (name == ov::hint::num_requests) { const auto perfHintNumRequests = config.perfHintsConfig.ovPerfHintNumRequests; - return perfHintNumRequests; + return decltype(ov::hint::num_requests)::value_type(perfHintNumRequests); } /* Internally legacy parameters are used with new API as part of migration procedure. * This fallback can be removed as soon as migration completed */ diff --git a/src/plugins/intel_cpu/src/graph_optimizer.cpp b/src/plugins/intel_cpu/src/graph_optimizer.cpp index fadac2d363e..ef87b76b9f3 100644 --- a/src/plugins/intel_cpu/src/graph_optimizer.cpp +++ b/src/plugins/intel_cpu/src/graph_optimizer.cpp @@ -337,25 +337,37 @@ void MKLDNNGraphOptimizer::FuseDeconvolutionAndSimpleOperation(MKLDNNGraph &grap void MKLDNNGraphOptimizer::FuseMultiplyAndAdd(MKLDNNGraph &graph) { auto& graphNodes = graph.GetNodes(); - auto isSuitableSecondInput = [](MKLDNNNodePtr node, VectorDims dataDims) { + auto isSuitableSecondInput = [](const MKLDNNNodePtr& node, VectorDims dataDims) { if (node->getType() != Input || !node->isConstant()) return false; - auto secondInputDims = node->getOutputShapeAtPort(0).getStaticDims(); + const auto secondInputDims = node->getOutputShapeAtPort(0).getStaticDims(); if (secondInputDims.size() != dataDims.size() || secondInputDims.size() < 2) return false; - if (secondInputDims[0] != 1 || !dimsEqualWeak(secondInputDims[1], dataDims[1])) + auto getChannelAxis = [](const VectorDims& dims) { + auto channelAxis = -1; + for (int i = 0; i < dims.size(); i ++) { + if (dims[i] != 1) { + if (channelAxis != -1) // more than one axis is != 1 + return -1; + else + channelAxis = i; + } + } + return channelAxis; + }; + + const auto channelAxis = getChannelAxis(secondInputDims); + if (channelAxis == -1) return false; - for (size_t i = 2; i < secondInputDims.size(); i++) { - if (secondInputDims[i] != 1) - return false; - } + if (secondInputDims[0] != 1 || !dimsEqualWeak(secondInputDims[channelAxis], dataDims[channelAxis])) + return false; return true; }; - auto isSuitableParentNode = [&](MKLDNNNodePtr node) { + auto isSuitableParentNode = [&](const MKLDNNNodePtr& node) { if (node->getAlgorithm() != EltwiseMultiply || !node->getFusedWith().empty() || node->getParentEdges().size() != 2 || node->getChildEdges().size() != 1) return false; @@ -363,7 +375,7 @@ void MKLDNNGraphOptimizer::FuseMultiplyAndAdd(MKLDNNGraph &graph) { return isSuitableSecondInput(node->getParentEdgesAtPort(1)[0]->getParent(), node->getInputShapeAtPort(0).getDims()); }; - auto isSuitableChildNode = [&](MKLDNNNodePtr parentNode, MKLDNNNodePtr childNode) { + auto isSuitableChildNode = [&](const MKLDNNNodePtr& parentNode, const MKLDNNNodePtr& childNode) { if (childNode->getAlgorithm() != EltwiseAdd || !childNode->getFusedWith().empty() || childNode->getParentEdges().size() != 2) return false; @@ -430,7 +442,7 @@ void MKLDNNGraphOptimizer::FuseMultiplyAndAdd(MKLDNNGraph &graph) { graph.RemoveEdge(remEdge); } - auto parentEltwise = parentNode; + auto& parentEltwise = parentNode; MKLDNNEdgePtr newEdge(new MKLDNNEdge(parent, parentEltwise, inNum, parentEltwise->getParentEdges().size())); auto &graphEdges = graph.GetEdges(); graphEdges.push_back(newEdge); @@ -1226,7 +1238,8 @@ void MKLDNNGraphOptimizer::FuseConvolutionSumAndConvolutionSumActivation(MKLDNNG // Merged with DW_conv. Shape may change mergedConv->inputShapes.push_back(mergedConv->fusedWith[0]->getOutputShapeAtPort(0)); } else { - mergedConv->inputShapes.push_back(sum->getInputShapeAtPort(1)); + size_t secondTermPort = sum->getFusingPort() == 0 ? 1 : 0; + mergedConv->inputShapes.push_back(sum->getInputShapeAtPort(secondTermPort)); } size_t childIdx = 0lu; @@ -1701,36 +1714,37 @@ void MKLDNNGraphOptimizer::FuseClampAndFakeQuantize(MKLDNNGraph &graph) { void MKLDNNGraphOptimizer::FusePerformedAsScaleShiftAndFakeQuantize(MKLDNNGraph &graph) { auto& graphNodes = graph.GetNodes(); - auto getConstPort = [](const MKLDNNNodePtr node) -> int { - if (node->getParentEdgesAtPort(0)[0]->getParent()->getType() == Input && node->getParentEdgesAtPort(0)[0]->getParent()->isConstant()) { - return 0; - } else if (node->getParentEdgesAtPort(1)[0]->getParent()->getType() == Input && node->getParentEdgesAtPort(1)[0]->getParent()->isConstant()) { - return 1; - } else { + auto getNonConstPort = [](const MKLDNNNodePtr& node) { + std::vector nonConstPorts; + for (size_t i = 0; i < node->getParentEdges().size(); i++) { + const auto& parent = node->getParentEdgeAt(i)->getParent(); + if (!(parent->getType() == Input && parent->isConstant())) + nonConstPorts.push_back(i); + } + // there are more than 1 nonconst port or missed + if (nonConstPorts.size() != 1) return -1; - } + + return nonConstPorts[0]; }; - auto isSuitableScaleShiftNode = [getConstPort](MKLDNNNodePtr node) { - if (one_of(node->getAlgorithm(), EltwiseAdd, EltwiseSubtract, EltwiseMultiply, EltwiseDivide, EltwiseMulAdd)) { - MKLDNNNode *parent = nullptr; - if (node->getAlgorithm() != EltwiseMulAdd) { - const auto constPort = getConstPort(node); - if (constPort == -1) { - return false; - } - parent = node->getParentEdgesAtPort(1 - constPort)[0]->getParent().get(); - } - return node->getType() == Eltwise && node->getChildEdges().size() == 1 && node->canBePerformedAsScaleShift(parent); - } - return false; + auto isSuitableScaleShiftNode = [getNonConstPort](const MKLDNNNodePtr& node) { + if (!one_of(node->getAlgorithm(), EltwiseAdd, EltwiseSubtract, EltwiseMultiply, EltwiseDivide, EltwiseMulAdd)) + return false; + + const auto nonConstPort = getNonConstPort(node); + if (nonConstPort == -1) + return false; + + const MKLDNNNodePtr eltwiseInput = node->getParentEdgeAt(nonConstPort)->getParent(); + return node->getChildEdges().size() == 1 && node->canBePerformedAsScaleShift(eltwiseInput.get()); }; - auto isSuitableFakeQuantizeNode = [](MKLDNNNodePtr node) { + auto isSuitableFakeQuantizeNode = [](const MKLDNNNodePtr& node) { return node->getType() == FakeQuantize && node->getAlgorithm() != FQBinarization; }; - auto fuseScaleShiftAndFakeQuantizeNodes = [getConstPort](MKLDNNNodePtr parent, MKLDNNNodePtr child) { + auto fuseScaleShiftAndFakeQuantizeNodes = [getNonConstPort](const MKLDNNNodePtr& parent, const MKLDNNNodePtr& child) { auto fakeQuantizeNode = std::dynamic_pointer_cast(child); if (fakeQuantizeNode == nullptr) IE_THROW() << "Cannot cast " << child->getName() << " to FakeQuantize node"; @@ -1742,11 +1756,13 @@ void MKLDNNGraphOptimizer::FusePerformedAsScaleShiftAndFakeQuantize(MKLDNNGraph IE_THROW() << "Cannot cast " << parent->getName() << " to Eltwise node"; } - std::tie(scalesBuffer, shiftsBuffer) = parentEltwise->getScalesAndShifts(parent->getParentEdgesAtPort(1 - getConstPort(parent))[0]->getParent().get()); + const MKLDNNNodePtr eltwiseInput = parentEltwise->getParentEdgeAt(getNonConstPort(parent))->getParent(); + std::tie(scalesBuffer, shiftsBuffer) = parentEltwise->getScalesAndShifts(eltwiseInput.get()); const auto &outputShape = child->getOutputShapeAtPort(0); VectorDims outputDims = outputShape.getDims(); - const size_t channelPos = outputDims.size() > 1 ? 1 : 0; + const size_t channelPos = parent->getParentEdgeAt(0)->getParent()->getFusingAxis(); + if (outputShape.isDynamic()) { if (outputDims[channelPos] == Shape::UNDEFINED_DIM) { if (scalesBuffer.size() > 1) { diff --git a/src/plugins/intel_cpu/src/ngraph_transformations/move_eltwise_up_data_movement.cpp b/src/plugins/intel_cpu/src/ngraph_transformations/move_eltwise_up_data_movement.cpp index 74eb8dc90cb..ab5717900f4 100644 --- a/src/plugins/intel_cpu/src/ngraph_transformations/move_eltwise_up_data_movement.cpp +++ b/src/plugins/intel_cpu/src/ngraph_transformations/move_eltwise_up_data_movement.cpp @@ -95,6 +95,9 @@ ov::intel_cpu::MoveEltwiseUpThroughDataMov::MoveEltwiseUpThroughDataMov() { ngraph::OutputVector eltwiseInputs = eltwise->input_values(); eltwiseInputs[0] = child->input_value(0); auto newEltwise = eltwise->clone_with_new_inputs(eltwiseInputs); + // WA: it's necessary to set empty friendly name here + // to avoid name duplication in TypeRelaxed cases + newEltwise->set_friendly_name(""); ngraph::copy_runtime_info(eltwise, newEltwise); ngraph::OutputVector childInputs = child->input_values(); diff --git a/src/plugins/intel_cpu/src/node.cpp b/src/plugins/intel_cpu/src/node.cpp index 02bbfe7f625..15945a2ecff 100644 --- a/src/plugins/intel_cpu/src/node.cpp +++ b/src/plugins/intel_cpu/src/node.cpp @@ -1263,11 +1263,12 @@ MKLDNNNode* MKLDNNNode::NodesFactory::create(const std::shared_ptr } bool MKLDNNNode::canBePerformedAsScaleShift(const MKLDNNNode *parentNode) const { - size_t fusingPort = 0; - // @todo graph optimizer can provide parentNode as nullptr. Should be avoided - const size_t channelAxis = parentNode ? parentNode->getFusingAxis() : MKLDNNNode::getFusingAxis(); + IE_ASSERT(parentNode); - for (size_t i = (parentNode == nullptr ? 1 : 0); i < getParentEdges().size(); i++) { + size_t fusingPort = 0; + const size_t channelAxis = parentNode->getFusingAxis(); + + for (size_t i = 0; i < getParentEdges().size(); i++) { MKLDNNNode *node = getParentEdgesAtPort(i)[0]->getParent().get(); if (node == nullptr) { IE_THROW() << "Cannot get parent node for " << getName() << " on " << i << " port"; diff --git a/src/plugins/intel_cpu/src/nodes/deconv.cpp b/src/plugins/intel_cpu/src/nodes/deconv.cpp index 1000c0850c8..adb012da93a 100644 --- a/src/plugins/intel_cpu/src/nodes/deconv.cpp +++ b/src/plugins/intel_cpu/src/nodes/deconv.cpp @@ -466,6 +466,20 @@ void MKLDNNDeconvolutionNode::setDynamicBatchLim(int lim) { MKLDNNNode::setDynamicBatchLim(lim); } +void MKLDNNDeconvolutionNode::cleanup() { + if (!isDynamicNode()) { + internalBlobs.clear(); + } + + for (auto it : fusedWith) { + it->cleanup(); + } + + for (auto it : mergedWith) { + it->cleanup(); + } +} + void MKLDNNDeconvolutionNode::execute(mkldnn::stream strm) { if (!execPtr) { IE_THROW() << "Can't execute Deconvolution node with name: " << getName() << ", because executor is not compiled"; diff --git a/src/plugins/intel_cpu/src/nodes/deconv.h b/src/plugins/intel_cpu/src/nodes/deconv.h index 25496851aad..151a737fcb7 100644 --- a/src/plugins/intel_cpu/src/nodes/deconv.h +++ b/src/plugins/intel_cpu/src/nodes/deconv.h @@ -56,6 +56,8 @@ public: void setDynamicBatchLim(int lim) override; + void cleanup() override; + protected: AttrPtr initPrimitiveAttr() override; AttrPtr makePrimitiveAttr(const VectorDims& dims); diff --git a/src/plugins/intel_cpu/src/nodes/def_conv.cpp b/src/plugins/intel_cpu/src/nodes/def_conv.cpp index fb82f296a80..87281c49beb 100644 --- a/src/plugins/intel_cpu/src/nodes/def_conv.cpp +++ b/src/plugins/intel_cpu/src/nodes/def_conv.cpp @@ -115,6 +115,11 @@ private: Xbyak::Label l_table; + inline void checkZeroWei(const Xbyak::Xmm &x1, Label &nullifyLabel) { + uni_vtestps(x1, x1); + jz(nullifyLabel); + } + void ow_loop() { Label ow_loop_main; Label ow_tail; @@ -280,6 +285,22 @@ private: Label ic_loop_main; Label ic_loop_tail; Label loop_end; + Label nullify_v1; + Label nullify_v2; + Label nullify_v3; + Label nullify_v4; + Label nullify_v1_end; + Label nullify_v2_end; + Label nullify_v3_end; + Label nullify_v4_end; + Label nullify_v1_tail; + Label nullify_v2_tail; + Label nullify_v3_tail; + Label nullify_v4_tail; + Label nullify_v1_end_tail; + Label nullify_v2_end_tail; + Label nullify_v3_end_tail; + Label nullify_v4_end_tail; mov(aux2_reg_input, aux_reg_input); add(aux2_reg_input, (ow * jcp_.stride_w * jcp_.ic) * jcp_.typesize_in); @@ -337,35 +358,69 @@ private: cmp(reg_ic_iter, simd_w); jl(ic_loop_tail, T_NEAR); + // check zero markers + uni_vbroadcastss(xmm_v1, dword[aux_reg_sampled_wei + ind_off_ll * jcp_.typesize_sampled_wei]); + uni_vbroadcastss(xmm_v2, dword[aux_reg_sampled_wei + ind_off_hl * jcp_.typesize_sampled_wei]); + uni_vbroadcastss(xmm_v3, dword[aux_reg_sampled_wei + ind_off_lh * jcp_.typesize_sampled_wei]); + uni_vbroadcastss(xmm_v4, dword[aux_reg_sampled_wei + ind_off_hh * jcp_.typesize_sampled_wei]); + size_t input_buffer_off = (size_t) kh * jcp_.kw * jcp_.ic + kw * jcp_.ic; uni_vpmovsxdq(xmm_v1_off, xmm_v1_off); uni_vmovq(reg_tmp_64, xmm_v1_off); imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in); add(reg_tmp_64, aux2_reg_input); + checkZeroWei(xmm_v1, nullify_v1); uni_vmovups(vmm_v1, ptr[reg_tmp_64]); uni_vmulps(vmm_v1, vmm_v1, vmm_w1); + jmp(nullify_v1_end, T_NEAR); + L(nullify_v1); + { + uni_vpxor(vmm_v1, vmm_v1, vmm_v1); + } + L(nullify_v1_end); uni_vpmovsxdq(xmm_v2_off, xmm_v2_off); uni_vmovq(reg_tmp_64, xmm_v2_off); imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in); add(reg_tmp_64, aux2_reg_input); + checkZeroWei(xmm_v2, nullify_v2); uni_vmovups(vmm_v2, ptr[reg_tmp_64]); uni_vmulps(vmm_v2, vmm_v2, vmm_w2); + jmp(nullify_v2_end, T_NEAR); + L(nullify_v2); + { + uni_vpxor(vmm_v2, vmm_v2, vmm_v2); + } + L(nullify_v2_end); uni_vpmovsxdq(xmm_v3_off, xmm_v3_off); uni_vmovq(reg_tmp_64, xmm_v3_off); imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in); add(reg_tmp_64, aux2_reg_input); + checkZeroWei(xmm_v3, nullify_v3); uni_vmovups(vmm_v3, ptr[reg_tmp_64]); uni_vmulps(vmm_v3, vmm_v3, vmm_w3); + jmp(nullify_v3_end, T_NEAR); + L(nullify_v3); + { + uni_vpxor(vmm_v3, vmm_v3, vmm_v3); + } + L(nullify_v3_end); uni_vpmovsxdq(xmm_v4_off, xmm_v4_off); uni_vmovq(reg_tmp_64, xmm_v4_off); imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in); add(reg_tmp_64, aux2_reg_input); + checkZeroWei(xmm_v4, nullify_v4); uni_vmovups(vmm_v4, ptr[reg_tmp_64]); uni_vmulps(vmm_v4, vmm_v4, vmm_w4); + jmp(nullify_v4_end, T_NEAR); + L(nullify_v4); + { + uni_vpxor(vmm_v4, vmm_v4, vmm_v4); + } + L(nullify_v4_end); uni_vaddps(vmm_v1, vmm_v1, vmm_v2); uni_vaddps(vmm_v1, vmm_v1, vmm_v3); @@ -383,34 +438,68 @@ private: cmp(reg_ic_iter, 1); jl(loop_end, T_NEAR); + // check zero markers + uni_vbroadcastss(xmm_v1, dword[aux_reg_sampled_wei + ind_off_ll * jcp_.typesize_sampled_wei]); + uni_vbroadcastss(xmm_v2, dword[aux_reg_sampled_wei + ind_off_hl * jcp_.typesize_sampled_wei]); + uni_vbroadcastss(xmm_v3, dword[aux_reg_sampled_wei + ind_off_lh * jcp_.typesize_sampled_wei]); + uni_vbroadcastss(xmm_v4, dword[aux_reg_sampled_wei + ind_off_hh * jcp_.typesize_sampled_wei]); + size_t input_buffer_off = (size_t) kh * jcp_.kw * jcp_.ic + kw * jcp_.ic; uni_vpmovsxdq(xmm_v1_off, xmm_v1_off); uni_vmovq(reg_tmp_64, xmm_v1_off); imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in); add(reg_tmp_64, aux2_reg_input); + checkZeroWei(xmm_v1, nullify_v1_tail); uni_vmovss(xmm_v1, ptr[reg_tmp_64]); uni_vmulss(xmm_v1, xmm_v1, xmm_w1); + jmp(nullify_v1_end_tail, T_NEAR); + L(nullify_v1_tail); + { + uni_vpxor(xmm_v1, xmm_v1, xmm_v1); + } + L(nullify_v1_end_tail); uni_vpmovsxdq(xmm_v2_off, xmm_v2_off); uni_vmovq(reg_tmp_64, xmm_v2_off); imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in); add(reg_tmp_64, aux2_reg_input); + checkZeroWei(xmm_v2, nullify_v2_tail); uni_vmovss(xmm_v2, ptr[reg_tmp_64]); uni_vmulss(xmm_v2, xmm_v2, xmm_w2); + jmp(nullify_v2_end_tail, T_NEAR); + L(nullify_v2_tail); + { + uni_vpxor(xmm_v2, xmm_v2, xmm_v2); + } + L(nullify_v2_end_tail); uni_vpmovsxdq(xmm_v3_off, xmm_v3_off); uni_vmovq(reg_tmp_64, xmm_v3_off); imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in); add(reg_tmp_64, aux2_reg_input); + checkZeroWei(xmm_v3, nullify_v3_tail); uni_vmovss(xmm_v3, ptr[reg_tmp_64]); uni_vmulss(xmm_v3, xmm_v3, xmm_w3); + jmp(nullify_v3_end_tail, T_NEAR); + L(nullify_v3_tail); + { + uni_vpxor(xmm_v3, xmm_v3, xmm_v3); + } + L(nullify_v3_end_tail); uni_vpmovsxdq(xmm_v4_off, xmm_v4_off); uni_vmovq(reg_tmp_64, xmm_v4_off); imul(reg_tmp_64, reg_tmp_64, jcp_.ic * jcp_.typesize_in); add(reg_tmp_64, aux2_reg_input); + checkZeroWei(xmm_v4, nullify_v4_tail); uni_vmovss(xmm_v4, ptr[reg_tmp_64]); uni_vmulss(xmm_v4, xmm_v4, xmm_w4); + jmp(nullify_v4_end_tail, T_NEAR); + L(nullify_v4_tail); + { + uni_vpxor(xmm_v4, xmm_v4, xmm_v4); + } + L(nullify_v4_end_tail); uni_vaddss(xmm_v1, xmm_v1, xmm_v2); uni_vaddss(xmm_v1, xmm_v1, xmm_v3); diff --git a/src/plugins/intel_cpu/src/nodes/input.cpp b/src/plugins/intel_cpu/src/nodes/input.cpp index 1425e70f77d..83f2d098ef2 100644 --- a/src/plugins/intel_cpu/src/nodes/input.cpp +++ b/src/plugins/intel_cpu/src/nodes/input.cpp @@ -249,12 +249,22 @@ MKLDNNInputNode::MKLDNNInputNode(const std::shared_ptr& op, const void MKLDNNInputNode::cloneBlobIfRequired() { Shape shape(constOp->get_shape().empty() ? ngraph::Shape(1, 1) : constOp->get_shape()); const auto prec = convertPrecision(constOp->get_element_type()); - const size_t size = shape.getRank(); + const size_t size = shape.getElementsCount(); DnnlBlockedMemoryDesc memDesc(prec, shape); auto cloneBlob = [&, this] () { MKLDNNMemory memory{ getEngine() }; - memory.Create(memDesc, constOp->get_data_ptr()); + + // CVS-74980 + // MKLDNN/oneDNN always allocate 1byte for element type with bitWidth < 8 (u4,u1...) + // but ngraph Constant uses actual bitWidth for data storage allocation + // in that case we make a copy to avoid overflow + if (constOp->get_byte_size() >= memDesc.getCurrentMemSize()) { + memory.Create(memDesc, constOp->get_data_ptr()); + } else { + memory.Create(memDesc); + memcpy(memory.GetPtr(), constOp->get_data_ptr(), constOp->get_byte_size()); + } MKLDNNMemoryPtr ptr = MKLDNNMemoryPtr(new MKLDNNMemory(getEngine())); ptr->Create(memDesc); diff --git a/src/plugins/intel_cpu/src/nodes/rnn.cpp b/src/plugins/intel_cpu/src/nodes/rnn.cpp index 5792057dd0e..08fc91c0ae1 100644 --- a/src/plugins/intel_cpu/src/nodes/rnn.cpp +++ b/src/plugins/intel_cpu/src/nodes/rnn.cpp @@ -810,16 +810,14 @@ void MKLDNNRNN::prepareParams() { bool wFormatWasChanged = false; // WA To avoid different weights layer and iter formats in FP32 case. - if (dataPrecision == Precision::FP32) { - if (SL != 1 || B < optimalBatchSize) { - if (wFormat != mkldnn::memory::format_tag::ldigo) { - wFormat = mkldnn::memory::format_tag::ldigo; - wFormatWasChanged = true; - } - } else if (wFormat != mkldnn::memory::format_tag::any) { - wFormat = mkldnn::memory::format_tag::any; + if (SL != 1 || B < optimalBatchSize) { + if (wFormat != mkldnn::memory::format_tag::ldigo) { + wFormat = mkldnn::memory::format_tag::ldigo; wFormatWasChanged = true; } + } else if (wFormat != mkldnn::memory::format_tag::any) { + wFormat = mkldnn::memory::format_tag::any; + wFormatWasChanged = true; } if (wFormatWasChanged) { auto weightsDims = MKLDNNExtensionUtils::convertToDnnlDims(VectorDims{ L, D, DC, G, SC }); diff --git a/src/plugins/intel_cpu/src/nodes/transpose.cpp b/src/plugins/intel_cpu/src/nodes/transpose.cpp index 996a49daf4d..2c3756105f6 100644 --- a/src/plugins/intel_cpu/src/nodes/transpose.cpp +++ b/src/plugins/intel_cpu/src/nodes/transpose.cpp @@ -14,6 +14,32 @@ using namespace mkldnn; using namespace ov::intel_cpu; using namespace InferenceEngine; +namespace { +struct TransposeAsReorderKey { + mkldnn::memory::desc src; + mkldnn::memory::desc dest; + size_t hash() const; + bool operator==(const TransposeAsReorderKey& rhs) const; +}; + +size_t TransposeAsReorderKey::hash() const { + using namespace dnnl::impl; + using namespace dnnl::impl::primitive_hashing; + + size_t seed = 0; + seed = hash_combine(seed, get_md_hash(src.data)); + seed = hash_combine(seed, get_md_hash(dest.data)); + + return seed; +} + +bool TransposeAsReorderKey::operator==(const TransposeAsReorderKey& rhs) const { + bool retVal = true; + retVal = src == rhs.src && dest == rhs.dest; + return retVal; +} +} // namespace + bool MKLDNNTransposeNode::isSupportedOperation(const std::shared_ptr& op, std::string& errorMessage) noexcept { try { if (!one_of(op->get_type_info(), @@ -122,6 +148,53 @@ void MKLDNNTransposeNode::prepareParams() { params.src_block_dims = srcDesc->getBlockDims(); auto dstDesc = getChildEdgeAt(0)->getMemory().GetDescWithType(); params.dst_block_dims = dstDesc->getBlockDims(); + + if (performAsReorder) { + mkldnn::primitive_attr attr; + const auto engine = getEngine(); + auto& dstMemPtr = getChildEdgeAt(0)->getMemoryPtr(); + auto& srcMemPtr = getParentEdgeAt(INPUT_DATA_IDX)->getMemoryPtr(); + MKLDNNMemoryPtr src_blocked = std::make_shared(engine); + MKLDNNMemoryPtr dst_blocked = std::make_shared(engine); + + dst_blocked->Create( + MKLDNNExtensionUtils::makeDescriptor(dstMemPtr->GetDescWithType()->getDnnlDesc()), + dstMemPtr->GetData(), false); + + const auto newDims = dst_blocked->getStaticDims(); + auto newDesc = mkldnn::memory::desc(MKLDNNExtensionUtils::convertToDnnlDims(newDims), + dst_blocked->GetDataType(), + memory::format_tag::acdb); + src_blocked->Create(MKLDNNExtensionUtils::makeDescriptor(newDesc), srcMemPtr->GetData(), false); + + impl_desc_type impl_type = getSelectedPrimitiveDescriptor()->getImplementationType(); + TransposeAsReorderKey key = {src_blocked->GetPrimitive().get_desc(), dst_blocked->GetPrimitive().get_desc()}; + auto builder = [&engine, &impl_type](const TransposeAsReorderKey& key) -> std::shared_ptr { + mkldnn::primitive_attr attr; + reorder::primitive_desc pd = mkldnn::reorder::primitive_desc(engine, key.src, engine, key.dest, attr, true); + + if (!pd) + return nullptr; + auto info = pd.impl_info_str(); + impl_type = parse_impl_name(info); + return std::make_shared(pd); + }; + + auto cache = getRuntimeCache(); + auto result = cache->getOrCreate(key, builder); + + if (!result.first) { + IE_THROW() << "Reorder primitive descriptor was not found for Transpose node " << getName() << "."; + } + + prim = result.first; + + supportedPrimitiveDescriptors[0].setImplementationType(impl_type); + primArgs = {{DNNL_ARG_SRC, getParentEdgesAtPort(INPUT_DATA_IDX)[0]->getMemoryPtr()->GetPrimitive()}, + {DNNL_ARG_DST, getChildEdgesAtPort(0)[0]->getMemoryPtr()->GetPrimitive()}}; + return; + } + if (!isInputOrderConst) { auto orderPtr = reinterpret_cast(getParentEdgeAt(0)->getMemoryPtr()->GetPtr()); auto orderLen = getParentEdgeAt(0)->getMemoryPtr()->GetSize(); @@ -154,6 +227,10 @@ void MKLDNNTransposeNode::createPrimitive() { IE_THROW() << "Preferable primitive descriptor was not set."; if (getParentEdgeAt(INPUT_DATA_IDX)->getMemory().getDesc().hasLayoutType(LayoutType::ncsp) && + getChildEdgeAt(0)->getMemory().getDesc().hasLayoutType(LayoutType::ncsp) && + order == std::vector{0, 3, 1, 2}) { + performAsReorder = true; + } else if (getParentEdgeAt(INPUT_DATA_IDX)->getMemory().getDesc().hasLayoutType(LayoutType::ncsp) && std::find(optimizedOrders.begin(), optimizedOrders.end(), order) != optimizedOrders.end()) { isOptimized = true; execPtr = std::make_shared(); @@ -276,7 +353,9 @@ void MKLDNNTransposeNode::optimizedExecute(const int MB, const MKLDNNMemoryPtr& } void MKLDNNTransposeNode::execute(mkldnn::stream strm) { - if (execPtr) { + if (prim) { + (*prim).execute(strm, primArgs); + } else if (execPtr) { auto &dstMemPtr = getChildEdgeAt(0)->getMemoryPtr(); auto &srcMemPtr = getParentEdgeAt(INPUT_DATA_IDX)->getMemoryPtr(); diff --git a/src/plugins/intel_cpu/src/nodes/transpose.h b/src/plugins/intel_cpu/src/nodes/transpose.h index 14e733d02f6..b9397315f17 100644 --- a/src/plugins/intel_cpu/src/nodes/transpose.h +++ b/src/plugins/intel_cpu/src/nodes/transpose.h @@ -92,6 +92,8 @@ private: static constexpr size_t INPUT_DATA_IDX = 0lu; static constexpr size_t INPUT_ORDER_IDX = 1lu; + + bool performAsReorder = false; }; } // namespace intel_cpu diff --git a/src/plugins/intel_cpu/src/plugin.cpp b/src/plugins/intel_cpu/src/plugin.cpp index b1e46577633..cad41cce996 100644 --- a/src/plugins/intel_cpu/src/plugin.cpp +++ b/src/plugins/intel_cpu/src/plugin.cpp @@ -106,6 +106,7 @@ #include #include #include "openvino/runtime/core.hpp" +#include "openvino/util/common_util.hpp" #include #include "performance_heuristics.hpp" @@ -284,6 +285,10 @@ static void TransformationUpToCPUSpecificOpSet(std::shared_ptr auto isSequencePrimitiveSupported = [](const_node_ptr &node) -> bool { const auto& data = node->input(0); const auto& data_pshape = data.get_partial_shape(); + // WA: dynamic shapes make impossible to check seq_len due to shapeOf subgraphs + // but the sequence is still supported in CPU and doesn't need to be decomposed + if (data_pshape.is_dynamic()) + return true; if (data_pshape.rank().is_static() && data_pshape.rank().get_length() > 1 && !data_pshape[1].is_static()) return false; auto max_seq_len = data.get_shape().at(1); @@ -568,6 +573,7 @@ void Engine::ApplyPerformanceHints(std::map &config, c if (mode_name == CONFIG_VALUE(LATENCY)) { config[CONFIG_KEY(CPU_THROUGHPUT_STREAMS)] = CONFIG_VALUE(CPU_THROUGHPUT_NUMA); + config[ov::num_streams.name()] = ov::util::to_string(ov::streams::NUMA); } else if (mode_name == CONFIG_VALUE(THROUGHPUT)) { const auto isa = dnnl::get_effective_cpu_isa(); float isaSpecificThreshold = 1.0f; @@ -625,6 +631,7 @@ void Engine::ApplyPerformanceHints(std::map &config, c engConfig.perfHintsConfig.ovPerfHintNumRequests); } config[CONFIG_KEY(CPU_THROUGHPUT_STREAMS)] = std::to_string(num_streams); + config[ov::num_streams.name()] = ov::util::to_string(num_streams); } } @@ -673,6 +680,16 @@ Engine::LoadExeNetworkImpl(const InferenceEngine::CNNNetwork &network, const std auto nGraphFunc = clonedNetwork.getFunction(); TransformationUpToCPUSpecificOpSet(nGraphFunc, enableLPT, enableSnippets, isLegacyAPI()); + // need to check that all outputs have static shapes + // checking that all inputs have static shapes is performed in the common part + if (isLegacyAPI()) { + for (const auto& res : nGraphFunc->get_results()) { + if (res->get_input_partial_shape(0).is_dynamic()) { + IE_THROW() << "CPU plug-in can't load a model with dynamic output shapes via legacy API."; + } + } + } + ApplyPerformanceHints(config, nGraphFunc); ConvertToCPUSpecificOpset(nGraphFunc); @@ -745,11 +762,11 @@ Parameter Engine::GetConfig(const std::string& name, const std::map(); + const auto perfHint = ov::util::from_string(engConfig.perfHintsConfig.ovPerfHint, ov::hint::performance_mode); + return perfHint; } else if (name == ov::hint::num_requests) { const auto perfHintNumRequests = engConfig.perfHintsConfig.ovPerfHintNumRequests; return decltype(ov::hint::num_requests)::value_type(perfHintNumRequests); @@ -840,12 +857,13 @@ Parameter Engine::GetMetric(const std::string& name, const std::map availableDevices = { "" }; - return availableDevices; + return decltype(ov::available_devices)::value_type(availableDevices); } else if (name == ov::device::capabilities) { std::vector capabilities; if (dnnl::impl::cpu::x64::mayiuse(dnnl::impl::cpu::x64::avx512_core_bf16)) @@ -857,13 +875,13 @@ Parameter Engine::GetMetric(const std::string& name, const std::map range = std::make_tuple(1, 1, 1); - return range; + return decltype(ov::range_for_async_infer_requests)::value_type(range); } else if (name == ov::range_for_streams) { const std::tuple range = std::make_tuple(1, parallel_get_max_threads()); - return range; + return decltype(ov::range_for_streams)::value_type(range); } /* Internally legacy parameters are used with new API as part of migration procedure. * This fallback can be removed as soon as migration completed */ diff --git a/src/plugins/intel_gna/backend/am_intel_dnn.cpp b/src/plugins/intel_gna/backend/am_intel_dnn.cpp index db1379916dc..b75cc3f265f 100644 --- a/src/plugins/intel_gna/backend/am_intel_dnn.cpp +++ b/src/plugins/intel_gna/backend/am_intel_dnn.cpp @@ -1518,6 +1518,9 @@ void GNAPluginNS::backend::AMIntelDNN::InitGNAStruct(Gna2Model *gnaModel, const if (i == 0) { THROW_GNA_EXCEPTION << "Pooling component with no preceeding component"; } else if (gnaOperation->Type == Gna2OperationTypeConvolution) { + if (gnaOperation->Operands == nullptr || gnaOperation->NumberOfOperands <= PwlOpIdx) { + THROW_GNA_EXCEPTION << "Number and details of operands are wrong"; + } auto pwlOperand = gnaOperation->Operands[PwlOpIdx]; if (pwlOperand != nullptr && pwlOperand->Shape.Dimensions[0] != 0 && gnaOperation->Operands[InOpIdx]->Shape.NumberOfDimensions == 2) { // kDnnConvolutional1dOp diff --git a/src/plugins/intel_gna/gna_graph_compiler.cpp b/src/plugins/intel_gna/gna_graph_compiler.cpp index 8cb40bb83e1..1e62ac6f15c 100644 --- a/src/plugins/intel_gna/gna_graph_compiler.cpp +++ b/src/plugins/intel_gna/gna_graph_compiler.cpp @@ -2392,6 +2392,9 @@ GNAPluginNS::ConnectionDetails GNAGraphCompiler::connectInput(CNNLayerPtr layer, auto prevLayer = CNNNetPrevLayerSkipCertain(layer, idx, [](CNNLayerPtr l) { return LayerInfo(l).isNonFunctional(); }); + if (!prevLayer) { + THROW_GNA_EXCEPTION << "Input layer was not found"; + } gnalog() << "Connecting input " << layer->name << " to " << prevLayer->name << " ...\n"; diff --git a/src/plugins/intel_gna/gna_plugin.cpp b/src/plugins/intel_gna/gna_plugin.cpp index 7d095ec4793..8c3b9fb3485 100644 --- a/src/plugins/intel_gna/gna_plugin.cpp +++ b/src/plugins/intel_gna/gna_plugin.cpp @@ -196,6 +196,9 @@ void GNAPlugin::ExportScores(void *ptr_dst, uint32_t num_vector_stride, Precision precision_in, Precision precision_out) { + if (ptr_src == nullptr || ptr_dst == nullptr) { + THROW_GNA_EXCEPTION << "Received null pointer arguments"; + } if (precision_out != Precision::I32 && precision_out != Precision::FP32) { THROW_GNA_EXCEPTION << "Unsupported target precision for infer : " << precision_out.name(); } diff --git a/src/plugins/intel_gpu/CMakeLists.txt b/src/plugins/intel_gpu/CMakeLists.txt index 06bf07bb4c8..9b30bbd67a5 100644 --- a/src/plugins/intel_gpu/CMakeLists.txt +++ b/src/plugins/intel_gpu/CMakeLists.txt @@ -46,7 +46,8 @@ set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_REL # Workaround to avoid warnings during LTO build if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set_target_properties(${TARGET_NAME} PROPERTIES LINK_FLAGS_RELEASE "-Wno-error=maybe-uninitialized -Wno-maybe-uninitialized") + set_target_properties(${TARGET_NAME} PROPERTIES LINK_FLAGS_RELEASE "-Wno-error=maybe-uninitialized -Wno-maybe-uninitialized" + LINK_FLAGS_RELWITHDEBINFO "-Wno-error=maybe-uninitialized -Wno-maybe-uninitialized") endif() if(ENABLE_TESTS) diff --git a/src/plugins/intel_gpu/include/intel_gpu/graph/program.hpp b/src/plugins/intel_gpu/include/intel_gpu/graph/program.hpp index c1e1e2ae6db..7c5cba3a587 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/graph/program.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/graph/program.hpp @@ -44,7 +44,6 @@ struct program { friend class prepare_conv_eltw_fusing; // to be removed when possible friend class reorder_inputs; // to be removed when possible friend class remove_redundant_reorders; // to be removed when possible - friend class handle_permute; // to be removed when possible friend class program_wrapper; // this class is intended to extend the interface of program for // the usage within tests_core_internal project only public: diff --git a/src/plugins/intel_gpu/include/intel_gpu/plugin/remote_context.hpp b/src/plugins/intel_gpu/include/intel_gpu/plugin/remote_context.hpp index c8334220e17..5ddefc3dc70 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/plugin/remote_context.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/plugin/remote_context.hpp @@ -89,6 +89,8 @@ protected: cldnn::memory::ptr m_memObject; + mutable std::mutex lockedMutex; + mutable size_t lockedCounter; mutable std::unique_ptr> lockedHolder; mutable void* _handle; mutable std::shared_ptr _allocator; diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/compile_graph.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/compile_graph.cpp index c749edc9784..5359ca294a3 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/compile_graph.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/compile_graph.cpp @@ -20,9 +20,8 @@ using namespace cldnn; void compile_graph::run(program& p) { OV_ITT_SCOPED_TASK(itt::domains::CLDNN, "CLDNN::pass::CompileGraph"); - size_t order_idx = 0; for (auto& node : p.get_processing_order()) { - node->set_unique_id(std::to_string(order_idx++)); + node->set_unique_id(); if (!node->is_type()) { node->get_output_layout(); } diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/handle_permute.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/handle_permute.cpp deleted file mode 100644 index 6359ccb2191..00000000000 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/handle_permute.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (C) 2022 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -#include "pass_manager.h" -#include "program_helpers.h" -#include "permute_inst.h" -#include "program_node.h" -#include "intel_gpu/graph/program.hpp" - -#include -#include -#include - -using namespace cldnn; - -void handle_permute::run(program& p) { - auto itr = p.get_processing_order().begin(); - while (itr != p.get_processing_order().end()) { - auto& node = (*itr++); - if (!node->is_type()) - continue; - - auto& perm_node = node->as(); - auto& prev_node = perm_node.get_dependencies().front(); - if (prev_node->get_output_layout().format == format::byxf && - perm_node.get_permute_order() == std::vector{ 0, 2, 3, 1 }) { - layout reorder_layout = perm_node.get_output_layout(); - reorder_layout.format = format::bfyx; - std::string reorder_name = perm_node.id() + "_converted_to_reorder"; - - auto new_reorder = std::make_shared(reorder_name, prev_node->id(), reorder_layout); - auto& new_reorder_node = p.get_or_create(new_reorder); - - p.replace(perm_node, new_reorder_node); - p.rename(new_reorder_node, reorder_name); - new_reorder_node.recalc_output_layout(); - } - } -} diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/post_input_reorder.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/post_input_reorder.cpp index a8ff3c4d739..d2ab615804d 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/post_input_reorder.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/post_input_reorder.cpp @@ -62,7 +62,7 @@ void post_input_reorder::run(program& p) { input_layout.size, input_layout.data_padding); auto& reorder = add_reorder(p, input, node, current_layout); - reorder.set_unique_id(node->get_unique_id() + "_input_reorder"); + reorder.set_unique_id(); reorder.get_output_layout(false); node->set_output_layout(previous_layout, false); reorder.set_selected_impl(reorder.type()->choose_impl(reorder)); diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp index 8a0579af82b..e2b3b6044ec 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp @@ -1076,7 +1076,7 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) { (!(user->is_type() && user->get_primitive()->input.size() == 2 && (std::find(supported_modes.begin(), supported_modes.end(), (user->as()).get_primitive()->mode) != supported_modes.end())) && - !(user->is_type() && user->get_primitive()->input.size() == 1))); + !(user->is_type() && user->get_dependency(0).get_users().size() == 1))); }); if (invalid_user_iter != curr_users.end()) { diff --git a/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp b/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp index eb6b8ca1ea6..a4476bf6b17 100644 --- a/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp +++ b/src/plugins/intel_gpu/src/graph/graph_optimizer/remove_redundant_reorders.cpp @@ -35,7 +35,7 @@ void remove_redundant_reorders::run(program& p) { if (!update_implementations) return; - node.set_unique_id(node.get_unique_id() + "_reorder"); + node.set_unique_id(); auto new_impl = node.type()->choose_impl(node); node.set_selected_impl(std::move(new_impl)); }; diff --git a/src/plugins/intel_gpu/src/graph/impls/ocl/cum_sum.cpp b/src/plugins/intel_gpu/src/graph/impls/ocl/cum_sum.cpp index b4f98213e41..3a6b08e365f 100644 --- a/src/plugins/intel_gpu/src/graph/impls/ocl/cum_sum.cpp +++ b/src/plugins/intel_gpu/src/graph/impls/ocl/cum_sum.cpp @@ -72,6 +72,12 @@ namespace detail { attach_cum_sum_impl::attach_cum_sum_impl() { implementation_map::add(impl_types::ocl, cum_sum_impl::create, { + std::make_tuple(data_types::i32, format::bfyx), + std::make_tuple(data_types::i32, format::bfzyx), + std::make_tuple(data_types::i32, format::bfwzyx), + std::make_tuple(data_types::i64, format::bfyx), + std::make_tuple(data_types::i64, format::bfzyx), + std::make_tuple(data_types::i64, format::bfwzyx), std::make_tuple(data_types::f16, format::bfyx), std::make_tuple(data_types::f16, format::bfzyx), std::make_tuple(data_types::f16, format::bfwzyx), diff --git a/src/plugins/intel_gpu/src/graph/include/pass_manager.h b/src/plugins/intel_gpu/src/graph/include/pass_manager.h index 59c89e79cef..7812dc481c0 100644 --- a/src/plugins/intel_gpu/src/graph/include/pass_manager.h +++ b/src/plugins/intel_gpu/src/graph/include/pass_manager.h @@ -132,14 +132,6 @@ private: void run(program& p) override; }; -class handle_permute : public base_pass { -public: - handle_permute() : base_pass("handle_permute") {} - -private: - void run(program& p) override; -}; - class mark_nodes : public base_pass { public: mark_nodes() : base_pass("analyzed_graph") {} diff --git a/src/plugins/intel_gpu/src/graph/include/program_node.h b/src/plugins/intel_gpu/src/graph/include/program_node.h index 6c80fc6958a..bc0d0c0607f 100644 --- a/src/plugins/intel_gpu/src/graph/include/program_node.h +++ b/src/plugins/intel_gpu/src/graph/include/program_node.h @@ -7,6 +7,7 @@ #include "intel_gpu/primitives/primitive.hpp" #include "intel_gpu/primitives/activation.hpp" #include "intel_gpu/primitives/implementation_desc.hpp" +#include "intel_gpu/graph/program.hpp" #include "kernel_selector_helper.h" #include "meta_utils.h" @@ -17,6 +18,7 @@ #include #include #include +#include namespace cldnn { @@ -350,11 +352,19 @@ public: bool need_lockable_memory() const; - std::string get_unique_id() const { return unique_id; } - void set_unique_id(std::string id) { unique_id = id; } + size_t get_unique_id() const { return unique_id; } + + void set_unique_id() { + unique_id = cur_id++; + } + + static void reset_unique_id() { + cur_id = 0; + } protected: - std::string unique_id; + size_t unique_id = 0; + static thread_local size_t cur_id; std::shared_ptr desc; program& myprog; diff --git a/src/plugins/intel_gpu/src/graph/kernel_selector_helper.cpp b/src/plugins/intel_gpu/src/graph/kernel_selector_helper.cpp index cab06f3e87e..66bc73c0afe 100644 --- a/src/plugins/intel_gpu/src/graph/kernel_selector_helper.cpp +++ b/src/plugins/intel_gpu/src/graph/kernel_selector_helper.cpp @@ -839,7 +839,7 @@ void set_params(const program_node& node, kernel_selector::params& params) { const auto& program = node.get_program(); const auto& device_info = program.get_engine().get_device_info(); - params.uniqueID = std::to_string(program.get_id()) + "_" + node.get_unique_id(); + params.uniqueID = std::to_string(node.get_unique_id()); params.engineInfo.bSubGroupSupport = device_info.supports_subgroups; params.engineInfo.bSubGroupShortSupport = device_info.supports_subgroups_short; params.engineInfo.bSubGroupCharSupport = device_info.supports_subgroups_char; diff --git a/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp b/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp index 646ee81a433..12278270388 100644 --- a/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp +++ b/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp @@ -765,16 +765,6 @@ static bool is_node_for_onednn(deconvolution_node const& node) { auto spatial_dims_num = input_layout.get_spatial_rank(); - // oneDNN doesn't support sum post ops for deconvolutions - for (auto& fused_op : node.get_fused_primitives()) { - if (fused_op.node->is_type() && fused_op.deps.size() == 1) { - auto eltw_in_layout = node.get_dependency(fused_op.dep_start_idx).get_output_layout(); - if (program_helpers::needs_onednn_sum_post_op(fused_op.node->as(), eltw_in_layout)) { - return false; - } - } - } - return onednn_valid_dt && onednn_valid_params && spatial_dims_num <= 3; } @@ -1501,6 +1491,12 @@ impl_types layout_optimizer::get_preferred_impl_type(program_node& node, format impl_candidate = impl_types::ocl; break; } + + if (fo.node->as().get_primitive()->mode == eltwise_mode::sum && + program_helpers::needs_onednn_sum_post_op(fo.node->as(), in_layout)) { + impl_candidate = impl_types::ocl; + break; + } // Gemm checkings // TODO: investigate why currently onednn gemm has some "sum" post-op restrictions // which don't correlate with fc checkings in the code above diff --git a/src/plugins/intel_gpu/src/graph/program.cpp b/src/plugins/intel_gpu/src/graph/program.cpp index 2518bd8903a..e9fdcca1890 100644 --- a/src/plugins/intel_gpu/src/graph/program.cpp +++ b/src/plugins/intel_gpu/src/graph/program.cpp @@ -107,6 +107,7 @@ program::program(engine& engine_ref, prepare_nodes(topology); _kernels_cache = std::unique_ptr(new kernels_cache(_engine, prog_id, kernel_selector::KernelBase::get_db().get_batch_header_str())); + program_node::reset_unique_id(); if (no_optimizations) { init_graph(); } else { @@ -482,8 +483,6 @@ void program::pre_optimize_graph(bool is_internal) { // handle symmetric and asymmetric padding for input apply_opt_pass(); - apply_opt_pass(); - processing_order.calculate_BFS_processing_order(); // this method makes sense only for OOOQ (out of order execution queue) apply_opt_pass(); diff --git a/src/plugins/intel_gpu/src/graph/program_node.cpp b/src/plugins/intel_gpu/src/graph/program_node.cpp index 61861e42bee..71a8beb99c7 100644 --- a/src/plugins/intel_gpu/src/graph/program_node.cpp +++ b/src/plugins/intel_gpu/src/graph/program_node.cpp @@ -3,10 +3,9 @@ // #include "program_node.h" -#include "intel_gpu/graph/program.hpp" #include "program_helpers.h" #include "primitive_inst.h" - +#include "loop_inst.h" #ifdef ENABLE_ONEDNN_FOR_GPU #include "convolution_inst.h" #include "quantize_inst.h" @@ -24,6 +23,8 @@ using namespace cldnn; +thread_local size_t program_node::cur_id = 0; + program_node::program_node(std::shared_ptr prim, program& prog) : desc(prim), myprog(prog), org_id(prim ? (prim->id) : 0) { if (prim) @@ -36,6 +37,11 @@ void program_node::replace_dependency(size_t idx, program_node& new_dep) { if (dependencies[idx] == &new_dep) return; + if (is_type()) { + loop_node& loop = *this; + loop.update_primitive_map(dependencies[idx]->id(), new_dep.id(), true); + } + auto it = std::find(dependencies[idx]->users.begin(), dependencies[idx]->users.end(), this); if (it != dependencies[idx]->users.end()) { dependencies[idx]->users.erase(it); @@ -387,7 +393,11 @@ dnnl::post_ops program_node::try_optimize_post_ops(dnnl::post_ops& p_ops, const float scale; dnnl::memory::data_type data_type; cur_p_ops.get_params_sum(idx, scale, data_type); - new_p_ops.append_sum(scale, data_type); + if (is_type()) { + new_p_ops.append_sum(scale, data_type); + } else { + new_p_ops.append_sum(scale); + } break; } @@ -664,7 +674,6 @@ dnnl::post_ops program_node::try_optimize_post_ops(dnnl::post_ops& p_ops, const } else if (sum_and_eltw) { dnnl::algorithm alg; float sum_scale, eltw_scale, alpha, beta; - dnnl::memory::data_type data_type; dnnl::algorithm next_alg; float next_scale, next_alpha, next_beta; @@ -685,14 +694,18 @@ dnnl::post_ops program_node::try_optimize_post_ops(dnnl::post_ops& p_ops, const // Try to optimize eltwise (any) + sum + eltwise_linear (with beta = 0) chain of operations if (can_optimize_eltw_and_sum) { + dnnl::memory::data_type data_type; p_ops.get_params_sum(cur_idx, sum_scale, data_type); p_ops.get_params_eltwise(prev_idx, eltw_scale, alg, alpha, beta); dnnl::post_ops eltw_p_op_prev, sum_p_op; eltw_p_op_prev.append_eltwise(eltw_scale * next_alpha * next_scale, alg, alpha, beta); - sum_p_op.append_sum(sum_scale * next_alpha, data_type); - + if (is_type()) { + sum_p_op.append_sum(sum_scale * next_alpha, data_type); + } else { + sum_p_op.append_sum(sum_scale * next_alpha); + } add_post_op(prev_type, eltw_p_op_prev, optimized_p_ops, 0); add_post_op(cur_type, sum_p_op, optimized_p_ops, 0); @@ -822,7 +835,11 @@ void program_node::init_onednn_primitive_attributes() { if (e_node.get_primitive()->mode == eltwise_mode::sum) { if (program_helpers::needs_onednn_sum_post_op(e_node, in)) { - post_ops.append_sum(1.0f, onednn::convert_data_type(in.data_type)); + if (is_type()) { + post_ops.append_sum(1.0f, onednn::convert_data_type(in.data_type)); + } else { + post_ops.append_sum(1.0f); + } update_onednn_post_op_list(onednn_post_op_type::sum, dep_idx); } else { dnnl::memory::desc in_desc = onednn::layout_to_memory_desc(in); diff --git a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/cum_sum/cum_sum_kernel_base.cpp b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/cum_sum/cum_sum_kernel_base.cpp index 7bd6402fb17..6268dd691bc 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/cum_sum/cum_sum_kernel_base.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/cum_sum/cum_sum_kernel_base.cpp @@ -42,8 +42,12 @@ ParamsKey CumSumKernelBase::GetSupportedKey() const { ParamsKey k; k.EnableInputDataType(Datatype::F16); k.EnableInputDataType(Datatype::F32); + k.EnableInputDataType(Datatype::INT32); + k.EnableInputDataType(Datatype::INT64); k.EnableOutputDataType(Datatype::F16); k.EnableOutputDataType(Datatype::F32); + k.EnableOutputDataType(Datatype::INT32); + k.EnableOutputDataType(Datatype::INT64); k.EnableInputLayout(DataLayout::bfyx); k.EnableInputLayout(DataLayout::bfzyx); diff --git a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/cum_sum/cum_sum_kernel_partial_sum.cpp b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/cum_sum/cum_sum_kernel_partial_sum.cpp index d61f244cfe4..e046bfebbf6 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/cum_sum/cum_sum_kernel_partial_sum.cpp +++ b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/cum_sum/cum_sum_kernel_partial_sum.cpp @@ -13,6 +13,26 @@ namespace kernel_selector { static constexpr size_t simd = 16; static constexpr size_t BLOCK_SIZE = 16; +ParamsKey CumSumKernelPartialSum::GetSupportedKey() const { + ParamsKey k; + k.EnableInputDataType(Datatype::F16); + k.EnableInputDataType(Datatype::F32); + k.EnableOutputDataType(Datatype::F16); + k.EnableOutputDataType(Datatype::F32); + + k.EnableInputLayout(DataLayout::bfyx); + k.EnableInputLayout(DataLayout::bfzyx); + k.EnableInputLayout(DataLayout::bfwzyx); + k.EnableOutputLayout(DataLayout::bfyx); + k.EnableOutputLayout(DataLayout::bfzyx); + k.EnableOutputLayout(DataLayout::bfwzyx); + + k.EnableTensorOffset(); + k.EnableTensorPitches(); + k.EnableBatching(); + return k; +} + JitConstants CumSumKernelPartialSum::GetJitConstants(const cum_sum_params& params, DispatchData dispatchData) const { auto jits = CumSumKernelBase::GetJitConstants(params, dispatchData); diff --git a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/cum_sum/cum_sum_kernel_partial_sum.h b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/cum_sum/cum_sum_kernel_partial_sum.h index 26277d9d9ba..1734e10c0cb 100644 --- a/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/cum_sum/cum_sum_kernel_partial_sum.h +++ b/src/plugins/intel_gpu/src/kernel_selector/core/actual_kernels/cum_sum/cum_sum_kernel_partial_sum.h @@ -11,6 +11,8 @@ class CumSumKernelPartialSum : public CumSumKernelBase { public: CumSumKernelPartialSum() : CumSumKernelBase("cum_sum_partial_sum") {} virtual ~CumSumKernelPartialSum() = default; + + ParamsKey GetSupportedKey() const override; protected: struct MultiDispatchData { DispatchData stage_1; diff --git a/src/plugins/intel_gpu/src/plugin/ops/transpose.cpp b/src/plugins/intel_gpu/src/plugin/ops/transpose.cpp index 85a70a93c52..8def02e4d3a 100644 --- a/src/plugins/intel_gpu/src/plugin/ops/transpose.cpp +++ b/src/plugins/intel_gpu/src/plugin/ops/transpose.cpp @@ -9,6 +9,7 @@ #include "ngraph/op/constant.hpp" #include "intel_gpu/primitives/permute.hpp" +#include "intel_gpu/primitives/reorder.hpp" namespace ov { namespace runtime { @@ -28,6 +29,30 @@ static void CreateTransposeOp(Program& p, const std::shared_ptrcast_vector(); } + auto is_convert_color_type = [](const std::shared_ptr &node) { + return ngraph::is_type(node) || + ngraph::is_type(node) || + ngraph::is_type(node) || + ngraph::is_type(node); + }; + + // Handle Transpose operation related to ConvertColor operation: + // In case of ConvertColor operation we have NHWC (byxf) input format which should be converted to + // NCHW (bfyx) by this Permute, so we replace Permute with Reorder (to bfyx) primitve + auto input = op->input(0).get_source_output().get_node_shared_ptr(); + if (is_convert_color_type(input) && ie_order == std::vector{0, 3, 1, 2}) { + auto precision = input->get_element_type(); + p.AddPrimitive(cldnn::reorder(layerName, + inputPrimitives[0], + cldnn::format::bfyx, + DataTypeFromPrecision(precision), + std::vector(), + cldnn::reorder_mean_mode::none, + op->get_friendly_name())); + p.AddPrimitiveToProfiler(op); + return; + } + int rank = std::max(4, static_cast(op->get_input_shape(0).size())); if (ie_order.empty()) { // if order size is less than 4 - fill the rest with just copy diff --git a/src/plugins/intel_gpu/src/plugin/remote_context.cpp b/src/plugins/intel_gpu/src/plugin/remote_context.cpp index 0720ec68eb4..1eaf01aea0b 100644 --- a/src/plugins/intel_gpu/src/plugin/remote_context.cpp +++ b/src/plugins/intel_gpu/src/plugin/remote_context.cpp @@ -25,7 +25,7 @@ RemoteBlobImpl::RemoteBlobImpl(ClContext::Ptr context, uint32_t plane, BlobType mem_type) : m_context(context), m_stream(stream), m_layout(layout), m_mem_type(mem_type), m_mem(mem), m_surf(surf), m_plane(plane), - _handle(nullptr), _allocator(nullptr), m_memObject(nullptr), lockedHolder(nullptr) { + _handle(nullptr), _allocator(nullptr), m_memObject(nullptr), lockedCounter(0), lockedHolder(nullptr) { auto _impl = getContextImpl(m_context.lock()); auto eng = _impl->GetEngine(); @@ -189,14 +189,22 @@ void RemoteBlobImpl::lock() const { if (!is_allocated()) { IE_THROW(NotAllocated) << "[GPU] Remote blob can't be locked as it's not allocated"; } - lockedHolder = std::unique_ptr>(new cldnn::mem_lock(m_memObject, m_stream)); - auto ptr = lockedHolder->data(); - _handle = reinterpret_cast(ptr); - m_allocator.regLockedBlob(_handle, this); + + std::lock_guard locker(lockedMutex); + if (lockedCounter == 0) { + lockedHolder = std::unique_ptr>(new cldnn::mem_lock(m_memObject, m_stream)); + auto ptr = lockedHolder->data(); + _handle = reinterpret_cast(ptr); + m_allocator.regLockedBlob(_handle, this); + } + lockedCounter++; } void RemoteBlobImpl::unlock() const { - lockedHolder.reset(); + std::lock_guard locker(lockedMutex); + lockedCounter--; + if (lockedCounter == 0) + lockedHolder.reset(); } LockedMemory RemoteBlobImpl::buffer() noexcept { diff --git a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp index 7fbc43833b1..ddcdaf5e71d 100644 --- a/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp +++ b/src/plugins/intel_gpu/src/plugin/transformations_pipeline.cpp @@ -233,6 +233,7 @@ void TransformationsPipeline::apply(std::shared_ptr func) { } else if (const auto &lstm_seq = std::dynamic_pointer_cast(node)) { return lstm_seq->get_clip() == 0.0f && lstm_seq->get_activations() == std::vector{"sigmoid", "tanh", "tanh"} && + max_seq_len < 16 && !ngraph::op::util::is_seq_len_provided(lstm_seq->get_input_node_shared_ptr(3), max_seq_len); } @@ -459,10 +460,9 @@ void TransformationsPipeline::apply(std::shared_ptr func) { [this](const std::shared_ptr &node) -> bool { auto sub_graph_op = std::dynamic_pointer_cast(node); int64_t num_iter = sub_graph_op->get_num_iterations(); - if (num_iter == 1) { - return false; - } - return !config.enable_loop_unrolling; + if (!config.enable_loop_unrolling) + return num_iter != 1; + return num_iter >= 16; }); manager.register_pass(); diff --git a/src/plugins/intel_gpu/tests/fusions/deconvolution_fusion_test.cpp b/src/plugins/intel_gpu/tests/fusions/deconvolution_fusion_test.cpp index b7b66adfb90..5a2effbbf82 100644 --- a/src/plugins/intel_gpu/tests/fusions/deconvolution_fusion_test.cpp +++ b/src/plugins/intel_gpu/tests/fusions/deconvolution_fusion_test.cpp @@ -700,7 +700,8 @@ INSTANTIATE_TEST_SUITE_P(fusings_gpu, deconv_scale_actv_quant_u8_eltw_scale_actv deconv_test_params{ CASE_DECONV_FP32_3D_1, 2, 9 }, // deconv_test_params{ CASE_DECONV_FP32_3D_2, 2, 9 }, - deconv_test_params{ CASE_DECONV_FP32_3D_3, 2, 9 }, + // Commented out due to sporadic CI failures + // deconv_test_params{ CASE_DECONV_FP32_3D_3, 2, 9 }, deconv_test_params{ CASE_DECONV_FP32_3D_4, 2, 9 }, deconv_test_params{ CASE_DECONV_FP32_3D_5, 2, 9 }, deconv_test_params{ CASE_DECONV_FP32_3D_6, 2, 9 }, diff --git a/src/plugins/intel_gpu/tests/test_cases/cum_sum_gpu_test.cpp b/src/plugins/intel_gpu/tests/test_cases/cum_sum_gpu_test.cpp index 0fda0dda745..ed4f0c8b962 100644 --- a/src/plugins/intel_gpu/tests/test_cases/cum_sum_gpu_test.cpp +++ b/src/plugins/intel_gpu/tests/test_cases/cum_sum_gpu_test.cpp @@ -26,6 +26,7 @@ static std::vector cumsum(const std::vector& input, std::vector output(input.size()); int dimNum = 0; std::vector reordered_shape = shape; + if (format == format::bfwzyx) { dimNum = 6; } else if (format == format::bfzyx) { @@ -39,6 +40,7 @@ static std::vector cumsum(const std::vector& input, reordered_shape[i] = shape[i + 2]; } } + std::vector sizeDim(dimNum); sizeDim[dimNum - 1] = 1; for (size_t i = dimNum - 1, mult = 1; i > 0; --i) { @@ -55,6 +57,7 @@ static std::vector cumsum(const std::vector& input, } return fullInd; }; + auto getIndex = [&sizeDim](std::vector fullInd) { size_t index = 0; for (size_t i = 0; i < fullInd.size(); ++i) { @@ -69,22 +72,23 @@ static std::vector cumsum(const std::vector& input, int stopInd = fullInd[axis] + 1; if (reverse) { stopInd = reordered_shape[axis]; - if (exclusive) + if (exclusive) { ++fullInd[axis]; - } - else { + } + } else { fullInd[axis] = 0; if (exclusive) { --stopInd; } } - float res = 0.f; + T res = (T)0; for (; fullInd[axis] < stopInd; ++fullInd[axis]) { auto ind = getIndex(fullInd); res += input[ind]; } - output[i] = res; + + output[i] = (T)res; } return output; } @@ -98,16 +102,6 @@ static std::vector vectorCast(const std::vector& vec) { return ret; } -template -static std::vector generateVector(size_t sz) { - std::vector vec(sz); - T n = 0; - std::generate(vec.begin(), vec.end(), [&n]() { - return n++; - }); - return vec; -} - static cldnn::cum_sum::cum_sum_axis getCumSumAxis(int axis, unsigned sz) { unsigned cldnn_axis = axis; if (axis >= 2) { @@ -133,6 +127,31 @@ static cldnn::cum_sum::cum_sum_axis getCumSumAxis(int axis, unsigned sz) { } } +#define CASE_CUM_SUM_AXIS_0 ::testing::Values(5), ::testing::Values(1), ::testing::Values(1), \ + ::testing::Values(1), ::testing::Values(1), ::testing::Values(1), \ + ::testing::Values(format::bfyx), ::testing::ValuesIn(axes[0]), \ + ::testing::ValuesIn(variants), ::testing::ValuesIn(variants) +#define CASE_CUM_SUM_AXIS_1 ::testing::Values(2), ::testing::Values(5), ::testing::Values(1), \ + ::testing::Values(1), ::testing::Values(1), ::testing::Values(1), \ + ::testing::Values(format::bfyx), ::testing::ValuesIn(axes[1]), \ + ::testing::ValuesIn(variants), ::testing::ValuesIn(variants) +#define CASE_CUM_SUM_AXIS_2 ::testing::Values(5), ::testing::Values(5), ::testing::Values(1), \ + ::testing::Values(1), ::testing::Values(5), ::testing::Values(1), \ + ::testing::Values(format::bfyx), ::testing::ValuesIn(axes[2]), \ + ::testing::ValuesIn(variants), ::testing::ValuesIn(variants) +#define CASE_CUM_SUM_AXIS_3 ::testing::Values(5), ::testing::Values(5), ::testing::Values(1), \ + ::testing::Values(1), ::testing::Values(5), ::testing::Values(5), \ + ::testing::Values(format::bfyx), ::testing::ValuesIn(axes[3]), \ + ::testing::ValuesIn(variants), ::testing::ValuesIn(variants) +#define CASE_CUM_SUM_AXIS_4 ::testing::Values(5), ::testing::Values(5), ::testing::Values(1), \ + ::testing::Values(5), ::testing::Values(5), ::testing::Values(5), \ + ::testing::Values(format::bfzyx), ::testing::ValuesIn(axes[4]), \ + ::testing::ValuesIn(variants), ::testing::ValuesIn(variants) +#define CASE_CUM_SUM_AXIS_5 ::testing::Values(5), ::testing::Values(5), ::testing::Values(5), \ + ::testing::Values(5), ::testing::Values(5), ::testing::Values(5), \ + ::testing::Values(format::bfwzyx), ::testing::ValuesIn(axes[5]), \ + ::testing::ValuesIn(variants), ::testing::ValuesIn(variants) + using cum_sum_test_params = std::tuple; // reverse -class cum_sum_gpu : public ::testing::TestWithParam {}; -TEST_P(cum_sum_gpu, basic_test) { - auto p = GetParam(); - auto& engine = get_test_engine(); +template +class cum_sum_gpu : public ::testing::TestWithParam { +public: - auto b = std::get<0>(p); - auto f = std::get<1>(p); - auto w = std::get<2>(p); - auto z = std::get<3>(p); - auto y = std::get<4>(p); - auto x = std::get<5>(p); - tensor shape = tensor{batch(b), feature(f), spatial(x, y, z, w)}; - auto in_out_format = std::get<6>(p); - auto axis = std::get<7>(p); - auto exclusive = std::get<8>(p); - auto reverse = std::get<9>(p); - auto size = 4; - if (in_out_format == format::bfwzyx) - size = 6; - else if (in_out_format == format::bfzyx) - size = 5; - - auto input = engine.allocate_memory({ data_types::f32, in_out_format, shape }); - const int inputSize = b * f * w * z * y * x; - auto inputVals = generateVector(inputSize); - - set_values(input, inputVals); - - topology topology; - topology.add(input_layout("Input0", input->get_layout())); - topology.add(cum_sum("cum_sum", "Input0", getCumSumAxis(axis, size), exclusive, reverse)); - - network network(engine, topology); - - network.set_input_data("Input0", input); - - auto outputs = network.execute(); - - EXPECT_EQ(outputs.size(), size_t(1)); - EXPECT_EQ(outputs.begin()->first, "cum_sum"); - - auto output = outputs.at("cum_sum").get_memory(); - cldnn::mem_lock output_ptr(output, get_test_stream()); - - auto answers = cumsum(inputVals, in_out_format, { b, f, w, z, y, x }, axis, exclusive, reverse); - ASSERT_EQ(output_ptr.size(), answers.size()); - for (size_t i = 0; i < answers.size(); ++i) - { - EXPECT_TRUE(are_equal(answers[i], output_ptr[i])) << i; + data_types get_alloc_data_type(void) { + if (std::is_same::value) + return data_types::f32; + else if (std::is_same::value) + return data_types::f16; + else if (std::is_same::value) + return data_types::i32; + else if (std::is_same::value) + return data_types::i64; + else + throw std::runtime_error("Unsupported cum sum data type in cum_sum_gpu_test.cpp"); } -} + + void execute(cum_sum_params& p) { + auto& engine = get_test_engine(); + + auto b = std::get<0>(p); + auto f = std::get<1>(p); + auto w = std::get<2>(p); + auto z = std::get<3>(p); + auto y = std::get<4>(p); + auto x = std::get<5>(p); + tensor shape = tensor{ batch(b), feature(f), spatial(x, y, z, w) }; + + auto in_out_format = std::get<6>(p); + auto axis = std::get<7>(p); + auto exclusive = std::get<8>(p); + auto reverse = std::get<9>(p); + + auto size = 4; + if (in_out_format == format::bfzyx) + size = 5; + else if (in_out_format == format::bfwzyx) + size = 6; + + auto input = engine.allocate_memory({ get_alloc_data_type(), in_out_format, shape }); + const int inputSize = b * f * w * z * y * x; + VF inputVals = std::is_same::value ? + generate_random_1d(inputSize, -1, 1, 1) : + generate_random_1d(inputSize, -100, 100, 8); + + set_values(input, inputVals); + + topology topology; + topology.add(input_layout("Input0", input->get_layout())); + topology.add(cum_sum("cum_sum", "Input0", getCumSumAxis(axis, size), exclusive, reverse)); + + network network(engine, topology); + + network.set_input_data("Input0", input); + + auto outputs = network.execute(); + + EXPECT_EQ(outputs.size(), size_t(1)); + EXPECT_EQ(outputs.begin()->first, "cum_sum"); + + auto output = outputs.at("cum_sum").get_memory(); + cldnn::mem_lock output_ptr(output, get_test_stream()); + + auto answers = cumsum(inputVals, in_out_format, { b, f, w, z, y, x }, axis, exclusive, reverse); + ASSERT_EQ(output_ptr.size(), answers.size()); + for (size_t i = 0; i < answers.size(); ++i) { + EXPECT_TRUE(are_equal(answers[i], output_ptr[i])) << i; + } + } +}; + +class cum_sum_gpu_fp16 : public ::cum_sum_gpu {}; +class cum_sum_gpu_fp32 : public ::cum_sum_gpu {}; +class cum_sum_gpu_int32 : public ::cum_sum_gpu {}; +class cum_sum_gpu_int64 : public ::cum_sum_gpu {}; + +TEST_P(cum_sum_gpu_fp16, basic) { auto p = GetParam(); execute(p); } +TEST_P(cum_sum_gpu_fp32, basic) { auto p = GetParam(); execute(p); } +TEST_P(cum_sum_gpu_int32, basic) { auto p = GetParam(); execute(p); } +TEST_P(cum_sum_gpu_int64, basic) { auto p = GetParam(); execute(p); } namespace { std::vector> axes = { - {0}, - {0, 1}, - {0, 1, 2}, - {0, 1, 2, 3}, - {0, 1, 2, 3, 4}, - {0, 1, 2, 3, 4, 5}, + { 0 }, + { 0, 1 }, + { 0, 1, 2 }, + { 0, 1, 2, 3 }, + { 0, 1, 2, 3, 4 }, + { 0, 1, 2, 3, 4, 5 }, }; - std::vector variants = {false, true}; + std::vector variants = { false, true }; } -INSTANTIATE_TEST_SUITE_P( - axis_0, - cum_sum_gpu, - ::testing::Combine( - ::testing::Values(5), - ::testing::Values(1), - ::testing::Values(1), - ::testing::Values(1), - ::testing::Values(1), - ::testing::Values(1), - ::testing::Values(format::bfyx), - ::testing::ValuesIn(axes[0]), - ::testing::ValuesIn(variants), - ::testing::ValuesIn(variants) - )); -INSTANTIATE_TEST_SUITE_P( - axis_1, - cum_sum_gpu, - ::testing::Combine( - ::testing::Values(2), - ::testing::Values(5), - ::testing::Values(1), - ::testing::Values(1), - ::testing::Values(1), - ::testing::Values(1), - ::testing::Values(format::bfyx), - ::testing::ValuesIn(axes[1]), - ::testing::ValuesIn(variants), - ::testing::ValuesIn(variants) - )); +INSTANTIATE_TEST_SUITE_P(axis_0, cum_sum_gpu_fp16, ::testing::Combine(CASE_CUM_SUM_AXIS_0)); +INSTANTIATE_TEST_SUITE_P(axis_0, cum_sum_gpu_fp32, ::testing::Combine(CASE_CUM_SUM_AXIS_0)); +INSTANTIATE_TEST_SUITE_P(axis_0, cum_sum_gpu_int32, ::testing::Combine(CASE_CUM_SUM_AXIS_0)); +INSTANTIATE_TEST_SUITE_P(axis_0, cum_sum_gpu_int64, ::testing::Combine(CASE_CUM_SUM_AXIS_0)); -INSTANTIATE_TEST_SUITE_P( - axis_2, - cum_sum_gpu, - ::testing::Combine( - ::testing::Values(5), - ::testing::Values(5), - ::testing::Values(1), - ::testing::Values(1), - ::testing::Values(5), - ::testing::Values(1), - ::testing::Values(format::bfyx), - ::testing::ValuesIn(axes[2]), - ::testing::ValuesIn(variants), - ::testing::ValuesIn(variants) - )); +INSTANTIATE_TEST_SUITE_P(axis_1, cum_sum_gpu_fp16, ::testing::Combine(CASE_CUM_SUM_AXIS_1)); +INSTANTIATE_TEST_SUITE_P(axis_1, cum_sum_gpu_fp32, ::testing::Combine(CASE_CUM_SUM_AXIS_1)); +INSTANTIATE_TEST_SUITE_P(axis_1, cum_sum_gpu_int32, ::testing::Combine(CASE_CUM_SUM_AXIS_1)); +INSTANTIATE_TEST_SUITE_P(axis_1, cum_sum_gpu_int64, ::testing::Combine(CASE_CUM_SUM_AXIS_1)); -INSTANTIATE_TEST_SUITE_P( - axis_3, - cum_sum_gpu, - ::testing::Combine( - ::testing::Values(5), - ::testing::Values(5), - ::testing::Values(1), - ::testing::Values(1), - ::testing::Values(5), - ::testing::Values(5), - ::testing::Values(format::bfyx), - ::testing::ValuesIn(axes[3]), - ::testing::ValuesIn(variants), - ::testing::ValuesIn(variants) - )); +INSTANTIATE_TEST_SUITE_P(axis_2, cum_sum_gpu_fp16, ::testing::Combine(CASE_CUM_SUM_AXIS_2)); +INSTANTIATE_TEST_SUITE_P(axis_2, cum_sum_gpu_fp32, ::testing::Combine(CASE_CUM_SUM_AXIS_2)); +INSTANTIATE_TEST_SUITE_P(axis_2, cum_sum_gpu_int32, ::testing::Combine(CASE_CUM_SUM_AXIS_2)); +INSTANTIATE_TEST_SUITE_P(axis_2, cum_sum_gpu_int64, ::testing::Combine(CASE_CUM_SUM_AXIS_2)); -INSTANTIATE_TEST_SUITE_P( - axis_4, - cum_sum_gpu, - ::testing::Combine( - ::testing::Values(5), - ::testing::Values(5), - ::testing::Values(1), - ::testing::Values(5), - ::testing::Values(5), - ::testing::Values(5), - ::testing::Values(format::bfzyx), - ::testing::ValuesIn(axes[4]), - ::testing::ValuesIn(variants), - ::testing::ValuesIn(variants) - )); +INSTANTIATE_TEST_SUITE_P(axis_3, cum_sum_gpu_fp16, ::testing::Combine(CASE_CUM_SUM_AXIS_3)); +INSTANTIATE_TEST_SUITE_P(axis_3, cum_sum_gpu_fp32, ::testing::Combine(CASE_CUM_SUM_AXIS_3)); +INSTANTIATE_TEST_SUITE_P(axis_3, cum_sum_gpu_int32, ::testing::Combine(CASE_CUM_SUM_AXIS_3)); +INSTANTIATE_TEST_SUITE_P(axis_3, cum_sum_gpu_int64, ::testing::Combine(CASE_CUM_SUM_AXIS_3)); -INSTANTIATE_TEST_SUITE_P( - axis_5, - cum_sum_gpu, - ::testing::Combine( - ::testing::Values(5), - ::testing::Values(5), - ::testing::Values(5), - ::testing::Values(5), - ::testing::Values(5), - ::testing::Values(5), - ::testing::Values(format::bfwzyx), - ::testing::ValuesIn(axes[5]), - ::testing::ValuesIn(variants), - ::testing::ValuesIn(variants) - )); +INSTANTIATE_TEST_SUITE_P(axis_4, cum_sum_gpu_fp16, ::testing::Combine(CASE_CUM_SUM_AXIS_4)); +INSTANTIATE_TEST_SUITE_P(axis_4, cum_sum_gpu_fp32, ::testing::Combine(CASE_CUM_SUM_AXIS_4)); +INSTANTIATE_TEST_SUITE_P(axis_4, cum_sum_gpu_int32, ::testing::Combine(CASE_CUM_SUM_AXIS_4)); +INSTANTIATE_TEST_SUITE_P(axis_4, cum_sum_gpu_int64, ::testing::Combine(CASE_CUM_SUM_AXIS_4)); + +INSTANTIATE_TEST_SUITE_P(axis_5, cum_sum_gpu_fp16, ::testing::Combine(CASE_CUM_SUM_AXIS_5)); +INSTANTIATE_TEST_SUITE_P(axis_5, cum_sum_gpu_fp32, ::testing::Combine(CASE_CUM_SUM_AXIS_5)); +INSTANTIATE_TEST_SUITE_P(axis_5, cum_sum_gpu_int32, ::testing::Combine(CASE_CUM_SUM_AXIS_5)); +INSTANTIATE_TEST_SUITE_P(axis_5, cum_sum_gpu_int64, ::testing::Combine(CASE_CUM_SUM_AXIS_5)); // FIXME: This test fails on some driver versions. Looks like UB in impl or driver issue TEST(cum_sum_gpu_f16, DISABLED_basic_1d) { diff --git a/src/plugins/intel_gpu/tests/test_cases/handle_permute_gpu_test.cpp b/src/plugins/intel_gpu/tests/test_cases/handle_permute_gpu_test.cpp deleted file mode 100644 index cee17b43627..00000000000 --- a/src/plugins/intel_gpu/tests/test_cases/handle_permute_gpu_test.cpp +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (C) 2022 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "test_utils.h" - -#include -#include -#include -#include - -using namespace cldnn; -using namespace ::tests; - -TEST(handle_permute, convert_permute_to_reorder) { - auto& engine = get_test_engine(); - - int32_t width = 224; - int32_t height = 448; - int32_t input_height = height + height / 2; - - auto input = engine.allocate_memory({ data_types::f32, format::byxf, { 1, 1, width, input_height } }); - - std::vector input_data = generate_random_1d(width * input_height, 0, 255); - set_values(input, input_data); - - layout output_layout(data_types::f32, cldnn::format::byxf, { 1, 3, width, height }); - - topology topology; - topology.add(input_layout("input", input->get_layout())); - topology.add(convert_color("convert_color", { "input" }, cldnn::convert_color::color_format::NV12, cldnn::convert_color::color_format::RGB, - cldnn::convert_color::memory_type::buffer, output_layout)); - topology.add(permute("permute", "convert_color", { 0, 2, 3, 1 })); - topology.add(resample("resample", "permute", { 1, 3, width, height })); - - network network(engine, topology); - network.set_input_data("input", input); - - auto outputs = network.execute(); - std::vector expected_shape = { 1, 3, width, height }; - std::vector output_shape = outputs.at("resample").get_memory()->get_layout().size.sizes(); - - for (size_t i = 0; i < expected_shape.size(); ++i) { - EXPECT_EQ(output_shape[i], expected_shape[i]); - } -} diff --git a/src/tests/README.md b/src/tests/README.md index 4dd2f8af5dc..32e5dfb5e08 100644 --- a/src/tests/README.md +++ b/src/tests/README.md @@ -15,7 +15,7 @@ This is OpenVINO Inference Engine testing framework. OpenVINO Inference Engine t files. > **Example**: We have `ie_reshaper.cpp` within the `src/shape_infer` subfolder of the tested module. In this case - new `shape_infer` subfolder should be created within the the root of the Unit Test folder for this module. And new + new `shape_infer` subfolder should be created within the root of the Unit Test folder for this module. And new `ie_reshaper_test.cpp` file should be created within this newly created subfolder. This test file should cover all the classes and methods from the original file. @@ -66,4 +66,4 @@ This is OpenVINO Inference Engine testing framework. OpenVINO Inference Engine t Internal namespaces (for example, `CommonTestUtils::`, `FuncTestUtils::` or `UnitTestUtils::`) must be used to separate utilities by domains. > **NOTE**: All the utilities libraries are added to the developer package and available for closed source - development. \ No newline at end of file + development. diff --git a/src/tests/functional/inference_engine/transformations/common_optimizations/dimension_tracking.cpp b/src/tests/functional/inference_engine/transformations/common_optimizations/dimension_tracking.cpp index df5b6343c1b..ca65499ef84 100644 --- a/src/tests/functional/inference_engine/transformations/common_optimizations/dimension_tracking.cpp +++ b/src/tests/functional/inference_engine/transformations/common_optimizations/dimension_tracking.cpp @@ -224,7 +224,7 @@ TEST(TransformationTests, AutoBatch_FindBatch_NegativeTracking) { ov::pass::Manager m; m.register_pass(); - m.register_pass(); + m.register_pass(false, false); m.run_passes(f); ASSERT_NO_THROW(check_rt_info(f)); @@ -238,6 +238,69 @@ TEST(TransformationTests, AutoBatch_FindBatch_NegativeTracking) { ASSERT_TRUE(!ov::DimensionTracker::get_label(out_shape[0])) << out_shape; } +TEST(TransformationTests, AutoBatch_FindBatch_AutoBatch_LabelPropagation_DO_detachment) { + const auto& data = std::make_shared(ov::element::f32, ov::Shape{1, 4, 10, 10}); + + const auto& constant_0 = std::make_shared(ov::element::f32, ov::Shape{1, 1, 1, 1}); + const auto& mul_0 = std::make_shared(data, constant_0); + + const auto& filters = std::make_shared(ov::element::f32, ov::Shape{1, 4, 1, 1}); + const auto& conv = std::make_shared( + mul_0, filters, ov::Strides{1, 1}, ov::CoordinateDiff{0, 0}, ov::CoordinateDiff{0, 0}, ov::Strides{1, 1}); + + const auto& box_logits_reshape = std::make_shared( + ov::element::i64, ov::Shape{2}, std::vector{0, -1}); + const auto& box_logits = std::make_shared(conv, box_logits_reshape, true); + + const auto& four_times = std::make_shared(box_logits, std::make_shared( + ov::element::i64, ov::Shape{2}, std::vector{1, 4})); + + const auto& third_input_reshape = std::make_shared( + ov::element::i64, ov::Shape{3}, std::vector{0, 1, -1}); + const auto& third_input = std::make_shared(four_times, third_input_reshape, true); + + ngraph::op::DetectionOutput::Attributes attr; + attr.num_classes = 4; + attr.background_label_id = 0; + attr.top_k = 75; + attr.variance_encoded_in_target = true; + attr.keep_top_k = {50}; + attr.code_type = std::string{"caffe.PriorBoxParameter.CORNER"}; + attr.share_location = true; + attr.nms_threshold = 0.5f; + attr.confidence_threshold = 0.5f; + attr.clip_after_nms = false; + attr.clip_before_nms = false; + attr.decrease_label_id = false; + attr.normalized = true; + attr.input_height = 1; + attr.input_width = 1; + attr.objectness_score = 0.4f; + + const auto& detection = std::make_shared(four_times, four_times, third_input, attr); + const auto& convert = std::make_shared(detection, ov::element::f32); + + const auto& f = std::make_shared(ov::NodeVector{convert}, ov::ParameterVector{data}); + + ov::pass::Manager m; + m.register_pass(); + m.register_pass(true); + m.run_passes(f); + ASSERT_NO_THROW(check_rt_info(f)); + + const auto& shape = data->get_partial_shape(); + ASSERT_TRUE(ov::DimensionTracker::get_label(shape[0])) << shape; + ASSERT_TRUE(!ov::DimensionTracker::get_label(shape[1])) << shape; + ASSERT_TRUE(!ov::DimensionTracker::get_label(shape[2])) << shape; + ASSERT_TRUE(!ov::DimensionTracker::get_label(shape[3])) << shape; + ASSERT_EQ(f->get_results().size(), 3); + for (const auto& result : f->get_results()) { + const auto& out_shape = result->get_output_partial_shape(0); + ASSERT_TRUE(ov::DimensionTracker::get_label(out_shape[0])) << out_shape; + ASSERT_TRUE(!ov::DimensionTracker::get_label(out_shape[1])) << out_shape; + } +} + TEST(partial_shape, cout_with_label) { ov::Dimension a = 5; ov::DimensionTracker::set_label(a, 100500); diff --git a/src/tests/functional/inference_engine/transformations/op_conversions/convert_interpolate1_to_interpolate4_test.cpp b/src/tests/functional/inference_engine/transformations/op_conversions/convert_interpolate1_to_interpolate4_test.cpp index 6650baffea6..8894b4c631e 100644 --- a/src/tests/functional/inference_engine/transformations/op_conversions/convert_interpolate1_to_interpolate4_test.cpp +++ b/src/tests/functional/inference_engine/transformations/op_conversions/convert_interpolate1_to_interpolate4_test.cpp @@ -56,6 +56,7 @@ TEST_F(TransformationTestsF, ConvertInterpolate1ToInterpolate4) { function_ref = std::make_shared(NodeVector{interpolate4}, ParameterVector{data_node}); } + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); } TEST_F(TransformationTestsF, ConvertInterpolate1ToInterpolate4_1) { @@ -93,4 +94,28 @@ TEST_F(TransformationTestsF, ConvertInterpolate1ToInterpolate4_1) { function_ref = std::make_shared(NodeVector{interpolate4}, ParameterVector{data_node}); } + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); } + +TEST(TransformationTests, DynamiShapeInterpolate1To4) { + auto data_node = std::make_shared(element::f32, PartialShape{-1, 5, {1, 10}, -1}); + auto out_shape_node = std::make_shared(element::i32, Shape{2}); + + auto interpolate1_attr = op::v0::InterpolateAttrs(); + interpolate1_attr.axes = AxisSet(std::vector{2, 3}); + interpolate1_attr.mode = "linear"; + interpolate1_attr.align_corners = false; + interpolate1_attr.antialias = true; + interpolate1_attr.pads_begin = std::vector{0, 0, 0, 0}; + interpolate1_attr.pads_end = std::vector{0, 0, 0, 0}; + + auto interpolate1 = std::make_shared(data_node, out_shape_node, interpolate1_attr); + auto f = std::make_shared(NodeVector{interpolate1}, ParameterVector{data_node, out_shape_node}); + + auto manager = ov::pass::Manager(); + manager.register_pass(); + manager.register_pass(); + manager.run_passes(f); + + ASSERT_TRUE(ngraph::op::util::has_op_with_type(f)); +} \ No newline at end of file diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/CMakeLists.txt b/src/tests/functional/plugin/conformance/subgraphs_dumper/CMakeLists.txt index 7570bcabcb9..a0cc8d349e8 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/CMakeLists.txt +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/CMakeLists.txt @@ -4,6 +4,13 @@ set(TARGET_NAME subgraphsDumper) +list(APPEND DEPENDENCIES + gflags + inference_engine + commonTestUtils + pugixml::static +) + addIeTargetTest( NAME ${TARGET_NAME} ROOT ${CMAKE_CURRENT_SOURCE_DIR} @@ -11,10 +18,9 @@ addIeTargetTest( ${CMAKE_CURRENT_SOURCE_DIR}/include LINK_LIBRARIES PRIVATE - gflags - inference_engine - commonTestUtils - pugixml::static + ${DEPENDENCIES} + DEPENDENCIES + ${DEPENDENCIES} ADD_CPPLINT ) diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/matchers/base_matcher.hpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/matchers/base_matcher.hpp index 4b6ce7d5adc..5954319880e 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/matchers/base_matcher.hpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/matchers/base_matcher.hpp @@ -37,7 +37,7 @@ public: bool is_fallback_config; bool ignore_matching = false; - virtual bool op_in_config(const std::shared_ptr &node) = 0; + virtual bool op_in_config(const std::shared_ptr &node) = 0; virtual ~iMatcherConfig() = default; }; @@ -54,8 +54,8 @@ public: MatcherConfig(bool ignore_matching) : iMatcherConfig({}, {}, sizeof...(OPTypes) == 0, ignore_matching) {} - bool op_in_config(const std::shared_ptr &node) override { - std::initializer_list vals{(ngraph::is_type(node))...}; + bool op_in_config(const std::shared_ptr &node) override { + std::initializer_list vals{(ov::is_type(node))...}; return std::any_of(vals.begin(), vals.end(), [](bool i) { return i; }); }; }; @@ -66,8 +66,8 @@ class Matcher { friend class MatchersManager; public: - virtual bool match(const std::shared_ptr &node, - const std::shared_ptr &ref, + virtual bool match(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) const = 0; virtual ~Matcher() = default; @@ -75,7 +75,7 @@ public: protected: virtual void configure(const pugi::xml_document &cfg) = 0; - iMatcherConfig::Ptr get_config(const std::shared_ptr &node) const; + iMatcherConfig::Ptr get_config(const std::shared_ptr &node) const; std::vector default_configs; diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/matchers/convolutions.hpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/matchers/convolutions.hpp index 29e0cd574e8..dff594200e1 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/matchers/convolutions.hpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/matchers/convolutions.hpp @@ -9,11 +9,11 @@ class ConvolutionsMatcher : public SingleOpMatcher { public: ConvolutionsMatcher(); - bool match_inputs(const std::shared_ptr &node, - const std::shared_ptr &ref, + bool match_inputs(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) const override; - bool match(const std::shared_ptr &node, - const std::shared_ptr &ref, + bool match(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) const override; protected: diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/matchers/matchers_manager.hpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/matchers/matchers_manager.hpp index 27446072cae..e48f117adc1 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/matchers/matchers_manager.hpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/matchers/matchers_manager.hpp @@ -19,20 +19,20 @@ public: explicit MatchersManager(const std::string &cfg_path = {}); - bool match_all(const std::shared_ptr &node, - const std::shared_ptr &ref, + bool match_all(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info); - bool match_any(const std::shared_ptr &node, - const std::shared_ptr &ref, + bool match_any(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info); // TODO: Implement default xml config file generation by Matchers void generate_config() {} private: - std::vector run_matchers(const std::shared_ptr &node, - const std::shared_ptr &ref, + std::vector run_matchers(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info); // TODO: No copy constructor for xml_document // pugi::xml_document m_cfg; diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/matchers/single_op.hpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/matchers/single_op.hpp index 490e582bae5..06e15cf99e2 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/matchers/single_op.hpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/matchers/single_op.hpp @@ -17,24 +17,24 @@ class SingleOpMatcher : public Matcher { public: SingleOpMatcher(); - bool match(const std::shared_ptr &node, - const std::shared_ptr &ref, + bool match(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) const override; - bool same_op_type(const std::shared_ptr &node, - const std::shared_ptr &ref, + bool same_op_type(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) const; - virtual bool match_inputs(const std::shared_ptr &node, - const std::shared_ptr &ref, + virtual bool match_inputs(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) const; - bool match_outputs(const std::shared_ptr &node, - const std::shared_ptr &ref, + bool match_outputs(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) const; - bool same_attrs(const std::shared_ptr &node, - const std::shared_ptr &ref, + bool same_attrs(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) const; - bool match_ports(const std::shared_ptr &node, - const std::shared_ptr &ref, + bool match_ports(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) const; protected: diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/op_cloner.hpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/op_cloner.hpp index a7fb23567c7..651105864bf 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/op_cloner.hpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/op_cloner.hpp @@ -14,9 +14,9 @@ namespace SubgraphsDumper { struct ClonersMap { - using clone_fn = std::function(const std::shared_ptr &, + using clone_fn = std::function(const std::shared_ptr &, LayerTestsUtils::OPInfo &meta)>; - using cloners_map_type = std::map; + using cloners_map_type = std::map; static float constant_size_threshold_mb; static const cloners_map_type cloners; diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/ops_cache.hpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/ops_cache.hpp index 79901e33b62..35d9dcc0247 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/include/ops_cache.hpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/include/ops_cache.hpp @@ -17,15 +17,15 @@ namespace SubgraphsDumper { class OPCache { public: OPCache() : num_neighbours_to_cache(0), manager(MatchersManager()), - m_ops_cache(std::map, LayerTestsUtils::OPInfo>()) {} + m_ops_cache(std::map, LayerTestsUtils::OPInfo>()) {} static std::unique_ptr make_cache() { return std::unique_ptr(new OPCache()); } - void update_ops_cache(const std::shared_ptr &op, const std::string &source_model = {}); + void update_ops_cache(const std::shared_ptr &op, const std::string &source_model = {}); - void update_ops_cache(const std::shared_ptr &func, const bool extract_body = true, const std::string &source_model = {}); + void update_ops_cache(const std::shared_ptr &func, const bool extract_body = true, const std::string &source_model = {}); void serialize_cached_ops(const std::string &serialization_dir); @@ -36,7 +36,7 @@ public: float get_size_of_cached_ops(); protected: - std::map, LayerTestsUtils::OPInfo> m_ops_cache; + std::map, LayerTestsUtils::OPInfo> m_ops_cache; MatchersManager manager; size_t num_neighbours_to_cache = 0; enum SerializationStatus { @@ -44,7 +44,7 @@ protected: FAILED = 1, RETRY = 2, }; - SerializationStatus serialize_function(const std::pair, LayerTestsUtils::OPInfo> &op_info, + SerializationStatus serialize_function(const std::pair, LayerTestsUtils::OPInfo> &op_info, const std::string &serialization_dir); }; } // namespace SubgraphsDumper diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/src/matchers/base_matcher.cpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/src/matchers/base_matcher.cpp index acb09dc7f02..fc40d4dccba 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/src/matchers/base_matcher.cpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/src/matchers/base_matcher.cpp @@ -6,7 +6,7 @@ #include "common_test_utils/common_utils.hpp" -SubgraphsDumper::iMatcherConfig::Ptr SubgraphsDumper::Matcher::get_config(const std::shared_ptr &node) const { +SubgraphsDumper::iMatcherConfig::Ptr SubgraphsDumper::Matcher::get_config(const std::shared_ptr &node) const { for (const auto &cfg : default_configs) { if (cfg->op_in_config(node)) { return cfg; diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/src/matchers/convolutions.cpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/src/matchers/convolutions.cpp index d346bad4942..668c07ddd8a 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/src/matchers/convolutions.cpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/src/matchers/convolutions.cpp @@ -8,15 +8,15 @@ using namespace SubgraphsDumper; ConvolutionsMatcher::ConvolutionsMatcher() { default_configs = { std::make_shared>(std::vector{}, std::vector{0, 1}) + ov::op::v1::Convolution, + ov::op::v1::ConvolutionBackpropData, + ov::op::v1::GroupConvolution, + ov::op::v1::GroupConvolutionBackpropData>>(std::vector{}, std::vector{0, 1}) }; } -bool ConvolutionsMatcher::match(const std::shared_ptr &node, - const std::shared_ptr &ref, +bool ConvolutionsMatcher::match(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) const { const auto &cfg = get_config(node); if (match_only_configured_ops() && cfg->is_fallback_config) { @@ -31,8 +31,8 @@ bool ConvolutionsMatcher::match(const std::shared_ptr &node, same_attrs(node, ref, op_info) && match_ports(node, ref, op_info); } -bool ConvolutionsMatcher::match_inputs(const std::shared_ptr &node, - const std::shared_ptr &ref, +bool ConvolutionsMatcher::match_inputs(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) const { if (node->get_input_size() != ref->get_input_size()) { return false; @@ -46,8 +46,8 @@ bool ConvolutionsMatcher::match_inputs(const std::shared_ptr &node if (!(rankIsEqual && elemTypeIsEqual && is_dynamic)) { return false; } - bool has_groups = std::dynamic_pointer_cast(node) != nullptr || - std::dynamic_pointer_cast(node); + bool has_groups = std::dynamic_pointer_cast(node) != nullptr || + std::dynamic_pointer_cast(node); size_t kernel_size_offset = has_groups ? 3 : 2; auto ref_weights_shape = ref->get_input_tensor(1).get_shape(); auto cur_weights_shape = node->get_input_tensor(1).get_shape(); diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/src/matchers/matchers_manager.cpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/src/matchers/matchers_manager.cpp index 33ad300b469..9c2a930e837 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/src/matchers/matchers_manager.cpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/src/matchers/matchers_manager.cpp @@ -6,8 +6,8 @@ using namespace SubgraphsDumper; -bool MatchersManager::match_any(const std::shared_ptr &node, - const std::shared_ptr &ref, +bool MatchersManager::match_any(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) { for (const auto &it : m_matchers) { if (it.second->match(node, ref, op_info)) return true; @@ -15,8 +15,8 @@ bool MatchersManager::match_any(const std::shared_ptr &node, return false; } -bool MatchersManager::match_all(const std::shared_ptr &node, - const std::shared_ptr &ref, +bool MatchersManager::match_all(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) { const auto matches = this->run_matchers(node, ref, op_info); return std::all_of(matches.begin(), matches.end(), [](bool i) { return i; }); @@ -31,8 +31,8 @@ MatchersManager::MatchersManager(const std::string &cfg_path) { } } -std::vector MatchersManager::run_matchers(const std::shared_ptr &node, - const std::shared_ptr &ref, +std::vector MatchersManager::run_matchers(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) { std::vector matches; for (const auto &it : m_matchers) { diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/src/matchers/single_op.cpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/src/matchers/single_op.cpp index dfb1b417734..06f779fe4c2 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/src/matchers/single_op.cpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/src/matchers/single_op.cpp @@ -10,10 +10,10 @@ using namespace SubgraphsDumper; template -bool compare_constants_data(const std::shared_ptr &op, - const std::shared_ptr &ref) { - size_t elements_count = ngraph::shape_size(op->get_shape()); - if (elements_count != ngraph::shape_size(ref->get_shape())) { +bool compare_constants_data(const std::shared_ptr &op, + const std::shared_ptr &ref) { + size_t elements_count = ov::shape_size(op->get_shape()); + if (elements_count != ov::shape_size(ref->get_shape())) { return false; } const auto &op_data = op->cast_vector(); @@ -29,37 +29,37 @@ bool compare_constants_data(const std::shared_ptr &op, } // TODO: Move to some utils? -bool compare_constants_data(const std::shared_ptr &op, - const std::shared_ptr &ref) { +bool compare_constants_data(const std::shared_ptr &op, + const std::shared_ptr &ref) { switch (op->get_element_type()) { - case ngraph::element::Type_t::boolean: + case ov::element::Type_t::boolean: return compare_constants_data(op, ref); - case ngraph::element::Type_t::bf16: - return compare_constants_data(op, ref); - case ngraph::element::Type_t::f16: - return compare_constants_data(op, ref); - case ngraph::element::Type_t::f32: + case ov::element::Type_t::bf16: + return compare_constants_data(op, ref); + case ov::element::Type_t::f16: + return compare_constants_data(op, ref); + case ov::element::Type_t::f32: return compare_constants_data(op, ref); - case ngraph::element::Type_t::f64: + case ov::element::Type_t::f64: return compare_constants_data(op, ref); - case ngraph::element::Type_t::i8: + case ov::element::Type_t::i8: return compare_constants_data(op, ref); - case ngraph::element::Type_t::i16: + case ov::element::Type_t::i16: return compare_constants_data(op, ref); - case ngraph::element::Type_t::i32: + case ov::element::Type_t::i32: return compare_constants_data(op, ref); - case ngraph::element::Type_t::i64: + case ov::element::Type_t::i64: return compare_constants_data(op, ref); // TODO cast_vector doesn't support u1 now -// case ngraph::element::Type_t::u1: +// case ov::element::Type_t::u1: // return compare_constants_data(op, ref); - case ngraph::element::Type_t::u8: + case ov::element::Type_t::u8: return compare_constants_data(op, ref); - case ngraph::element::Type_t::u16: + case ov::element::Type_t::u16: return compare_constants_data(op, ref); - case ngraph::element::Type_t::u32: + case ov::element::Type_t::u32: return compare_constants_data(op, ref); - case ngraph::element::Type_t::u64: + case ov::element::Type_t::u64: return compare_constants_data(op, ref); default: std::cout << "Can't compare constants" << op << " with " << ref << "\n" << "Unsupported data type"; @@ -67,15 +67,15 @@ bool compare_constants_data(const std::shared_ptr &op, } } -bool SingleOpMatcher::same_op_type(const std::shared_ptr &node, - const std::shared_ptr &ref, +bool SingleOpMatcher::same_op_type(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) const { return node->get_type_info().name == ref->get_type_info().name && node->get_type_info().version == ref->get_type_info().version; } -bool SingleOpMatcher::match_inputs(const std::shared_ptr &node, - const std::shared_ptr &ref, +bool SingleOpMatcher::match_inputs(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) const { if (node->get_input_size() != ref->get_input_size()) { return false; @@ -96,8 +96,8 @@ bool SingleOpMatcher::match_inputs(const std::shared_ptr &node, } bool -SingleOpMatcher::match_outputs(const std::shared_ptr &node, - const std::shared_ptr &ref, +SingleOpMatcher::match_outputs(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) const { if (node->get_output_size() != ref->get_output_size()) { return false; @@ -121,14 +121,14 @@ SingleOpMatcher::match_outputs(const std::shared_ptr &node, return true; } -bool SingleOpMatcher::same_attrs(const std::shared_ptr &node, - const std::shared_ptr &ref, +bool SingleOpMatcher::same_attrs(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) const { return attributes::compare(node.get(), ref.get(), Comparator::CmpValues::ATTRIBUTES).valid; } -bool SingleOpMatcher::match_ports(const std::shared_ptr &node, - const std::shared_ptr &ref, +bool SingleOpMatcher::match_ports(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) const { const auto &cfg = get_config(node); const std::vector &ignored_ports = cfg->ignored_ports; @@ -140,8 +140,8 @@ bool SingleOpMatcher::match_ports(const std::shared_ptr &node, const auto &cur_node_input = node->input_value(port_id); const auto &ref_node_input = ref->input_value(port_id); - const auto &cur_const_input = ngraph::get_constant_from_source(cur_node_input); - const auto &ref_const_input = ngraph::get_constant_from_source(ref_node_input); + const auto &cur_const_input = ov::get_constant_from_source(cur_node_input); + const auto &ref_const_input = ov::get_constant_from_source(ref_node_input); // Check that both OP an reference port inputs are constant and have same data if (cur_const_input && ref_const_input && @@ -155,8 +155,8 @@ bool SingleOpMatcher::match_ports(const std::shared_ptr &node, return true; } -bool SingleOpMatcher::match(const std::shared_ptr &node, - const std::shared_ptr &ref, +bool SingleOpMatcher::match(const std::shared_ptr &node, + const std::shared_ptr &ref, const LayerTestsUtils::OPInfo &op_info) const { for (const auto& input_node : node->inputs()) { if (input_node.get_partial_shape().is_dynamic()) { @@ -181,19 +181,19 @@ bool SingleOpMatcher::match(const std::shared_ptr &node, SingleOpMatcher::SingleOpMatcher() { default_configs = { std::make_shared>(std::vector{}, std::vector{0}), - std::make_shared>(std::vector{}, - std::vector{0, 1, 2, 3, 4}), + std::make_shared>(std::vector{}, + std::vector{0, 1, 2, 3, 4}), std::make_shared>(std::vector{}, std::vector{0, 1}), + ov::op::v0::MatMul, + ov::op::v1::Add, + ov::op::v1::Multiply, + ov::op::v1::Subtract, + ov::op::v1::Power>>(std::vector{}, std::vector{0, 1}), std::make_shared>(true) + ov::op::v1::Convolution, + ov::op::v1::ConvolutionBackpropData, + ov::op::v1::GroupConvolution, + ov::op::v1::GroupConvolutionBackpropData>>(true) }; } \ No newline at end of file diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/src/op_cloner.cpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/src/op_cloner.cpp index 2680b7f84d0..fbfd74abc1c 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/src/op_cloner.cpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/src/op_cloner.cpp @@ -11,7 +11,7 @@ namespace { template -void get_port_range(const std::shared_ptr &const_node, LayerTestsUtils::PortInfo &port_info) { +void get_port_range(const std::shared_ptr &const_node, LayerTestsUtils::PortInfo &port_info) { std::vector data = const_node->cast_vector(); if (!data.empty()) { auto min_max = std::minmax_element(data.begin(), data.end()); @@ -21,48 +21,48 @@ void get_port_range(const std::shared_ptr &const_node, Lay } -void get_port_range(const std::shared_ptr &constant_input, LayerTestsUtils::PortInfo &port_info) { +void get_port_range(const std::shared_ptr &constant_input, LayerTestsUtils::PortInfo &port_info) { switch (constant_input->get_element_type()) { - case ngraph::element::Type_t::boolean: + case ov::element::Type_t::boolean: get_port_range(constant_input, port_info); break; - case ngraph::element::Type_t::bf16: - get_port_range(constant_input, port_info); + case ov::element::Type_t::bf16: + get_port_range(constant_input, port_info); break; - case ngraph::element::Type_t::f16: - get_port_range(constant_input, port_info); + case ov::element::Type_t::f16: + get_port_range(constant_input, port_info); break; - case ngraph::element::Type_t::f32: + case ov::element::Type_t::f32: get_port_range(constant_input, port_info); break; - case ngraph::element::Type_t::f64: + case ov::element::Type_t::f64: get_port_range(constant_input, port_info); break; - case ngraph::element::Type_t::i8: + case ov::element::Type_t::i8: get_port_range(constant_input, port_info); break; - case ngraph::element::Type_t::i16: + case ov::element::Type_t::i16: get_port_range(constant_input, port_info); break; - case ngraph::element::Type_t::i32: + case ov::element::Type_t::i32: get_port_range(constant_input, port_info); break; - case ngraph::element::Type_t::i64: + case ov::element::Type_t::i64: get_port_range(constant_input, port_info); break; - case ngraph::element::Type_t::u1: + case ov::element::Type_t::u1: get_port_range(constant_input, port_info); break; - case ngraph::element::Type_t::u8: + case ov::element::Type_t::u8: get_port_range(constant_input, port_info); break; - case ngraph::element::Type_t::u16: + case ov::element::Type_t::u16: get_port_range(constant_input, port_info); break; - case ngraph::element::Type_t::u32: + case ov::element::Type_t::u32: get_port_range(constant_input, port_info); break; - case ngraph::element::Type_t::u64: + case ov::element::Type_t::u64: get_port_range(constant_input, port_info); break; default: @@ -70,36 +70,36 @@ void get_port_range(const std::shared_ptr &constant_input, } } -std::shared_ptr clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { - ngraph::OutputVector op_inputs; +std::shared_ptr clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { + ov::OutputVector op_inputs; bool static_inputs = true; for (size_t i = 0; i < node->get_input_size(); ++i) { const auto input = node->input(i).get_source_output(); static_inputs &= input.get_partial_shape().is_static(); auto port_info = LayerTestsUtils::PortInfo(); - const auto constant = ngraph::get_constant_from_source(input); + const auto constant = ov::get_constant_from_source(input); if (constant != nullptr) { get_port_range(constant, port_info); float weights_size = - static_cast(ngraph::shape_size(constant->get_shape()) * + static_cast(ov::shape_size(constant->get_shape()) * constant->get_element_type().size()) / (1024 * 1024); if (weights_size > ClonersMap::constant_size_threshold_mb) { std::cout << "Constant with size " << weights_size << " detected on port " << i << " of OP " << node << std::endl << "The constant will be replaced with parameter and initial data ranges meta info" << std::endl; - auto param = std::make_shared(constant->get_element_type(), - constant->get_shape()); + auto param = std::make_shared(constant->get_element_type(), + constant->get_shape()); op_inputs.push_back(param); } else { - const auto clone = std::make_shared(constant->get_element_type(), + const auto clone = std::make_shared(constant->get_element_type(), constant->get_shape(), constant->get_data_ptr()); op_inputs.push_back(clone); } } else { - auto param = std::make_shared(input.get_element_type(), + auto param = std::make_shared(input.get_element_type(), input.get_partial_shape()); op_inputs.push_back(param); } @@ -109,17 +109,17 @@ std::shared_ptr clone(const std::shared_ptr &node, L return op_clone; } -std::shared_ptr clone_weightable_node(const std::shared_ptr &node, +std::shared_ptr clone_weightable_node(const std::shared_ptr &node, const std::vector &weight_ports, LayerTestsUtils::OPInfo &meta) { - ngraph::OutputVector op_inputs; + ov::OutputVector op_inputs; for (size_t i = 0; i < node->get_input_size(); ++i) { const auto input = node->input(i).get_source_output(); - const auto constant_input = ngraph::get_constant_from_source(input); + const auto constant_input = ov::get_constant_from_source(input); auto port_info = LayerTestsUtils::PortInfo(); // Input is Parameter or dynamic data pass if (!constant_input) { - auto param = std::make_shared(input.get_element_type(), + auto param = std::make_shared(input.get_element_type(), input.get_partial_shape()); op_inputs.push_back(param); meta.ports_info[i] = port_info; @@ -129,18 +129,18 @@ std::shared_ptr clone_weightable_node(const std::shared_ptr(ngraph::shape_size(constant_input->get_shape()) * + static_cast(ov::shape_size(constant_input->get_shape()) * constant_input->get_element_type().size()) / (1024 * 1024); if (weights_size > ClonersMap::constant_size_threshold_mb) { std::cout << "Constant with size " << weights_size << " detected on port " << i << " of OP " << node << std::endl << "The constant will be replaced with parameter and initial data ranges meta info" << std::endl; - auto param = std::make_shared(constant_input->get_element_type(), - constant_input->get_shape()); + auto param = std::make_shared(constant_input->get_element_type(), + constant_input->get_shape()); op_inputs.push_back(param); } else { - const auto clone = std::make_shared(constant_input->get_element_type(), + const auto clone = std::make_shared(constant_input->get_element_type(), constant_input->get_shape(), constant_input->get_data_ptr()); op_inputs.push_back(clone); @@ -149,8 +149,8 @@ std::shared_ptr clone_weightable_node(const std::shared_ptr(constant_input->get_element_type(), - constant_input->get_shape()); + auto param = std::make_shared(constant_input->get_element_type(), + constant_input->get_shape()); port_info.convert_to_const = true; meta.ports_info[i] = port_info; op_inputs.push_back(param); @@ -160,52 +160,52 @@ std::shared_ptr clone_weightable_node(const std::shared_ptr -clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { +std::shared_ptr +clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { return clone_weightable_node(node, {1}, meta); } -std::shared_ptr -clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { +std::shared_ptr +clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { return clone_weightable_node(node, {1}, meta); } -std::shared_ptr -clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { +std::shared_ptr +clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { return clone_weightable_node(node, {1}, meta); } -std::shared_ptr -clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { +std::shared_ptr +clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { return clone_weightable_node(node, {1}, meta); } -std::shared_ptr -clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { +std::shared_ptr +clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { return clone_weightable_node(node, {0, 1}, meta); } -std::shared_ptr clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { +std::shared_ptr clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { return clone_weightable_node(node, {0, 1}, meta); } -std::shared_ptr -clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { +std::shared_ptr +clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { return clone_weightable_node(node, {0, 1}, meta); } -std::shared_ptr -clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { +std::shared_ptr +clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { return clone_weightable_node(node, {0, 1}, meta); } -std::shared_ptr clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { +std::shared_ptr clone(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { return clone_weightable_node(node, {0, 1}, meta); } template -std::shared_ptr clone_node(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { - return clone(ngraph::as_type_ptr(node), meta); +std::shared_ptr clone_node(const std::shared_ptr &node, LayerTestsUtils::OPInfo &meta) { + return clone(ov::as_type_ptr(node), meta); } } // namespace diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/src/ops_cache.cpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/src/ops_cache.cpp index 85770913eeb..af72f961e34 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/src/ops_cache.cpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/src/ops_cache.cpp @@ -13,7 +13,7 @@ using namespace SubgraphsDumper; -void OPCache::update_ops_cache(const std::shared_ptr &op, +void OPCache::update_ops_cache(const std::shared_ptr &op, const std::string &source_model) { const bool op_found = [&] { for (auto &&it : m_ops_cache) { @@ -28,7 +28,7 @@ void OPCache::update_ops_cache(const std::shared_ptr &op, const auto &clone_fn = SubgraphsDumper::ClonersMap::cloners.at(op->get_type_info()); LayerTestsUtils::OPInfo meta(source_model); try { - const std::shared_ptr op_clone = clone_fn(op, meta); + const std::shared_ptr op_clone = clone_fn(op, meta); op_clone->set_friendly_name(op_clone->get_friendly_name() + "_cached"); m_ops_cache.insert({op_clone, meta}); } catch (std::exception &e) { @@ -37,33 +37,33 @@ void OPCache::update_ops_cache(const std::shared_ptr &op, } } -void OPCache::update_ops_cache(const std::shared_ptr &func, const bool extract_body, const std::string &source_model) { +void OPCache::update_ops_cache(const std::shared_ptr &func, const bool extract_body, const std::string &source_model) { size_t cached_ops_count = m_ops_cache.size(); for (const auto &op : func->get_ordered_ops()) { - if (ngraph::is_type(op) || - ngraph::is_type(op) || - ngraph::is_type(op) || + if (ov::is_type(op) || + ov::is_type(op) || + ov::is_type(op) || // ReadValue and Assign have to be handled in pair // Will be handled as part of 48838 - ngraph::is_type(op) || - ngraph::is_type(op) + ov::is_type(op) || + ov::is_type(op) ) { continue; } if (extract_body) { - if (ngraph::is_type(op)) { - auto if_op = std::dynamic_pointer_cast(op); - std::vector> bodies; + if (ov::is_type(op)) { + auto if_op = std::dynamic_pointer_cast(op); + std::vector> bodies; for (size_t i = 0; i < if_op->get_internal_subgraphs_size(); i++) { auto if_body = if_op->get_function(i); update_ops_cache(if_body, extract_body, source_model); } - } else if (ngraph::is_type(op)) { - auto loop = std::dynamic_pointer_cast(op); + } else if (ov::is_type(op)) { + auto loop = std::dynamic_pointer_cast(op); auto loop_body = loop->get_function(); update_ops_cache(loop_body, extract_body, source_model); - } else if (ngraph::is_type(op)) { - auto ti = std::dynamic_pointer_cast(op); + } else if (ov::is_type(op)) { + auto ti = std::dynamic_pointer_cast(op); auto ti_body = ti->get_body(); update_ops_cache(ti_body, extract_body, source_model); } @@ -123,10 +123,10 @@ float OPCache::get_size_of_cached_ops() { float size = 0; for (const auto &op : m_ops_cache) { for (size_t i = 0; i < op.first->get_input_size(); ++i) { - const auto constant = std::dynamic_pointer_cast( + const auto constant = std::dynamic_pointer_cast( op.first->get_input_node_shared_ptr(i)); if (constant != nullptr) { - size += static_cast(ngraph::shape_size(constant->get_shape()) * + size += static_cast(ov::shape_size(constant->get_shape()) * constant->get_element_type().size()) / (1024 * 1024); } } @@ -135,7 +135,7 @@ float OPCache::get_size_of_cached_ops() { } OPCache::SerializationStatus -OPCache::serialize_function(const std::pair, LayerTestsUtils::OPInfo> &op, +OPCache::serialize_function(const std::pair, LayerTestsUtils::OPInfo> &op, const std::string &serialization_dir) { try { if (op.first->get_friendly_name() == "Relu_8793_cached") { @@ -144,19 +144,19 @@ OPCache::serialize_function(const std::pair, Layer std::cout << "Serializing function wrapping op " << op.first << std::endl; std::cout << "Taken from model: " << op.second.source_model << std::endl; - ngraph::ParameterVector params; + ov::ParameterVector params; for (size_t i = 0; i < op.first->get_input_size(); ++i) { - if (ngraph::op::is_parameter(op.first->get_input_node_ptr(i))) { - auto param = std::dynamic_pointer_cast( + if (ov::op::util::is_parameter(op.first->get_input_node_ptr(i))) { + auto param = std::dynamic_pointer_cast( op.first->get_input_node_shared_ptr(i)); params.push_back(param); } } - ngraph::ResultVector results; + ov::ResultVector results; for (auto &out : op.first->outputs()) { - results.push_back(std::make_shared(out)); + results.push_back(std::make_shared(out)); } - auto function = std::make_shared(results, params); + auto function = std::make_shared(results, params); // TODO: How to define element type for multi-output ops auto op_el_type = op.first->get_output_element_type(0).get_type_name(); diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/tests/matchers/convolutions_matcher.cpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/tests/matchers/convolutions_matcher.cpp index 141d18c9952..f8b1cafb907 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/tests/matchers/convolutions_matcher.cpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/tests/matchers/convolutions_matcher.cpp @@ -7,10 +7,6 @@ #include "ngraph/ops.hpp" #include "functional_test_utils/include/functional_test_utils/layer_test_utils/op_info.hpp" -using namespace ngraph::op; -using namespace ngraph; -using ngraph::element::Type_t; - class ConvolutionMatcherTest : public ::testing::Test { protected: void SetUp() override { @@ -23,58 +19,58 @@ protected: }; -// Check that two convolutions with different input shapes but same kernel size are match each other +// Check that two convolutions with different input ov::Shapes but same kernel size are match each other TEST_F(ConvolutionMatcherTest, ConvsSameKernelSize) { - const auto param = std::make_shared(Type_t::f32, Shape({1, 3, 10, 10})); - const auto weights = std::make_shared(Type_t::f32, Shape({10, 3, 3, 3}), 1); - const auto op1 = std::make_shared(param, weights, Strides(0, 0), CoordinateDiff(0, 0), - CoordinateDiff(0, 0), Strides(0, 0)); + const auto param = std::make_shared(ov::element::Type_t::f32, ov::Shape({1, 3, 10, 10})); + const auto weights = std::make_shared(ov::element::Type_t::f32, ov::Shape({10, 3, 3, 3}), 1); + const auto op1 = std::make_shared(param, weights, ov::Strides(0, 0), ov::CoordinateDiff(0, 0), + ov::CoordinateDiff(0, 0), ov::Strides(0, 0)); - const auto param2 = std::make_shared(Type_t::f32, Shape({1, 5, 20, 20})); - const auto weights2 = std::make_shared(Type_t::f32, Shape({10, 5, 3, 3}), 1); - const auto op2 = std::make_shared(param2, weights2, Strides(0, 0), CoordinateDiff(0, 0), - CoordinateDiff(0, 0), Strides(0, 0)); + const auto param2 = std::make_shared(ov::element::Type_t::f32, ov::Shape({1, 5, 20, 20})); + const auto weights2 = std::make_shared(ov::element::Type_t::f32, ov::Shape({10, 5, 3, 3}), 1); + const auto op2 = std::make_shared(param2, weights2, ov::Strides(0, 0), ov::CoordinateDiff(0, 0), + ov::CoordinateDiff(0, 0), ov::Strides(0, 0)); ASSERT_TRUE(matcher.match(op1, op2, op_info)); } -// Check that two convolutions with different input shapes but same kernel size are match each other +// Check that two convolutions with different input ov::Shapes but same kernel size are match each other TEST_F(ConvolutionMatcherTest, ConvsDifferentKernelSize) { - const auto param = std::make_shared(Type_t::f32, Shape({1, 3, 10, 10})); - const auto weights = std::make_shared(Type_t::f32, Shape({10, 3, 3, 5}), 1); - const auto op1 = std::make_shared(param, weights, Strides(0, 0), CoordinateDiff(0, 0), - CoordinateDiff(0, 0), Strides(0, 0)); + const auto param = std::make_shared(ov::element::Type_t::f32, ov::Shape({1, 3, 10, 10})); + const auto weights = std::make_shared(ov::element::Type_t::f32, ov::Shape({10, 3, 3, 5}), 1); + const auto op1 = std::make_shared(param, weights, ov::Strides(0, 0), ov::CoordinateDiff(0, 0), + ov::CoordinateDiff(0, 0), ov::Strides(0, 0)); - const auto param2 = std::make_shared(Type_t::f32, Shape({1, 5, 20, 20})); - const auto weights2 = std::make_shared(Type_t::f32, Shape({10, 5, 3, 3}), 1); - const auto op2 = std::make_shared(param2, weights2, Strides(0, 0), CoordinateDiff(0, 0), - CoordinateDiff(0, 0), Strides(0, 0)); + const auto param2 = std::make_shared(ov::element::Type_t::f32, ov::Shape({1, 5, 20, 20})); + const auto weights2 = std::make_shared(ov::element::Type_t::f32, ov::Shape({10, 5, 3, 3}), 1); + const auto op2 = std::make_shared(param2, weights2, ov::Strides(0, 0), ov::CoordinateDiff(0, 0), + ov::CoordinateDiff(0, 0), ov::Strides(0, 0)); ASSERT_FALSE(matcher.match(op1, op2, op_info)); } -// Check that two group convolutions with different input shapes but same kernel size are match each other +// Check that two group convolutions with different input ov::Shapes but same kernel size are match each other TEST_F(ConvolutionMatcherTest, GroupConvsSameKernelSize) { - const auto param = std::make_shared(Type_t::f32, Shape({1, 4, 10, 10})); - const auto weights = std::make_shared(Type_t::f32, Shape({2, 10, 2, 3, 3}), 1); - const auto op1 = std::make_shared(param, weights, Strides(0, 0), CoordinateDiff(0, 0), - CoordinateDiff(0, 0), Strides(0, 0)); + const auto param = std::make_shared(ov::element::Type_t::f32, ov::Shape({1, 4, 10, 10})); + const auto weights = std::make_shared(ov::element::Type_t::f32, ov::Shape({2, 10, 2, 3, 3}), 1); + const auto op1 = std::make_shared(param, weights, ov::Strides(0, 0), ov::CoordinateDiff(0, 0), + ov::CoordinateDiff(0, 0), ov::Strides(0, 0)); - const auto param2 = std::make_shared(Type_t::f32, Shape({1, 6, 20, 20})); - const auto weights2 = std::make_shared(Type_t::f32, Shape({2, 10, 3, 3, 3}), 1); - const auto op2 = std::make_shared(param2, weights2, Strides(0, 0), CoordinateDiff(0, 0), - CoordinateDiff(0, 0), Strides(0, 0)); + const auto param2 = std::make_shared(ov::element::Type_t::f32, ov::Shape({1, 6, 20, 20})); + const auto weights2 = std::make_shared(ov::element::Type_t::f32, ov::Shape({2, 10, 3, 3, 3}), 1); + const auto op2 = std::make_shared(param2, weights2, ov::Strides(0, 0), ov::CoordinateDiff(0, 0), + ov::CoordinateDiff(0, 0), ov::Strides(0, 0)); ASSERT_TRUE(matcher.match(op1, op2, op_info)); } -// Check that two group convolutions with different input shapes but same kernel size are match each other +// Check that two group convolutions with different input ov::Shapes but same kernel size are match each other TEST_F(ConvolutionMatcherTest, GroupConvsDifferentKernelSize) { - const auto param = std::make_shared(Type_t::f32, Shape({1, 4, 10, 10})); - const auto weights = std::make_shared(Type_t::f32, Shape({2, 10, 2, 3, 5}), 1); - const auto op1 = std::make_shared(param, weights, Strides(0, 0), CoordinateDiff(0, 0), - CoordinateDiff(0, 0), Strides(0, 0)); + const auto param = std::make_shared(ov::element::Type_t::f32, ov::Shape({1, 4, 10, 10})); + const auto weights = std::make_shared(ov::element::Type_t::f32, ov::Shape({2, 10, 2, 3, 5}), 1); + const auto op1 = std::make_shared(param, weights, ov::Strides(0, 0), ov::CoordinateDiff(0, 0), + ov::CoordinateDiff(0, 0), ov::Strides(0, 0)); - const auto param2 = std::make_shared(Type_t::f32, Shape({1, 6, 20, 20})); - const auto weights2 = std::make_shared(Type_t::f32, Shape({2, 10, 3, 3, 3}), 1); - const auto op2 = std::make_shared(param2, weights2, Strides(0, 0), CoordinateDiff(0, 0), - CoordinateDiff(0, 0), Strides(0, 0)); + const auto param2 = std::make_shared(ov::element::Type_t::f32, ov::Shape({1, 6, 20, 20})); + const auto weights2 = std::make_shared(ov::element::Type_t::f32, ov::Shape({2, 10, 3, 3, 3}), 1); + const auto op2 = std::make_shared(param2, weights2, ov::Strides(0, 0), ov::CoordinateDiff(0, 0), + ov::CoordinateDiff(0, 0), ov::Strides(0, 0)); ASSERT_FALSE(matcher.match(op1, op2, op_info)); } \ No newline at end of file diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/tests/matchers/generic_single_op.cpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/tests/matchers/generic_single_op.cpp index 7907cc1b41e..c40a030309d 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/tests/matchers/generic_single_op.cpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/tests/matchers/generic_single_op.cpp @@ -7,10 +7,6 @@ #include "ngraph/ops.hpp" #include "functional_test_utils/include/functional_test_utils/layer_test_utils/op_info.hpp" -using namespace ngraph::op; -using namespace ngraph; -using ngraph::element::Type_t; - class SingleOpMatcherTest : public ::testing::Test { protected: void SetUp() override { @@ -25,68 +21,68 @@ protected: // Check that different values of constant nodes on port 0 (default value) are ignored in match() TEST_F(SingleOpMatcherTest, AllPortsAreConsts_IgnoreConstPortVals) { - const auto const1 = std::make_shared(Type_t::f32, Shape({5, 5}), 1); - const auto shape_pattern = std::make_shared(Type_t::i64, Shape({2}), std::vector{1, 25}); - const auto op1 = std::make_shared(const1, shape_pattern, false); + const auto const1 = std::make_shared(ov::element::Type_t::f32, ov::Shape({5, 5}), 1); + const auto shape_pattern = std::make_shared(ov::element::Type_t::i64, ov::Shape({2}), std::vector{1, 25}); + const auto op1 = std::make_shared(const1, shape_pattern, false); - const auto const2 = std::make_shared(Type_t::f32, Shape({5, 5}), 2); - const auto op2 = std::make_shared(const2, shape_pattern, false); + const auto const2 = std::make_shared(ov::element::Type_t::f32, ov::Shape({5, 5}), 2); + const auto op2 = std::make_shared(const2, shape_pattern, false); ASSERT_TRUE(matcher.match(op1, op2, op_info)); } // Check match of equal nodes TEST_F(SingleOpMatcherTest, AllPortsAreParams_NodesEqual) { - const auto param1 = std::make_shared(element::Type_t::f32, Shape({10, 10})); - const auto param2 = std::make_shared(element::Type_t::f32, Shape({10, 20})); - const auto op1 = std::make_shared<::Concat>(OutputVector({param1, param2}), 1); - const auto op2 = std::make_shared<::Concat>(OutputVector({param1, param2}), 1); + const auto param1 = std::make_shared(ov::element::Type_t::f32, ov::Shape({10, 10})); + const auto param2 = std::make_shared(ov::element::Type_t::f32, ov::Shape({10, 20})); + const auto op1 = std::make_shared(ov::OutputVector({param1, param2}), 1); + const auto op2 = std::make_shared(ov::OutputVector({param1, param2}), 1); ASSERT_TRUE(matcher.match(op1, op2, op_info)); } // Check nodes doesn't match - different input ranks TEST_F(SingleOpMatcherTest, AllPortsAreParams_RanksNotEqual) { - const auto param1 = std::make_shared(element::Type_t::f32, Shape({10, 10})); - const auto param2 = std::make_shared(element::Type_t::f32, Shape({10, 20})); - const auto op1 = std::make_shared<::Concat>(OutputVector({param1, param2}), 1); + const auto param1 = std::make_shared(ov::element::Type_t::f32, ov::Shape({10, 10})); + const auto param2 = std::make_shared(ov::element::Type_t::f32, ov::Shape({10, 20})); + const auto op1 = std::make_shared(ov::OutputVector({param1, param2}), 1); - const auto param3 = std::make_shared(element::Type_t::f32, Shape({10, 40, 10})); - const auto param4 = std::make_shared(element::Type_t::f32, Shape({10, 40, 10})); - const auto op2 = std::make_shared<::Concat>(OutputVector({param3, param4}), 1); + const auto param3 = std::make_shared(ov::element::Type_t::f32, ov::Shape({10, 40, 10})); + const auto param4 = std::make_shared(ov::element::Type_t::f32, ov::Shape({10, 40, 10})); + const auto op2 = std::make_shared(ov::OutputVector({param3, param4}), 1); ASSERT_FALSE(matcher.match(op1, op2, op_info)); } // Check nodes doesn't match - different input element types TEST_F(SingleOpMatcherTest, AllPortsAreParams_TypesNotEqual) { - const auto param1 = std::make_shared(element::Type_t::f32, Shape({10, 10})); - const auto param2 = std::make_shared(element::Type_t::f32, Shape({10, 20})); - const auto op1 = std::make_shared<::Concat>(OutputVector({param1, param2}), 1); + const auto param1 = std::make_shared(ov::element::Type_t::f32, ov::Shape({10, 10})); + const auto param2 = std::make_shared(ov::element::Type_t::f32, ov::Shape({10, 20})); + const auto op1 = std::make_shared(ov::OutputVector({param1, param2}), 1); - const auto param3 = std::make_shared(element::Type_t::f16, Shape({10, 10})); - const auto param4 = std::make_shared(element::Type_t::f16, Shape({10, 20})); - const auto op2 = std::make_shared<::Concat>(OutputVector({param3, param4}), 1); + const auto param3 = std::make_shared(ov::element::Type_t::f16, ov::Shape({10, 10})); + const auto param4 = std::make_shared(ov::element::Type_t::f16, ov::Shape({10, 20})); + const auto op2 = std::make_shared(ov::OutputVector({param3, param4}), 1); ASSERT_FALSE(matcher.match(op1, op2, op_info)); } // Check nodes doesn't match - different input element types TEST_F(SingleOpMatcherTest, AllPortsAreParams_AttrsNotEqual) { - const auto param1 = std::make_shared(element::Type_t::f32, Shape({10, 10, 10})); - const auto param2 = std::make_shared(element::Type_t::f32, Shape({10, 10, 10})); - const auto op1 = std::make_shared<::Concat>(OutputVector({param1, param2}), 1); + const auto param1 = std::make_shared(ov::element::Type_t::f32, ov::Shape({10, 10, 10})); + const auto param2 = std::make_shared(ov::element::Type_t::f32, ov::Shape({10, 10, 10})); + const auto op1 = std::make_shared(ov::OutputVector({param1, param2}), 1); - const auto param3 = std::make_shared(element::Type_t::f32, Shape({10, 10, 10})); - const auto param4 = std::make_shared(element::Type_t::f32, Shape({10, 10, 10})); - const auto op2 = std::make_shared<::Concat>(OutputVector({param3, param4}), 2); + const auto param3 = std::make_shared(ov::element::Type_t::f32, ov::Shape({10, 10, 10})); + const auto param4 = std::make_shared(ov::element::Type_t::f32, ov::Shape({10, 10, 10})); + const auto op2 = std::make_shared(ov::OutputVector({param3, param4}), 2); ASSERT_FALSE(matcher.match(op1, op2, op_info)); } // Check nodes Add OPs match with different constants on ports TEST_F(SingleOpMatcherTest, ChecAddOpConfiguration) { - const auto const1 = std::make_shared(Type_t::f32, Shape({5, 5}), 1); - const auto const2 = std::make_shared(Type_t::f32, Shape({5, 5}), 2); - const auto op1 = std::make_shared(const1, const2); + const auto const1 = std::make_shared(ov::element::Type_t::f32, ov::Shape({5, 5}), 1); + const auto const2 = std::make_shared(ov::element::Type_t::f32, ov::Shape({5, 5}), 2); + const auto op1 = std::make_shared(const1, const2); - const auto const3 = std::make_shared(Type_t::f32, Shape({5, 5}), 3); - const auto const4 = std::make_shared(Type_t::f32, Shape({5, 5}), 4); - const auto op2 = std::make_shared(const1, const2); + const auto const3 = std::make_shared(ov::element::Type_t::f32, ov::Shape({5, 5}), 3); + const auto const4 = std::make_shared(ov::element::Type_t::f32, ov::Shape({5, 5}), 4); + const auto op2 = std::make_shared(const1, const2); ASSERT_TRUE(matcher.match(op1, op2, op_info)); } \ No newline at end of file diff --git a/src/tests/functional/plugin/conformance/subgraphs_dumper/tests/matchers/matchers_config.cpp b/src/tests/functional/plugin/conformance/subgraphs_dumper/tests/matchers/matchers_config.cpp index 322a99fbec5..029c746e205 100644 --- a/src/tests/functional/plugin/conformance/subgraphs_dumper/tests/matchers/matchers_config.cpp +++ b/src/tests/functional/plugin/conformance/subgraphs_dumper/tests/matchers/matchers_config.cpp @@ -6,15 +6,15 @@ #include "matchers/base_matcher.hpp" #include "ngraph/ops.hpp" -using namespace ngraph::op; +using namespace ov::op; using namespace ngraph; -using ngraph::element::Type_t; +using ov::element::Type_t; class MatcherConfigTest : public ::testing::Test { protected: void SetUp() override { - const auto const1 = std::make_shared(Type_t::f32, Shape({5, 5}), 1); - const auto const2 = std::make_shared(Type_t::f32, Shape({5, 5}), 2); + const auto const1 = std::make_shared(Type_t::f32, Shape({5, 5}), 1); + const auto const2 = std::make_shared(Type_t::f32, Shape({5, 5}), 2); node = std::make_shared(const1, const2); } diff --git a/src/tests/functional/plugin/conformance/test_runner/README.md b/src/tests/functional/plugin/conformance/test_runner/README.md index 2e05c79aaec..1e56d86461d 100644 --- a/src/tests/functional/plugin/conformance/test_runner/README.md +++ b/src/tests/functional/plugin/conformance/test_runner/README.md @@ -45,6 +45,8 @@ The target is able to take the following command-line arguments: * `--save_report_timeout` allows to try to save report in cycle using timeout (in seconds). * `--output_folder` Paths to the output folder to save report. * `--extract_body` allows to count extracted operation bodies to report. +* `--shape_mode` Optional. Allows to run `static`, `dynamic` or both scenarios. Default value is empty string allows to run both scenarios. Possible values + are `static`, `dynamic`, `` * All `gtest` command-line parameters The result of execution is `report.xml` file. It demonstrates tests statistic like pass rate, passed, crashed, skipped failed tests and plugin implementation diff --git a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/gflag_config.hpp b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/gflag_config.hpp index 8a0c9f6b67b..836bf0a0c4f 100644 --- a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/gflag_config.hpp +++ b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/gflag_config.hpp @@ -33,6 +33,8 @@ static const char skip_config_path_message[] = "Optional. Allows to specify path static const char config_path_message[] = "Optional. Allows to specify path to file contains plugin config. " "Default value is empty string."; static const char extract_body_message[] = "Optional. Allows to count extracted operation bodies to report. Default value is false."; +static const char shape_mode_message[] = "Optional. Allows to run `static`, `dynamic` or both scenarios. Default value is empty string allows to run both" + " scenarios. Possible values are `static`, `dynamic`, ``"; DEFINE_bool(h, false, help_message); @@ -47,6 +49,7 @@ DEFINE_bool(disable_test_config, true, disable_test_config_message); DEFINE_bool(extend_report, false, extend_report_config_message); DEFINE_bool(report_unique_name, false, report_unique_name_message); DEFINE_bool(extract_body, false, extract_body_message); +DEFINE_string(shape_mode, "", shape_mode_message); /** * @brief This function shows a help message @@ -68,6 +71,7 @@ static void showUsage() { std::cout << " --input_folders \"\" " << input_folders_message << std::endl; std::cout << " --output_folder \"\" " << output_folder_message << std::endl; std::cout << " --plugin_lib_name " << output_folder_message << std::endl; + std::cout << " --shape_mode \"\" " << shape_mode_message << std::endl; } } // namespace conformance diff --git a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/read_ir_test/read_ir.hpp b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/read_ir_test/read_ir.hpp index f2c7ea1903b..c0dab47a33c 100644 --- a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/read_ir_test/read_ir.hpp +++ b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/include/read_ir_test/read_ir.hpp @@ -10,6 +10,14 @@ namespace ov { namespace test { namespace subgraph { +enum ShapeMode { + DYNAMIC, + STATIC, + BOTH +}; + +extern ShapeMode shapeMode; + using ReadIRParams = std::tuple< std::string, // IR path std::string, // Target Device diff --git a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/main.cpp b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/main.cpp index 00d5b4a35ff..3ec2b46441b 100644 --- a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/main.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/main.cpp @@ -2,12 +2,18 @@ // SPDX-License-Identifier: Apache-2.0 // +#include +#ifdef _WIN32 +#include +#endif + #include "gtest/gtest.h" #include "common_test_utils/file_utils.hpp" #include "functional_test_utils/skip_tests_config.hpp" -#include "shared_test_classes/base/layer_test_utils.hpp" +#include "functional_test_utils/layer_test_utils/environment.hpp" +#include "read_ir_test/read_ir.hpp" #include "gflag_config.hpp" #include "conformance.hpp" @@ -42,6 +48,13 @@ int main(int argc, char* argv[]) { LayerTestsUtils::Summary::setSaveReportWithUniqueName(FLAGS_report_unique_name); LayerTestsUtils::Summary::setOutputFolder(FLAGS_output_folder); LayerTestsUtils::Summary::setSaveReportTimeout(FLAGS_save_report_timeout); + if (FLAGS_shape_mode == std::string("static")) { + ov::test::subgraph::shapeMode = ov::test::subgraph::ShapeMode::STATIC; + } else if (FLAGS_shape_mode == std::string("dynamic")) { + ov::test::subgraph::shapeMode = ov::test::subgraph::ShapeMode::DYNAMIC; + } else if (FLAGS_shape_mode != std::string("")) { + throw std::runtime_error("Incorrect value for `--shape_mode`. Should be `dynamic`, `static` or ``. Current value is `" + FLAGS_shape_mode + "`"); + } // ---------------------------Initialization of Gtest env ----------------------------------------------- ov::test::conformance::targetDevice = FLAGS_device.c_str(); @@ -59,5 +72,24 @@ int main(int argc, char* argv[]) { ::testing::InitGoogleTest(&argc, argv); ::testing::AddGlobalTestEnvironment(new LayerTestsUtils::TestEnvironment); + + auto exernalSignalHandler = [](int errCode) { + std::cerr << "Unexpected application crash with code: " << errCode << std::endl; + + // set default handler for crash + signal(SIGINT, SIG_DFL); + signal(SIGTERM, SIG_DFL); + + if (errCode == SIGINT || errCode == SIGTERM) { + auto& s = LayerTestsUtils::Summary::getInstance(); + s.saveReport(); + exit(1); + } + }; + + // killed by extarnal + signal(SIGINT, exernalSignalHandler); + signal(SIGTERM , exernalSignalHandler); + return RUN_ALL_TESTS(); } diff --git a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp index 812de4b83ce..551bf062d28 100644 --- a/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/conformance_infra/src/read_ir_test/read_ir.cpp @@ -13,14 +13,20 @@ #include "common_test_utils/file_utils.hpp" #include "common_test_utils/data_utils.hpp" #include "common_test_utils/common_utils.hpp" +#include "common_test_utils/crash_handler.hpp" #include "functional_test_utils/layer_test_utils/op_info.hpp" #include "functional_test_utils/skip_tests_config.hpp" #include "read_ir_test/read_ir.hpp" +#include + namespace ov { namespace test { namespace subgraph { + +ShapeMode shapeMode = ShapeMode::BOTH; + std::string ReadIRTest::getTestCaseName(const testing::TestParamInfo &obj) { using namespace CommonTestUtils; std::string pathToModel, deviceName; @@ -48,118 +54,141 @@ std::string ReadIRTest::getTestCaseName(const testing::TestParamInfoset_friendly_name("refFunction"); - } - auto crashHandler = [](int errCode) { + // in case of crash jump will be made and work will be continued + auto crashHandler = std::unique_ptr(new CommonTestUtils::CrashHandler()); + + // place to jump in case of a crash +#ifdef _WIN32 + if (setjmp(CommonTestUtils::env) == 0) { +#else + if (sigsetjmp(CommonTestUtils::env, 1) == 0) { +#endif + if (functionRefs == nullptr) { + functionRefs = ngraph::clone_function(*function); + functionRefs->set_friendly_name("refFunction"); + } + auto &s = LayerTestsUtils::Summary::getInstance(); - s.saveReport(); - std::cout << "Unexpected application crash!" << std::endl; - std::abort(); - }; - signal(SIGSEGV, crashHandler); + s.setDeviceName(targetDevice); - auto &s = LayerTestsUtils::Summary::getInstance(); - s.setDeviceName(targetDevice); - - if (FuncTestUtils::SkipTestsConfig::currentTestIsDisabled()) { - s.updateOPsStats(functionRefs, LayerTestsUtils::PassRate::Statuses::SKIPPED); - GTEST_SKIP() << "Disabled test due to configuration" << std::endl; + if (FuncTestUtils::SkipTestsConfig::currentTestIsDisabled()) { + s.updateOPsStats(functionRefs, LayerTestsUtils::PassRate::Statuses::SKIPPED); + GTEST_SKIP() << "Disabled test due to configuration" << std::endl; + } else { + s.updateOPsStats(functionRefs, LayerTestsUtils::PassRate::Statuses::CRASHED); + } + try { + SubgraphBaseTest::query_model(); + s.updateOPsStats(functionRefs, LayerTestsUtils::PassRate::Statuses::PASSED); + } catch (...) { + s.updateOPsStats(functionRefs, LayerTestsUtils::PassRate::Statuses::FAILED); + } } else { - s.updateOPsStats(functionRefs, LayerTestsUtils::PassRate::Statuses::CRASHED); - } - try { - SubgraphBaseTest::query_model(); - s.updateOPsStats(functionRefs, LayerTestsUtils::PassRate::Statuses::PASSED); - } catch (...) { - s.updateOPsStats(functionRefs, LayerTestsUtils::PassRate::Statuses::FAILED); + IE_THROW() << "Crash happens"; } } void ReadIRTest::SetUp() { - std::tie(pathToModel, targetDevice, configuration) = this->GetParam(); - function = core->read_model(pathToModel); - const auto metaFile = CommonTestUtils::replaceExt(pathToModel, "meta"); - if (CommonTestUtils::fileExists(metaFile)) { - pugi::xml_document doc; - doc.load_file(metaFile.c_str()); - auto models = doc.child("meta_info").child("models"); - sourceModel = models.child("initial_model").attribute("name").as_string(); - for (const auto &model : models.children("model")) { - ocuranceInModels.push_back({model.attribute("name").as_string(), model.attribute("count").as_uint()}); - } - auto portsInfo = doc.child("meta_info").child("ports_info"); - auto getPortInfo = [&](size_t id) { - LayerTestsUtils::PortInfo info; - for (const auto &p : portsInfo.children()) { - if (p.attribute("id").as_uint() == id) { - info.convert_to_const = p.attribute("convert_to_const").as_bool(); - if (std::strcmp(p.attribute("min").as_string(), "undefined") != 0) { - info.min = p.attribute("min").as_double(); - } else { - info.min = -10; - } - if (std::strcmp(p.attribute("max").as_string(), "undefined") != 0) { - info.max = p.attribute("max").as_double(); - } else { - info.max = 10; - } - break; - } - } - return info; - }; + // in case of crash jump will be made and work will be continued + auto crashHandler = std::unique_ptr(new CommonTestUtils::CrashHandler()); - auto params = function->get_parameters(); - for (const auto ¶m : params) { - auto idx = -1; - for (size_t i = 0; i < param->get_output_size(); i++) { - for (const auto &node : param->get_output_target_inputs(i)) { - const auto nodePtr = node.get_node()->shared_from_this(); - for (size_t port = 0; port < nodePtr->get_input_size(); ++port) { - if (nodePtr->get_input_node_ptr(port)->shared_from_this() == param->shared_from_this()) { - idx = port; - break; + // place to jump in case of a crash +#ifdef _WIN32 + if (setjmp(CommonTestUtils::env) == 0) { +#else + if (sigsetjmp(CommonTestUtils::env, 1) == 0) { +#endif + std::tie(pathToModel, targetDevice, configuration) = this->GetParam(); + function = core->read_model(pathToModel); + const auto metaFile = CommonTestUtils::replaceExt(pathToModel, "meta"); + if (CommonTestUtils::fileExists(metaFile)) { + pugi::xml_document doc; + doc.load_file(metaFile.c_str()); + auto models = doc.child("meta_info").child("models"); + sourceModel = models.child("initial_model").attribute("name").as_string(); + for (const auto &model : models.children("model")) { + ocuranceInModels.push_back({model.attribute("name").as_string(), model.attribute("count").as_uint()}); + } + auto portsInfo = doc.child("meta_info").child("ports_info"); + auto getPortInfo = [&](size_t id) { + LayerTestsUtils::PortInfo info; + for (const auto &p : portsInfo.children()) { + if (p.attribute("id").as_uint() == id) { + info.convert_to_const = p.attribute("convert_to_const").as_bool(); + if (std::strcmp(p.attribute("min").as_string(), "undefined") != 0) { + info.min = p.attribute("min").as_double(); + } else { + info.min = -10; + } + if (std::strcmp(p.attribute("max").as_string(), "undefined") != 0) { + info.max = p.attribute("max").as_double(); + } else { + info.max = 10; + } + break; + } + } + return info; + }; + + auto params = function->get_parameters(); + for (const auto ¶m : params) { + auto idx = -1; + for (size_t i = 0; i < param->get_output_size(); i++) { + for (const auto &node : param->get_output_target_inputs(i)) { + const auto nodePtr = node.get_node()->shared_from_this(); + for (size_t port = 0; port < nodePtr->get_input_size(); ++port) { + if (nodePtr->get_input_node_ptr(port)->shared_from_this() == param->shared_from_this()) { + idx = port; + break; + } } } } - } - EXPECT_GE(idx, 0); + EXPECT_GE(idx, 0); - auto info = getPortInfo(idx); - if (info.convert_to_const) { - const auto constant = ngraph::builder::makeConstant(param->get_element_type(), - param->get_shape(), - std::vector{}, - true, - info.max, - info.min, - 1); - ov::replace_node(param, constant); - function->remove_parameter(param); - } - } - } - std::vector inputShapes; - for (const auto& param : function -> get_parameters()) { - if (param->get_partial_shape().is_static()) { - inputShapes.push_back(InputShape{{}, {param->get_shape()}}); - } else { - ov::Shape midShape; - for (const auto s : param->get_partial_shape()) { - int dimValue = s.get_length(); - if (s.is_dynamic()) { - CommonTestUtils::fill_data_random(&dimValue, 1, s.get_max_length() - s.get_min_length(), s.get_min_length(), 1); + auto info = getPortInfo(idx); + if (info.convert_to_const) { + const auto constant = ngraph::builder::makeConstant(param->get_element_type(), + param->get_shape(), + std::vector{}, + true, + info.max, + info.min, + 1); + ov::replace_node(param, constant); + function->remove_parameter(param); } - midShape.push_back(dimValue); } - inputShapes.push_back(InputShape{param->get_partial_shape(), { param->get_partial_shape().get_min_shape(), - param->get_partial_shape().get_max_shape(), - midShape }}); } + std::vector inputShapes; + for (const auto& param : function -> get_parameters()) { + if (param->get_partial_shape().is_static()) { + if (ov::test::subgraph::shapeMode == ov::test::subgraph::ShapeMode::DYNAMIC) { + GTEST_SKIP() << "Static cases are skipped according `shape_mode`"; + } + inputShapes.push_back(InputShape{{}, {param->get_shape()}}); + } else { + if (ov::test::subgraph::shapeMode == ov::test::subgraph::ShapeMode::STATIC) { + GTEST_SKIP() << "Dynamic cases are skipped according `shape_mode`"; + } + ov::Shape midShape; + for (const auto s : param->get_partial_shape()) { + int dimValue = s.get_length(); + if (s.is_dynamic()) { + CommonTestUtils::fill_data_random(&dimValue, 1, s.get_max_length() - s.get_min_length(), s.get_min_length(), 1); + } + midShape.push_back(dimValue); + } + inputShapes.push_back(InputShape{param->get_partial_shape(), { param->get_partial_shape().get_min_shape(), + param->get_partial_shape().get_max_shape(), + midShape }}); + } + } + init_input_shapes(inputShapes); + } else { + IE_THROW() << "Crash happens"; } - init_input_shapes(inputShapes); } } // namespace subgraph diff --git a/src/tests/functional/plugin/cpu/CMakeLists.txt b/src/tests/functional/plugin/cpu/CMakeLists.txt index 9ba753bea29..2790bf21525 100644 --- a/src/tests/functional/plugin/cpu/CMakeLists.txt +++ b/src/tests/functional/plugin/cpu/CMakeLists.txt @@ -12,9 +12,6 @@ set(INCLUDES ${CMAKE_CURRENT_SOURCE_DIR} $ #include #include -#include #include #include -#include - class CustomAbsKernel : public InferenceEngine::ILayerExecImpl { public: @@ -152,23 +149,6 @@ static std::string model_full_path(const char* path) { return FileUtils::makePath(TEST_MODELS, path); } -TEST(Extension, OnnxModelWithCustomAbs) { - std::vector input_values{1, -2, 3, -4, 5, -6, 7, -8, 9, -10}; - std::vector expected{1, 4, 3, 8, 5, 12, 7, 16, 9, 20}; - InferenceEngine::Core ie; - ie.AddExtension(std::make_shared()); - ngraph::onnx_import::register_operator( - CustomAbs::get_type_info_static().name, 1, "custom_domain", [](const ngraph::onnx_import::Node& node) -> ngraph::OutputVector { - ngraph::OutputVector ng_inputs{node.get_ng_inputs()}; - return {std::make_shared(ng_inputs.at(0))}; - }); - - auto network = ie.ReadNetwork(model_full_path("func_tests/models/custom_abs_op.onnx")); - infer_model(ie, network, input_values, expected); - ngraph::onnx_import::unregister_operator(CustomAbs::get_type_info_static().name, 1, "custom_domain"); -} - - TEST(Extension, XmlModelWithCustomAbs) { std::string model = R"V0G0N( @@ -293,15 +273,3 @@ TEST(Extension, OnnxModelWithExtensionFromDSO) { auto network = ie.ReadNetwork(model_full_path("func_tests/models/custom_template_op.onnx")); infer_model(ie, network, input_values, expected); } - - -TEST(Extension, OnnxModelWithCustomReluDocsExample) { - std::vector input_values{0, -1, 2, -3, 4, -5, 6, -7}; - std::vector expected{0, -3, 4, -9, 8, -15, 12, -21}; - - register_custom_relu_operator(); - InferenceEngine::Core ie; - auto network = ie.ReadNetwork(model_full_path("docs/models/custom_relu_model.onnx")); - infer_model(ie, network, input_values, expected); - unregister_custom_relu_operator(); -} diff --git a/src/tests/functional/plugin/cpu/shared_tests_instances/blob_tests/dynamic_batch.cpp b/src/tests/functional/plugin/cpu/shared_tests_instances/behavior/infer_request/dynamic_batch.cpp similarity index 95% rename from src/tests/functional/plugin/cpu/shared_tests_instances/blob_tests/dynamic_batch.cpp rename to src/tests/functional/plugin/cpu/shared_tests_instances/behavior/infer_request/dynamic_batch.cpp index d51c6da9044..0b8582578b8 100644 --- a/src/tests/functional/plugin/cpu/shared_tests_instances/blob_tests/dynamic_batch.cpp +++ b/src/tests/functional/plugin/cpu/shared_tests_instances/behavior/infer_request/dynamic_batch.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include +#include #include "common_test_utils/test_constants.hpp" namespace ConfigurationTestsDefinitions { diff --git a/src/tests/functional/plugin/cpu/shared_tests_instances/behavior/ov_executable_network/properties.cpp b/src/tests/functional/plugin/cpu/shared_tests_instances/behavior/ov_executable_network/properties.cpp index 96189a934cc..22705a5d63f 100644 --- a/src/tests/functional/plugin/cpu/shared_tests_instances/behavior/ov_executable_network/properties.cpp +++ b/src/tests/functional/plugin/cpu/shared_tests_instances/behavior/ov_executable_network/properties.cpp @@ -30,6 +30,7 @@ const std::vector auto_inproperties = { const std::vector auto_batch_inproperties = { {ov::num_streams(-100)}, + {{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , std::string(CommonTestUtils::DEVICE_CPU) + "(4)"}, {ov::auto_batch_timeout(-1)}}, }; INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVCompiledModelPropertiesIncorrectTests, @@ -86,32 +87,30 @@ INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVCompiledModelPropertiesDefaultTe OVCompiledModelPropertiesDefaultTests::getTestCaseName); const std::vector properties = { - {ov::num_streams(ov::NumStreams::AUTO)}, - {ov::num_streams(ov::NumStreams::NUMA)}, + {ov::num_streams(ov::streams::NUMA)}, + {ov::num_streams(ov::streams::AUTO)}, {ov::num_streams(0), ov::inference_num_threads(1)}, {ov::num_streams(1), ov::inference_num_threads(1)}, {{InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, InferenceEngine::PluginConfigParams::CPU_THROUGHPUT_AUTO}} }; const std::vector hetero_properties = { - {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::num_streams(ov::NumStreams::AUTO)}, + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::num_streams(ov::streams::AUTO)}, {ov::device::priorities(CommonTestUtils::DEVICE_CPU), {InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, InferenceEngine::PluginConfigParams::CPU_THROUGHPUT_AUTO}}, }; const std::vector multi_properties = { - {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::num_streams(ov::NumStreams::AUTO)}, + {ov::device::priorities(CommonTestUtils::DEVICE_CPU), ov::num_streams(ov::streams::AUTO)}, {ov::device::priorities(CommonTestUtils::DEVICE_CPU), {InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, InferenceEngine::PluginConfigParams::CPU_THROUGHPUT_AUTO}}, }; const std::vector auto_batch_properties = { - {{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , CommonTestUtils::DEVICE_CPU}, ov::num_streams(ov::NumStreams::AUTO)}, - {{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , CommonTestUtils::DEVICE_CPU}, - {InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, InferenceEngine::PluginConfigParams::CPU_THROUGHPUT_AUTO}}, - {{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , CommonTestUtils::DEVICE_CPU}, - {CONFIG_KEY(AUTO_BATCH_TIMEOUT) , "1"}, ov::num_streams(ov::NumStreams::AUTO)}, + {{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , std::string(CommonTestUtils::DEVICE_CPU) + "(4)"}}, + {{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , std::string(CommonTestUtils::DEVICE_CPU) + "(4)"}, {CONFIG_KEY(AUTO_BATCH_TIMEOUT) , "1"}}, + {{CONFIG_KEY(AUTO_BATCH_DEVICE_CONFIG) , std::string(CommonTestUtils::DEVICE_CPU) + "(4)"}, {ov::auto_batch_timeout(10)}}, }; INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, OVCompiledModelPropertiesTests, diff --git a/src/tests/functional/plugin/cpu/shared_tests_instances/auto_batching/auto_batching_tests.cpp b/src/tests/functional/plugin/cpu/shared_tests_instances/behavior/plugin/auto_batching_tests.cpp similarity index 95% rename from src/tests/functional/plugin/cpu/shared_tests_instances/auto_batching/auto_batching_tests.cpp rename to src/tests/functional/plugin/cpu/shared_tests_instances/behavior/plugin/auto_batching_tests.cpp index 37ff55e3c73..b1f1432a14e 100644 --- a/src/tests/functional/plugin/cpu/shared_tests_instances/auto_batching/auto_batching_tests.cpp +++ b/src/tests/functional/plugin/cpu/shared_tests_instances/behavior/plugin/auto_batching_tests.cpp @@ -1,7 +1,7 @@ // Copyright (C) 2018-2022 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // -#include +#include const std::vector get_vs_set{ true, false }; const std::vector num_streams{ 1, 2 }; diff --git a/src/tests/functional/plugin/cpu/shared_tests_instances/hetero/synthetic.cpp b/src/tests/functional/plugin/cpu/shared_tests_instances/behavior/plugin/hetero_synthetic.cpp similarity index 96% rename from src/tests/functional/plugin/cpu/shared_tests_instances/hetero/synthetic.cpp rename to src/tests/functional/plugin/cpu/shared_tests_instances/behavior/plugin/hetero_synthetic.cpp index e3bbe500f9a..402b986265e 100644 --- a/src/tests/functional/plugin/cpu/shared_tests_instances/hetero/synthetic.cpp +++ b/src/tests/functional/plugin/cpu/shared_tests_instances/behavior/plugin/hetero_synthetic.cpp @@ -4,7 +4,7 @@ #include -#include "hetero/synthetic.hpp" +#include "behavior/plugin/hetero_synthetic.hpp" #include "ngraph_functions/builders.hpp" #include "ngraph_functions/subgraph_builders.hpp" diff --git a/src/tests/functional/plugin/cpu/shared_tests_instances/skip_tests_config.cpp b/src/tests/functional/plugin/cpu/shared_tests_instances/skip_tests_config.cpp index f0834a5fb55..d98c578ee67 100644 --- a/src/tests/functional/plugin/cpu/shared_tests_instances/skip_tests_config.cpp +++ b/src/tests/functional/plugin/cpu/shared_tests_instances/skip_tests_config.cpp @@ -66,8 +66,6 @@ std::vector disabledTestPatterns() { // TODO: 57562 No dynamic output shape support R"(.*NonZeroLayerTest.*)", - // TODO: 74961. Enforce precision via inType and outType does not work properly. - R"(.*(RNN|GRU|LSTM).*ENFORCE_BF16=YES.*)", // Not expected behavior R"(.*Behavior.*InferRequestIOBBlobSetLayoutTest.*layout=(95|OIHW).*)", R"(.*Behavior.*InferRequestIOBBlobSetLayoutTest.*layout=(95|OIHW).*)", @@ -143,9 +141,6 @@ std::vector disabledTestPatterns() { *IS=_TS=\(\(4\.5\.6\.7\)\)_RS=\(\(1\.1\.6\.1\)\)_\(\(1\.5\.6\.1\)\)_\(\(1\.1\.1\.1\)\)_\(\(1\.1\.6\.1\)\).*)", // Issue: 69222 R"(.*smoke_PriorBoxClustered.*PriorBoxClusteredLayerCPUTest.*_netPRC=f16_.*)", - // Issue: 74817 - // Sporadic failings with NAN on Dynamic shape cases with jit implementation - R"(.*DefConvLayoutTest7.*)", // Issue: 71968 R"(.*LSTMSequenceCommonZeroClip.*PURE.*CONST.*hidden_size=10.*sigmoid.sigmoid.sigmoid.*reverse.*FP32_targetDevice=CPU.*)", // Issue: 72005 diff --git a/src/tests/functional/plugin/cpu/single_layer_tests/convolution.cpp b/src/tests/functional/plugin/cpu/single_layer_tests/convolution.cpp index 00e2da6c101..bbf30caeba1 100755 --- a/src/tests/functional/plugin/cpu/single_layer_tests/convolution.cpp +++ b/src/tests/functional/plugin/cpu/single_layer_tests/convolution.cpp @@ -1156,6 +1156,20 @@ INSTANTIATE_TEST_SUITE_P(smoke_Conv_3D_FP32, ConvolutionLayerCPUTest, ::testing::Values(cpuEmptyPluginConfig)), ConvolutionLayerCPUTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_Conv_3D_FP32_fusingScaleShiftAndFakeQuantizePerChannel, ConvolutionLayerCPUTest, + ::testing::Combine( + ::testing::Combine( + convParams_ExplicitPadding_3D, + ::testing::Values(ElementType::f32), + ::testing::Values(ElementType::undefined), + ::testing::Values(ElementType::undefined), + ::testing::ValuesIn(inputShapes3d), + ::testing::Values(CommonTestUtils::DEVICE_CPU)), + ::testing::ValuesIn(filterCPUInfoForDevice(CPUParams_3D)), + ::testing::Values(fusingScaleShiftAndFakeQuantizePerChannel), + ::testing::Values(cpuEmptyPluginConfig)), + ConvolutionLayerCPUTest::getTestCaseName); + INSTANTIATE_TEST_SUITE_P(smoke_Conv_3D_BF16, ConvolutionLayerCPUTest, ::testing::Combine( ::testing::Combine( diff --git a/src/tests/functional/plugin/cpu/single_layer_tests/gru_cell.cpp b/src/tests/functional/plugin/cpu/single_layer_tests/gru_cell.cpp index 1c244c8d2c6..03378b40de9 100644 --- a/src/tests/functional/plugin/cpu/single_layer_tests/gru_cell.cpp +++ b/src/tests/functional/plugin/cpu/single_layer_tests/gru_cell.cpp @@ -88,11 +88,10 @@ protected: configuration.insert(additionalConfig.begin(), additionalConfig.end()); if (additionalConfig[InferenceEngine::PluginConfigParams::KEY_ENFORCE_BF16] == InferenceEngine::PluginConfigParams::YES) { - inType = outType = ElementType::bf16; + selectedType = makeSelectedTypeStr(selectedType, ElementType::bf16); } else { - inType = outType = netPrecision; + selectedType = makeSelectedTypeStr(selectedType, netPrecision); } - selectedType = makeSelectedTypeStr(selectedType, outType); auto params = ngraph::builder::makeDynamicParams(netPrecision, inputDynamicShapes); std::vector WRB = {{3 * hiddenSize, inputSize}, {3 * hiddenSize, hiddenSize}, {(linearBeforeReset ? 4 : 3) * hiddenSize}}; diff --git a/src/tests/functional/plugin/cpu/single_layer_tests/gru_sequence.cpp b/src/tests/functional/plugin/cpu/single_layer_tests/gru_sequence.cpp index c2c035697ee..34291d4306d 100644 --- a/src/tests/functional/plugin/cpu/single_layer_tests/gru_sequence.cpp +++ b/src/tests/functional/plugin/cpu/single_layer_tests/gru_sequence.cpp @@ -105,15 +105,11 @@ protected: const size_t inputSize = targetStaticShapes.front()[0][2]; const size_t numDirections = direction == ov::op::RecurrentSequenceDirection::BIDIRECTIONAL ? 2 : 1; - // 3rd input type must be an integer, thus it cannot be forced to BF16. if (additionalConfig[InferenceEngine::PluginConfigParams::KEY_ENFORCE_BF16] == InferenceEngine::PluginConfigParams::YES) { - if (inputDynamicShapes.size() > 2) - throw std::runtime_error("Invalid test case. Cannot enforce integer input to BF16."); - inType = outType = ElementType::bf16; + selectedType = makeSelectedTypeStr(selectedType, ElementType::bf16); } else { - outType = netPrecision; + selectedType = makeSelectedTypeStr(selectedType, netPrecision); } - selectedType = makeSelectedTypeStr(selectedType, outType); auto params = ngraph::builder::makeDynamicParams(netPrecision, inputDynamicShapes); const size_t batchSize = inputDynamicShapes[0][0].is_static() ? inputDynamicShapes[0][0].get_length() : @@ -295,7 +291,7 @@ const std::vector> dynamicShapes = { { {1, 2, 10}, {1, 4, 10}, {1, 8, 10} } }, // Target shapes { {1, 1, 10}, // Dynamic shape 1 { {1, 1, 10}, {1, 1, 10}, {1, 1, 10} } }, // Target shapes - { {-1}, // Dynamic shape 2 + { {-1}, // Dynamic shape 2 { {1}, {1}, {1} } } }, // Target shapes { { {-1, -1, -1}, // #5. Dynamic shape 0 { {1, 2, 10}, {1, 4, 10}, {1, 8, 10} } }, // Target shapes @@ -304,7 +300,7 @@ const std::vector> dynamicShapes = { { {-1}, // Dynamic shape 2 { {1}, {1}, {1} } } }, // Target shapes { { {2, {1, 5}, 10}, // #6. Dynamic shape 0 - { {10, 2, 10}, {2, 3, 10}, {2, 4, 10} } }, // Target shapes + { {2, 2, 10}, {2, 3, 10}, {2, 4, 10} } }, // Target shapes { {2, 1, 1}, // Dynamic shape 1 { {2, 1, 1}, {2, 1, 1}, {2, 1, 1} } } }, // Target shapes { { {5, -1, 10}, // #7. Dynamic shape 0 diff --git a/src/tests/functional/plugin/cpu/single_layer_tests/lstm_cell.cpp b/src/tests/functional/plugin/cpu/single_layer_tests/lstm_cell.cpp index e4e85a776d7..3a0d6bbd79d 100644 --- a/src/tests/functional/plugin/cpu/single_layer_tests/lstm_cell.cpp +++ b/src/tests/functional/plugin/cpu/single_layer_tests/lstm_cell.cpp @@ -87,11 +87,10 @@ protected: const size_t inputSize = targetStaticShapes.front()[0][1]; if (additionalConfig[InferenceEngine::PluginConfigParams::KEY_ENFORCE_BF16] == InferenceEngine::PluginConfigParams::YES) { - inType = outType = ElementType::bf16; + selectedType = makeSelectedTypeStr(selectedType, ElementType::bf16); } else { - inType = outType = netPrecision; + selectedType = makeSelectedTypeStr(selectedType, netPrecision); } - selectedType = makeSelectedTypeStr(selectedType, outType); auto params = ngraph::builder::makeDynamicParams(netPrecision, inputDynamicShapes); auto paramsOuts = ngraph::helpers::convert2OutputVector(ngraph::helpers::castOps2Nodes(params)); diff --git a/src/tests/functional/plugin/cpu/single_layer_tests/lstm_sequence.cpp b/src/tests/functional/plugin/cpu/single_layer_tests/lstm_sequence.cpp index 2de06f59a05..cc3c3db39c6 100644 --- a/src/tests/functional/plugin/cpu/single_layer_tests/lstm_sequence.cpp +++ b/src/tests/functional/plugin/cpu/single_layer_tests/lstm_sequence.cpp @@ -105,15 +105,11 @@ protected: configuration.insert(additionalConfig.begin(), additionalConfig.end()); - // 4th input type must be integer, thus it cannot be forced to BF16. if (additionalConfig[InferenceEngine::PluginConfigParams::KEY_ENFORCE_BF16] == InferenceEngine::PluginConfigParams::YES) { - if (inputDynamicShapes.size() > 3) - throw std::runtime_error("Invalid test case. Cannot enforce integer input to BF16."); - inType = outType = ElementType::bf16; + selectedType = makeSelectedTypeStr(selectedType, ElementType::bf16); } else { - outType = netPrecision; + selectedType = makeSelectedTypeStr(selectedType, netPrecision); } - selectedType = makeSelectedTypeStr(selectedType, outType); auto params = ngraph::builder::makeDynamicParams(netPrecision, inputDynamicShapes); const size_t batchSize = inputDynamicShapes[0][0].is_static() ? inputDynamicShapes[0][0].get_length() : diff --git a/src/tests/functional/plugin/cpu/single_layer_tests/matmul.cpp b/src/tests/functional/plugin/cpu/single_layer_tests/matmul.cpp index fb311d6eaf2..49f6a649d1f 100644 --- a/src/tests/functional/plugin/cpu/single_layer_tests/matmul.cpp +++ b/src/tests/functional/plugin/cpu/single_layer_tests/matmul.cpp @@ -427,6 +427,7 @@ std::vector fusingParamsSet3D_smoke { fusingBias, fusingMultiplyPerChannel, fusingFakeQuantizePerChannel, + fusingScaleShiftAndFakeQuantizePerChannel, }; std::vector fusingParamsSet3D_nightly { @@ -866,6 +867,7 @@ std::vector matmulFusingParams { * so Relu cannot be fused in this case. Should be analysed */ // fusingFakeQuantizePerChannelRelu, fusingFakeQuantizePerTensorRelu, + fusingScaleShiftAndFakeQuantizePerChannel, }; const auto matMulParams = ::testing::Combine(::testing::ValuesIn(IS), diff --git a/src/tests/functional/plugin/cpu/single_layer_tests/rnn_cell.cpp b/src/tests/functional/plugin/cpu/single_layer_tests/rnn_cell.cpp index 0ec1d3f9789..3d2097f7b23 100644 --- a/src/tests/functional/plugin/cpu/single_layer_tests/rnn_cell.cpp +++ b/src/tests/functional/plugin/cpu/single_layer_tests/rnn_cell.cpp @@ -83,11 +83,10 @@ protected: configuration.insert(additionalConfig.begin(), additionalConfig.end()); if (additionalConfig[InferenceEngine::PluginConfigParams::KEY_ENFORCE_BF16] == InferenceEngine::PluginConfigParams::YES) { - inType = outType = ElementType::bf16; + selectedType = makeSelectedTypeStr(selectedType, ElementType::bf16); } else { - inType = outType = netPrecision; + selectedType = makeSelectedTypeStr(selectedType, netPrecision); } - selectedType = makeSelectedTypeStr(selectedType, outType); auto params = ngraph::builder::makeDynamicParams(netPrecision, inputDynamicShapes); auto paramsOuts = ngraph::helpers::convert2OutputVector(ngraph::helpers::castOps2Nodes(params)); diff --git a/src/tests/functional/plugin/cpu/single_layer_tests/rnn_sequence.cpp b/src/tests/functional/plugin/cpu/single_layer_tests/rnn_sequence.cpp index 31963f9fa05..baeb99589c3 100644 --- a/src/tests/functional/plugin/cpu/single_layer_tests/rnn_sequence.cpp +++ b/src/tests/functional/plugin/cpu/single_layer_tests/rnn_sequence.cpp @@ -94,15 +94,11 @@ protected: configuration.insert(additionalConfig.begin(), additionalConfig.end()); - // 3rd input type must be integer, thus it cannot be forced to BF16. if (additionalConfig[InferenceEngine::PluginConfigParams::KEY_ENFORCE_BF16] == InferenceEngine::PluginConfigParams::YES) { - if (inputDynamicShapes.size() > 2) - throw std::runtime_error("Invalid test case. Cannot enforce integer input to BF16."); - inType = outType = ElementType::bf16; + selectedType = makeSelectedTypeStr(selectedType, ElementType::bf16); } else { - outType = netPrecision; + selectedType = makeSelectedTypeStr(selectedType, netPrecision); } - selectedType = makeSelectedTypeStr(selectedType, outType); auto params = ngraph::builder::makeDynamicParams(netPrecision, inputDynamicShapes); const size_t batchSize = inputDynamicShapes[0][0].is_static() ? inputDynamicShapes[0][0].get_length() : diff --git a/src/tests/functional/plugin/cpu/subgraph_tests/src/conv_sum_broadcast.cpp b/src/tests/functional/plugin/cpu/subgraph_tests/src/conv_sum_broadcast.cpp index 4a82b0adcf2..41a6ffc29d4 100644 --- a/src/tests/functional/plugin/cpu/subgraph_tests/src/conv_sum_broadcast.cpp +++ b/src/tests/functional/plugin/cpu/subgraph_tests/src/conv_sum_broadcast.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "test_utils/cpu_test_utils.hpp" +#include #include "test_utils/fusing_test_utils.hpp" #include "test_utils/convolution_params.hpp" #include "shared_test_classes/base/ov_subgraph.hpp" @@ -60,6 +60,28 @@ public: return result.str(); } + virtual ngraph::ParameterVector makeParams() { + return ngraph::builder::makeDynamicParams(ngraph::element::f32, inputDynamicShapes); + } + + virtual std::shared_ptr makeConv(const ngraph::ParameterVector& inputParams) { + auto conv = ngraph::builder::makeConvolution(inputParams[0], ngraph::element::f32, _kernel, _stride, _padBegin, + _padEnd, _dilation, ngraph::op::PadType::EXPLICIT, _convOutChannels); + + return conv; + } + + virtual std::shared_ptr addSum(std::shared_ptr lastNode, const ngraph::ParameterVector& inputParams) { + auto sum = std::make_shared(lastNode, inputParams[1]); + + fusedOps.insert(fusedOps.begin(), "Add"); // as we always fuse the sum first + return sum; + } + + virtual ov::element::Type getNetType() const { + return ov::element::Type_t::f32; + } + void SetUp() override { InputShape convShape; InputShape secondShape; @@ -75,38 +97,41 @@ public: init_input_shapes({convShape, secondShape}); - const InferenceEngine::SizeVector kernel = {3, 3}; - const InferenceEngine::SizeVector stride = {1, 1}; - const InferenceEngine::SizeVector dilation = {1, 1}; - const std::vector padBegin = {0, 0}; - const std::vector padEnd = {0, 0}; - const size_t convOutChannels = 64; + auto inputParams = makeParams(); - auto netType = ngraph::element::f32; - auto inputParams = ngraph::builder::makeDynamicParams(netType, inputDynamicShapes); + auto conv = makeConv(inputParams); - auto conv = ngraph::builder::makeConvolution(inputParams[0], ngraph::element::f32, kernel, stride, padBegin, - padEnd, dilation, ngraph::op::PadType::EXPLICIT, convOutChannels); if (bias) { - auto biasNode = ngraph::builder::makeConstant(ngraph::element::Type_t::f32, ngraph::Shape({1, convOutChannels, 1, 1}), {}, true); + auto biasNode = ngraph::builder::makeConstant(ngraph::element::Type_t::f32, ngraph::Shape({1, _convOutChannels, 1, 1}), {}, true); conv = std::make_shared(conv, biasNode); } - auto sum = std::make_shared(conv, inputParams[1]); + auto sum = addSum(conv, inputParams); - fusedOps.insert(fusedOps.begin(), "Add"); // as we always fuse the sum first - - auto runtimeType = netType; + auto runtimeType = getNetType(); if (configuration.count(PluginConfigParams::KEY_ENFORCE_BF16) && PluginConfigParams::YES == configuration[PluginConfigParams::KEY_ENFORCE_BF16].as()) { runtimeType = ngraph::element::Type_t::bf16; } + if (inputParams.front()->get_element_type() == ngraph::element::i8 || inputParams.front()->get_element_type() == ngraph::element::u8) { + runtimeType = ngraph::element::i8; + } + selectedType = makeSelectedTypeStr(getPrimitiveType(), runtimeType); - function = makeNgraphFunction(netType, inputParams, sum, "ConvolutionSumBroadcast"); + function = makeNgraphFunction(getNetType(), inputParams, sum, "ConvolutionSumBroadcast"); + targetDevice = CommonTestUtils::DEVICE_CPU; } + +protected: + const InferenceEngine::SizeVector _kernel = {3, 3}; + const InferenceEngine::SizeVector _stride = {1, 1}; + const InferenceEngine::SizeVector _dilation = {1, 1}; + const std::vector _padBegin = {0, 0}; + const std::vector _padEnd = {0, 0}; + const size_t _convOutChannels = 64; }; TEST_P(ConcatConvSumInPlaceTest, CompareWithRefs) { @@ -117,74 +142,140 @@ TEST_P(ConcatConvSumInPlaceTest, CompareWithRefs) { CheckPluginRelatedResults(compiledModel, "Convolution"); } +class ConcatConvSumInPlaceTestInt8 : public ConcatConvSumInPlaceTest { +public: + ngraph::ParameterVector makeParams() override { + ngraph::ParameterVector outs(2); + outs[0] = std::make_shared(ngraph::element::u8, inputDynamicShapes[0]); + outs[1] = std::make_shared(ngraph::element::f32, inputDynamicShapes[1]); + return outs; + } + + std::shared_ptr makeConv(const ngraph::ParameterVector& inputParams) override { + using namespace ngraph; + auto inputParamsFP32 = builder::makeDynamicParams(element::f32, { inputParams.front()->get_partial_shape() }); + + auto convolutionNodeRelaxed = std::make_shared>( + *as_type_ptr(builder::makeConvolution(inputParamsFP32.front(), element::f32, _kernel, _stride, _padBegin, + _padEnd, _dilation, ngraph::op::PadType::EXPLICIT, _convOutChannels)), + element::f32); + + auto inpShape = inputParams.front()->get_partial_shape(); + Shape filterShape = {_convOutChannels, static_cast(inpShape[1].get_length())}; + filterShape.insert(filterShape.end(), _kernel.begin(), _kernel.end()); + auto filterWeightsNode = builder::makeConstant(element::i8, filterShape, {}, true); + + auto conv = convolutionNodeRelaxed->copy_with_new_inputs({inputParams.front(), filterWeightsNode}); + + return conv; + } + + std::shared_ptr addSum(std::shared_ptr lastNode, const ngraph::ParameterVector& inputParams) override { + std::vector additionalFusedOps; + + lastNode = ngraph::builder::makeActivation(lastNode, ngraph::element::f32, ngraph::helpers::Relu); + //additionalFusedOps.push_back("Relu"); + + auto fqShape = ngraph::Shape(lastNode->get_output_partial_shape(0).size(), 1); + lastNode = ngraph::builder::makeFakeQuantize(lastNode, ngraph::element::f32, 256, fqShape); + additionalFusedOps.push_back("FakeQuantize"); + + auto secondTerm = ngraph::builder::makeFakeQuantize(inputParams[1], ngraph::element::f32, 256, fqShape); + + auto sum = std::make_shared(lastNode, secondTerm); + additionalFusedOps.push_back("Add"); + + fusedOps.insert(fusedOps.begin(), additionalFusedOps.begin(), additionalFusedOps.end()); + return sum; + } + + void SetUp() override { + abs_threshold = 1.001f; + using ngraph::pass::ConvertPrecision; + ConcatConvSumInPlaceTest::SetUp(); + functionRefs = ov::clone_model(*function); + ngraph::pass::ConvertPrecision().run_on_function(functionRefs); + ngraph::pass::ConvertPrecision().run_on_function(functionRefs); + functionRefs->validate_nodes_and_infer_types(); + } +}; + +TEST_P(ConcatConvSumInPlaceTestInt8, CompareWithRefs) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + + run(); + + CheckPluginRelatedResults(compiledModel, "Convolution"); +} + namespace { const auto fusingMulAddFQMullAdd = fusingSpecificParams{ std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params) { - ngraph::Shape newShape = generatePerChannelShape(inpNode); - auto constNode = ngraph::builder::makeConstant(ngPrc, newShape, std::vector{}, true); - return std::make_shared(inpNode, constNode); + {[](postNodeConfig& cfg) { + ngraph::Shape newShape = generatePerChannelShape(cfg.input); + auto constNode = ngraph::builder::makeConstant(cfg.type, newShape, std::vector{}, true); + return std::make_shared(cfg.input, constNode); }, "Multiply(PerChannel)"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params) { - ngraph::Shape newShape = generatePerChannelShape(inpNode); - auto constNode = ngraph::builder::makeConstant(ngPrc, newShape, std::vector{}, true); - return std::make_shared(inpNode, constNode); + {[](postNodeConfig& cfg) { + ngraph::Shape newShape = generatePerChannelShape(cfg.input); + auto constNode = ngraph::builder::makeConstant(cfg.type, newShape, std::vector{}, true); + return std::make_shared(cfg.input, constNode); }, "Add(PerChannel)"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - auto localPrc = inpNode->get_element_type(); - ngraph::Shape newShape = generatePerChannelShape(inpNode); - return ngraph::builder::makeFakeQuantize(inpNode, localPrc, 256, newShape); + {[](postNodeConfig& cfg){ + auto localPrc = cfg.input->get_element_type(); + ngraph::Shape newShape = generatePerChannelShape(cfg.input); + return ngraph::builder::makeFakeQuantize(cfg.input, localPrc, 256, newShape); }, "FakeQuantize(PerChannel)"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params) { - ngraph::Shape newShape = generatePerChannelShape(inpNode); - auto constNode = ngraph::builder::makeConstant(ngPrc, newShape, std::vector{}, true); - return std::make_shared(inpNode, constNode); + {[](postNodeConfig& cfg) { + ngraph::Shape newShape = generatePerChannelShape(cfg.input); + auto constNode = ngraph::builder::makeConstant(cfg.type, newShape, std::vector{}, true); + return std::make_shared(cfg.input, constNode); }, "Multiply(PerChannel)"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params) { - ngraph::Shape newShape = generatePerChannelShape(inpNode); - auto constNode = ngraph::builder::makeConstant(ngPrc, newShape, std::vector{}, true); - return std::make_shared(inpNode, constNode); + {[](postNodeConfig& cfg) { + ngraph::Shape newShape = generatePerChannelShape(cfg.input); + auto constNode = ngraph::builder::makeConstant(cfg.type, newShape, std::vector{}, true); + return std::make_shared(cfg.input, constNode); }, "Add(PerChannel)"}}), {"Add"} }; const auto fusingDivSubFQ = fusingSpecificParams{ std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - ngraph::Shape secondMultInShape = generatePerChannelShape(inpNode); - auto secondMultInput = ngraph::builder::makeConstant(ngPrc, secondMultInShape, std::vector{}, true); - return std::make_shared(inpNode, secondMultInput); + {[](postNodeConfig& cfg){ + ngraph::Shape secondMultInShape = generatePerChannelShape(cfg.input); + auto secondMultInput = ngraph::builder::makeConstant(cfg.type, secondMultInShape, std::vector{}, true); + return std::make_shared(cfg.input, secondMultInput); }, "Divide(PerChannel)"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - ngraph::Shape secondMultInShape = generatePerChannelShape(inpNode); - auto secondMultInput = ngraph::builder::makeConstant(ngPrc, secondMultInShape, std::vector{}, true); - return std::make_shared(inpNode, secondMultInput); + {[](postNodeConfig& cfg){ + ngraph::Shape secondMultInShape = generatePerChannelShape(cfg.input); + auto secondMultInput = ngraph::builder::makeConstant(cfg.type, secondMultInShape, std::vector{}, true); + return std::make_shared(cfg.input, secondMultInput); }, "Subtract(PerChannel)"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - auto localPrc = inpNode->get_element_type(); - ngraph::Shape newShape = generatePerChannelShape(inpNode); - return ngraph::builder::makeFakeQuantize(inpNode, localPrc, 256, newShape); + {[](postNodeConfig& cfg){ + auto localPrc = cfg.input->get_element_type(); + ngraph::Shape newShape = generatePerChannelShape(cfg.input); + return ngraph::builder::makeFakeQuantize(cfg.input, localPrc, 256, newShape); }, "FakeQuantize(PerChannel)"}}), {"FakeQuantize"} }; const auto fusingSigmoidFQFQ = fusingSpecificParams{ std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Sigmoid); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::Sigmoid); }, "Sigmoid"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - auto localPrc = inpNode->get_element_type(); - ngraph::Shape newShape = generatePerChannelShape(inpNode); - return ngraph::builder::makeFakeQuantize(inpNode, localPrc, 256, newShape); + {[](postNodeConfig& cfg){ + auto localPrc = cfg.input->get_element_type(); + ngraph::Shape newShape = generatePerChannelShape(cfg.input); + return ngraph::builder::makeFakeQuantize(cfg.input, localPrc, 256, newShape); }, "FakeQuantize(PerChannel)"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - auto localPrc = inpNode->get_element_type(); - ngraph::Shape newShape = generatePerChannelShape(inpNode); - return ngraph::builder::makeFakeQuantize(inpNode, localPrc, 256, newShape); + {[](postNodeConfig& cfg){ + auto localPrc = cfg.input->get_element_type(); + ngraph::Shape newShape = generatePerChannelShape(cfg.input); + return ngraph::builder::makeFakeQuantize(cfg.input, localPrc, 256, newShape); }, "FakeQuantize(PerChannel)"}}), {"Sigmoid", "FakeQuantize", "FakeQuantize"} }; const auto fusingClampFQ = fusingSpecificParams{ std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Clamp, {}, {3.0f, 6.0f}); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::Clamp, {}, {3.0f, 6.0f}); }, "Clamp"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - auto localPrc = inpNode->get_element_type(); - ngraph::Shape newShape = generatePerChannelShape(inpNode); - return ngraph::builder::makeFakeQuantize(inpNode, localPrc, 256, newShape); + {[](postNodeConfig& cfg){ + auto localPrc = cfg.input->get_element_type(); + ngraph::Shape newShape = generatePerChannelShape(cfg.input); + return ngraph::builder::makeFakeQuantize(cfg.input, localPrc, 256, newShape); }, "FakeQuantize(PerChannel)"}}), {"FakeQuantize"} }; @@ -198,7 +289,6 @@ const std::vector fusingParamsSet{ fusingReluScaleShift, fusingMulAddFQMullAdd, fusingSigmoidFQFQ, -// fusingClampFQ // TODO: we need investigation, this particular pattern does not work even in static case fusingDivSubFQ }; @@ -250,5 +340,14 @@ INSTANTIATE_TEST_SUITE_P(smoke_Conv_Sum_Broadcast_BF16, ConcatConvSumInPlaceTest ::testing::Values(cpuBF16PluginConfig)), ConcatConvSumInPlaceTest::getTestCaseName); +INSTANTIATE_TEST_SUITE_P(smoke_Conv_Sum_Broadcast_INT8, ConcatConvSumInPlaceTestInt8, + ::testing::Combine( + ::testing::Values(convInpShape), + ::testing::Values(secondInp), + ::testing::Values(true, false), + ::testing::ValuesIn(fusingParamsSet), + ::testing::Values(cpuEmptyPluginConfig)), + ConcatConvSumInPlaceTest::getTestCaseName); + } // namespace } // namespace SubgraphTestsDefinitions diff --git a/src/tests/functional/plugin/cpu/subgraph_tests/src/denormal_check.cpp b/src/tests/functional/plugin/cpu/subgraph_tests/src/denormal_check.cpp new file mode 100644 index 00000000000..6127fb524fb --- /dev/null +++ b/src/tests/functional/plugin/cpu/subgraph_tests/src/denormal_check.cpp @@ -0,0 +1,110 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "shared_test_classes/base/layer_test_utils.hpp" +#include "shared_test_classes/base/ov_subgraph.hpp" +#include "ngraph_functions/utils/ngraph_helpers.hpp" +#include "ngraph_functions/builders.hpp" +#include "ngraph/runtime/aligned_buffer.hpp" + +using namespace InferenceEngine; +using namespace ov::test; +namespace SubgraphTestsDefinitions { + +template +class AlignedBufferWrapper { +public: + AlignedBufferWrapper(size_t size, size_t alignment) { + _buffer.reset(new ngraph::runtime::AlignedBuffer(size * sizeof(T), alignment)); + } + AlignedBufferWrapper(const AlignedBufferWrapper&) = delete; + AlignedBufferWrapper& operator=(const AlignedBufferWrapper&) = delete; + AlignedBufferWrapper(AlignedBufferWrapper&&) = default; + AlignedBufferWrapper& operator=(AlignedBufferWrapper&&) = default; + + T* get_ptr() { + return _buffer->get_ptr(); + } + + size_t size() const { + return _buffer->size() / sizeof(T); + } +private: + std::unique_ptr _buffer = nullptr; +}; + +class DenormalNullifyCheck : public SubgraphBaseTest { +protected: +std::unique_ptr> pConstStorage; + +void validate() override { + const auto& actualOutputs = get_plugin_outputs(); + ASSERT_FALSE(actualOutputs.empty()); + auto& outTensor = actualOutputs.front(); + ASSERT_EQ(ov::element::f32, outTensor.get_element_type()) << "Unexpected element type"; + const uint32_t* data = reinterpret_cast(outTensor.data()); + bool hasDenormals = false; + for (size_t i = 0; i < outTensor.get_size(); ++i) { + if (data[i] && (data[i] & (0xff << 23)) == 0) { + hasDenormals = true; + } + } + ASSERT_FALSE(hasDenormals); +} + + +void SetUp() override { + constexpr size_t alignment = 64; // bytes cache line size, to avoid denormals zeroing due to memory reallocation in the input node implementation + const ov::Shape inpShape = {1, 24, 3, 3}; + targetStaticShapes.push_back({inpShape}); + targetDevice = CommonTestUtils::DEVICE_CPU; + + const auto elemsCount = shape_size(inpShape); + const auto rtPrc = ov::element::f32; + auto params = ngraph::builder::makeParams(rtPrc, {inpShape}); + pConstStorage.reset(new AlignedBufferWrapper(elemsCount, alignment)); + + auto constTensor = std::make_shared(rtPrc, inpShape, pConstStorage->get_ptr()); + auto constNode = std::make_shared(constTensor); + ov::NodeVector input = {params[0], constNode}; + auto concat = std::make_shared(input, 1); + + ov::ResultVector results{std::make_shared(concat->output(0))}; + + function = std::make_shared(results, params, "denormal_check"); +} +}; + +TEST_F(DenormalNullifyCheck, smoke_CPU_Denormal_Check) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + using indexInterval = std::pair; + size_t elemsCount = pConstStorage->size(); + const indexInterval intervals[] = { + {0, elemsCount/2}, + {elemsCount/2, elemsCount}, + {0, elemsCount} + }; + + constexpr unsigned seed = 1u; + constexpr unsigned denormalsCount = 15u; + constexpr uint32_t denormalsRange = (0xffffffffu >> 9u) - 1; + testing::internal::Random random(seed); + auto randomRange = NGraphFunctions::Utils::generateVector(elemsCount, 10, -10); + + for (auto& interval : intervals) { + auto randomIndices = NGraphFunctions::Utils::generateVector(denormalsCount, interval.second, interval.first); + std::unordered_set randomIndexSet(randomIndices.begin(), randomIndices.end()); + for (size_t i = 0; i < elemsCount; ++i) { + if (randomIndexSet.count(i)) { + auto denormal = random.Generate(denormalsRange) + 1; + pConstStorage->get_ptr()[i] = *(reinterpret_cast(&denormal)); + } else { + pConstStorage->get_ptr()[i] = randomRange[i]; + } + } + + run(); + } +} +}// namespace SubgraphTestsDefinitions \ No newline at end of file diff --git a/src/tests/functional/plugin/cpu/test_utils/cpu_test_utils.cpp b/src/tests/functional/plugin/cpu/test_utils/cpu_test_utils.cpp index 502779f21b2..ce4a4edd8cc 100644 --- a/src/tests/functional/plugin/cpu/test_utils/cpu_test_utils.cpp +++ b/src/tests/functional/plugin/cpu/test_utils/cpu_test_utils.cpp @@ -114,24 +114,24 @@ std::string CPUTestsBase::impls2str(const std::vector &priority) { return str; } -void CPUTestsBase::CheckPluginRelatedResults(InferenceEngine::ExecutableNetwork &execNet, std::string nodeType) const { +void CPUTestsBase::CheckPluginRelatedResults(InferenceEngine::ExecutableNetwork &execNet, const std::string& nodeType) const { if (nodeType.empty()) return; ASSERT_TRUE(!selectedType.empty()) << "Node type is not defined."; InferenceEngine::CNNNetwork execGraphInfo = execNet.GetExecGraphInfo(); auto function = execGraphInfo.getFunction(); - CheckPluginRelatedResultsImpl(function, std::move(nodeType)); + CheckPluginRelatedResultsImpl(function, nodeType); } -void CPUTestsBase::CheckPluginRelatedResults(ov::CompiledModel &execNet, std::string nodeType) const { +void CPUTestsBase::CheckPluginRelatedResults(const ov::CompiledModel &execNet, const std::string& nodeType) const { if (nodeType.empty()) return; ASSERT_TRUE(!selectedType.empty()) << "Node type is not defined."; auto function = execNet.get_runtime_model(); - CheckPluginRelatedResultsImpl(function, std::move(nodeType)); + CheckPluginRelatedResultsImpl(function, nodeType); } -void CPUTestsBase::CheckPluginRelatedResultsImpl(std::shared_ptr function, std::string nodeType) const { +void CPUTestsBase::CheckPluginRelatedResultsImpl(const std::shared_ptr& function, const std::string& nodeType) const { ASSERT_NE(nullptr, function); for (const auto &node : function->get_ops()) { const auto & rtInfo = node->get_rt_info(); @@ -140,7 +140,7 @@ void CPUTestsBase::CheckPluginRelatedResultsImpl(std::shared_ptrsecond.as(); }; - auto getExecValueOutputsLayout = [] (std::shared_ptr node) -> std::string { + auto getExecValueOutputsLayout = [] (const std::shared_ptr& node) -> std::string { auto rtInfo = node->get_rt_info(); auto it = rtInfo.find(ExecGraphInfoSerialization::OUTPUT_LAYOUTS); IE_ASSERT(rtInfo.end() != it); @@ -261,7 +261,9 @@ std::string CPUTestsBase::getPrimitiveType() const { } CPUTestsBase::CPUInfo -CPUTestsBase::makeCPUInfo(std::vector inFmts, std::vector outFmts, std::vector priority) { +CPUTestsBase::makeCPUInfo(const std::vector& inFmts, + const std::vector& outFmts, + const std::vector& priority) { CPUInfo cpuInfo; if (!inFmts.empty()) { diff --git a/src/tests/functional/plugin/cpu/test_utils/cpu_test_utils.hpp b/src/tests/functional/plugin/cpu/test_utils/cpu_test_utils.hpp index 14d5fa04711..a95d2b3857f 100644 --- a/src/tests/functional/plugin/cpu/test_utils/cpu_test_utils.hpp +++ b/src/tests/functional/plugin/cpu/test_utils/cpu_test_utils.hpp @@ -124,9 +124,9 @@ public: static cpu_memory_format_t cpu_str2fmt(const char *str); static std::string fmts2str(const std::vector &fmts, const std::string &prefix); static std::string impls2str(const std::vector &priority); - static CPUInfo makeCPUInfo(std::vector inFmts, - std::vector outFmts, - std::vector priority); + static CPUInfo makeCPUInfo(const std::vector& inFmts, + const std::vector& outFmts, + const std::vector& priority); //TODO: change to setter method static std::string makeSelectedTypeStr(std::string implString, ngraph::element::Type_t elType); @@ -136,11 +136,11 @@ public: const std::shared_ptr &lastNode, std::string name); - void CheckPluginRelatedResults(InferenceEngine::ExecutableNetwork &execNet, std::string nodeType) const; - void CheckPluginRelatedResults(ov::CompiledModel &execNet, std::string nodeType) const; + void CheckPluginRelatedResults(InferenceEngine::ExecutableNetwork &execNet, const std::string& nodeType) const; + void CheckPluginRelatedResults(const ov::CompiledModel &execNet, const std::string& nodeType) const; protected: - virtual void CheckPluginRelatedResultsImpl(std::shared_ptr function, std::string nodeType) const; + virtual void CheckPluginRelatedResultsImpl(const std::shared_ptr& function, const std::string& nodeType) const; /** * @brief This function modifies the initial single layer test graph to add any necessary modifications that are specific to the cpu test scope. * @param ngPrc Graph precision. diff --git a/src/tests/functional/plugin/cpu/test_utils/fusing_test_utils.cpp b/src/tests/functional/plugin/cpu/test_utils/fusing_test_utils.cpp index 4b16a5e131a..2b0e2106974 100644 --- a/src/tests/functional/plugin/cpu/test_utils/fusing_test_utils.cpp +++ b/src/tests/functional/plugin/cpu/test_utils/fusing_test_utils.cpp @@ -36,7 +36,7 @@ CpuTestWithFusing::modifyGraph(const ngraph::element::Type &ngPrc, ngraph::Param return retNode; } -void CpuTestWithFusing::CheckFusingResults(std::shared_ptr function, std::string nodeType) const { +void CpuTestWithFusing::CheckFusingResults(const std::shared_ptr& function, const std::string& nodeType) const { ASSERT_NE(nullptr, function); bool isNodeFound = false; for (const auto & op : function->get_ops()) { @@ -55,7 +55,7 @@ void CpuTestWithFusing::CheckFusingResults(std::shared_ptr func std::string opFriendlyName = op->get_friendly_name(); auto pos = originalLayersNames.find(opFriendlyName); ASSERT_TRUE(pos != std::string::npos) << "Operation name " << op->get_friendly_name() << " has not been found in originalLayersNames!"; - for (auto fusedOp : fusedOps) { + for (const auto& fusedOp : fusedOps) { pos = originalLayersNames.find(fusedOp, checkFusingPosition ? pos : 0); ASSERT_TRUE(pos != std::string::npos) << "Fused op " << fusedOp << " has not been found!"; } @@ -64,7 +64,7 @@ void CpuTestWithFusing::CheckFusingResults(std::shared_ptr func ASSERT_TRUE(isNodeFound) << "Node type name: \"" << nodeType << "\" has not been found."; } -void CpuTestWithFusing::CheckPluginRelatedResultsImpl(std::shared_ptr function, std::string nodeType) const { +void CpuTestWithFusing::CheckPluginRelatedResultsImpl(const std::shared_ptr& function, const std::string& nodeType) const { CPUTestsBase::CheckPluginRelatedResultsImpl(function, nodeType); CheckFusingResults(function, nodeType); } @@ -87,8 +87,11 @@ std::shared_ptr postNodesMgr::addPostOps(const ngraph::element::Type &ngPrc, ngraph::ParameterVector ¶ms, const std::shared_ptr &lastNode) const { std::shared_ptr tmpNode = lastNode; - for (auto postNode : _postNodes) { - tmpNode = postNode.makeNode(tmpNode, ngPrc, params); + postNodeConfig cfg{lastNode, tmpNode, ngPrc, params}; + + for (const auto& postNode : _postNodes) { + cfg.input = tmpNode; + tmpNode = postNode.makeNode(cfg); } return tmpNode; } diff --git a/src/tests/functional/plugin/cpu/test_utils/fusing_test_utils.hpp b/src/tests/functional/plugin/cpu/test_utils/fusing_test_utils.hpp index b51c986d9be..9a1a0e55678 100644 --- a/src/tests/functional/plugin/cpu/test_utils/fusing_test_utils.hpp +++ b/src/tests/functional/plugin/cpu/test_utils/fusing_test_utils.hpp @@ -10,8 +10,15 @@ namespace CPUTestUtils { +struct postNodeConfig { + const std::shared_ptr target; + std::shared_ptr input; + const ngraph::element::Type& type; + ngraph::ParameterVector& params; +}; + struct postNodeBuilder { - std::function(std::shared_ptr, const ngraph::element::Type&, ngraph::ParameterVector&)> makeNode; + std::function(postNodeConfig& cfg)> makeNode; std::string name; }; @@ -26,7 +33,7 @@ public: class postFunctionMgr : public postOpMgr { public: - postFunctionMgr(std::shared_ptr function) : _pFunction(function) {} + postFunctionMgr(std::shared_ptr function) : _pFunction(std::move(function)) {} std::shared_ptr addPostOps(const ngraph::element::Type &ngPrc, ngraph::ParameterVector ¶ms, const std::shared_ptr &lastNode) const override; @@ -65,10 +72,10 @@ protected: ngraph::ParameterVector ¶ms, const std::shared_ptr &lastNode) override; - void CheckPluginRelatedResultsImpl(std::shared_ptr function, std::string nodeType) const override; + void CheckPluginRelatedResultsImpl(const std::shared_ptr& function, const std::string& nodeType) const override; private: - void CheckFusingResults(std::shared_ptr function, std::string nodeType) const; + void CheckFusingResults(const std::shared_ptr& function, const std::string& nodeType) const; protected: std::shared_ptr postOpMgrPtr; @@ -98,292 +105,311 @@ static ngraph::Shape generatePerChannelShape(const std::shared_ptr const auto emptyFusingSpec = fusingSpecificParams{nullptr, {}}; const auto fusingRelu = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Relu); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::Relu); }, "Relu"}}), {"Relu"}}; const auto fusingElu = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Elu, {}, {2.0f}); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::Elu, {}, {2.0f}); }, "Elu"}}), {"Elu"}}; const auto fusingGelu = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Gelu); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::Gelu); }, "Gelu"}}), {"Gelu"}}; const auto fusingSigmoid = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Sigmoid); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::Sigmoid); }, "Sigmoid"}}), {"Sigmoid"}}; const auto fusingClamp = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Clamp, {}, {3.0f, 6.0f}); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::Clamp, {}, {3.0f, 6.0f}); }, "Clamp"}}), {"Clamp"}}; const auto fusingTanh = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Tanh); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::Tanh); }, "Tanh"}}), {"Tanh"}}; const auto fusingAbs = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Abs); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::Abs); }, "Abs"}}), {"Abs"}}; const auto fusingSqrt = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Sqrt); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::Sqrt); }, "Sqrt"}}), {"Sqrt"}}; const auto fusingPReluPerChannel = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - ngraph::Shape newShape = generatePerChannelShape(inpNode); + {[](postNodeConfig& cfg){ + ngraph::Shape newShape = generatePerChannelShape(cfg.target); auto data = NGraphFunctions::Utils::generateVector(ngraph::shape_size(newShape)); - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::LeakyRelu, newShape, data); + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::LeakyRelu, newShape, data); }, "PRelu(PerChannel)"}}), {"PRelu"}}; const auto fusingPReluPerTensor = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ + {[](postNodeConfig& cfg){ ngraph::Shape shape(1, 1); auto data = NGraphFunctions::Utils::generateVector(ngraph::shape_size(shape)); - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::LeakyRelu, shape, data); + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::LeakyRelu, shape, data); }, "PRelu(PerTensor)"}}), {"PRelu"}}; const auto fusingSwish = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Swish, {}, {1.0f}); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::Swish, {}, {1.0f}); }, "Swish"}}), {"Swish"}}; const auto fusingSoftPlus = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::SoftPlus, {}, {}); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::SoftPlus, {}, {}); }, "SoftPlus"}}), {"SoftPlus"}}; const auto fusingHSwish = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::HSwish, {}, {}); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::HSwish, {}, {}); }, "HSwish"}}), {"HSwish"}}; const auto fusingMish = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Mish, {}, {}); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::Mish, {}, {}); }, "Mish"}}), {"Mish"}}; const auto fusingHSigmoid = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::HSigmoid); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::HSigmoid); }, "HSigmoid"}}), {"HSigmoid"}}; const auto fusingReluAdd = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Relu); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::Relu); }, "Relu"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - ngraph::Shape newShape = generatePerChannelShape(inpNode); - auto constNode = ngraph::builder::makeConstant(ngPrc, newShape, std::vector{}, true); - return std::make_shared(inpNode, constNode); + {[](postNodeConfig& cfg){ + ngraph::Shape newShape = generatePerChannelShape(cfg.target); + auto constNode = ngraph::builder::makeConstant(cfg.type, newShape, std::vector{}, true); + return std::make_shared(cfg.input, constNode); }, "Add(PerChannel)"}}), {"Relu", "Add"}}; const auto fusingReluScaleShift = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Relu); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::Relu); }, "Relu"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - ngraph::Shape newShape = generatePerChannelShape(inpNode); - auto constNode = ngraph::builder::makeConstant(ngPrc, newShape, std::vector{}, true); - return std::make_shared(inpNode, constNode); + {[](postNodeConfig& cfg){ + ngraph::Shape newShape = generatePerChannelShape(cfg.target); + auto constNode = ngraph::builder::makeConstant(cfg.type, newShape, std::vector{}, true); + return std::make_shared(cfg.input, constNode); }, "Multiply(PerChannel)"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - ngraph::Shape newShape = generatePerChannelShape(inpNode); - auto constNode = ngraph::builder::makeConstant(ngPrc, newShape, std::vector{}, true); - return std::make_shared(inpNode, constNode); + {[](postNodeConfig& cfg){ + ngraph::Shape newShape = generatePerChannelShape(cfg.target); + auto constNode = ngraph::builder::makeConstant(cfg.type, newShape, std::vector{}, true); + return std::make_shared(cfg.input, constNode); }, "Add(PerChannel)"}}), {"Relu", "Add"}}; const auto fusingScaleShift = fusingSpecificParams{ std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params) { - ngraph::Shape newShape = generatePerChannelShape(inpNode); - auto constNode = ngraph::builder::makeConstant(ngPrc, newShape, std::vector{}, true); - return std::make_shared(inpNode, constNode); + {[](postNodeConfig& cfg) { + ngraph::Shape newShape = generatePerChannelShape(cfg.target); + auto constNode = ngraph::builder::makeConstant(cfg.type, newShape, std::vector{}, true); + return std::make_shared(cfg.input, constNode); }, "Multiply(PerChannel)"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params) { - ngraph::Shape newShape = generatePerChannelShape(inpNode); - auto constNode = ngraph::builder::makeConstant(ngPrc, newShape, std::vector{}, true); - return std::make_shared(inpNode, constNode); + {[](postNodeConfig& cfg) { + ngraph::Shape newShape = generatePerChannelShape(cfg.target); + auto constNode = ngraph::builder::makeConstant(cfg.type, newShape, std::vector{}, true); + return std::make_shared(cfg.input, constNode); }, "Add(PerChannel)"}}), {"Add"} }; +const auto fusingScaleShiftAndFakeQuantizePerChannel = fusingSpecificParams{ std::make_shared(std::vector{ + {[](postNodeConfig& cfg) { + ngraph::Shape newShape = generatePerChannelShape(cfg.target); + auto constNode = ngraph::builder::makeConstant(cfg.type, newShape, std::vector{}, true); + return std::make_shared(cfg.input, constNode); + }, "Multiply(PerChannel)"}, + {[](postNodeConfig& cfg) { + ngraph::Shape newShape = generatePerChannelShape(cfg.target); + auto constNode = ngraph::builder::makeConstant(cfg.type, newShape, std::vector{}, true); + return std::make_shared(cfg.input, constNode); + }, "Add(PerChannel)"}, + {[](postNodeConfig& cfg){ + auto localPrc = cfg.input->get_element_type(); + ngraph::Shape newShape = generatePerChannelShape(cfg.target); + // auto newShape = ngraph::Shape(cfg.inputNode->get_output_partial_shape(0).size(), 1); + return ngraph::builder::makeFakeQuantize(cfg.input, localPrc, 256, newShape); + }, "FakeQuantize(PerChannel)"}}), {"FakeQuantize"}}; + const auto fusingFakeQuantizePerTensor = fusingSpecificParams{ std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - auto localPrc = inpNode->get_element_type(); - ngraph::Shape newShape(inpNode->get_output_partial_shape(0).size(), 1); - return ngraph::builder::makeFakeQuantize(inpNode, localPrc, 256, newShape); + {[](postNodeConfig& cfg){ + auto localPrc = cfg.input->get_element_type(); + ngraph::Shape newShape(cfg.input->get_output_partial_shape(0).size(), 1); + return ngraph::builder::makeFakeQuantize(cfg.input, localPrc, 256, newShape); }, "FakeQuantize(PerTensor)"}}), {"FakeQuantize"} }; const auto fusingFakeQuantizePerChannel = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - auto localPrc = inpNode->get_element_type(); - ngraph::Shape newShape = generatePerChannelShape(inpNode); - return ngraph::builder::makeFakeQuantize(inpNode, localPrc, 256, newShape); + {[](postNodeConfig& cfg){ + auto localPrc = cfg.input->get_element_type(); + ngraph::Shape newShape = generatePerChannelShape(cfg.target); + return ngraph::builder::makeFakeQuantize(cfg.input, localPrc, 256, newShape); }, "FakeQuantize(PerChannel)"}}), {"FakeQuantize"}}; const auto fusingFakeQuantizePerChannelRelu = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - auto localPrc = inpNode->get_element_type(); - ngraph::Shape newShape = generatePerChannelShape(inpNode); - return ngraph::builder::makeFakeQuantize(inpNode, localPrc, 256, newShape); + {[](postNodeConfig& cfg){ + auto localPrc = cfg.input->get_element_type(); + ngraph::Shape newShape = generatePerChannelShape(cfg.target); + return ngraph::builder::makeFakeQuantize(cfg.input, localPrc, 256, newShape); }, "FakeQuantize(PerChannel)"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Relu); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::Relu); }, "Relu"}}), {"FakeQuantize", "Relu"}}; const auto fusingFQPerChannelSigmoidFQPerChannel = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - auto localPrc = inpNode->get_element_type(); - auto shape = inpNode->get_output_partial_shape(0); + {[](postNodeConfig& cfg){ + auto localPrc = cfg.input->get_element_type(); + auto shape = cfg.input->get_output_partial_shape(0); if (shape.size() == 1) IE_THROW() << "If shape.size() == 1 then Granularity can be PerTensor only"; ngraph::Shape newShape(shape.size(), 1); newShape[1] = shape[1].get_length(); - return ngraph::builder::makeFakeQuantize(inpNode, localPrc, 256, newShape); + return ngraph::builder::makeFakeQuantize(cfg.input, localPrc, 256, newShape); }, "FakeQuantize(PerChannel)"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Sigmoid); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::Sigmoid); }, "Sigmoid"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - auto localPrc = inpNode->get_element_type(); - auto shape = inpNode->get_output_partial_shape(0); + {[](postNodeConfig& cfg){ + auto localPrc = cfg.input->get_element_type(); + auto shape = cfg.input->get_output_partial_shape(0); if (shape.size() == 1) IE_THROW() << "If shape.size() == 1 then Granularity can be PerTensor only"; ngraph::Shape newShape(shape.size(), 1); newShape[1] = shape[1].get_length(); - return ngraph::builder::makeFakeQuantize(inpNode, localPrc, 256, newShape); + return ngraph::builder::makeFakeQuantize(cfg.input, localPrc, 256, newShape); }, "FakeQuantize(PerChannel)"}}), {"FakeQuantize", "Sigmoid", "FakeQuantize"}}; const auto fusingFakeQuantizePerTensorRelu = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params) { - auto localPrc = inpNode->get_element_type(); - auto newShape = ngraph::Shape(inpNode->get_output_partial_shape(0).size(), 1); - return ngraph::builder::makeFakeQuantize(inpNode, localPrc, 256, newShape); + {[](postNodeConfig& cfg) { + auto localPrc = cfg.input->get_element_type(); + auto newShape = ngraph::Shape(cfg.input->get_output_partial_shape(0).size(), 1); + return ngraph::builder::makeFakeQuantize(cfg.input, localPrc, 256, newShape); }, "FakeQuantize(PerTensor)"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Relu); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::Relu); }, "Relu"}}), {"FakeQuantize", "Relu"}}; const auto fusingSum = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - auto shape = inpNode->get_output_partial_shape(0); - ngraph::ParameterVector newParams = ngraph::builder::makeDynamicParams(ngPrc, {shape}); - params.insert(params.end(), newParams.begin(), newParams.end()); + {[](postNodeConfig& cfg){ + auto shape = cfg.input->get_output_partial_shape(0); + ngraph::ParameterVector newParams = ngraph::builder::makeDynamicParams(cfg.type, {shape}); + cfg.params.insert(cfg.params.end(), newParams.begin(), newParams.end()); auto newParamOuts = ngraph::helpers::convert2OutputVector( ngraph::helpers::castOps2Nodes(newParams)); - return std::make_shared(inpNode, newParamOuts[0]); + return std::make_shared(cfg.input, newParamOuts[0]); }, "Add(Parameters)"}}), {"Add"}}; const auto fusingSumEluFQ = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - auto shape = inpNode->get_output_partial_shape(0); - ngraph::ParameterVector newParams = ngraph::builder::makeDynamicParams(ngPrc, {shape}); - params.insert(params.end(), newParams.begin(), newParams.end()); + {[](postNodeConfig& cfg){ + auto shape = cfg.input->get_output_partial_shape(0); + ngraph::ParameterVector newParams = ngraph::builder::makeDynamicParams(cfg.type, {shape}); + cfg.params.insert(cfg.params.end(), newParams.begin(), newParams.end()); auto newParamOuts = ngraph::helpers::convert2OutputVector( ngraph::helpers::castOps2Nodes(newParams)); - return std::make_shared(inpNode, newParamOuts[0]); + return std::make_shared(cfg.input, newParamOuts[0]); }, "Add(Parameters)"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::Elu, {}, {2.0f}); + {[](postNodeConfig& cfg){ + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::Elu, {}, {2.0f}); }, "Elu"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params) { - auto localPrc = inpNode->get_element_type(); - auto newShape = ngraph::Shape(inpNode->get_output_partial_shape(0).size(), 1); - return ngraph::builder::makeFakeQuantize(inpNode, localPrc, 256, newShape); + {[](postNodeConfig& cfg) { + auto localPrc = cfg.input->get_element_type(); + auto newShape = ngraph::Shape(cfg.input->get_output_partial_shape(0).size(), 1); + return ngraph::builder::makeFakeQuantize(cfg.input, localPrc, 256, newShape); }, "FakeQuantize(PerTensor)"}}), {"Add", "Elu", "FakeQuantize"}}; const auto fusingMultiplyPerTensor = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ + {[](postNodeConfig& cfg){ ngraph::Shape secondMultInShape(1, 1); - auto secondMultInput = ngraph::builder::makeConstant(ngPrc, secondMultInShape, std::vector{}, true); - return std::make_shared(inpNode, secondMultInput); + auto secondMultInput = ngraph::builder::makeConstant(cfg.type, secondMultInShape, std::vector{}, true); + return std::make_shared(cfg.input, secondMultInput); }, "Multiply(PerTensor)"}}), {"Multiply"}}; const auto fusingMultiplyPerChannel = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - ngraph::Shape secondMultInShape = generatePerChannelShape(inpNode); - auto secondMultInput = ngraph::builder::makeConstant(ngPrc, secondMultInShape, std::vector{}, true); - return std::make_shared(inpNode, secondMultInput); + {[](postNodeConfig& cfg){ + ngraph::Shape secondMultInShape = generatePerChannelShape(cfg.target); + auto secondMultInput = ngraph::builder::makeConstant(cfg.type, secondMultInShape, std::vector{}, true); + return std::make_shared(cfg.input, secondMultInput); }, "Multiply(PerChannel)"}}), {"Multiply"}}; const auto fusingAddPerTensor = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ + {[](postNodeConfig& cfg){ ngraph::Shape secondMultInShape(1, 1); - auto secondMultInput = ngraph::builder::makeConstant(ngPrc, secondMultInShape, std::vector{}, true); - return std::make_shared(inpNode, secondMultInput); + auto secondMultInput = ngraph::builder::makeConstant(cfg.type, secondMultInShape, std::vector{}, true); + return std::make_shared(cfg.input, secondMultInput); }, "Add(PerTensor)"}}), {"Add"}}; const auto fusingAddPerChannel = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - ngraph::Shape secondMultInShape = generatePerChannelShape(inpNode); - auto secondMultInput = ngraph::builder::makeConstant(ngPrc, secondMultInShape, std::vector{}, true); - return std::make_shared(inpNode, secondMultInput); + {[](postNodeConfig& cfg){ + ngraph::Shape secondMultInShape = generatePerChannelShape(cfg.target); + auto secondMultInput = ngraph::builder::makeConstant(cfg.type, secondMultInShape, std::vector{}, true); + return std::make_shared(cfg.input, secondMultInput); }, "Add(PerChannel)"}}), {"Add"}}; const auto fusingSubtractPerTensor = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ + {[](postNodeConfig& cfg){ ngraph::Shape secondMultInShape(1, 1); - auto secondMultInput = ngraph::builder::makeConstant(ngPrc, secondMultInShape, std::vector{}, true); - return std::make_shared(inpNode, secondMultInput); + auto secondMultInput = ngraph::builder::makeConstant(cfg.type, secondMultInShape, std::vector{}, true); + return std::make_shared(cfg.input, secondMultInput); }, "Subtract(PerTensor)"}}), {"Subtract"}}; const auto fusingSubtractPerChannel = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - ngraph::Shape secondMultInShape = generatePerChannelShape(inpNode); - auto secondMultInput = ngraph::builder::makeConstant(ngPrc, secondMultInShape, std::vector{}, true); - return std::make_shared(inpNode, secondMultInput); + {[](postNodeConfig& cfg){ + ngraph::Shape secondMultInShape = generatePerChannelShape(cfg.target); + auto secondMultInput = ngraph::builder::makeConstant(cfg.type, secondMultInShape, std::vector{}, true); + return std::make_shared(cfg.input, secondMultInput); }, "Subtract(PerChannel)"}}), {"Subtract"}}; const auto fusingDividePerTensor = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ + {[](postNodeConfig& cfg){ ngraph::Shape secondMultInShape(1, 1); - auto secondMultInput = ngraph::builder::makeConstant(ngPrc, secondMultInShape, std::vector{}, true); - return std::make_shared(inpNode, secondMultInput); + auto secondMultInput = ngraph::builder::makeConstant(cfg.type, secondMultInShape, std::vector{}, true); + return std::make_shared(cfg.input, secondMultInput); }, "Divide(PerTensor)"}}), {"Divide"}}; const auto fusingDividePerChannel = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - ngraph::Shape secondMultInShape = generatePerChannelShape(inpNode); - auto secondMultInput = ngraph::builder::makeConstant(ngPrc, secondMultInShape, std::vector{}, true); - return std::make_shared(inpNode, secondMultInput); + {[](postNodeConfig& cfg){ + ngraph::Shape secondMultInShape = generatePerChannelShape(cfg.target); + auto secondMultInput = ngraph::builder::makeConstant(cfg.type, secondMultInShape, std::vector{}, true); + return std::make_shared(cfg.input, secondMultInput); }, "Divide(PerChannel)"}}), {"Divide"}}; const auto fusingPRelu1D = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - auto shape = inpNode->get_output_partial_shape(0); + {[](postNodeConfig& cfg){ + auto shape = cfg.input->get_output_partial_shape(0); ngraph::Shape newShape({static_cast(shape[1].get_length())}); auto data = NGraphFunctions::Utils::generateVector(ngraph::shape_size(newShape)); - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::LeakyRelu, newShape, data); + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::LeakyRelu, newShape, data); }, "PRelu1D"}}), {"PRelu"}}; + const auto fusingPRelu1DScaleShift = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params){ - auto shape = inpNode->get_output_partial_shape(0); + {[](postNodeConfig& cfg){ + auto shape = cfg.input->get_output_partial_shape(0); ngraph::Shape newShape({static_cast(shape[1].get_length())}); auto data = NGraphFunctions::Utils::generateVector(ngraph::shape_size(newShape)); - return ngraph::builder::makeActivation(inpNode, ngPrc, ngraph::helpers::LeakyRelu, newShape, data); + return ngraph::builder::makeActivation(cfg.input, cfg.type, ngraph::helpers::LeakyRelu, newShape, data); }, "PRelu1D"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params) { - ngraph::Shape newShape = generatePerChannelShape(inpNode); - auto constNode = ngraph::builder::makeConstant(ngPrc, newShape, std::vector{}, true); - return std::make_shared(inpNode, constNode); + {[](postNodeConfig& cfg) { + ngraph::Shape newShape = generatePerChannelShape(cfg.input); + auto constNode = ngraph::builder::makeConstant(cfg.type, newShape, std::vector{}, true); + return std::make_shared(cfg.input, constNode); }, "Multiply(PerChannel)"}, - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params) { - ngraph::Shape newShape = generatePerChannelShape(inpNode); - auto constNode = ngraph::builder::makeConstant(ngPrc, newShape, std::vector{}, true); - return std::make_shared(inpNode, constNode); + {[](postNodeConfig& cfg) { + ngraph::Shape newShape = generatePerChannelShape(cfg.input); + auto constNode = ngraph::builder::makeConstant(cfg.type, newShape, std::vector{}, true); + return std::make_shared(cfg.input, constNode); }, "Add(PerChannel)"}}), {"Add"} }; const auto fusingBias = fusingSpecificParams{std::make_shared(std::vector{ - {[](std::shared_ptr inpNode, const ngraph::element::Type& ngPrc, ngraph::ParameterVector& params) { - size_t last_dim = inpNode->get_output_partial_shape(0).rbegin()->get_length(); - auto bias = ngraph::builder::makeConstant(ngPrc, ngraph::Shape{last_dim}, std::vector{}, true); - return std::make_shared(inpNode, bias); - }, "fusingBias"}}), {"Add"}}; + {[](postNodeConfig& cfg) { + size_t last_dim = cfg.input->get_output_partial_shape(0).rbegin()->get_length(); + auto bias = ngraph::builder::makeConstant(cfg.type, ngraph::Shape{last_dim}, std::vector{}, true); + return std::make_shared(cfg.input, bias); + }, "fusingBias"}}), {"Add"}}; } // namespace CPUTestUtils diff --git a/src/tests/functional/plugin/gpu/behavior/cache.cpp b/src/tests/functional/plugin/gpu/behavior/cache.cpp index 2d64e8812b9..853e2612497 100644 --- a/src/tests/functional/plugin/gpu/behavior/cache.cpp +++ b/src/tests/functional/plugin/gpu/behavior/cache.cpp @@ -49,6 +49,39 @@ TEST_F(CompiledKernelsCacheTest, CanCreateCacheDirAndDumpBinaries) { } } +TEST_F(CompiledKernelsCacheTest, TwoNetworksWithSameModelCreatesSameCache) { + std::shared_ptr ie = PluginCache::get().ie(); + // Create two CNNNetwork from same ngraph::Function + InferenceEngine::CNNNetwork cnnNet1(function); + InferenceEngine::CNNNetwork cnnNet2(function); + std::map config = {{ CONFIG_KEY(CACHE_DIR), cache_path }}; + try { + // Load 1st CNNNetwork + auto execNet1 = ie->LoadNetwork(cnnNet1, "GPU", config); + auto n_cache_files = CommonTestUtils::listFilesWithExt(cache_path, "cl_cache").size(); + + // Check that directory with cached kernels exists after loading network + ASSERT_TRUE(CommonTestUtils::directoryExists(cache_path)) << "Directory with cached kernels doesn't exist"; + // Load 2nd CNNNetwork + auto execNet2 = ie->LoadNetwork(cnnNet2, "GPU", config); + + // Check that two loaded networks with same function creates same caches + ASSERT_EQ(CommonTestUtils::removeFilesWithExt(cache_path, "cl_cache"), n_cache_files); + + // Remove directory and check that it doesn't exist anymore + ASSERT_EQ(CommonTestUtils::removeDir(cache_path), 0); + ASSERT_FALSE(CommonTestUtils::directoryExists(cache_path)); + } catch (std::exception& ex) { + // Cleanup in case of any exception + if (CommonTestUtils::directoryExists(cache_path)) { + ASSERT_GE(CommonTestUtils::removeFilesWithExt(cache_path, "cl_cache"), 0); + ASSERT_EQ(CommonTestUtils::removeDir(cache_path), 0); + } + FAIL() << ex.what() << std::endl; + } +} + + #ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT TEST_F(CompiledKernelsCacheTest, CanCreateCacheDirAndDumpBinariesUnicodePath) { diff --git a/src/tests/functional/plugin/gpu/remote_blob_tests/cldnn_remote_blob_tests.cpp b/src/tests/functional/plugin/gpu/remote_blob_tests/cldnn_remote_blob_tests.cpp index c3f30667002..0cab1150cf9 100644 --- a/src/tests/functional/plugin/gpu/remote_blob_tests/cldnn_remote_blob_tests.cpp +++ b/src/tests/functional/plugin/gpu/remote_blob_tests/cldnn_remote_blob_tests.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include @@ -98,6 +99,64 @@ TEST_P(RemoteBlob_Test, smoke_canInputUserBlob) { } } +TEST_P(RemoteBlob_Test, smoke_canUseRemoteBlobSimultaneously) { +#if defined(ANDROID) + GTEST_SKIP(); +#endif + const int batch = 2; + const int channels = 3; + const int height = 512; + const int width = 512; + const size_t img_size = batch * channels * height * width; + cl_int err; + + const InferenceEngine::TensorDesc tensor_desc{InferenceEngine::Precision::U8, + {batch, channels, height, width}, + InferenceEngine::Layout::NHWC}; + + InferenceEngine::Blob::Ptr ref_blob = FuncTestUtils::createAndFillBlob(tensor_desc); + + auto ie = PluginCache::get().ie(); + auto ocl_instance = std::make_shared(); + ocl_instance->_queue = cl::CommandQueue(ocl_instance->_context, ocl_instance->_device); + + // Allocate OpenCL buffer for data + cl::Buffer shared_buffer(ocl_instance->_context, CL_MEM_READ_WRITE, img_size, NULL, &err); + + // Create shared context + auto remote_context = make_shared_context(*ie, deviceName, ocl_instance->_queue.get()); + + // Wrap buffer above with IE blob + Blob::Ptr shared_blob = make_shared_blob(tensor_desc, remote_context, shared_buffer); + // Allocate is needed to actually trigger memory handle sharing. For other buffers it's called inside SetBlob impl + // TODO: Why do we need to call it explicitly? Consider doing it internally + shared_blob->allocate(); + + // Copy data from ordinary blob to OpenCL buffer + { + void* buffer = ref_blob->buffer(); + ocl_instance->_queue.enqueueWriteBuffer(shared_buffer, true, 0, img_size, buffer); + } + + // Lock remote buffer in multiple threads and compare data with ordinary one + const int threads_num = 8; + std::vector threads; + for (int i = 0; i < threads_num; i++) { + threads.emplace_back(std::thread{[&] { + auto ref_blob_buf = ref_blob->cbuffer(); + auto ref_blob_ptr = ref_blob_buf.as(); + auto remote_blob_buf = shared_blob->cbuffer(); + auto remote_blob_ptr = remote_blob_buf.as(); + ASSERT_EQ(ref_blob->size(), shared_blob->size()); + for (size_t j = 0; j < ref_blob->size(); j++) { + ASSERT_EQ(ref_blob_ptr[j], remote_blob_ptr[j]); + } + }}); + } + + for (auto& t : threads) + t.join(); +} TEST_P(RemoteBlob_Test, smoke_canInputPluginRemoteBlob) { #if defined(ANDROID) diff --git a/src/tests/functional/plugin/gpu/shared_tests_instances/blob_tests/dynamic_batch.cpp b/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/infer_request/dynamic_batch.cpp similarity index 95% rename from src/tests/functional/plugin/gpu/shared_tests_instances/blob_tests/dynamic_batch.cpp rename to src/tests/functional/plugin/gpu/shared_tests_instances/behavior/infer_request/dynamic_batch.cpp index 83c06de48b3..409511ea9fa 100644 --- a/src/tests/functional/plugin/gpu/shared_tests_instances/blob_tests/dynamic_batch.cpp +++ b/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/infer_request/dynamic_batch.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include +#include #include "common_test_utils/test_constants.hpp" namespace ConfigurationTestsDefinitions { diff --git a/src/tests/functional/plugin/gpu/shared_tests_instances/auto_batching/auto_batching_tests.cpp b/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/plugin/auto_batching_tests.cpp similarity index 65% rename from src/tests/functional/plugin/gpu/shared_tests_instances/auto_batching/auto_batching_tests.cpp rename to src/tests/functional/plugin/gpu/shared_tests_instances/behavior/plugin/auto_batching_tests.cpp index bb1460fcdb7..a67ef66b2ac 100644 --- a/src/tests/functional/plugin/gpu/shared_tests_instances/auto_batching/auto_batching_tests.cpp +++ b/src/tests/functional/plugin/gpu/shared_tests_instances/behavior/plugin/auto_batching_tests.cpp @@ -1,8 +1,9 @@ // Copyright (C) 2018-2022 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // -#include "auto_batching/auto_batching_tests.hpp" +#include "behavior/plugin/auto_batching_tests.hpp" #include "behavior/plugin/configuration_tests.hpp" +#include "openvino/runtime/properties.hpp" const std::vector num_streams{ 2 }; @@ -40,4 +41,22 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values(DefaultParameter{CONFIG_KEY(AUTO_BATCH_TIMEOUT), InferenceEngine::Parameter{"1000"}})), DefaultConfigurationTest::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P( + smoke_AutoBatching_GPU_2_0_string, + DefaultConfigurationTest, + ::testing::Combine( + ::testing::Values(std::string(CommonTestUtils::DEVICE_BATCH) + ":" + CommonTestUtils::DEVICE_GPU), + ::testing::Values(DefaultParameter{ov::auto_batch_timeout.name(), + InferenceEngine::Parameter{"1000"}})), + DefaultConfigurationTest::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P( + smoke_AutoBatching_GPU_2_0_uint, + DefaultConfigurationTest, + ::testing::Combine( + ::testing::Values(std::string(CommonTestUtils::DEVICE_BATCH) + ":" + CommonTestUtils::DEVICE_GPU), + ::testing::Values(DefaultParameter{ov::auto_batch_timeout.name(), + InferenceEngine::Parameter{1000}})), + DefaultConfigurationTest::getTestCaseName); } // namespace AutoBatchingTests \ No newline at end of file diff --git a/src/tests/functional/plugin/shared/src/blob_tests/detect_network_batch_test.cpp b/src/tests/functional/plugin/myriad/set_batch/detect_network_batch_test.cpp similarity index 87% rename from src/tests/functional/plugin/shared/src/blob_tests/detect_network_batch_test.cpp rename to src/tests/functional/plugin/myriad/set_batch/detect_network_batch_test.cpp index 0ba6f0f166a..3ccee366dad 100644 --- a/src/tests/functional/plugin/shared/src/blob_tests/detect_network_batch_test.cpp +++ b/src/tests/functional/plugin/myriad/set_batch/detect_network_batch_test.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "blob_tests/detect_network_batch_test.hpp" +#include "detect_network_batch_test.hpp" #include "ngraph_functions/subgraph_builders.hpp" #include "ngraph_functions/builders.hpp" @@ -78,3 +78,19 @@ TEST_P(DetectNetworkBatch, InferWithMultipleInputs_SameDims) { }; } // namespace LayerTestsDefinitions + +using namespace LayerTestsDefinitions; + +const std::vector batchSizes = { + 2, + 4, + 8, +}; + +namespace { +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, DetectNetworkBatch, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_MYRIAD), + ::testing::ValuesIn(batchSizes)), + DetectNetworkBatch::getTestCaseName); +} // namespace diff --git a/src/tests/functional/plugin/shared/include/blob_tests/detect_network_batch_test.hpp b/src/tests/functional/plugin/myriad/set_batch/detect_network_batch_test.hpp similarity index 100% rename from src/tests/functional/plugin/shared/include/blob_tests/detect_network_batch_test.hpp rename to src/tests/functional/plugin/myriad/set_batch/detect_network_batch_test.hpp diff --git a/src/tests/functional/plugin/myriad/shared_tests_instances/stress_tests/stress_tests.cpp b/src/tests/functional/plugin/myriad/shared_tests_instances/behavior/plugin/stress_tests.cpp similarity index 92% rename from src/tests/functional/plugin/myriad/shared_tests_instances/stress_tests/stress_tests.cpp rename to src/tests/functional/plugin/myriad/shared_tests_instances/behavior/plugin/stress_tests.cpp index 2dde4184585..c45f6a2f9e7 100644 --- a/src/tests/functional/plugin/myriad/shared_tests_instances/stress_tests/stress_tests.cpp +++ b/src/tests/functional/plugin/myriad/shared_tests_instances/behavior/plugin/stress_tests.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "stress_tests/stress_tests.hpp" +#include "behavior/plugin/stress_tests.hpp" using namespace LayerTestsDefinitions; diff --git a/src/tests/functional/plugin/myriad/shared_tests_instances/blob_tests/detect_network_batch_test.cpp b/src/tests/functional/plugin/myriad/shared_tests_instances/blob_tests/detect_network_batch_test.cpp deleted file mode 100644 index c66d9ef3657..00000000000 --- a/src/tests/functional/plugin/myriad/shared_tests_instances/blob_tests/detect_network_batch_test.cpp +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (C) 2018-2022 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#include "blob_tests/detect_network_batch_test.hpp" - -using namespace LayerTestsDefinitions; - -const std::vector batchSizes = { - 2, - 4, - 8, -}; - -namespace { - INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, DetectNetworkBatch, - ::testing::Combine( - ::testing::Values(CommonTestUtils::DEVICE_MYRIAD), - ::testing::ValuesIn(batchSizes)), - DetectNetworkBatch::getTestCaseName); -} // namespace diff --git a/src/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp b/src/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp index e892cecbc8b..af3fe8cbb26 100644 --- a/src/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp +++ b/src/tests/functional/plugin/myriad/shared_tests_instances/skip_tests_config.cpp @@ -82,5 +82,8 @@ std::vector disabledTestPatterns() { ".*InferRequestCheckTensorPrecision.*type=u8.*", ".*InferRequestCheckTensorPrecision.*type=u16.*", ".*InferRequestCheckTensorPrecision.*type=u64.*", + + // TODO: Issue 76209 + R"(.*MultithreadingTests.*canRun3SyncRequestsConsistentlyFromThreads.*MYRIAD.*)", }; } diff --git a/src/tests/functional/plugin/shared/include/blob_tests/dynamic_batch.hpp b/src/tests/functional/plugin/shared/include/behavior/infer_request/dynamic_batch.hpp similarity index 100% rename from src/tests/functional/plugin/shared/include/blob_tests/dynamic_batch.hpp rename to src/tests/functional/plugin/shared/include/behavior/infer_request/dynamic_batch.hpp diff --git a/src/tests/functional/plugin/shared/include/blob_tests/set_blob.hpp b/src/tests/functional/plugin/shared/include/behavior/infer_request/set_io_blob_precision.hpp similarity index 100% rename from src/tests/functional/plugin/shared/include/blob_tests/set_blob.hpp rename to src/tests/functional/plugin/shared/include/behavior/infer_request/set_io_blob_precision.hpp diff --git a/src/tests/functional/plugin/shared/include/behavior/ov_plugin/properties_tests.hpp b/src/tests/functional/plugin/shared/include/behavior/ov_plugin/properties_tests.hpp new file mode 100644 index 00000000000..3f90e4b68ca --- /dev/null +++ b/src/tests/functional/plugin/shared/include/behavior/ov_plugin/properties_tests.hpp @@ -0,0 +1,52 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "base/ov_behavior_test_utils.hpp" +#include "openvino/runtime/properties.hpp" +#include "openvino/util/common_util.hpp" + +#include "common_test_utils/test_assertions.hpp" +#include "common_test_utils/file_utils.hpp" +#include "common_test_utils/unicode_utils.hpp" + +namespace ov { +namespace test { +namespace behavior { + +class OVPropertiesBase : public CommonTestUtils::TestsCommon { +public: + std::shared_ptr core = utils::PluginCache::get().core(); + std::shared_ptr model; + std::string device_name; + AnyMap properties; +}; + +class OVEmptyPropertiesTests : public testing::WithParamInterface, + public OVPropertiesBase { +public: + static std::string getTestCaseName(testing::TestParamInfo obj); + + void SetUp() override; +}; + +using PropertiesParams = std::tuple; + +class OVPropertiesTests : public testing::WithParamInterface, + public OVPropertiesBase { +public: + static std::string getTestCaseName(testing::TestParamInfo obj); + + void SetUp() override; + + void TearDown() override; +}; + +using OVPropertiesIncorrectTests = OVPropertiesTests; +using OVPropertiesDefaultTests = OVPropertiesTests; + +} // namespace behavior +} // namespace test +} // namespace ov \ No newline at end of file diff --git a/src/tests/functional/plugin/shared/include/auto_batching/auto_batching_tests.hpp b/src/tests/functional/plugin/shared/include/behavior/plugin/auto_batching_tests.hpp similarity index 100% rename from src/tests/functional/plugin/shared/include/auto_batching/auto_batching_tests.hpp rename to src/tests/functional/plugin/shared/include/behavior/plugin/auto_batching_tests.hpp diff --git a/src/tests/functional/plugin/shared/include/hetero/synthetic.hpp b/src/tests/functional/plugin/shared/include/behavior/plugin/hetero_synthetic.hpp similarity index 100% rename from src/tests/functional/plugin/shared/include/hetero/synthetic.hpp rename to src/tests/functional/plugin/shared/include/behavior/plugin/hetero_synthetic.hpp diff --git a/src/tests/functional/plugin/shared/include/behavior/plugin/life_time.hpp b/src/tests/functional/plugin/shared/include/behavior/plugin/life_time.hpp index 39cb28f6cbd..ca6c2ff4bdf 100644 --- a/src/tests/functional/plugin/shared/include/behavior/plugin/life_time.hpp +++ b/src/tests/functional/plugin/shared/include/behavior/plugin/life_time.hpp @@ -16,6 +16,7 @@ #include #include "gtest/gtest.h" #include "common_test_utils/test_common.hpp" +#include "common_test_utils/crash_handler.hpp" #include "functional_test_utils/skip_tests_config.hpp" #include "functional_test_utils/precision_utils.hpp" #include diff --git a/src/tests/functional/plugin/shared/include/stress_tests/stress_tests.hpp b/src/tests/functional/plugin/shared/include/behavior/plugin/stress_tests.hpp similarity index 100% rename from src/tests/functional/plugin/shared/include/stress_tests/stress_tests.hpp rename to src/tests/functional/plugin/shared/include/behavior/plugin/stress_tests.hpp diff --git a/src/tests/functional/plugin/shared/src/blob_tests/dynamic_batch.cpp b/src/tests/functional/plugin/shared/src/behavior/infer_request/dynamic_batch.cpp similarity index 99% rename from src/tests/functional/plugin/shared/src/blob_tests/dynamic_batch.cpp rename to src/tests/functional/plugin/shared/src/behavior/infer_request/dynamic_batch.cpp index e553e2bb703..67ec9088099 100644 --- a/src/tests/functional/plugin/shared/src/blob_tests/dynamic_batch.cpp +++ b/src/tests/functional/plugin/shared/src/behavior/infer_request/dynamic_batch.cpp @@ -12,7 +12,7 @@ #include "ie_transformations.hpp" #include "common_test_utils/common_utils.hpp" #include "functional_test_utils/skip_tests_config.hpp" -#include "blob_tests/dynamic_batch.hpp" +#include "behavior/infer_request/dynamic_batch.hpp" #include "ngraph_functions/subgraph_builders.hpp" diff --git a/src/tests/functional/plugin/shared/src/blob_tests/set_blob.cpp b/src/tests/functional/plugin/shared/src/behavior/infer_request/set_io_blob_precision.cpp similarity index 80% rename from src/tests/functional/plugin/shared/src/blob_tests/set_blob.cpp rename to src/tests/functional/plugin/shared/src/behavior/infer_request/set_io_blob_precision.cpp index 5fe7a17c4d7..717c5979336 100644 --- a/src/tests/functional/plugin/shared/src/blob_tests/set_blob.cpp +++ b/src/tests/functional/plugin/shared/src/behavior/infer_request/set_io_blob_precision.cpp @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "blob_tests/set_blob.hpp" -#include +#include "behavior/infer_request/set_io_blob_precision.hpp" +#include "ngraph_functions/builders.hpp" using namespace InferenceEngine; @@ -11,17 +11,17 @@ namespace BehaviorTestsDefinitions { std::ostream& operator<<(std::ostream & os, setType type) { switch (type) { - case setType::INPUT: - os << "INPUT"; - break; - case setType::OUTPUT: - os << "OUTPUT"; - break; - case setType::BOTH: - os << "BOTH"; - break; - default: - IE_THROW() << "Not supported type for SetBlob"; + case setType::INPUT: + os << "INPUT"; + break; + case setType::OUTPUT: + os << "OUTPUT"; + break; + case setType::BOTH: + os << "BOTH"; + break; + default: + IE_THROW() << "Not supported type for SetBlob"; } return os; } @@ -43,19 +43,19 @@ std::string SetBlobTest::getTestCaseName(testing::TestParamInfo o inline void fillBlob(Blob::Ptr &blob) { switch (blob->getTensorDesc().getPrecision()) { #define CASE(X) case X: CommonTestUtils::fill_data_random(blob); break; - CASE(Precision::U8) - CASE(Precision::I8) - CASE(Precision::U16) - CASE(Precision::I16) - CASE(Precision::U32) - CASE(Precision::I32) - CASE(Precision::U64) - CASE(Precision::I64) - CASE(Precision::BF16) - CASE(Precision::FP16) - CASE(Precision::FP32) - CASE(Precision::FP64) - CASE(Precision::BOOL) + CASE(Precision::U8) + CASE(Precision::I8) + CASE(Precision::U16) + CASE(Precision::I16) + CASE(Precision::U32) + CASE(Precision::I32) + CASE(Precision::U64) + CASE(Precision::I64) + CASE(Precision::BF16) + CASE(Precision::FP16) + CASE(Precision::FP32) + CASE(Precision::FP64) + CASE(Precision::BOOL) #undef CASE default: IE_THROW() << "Can't fill blob with precision: " << blob->getTensorDesc().getPrecision(); diff --git a/src/tests/functional/plugin/shared/src/behavior/ov_executable_network/properties.cpp b/src/tests/functional/plugin/shared/src/behavior/ov_executable_network/properties.cpp index 6263ea0ed1c..7068000f690 100644 --- a/src/tests/functional/plugin/shared/src/behavior/ov_executable_network/properties.cpp +++ b/src/tests/functional/plugin/shared/src/behavior/ov_executable_network/properties.cpp @@ -17,7 +17,7 @@ std::string OVCompiledModelEmptyPropertiesTests::getTestCaseName(testing::TestPa void OVCompiledModelEmptyPropertiesTests::SetUp() { SKIP_IF_CURRENT_TEST_IS_DISABLED() device_name = this->GetParam(); - model = ngraph::builder::subgraph::makeConvPoolRelu(); + model = ov::test::behavior::getDefaultNGraphFunctionForTheDevice(device_name); } std::string OVCompiledModelPropertiesTests::getTestCaseName(testing::TestParamInfo obj) { @@ -35,7 +35,7 @@ std::string OVCompiledModelPropertiesTests::getTestCaseName(testing::TestParamIn void OVCompiledModelPropertiesTests::SetUp() { SKIP_IF_CURRENT_TEST_IS_DISABLED(); std::tie(device_name, properties) = this->GetParam(); - model = ngraph::builder::subgraph::makeConvPoolRelu(); + model = ov::test::behavior::getDefaultNGraphFunctionForTheDevice(device_name); } void OVCompiledModelPropertiesTests::TearDown() { diff --git a/src/tests/functional/plugin/shared/src/behavior/ov_plugin/properties_tests.cpp b/src/tests/functional/plugin/shared/src/behavior/ov_plugin/properties_tests.cpp new file mode 100644 index 00000000000..a7d4083c99a --- /dev/null +++ b/src/tests/functional/plugin/shared/src/behavior/ov_plugin/properties_tests.cpp @@ -0,0 +1,101 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "behavior/ov_plugin/properties_tests.hpp" +#include "openvino/runtime/properties.hpp" +#include + +namespace ov { +namespace test { +namespace behavior { + +std::string OVEmptyPropertiesTests::getTestCaseName(testing::TestParamInfo obj) { + return "device_name=" + obj.param; +} + +void OVEmptyPropertiesTests::SetUp() { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + device_name = this->GetParam(); + model = ngraph::builder::subgraph::makeConvPoolRelu(); +} + +std::string OVPropertiesTests::getTestCaseName(testing::TestParamInfo obj) { + std::string device_name; + AnyMap properties; + std::tie(device_name, properties) = obj.param; + std::ostringstream result; + result << "device_name=" << device_name << "_"; + if (!properties.empty()) { + result << "properties=" << util::join(util::split(util::to_string(properties), ' '), "_"); + } + return result.str(); +} + +void OVPropertiesTests::SetUp() { + SKIP_IF_CURRENT_TEST_IS_DISABLED(); + std::tie(device_name, properties) = this->GetParam(); + model = ngraph::builder::subgraph::makeConvPoolRelu(); +} + +void OVPropertiesTests::TearDown() { + if (!properties.empty()) { + utils::PluginCache::get().reset(); + } +} + +TEST_P(OVEmptyPropertiesTests, SetEmptyProperties) { + OV_ASSERT_NO_THROW(core->get_property(device_name, ov::supported_properties)); + OV_ASSERT_NO_THROW(core->set_property(device_name, AnyMap{})); +} + +// Setting correct properties doesn't throw +TEST_P(OVPropertiesTests, SetCorrectProperties) { + OV_ASSERT_NO_THROW(core->set_property(device_name, properties)); +} + +TEST_P(OVPropertiesTests, canSetPropertyAndCheckGetProperty) { + core->set_property(device_name, properties); + for (const auto& property_item : properties) { + Any property; + OV_ASSERT_NO_THROW(property = core->get_property(device_name, property_item.first)); + ASSERT_FALSE(property.empty()); + std::cout << property_item.first << ":" << property.as() << std::endl; + } +} + +TEST_P(OVPropertiesIncorrectTests, SetPropertiesWithIncorrectKey) { + ASSERT_THROW(core->set_property(device_name, properties), ov::Exception); +} + +TEST_P(OVPropertiesIncorrectTests, CanNotCompileModelWithIncorrectProperties) { + ASSERT_THROW(core->compile_model(model, device_name, properties), ov::Exception); +} + +TEST_P(OVPropertiesDefaultTests, CanSetDefaultValueBackToPlugin) { + std::vector supported_properties; + OV_ASSERT_NO_THROW(supported_properties = core->get_property(device_name, ov::supported_properties)); + for (auto& supported_property : supported_properties) { + Any property; + OV_ASSERT_NO_THROW(property = core->get_property(device_name, supported_property)); + if (supported_property.is_mutable()) { + OV_ASSERT_NO_THROW(core->set_property(device_name, {{ supported_property, property}})); + } + } +} + +TEST_P(OVPropertiesDefaultTests, CheckDefaultValues) { + std::vector supported_properties; + OV_ASSERT_NO_THROW(supported_properties = core->get_property(device_name, ov::supported_properties)); + for (auto&& default_property : properties) { + auto supported = util::contains(supported_properties, default_property.first); + ASSERT_TRUE(supported) << "default_property=" << default_property.first; + Any property; + OV_ASSERT_NO_THROW(property = core->get_property(device_name, default_property.first)); + ASSERT_EQ(default_property.second, property); + } +} + +} // namespace behavior +} // namespace test +} // namespace ov diff --git a/src/tests/functional/plugin/shared/src/hetero/synthetic.cpp b/src/tests/functional/plugin/shared/src/behavior/plugin/hetero_synthetic.cpp similarity index 99% rename from src/tests/functional/plugin/shared/src/hetero/synthetic.cpp rename to src/tests/functional/plugin/shared/src/behavior/plugin/hetero_synthetic.cpp index 6165aef7cf0..040e22bf53d 100644 --- a/src/tests/functional/plugin/shared/src/hetero/synthetic.cpp +++ b/src/tests/functional/plugin/shared/src/behavior/plugin/hetero_synthetic.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "hetero/synthetic.hpp" +#include "behavior/plugin/hetero_synthetic.hpp" #include #include #include "ngraph_functions/builders.hpp" diff --git a/src/tests/functional/plugin/shared/src/behavior/plugin/life_time.cpp b/src/tests/functional/plugin/shared/src/behavior/plugin/life_time.cpp index df57618e916..149620d7b46 100644 --- a/src/tests/functional/plugin/shared/src/behavior/plugin/life_time.cpp +++ b/src/tests/functional/plugin/shared/src/behavior/plugin/life_time.cpp @@ -8,17 +8,9 @@ #include #include "behavior/plugin/life_time.hpp" -#ifndef _WIN32 - #include - #include -#endif +#include namespace BehaviorTestsDefinitions { - -#ifndef _WIN32 - static jmp_buf env; -#endif - std::string HoldersTest::getTestCaseName(testing::TestParamInfo obj) { std::string targetDevice; std::vector order; @@ -38,19 +30,6 @@ namespace BehaviorTestsDefinitions { SKIP_IF_CURRENT_TEST_IS_DISABLED(); std::tie(targetDevice, order) = this->GetParam(); function = ngraph::builder::subgraph::makeConvPoolRelu(); - -#ifndef _WIN32 - // configure handling of crash - auto crashHandler = [](int errCode) { - std::cerr << "Unexpected application crash with code: " << errCode << std::endl; - siglongjmp(env, 1); - }; - struct sigaction act; - act.sa_handler = crashHandler; - sigemptyset(&act.sa_mask); - act.sa_flags = 0; - sigaction(SIGSEGV, &act, 0); -#endif } void release_order_test(std::vector order, const std::string &deviceName, @@ -90,29 +69,35 @@ namespace BehaviorTestsDefinitions { } TEST_P(HoldersTest, Orders) { + // in case of crash jump will be made and work will be continued + auto crashHandler = std::unique_ptr(new CommonTestUtils::CrashHandler()); + // Test failed if crash happens #ifdef _WIN32 - EXPECT_NO_THROW(release_order_test(order, targetDevice, function)); + if (setjmp(CommonTestUtils::env) == 0) { #else - if (sigsetjmp(env, 1) == 0) { - release_order_test(order, targetDevice, function); + if (sigsetjmp(CommonTestUtils::env, 1) == 0) { +#endif + EXPECT_NO_THROW(release_order_test(order, targetDevice, function)); } else { IE_THROW() << "Crash happens"; } -#endif } TEST_P(HoldersTestImportNetwork, Orders) { + // in case of crash jump will be made and work will be continued + auto crashHandler = std::unique_ptr(new CommonTestUtils::CrashHandler()); + // Test failed if crash happens #ifdef _WIN32 - EXPECT_NO_THROW(release_order_test(order, targetDevice, function)); + if (setjmp(CommonTestUtils::env) == 0) { #else - if (sigsetjmp(env, 1) == 0) { - release_order_test(order, targetDevice, function); + if (sigsetjmp(CommonTestUtils::env, 1) == 0) { +#endif + EXPECT_NO_THROW(release_order_test(order, targetDevice, function)); } else { IE_THROW() << "Crash happens"; } -#endif } std::string HoldersTestOnImportedNetwork::getTestCaseName(testing::TestParamInfo obj) { diff --git a/src/tests/functional/plugin/shared/src/stress_tests/stress_tests.cpp b/src/tests/functional/plugin/shared/src/behavior/plugin/stress_tests.cpp similarity index 97% rename from src/tests/functional/plugin/shared/src/stress_tests/stress_tests.cpp rename to src/tests/functional/plugin/shared/src/behavior/plugin/stress_tests.cpp index 8119c9b55c6..11b8317e7f9 100644 --- a/src/tests/functional/plugin/shared/src/stress_tests/stress_tests.cpp +++ b/src/tests/functional/plugin/shared/src/behavior/plugin/stress_tests.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "stress_tests/stress_tests.hpp" +#include "behavior/plugin/stress_tests.hpp" #include "ngraph_functions/subgraph_builders.hpp" namespace LayerTestsDefinitions { diff --git a/src/tests/functional/plugin/shared/src/single_layer_tests/op_impl_check/op_impl_check.cpp b/src/tests/functional/plugin/shared/src/single_layer_tests/op_impl_check/op_impl_check.cpp index cef5aee662a..3a3b8536598 100644 --- a/src/tests/functional/plugin/shared/src/single_layer_tests/op_impl_check/op_impl_check.cpp +++ b/src/tests/functional/plugin/shared/src/single_layer_tests/op_impl_check/op_impl_check.cpp @@ -7,6 +7,7 @@ #endif #include "single_layer_tests/op_impl_check/op_impl_check.hpp" +#include "common_test_utils/crash_handler.hpp" namespace ov { namespace test { @@ -16,21 +17,26 @@ void OpImplCheckTest::run() { if (function == nullptr) { GTEST_FAIL() << "Target function is empty!"; } - auto crashHandler = [](int errCode) { - auto& s = LayerTestsUtils::Summary::getInstance(); - s.saveReport(); - std::cerr << "Unexpected application crash with code: " << errCode << std::endl; - std::abort(); - }; - signal(SIGSEGV, crashHandler); - summary.setDeviceName(targetDevice); - try { - auto executableNetwork = core->compile_model(function, targetDevice, configuration); - summary.updateOPsImplStatus(function, true); - } catch (...) { - summary.updateOPsImplStatus(function, false); - GTEST_FAIL() << "Error in the LoadNetwork!"; + // in case of crash jump will be made and work will be continued + auto crashHandler = std::unique_ptr(new CommonTestUtils::CrashHandler()); + + // place to jump in case of a crash +#ifdef _WIN32 + if (setjmp(CommonTestUtils::env) == 0) { +#else + if (sigsetjmp(CommonTestUtils::env, 1) == 0) { +#endif + summary.setDeviceName(targetDevice); + try { + auto executableNetwork = core->compile_model(function, targetDevice, configuration); + summary.updateOPsImplStatus(function, true); + } catch (...) { + summary.updateOPsImplStatus(function, false); + GTEST_FAIL() << "Error in the LoadNetwork!"; + } + } else { + IE_THROW() << "Crash happens"; } } diff --git a/src/tests/functional/plugin/shared/src/single_layer_tests/op_impl_check/single_op_graph.cpp b/src/tests/functional/plugin/shared/src/single_layer_tests/op_impl_check/single_op_graph.cpp index f0e5cbedde6..c78823e79f6 100644 --- a/src/tests/functional/plugin/shared/src/single_layer_tests/op_impl_check/single_op_graph.cpp +++ b/src/tests/functional/plugin/shared/src/single_layer_tests/op_impl_check/single_op_graph.cpp @@ -14,9 +14,921 @@ std::shared_ptr generate(const std::shared_ptr &node) { return nullptr; } +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{3, 2}}); + const auto elu = std::make_shared(params[0], 0.5f); + ov::ResultVector results{std::make_shared(elu)}; + return std::make_shared(results, params, "ElueGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{5, 2}}); + const auto indices = ngraph::builder::makeConstant(ov::element::i32, {4}, {0, 2, 3, 4}); + const auto segment_ids = ngraph::builder::makeConstant(ov::element::i32, {4}, {0, 0, 2, 2}); + const auto num_segments = ngraph::builder::makeConstant(ov::element::i32, {}, {3}); + const auto default_index = ngraph::builder::makeConstant(ov::element::i32, {}, {0}); + const auto per_sample_weights = + ngraph::builder::makeConstant(ov::element::f32, {4}, {0.5, 0.5, 0.5, 0.5}); + const auto embed_seg_sum = std::make_shared(params[0], + indices, + segment_ids, + num_segments, + default_index, + per_sample_weights); + ov::ResultVector results{std::make_shared(embed_seg_sum)}; + return std::make_shared(results, params, "EmbeddingSegmentsSum"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto rois = ngraph::builder::makeConstant( + ov::element::f32, + {{16, 4}}, + {1.0f, 1.0f, 10.0f, 10.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 4.0f, 1.0f, 8.0f, 5.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f}); + const auto deltas = ngraph::builder::makeConstant( + ov::element::f32, + {{16, 8}}, + {5.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 4.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 8.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f}); + const auto scores = ngraph::builder::makeConstant( + ov::element::f32, + {{16, 2}}, + {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f}); + const auto im_info = ngraph::builder::makeConstant(ov::element::f32, {{1, 3}}, {1.0f, 1.0f, 1.0f}); + const auto attrs = ov::op::v6::ExperimentalDetectronDetectionOutput::Attributes{0.01000000074505806f, + 0.2f, + 2.0f, + 2, + 500, + 5, + true, + {10.0f, 10.0f, 5.0f, 5.0f}}; + const auto exp_detection_output = + std::make_shared(rois, deltas, scores, im_info, attrs); + ov::ResultVector results{std::make_shared(exp_detection_output)}; + return std::make_shared(results, + ngraph::ParameterVector{}, + "ExperimentalDetectronDetectionOutput"); +} + +std::shared_ptr generate( + const std::shared_ptr &node) { + const auto im_info = ngraph::builder::makeConstant(ov::element::f32, {3}, {1.0f, 1.0f, 1.0f}); + const auto anchors = ngraph::builder::makeConstant( + ov::element::f32, + {{36, 4}}, + {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f}); + const auto deltas = ngraph::builder::makeConstant( + ov::element::f32, + {{12, 2, 6}}, + {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f}); + const auto scores = ngraph::builder::makeConstant( + ov::element::f32, + {{3, 2, 6}}, + {5.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 4.0f, 1.0f, 1.0f, 1.0f, + 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 8.0f, 1.0f}); + const auto attrs = + ov::op::v6::ExperimentalDetectronGenerateProposalsSingleImage::Attributes{0, 0.699999988079071, 6, 1000}; + const auto exp_gen_prop_sing_img = + std::make_shared(im_info, + anchors, + deltas, + scores, + attrs); + ov::ResultVector results{std::make_shared(exp_gen_prop_sing_img)}; + return std::make_shared(results, + ngraph::ParameterVector{}, + "ExperimentalDetectronGenerateProposalsSingleImage"); +} + +std::shared_ptr generate( + const std::shared_ptr &node) { + const auto params = + ngraph::builder::makeDynamicParams(ov::element::f32, {{3, 4}, {1, 16, 4, 5}, {1, 3, 100, 200}}); + const auto attrs = ov::op::v6::ExperimentalDetectronPriorGridGenerator::Attributes{true, 0, 0, 4.0f, 4.0f}; + const auto exp_prior_grid_gen = std::make_shared(params[0], + params[1], + params[2], + attrs); + ov::ResultVector results{std::make_shared(exp_prior_grid_gen)}; + return std::make_shared(results, params, "ExperimentalDetectronPriorGridGenerator"); +} + +std::shared_ptr generate( + const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 4}, {1, 2, 2, 3}}); + const auto attrs = ov::op::v6::ExperimentalDetectronROIFeatureExtractor::Attributes{3, 2, {4}, false}; + const auto exp_roi_feature_ext = + std::make_shared(NodeVector{params[0], params[1]}, + attrs); + ov::ResultVector results{std::make_shared(exp_roi_feature_ext)}; + return std::make_shared(results, params, "ExperimentalDetectronROIFeatureExtractor"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 4}, {2}}); + const auto exp_topk_rois = std::make_shared(params[0], params[1], 1); + ov::ResultVector results{std::make_shared(exp_topk_rois)}; + return std::make_shared(results, params, "ExperimentalDetectronTopKROIs"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 1, 10, 10}}); + const auto ext_img_patch = std::make_shared(params[0], + ov::Shape{3, 3}, + ov::Strides{5, 5}, + ov::Shape{1, 1}, + ov::op::PadType::VALID); + ov::ResultVector results{std::make_shared(ext_img_patch)}; + return std::make_shared(results, params, "ExtractImagePatches"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 2, 3, 4}}); + const auto input_low = ngraph::builder::makeConstant(ov::element::f32, {}, {0.f}); + const auto input_high = ngraph::builder::makeConstant(ov::element::f32, {}, {23.f}); + const auto output_low = ngraph::builder::makeConstant(ov::element::f32, {}, {2.f}); + const auto output_high = ngraph::builder::makeConstant(ov::element::f32, {}, {16.f}); + const auto fake_quantize = std::make_shared(params[0], input_low, input_high, output_low, output_high, 4); + ov::ResultVector results{std::make_shared(fake_quantize)}; + return std::make_shared(results, params, "FakeQuantize"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{3, 4}}); + const auto grn = std::make_shared(params[0], 1e-6); + ov::ResultVector results{std::make_shared(grn)}; + return std::make_shared(results, params, "GRN"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = + ngraph::builder::makeDynamicParams({ov::element::f32, ov::element::f32, ov::element::i64}, + {{5, 10, 10}, {5, 1, 10}, {5}}); + const auto W = ngraph::builder::makeConstant(ov::element::f32, {1, 30, 10}, {}, true); + const auto R = ngraph::builder::makeConstant(ov::element::f32, {1, 30, 10}, {}, true); + const auto B = ngraph::builder::makeConstant(ov::element::f32, {1, 30}, {}, true); + const size_t hidden_size = 10; + const auto gru_sequence = + std::make_shared(params[0], + params[1], + params[2], + W, + R, + B, + hidden_size, + ov::op::RecurrentSequenceDirection::FORWARD); + ov::ResultVector results{std::make_shared(gru_sequence)}; + return std::make_shared(results, params, "GRUSequence"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams({ov::element::f32, ov::element::i32}, {{3}, {7}}); + const auto gather_elements = std::make_shared(params[0], params[1], 0); + ov::ResultVector results{std::make_shared(gather_elements)}; + return std::make_shared(results, params, "GatherElements"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 1, 10}, {1, 1, 10}, {1}, {}}); + const auto gather_tree = std::make_shared(params[0], params[1], params[2], params[3]); + ov::ResultVector results{std::make_shared(gather_tree)}; + return std::make_shared(results, params, "GatherTree"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{8}}); + const auto gelu = std::make_shared(params[0]); + ov::ResultVector results{std::make_shared(gelu)}; + return std::make_shared(results, params, "Gelu"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 1, 6}, {1, 1, 1, 3}}); + const auto group_convolution = std::make_shared(params[0], + params[1], + ov::Strides{1}, + ov::CoordinateDiff{0}, + ov::CoordinateDiff{0}, + ov::Strides{1}); + ov::ResultVector results{std::make_shared(group_convolution)}; + return std::make_shared(results, params, "GroupConvolution"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 1, 4}, {1, 1, 1, 3}}); + const auto group_convolution = std::make_shared(params[0], + params[1], + ov::Strides{1}, + ov::CoordinateDiff{0}, + ov::CoordinateDiff{0}, + ov::Strides{1}, + ov::op::PadType{ov::op::PadType::EXPLICIT}); + ov::ResultVector results{std::make_shared(group_convolution)}; + return std::make_shared(results, params, "GroupConvolutionBackpropData"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{3}}); + const auto alpha = ngraph::builder::makeConstant(ov::element::f32, {}, {std::vector{0.5}}); + const auto beta = ngraph::builder::makeConstant(ov::element::f32, {}, {std::vector{0.6}}); + const auto hard_sigmoid = std::make_shared(params[0], alpha, beta); + ov::ResultVector results{std::make_shared(hard_sigmoid)}; + return std::make_shared(results, params, "HardSigmoid"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 1, 2, 4}}); + const auto out_shape_in = ngraph::builder::makeConstant(ov::element::i64, {4}, {1, 1, 1, 2}); + ov::op::v0::Interpolate::Attributes attrs; + attrs.axes = ov::AxisSet{0, 1, 2, 3}; + attrs.mode = "nearest"; + attrs.align_corners = false; + attrs.antialias = false; + attrs.pads_begin = std::vector{0, 0, 0, 0}; + attrs.pads_end = std::vector{0, 0, 0, 0}; + const auto interpolate = std::make_shared(params[0], out_shape_in, attrs); + ov::ResultVector results{std::make_shared(interpolate)}; + return std::make_shared(results, params, "Interpolat-1"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + using InterpolateAttrs = op::v4::Interpolate::InterpolateAttrs; + using InterpolateMode = op::v4::Interpolate::InterpolateMode; + using ShapeCalcMode = op::v4::Interpolate::ShapeCalcMode; + using TransformMode = op::v4::Interpolate::CoordinateTransformMode; + using NearestMode = op::v4::Interpolate::NearestMode; + + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 1, 2, 4}}); + const auto out_shape_in = ngraph::builder::makeConstant(ov::element::i64, {4}, {1, 1, 1, 2}); + const auto scales = ngraph::builder::makeConstant(ov::element::f32, {1}, {1.0}); + const InterpolateAttrs attrs{InterpolateMode::NEAREST, + ShapeCalcMode::SIZES, + std::vector{0, 0, 0, 0}, + std::vector{0, 0, 0, 0}, + TransformMode::HALF_PIXEL, + NearestMode::ROUND_PREFER_FLOOR, + false, + -0.75}; + const auto interpolate = std::make_shared(params[0], out_shape_in, scales, attrs); + ov::ResultVector results{std::make_shared(interpolate)}; + return std::make_shared(results, params, "Interpolate-4"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1}}); + auto read_value = std::make_shared(params[0], "v0"); + auto add = std::make_shared(read_value, params[0]); + auto assign = std::make_shared(add, "v0"); + ov::ResultVector results{std::make_shared(add)}; + return std::make_shared(results, SinkVector{assign}, params, "Assign-3"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1}}); + auto variable = std::make_shared( + ov::op::util::VariableInfo{ov::PartialShape::dynamic(), ov::element::dynamic, "v0"}); + auto read_value = std::make_shared(params[0], variable); + auto add = std::make_shared(read_value, params[0]); + auto assign = std::make_shared(add, variable); + ov::ResultVector results{std::make_shared(add)}; + return std::make_shared(results, SinkVector{assign}, params, "Assign-6"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 3, 2, 1}}); + const auto axes = ngraph::builder::makeConstant(ov::element::i64, {1}, std::vector{2}); + const auto lrn = std::make_shared(params[0], axes, 3, 0.5, 1, 3); + ov::ResultVector results{std::make_shared(lrn)}; + return std::make_shared(results, params, "LRN"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = + ngraph::builder::makeDynamicParams({ov::element::f32, ov::element::f32, ov::element::f32, ov::element::i32}, + {{5, 10, 10}, {5, 1, 10}, {5, 1, 10}, {5}}); + const auto W = ngraph::builder::makeConstant(ov::element::f32, {1, 40, 10}, {}, true); + const auto R = ngraph::builder::makeConstant(ov::element::f32, {1, 40, 10}, {}, true); + const auto B = ngraph::builder::makeConstant(ov::element::f32, {1, 40}, {}, true); + const auto P = ngraph::builder::makeConstant(ov::element::f32, {1, 30}, {}, true); + const int64_t hidden_size = 10; + const auto lstm_sequence = + std::make_shared(params[0], + params[1], + params[2], + params[3], + W, + R, + B, + P, + hidden_size, + ov::op::RecurrentSequenceDirection::FORWARD); + ov::ResultVector results{std::make_shared(lstm_sequence->output(0)), + std::make_shared(lstm_sequence->output(1)), + std::make_shared(lstm_sequence->output(2))}; + return std::make_shared(results, params, "LSTMSequence"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1}}); + const auto lsm = std::make_shared(params[0], 0); + ov::ResultVector results{std::make_shared(lsm)}; + return std::make_shared(results, params, "LogSoftmax"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::boolean, {{1, 2}}); + const auto logical_not = std::make_shared(params[0]); + ov::ResultVector results{std::make_shared(logical_not)}; + return std::make_shared(results, params, "LogicalNot"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 3, 3, 3}}); + const auto mvn = std::make_shared(params[0], false, false, 1e-9); + ov::ResultVector results{std::make_shared(mvn)}; + return std::make_shared(results, params, "MVN-2"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 3, 3, 3}}); + const auto axes = ngraph::builder::makeConstant(ov::element::i64, {2}, std::vector{2, 3}); + const auto mvn = std::make_shared(params[0], axes, false, 1e-9, ov::op::MVNEpsMode::OUTSIDE_SQRT); + ov::ResultVector results{std::make_shared(mvn)}; + return std::make_shared(results, params, "MVN-6"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 2}, {2, 2}}); + const auto matmul = std::make_shared(params[0], params[1], false, false); + ov::ResultVector results{std::make_shared(matmul)}; + return std::make_shared(results, params, "MatMul-1"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 2}}); + const auto mish = std::make_shared(params[0]); + ov::ResultVector results{std::make_shared(mish)}; + return std::make_shared(results, params, "Mish-4"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams( + {ov::element::f32, ov::element::f32, ov::element::i32, ov::element::f32, ov::element::f32}, + {{1, 6, 4}, {1, 1, 6}, {}, {}, {}}); + auto nms = std::make_shared(params[0], + params[1], + params[2], + params[3], + params[4], + ov::op::v1::NonMaxSuppression::BoxEncodingType::CENTER, + false); + ov::ResultVector results{std::make_shared(nms)}; + return std::make_shared(results, params, "NonMaxSuppression-1"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams( + {ov::element::f32, ov::element::f32, ov::element::i32, ov::element::f32, ov::element::f32}, + {{1, 6, 4}, {1, 1, 6}, {}, {}, {}}); + auto nms = std::make_shared(params[0], + params[1], + params[2], + params[3], + params[4], + ov::op::v3::NonMaxSuppression::BoxEncodingType::CENTER, + false); + ov::ResultVector results{std::make_shared(nms)}; + return std::make_shared(results, params, "NonMaxSuppression-1"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams( + {ov::element::f32, ov::element::f32, ov::element::i32, ov::element::f32, ov::element::f32}, + {{1, 6, 4}, {1, 1, 6}, {}, {}, {}}); + auto nms = std::make_shared(params[0], + params[1], + params[2], + params[3], + params[4], + ov::op::v4::NonMaxSuppression::BoxEncodingType::CENTER, + false); + ov::ResultVector results{std::make_shared(nms)}; + return std::make_shared(results, params, "NonMaxSuppression-1"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams( + {ov::element::f32, ov::element::f32, ov::element::i32, ov::element::f32, ov::element::f32}, + {{1, 6, 4}, {1, 1, 6}, {}, {}, {}}); + auto nms = std::make_shared(params[0], + params[1], + params[2], + params[3], + params[4], + ov::op::v5::NonMaxSuppression::BoxEncodingType::CENTER, + false); + ov::ResultVector results{std::make_shared(nms)}; + return std::make_shared(results, params, "NonMaxSuppression-1"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{3, 2}}); + auto nonzero = std::make_shared(params[0], ov::element::i32); + ov::ResultVector results{std::make_shared(nonzero)}; + return std::make_shared(results, params, "NonZero-3"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{4}}); + const auto axes = ngraph::builder::makeConstant(ov::element::i64, {0}, std::vector{}); + auto normalize = std::make_shared(params[0], axes, 1e-7, ov::op::EpsMode::ADD); + ov::ResultVector results{std::make_shared(normalize)}; + return std::make_shared(results, params, "NormalizeL2-1"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::i32, {{}}); + const auto depth = ngraph::builder::makeConstant(ov::element::i32, {}, std::vector{3}); + const auto onvalue = ngraph::builder::makeConstant(ov::element::i32, {}, std::vector{1}); + const auto offvalue = ngraph::builder::makeConstant(ov::element::i32, {}, std::vector{0}); + const int32_t axes = 0; + const auto onehot = std::make_shared(params[0], depth, onvalue, offvalue, axes); + ov::ResultVector results{std::make_shared(onehot)}; + return std::make_shared(results, params, "OneHot-1"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{6}}); + const auto slope = ngraph::builder::makeConstant(ov::element::f32, {1}, {2}); + const auto prelu = std::make_shared(params[0], slope); + ov::ResultVector results{std::make_shared(prelu)}; + return std::make_shared(results, params, "PRelu-1"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const std::string mode = "average"; + const size_t n_channel = 8; + const size_t n_group = 2; + const size_t n_boxes = 3; + const size_t spatial_bin_x = 1; + const size_t spatial_bin_y = 1; + const float spatial_scale = 1; + const size_t output_dim = n_channel / (n_group * n_group); + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, n_channel, 20, 20}}); + const auto coordi = ngraph::builder::makeConstant(ov::element::f32, + {n_boxes, 5}, + {0, 1, 2, 4, 6, 1, 0, 3, 10, 4, 0, 10, 7, 11, 13}); + const auto psroi_pooling = std::make_shared(params[0], + coordi, + output_dim, + n_group, + spatial_scale, + spatial_bin_x, + spatial_bin_y, + mode); + ov::ResultVector results{std::make_shared(psroi_pooling)}; + return std::make_shared(results, params, "PSROIPooling-1"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{6}}); + const auto pad_begin = ngraph::builder::makeConstant(ov::element::i64, {1}, {4}); + const auto pad_end = ngraph::builder::makeConstant(ov::element::i64, {1}, {5}); + const auto pad = std::make_shared(params[0], pad_begin, pad_end, ov::op::PadMode::CONSTANT); + ov::ResultVector results{std::make_shared(pad)}; + return std::make_shared(results, params, "Pad-1"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto in = std::make_shared(ov::element::f32, Shape{3, 4}); + return std::make_shared(in, ParameterVector{in}, "Parameter-1"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + ov::op::v0::PriorBox::Attributes attrs; + attrs.min_size = {2.0f}; + attrs.aspect_ratio = {1.5f}; + attrs.scale_all_sizes = false; + const auto LS = ngraph::builder::makeConstant(ov::element::i32, {2}, {2, 2}); + const auto IS = ngraph::builder::makeConstant(ov::element::i32, {2}, {10, 10}); + auto Node = std::make_shared(LS, IS, attrs); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, ov::ParameterVector{}, "PrioBoxGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + ov::op::v8::PriorBox::Attributes attrs; + attrs.min_size = {2.0f}; + attrs.max_size = {5.0f}; + attrs.aspect_ratio = {1.5f}; + attrs.scale_all_sizes = true; + attrs.min_max_aspect_ratios_order = false; + const auto LS = ngraph::builder::makeConstant(ov::element::i32, {2}, {2, 2}); + const auto IS = ngraph::builder::makeConstant(ov::element::i32, {2}, {10, 10}); + auto Node = std::make_shared(LS, IS, attrs); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, ov::ParameterVector{}, "PrioBoxGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + ov::op::v0::PriorBoxClustered::Attributes attrs; + attrs.widths = {3.0f}; + attrs.heights = {3.0f}; + attrs.clip = true; + const auto LS = ngraph::builder::makeConstant(ov::element::i32, {2}, {2, 2}); + const auto IS = ngraph::builder::makeConstant(ov::element::i32, {2}, {10, 10}); + auto Node = std::make_shared(LS, IS, attrs); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, ov::ParameterVector{}, "PrioBoxClustedGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + ov::op::v0::Proposal::Attributes attrs; + attrs.base_size = 16; + attrs.min_size = 16; + attrs.pre_nms_topn = 6000; + attrs.post_nms_topn = 10; + attrs.nms_thresh = 0.7f; + attrs.feat_stride = 16; + attrs.min_size = 16; + attrs.ratio = {0.5f}; + attrs.scale = {32.0f}; + attrs.clip_before_nms = true; + attrs.clip_after_nms = false; + attrs.normalize = false; + attrs.box_size_scale = 1.0f; + attrs.box_coordinate_scale = 1.0f; + attrs.framework = ""; + attrs.infer_probs = false; + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 2, 10, 10}, + {1, 4, 10, 10}, + {3}}); + auto Node = std::make_shared(params.at(0), params.at(1), params.at(2), attrs); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "ProposalGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + ov::op::v4::Proposal::Attributes attrs; + attrs.base_size = 16; + attrs.min_size = 16; + attrs.pre_nms_topn = 6000; + attrs.post_nms_topn = 10; + attrs.nms_thresh = 0.7f; + attrs.feat_stride = 16; + attrs.min_size = 16; + attrs.ratio = {0.5f}; + attrs.scale = {32.0f}; + attrs.clip_before_nms = true; + attrs.clip_after_nms = false; + attrs.normalize = false; + attrs.box_size_scale = 1.0f; + attrs.box_coordinate_scale = 1.0f; + attrs.framework = ""; + attrs.infer_probs = true; + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 2, 10, 10}, + {1, 4, 10, 10}, + {3}}); + auto Node = std::make_shared(params.at(0), params.at(1), params.at(2), attrs); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "ProposalGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 1, 16, 16}}); + const auto coords = ngraph::builder::makeConstant(ov::element::f32, {2, 4}, {2, 2, 8, 8, 2, 2, 8, 8}); + const auto roisIdx = ngraph::builder::makeConstant(ov::element::i32, {2}, {0, 1}); + auto Node = std::make_shared(params.at(0), coords, roisIdx, 2, 2, 2, 1, "avg"); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "ROIAlignGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 3, 8, 8}, + {1, 5}}); + auto Node = std::make_shared(params.at(0), params.at(1), Shape{1, 1}, 1); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "ROIPoolingGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto out_shape_ = ngraph::builder::makeConstant(ov::element::i64, {4}, {1, 3, 3, 3}); + const auto min_value = ngraph::builder::makeConstant(ov::element::f32, {}, {0.f}); + const auto max_value = ngraph::builder::makeConstant(ov::element::f32, {}, {1.f}); + auto Node = std::make_shared(out_shape_, min_value, max_value, ov::element::f32, 10, 10); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, ov::ParameterVector{}, "RandomUniformGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto start = ngraph::builder::makeConstant(ov::element::f32, {}, {1.f}); + const auto stop = ngraph::builder::makeConstant(ov::element::f32, {}, {5.f}); + const auto step = ngraph::builder::makeConstant(ov::element::f32, {}, {1.f}); + auto Node = std::make_shared(start, stop, step); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, ov::ParameterVector{}, "RangeGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto start = ngraph::builder::makeConstant(ov::element::f32, {}, {1.f}); + const auto stop = ngraph::builder::makeConstant(ov::element::f32, {}, {5.f}); + const auto step = ngraph::builder::makeConstant(ov::element::f32, {}, {1.f}); + auto Node = std::make_shared(start, stop, step, ov::element::f32); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, ov::ParameterVector{}, "RangeGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 8, 2, 2}}); + auto Node = std::make_shared(params.at(0), 4, 1, 1, true, std::vector{0}, 1, 3); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "RegionYoloGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 8, 4, 4}}); + auto Node = std::make_shared(params.at(0), ov::Strides{2}); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "ReorgYoloGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 2, 3}}); + const auto shape = ngraph::builder::makeConstant(ov::element::i64, {1}, {12}); + auto Node = std::make_shared(params.at(0), shape, false); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "ReshapeGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeParams(ov::element::f32, {{2, 2}}); + ov::ResultVector results{std::make_shared(params.at(0))}; + return std::make_shared(results, params, "ResultGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 4, 3}}); + const auto axis = ngraph::builder::makeConstant(ov::element::i64, {3}, {0, 1, 2}); + auto Node = std::make_shared(params.at(0), axis, op::v1::Reverse::Mode::INDEX); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "ReverseGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams({ov::element::f32, ov::element::i32}, + {{3, 10}, {3}}); + auto Node = std::make_shared(params.at(0), params.at(1), 0, 1); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "ReverseSequenceGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{4, 2, 3}}); + const auto shift = ngraph::builder::makeConstant(ov::element::i64, {3}, {2, 1, 3}); + const auto axes = ngraph::builder::makeConstant(ov::element::i64, {3}, {0, 1, 2}); + auto Node = std::make_shared(params.at(0), shift, axes); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "RollGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{10}}); + auto Node = std::make_shared(params.at(0), op::v5::Round::RoundMode::HALF_TO_EVEN); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "RoundGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 2}, {2, 2}}); + const auto indices = ngraph::builder::makeConstant(ov::element::i64, {2, 2}, {1, 1, 0, 0}); + const auto axis = ngraph::builder::makeConstant(ov::element::i64, {1}, {0}); + auto Node = std::make_shared(params.at(0), indices, params.at(1), axis); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "ScatterElementsUpdateGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams({ov::element::boolean, ov::element::f32, ov::element::f32}, + {{2, 2, 2}, {2, 2, 2}, {2, 2, 2}}); + auto Node = std::make_shared(params.at(0), params.at(1), params.at(2), op::AutoBroadcastType::NONE); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "SelectGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{3}}); + const auto alpha = ngraph::builder::makeConstant(ov::element::f32, {1}, {1.67326324}); + const auto lambda = ngraph::builder::makeConstant(ov::element::f32, {1}, {1.05070098}); + auto Node = std::make_shared(params.at(0), alpha, lambda); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "SeluGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 4, 8, 16, 64}}); + auto Node = std::make_shared(params.at(0)); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "ShapeOfGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 4, 8, 16, 64}}); + auto Node = std::make_shared(params.at(0)); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "ShapeOfGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 15, 2, 2}}); + auto Node = std::make_shared(params.at(0), 1, 5); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "ShuffleChannelsGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 4, 3}}); + const auto start = ngraph::builder::makeConstant(ov::element::i64, {3}, {0, 0, 4}); + const auto stop = ngraph::builder::makeConstant(ov::element::i64, {3}, {2, 4, -5}); + const auto step = ngraph::builder::makeConstant(ov::element::i64, {3}, {3, 2, -2}); + const auto axes = ngraph::builder::makeConstant(ov::element::i64, {3}, {0, 1, 2}); + auto Node = std::make_shared(params.at(0), start, stop, step, axes); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "SliceGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{4, 4}}); + auto Node = std::make_shared(params.at(0)); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "SoftPlusGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 2, 3}}); + auto Node = std::make_shared(params.at(0), 0); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "SoftmaxGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 2, 3}}); + auto Node = std::make_shared(params.at(0), 0); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "SoftmaxGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 1, 3, 2, 1}}); + const auto blockShape = ngraph::builder::makeConstant(ov::element::i64, {5}, {1, 1, 3, 2, 2}); + const auto padsBegin = ngraph::builder::makeConstant(ov::element::i64, {5}, {0, 0, 1, 0, 3}); + const auto padsEnd = ngraph::builder::makeConstant(ov::element::i64, {5}, {0, 0, 2, 0, 0}); + auto Node = std::make_shared(params.at(0), blockShape, padsBegin, padsEnd); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "SpaceToBatchGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 2, 4, 4}}); + auto Node = std::make_shared(params.at(0), "BLOCKS_FIRST", 2); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "SpaceToDepthGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 8, 2}}); + const auto axis = ngraph::builder::makeConstant(ov::element::i64, {}, {1}); + auto Node = std::make_shared(params.at(0), axis, 4); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "SplitGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 4, 1, 1, 2}}); + const auto axes = ngraph::builder::makeConstant(ov::element::i64, {2}, {0, 2}); + auto Node = std::make_shared(params.at(0), axes); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "SqueezeGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{128, 1}}); + const auto begin = ngraph::builder::makeConstant(ov::element::i64, {3}, {0, 0, 0}); + const auto end = ngraph::builder::makeConstant(ov::element::i64, {3}, {0, 0, 0}); + const auto stride = ngraph::builder::makeConstant(ov::element::i64, {3}, {1, 1, 1}); + auto Node = std::make_shared(params.at(0), begin, end, stride, + std::vector{0, 1, 1}, + std::vector{0, 1, 1}, + std::vector{1, 0, 0}, + std::vector{1, 0, 0}, + std::vector{0, 0, 0}); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "StridedSliceGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 4}}); + const auto beta = ngraph::builder::makeConstant(ov::element::f32, {}, {0.6f}); + auto Node = std::make_shared(params.at(0), beta); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "SwishGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 1, 3}}); + const auto repeats = ngraph::builder::makeConstant(ov::element::i64, {2}, {2, 1}); + auto Node = std::make_shared(params.at(0), repeats); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "TileGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 3, 2}}); + const auto k = ngraph::builder::makeConstant(ov::element::i64, {}, {3}); + auto Node = std::make_shared(params.at(0), + k, + 1, + ov::op::v1::TopK::Mode::MAX, + ov::op::v1::TopK::SortType::SORT_VALUES); + ov::ResultVector results{std::make_shared(Node->output(0)), + std::make_shared(Node->output(1))}; + return std::make_shared(results, params, "TopKGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 3, 2}}); + const auto k = ngraph::builder::makeConstant(ov::element::i64, {}, {3}); + auto Node = std::make_shared(params.at(0), + k, + 1, + ov::op::v3::TopK::Mode::MAX, + ov::op::v3::TopK::SortType::SORT_VALUES); + ov::ResultVector results{std::make_shared(Node->output(0)), + std::make_shared(Node->output(1))}; + return std::make_shared(results, params, "TopKGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 2, 3}}); + const auto inputOrder = ngraph::builder::makeConstant(ov::element::i64, {3}, {2, 1, 0}); + auto Node = std::make_shared(params.at(0), inputOrder); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "TransposeGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{4, 2}}); + const auto axes = ngraph::builder::makeConstant(ov::element::i64, {2}, {1, -1}); + auto Node = std::make_shared(params.at(0), axes); + ov::ResultVector results{std::make_shared(Node)}; + return std::make_shared(results, params, "UnsqueezeGraph"); +} + +std::shared_ptr generate(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 8, 2, 2}}); + const auto axis = ngraph::builder::makeConstant(ov::element::i64, {1}, {1}); + const auto splitLengths = ngraph::builder::makeConstant(ov::element::i64, {4}, {1, 3, 2, 2}); + auto Node = std::make_shared(params.at(0), axis, splitLengths); + ov::ResultVector results{std::make_shared(Node->output(0)), + std::make_shared(Node->output(1)), + std::make_shared(Node->output(2)), + std::make_shared(Node->output(3))}; + return std::make_shared(results, params, "VariadicSplitGraph"); +} + std::shared_ptr generateBinaryEltwise(const std::shared_ptr &node) { - const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 2}, - {1, 2}}); + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 2}, {1, 2}}); std::shared_ptr eltwiseNode; if (ov::is_type(node)) { eltwiseNode = std::make_shared(params.front(), params.back()); @@ -36,21 +948,449 @@ std::shared_ptr generateBinaryEltwise(const std::shared_ptr(params.front(), params.back()); } else if (ov::is_type(node)) { eltwiseNode = std::make_shared(params.front(), params.back()); + } else if (ov::is_type(node)) { + eltwiseNode = std::make_shared(params.front(), params.back()); } else { return nullptr; } - ngraph::ResultVector results{std::make_shared(eltwiseNode)}; - return std::make_shared(results, params, "BinaryEltwiseGraph"); + ov::ResultVector results{std::make_shared(eltwiseNode)}; + return std::make_shared(results, params, "BinaryEltwiseGraph"); } -} // namespace + +std::shared_ptr generateBinaryEltwiseComp(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2}, {2}}); + std::shared_ptr eltwise; + if (ov::is_type(node)) { + eltwise = std::make_shared(params[0], params[1]); + } else if (ov::is_type(node)) { + eltwise = std::make_shared(params[0], params[1]); + } else if (ov::is_type(node)) { + eltwise = std::make_shared(params[0], params[1]); + } else if (ov::is_type(node)) { + eltwise = std::make_shared(params[0], params[1]); + } else if (ov::is_type(node)) { + eltwise = std::make_shared(params[0], params[1]); + } else if (ov::is_type(node)) { + eltwise = std::make_shared(params[0], params[1]); + } else { + return nullptr; + } + + ov::ResultVector results{std::make_shared(eltwise)}; + return std::make_shared(results, params, "BinaryEltwiseComparisonGraph"); +} + +std::shared_ptr generateBinaryEltwiseLogical(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::boolean, {{1}, {1}}); + std::shared_ptr eltwise; + if (ov::is_type(node)) { + eltwise = std::make_shared(params[0], params[1]); + } else if (ov::is_type(node)) { + eltwise = std::make_shared(params[0], params[1]); + } else if (ov::is_type(node)) { + eltwise = std::make_shared(params[0], params[1]); + } else if (ov::is_type(node)) { + eltwise = std::make_shared(params[0], params[1]); + } else { + return nullptr; + } + + ov::ResultVector results{std::make_shared(eltwise)}; + return std::make_shared(results, ngraph::ParameterVector{params}, "BinaryEltwiseLogicalGraph"); +} + +std::shared_ptr generateBroadcast(const std::shared_ptr &node) { + const ov::Shape input_shape{}; + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {input_shape}); + const auto shape_const = + ngraph::builder::makeConstant(ov::element::u64, {4}, {5, 4, 3, 2}); + std::shared_ptr broadcast; + if (ov::is_type(node)) { + broadcast = std::make_shared(params[0], shape_const); + } else if (ov::is_type(node)) { + broadcast = std::make_shared(params[0], shape_const); + } else { + return nullptr; + } + + return std::make_shared(broadcast, ParameterVector{params}, "BroadcastGraph"); +} + +std::shared_ptr generateConvertColor(const std::shared_ptr &node) { + const auto params = std::make_shared(ov::element::u8, Shape{1, 3, 2, 1}); + std::shared_ptr convert; + if (ov::is_type(node)) { + convert = std::make_shared(params); + } else if (ov::is_type(node)) { + convert = std::make_shared(params); + } else if (ov::is_type(node)) { + convert = std::make_shared(params); + } else if (ov::is_type(node)) { + convert = std::make_shared(params); + } else { + return nullptr; + } + + ov::ResultVector results{std::make_shared(convert)}; + return std::make_shared(results, ParameterVector{params}, "ConvertColorGraph"); +} + +std::shared_ptr generateMultiSubGraph(const std::shared_ptr &node) { + if (ov::is_type(node)) { + auto cond = std::make_shared(ov::element::boolean, Shape{1}); + auto A = std::make_shared(ov::element::f32, ov::Shape{1}, 8.0); + auto B = std::make_shared(element::f32, ov::Shape{1}, 2.0); + auto A_res = std::make_shared(A); + auto B_res = std::make_shared(B); + auto then_body = std::make_shared(OutputVector{A_res}, ParameterVector{}); + auto else_body = std::make_shared(OutputVector{B_res}, ParameterVector{}); + auto if_op = std::make_shared(cond); + if_op->set_then_body(then_body); + if_op->set_else_body(else_body); + auto res = if_op->set_output(A_res, B_res); + return std::make_shared(OutputVector{res}, ParameterVector{cond}, "MultiSubGraphOp"); + } else { + return nullptr; + } +} + +std::shared_ptr generateNmsBase(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 2, 4}, {1, 2, 2}}); + const auto outputs = + ngraph::helpers::convert2OutputVector(ngraph::helpers::castOps2Nodes(params)); + if (ov::is_type(node)) { + const auto nms = + std::make_shared(outputs[0], outputs[1], ov::op::v8::MatrixNms::Attributes()); + ov::ResultVector results{std::make_shared(nms)}; + return std::make_shared(results, params, "MatrixNms"); + } else if (ov::is_type(node)) { + const auto nms = std::make_shared(outputs[0], outputs[1], ov::op::v8::MulticlassNms::Attributes()); + ov::ResultVector results{std::make_shared(nms)}; + return std::make_shared(results, params, "MulticlassNms"); + } else { + return nullptr; + } +} + +std::shared_ptr generateReadValueBase(const std::shared_ptr &node) { + auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1}}); + if (ov::is_type(node)) { + auto read_value = std::make_shared(params[0], "v0"); + auto add = std::make_shared(read_value, params[0]); + auto assign = std::make_shared(add, "v0"); + ov::ResultVector results{std::make_shared(add)}; + return std::make_shared(results, SinkVector{assign}, params, "ReadValue-3"); + } else if (ov::is_type(node)) { + auto variable = std::make_shared( + ov::op::util::VariableInfo{ov::PartialShape::dynamic(), ov::element::dynamic, "v0"}); + auto read_value = std::make_shared(params[0], variable); + auto add = std::make_shared(read_value, params[0]); + auto assign = std::make_shared(add, variable); + ov::ResultVector results{std::make_shared(add)}; + return std::make_shared(results, SinkVector{assign}, params, "ReadValue-6"); + } else { + return nullptr; + } +} + + +std::shared_ptr generateDeformableConvolutionBase(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{1, 2, 4, 4}, + {1, 18, 2, 2}, + {1, 2, 3, 3}}); + std::shared_ptr deformableConvolutionNode; + if (ov::is_type(node)) { + deformableConvolutionNode = std::make_shared(params.at(0), params.at(1), params.at(2), + ov::Strides {1, 1}, + ov::CoordinateDiff {0, 0}, + ov::CoordinateDiff {0, 0}, + ov::Strides {1, 1}); + } else if (ov::is_type(node)) { + deformableConvolutionNode = std::make_shared(params.at(0), params.at(1), params.at(2), + ov::Strides {1, 1}, + ov::CoordinateDiff {0, 0}, + ov::CoordinateDiff {0, 0}, + ov::Strides {1, 1}); + } else { + return nullptr; + } + + ov::ResultVector results{std::make_shared(deformableConvolutionNode)}; + return std::make_shared(results, params, "DeformableConvolutionBaseGraph"); +} + +std::shared_ptr generateDetectionOutputBase(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 8}, + {2, 6}, + {2, 1, 8}}); + ov::op::v0::DetectionOutput::Attributes attrs; + ov::op::v8::DetectionOutput::Attributes attrs_v8; + attrs.num_classes = 3; + attrs_v8.background_label_id = attrs.background_label_id = -1; + attrs_v8.top_k = attrs.top_k = -1; + attrs_v8.variance_encoded_in_target = attrs.variance_encoded_in_target = true; + attrs_v8.keep_top_k = attrs.keep_top_k = {2}; + attrs_v8.code_type = attrs.code_type = "caffe.PriorBoxParameter.CORNER"; + attrs_v8.share_location = attrs.share_location = true; + attrs_v8.nms_threshold = attrs.nms_threshold = 0.5; + attrs_v8.confidence_threshold = attrs.confidence_threshold = 0.3; + attrs_v8.clip_after_nms = attrs.clip_after_nms = false; + attrs_v8.clip_before_nms = attrs.clip_before_nms = true; + attrs_v8.decrease_label_id = attrs.decrease_label_id = false; + attrs_v8.normalized = attrs.normalized = true; + attrs_v8.input_height = attrs.input_height = 0; + attrs_v8.input_width = attrs.input_width = 0; + attrs_v8.objectness_score = attrs.objectness_score = 0; + + std::shared_ptr DetectionOutputNode; + if (ov::is_type(node)) { + DetectionOutputNode = std::make_shared(params.at(0), params.at(1), params.at(2), attrs); + } else if (ov::is_type(node)) { + DetectionOutputNode = std::make_shared(params.at(0), params.at(1), params.at(2), attrs_v8); + } else { + return nullptr; + } + + ov::ResultVector results{std::make_shared(DetectionOutputNode)}; + return std::make_shared(results, params, "DetectionOutputBaseGraph"); +} + +std::shared_ptr generateEmbeddingBagOffsetsBase(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{5, 2}}); + const auto indices = ngraph::builder::makeConstant(ov::element::i32, {4}, {}, true); + const auto offsets = ngraph::builder::makeConstant(ov::element::i32, {3}, {}, true); + const auto default_index = ngraph::builder::makeConstant(ov::element::i32, ov::Shape(), std::vector{0}); + + std::shared_ptr EmbeddingBagOffsetsSumNode; + if (ov::is_type(node)) { + EmbeddingBagOffsetsSumNode = std::make_shared(params.at(0), indices, offsets, default_index); + } else { + return nullptr; + } + + ov::ResultVector results{std::make_shared(EmbeddingBagOffsetsSumNode)}; + return std::make_shared(results, params, "EmbeddingBagOffsetsBaseGraph"); +} + +std::shared_ptr generateEmbeddingBagPackedBase(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{5, 2}}); + const auto indices = ngraph::builder::makeConstant(ov::element::i32, {2, 3}, {}, true); + + std::shared_ptr EmbeddingBagPackedSumNode; + if (ov::is_type(node)) { + EmbeddingBagPackedSumNode = std::make_shared(params.at(0), indices); + } else { + return nullptr; + } + + ov::ResultVector results{std::make_shared(EmbeddingBagPackedSumNode)}; + return std::make_shared(results, params, "EmbeddingBagPackedBaseGraph"); +} + +std::shared_ptr generateFFTBase(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 10, 10, 2}}); + const auto axes = ngraph::builder::makeConstant(ov::element::i32, {1}, {2}); + + std::shared_ptr FFTBaseNode; + if (ov::is_type(node)) { + FFTBaseNode = std::make_shared(params.at(0), axes); + } else if (ov::is_type(node)) { + FFTBaseNode = std::make_shared(params.at(0), axes); + } else { + return nullptr; + } + + ov::ResultVector results{std::make_shared(FFTBaseNode)}; + return std::make_shared(results, params, "FFTBaseGraph"); +} + +std::shared_ptr generateGatherBase(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::i32, {{2, 2, 3, 3}, {2}}); + const auto axis = ngraph::builder::makeConstant(ov::element::i64, ov::Shape(), std::vector{2}); + + std::shared_ptr GatherBaseNode; + if (ov::is_type(node)) { + GatherBaseNode = std::make_shared(params.at(0), params.at(1), axis); + } else if (ov::is_type(node)) { + GatherBaseNode = std::make_shared(params.at(0), params.at(1), axis); + } else if (ov::is_type(node)) { + GatherBaseNode = std::make_shared(params.at(0), params.at(1), axis); + } else { + return nullptr; + } + + ov::ResultVector results{std::make_shared(GatherBaseNode)}; + return std::make_shared(results, params, "GatherBaseGraph"); +} + +std::shared_ptr generateGatherNDBase(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::i32, {{2, 3, 4, 2}, {2, 3, 3, 2}}); + + std::shared_ptr GatherNDBaseNode; + if (ov::is_type(node)) { + GatherNDBaseNode = std::make_shared(params.at(0), params.at(1)); + } else if (ov::is_type(node)) { + GatherNDBaseNode = std::make_shared(params.at(0), params.at(1)); + } else { + return nullptr; + } + + ov::ResultVector results{std::make_shared(GatherNDBaseNode)}; + return std::make_shared(results, params, "GatherNDBaseGraph"); +} + +std::shared_ptr generateRNNCellBase(const std::shared_ptr &node) { + std::shared_ptr RNNCellBaseNode; + if (ov::is_type(node)) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 3}, {2, 3}}); + const auto W = ngraph::builder::makeConstant(ov::element::f32, {9, 3}, {}, true); + const auto R = ngraph::builder::makeConstant(ov::element::f32, {9, 3}, {}, true); + const auto B = ngraph::builder::makeConstant(ov::element::f32, {9}, {}, true); + RNNCellBaseNode = std::make_shared(params.at(0), params.at(1), + W, R, B, 3); + ov::ResultVector results{std::make_shared(RNNCellBaseNode)}; + return std::make_shared(results, params, "RNNCellBaseGraph"); + } else if (ov::is_type(node)) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 3}, {2, 3}, {2, 3}}); + const auto W = ngraph::builder::makeConstant(ov::element::f32, {12, 3}, {}, true); + const auto R = ngraph::builder::makeConstant(ov::element::f32, {12, 3}, {}, true); + const auto B = ngraph::builder::makeConstant(ov::element::f32, {12}, {}, true); + const auto P = ngraph::builder::makeConstant(ov::element::f32, {9}, {}, true); + RNNCellBaseNode = std::make_shared(params.at(0), params.at(1), params.at(2), + W, R, B, P, 3); + ov::ResultVector results{std::make_shared(RNNCellBaseNode->output(0)), + std::make_shared(RNNCellBaseNode->output(1))}; + //return std::make_shared(results, params, "RNNCellBaseGraph"); + return nullptr; // Temporary nullptr return due to crash in mkldnn + } else if (ov::is_type(node)) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 3}, {2, 3}, {2, 3}}); + const auto W = ngraph::builder::makeConstant(ov::element::f32, {12, 3}, {}, true); + const auto R = ngraph::builder::makeConstant(ov::element::f32, {12, 3}, {}, true); + const auto B = ngraph::builder::makeConstant(ov::element::f32, {12}, {}, true); + RNNCellBaseNode = std::make_shared(params.at(0), params.at(1), params.at(2), + W, R, B, 3); + ov::ResultVector results{std::make_shared(RNNCellBaseNode->output(0)), + std::make_shared(RNNCellBaseNode->output(1))};; + return std::make_shared(results, params, "RNNCellBaseGraph"); + } else if (ov::is_type(node)) { + const auto params = ngraph::builder::makeDynamicParams({ov::element::f32, ov::element::f32, ov::element::f32, ov::element::i64}, + {{5, 10, 10}, {5, 1, 10}, {5, 1, 10}, {5}}); + const auto W = ngraph::builder::makeConstant(ov::element::f32, {1, 40, 10}, {}, true); + const auto R = ngraph::builder::makeConstant(ov::element::f32, {1, 40, 10}, {}, true); + const auto B = ngraph::builder::makeConstant(ov::element::f32, {1, 40}, {}, true); + RNNCellBaseNode = std::make_shared(params.at(0), params.at(1), params.at(2), params.at(3), + W, R, B, 10, ov::op::RecurrentSequenceDirection::FORWARD); + ov::ResultVector results{std::make_shared(RNNCellBaseNode->output(0)), + std::make_shared(RNNCellBaseNode->output(1)), + std::make_shared(RNNCellBaseNode->output(2))}; + return std::make_shared(results, params, "RNNCellBaseGraph"); + } else if (ov::is_type(node)) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 3}, {2, 3}}); + const auto W = ngraph::builder::makeConstant(ov::element::f32, {3, 3}, {}, true); + const auto R = ngraph::builder::makeConstant(ov::element::f32, {3, 3}, {}, true); + const auto B = ngraph::builder::makeConstant(ov::element::f32, {3}, {}, true); + RNNCellBaseNode = std::make_shared(params.at(0), params.at(1), + W, R, B, 3); + ov::ResultVector results{std::make_shared(RNNCellBaseNode)}; + return std::make_shared(results, params, "RNNCellBaseGraph"); + } else if (ov::is_type(node)) { + const auto params = ngraph::builder::makeDynamicParams({ov::element::f32, ov::element::f32, ov::element::i64}, + {{2, 5, 3}, {2, 1, 3}, {2}}); + const auto W = ngraph::builder::makeConstant(ov::element::f32, {1, 3, 3}, {}, true); + const auto R = ngraph::builder::makeConstant(ov::element::f32, {1, 3, 3}, {}, true); + const auto B = ngraph::builder::makeConstant(ov::element::f32, {1, 3}, {}, true); + RNNCellBaseNode = std::make_shared(params.at(0), params.at(1), params.at(2), + W, R, B, 3, ov::op::RecurrentSequenceDirection::FORWARD); + ov::ResultVector results{std::make_shared(RNNCellBaseNode->output(0)), + std::make_shared(RNNCellBaseNode->output(1))}; + return std::make_shared(results, params, "RNNCellBaseGraph"); + } else { + return nullptr; + } +} + +std::shared_ptr generateSubGraphOp(const std::shared_ptr &node) { + const auto params = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 2}, {2, 2}, {2, 2}}); + const auto params_body = ngraph::builder::makeDynamicParams(ov::element::f32, {{2, 2}, {2, 2}, {2, 2}}); + const auto body_condition = ngraph::builder::makeConstant(ov::element::boolean, ov::Shape{1}, {true}); + const auto trip_count = ngraph::builder::makeConstant(ngraph::element::i64, ov::Shape{1}, {3}); + const auto exec_condition = ngraph::builder::makeConstant(ov::element::boolean, ov::Shape{1}, {true}); + // Body + auto sum = std::make_shared(params_body.at(0), params_body.at(1)); + auto Zo = std::make_shared(sum, params_body.at(2)); + auto body = std::make_shared(ov::OutputVector{body_condition, Zo}, params_body); + + ov::Output SubGraphOpNode; + if (ov::is_type(node)) { + auto tensor_iterator = std::make_shared(); + tensor_iterator->set_function(body); + + tensor_iterator->set_sliced_input(params_body.at(0), params.at(0), 0, 1, 1, -1, 1); + tensor_iterator->set_sliced_input(params_body.at(1), params.at(1), 0, 1, 1, -1, 0); + tensor_iterator->set_merged_input(params_body.at(2), params.at(2), Zo); + + // Output 0 is last Zo + SubGraphOpNode = tensor_iterator->get_iter_value(Zo, -1); + } else if (ov::is_type(node)) { + auto loop = std::make_shared(trip_count, exec_condition); + loop->set_function(body); + + loop->set_invariant_input(params_body.at(0), params.at(0)); + loop->set_invariant_input(params_body.at(1), params.at(1)); + loop->set_merged_input(params_body.at(2), params.at(2), Zo); + + loop->set_special_body_ports(ov::op::v5::Loop::SpecialBodyPorts{-1, 0}); + SubGraphOpNode = loop->get_iter_value(Zo, -1); + } else { + return nullptr; + } + + ov::ResultVector results{std::make_shared(SubGraphOpNode)}; + return std::make_shared(results, params, "SubGraphOpGraph"); +} +} // namespace template std::shared_ptr generateGraph() { - std::shared_ptr node = std::shared_ptr(new T); + std::shared_ptr node = std::shared_ptr(new T); if (ov::is_type(node)) { return generateBinaryEltwise(node); + } else if (ov::is_type(node)) { + return generateBinaryEltwiseComp(node); + } else if (ov::is_type(node)) { + return generateBinaryEltwiseLogical(node); + } else if (ov::is_type(node)) { + return generateBroadcast(node); + } else if (ov::is_type(node) || + ov::is_type(node)) { + return generateConvertColor(node); + } else if (ov::is_type(node)) { + return generateMultiSubGraph(node); + } else if (ov::is_type(node)) { + return generateNmsBase(node); + } else if (ov::is_type(node)) { + return generateReadValueBase(node); + } else if (ov::is_type(node)) { + return generateDeformableConvolutionBase(node); + } else if (ov::is_type(node)) { + return generateDetectionOutputBase(node); + } else if (ov::is_type(node)) { + return generateEmbeddingBagOffsetsBase(node); + } else if (ov::is_type(node)) { + return generateEmbeddingBagPackedBase(node); + } else if (ov::is_type(node)) { + return generateFFTBase(node); + } else if (ov::is_type(node)) { + return generateGatherBase(node); + } else if (ov::is_type(node)) { + return generateGatherNDBase(node); + } else if (ov::is_type(node)) { + return generateRNNCellBase(node); + } else if (ov::is_type(node)) { + return generateSubGraphOp(node); } + return generate(node); } @@ -72,4 +1412,4 @@ OpGenerator getOpGeneratorMap() { } // namespace subgraph } // namespace test -} // namespace ov +} // namespace ov \ No newline at end of file diff --git a/src/tests/functional/shared_test_classes/include/shared_test_classes/base/layer_test_utils.hpp b/src/tests/functional/shared_test_classes/include/shared_test_classes/base/layer_test_utils.hpp index 4762844086d..3ead1299065 100644 --- a/src/tests/functional/shared_test_classes/include/shared_test_classes/base/layer_test_utils.hpp +++ b/src/tests/functional/shared_test_classes/include/shared_test_classes/base/layer_test_utils.hpp @@ -21,6 +21,7 @@ #include "common_test_utils/ngraph_test_utils.hpp" #include "common_test_utils/common_utils.hpp" #include "common_test_utils/test_common.hpp" +#include "common_test_utils/crash_handler.hpp" #include "functional_test_utils/skip_tests_config.hpp" #include "functional_test_utils/plugin_cache.hpp" diff --git a/src/tests/functional/shared_test_classes/include/shared_test_classes/base/ov_subgraph.hpp b/src/tests/functional/shared_test_classes/include/shared_test_classes/base/ov_subgraph.hpp index 4739fa1045f..b5b6713307c 100644 --- a/src/tests/functional/shared_test_classes/include/shared_test_classes/base/ov_subgraph.hpp +++ b/src/tests/functional/shared_test_classes/include/shared_test_classes/base/ov_subgraph.hpp @@ -26,12 +26,6 @@ public: virtual void serialize(); virtual void query_model(); - void TearDown() override { - if (!configuration.empty()) { - ov::test::utils::PluginCache::get().core().reset(); - } - } - protected: virtual void compare(const std::vector &expected, const std::vector &actual); diff --git a/src/tests/functional/shared_test_classes/src/base/layer_test_utils.cpp b/src/tests/functional/shared_test_classes/src/base/layer_test_utils.cpp index cefc0234fa7..5d815d69c98 100644 --- a/src/tests/functional/shared_test_classes/src/base/layer_test_utils.cpp +++ b/src/tests/functional/shared_test_classes/src/base/layer_test_utils.cpp @@ -30,40 +30,45 @@ void LayerTestsCommon::Run() { functionRefs = ngraph::clone_function(*function); functionRefs->set_friendly_name("refFunction"); } - auto crashHandler = [](int errCode) { + + // in case of crash jump will be made and work will be continued + auto crashHandler = std::unique_ptr(new CommonTestUtils::CrashHandler()); + + // place to jump in case of a crash +#ifdef _WIN32 + if (setjmp(CommonTestUtils::env) == 0) { +#else + if (sigsetjmp(CommonTestUtils::env, 1) == 0) { +#endif auto &s = Summary::getInstance(); - s.saveReport(); - std::cout << "Unexpected application crash!" << std::endl; - std::abort(); - }; - signal(SIGSEGV, crashHandler); + s.setDeviceName(targetDevice); - auto &s = Summary::getInstance(); - s.setDeviceName(targetDevice); + if (FuncTestUtils::SkipTestsConfig::currentTestIsDisabled()) { + s.updateOPsStats(functionRefs, PassRate::Statuses::SKIPPED); + GTEST_SKIP() << "Disabled test due to configuration" << std::endl; + } else { + s.updateOPsStats(functionRefs, PassRate::Statuses::CRASHED); + } - if (FuncTestUtils::SkipTestsConfig::currentTestIsDisabled()) { - s.updateOPsStats(functionRefs, PassRate::Statuses::SKIPPED); - GTEST_SKIP() << "Disabled test due to configuration" << std::endl; + try { + LoadNetwork(); + GenerateInputs(); + Infer(); + Validate(); + s.updateOPsStats(functionRefs, PassRate::Statuses::PASSED); + } + catch (const std::runtime_error &re) { + s.updateOPsStats(functionRefs, PassRate::Statuses::FAILED); + GTEST_FATAL_FAILURE_(re.what()); + } catch (const std::exception &ex) { + s.updateOPsStats(functionRefs, PassRate::Statuses::FAILED); + GTEST_FATAL_FAILURE_(ex.what()); + } catch (...) { + s.updateOPsStats(functionRefs, PassRate::Statuses::FAILED); + GTEST_FATAL_FAILURE_("Unknown failure occurred."); + } } else { - s.updateOPsStats(functionRefs, PassRate::Statuses::CRASHED); - } - - try { - LoadNetwork(); - GenerateInputs(); - Infer(); - Validate(); - s.updateOPsStats(functionRefs, PassRate::Statuses::PASSED); - } - catch (const std::runtime_error &re) { - s.updateOPsStats(functionRefs, PassRate::Statuses::FAILED); - GTEST_FATAL_FAILURE_(re.what()); - } catch (const std::exception &ex) { - s.updateOPsStats(functionRefs, PassRate::Statuses::FAILED); - GTEST_FATAL_FAILURE_(ex.what()); - } catch (...) { - s.updateOPsStats(functionRefs, PassRate::Statuses::FAILED); - GTEST_FATAL_FAILURE_("Unknown failure occurred."); + IE_THROW() << "Crash happens"; } } diff --git a/src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp b/src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp index 6db97b61d6b..2700cf0eb45 100644 --- a/src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp +++ b/src/tests/functional/shared_test_classes/src/base/ov_subgraph.cpp @@ -18,6 +18,7 @@ #include "ngraph_functions/utils/ngraph_helpers.hpp" #include "common_test_utils/file_utils.hpp" +#include "common_test_utils/crash_handler.hpp" #include "functional_test_utils/ov_tensor_utils.hpp" #include "functional_test_utils/skip_tests_config.hpp" @@ -25,6 +26,8 @@ #include "shared_test_classes/base/utils/generate_inputs.hpp" #include "shared_test_classes/base/utils/compare_results.hpp" +#include + namespace ov { namespace test { @@ -34,52 +37,60 @@ std::ostream& operator <<(std::ostream& os, const InputShape& inputShape) { } void SubgraphBaseTest::run() { - auto crashHandler = [](int errCode) { - auto& s = LayerTestsUtils::Summary::getInstance(); - s.saveReport(); - std::cerr << "Unexpected application crash with code: " << errCode << std::endl; - std::abort(); - }; - signal(SIGSEGV, crashHandler); + // in case of crash jump will be made and work will be continued + auto crashHandler = std::unique_ptr(new CommonTestUtils::CrashHandler()); - LayerTestsUtils::PassRate::Statuses status = FuncTestUtils::SkipTestsConfig::currentTestIsDisabled() - ? LayerTestsUtils::PassRate::Statuses::SKIPPED - : LayerTestsUtils::PassRate::Statuses::CRASHED; - summary.setDeviceName(targetDevice); - summary.updateOPsStats(function, status); - SKIP_IF_CURRENT_TEST_IS_DISABLED(); + // place to jump in case of a crash +#ifdef _WIN32 + if (setjmp(CommonTestUtils::env) == 0) { +#else + if (sigsetjmp(CommonTestUtils::env, 1) == 0) { +#endif + bool isCurrentTestDisabled = FuncTestUtils::SkipTestsConfig::currentTestIsDisabled(); - ASSERT_FALSE(targetStaticShapes.empty()) << "Target Static Shape is empty!!!"; - std::string errorMessage; - try { - compile_model(); - for (const auto& targetStaticShapeVec : targetStaticShapes) { - try { - if (!inputDynamicShapes.empty()) { - // resize ngraph function according new target shape - // Note: output shapes of some nodes depend on the input data - // so for some tests we need to override this function and replace parameter with constant node to get correct output shapes - init_ref_function(functionRefs, targetStaticShapeVec); + LayerTestsUtils::PassRate::Statuses status = isCurrentTestDisabled ? + LayerTestsUtils::PassRate::Statuses::SKIPPED : + LayerTestsUtils::PassRate::Statuses::CRASHED; + summary.setDeviceName(targetDevice); + summary.updateOPsStats(function, status); + + if (isCurrentTestDisabled) + GTEST_SKIP() << "Disabled test due to configuration" << std::endl; + + ASSERT_FALSE(targetStaticShapes.empty()) << "Target Static Shape is empty!!!"; + std::string errorMessage; + try { + compile_model(); + for (const auto& targetStaticShapeVec : targetStaticShapes) { + try { + if (!inputDynamicShapes.empty()) { + // resize ngraph function according new target shape + // Note: output shapes of some nodes depend on the input data + // so for some tests we need to override this function and replace parameter with constant node to get correct output shapes + init_ref_function(functionRefs, targetStaticShapeVec); + } + generate_inputs(targetStaticShapeVec); + } catch (const std::exception& ex) { + throw std::runtime_error("Incorrect target static shape: " + + CommonTestUtils::vec2str(targetStaticShapeVec) + " " + ex.what()); } - generate_inputs(targetStaticShapeVec); - } catch (const std::exception& ex) { - throw std::runtime_error("Incorrect target static shape: " + - CommonTestUtils::vec2str(targetStaticShapeVec) + " " + ex.what()); + infer(); + validate(); } - infer(); - validate(); + status = LayerTestsUtils::PassRate::Statuses::PASSED; + } catch (const std::exception& ex) { + status = LayerTestsUtils::PassRate::Statuses::FAILED; + errorMessage = ex.what(); + } catch (...) { + status = LayerTestsUtils::PassRate::Statuses::FAILED; + errorMessage = "Unknown failure occurred."; } - status = LayerTestsUtils::PassRate::Statuses::PASSED; - } catch (const std::exception& ex) { - status = LayerTestsUtils::PassRate::Statuses::FAILED; - errorMessage = ex.what(); - } catch (...) { - status = LayerTestsUtils::PassRate::Statuses::FAILED; - errorMessage = "Unknown failure occurred."; - } - summary.updateOPsStats(function, status); - if (status != LayerTestsUtils::PassRate::Statuses::PASSED) { - GTEST_FATAL_FAILURE_(errorMessage.c_str()); + summary.updateOPsStats(function, status); + if (status != LayerTestsUtils::PassRate::Statuses::PASSED) { + GTEST_FATAL_FAILURE_(errorMessage.c_str()); + } + } else { + IE_THROW() << "Crash happens"; } } @@ -228,11 +239,24 @@ std::vector SubgraphBaseTest::calculate_refs() { auto functionToProcess = ov::clone_model(*functionRefs); //TODO: remove this conversions as soon as function interpreter fully support bf16 and f16 - static const precisions_array precisions = { - { ngraph::element::bf16, ngraph::element::f32 }, - { ngraph::element::f16, ngraph::element::f32} + precisions_array precisions = { + { ngraph::element::bf16, ngraph::element::f32 } }; - + auto convert_added = false; + for (const auto ¶m : function->get_parameters()) { + for (size_t i = 0; i < param->get_output_size(); i++) { + for (const auto &node : param->get_output_target_inputs(i)) { + std::shared_ptr nodePtr = node.get_node()->shared_from_this(); + if (std::dynamic_pointer_cast(nodePtr)) { + convert_added = true; + break; + } + } + } + } + if (!convert_added) { + precisions.push_back({ ngraph::element::f16, ngraph::element::f32}); + } pass::Manager manager; manager.register_pass(precisions); manager.run_passes(functionToProcess); diff --git a/src/tests/ie_test_utils/common_test_utils/crash_handler.cpp b/src/tests/ie_test_utils/common_test_utils/crash_handler.cpp new file mode 100644 index 00000000000..ed3a1e446c4 --- /dev/null +++ b/src/tests/ie_test_utils/common_test_utils/crash_handler.cpp @@ -0,0 +1,55 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "crash_handler.hpp" + +namespace CommonTestUtils { + +// enviroment to restore in case of crash +jmp_buf env; + +CrashHandler::CrashHandler() { + auto crashHandler = [](int errCode) { + std::cerr << "Unexpected application crash with code: " << errCode << std::endl; + + // reset custom signal handler to avoid infinit loop + // if for some reasons sigsetjmp will not be available + signal(SIGABRT, SIG_DFL); + signal(SIGSEGV, SIG_DFL); + signal(SIGILL, SIG_DFL); +#ifndef _WIN32 + signal(SIGBUS, SIG_DFL); + signal(SIGFPE, SIG_DFL); +#endif + + // goto sigsetjmp +#ifdef _WIN32 + longjmp(env, 1); +#else + siglongjmp(env, 1); +#endif + }; + + // setup custom handler for signals + signal(SIGABRT, crashHandler); + signal(SIGSEGV, crashHandler); + signal(SIGILL, crashHandler); +#ifndef _WIN32 + signal(SIGFPE, crashHandler); + signal(SIGBUS, crashHandler); +#endif +} + +CrashHandler::~CrashHandler() { + // reset custom signal handler to avoid infinit loop + signal(SIGABRT, SIG_DFL); + signal(SIGSEGV, SIG_DFL); + signal(SIGILL, SIG_DFL); +#ifndef _WIN32 + signal(SIGFPE, SIG_DFL); + signal(SIGBUS, SIG_DFL); +#endif +} + +} // namespace CommonTestUtils \ No newline at end of file diff --git a/src/tests/ie_test_utils/common_test_utils/crash_handler.hpp b/src/tests/ie_test_utils/common_test_utils/crash_handler.hpp new file mode 100644 index 00000000000..8e624671951 --- /dev/null +++ b/src/tests/ie_test_utils/common_test_utils/crash_handler.hpp @@ -0,0 +1,24 @@ +// Copyright (C) 2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include "common_utils.hpp" + +#include +#include + +namespace CommonTestUtils { + +extern jmp_buf env; + +class CrashHandler { +public: + CrashHandler(); + ~CrashHandler(); +}; + +} // namespace CommonTestUtils \ No newline at end of file diff --git a/src/tests/ie_test_utils/functional_test_utils/src/skip_tests_config.cpp b/src/tests/ie_test_utils/functional_test_utils/src/skip_tests_config.cpp index 56e90a7ce97..0eac0439e96 100644 --- a/src/tests/ie_test_utils/functional_test_utils/src/skip_tests_config.cpp +++ b/src/tests/ie_test_utils/functional_test_utils/src/skip_tests_config.cpp @@ -14,15 +14,19 @@ namespace SkipTestsConfig { bool disable_tests_skipping = false; bool currentTestIsDisabled() { - bool skip_test = false; + if (disable_tests_skipping) + return false; + const auto fullName = ::testing::UnitTest::GetInstance()->current_test_info()->test_case_name() + std::string(".") + ::testing::UnitTest::GetInstance()->current_test_info()->name(); + for (const auto &pattern : disabledTestPatterns()) { std::regex re(pattern); if (std::regex_match(fullName, re)) - skip_test = true; + return true; } - return skip_test && !disable_tests_skipping; + + return false; } std::vector readSkipTestConfigFiles(const std::vector& filePaths) { diff --git a/src/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/builders.hpp b/src/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/builders.hpp index 1b41cfa9dcf..24a8e5ebe90 100644 --- a/src/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/builders.hpp +++ b/src/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/builders.hpp @@ -41,22 +41,16 @@ std::shared_ptr makeConstant(const element::Type &type, const std::vector< case TYPE: \ weightsNode = std::make_shared( \ type, shape, \ - random ? NGraphFunctions::Utils::generateVector(ngraph::shape_size(shape), upTo, startFrom, seed) : \ - NGraphFunctions::Utils::castVector::value_type >(data)); \ + random ? NGraphFunctions::Utils::generateVector( \ + ngraph::shape_size(shape), \ + ngraph::helpers::nGraphTypesTrait::value_type(upTo), \ + ngraph::helpers::nGraphTypesTrait::value_type(startFrom), \ + seed) \ + : NGraphFunctions::Utils::castVector::value_type >(data)); \ break; switch (type) { - case ngraph::element::Type_t::bf16: - weightsNode = std::make_shared( - type, shape, - random ? NGraphFunctions::Utils::generateBF16Vector(ngraph::shape_size(shape), upTo, startFrom) : - NGraphFunctions::Utils::castVector(data)); - break; - case ngraph::element::Type_t::f16: - weightsNode = std::make_shared( - type, shape, - random ? NGraphFunctions::Utils::generateF16Vector(ngraph::shape_size(shape), upTo, startFrom) : - NGraphFunctions::Utils::castVector(data)); - break; + makeNode(ngraph::element::Type_t::bf16); + makeNode(ngraph::element::Type_t::f16); makeNode(ngraph::element::Type_t::f32); makeNode(ngraph::element::Type_t::f64); makeNode(ngraph::element::Type_t::i8); diff --git a/src/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/utils/data_utils.hpp b/src/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/utils/data_utils.hpp index ee7d6ed7184..3c6390ec965 100644 --- a/src/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/utils/data_utils.hpp +++ b/src/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/utils/data_utils.hpp @@ -54,8 +54,12 @@ generateVector(size_t vec_len, } } +template<> std::vector inline -generateF16Vector(size_t vec_len, ngraph::float16 upTo = 10, ngraph::float16 startFrom = 1, int32_t seed = 1) { +generateVector(size_t vec_len, + ngraph::float16 upTo, + ngraph::float16 startFrom, + int32_t seed) { std::vector res(vec_len); std::mt19937 gen(seed); // chose values between this range to avoid type overrun (e.g. in case of I8 precision) @@ -69,8 +73,12 @@ generateF16Vector(size_t vec_len, ngraph::float16 upTo = 10, ngraph::float16 sta return res; } +template<> std::vector inline -generateBF16Vector(size_t vec_len, ngraph::bfloat16 upTo = 10, ngraph::bfloat16 startFrom = 1, int32_t seed = 1) { +generateVector(size_t vec_len, + ngraph::bfloat16 upTo, + ngraph::bfloat16 startFrom, + int32_t seed) { std::vector res(vec_len); std::mt19937 gen(seed); diff --git a/src/tests/unit/cpu/ngraph_transformations/move_eltwise_up_data_movement_test.cpp b/src/tests/unit/cpu/ngraph_transformations/move_eltwise_up_data_movement_test.cpp index 1358440ec98..969aa008c98 100644 --- a/src/tests/unit/cpu/ngraph_transformations/move_eltwise_up_data_movement_test.cpp +++ b/src/tests/unit/cpu/ngraph_transformations/move_eltwise_up_data_movement_test.cpp @@ -6,6 +6,7 @@ #include #include +#include "ngraph_ops/type_relaxed.hpp" #include #include @@ -49,6 +50,38 @@ TEST_F(MoveEltwiseUpThroughDataMovTest, SingleUnaryEltwise) { } } +TEST_F(MoveEltwiseUpThroughDataMovTest, TypeRelaxedEltwise) { + const ngraph::Shape shape{1, 3, 224, 224}; + const std::vector input_order = {3, 2, 1, 0}; + { + auto input = std::make_shared(ngraph::element::f32, shape); + auto intermediate_op = std::make_shared(input, 0, 6); + + auto transpose_const = + ngraph::opset8::Constant::create(ngraph::element::i64, ngraph::Shape{input_order.size()}, input_order); + auto transpose = std::make_shared(intermediate_op, transpose_const); + + auto mul_const = ngraph::opset8::Constant::create(ngraph::element::f32, {}, {2.f}); + auto multiply = std::make_shared>(transpose, mul_const); + + function = std::make_shared(ngraph::NodeVector{multiply}, ngraph::ParameterVector{input}); + manager.register_pass(); + } + { + auto input = std::make_shared(ngraph::element::f32, shape); + auto intermediate_op = std::make_shared(input, 0, 6); + + auto mul_const = ngraph::opset8::Constant::create(ngraph::element::f32, {}, {2.f}); + auto multiply = std::make_shared>(intermediate_op, mul_const); + + auto transpose_const = ngraph::opset8::Constant::create(ngraph::element::i64, ngraph::Shape{input_order.size()}, input_order); + auto transpose = std::make_shared(multiply, transpose_const); + + function_ref = + std::make_shared(ngraph::NodeVector{transpose}, ngraph::ParameterVector{input}); + } +} + TEST_F(MoveEltwiseUpThroughDataMovTest, EltwiseSequence) { const ngraph::Shape shape{1, 3, 224, 224}; const std::vector input_order = {1, 2, 0, 3}; diff --git a/tests/stress_tests/.automation/memcheck_tests/nightly_configs/desktop_references_config.xml b/tests/stress_tests/.automation/memcheck_tests/nightly_configs/desktop_references_config.xml index 9e68f060112..75a085a2642 100644 --- a/tests/stress_tests/.automation/memcheck_tests/nightly_configs/desktop_references_config.xml +++ b/tests/stress_tests/.automation/memcheck_tests/nightly_configs/desktop_references_config.xml @@ -2,30 +2,12 @@ # Values from {"commit_id": "a92a737ba0ce5afd6df2da8d80c8deec7e11b1d9", "commit_date": "2021-07-16 13:30"} and *= 1.3 - - - - - - - - - - - - - - - - - - @@ -414,7 +396,7 @@ - + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 @@ -424,7 +406,7 @@ # Values from {"commit_id": "403339f8f470c90dee6f6d94ed58644b2787f66b", "commit_date": "2022-01-19 14:13"} and *= 1.3 - + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 # Values from {"commit_id": "403339f8f470c90dee6f6d94ed58644b2787f66b", "commit_date": "2022-01-19 14:13"} and *= 1.3 @@ -582,5 +564,41 @@ + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 + # Values from {"commit_id": "8ca1abed0d309d4c4981f00786adbe6106ca46b3", "commit_date": "2022-03-01 14:04"} and *= 1.3 diff --git a/tests/stress_tests/.automation/memcheck_tests/nightly_configs/desktop_test_config.xml b/tests/stress_tests/.automation/memcheck_tests/nightly_configs/desktop_test_config.xml index a4456881c73..3c91115a4c2 100644 --- a/tests/stress_tests/.automation/memcheck_tests/nightly_configs/desktop_test_config.xml +++ b/tests/stress_tests/.automation/memcheck_tests/nightly_configs/desktop_test_config.xml @@ -43,6 +43,9 @@ + + + @@ -78,6 +81,9 @@ + + + @@ -87,14 +93,12 @@ - - @@ -103,7 +107,6 @@ - diff --git a/tests/stress_tests/.automation/memcheck_tests/precommit_configs/desktop_references_config.xml b/tests/stress_tests/.automation/memcheck_tests/precommit_configs/desktop_references_config.xml index b0db2e425f4..6bc58fb4ce8 100644 --- a/tests/stress_tests/.automation/memcheck_tests/precommit_configs/desktop_references_config.xml +++ b/tests/stress_tests/.automation/memcheck_tests/precommit_configs/desktop_references_config.xml @@ -46,18 +46,18 @@ - + # values from {"commit_id": "25c76471d78628aa772f3a7e341ae915bdce6026", "commit_date": "2022-02-23 15:55"} - + # values from {"commit_id": "25c76471d78628aa772f3a7e341ae915bdce6026", "commit_date": "2022-02-23 15:55"} - + # values from {"commit_id": "25c76471d78628aa772f3a7e341ae915bdce6026", "commit_date": "2022-02-23 15:55"} diff --git a/tests/stress_tests/scripts/get_testdata.py b/tests/stress_tests/scripts/get_testdata.py index 176472649c1..3190f7b19e3 100755 --- a/tests/stress_tests/scripts/get_testdata.py +++ b/tests/stress_tests/scripts/get_testdata.py @@ -181,8 +181,8 @@ def main(): # check selected precision with model info from Open Model Zoo if precision not in model_info['precisions']: log.warning("Please specify precision for the model " - "{model_name} from the list: {model_info}".format(model_name=model_name, - model_info=model_info['precisions'])) + f"{model_name} from the list: {model_info['precisions']}") + model_recs.remove(model_rec) continue model_rec.attrib.update(info_to_add) model_rec.attrib["path"] = str( @@ -216,6 +216,12 @@ def main(): models_dir=args.omz_models_out_dir, mo_tool=args.mo_tool) run_in_subprocess(cmd, check_call=not args.skip_omz_errors) + for model_rec in model_recs: + if model_rec.attrib.get("full_path") is None: + log.warning(f"Model {model_rec.attrib['name']} does not have 'full_path' attribute! " + f"This model will not be verified in this run.") + model_recs.remove(model_rec) + # rewrite test config with updated records test_conf_obj.write(args.test_conf) diff --git a/thirdparty/open_model_zoo b/thirdparty/open_model_zoo index d4f504bf6f9..bc92ceb1d3f 160000 --- a/thirdparty/open_model_zoo +++ b/thirdparty/open_model_zoo @@ -1 +1 @@ -Subproject commit d4f504bf6f96647c0a5727bf69cbfa2721a8717d +Subproject commit bc92ceb1d3f685f4c98eccd9320aff25401feee2 diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 0afffbb05c9..c8638f7fd39 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -7,6 +7,10 @@ add_subdirectory(compile_tool) +if(ENABLE_SAMPLES) + add_subdirectory(legacy/benchmark_app) +endif() + # # Python tools # diff --git a/tools/benchmark_tool/README.md b/tools/benchmark_tool/README.md index 9bacb159065..fd2d542fb6c 100644 --- a/tools/benchmark_tool/README.md +++ b/tools/benchmark_tool/README.md @@ -229,7 +229,7 @@ This section provides step-by-step instructions on how to run the Benchmark Tool > **NOTE**: The Internet access is required to execute the following steps successfully. If you have access to the Internet through the proxy server only, please make sure that it is configured in your OS environment. -1. Download the model. Go to the the Model Downloader directory and run the `downloader.py` script with the model name and directory to download the model to: +1. Download the model. Go to the Model Downloader directory and run the `downloader.py` script with the model name and directory to download the model to: ```sh cd /extras/open_model_zoo/tools/downloader ``` diff --git a/tools/benchmark_tool/openvino/tools/benchmark/utils/utils.py b/tools/benchmark_tool/openvino/tools/benchmark/utils/utils.py index ce39d5d49f4..d59fe2c11e1 100644 --- a/tools/benchmark_tool/openvino/tools/benchmark/utils/utils.py +++ b/tools/benchmark_tool/openvino/tools/benchmark/utils/utils.py @@ -375,9 +375,9 @@ def get_data_shapes_map(data_shape_string, input_names): input_name = match[:match.find('[')] shapes = re.findall(r'\[(.*?)\]', match[len(input_name):]) if input_name: - return_value[input_name] = list(parse_partial_shape(shape_str) for shape_str in shapes) + return_value[input_name] = list(PartialShape(shape_str) for shape_str in shapes) else: - data_shapes = list(parse_partial_shape(shape_str) for shape_str in shapes) + data_shapes = list(PartialShape(shape_str) for shape_str in shapes) num_inputs, num_shapes = len(input_names), len(data_shapes) if num_shapes != 1 and num_shapes % num_inputs != 0: raise Exception(f"Number of provided data_shapes is not a multiple of the number of model inputs!") @@ -505,52 +505,13 @@ class AppInputInfo: return self.partial_shape.is_dynamic -def parse_partial_shape(shape_str): - dims = [] - for dim in shape_str.split(','): - if '.. ' in dim: - range = list(int(d) for d in dim.split('..')) - assert len(range) == 2 - dims.append(Dimension(range)) - elif dim == '?': - dims.append(Dimension()) - else: - dims.append(Dimension(int(dim))) - return PartialShape(dims) - - -def parse_batch_size(batch_size_str): - if batch_size_str: - error_message = f"Can't parse batch size '{batch_size_str}'" - dims = batch_size_str.split("..") - if len(dims) > 2: - raise Exception(error_message) - elif len(dims) == 2: - range = [] - for d in dims: - if d.isnumeric(): - range.append(int(d)) - else: - raise Exception(error_message) - return Dimension(*range) - else: - if dims[0].lstrip("-").isnumeric(): - return Dimension(int(dims[0])) - elif dims[0] == "?": - return Dimension() - else: - raise Exception(error_message) - else: - return Dimension(0) - - def get_inputs_info(shape_string, data_shape_string, layout_string, batch_size, scale_string, mean_string, inputs): input_names = get_input_output_names(inputs) input_node_names = get_node_names(inputs) shape_map = parse_input_parameters(shape_string, input_names) data_shape_map = get_data_shapes_map(data_shape_string, input_names) layout_map = parse_input_parameters(layout_string, input_names) - batch_size = parse_batch_size(batch_size) + batch_size = Dimension(batch_size) reshape = False batch_found = False input_info = [] @@ -565,10 +526,10 @@ def get_inputs_info(shape_string, data_shape_string, layout_string, batch_size, # Shape info.original_shape = inputs[i].partial_shape if info.name in shape_map: - info.partial_shape = parse_partial_shape(shape_map[info.name]) + info.partial_shape = PartialShape(shape_map[info.name]) reshape = True elif info.node_name in shape_map: - info.partial_shape = parse_partial_shape(shape_map[info.node_name]) + info.partial_shape = PartialShape(shape_map[info.node_name]) reshape = True else: info.partial_shape = inputs[i].partial_shape diff --git a/tools/legacy/benchmark_app/CMakeLists.txt b/tools/legacy/benchmark_app/CMakeLists.txt new file mode 100644 index 00000000000..732058d89b1 --- /dev/null +++ b/tools/legacy/benchmark_app/CMakeLists.txt @@ -0,0 +1,37 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +set(TARGET_NAME benchmark_app_legacy) + +file (GLOB SRC ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) +file (GLOB HDR ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp) + +if(OpenVINO_SOURCE_DIR) + set(OpenVINO_DIR "${CMAKE_BINARY_DIR}") +endif() + +source_group("src" FILES ${SRC}) +source_group("include" FILES ${HDR}) + +add_executable(${TARGET_NAME} ${SRC} ${HDR}) + +find_package(OpenVINO REQUIRED COMPONENTS Runtime) + +set_target_properties(${TARGET_NAME} PROPERTIES + COMPILE_PDB_NAME ${TARGET_NAME} + FOLDER tools +) + +if (CMAKE_COMPILER_IS_GNUCXX) + target_compile_options(${TARGET_NAME} PRIVATE -Wall) +endif() + +find_package(OpenCV COMPONENTS core QUIET) +if(NOT OpenCV_FOUND) + message(WARNING "OpenCV is disabled or not found, ${TARGET_NAME} will be built without OpenCV support. Set OpenCV_DIR") + target_link_libraries(${TARGET_NAME} PRIVATE nlohmann_json_schema_validator openvino::runtime format_reader gflags) +else() + target_compile_definitions(${TARGET_NAME} PRIVATE USE_OPENCV) + target_link_libraries(${TARGET_NAME} PRIVATE opencv_core openvino::runtime format_reader gflags) +endif() diff --git a/tools/legacy/benchmark_app/README.md b/tools/legacy/benchmark_app/README.md new file mode 100644 index 00000000000..4651447ce7b --- /dev/null +++ b/tools/legacy/benchmark_app/README.md @@ -0,0 +1,186 @@ +# Benchmark C++ Tool + +This topic demonstrates how to use the Benchmark C++ Tool to estimate deep learning inference performance on supported devices. Performance can be measured for two inference modes: synchronous (latency-oriented) and asynchronous (throughput-oriented). + +> **NOTE**: This topic describes usage of C++ implementation of the Benchmark Tool. For the Python* implementation, refer to [Benchmark Python* Tool](../../benchmark_tool/README.md). + + +## How It Works + +Upon start-up, the application reads command-line parameters and loads a network and images/binary files to the Inference Engine plugin, which is chosen depending on a specified device. The number of infer requests and execution approach depend on the mode defined with the `-api` command-line parameter. + +> **NOTE**: By default, Inference Engine samples, tools and demos expect input with BGR channels order. If you trained your model to work with RGB order, you need to manually rearrange the default channels order in the sample or demo application or reconvert your model using the Model Optimizer tool with `--reverse_input_channels` argument specified. For more information about the argument, refer to **When to Reverse Input Channels** section of [Converting a Model to Intermediate Representation (IR)](../../../docs/MO_DG/prepare_model/convert_model/Converting_Model.md). + +If you run the application in the synchronous mode, it creates one infer request and executes the `Infer` method. +If you run the application in the asynchronous mode, it creates as many infer requests as specified in the `-nireq` command-line parameter and executes the `StartAsync` method for each of them. If `-nireq` is not set, the application will use the default value for specified device. + +A number of execution steps is defined by one of the following parameters: +* Number of iterations specified with the `-niter` command-line argument +* Time duration specified with the `-t` command-line argument +* Both of them (execution will continue until both conditions are met) +* Predefined duration if `-niter` and `-t` are not specified. Predefined duration value depends on a device. + +During the execution, the application collects latency for each executed infer request. + +Reported latency value is calculated as a median value of all collected latencies. Reported throughput value is reported +in frames per second (FPS) and calculated as a derivative from: +* Reported latency in the Sync mode +* The total execution time in the Async mode + +Throughput value also depends on batch size. + +The application also collects per-layer Performance Measurement (PM) counters for each executed infer request if you +enable statistics dumping by setting the `-report_type` parameter to one of the possible values: +* `no_counters` report includes configuration options specified, resulting FPS and latency. +* `average_counters` report extends the `no_counters` report and additionally includes average PM counters values for each layer from the network. +* `detailed_counters` report extends the `average_counters` report and additionally includes per-layer PM counters and latency for each executed infer request. + +Depending on the type, the report is stored to `benchmark_no_counters_report.csv`, `benchmark_average_counters_report.csv`, +or `benchmark_detailed_counters_report.csv` file located in the path specified in `-report_folder`. + +The application also saves executable graph information serialized to an XML file if you specify a path to it with the +`-exec_graph_path` parameter. + +## Building +This tools can be built as part of OpenVINO, during standard building process. More information about building OpenVINO can be found here[Build OpenVINO Inference Engine](https://github.com/openvinotoolkit/openvino/wiki/BuildingCode) + +## Run the Tool + +Note that the benchmark_app usually produces optimal performance for any device out of the box. + +**So in most cases you don't need to play the app options explicitly and the plain device name is enough**, for example, for CPU: +```sh +./benchmark_app -m -i -d CPU +``` + +But it is still may be sub-optimal for some cases, especially for very small networks. More details can read in [Performance Optimization Guide](../../../docs/optimization_guide/dldt_optimization_guide.md). + +As explained in the [Performance Optimization Guide](../../../docs/optimization_guide/dldt_optimization_guide.md) section, for all devices, including new [MULTI device](../../../docs/OV_Runtime_UG/supported_plugins/MULTI.md) it is preferable to use the FP16 IR for the model. +Also if latency of the CPU inference on the multi-socket machines is of concern, please refer to the same +[Performance Optimization Guide](../../../docs/optimization_guide/dldt_optimization_guide.md). + +Running the application with the `-h` option yields the following usage message: +``` +./benchmark_app -h +InferenceEngine: + API version ............ + Build .................. +[ INFO ] Parsing input parameters + +benchmark_app [OPTION] +Options: + + -h, --help Print a usage message + -m "" Required. Path to an .xml/.onnx/.prototxt file with a trained model or to a .blob files with a trained compiled model. + -i "" Optional. Path to a folder with images and/or binaries or to specific image or binary file. + -d "" Optional. Specify a target device to infer on (the list of available devices is shown below). Default value is CPU. + Use "-d HETERO:" format to specify HETERO plugin. + Use "-d MULTI:" format to specify MULTI plugin. + The application looks for a suitable plugin for the specified device. + -l "" Required for CPU custom layers. Absolute path to a shared library with the kernels implementations. + Or + -c "" Required for GPU custom kernels. Absolute path to an .xml file with the kernels description. + -api "" Optional. Enable Sync/Async API. Default value is "async". + -niter "" Optional. Number of iterations. If not specified, the number of iterations is calculated depending on a device. + -nireq "" Optional. Number of infer requests. Default value is determined automatically for a device. + -b "" Optional. Batch size value. If not specified, the batch size value is determined from Intermediate Representation. + -stream_output Optional. Print progress as a plain text. When specified, an interactive progress bar is replaced with a multiline output. + -t Optional. Time, in seconds, to execute topology. + -progress Optional. Show progress bar (can affect performance measurement). Default values is "false". + -shape Optional. Set shape for input. For example, "input1[1,3,224,224],input2[1,4]" or "[1,3,224,224]" in case of one input size. + -layout Optional. Prompts how network layouts should be treated by application. For example, "input1[NCHW],input2[NC]" or "[NCHW]" in case of one input size. + -cache_dir "" Optional. Enables caching of loaded models to specified directory. + -load_from_file Optional. Loads model from file directly without ReadNetwork. + + CPU-specific performance options: + -nstreams "" Optional. Number of streams to use for inference on the CPU, GPU or MYRIAD devices + (for HETERO and MULTI device cases use format :,: or just ). + Default value is determined automatically for a device. + Please note that although the automatic selection usually provides a reasonable performance, + it still may be non-optimal for some cases, especially for very small networks. + Also, using nstreams>1 is inherently throughput-oriented option, while for the best-latency + estimations the number of streams should be set to 1. + -nthreads "" Optional. Number of threads to use for inference on the CPU (including HETERO and MULTI cases). + -enforcebf16="" Optional. By default floating point operations execution in bfloat16 precision are enforced if supported by platform. + -pin "YES"/"HYBRID_AWARE"/"NUMA"/"NO" + Optional. Explicit inference threads binding options (leave empty to let the OpenVINO to make a choice): + enabling threads->cores pinning ("YES", which is already default for a conventional CPU), + letting the runtime to decide on the threads->different core types ("HYBRID_AWARE", which is default on the hybrid CPUs) + threads->(NUMA)nodes ("NUMA") or + completely disable ("NO") CPU inference threads pinning. + -ip "U8"/"FP16"/"FP32" Optional. Specifies precision for all input layers of the network. + -op "U8"/"FP16"/"FP32" Optional. Specifies precision for all output layers of the network. + -iop Optional. Specifies precision for input and output layers by name. Example: -iop "input:FP16, output:FP16". Notice that quotes are required. Overwrites precision from ip and op options for specified layers. + + Statistics dumping options: + -report_type "" Optional. Enable collecting statistics report. "no_counters" report contains configuration options specified, resulting FPS and latency. "average_counters" report extends "no_counters" report and additionally includes average PM counters values for each layer from the network. "detailed_counters" report extends "average_counters" report and additionally includes per-layer PM counters and latency for each executed infer request. + -report_folder Optional. Path to a folder where statistics report is stored. + -exec_graph_path Optional. Path to a file where to store executable graph information serialized. + -pc Optional. Report performance counters. + -dump_config Optional. Path to XML/YAML/JSON file to dump IE parameters, which were set by application. + -load_config Optional. Path to XML/YAML/JSON file to load custom IE parameters. Please note, command line parameters have higher priority then parameters from configuration file. +``` + +Running the application with the empty list of options yields the usage message given above and an error message. + +Application supports topologies with one or more inputs. If a topology is not data-sensitive, you can skip the input parameter. In this case, inputs are filled with random values. +If a model has only image input(s), please provide a folder with images or a path to an image as input. +If a model has some specific input(s) (not images), please prepare a binary file(s) that is filled with data of appropriate precision and provide a path to them as input. +If a model has mixed input types, input folder should contain all required files. Image inputs are filled with image files one by one. Binary inputs are filled with binary inputs one by one. + +To run the tool, you can use [public](@ref omz_models_group_public) or [Intel's](@ref omz_models_group_intel) pre-trained models from the Open Model Zoo. The models can be downloaded using the [Model Downloader](@ref omz_tools_downloader). + +> **NOTE**: Before running the tool with a trained model, make sure the model is converted to the Inference Engine format (\*.xml + \*.bin) using the [Model Optimizer tool](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md). +> +> The sample accepts models in ONNX format (.onnx) that do not require preprocessing. + +## Examples of Running the Tool + +This section provides step-by-step instructions on how to run the Benchmark Tool with the `googlenet-v1` public model on CPU or GPU devices. As an input, the `car.png` file from the `/samples/scripts/` directory is used. + +> **NOTE**: The Internet access is required to execute the following steps successfully. If you have access to the Internet through the proxy server only, please make sure that it is configured in your OS environment. + +1. Download the model. Install openvino-dev package into Python virtual environment from PyPi and run omz_downloader: + ```sh + omz_downloader --name googlenet-v1 -o +2. Convert the model to the Inference Engine IR format. Run the Model Optimizer using the `mo` command with the path to the model, model format (which must be FP32 for CPU and FPG) and output directory to generate the IR files: + ```sh + mo --input_model /public/googlenet-v1/googlenet-v1.caffemodel --data_type FP32 --output_dir + ``` +3. Run the tool with specifying the `/samples/scripts/car.png` file as an input image, the IR of the `googlenet-v1` model and a device to perform inference on. The following commands demonstrate running the Benchmark Tool in the asynchronous mode on CPU and GPU devices: + + * On CPU: + ```sh + ./benchmark_app -m /googlenet-v1.xml -i /samples/scripts/car.png -d CPU -api async --progress true + ``` + * On GPU: + ```sh + ./benchmark_app -m /googlenet-v1.xml -i /samples/scripts/car.png -d GPU -api async --progress true + ``` + +The application outputs the number of executed iterations, total duration of execution, latency, and throughput. +Additionally, if you set the `-report_type` parameter, the application outputs statistics report. If you set the `-pc` parameter, the application outputs performance counters. If you set `-exec_graph_path`, the application reports executable graph information serialized. All measurements including per-layer PM counters are reported in milliseconds. + +Below are fragments of sample output: + + ``` + [Step 10/11] Measuring performance (Start inference asynchronously, 4 inference requests using 4 streams for CPU, limits: 60000 ms duration) + [ INFO ] BENCHMARK IS IN INFERENCE ONLY MODE. + [ INFO ] Input blobs will be filled once before performance measurements. + [ INFO ] First inference took 26.26 ms + Progress: [................... ] 99% done + + [Step 11/11] Dumping statistics report + [ INFO ] Count: 6640 iterations + [ INFO ] Duration: 60039.70 ms + [ INFO ] Latency: + [ INFO ] Median: 35.36 ms + [ INFO ] Avg: 36.12 ms + [ INFO ] Min: 18.55 ms + [ INFO ] Max: 88.96 ms + [ INFO ] Throughput: 110.59 FPS + ``` + +## See Also +* [Model Optimizer](../../../docs/MO_DG/Deep_Learning_Model_Optimizer_DevGuide.md) +* [Model Downloader](@ref omz_tools_downloader) diff --git a/tools/legacy/benchmark_app/args_helper.cpp b/tools/legacy/benchmark_app/args_helper.cpp new file mode 100644 index 00000000000..ba6e2fe9c65 --- /dev/null +++ b/tools/legacy/benchmark_app/args_helper.cpp @@ -0,0 +1,216 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "args_helper.hpp" + +#include +#include + +#include +#include "slog.hpp" + +#ifdef _WIN32 + #include "w_dirent.h" +#else + #include +#endif + +/** + * @brief Checks input file argument and add it to files vector + * @param files reference to vector to store file names + * @param arg file or folder name + * @return none + */ +void readInputFilesArguments(std::vector& files, const std::string& arg) { + struct stat sb; + if (stat(arg.c_str(), &sb) != 0) { + slog::warn << "File " << arg << " cannot be opened!" << slog::endl; + return; + } + if (S_ISDIR(sb.st_mode)) { + struct CloseDir { + void operator()(DIR* d) const noexcept { + if (d) { + closedir(d); + } + } + }; + using Dir = std::unique_ptr; + Dir dp(opendir(arg.c_str())); + if (dp == nullptr) { + slog::warn << "Directory " << arg << " cannot be opened!" << slog::endl; + return; + } + + struct dirent* ep; + while (nullptr != (ep = readdir(dp.get()))) { + std::string fileName = ep->d_name; + if (fileName == "." || fileName == "..") + continue; + files.push_back(arg + "/" + ep->d_name); + } + } else { + files.push_back(arg); + } +} + +/** + * @brief This function find -i key in input args. It's necessary to process multiple values for + * single key + * @param files reference to vector + * @return none. + */ +void parseInputFilesArguments(std::vector& files) { + std::vector args = gflags::GetArgvs(); + auto args_it = begin(args); + const auto is_image_arg = [](const std::string& s) { + return s == "-i" || s == "--images"; + }; + const auto is_arg = [](const std::string& s) { + return s.front() == '-'; + }; + + while (args_it != args.end()) { + const auto img_start = std::find_if(args_it, end(args), is_image_arg); + if (img_start == end(args)) { + break; + } + const auto img_begin = std::next(img_start); + const auto img_end = std::find_if(img_begin, end(args), is_arg); + for (auto img = img_begin; img != img_end; ++img) { + readInputFilesArguments(files, *img); + } + args_it = img_end; + } + + if (files.empty()) { + return; + } + size_t max_files = 20; + if (files.size() < max_files) { + slog::info << "Files were added: " << files.size() << slog::endl; + for (const auto& filePath : files) { + slog::info << " " << filePath << slog::endl; + } + } else { + slog::info << "Files were added: " << files.size() << ". Too many to display each of them." << slog::endl; + } +} + +namespace { +std::vector splitStringList(const std::string& str, char delim) { + if (str.empty()) + return {}; + + std::istringstream istr(str); + + std::vector result; + std::string elem; + while (std::getline(istr, elem, delim)) { + if (elem.empty()) { + continue; + } + result.emplace_back(std::move(elem)); + } + + return result; +} + +std::map parseArgMap(std::string argMap) { + argMap.erase(std::remove_if(argMap.begin(), argMap.end(), ::isspace), argMap.end()); + + const auto pairs = splitStringList(argMap, ','); + + std::map parsedMap; + for (auto&& pair : pairs) { + const auto keyValue = splitStringList(pair, ':'); + if (keyValue.size() != 2) { + throw std::invalid_argument("Invalid key/value pair " + pair + ". Expected :"); + } + + parsedMap[keyValue[0]] = keyValue[1]; + } + + return parsedMap; +} + +using supported_precisions_t = std::unordered_map; + +InferenceEngine::Precision getPrecision(std::string value, const supported_precisions_t& supported_precisions) { + std::transform(value.begin(), value.end(), value.begin(), ::toupper); + + const auto precision = supported_precisions.find(value); + if (precision == supported_precisions.end()) { + throw std::logic_error("\"" + value + "\"" + " is not a valid precision"); + } + + return precision->second; +} + +InferenceEngine::Precision getPrecision(const std::string& value) { + static const supported_precisions_t supported_precisions = { + {"FP32", InferenceEngine::Precision::FP32}, {"FP16", InferenceEngine::Precision::FP16}, {"BF16", InferenceEngine::Precision::BF16}, + {"U64", InferenceEngine::Precision::U64}, {"I64", InferenceEngine::Precision::I64}, {"U32", InferenceEngine::Precision::U32}, + {"I32", InferenceEngine::Precision::I32}, {"U16", InferenceEngine::Precision::U16}, {"I16", InferenceEngine::Precision::I16}, + {"U8", InferenceEngine::Precision::U8}, {"I8", InferenceEngine::Precision::I8}, {"BOOL", InferenceEngine::Precision::BOOL}, + }; + + return getPrecision(value, supported_precisions); +} + +void setPrecisions(const InferenceEngine::CNNNetwork& network, const std::string& iop) { + const auto user_precisions_map = parseArgMap(iop); + + auto inputs = network.getInputsInfo(); + auto outputs = network.getOutputsInfo(); + + for (auto&& item : user_precisions_map) { + const auto& layer_name = item.first; + const auto& user_precision = item.second; + + const auto input = inputs.find(layer_name); + const auto output = outputs.find(layer_name); + + if (input != inputs.end()) { + input->second->setPrecision(getPrecision(user_precision)); + } else if (output != outputs.end()) { + output->second->setPrecision(getPrecision(user_precision)); + } else { + throw std::logic_error(layer_name + " is not an input neither output"); + } + } +} + +} // namespace + +void processPrecision(InferenceEngine::CNNNetwork& network, const std::string& ip, const std::string& op, const std::string& iop) { + if (!ip.empty()) { + const auto user_precision = getPrecision(ip); + for (auto&& layer : network.getInputsInfo()) { + layer.second->setPrecision(user_precision); + } + } + + if (!op.empty()) { + auto user_precision = getPrecision(op); + for (auto&& layer : network.getOutputsInfo()) { + layer.second->setPrecision(user_precision); + } + } + + if (!iop.empty()) { + setPrecisions(network, iop); + } +} + +void printInputAndOutputsInfo(const InferenceEngine::CNNNetwork& network) { + std::cout << "Network inputs:" << std::endl; + for (auto&& layer : network.getInputsInfo()) { + std::cout << " " << layer.first << " : " << layer.second->getPrecision() << " / " << layer.second->getLayout() << std::endl; + } + std::cout << "Network outputs:" << std::endl; + for (auto&& layer : network.getOutputsInfo()) { + std::cout << " " << layer.first << " : " << layer.second->getPrecision() << " / " << layer.second->getLayout() << std::endl; + } +} diff --git a/tools/legacy/benchmark_app/args_helper.hpp b/tools/legacy/benchmark_app/args_helper.hpp new file mode 100644 index 00000000000..2ffabc0a90a --- /dev/null +++ b/tools/legacy/benchmark_app/args_helper.hpp @@ -0,0 +1,32 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief a header file with common samples functionality + * @file args_helper.hpp + */ + +#pragma once + +#include +#include +#include + +/** + * @brief This function checks input args and existence of specified files in a given folder + * @param arg path to a file to be checked for existence + * @return files updated vector of verified input files + */ +void readInputFilesArguments(std::vector& files, const std::string& arg); + +/** + * @brief This function find -i/--images key in input args + * It's necessary to process multiple values for single key + * @return files updated vector of verified input files + */ +void parseInputFilesArguments(std::vector& files); + +void processPrecision(InferenceEngine::CNNNetwork& network, const std::string& ip, const std::string& op, const std::string& iop); + +void printInputAndOutputsInfo(const InferenceEngine::CNNNetwork& network); diff --git a/tools/legacy/benchmark_app/benchmark_app.hpp b/tools/legacy/benchmark_app/benchmark_app.hpp new file mode 100644 index 00000000000..af18c908e31 --- /dev/null +++ b/tools/legacy/benchmark_app/benchmark_app.hpp @@ -0,0 +1,299 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include +#include +#include + +/// @brief message for help argument +static const char help_message[] = "Print a usage message"; + +/// @brief message for images argument +static const char input_message[] = "Optional. Path to a folder with images and/or binaries or to specific image or binary file."; + +/// @brief message for model argument +static const char model_message[] = "Required. Path to an .xml/.onnx/.prototxt file with a trained model or to a .blob files with " + "a trained compiled model."; + +/// @brief message for execution mode +static const char api_message[] = "Optional. Enable Sync/Async API. Default value is \"async\"."; + +/// @brief message for assigning cnn calculation to device +static const char target_device_message[] = "Optional. Specify a target device to infer on (the list of available devices is shown below). " + "Default value is CPU. Use \"-d HETERO:\" format to specify " + "HETERO plugin. " + "Use \"-d MULTI:\" format to specify MULTI plugin. " + "The application looks for a suitable plugin for the specified device."; + +/// @brief message for iterations count +static const char iterations_count_message[] = "Optional. Number of iterations. " + "If not specified, the number of iterations is calculated depending on a device."; + +/// @brief message for requests count +static const char infer_requests_count_message[] = "Optional. Number of infer requests. Default value is determined automatically for device."; + +/// @brief message for execution time +static const char execution_time_message[] = "Optional. Time in seconds to execute topology."; + +/// @brief message for #threads for CPU inference +static const char infer_num_threads_message[] = "Optional. Number of threads to use for inference on the CPU " + "(including HETERO and MULTI cases)."; + +/// @brief message for #streams for CPU inference +static const char infer_num_streams_message[] = "Optional. Number of streams to use for inference on the CPU, GPU or MYRIAD devices " + "(for HETERO and MULTI device cases use format :,: or just " + "). " + "Default value is determined automatically for a device.Please note that although the " + "automatic selection " + "usually provides a reasonable performance, it still may be non - optimal for some cases, " + "especially for " + "very small networks. See sample's README for more details. " + "Also, using nstreams>1 is inherently throughput-oriented option, " + "while for the best-latency estimations the number of streams should be set to 1."; + +/// @brief message for enforcing of BF16 execution where it is possible +static const char enforce_bf16_message[] = "Optional. By default floating point operations execution in bfloat16 precision are enforced " + "if supported by platform.\n" + " 'true' - enable bfloat16 regardless of platform support\n" + " 'false' - disable bfloat16 regardless of platform support"; + +/// @brief message for user library argument +static const char custom_cpu_library_message[] = "Required for CPU custom layers. Absolute path to a shared library with the kernels " + "implementations."; + +/// @brief message for clDNN custom kernels desc +static const char custom_cldnn_message[] = "Required for GPU custom kernels. Absolute path to an .xml file with the kernels description."; + +static const char batch_size_message[] = "Optional. Batch size value. If not specified, the batch size value is determined from " + "Intermediate Representation."; + +// @brief message for CPU threads pinning option +static const char infer_threads_pinning_message[] = + "Optional. Explicit inference threads binding options (leave empty to let the OpenVINO to make a choice):\n" + "\t\t\t\tenabling threads->cores pinning(\"YES\", which is already default for any conventional CPU), \n" + "\t\t\t\tletting the runtime to decide on the threads->different core types(\"HYBRID_AWARE\", which is default on the hybrid CPUs) \n" + "\t\t\t\tthreads->(NUMA)nodes(\"NUMA\") or \n" + "\t\t\t\tcompletely disable(\"NO\") CPU inference threads pinning"; +// @brief message for stream_output option +static const char stream_output_message[] = "Optional. Print progress as a plain text. When specified, an interactive progress bar is " + "replaced with a " + "multiline output."; + +// @brief message for report_type option +static const char report_type_message[] = "Optional. Enable collecting statistics report. \"no_counters\" report contains " + "configuration options specified, resulting FPS and latency. \"average_counters\" " + "report extends \"no_counters\" report and additionally includes average PM " + "counters values for each layer from the network. \"detailed_counters\" report " + "extends \"average_counters\" report and additionally includes per-layer PM " + "counters and latency for each executed infer request."; + +// @brief message for report_folder option +static const char report_folder_message[] = "Optional. Path to a folder where statistics report is stored."; + +// @brief message for exec_graph_path option +static const char exec_graph_path_message[] = "Optional. Path to a file where to store executable graph information serialized."; + +// @brief message for progress bar option +static const char progress_message[] = "Optional. Show progress bar (can affect performance measurement). Default values is " + "\"false\"."; + +// @brief message for performance counters option +static const char pc_message[] = "Optional. Report performance counters."; + +#ifdef USE_OPENCV +// @brief message for load config option +static const char load_config_message[] = "Optional. Path to XML/YAML/JSON file to load custom IE parameters." + " Please note, command line parameters have higher priority then parameters from configuration " + "file."; + +// @brief message for dump config option +static const char dump_config_message[] = "Optional. Path to XML/YAML/JSON file to dump IE parameters, which were set by application."; +#endif + +static const char shape_message[] = "Optional. Set shape for input. For example, \"input1[1,3,224,224],input2[1,4]\" or " + "\"[1,3,224,224]\"" + " in case of one input size."; + +static const char layout_message[] = "Optional. Prompts how network layouts should be treated by application. " + "For example, \"input1[NCHW],input2[NC]\" or \"[NCHW]\" in case of one input size."; + +// @brief message for enabling caching +static const char cache_dir_message[] = "Optional. Enables caching of loaded models to specified directory. " + "List of devices which support caching is shown at the end of this message."; + +// @brief message for single load network +static const char load_from_file_message[] = "Optional. Loads model from file directly without ReadNetwork." + "All CNNNetwork options (like re-shape) will be ignored"; + +// @brief message for quantization bits +static const char gna_qb_message[] = "Optional. Weight bits for quantization: 8 or 16 (default)"; + +static constexpr char inputs_precision_message[] = "Optional. Specifies precision for all input layers of the network."; + +static constexpr char outputs_precision_message[] = "Optional. Specifies precision for all output layers of the network."; + +static constexpr char iop_message[] = "Optional. Specifies precision for input and output layers by name.\n" + " Example: -iop \"input:FP16, output:FP16\".\n" + " Notice that quotes are required.\n" + " Overwrites precision from ip and op options for " + "specified layers."; + +/// @brief Define flag for showing help message
+DEFINE_bool(h, false, help_message); + +/// @brief Declare flag for showing help message
+DECLARE_bool(help); + +/// @brief Define parameter for set image file
+/// i or mif is a required parameter +DEFINE_string(i, "", input_message); + +/// @brief Define parameter for set model file
+/// It is a required parameter +DEFINE_string(m, "", model_message); + +/// @brief Define execution mode +DEFINE_string(api, "async", api_message); + +/// @brief device the target device to infer on
+DEFINE_string(d, "CPU", target_device_message); + +/// @brief Absolute path to CPU library with user layers
+/// It is a required parameter +DEFINE_string(l, "", custom_cpu_library_message); + +/// @brief Define parameter for clDNN custom kernels path
+/// Default is ./lib +DEFINE_string(c, "", custom_cldnn_message); + +/// @brief Iterations count (default 0) +/// Sync mode: iterations count +/// Async mode: StartAsync counts +DEFINE_uint32(niter, 0, iterations_count_message); + +/// @brief Time to execute topology in seconds +DEFINE_uint32(t, 0, execution_time_message); + +/// @brief Number of infer requests in parallel +DEFINE_uint32(nireq, 0, infer_requests_count_message); + +/// @brief Number of threads to use for inference on the CPU in throughput mode (also affects Hetero +/// cases) +DEFINE_uint32(nthreads, 0, infer_num_threads_message); + +/// @brief Number of streams to use for inference on the CPU (also affects Hetero cases) +DEFINE_string(nstreams, "", infer_num_streams_message); + +/// @brief Enforces bf16 execution with bfloat16 precision on systems having this capability +DEFINE_bool(enforcebf16, false, enforce_bf16_message); + +/// @brief Define parameter for batch size
+/// Default is 0 (that means don't specify) +DEFINE_uint32(b, 0, batch_size_message); + +// @brief Enable plugin messages +DEFINE_string(pin, "", infer_threads_pinning_message); + +/// @brief Enables multiline text output instead of progress bar +DEFINE_bool(stream_output, false, stream_output_message); + +/// @brief Enables statistics report collecting +DEFINE_string(report_type, "", report_type_message); + +/// @brief Path to a folder where statistics report is stored +DEFINE_string(report_folder, "", report_folder_message); + +/// @brief Path to a file where to store executable graph information serialized +DEFINE_string(exec_graph_path, "", exec_graph_path_message); + +/// @brief Define flag for showing progress bar
+DEFINE_bool(progress, false, progress_message); + +/// @brief Define flag for showing performance counters
+DEFINE_bool(pc, false, pc_message); + +#ifdef USE_OPENCV +/// @brief Define flag for loading configuration file
+DEFINE_string(load_config, "", load_config_message); + +/// @brief Define flag for dumping configuration file
+DEFINE_string(dump_config, "", dump_config_message); +#endif + +/// @brief Define flag for input shape
+DEFINE_string(shape, "", shape_message); + +/// @brief Define flag for layout shape
+DEFINE_string(layout, "", layout_message); + +/// @brief Define flag for quantization bits (default 16) +DEFINE_int32(qb, 16, gna_qb_message); + +/// @brief Specify precision for all input layers of the network +DEFINE_string(ip, "", inputs_precision_message); + +/// @brief Specify precision for all ouput layers of the network +DEFINE_string(op, "", outputs_precision_message); + +/// @brief Specify precision for input and output layers by name.\n" +/// Example: -iop \"input:FP16, output:FP16\".\n" +/// Notice that quotes are required.\n" +/// Overwrites layout from ip and op options for specified layers."; +DEFINE_string(iop, "", iop_message); + +/// @brief Define parameter for cache model dir
+DEFINE_string(cache_dir, "", cache_dir_message); + +/// @brief Define flag for load network from model file by name without ReadNetwork
+DEFINE_bool(load_from_file, false, load_from_file_message); + +/** + * @brief This function show a help message + */ +static void showUsage() { + std::cout << std::endl; + std::cout << "benchmark_app [OPTION]" << std::endl; + std::cout << "Options:" << std::endl; + std::cout << std::endl; + std::cout << " -h, --help " << help_message << std::endl; + std::cout << " -m \"\" " << model_message << std::endl; + std::cout << " -i \"\" " << input_message << std::endl; + std::cout << " -d \"\" " << target_device_message << std::endl; + std::cout << " -l \"\" " << custom_cpu_library_message << std::endl; + std::cout << " Or" << std::endl; + std::cout << " -c \"\" " << custom_cldnn_message << std::endl; + std::cout << " -api \"\" " << api_message << std::endl; + std::cout << " -niter \"\" " << iterations_count_message << std::endl; + std::cout << " -nireq \"\" " << infer_requests_count_message << std::endl; + std::cout << " -b \"\" " << batch_size_message << std::endl; + std::cout << " -stream_output " << stream_output_message << std::endl; + std::cout << " -t " << execution_time_message << std::endl; + std::cout << " -progress " << progress_message << std::endl; + std::cout << " -shape " << shape_message << std::endl; + std::cout << " -layout " << layout_message << std::endl; + std::cout << " -cache_dir \"\" " << cache_dir_message << std::endl; + std::cout << " -load_from_file " << load_from_file_message << std::endl; + std::cout << std::endl << " device-specific performance options:" << std::endl; + std::cout << " -nstreams \"\" " << infer_num_streams_message << std::endl; + std::cout << " -nthreads \"\" " << infer_num_threads_message << std::endl; + std::cout << " -enforcebf16= " << enforce_bf16_message << std::endl; + std::cout << " -pin \"YES\"/\"HYBRID_AWARE\"/\"NO\"/\"NUMA\" " << infer_threads_pinning_message << std::endl; + std::cout << std::endl << " Statistics dumping options:" << std::endl; + std::cout << " -report_type \"\" " << report_type_message << std::endl; + std::cout << " -report_folder " << report_folder_message << std::endl; + std::cout << " -exec_graph_path " << exec_graph_path_message << std::endl; + std::cout << " -pc " << pc_message << std::endl; +#ifdef USE_OPENCV + std::cout << " -dump_config " << dump_config_message << std::endl; + std::cout << " -load_config " << load_config_message << std::endl; +#endif + std::cout << " -qb " << gna_qb_message << std::endl; + std::cout << " -ip " << inputs_precision_message << std::endl; + std::cout << " -op " << outputs_precision_message << std::endl; + std::cout << " -iop \"\" " << iop_message << std::endl; +} diff --git a/tools/legacy/benchmark_app/common.hpp b/tools/legacy/benchmark_app/common.hpp new file mode 100644 index 00000000000..ac423db96c8 --- /dev/null +++ b/tools/legacy/benchmark_app/common.hpp @@ -0,0 +1,166 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief a header file with common samples functionality + * @file common.hpp + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef UNUSED + #if defined(_MSC_VER) && !defined(__clang__) + #define UNUSED + #else + #define UNUSED __attribute__((unused)) + #endif +#endif +\ + +/** + * @brief Get extension from filename + * @param filename - name of the file which extension should be extracted + * @return string with extracted file extension + */ +inline std::string fileExt(const std::string& filename) { + auto pos = filename.rfind('.'); + if (pos == std::string::npos) + return ""; + return filename.substr(pos + 1); +} + +inline std::ostream& operator<<(std::ostream& os, const InferenceEngine::Version& version) { + os << "\t" << version.description << " version ......... "; + os << IE_VERSION_MAJOR << "." << IE_VERSION_MINOR << "." << IE_VERSION_PATCH; + + os << "\n\tBuild ........... "; + os << version.buildNumber; + + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const InferenceEngine::Version* version) { + if (nullptr != version) { + os << std::endl << *version; + } + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const std::map& versions) { + for (auto&& version : versions) { + os << "\t" << version.first << std::endl; + os << version.second << std::endl; + } + + return os; +} + +static std::vector> perfCountersSorted( + std::map perfMap) { + using perfItem = std::pair; + std::vector sorted; + for (auto& kvp : perfMap) + sorted.push_back(kvp); + + std::stable_sort(sorted.begin(), sorted.end(), [](const perfItem& l, const perfItem& r) { + return l.second.execution_index < r.second.execution_index; + }); + + return sorted; +} + +static UNUSED void printPerformanceCounts(const std::map& performanceMap, std::ostream& stream, + std::string deviceName, bool bshowHeader = true) { + long long totalTime = 0; + // Print performance counts + if (bshowHeader) { + stream << std::endl << "performance counts:" << std::endl << std::endl; + } + + auto performanceMapSorted = perfCountersSorted(performanceMap); + + for (const auto& it : performanceMapSorted) { + std::string toPrint(it.first); + const int maxLayerName = 30; + + if (it.first.length() >= maxLayerName) { + toPrint = it.first.substr(0, maxLayerName - 4); + toPrint += "..."; + } + + stream << std::setw(maxLayerName) << std::left << toPrint; + switch (it.second.status) { + case InferenceEngine::InferenceEngineProfileInfo::EXECUTED: + stream << std::setw(15) << std::left << "EXECUTED"; + break; + case InferenceEngine::InferenceEngineProfileInfo::NOT_RUN: + stream << std::setw(15) << std::left << "NOT_RUN"; + break; + case InferenceEngine::InferenceEngineProfileInfo::OPTIMIZED_OUT: + stream << std::setw(15) << std::left << "OPTIMIZED_OUT"; + break; + } + stream << std::setw(30) << std::left << "layerType: " + std::string(it.second.layer_type) + " "; + stream << std::setw(20) << std::left << "realTime: " + std::to_string(it.second.realTime_uSec); + stream << std::setw(20) << std::left << "cpu: " + std::to_string(it.second.cpu_uSec); + stream << " execType: " << it.second.exec_type << std::endl; + if (it.second.realTime_uSec > 0) { + totalTime += it.second.realTime_uSec; + } + } + stream << std::setw(20) << std::left << "Total time: " + std::to_string(totalTime) << " microseconds" << std::endl; + std::cout << std::endl; + std::cout << "Full device name: " << deviceName << std::endl; + std::cout << std::endl; +} + +static UNUSED void printPerformanceCounts(InferenceEngine::InferRequest request, std::ostream& stream, std::string deviceName, bool bshowHeader = true) { + auto performanceMap = request.GetPerformanceCounts(); + printPerformanceCounts(performanceMap, stream, deviceName, bshowHeader); +} + +inline std::string getFullDeviceName(std::map& devicesMap, std::string device) { + std::map::iterator it = devicesMap.find(device); + if (it != devicesMap.end()) { + return it->second; + } else { + return ""; + } +} + +inline std::string getFullDeviceName(InferenceEngine::Core& ie, std::string device) { + InferenceEngine::Parameter p; + try { + p = ie.GetMetric(device, METRIC_KEY(FULL_DEVICE_NAME)); + return p.as(); + } catch (InferenceEngine::Exception&) { + return ""; + } +} + +inline void showAvailableDevices() { + InferenceEngine::Core ie; + std::vector devices = ie.GetAvailableDevices(); + + std::cout << std::endl; + std::cout << "Available target devices:"; + for (const auto& device : devices) { + std::cout << " " << device; + } + std::cout << std::endl; +} diff --git a/tools/legacy/benchmark_app/console_progress.hpp b/tools/legacy/benchmark_app/console_progress.hpp new file mode 100644 index 00000000000..f62aeed37d1 --- /dev/null +++ b/tools/legacy/benchmark_app/console_progress.hpp @@ -0,0 +1,107 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include + +/** + * @class ConsoleProgress + * @brief A ConsoleProgress class provides functionality for printing progress dynamics + */ +class ConsoleProgress { + static const size_t DEFAULT_DETALIZATION = 20; + static const size_t DEFAULT_PERCENT_TO_UPDATE_PROGRESS = 1; + + size_t total; + size_t cur_progress = 0; + size_t prev_progress = 0; + bool stream_output; + size_t detalization; + size_t percent_to_update; + +public: + /** + * @brief A constructor of ConsoleProgress class + * @param _total - maximum value that is correspondent to 100% + * @param _detalization - number of symbols(.) to use to represent progress + */ + explicit ConsoleProgress(size_t _total, + bool _stream_output = false, + size_t _percent_to_update = DEFAULT_PERCENT_TO_UPDATE_PROGRESS, + size_t _detalization = DEFAULT_DETALIZATION) + : total(_total), + detalization(_detalization), + percent_to_update(_percent_to_update) { + stream_output = _stream_output; + if (total == 0) { + total = 1; + } + } + + /** + * @brief Shows progress with current data. Progress is shown from the beginning of the current + * line. + */ + void showProgress() const { + std::stringstream strm; + if (!stream_output) { + strm << '\r'; + } + strm << "Progress: ["; + size_t i = 0; + for (; i < detalization * cur_progress / total; i++) { + strm << "."; + } + for (; i < detalization; i++) { + strm << " "; + } + strm << "] " << std::setw(3) << 100 * cur_progress / total << "% done"; + if (stream_output) { + strm << std::endl; + } + std::fputs(strm.str().c_str(), stdout); + std::fflush(stdout); + } + + /** + * @brief Updates current value and progressbar + */ + void updateProgress() { + if (cur_progress > total) + cur_progress = total; + size_t prev_percent = 100 * prev_progress / total; + size_t cur_percent = 100 * cur_progress / total; + + if (prev_progress == 0 || cur_progress == total || prev_percent + percent_to_update <= cur_percent) { + showProgress(); + prev_progress = cur_progress; + } + } + + /** + * @brief Adds value to currently represented and redraw progressbar + * @param add - value to add + */ + void addProgress(int add) { + if (add < 0 && -add > static_cast(cur_progress)) { + add = -static_cast(cur_progress); + } + cur_progress += add; + updateProgress(); + } + + /** + * @brief Output end line. + * @return + */ + void finish() { + std::stringstream strm; + strm << std::endl; + std::fputs(strm.str().c_str(), stdout); + std::fflush(stdout); + } +}; diff --git a/tools/legacy/benchmark_app/csv_dumper.hpp b/tools/legacy/benchmark_app/csv_dumper.hpp new file mode 100644 index 00000000000..5e0bc039eae --- /dev/null +++ b/tools/legacy/benchmark_app/csv_dumper.hpp @@ -0,0 +1,98 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include + +#include +#include +#include "slog.hpp" +#include +#include + +/** + * @class CsvDumper + * @brief A CsvDumper class provides functionality for dumping the values in CSV files + */ +class CsvDumper { + std::ofstream file; + std::string filename; + bool canDump = true; + char delimiter = ';'; + + std::string generateFilename() { + std::stringstream filename; + filename << "dumpfile-"; + filename << time(nullptr); + filename << ".csv"; + return filename.str(); + } + +public: + /** + * @brief A constructor. Disables dumping in case dump file cannot be created + * @param enabled - True if dumping is enabled by default. + * @param name - name of file to dump to. File won't be created if first parameter is false. + */ + explicit CsvDumper(bool enabled = true, const std::string& name = ""): canDump(enabled) { + if (!canDump) { + return; + } + filename = (name == "" ? generateFilename() : name); + file.open(filename, std::ios::out); + if (!file) { + slog::warn << "Cannot create dump file! Disabling dump." << slog::endl; + canDump = false; + } + } + + /** + * @brief Sets a delimiter to use in csv file + * @param c - Delimiter char + * @return + */ + void setDelimiter(char c) { + delimiter = c; + } + + /** + * @brief Overloads operator to organize streaming values to file. Does nothing if dumping is + * disabled Adds delimiter at the end of value provided + * @param add - value to add to dump + * @return reference to same object + */ + template + CsvDumper& operator<<(const T& add) { + if (canDump) { + file << add << delimiter; + } + return *this; + } + + /** + * @brief Finishes line in dump file. Does nothing if dumping is disabled + */ + void endLine() { + if (canDump) { + file << "\n"; + } + } + + /** + * @brief Gets information if dump is enabled. + * @return true if dump is enabled and file was successfully created + */ + bool dumpEnabled() { + return canDump; + } + + /** + * @brief Gets name of a dump file + * @return name of a dump file + */ + std::string getFilename() const { + return filename; + } +}; diff --git a/tools/legacy/benchmark_app/infer_request_wrap.hpp b/tools/legacy/benchmark_app/infer_request_wrap.hpp new file mode 100644 index 00000000000..741b2ad7f13 --- /dev/null +++ b/tools/legacy/benchmark_app/infer_request_wrap.hpp @@ -0,0 +1,146 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "statistics_report.hpp" + +typedef std::chrono::high_resolution_clock Time; +typedef std::chrono::nanoseconds ns; + +typedef std::function QueueCallbackFunction; + +/// @brief Wrapper class for InferenceEngine::InferRequest. Handles asynchronous callbacks and calculates execution time. +class InferReqWrap final { +public: + using Ptr = std::shared_ptr; + + ~InferReqWrap() = default; + + explicit InferReqWrap(InferenceEngine::ExecutableNetwork& net, size_t id, QueueCallbackFunction callbackQueue) + : _request(net.CreateInferRequest()), _id(id), _callbackQueue(callbackQueue) { + _request.SetCompletionCallback([&]() { + _endTime = Time::now(); + _callbackQueue(_id, getExecutionTimeInMilliseconds()); + }); + } + + void startAsync() { + _startTime = Time::now(); + _request.StartAsync(); + } + + void wait() { + _request.Wait(InferenceEngine::InferRequest::RESULT_READY); + } + + void infer() { + _startTime = Time::now(); + _request.Infer(); + _endTime = Time::now(); + _callbackQueue(_id, getExecutionTimeInMilliseconds()); + } + + std::map getPerformanceCounts() { + return _request.GetPerformanceCounts(); + } + + InferenceEngine::Blob::Ptr getBlob(const std::string& name) { + return _request.GetBlob(name); + } + + double getExecutionTimeInMilliseconds() const { + auto execTime = std::chrono::duration_cast(_endTime - _startTime); + return static_cast(execTime.count()) * 0.000001; + } + +private: + InferenceEngine::InferRequest _request; + Time::time_point _startTime; + Time::time_point _endTime; + size_t _id; + QueueCallbackFunction _callbackQueue; +}; + +class InferRequestsQueue final { +public: + InferRequestsQueue(InferenceEngine::ExecutableNetwork& net, size_t nireq) { + for (size_t id = 0; id < nireq; id++) { + requests.push_back( + std::make_shared(net, id, std::bind(&InferRequestsQueue::putIdleRequest, this, std::placeholders::_1, std::placeholders::_2))); + _idleIds.push(id); + } + resetTimes(); + } + ~InferRequestsQueue() { + // Inference Request guarantee that it will wait for all asynchronous internal tasks in destructor + // So it should be released before any context that the request can use inside internal asynchronous tasks + // For example all members of InferRequestsQueue would be destroyed before `requests` vector + // So requests can try to use this members from `putIdleRequest()` that would be called from request callback + // To avoid this we should move this vector declaration after all members declaration or just clear it manually in destructor + requests.clear(); + } + + void resetTimes() { + _startTime = Time::time_point::max(); + _endTime = Time::time_point::min(); + _latencies.clear(); + } + + double getDurationInMilliseconds() { + return std::chrono::duration_cast(_endTime - _startTime).count() * 0.000001; + } + + void putIdleRequest(size_t id, const double latency) { + std::unique_lock lock(_mutex); + _latencies.push_back(latency); + _idleIds.push(id); + _endTime = std::max(Time::now(), _endTime); + _cv.notify_one(); + } + + InferReqWrap::Ptr getIdleRequest() { + std::unique_lock lock(_mutex); + _cv.wait(lock, [this] { + return _idleIds.size() > 0; + }); + auto request = requests.at(_idleIds.front()); + _idleIds.pop(); + _startTime = std::min(Time::now(), _startTime); + return request; + } + + void waitAll() { + std::unique_lock lock(_mutex); + _cv.wait(lock, [this] { + return _idleIds.size() == requests.size(); + }); + } + + std::vector getLatencies() { + return _latencies; + } + + std::vector requests; + +private: + std::queue _idleIds; + std::mutex _mutex; + std::condition_variable _cv; + Time::time_point _startTime; + Time::time_point _endTime; + std::vector _latencies; +}; diff --git a/tools/legacy/benchmark_app/inputs_filling.cpp b/tools/legacy/benchmark_app/inputs_filling.cpp new file mode 100644 index 00000000000..4ffa8f8ffdd --- /dev/null +++ b/tools/legacy/benchmark_app/inputs_filling.cpp @@ -0,0 +1,356 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "inputs_filling.hpp" + +#include + +#include +#include +#include "slog.hpp" +#include +#include +#include + +using namespace InferenceEngine; + +#ifdef USE_OPENCV +static const std::vector supported_image_extensions = {"bmp", "dib", "jpeg", "jpg", "jpe", "jp2", "png", + "pbm", "pgm", "ppm", "sr", "ras", "tiff", "tif"}; +#else +static const std::vector supported_image_extensions = {"bmp"}; +#endif +static const std::vector supported_binary_extensions = {"bin"}; + +std::vector filterFilesByExtensions(const std::vector& filePaths, const std::vector& extensions) { + std::vector filtered; + auto getExtension = [](const std::string& name) { + auto extensionPosition = name.rfind('.', name.size()); + return extensionPosition == std::string::npos ? "" : name.substr(extensionPosition + 1, name.size() - 1); + }; + for (auto& filePath : filePaths) { + auto extension = getExtension(filePath); + std::transform(extension.begin(), extension.end(), extension.begin(), ::tolower); + if (std::find(extensions.begin(), extensions.end(), extension) != extensions.end()) { + filtered.push_back(filePath); + } + } + return filtered; +} + +template +void fillBlobImage(Blob::Ptr& inputBlob, const std::vector& filePaths, const size_t& batchSize, const benchmark_app::InputInfo& app_info, + const size_t& requestId, const size_t& inputId, const size_t& inputSize) { + MemoryBlob::Ptr minput = as(inputBlob); + if (!minput) { + IE_THROW() << "We expect inputBlob to be inherited from MemoryBlob in " + "fillBlobImage, " + << "but by fact we were not able to cast inputBlob to MemoryBlob"; + } + // locked memory holder should be alive all time while access to its buffer + // happens + auto minputHolder = minput->wmap(); + auto inputBlobData = minputHolder.as(); + + /** Collect images data ptrs **/ + std::vector> vreader; + vreader.reserve(batchSize); + + for (size_t i = 0ULL, inputIndex = requestId * batchSize * inputSize + inputId; i < batchSize; i++, inputIndex += inputSize) { + inputIndex %= filePaths.size(); + + slog::info << "Prepare image " << filePaths[inputIndex] << slog::endl; + FormatReader::ReaderPtr reader(filePaths[inputIndex].c_str()); + if (reader.get() == nullptr) { + slog::warn << "Image " << filePaths[inputIndex] << " cannot be read!" << slog::endl << slog::endl; + continue; + } + + /** Getting image data **/ + std::shared_ptr imageData(reader->getData(app_info.width(), app_info.height())); + if (imageData) { + vreader.push_back(imageData); + } + } + + /** Fill input tensor with images. First b channel, then g and r channels **/ + const size_t numChannels = app_info.channels(); + const size_t width = app_info.width(); + const size_t height = app_info.height(); + /** Iterate over all input images **/ + for (size_t imageId = 0; imageId < vreader.size(); ++imageId) { + /** Iterate over all width **/ + for (size_t w = 0; w < app_info.width(); ++w) { + /** Iterate over all height **/ + for (size_t h = 0; h < app_info.height(); ++h) { + /** Iterate over all channels **/ + for (size_t ch = 0; ch < numChannels; ++ch) { + /** [images stride + channels stride + pixel id ] all in + * bytes **/ + size_t offset = imageId * numChannels * width * height + (((app_info.layout == "NCHW") || (app_info.layout == "CHW")) + ? (ch * width * height + h * width + w) + : (h * width * numChannels + w * numChannels + ch)); + inputBlobData[offset] = static_cast(vreader.at(imageId).get()[h * width * numChannels + w * numChannels + ch]); + } + } + } + } +} + +template +void fillBlobBinary(Blob::Ptr& inputBlob, const std::vector& filePaths, const size_t& batchSize, const size_t& requestId, const size_t& inputId, + const size_t& inputSize) { + MemoryBlob::Ptr minput = as(inputBlob); + if (!minput) { + IE_THROW() << "We expect inputBlob to be inherited from MemoryBlob in " + "fillBlobBinary, " + << "but by fact we were not able to cast inputBlob to MemoryBlob"; + } + // locked memory holder should be alive all time while access to its buffer + // happens + auto minputHolder = minput->wmap(); + + auto inputBlobData = minputHolder.as(); + for (size_t i = 0ULL, inputIndex = requestId * batchSize * inputSize + inputId; i < batchSize; i++, inputIndex += inputSize) { + inputIndex %= filePaths.size(); + + slog::info << "Prepare binary file " << filePaths[inputIndex] << slog::endl; + std::ifstream binaryFile(filePaths[inputIndex], std::ios_base::binary | std::ios_base::ate); + if (!binaryFile) { + IE_THROW() << "Cannot open " << filePaths[inputIndex]; + } + + auto fileSize = static_cast(binaryFile.tellg()); + binaryFile.seekg(0, std::ios_base::beg); + if (!binaryFile.good()) { + IE_THROW() << "Can not read " << filePaths[inputIndex]; + } + auto inputSize = inputBlob->size() * sizeof(T) / batchSize; + if (fileSize != inputSize) { + IE_THROW() << "File " << filePaths[inputIndex] << " contains " << std::to_string(fileSize) + << " bytes " + "but the network expects " + << std::to_string(inputSize); + } + binaryFile.read(&inputBlobData[i * inputSize], inputSize); + } +} + +template +using uniformDistribution = + typename std::conditional::value, std::uniform_real_distribution, + typename std::conditional::value, std::uniform_int_distribution, void>::type>::type; + +template +void fillBlobRandom(Blob::Ptr& inputBlob, T rand_min = std::numeric_limits::min(), T rand_max = std::numeric_limits::max()) { + MemoryBlob::Ptr minput = as(inputBlob); + if (!minput) { + IE_THROW() << "We expect inputBlob to be inherited from MemoryBlob in " + "fillBlobRandom, " + << "but by fact we were not able to cast inputBlob to MemoryBlob"; + } + // locked memory holder should be alive all time while access to its buffer + // happens + auto minputHolder = minput->wmap(); + + auto inputBlobData = minputHolder.as(); + std::mt19937 gen(0); + uniformDistribution distribution(rand_min, rand_max); + for (size_t i = 0; i < inputBlob->size(); i++) { + inputBlobData[i] = static_cast(distribution(gen)); + } +} + +template +void fillBlobImInfo(Blob::Ptr& inputBlob, const size_t& batchSize, std::pair image_size) { + MemoryBlob::Ptr minput = as(inputBlob); + if (!minput) { + IE_THROW() << "We expect inputBlob to be inherited from MemoryBlob in " + "fillBlobImInfo, " + << "but by fact we were not able to cast inputBlob to MemoryBlob"; + } + // locked memory holder should be alive all time while access to its buffer + // happens + auto minputHolder = minput->wmap(); + + auto inputBlobData = minputHolder.as(); + for (size_t b = 0; b < batchSize; b++) { + size_t iminfoSize = inputBlob->size() / batchSize; + for (size_t i = 0; i < iminfoSize; i++) { + size_t index = b * iminfoSize + i; + if (0 == i) + inputBlobData[index] = static_cast(image_size.first); + else if (1 == i) + inputBlobData[index] = static_cast(image_size.second); + else + inputBlobData[index] = 1; + } + } +} + +void fillBlobs(const std::vector& inputFiles, const size_t& batchSize, benchmark_app::InputsInfo& app_inputs_info, + std::vector requests) { + std::vector> input_image_sizes; + for (auto& item : app_inputs_info) { + if (item.second.isImage()) { + input_image_sizes.push_back(std::make_pair(item.second.width(), item.second.height())); + } + slog::info << "Network input '" << item.first << "' precision " << item.second.precision << ", dimensions (" << item.second.layout << "): "; + for (const auto& i : item.second.shape) { + slog::info << i << " "; + } + slog::info << slog::endl; + } + + size_t imageInputCount = input_image_sizes.size(); + size_t binaryInputCount = app_inputs_info.size() - imageInputCount; + + std::vector binaryFiles; + std::vector imageFiles; + + if (inputFiles.empty()) { + slog::warn << "No input files were given: all inputs will be filled with " + "random values!" + << slog::endl; + } else { + binaryFiles = filterFilesByExtensions(inputFiles, supported_binary_extensions); + std::sort(std::begin(binaryFiles), std::end(binaryFiles)); + + auto binaryToBeUsed = binaryInputCount * batchSize * requests.size(); + if (binaryToBeUsed > 0 && binaryFiles.empty()) { + std::stringstream ss; + for (auto& ext : supported_binary_extensions) { + if (!ss.str().empty()) { + ss << ", "; + } + ss << ext; + } + slog::warn << "No supported binary inputs found! Please check your file " + "extensions: " + << ss.str() << slog::endl; + } else if (binaryToBeUsed > binaryFiles.size()) { + slog::warn << "Some binary input files will be duplicated: " << binaryToBeUsed << " files are required but only " << binaryFiles.size() + << " are provided" << slog::endl; + } else if (binaryToBeUsed < binaryFiles.size()) { + slog::warn << "Some binary input files will be ignored: only " << binaryToBeUsed << " are required from " << binaryFiles.size() << slog::endl; + } + + imageFiles = filterFilesByExtensions(inputFiles, supported_image_extensions); + std::sort(std::begin(imageFiles), std::end(imageFiles)); + + auto imagesToBeUsed = imageInputCount * batchSize * requests.size(); + if (imagesToBeUsed > 0 && imageFiles.empty()) { + std::stringstream ss; + for (auto& ext : supported_image_extensions) { + if (!ss.str().empty()) { + ss << ", "; + } + ss << ext; + } + slog::warn << "No supported image inputs found! Please check your file " + "extensions: " + << ss.str() << slog::endl; + } else if (imagesToBeUsed > imageFiles.size()) { + slog::warn << "Some image input files will be duplicated: " << imagesToBeUsed << " files are required but only " << imageFiles.size() + << " are provided" << slog::endl; + } else if (imagesToBeUsed < imageFiles.size()) { + slog::warn << "Some image input files will be ignored: only " << imagesToBeUsed << " are required from " << imageFiles.size() << slog::endl; + } + } + + for (size_t requestId = 0; requestId < requests.size(); requestId++) { + slog::info << "Infer Request " << requestId << " filling" << slog::endl; + + size_t imageInputId = 0; + size_t binaryInputId = 0; + for (auto& item : app_inputs_info) { + Blob::Ptr inputBlob = requests.at(requestId)->getBlob(item.first); + auto app_info = app_inputs_info.at(item.first); + auto precision = app_info.precision; + if (app_info.isImage()) { + if (!imageFiles.empty()) { + // Fill with Images + if (precision == InferenceEngine::Precision::FP32) { + fillBlobImage(inputBlob, imageFiles, batchSize, app_info, requestId, imageInputId++, imageInputCount); + } else if (precision == InferenceEngine::Precision::FP16) { + fillBlobImage(inputBlob, imageFiles, batchSize, app_info, requestId, imageInputId++, imageInputCount); + } else if (precision == InferenceEngine::Precision::I32) { + fillBlobImage(inputBlob, imageFiles, batchSize, app_info, requestId, imageInputId++, imageInputCount); + } else if (precision == InferenceEngine::Precision::I64) { + fillBlobImage(inputBlob, imageFiles, batchSize, app_info, requestId, imageInputId++, imageInputCount); + } else if (precision == InferenceEngine::Precision::U8) { + fillBlobImage(inputBlob, imageFiles, batchSize, app_info, requestId, imageInputId++, imageInputCount); + } else { + IE_THROW() << "Input precision is not supported for " << item.first; + } + continue; + } + } else { + if (!binaryFiles.empty()) { + // Fill with binary files + if (precision == InferenceEngine::Precision::FP32) { + fillBlobBinary(inputBlob, binaryFiles, batchSize, requestId, binaryInputId++, binaryInputCount); + } else if (precision == InferenceEngine::Precision::FP16) { + fillBlobBinary(inputBlob, binaryFiles, batchSize, requestId, binaryInputId++, binaryInputCount); + } else if (precision == InferenceEngine::Precision::I32) { + fillBlobBinary(inputBlob, binaryFiles, batchSize, requestId, binaryInputId++, binaryInputCount); + } else if (precision == InferenceEngine::Precision::I64) { + fillBlobBinary(inputBlob, binaryFiles, batchSize, requestId, binaryInputId++, binaryInputCount); + } else if ((precision == InferenceEngine::Precision::U8) || (precision == InferenceEngine::Precision::BOOL)) { + fillBlobBinary(inputBlob, binaryFiles, batchSize, requestId, binaryInputId++, binaryInputCount); + } else { + IE_THROW() << "Input precision is not supported for " << item.first; + } + continue; + } + + if (app_info.isImageInfo() && (input_image_sizes.size() == 1)) { + // Most likely it is image info: fill with image information + auto image_size = input_image_sizes.at(0); + slog::info << "Fill input '" << item.first << "' with image size " << image_size.first << "x" << image_size.second << slog::endl; + if (precision == InferenceEngine::Precision::FP32) { + fillBlobImInfo(inputBlob, batchSize, image_size); + } else if (precision == InferenceEngine::Precision::FP16) { + fillBlobImInfo(inputBlob, batchSize, image_size); + } else if (precision == InferenceEngine::Precision::I32) { + fillBlobImInfo(inputBlob, batchSize, image_size); + } else if (precision == InferenceEngine::Precision::I64) { + fillBlobImInfo(inputBlob, batchSize, image_size); + } else { + IE_THROW() << "Input precision is not supported for image info!"; + } + continue; + } + } + // Fill random + slog::info << "Fill input '" << item.first << "' with random values (" << std::string((app_info.isImage() ? "image" : "some binary data")) + << " is expected)" << slog::endl; + if (precision == InferenceEngine::Precision::FP32) { + fillBlobRandom(inputBlob); + } else if (precision == InferenceEngine::Precision::FP16) { + fillBlobRandom(inputBlob); + } else if (precision == InferenceEngine::Precision::I32) { + fillBlobRandom(inputBlob); + } else if (precision == InferenceEngine::Precision::I64) { + fillBlobRandom(inputBlob); + } else if (precision == InferenceEngine::Precision::U8) { + // uniform_int_distribution is not allowed in the C++17 + // standard and vs2017/19 + fillBlobRandom(inputBlob); + } else if (precision == InferenceEngine::Precision::I8) { + // uniform_int_distribution is not allowed in the C++17 standard + // and vs2017/19 + fillBlobRandom(inputBlob); + } else if (precision == InferenceEngine::Precision::U16) { + fillBlobRandom(inputBlob); + } else if (precision == InferenceEngine::Precision::I16) { + fillBlobRandom(inputBlob); + } else if (precision == InferenceEngine::Precision::BOOL) { + fillBlobRandom(inputBlob, 0, 1); + } else { + IE_THROW() << "Input precision is not supported for " << item.first; + } + } + } +} diff --git a/tools/legacy/benchmark_app/inputs_filling.hpp b/tools/legacy/benchmark_app/inputs_filling.hpp new file mode 100644 index 00000000000..4410faae11e --- /dev/null +++ b/tools/legacy/benchmark_app/inputs_filling.hpp @@ -0,0 +1,15 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include + +#include "infer_request_wrap.hpp" +#include "utils.hpp" + +void fillBlobs(const std::vector& inputFiles, const size_t& batchSize, benchmark_app::InputsInfo& app_inputs_info, + std::vector requests); diff --git a/tools/legacy/benchmark_app/main.cpp b/tools/legacy/benchmark_app/main.cpp new file mode 100644 index 00000000000..51ccba5d39a --- /dev/null +++ b/tools/legacy/benchmark_app/main.cpp @@ -0,0 +1,704 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include +#include +#include +#include +#include +#include +#include "args_helper.hpp" +#include "common.hpp" +#include "slog.hpp" +#include +#include +#include +#include + +#include "benchmark_app.hpp" +#include "infer_request_wrap.hpp" +#include "inputs_filling.hpp" +#include "progress_bar.hpp" +#include "statistics_report.hpp" +#include "utils.hpp" + +using namespace InferenceEngine; + +static const size_t progressBarDefaultTotalCount = 1000; + +uint64_t getDurationInMilliseconds(uint32_t duration) { + return duration * 1000LL; +} + +uint64_t getDurationInNanoseconds(uint32_t duration) { + return duration * 1000000000LL; +} + +bool ParseAndCheckCommandLine(int argc, char* argv[]) { + // ---------------------------Parsing and validating input + // arguments-------------------------------------- + slog::info << "Parsing input parameters" << slog::endl; + gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true); + if (FLAGS_help || FLAGS_h) { + showUsage(); + showAvailableDevices(); + return false; + } + + if (FLAGS_m.empty()) { + showUsage(); + throw std::logic_error("Model is required but not set. Please set -m option."); + } + + if (FLAGS_api != "async" && FLAGS_api != "sync") { + throw std::logic_error("Incorrect API. Please set -api option to `sync` or `async` value."); + } + + if (!FLAGS_report_type.empty() && FLAGS_report_type != noCntReport && FLAGS_report_type != averageCntReport && FLAGS_report_type != detailedCntReport) { + std::string err = "only " + std::string(noCntReport) + "/" + std::string(averageCntReport) + "/" + std::string(detailedCntReport) + + " report types are supported (invalid -report_type option value)"; + throw std::logic_error(err); + } + + if ((FLAGS_report_type == averageCntReport) && ((FLAGS_d.find("MULTI") != std::string::npos))) { + throw std::logic_error("only " + std::string(detailedCntReport) + " report type is supported for MULTI device"); + } + + bool isNetworkCompiled = fileExt(FLAGS_m) == "blob"; + bool isPrecisionSet = !(FLAGS_ip.empty() && FLAGS_op.empty() && FLAGS_iop.empty()); + if (isNetworkCompiled && isPrecisionSet) { + std::string err = std::string("Cannot set precision for a compiled network. ") + std::string("Please re-compile your network with required precision " + "using compile_tool"); + + throw std::logic_error(err); + } + return true; +} + +static void next_step(const std::string additional_info = "") { + static size_t step_id = 0; + static const std::map step_names = {{1, "Parsing and validating input arguments"}, + {2, "Loading Inference Engine"}, + {3, "Setting device configuration"}, + {4, "Reading network files"}, + {5, "Resizing network to match image sizes and given batch"}, + {6, "Configuring input of the model"}, + {7, "Loading the model to the device"}, + {8, "Setting optimal runtime parameters"}, + {9, "Creating infer requests and filling input blobs with images"}, + {10, "Measuring performance"}, + {11, "Dumping statistics report"}}; + + step_id++; + if (step_names.count(step_id) == 0) + IE_THROW() << "Step ID " << step_id << " is out of total steps number " << step_names.size(); + + std::cout << "[Step " << step_id << "/" << step_names.size() << "] " << step_names.at(step_id) + << (additional_info.empty() ? "" : " (" + additional_info + ")") << std::endl; +} + +template +T getMedianValue(const std::vector& vec) { + std::vector sortedVec(vec); + std::sort(sortedVec.begin(), sortedVec.end()); + return (sortedVec.size() % 2 != 0) ? sortedVec[sortedVec.size() / 2ULL] + : (sortedVec[sortedVec.size() / 2ULL] + sortedVec[sortedVec.size() / 2ULL - 1ULL]) / static_cast(2.0); +} + +/** + * @brief The entry point of the benchmark application + */ +int main(int argc, char* argv[]) { + std::shared_ptr statistics; + try { + ExecutableNetwork exeNetwork; + + // ----------------- 1. Parsing and validating input arguments + // ------------------------------------------------- + next_step(); + + if (!ParseAndCheckCommandLine(argc, argv)) { + return 0; + } + + bool isNetworkCompiled = fileExt(FLAGS_m) == "blob"; + if (isNetworkCompiled) { + slog::info << "Network is compiled" << slog::endl; + } + + std::vector flags; + StatisticsReport::Parameters command_line_arguments; + gflags::GetAllFlags(&flags); + for (auto& flag : flags) { + if (!flag.is_default) { + command_line_arguments.push_back({flag.name, flag.current_value}); + } + } + if (!FLAGS_report_type.empty()) { + statistics = std::make_shared(StatisticsReport::Config {FLAGS_report_type, FLAGS_report_folder}); + statistics->addParameters(StatisticsReport::Category::COMMAND_LINE_PARAMETERS, command_line_arguments); + } + auto isFlagSetInCommandLine = [&command_line_arguments](const std::string& name) { + return (std::find_if(command_line_arguments.begin(), command_line_arguments.end(), [name](const std::pair& p) { + return p.first == name; + }) != command_line_arguments.end()); + }; + + std::string device_name = FLAGS_d; + + // Parse devices + auto devices = parseDevices(device_name); + + // Parse nstreams per device + std::map device_nstreams = parseNStreamsValuePerDevice(devices, FLAGS_nstreams); + + // Load device config file if specified + std::map> config; +#ifdef USE_OPENCV + if (!FLAGS_load_config.empty()) { + load_config(FLAGS_load_config, config); + } +#endif + /** This vector stores paths to the processed images **/ + std::vector inputFiles; + parseInputFilesArguments(inputFiles); + + // ----------------- 2. Loading the Inference Engine + // ----------------------------------------------------------- + next_step(); + + Core ie; + if (FLAGS_d.find("CPU") != std::string::npos && !FLAGS_l.empty()) { + // CPU (MKLDNN) extensions is loaded as a shared library and passed as a + // pointer to base extension + const auto extension_ptr = std::make_shared(FLAGS_l); + ie.AddExtension(extension_ptr); + slog::info << "CPU (MKLDNN) extensions is loaded " << FLAGS_l << slog::endl; + } + + // Load clDNN Extensions + if ((FLAGS_d.find("GPU") != std::string::npos) && !FLAGS_c.empty()) { + // Override config if command line parameter is specified + if (!config.count("GPU")) + config["GPU"] = {}; + config["GPU"][CONFIG_KEY(CONFIG_FILE)] = FLAGS_c; + } + if (config.count("GPU") && config.at("GPU").count(CONFIG_KEY(CONFIG_FILE))) { + auto ext = config.at("GPU").at(CONFIG_KEY(CONFIG_FILE)); + ie.SetConfig({{CONFIG_KEY(CONFIG_FILE), ext}}, "GPU"); + slog::info << "GPU extensions is loaded " << ext << slog::endl; + } + + slog::info << "InferenceEngine: " << GetInferenceEngineVersion() << slog::endl; + slog::info << "Device info: " << slog::endl; + std::cout << ie.GetVersions(device_name) << std::endl; + + // ----------------- 3. Setting device configuration + // ----------------------------------------------------------- + next_step(); + + bool perf_counts = false; + // Update config per device according to command line parameters + for (auto& device : devices) { + if (!config.count(device)) + config[device] = {}; + std::map& device_config = config.at(device); + + // Set performance counter + if (isFlagSetInCommandLine("pc")) { + // set to user defined value + device_config[CONFIG_KEY(PERF_COUNT)] = FLAGS_pc ? CONFIG_VALUE(YES) : CONFIG_VALUE(NO); + } else if (device_config.count(CONFIG_KEY(PERF_COUNT)) && (device_config.at(CONFIG_KEY(PERF_COUNT)) == "YES")) { + slog::warn << "Performance counters for " << device << " device is turned on. To print results use -pc option." << slog::endl; + } else if (FLAGS_report_type == detailedCntReport || FLAGS_report_type == averageCntReport) { + slog::warn << "Turn on performance counters for " << device << " device since report type is " << FLAGS_report_type << "." << slog::endl; + device_config[CONFIG_KEY(PERF_COUNT)] = CONFIG_VALUE(YES); + } else if (!FLAGS_exec_graph_path.empty()) { + slog::warn << "Turn on performance counters for " << device << " device due to execution graph dumping." << slog::endl; + device_config[CONFIG_KEY(PERF_COUNT)] = CONFIG_VALUE(YES); + } else { + // set to default value + device_config[CONFIG_KEY(PERF_COUNT)] = FLAGS_pc ? CONFIG_VALUE(YES) : CONFIG_VALUE(NO); + } + perf_counts = (device_config.at(CONFIG_KEY(PERF_COUNT)) == CONFIG_VALUE(YES)) ? true : perf_counts; + + auto setThroughputStreams = [&]() { + const std::string key = device + "_THROUGHPUT_STREAMS"; + if (device_nstreams.count(device)) { + // set to user defined value + std::vector supported_config_keys = ie.GetMetric(device, METRIC_KEY(SUPPORTED_CONFIG_KEYS)); + if (std::find(supported_config_keys.begin(), supported_config_keys.end(), key) == supported_config_keys.end()) { + throw std::logic_error("Device " + device + " doesn't support config key '" + key + "'! " + + "Please specify -nstreams for correct devices in format " + ":,:" + + " or via configuration file."); + } + device_config[key] = device_nstreams.at(device); + } else if (!device_config.count(key) && (FLAGS_api == "async")) { + slog::warn << "-nstreams default value is determined automatically for " << device + << " device. " + "Although the automatic selection usually provides a " + "reasonable performance, " + "but it still may be non-optimal for some cases, for more " + "information look at README." + << slog::endl; + if (std::string::npos == device.find("MYRIAD")) // MYRIAD sets the default number of + // streams implicitly (without _AUTO) + device_config[key] = std::string(device + "_THROUGHPUT_AUTO"); + } + if (device_config.count(key)) + device_nstreams[device] = device_config.at(key); + }; + + if (device == "CPU") { // CPU supports few special performance-oriented keys + // limit threading for CPU portion of inference + if (isFlagSetInCommandLine("nthreads")) + device_config[CONFIG_KEY(CPU_THREADS_NUM)] = std::to_string(FLAGS_nthreads); + + if (isFlagSetInCommandLine("enforcebf16")) + device_config[CONFIG_KEY(ENFORCE_BF16)] = FLAGS_enforcebf16 ? CONFIG_VALUE(YES) : CONFIG_VALUE(NO); + + if (isFlagSetInCommandLine("pin")) { + // set to user defined value + device_config[CONFIG_KEY(CPU_BIND_THREAD)] = FLAGS_pin; + } else if (!device_config.count(CONFIG_KEY(CPU_BIND_THREAD))) { + if ((device_name.find("MULTI") != std::string::npos) && (device_name.find("GPU") != std::string::npos)) { + slog::warn << "Turn off threads pinning for " << device << " device since multi-scenario with GPU device is used." << slog::endl; + device_config[CONFIG_KEY(CPU_BIND_THREAD)] = CONFIG_VALUE(NO); + } + } + + // for CPU execution, more throughput-oriented execution via streams + setThroughputStreams(); + } else if (device == ("GPU")) { + // for GPU execution, more throughput-oriented execution via streams + setThroughputStreams(); + + if ((device_name.find("MULTI") != std::string::npos) && (device_name.find("CPU") != std::string::npos)) { + slog::warn << "Turn on GPU trottling. Multi-device execution with " + "the CPU + GPU performs best with GPU trottling hint," + << "which releases another CPU thread (that is otherwise " + "used by the GPU driver for active polling)" + << slog::endl; + device_config[GPU_CONFIG_KEY(PLUGIN_THROTTLE)] = "1"; + } + } else if (device == "MYRIAD") { + device_config[CONFIG_KEY(LOG_LEVEL)] = CONFIG_VALUE(LOG_WARNING); + setThroughputStreams(); + } else if (device == "GNA") { + if (FLAGS_qb == 8) + device_config[GNA_CONFIG_KEY(PRECISION)] = "I8"; + else + device_config[GNA_CONFIG_KEY(PRECISION)] = "I16"; + + if (isFlagSetInCommandLine("nthreads")) + device_config[GNA_CONFIG_KEY(LIB_N_THREADS)] = std::to_string(FLAGS_nthreads); + } else { + std::vector supported_config_keys = ie.GetMetric(device, METRIC_KEY(SUPPORTED_CONFIG_KEYS)); + auto supported = [&](const std::string& key) { + return std::find(std::begin(supported_config_keys), std::end(supported_config_keys), key) != std::end(supported_config_keys); + }; + if (supported(CONFIG_KEY(CPU_THREADS_NUM)) && isFlagSetInCommandLine("nthreads")) { + device_config[CONFIG_KEY(CPU_THREADS_NUM)] = std::to_string(FLAGS_nthreads); + } + if (supported(CONFIG_KEY(CPU_THROUGHPUT_STREAMS)) && isFlagSetInCommandLine("nstreams")) { + device_config[CONFIG_KEY(CPU_THROUGHPUT_STREAMS)] = FLAGS_nstreams; + } + if (supported(CONFIG_KEY(CPU_BIND_THREAD)) && isFlagSetInCommandLine("pin")) { + device_config[CONFIG_KEY(CPU_BIND_THREAD)] = FLAGS_pin; + } + } + } + + for (auto&& item : config) { + ie.SetConfig(item.second, item.first); + } + + auto double_to_string = [](const double number) { + std::stringstream ss; + ss << std::fixed << std::setprecision(2) << number; + return ss.str(); + }; + auto get_total_ms_time = [](Time::time_point& startTime) { + return std::chrono::duration_cast(Time::now() - startTime).count() * 0.000001; + }; + + size_t batchSize = FLAGS_b; + Precision precision = Precision::UNSPECIFIED; + std::string topology_name = ""; + benchmark_app::InputsInfo app_inputs_info; + std::string output_name; + + // Takes priority over config from file + if (!FLAGS_cache_dir.empty()) { + ie.SetConfig({{CONFIG_KEY(CACHE_DIR), FLAGS_cache_dir}}); + } + + if (FLAGS_load_from_file && !isNetworkCompiled) { + next_step(); + slog::info << "Skipping the step for loading network from file" << slog::endl; + next_step(); + slog::info << "Skipping the step for loading network from file" << slog::endl; + next_step(); + slog::info << "Skipping the step for loading network from file" << slog::endl; + auto startTime = Time::now(); + exeNetwork = ie.LoadNetwork(FLAGS_m, device_name); + auto duration_ms = double_to_string(get_total_ms_time(startTime)); + slog::info << "Load network took " << duration_ms << " ms" << slog::endl; + if (statistics) + statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"load network time (ms)", duration_ms}}); + if (batchSize == 0) { + batchSize = 1; + } + } else if (!isNetworkCompiled) { + // ----------------- 4. Reading the Intermediate Representation network + // ---------------------------------------- + next_step(); + + slog::info << "Loading network files" << slog::endl; + + auto startTime = Time::now(); + CNNNetwork cnnNetwork = ie.ReadNetwork(FLAGS_m); + auto duration_ms = double_to_string(get_total_ms_time(startTime)); + slog::info << "Read network took " << duration_ms << " ms" << slog::endl; + if (statistics) + statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"read network time (ms)", duration_ms}}); + + const InputsDataMap inputInfo(cnnNetwork.getInputsInfo()); + if (inputInfo.empty()) { + throw std::logic_error("no inputs info is provided"); + } + + // ----------------- 5. Resizing network to match image sizes and given + // batch ---------------------------------- + next_step(); + batchSize = cnnNetwork.getBatchSize(); + // Parse input shapes if specified + bool reshape = false; + app_inputs_info = getInputsInfo(FLAGS_shape, FLAGS_layout, FLAGS_b, inputInfo, reshape); + if (reshape) { + InferenceEngine::ICNNNetwork::InputShapes shapes = {}; + for (auto& item : app_inputs_info) + shapes[item.first] = item.second.shape; + slog::info << "Reshaping network: " << getShapesString(shapes) << slog::endl; + startTime = Time::now(); + cnnNetwork.reshape(shapes); + duration_ms = double_to_string(get_total_ms_time(startTime)); + slog::info << "Reshape network took " << duration_ms << " ms" << slog::endl; + if (statistics) + statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"reshape network time (ms)", duration_ms}}); + } + // use batch size according to provided layout and shapes + batchSize = (!FLAGS_layout.empty()) ? getBatchSize(app_inputs_info) : cnnNetwork.getBatchSize(); + + topology_name = cnnNetwork.getName(); + slog::info << (FLAGS_b != 0 ? "Network batch size was changed to: " : "Network batch size: ") << batchSize << slog::endl; + + // ----------------- 6. Configuring inputs and outputs + // ---------------------------------------------------------------------- + next_step(); + + processPrecision(cnnNetwork, FLAGS_ip, FLAGS_op, FLAGS_iop); + for (auto& item : cnnNetwork.getInputsInfo()) { + // if precision for input set by user, then set it to app_inputs + // if it an image, set U8 + if (!FLAGS_ip.empty() || FLAGS_iop.find(item.first) != std::string::npos) { + app_inputs_info.at(item.first).precision = item.second->getPrecision(); + } else if (app_inputs_info.at(item.first).isImage()) { + app_inputs_info.at(item.first).precision = Precision::U8; + item.second->setPrecision(app_inputs_info.at(item.first).precision); + } + } + + printInputAndOutputsInfo(cnnNetwork); + // ----------------- 7. Loading the model to the device + // -------------------------------------------------------- + next_step(); + startTime = Time::now(); + exeNetwork = ie.LoadNetwork(cnnNetwork, device_name); + duration_ms = double_to_string(get_total_ms_time(startTime)); + slog::info << "Load network took " << duration_ms << " ms" << slog::endl; + if (statistics) + statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"load network time (ms)", duration_ms}}); + } else { + next_step(); + slog::info << "Skipping the step for compiled network" << slog::endl; + next_step(); + slog::info << "Skipping the step for compiled network" << slog::endl; + next_step(); + slog::info << "Skipping the step for compiled network" << slog::endl; + // ----------------- 7. Loading the model to the device + // -------------------------------------------------------- + next_step(); + auto startTime = Time::now(); + exeNetwork = ie.ImportNetwork(FLAGS_m, device_name, {}); + auto duration_ms = double_to_string(get_total_ms_time(startTime)); + slog::info << "Import network took " << duration_ms << " ms" << slog::endl; + if (statistics) + statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"import network time (ms)", duration_ms}}); + app_inputs_info = getInputsInfo(FLAGS_shape, FLAGS_layout, FLAGS_b, exeNetwork.GetInputsInfo()); + if (batchSize == 0) { + batchSize = 1; + } + } + // ----------------- 8. Setting optimal runtime parameters + // ----------------------------------------------------- + next_step(); + + // Update number of streams + for (auto&& ds : device_nstreams) { + const std::string key = ds.first + "_THROUGHPUT_STREAMS"; + device_nstreams[ds.first] = ie.GetConfig(ds.first, key).as(); + } + + // Number of requests + uint32_t nireq = FLAGS_nireq; + if (nireq == 0) { + if (FLAGS_api == "sync") { + nireq = 1; + } else { + std::string key = METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS); + try { + nireq = exeNetwork.GetMetric(key).as(); + } catch (const std::exception& ex) { + IE_THROW() << "Every device used with the benchmark_app should " + << "support OPTIMAL_NUMBER_OF_INFER_REQUESTS " + "ExecutableNetwork metric. " + << "Failed to query the metric for the " << device_name << " with error:" << ex.what(); + } + } + } + + // Iteration limit + uint32_t niter = FLAGS_niter; + if ((niter > 0) && (FLAGS_api == "async")) { + niter = ((niter + nireq - 1) / nireq) * nireq; + if (FLAGS_niter != niter) { + slog::warn << "Number of iterations was aligned by request number from " << FLAGS_niter << " to " << niter << " using number of requests " + << nireq << slog::endl; + } + } + + // Time limit + uint32_t duration_seconds = 0; + if (FLAGS_t != 0) { + // time limit + duration_seconds = FLAGS_t; + } else if (FLAGS_niter == 0) { + // default time limit + duration_seconds = deviceDefaultDeviceDurationInSeconds(device_name); + } + uint64_t duration_nanoseconds = getDurationInNanoseconds(duration_seconds); + + if (statistics) { + statistics->addParameters(StatisticsReport::Category::RUNTIME_CONFIG, + { + {"topology", topology_name}, + {"target device", device_name}, + {"API", FLAGS_api}, + {"precision", std::string(precision.name())}, + {"batch size", std::to_string(batchSize)}, + {"number of iterations", std::to_string(niter)}, + {"number of parallel infer requests", std::to_string(nireq)}, + {"duration (ms)", std::to_string(getDurationInMilliseconds(duration_seconds))}, + }); + for (auto& nstreams : device_nstreams) { + std::stringstream ss; + ss << "number of " << nstreams.first << " streams"; + statistics->addParameters(StatisticsReport::Category::RUNTIME_CONFIG, { + {ss.str(), nstreams.second}, + }); + } + } + + // ----------------- 9. Creating infer requests and filling input blobs + // ---------------------------------------- + next_step(); + + InferRequestsQueue inferRequestsQueue(exeNetwork, nireq); + fillBlobs(inputFiles, batchSize, app_inputs_info, inferRequestsQueue.requests); + + // ----------------- 10. Measuring performance + // ------------------------------------------------------------------ + size_t progressCnt = 0; + size_t progressBarTotalCount = progressBarDefaultTotalCount; + size_t iteration = 0; + + std::stringstream ss; + ss << "Start inference " << FLAGS_api << "hronously"; + if (FLAGS_api == "async") { + if (!ss.str().empty()) { + ss << ", "; + } + ss << nireq << " inference requests"; + std::stringstream device_ss; + for (auto& nstreams : device_nstreams) { + if (!device_ss.str().empty()) { + device_ss << ", "; + } + device_ss << nstreams.second << " streams for " << nstreams.first; + } + if (!device_ss.str().empty()) { + ss << " using " << device_ss.str(); + } + } + ss << ", limits: "; + if (duration_seconds > 0) { + ss << getDurationInMilliseconds(duration_seconds) << " ms duration"; + } + if (niter != 0) { + if (duration_seconds == 0) { + progressBarTotalCount = niter; + } + if (duration_seconds > 0) { + ss << ", "; + } + ss << niter << " iterations"; + } + next_step(ss.str()); + + // warming up - out of scope + auto inferRequest = inferRequestsQueue.getIdleRequest(); + if (!inferRequest) { + IE_THROW() << "No idle Infer Requests!"; + } + if (FLAGS_api == "sync") { + inferRequest->infer(); + } else { + inferRequest->startAsync(); + } + inferRequestsQueue.waitAll(); + auto duration_ms = double_to_string(inferRequestsQueue.getLatencies()[0]); + slog::info << "First inference took " << duration_ms << " ms" << slog::endl; + if (statistics) + statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"first inference time (ms)", duration_ms}}); + inferRequestsQueue.resetTimes(); + + auto startTime = Time::now(); + auto execTime = std::chrono::duration_cast(Time::now() - startTime).count(); + + /** Start inference & calculate performance **/ + /** to align number if iterations to guarantee that last infer requests are + * executed in the same conditions **/ + ProgressBar progressBar(progressBarTotalCount, FLAGS_stream_output, FLAGS_progress); + + while ((niter != 0LL && iteration < niter) || (duration_nanoseconds != 0LL && (uint64_t)execTime < duration_nanoseconds) || + (FLAGS_api == "async" && iteration % nireq != 0)) { + inferRequest = inferRequestsQueue.getIdleRequest(); + if (!inferRequest) { + IE_THROW() << "No idle Infer Requests!"; + } + + if (FLAGS_api == "sync") { + inferRequest->infer(); + } else { + // As the inference request is currently idle, the wait() adds no + // additional overhead (and should return immediately). The primary + // reason for calling the method is exception checking/re-throwing. + // Callback, that governs the actual execution can handle errors as + // well, but as it uses just error codes it has no details like ‘what()’ + // method of `std::exception` So, rechecking for any exceptions here. + inferRequest->wait(); + inferRequest->startAsync(); + } + iteration++; + + execTime = std::chrono::duration_cast(Time::now() - startTime).count(); + + if (niter > 0) { + progressBar.addProgress(1); + } else { + // calculate how many progress intervals are covered by current + // iteration. depends on the current iteration time and time of each + // progress interval. Previously covered progress intervals must be + // skipped. + auto progressIntervalTime = duration_nanoseconds / progressBarTotalCount; + size_t newProgress = execTime / progressIntervalTime - progressCnt; + progressBar.addProgress(newProgress); + progressCnt += newProgress; + } + } + + // wait the latest inference executions + inferRequestsQueue.waitAll(); + + double latency = getMedianValue(inferRequestsQueue.getLatencies()); + double totalDuration = inferRequestsQueue.getDurationInMilliseconds(); + double fps = (FLAGS_api == "sync") ? batchSize * 1000.0 / latency : batchSize * 1000.0 * iteration / totalDuration; + + if (statistics) { + statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, { + {"total execution time (ms)", double_to_string(totalDuration)}, + {"total number of iterations", std::to_string(iteration)}, + }); + if (device_name.find("MULTI") == std::string::npos) { + statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, { + {"latency (ms)", double_to_string(latency)}, + }); + } + statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"throughput", double_to_string(fps)}}); + } + + progressBar.finish(); + + // ----------------- 11. Dumping statistics report + // ------------------------------------------------------------- + next_step(); + +#ifdef USE_OPENCV + if (!FLAGS_dump_config.empty()) { + dump_config(FLAGS_dump_config, config); + slog::info << "Inference Engine configuration settings were dumped to " << FLAGS_dump_config << slog::endl; + } +#endif + + if (!FLAGS_exec_graph_path.empty()) { + try { + CNNNetwork execGraphInfo = exeNetwork.GetExecGraphInfo(); + execGraphInfo.serialize(FLAGS_exec_graph_path); + slog::info << "executable graph is stored to " << FLAGS_exec_graph_path << slog::endl; + } catch (const std::exception& ex) { + slog::err << "Can't get executable graph: " << ex.what() << slog::endl; + } + } + + if (perf_counts) { + std::vector> perfCounts; + for (size_t ireq = 0; ireq < nireq; ireq++) { + auto reqPerfCounts = inferRequestsQueue.requests[ireq]->getPerformanceCounts(); + if (FLAGS_pc) { + slog::info << "Performance counts for " << ireq << "-th infer request:" << slog::endl; + printPerformanceCounts(reqPerfCounts, std::cout, getFullDeviceName(ie, FLAGS_d), false); + } + perfCounts.push_back(reqPerfCounts); + } + if (statistics) { + statistics->dumpPerformanceCounters(perfCounts); + } + } + + if (statistics) + statistics->dump(); + + std::cout << "Count: " << iteration << " iterations" << std::endl; + std::cout << "Duration: " << double_to_string(totalDuration) << " ms" << std::endl; + if (device_name.find("MULTI") == std::string::npos) + std::cout << "Latency: " << double_to_string(latency) << " ms" << std::endl; + std::cout << "Throughput: " << double_to_string(fps) << " FPS" << std::endl; + } catch (const std::exception& ex) { + slog::err << ex.what() << slog::endl; + + if (statistics) { + statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, { + {"error", ex.what()}, + }); + statistics->dump(); + } + + return 3; + } + + return 0; +} diff --git a/tools/legacy/benchmark_app/progress_bar.hpp b/tools/legacy/benchmark_app/progress_bar.hpp new file mode 100644 index 00000000000..2cd1ebaa69e --- /dev/null +++ b/tools/legacy/benchmark_app/progress_bar.hpp @@ -0,0 +1,51 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include "console_progress.hpp" + +/// @brief Responsible for progress bar handling within the benchmark_app +class ProgressBar { +public: + explicit ProgressBar(size_t totalNum, bool streamOutput = false, bool progressEnabled = false) { + _bar.reset(new ConsoleProgress(totalNum, streamOutput)); + _streamOutput = streamOutput; + _isFinished = true; + _progressEnabled = progressEnabled; + } + + void addProgress(size_t num) { + _isFinished = false; + if (_progressEnabled) { + _bar->addProgress(num); + } + } + + void finish(size_t num = 0) { + if (num > 0) { + addProgress(num); + } + _isFinished = true; + _bar->finish(); + if (_progressEnabled) { + std::cout << std::endl; + } + } + + void newBar(size_t totalNum) { + if (_isFinished) { + _bar.reset(new ConsoleProgress(totalNum, _streamOutput)); + } else { + throw std::logic_error("Cannot create a new bar. Current bar is still in progress"); + } + } + +private: + std::unique_ptr _bar; + bool _streamOutput; + bool _isFinished; + bool _progressEnabled; +}; diff --git a/tools/legacy/benchmark_app/slog.cpp b/tools/legacy/benchmark_app/slog.cpp new file mode 100644 index 00000000000..0fc7d391ef3 --- /dev/null +++ b/tools/legacy/benchmark_app/slog.cpp @@ -0,0 +1,33 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "slog.hpp" + +#include + +namespace slog { + +LogStream info("INFO", std::cout); +LogStream warn("WARNING", std::cout); +LogStream err("ERROR", std::cerr); + +LogStream::LogStream(const std::string& prefix, std::ostream& log_stream): _prefix(prefix), _new_line(true) { + _log_stream = &log_stream; +} + +// Specializing for LogStreamEndLine to support slog::endl +LogStream& LogStream::operator<<(const LogStreamEndLine& /*arg*/) { + _new_line = true; + + (*_log_stream) << std::endl; + return *this; +} + +// Specializing for LogStreamBoolAlpha to support slog::boolalpha +LogStream& LogStream::operator<<(const LogStreamBoolAlpha& /*arg*/) { + (*_log_stream) << std::boolalpha; + return *this; +} + +} // namespace slog \ No newline at end of file diff --git a/tools/legacy/benchmark_app/slog.hpp b/tools/legacy/benchmark_app/slog.hpp new file mode 100644 index 00000000000..0f5150b8ef0 --- /dev/null +++ b/tools/legacy/benchmark_app/slog.hpp @@ -0,0 +1,74 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief a header file with logging facility for common samples + * @file log.hpp + */ + +#pragma once + +#include +#include + +namespace slog { +/** + * @class LogStreamEndLine + * @brief The LogStreamEndLine class implements an end line marker for a log stream + */ +class LogStreamEndLine {}; + +static constexpr LogStreamEndLine endl; + +/** + * @class LogStreamBoolAlpha + * @brief The LogStreamBoolAlpha class implements bool printing for a log stream + */ +class LogStreamBoolAlpha {}; + +static constexpr LogStreamBoolAlpha boolalpha; + +/** + * @class LogStream + * @brief The LogStream class implements a stream for sample logging + */ +class LogStream { + std::string _prefix; + std::ostream* _log_stream; + bool _new_line; + +public: + /** + * @brief A constructor. Creates an LogStream object + * @param prefix The prefix to print + */ + LogStream(const std::string& prefix, std::ostream& log_stream); + + /** + * @brief A stream output operator to be used within the logger + * @param arg Object for serialization in the logger message + */ + template + LogStream& operator<<(const T& arg) { + if (_new_line) { + (*_log_stream) << "[ " << _prefix << " ] "; + _new_line = false; + } + + (*_log_stream) << arg; + return *this; + } + + // Specializing for LogStreamEndLine to support slog::endl + LogStream& operator<<(const LogStreamEndLine&); + + // Specializing for LogStreamBoolAlpha to support slog::boolalpha + LogStream& operator<<(const LogStreamBoolAlpha&); +}; + +extern LogStream info; +extern LogStream warn; +extern LogStream err; + +} // namespace slog diff --git a/tools/legacy/benchmark_app/statistics_report.cpp b/tools/legacy/benchmark_app/statistics_report.cpp new file mode 100644 index 00000000000..8ad8443804f --- /dev/null +++ b/tools/legacy/benchmark_app/statistics_report.cpp @@ -0,0 +1,143 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "statistics_report.hpp" + +#include +#include +#include +#include +#include + +void StatisticsReport::addParameters(const Category& category, const Parameters& parameters) { + if (_parameters.count(category) == 0) + _parameters[category] = parameters; + else + _parameters[category].insert(_parameters[category].end(), parameters.begin(), parameters.end()); +} + +void StatisticsReport::dump() { + CsvDumper dumper(true, _config.report_folder + _separator + "benchmark_report.csv"); + + auto dump_parameters = [&dumper](const Parameters& parameters) { + for (auto& parameter : parameters) { + dumper << parameter.first << parameter.second; + dumper.endLine(); + } + }; + if (_parameters.count(Category::COMMAND_LINE_PARAMETERS)) { + dumper << "Command line parameters"; + dumper.endLine(); + + dump_parameters(_parameters.at(Category::COMMAND_LINE_PARAMETERS)); + dumper.endLine(); + } + + if (_parameters.count(Category::RUNTIME_CONFIG)) { + dumper << "Configuration setup"; + dumper.endLine(); + + dump_parameters(_parameters.at(Category::RUNTIME_CONFIG)); + dumper.endLine(); + } + + if (_parameters.count(Category::EXECUTION_RESULTS)) { + dumper << "Execution results"; + dumper.endLine(); + + dump_parameters(_parameters.at(Category::EXECUTION_RESULTS)); + dumper.endLine(); + } + + slog::info << "Statistics report is stored to " << dumper.getFilename() << slog::endl; +} + +void StatisticsReport::dumpPerformanceCountersRequest(CsvDumper& dumper, const PerformaceCounters& perfCounts) { + auto performanceMapSorted = perfCountersSorted(perfCounts); + + long long total = 0L; + long long total_cpu = 0L; + + dumper << "layerName" + << "execStatus" + << "layerType" + << "execType"; + dumper << "realTime (ms)" + << "cpuTime (ms)"; + dumper.endLine(); + + for (const auto& layer : performanceMapSorted) { + dumper << layer.first; // layer name + + switch (layer.second.status) { + case InferenceEngine::InferenceEngineProfileInfo::EXECUTED: + dumper << "EXECUTED"; + break; + case InferenceEngine::InferenceEngineProfileInfo::NOT_RUN: + dumper << "NOT_RUN"; + break; + case InferenceEngine::InferenceEngineProfileInfo::OPTIMIZED_OUT: + dumper << "OPTIMIZED_OUT"; + break; + } + dumper << layer.second.layer_type << layer.second.exec_type; + dumper << std::to_string(layer.second.realTime_uSec / 1000.0) << std::to_string(layer.second.cpu_uSec / 1000.0); + total += layer.second.realTime_uSec; + total_cpu += layer.second.cpu_uSec; + dumper.endLine(); + } + dumper << "Total" + << "" + << "" + << ""; + dumper << total / 1000.0 << total_cpu / 1000.0; + dumper.endLine(); + dumper.endLine(); +} + +void StatisticsReport::dumpPerformanceCounters(const std::vector& perfCounts) { + if ((_config.report_type.empty()) || (_config.report_type == noCntReport)) { + slog::info << "Statistics collecting for performance counters was not " + "requested. No reports are dumped." + << slog::endl; + return; + } + if (perfCounts.empty()) { + slog::info << "Performance counters are empty. No reports are dumped." << slog::endl; + return; + } + CsvDumper dumper(true, _config.report_folder + _separator + "benchmark_" + _config.report_type + "_report.csv"); + if (_config.report_type == detailedCntReport) { + for (auto& pc : perfCounts) { + dumpPerformanceCountersRequest(dumper, pc); + } + } else if (_config.report_type == averageCntReport) { + auto getAveragePerformanceCounters = [&perfCounts]() { + std::map performanceCountersAvg; + // iterate over each processed infer request and handle its PM data + for (size_t i = 0; i < perfCounts.size(); i++) { + auto performanceMapSorted = perfCountersSorted(perfCounts[i]); + // iterate over each layer from sorted vector and add required PM data + // to the per-layer maps + for (const auto& pm : performanceMapSorted) { + if (performanceCountersAvg.count(pm.first) == 0) { + performanceCountersAvg[pm.first] = perfCounts.at(i).at(pm.first); + } else { + performanceCountersAvg[pm.first].realTime_uSec += perfCounts.at(i).at(pm.first).realTime_uSec; + performanceCountersAvg[pm.first].cpu_uSec += perfCounts.at(i).at(pm.first).cpu_uSec; + } + } + } + for (auto& pm : performanceCountersAvg) { + pm.second.realTime_uSec /= perfCounts.size(); + pm.second.cpu_uSec /= perfCounts.size(); + } + return performanceCountersAvg; + }; + dumpPerformanceCountersRequest(dumper, getAveragePerformanceCounters()); + } else { + throw std::logic_error("PM data can only be collected for average or detailed report types"); + } + slog::info << "Performance counters report is stored to " << dumper.getFilename() << slog::endl; +} diff --git a/tools/legacy/benchmark_app/statistics_report.hpp b/tools/legacy/benchmark_app/statistics_report.hpp new file mode 100644 index 00000000000..bb4877da89a --- /dev/null +++ b/tools/legacy/benchmark_app/statistics_report.hpp @@ -0,0 +1,70 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include "common.hpp" +#include "csv_dumper.hpp" +#include "slog.hpp" +#include +#include +#include + +// @brief statistics reports types +static constexpr char noCntReport[] = "no_counters"; +static constexpr char averageCntReport[] = "average_counters"; +static constexpr char detailedCntReport[] = "detailed_counters"; + +/// @brief Responsible for collecting of statistics and dumping to .csv file +class StatisticsReport { +public: + typedef std::map PerformaceCounters; + typedef std::vector> Parameters; + + struct Config { + std::string report_type; + std::string report_folder; + }; + + enum class Category { + COMMAND_LINE_PARAMETERS, + RUNTIME_CONFIG, + EXECUTION_RESULTS, + }; + + explicit StatisticsReport(Config config): _config(std::move(config)) { + _separator = +#if defined _WIN32 || defined __CYGWIN__ + #if defined UNICODE + L"\\"; + #else + "\\"; + #endif +#else + "/"; +#endif + if (_config.report_folder.empty()) + _separator = ""; + } + + void addParameters(const Category& category, const Parameters& parameters); + + void dump(); + + void dumpPerformanceCounters(const std::vector& perfCounts); + +private: + void dumpPerformanceCountersRequest(CsvDumper& dumper, const PerformaceCounters& perfCounts); + + // configuration of current benchmark execution + const Config _config; + + // parameters + std::map _parameters; + + // csv separator + std::string _separator; +}; diff --git a/tools/legacy/benchmark_app/utils.cpp b/tools/legacy/benchmark_app/utils.cpp new file mode 100644 index 00000000000..43e2d27fc19 --- /dev/null +++ b/tools/legacy/benchmark_app/utils.cpp @@ -0,0 +1,194 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +// clang-format off +#include +#include +#include +#include "common.hpp" +#include "slog.hpp" +#include +#include +#include + +#include "utils.hpp" +// clang-format on + +#ifdef USE_OPENCV + #include +#endif + +namespace benchmark_app { +bool InputInfo::isImage() const { + if ((layout != "NCHW") && (layout != "NHWC") && (layout != "CHW") && (layout != "HWC")) + return false; + return (channels() == 3); +} +bool InputInfo::isImageInfo() const { + if (layout != "NC") + return false; + return (channels() >= 2); +} +size_t InputInfo::getDimentionByLayout(char character) const { + size_t pos = layout.find(character); + if (pos == std::string::npos) + throw std::runtime_error("Error: Can't get " + std::string(character, 1) + " from layout " + layout); + return shape.at(pos); +} +size_t InputInfo::width() const { + return getDimentionByLayout('W'); +} +size_t InputInfo::height() const { + return getDimentionByLayout('H'); +} +size_t InputInfo::channels() const { + return getDimentionByLayout('C'); +} +size_t InputInfo::batch() const { + return getDimentionByLayout('N'); +} +size_t InputInfo::depth() const { + return getDimentionByLayout('D'); +} +} // namespace benchmark_app + +uint32_t deviceDefaultDeviceDurationInSeconds(const std::string& device) { + static const std::map deviceDefaultDurationInSeconds {{"CPU", 60}, {"GPU", 60}, {"VPU", 60}, {"MYRIAD", 60}, + {"HDDL", 60}, {"FPGA", 120}, {"UNKNOWN", 120}}; + uint32_t duration = 0; + for (const auto& deviceDurationInSeconds : deviceDefaultDurationInSeconds) { + if (device.find(deviceDurationInSeconds.first) != std::string::npos) { + duration = std::max(duration, deviceDurationInSeconds.second); + } + } + if (duration == 0) { + const auto unknownDeviceIt = + find_if(deviceDefaultDurationInSeconds.begin(), deviceDefaultDurationInSeconds.end(), [](std::pair deviceDuration) { + return deviceDuration.first == "UNKNOWN"; + }); + + if (unknownDeviceIt == deviceDefaultDurationInSeconds.end()) { + throw std::logic_error("UNKNOWN device was not found in the device duration list"); + } + duration = unknownDeviceIt->second; + slog::warn << "Default duration " << duration << " seconds for unknown device '" << device << "' is used" << slog::endl; + } + return duration; +} + +std::vector split(const std::string& s, char delim) { + std::vector result; + std::stringstream ss(s); + std::string item; + + while (getline(ss, item, delim)) { + result.push_back(item); + } + return result; +} + +std::vector parseDevices(const std::string& device_string) { + std::string comma_separated_devices = device_string; + if (comma_separated_devices.find(":") != std::string::npos) { + comma_separated_devices = comma_separated_devices.substr(comma_separated_devices.find(":") + 1); + } + if ((comma_separated_devices == "MULTI") || (comma_separated_devices == "HETERO")) + return std::vector(); + auto devices = split(comma_separated_devices, ','); + for (auto& device : devices) + device = device.substr(0, device.find_first_of(".(")); + return devices; +} + +std::map parseNStreamsValuePerDevice(const std::vector& devices, const std::string& values_string) { + // Format: :,: or just + std::map result; + auto device_value_strings = split(values_string, ','); + for (auto& device_value_string : device_value_strings) { + auto device_value_vec = split(device_value_string, ':'); + if (device_value_vec.size() == 2) { + auto device_name = device_value_vec.at(0); + auto nstreams = device_value_vec.at(1); + auto it = std::find(devices.begin(), devices.end(), device_name); + if (it != devices.end()) { + result[device_name] = nstreams; + } else { + throw std::logic_error("Can't set nstreams value " + std::string(nstreams) + " for device '" + device_name + "'! Incorrect device name!"); + } + } else if (device_value_vec.size() == 1) { + auto value = device_value_vec.at(0); + for (auto& device : devices) { + result[device] = value; + } + } else if (device_value_vec.size() != 0) { + throw std::runtime_error("Unknown string format: " + values_string); + } + } + return result; +} + +size_t getBatchSize(const benchmark_app::InputsInfo& inputs_info) { + size_t batch_size = 0; + for (auto& info : inputs_info) { + std::size_t batch_index = info.second.layout.find("N"); + if (batch_index != std::string::npos) { + if (batch_size == 0) + batch_size = info.second.shape[batch_index]; + else if (batch_size != info.second.shape[batch_index]) + throw std::logic_error("Can't deterimine batch size: batch is " + "different for different inputs!"); + } + } + if (batch_size == 0) + batch_size = 1; + return batch_size; +} + +std::string getShapesString(const InferenceEngine::ICNNNetwork::InputShapes& shapes) { + std::stringstream ss; + for (auto& shape : shapes) { + if (!ss.str().empty()) + ss << ", "; + ss << "\'" << shape.first << "': ["; + for (size_t i = 0; i < shape.second.size(); i++) { + if (i > 0) + ss << ", "; + ss << shape.second.at(i); + } + ss << "]"; + } + return ss.str(); +} + +#ifdef USE_OPENCV +void dump_config(const std::string& filename, const std::map>& config) { + cv::FileStorage fs(filename, cv::FileStorage::WRITE); + if (!fs.isOpened()) + throw std::runtime_error("Error: Can't open config file : " + filename); + for (auto device_it = config.begin(); device_it != config.end(); ++device_it) { + fs << device_it->first << "{:"; + for (auto param_it = device_it->second.begin(); param_it != device_it->second.end(); ++param_it) + fs << param_it->first << param_it->second; + fs << "}"; + } + fs.release(); +} + +void load_config(const std::string& filename, std::map>& config) { + cv::FileStorage fs(filename, cv::FileStorage::READ); + if (!fs.isOpened()) + throw std::runtime_error("Error: Can't load config file : " + filename); + cv::FileNode root = fs.root(); + for (auto it = root.begin(); it != root.end(); ++it) { + auto device = *it; + if (!device.isMap()) { + throw std::runtime_error("Error: Can't parse config file : " + filename); + } + for (auto iit = device.begin(); iit != device.end(); ++iit) { + auto item = *iit; + config[device.name()][item.name()] = item.string(); + } + } +} +#endif \ No newline at end of file diff --git a/tools/legacy/benchmark_app/utils.hpp b/tools/legacy/benchmark_app/utils.hpp new file mode 100644 index 00000000000..d2923002242 --- /dev/null +++ b/tools/legacy/benchmark_app/utils.hpp @@ -0,0 +1,123 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include + +namespace benchmark_app { +struct InputInfo { + InferenceEngine::Precision precision; + InferenceEngine::SizeVector shape; + std::string layout; + bool isImage() const; + bool isImageInfo() const; + size_t getDimentionByLayout(char character) const; + size_t width() const; + size_t height() const; + size_t channels() const; + size_t batch() const; + size_t depth() const; +}; +using InputsInfo = std::map; +} // namespace benchmark_app + +std::vector parseDevices(const std::string& device_string); +uint32_t deviceDefaultDeviceDurationInSeconds(const std::string& device); +std::map parseNStreamsValuePerDevice(const std::vector& devices, const std::string& values_string); +std::string getShapesString(const InferenceEngine::ICNNNetwork::InputShapes& shapes); +size_t getBatchSize(const benchmark_app::InputsInfo& inputs_info); +std::vector split(const std::string& s, char delim); + +template +std::map parseInputParameters(const std::string parameter_string, const std::map& input_info) { + // Parse parameter string like "input0[value0],input1[value1]" or "[value]" (applied to all + // inputs) + std::map return_value; + std::string search_string = parameter_string; + auto start_pos = search_string.find_first_of('['); + while (start_pos != std::string::npos) { + auto end_pos = search_string.find_first_of(']'); + if (end_pos == std::string::npos) + break; + auto input_name = search_string.substr(0, start_pos); + auto input_value = search_string.substr(start_pos + 1, end_pos - start_pos - 1); + if (!input_name.empty()) { + return_value[input_name] = input_value; + } else { + for (auto& item : input_info) { + return_value[item.first] = input_value; + } + } + search_string = search_string.substr(end_pos + 1); + if (search_string.empty() || search_string.front() != ',') + break; + search_string = search_string.substr(1); + start_pos = search_string.find_first_of('['); + } + if (!search_string.empty()) + throw std::logic_error("Can't parse input parameter string: " + parameter_string); + return return_value; +} + +template +benchmark_app::InputsInfo getInputsInfo(const std::string& shape_string, const std::string& layout_string, const size_t batch_size, + const std::map& input_info, bool& reshape_required) { + std::map shape_map = parseInputParameters(shape_string, input_info); + std::map layout_map = parseInputParameters(layout_string, input_info); + reshape_required = false; + benchmark_app::InputsInfo info_map; + for (auto& item : input_info) { + benchmark_app::InputInfo info; + auto name = item.first; + auto descriptor = item.second->getTensorDesc(); + // Precision + info.precision = descriptor.getPrecision(); + // Shape + if (shape_map.count(name)) { + std::vector parsed_shape; + for (auto& dim : split(shape_map.at(name), ',')) { + parsed_shape.push_back(std::stoi(dim)); + } + info.shape = parsed_shape; + reshape_required = true; + } else { + info.shape = descriptor.getDims(); + } + // Layout + if (layout_map.count(name)) { + info.layout = layout_map.at(name); + std::transform(info.layout.begin(), info.layout.end(), info.layout.begin(), ::toupper); + } else { + std::stringstream ss; + ss << descriptor.getLayout(); + info.layout = ss.str(); + } + // Update shape with batch if needed + if (batch_size != 0) { + std::size_t batch_index = info.layout.find("N"); + if ((batch_index != std::string::npos) && (info.shape.at(batch_index) != batch_size)) { + info.shape[batch_index] = batch_size; + reshape_required = true; + } + } + info_map[name] = info; + } + return info_map; +} + +template +benchmark_app::InputsInfo getInputsInfo(const std::string& shape_string, const std::string& layout_string, const size_t batch_size, + const std::map& input_info) { + bool reshape_required = false; + return getInputsInfo(shape_string, layout_string, batch_size, input_info, reshape_required); +} + +#ifdef USE_OPENCV +void dump_config(const std::string& filename, const std::map>& config); +void load_config(const std::string& filename, std::map>& config); +#endif \ No newline at end of file diff --git a/tools/legacy/benchmark_app/w_dirent.h b/tools/legacy/benchmark_app/w_dirent.h new file mode 100644 index 00000000000..5352a8f8b13 --- /dev/null +++ b/tools/legacy/benchmark_app/w_dirent.h @@ -0,0 +1,176 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#if defined(_WIN32) + + #ifndef WIN32_LEAN_AND_MEAN + #define WIN32_LEAN_AND_MEAN + #define WIN32_LEAN_AND_MEAN_UNDEF + #endif + + #ifndef NOMINMAX + #define NOMINMAX + #define NOMINMAX_UNDEF + #endif + + #if defined(_M_IX86) && !defined(_X86_) && !defined(_AMD64_) + #define _X86_ + #endif + + #if defined(_M_X64) && !defined(_X86_) && !defined(_AMD64_) + #define _AMD64_ + #endif + + #if defined(_M_ARM) && !defined(_ARM_) && !defined(_ARM64_) + #define _ARM_ + #endif + + #if defined(_M_ARM64) && !defined(_ARM_) && !defined(_ARM64_) + #define _ARM64_ + #endif + + // clang-format off + #include + #include + #include + #include + #include + // clang-format on + + // Copied from linux libc sys/stat.h: + #define S_ISREG(m) (((m)&S_IFMT) == S_IFREG) + #define S_ISDIR(m) (((m)&S_IFMT) == S_IFDIR) + +/// @brief structure to store directory names +struct dirent { + char* d_name; + + explicit dirent(const wchar_t* wsFilePath) { + size_t i; + auto slen = wcslen(wsFilePath); + d_name = static_cast(malloc(slen + 1)); + wcstombs_s(&i, d_name, slen + 1, wsFilePath, slen); + } + ~dirent() { + free(d_name); + } +}; + +/// @brief class to store directory data (files meta) +class DIR { + WIN32_FIND_DATAA FindFileData; + HANDLE hFind; + dirent* next; + + static inline bool endsWith(const std::string& src, const char* with) { + int wl = static_cast(strlen(with)); + int so = static_cast(src.length()) - wl; + if (so < 0) + return false; + return 0 == strncmp(with, &src[so], wl); + } + +public: + DIR(const DIR& other) = delete; + DIR(DIR&& other) = delete; + DIR& operator=(const DIR& other) = delete; + DIR& operator=(DIR&& other) = delete; + + explicit DIR(const char* dirPath): next(nullptr) { + std::string ws = dirPath; + if (endsWith(ws, "\\")) + ws += "*"; + else + ws += "\\*"; + hFind = FindFirstFileA(ws.c_str(), &FindFileData); + FindFileData.dwReserved0 = hFind != INVALID_HANDLE_VALUE; + } + + ~DIR() { + if (!next) + delete next; + next = nullptr; + FindClose(hFind); + } + + /** + * @brief Check file handler is valid + * @return status True(success) or False(fail) + */ + bool isValid() const { + return (hFind != INVALID_HANDLE_VALUE && FindFileData.dwReserved0); + } + + /** + * @brief Add directory to directory names struct + * @return pointer to directory names struct + */ + dirent* nextEnt() { + if (next != nullptr) + delete next; + next = nullptr; + + if (!FindFileData.dwReserved0) + return nullptr; + + wchar_t wbuf[4096]; + + size_t outSize; + mbstowcs_s(&outSize, wbuf, 4094, FindFileData.cFileName, 4094); + next = new dirent(wbuf); + FindFileData.dwReserved0 = FindNextFileA(hFind, &FindFileData); + return next; + } +}; + +/** + * @brief Create directory data struct element + * @param string directory path + * @return pointer to directory data struct element + */ +static DIR* opendir(const char* dirPath) { + auto dp = new DIR(dirPath); + if (!dp->isValid()) { + delete dp; + return nullptr; + } + return dp; +} + +/** + * @brief Walk throw directory data struct + * @param pointer to directory data struct + * @return pointer to directory data struct next element + */ +static struct dirent* readdir(DIR* dp) { + return dp->nextEnt(); +} + +/** + * @brief Remove directory data struct + * @param pointer to struct directory data + * @return void + */ +static void closedir(DIR* dp) { + delete dp; +} + + #ifdef WIN32_LEAN_AND_MEAN_UNDEF + #undef WIN32_LEAN_AND_MEAN + #undef WIN32_LEAN_AND_MEAN_UNDEF + #endif + + #ifdef NOMINMAX_UNDEF + #undef NOMINMAX_UNDEF + #undef NOMINMAX + #endif + +#else + + #include + #include + +#endif diff --git a/tools/mo/openvino/tools/mo/main.py b/tools/mo/openvino/tools/mo/main.py index b2d79017c60..208877049cc 100644 --- a/tools/mo/openvino/tools/mo/main.py +++ b/tools/mo/openvino/tools/mo/main.py @@ -28,13 +28,14 @@ from openvino.tools.mo.middle.pattern_match import for_graph_and_each_sub_graph_ from openvino.tools.mo.pipeline.common import prepare_emit_ir, get_ir_version from openvino.tools.mo.pipeline.unified import unified_pipeline from openvino.tools.mo.utils import import_extensions -from openvino.tools.mo.utils.cli_parser import check_available_transforms, get_caffe_cli_options, \ +from openvino.tools.mo.utils.cli_parser import check_available_transforms, \ + get_advanced_cli_options, get_caffe_cli_options, \ get_common_cli_options, get_freeze_placeholder_values, get_kaldi_cli_options, get_layout_values, \ get_mean_scale_dictionary, get_meta_info, get_model_name, get_mxnet_cli_options, get_onnx_cli_options, \ get_placeholder_shapes, get_tf_cli_options, get_tuple_values, parse_transform, parse_tuple_pairs from openvino.tools.mo.utils.error import Error, FrameworkError from openvino.tools.mo.utils.find_ie_version import find_ie_version -from openvino.tools.mo.utils.get_ov_update_message import get_ov_update_message +from openvino.tools.mo.utils.get_ov_update_message import get_ov_update_message, get_ov_api20_message from openvino.tools.mo.utils.guess_framework import deduce_legacy_frontend_by_namespace from openvino.tools.mo.utils.logger import init_logger, progress_printer from openvino.tools.mo.utils.model_analysis import AnalysisResults @@ -61,6 +62,7 @@ def print_argv(argv: argparse.Namespace, is_caffe: bool, is_tf: bool, is_mxnet: print('Model Optimizer arguments:') props = OrderedDict() props['common_args'] = get_common_cli_options(model_name) + props['advanced_args'] = get_advanced_cli_options() if is_caffe: props['caffe_args'] = get_caffe_cli_options() if is_tf: @@ -74,6 +76,7 @@ def print_argv(argv: argparse.Namespace, is_caffe: bool, is_tf: bool, is_mxnet: framework_specifics_map = { 'common_args': 'Common parameters:', + 'advanced_args': 'Advanced parameters:', 'caffe_args': 'Caffe specific parameters:', 'tf_args': 'TensorFlow specific parameters:', 'mxnet_args': 'MXNet specific parameters:', @@ -245,9 +248,6 @@ def arguments_post_parsing(argv: argparse.Namespace): # This is just to check that transform key is valid and transformations are available check_available_transforms(parse_transform(argv.transform)) - if argv.legacy_ir_generation and len(argv.transform) != 0: - raise Error("--legacy_ir_generation and --transform keys can not be used at the same time.") - # For C++ frontends there are no specific Python installation requirements, check only generic ones if moc_front_end: ret_code = check_requirements() @@ -434,13 +434,12 @@ def emit_ir(graph: Graph, argv: argparse.Namespace): return_code = "not executed" try: - if not argv.legacy_ir_generation: - from openvino.tools.mo.back.offline_transformations import apply_offline_transformations - apply_offline_transformations(orig_model_name, argv) - if "compress_fp16" in argv and argv.compress_fp16: - # restore data_type cmd parameter - argv.data_type = 'FP16' - return_code = 0 + from openvino.tools.mo.back.offline_transformations import apply_offline_transformations + apply_offline_transformations(orig_model_name, argv) + if "compress_fp16" in argv and argv.compress_fp16: + # restore data_type cmd parameter + argv.data_type = 'FP16' + return_code = 0 except Exception as e: return_code = "failed" log.error(e) @@ -523,11 +522,15 @@ def main(cli_parser: argparse.ArgumentParser, fem: FrontEndManager, framework: s argv.feManager = fem ov_update_message = None + ov_api20_message = None if not hasattr(argv, 'silent') or not argv.silent: ov_update_message = get_ov_update_message() + ov_api20_message = get_ov_api20_message() ret_code = driver(argv) if ov_update_message: print(ov_update_message) + if ov_api20_message and ret_code == 0: + print(ov_api20_message) telemetry.send_event('mo', 'conversion_result', 'success') telemetry.end_session('mo') telemetry.force_shutdown(1.0) diff --git a/tools/mo/openvino/tools/mo/middle/TensorIteratorCondition.py b/tools/mo/openvino/tools/mo/middle/TensorIteratorCondition.py index df98f2ca05f..cf739e5c360 100644 --- a/tools/mo/openvino/tools/mo/middle/TensorIteratorCondition.py +++ b/tools/mo/openvino/tools/mo/middle/TensorIteratorCondition.py @@ -5,6 +5,7 @@ import logging as log import numpy as np +from openvino.tools.mo.middle.pattern_match import apply_pattern from openvino.tools.mo.middle.TensorIterator_utils import delete_selects_from from openvino.tools.mo.ops.TensorIterator_ops import TensorIteratorCondition, TensorIteratorBackEdge from openvino.tools.mo.ops.identity import Identity @@ -66,134 +67,140 @@ Shape -> StridedSlice -> Enter -| LogicalAnd --> LoopCond (data) return [TensorIteratorMerge] @staticmethod - def pattern(): + def pattern(variation): log.debug('+++++++++++++++ ConditionMatching ++++++++++++++++') - return dict( - nodes=[ - ('Enter_1_less', dict(kind='op', op='Enter')), - ('Strided_slice', dict(kind='op', op='StridedSlice')), - ('Strided_slice_data', dict(kind='data')), - ('Enter_1_less_data', dict(kind='data')), + nodes = [ + ('Enter_1_less', dict(kind='op', op='Enter')), + ('Strided_slice', dict(kind='op', op='StridedSlice')), + ('Strided_slice_data', dict(kind='data')), + ('Enter_1_less_data', dict(kind='data')), - ('Less_1', dict(kind='op', op='Less')), - ('Merge_1', dict(kind='op', op='Merge')), - ('Merge_1_data', dict(kind='data')), - ('Less_1_data', dict(kind='data')), + ('Less_1', dict(kind='op', op='Less')), + ('Merge_1', dict(kind='op', op='Merge')), + ('Merge_1_data', dict(kind='data')), + ('Less_1_data', dict(kind='data')), - ('Less_2', dict(kind='op', op='Less')), - ('Merge_2', dict(kind='op', op='Merge')), - ('Merge_2_data', dict(kind='data')), - ('Less_2_data', dict(kind='data')), + ('Less_2', dict(kind='op', op='Less')), + ('Merge_2', dict(kind='op', op='Merge')), + ('Merge_2_data', dict(kind='data')), + ('Less_2_data', dict(kind='data')), + ('and', dict(kind='op', op='LogicalAnd')), + ('and_data', dict(kind='data')), + ('loop_cond', dict(kind='op', op='LoopCond')), + ('loop_cond_data', dict(kind='data')), + + ('init_1', dict(kind='op', op='Const')), + ('init_1_data', dict(kind='data')), + ('Enter_1', dict(kind='op', op='Enter')), + ('Enter_1_data', dict(kind='data')), + + ('init_2', dict(kind='op', op='Const')), + ('init_2_data', dict(kind='data')), + ('Enter_2', dict(kind='op', op='Enter')), + ('Enter_2_data', dict(kind='data')), + + ('Switch_1', dict(kind='op', op='Switch')), + ('Switch_1_data', dict(kind='data')), + ('Identity_1', dict(kind='op', op='Identity')), + ('Identity_1_data', dict(kind='data')), + ('add_1', dict(kind='op', op='Add')), + ('add_1_y', dict(kind='op', op='Const')), + ('add_1_y_data', dict(kind='data')), + ('add_1_data', dict(kind='data')), + ('NextIteration_1', dict(kind='op', op='NextIteration')), + + ('Switch_2', dict(kind='op', op='Switch')), + ('Switch_2_data', dict(kind='data')), + ('Identity_2', dict(kind='op', op='Identity')), + ('Identity_2_data', dict(kind='data')), + ('add_2', dict(kind='op', op='Add')), + ('add_2_y', dict(kind='op', op='Const')), + ('add_2_y_data', dict(kind='data')), + ('add_2_data', dict(kind='data')), + ('NextIteration_2', dict(kind='op', op='NextIteration')), + + ] + edges = [ + ('Strided_slice', 'Strided_slice_data'), + ('Strided_slice_data', 'Enter_1_less'), + ('Enter_1_less', 'Enter_1_less_data'), + ('Enter_1_less_data', 'Less_1'), + ('Less_1', 'Less_1_data'), + ('Less_1_data', 'and'), + + ('and', 'and_data'), + ('and_data', 'loop_cond'), + ('loop_cond', 'loop_cond_data'), + ('loop_cond_data', 'Switch_1'), + ('loop_cond_data', 'Switch_2'), + + ('init_1', 'init_1_data'), + ('init_1_data', 'Enter_1'), + ('Enter_1', 'Enter_1_data'), + ('Enter_1_data', 'Merge_1'), + ('Merge_1', 'Merge_1_data'), + ('Merge_1_data', 'Less_1'), + + ('Merge_1_data', 'Switch_1'), + ('Switch_1', 'Switch_1_data'), + ('Switch_1_data', 'Identity_1'), + ('Identity_1', 'Identity_1_data'), + ('Identity_1_data', 'add_1'), + ('add_1_y', 'add_1_y_data'), + ('add_1_y_data', 'add_1'), + ('add_1', 'add_1_data'), + ('add_1_data', 'NextIteration_1'), + + ('Merge_2_data', 'Switch_2'), + ('Switch_2', 'Switch_2_data'), + ('Switch_2_data', 'Identity_2'), + ('Identity_2', 'Identity_2_data'), + ('Identity_2_data', 'add_2'), + ('add_2_y', 'add_2_y_data'), + ('add_2_y_data', 'add_2'), + ('add_2', 'add_2_data'), + ('add_2_data', 'NextIteration_2'), + + ('init_2', 'init_2_data'), + ('init_2_data', 'Enter_2'), + ('Enter_2', 'Enter_2_data'), + ('Enter_2_data', 'Merge_2'), + + ('Merge_2', 'Merge_2_data'), + ('Merge_2_data', 'Less_2'), + ('Less_2', 'Less_2_data'), + ('Less_2_data', 'and'), + ] + if variation == 1: + nodes.extend([ ('Enter_2_less', dict(kind='op', op='Enter')), ('Enter_2_less_data', dict(kind='data')), - ('minimum_data', dict(kind='data')), - - ('and', dict(kind='op', op='LogicalAnd')), - ('and_data', dict(kind='data')), - ('loop_cond', dict(kind='op', op='LoopCond')), - ('loop_cond_data', dict(kind='data')), - - ('init_1', dict(kind='op', op='Const')), - ('init_1_data', dict(kind='data')), - ('Enter_1', dict(kind='op', op='Enter')), - ('Enter_1_data', dict(kind='data')), - - ('init_2', dict(kind='op', op='Const')), - ('init_2_data', dict(kind='data')), - ('Enter_2', dict(kind='op', op='Enter')), - ('Enter_2_data', dict(kind='data')), - - ('Switch_1', dict(kind='op', op='Switch')), - ('Switch_1_data', dict(kind='data')), - ('Identity_1', dict(kind='op', op='Identity')), - ('Identity_1_data', dict(kind='data')), - ('add_1', dict(kind='op', op='Add')), - ('add_1_y', dict(kind='op', op='Const')), - ('add_1_y_data', dict(kind='data')), - ('add_1_data', dict(kind='data')), - ('NextIteration_1', dict(kind='op', op='NextIteration')), - - ('Switch_2', dict(kind='op', op='Switch')), - ('Switch_2_data', dict(kind='data')), - ('Identity_2', dict(kind='op', op='Identity')), - ('Identity_2_data', dict(kind='data')), - ('add_2', dict(kind='op', op='Add')), - ('add_2_y', dict(kind='op', op='Const')), - ('add_2_y_data', dict(kind='data')), - ('add_2_data', dict(kind='data')), - ('NextIteration_2', dict(kind='op', op='NextIteration')), - - ], - edges=[ - ('Strided_slice', 'Strided_slice_data'), - ('Strided_slice_data', 'Enter_1_less'), - ('Enter_1_less', 'Enter_1_less_data'), - ('Enter_1_less_data', 'Less_1'), - ('Less_1', 'Less_1_data'), - ('Less_1_data', 'and'), - - ('and', 'and_data'), - ('and_data', 'loop_cond'), - ('loop_cond', 'loop_cond_data'), - ('loop_cond_data', 'Switch_1'), - ('loop_cond_data', 'Switch_2'), - - ('init_1', 'init_1_data'), - ('init_1_data', 'Enter_1'), - ('Enter_1', 'Enter_1_data'), - ('Enter_1_data', 'Merge_1'), - ('Merge_1', 'Merge_1_data'), - ('Merge_1_data', 'Less_1'), - - ('Merge_1_data', 'Switch_1'), - ('Switch_1', 'Switch_1_data'), - ('Switch_1_data', 'Identity_1'), - ('Identity_1', 'Identity_1_data'), - ('Identity_1_data', 'add_1'), - ('add_1_y', 'add_1_y_data'), - ('add_1_y_data', 'add_1'), - ('add_1', 'add_1_data'), - ('add_1_data', 'NextIteration_1'), - - ('Merge_2_data', 'Switch_2'), - ('Switch_2', 'Switch_2_data'), - ('Switch_2_data', 'Identity_2'), - ('Identity_2', 'Identity_2_data'), - ('Identity_2_data', 'add_2'), - ('add_2_y', 'add_2_y_data'), - ('add_2_y_data', 'add_2'), - ('add_2', 'add_2_data'), - ('add_2_data', 'NextIteration_2'), - + ('minimum_data', dict(kind='data')) + ]) + edges.extend([ ('minimum_data', 'Enter_2_less'), ('Enter_2_less', 'Enter_2_less_data'), ('Enter_2_less_data', 'Less_2'), - - ('init_2', 'init_2_data'), - ('init_2_data', 'Enter_2'), - ('Enter_2', 'Enter_2_data'), - ('Enter_2_data', 'Merge_2'), - - ('Merge_2', 'Merge_2_data'), - ('Merge_2_data', 'Less_2'), - ('Less_2', 'Less_2_data'), - ('Less_2_data', 'and'), - ], - ) + ]) + elif variation == 2: + edges.append(('Enter_1_less_data', 'Less_2')) + else: + raise Exception('Wrong pattern variation') + return dict(nodes=nodes, edges=edges) @staticmethod def looking_for_iteration_counter(graph: Graph, match: dict): types = ['TensorIteratorInput', 'TensorIteratorOutput'] - candidates = mo_array([match['Identity_1_data'], match['Identity_2_data']]) - results = mo_array([False for i in range(len(candidates))]) - for i, candidat in enumerate(candidates): - for node in candidat.out_nodes(): + candidates = [match['Identity_1_data'], match['Identity_2_data']] + results = [] + for candidate in candidates: + for node in candidate.out_nodes(): if node['op'] in types: - results[i] = True - assert not np.all(results) - assert sum(results) == 1 - return candidates[results == True][0] + results.append(candidate) + break + assert len(results) == 1 + return results[0] @staticmethod def check_dynamic_seq_len(graph: Graph, match: dict): @@ -201,11 +208,17 @@ Shape -> StridedSlice -> Enter -| LogicalAnd --> LoopCond (data) Cycle is dynamic if at least one of the boundaries isn't constant OR this boundaries is different from tensor shape. """ - dynamic_seq_len = match['Enter_1_less_data'].value is None or match['Enter_2_less_data'].value is None or \ - not np.array_equal(match['Enter_1_less_data'].value, match['Enter_2_less_data'].value) + dynamic_seq_len = match['Enter_1_less_data'].value is None + if 'Enter_2_less_data' in match: + dynamic_seq_len = dynamic_seq_len or match['Enter_2_less_data'].value is None or \ + not np.array_equal(match['Enter_1_less_data'].value, match['Enter_2_less_data'].value) return dynamic_seq_len + def find_and_replace_pattern(self, graph: Graph): + apply_pattern(graph, **self.pattern(1), action=self.replace_pattern) # pylint: disable=no-member + apply_pattern(graph, **self.pattern(2), action=self.replace_pattern) # pylint: disable=no-member + def replace_pattern(self, graph: Graph, match: dict): log.debug('================== ConditionFind ===============') # init_1 @@ -235,7 +248,11 @@ Shape -> StridedSlice -> Enter -| LogicalAnd --> LoopCond (data) condition_attrs = dict(time=dict(init=init_2, step=step_2), iter=dict(init=init_1, step=step_1), name=match['loop_cond'].name + '/TensorIteratorCondition_') condition = TensorIteratorCondition(graph, attrs=condition_attrs) - condition_data = condition.create_node_with_data(inputs=[match['Strided_slice_data'], match['minimum_data']], + if 'minimum_data' in match: + condition_inp = [match['Strided_slice_data'], match['minimum_data']] + else: + condition_inp = [match['Strided_slice_data']] + condition_data = condition.create_node_with_data(inputs=condition_inp, data_nodes=[loop_condition, iterator_data]) safe_nodes = ['loop_cond_data', 'Identity_1_data', 'Identity_2_data', 'Strided_slice', 'Strided_slice_data', diff --git a/tools/mo/openvino/tools/mo/middle/fusings.py b/tools/mo/openvino/tools/mo/middle/fusings.py index 238dfed3260..c313e6bfc2e 100644 --- a/tools/mo/openvino/tools/mo/middle/fusings.py +++ b/tools/mo/openvino/tools/mo/middle/fusings.py @@ -77,12 +77,11 @@ class Fusing(MiddleReplacementPattern): for_graph_and_each_sub_graph_recursively(graph, fuse_linear_ops) for_graph_and_each_sub_graph_recursively(graph, lambda G: G.clean_up()) - if not argv.disable_gfusing: - for_graph_and_each_sub_graph_recursively(graph, grouped_convolutions_fusing) + for_graph_and_each_sub_graph_recursively(graph, grouped_convolutions_fusing) + for_graph_and_each_sub_graph_recursively(graph, lambda G: G.clean_up()) + if not argv.disable_fusing: + for_graph_and_each_sub_graph_recursively(graph, fuse_linear_ops) for_graph_and_each_sub_graph_recursively(graph, lambda G: G.clean_up()) - if not argv.disable_fusing: - for_graph_and_each_sub_graph_recursively(graph, fuse_linear_ops) - for_graph_and_each_sub_graph_recursively(graph, lambda G: G.clean_up()) for_graph_and_each_sub_graph_recursively(graph, normalize_eltwise_inputs) for_graph_and_each_sub_graph_recursively(graph, lambda G: G.clean_up()) diff --git a/tools/mo/openvino/tools/mo/middle/passes/infer.py b/tools/mo/openvino/tools/mo/middle/passes/infer.py index e1116f34d17..97c263c8ecd 100644 --- a/tools/mo/openvino/tools/mo/middle/passes/infer.py +++ b/tools/mo/openvino/tools/mo/middle/passes/infer.py @@ -353,3 +353,20 @@ def reverse_infer(graph: Graph, nodes: list): log.debug('Inputs:') log_debug_dict(node.in_nodes(), 'inputs') + parameters_with_no_shape = [] + for node in graph.get_op_nodes(op='Parameter'): + if not node.has_valid('shape'): + parameters_with_no_shape.append(node) + + if len(parameters_with_no_shape) == 0: + return + + parameters_names = '' + for idx, node in enumerate(parameters_with_no_shape): + parameters_names += "'{}'".format(node.soft_get('name', node.id)) + if idx < len(parameters_with_no_shape) - 1: + parameters_names += ', ' + + if len(parameters_with_no_shape) > 0: + raise Error("Model Optimizer is unable to deduce input shapes for the following Parameter nodes: {}. " + "Please use cli options --input or --input_shape to set model input shape.".format(parameters_names)) diff --git a/tools/mo/openvino/tools/mo/ops/broadcast.py b/tools/mo/openvino/tools/mo/ops/broadcast.py index 746fe4df0fa..76426146f18 100644 --- a/tools/mo/openvino/tools/mo/ops/broadcast.py +++ b/tools/mo/openvino/tools/mo/ops/broadcast.py @@ -1,7 +1,7 @@ # Copyright (C) 2018-2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -from openvino.tools.mo.front.common.partial_infer.utils import is_fully_defined +from openvino.tools.mo.front.common.partial_infer.utils import is_fully_defined, shape_array, undefined_shape_of_rank from openvino.tools.mo.graph.graph import Node, Graph from openvino.tools.mo.graph.perm_inputs import PermuteInputs from openvino.tools.mo.ops.op import Op @@ -46,9 +46,16 @@ class Broadcast(Op): input_shape = node.in_port(0).data.get_shape() input_value = node.in_port(0).data.get_value() + target_shape_shape = node.in_port(1).data.get_shape() target_shape = node.in_port(1).data.get_value() - assert target_shape is not None, 'Output shape is not defined for node "{}"'.format(node_name) assert node.has_and_set('mode'), 'Broadcasting mode is not defined for node "{}"'.format(node_name) + # Dynamic target shape is possible to infer only if shape of target shape is static and 1D + if target_shape is None and len(target_shape_shape) == 1 and (len(input_shape) <= 1 or node.mode == 'explicit'): + assert is_fully_defined(target_shape_shape) + new_shape = undefined_shape_of_rank(target_shape_shape.item(0)) + node.out_port(0).data.set_shape(new_shape) + return + assert target_shape is not None, 'Output shape is not defined for node "{}"'.format(node_name) PermuteInputs().set_input_permutation(node.in_node(1), node, 'output:0', 'shape') diff --git a/tools/mo/openvino/tools/mo/utils/check_ie_bindings.py b/tools/mo/openvino/tools/mo/utils/check_ie_bindings.py index acc20097c14..22d3ac39c40 100644 --- a/tools/mo/openvino/tools/mo/utils/check_ie_bindings.py +++ b/tools/mo/openvino/tools/mo/utils/check_ie_bindings.py @@ -66,7 +66,7 @@ def import_core_modules(silent: bool, path_to_module: str): ie_version = str(get_version()) mo_version = str(v.get_version()) # pylint: disable=no-member,no-name-in-module - print("\t- {}: \t{}".format("OpenVINO runtime found in", os.path.dirname(openvino.__file__))) + print("{}: \t{}".format("OpenVINO runtime found in", os.path.dirname(openvino.__file__))) print("{}: \t{}".format("OpenVINO runtime version", ie_version)) print("{}: \t{}".format("Model Optimizer version", mo_version)) diff --git a/tools/mo/openvino/tools/mo/utils/cli_parser.py b/tools/mo/openvino/tools/mo/utils/cli_parser.py index 0b75e063fd4..29b2ff096dc 100644 --- a/tools/mo/openvino/tools/mo/utils/cli_parser.py +++ b/tools/mo/openvino/tools/mo/utils/cli_parser.py @@ -34,6 +34,13 @@ class DeprecatedStoreTrue(argparse.Action): setattr(namespace, self.dest, True) +class DeprecatedOptionCommon(argparse.Action): + def __call__(self, parser, args, values, option_string): + dep_msg = "Use of deprecated cli option {} detected. Option use in the following releases will be fatal. ".format(option_string) + log.error(dep_msg, extra={'is_warning': True}) + setattr(args, self.dest, values) + + class IgnoredAction(argparse.Action): def __init__(self, nargs=0, **kw): super().__init__(nargs=nargs, **kw) @@ -254,10 +261,8 @@ def get_common_cli_parser(parser: argparse.ArgumentParser = None): 'Shape is defined as a comma-separated list of integer numbers enclosed in ' 'parentheses or square brackets, for example [1,3,227,227] or (1,227,227,3), where ' 'the order of dimensions depends on the framework input layout of the model. ' - 'For example, [N,C,H,W] is used for Caffe* models and [N,H,W,C] for TensorFlow* ' - 'models. Model Optimizer performs necessary transformations to convert the shape to ' - 'the layout required by Inference Engine (N,C,H,W). The shape could contain ' - 'undefined dimensions (-1) and should fit the dimensions defined in the input ' + 'For example, [N,C,H,W] is used for ONNX* models and [N,H,W,C] for TensorFlow* ' + 'models. The shape can contain undefined dimensions (? or -1) and should fit the dimensions defined in the input ' 'operation of the graph. Boundaries of undefined dimension can be specified with ' 'ellipsis, for example [1,1..10,128,128]. One boundary can be undefined, for ' 'example [1,..100] or [1,3,1..,1..]. If there are multiple inputs in the model, ' @@ -371,23 +376,18 @@ def get_common_cli_parser(parser: argparse.ArgumentParser = None): 'Available transformations: "LowLatency2", "MakeStateful"', default="") common_group.add_argument('--disable_fusing', - help='Turn off fusing of linear operations to Convolution', + help='[DEPRECATED] Turn off fusing of linear operations to Convolution.', action=DeprecatedStoreTrue) common_group.add_argument('--disable_resnet_optimization', - help='Turn off resnet optimization', - action='store_true') + help='[DEPRECATED] Turn off ResNet optimization.', + action=DeprecatedStoreTrue, default=False) common_group.add_argument('--finegrain_fusing', - help='Regex for layers/operations that won\'t be fused. ' + - 'Example: --finegrain_fusing Convolution1,.*Scale.*') - common_group.add_argument('--disable_gfusing', - help='Turn off fusing of grouped convolutions', - action=DeprecatedStoreTrue) + help='[DEPRECATED] Regex for layers/operations that won\'t be fused. ' + + 'Example: --finegrain_fusing Convolution1,.*Scale.*', + action=DeprecatedOptionCommon) common_group.add_argument('--enable_concat_optimization', - help='Turn on Concat optimization.', - action='store_true') - common_group.add_argument('--move_to_preprocess', - help='Move mean values to IR preprocess section', - action=DeprecatedStoreTrue) + help='[DEPRECATED] Turn on Concat optimization.', + action=DeprecatedStoreTrue, default=False) # we use CanonicalizeDirCheckExistenceAction instead of readable_dirs to handle empty strings common_group.add_argument("--extensions", help="Directory or a comma separated list of directories with extensions. To disable all " @@ -417,19 +417,14 @@ def get_common_cli_parser(parser: argparse.ArgumentParser = None): 'It will be DEPRECATED in future releases. ' 'Use --input option to specify a value for freezing.', default=None) - common_group.add_argument('--generate_deprecated_IR_V7', - help=argparse.SUPPRESS, action=IgnoredAction, default=False) common_group.add_argument('--static_shape', help='Enables IR generation for fixed input shape (folding `ShapeOf` operations and ' 'shape-calculating sub-graphs to `Constant`). Changing model input shape using ' - 'the Inference Engine API in runtime may fail for such an IR.', + 'the OpenVINO Runtime API in runtime may fail for such an IR.', action='store_true', default=False) - common_group.add_argument('--keep_shape_ops', - help=argparse.SUPPRESS, - action=IgnoredAction, default=True) common_group.add_argument('--disable_weights_compression', - help='Disable compression and store weights with original precision.', - action='store_true', default=False) + help='[DEPRECATED] Disable compression and store weights with original precision.', + action=DeprecatedStoreTrue, default=False) common_group.add_argument('--progress', help='Enable model conversion progress display.', action='store_true', default=False) @@ -442,13 +437,16 @@ def get_common_cli_parser(parser: argparse.ArgumentParser = None): 'from the current directory, as absolute path or as a' 'relative path from the mo root directory', action=CanonicalizeTransformationPathCheckExistenceAction) - common_group.add_argument('--legacy_ir_generation', - help=argparse.SUPPRESS, action=DeprecatedStoreTrue, default=False) common_group.add_argument("--use_new_frontend", - help="Force the usage of new frontend API for model processing", + help='Force the usage of new Frontend of Model Optimizer for model conversion into IR. ' + 'The new Frontend is C++ based and is available for ONNX* and PaddlePaddle* models. ' + 'Model optimizer uses new Frontend for ONNX* and PaddlePaddle* by default that means ' + '`--use_new_frontend` and `--use_legacy_frontend` options are not specified.', action='store_true', default=False) common_group.add_argument("--use_legacy_frontend", - help="Force the usage of legacy API for model processing", + help='Force the usage of legacy Frontend of Model Optimizer for model conversion into IR. ' + 'The legacy Frontend is Python based and is available for TensorFlow*, ONNX*, MXNet*, ' + 'Caffe*, and Kaldi* models.', action='store_true', default=False) return parser @@ -471,14 +469,20 @@ def get_common_cli_options(model_name): d['scale'] = ['- Scale factor', lambda x: x if x else 'Not specified'] d['data_type'] = ['- Precision of IR', lambda x: 'FP32' if x == 'float' else 'FP16' if x == 'half' else x] d['disable_fusing'] = ['- Enable fusing', lambda x: not x] - d['disable_gfusing'] = ['- Enable grouped convolutions fusing', lambda x: not x] - d['move_to_preprocess'] = '- Move mean values to preprocess section' + d['transform'] = ['- User transformations', lambda x: x if x else 'Not specified'] d['reverse_input_channels'] = '- Reverse input channels' - d['use_legacy_frontend'] = '- Use legacy API for model processing' + d['static_shape'] = '- Enable IR generation for fixed input shape' d['transformations_config'] = '- Use the transformations config file' return d +def get_advanced_cli_options(): + d = OrderedDict() + d['use_legacy_frontend'] = '- Force the usage of legacy Frontend of Model Optimizer for model conversion into IR' + d['use_new_frontend'] = '- Force the usage of new Frontend of Model Optimizer for model conversion into IR' + return d + + def get_caffe_cli_options(): d = { 'input_proto': ['- Path to the Input prototxt', lambda x: x], @@ -657,7 +661,7 @@ def get_tf_cli_parser(parser: argparse.ArgumentParser = None): tf_group.add_argument('--disable_nhwc_to_nchw', help='[DEPRECATED] Disables the default translation from NHWC to NCHW. Since 2022.1 this option ' 'is deprecated and used only to maintain backward compatibility with previous releases.', - action='store_true') + action=DeprecatedStoreTrue, default=False) return parser @@ -996,8 +1000,14 @@ def parse_layouts_by_destination(s: str, parsed: dict, dest: str = None) -> None elif m2: found_g = m2.groups() else: - raise Error("More then one layout provided for --{}layout without providing name.".format( - dest + '_' if dest else '')) + error_msg = "Invalid usage of --{}layout parameter. Please use following syntax for each tensor " \ + "or operation name:" \ + "\n name(nchw)" \ + "\n name[n,c,h,w]".format(dest + '_' if dest else '') + if dest is None: + error_msg += "\n name(nhwc->[n,h,w,c])" \ + "\n name[n,h,w,c]->[n,c,h,w]" + raise Error(error_msg) write_found_layout(found_g[0], found_g[1], parsed, dest) diff --git a/tools/mo/openvino/tools/mo/utils/get_ov_update_message.py b/tools/mo/openvino/tools/mo/utils/get_ov_update_message.py index 92311409c73..388082ebac7 100644 --- a/tools/mo/openvino/tools/mo/utils/get_ov_update_message.py +++ b/tools/mo/openvino/tools/mo/utils/get_ov_update_message.py @@ -14,3 +14,13 @@ def get_ov_update_message(): link = 'https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit/download.html?cid=other&source=prod&campid=ww_2022_bu_IOTG_OpenVINO-2022-1&content=upg_all&medium=organic' return msg_fmt.format(link) if current_date >= expected_update_date else None + + +def get_ov_api20_message(): + link = "https://docs.openvino.ai" + message = '[ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework ' \ + 'input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, ' \ + 'please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11.\n' \ + 'Find more information about API v2.0 and IR v11 at {}'.format(link) + + return message diff --git a/tools/mo/requirements.txt b/tools/mo/requirements.txt index b672d62af54..7f5243164f5 100644 --- a/tools/mo/requirements.txt +++ b/tools/mo/requirements.txt @@ -1,5 +1,4 @@ -tensorflow~=2.5.0; python_version <= "3.6" -tensorflow~=2.5.3; python_version > "3.6" +tensorflow>=1.15.5,<2.6 mxnet~=1.2.0; sys_platform == 'win32' mxnet~=1.7.0.post2; sys_platform != 'win32' networkx~=2.5; python_version <= "3.6" diff --git a/tools/mo/requirements_tf.txt b/tools/mo/requirements_tf.txt index 6ba68c986cc..86a0c421c4d 100644 --- a/tools/mo/requirements_tf.txt +++ b/tools/mo/requirements_tf.txt @@ -1,5 +1,4 @@ -tensorflow~=2.5.0; python_version <= "3.6" -tensorflow~=2.5.3; python_version > "3.6" +tensorflow>=1.15.5,<2.6 networkx~=2.5; python_version <= "3.6" networkx~=2.6; python_version > "3.6" numpy>=1.16.6,<1.20 diff --git a/tools/mo/unit_tests/mo/front/common/partial_infer/eltwise_test.py b/tools/mo/unit_tests/mo/front/common/partial_infer/eltwise_test.py index 6b459682043..099e5c434f5 100644 --- a/tools/mo/unit_tests/mo/front/common/partial_infer/eltwise_test.py +++ b/tools/mo/unit_tests/mo/front/common/partial_infer/eltwise_test.py @@ -180,7 +180,7 @@ class TestElementwiseReverseInfer(unittest.TestCase): def test_reverse_infer_6(self): # both output and input has the same rank, cannot deduce other inputs rank - with self.assertRaisesRegex(Error, 'Stopped shape/value propagation'): + with self.assertRaisesRegex(Error, "Model Optimizer is unable to deduce input shapes"): self.build_and_test_reverse_inference(inp_shape_1=[dyn, dyn, dyn, dyn], inp_shape_2=None, out_shape=[dyn, dyn, 4, 1], diff --git a/tools/mo/unit_tests/mo/frontend_ngraph_test_actual.py b/tools/mo/unit_tests/mo/frontend_ngraph_test_actual.py index 8653190f1b3..97f2f2c6397 100644 --- a/tools/mo/unit_tests/mo/frontend_ngraph_test_actual.py +++ b/tools/mo/unit_tests/mo/frontend_ngraph_test_actual.py @@ -48,7 +48,6 @@ def replaceArgsHelper(log_level='DEBUG', model_name='abc', input_model='abc.test_mo_mock_mdl', transform=[], - legacy_ir_generation=False, scale=None, output=None, _input=None, @@ -67,7 +66,6 @@ def replaceArgsHelper(log_level='DEBUG', model_name=model_name, input_model=input_model, transform=transform, - legacy_ir_generation=legacy_ir_generation, scale=scale, output=output, input=_input, diff --git a/tools/mo/unit_tests/mo/middle/TensorIteratorCondition_test.py b/tools/mo/unit_tests/mo/middle/TensorIteratorCondition_test.py index 8a574abfdcd..a3313ff83ca 100644 --- a/tools/mo/unit_tests/mo/middle/TensorIteratorCondition_test.py +++ b/tools/mo/unit_tests/mo/middle/TensorIteratorCondition_test.py @@ -7,55 +7,85 @@ import numpy as np from openvino.tools.mo.middle.TensorIteratorCondition import LoopConditionMatcher from openvino.tools.mo.utils.ir_engine.compare_graphs import compare_graphs -from unit_tests.utils.graph import build_graph_with_attrs +from unit_tests.utils.graph import build_graph_with_attrs, regular_op_with_empty_data, connect, build_graph class TensorIteratorConditionTests(unittest.TestCase): - def test_not_dynamic(self): + def test_not_dynamic_1(self): pattern_matcher = LoopConditionMatcher() - pattern = pattern_matcher.pattern() + pattern = pattern_matcher.pattern(1) graph = build_graph_with_attrs(nodes_with_attrs=pattern['nodes'], edges_with_attrs=pattern['edges'], - new_nodes_with_attrs=[('maximum', {'kind': 'op', 'op': 'Maximum'}), - ('maximum_data', {'kind': 'data'}), - ('TensorIteratorInput', {'kind': 'op', 'op': 'TensorIteratorInput'})], - new_edges_with_attrs=[('maximum', 'maximum_data'), - ('Identity_1_data', 'TensorIteratorInput')], - update_nodes_attributes=[('init_1_data', {'value': np.array([0])}), - ('init_2_data', {'value': np.array([0])}), - ('add_1_y_data', {'value': np.array(1)}), - ('add_2_y_data', {'value': np.array(1)}), - ('loop_cond_data', {'value': None}), - ('Identity_2_data', {'value': None}, ), - ('Enter_1_less_data', {'value': None},), - ('Enter_2_less_data', {'value': None},), - ]) + new_nodes_with_attrs=[ + ('TensorIteratorInput', {'kind': 'op', 'op': 'TensorIteratorInput'})], + new_edges_with_attrs=[ + ('Identity_1_data', 'TensorIteratorInput')], + update_nodes_attributes=[ + ('init_1_data', {'value': np.array([0])}), + ('init_2_data', {'value': np.array([0])}), + ('add_1_y_data', {'value': np.array(1)}), + ('add_2_y_data', {'value': np.array(1)}), + ('loop_cond_data', {'value': None}), + ('Identity_2_data', {'value': None},), + ('Enter_1_less_data', {'value': None},), + ]) pattern_matcher.find_and_replace_pattern(graph) - graph_ref = build_graph_with_attrs( - nodes_with_attrs=[('TensorIteratorCondition', {'kind': 'op', 'op': 'TensorIteratorCondition'}), - ('loop_cond_data', {'kind': 'data'}), - ('identity_data', {'kind': 'data'}), - ('StridedSlice', {'kind': 'op', 'op':'StridedSlice'}), - ('StridedSlice_data', {'kind': 'data'}), - ('Maximum', {'kind': 'op', 'op': 'Maximum'}), - ('Maximum_data', {'kind': 'data'}), - ('minimum_data', {'kind': 'data'}), - ('TensorIteratorInput', {'kind': 'op', 'op': 'TensorIteratorInput'}) - ], - edges_with_attrs=[('Maximum', 'Maximum_data'), - ('StridedSlice', 'StridedSlice_data'), - ('StridedSlice_data', 'TensorIteratorCondition', {'in':0}), - ('minimum_data', 'TensorIteratorCondition', {'in':1}), - ('TensorIteratorCondition', 'loop_cond_data'), - ('TensorIteratorCondition', 'identity_data'), - ('identity_data', 'TensorIteratorInput'), - ], - update_edge_attrs=None, - new_nodes_with_attrs=[], - new_edges_with_attrs=[], - ) + nodes_attributes = { + **regular_op_with_empty_data('StridedSlice', {'op': 'StridedSlice', 'type': None}), + 'TensorIteratorCondition': {'kind': 'op', 'op': 'TensorIteratorCondition'}, + 'loop_cond_data': {'kind': 'data'}, + 'identity_data': {'kind': 'data'}, + 'minimum_data': {'kind': 'data'}, + 'TensorIteratorInput': {'kind': 'op', 'op': 'TensorIteratorInput'} + } + edges = [ + *connect('StridedSlice', '0:TensorIteratorCondition'), + ('minimum_data', 'TensorIteratorCondition', {'in':1}), + ('TensorIteratorCondition', 'loop_cond_data'), + ('TensorIteratorCondition', 'identity_data'), + ('identity_data', 'TensorIteratorInput') + ] + graph_ref = build_graph(nodes_attributes, edges) (flag, resp) = compare_graphs(graph, graph_ref, 'loop_cond_data', check_op_attrs=True) self.assertTrue(flag, resp) + def test_not_dynamic_2(self): + pattern_matcher = LoopConditionMatcher() + pattern = pattern_matcher.pattern(2) + graph = build_graph_with_attrs(nodes_with_attrs=pattern['nodes'], edges_with_attrs=pattern['edges'], + new_nodes_with_attrs=[ + ('TensorIteratorInput', {'kind': 'op', 'op': 'TensorIteratorInput'}), + ('some_op', {'kind': 'op', 'op': 'Add'})], + new_edges_with_attrs=[ + ('Identity_1_data', 'TensorIteratorInput'), + ('loop_cond_data', 'some_op'), + ], + update_nodes_attributes=[ + ('init_1_data', {'value': np.array([0])}), + ('init_2_data', {'value': np.array([0])}), + ('add_1_y_data', {'value': np.array(1)}), + ('add_2_y_data', {'value': np.array(1)}), + ('loop_cond_data', {'value': None}), + ('Identity_2_data', {'value': None},), + ('Enter_1_less_data', {'value': None},), + ]) + + pattern_matcher.find_and_replace_pattern(graph) + nodes_attributes = { + **regular_op_with_empty_data('loop_cond', {'op': 'TensorIteratorCondition', 'type': None}), + **regular_op_with_empty_data('StridedSlice', {'op': 'StridedSlice', 'type': None}), + 'some_op': {'kind': 'op', 'op': 'Add'}, + 'identity_data': {'kind': 'data'}, + 'TensorIteratorInput': {'kind': 'op', 'op': 'TensorIteratorInput'} + } + edges = [ + *connect('StridedSlice', 'loop_cond'), + *connect('loop_cond', 'some_op'), + ('loop_cond', 'identity_data'), + ('identity_data', 'TensorIteratorInput') + ] + graph_ref = build_graph(nodes_attributes, edges) + (flag, resp) = compare_graphs(graph, graph_ref, 'some_op', check_op_attrs=True) + self.assertTrue(flag, resp) diff --git a/tools/mo/unit_tests/mo/ops/broadcast_test.py b/tools/mo/unit_tests/mo/ops/broadcast_test.py index 55e8b99c72c..405411ffafb 100644 --- a/tools/mo/unit_tests/mo/ops/broadcast_test.py +++ b/tools/mo/unit_tests/mo/ops/broadcast_test.py @@ -6,7 +6,7 @@ import unittest import numpy as np from generator import generator, generate -from openvino.tools.mo.front.common.partial_infer.utils import int64_array +from openvino.tools.mo.front.common.partial_infer.utils import int64_array, undefined_shape_of_rank from openvino.tools.mo.graph.graph import Node from openvino.tools.mo.ops.broadcast import Broadcast from unit_tests.utils.graph import build_graph, valued_const_with_data, regular_op_with_empty_data, \ @@ -34,10 +34,10 @@ class BroadcastTest(unittest.TestCase): ([[3, 1]], [2, 1, 2], [-2, -1], 'explicit', [[[3, 1]], [[3, 1]]]), # ref_shape (2, 1, 2) ([[[9, 5, 7]], [[9, 5, 7]]], [2, 2, 1, 3], [1, 2, 3], 'explicit', # in_shape (2, 1, 3) - [[[[9, 5, 7]], [[9, 5, 7]]], [[[9, 5, 7]], [[9, 5, 7]]]]), # ref_out_shape (2, 2, 1, 3) + [[[[9, 5, 7]], [[9, 5, 7]]], [[[9, 5, 7]], [[9, 5, 7]]]]), # ref_out_shape (2, 2, 1, 3) - ([[[9, 5, 7]], [[3, 4, 8]]], [2, 1, 3, 3], [0, 1, 2], 'explicit', # in_shape (2, 1, 3) - [[[[9, 9, 9], [5, 5, 5], [7, 7, 7]]], [[[3, 3, 3], [4, 4, 4], [8, 8, 8]]]]), # ref_out_shape (2, 1, 3, 3) + ([[[9, 5, 7]], [[3, 4, 8]]], [2, 1, 3, 3], [0, 1, 2], 'explicit', # in_shape (2, 1, 3) + [[[[9, 9, 9], [5, 5, 5], [7, 7, 7]]], [[[3, 3, 3], [4, 4, 4], [8, 8, 8]]]]), # ref_out_shape (2, 1, 3, 3) # negative tests ([1], [2, 2], [0], 'explicit', None, True), @@ -76,3 +76,29 @@ class BroadcastTest(unittest.TestCase): self.assertTrue(np.array_equal(broadcast_node.out_node().value, np.array(ref_out))) else: self.assertTrue(np.array_equal(broadcast_node.out_node().shape, np.array(target_shape))) + + @generate(*[ + ([1], [3], 'numpy', undefined_shape_of_rank(3)), + ([1], [3], 'explicit', undefined_shape_of_rank(3)), + ([1, 2], [3], 'numpy', None, True), + ]) + def test_broadcast_dynamic(self, data, target_shape_shape, mode='numpy', ref_out_shape=None, test_raising=False): + nodes = { + **shaped_data('data', int64_array(data)), + **shaped_data('target_shape', int64_array(target_shape_shape)), + **regular_op_with_empty_data('broadcast', {'op': 'Broadcast', 'mode': mode}), + } + + edges = [('data', 'broadcast'), + ('target_shape', 'broadcast'), + ('broadcast', 'broadcast_d')] + + graph = build_graph(nodes, edges) + + broadcast_node = Node(graph, 'broadcast') + if test_raising: + self.assertRaises(AssertionError, Broadcast.infer, broadcast_node) + return + + Broadcast.infer(broadcast_node) + self.assertTrue(np.array_equal(broadcast_node.out_node().shape, ref_out_shape)) diff --git a/tools/mo/unit_tests/mo/utils/mo_fallback_test_actual.py b/tools/mo/unit_tests/mo/utils/mo_fallback_test_actual.py index 168c35a54ae..87297b7005c 100644 --- a/tools/mo/unit_tests/mo/utils/mo_fallback_test_actual.py +++ b/tools/mo/unit_tests/mo/utils/mo_fallback_test_actual.py @@ -34,7 +34,6 @@ def base_args_config(use_legacy_fe:bool=None, use_new_fe:bool=None): args.input_model = None args.silent = True args.transform=[] - args.legacy_ir_generation = False args.scale = None args.output=None args.input=None @@ -47,7 +46,6 @@ def base_args_config(use_legacy_fe:bool=None, use_new_fe:bool=None): args.transformations_config = None args.disable_fusing = None args.finegrain_fusing = None - args.disable_gfusing = None args.disable_resnet_optimization = None args.enable_concat_optimization = None args.static_shape = None diff --git a/tools/mo/unit_tests/mo/utils/test_mo_model_analysis_actual.py b/tools/mo/unit_tests/mo/utils/test_mo_model_analysis_actual.py index 9a8ec8a741d..19d7f782b84 100644 --- a/tools/mo/unit_tests/mo/utils/test_mo_model_analysis_actual.py +++ b/tools/mo/unit_tests/mo/utils/test_mo_model_analysis_actual.py @@ -29,7 +29,6 @@ def base_args_config(): args.input_model = None args.silent = True args.transform=[] - args.legacy_ir_generation = False args.scale = None args.output=None args.input=None @@ -42,7 +41,6 @@ def base_args_config(): args.transformations_config = None args.disable_fusing = None args.finegrain_fusing = None - args.disable_gfusing = None args.disable_resnet_optimization = None args.enable_concat_optimization = None args.static_shape = None diff --git a/tools/pot/configs/data_free_mode_template.json b/tools/pot/configs/data_free_mode_template.json deleted file mode 100644 index 8c621fb7b6b..00000000000 --- a/tools/pot/configs/data_free_mode_template.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "model": { - "model_name": "model_name", // Model name - "model": "", // Path to model (.xml format) - "weights": "" // Path to weights (.bin format) - }, - "engine": { - - "type": "data_free", // Engine type​ - "generate_data": "True", // (Optional) If True, generate synthetic data and store to `data_source`​ - // Otherwise, the dataset from `--data-source` will be used'​ - "layout": "NCHW", // (Optional) Layout of input data. Supported: ["NCHW", "NHWC", "CHW", "CWH"]​ - "shape": "[None, None, None, None]", // (Optional) if model has dynamic shapes, input shapes must be provided​ - "data_type": "image", // (Optional) You can specify the type of data to be generated.​ - // Currently only `image` is supported.​ - // It is planned to add 'text` and 'audio' cases​ - "data_source": "PATH_TO_SOURCE" // (Optional) You can specify path to directory​ - // where synthetic dataset is located or will be generated and saved​ - }, - "compression": { - "algorithms": [ - { - "name": "DefaultQuantization", // Optimization algorithm name - "params": { - "preset": "performance", // Preset [performance, mixed, accuracy] which control the quantization - // mode (symmetric, mixed (weights symmetric and activations asymmetric) - // and fully asymmetric respectively) - "stat_subset_size": 300 // Size of subset to calculate activations statistics that can be used - // for quantization parameters calculation - } - } - ] - } -} diff --git a/tools/pot/openvino/tools/pot/algorithms/quantization/utils.py b/tools/pot/openvino/tools/pot/algorithms/quantization/utils.py index 365dc814347..94bf92e6009 100644 --- a/tools/pot/openvino/tools/pot/algorithms/quantization/utils.py +++ b/tools/pot/openvino/tools/pot/algorithms/quantization/utils.py @@ -293,7 +293,8 @@ def get_tensor_statistics(range_estimator_config, for_weights, **kwargs): stat_mod_name = get_stat_name_by_config(range_estimator_config, stats_name) if fn_type in ['quantile', 'abs_quantile']: q_value = range_estimator_config[stats_name]['outlier_prob'] - ts_args['inplace_statistics'] = False + if not for_weights: + ts_args['inplace_statistics'] = False if stats_name == 'max': q_value = 1 - q_value ts_args.update({'q': q_value}) diff --git a/tools/pot/openvino/tools/pot/api/README.md b/tools/pot/openvino/tools/pot/api/README.md index 728711284f3..06b9af6a165 100644 --- a/tools/pot/openvino/tools/pot/api/README.md +++ b/tools/pot/openvino/tools/pot/api/README.md @@ -52,7 +52,7 @@ The POT Python* API for model optimization can be used in the following cases: - [Accuracy Checker](@ref omz_tools_accuracy_checker) tool does not support the model or dataset. - POT does not support the model in the [Simplified Mode](@ref pot_docs_BestPractices) or produces the optimized model with low accuracy in this mode. -- You already have the Python* script to validate the accuracy of the model using the [OpenVINO™ Runtime](@ref openvino_docs_OV_Runtime_User_Guide). +- You already have the Python* script to validate the accuracy of the model using the [OpenVINO™ Runtime](@ref openvino_docs_OV_Runtime_User_Guide). ## Examples diff --git a/tools/pot/openvino/tools/pot/api/metric.py b/tools/pot/openvino/tools/pot/api/metric.py index f5cb021cd2c..bff089f5b64 100644 --- a/tools/pot/openvino/tools/pot/api/metric.py +++ b/tools/pot/openvino/tools/pot/api/metric.py @@ -12,9 +12,10 @@ class Metric(ABC): self.reset() @property - @abstractmethod def value(self): """ Returns accuracy metric value for the last model output. """ + raise Exception('The value() property should be implemented to use this metric ' + 'with AccuracyAwareQuantization algorithm!') @property @abstractmethod diff --git a/tools/pot/openvino/tools/pot/api/samples/3d_segmentation/3d_segmentation_sample.py b/tools/pot/openvino/tools/pot/api/samples/3d_segmentation/3d_segmentation_sample.py index 2056d7e215a..4d0906f68fc 100644 --- a/tools/pot/openvino/tools/pot/api/samples/3d_segmentation/3d_segmentation_sample.py +++ b/tools/pot/openvino/tools/pot/api/samples/3d_segmentation/3d_segmentation_sample.py @@ -38,9 +38,8 @@ class BRATSDataLoader(DataLoader): mask_path = os.path.join(self.config.mask_dir, self._img_ids[index]) image_path = os.path.join(self.config.data_source, self._img_ids[index]) - annotation = (index, self._read_image(mask_path)) image, image_meta = self._preprocess_image(self._read_image(image_path)) - return annotation, image, image_meta + return image, self._read_image(mask_path), image_meta def __len__(self): """ Returns size of the dataset """ @@ -120,13 +119,6 @@ class DiceIndex(Metric): self._name = 'dice_index' self._overall_metric = [] - @property - def value(self): - """ Returns accuracy metric value for the last model output. - Possible format: {metric_name: [metric_values_per_image]} - """ - return {self._name: [np.mean(self._overall_metric[-1])]} - @property def avg_value(self): """ Returns accuracy metric value for all model outputs. @@ -195,6 +187,7 @@ class SegmentationEngine(IEEngine): processed_outputs = [] for output, meta in zip(outputs.values(), metadata): # Resize to bounding box size and extend to mask size + output = output[0] low = meta['bbox'][0] high = meta['bbox'][1] box_shape = tuple((high - low).astype(np.int32)) diff --git a/tools/pot/openvino/tools/pot/api/samples/classification/classification_sample.py b/tools/pot/openvino/tools/pot/api/samples/classification/classification_sample.py index 4cc2b7db1d5..33db1bdf51f 100644 --- a/tools/pot/openvino/tools/pot/api/samples/classification/classification_sample.py +++ b/tools/pot/openvino/tools/pot/api/samples/classification/classification_sample.py @@ -29,9 +29,8 @@ class ImageNetDataLoader(DataLoader): if index >= len(self): raise IndexError - annotation = (index, self._annotations[self._img_ids[index]])\ - if self._annotations else (index, None) - return annotation, self._read_image(self._img_ids[index]) + annotation = self._annotations[self._img_ids[index]] if self._annotations else None + return self._read_image(self._img_ids[index]), annotation # Methods specific to the current implementation @staticmethod @@ -84,11 +83,6 @@ class Accuracy(Metric): self._name = 'accuracy@top{}'.format(self._top_k) self._matches = [] - @property - def value(self): - """ Returns accuracy metric value for the last model output. """ - return {self._name: self._matches[-1]} - @property def avg_value(self): """ Returns accuracy metric value for all model outputs. """ diff --git a/tools/pot/openvino/tools/pot/api/samples/face_detection/face_detection_sample.py b/tools/pot/openvino/tools/pot/api/samples/face_detection/face_detection_sample.py index c22adefbd9c..c7051ba090e 100644 --- a/tools/pot/openvino/tools/pot/api/samples/face_detection/face_detection_sample.py +++ b/tools/pot/openvino/tools/pot/api/samples/face_detection/face_detection_sample.py @@ -6,12 +6,18 @@ from argparse import ArgumentParser from functools import partial from time import time +import copy import cv2 import numpy as np +from openvino.runtime import PartialShape # pylint: disable=E0611,E0401 from openvino.tools.pot import Metric, DataLoader, IEEngine, \ load_model, compress_model_weights, create_pipeline from openvino.tools.pot.graph.model_utils import add_outputs +from openvino.tools.pot.samplers.batch_sampler import BatchSampler +from openvino.tools.pot.engines.utils import process_accumulated_stats, \ + restore_original_node_names, align_stat_names_with_results, \ + add_tensor_names, collect_model_outputs from openvino.tools.pot.utils.logger import init_logger, get_logger from openvino.tools.pot.api.samples.face_detection import utils @@ -28,7 +34,7 @@ class WiderFaceLoader(DataLoader): def __init__(self, config): super().__init__(config) self._min_height_ann = 60 - self._img_ids, self._annotations = self._read_image_ids_annotations(config.annotation_file) + self._img_ids, self._annotations = self._read_image_ids_annotations(self.config.annotation_file) def __getitem__(self, index): """ @@ -40,8 +46,7 @@ class WiderFaceLoader(DataLoader): if index >= len(self): raise IndexError - annotation = (index, self._annotations[self._img_ids[index]]) - return annotation, self._read_image(self._img_ids[index]) + return self._read_image(self._img_ids[index]), self._annotations[self._img_ids[index]] def __len__(self): """ Returns size of the dataset """ @@ -98,14 +103,68 @@ class MTCNNEngine(IEEngine): self._model = self._set_model(model) self._output_layers = {} stage_names = ['pnet', 'rnet', 'onet'] - for stage, model_dict in enumerate(model.models): + for stage, _ in enumerate(model.models): self._output_layers[stage_names[stage]] = { - 'probabilities': model_dict['name'] + '_' + self.config['outputs']['probabilities'][stage], - 'regions': model_dict['name'] + '_' + self.config['outputs']['regions'][stage], + 'probabilities': self.config['outputs']['probabilities'][stage], + 'regions': self.config['outputs']['regions'][stage], } def _add_outputs(self, nodes_name): - add_outputs(self._model, nodes_name) + return add_outputs(self._model, nodes_name) + + def predict(self, stats_layout=None, sampler=None, stat_aliases=None, + metric_per_sample=False, print_progress=False): + stat_names_aliases = None + if sampler is None: + sampler = BatchSampler(self.data_loader) + if stats_layout: + model_with_stat_op, nodes_names_map, output_to_node_names = self._statistic_graph_builder. \ + insert_statistic(copy.deepcopy(self._nx_model), + stats_layout, stat_aliases) + self.set_model(model_with_stat_op) + + nodes_name = [] + for names_map in nodes_names_map.values(): + nodes_name.extend(list(names_map.keys())) + + outputs = self._add_outputs(nodes_names_map) + for model_name, outputs_data in outputs.items(): + add_tensor_names(outputs_data, nodes_names_map[model_name].keys()) + + model_output_names = [] + for model in self._model: + model_output_names.extend(collect_model_outputs(model['model'])) + + align_stat_names_with_results(model_output_names, + nodes_name, + output_to_node_names, + stats_layout, + stat_aliases) + + # Creating statistics layout with IE-like names + stats_layout, stat_names_aliases = self._convert_stats_names(stats_layout) + + self._predict(stats_layout=stats_layout, + sampler=sampler, + print_progress=print_progress, + need_metrics_per_sample=metric_per_sample) + + accumulated_stats = \ + process_accumulated_stats(stat_names_aliases=stat_names_aliases, + accumulated_stats=self._accumulated_layer_stats) + + if stats_layout: + restore_original_node_names(output_to_node_names, accumulated_stats, stats_layout, stat_aliases) + + metrics = None + if self._metric: + metrics = self._metric.avg_value + if metric_per_sample: + metrics = (sorted(self._per_sample_metrics, key=lambda i: i['sample_id']), metrics) + + self._reset() + + return metrics, accumulated_stats def _predict(self, stats_layout, sampler, print_progress=False, need_metrics_per_sample=False): @@ -142,12 +201,12 @@ class MTCNNEngine(IEEngine): progress_log_fn('Inference finished') def _infer(self, data, ie_network, stats_collect_callback=None): + ie_network.reshape(PartialShape(data.shape)) filled_input = self._fill_input(ie_network, data) - input_shapes = {layer_name: data.shape for layer_name, data in filled_input.items()} - ie_network.reshape(input_shapes) - exec_model = self._ie.load_network(network=ie_network, - device_name=self.config.device) - result = exec_model.infer(filled_input) + compiled_model = self._ie.compile_model(model=ie_network, + device_name=self.config.device) + infer_request = compiled_model.create_infer_request() + result = infer_request.infer(filled_input) # Collect statistics if stats_collect_callback: stats_collect_callback(self._transform_for_callback(result)) @@ -173,9 +232,12 @@ class MTCNNEngine(IEEngine): total_boxes = np.zeros((0, 9), np.float) for idx, outputs in enumerate(output): scales = input_meta['scales'][idx] + mapping = outputs[[i for i, _ in outputs.items() + if i.any_name == self._output_layers['pnet']['probabilities']][0]][0, 1] + + regions = outputs[[i for i, _ in outputs.items() + if i.any_name == self._output_layers['pnet']['regions']][0]][0] - mapping = outputs[self._output_layers['pnet']['probabilities']][0, 1] - regions = outputs[self._output_layers['pnet']['regions']][0] boxes = utils.generate_bounding_box(mapping, regions, scales, 0.6) if len(boxes) != 0: pick = utils.nms(boxes, 0.5) @@ -207,8 +269,9 @@ class MTCNNEngine(IEEngine): return np.transpose(img, [0, 3, 2, 1]) def postprocess(output): - score = output[self._output_layers['rnet']['probabilities']][:, 1] - regions = output[self._output_layers['rnet']['regions']] + score = output[[i for i, _ in output.items() + if i.any_name == self._output_layers['rnet']['probabilities']][0]][:, 1] + regions = output[[i for i, _ in output.items() if i.any_name == self._output_layers['rnet']['regions']][0]] return utils.calibrate_bboxes(prev_stage_output, score, regions, nms_type='union') ie_network = self._model[1]['model'] @@ -223,8 +286,9 @@ class MTCNNEngine(IEEngine): return np.transpose(img, [0, 3, 2, 1]) def postprocess(output): - score = output[self._output_layers['onet']['probabilities']][:, 1] - regions = output[self._output_layers['onet']['regions']] + score = output[[i for i, _ in output.items() + if i.any_name == self._output_layers['onet']['probabilities']][0]][:, 1] + regions = output[[i for i, _ in output.items() if i.any_name == self._output_layers['onet']['regions']][0]] bboxes = utils.calibrate_bboxes(prev_stage_output, score, regions) pick = utils.nms(bboxes, 0.7, 'min') bboxes_to_remove = np.setdiff1d(np.arange(len(bboxes)), pick) @@ -247,15 +311,6 @@ class Recall(Metric): self._n_recorded_faces = [] self._n_total_preds = [] - @property - def value(self): - """ Returns metric value for the last model output. - Possible format: {metric_name: [metric_values_per_image]} - """ - tp = np.cumsum(self._true_positives[-1])[np.arange(self._n_total_preds[-1])] - recalls = tp / np.maximum(self._n_recorded_faces[-1], np.finfo(np.float64).eps) - return {self._name: [recalls[-1]]} - @property def avg_value(self): """ Returns average metric value for all model outputs. diff --git a/tools/pot/openvino/tools/pot/api/samples/object_detection/data_loader.py b/tools/pot/openvino/tools/pot/api/samples/object_detection/data_loader.py index 0a400fd9919..3c2856d794d 100644 --- a/tools/pot/openvino/tools/pot/api/samples/object_detection/data_loader.py +++ b/tools/pot/openvino/tools/pot/api/samples/object_detection/data_loader.py @@ -12,8 +12,8 @@ from openvino.tools.pot import DataLoader class COCOLoader(DataLoader): def __init__(self, config): super().__init__(config) - self.images_path = config.images_path - self.annotation_path = config.annotation_path + self.images_path = self.config.images_path + self.annotation_path = self.config.annotation_path self.images = os.listdir(self.images_path) self.labels = None self.data, self.bbox = self.prepare_annotation() @@ -61,8 +61,8 @@ class COCOLoader(DataLoader): annotation = {'boxes': bbox, 'labels': labels, 'iscrowd': iscrowd, 'x_maxs': x_maxs, 'x_mins': x_mins, 'y_maxs': y_maxs, 'y_mins': y_mins} - annotation = (index, [annotation, shape_image]) - return annotation, self._read_and_preprocess_image(self.images_path + self.data[index]['file_name']) + annotation = [annotation, shape_image] + return self._read_and_preprocess_image(self.images_path + self.data[index]['file_name']), annotation def __len__(self): return len(self.images) diff --git a/tools/pot/openvino/tools/pot/api/samples/segmentation/segmentation_sample.py b/tools/pot/openvino/tools/pot/api/samples/segmentation/segmentation_sample.py index 0eacbe8937d..ec20ee4615a 100644 --- a/tools/pot/openvino/tools/pot/api/samples/segmentation/segmentation_sample.py +++ b/tools/pot/openvino/tools/pot/api/samples/segmentation/segmentation_sample.py @@ -33,8 +33,8 @@ class VOCSegmentationLoader(DataLoader): # Required methods: def __init__(self, config): super().__init__(config) - self._image_size = config.image_size - self._img_ids = self._read_img_ids(config) + self._image_size = self.config.image_size + self._img_ids = self._read_img_ids(self.config) def __getitem__(self, index): """ @@ -49,8 +49,7 @@ class VOCSegmentationLoader(DataLoader): mask_path = os.path.join(self.config.mask_dir, self._img_ids[index] + '.png') image_path = os.path.join(self.config.data_source, self._img_ids[index] + '.jpg') - annotation = (index, self._read_and_preprocess_mask(mask_path)) - return annotation, self._read_and_preprocess_image(image_path) + return self._read_and_preprocess_image(image_path), self._read_and_preprocess_mask(mask_path) def __len__(self): """ Returns size of the dataset """ @@ -93,13 +92,6 @@ class MeanIOU(Metric): self._current_cm = [] self._total_cm = np.zeros((self._classes_num, self._classes_num)) - @property - def value(self): - """ Returns metric value for the last model output. - Possible format: {metric_name: [metric_values_per_image]} - """ - return {self._name: [self._evaluate(cm) for cm in self._current_cm]} - @property def avg_value(self): """ Returns average metric value for all model outputs. diff --git a/tools/pot/openvino/tools/pot/api/samples/speech/README.md b/tools/pot/openvino/tools/pot/api/samples/speech/README.md index d25bc863b98..d149cedcf41 100644 --- a/tools/pot/openvino/tools/pot/api/samples/speech/README.md +++ b/tools/pot/openvino/tools/pot/api/samples/speech/README.md @@ -1,6 +1,6 @@ # API usage sample for speech task on GNA {#pot_sample_speech_README} -This sample demonstrates the use of the [Post-training Optimization Tool API](@ref pot_compression_api_README) for the task of quantizing a speech model for [GNA](@ref openvino_docs_IE_DG_supported_plugins_GNA) device. +This sample demonstrates the use of the [Post-training Optimization Tool API](@ref pot_compression_api_README) for the task of quantizing a speech model for [GNA](@ref openvino_docs_OV_UG_supported_plugins_GNA) device. Quantization for GNA is different from CPU quantization due to device specific: GNA supports quantized inputs in INT16 and INT32 (for activations) precision and quantized weights in INT8 and INT16 precision. This sample contains pre-selected quantization options based on the DefaultQuantization algorithm and created for models from [Kaldi](http://kaldi-asr.org/doc/) framework, and its data format. diff --git a/tools/pot/openvino/tools/pot/app/argparser.py b/tools/pot/openvino/tools/pot/app/argparser.py index bd919246e88..d5f8fdf8f77 100644 --- a/tools/pot/openvino/tools/pot/app/argparser.py +++ b/tools/pot/openvino/tools/pot/app/argparser.py @@ -50,7 +50,7 @@ def get_common_argument_parser(): parser.add_argument( '--engine', - choices=['accuracy_checker', 'data_free', 'simplified'], + choices=['accuracy_checker', 'simplified'], type=str, help='Engine type. Default: `accuracy_checker`') @@ -114,35 +114,7 @@ def get_common_argument_parser(): parser.add_argument( '--data-source', - help='Valid for DataFree and Simplified modes. For Simplified mode path to dataset dir is required. ' - 'For DataFree mode specify path to directory ' - 'where syntetic dataset is located or will be generated and saved. ' - 'For DataFree mode default: `./pot_dataset`') - - data_free_opt = parser.add_argument_group('DataFree mode options') - - data_free_opt.add_argument( - '--shape', - type=str, - help='Required for models with dynamic shapes. ' - 'Input shape that should be fed to an input node of the model. ' - 'Shape is defined as a comma-separated list of integer numbers enclosed in ' - 'parentheses or square brackets, for example [1,3,227,227] or (1,227,227,3), where ' - 'the order of dimensions depends on the framework input layout of the model.') - - data_free_opt.add_argument( - '--data-type', - type=str, - default='image', - choices=['image'], - help='Type of data for generation. Dafault: `image`') - - data_free_opt.add_argument( - '--generate-data', - action='store_true', - default=False, - help='If specified, generate synthetic data and store to `data-source`. ' - 'Otherwise, the dataset from `--data-source` will be used') + help='Valid only for Simplified modes. Path to dataset dir is required.') return parser @@ -160,7 +132,7 @@ def check_dependencies(args): (args.engine == 'accuracy_checker' or args.engine is None)): raise ValueError( '--quantize option requires AC config to be specified ' - 'or --engine should be `data_free` or `simplified`.') + 'or --engine should be `simplified`.') if args.quantize == 'accuracy_aware' and args.max_drop is None: raise ValueError('For AccuracyAwareQuantization --max-drop should be specified') if args.config is None and args.engine == 'simplified' and args.data_source is None: diff --git a/tools/pot/openvino/tools/pot/app/run.py b/tools/pot/openvino/tools/pot/app/run.py index f7677237d5d..3c3f2694c22 100644 --- a/tools/pot/openvino/tools/pot/app/run.py +++ b/tools/pot/openvino/tools/pot/app/run.py @@ -39,15 +39,13 @@ def app(argv): if args.engine: config.engine['type'] = args.engine if args.engine else 'accuracy_checker' if 'data_source' not in config.engine: - if args.data_source is None and config.engine.type == 'data_free': - args.data_source = 'pot_dataset' config.engine['data_source'] = args.data_source config.configure_params(args.ac_config) config.update_from_args(args) if config.engine.type != 'accuracy_checker' and args.evaluate: - raise Exception('Can not make evaluation in simplified or data_free mode') + raise Exception('Can not make evaluation in simplified mode') log_dir = _create_log_path(config) init_logger(level=args.log_level, diff --git a/tools/pot/openvino/tools/pot/configs/config.py b/tools/pot/openvino/tools/pot/configs/config.py index f97af55e656..72724cfbd6b 100644 --- a/tools/pot/openvino/tools/pot/configs/config.py +++ b/tools/pot/openvino/tools/pot/configs/config.py @@ -63,19 +63,6 @@ class Config(Dict): self.model['output_dir'] = args.output_dir self.model['direct_dump'] = args.direct_dump self.engine['evaluate'] = args.evaluate - if self.engine.type == 'data_free': - if 'data_type' not in self.engine: - self.engine['data_type'] = args.data_type - if 'generate_data' not in self.engine: - self.engine['generate_data'] = args.generate_data - if 'shape' not in self.engine: - self.engine['shape'] = args.shape - if self.engine['generate_data']: - subset_size = 0 - for algo in self.compression['algorithms']: - subset_size = max(subset_size, algo.get('stat_subset_size', 300)) - self.engine['subset_size'] = subset_size - self.model['keep_uncompressed_weights'] = args.keep_uncompressed_weights if 'optimizer' in self: self.optimizer.params['keep_uncompressed_weights'] = args.keep_uncompressed_weights @@ -309,9 +296,9 @@ class Config(Dict): if 'type' not in engine or engine.type == 'accuracy_checker': self._configure_ac_params() self.engine.type = 'accuracy_checker' - elif engine.type == 'simplified' or engine.type == 'data_free': + elif engine.type == 'simplified': if engine.data_source is None: - raise KeyError(f'Missed data dir for {engine.type} engine') + raise KeyError('Missed data dir for simplified engine') self.engine.device = engine.device if engine.device else 'CPU' engine.data_source = Path(engine.data_source) else: diff --git a/tools/pot/openvino/tools/pot/data_loaders/creator.py b/tools/pot/openvino/tools/pot/data_loaders/creator.py index bdce5a3ccb0..54cdf5935da 100644 --- a/tools/pot/openvino/tools/pot/data_loaders/creator.py +++ b/tools/pot/openvino/tools/pot/data_loaders/creator.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 from openvino.tools.pot.data_loaders.image_loader import ImageLoader -from openvino.tools.pot.data_loaders.synthetic_image_loader import SyntheticImageLoader from openvino.tools.pot.graph.model_utils import get_nodes_by_type @@ -25,14 +24,9 @@ def create_data_loader(config, model): data_loader = None for in_node in inputs: if tuple(in_node.shape) != (1, 3): - if config.type == 'simplified': - data_loader = ImageLoader(config) - data_loader.shape = in_node.shape - data_loader.get_layout(in_node) - elif config.type == 'data_free': - if not config.shape: - config.shape = in_node.shape - data_loader = SyntheticImageLoader(config) + data_loader = ImageLoader(config) + data_loader.shape = in_node.shape + data_loader.get_layout(in_node) return data_loader if data_loader is None: diff --git a/tools/pot/openvino/tools/pot/data_loaders/image_loader.py b/tools/pot/openvino/tools/pot/data_loaders/image_loader.py index 54e90950403..2a0f16fe118 100644 --- a/tools/pot/openvino/tools/pot/data_loaders/image_loader.py +++ b/tools/pot/openvino/tools/pot/data_loaders/image_loader.py @@ -64,9 +64,6 @@ class ImageLoader(DataLoader): layout_from_ir = get_layout_values(input_node.graph.meta_data.get('layout', None)) if layout_from_ir is not None: layout_from_ir = layout_from_ir[next(iter(layout_from_ir))].get('source_layout', None) - # SyntheticImageLoader uses only H,W,C dimensions - if self._shape is not None and 'N' in layout_from_ir and len(self._shape) == 3: - layout_from_ir = layout_from_ir[1:] self._layout = Layout(layout_from_ir) return diff --git a/tools/pot/openvino/tools/pot/data_loaders/synthetic_background.npy b/tools/pot/openvino/tools/pot/data_loaders/synthetic_background.npy deleted file mode 100644 index 28da19dee25..00000000000 Binary files a/tools/pot/openvino/tools/pot/data_loaders/synthetic_background.npy and /dev/null differ diff --git a/tools/pot/openvino/tools/pot/data_loaders/synthetic_image_loader.py b/tools/pot/openvino/tools/pot/data_loaders/synthetic_image_loader.py deleted file mode 100644 index 9fb7e2ed41e..00000000000 --- a/tools/pot/openvino/tools/pot/data_loaders/synthetic_image_loader.py +++ /dev/null @@ -1,327 +0,0 @@ -# Copyright (C) 2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -from multiprocessing import Pool -from pathlib import Path -import os -import re -import requests - -import cv2 as cv -import numpy as np - -from openvino.runtime import Layout # pylint: disable=E0611,E0401 -from openvino.tools.pot.utils.logger import get_logger -from openvino.tools.pot.data_loaders.image_loader import ImageLoader -from .utils import collect_img_files - -logger = get_logger(__name__) - - -class IFSFunction: - def __init__(self, prev_x, prev_y): - self.function = [] - self.xs, self.ys = [prev_x], [prev_y] - self.select_function = [] - self.cum_proba = 0.0 - - def set_param(self, params, proba, weights=None): - if weights is not None: - params = list(np.array(params) * np.array(weights)) - - self.function.append(params) - self.cum_proba += proba - self.select_function.append(self.cum_proba) - - def calculate(self, iteration): - rand = np.random.random(iteration) - prev_x, prev_y = 0, 0 - next_x, next_y = 0, 0 - - for i in range(iteration): - for func_params, select_func in zip(self.function, self.select_function): - a, b, c, d, e, f = func_params - if rand[i] <= select_func: - next_x = prev_x * a + prev_y * b + e - next_y = prev_x * c + prev_y * d + f - break - - self.xs.append(next_x) - self.ys.append(next_y) - prev_x = next_x - prev_y = next_y - - @staticmethod - def process_nans(data): - nan_index = np.where(np.isnan(data)) - extend = np.array(range(nan_index[0][0] - 100, nan_index[0][0])) - delete_row = np.append(extend, nan_index) - return delete_row - - def rescale(self, image_x, image_y, pad_x, pad_y): - xs = np.array(self.xs) - ys = np.array(self.ys) - if np.any(np.isnan(xs)): - delete_row = self.process_nans(xs) - xs = np.delete(xs, delete_row, axis=0) - ys = np.delete(ys, delete_row, axis=0) - - if np.any(np.isnan(ys)): - delete_row = self.process_nans(ys) - xs = np.delete(xs, delete_row, axis=0) - ys = np.delete(ys, delete_row, axis=0) - - if np.min(xs) < 0.0: - xs -= np.min(xs) - if np.min(ys) < 0.0: - ys -= np.min(ys) - xmax, xmin = np.max(xs), np.min(xs) - ymax, ymin = np.max(ys), np.min(ys) - self.xs = np.uint16(xs / (xmax - xmin) * (image_x - 2 * pad_x) + pad_x) - self.ys = np.uint16(ys / (ymax - ymin) * (image_y - 2 * pad_y) + pad_y) - - def draw(self, draw_type, image_x, image_y, pad_x=6, pad_y=6): - self.rescale(image_x, image_y, pad_x, pad_y) - image = np.zeros((image_x, image_y), dtype=np.uint8) - - for i in range(len(self.xs)): - if draw_type == 'point': - image[self.ys[i], self.xs[i]] = 127 - else: - mask = '{:09b}'.format(np.random.randint(1, 512)) - patch = 127 * np.array(list(map(int, list(mask))), dtype=np.uint8).reshape(3, 3) - x_start = self.xs[i] + 1 - y_start = self.ys[i] + 1 - image[x_start:x_start+3, y_start:y_start+3] = patch - - return image - - -class SyntheticImageLoader(ImageLoader): - def __init__(self, config): - super().__init__(config) - - np.random.seed(seed=1) - self.subset_size = config.get('subset_size', 300) - self._cpu_count = min(os.cpu_count(), self.subset_size) - self._shape = config.get('shape', None) - self.data_source = config.get('data_source', None) - self._weights = np.array([ - 0.2, 1, 1, 1, 1, 1, - 0.6, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, - 1.4, 1, 1, 1, 1, 1, - 1.8, 1, 1, 1, 1, 1, - 1, 0.2, 1, 1, 1, 1, - 1, 0.6, 1, 1, 1, 1, - 1, 1.4, 1, 1, 1, 1, - 1, 1.8, 1, 1, 1, 1, - 1, 1, 0.2, 1, 1, 1, - 1, 1, 0.6, 1, 1, 1, - 1, 1, 1.4, 1, 1, 1, - 1, 1, 1.8, 1, 1, 1, - 1, 1, 1, 0.2, 1, 1, - 1, 1, 1, 0.6, 1, 1, - 1, 1, 1, 1.4, 1, 1, - 1, 1, 1, 1.8, 1, 1, - 1, 1, 1, 1, 0.2, 1, - 1, 1, 1, 1, 0.6, 1, - 1, 1, 1, 1, 1.4, 1, - 1, 1, 1, 1, 1.8, 1, - 1, 1, 1, 1, 1, 0.2, - 1, 1, 1, 1, 1, 0.6, - 1, 1, 1, 1, 1, 1.4, - 1, 1, 1, 1, 1, 1.8, - ]).reshape(-1, 6) - self._threshold = 0.2 - self._iterations = 200000 - self._num_of_points = None - self._instances = None - self._categories = None - if isinstance(self._shape, str): - self._shape = list(map(int, re.findall(r'\d+', self._shape))) - - super().get_layout() - self._check_input_shape() - - if os.path.exists(self.data_source) and os.listdir(self.data_source) and not config.generate_data: - logger.info(f'Dataset was found in `{self.data_source}`') - else: - logger.info(f'Synthetic dataset will be stored in `{self.data_source}`') - if not os.path.exists(self.data_source): - os.mkdir(self.data_source) - - assert os.path.isdir(self.data_source) - if config.generate_data or not os.listdir(self.data_source): - self._download_colorization_model() - logger.info(f'Start generating {self.subset_size} synthetic images') - self.generate_dataset() - - self._img_files = collect_img_files(self.data_source) - - def _check_input_shape(self): - if self._shape is None: - raise ValueError('Input shape should be specified. Please, use `--shape`') - if len(self._shape) < 3 or len(self._shape) > 4: - raise ValueError(f'Input shape should have 3 or 4 dimensions, but provided {self._shape}') - if self._shape[self._layout.get_index_by_name('C')] != 3: - raise ValueError('SyntheticImageLoader can generate images with only channels == 3') - - def _download_colorization_model(self): - proto_name = 'colorization_deploy_v2.prototxt' - model_name = 'colorization_release_v2.caffemodel' - npy_name = 'pts_in_hull.npy' - - if not os.path.exists(proto_name): - url = 'https://raw.githubusercontent.com/richzhang/colorization/caffe/colorization/models/' - proto = requests.get(url + proto_name) - open(proto_name, 'wb').write(proto.content) - if not os.path.exists(model_name): - url = 'http://eecs.berkeley.edu/~rich.zhang/projects/2016_colorization/files/demo_v2/' - model = requests.get(url + model_name) - open(model_name, 'wb').write(model.content) - if not os.path.exists(npy_name): - url = 'https://github.com/richzhang/colorization/raw/caffe/colorization/resources/' - pts_in_hull = requests.get(url + npy_name) - open(npy_name, 'wb').write(pts_in_hull.content) - - def _initialize_params(self, height, width): - default_img_size = 362 * 362 - points_coeff = max(1, int(np.round(height * width / default_img_size))) - self._num_of_points = 100000 * points_coeff - - if self.subset_size < len(self._weights): - self._instances = 1 - self._categories = 1 - self._weights = self._weights[:self.subset_size, :] - else: - self._instances = np.ceil(0.25 * self.subset_size / self._weights.shape[0]).astype(int) - self._categories = np.ceil(self.subset_size / (self._instances * self._weights.shape[0])).astype(int) - - def generate_dataset(self): - height = self._shape[self._layout.get_index_by_name('H')] - width = self._shape[self._layout.get_index_by_name('W')] - self._initialize_params(height, width) - - # to avoid multiprocessing error: can't pickle openvino.pyopenvino.Layout objects - self._layout = str(self._layout) - - with Pool(processes=self._cpu_count) as pool: - params = pool.map(self._generate_category, [1e-5] * self._categories) - - instances_weights = np.repeat(self._weights, self._instances, axis=0) - weight_per_img = np.tile(instances_weights, (self._categories, 1)) - repeated_params = np.repeat(params, self._weights.shape[0] * self._instances, axis=0) - repeated_params = repeated_params[:self.subset_size] - weight_per_img = weight_per_img[:self.subset_size] - assert weight_per_img.shape[0] == len(repeated_params) == self.subset_size - - splits = min(self._cpu_count, self.subset_size) - params_per_proc = np.array_split(repeated_params, splits) - weights_per_proc = np.array_split(weight_per_img, splits) - - generation_params = [] - offset = 0 - for param, w in zip(params_per_proc, weights_per_proc): - indices = list(range(offset, offset + len(param))) - offset += len(param) - generation_params.append((param, w, height, width, indices)) - - with Pool(processes=self._cpu_count) as pool: - pool.starmap(self._generate_image_batch, generation_params) - - self._layout = Layout(self._layout) - - def _generate_image_batch(self, params, weights, height, width, indices): - pts_in_hull = np.load('pts_in_hull.npy').transpose().reshape(2, 313, 1, 1).astype(np.float32) - net = cv.dnn.readNetFromCaffe('colorization_deploy_v2.prototxt', 'colorization_release_v2.caffemodel') - net.getLayer(net.getLayerId('class8_ab')).blobs = [pts_in_hull] - net.getLayer(net.getLayerId('conv8_313_rh')).blobs = [np.full([1, 313], 2.606, np.float32)] - - for i, param, weight in zip(indices, params, weights): - image = self._generator(param, 'gray', self._iterations, height, width, weight) - color_image = self._colorize(image, net) - aug_image = self._augment(color_image) - cv.imwrite(os.path.join(self.data_source, "{:06d}.png".format(i)), aug_image) - - @staticmethod - def _generator(params, draw_type, iterations, height=512, width=512, weight=None): - generators = IFSFunction(prev_x=0.0, prev_y=0.0) - for param in params: - generators.set_param(param[:6], param[6], weight) - generators.calculate(iterations) - img = generators.draw(draw_type, height, width) - return img - - def _generate_category(self, eps, height=512, width=512): - pixels = -1 - while pixels < self._threshold: - param_size = np.random.randint(2, 8) - params = np.zeros((param_size, 7), dtype=np.float32) - - sum_proba = eps - for i in range(param_size): - a, b, c, d, e, f = np.random.uniform(-1.0, 1.0, 6) - prob = abs(a * d - b * c) - sum_proba += prob - params[i] = a, b, c, d, e, f, prob - params[:, 6] /= sum_proba - - fracral_img = self._generator(params, 'point', self._num_of_points, height, width) - pixels = np.count_nonzero(fracral_img) / (height * width) - return params - - @staticmethod - def _rgb2lab(frame): - y_coeffs = np.array([0.212671, 0.715160, 0.072169], dtype=np.float32) - frame = np.where(frame > 0.04045, np.power((frame + 0.055) / 1.055, 2.4), frame / 12.92) - y = frame @ y_coeffs.T - L = np.where(y > 0.008856, 116 * np.cbrt(y) - 16, 903.3 * y) - return L - - def _colorize(self, frame, net): - H_orig, W_orig = frame.shape[:2] # original image size - if len(frame.shape) == 2 or frame.shape[-1] == 1: - frame = np.tile(frame.reshape(H_orig, W_orig, 1), (1, 1, 3)) - - frame = frame.astype(np.float32) / 255 - img_l = self._rgb2lab(frame) # get L from Lab image - img_rs = cv.resize(img_l, (224, 224)) # resize image to network input size - img_l_rs = img_rs - 50 # subtract 50 for mean-centering - - net.setInput(cv.dnn.blobFromImage(img_l_rs)) - ab_dec = net.forward()[0, :, :, :].transpose((1, 2, 0)) - - ab_dec_us = cv.resize(ab_dec, (W_orig, H_orig)) - img_lab_out = np.concatenate((img_l[..., np.newaxis], ab_dec_us), axis=2) # concatenate with original image L - img_bgr_out = np.clip(cv.cvtColor(img_lab_out, cv.COLOR_Lab2BGR), 0, 1) - frame_normed = 255 * (img_bgr_out - img_bgr_out.min()) / (img_bgr_out.max() - img_bgr_out.min()) - frame_normed = np.array(frame_normed, dtype=np.uint8) - return cv.resize(frame_normed, (H_orig, W_orig)) - - def _augment(self, image): - if np.random.random(1) >= 0.5: - image = cv.flip(image, 1) - - if np.random.random(1) >= 0.5: - image = cv.flip(image, 0) - - height, width = image.shape[:2] - angle = np.random.uniform(-30, 30) - rotate_matrix = cv.getRotationMatrix2D(center=(width / 2, height / 2), angle=angle, scale=1) - image = cv.warpAffine(src=image, M=rotate_matrix, dsize=(width, height)) - - image = self._fill_background(image) - - k_size = np.random.choice(list(range(1, 16, 2))) - image = cv.GaussianBlur(image, (k_size, k_size), 0) - return image - - @staticmethod - def _fill_background(image): - synthetic_background = Path(__file__).parent / 'synthetic_background.npy' - imagenet_means = np.load(synthetic_background) - class_id = np.random.randint(0, imagenet_means.shape[0]) - rows, cols = np.where(~np.any(image, axis=-1)) # background color = [0, 0, 0] - image[rows, cols] = imagenet_means[class_id] - return image diff --git a/tools/pot/openvino/tools/pot/engines/creator.py b/tools/pot/openvino/tools/pot/engines/creator.py index 895da98e7aa..563e983c2dd 100644 --- a/tools/pot/openvino/tools/pot/engines/creator.py +++ b/tools/pot/openvino/tools/pot/engines/creator.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 from openvino.tools.pot.engines.ac_engine import ACEngine -from openvino.tools.pot.engines.data_free_engine import DataFreeEngine from openvino.tools.pot.engines.simplified_engine import SimplifiedEngine @@ -17,6 +16,4 @@ def create_engine(config, **kwargs): return ACEngine(config) if config.type == 'simplified': return SimplifiedEngine(config, **kwargs) - if config.type == 'data_free': - return DataFreeEngine(config, **kwargs) raise RuntimeError('Unsupported engine type') diff --git a/tools/pot/openvino/tools/pot/engines/data_free_engine.py b/tools/pot/openvino/tools/pot/engines/data_free_engine.py deleted file mode 100644 index 35789b36cb8..00000000000 --- a/tools/pot/openvino/tools/pot/engines/data_free_engine.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (C) 2021-2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -from openvino.tools.pot.data_loaders.synthetic_image_loader import SyntheticImageLoader -from openvino.tools.pot.engines.simplified_engine import SimplifiedEngine - -class DataFreeEngine(SimplifiedEngine): - def __init__(self, config, data_loader=None, metric=None): - super().__init__(config) - if not data_loader: - self._data_loader = self.get_data_loader(config) - else: - self._data_loader = data_loader - - def get_data_loader(self, config): - if config.data_type == 'image': - return SyntheticImageLoader(config) - - raise NotImplementedError("Currently data-free optimization is available for Computer Vision models only") diff --git a/tools/pot/openvino/tools/pot/engines/ie_engine.py b/tools/pot/openvino/tools/pot/engines/ie_engine.py index bff5a1e0d4d..d5d80c3055f 100644 --- a/tools/pot/openvino/tools/pot/engines/ie_engine.py +++ b/tools/pot/openvino/tools/pot/engines/ie_engine.py @@ -403,11 +403,15 @@ class IEEngine(Engine): raise RuntimeError('Inconsistent data in the batch. ' 'Some items contain annotation, and some do not.') + if not all([isinstance(item[0], tuple) for item in batch]): + images, image_annotation = [data[0] for data in batch], [(idx, data[1]) for idx, data in enumerate(batch)] + else: + images, image_annotation = [data[1] for data in batch], [data[0] for data in batch] + if all([len(item) == 2 for item in batch]): - image_annotation, images = map(list, zip(*batch)) meta_data = [{}]*len(images) elif all([len(item) == 3 for item in batch]): - image_annotation, images, meta_data = map(list, zip(*batch)) + meta_data = [data[2] for data in batch] else: raise RuntimeError('Inconsistent data in the batch. ' 'Some items contain meta data, and some do not.') diff --git a/tools/pot/setup.py b/tools/pot/setup.py index 1c855e6d50f..bcc6c6bdd2b 100644 --- a/tools/pot/setup.py +++ b/tools/pot/setup.py @@ -25,6 +25,11 @@ class InstallCmd(install): # install requires self.do_egg_install() + def_quant_path = os.path.join("configs", "default_quantization_template.json") + aa_quant_path = os.path.join("configs", "accuracy_aware_quantization_template.json") + copyfile(def_quant_path, os.path.join(self.install_purelib, prefix, "pot", def_quant_path)) + copyfile(aa_quant_path, os.path.join(self.install_purelib, prefix, "pot", aa_quant_path)) + version_txt = os.path.join(prefix, "pot", "version.txt") if os.path.exists(version_txt): copyfile(os.path.join(version_txt), diff --git a/tools/pot/tests/test_cmd_params.py b/tools/pot/tests/test_cmd_params.py index a699573163c..e7f437ca78e 100644 --- a/tools/pot/tests/test_cmd_params.py +++ b/tools/pot/tests/test_cmd_params.py @@ -21,7 +21,7 @@ test_params = [('', 'Either --config or --quantize option should be specified', ('-e -m path_model', 'Either --config or --quantize option should be specified', ValueError), ('--quantize default -w path_weights -m path_model', '--quantize option requires AC config to be specified ' - 'or --engine should be `data_free` or `simplified`.', ValueError), + 'or --engine should be `simplified`.', ValueError), ('--quantize accuracy_aware -m path_model --ac-config path_config', '--quantize option requires model and weights to be specified.', ValueError), ('-c path_config -m path_model', 'Either --config or --model option should be specified', ValueError), diff --git a/tools/pot/tests/test_data_generation.py b/tools/pot/tests/test_data_generation.py deleted file mode 100644 index 75c7a19ba9f..00000000000 --- a/tools/pot/tests/test_data_generation.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (C) 2022 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -import os -from addict import Dict - -import pytest - -from openvino.tools.pot.data_loaders.creator import create_data_loader -from openvino.tools.pot.graph import load_model -from openvino.tools.pot.graph.model_utils import get_nodes_by_type - - -TEST_MODELS = [ - ('mobilenet-v2-pytorch', 'pytorch', None, None), - ('mobilenet-v2-pytorch', 'pytorch', None, (3, 640, 720)), - ('mobilenet-v2-pytorch', 'pytorch', 'HWC', (224, 224, 3)), - ('mobilenet-v2-pytorch', 'pytorch', 'NHWC', (1, 224, 224, 3)), - ('mobilenet-v2-pytorch', 'pytorch', 'CHW', (3, 224, 224)), - ('mobilenet-v2-pytorch', 'pytorch', 'NCHW', (1, 3, 224, 224)), -] - -@pytest.mark.parametrize( - 'model_name, model_framework, layout, input_shape', TEST_MODELS, - ids=['{}_{}_{}_{}'.format(m[0], m[1], m[2], m[3]) for m in TEST_MODELS]) -def test_generate_image(tmp_path, models, model_name, model_framework, layout, input_shape): - path_image_data = os.path.join(tmp_path, 'pot_dataset') - stat_subset_size = 5 - engine_config = Dict({'device': 'CPU', - 'type': 'data_free', - 'data_source': path_image_data, - 'subset_size': stat_subset_size, - 'layout': layout, - 'shape': input_shape, - 'generate_data': 'True'}) - model = models.get(model_name, model_framework, tmp_path) - model = load_model(model.model_params) - data_loader = create_data_loader(engine_config, model) - - num_images_from_data_loader = len(list(data_loader)) - num_images_in_dir = len(os.listdir(path_image_data)) - assert num_images_from_data_loader == num_images_in_dir == stat_subset_size - - image = data_loader[0] - if input_shape is None: - in_node = get_nodes_by_type(model, ['Parameter'], recursively=False)[0] - input_shape = tuple(in_node.shape[1:]) - elif len(input_shape) == 4: - input_shape = input_shape[1:] - - assert image.shape == input_shape diff --git a/tools/pot/tests/test_sanity.py b/tools/pot/tests/test_sanity.py index f29334c1e0b..c8bc9a4eba3 100644 --- a/tools/pot/tests/test_sanity.py +++ b/tools/pot/tests/test_sanity.py @@ -219,24 +219,6 @@ def test_simplified_mode(tmp_path, models): assert metrics == pytest.approx(expected_accuracy, abs=0.006) -DATAFREE_TEST_MODELS = [ - ('mobilenet-v2-pytorch', 'pytorch', 'DefaultQuantization', 'performance', - {'accuracy@top1': 0.679, 'accuracy@top5': 0.888}) -] - - -def test_datafree_mode(tmp_path, models): - engine_config = Dict({'type': 'data_free', - 'data_source': os.path.join(tmp_path, 'pot_dataset'), - 'generate_data': 'True', - 'subset_size': 30, - 'device': 'CPU'}) - - _, _, _, _, expected_accuracy = DATAFREE_TEST_MODELS[0] - metrics = launch_simplified_mode(tmp_path, models, engine_config) - assert metrics == pytest.approx(expected_accuracy, abs=0.06) - - def test_frame_extractor_tool(): # hack due to strange python imports (same as in sample test) pot_dir = Path(__file__).parent.parent