Merge remote-tracking branch 'upstream/master' into debian-packages
This commit is contained in:
Ilya Lavrenov committed
commit
be78da4a88
417 files changed
+4184
-3175
No files matched your search
+2
-2
@@ -1,5 +1,5 @@
|
||||
[submodule "src/plugins/intel_cpu/thirdparty/mkl-dnn"]
|
||||
path = src/plugins/intel_cpu/thirdparty/mkl-dnn
|
||||
[submodule "src/plugins/intel_cpu/thirdparty/onednn"]
|
||||
path = src/plugins/intel_cpu/thirdparty/onednn
|
||||
url = https://github.com/openvinotoolkit/oneDNN.git
|
||||
ignore = dirty
|
||||
[submodule "thirdparty/xbyak"]
|
||||
|
||||
@@ -51,6 +51,7 @@ file(REMOVE "${CMAKE_BINARY_DIR}/InferenceEngineTargets.cmake")
|
||||
file(REMOVE "${CMAKE_BINARY_DIR}/OpenVINOTargets.cmake")
|
||||
foreach(component IN LISTS openvino_export_components)
|
||||
file(REMOVE "${CMAKE_BINARY_DIR}/${component}_dev_targets.cmake")
|
||||
file(REMOVE "${CMAKE_BINARY_DIR}/ov_${component}_dev_targets.cmake")
|
||||
unset(${component} CACHE)
|
||||
endforeach()
|
||||
unset(openvino_export_components CACHE)
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@ Jenkinsfile @openvinotoolkit/openvino-admins
|
||||
|
||||
# IE CPU:
|
||||
/src/plugins/intel_cpu/ @openvinotoolkit/openvino-ie-cpu-maintainers @openvinotoolkit/openvino-ie-cpu-developers
|
||||
/src/plugins/intel_cpu/thirdparty/mkl-dnn/ @openvinotoolkit/openvino-ie-cpu-maintainers @openvinotoolkit/openvino-ie-cpu-developers
|
||||
/src/plugins/intel_cpu/thirdparty/onednn/ @openvinotoolkit/openvino-ie-cpu-maintainers @openvinotoolkit/openvino-ie-cpu-developers
|
||||
|
||||
#IE LPT
|
||||
/src/common/low_precision_transformations/ @openvinotoolkit/openvino-ie-lpt-maintainers
|
||||
|
||||
@@ -1,43 +1,203 @@
|
||||
# OpenVINO™ Toolkit
|
||||
<div align="center">
|
||||
|
||||
<img src="docs/img/openvino-logo-purple-black.png" width="400px">
|
||||
|
||||
[](https://github.com/openvinotoolkit/openvino/releases/tag/2022.1)
|
||||
[](LICENSE)
|
||||
|
||||
|
||||
[](https://badge.fury.io/py/openvino)
|
||||
[](https://pepy.tech/project/openvino)
|
||||
|
||||
</div>
|
||||
|
||||
## Contents:
|
||||
|
||||
- [What is OpenVINO?](#what-is-openvino-toolkit)
|
||||
- [Components](#components)
|
||||
- [Supported Hardware matrix](#supported-hardware-matrix)
|
||||
- [License](#license)
|
||||
- [Documentation](#documentation)
|
||||
- [Tutorials](#tutorials)
|
||||
- [Products which use OpenVINO](#products-which-use-openvino)
|
||||
- [System requirements](#system-requirements)
|
||||
- [How to build](#how-to-build)
|
||||
- [How to contribute](#how-to-contribute)
|
||||
- [Get a support](#get-a-support)
|
||||
- [See also](#see-also)
|
||||
|
||||
## What is OpenVINO toolkit?
|
||||
|
||||
OpenVINO™ is an open-source toolkit for optimizing and deploying AI inference.
|
||||
- Boost deep learning performance in computer vision, automatic speech recognition, natural language processing and other common tasks
|
||||
- Use models trained with popular frameworks like TensorFlow, PyTorch and more
|
||||
- Reduce resource demands and efficiently deploy on a range of Intel® platforms from edge to cloud
|
||||
|
||||
This toolkit allows developers to deploy pre-trained deep learning models
|
||||
through a high-level OpenVINO™ Runtime C++ and Python APIs integrated with application logic.
|
||||
|
||||
This open source version includes several components: namely [Model Optimizer], [OpenVINO™ Runtime], [Post-Training Optimization Tool], as well as CPU, GPU, MYRIAD, multi device and heterogeneous plugins to accelerate deep learning inferencing on Intel® CPUs and Intel® Processor Graphics.
|
||||
It supports pre-trained models from the [Open Model Zoo], along with 100+ open
|
||||
source and public models in popular formats such as TensorFlow, ONNX, PaddlePaddle, MXNet, Caffe, Kaldi.
|
||||
|
||||
## Repository components
|
||||
* [OpenVINO™ Runtime]
|
||||
* [Model Optimizer]
|
||||
* [Post-Training Optimization Tool]
|
||||
* [Samples]
|
||||
### Components
|
||||
* [OpenVINO™ Runtime] - is a set of C++ libraries with C and Python bindings providing a common API to deliver inference solutions on the platform of your choice.
|
||||
* [core](https://github.com/openvinotoolkit/openvino/tree/master/src/core) - provides the base API for model representation and modification.
|
||||
* [inference](https://github.com/openvinotoolkit/openvino/tree/master/src/inference) - provides an API to infer models on device.
|
||||
* [transformations](https://github.com/openvinotoolkit/openvino/tree/master/src/common/transformations) - contains the set of common transformations which are used in OpenVINO plugins.
|
||||
* [low precision transformations](https://github.com/openvinotoolkit/openvino/tree/master/src/common/low_precision_transformations) - contains the set of transformations which are used in low precision models
|
||||
* [bindings](https://github.com/openvinotoolkit/openvino/tree/master/src/bindings) - contains all awailable OpenVINO bindings which are maintained by OpenVINO team.
|
||||
* [c](https://github.com/openvinotoolkit/openvino/tree/master/src/bindings/c) - provides C API for OpenVINO™ Runtime
|
||||
* [python](https://github.com/openvinotoolkit/openvino/tree/master/src/bindings/python) - Python API for OpenVINO™ Runtime
|
||||
* [Plugins](https://github.com/openvinotoolkit/openvino/tree/master/src/plugins) - contains OpenVINO plugins which are maintained in open-source by OpenVINO team. For more information please taje a look to the [list of supported devices](#supported-hardware-matrix).
|
||||
* [Frontends](https://github.com/openvinotoolkit/openvino/tree/master/src/frontends) - contains available OpenVINO frontends which allow to read model from native framework format.
|
||||
* [Model Optimizer] - is a cross-platform command-line tool that facilitates the transition between training and deployment environments, performs static model analysis, and adjusts deep learning models for optimal execution on end-point target devices.
|
||||
* [Post-Training Optimization Tool] - is designed to accelerate the inference of deep learning models by applying special methods without model retraining or fine-tuning, for example, post-training 8-bit quantization.
|
||||
* [Samples] - applications on C, C++ and Python languages which shows basic use cases of OpenVINO usages.
|
||||
|
||||
## Supported Hardware matrix
|
||||
|
||||
The OpenVINO™ Runtime can infer models on different hardware devices. This section provides the list of supported devices.
|
||||
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Device</th>
|
||||
<th>Plugin</th>
|
||||
<th>Library</th>
|
||||
<th>ShortDescription</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td rowspan=2>CPU</td>
|
||||
<td> <a href="https://docs.openvino.ai/nightly/openvino_docs_OV_UG_supported_plugins_CPU.html#doxid-openvino-docs-o-v-u-g-supported-plugins-c-p-u">Intel CPU</a></tb>
|
||||
<td><b><i><a href="https://github.com/openvinotoolkit/openvino/tree/master/src/plugins/intel_cpu">openvino_intel_cpu_plugin</a></i></b></td>
|
||||
<td>Intel Xeon with Intel® Advanced Vector Extensions 2 (Intel® AVX2), Intel® Advanced Vector Extensions 512 (Intel® AVX-512), and AVX512_BF16, Intel Core Processors with Intel AVX2, Intel Atom Processors with Intel® Streaming SIMD Extensions (Intel® SSE)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td> <a href="https://docs.openvino.ai/nightly/openvino_docs_OV_UG_supported_plugins_ARM_CPU.html">ARM CPU</a></tb>
|
||||
<td><b><i><a href="https://github.com/openvinotoolkit/openvino_contrib/tree/master/modules/arm_plugin">openvino_arm_cpu_plugin</a></i></b></td>
|
||||
<td>Raspberry Pi™ 4 Model B, Apple® Mac mini with M1 chip, NVIDIA® Jetson Nano™, Android™ devices
|
||||
</tr>
|
||||
<tr>
|
||||
<td>GPU</td>
|
||||
<td><a href="https://docs.openvino.ai/nightly/openvino_docs_OV_UG_supported_plugins_GPU.html#doxid-openvino-docs-o-v-u-g-supported-plugins-g-p-u">Intel GPU</a></td>
|
||||
<td><b><i><a href="https://github.com/openvinotoolkit/openvino/tree/master/src/plugins/intel_gpu">openvino_intel_gpu_plugin</a></i></b></td>
|
||||
<td>Intel Processor Graphics, including Intel HD Graphics and Intel Iris Graphics</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>GNA</td>
|
||||
<td><a href="https://docs.openvino.ai/nightly/openvino_docs_OV_UG_supported_plugins_GNA.html#doxid-openvino-docs-o-v-u-g-supported-plugins-g-n-a">Intel GNA</a></td>
|
||||
<td><b><i><a href="https://github.com/openvinotoolkit/openvino/tree/master/src/plugins/intel_gna">openvino_intel_gna_plugin</a></i></b></td>
|
||||
<td>Intel Speech Enabling Developer Kit, Amazon Alexa* Premium Far-Field Developer Kit, Intel Pentium Silver J5005 Processor, Intel Pentium Silver N5000 Processor, Intel Celeron J4005 Processor, Intel Celeron J4105 Processor, Intel Celeron Processor N4100, Intel Celeron Processor N4000, Intel Core i3-8121U Processor, Intel Core i7-1065G7 Processor, Intel Core i7-1060G7 Processor, Intel Core i5-1035G4 Processor, Intel Core i5-1035G7 Processor, Intel Core i5-1035G1 Processor, Intel Core i5-1030G7 Processor, Intel Core i5-1030G4 Processor, Intel Core i3-1005G1 Processor, Intel Core i3-1000G1 Processor, Intel Core i3-1000G4 Processor</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>VPU</td>
|
||||
<td><a href="https://docs.openvino.ai/nightly/openvino_docs_IE_DG_supported_plugins_VPU.html#doxid-openvino-docs-i-e-d-g-supported-plugins-v-p-u">Myriad plugin</a></td>
|
||||
<td><b><i><a href="https://github.com/openvinotoolkit/openvino/tree/master/src/plugins/intel_myriad">openvino_intel_myriad_plugin</a></i></b></td>
|
||||
<td>Intel® Neural Compute Stick 2 powered by the Intel® Movidius™ Myriad™ X</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
Also OpenVINO™ Toolkit contains several plugins which should simplify to load model on several hardware devices:
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Plugin</th>
|
||||
<th>Library</th>
|
||||
<th>ShortDescription</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><a href="https://docs.openvino.ai/nightly/openvino_docs_IE_DG_supported_plugins_AUTO.html#doxid-openvino-docs-i-e-d-g-supported-plugins-a-u-t-o">Auto</a></td>
|
||||
<td><b><i><a href="https://github.com/openvinotoolkit/openvino/tree/master/src/plugins/auto">openvino_auto_plugin</a></i></b></td>
|
||||
<td>Auto plugin enables selecting Intel device for inference automatically</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a href="https://docs.openvino.ai/nightly/openvino_docs_OV_UG_Automatic_Batching.html">Auto Batch</a></td>
|
||||
<td><b><i><a href="https://github.com/openvinotoolkit/openvino/tree/master/src/plugins/auto_batch">openvino_auto_batch_plugin</a></i></b></td>
|
||||
<td>Auto batch plugin performs on-the-fly automatic batching (i.e. grouping inference requests together) to improve device utilization, with no programming effort from the user</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a href="https://docs.openvino.ai/nightly/openvino_docs_OV_UG_Hetero_execution.html#doxid-openvino-docs-o-v-u-g-hetero-execution">Hetero</a></td>
|
||||
<td><b><i><a href="https://github.com/openvinotoolkit/openvino/tree/master/src/plugins/hetero">openvino_hetero_plugin</a></i></b></td>
|
||||
<td>Heterogeneous execution enables automatic inference splitting between several devices</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><a href="https://docs.openvino.ai/nightly/openvino_docs_OV_UG_Running_on_multiple_devices.html#doxid-openvino-docs-o-v-u-g-running-on-multiple-devices">Multi</a></td>
|
||||
<td><b><i><a href="https://github.com/openvinotoolkit/openvino/tree/master/src/plugins/auto">openvino_auto_plugin</a></i></b></td>
|
||||
<td>Multi plugin enables simultaneous inference of the same model on several devices in parallel</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
## License
|
||||
OpenVINO™ Toolkit is licensed under [Apache License Version 2.0](LICENSE).
|
||||
By contributing to the project, you agree to the license and copyright terms therein and release your contribution under these terms.
|
||||
|
||||
## Resources
|
||||
* Docs: https://docs.openvino.ai/
|
||||
* Wiki: https://github.com/openvinotoolkit/openvino/wiki
|
||||
* Issue tracking: https://github.com/openvinotoolkit/openvino/issues
|
||||
* Storage: https://storage.openvinotoolkit.org/
|
||||
* Additional OpenVINO™ toolkit modules: https://github.com/openvinotoolkit/openvino_contrib
|
||||
* [Intel® Distribution of OpenVINO™ toolkit Product Page](https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit.html)
|
||||
* [Intel® Distribution of OpenVINO™ toolkit Release Notes](https://software.intel.com/en-us/articles/OpenVINO-RelNotes)
|
||||
## Documentation
|
||||
|
||||
### User documentation
|
||||
|
||||
The latest documentation for OpenVINO™ Toolkit is availabe [here](https://docs.openvino.ai/). This documentation contains detailed information about all OpenVINO components and provides all important information which could be needed if you create an application which is based on binary OpenVINO distribution or own OpenVINO version without source code modification.
|
||||
|
||||
### Developer documentation
|
||||
|
||||
[Developer documentation](#todo-add) contains information about architectural decisions which are applied inside the OpenVINO components. This documentation has all necessary information which could be needed in order to contribute to OpenVINO.
|
||||
|
||||
## Tutorials
|
||||
|
||||
The list of OpenVINO tutorials:
|
||||
|
||||
- [Jupiter notebooks](https://github.com/openvinotoolkit/openvino_notebooks)
|
||||
|
||||
## Products which use OpenVINO
|
||||
|
||||
- [OpenCV](https://opencv.org/)
|
||||
- [ONNX Runtime](https://onnxruntime.ai/)
|
||||
- [OpenVINO™ Integration with TensorFlow](https://www.intel.com/content/www/us/en/developer/tools/devcloud/edge/build/ovtfoverview.html)
|
||||
- [TNN](https://github.com/Tencent/TNN/tree/master)
|
||||
|
||||
## System requirements
|
||||
|
||||
The full information about system requirements depends on platform and available in section `System requirement` on dedicated pages:
|
||||
- [Linux](https://docs.openvino.ai/latest/openvino_docs_install_guides_installing_openvino_linux.html)
|
||||
- [Windows](https://docs.openvino.ai/latest/openvino_docs_install_guides_installing_openvino_windows.html)
|
||||
- [macOS](https://docs.openvino.ai/latest/openvino_docs_install_guides_installing_openvino_macos.html)
|
||||
- [Raspbian](https://docs.openvino.ai/latest/openvino_docs_install_guides_installing_openvino_raspbian.html)
|
||||
|
||||
## How to build
|
||||
|
||||
Please take a look to [OpenVINO Wiki](https://github.com/openvinotoolkit/openvino/wiki#how-to-build) to get more information about OpenVINO build process.
|
||||
|
||||
## How to contribute
|
||||
|
||||
See [CONTRIBUTING](./CONTRIBUTING.md) for details. Thank you!
|
||||
|
||||
## Get a support
|
||||
|
||||
## Support
|
||||
Please report questions, issues and suggestions using:
|
||||
|
||||
* The [`openvino`](https://stackoverflow.com/questions/tagged/openvino) tag on StackOverflow\*
|
||||
* [GitHub* Issues](https://github.com/openvinotoolkit/openvino/issues)
|
||||
* The [`openvino`](https://stackoverflow.com/questions/tagged/openvino) tag on StackOverflow\*
|
||||
* [Forum](https://software.intel.com/en-us/forums/computer-vision)
|
||||
|
||||
## See also
|
||||
|
||||
* [OpenVINO Wiki](https://github.com/openvinotoolkit/openvino/wiki)
|
||||
* [OpenVINO Storage](https://storage.openvinotoolkit.org/)
|
||||
* Additional OpenVINO™ toolkit modules:
|
||||
* [openvino_contrib](https://github.com/openvinotoolkit/openvino_contrib)
|
||||
* [Intel® Distribution of OpenVINO™ toolkit Product Page](https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit.html)
|
||||
* [Intel® Distribution of OpenVINO™ toolkit Release Notes](https://software.intel.com/en-us/articles/OpenVINO-RelNotes)
|
||||
* [Neural Network Compression Framework (NNCF)](https://github.com/openvinotoolkit/nncf) - a suite of advanced algorithms for model inference optimization including quantization, filter pruning, binarization and sparsity
|
||||
* [OpenVINO™ Training Extensions (OTE)](https://github.com/openvinotoolkit/training_extensions) - convenient environment to train Deep Learning models and convert them using OpenVINO for optimized inference.
|
||||
* [OpenVINO™ Model Server (OVMS)](https://github.com/openvinotoolkit/model_server) - a scalable, high-performance solution for serving deep learning models optimized for Intel architectures
|
||||
* [DL Workbench](https://docs.openvino.ai/nightly/workbench_docs_Workbench_DG_Introduction.html) - An alternative, web-based version of OpenVINO designed to make production of pretrained deep learning models significantly easier.
|
||||
* [Computer Vision Annotation Tool (CVAT)](https://github.com/openvinotoolkit/cvat) - an online, interactive video and image annotation tool for computer vision purposes.
|
||||
* [Dataset Management Framework (Datumaro)](https://github.com/openvinotoolkit/datumaro) - a framework and CLI tool to build, transform, and analyze datasets.
|
||||
|
||||
---
|
||||
\* Other names and brands may be claimed as the property of others.
|
||||
|
||||
@@ -46,5 +206,3 @@ Please report questions, issues and suggestions using:
|
||||
[Model Optimizer]:https://docs.openvino.ai/latest/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html
|
||||
[Post-Training Optimization Tool]:https://docs.openvino.ai/latest/pot_introduction.html
|
||||
[Samples]:https://github.com/openvinotoolkit/openvino/tree/master/samples
|
||||
[tag on StackOverflow]:https://stackoverflow.com/search?q=%23openvino
|
||||
|
||||
@@ -7,6 +7,7 @@ set(CMAKE_SYSTEM_PROCESSOR armv7l)
|
||||
|
||||
set(CMAKE_C_COMPILER arm-linux-gnueabihf-gcc)
|
||||
set(CMAKE_CXX_COMPILER arm-linux-gnueabihf-g++)
|
||||
set(PKG_CONFIG_EXECUTABLE arm-linux-gnueabihf-pkg-config CACHE PATH "Path to ARM pkg-config")
|
||||
|
||||
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
|
||||
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
|
||||
|
||||
@@ -7,6 +7,7 @@ set(CMAKE_SYSTEM_PROCESSOR aarch64)
|
||||
|
||||
set(CMAKE_C_COMPILER aarch64-linux-gnu-gcc)
|
||||
set(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++)
|
||||
set(PKG_CONFIG_EXECUTABLE aarch64-linux-gnu-pkg-config CACHE PATH "Path to ARM64 pkg-config")
|
||||
|
||||
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
|
||||
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
|
||||
|
||||
+31
-10
@@ -91,7 +91,19 @@ if(THREADING STREQUAL "OMP")
|
||||
endif()
|
||||
|
||||
## TBB package
|
||||
if(THREADING STREQUAL "TBB" OR THREADING STREQUAL "TBB_AUTO" AND NOT ENABLE_SYSTEM_TBB)
|
||||
unset(_ov_download_tbb_done CACHE)
|
||||
|
||||
#
|
||||
# The function downloads prebuilt TBB package
|
||||
# NOTE: the function should be used if system TBB is not found
|
||||
# or ENABLE_SYSTEM_TBB is OFF
|
||||
#
|
||||
function(ov_download_tbb)
|
||||
if(_ov_download_tbb_done OR NOT THREADING MATCHES "^(TBB|TBB_AUTO)$")
|
||||
return()
|
||||
endif()
|
||||
set(_ov_download_tbb_done ON CACHE BOOL "Whether prebuilt TBB is already downloaded")
|
||||
|
||||
reset_deps_cache(TBBROOT TBB_DIR)
|
||||
|
||||
if(DEFINED ENV{THIRDPARTY_SERVER_PATH})
|
||||
@@ -153,13 +165,24 @@ if(THREADING STREQUAL "TBB" OR THREADING STREQUAL "TBB_AUTO" AND NOT ENABLE_SYST
|
||||
debug_message(STATUS "tbb_dir=" ${TBB_DIR})
|
||||
debug_message(STATUS "tbbroot=" ${TBBROOT})
|
||||
|
||||
if(DEFINED IE_PATH_TO_DEPS)
|
||||
unset(IE_PATH_TO_DEPS)
|
||||
endif()
|
||||
endif()
|
||||
set(TBB "${TBB}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
## TBBBind_2_5 package
|
||||
if(ENABLE_TBBBIND_2_5)
|
||||
unset(_ov_download_tbbbind_2_5_done CACHE)
|
||||
|
||||
#
|
||||
# The function downloads static prebuilt TBBBind_2_5 package
|
||||
# NOTE: the function should be called only we have TBB with version less 2021
|
||||
#
|
||||
function(ov_download_tbbbind_2_5)
|
||||
if(_ov_download_tbbbind_2_5_done OR NOT ENABLE_TBBBIND_2_5)
|
||||
return()
|
||||
endif()
|
||||
set(_ov_download_tbbbind_2_5_done ON CACHE BOOL "Whether prebuilt TBBBind_2_5 is already downloaded")
|
||||
|
||||
reset_deps_cache(TBBBIND_2_5_DIR)
|
||||
|
||||
if(DEFINED ENV{THIRDPARTY_SERVER_PATH})
|
||||
set(IE_PATH_TO_DEPS "$ENV{THIRDPARTY_SERVER_PATH}")
|
||||
elseif(DEFINED THIRDPARTY_SERVER_PATH)
|
||||
@@ -187,10 +210,8 @@ Build oneTBB from sources and set TBBROOT environment var before OpenVINO cmake
|
||||
|
||||
update_deps_cache(TBBBIND_2_5_DIR "${TBBBIND_2_5}/cmake" "Path to TBBBIND_2_5 cmake folder")
|
||||
|
||||
if(DEFINED IE_PATH_TO_DEPS)
|
||||
unset(IE_PATH_TO_DEPS)
|
||||
endif()
|
||||
endif()
|
||||
set(TBBBIND_2_5 "${TBBBIND_2_5}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
## OpenCV
|
||||
if(ENABLE_OPENCV)
|
||||
|
||||
@@ -186,6 +186,8 @@ endif()
|
||||
# Use solution folders
|
||||
set_property(GLOBAL PROPERTY USE_FOLDERS ON)
|
||||
|
||||
# cmake_dependent_option() supports full Condition Syntax
|
||||
set(CMAKE_POLICY_DEFAULT_CMP0127 NEW)
|
||||
# Enable CMAKE_<LANG>_COMPILER_ID AppleClang
|
||||
set(CMAKE_POLICY_DEFAULT_CMP0025 NEW)
|
||||
|
||||
|
||||
@@ -13,8 +13,8 @@ function(ie_plugin_get_file_name target_name library_name)
|
||||
set("${library_name}" "${LIB_PREFIX}${target_name}${LIB_SUFFIX}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
if(NOT TARGET ie_plugins)
|
||||
add_custom_target(ie_plugins)
|
||||
if(NOT TARGET ov_plugins)
|
||||
add_custom_target(ov_plugins)
|
||||
endif()
|
||||
|
||||
#
|
||||
@@ -113,7 +113,7 @@ function(ie_add_plugin)
|
||||
add_cpplint_target(${IE_PLUGIN_NAME}_cpplint FOR_TARGETS ${IE_PLUGIN_NAME} CUSTOM_FILTERS ${custom_filter})
|
||||
endif()
|
||||
|
||||
add_dependencies(ie_plugins ${IE_PLUGIN_NAME})
|
||||
add_dependencies(ov_plugins ${IE_PLUGIN_NAME})
|
||||
if(TARGET openvino_gapi_preproc)
|
||||
if(BUILD_SHARED_LIBS)
|
||||
add_dependencies(${IE_PLUGIN_NAME} openvino_gapi_preproc)
|
||||
|
||||
+17
-13
@@ -6,15 +6,19 @@ function(ie_generate_dev_package_config)
|
||||
# dummy check that OpenCV is here
|
||||
find_package(OpenCV QUIET)
|
||||
|
||||
set(all_dev_targets ov_runtime_libraries)
|
||||
foreach(component IN LISTS openvino_export_components)
|
||||
# export all targets with prefix and use them during extra modules build
|
||||
export(TARGETS ${${component}} NAMESPACE IE::
|
||||
APPEND FILE "${CMAKE_BINARY_DIR}/${component}_dev_targets.cmake")
|
||||
APPEND FILE "${CMAKE_BINARY_DIR}/${component}_dev_targets.cmake")
|
||||
list(APPEND all_dev_targets ${${component}})
|
||||
endforeach()
|
||||
add_custom_target(ie_dev_targets DEPENDS ${all_dev_targets})
|
||||
|
||||
# if we've found system gflags
|
||||
if(gflags_DIR)
|
||||
set(gflags_BINARY_DIR "${gflags_DIR}")
|
||||
endif()
|
||||
|
||||
configure_package_config_file("${OpenVINO_SOURCE_DIR}/cmake/templates/InferenceEngineDeveloperPackageConfig.cmake.in"
|
||||
"${CMAKE_BINARY_DIR}/InferenceEngineDeveloperPackageConfig.cmake"
|
||||
INSTALL_DESTINATION share # not used
|
||||
@@ -30,18 +34,22 @@ function(ov_generate_dev_package_config)
|
||||
# dummy check that OpenCV is here
|
||||
find_package(OpenCV QUIET)
|
||||
|
||||
set(all_dev_targets ov_runtime_libraries)
|
||||
foreach(component IN LISTS openvino_export_components)
|
||||
string(FIND "${component}" "_legacy" index)
|
||||
if (index EQUAL -1)
|
||||
if(index EQUAL -1)
|
||||
# export all targets with prefix and use them during extra modules build
|
||||
export(TARGETS ${${component}} NAMESPACE openvino::
|
||||
APPEND FILE "${CMAKE_BINARY_DIR}/ov_${component}_dev_targets.cmake")
|
||||
APPEND FILE "${CMAKE_BINARY_DIR}/ov_${component}_dev_targets.cmake")
|
||||
list(APPEND all_dev_targets ${${component}})
|
||||
endif()
|
||||
endforeach()
|
||||
add_custom_target(ov_dev_targets DEPENDS ${all_dev_targets})
|
||||
|
||||
# if we've found system gflags
|
||||
if(gflags_DIR)
|
||||
set(gflags_BINARY_DIR "${gflags_DIR}")
|
||||
endif()
|
||||
|
||||
configure_package_config_file("${OpenVINO_SOURCE_DIR}/cmake/templates/OpenVINODeveloperPackageConfig.cmake.in"
|
||||
"${CMAKE_BINARY_DIR}/OpenVINODeveloperPackageConfig.cmake"
|
||||
INSTALL_DESTINATION share # not used
|
||||
@@ -59,14 +67,14 @@ endfunction()
|
||||
|
||||
function(register_extra_modules)
|
||||
# post export
|
||||
openvino_developer_export_targets(COMPONENT core TARGETS inference_engine)
|
||||
openvino_developer_export_targets(COMPONENT core TARGETS ngraph)
|
||||
openvino_developer_export_targets(COMPONENT core_legacy TARGETS inference_engine)
|
||||
openvino_developer_export_targets(COMPONENT core_legacy TARGETS ngraph)
|
||||
|
||||
set(InferenceEngineDeveloperPackage_DIR "${CMAKE_CURRENT_BINARY_DIR}/runtime")
|
||||
set(OpenVINODeveloperPackage_DIR "${CMAKE_BINARY_DIR}/runtime")
|
||||
|
||||
function(generate_fake_dev_package NS)
|
||||
if (NS STREQUAL "openvino")
|
||||
if(NS STREQUAL "openvino")
|
||||
set(devconfig_file "${OpenVINODeveloperPackage_DIR}/OpenVINODeveloperPackageConfig.cmake")
|
||||
else()
|
||||
set(devconfig_file "${InferenceEngineDeveloperPackage_DIR}/InferenceEngineDeveloperPackageConfig.cmake")
|
||||
@@ -81,10 +89,6 @@ function(register_extra_modules)
|
||||
file(APPEND "${devconfig_file}" "add_library(${NS}::${target} ALIAS ${target})\n")
|
||||
endif()
|
||||
endforeach()
|
||||
if ("${NS}" STREQUAL "openvino")
|
||||
file(APPEND "${devconfig_file}" "add_library(${NS}::runtime ALIAS openvino)\n")
|
||||
file(APPEND "${devconfig_file}" "add_library(${NS}::runtime::dev ALIAS openvino_dev)\n")
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
generate_fake_dev_package("openvino")
|
||||
@@ -137,7 +141,7 @@ ie_generate_dev_package_config()
|
||||
ov_generate_dev_package_config()
|
||||
|
||||
# extra modules must be registered after inference_engine library
|
||||
# and all other IE common libraries (ov_runtime_libraries) are creared
|
||||
# and all other OpenVINO Core libraries are creared
|
||||
# because 'register_extra_modules' creates fake InferenceEngineDeveloperPackageConfig.cmake
|
||||
# with all imported developer targets
|
||||
register_extra_modules()
|
||||
|
||||
+2
-16
@@ -59,7 +59,7 @@ cmake_dependent_option (ENABLE_WHEEL "Build wheel packages for PyPi" OFF
|
||||
# Inference Engine specific options
|
||||
#
|
||||
|
||||
# "MKL-DNN library based on OMP or TBB or Sequential implementation: TBB|OMP|SEQ"
|
||||
# "OneDNN library based on OMP or TBB or Sequential implementation: TBB|OMP|SEQ"
|
||||
if(X86 OR ARM OR (MSVC AND (ARM OR AARCH64)) )
|
||||
set(THREADING_DEFAULT "SEQ")
|
||||
else()
|
||||
@@ -139,22 +139,8 @@ ie_dependent_option (ENABLE_SYSTEM_PUGIXML "use the system copy of pugixml" OFF
|
||||
get_linux_name(LINUX_OS_NAME)
|
||||
if(LINUX_OS_NAME MATCHES "^Ubuntu [0-9]+\.[0-9]+$" AND NOT DEFINED ENV{TBBROOT})
|
||||
# Debian packages are enabled on Ubuntu systems
|
||||
# so, system TBB can be tried for usage
|
||||
set(ENABLE_SYSTEM_TBB_DEFAULT ON)
|
||||
|
||||
# check whether "default" (customly provided or system one) is available
|
||||
find_package(TBB QUIET)
|
||||
if(NOT TBB_FOUND)
|
||||
message(WARNING "System TBB is not found, custom TBB version will be downloaded from shared drive")
|
||||
# we still need to download prebuilt version of TBB
|
||||
set(ENABLE_SYSTEM_TBB_DEFAULT OFF)
|
||||
endif()
|
||||
|
||||
# remove found TBB (or invalid TBB_DIR-NOTFOUND) from cache
|
||||
unset(TBB_DIR CACHE)
|
||||
unset(TBB_DIR)
|
||||
unset(TBB_FOUND)
|
||||
unset(TBB_IMPORTED_TARGETS)
|
||||
unset(TBB_VERSION)
|
||||
else()
|
||||
set(ENABLE_SYSTEM_TBB_DEFAULT OFF)
|
||||
endif()
|
||||
|
||||
@@ -18,14 +18,14 @@ list(APPEND ie_options CMAKE_CXX_COMPILER_LAUNCHER CMAKE_C_COMPILER_LAUNCHER
|
||||
file(TO_CMAKE_PATH "${CMAKE_CURRENT_LIST_DIR}" cache_path)
|
||||
|
||||
message(STATUS "The following CMake options are exported from Inference Engine Developer package")
|
||||
message("")
|
||||
message(" ")
|
||||
foreach(option IN LISTS ie_options)
|
||||
if(NOT DEFINED "${option}")
|
||||
load_cache("${cache_path}" READ_WITH_PREFIX "" ${option})
|
||||
endif()
|
||||
message(" ${option}: ${${option}}")
|
||||
endforeach()
|
||||
message("")
|
||||
message(" ")
|
||||
|
||||
# for samples in 3rd party projects
|
||||
set_and_check(gflags_DIR "@gflags_BINARY_DIR@")
|
||||
@@ -66,6 +66,18 @@ foreach(component @openvino_export_components@)
|
||||
include("${CMAKE_CURRENT_LIST_DIR}/${component}_dev_targets.cmake")
|
||||
endforeach()
|
||||
|
||||
if(TARGET IE::ov_core_dev AND NOT TARGET openvino::core::dev)
|
||||
add_library(openvino::core::dev INTERFACE IMPORTED)
|
||||
set_target_properties(openvino::core::dev PROPERTIES
|
||||
INTERFACE_LINK_LIBRARIES IE::ov_core_dev)
|
||||
endif()
|
||||
|
||||
if(TARGET IE::runtime::dev AND NOT TARGET openvino::runtime::dev)
|
||||
add_library(openvino::runtime::dev INTERFACE IMPORTED)
|
||||
set_target_properties(openvino::runtime::dev PROPERTIES
|
||||
INTERFACE_LINK_LIBRARIES IE::runtime::dev)
|
||||
endif()
|
||||
|
||||
if(ENABLE_SYSTEM_PUGIXML)
|
||||
find_dependency(PugiXML)
|
||||
set_property(TARGET pugixml PROPERTY IMPORTED_GLOBAL TRUE)
|
||||
|
||||
@@ -16,14 +16,14 @@ list(APPEND ov_options CMAKE_CXX_COMPILER_LAUNCHER CMAKE_C_COMPILER_LAUNCHER
|
||||
file(TO_CMAKE_PATH "${CMAKE_CURRENT_LIST_DIR}" cache_path)
|
||||
|
||||
message(STATUS "The following CMake options are exported from OpenVINO Developer package")
|
||||
message("")
|
||||
message(" ")
|
||||
foreach(option IN LISTS ov_options)
|
||||
if(NOT DEFINED "${option}")
|
||||
load_cache("${cache_path}" READ_WITH_PREFIX "" ${option})
|
||||
endif()
|
||||
message(" ${option}: ${${option}}")
|
||||
endforeach()
|
||||
message("")
|
||||
message(" ")
|
||||
|
||||
# for samples in 3rd party projects
|
||||
set_and_check(gflags_DIR "@gflags_BINARY_DIR@")
|
||||
|
||||
@@ -9,10 +9,10 @@
|
||||
<tab type="user" title="Attributes" url="@ref openvino_docs_OV_UG_lpt_attributes">
|
||||
<tab type="user" title="AvgPoolPrecisionPreserved" url="@ref openvino_docs_OV_UG_lpt_AvgPoolPrecisionPreserved"/>
|
||||
<tab type="user" title="IntervalsAlignment" url="@ref openvino_docs_OV_UG_lpt_IntervalsAlignment"/>
|
||||
<tab type="user" title="PerTensorQuantization" url="@ref openvino_docs_OV_UG_lpt_PerTensorQuantization"/>
|
||||
<tab type="user" title="PrecisionPreserved" url="@ref openvino_docs_OV_UG_lpt_PrecisionPreserved"/>
|
||||
<tab type="user" title="Precisions" url="@ref openvino_docs_OV_UG_lpt_Precisions"/>
|
||||
<tab type="user" title="QuantizationAlignment" url="@ref openvino_docs_OV_UG_lpt_QuantizationAlignment"/>
|
||||
<tab type="user" title="QuantizationGranularity" url="@ref openvino_docs_OV_UG_lpt_QuantizationGranularity"/>
|
||||
</tab>
|
||||
<tab type="user" title="Step 1. Prerequisites transformations" url="@ref openvino_docs_OV_UG_lpt_step1_prerequisites">
|
||||
<tab type="user" title="LinOpSequenceFusion" url="@ref openvino_docs_OV_UG_lpt_LinOpSequenceFusion"/>
|
||||
|
||||
-11
@@ -1,11 +0,0 @@
|
||||
# PerTensorQuantization attribute {#openvino_docs_OV_UG_lpt_PerTensorQuantization}
|
||||
|
||||
ngraph::PerTensorQuantizationAttribute class represents the `PerTensorQuantization` attribute.
|
||||
|
||||
The attribute defines if the operation input port requires per-tensor quantization.
|
||||
|
||||
| Property name | Values |
|
||||
|---------------|----------------------------------------------|
|
||||
| Required | Yes |
|
||||
| Defined | Operation, input ports |
|
||||
| Properties | |
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
# QuantizationGranularity attribute {#openvino_docs_OV_UG_lpt_QuantizationGranularity}
|
||||
|
||||
ngraph::QuantizationAttribute class represents the `QuantizationGranularity` attribute.
|
||||
|
||||
The attribute defines quantization granularity of operation inputs.
|
||||
|
||||
| Property name | Values |
|
||||
|---------------|----------------------------------------------|
|
||||
| Required | No |
|
||||
| Defined | Input ports |
|
||||
| Properties | Quantization granularity |
|
||||
+12
-11
@@ -8,29 +8,30 @@
|
||||
:hidden:
|
||||
|
||||
AvgPoolPrecisionPreserved <openvino_docs_OV_UG_lpt_AvgPoolPrecisionPreserved>
|
||||
IntervalsAlignment <openvino_docs_OV_UG_lpt_IntervalsAlignment>
|
||||
PerTensorQuantization <openvino_docs_OV_UG_lpt_PerTensorQuantization>
|
||||
IntervalsAlignment <openvino_docs_OV_UG_lpt_IntervalsAlignment>
|
||||
PrecisionPreserved <openvino_docs_OV_UG_lpt_PrecisionPreserved>
|
||||
Precisions <openvino_docs_OV_UG_lpt_Precisions>
|
||||
QuantizationAlignment <openvino_docs_OV_UG_lpt_QuantizationAlignment>
|
||||
QuantizationGranularity <openvino_docs_OV_UG_lpt_QuantizationGranularity>
|
||||
|
||||
@endsphinxdirective
|
||||
|
||||
## Introduction
|
||||
|
||||
| Name | Target | Required | Mutable |
|
||||
|-------------------------------------------------------------------------------------|------------------------|----------|---------|
|
||||
| [AvgPoolPrecisionPreserved](@ref openvino_docs_OV_UG_lpt_AvgPoolPrecisionPreserved) | Precision | No | Yes |
|
||||
| [IntervalsAlignment](@ref openvino_docs_OV_UG_lpt_IntervalsAlignment) | Quantization interval | Yes | Yes |
|
||||
| [PerTensorQuantization](@ref openvino_docs_OV_UG_lpt_PerTensorQuantization) | Precision | Yes | No |
|
||||
| [PrecisionPreserved](@ref openvino_docs_OV_UG_lpt_PrecisionPreserved) | Precision | Yes | Yes |
|
||||
| [Precisions](@ref openvino_docs_OV_UG_lpt_Precisions) | Precision | Yes | Yes |
|
||||
| [QuantizationAlignment](@ref openvino_docs_OV_UG_lpt_QuantizationAlignment) | Quantization alignment | Yes | Yes |
|
||||
| Name | Target | Required | Mutable |
|
||||
|-------------------------------------------------------------------------------------|--------------------------|----------|---------|
|
||||
| [AvgPoolPrecisionPreserved](@ref openvino_docs_OV_UG_lpt_AvgPoolPrecisionPreserved) | Precision | No | Yes |
|
||||
| [IntervalsAlignment](@ref openvino_docs_OV_UG_lpt_IntervalsAlignment) | Quantization interval | Yes | Yes |
|
||||
| [PrecisionPreserved](@ref openvino_docs_OV_UG_lpt_PrecisionPreserved) | Precision | Yes | Yes |
|
||||
| [Precisions](@ref openvino_docs_OV_UG_lpt_Precisions) | Precision | Yes | Yes |
|
||||
| [QuantizationAlignment](@ref openvino_docs_OV_UG_lpt_QuantizationAlignment) | Quantization granularity | Yes | Yes |
|
||||
| [QuantizationGranularity](@ref openvino_docs_OV_UG_lpt_QuantizationGranularity) | Quantization granularity | Yes | No |
|
||||
|
||||
> `Target` attribute group defines attribute usage during model transformation for the best performance:
|
||||
> - `Precision` - the attribute defines the most optimal output port precision.
|
||||
> - `Quantization interval` - the attribute defines quantization interval.
|
||||
> - `Quantization alignment` - the attribute defines quantization alignment: per-channel or per-tensor quantization.
|
||||
> - `Quantization alignment` - the attribute defines quantization granularity in runtime: per-channel or per-tensor quantization.
|
||||
> - `Quantization granularity` - the attribute is set by plugin to define quantization granularity: per-channel or per-tensor quantization.
|
||||
>
|
||||
> `Required` attribute group defines if attribute usage is required to get an optimal model during transformation:
|
||||
> - `Yes` - the attribute is used by all OpenVINO plugins for low-precision optimization.
|
||||
|
||||
@@ -10,38 +10,45 @@
|
||||
|
||||
@endsphinxdirective
|
||||
|
||||
Auto Device (or `AUTO` in short) is a new special "virtual" or "proxy" device in the OpenVINO toolkit, it doesn’t bind to a specific type of HW device. AUTO solves the complexity in application required to code a logic for the HW device selection (through HW devices) and then, on the deducing the best optimization settings on that device. It does this by self-discovering all available accelerators & capabilities in the system, matching to the user’s performance requirements by respecting new “hints” configuration API to dynamically optimize for latency or throughput respectively. Developer can write application once and deploy anywhere.
|
||||
For developer who want to limit inference on specific HW candidates, AUTO also provide device priority list as optional property. After developer set device priority list, AUTO will not discover all available accelerators in the system and only try device in list with priority order.
|
||||
The Automatic Device Selection mode, or AUTO for short, uses a "virtual" or a "proxy" device,
|
||||
which does not bind to a specific type of hardware, but rather selects the processing unit for inference automatically.
|
||||
It detects available devices, picks the one best-suited for the task, and configures its optimization settings.
|
||||
This way, you can write the application once and deploy it anywhere.
|
||||
|
||||
AUTO always choose the best device, if compiling model fails on this device, AUTO will try to compile it on next best device until one of them succeeds.
|
||||
If priority list is set, AUTO only select devices according to the list.
|
||||
The selection also depends on your performance requirements, defined by the “hints” configuration API, as well as
|
||||
device priority list limitations, if you choose to exclude some hardware from the process.
|
||||
|
||||
The best device is chosen using the following logic:
|
||||
|
||||
1. Check which supported devices are available.
|
||||
2. Check the precision of the input model (for detailed information on precisions read more on the `ov::device::capabilities`)
|
||||
3. Select the first device capable of supporting the given precision, as presented in the table below.
|
||||
4. If the model’s precision is FP32 but there is no device capable of supporting it, offload the model to a device supporting FP16.
|
||||
The logic behind the choice is as follows:
|
||||
1. Check what supported devices are available.
|
||||
2. Check precisions of the input model (for detailed information on precisions read more on the `ov::device::capabilities`)
|
||||
3. Select the highest-priority device capable of supporting the given model, as listed in the table below.
|
||||
4. If model’s precision is FP32 but there is no device capable of supporting it, offload the model to a device supporting FP16.
|
||||
|
||||
+----------+------------------------------------------------------+-------------------------------------+
|
||||
| Choice || Supported || Supported |
|
||||
| Device || Supported || Supported |
|
||||
| Priority || Device || model precision |
|
||||
+==========+======================================================+=====================================+
|
||||
| 1 || dGPU || FP32, FP16, INT8, BIN |
|
||||
| || (e.g. Intel® Iris® Xe MAX) || |
|
||||
| 1 || dGPU | FP32, FP16, INT8, BIN |
|
||||
| || (e.g. Intel® Iris® Xe MAX) | |
|
||||
+----------+------------------------------------------------------+-------------------------------------+
|
||||
| 2 || iGPU || FP32, FP16, BIN |
|
||||
| || (e.g. Intel® UHD Graphics 620 (iGPU)) || |
|
||||
| 2 || iGPU | FP32, FP16, BIN |
|
||||
| || (e.g. Intel® UHD Graphics 620 (iGPU)) | |
|
||||
+----------+------------------------------------------------------+-------------------------------------+
|
||||
| 3 || Intel® Movidius™ Myriad™ X VPU || FP16 |
|
||||
| || (e.g. Intel® Neural Compute Stick 2 (Intel® NCS2)) || |
|
||||
| 3 || Intel® Movidius™ Myriad™ X VPU | FP16 |
|
||||
| || (e.g. Intel® Neural Compute Stick 2 (Intel® NCS2)) | |
|
||||
+----------+------------------------------------------------------+-------------------------------------+
|
||||
| 4 || Intel® CPU || FP32, FP16, INT8, BIN |
|
||||
| || (e.g. Intel® Core™ i7-1165G7) || |
|
||||
| 4 || Intel® CPU | FP32, FP16, INT8, BIN |
|
||||
| || (e.g. Intel® Core™ i7-1165G7) | |
|
||||
+----------+------------------------------------------------------+-------------------------------------+
|
||||
|
||||
What is important, **AUTO starts inference with the CPU by default except the priority list is set and there is no CPU in it**. CPU provides very low latency and can start inference with no additional delays. While it performs inference, the Auto-Device plugin continues to load the model to the device best suited for the purpose and transfers the task to it when ready. This way, the devices which are much slower in compile the model, GPU being the best example, do not impede inference at its initial stages.
|
||||
To put it simply, when loading the model to the first device on the list fails, AUTO will try to load it to the next device in line, until one of them succeeds.
|
||||
What is important, **AUTO always starts inference with the CPU**, as it provides very low latency and can start inference with no additional delays.
|
||||
While the CPU is performing inference, AUTO continues to load the model to the device best suited for the purpose and transfers the task to it when ready.
|
||||
This way, the devices which are much slower in compiling models, GPU being the best example, do not impede inference at its initial stages.
|
||||
For example, if you use a CPU and a GPU, first-inference latency of AUTO will be better than GPU itself.
|
||||
|
||||
Note that if you choose to exclude the CPU from the priority list, it will also be unable to support the initial model compilation stage.
|
||||
|
||||
![autoplugin_accelerate]
|
||||
|
||||
This mechanism can be easily observed in our Benchmark Application sample ([see here](#Benchmark App Info)), showing how the first-inference latency (the time it takes to compile the model and perform the first inference) is reduced when using AUTO. For example:
|
||||
@@ -58,20 +65,19 @@ This mechanism can be easily observed in our Benchmark Application sample ([see
|
||||
./benchmark_app -m ../public/alexnet/FP32/alexnet.xml -d AUTO -niter 128
|
||||
@endsphinxdirective
|
||||
|
||||
Assume there are CPU and GPU on the machine, first-inference latency of "AUTO" will be better than "GPU".
|
||||
|
||||
@sphinxdirective
|
||||
.. note::
|
||||
The realtime performance will be closer to the best suited device the longer the process runs.
|
||||
The longer the process runs, the closer realtime performance will be to that of the best-suited device.
|
||||
@endsphinxdirective
|
||||
|
||||
## Using the Auto-Device Plugin
|
||||
|
||||
Inference with AUTO is configured similarly to other plugins: compile the model on the plugin whth configuration, and finally, execute inference.
|
||||
|
||||
Following the OpenVINO™ naming convention, the Auto-Device plugin is assigned the label of “AUTO.” It may be defined with no additional parameters, resulting in defaults being used, or configured further with the following setup options:
|
||||
Following the OpenVINO™ naming convention, the Automatic Device Selection mode is assigned the label of “AUTO.” It may be defined with no additional parameters, resulting in defaults being used, or configured further with the following setup options:
|
||||
|
||||
@sphinxdirective
|
||||
|
||||
+---------------------------+-----------------------------------------------+-----------------------------------------------------------+
|
||||
| Property | Property values | Description |
|
||||
+===========================+===============================================+===========================================================+
|
||||
@@ -93,8 +99,12 @@ Following the OpenVINO™ naming convention, the Auto-Device plugin is assigned
|
||||
| | | ov::hint::Priority::MEDIUM | | Importantly! |
|
||||
| | | ov::hint::Priority::LOW | | This property is still not fully supported |
|
||||
+---------------------------+-----------------------------------------------+-----------------------------------------------------------+
|
||||
|
||||
@endsphinxdirective
|
||||
|
||||
Inference with AUTO is configured similarly to when device plugins are used:
|
||||
you compile the model on the plugin with configuration and execute inference.
|
||||
|
||||
### Device candidate list
|
||||
The device candidate list allows users to customize the priority and limit the choice of devices available to the AUTO plugin. If not specified, the plugin assumes all the devices present in the system can be used. Note, that OpenVINO™ Runtime lets you use “GPU” as an alias for “GPU.0” in function calls.
|
||||
The following commands are accepted by the API:
|
||||
@@ -179,7 +189,7 @@ The property enables you to control the priorities of models in the Auto-Device
|
||||
@endsphinxdirective
|
||||
|
||||
## Configuring Individual Devices and Creating the Auto-Device plugin on Top
|
||||
Although the methods described above are currently the preferred way to execute inference with AUTO, the following steps can be also used as an alternative. It is currently available as a legacy feature and used if the device candidate list includes Myriad (devices uncapable of utilizing the Performance Hints option).
|
||||
Although the methods described above are currently the preferred way to execute inference with AUTO, the following steps can be also used as an alternative. It is currently available as a legacy feature and used if the device candidate list includes Myriad devices, uncapable of utilizing the Performance Hints option.
|
||||
|
||||
@sphinxdirective
|
||||
|
||||
|
||||
+1
-1
@@ -26,7 +26,7 @@
|
||||
| LPR | License-Plate Recognition |
|
||||
| LRN | Local Response Normalization |
|
||||
| mAP | Mean Average Precision |
|
||||
| Intel(R) MKL-DNN | Intel(R) Math Kernel Library Deep Neural Networks |
|
||||
| Intel(R) OneDNN | Intel(R) OneAPI Deep Neural Network Library |
|
||||
| MO | Model Optimizer |
|
||||
| MVN | Mean Variance Normalization |
|
||||
| NCDHW | Number of images, Channels, Depth, Height, Width |
|
||||
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:61e237b3ced7eaa0cf1f8c2688753867b172712925068a4a47e07b5c71e48bdf
|
||||
size 89866
|
||||
@@ -1,6 +1,9 @@
|
||||
# General Optimizations {#openvino_docs_deployment_optimization_guide_common}
|
||||
|
||||
## Inputs Pre-processing with OpenVINO
|
||||
This chapter covers application-level optimization techniques such as asynchronous execution to improve data pipelining, pre-processing acceleration and so on.
|
||||
While the techniques (e.g. pre-processing) can be specific to end-user applications, the associated performance improvements are general and shall improve any target scenario (both latency and throughput).
|
||||
|
||||
## Inputs Pre-Processing with OpenVINO
|
||||
|
||||
In many cases, a network expects a pre-processed image, so make sure you do not perform unnecessary steps in your code:
|
||||
- Model Optimizer can efficiently bake the mean and normalization (scale) values into the model (for example, to the weights of the first convolution). Please see [relevant Model Optimizer command-line options](../MO_DG/prepare_model/Additional_Optimizations.md).
|
||||
|
||||
@@ -9,36 +9,53 @@
|
||||
openvino_docs_deployment_optimization_guide_common
|
||||
openvino_docs_deployment_optimization_guide_latency
|
||||
openvino_docs_deployment_optimization_guide_tput
|
||||
openvino_docs_deployment_optimization_guide_hints
|
||||
openvino_docs_deployment_optimization_guide_tput_advanced
|
||||
openvino_docs_deployment_optimization_guide_internals
|
||||
|
||||
@endsphinxdirective
|
||||
|
||||
## Deployment Optimizations Overview {#openvino_docs_deployment_optimization_guide_overview}
|
||||
Runtime or deployment optimizations are focused on tuning of the inference _parameters_ (e.g. optimal number of the requests executed simultaneously) and other means of how a model is _executed_.
|
||||
Runtime or deployment optimizations are focused on tuning of the inference _parameters_ (e.g. optimal number of the requests executed simultaneously) and other means of how a model is _executed_.
|
||||
|
||||
As referenced in the parent [performance introduction topic](./dldt_optimization_guide.md), the [dedicated document](./model_optimization_guide.md) covers the **model-level optimizations** like quantization that unlocks the 8-bit inference. Model-optimizations are most general and help any scenario and any device (that e.g. accelerates the quantized models). The relevant _runtime_ configuration is `ov::hint::inference_precision` allowing the devices to trade the accuracy for the performance (e.g. by allowing the fp16/bf16 execution for the layers that remain in fp32 after quantization of the original fp32 model).
|
||||
As referenced in the parent [performance introduction topic](./dldt_optimization_guide.md), the [dedicated document](./model_optimization_guide.md) covers the **model-level optimizations** like quantization that unlocks the 8-bit inference. Model-optimizations are most general and help any scenario and any device (that e.g. accelerates the quantized models). The relevant _runtime_ configuration is `ov::hint::inference_precision` which trades the accuracy for the performance (e.g. by allowing the fp16/bf16 execution for the layers that remain in fp32 after quantization of the original fp32 model).
|
||||
|
||||
Then, possible optimization should start with defining the use-case. For example, whether the target scenario emphasizes throughput over latency like processing millions of samples by overnight jobs in the data centers.
|
||||
In contrast, real-time usages would likely trade off the throughput to deliver the results at minimal latency. Often this is a combined scenario that targets highest possible throughput while maintaining a specific latency threshold.
|
||||
Below you can find summary on the associated tips.
|
||||
Below you can find summary on the associated tips.
|
||||
|
||||
How the full-stack application uses the inference component _end-to-end_ is also important. For example, what are the stages that needs to be orchestrated? In some cases a significant part of the workload time is spent on bringing and preparing the input data. Below you can find multiple tips on connecting the data input pipeline and the model inference efficiently.
|
||||
These are also common performance tricks that help both latency and throughput scenarios.
|
||||
|
||||
Further documents cover the associated _runtime_ performance optimizations topics. Please also consider [matrix support of the features by the individual devices](@ref features_support_matrix).
|
||||
|
||||
**General, application-level optimizations**, and specifically:
|
||||
Further documents cover the associated _runtime_ performance optimizations topics. Please also consider [matrix support of the features by the individual devices](@ref features_support_matrix).
|
||||
|
||||
[General, application-level optimizations](dldt_deployment_optimization_common.md), and specifically:
|
||||
|
||||
* [Inputs Pre-processing with the OpenVINO](../OV_Runtime_UG/preprocessing_overview.md)
|
||||
|
||||
* [Async API and 'get_tensor' Idiom](./dldt_deployment_optimization_common.md)
|
||||
* [Async API and 'get_tensor' Idiom](dldt_deployment_optimization_common.md)
|
||||
|
||||
* For variably-sized inputs, consider [dynamic shapes](../OV_Runtime_UG/ov_dynamic_shapes.md)
|
||||
|
||||
**Use-case specific optimizations** along with some implementation details:
|
||||
**Use-case specific optimizations** such as optimizing for [latency](./dldt_deployment_optimization_latency.md) or [throughput](./dldt_deployment_optimization_tput.md)
|
||||
|
||||
* Optimizing for [throughput](./dldt_deployment_optimization_tput.md) and [latency](./dldt_deployment_optimization_latency.md)
|
||||
## Writing Performance Portable Inference Application
|
||||
Each of the OpenVINO's [supported devices](../OV_Runtime_UG/supported_plugins/Supported_Devices.md) offers a bunch of low-level performance settings.
|
||||
Tweaking this detailed configuration requires deep architecture understanding.
|
||||
|
||||
* [OpenVINO's high-level performance hints](./dldt_deployment_optimization_hints.md) as the portable, future-proof approach for performance configuration, thar does not requires re-tuning when the model or device has changed.
|
||||
* **If the performance portability is of concern, consider using the [hints](../OV_Runtime_UG/performance_hints.md) first.**
|
||||
Also, while the resulting performance may be optimal for the specific combination of the device and the model that is inferred, it is actually neither device/model nor future-proof:
|
||||
- Even within a family of the devices (like various CPUs), different instruction set, or number of CPU cores would eventually result in different execution configuration to be optimal.
|
||||
- Similarly the optimal batch size is very much specific to the particular instance of the GPU.
|
||||
- Compute vs memory-bandwidth requirements for the model being inferenced, as well as inference precision, possible model's quantization also contribute to the optimal parameters selection.
|
||||
- Finally, the optimal execution parameters of one device do not transparently map to another device type, for example:
|
||||
- Both the CPU and GPU devices support the notion of the [streams](./dldt_deployment_optimization_tput_advanced.md), yet the optimal number of the streams is deduced very differently.
|
||||
|
||||
Here, to mitigate the performance configuration complexity the **Performance Hints** offer the high-level "presets" for the **latency** and **throughput**, as detailed in the [Performance Hints usage document](../OV_Runtime_UG/performance_hints.md).
|
||||
|
||||
Beyond execution _parameters_ there is a device-specific _scheduling_ that greatly affects the performance.
|
||||
Specifically, GPU-oriented optimizations like batching, which combines many (potentially tens) of inputs to achieve optimal throughput, do not always map well to the CPU, as e.g. detailed in the [further internals](dldt_deployment_optimization_internals.md) sections.
|
||||
|
||||
The hints really hide the _execution_ specifics required to saturate the device. In the [internals](dldt_deployment_optimization_internals.md) sections you can find the implementation details (particularly how the OpenVINO implements the 'throughput' approach) for the specific devices. Keep in mind that the hints make this transparent to the application. For example, the hints obviates the need for explicit (application-side) batching or streams.
|
||||
|
||||
With the hints, it is enough to keep separate infer requests per camera or another source of input and process the requests in parallel using Async API as explained in the [application design considerations section](@ref throughput_app_design). The main requirement for the application to leverage the throughput is **running multiple inference requests in parallel**.
|
||||
|
||||
|
||||
In summary, when the performance _portability_ is of concern, consider the Performance Hints as a solution. You may find further details and API examples [here](../OV_Runtime_UG/performance_hints.md).
|
||||
@@ -1,22 +0,0 @@
|
||||
# High-level Performance Hints (Presets) {#openvino_docs_deployment_optimization_guide_hints}
|
||||
|
||||
Traditionally, each of the OpenVINO's [supported devices](../OV_Runtime_UG/supported_plugins/Supported_Devices.md) offers a bunch of low-level performance settings.
|
||||
Tweaking this detailed configuration requires deep architecture understanding.
|
||||
Also, while the resulting performance may be optimal for the specific combination of the device and the model that is inferred, it is actually neither device/model nor future-proof:
|
||||
- Even within a family of the devices (like various CPUs), things like different number of CPU cores would eventually result in different execution configuration to be optimal.
|
||||
- Similarly the optimal batch size is very much specific to the particular instance of the GPU.
|
||||
- Compute vs memory-bandwidth requirements for the model being inferenced, as well as inference precision, possible model's quantization and other factors add more unknowns to the resulting performance equation.
|
||||
- Finally, the optimal execution parameters of one device do not transparently map to another device type, for example:
|
||||
- Both the CPU and GPU devices support the notion of the 'streams' (i.e. inference instances that are executed in parallel, please see `ov::num_streams`), yet the optimal number of the streams is deduced very differently.
|
||||
|
||||
Beyond execution _parameters_ there are potentially many device-specific details like _scheduling_ that greatly affect the performance.
|
||||
Specifically, GPU-oriented tricks like batching, which combines many (potentially tens) of input images to achieve optimal throughput, do not always map well to the CPU, as e.g. detailed in the next sections.
|
||||
The hints allow to really hide _execution_ specifics required to saturate the device. For example, no need to explicitly combine multiple inputs into a batch to achieve good GPU performance.
|
||||
Instead, it is possible to keep a separate infer request per camera or another source of input and process the requests in parallel using Async API as explained in the [common-optimizations section](@ref openvino_docs_deployment_optimization_guide_common).
|
||||
|
||||
The only requirement for the application to leverage the throughput is about **running multiple inference requests in parallel**.
|
||||
OpenVINO's device-specific implementation of the hints will take care of the rest. This allows a developer to greatly simplify the app-logic.
|
||||
|
||||
In summary, when the performance _portability_ is of concern, consider the [High-Level Performance Hints](../OV_Runtime_UG/performance_hints.md).
|
||||
Below you can find the implementation details (particularly how the OpenVINO implements the 'throughput' approach) for the specific devices.
|
||||
Keep in mind that while different throughput-oriented scheduling approaches ([like the batching or other means of executing individual inference requests](./dldt_deployment_optimization_tput.md)) can work together, the hints make these decisions to be transparent to the application.
|
||||
@@ -20,9 +20,9 @@ As expected, the easiest way to achieve the lowest latency is **running only one
|
||||
|
||||
However, some conventional "root" devices (e.g. CPU or GPU) can be in fact internally composed of several "sub-devices". In many cases letting the OpenVINO to transparently leverage the "sub-devices" helps to improve the application throughput (e.g. serve multiple clients simultaneously) without degrading the latency. For example, multi-socket CPUs can deliver as high number of requests (at the same minimal latency) as there are NUMA nodes in the machine. Similarly, a multi-tile GPU (which is essentially multiple GPUs in a single package), can deliver a multi-tile scalability with the number of inference requests, while preserving the single-tile latency.
|
||||
|
||||
Thus, human expertise is required to get more _throughput_ out of the device even in the inherently latency-oriented cases. OpenVINO can take this configuration burden via [high-level performance hints](../OV_Runtime_UG/performance_hints.md).
|
||||
Thus, human expertise is required to get more _throughput_ out of the device even in the inherently latency-oriented cases. OpenVINO can take this configuration burden via [high-level performance hints](../OV_Runtime_UG/performance_hints.md), via `ov::hint::PerformanceMode::LATENCY` specified for the `ov::hint::performance_mode` property for the compile_model.
|
||||
|
||||
> **NOTE**: [OpenVINO performance hints](./dldt_deployment_optimization_hints.md) is a recommended way for performance configuration, which is both device-agnostic and future-proof.
|
||||
> **NOTE**: [OpenVINO performance hints](../OV_Runtime_UG/performance_hints.md) is a recommended way for performance configuration, which is both device-agnostic and future-proof.
|
||||
|
||||
In the case when there are multiple models to be used simultaneously, consider using different devices for inferencing the different models. Finally, when multiple models are executed in parallel on the device, using additional `ov::hint::model_priority` may help to define relative priorities of the models (please refer to the documentation on the [matrix features support for OpenVINO devices](@ref features_support_matrix) to check for the support of the feature by the specific device).
|
||||
|
||||
|
||||
@@ -1,79 +1,50 @@
|
||||
# Optimizing for Throughput {#openvino_docs_deployment_optimization_guide_tput}
|
||||
|
||||
## General Throughput Considerations
|
||||
As described in the section on the [latency-specific considerations](./dldt_deployment_optimization_latency.md) one possible use-case is delivering every single request at the minimal delay.
|
||||
As described in the section on the [latency-specific considerations](./dldt_deployment_optimization_latency.md) one possible use-case is _delivering every single request at the minimal delay_.
|
||||
Throughput on the other hand, is about inference scenarios in which potentially large **number of inference requests are served simultaneously to improve the device utilization**.
|
||||
|
||||
Here, the overall application inference rate can be significantly improved with the right performance configuration.
|
||||
Also, if the model is not already memory bandwidth-limited, the associated increase in latency is not linearly dependent on the number of requests executed in parallel.
|
||||
With the OpenVINO there are two major means of processing multiple inputs simultaneously: **batching** and **streams**, explained in this document.
|
||||
The associated increase in latency is not linearly dependent on the number of requests executed in parallel.
|
||||
Here, a trade-off between overall throughput and serial performance of individual requests can be achieved with the right OpenVINO performance configuration.
|
||||
|
||||
## OpenVINO Streams
|
||||
As detailed in the [common-optimizations section](@ref openvino_docs_deployment_optimization_guide_common) running multiple inference requests asynchronously is important for general application efficiency.
|
||||
The [Asynchronous API](./dldt_deployment_optimization_common.md) is in fact the "application side" of scheduling, as every device internally implements a queue. The queue acts as a buffer, storing the inference requests until retrieved by the device at its own pace.
|
||||
## Basic and Advanced Ways of Leveraging Throughput
|
||||
With the OpenVINO there are two means of leveraging the throughput with the individual device:
|
||||
* **Basic (high-level)** flow with [OpenVINO performance hints](../OV_Runtime_UG/performance_hints.md) which is inherently **portable and future-proof**
|
||||
* **Advanced (low-level)** approach of explicit **batching** and **streams**, explained in the separate [document](dldt_deployment_optimization_tput_advanced.md).
|
||||
|
||||
Further, the devices may actually process multiple inference requests in parallel in order to improve the device utilization and overall throughput. This parallelism is commonly referred as 'streams'. Some devices (like GPU) may run several requests per stream to amortize the host-side costs.
|
||||
Notice that streams are **really executing the requests in parallel, but not in the lock step** (as e.g. the batching does), which makes the streams fully compatible with [dynamically-shaped inputs](../OV_Runtime_UG/ov_dynamic_shapes.md) when individual requests can have different shapes.
|
||||
In both cases application should be designed to execute multiple inference requests in parallel as detailed in the [next section](@ref throughput_app_design).
|
||||
|
||||
For efficient asynchronous execution, the streams are actually handling inference with special pool of the threads.
|
||||
So each time you start inference requests (potentially from different application threads), they are actually muxed into a inference queue of the particular `ov:Compiled_Model`.
|
||||
If there is a vacant stream, it pops the request from the queue and actually expedites that to the on-device execution.
|
||||
Finally, consider the _automatic_ multi-device execution covered below.
|
||||
|
||||
The multi-streams approach is inherently throughput-oriented, as every stream requires a dedicated device memory to do inference in parallel to the rest of streams.
|
||||
Although similar, the streams are always preferable compared to creating multiple `ov:Compiled_Model` instances for the same model, as weights memory is shared across streams, reducing the overall memory consumption.
|
||||
Notice that the streams inflate the model load/compilation time.
|
||||
Finally, using streams does increase the latency of an individual request, this is why for example the [latency hint](./dldt_deployment_optimization_hints.md) governs a device to create a bare minimum of streams (usually just one).
|
||||
Please find the considerations for the optimal number of the streams in the later sections.
|
||||
@anchor throughput_app_design
|
||||
## Throughput-Oriented Application Design
|
||||
Most generally, throughput-oriented inference applications should:
|
||||
* Expose substantial amounts of _inputs_ parallelism (e.g. process multiple video- or audio- sources, text documents, etc)
|
||||
* Decompose the data flow into a collection of concurrent inference requests that are aggressively scheduled to be executed in parallel
|
||||
* Setup the configuration for the _device_ (e.g. as parameters of the `ov::Core::compile_model`) via either [low-level explicit options](dldt_deployment_optimization_tput_advanced.md), introduced in the previous section or [OpenVINO performance hints](../OV_Runtime_UG/performance_hints.md) (**preferable**):
|
||||
@sphinxdirective
|
||||
|
||||
## Batching
|
||||
Hardware accelerators like GPUs are optimized for massive compute parallelism, so the batching helps to saturate the device and leads to higher throughput.
|
||||
While the streams (described) earlier already allow to hide the communication overheads and certain bubbles in the scheduling, running multiple OpenCL kernels simultaneously is less GPU-efficient, compared to calling a kernel on the multiple inputs at once.
|
||||
As explained in the next section, the batching is a must to leverage maximum throughput on the GPUs.
|
||||
.. tab:: C++
|
||||
|
||||
There are two primary ways of using the batching to help application performance:
|
||||
* Collecting the inputs explicitly on the application side and then _sending these batched requests to the OpenVINO_
|
||||
* Although this gives flexibility with the possible batching strategies, the approach requires redesigning the application logic
|
||||
* _Sending individual requests_, while configuring the OpenVINO to collect and perform inference on the requests in batch [automatically](../OV_Runtime_UG/automatic_batching.md).
|
||||
In both cases, optimal batch size is very device-specific. Also as explained below, the optimal batch size depends on the model, inference precision and other factors.
|
||||
.. doxygensnippet:: docs/snippets/ov_auto_batching.cpp
|
||||
:language: cpp
|
||||
:fragment: [compile_model]
|
||||
|
||||
## Choosing the Batch Size and Number of Streams
|
||||
Predicting the inference performance is difficult and finding optimal execution parameters requires direct experiments with measurements.
|
||||
One possible throughput optimization strategy is to **set an upper bound for latency and then increase the batch size or number of the streams until that tail latency is met (or the throughput is not growing anymore)**.
|
||||
Also, consider [Deep Learning Workbench](@ref workbench_docs_Workbench_DG_Introduction) that builds handy latency vs throughput charts, iterating over possible values of the batch size and number of streams.
|
||||
.. tab:: Python
|
||||
|
||||
Different devices behave differently with the batch sizes. The optimal batch size depends on the model, inference precision and other factors. Similarly, different devices require different number of execution streams to maximize the throughput.
|
||||
Below are general recommendations:
|
||||
* For the **CPU always prefer the streams** over the batching
|
||||
* Create as many streams as you application runs the requests simultaneously
|
||||
* Number of streams should be enough to meet the _average_ parallel slack rather than the peak load
|
||||
* _Maximum number of streams_ equals **total number of CPU cores**
|
||||
* As explained in the [CPU streams internals](dldt_deployment_optimization_internals.md), the CPU cores are evenly distributed between streams, so one core per stream is the finest-grained configuration
|
||||
* For the **GPU**:
|
||||
* When the parallel slack is small (e.g. only 2-4 requests executed simultaneously), then using the streams for the GPU may suffice
|
||||
* Notice that the GPU runs 2 request per stream
|
||||
* _Maximum number of streams_ is usually 2, for more portability consider using the `ov::streams::AUTO` (`GPU_THROUGHPUT_AUTO` in the pre-OpenVINO 2.0 parlance)
|
||||
* Typically, for 4 and more requests the batching delivers better throughput for the GPUs
|
||||
* Batch size can be calculated as "number of inference requests executed _in parallel_" divided by the "number of requests that the streams consume"
|
||||
* E.g. if you process 16 cameras (by 16 requests inferenced _simultaneously_) with 2 GPU streams (each can process 2 requests), the batch size per request is 16/(2*2)=4
|
||||
.. doxygensnippet:: docs/snippets/ov_auto_batching.py
|
||||
:language: python
|
||||
:fragment: [compile_model]
|
||||
|
||||
> **NOTE**: When playing with [dynamically-shaped inputs](../OV_Runtime_UG/ov_dynamic_shapes.md) use only the streams (no batching), as they tolerate individual requests having different shapes.
|
||||
|
||||
> **NOTE**: Using the [High-Level Performance Hints](../OV_Runtime_UG/performance_hints.md) explained in the next section, is the most portable and future-proof option, allowing the OpenVINO to find best combination of streams and batching for a given scenario and model.
|
||||
|
||||
## OpenVINO Hints: Selecting Optimal Execution and Parameters **Automatically**
|
||||
Overall, the latency-throughput is not linearly dependent and very _device_ specific. It is also tightly integrated with _model_ characteristics.
|
||||
As for the possible inference devices the scenery had already become pretty diverse, the OpenVINO has introduced the dedicated notion of the high-level performance configuration "hints" to describe the target application scenarios.
|
||||
The hints are described [here](./dldt_deployment_optimization_hints.md).
|
||||
|
||||
The hints also obviates the need for explicit (application-side) batching. With the hints, the only requirement for the application is to run multiple individual requests using [Async API](./dldt_deployment_optimization_common.md) and let the OpenVINO decide whether to collect the requests and execute them in batch, streams, or both.
|
||||
|
||||
> **NOTE**: [OpenVINO performance hints](./dldt_deployment_optimization_hints.md) is a recommended way for performance configuration, which is both device-agnostic and future-proof.
|
||||
@endsphinxdirective
|
||||
* Query the `ov::optimal_number_of_infer_requests` from the `ov::CompiledModel` (resulted from compilation of the model for a device) to create the number of the requests required to saturate the device
|
||||
* Use the Async API with callbacks, to avoid any dependency on the requests' completion order and possible device starvation, as explained in the [common-optimizations section](@ref openvino_docs_deployment_optimization_guide_common)
|
||||
|
||||
## Multi-Device Execution
|
||||
OpenVINO offers _automatic_, [scalable multi-device inference](../OV_Runtime_UG/multi_device.md). This is simple _application-transparent_ way to improve the throughput. No need to re-architecture existing applications for any explicit multi-device support: no explicit network loading to each device, no separate per-device queues, no additional logic to balance the inference requests between devices, etc. From the application point of view, it is communicating to the single device that internally handles the actual machinery.
|
||||
OpenVINO offers automatic, [scalable multi-device inference](../OV_Runtime_UG/multi_device.md). This is simple _application-transparent_ way to improve the throughput. No need to re-architecture existing applications for any explicit multi-device support: no explicit network loading to each device, no separate per-device queues, no additional logic to balance the inference requests between devices, etc. From the application point of view, it is communicating to the single device that internally handles the actual machinery.
|
||||
Just like with other throughput-oriented scenarios, there are two major pre-requisites for optimal multi-device performance:
|
||||
* Using the [Asynchronous API](@ref openvino_docs_deployment_optimization_guide_common) and [callbacks](../OV_Runtime_UG/ov_infer_request.md) in particular
|
||||
* Providing the multi-device (and hence the underlying devices) with enough data to crunch. As the inference requests are naturally independent data pieces, the multi-device performs load-balancing at the “requests” (outermost) level to minimize the scheduling overhead.
|
||||
|
||||
Notice that the resulting performance is usually a fraction of the “ideal” (plain sum) value, when the devices compete for a certain resources, like the memory-bandwidth which is shared between CPU and iGPU.
|
||||
> **NOTE**: While the legacy approach of optimizing the parameters of each device separately works, the [OpenVINO performance hints](./dldt_deployment_optimization_hints.md) allow to configure all devices (that are part of the specific multi-device configuration) at once.
|
||||
> **NOTE**: While the legacy approach of optimizing the parameters of each device separately works, the [OpenVINO performance hints](../OV_Runtime_UG/performance_hints.md) allow to configure all devices (that are part of the specific multi-device configuration) at once.
|
||||
@@ -0,0 +1,77 @@
|
||||
# Using Advanced Throughput Options: Streams and Batching {#openvino_docs_deployment_optimization_guide_tput_advanced}
|
||||
|
||||
## OpenVINO Streams
|
||||
As detailed in the [common-optimizations section](@ref openvino_docs_deployment_optimization_guide_common) running multiple inference requests asynchronously is important for general application efficiency.
|
||||
Internally, every device implements a queue. The queue acts as a buffer, storing the inference requests until retrieved by the device at its own pace.
|
||||
The devices may actually process multiple inference requests in parallel in order to improve the device utilization and overall throughput.
|
||||
This configurable mean of this device-side parallelism is commonly referred as **streams**.
|
||||
|
||||
> **NOTE**: Notice that streams are **really executing the requests in parallel, but not in the lock step** (as e.g. the batching does), which makes the streams fully compatible with [dynamically-shaped inputs](../OV_Runtime_UG/ov_dynamic_shapes.md) when individual requests can have different shapes.
|
||||
|
||||
> **NOTE**: Most OpenVINO devices (including CPU, GPU and VPU) support the streams, yet the _optimal_ number of the streams is deduced very differently, please see the a dedicated section below.
|
||||
|
||||
Few general considerations:
|
||||
* Using the streams does increase the latency of an individual request
|
||||
* When no number of streams is not specified, a device creates a bare minimum of streams (usually just one), as the latency-oriented case is default
|
||||
* Please find further tips for the optimal number of the streams [below](@ref throughput_advanced)
|
||||
* Streams are memory-hungry, as every stream duplicates the intermediate buffers to do inference in parallel to the rest of streams
|
||||
* Always prefer streams over creating multiple `ov:Compiled_Model` instances for the same model, as weights memory is shared across streams, reducing the memory consumption
|
||||
* Notice that the streams also inflate the model load (compilation) time.
|
||||
|
||||
For efficient asynchronous execution, the streams are actually handling the inference with a special pool of the threads (a thread per stream).
|
||||
Each time you start inference requests (potentially from different application threads), they are actually muxed into a inference queue of the particular `ov:Compiled_Model`.
|
||||
If there is a vacant stream, it pops the request from the queue and actually expedites that to the on-device execution.
|
||||
There are further device-specific details e.g. for the CPU, that you may find in the [internals](dldt_deployment_optimization_internals.md) section.
|
||||
|
||||
## Batching
|
||||
Hardware accelerators like GPUs are optimized for massive compute parallelism, so the batching helps to saturate the device and leads to higher throughput.
|
||||
While the streams (described earlier) already help to hide the communication overheads and certain bubbles in the scheduling, running multiple OpenCL kernels simultaneously is less GPU-efficient, compared to calling a kernel on the multiple inputs at once.
|
||||
As explained in the next section, the batching is a must to leverage maximum throughput on the GPUs.
|
||||
|
||||
There are two primary ways of using the batching to help application performance:
|
||||
* Collecting the inputs explicitly on the application side and then _sending these batched requests to the OpenVINO_
|
||||
* Although this gives flexibility with the possible batching strategies, the approach requires redesigning the application logic
|
||||
* _Sending individual requests_, while configuring the OpenVINO to collect and perform inference on the requests in batch [automatically](../OV_Runtime_UG/automatic_batching.md).
|
||||
In both cases, optimal batch size is very device-specific. Also as explained below, the optimal batch size depends on the model, inference precision and other factors.
|
||||
|
||||
@anchor throughput_advanced
|
||||
## Choosing the Number of Streams and/or Batch Size
|
||||
Predicting the inference performance is difficult and finding optimal execution parameters requires direct experiments with measurements.
|
||||
Run performance testing in the scope of development, and make sure to validate overall (end-to-end) application performance.
|
||||
|
||||
Different devices behave differently with the batch sizes. The optimal batch size depends on the model, inference precision and other factors.
|
||||
Similarly, different devices require different number of execution streams to saturate.
|
||||
Finally, in some cases combination of streams and batching may be required to maximize the throughput.
|
||||
|
||||
One possible throughput optimization strategy is to **set an upper bound for latency and then increase the batch size and/or number of the streams until that tail latency is met (or the throughput is not growing anymore)**.
|
||||
Also, consider [OpenVINO Deep Learning Workbench](@ref workbench_docs_Workbench_DG_Introduction) that builds handy latency vs throughput charts, iterating over possible values of the batch size and number of streams.
|
||||
|
||||
> **NOTE**: When playing with [dynamically-shaped inputs](../OV_Runtime_UG/ov_dynamic_shapes.md) use only the streams (no batching), as they tolerate individual requests having different shapes.
|
||||
|
||||
> **NOTE**: Using the [High-Level Performance Hints](../OV_Runtime_UG/performance_hints.md) is the alternative, portable and future-proof option, allowing the OpenVINO to find best combination of streams and batching for a given scenario and model.
|
||||
|
||||
### Number of Streams Considerations
|
||||
* Select the number of streams is it is **less or equal** to the number of requests that your application would be able to runs simultaneously
|
||||
* To avoid wasting resources, the number of streams should be enough to meet the _average_ parallel slack rather than the peak load
|
||||
* As a more portable option (that also respects the underlying hardware configuration) use the `ov::streams::AUTO`
|
||||
* It is very important to keep these streams busy, by running as many inference requests as possible (e.g. start the newly-arrived inputs immediately)
|
||||
* Bare minimum of requests to saturate the device can be queried as `ov::optimal_number_of_infer_requests` of the `ov:Compiled_Model`
|
||||
* _Maximum number of streams_ for the device (per model) can be queried as the `ov::range_for_streams`
|
||||
|
||||
### Batch Size Considerations
|
||||
* Select the batch size that is **equal** to the number of requests that your application is able to runs simultaneously
|
||||
* Otherwise (or if the number of "available" requests fluctuates), you may need to keep several instances of the network (reshaped to the different batch size) and select the properly sized instance in the runtime accordingly
|
||||
* For OpenVINO devices that internally implement a dedicated heuristic, the `ov::optimal_batch_size` is a _device_ property (that accepts the actual model as a parameter) to query the recommended batch size for the model.
|
||||
|
||||
|
||||
### Few Device Specific Details
|
||||
* For the **GPU**:
|
||||
* When the parallel slack is small (e.g. only 2-4 requests executed simultaneously), then using only the streams for the GPU may suffice
|
||||
* Notice that the GPU runs 2 request per stream, so 4 requests can be served by 2 streams
|
||||
* Alternatively, consider single stream with with 2 requests (each with a small batch size like 2), which would total the same 4 inputs in flight
|
||||
* Typically, for 4 and more requests the batching delivers better throughput
|
||||
* Batch size can be calculated as "number of inference requests executed in parallel" divided by the "number of requests that the streams consume"
|
||||
* E.g. if you process 16 cameras (by 16 requests inferenced _simultaneously_) by the two GPU streams (each can process two requests), the batch size per request is 16/(2*2)=4
|
||||
|
||||
* For the **CPU always use the streams first**
|
||||
* On the high-end CPUs, using moderate (2-8) batch size _in addition_ to the maximum number of streams, may further improve the performance.
|
||||
@@ -9,11 +9,13 @@ Generally, performance means how fast the model processes the live data. Two key
|
||||
|
||||
|
||||
|
||||
**Latency** measures inference time (ms) required to process a single input. When it comes to the executing multiple inputs executed simultaneously (e.g. via batching) then the overall throughput (inferences per second, or frames per second, FPS, in the specific case of visual processing) is usually of more concern.
|
||||
**Latency** measures inference time (ms) required to process a single input. When it comes to the executing multiple inputs simultaneously (e.g. via batching) then the overall throughput (inferences per second, or frames per second, FPS, in the specific case of visual processing) is usually of more concern.
|
||||
To calculate **throughput**, divide number of inputs that were processed by the processing time.
|
||||
|
||||
## End-to-End Application Performance
|
||||
It is important to separate the "pure" inference time of a neural network and the end-to-end application performance. For example data transfers between the host and a device may unintentionally affect the performance when a host input tensor is processed on the accelerator like dGPU. Similarly, the image-preprocessing may also contribute significantly to the to inference time. As detailed in the [getting performance numbers](../MO_DG/prepare_model/Getting_performance_numbers.md) section, when drilling into _inference_ performance, one option is to measure all such items separately.
|
||||
It is important to separate the "pure" inference time of a neural network and the end-to-end application performance. For example data transfers between the host and a device may unintentionally affect the performance when a host input tensor is processed on the accelerator like dGPU.
|
||||
|
||||
Similarly, the input-preprocessing contributes significantly to the to inference time. As detailed in the [getting performance numbers](../MO_DG/prepare_model/Getting_performance_numbers.md) section, when drilling into _inference_ performance, one option is to measure all such items separately.
|
||||
For the **end-to-end scenario** though, consider the image pre-processing thru the OpenVINO and the asynchronous execution as a way to amortize the communication costs like data transfers. You can find further details in the [general optimizations document](./dldt_deployment_optimization_common.md).
|
||||
|
||||
**First-inference latency** is another specific case (e.g. when fast application start-up is required) where the resulting performance may be well dominated by the model loading time. Consider [model caching](../OV_Runtime_UG/Model_caching_overview.md) as a way to improve model loading/compilation time.
|
||||
@@ -29,7 +31,8 @@ Finally, **memory footprint** restrictions is another possible concern when desi
|
||||
|
||||
With the OpenVINO there are two primary ways of improving the inference performance, namely model- and runtime-level optimizations. **These two optimizations directions are fully compatible**.
|
||||
|
||||
- **Model optimizations** includes model modification, such as quantization, pruning, optimization of preprocessing, etc. Fore more details, refer to this [document](./model_optimization_guide.md).
|
||||
- **Model optimizations** includes model modification, such as quantization, pruning, optimization of preprocessing, etc. Fore more details, refer to this [document](./model_optimization_guide.md).
|
||||
- Notice that the model optimizations directly improve the inference time, even without runtime parameters tuning, described below
|
||||
|
||||
- **Runtime (Deployment) optimizations** includes tuning of model _execution_ parameters. To read more visit the [Runtime Inference Optimizations](../optimization_guide/dldt_deployment_optimization_guide.md).
|
||||
|
||||
|
||||
@@ -53,14 +53,11 @@ if (NOT (OpenCL_FOUND AND OpenCL_HEADERS))
|
||||
endif()
|
||||
|
||||
# try to find VA libraries
|
||||
find_package(PkgConfig QUIET)
|
||||
if(PkgConfig_FOUND)
|
||||
pkg_search_module(LIBVA QUIET libva)
|
||||
endif()
|
||||
|
||||
# TODO: pkg_search_module finds libva not in sysroot
|
||||
if(ANDROID)
|
||||
set(LIBVA_FOUND OFF CACHE BOOL "" FORCE)
|
||||
if(NOT CMAKE_CROSSCOMPILING)
|
||||
find_host_package(PkgConfig QUIET)
|
||||
if(PkgConfig_FOUND)
|
||||
pkg_search_module(libva QUIET IMPORTED_TARGET libva)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# remove OpenCV related sources
|
||||
@@ -89,10 +86,9 @@ if(OpenCL_FOUND AND OpenCL_HEADERS)
|
||||
target_link_libraries(${TARGET_NAME} PRIVATE ${OpenCL_LIB})
|
||||
target_include_directories(${TARGET_NAME} PRIVATE ${OpenCL_HEADERS})
|
||||
|
||||
if(LIBVA_FOUND)
|
||||
if(libva_FOUND)
|
||||
target_compile_definitions(${TARGET_NAME} PRIVATE ENABLE_LIBVA)
|
||||
target_include_directories(${TARGET_NAME} PRIVATE ${LIBVA_INCLUDE_DIRS})
|
||||
target_link_libraries(${TARGET_NAME} PRIVATE ${LIBVA_LINK_LIBRARIES})
|
||||
target_link_libraries(${TARGET_NAME} PRIVATE PkgConfig::libva)
|
||||
endif()
|
||||
|
||||
if(WIN32)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
#include <transformations/low_precision/disable_convert_constant_folding_on_const_path.hpp>
|
||||
|
||||
#include <low_precision/common/operation_per_tensor_quantization_restriction.hpp>
|
||||
#include <low_precision/common/quantization_granularity_restriction.hpp>
|
||||
#include <low_precision/convert_subtract_constant.hpp>
|
||||
#include <low_precision/convolution.hpp>
|
||||
#include <low_precision/convolution_backprop_data.hpp>
|
||||
@@ -45,7 +45,7 @@ if (useLpt) {
|
||||
// nGraph common transformations happen here
|
||||
|
||||
if (useLpt) {
|
||||
// convert subtract constant to INT8 to prevent unnecessary FP16 to FP32 conversion
|
||||
// convert subtract constant to INT8 to prevent unnecessary FP16 to FP32 conversion
|
||||
manager.register_pass<ngraph::pass::low_precision::ConvertSubtractConstant>(defaultPrecisions);
|
||||
}
|
||||
|
||||
@@ -70,29 +70,29 @@ manager.run_passes(nGraphFunc);
|
||||
using namespace ngraph::pass::low_precision;
|
||||
if (useLpt) {
|
||||
// Low precision transformations plugin specific configuration: restrictions definition
|
||||
auto supportedPrecisions = std::vector<OperationPrecisionRestriction>({
|
||||
OperationPrecisionRestriction::create<ngraph::opset1::Convolution>({
|
||||
auto supportedPrecisions = std::vector<PrecisionsRestriction>({
|
||||
PrecisionsRestriction::create<ngraph::opset1::Convolution>({
|
||||
{0, {ngraph::element::u8}},
|
||||
{1, {ngraph::element::i8}},
|
||||
}),
|
||||
OperationPrecisionRestriction::create<ngraph::opset1::ConvolutionBackpropData>({
|
||||
PrecisionsRestriction::create<ngraph::opset1::ConvolutionBackpropData>({
|
||||
{0, {ngraph::element::u8, ngraph::element::i8}},
|
||||
{1, {ngraph::element::i8}}
|
||||
}),
|
||||
OperationPrecisionRestriction::create<ngraph::opset1::GroupConvolution>({
|
||||
PrecisionsRestriction::create<ngraph::opset1::GroupConvolution>({
|
||||
{0, {ngraph::element::u8}},
|
||||
{1, {ngraph::element::i8}}
|
||||
}),
|
||||
OperationPrecisionRestriction::create<ngraph::opset1::Multiply>({
|
||||
PrecisionsRestriction::create<ngraph::opset1::Multiply>({
|
||||
{0, {ngraph::element::u8}},
|
||||
{1, {ngraph::element::i8}},
|
||||
}),
|
||||
});
|
||||
|
||||
// Low precision transformations plugin specific configuration: per-tensor quantization operations definition
|
||||
auto perTensorQuantization = std::vector<OperationPerTensorQuantizationRestriction>({
|
||||
OperationPerTensorQuantizationRestriction::create<ngraph::opset1::Convolution>({0}),
|
||||
OperationPerTensorQuantizationRestriction::create<ngraph::opset1::ConvolutionBackpropData>({0})
|
||||
auto perTensorQuantization = std::vector<QuantizationGranularityRestriction>({
|
||||
QuantizationGranularityRestriction::create<ngraph::opset1::Convolution>({0}),
|
||||
QuantizationGranularityRestriction::create<ngraph::opset1::ConvolutionBackpropData>({0})
|
||||
});
|
||||
|
||||
// Low precision transformations instantiation and registration in pass manager
|
||||
@@ -133,8 +133,8 @@ ngraph::pass::Manager manager;
|
||||
|
||||
using namespace ngraph::pass::low_precision;
|
||||
//! [lpt_supported_precisions]
|
||||
auto supportedPrecisions = std::vector<OperationPrecisionRestriction>({
|
||||
OperationPrecisionRestriction::create<ngraph::opset1::Convolution>({
|
||||
auto supportedPrecisions = std::vector<PrecisionsRestriction>({
|
||||
PrecisionsRestriction::create<ngraph::opset1::Convolution>({
|
||||
{0, {ngraph::element::u8}},
|
||||
{1, {ngraph::element::i8}},
|
||||
}),
|
||||
@@ -157,10 +157,10 @@ std::shared_ptr<ov::Model> nGraphFunc;
|
||||
//! [per_tensor_quantization]
|
||||
using namespace ngraph::pass::low_precision;
|
||||
|
||||
const std::vector<OperationPrecisionRestriction> emptyRestrictions;
|
||||
const std::vector<PrecisionsRestriction> emptyRestrictions;
|
||||
|
||||
auto perTensorQuantization = std::vector<OperationPerTensorQuantizationRestriction>({
|
||||
OperationPerTensorQuantizationRestriction::create<ngraph::opset1::Convolution>({0})
|
||||
auto perTensorQuantization = std::vector<QuantizationGranularityRestriction>({
|
||||
QuantizationGranularityRestriction::create<ngraph::opset1::Convolution>({0})
|
||||
});
|
||||
|
||||
ngraph::pass::Manager lptManager;
|
||||
@@ -197,15 +197,15 @@ ngraph::pass::Manager manager;
|
||||
|
||||
using namespace ngraph::pass::low_precision;
|
||||
//! [lpt_markup_pipeline]
|
||||
auto supportedPrecisions = std::vector<OperationPrecisionRestriction>({
|
||||
OperationPrecisionRestriction::create<ngraph::opset1::Convolution>({
|
||||
auto supportedPrecisions = std::vector<PrecisionsRestriction>({
|
||||
PrecisionsRestriction::create<ngraph::opset1::Convolution>({
|
||||
{0, {ngraph::element::u8}},
|
||||
{1, {ngraph::element::i8}},
|
||||
}),
|
||||
});
|
||||
|
||||
auto perTensorQuantization = std::vector<OperationPerTensorQuantizationRestriction>({
|
||||
OperationPerTensorQuantizationRestriction::create<ngraph::opset1::Convolution>({0})
|
||||
auto perTensorQuantization = std::vector<QuantizationGranularityRestriction>({
|
||||
QuantizationGranularityRestriction::create<ngraph::opset1::Convolution>({0})
|
||||
});
|
||||
|
||||
ngraph::pass::Manager lptManager;
|
||||
|
||||
@@ -44,7 +44,7 @@ if [ -f /etc/lsb-release ]; then
|
||||
$x86_64_specific_packages \
|
||||
libgtk2.0-dev \
|
||||
pkg-config \
|
||||
libflags-dev \
|
||||
libgflags-dev \
|
||||
zlib1g-dev \
|
||||
unzip \
|
||||
automake \
|
||||
@@ -79,8 +79,13 @@ if [ -f /etc/lsb-release ]; then
|
||||
else
|
||||
sudo -E apt-get install -y libpng-dev
|
||||
fi
|
||||
if apt-cache search --names-only '^nlohmann-json3-dev'| grep -q nlohmann-json3; then
|
||||
sudo -E apt-get install -y nlohmann-json3-dev
|
||||
else
|
||||
sudo -E apt-get install -y nlohmann-json-dev
|
||||
fi
|
||||
elif [ -f /etc/redhat-release ]; then
|
||||
# CentOS 7.x
|
||||
# RHEL 8
|
||||
sudo -E yum install -y centos-release-scl epel-release
|
||||
sudo -E yum install -y \
|
||||
wget \
|
||||
@@ -100,8 +105,8 @@ elif [ -f /etc/redhat-release ]; then
|
||||
gcc-c++ \
|
||||
make \
|
||||
pkg-config \
|
||||
libflags-dev \
|
||||
zlib1g-dev \
|
||||
gflags-devel.i686 \
|
||||
zlib-devel.i686 \
|
||||
glibc-static \
|
||||
glibc-devel \
|
||||
libstdc++-static \
|
||||
@@ -157,8 +162,9 @@ elif [ -f /etc/os-release ] && grep -q "raspbian" /etc/os-release; then
|
||||
libboost-regex-dev \
|
||||
libgtk2.0-dev \
|
||||
pkg-config \
|
||||
libflags-dev \
|
||||
libgflags-dev \
|
||||
zlib1g-dev \
|
||||
nlohmann-json-dev \
|
||||
unzip \
|
||||
automake \
|
||||
libtool \
|
||||
|
||||
+17
-10
@@ -91,6 +91,12 @@ if(ARM AND NOT CMAKE_CROSSCOMPILING)
|
||||
add_compile_options(-march=armv7-a)
|
||||
endif()
|
||||
|
||||
if(NOT COMMAND find_host_package)
|
||||
macro(find_host_package)
|
||||
find_package(${ARGN})
|
||||
endmacro()
|
||||
endif()
|
||||
|
||||
set(CMAKE_POLICY_DEFAULT_CMP0063 NEW)
|
||||
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
|
||||
set(CMAKE_CXX_VISIBILITY_PRESET hidden)
|
||||
@@ -106,8 +112,6 @@ if(NOT DEFINED CMAKE_CXX_STANDARD)
|
||||
endif()
|
||||
####################################
|
||||
|
||||
# TODO: add find_package calls for zlib and json
|
||||
|
||||
if(NOT TARGET gflags)
|
||||
find_package(gflags QUIET COMPONENTS nothreads_shared)
|
||||
|
||||
@@ -119,13 +123,16 @@ if(NOT TARGET gflags)
|
||||
endif()
|
||||
|
||||
if(NOT TARGET zlib::zlib)
|
||||
find_package(PkgConfig QUIET)
|
||||
if(PkgConfig_FOUND)
|
||||
pkg_search_module(zlib QUIET
|
||||
IMPORTED_TARGET
|
||||
zlib)
|
||||
if(zlib_FOUND)
|
||||
add_library(zlib::zlib ALIAS PkgConfig::zlib)
|
||||
if(NOT CMAKE_CROSSCOMPILING)
|
||||
find_host_package(PkgConfig QUIET)
|
||||
if(PkgConfig_FOUND)
|
||||
pkg_search_module(zlib QUIET
|
||||
IMPORTED_TARGET
|
||||
zlib)
|
||||
if(zlib_FOUND)
|
||||
set_target_properties(PkgConfig::zlib PROPERTIES IMPORTED_GLOBAL ON)
|
||||
add_library(zlib::zlib ALIAS PkgConfig::zlib)
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
@@ -144,7 +151,7 @@ if(NOT TARGET nlohmann_json::nlohmann_json)
|
||||
find_package(nlohmann_json QUIET)
|
||||
if(nlohmann_json_FOUND)
|
||||
message(STATUS "nlohmann_json (${nlohmann_json_VERSION}) is found at ${nlohmann_json_DIR}")
|
||||
elseif(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/nlohmann_json")
|
||||
elseif(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/json")
|
||||
add_subdirectory(thirdparty/json EXCLUDE_FROM_ALL)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
@@ -69,7 +69,7 @@ Options:
|
||||
In case of dynamic shapes networks with several inputs provide the same number of files for each input (except cases with single file for any input):"input1:1.jpg input2:1.bin", "input1:1.bin,2.bin input2:3.bin input3:4.bin,5.bin ". Also you can pass specific keys for inputs: "random" - for fillling input with random data, "image_info" - for filling input with image size.
|
||||
You should specify either one files set to be used for all inputs (without providing input names) or separate files sets for every input of model (providing inputs names).
|
||||
-d "<device>" Optional. Specify a target device to infer on (the list of available devices is shown below). Default value is CPU. Use "-d HETERO:<comma-separated_devices_list>" format to specify HETERO plugin. Use "-d MULTI:<comma-separated_devices_list>" format to specify MULTI plugin. The application looks for a suitable plugin for the specified device.
|
||||
-l "<absolute_path>" Required for CPU custom layers. Absolute path to a shared library with the kernels implementations.
|
||||
-extensions "<absolute_path>" Required for custom layers (extensions). Absolute path to a shared library with the kernels implementations.
|
||||
Or
|
||||
-c "<absolute_path>" Required for GPU custom kernels. Absolute path to an .xml file with the kernels description.
|
||||
-hint "performance hint (latency or throughput or none)" Optional. Performance hint allows the OpenVINO device to select the right network-specific settings.
|
||||
|
||||
@@ -96,8 +96,8 @@ static const char enforce_bf16_message[] =
|
||||
" 'false' - disable bfloat16 regardless of platform support";
|
||||
|
||||
/// @brief message for user library argument
|
||||
static const char custom_cpu_library_message[] =
|
||||
"Required for CPU custom layers. Absolute path to a shared library with the kernels "
|
||||
static const char custom_extensions_library_message[] =
|
||||
"Required for custom layers (extensions). Absolute path to a shared library with the kernels "
|
||||
"implementations.";
|
||||
|
||||
/// @brief message for clDNN custom kernels desc
|
||||
@@ -251,9 +251,9 @@ DEFINE_string(api, "async", api_message);
|
||||
/// @brief device the target device to infer on <br>
|
||||
DEFINE_string(d, "CPU", target_device_message);
|
||||
|
||||
/// @brief Absolute path to CPU library with user layers <br>
|
||||
/// @brief Absolute path to extensions library with user layers <br>
|
||||
/// It is a required parameter
|
||||
DEFINE_string(l, "", custom_cpu_library_message);
|
||||
DEFINE_string(extensions, "", custom_extensions_library_message);
|
||||
|
||||
/// @brief Define parameter for clDNN custom kernels path <br>
|
||||
/// Default is ./lib
|
||||
@@ -373,8 +373,7 @@ static void show_usage() {
|
||||
std::cout << " -m \"<path>\" " << model_message << std::endl;
|
||||
std::cout << " -i \"<path>\" " << input_message << std::endl;
|
||||
std::cout << " -d \"<device>\" " << target_device_message << std::endl;
|
||||
std::cout << " -l \"<absolute_path>\" " << custom_cpu_library_message << std::endl;
|
||||
std::cout << " Or" << std::endl;
|
||||
std::cout << " -extensions \"<absolute_path>\" " << custom_extensions_library_message << std::endl;
|
||||
std::cout << " -c \"<absolute_path>\" " << custom_cldnn_message << std::endl;
|
||||
std::cout << " -hint \"performance hint (latency or throughput or none)\" " << hint_message << std::endl;
|
||||
std::cout << " -api \"<sync/async>\" " << api_message << std::endl;
|
||||
|
||||
@@ -209,10 +209,10 @@ int main(int argc, char* argv[]) {
|
||||
|
||||
ov::Core core;
|
||||
|
||||
if (FLAGS_d.find("CPU") != std::string::npos && !FLAGS_l.empty()) {
|
||||
// CPU plugin extensions is loaded as a shared library
|
||||
core.add_extension(FLAGS_l);
|
||||
slog::info << "CPU plugin extensions is loaded " << FLAGS_l << slog::endl;
|
||||
if (!FLAGS_extensions.empty()) {
|
||||
// Extensions are loaded as a shared library
|
||||
core.add_extension(FLAGS_extensions);
|
||||
slog::info << "Extensions are loaded: " << FLAGS_extensions << slog::endl;
|
||||
}
|
||||
|
||||
// Load clDNN Extensions
|
||||
@@ -225,7 +225,7 @@ int main(int argc, char* argv[]) {
|
||||
if (config.count("GPU") && config.at("GPU").count(CONFIG_KEY(CONFIG_FILE))) {
|
||||
auto ext = config.at("GPU").at(CONFIG_KEY(CONFIG_FILE)).as<std::string>();
|
||||
core.set_property("GPU", {{CONFIG_KEY(CONFIG_FILE), ext}});
|
||||
slog::info << "GPU extensions is loaded " << ext << slog::endl;
|
||||
slog::info << "GPU extensions are loaded: " << ext << slog::endl;
|
||||
}
|
||||
|
||||
slog::info << "OpenVINO: " << ov::get_openvino_version() << slog::endl;
|
||||
|
||||
@@ -14,7 +14,7 @@ usage() {
|
||||
exit 1
|
||||
}
|
||||
|
||||
samples_type=$(basename "$PWD")
|
||||
samples_type=$(basename "$( dirname "${BASH_SOURCE[0]-$0}" )" )
|
||||
build_dir="$HOME/inference_engine_${samples_type}_samples_build"
|
||||
sample_install_dir=""
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ target_include_directories(${TARGET_NAME}
|
||||
|
||||
find_package(OpenVINO REQUIRED COMPONENTS Runtime)
|
||||
|
||||
target_link_libraries(${TARGET_NAME} PUBLIC openvino::runtime gflags)
|
||||
target_link_libraries(${TARGET_NAME} PUBLIC openvino::runtime PRIVATE gflags)
|
||||
|
||||
if(COMMAND add_clang_format_target)
|
||||
add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME})
|
||||
|
||||
@@ -243,7 +243,7 @@ int main(int argc, char* argv[]) {
|
||||
};
|
||||
gnaPluginConfig[ov::intel_gna::execution_target.name()] = parse_target(FLAGS_exec_target);
|
||||
gnaPluginConfig[ov::intel_gna::compile_target.name()] = parse_target(FLAGS_compile_target);
|
||||
gnaPluginConfig[ov::intel_gna::memory_reuse.name()] = false;
|
||||
gnaPluginConfig[ov::intel_gna::memory_reuse.name()] = !FLAGS_memory_reuse_off;
|
||||
gnaPluginConfig[ov::intel_gna::pwl_max_error_percent.name()] = FLAGS_pwl_me;
|
||||
// -----------------------------------------------------------------------------------------------------
|
||||
// --------------------------- Write model to file --------------------------------------------------
|
||||
|
||||
@@ -51,6 +51,9 @@ static const char enable_log_message[] = "Optional. Enable GNA logging, which ma
|
||||
/// @brief message for performance counters
|
||||
static const char performance_counter_message[] = "Optional. Enables per-layer performance report.";
|
||||
|
||||
/// @brief message for disabling of compact (memory_reuse) mode
|
||||
static const char memory_reuse_message[] = "Optional. Disables memory optimizations for compiled model.";
|
||||
|
||||
/// @brief message for user library argument
|
||||
static const char custom_cpu_library_message[] = "Required for CPU plugin custom layers."
|
||||
"Absolute path to a shared library with the kernels implementations.";
|
||||
@@ -130,6 +133,9 @@ static const char pwl_max_error_percent_message[] = "Optional. The maximum perce
|
||||
/// \brief Define flag for showing help message <br>
|
||||
DEFINE_bool(h, false, help_message);
|
||||
|
||||
/// \brief Define flag for disabling compact (memory_reuse) mode <br>
|
||||
DEFINE_bool(memory_reuse_off, false, memory_reuse_message);
|
||||
|
||||
/// \brief Define parameter for set image file <br>
|
||||
/// It is a required parameter
|
||||
DEFINE_string(i, "", input_message);
|
||||
@@ -228,6 +234,7 @@ static void show_usage() {
|
||||
std::cout << " -pwl_me \"<double>\" " << pwl_max_error_percent_message << std::endl;
|
||||
std::cout << " -exec_target \"<string>\" " << execution_target_message << std::endl;
|
||||
std::cout << " -compile_target \"<string>\" " << compile_target_message << std::endl;
|
||||
std::cout << " -memory_reuse_off " << memory_reuse_message << std::endl;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
# Copyright (C) 2018-2022 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import argparse
|
||||
import re
|
||||
from typing import List, Tuple, Union
|
||||
|
||||
|
||||
def build_arg_parser() -> argparse.ArgumentParser:
|
||||
@@ -10,11 +12,12 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
args = parser.add_argument_group('Options')
|
||||
model = parser.add_mutually_exclusive_group(required=True)
|
||||
|
||||
args.add_argument('-h', '--help', action='help', help='Show this help message and exit.')
|
||||
model.add_argument('-m', '--model', type=str,
|
||||
help='Path to an .xml file with a trained model (required if -rg is missing).')
|
||||
model.add_argument('-rg', '--import_gna_model', type=str,
|
||||
help='Read GNA model from file using path/filename provided (required if -m is missing).')
|
||||
|
||||
args.add_argument('-h', '--help', action='help', help='Show this help message and exit.')
|
||||
args.add_argument('-i', '--input', required=True, type=str, help='Required. Path to an input file (.ark or .npz).')
|
||||
args.add_argument('-o', '--output', type=str,
|
||||
help='Optional. Output file name to save inference results (.ark or .npz).')
|
||||
@@ -32,8 +35,7 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
args.add_argument('-qb', '--quantization_bits', default=16, type=int, choices=(8, 16), metavar='[8, 16]',
|
||||
help='Optional. Weight bits for quantization: 8 or 16 (default 16).')
|
||||
args.add_argument('-sf', '--scale_factor', type=str,
|
||||
help='Optional. The user-specified input scale factor for quantization. '
|
||||
'If the model contains multiple inputs, provide scale factors by separating them with commas.')
|
||||
help='Optional. The user-specified input scale factor for quantization.')
|
||||
args.add_argument('-wg', '--export_gna_model', type=str,
|
||||
help='Optional. Write GNA model to file using path/filename provided.')
|
||||
args.add_argument('-we', '--export_embedded_gna_model', type=str,
|
||||
@@ -51,12 +53,6 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
help='Optional. Enables performance report (specify -a to ensure arch accurate results).')
|
||||
args.add_argument('-a', '--arch', default='CORE', type=str.upper, choices=('CORE', 'ATOM'), metavar='[CORE, ATOM]',
|
||||
help='Optional. Specify architecture. CORE, ATOM with the combination of -pc.')
|
||||
args.add_argument('-iname', '--input_layers', type=str,
|
||||
help='Optional. Layer names for input blobs. The names are separated with ",". '
|
||||
'Allows to change the order of input layers for -i flag. Example: Input1,Input2')
|
||||
args.add_argument('-oname', '--output_layers', type=str,
|
||||
help='Optional. Layer names for output blobs. The names are separated with ",". '
|
||||
'Allows to change the order of output layers for -o flag. Example: Output1:port,Output2:port.')
|
||||
args.add_argument('-cw_l', '--context_window_left', type=int, default=0,
|
||||
help='Optional. Number of frames for left context windows (default is 0). '
|
||||
'Works only with context window models. '
|
||||
@@ -72,7 +68,27 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
return parser
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
def parse_arg_with_names(arg_string: Union[str, None], separator: str = '=') -> Tuple[List[str], List[str]]:
|
||||
keys = []
|
||||
values = []
|
||||
|
||||
if isinstance(arg_string, str):
|
||||
for parameter in re.split(', |,', arg_string):
|
||||
if separator in parameter:
|
||||
key, value = parameter.split(separator)
|
||||
keys.append(key)
|
||||
values.append(value)
|
||||
else:
|
||||
values.append(parameter)
|
||||
|
||||
return keys, values
|
||||
|
||||
|
||||
def check_arg_with_names(arg: Tuple[List[str], List[str]]) -> bool:
|
||||
return True if len(arg[0]) == 0 and len(arg[1]) > 1 else False
|
||||
|
||||
|
||||
def parse_args(separator: str = '=') -> argparse.Namespace:
|
||||
"""Parse and validate command-line arguments."""
|
||||
parser = build_arg_parser()
|
||||
args = parser.parse_args()
|
||||
@@ -86,4 +102,32 @@ def parse_args() -> argparse.Namespace:
|
||||
if args.pwl_me < 0.0 or args.pwl_me > 100.0:
|
||||
parser.error('Invalid value for -pwl_me argument. It must be greater than 0.0 and less than 100.0')
|
||||
|
||||
args.input = parse_arg_with_names(args.input, separator)
|
||||
if check_arg_with_names(args.input):
|
||||
parser.error(
|
||||
'Invalid format for -i/--input argment. Please specify the parameter like this '
|
||||
f'<input_name1>{separator}<file1.ark/.npz>,<input_name2>{separator}<file2.ark/.npz> or just <file.ark/.npz> in case of one input.',
|
||||
)
|
||||
|
||||
args.scale_factor = parse_arg_with_names(args.scale_factor, separator)
|
||||
if check_arg_with_names(args.scale_factor):
|
||||
parser.error(
|
||||
'Invalid format for -sf/--scale_factor argment. Please specify the parameter like this '
|
||||
f'<input_name1>{separator}<sf1>,<input_name2>{separator}<sf2> or just <sf> to be applied to all inputs.',
|
||||
)
|
||||
|
||||
args.output = parse_arg_with_names(args.output, separator)
|
||||
if check_arg_with_names(args.output):
|
||||
parser.error(
|
||||
'Invalid format for -o/--output argment. Please specify the parameter like this '
|
||||
f'<output_name1>{separator}<output1.ark/.npz>,<output_name2>{separator}<output2.ark/.npz> or just <output.ark/.npz> in case of one output.',
|
||||
)
|
||||
|
||||
args.reference = parse_arg_with_names(args.reference, separator)
|
||||
if check_arg_with_names(args.reference):
|
||||
parser.error(
|
||||
'Invalid format for -r/--reference argment. Please specify the parameter like this '
|
||||
f'<output_name1>{separator}<reference1.ark/.npz>,<output_name2>{separator}<reference2.ark/.npz> or <reference.ark/.npz> in case of one output.',
|
||||
)
|
||||
|
||||
return args
|
||||
@@ -3,7 +3,6 @@
|
||||
# Copyright (C) 2018-2022 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import re
|
||||
import sys
|
||||
from io import BytesIO
|
||||
from timeit import default_timer
|
||||
@@ -16,9 +15,9 @@ from openvino.runtime import Core, InferRequest, Layout, Type, set_batch
|
||||
from arg_parser import parse_args
|
||||
from file_options import read_utterance_file, write_utterance_file
|
||||
from utils import (GNA_ATOM_FREQUENCY, GNA_CORE_FREQUENCY,
|
||||
compare_with_reference, get_scale_factor, log,
|
||||
parse_input_layouts, parse_outputs_from_args,
|
||||
parse_scale_factors, set_scale_factors)
|
||||
calculate_scale_factor, compare_with_reference,
|
||||
get_input_layouts, get_sorted_scale_factors, log,
|
||||
set_scale_factors)
|
||||
|
||||
|
||||
def do_inference(data: Dict[str, np.ndarray], infer_request: InferRequest, cw_l: int = 0, cw_r: int = 0) -> np.ndarray:
|
||||
@@ -79,12 +78,10 @@ def main():
|
||||
model = core.read_model(args.model)
|
||||
|
||||
# --------------------------- Step 3. Apply preprocessing -------------------------------------------------------------
|
||||
if args.output_layers:
|
||||
output_layer_names, output_layer_ports = parse_outputs_from_args(args)
|
||||
model.add_outputs(list(zip(output_layer_names, output_layer_ports)))
|
||||
model.add_outputs(args.output[0] + args.reference[0])
|
||||
|
||||
if args.layout:
|
||||
layouts = parse_input_layouts(args, model.inputs)
|
||||
layouts = get_input_layouts(args.layout, model.inputs)
|
||||
|
||||
ppp = PrePostProcessor(model)
|
||||
|
||||
@@ -125,23 +122,24 @@ def main():
|
||||
|
||||
# Set a GNA scale factor
|
||||
if args.import_gna_model:
|
||||
if args.scale_factor:
|
||||
log.warning(f'Custom scale factor will be used for imported GNA model: {args.import_gna_model}')
|
||||
set_scale_factors(plugin_config, parse_scale_factors(args))
|
||||
if args.scale_factor[1]:
|
||||
log.error(f'Custom scale factor can not be set for imported gna model: {args.import_gna_model}')
|
||||
return 1
|
||||
else:
|
||||
log.info(f'Using scale factor from the imported GNA model: {args.import_gna_model}')
|
||||
log.info(f'Using scale factor from provided imported gna model: {args.import_gna_model}')
|
||||
else:
|
||||
if args.scale_factor:
|
||||
set_scale_factors(plugin_config, parse_scale_factors(args))
|
||||
if args.scale_factor[1]:
|
||||
scale_factors = get_sorted_scale_factors(args.scale_factor, model.inputs)
|
||||
else:
|
||||
scale_factors = []
|
||||
|
||||
for file_name in re.split(', |,', args.input):
|
||||
for file_name in args.input[1]:
|
||||
_, utterances = read_utterance_file(file_name)
|
||||
scale_factors.append(get_scale_factor(utterances[0]))
|
||||
scale_factor = calculate_scale_factor(utterances[0])
|
||||
log.info('Using scale factor(s) calculated from first utterance')
|
||||
scale_factors.append(str(scale_factor))
|
||||
|
||||
log.info('Using scale factor(s) calculated from first utterance')
|
||||
set_scale_factors(plugin_config, scale_factors)
|
||||
set_scale_factors(plugin_config, scale_factors, model.inputs)
|
||||
|
||||
if args.export_embedded_gna_model:
|
||||
plugin_config['GNA_FIRMWARE_MODEL_IMAGE'] = args.export_embedded_gna_model
|
||||
@@ -175,59 +173,39 @@ def main():
|
||||
return 0
|
||||
|
||||
# --------------------------- Step 6. Set up input --------------------------------------------------------------------
|
||||
if args.input_layers:
|
||||
input_layer_names = re.split(', |,', args.input_layers)
|
||||
else:
|
||||
input_layer_names = [_input.any_name for _input in compiled_model.inputs]
|
||||
|
||||
input_file_names = re.split(', |,', args.input)
|
||||
input_layer_names = args.input[0] if args.input[0] else [_input.any_name for _input in compiled_model.inputs]
|
||||
input_file_names = args.input[1]
|
||||
|
||||
if len(input_layer_names) != len(input_file_names):
|
||||
log.error(f'Number of model inputs ({len(compiled_model.inputs)}) is not equal '
|
||||
f'to number of ark files ({len(input_file_names)})')
|
||||
sys.exit(-3)
|
||||
return 3
|
||||
|
||||
input_file_data = [read_utterance_file(file_name) for file_name in input_file_names]
|
||||
|
||||
infer_data = [
|
||||
{
|
||||
input_layer_names[j]: input_file_data[j].utterances[i]
|
||||
for j in range(len(input_layer_names))
|
||||
for j in range(len(input_file_data))
|
||||
}
|
||||
for i in range(len(input_file_data[0].utterances))
|
||||
]
|
||||
|
||||
if args.output_layers:
|
||||
output_layer_names, output_layer_ports = parse_outputs_from_args(args)
|
||||
# If a name of output layer contains a port number then concatenate output_layer_names and output_layer_ports
|
||||
if ':' in compiled_model.outputs[0].any_name:
|
||||
output_layer_names = [f'{output_layer_names[i]}:{output_layer_ports[i]}' for i in range(len(output_layer_names))]
|
||||
else:
|
||||
output_layer_names = [compiled_model.outputs[0].any_name]
|
||||
output_layer_names = args.output[0] if args.output[0] else [compiled_model.outputs[0].any_name]
|
||||
output_file_names = args.output[1]
|
||||
|
||||
if args.output:
|
||||
output_file_names = re.split(', |,', args.output)
|
||||
reference_layer_names = args.reference[0] if args.reference[0] else [compiled_model.outputs[0].any_name]
|
||||
reference_file_names = args.reference[1]
|
||||
|
||||
if len(output_layer_names) != len(output_file_names):
|
||||
log.error('The number of output files is not equal to the number of model outputs.')
|
||||
sys.exit(-6)
|
||||
reference_file_data = [read_utterance_file(file_name) for file_name in reference_file_names]
|
||||
|
||||
if args.reference:
|
||||
reference_file_names = re.split(', |,', args.reference)
|
||||
|
||||
if len(output_layer_names) != len(reference_file_names):
|
||||
log.error('The number of reference files is not equal to the number of model outputs.')
|
||||
sys.exit(-5)
|
||||
|
||||
reference_file_data = [read_utterance_file(file_name) for file_name in reference_file_names]
|
||||
|
||||
references = [
|
||||
{
|
||||
output_layer_names[j]: reference_file_data[j].utterances[i]
|
||||
for j in range(len(output_layer_names))
|
||||
}
|
||||
for i in range(len(input_file_data[0].utterances))
|
||||
]
|
||||
references = [
|
||||
{
|
||||
reference_layer_names[j]: reference_file_data[j].utterances[i]
|
||||
for j in range(len(reference_file_data))
|
||||
}
|
||||
for i in range(len(input_file_data[0].utterances))
|
||||
]
|
||||
|
||||
# --------------------------- Step 7. Create infer request ------------------------------------------------------------
|
||||
infer_request = compiled_model.create_infer_request()
|
||||
@@ -263,12 +241,12 @@ def main():
|
||||
log.info(f'Frames in utterance: {num_of_frames}')
|
||||
log.info(f'Average Infer time per frame: {avg_infer_time_per_frame * 1000:.2f}ms')
|
||||
|
||||
for name in output_layer_names:
|
||||
for name in set(reference_layer_names + output_layer_names):
|
||||
log.info('')
|
||||
log.info(f'Output blob name: {name}')
|
||||
log.info(f'Output layer name: {name}')
|
||||
log.info(f'Number scores per frame: {results[i][name].shape[1]}')
|
||||
|
||||
if args.reference:
|
||||
if name in references[i].keys():
|
||||
log.info('')
|
||||
compare_with_reference(results[i][name], references[i][name])
|
||||
|
||||
@@ -294,11 +272,10 @@ def main():
|
||||
log.info('')
|
||||
log.info(f'Total sample time: {total_infer_time * 1000:.2f}ms')
|
||||
|
||||
if args.output:
|
||||
for i, name in enumerate(output_layer_names):
|
||||
data = [results[i][name] for i in range(len(input_file_data[0].utterances))]
|
||||
write_utterance_file(output_file_names[i], input_file_data[0].keys, data)
|
||||
log.info(f'File {output_file_names[i]} was created!')
|
||||
for i in range(len(output_file_names)):
|
||||
log.info(f'Saving results from "{output_layer_names[i]}" layer to {output_file_names[i]}')
|
||||
data = [results[j][output_layer_names[i]] for j in range(len(input_file_data[0].utterances))]
|
||||
write_utterance_file(output_file_names[i], input_file_data[0].keys, data)
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------
|
||||
log.info('This sample is an API example, '
|
||||
|
||||
@@ -2,9 +2,7 @@
|
||||
# Copyright (C) 2018-2022 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import argparse
|
||||
import logging as log
|
||||
import re
|
||||
import sys
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
@@ -34,7 +32,7 @@ def compare_with_reference(result: np.ndarray, reference: np.ndarray):
|
||||
log.info(f'stdev error: {stdev_error:.7f}')
|
||||
|
||||
|
||||
def get_scale_factor(matrix: np.ndarray) -> float:
|
||||
def calculate_scale_factor(matrix: np.ndarray) -> float:
|
||||
"""Get scale factor for quantization using utterance matrix."""
|
||||
# Max to find scale factor
|
||||
target_max = 16384
|
||||
@@ -45,45 +43,31 @@ def get_scale_factor(matrix: np.ndarray) -> float:
|
||||
return target_max / max_val
|
||||
|
||||
|
||||
def set_scale_factors(plugin_config: dict, scale_factors: list):
|
||||
def set_scale_factors(plugin_config: Dict[str, str], scale_factors: List[str], inputs: List[Output]):
|
||||
"""Set a scale factor provided for each input."""
|
||||
for i, scale_factor in enumerate(scale_factors):
|
||||
log.info(f'For input {i} using scale factor of {scale_factor:.7f}')
|
||||
plugin_config[f'GNA_SCALE_FACTOR_{i}'] = str(scale_factor)
|
||||
for i in range(len(inputs)):
|
||||
log.info(f'For input {inputs[i].get_any_name()} using scale factor of {scale_factors[i]}')
|
||||
plugin_config[f'GNA_SCALE_FACTOR_{i}'] = scale_factors[i]
|
||||
|
||||
|
||||
def parse_scale_factors(args: argparse.Namespace) -> list:
|
||||
"""Get a list of scale factors for input files."""
|
||||
input_files = re.split(', |,', args.input)
|
||||
scale_factors = re.split(', |,', str(args.scale_factor))
|
||||
scale_factors = list(map(float, scale_factors))
|
||||
|
||||
if len(input_files) != len(scale_factors):
|
||||
log.error(f'Incorrect command line for multiple inputs: {len(scale_factors)} scale factors provided for '
|
||||
f'{len(input_files)} input files.')
|
||||
sys.exit(-7)
|
||||
|
||||
for i, scale_factor in enumerate(scale_factors):
|
||||
if float(scale_factor) < 0:
|
||||
log.error(f'Scale factor for input #{i} (counting from zero) is out of range (must be positive).')
|
||||
sys.exit(-8)
|
||||
|
||||
return scale_factors
|
||||
|
||||
|
||||
def parse_outputs_from_args(args: argparse.Namespace) -> Tuple[List[str], List[int]]:
|
||||
"""Get a list of outputs specified in the args."""
|
||||
name_and_port = [output.split(':') for output in re.split(', |,', args.output_layers)]
|
||||
try:
|
||||
return [name for name, _ in name_and_port], [int(port) for _, port in name_and_port]
|
||||
except ValueError:
|
||||
log.error('Incorrect value for -oname/--output_layers option, please specify a port for each output layer.')
|
||||
sys.exit(-4)
|
||||
|
||||
|
||||
def parse_input_layouts(args: argparse.Namespace, inputs: List[Output]) -> Dict[str, str]:
|
||||
if args.layout[0] == '[':
|
||||
return {_input.get_any_name(): args.layout[1:-1] for _input in inputs}
|
||||
def get_input_layouts(layout_string: str, inputs: List[Output]) -> Dict[str, str]:
|
||||
if layout_string[0] == '[':
|
||||
return {_input.get_any_name(): layout_string[1:-1] for _input in inputs}
|
||||
else:
|
||||
sep = '],' if ',' in args.layout else ']'
|
||||
return dict([_input.split('[') for _input in args.layout[:-1].split(sep)])
|
||||
sep = '],' if ',' in layout_string else ']'
|
||||
return dict([_input.split('[') for _input in layout_string[:-1].split(sep)])
|
||||
|
||||
|
||||
def get_sorted_scale_factors(scale_factor_arg: Tuple[List[str], List[str]], inputs: List[Output]) -> List[str]:
|
||||
if scale_factor_arg[0]:
|
||||
res = [1 for _ in range(len(inputs))]
|
||||
input_names = [_input.get_any_name() for _input in inputs]
|
||||
|
||||
for i in range(len(scale_factor_arg[0])):
|
||||
input_index = input_names.index(scale_factor_arg[0][i])
|
||||
res[input_index] = scale_factor_arg[1][i]
|
||||
|
||||
return res
|
||||
|
||||
else:
|
||||
return [scale_factor_arg[1][0] for _ in range(len(inputs))]
|
||||
@@ -101,7 +101,7 @@ if [ "$os" == "ubuntu18.04" ] ; then
|
||||
|
||||
pkgs_opencv_req=(libgtk-3-0 libgl1)
|
||||
pkgs_python=(python3 python3-dev python3-venv python3-setuptools python3-pip)
|
||||
pkgs_dev=(cmake pkg-config libflags-dev zlib1g-dev g++ gcc libc6-dev make curl sudo)
|
||||
pkgs_dev=(cmake pkg-config libgflags-dev zlib1g-dev nlohmann-json-dev g++ gcc libc6-dev make curl sudo)
|
||||
pkgs_myriad=(libusb-1.0-0)
|
||||
pkgs_cl_compiler=(libtinfo5)
|
||||
pkgs_opencv_opt=(
|
||||
@@ -122,7 +122,7 @@ elif [ "$os" == "ubuntu20.04" ] ; then
|
||||
|
||||
pkgs_opencv_req=(libgtk-3-0 libgl1)
|
||||
pkgs_python=(python3 python3-dev python3-venv python3-setuptools python3-pip)
|
||||
pkgs_dev=(cmake pkg-config g++ gcc libc6-dev libflags-dev zlib1g-dev make curl sudo)
|
||||
pkgs_dev=(cmake pkg-config g++ gcc libc6-dev libgflags-dev zlib1g-dev nlohmann-json3-dev make curl sudo)
|
||||
pkgs_myriad=(libusb-1.0-0)
|
||||
pkgs_cl_compiler=(libtinfo5)
|
||||
pkgs_opencv_opt=(
|
||||
@@ -143,7 +143,7 @@ elif [ "$os" == "rhel8" ] ; then
|
||||
|
||||
pkgs_opencv_req=(gtk3)
|
||||
pkgs_python=(python3 python3-devel python3-setuptools python3-pip)
|
||||
pkgs_dev=(gcc gcc-c++ make glibc libstdc++ libgcc cmake pkg-config libflags-dev zlib1g-dev curl sudo)
|
||||
pkgs_dev=(gcc gcc-c++ make glibc libstdc++ libgcc cmake pkg-config gflags-devel.i686 zlib-devel.i686 curl sudo)
|
||||
pkgs_myriad=()
|
||||
pkgs_opencv_opt=(
|
||||
gstreamer1
|
||||
|
||||
+1
-3
@@ -4,8 +4,6 @@
|
||||
|
||||
add_definitions(-DPROJECT_ROOT_DIR="${CMAKE_CURRENT_SOURCE_DIR}")
|
||||
|
||||
add_custom_target(ov_runtime_libraries)
|
||||
|
||||
include(cmake/install_tbb.cmake)
|
||||
|
||||
# CC library should be registered before other cc targets
|
||||
@@ -27,7 +25,7 @@ endif()
|
||||
add_subdirectory(plugins)
|
||||
add_subdirectory(inference)
|
||||
include(cmake/openvino.cmake)
|
||||
# preprocessing has dependency on ov_runtime for static build
|
||||
# preprocessing has dependency on `openvino` for static build
|
||||
add_subdirectory(common/preprocessing)
|
||||
|
||||
if(ENABLE_TESTS)
|
||||
|
||||
@@ -28,7 +28,6 @@ else()
|
||||
|
||||
add_subdirectory(src/compatibility/pyngraph)
|
||||
add_subdirectory(src/pyopenvino)
|
||||
|
||||
endif()
|
||||
|
||||
if(ENABLE_WHEEL)
|
||||
|
||||
@@ -2,6 +2,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
if(NOT TARGET py_ov_frontends)
|
||||
add_custom_target(py_ov_frontends)
|
||||
endif()
|
||||
|
||||
function(frontend_module TARGET FRAMEWORK INSTALL_COMPONENT)
|
||||
set(TARGET_NAME ${TARGET})
|
||||
|
||||
@@ -18,6 +22,7 @@ function(frontend_module TARGET FRAMEWORK INSTALL_COMPONENT)
|
||||
pybind11_add_module(${TARGET_NAME} MODULE ${SOURCES})
|
||||
|
||||
add_dependencies(${TARGET_NAME} pyopenvino)
|
||||
add_dependencies(py_ov_frontends ${TARGET_NAME})
|
||||
|
||||
target_include_directories(${TARGET_NAME} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}"
|
||||
"${PYTHON_SOURCE_DIR}/pyopenvino/utils/")
|
||||
|
||||
@@ -25,7 +25,7 @@ endif()
|
||||
# create target for openvino.wheel
|
||||
|
||||
set(openvino_wheel_deps ie_api)
|
||||
foreach(_target ov_runtime_libraries ie_plugins _pyngraph pyopenvino py_paddle_frontend py_tensorflow_frontend py_onnx_frontend)
|
||||
foreach(_target ie_api constants _pyngraph pyopenvino ov_plugins py_ov_frontends)
|
||||
if(TARGET ${_target})
|
||||
list(APPEND openvino_wheel_deps ${_target})
|
||||
endif()
|
||||
|
||||
+41
-22
@@ -2,25 +2,35 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
macro(ov_find_tbb)
|
||||
macro(ov_find_package_tbb)
|
||||
if(THREADING STREQUAL "TBB" OR THREADING STREQUAL "TBB_AUTO" AND NOT TBB_FOUND)
|
||||
find_package(TBB QUIET COMPONENTS tbb tbbmalloc)
|
||||
if(NOT ENABLE_SYSTEM_TBB)
|
||||
set(_find_package_no_args NO_SYSTEM_ENVIRONMENT_PATH
|
||||
NO_CMAKE_SYSTEM_PATH)
|
||||
endif()
|
||||
|
||||
find_package(TBB QUIET COMPONENTS tbb tbbmalloc
|
||||
${_find_package_no_args})
|
||||
|
||||
if(NOT TBB_FOUND)
|
||||
# system TBB failed to be found
|
||||
set(ENABLE_SYSTEM_TBB OFF)
|
||||
|
||||
# try to find TBB via custom scripts if have not found by default
|
||||
if(NOT TBB_FOUND AND IEDevScripts_DIR)
|
||||
# remove invalid TBB_DIR=TBB_DIR-NOTFOUND from cache
|
||||
unset(TBB_DIR CACHE)
|
||||
unset(TBB_DIR)
|
||||
|
||||
# use our custom scripts for old TBB versions
|
||||
# which are exposed via `export TBBROOT=<tbbroot>`
|
||||
# see https://github.com/openvinotoolkit/openvino/pull/1288
|
||||
find_package(TBB COMPONENTS tbb tbbmalloc
|
||||
# TBB on system is not found, download prebuilt one
|
||||
# if TBBROOT env variable is not defined
|
||||
ov_download_tbb()
|
||||
|
||||
# try to find one more time
|
||||
find_package(TBB QUIET COMPONENTS tbb tbbmalloc
|
||||
# can be provided by ov_download_tbb
|
||||
HINTS ${TBB_DIR}
|
||||
# fallback variant for TBB 2018 and older
|
||||
PATHS ${IEDevScripts_DIR}
|
||||
NO_CMAKE_FIND_ROOT_PATH
|
||||
NO_DEFAULT_PATH)
|
||||
else()
|
||||
message(STATUS "TBB (${TBB_VERSION}) is found at ${TBB_DIR}")
|
||||
${_find_package_no_args})
|
||||
endif()
|
||||
|
||||
# WA for oneTBB: it does not define TBB_IMPORTED_TARGETS
|
||||
@@ -32,22 +42,31 @@ macro(ov_find_tbb)
|
||||
endforeach()
|
||||
endif()
|
||||
|
||||
if (NOT TBB_FOUND)
|
||||
set(THREADING "SEQ" PARENT_SCOPE)
|
||||
message(WARNING "TBB was not found by the configured TBB_DIR/TBBROOT path.\
|
||||
SEQ method will be used.")
|
||||
else()
|
||||
message(STATUS "TBB (${TBB_VERSION}) is found at ${TBB_DIR}")
|
||||
endif()
|
||||
|
||||
unset(_find_package_no_args)
|
||||
endif()
|
||||
endmacro()
|
||||
|
||||
function(set_ie_threading_interface_for TARGET_NAME)
|
||||
if(THREADING STREQUAL "TBB" OR THREADING STREQUAL "TBB_AUTO" AND NOT TBB_FOUND)
|
||||
# find TBB
|
||||
ov_find_package_tbb()
|
||||
|
||||
# set variables to parent scope to prevent multiple invocations of find_package(TBB)
|
||||
# at the same CMakeLists.txt; invocations in different directories are allowed
|
||||
set(TBB_FOUND ${TBB_FOUND} PARENT_SCOPE)
|
||||
set(TBB_IMPORTED_TARGETS ${TBB_IMPORTED_TARGETS} PARENT_SCOPE)
|
||||
set(TBB_VERSION ${TBB_VERSION} PARENT_SCOPE)
|
||||
if (NOT TBB_FOUND)
|
||||
set(THREADING "SEQ" PARENT_SCOPE)
|
||||
message(WARNING "TBB was not found by the configured TBB_DIR/TBBROOT path.\
|
||||
SEQ method will be used.")
|
||||
endif ()
|
||||
set(TBB_DIR ${TBB_DIR} PARENT_SCOPE)
|
||||
set(ENABLE_SYSTEM_TBB ${ENABLE_SYSTEM_TBB} PARENT_SCOPE)
|
||||
endif()
|
||||
endmacro()
|
||||
|
||||
function(set_ie_threading_interface_for TARGET_NAME)
|
||||
# find TBB
|
||||
ov_find_tbb()
|
||||
|
||||
get_target_property(target_type ${TARGET_NAME} TYPE)
|
||||
|
||||
|
||||
@@ -5,16 +5,16 @@
|
||||
include(cmake/ie_parallel.cmake)
|
||||
|
||||
# pre-find TBB: need to provide TBB_IMPORTED_TARGETS used for installation
|
||||
ov_find_tbb()
|
||||
ov_find_package_tbb()
|
||||
|
||||
if(TBB_FOUND AND TBB_VERSION VERSION_GREATER_EQUAL 2021)
|
||||
message(STATUS "Static tbbbind_2_5 package usage is disabled, since oneTBB is used")
|
||||
set(ENABLE_TBBBIND_2_5 OFF)
|
||||
endif()
|
||||
|
||||
if(ENABLE_TBBBIND_2_5)
|
||||
# try to find prebuilt version of tbbbind_2_5
|
||||
elseif(ENABLE_TBBBIND_2_5)
|
||||
# download and find a prebuilt version of TBBBind_2_5
|
||||
ov_download_tbbbind_2_5()
|
||||
find_package(TBBBIND_2_5 QUIET)
|
||||
|
||||
if(TBBBIND_2_5_FOUND)
|
||||
message(STATUS "Static tbbbind_2_5 package is found")
|
||||
set_target_properties(${TBBBIND_2_5_IMPORTED_TARGETS} PROPERTIES
|
||||
@@ -22,8 +22,6 @@ if(ENABLE_TBBBIND_2_5)
|
||||
if(NOT BUILD_SHARED_LIBS)
|
||||
set(install_tbbbind ON)
|
||||
endif()
|
||||
else()
|
||||
message(WARNING "Static tbbbind_2_5 package is not found")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
@@ -36,7 +34,7 @@ if(THREADING MATCHES "^(TBB|TBB_AUTO)$")
|
||||
endif()
|
||||
|
||||
if(install_tbbbind)
|
||||
set(IE_TBBBIND_DIR "${TBBBIND_2_5}")
|
||||
set(IE_TBBBIND_DIR "${TBBBIND_2_5_DIR}")
|
||||
list(APPEND PATH_VARS "IE_TBBBIND_DIR")
|
||||
endif()
|
||||
|
||||
|
||||
@@ -91,7 +91,7 @@ target_link_libraries(${TARGET_NAME}_dev INTERFACE ${TARGET_NAME} pugixml::stati
|
||||
set_ie_threading_interface_for(${TARGET_NAME}_dev)
|
||||
set_target_properties(${TARGET_NAME}_dev PROPERTIES EXPORT_NAME runtime::dev)
|
||||
|
||||
openvino_developer_export_targets(COMPONENT core TARGETS ${TARGET_NAME}_dev)
|
||||
openvino_developer_export_targets(COMPONENT core TARGETS openvino::runtime::dev)
|
||||
|
||||
# Install static libraries for case BUILD_SHARED_LIBS=OFF
|
||||
ov_install_static_lib(${TARGET_NAME}_dev core)
|
||||
|
||||
@@ -11,5 +11,3 @@ add_subdirectory(legacy)
|
||||
add_subdirectory(transformations)
|
||||
add_subdirectory(offline_transformations)
|
||||
add_subdirectory(low_precision_transformations)
|
||||
|
||||
add_dependencies(ov_runtime_libraries openvino_gapi_preproc)
|
||||
@@ -50,4 +50,4 @@ ov_install_static_lib(${TARGET_NAME} core)
|
||||
|
||||
file(GLOB_RECURSE hdrs ${CMAKE_CURRENT_SOURCE_DIR}/include/*.h ${CMAKE_CURRENT_SOURCE_DIR}/include/*.hpp)
|
||||
add_clang_format_target(${TARGET_NAME}_clang FOR_SOURCES ${hdrs})
|
||||
openvino_developer_export_targets(COMPONENT core TARGETS openvino::conditional_compilation)
|
||||
openvino_developer_export_targets(COMPONENT openvino_common TARGETS openvino::conditional_compilation)
|
||||
@@ -36,4 +36,4 @@ target_include_directories(${TARGET_NAME} PUBLIC
|
||||
ov_install_static_lib(${TARGET_NAME} openvino_common)
|
||||
|
||||
add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME})
|
||||
openvino_developer_export_targets(COMPONENT core TARGETS openvino::itt)
|
||||
openvino_developer_export_targets(COMPONENT openvino_common TARGETS openvino::itt)
|
||||
@@ -65,7 +65,7 @@ set_target_properties(${TARGET_NAME} ${TARGET_NAME}_obj
|
||||
|
||||
# developer package
|
||||
|
||||
openvino_developer_export_targets(COMPONENT core TARGETS ${TARGET_NAME})
|
||||
openvino_developer_export_targets(COMPONENT core_legacy TARGETS ${TARGET_NAME})
|
||||
|
||||
# install
|
||||
|
||||
|
||||
-56
@@ -1,56 +0,0 @@
|
||||
// Copyright (C) 2018-2022 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include <ngraph/node.hpp>
|
||||
#include <ngraph/variant.hpp>
|
||||
|
||||
#include <low_precision/lpt_visibility.hpp>
|
||||
#include <ngraph/pass/graph_rewrite.hpp>
|
||||
|
||||
namespace ngraph {
|
||||
namespace pass {
|
||||
namespace low_precision {
|
||||
|
||||
class OperationPerTensorQuantizationRestriction {
|
||||
public:
|
||||
using RestrictedPorts = std::vector<size_t>;
|
||||
|
||||
ngraph::Node::type_info_t operationType;
|
||||
bool specifyVersion;
|
||||
std::vector<size_t> restrictedPorts;
|
||||
|
||||
OperationPerTensorQuantizationRestriction() = default;
|
||||
OperationPerTensorQuantizationRestriction(
|
||||
const ngraph::Node::type_info_t operationType,
|
||||
const bool specifyVersion,
|
||||
const RestrictedPorts& restrictedPorts) :
|
||||
operationType(operationType),
|
||||
specifyVersion(specifyVersion),
|
||||
restrictedPorts(restrictedPorts) {}
|
||||
|
||||
template <typename T>
|
||||
static OperationPerTensorQuantizationRestriction create(
|
||||
const RestrictedPorts& restrictedPorts = {},
|
||||
const bool specifyVersion = false) {
|
||||
return OperationPerTensorQuantizationRestriction(T::get_type_info_static(), specifyVersion, restrictedPorts);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static RestrictedPorts getPrecisionsByOperationType(std::vector<OperationPerTensorQuantizationRestriction>& restrictions) {
|
||||
for (const auto& restriction : restrictions) {
|
||||
if (restriction.operationType == T::get_type_info_static()) {
|
||||
return restriction.restrictedPorts;
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace low_precision
|
||||
} // namespace pass
|
||||
} // namespace ngraph
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
// Copyright (C) 2022 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include <ngraph/pass/pass.hpp>
|
||||
#include "low_precision/lpt_visibility.hpp"
|
||||
#include "low_precision/rt_info/quantization_granularity_attribute.hpp"
|
||||
|
||||
namespace ngraph {
|
||||
namespace pass {
|
||||
namespace low_precision {
|
||||
|
||||
class LP_TRANSFORMATIONS_API PortQuantizationGranularityRestriction {
|
||||
public:
|
||||
PortQuantizationGranularityRestriction(const size_t port, QuantizationGranularityAttribute::Granularity granularity) :
|
||||
port(port),
|
||||
granularity(granularity) {}
|
||||
|
||||
size_t port;
|
||||
QuantizationGranularityAttribute::Granularity granularity;
|
||||
};
|
||||
|
||||
} // namespace low_precision
|
||||
} // namespace pass
|
||||
} // namespace ngraph
|
||||
+6
-6
@@ -19,7 +19,7 @@ namespace ngraph {
|
||||
namespace pass {
|
||||
namespace low_precision {
|
||||
|
||||
class OperationPrecisionRestriction {
|
||||
class PrecisionsRestriction {
|
||||
public:
|
||||
using PrecisionsByPort = std::vector<std::pair<size_t, std::vector<ngraph::element::Type>>>;
|
||||
|
||||
@@ -27,8 +27,8 @@ public:
|
||||
bool specifyVersion;
|
||||
std::vector<std::pair<size_t, std::vector<ngraph::element::Type>>> precisionsByPort;
|
||||
|
||||
OperationPrecisionRestriction() = default;
|
||||
OperationPrecisionRestriction(
|
||||
PrecisionsRestriction() = default;
|
||||
PrecisionsRestriction(
|
||||
const ngraph::Node::type_info_t operationType,
|
||||
const bool specifyVersion,
|
||||
const PrecisionsByPort& precisionsByPort) :
|
||||
@@ -37,14 +37,14 @@ public:
|
||||
precisionsByPort(precisionsByPort) {}
|
||||
|
||||
template <typename T>
|
||||
static OperationPrecisionRestriction create(
|
||||
static PrecisionsRestriction create(
|
||||
const PrecisionsByPort& precisionsByPort,
|
||||
const bool specifyVersion = false) {
|
||||
return OperationPrecisionRestriction(T::get_type_info_static(), specifyVersion, precisionsByPort);
|
||||
return PrecisionsRestriction(T::get_type_info_static(), specifyVersion, precisionsByPort);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static PrecisionsByPort getPrecisionsByOperationType(std::vector<OperationPrecisionRestriction>& restrictions) {
|
||||
static PrecisionsByPort getPrecisionsByOperationType(std::vector<PrecisionsRestriction>& restrictions) {
|
||||
for (const auto& restriction : restrictions) {
|
||||
if (restriction.operationType == T::get_type_info_static()) {
|
||||
return restriction.precisionsByPort;
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
// Copyright (C) 2022 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include <ngraph/node.hpp>
|
||||
#include <ngraph/variant.hpp>
|
||||
|
||||
#include <low_precision/lpt_visibility.hpp>
|
||||
#include <ngraph/pass/graph_rewrite.hpp>
|
||||
#include <low_precision/rt_info/quantization_granularity_attribute.hpp>
|
||||
#include <low_precision/common/port_quantization_granularity_restriction.hpp>
|
||||
|
||||
|
||||
namespace ngraph {
|
||||
namespace pass {
|
||||
namespace low_precision {
|
||||
|
||||
class LP_TRANSFORMATIONS_API QuantizationGranularityRestriction {
|
||||
public:
|
||||
ngraph::Node::type_info_t operationType;
|
||||
bool specifyVersion;
|
||||
std::vector<PortQuantizationGranularityRestriction> restrictions;
|
||||
|
||||
QuantizationGranularityRestriction() = default;
|
||||
QuantizationGranularityRestriction(
|
||||
const ngraph::Node::type_info_t operationType,
|
||||
const bool specifyVersion,
|
||||
const std::vector<PortQuantizationGranularityRestriction>& restrictions) :
|
||||
operationType(operationType),
|
||||
specifyVersion(specifyVersion),
|
||||
restrictions(restrictions) {}
|
||||
|
||||
template <typename T>
|
||||
static QuantizationGranularityRestriction create(
|
||||
const std::vector<PortQuantizationGranularityRestriction>& restrictions,
|
||||
const bool specifyVersion) {
|
||||
return QuantizationGranularityRestriction(T::get_type_info_static(), specifyVersion, restrictions);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static QuantizationGranularityRestriction create(
|
||||
const std::vector<size_t>& restrictedPorts = {},
|
||||
const bool specifyVersion = false) {
|
||||
std::vector<PortQuantizationGranularityRestriction> restrictions;
|
||||
restrictions.reserve(restrictedPorts.size());
|
||||
for (auto i = 0ul; i < restrictedPorts.size(); ++i) {
|
||||
restrictions.push_back(PortQuantizationGranularityRestriction(
|
||||
restrictedPorts[i],
|
||||
ngraph::QuantizationGranularityAttribute::Granularity::PerTensor));
|
||||
}
|
||||
return QuantizationGranularityRestriction(T::get_type_info_static(), specifyVersion, restrictions);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static std::vector<PortQuantizationGranularityRestriction> getPrecisionsByOperationType(std::vector<QuantizationGranularityRestriction>& restrictions) {
|
||||
for (const auto& restriction : restrictions) {
|
||||
if (restriction.operationType == T::get_type_info_static()) {
|
||||
return restriction.restrictions;
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace low_precision
|
||||
} // namespace pass
|
||||
} // namespace ngraph
|
||||
@@ -21,7 +21,8 @@
|
||||
|
||||
#include <low_precision/lpt_visibility.hpp>
|
||||
#include <ngraph/pass/graph_rewrite.hpp>
|
||||
#include <low_precision/common/operation_per_tensor_quantization_restriction.hpp>
|
||||
#include <low_precision/common/quantization_granularity_restriction.hpp>
|
||||
#include <low_precision/common/precisions_restriction.hpp>
|
||||
#include "low_precision/layer_transformation.hpp"
|
||||
#include "low_precision/markup_precisions.hpp"
|
||||
|
||||
@@ -41,13 +42,13 @@ class ngraph::pass::low_precision::MarkupOptimizations : public ngraph::pass::Fu
|
||||
public:
|
||||
OPENVINO_RTTI("MarkupOptimizations", "0");
|
||||
MarkupOptimizations(
|
||||
const std::vector<OperationPrecisionRestriction>& precisionRestrictions,
|
||||
const std::vector<OperationPerTensorQuantizationRestriction>& quantizationRestrictions,
|
||||
const std::vector<PrecisionsRestriction>& precisionRestrictions,
|
||||
const std::vector<QuantizationGranularityRestriction>& quantizationRestrictions,
|
||||
const AttributeParameters& params);
|
||||
bool run_on_model(const std::shared_ptr<ngraph::Function>& m) override;
|
||||
private:
|
||||
const std::vector<OperationPrecisionRestriction>& precisionRestrictions;
|
||||
const std::vector<OperationPerTensorQuantizationRestriction>& quantizationRestrictions;
|
||||
const std::vector<PrecisionsRestriction>& precisionRestrictions;
|
||||
const std::vector<QuantizationGranularityRestriction>& quantizationRestrictions;
|
||||
const AttributeParameters& params;
|
||||
};
|
||||
|
||||
@@ -61,8 +62,8 @@ class ngraph::pass::low_precision::LowPrecision : public ngraph::pass::FunctionP
|
||||
public:
|
||||
OPENVINO_RTTI("LowPrecision", "0");
|
||||
LowPrecision(
|
||||
const std::vector<OperationPrecisionRestriction>& precisionRestrictions = {},
|
||||
const std::vector<OperationPerTensorQuantizationRestriction>& quantizationRestrictions = {},
|
||||
const std::vector<PrecisionsRestriction>& precisionRestrictions = {},
|
||||
const std::vector<QuantizationGranularityRestriction>& quantizationRestrictions = {},
|
||||
const LayerTransformation::Params = LayerTransformation::Params());
|
||||
bool run_on_model(const std::shared_ptr<ngraph::Function>& m) override;
|
||||
|
||||
@@ -70,8 +71,8 @@ public:
|
||||
static bool isFQLevelsPresent(const std::shared_ptr<const ngraph::Function>& function, const std::set<size_t>& levels);
|
||||
|
||||
protected:
|
||||
std::vector<OperationPrecisionRestriction> precisionRestrictions;
|
||||
std::vector<OperationPerTensorQuantizationRestriction> quantizationRestrictions;
|
||||
std::vector<PrecisionsRestriction> precisionRestrictions;
|
||||
std::vector<QuantizationGranularityRestriction> quantizationRestrictions;
|
||||
// remove
|
||||
LayerTransformation::Params params;
|
||||
};
|
||||
+2
-2
@@ -10,7 +10,7 @@
|
||||
|
||||
#include <ngraph/pass/pass.hpp>
|
||||
#include "low_precision/lpt_visibility.hpp"
|
||||
#include "low_precision/common/operation_precision_restriction.hpp"
|
||||
#include "low_precision/common/precisions_restriction.hpp"
|
||||
|
||||
namespace ngraph {
|
||||
namespace pass {
|
||||
@@ -48,7 +48,7 @@ public:
|
||||
};
|
||||
|
||||
OPENVINO_RTTI("MarkupPrecisions", "0");
|
||||
explicit MarkupPrecisions(const std::vector<OperationPrecisionRestriction>& restrictions = {},
|
||||
explicit MarkupPrecisions(const std::vector<PrecisionsRestriction>& restrictions = {},
|
||||
const std::vector<ngraph::element::Type>& defaultPrecisions = { ngraph::element::u8, ngraph::element::i8 });
|
||||
bool run_on_model(const std::shared_ptr<ngraph::Function>& m) override;
|
||||
|
||||
|
||||
+8
-7
@@ -9,14 +9,15 @@
|
||||
#include <vector>
|
||||
|
||||
#include <ngraph/pass/pass.hpp>
|
||||
#include "common/operation_per_tensor_quantization_restriction.hpp"
|
||||
#include "low_precision/common/port_quantization_granularity_restriction.hpp"
|
||||
#include "low_precision/common/quantization_granularity_restriction.hpp"
|
||||
#include "low_precision/lpt_visibility.hpp"
|
||||
|
||||
namespace ngraph {
|
||||
namespace pass {
|
||||
namespace low_precision {
|
||||
|
||||
class LP_TRANSFORMATIONS_API MarkupPerTensorQuantization;
|
||||
class LP_TRANSFORMATIONS_API MarkupQuantizationGranularity;
|
||||
|
||||
} // namespace low_precision
|
||||
} // namespace pass
|
||||
@@ -31,21 +32,21 @@ class LP_TRANSFORMATIONS_API MarkupPerTensorQuantization;
|
||||
* [MarkupPerTensorQuantization](@ref openvino_docs_OV_UG_lpt_MarkupPerTensorQuantization) page
|
||||
* in the Inference Engine Developer Guide.
|
||||
*/
|
||||
class ngraph::pass::low_precision::MarkupPerTensorQuantization : public ngraph::pass::FunctionPass {
|
||||
class ngraph::pass::low_precision::MarkupQuantizationGranularity : public ngraph::pass::FunctionPass {
|
||||
public:
|
||||
class PerTensorQuantization {
|
||||
public:
|
||||
explicit PerTensorQuantization(const bool versionIsRequired) : versionIsRequired(versionIsRequired) {}
|
||||
void add(const uint64_t version, const std::vector<size_t>& ports) {
|
||||
portsByVersion.emplace(version, ports);
|
||||
void add(const uint64_t version, const std::vector<PortQuantizationGranularityRestriction>& restrictions) {
|
||||
portsByVersion.emplace(version, restrictions);
|
||||
}
|
||||
|
||||
bool versionIsRequired;
|
||||
std::unordered_map<uint64_t, std::vector<size_t>> portsByVersion;
|
||||
std::unordered_map<uint64_t, std::vector<PortQuantizationGranularityRestriction>> portsByVersion;
|
||||
};
|
||||
|
||||
OPENVINO_RTTI("MarkupPerTensorQuantization", "0");
|
||||
explicit MarkupPerTensorQuantization(const std::vector<OperationPerTensorQuantizationRestriction>& restrictions = {});
|
||||
explicit MarkupQuantizationGranularity(const std::vector<QuantizationGranularityRestriction>& restrictions = {});
|
||||
bool run_on_model(const std::shared_ptr<ngraph::Function>& m) override;
|
||||
|
||||
private:
|
||||
+3
-3
@@ -7,7 +7,7 @@
|
||||
#include <memory>
|
||||
#include <ngraph/ngraph.hpp>
|
||||
#include "low_precision/layer_transformation.hpp"
|
||||
#include "common/operation_precision_restriction.hpp"
|
||||
#include "common/precisions_restriction.hpp"
|
||||
|
||||
namespace ngraph {
|
||||
namespace pass {
|
||||
@@ -26,7 +26,7 @@ public:
|
||||
OPENVINO_RTTI("MultiplyToGroupConvolutionTransformation", "0");
|
||||
MultiplyToGroupConvolutionTransformation(
|
||||
const Params& params = Params(),
|
||||
const OperationPrecisionRestriction::PrecisionsByPort& restrictions = {});
|
||||
const PrecisionsRestriction::PrecisionsByPort& restrictions = {});
|
||||
~MultiplyToGroupConvolutionTransformation() override {}
|
||||
bool transform(TransformationContext& context, ngraph::pattern::Matcher &m) override;
|
||||
bool canBeTransformed(const TransformationContext& context, std::shared_ptr<Node> layer) const override;
|
||||
@@ -39,7 +39,7 @@ public:
|
||||
void setGroupSize(const size_t groupSize);
|
||||
size_t getGroupSize() const;
|
||||
private:
|
||||
OperationPrecisionRestriction::PrecisionsByPort restrictions;
|
||||
PrecisionsRestriction::PrecisionsByPort restrictions;
|
||||
size_t groupSize;
|
||||
};
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
|
||||
#include "rt_info/shared_value_attribute.hpp"
|
||||
#include "rt_info/precisions_attribute.hpp"
|
||||
#include "rt_info/per_tensor_quantization_attribute.hpp"
|
||||
#include "rt_info/quantization_granularity_attribute.hpp"
|
||||
#include "rt_info/intervals_alignment_attribute.hpp"
|
||||
#include "transformation_context.hpp"
|
||||
#include "quantization_details.hpp"
|
||||
|
||||
+1
@@ -27,6 +27,7 @@ public:
|
||||
OPENVINO_RTTI("LowPrecision::AvgPoolPrecisionPreserved", "", ov::RuntimeAttribute, 0);
|
||||
using PrecisionPreservedAttribute::PrecisionPreservedAttribute;
|
||||
void merge(std::vector<ov::Any>& attributes);
|
||||
bool is_skipped() const;
|
||||
std::string to_string() const override;
|
||||
};
|
||||
|
||||
|
||||
-29
@@ -1,29 +0,0 @@
|
||||
// Copyright (C) 2018-2022 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ngraph/node.hpp>
|
||||
#include <ngraph/variant.hpp>
|
||||
|
||||
#include <low_precision/lpt_visibility.hpp>
|
||||
#include <ngraph/pass/graph_rewrite.hpp>
|
||||
#include "low_precision/rt_info/shared_value_attribute.hpp"
|
||||
#include "low_precision/layer_transformation.hpp"
|
||||
#include "attribute_parameters.hpp"
|
||||
|
||||
namespace ngraph {
|
||||
/**
|
||||
* @ingroup ie_transformation_common_api
|
||||
* @brief PerTensorQuantizationAttribute defines if operation input port requires per-tensor quantization.
|
||||
*
|
||||
* For more details about the attribute, refer to
|
||||
* [PerTensorQuantizationAttribute](@ref openvino_docs_OV_UG_lpt_PerTensorQuantization) page in the Inference Engine Developer Guide.
|
||||
*/
|
||||
class LP_TRANSFORMATIONS_API PerTensorQuantizationAttribute : public ov::RuntimeAttribute {
|
||||
public:
|
||||
OPENVINO_RTTI("LowPrecision::PerTensorQuantization", "", ov::RuntimeAttribute, 0);
|
||||
~PerTensorQuantizationAttribute();
|
||||
};
|
||||
} // namespace ngraph
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
// Copyright (C) 2018-2022 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ngraph/node.hpp>
|
||||
#include <ngraph/variant.hpp>
|
||||
|
||||
#include <low_precision/lpt_visibility.hpp>
|
||||
#include <ngraph/pass/graph_rewrite.hpp>
|
||||
#include "low_precision/rt_info/shared_value_attribute.hpp"
|
||||
#include "low_precision/layer_transformation.hpp"
|
||||
#include "attribute_parameters.hpp"
|
||||
|
||||
namespace ngraph {
|
||||
/**
|
||||
* @ingroup ie_transformation_common_api
|
||||
* @brief QuantizationGranularityAttribute defines quantization granularity of operation inputs.
|
||||
*
|
||||
* For more details about the attribute, refer to
|
||||
* [QuantizationGranularityAttribute](@ref openvino_docs_OV_UG_lpt_QuantizationGranularity) page in the Inference Engine Developer Guide.
|
||||
*/
|
||||
class LP_TRANSFORMATIONS_API QuantizationGranularityAttribute : public ov::RuntimeAttribute {
|
||||
public:
|
||||
OPENVINO_RTTI("LowPrecision::QuantizationGranularity", "", ov::RuntimeAttribute, 0);
|
||||
|
||||
enum class Granularity {
|
||||
PerChannel,
|
||||
PerTensor
|
||||
};
|
||||
|
||||
QuantizationGranularityAttribute() : granularity(Granularity::PerChannel) {}
|
||||
QuantizationGranularityAttribute(const Granularity granularity) : granularity(granularity) {}
|
||||
|
||||
bool operator==(const QuantizationGranularityAttribute& attribute) const {
|
||||
return this->granularity == attribute.granularity;
|
||||
}
|
||||
|
||||
bool is_skipped() const;
|
||||
std::string to_string() const override;
|
||||
|
||||
Granularity granularity;
|
||||
};
|
||||
} // namespace ngraph
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
// Copyright (C) 2022 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
#include <ngraph/variant.hpp>
|
||||
#include <low_precision/lpt_visibility.hpp>
|
||||
|
||||
namespace ngraph {
|
||||
|
||||
class LP_TRANSFORMATIONS_API QuantizationModeAttribute : public ov::RuntimeAttribute {
|
||||
public:
|
||||
OPENVINO_RTTI("LowPrecision::QuantizationModeAttribute", "", ov::RuntimeAttribute, 0);
|
||||
|
||||
enum class Mode {
|
||||
Asymmetric,
|
||||
Symmetric
|
||||
};
|
||||
|
||||
QuantizationModeAttribute() : mode(Mode::Asymmetric) {}
|
||||
QuantizationModeAttribute(const Mode mode) : mode(mode) {}
|
||||
|
||||
bool operator==(const QuantizationModeAttribute& attribute) const {
|
||||
return this->mode == attribute.mode;
|
||||
}
|
||||
|
||||
std::string to_string() const override;
|
||||
|
||||
Mode mode;
|
||||
};
|
||||
} // namespace ngraph
|
||||
+7
-1
@@ -70,7 +70,13 @@ public:
|
||||
|
||||
for (auto input : node->inputs()) {
|
||||
if (needToCheckExpectedAttributeType) {
|
||||
if (getAttribute<ExpectedAttributeType>(input).empty()) {
|
||||
const auto& attribute = getAttribute<ExpectedAttributeType>(input);
|
||||
if (attribute.empty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto& expectedAttribute = attribute.template as<ExpectedAttributeType>();
|
||||
if (expectedAttribute.is_skipped()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,10 +6,10 @@
|
||||
#include <memory>
|
||||
#include "low_precision/create_attribute.hpp"
|
||||
#include "low_precision/propagate_through_precision_preserved.hpp"
|
||||
#include "low_precision/rt_info/quantization_alignment_attribute.hpp"
|
||||
#include "low_precision/rt_info/per_tensor_quantization_attribute.hpp"
|
||||
#include "low_precision/update_shared_precision_preserved.hpp"
|
||||
#include "low_precision/rt_info/attribute_parameters.hpp"
|
||||
#include "low_precision/rt_info/quantization_alignment_attribute.hpp"
|
||||
#include "low_precision/rt_info/quantization_granularity_attribute.hpp"
|
||||
#include "low_precision/update_shared_precision_preserved.hpp"
|
||||
#include "itt.hpp"
|
||||
|
||||
using namespace ngraph;
|
||||
@@ -25,7 +25,7 @@ bool ngraph::pass::low_precision::AlignQuantizationParameters::run_on_model(cons
|
||||
std::shared_ptr<ngraph::pass::GraphRewrite> propagation = manager.register_pass<ngraph::pass::GraphRewrite>();
|
||||
propagation->add_matcher<low_precision::CreateAttribute<QuantizationAlignmentAttribute>>();
|
||||
propagation->add_matcher<low_precision::PropagateThroughPrecisionPreserved<QuantizationAlignmentAttribute>>();
|
||||
propagation->add_matcher<low_precision::UpdateSharedPrecisionPreserved<QuantizationAlignmentAttribute, PerTensorQuantizationAttribute>>();
|
||||
propagation->add_matcher<low_precision::UpdateSharedPrecisionPreserved<QuantizationAlignmentAttribute, QuantizationGranularityAttribute>>();
|
||||
manager.run_passes(f);
|
||||
return false;
|
||||
}
|
||||
@@ -14,7 +14,6 @@
|
||||
#include <ngraph/opsets/opset4.hpp>
|
||||
#include <ngraph/opsets/opset6.hpp>
|
||||
#include <transformations/utils/utils.hpp>
|
||||
#include <low_precision/markup_per_tensor_quantization.hpp>
|
||||
#include <low_precision/lpt_itt.hpp>
|
||||
|
||||
#include "low_precision/align_quantization_intervals.hpp"
|
||||
@@ -22,6 +21,7 @@
|
||||
#include "low_precision/markup_precisions.hpp"
|
||||
#include "low_precision/markup_can_be_quantized.hpp"
|
||||
#include "low_precision/markup_avg_pool_precision_preserved.hpp"
|
||||
#include <low_precision/markup_quantization_granularity.hpp>
|
||||
#include "low_precision/propagate_precisions.hpp"
|
||||
#include "low_precision/align_quantization_parameters.hpp"
|
||||
|
||||
@@ -81,8 +81,8 @@
|
||||
#include "itt.hpp"
|
||||
|
||||
ngraph::pass::low_precision::LowPrecision::LowPrecision(
|
||||
const std::vector<OperationPrecisionRestriction>& precisionRestrictions,
|
||||
const std::vector<OperationPerTensorQuantizationRestriction>& quantizationRestrictions,
|
||||
const std::vector<PrecisionsRestriction>& precisionRestrictions,
|
||||
const std::vector<QuantizationGranularityRestriction>& quantizationRestrictions,
|
||||
const LayerTransformation::Params params) :
|
||||
precisionRestrictions(precisionRestrictions),
|
||||
quantizationRestrictions(quantizationRestrictions),
|
||||
@@ -158,8 +158,8 @@ ngraph::pass::low_precision::TypeRelaxedReplacer::TypeRelaxedReplacer() {
|
||||
}
|
||||
|
||||
MarkupOptimizations::MarkupOptimizations(
|
||||
const std::vector<OperationPrecisionRestriction>& precisionRestrictions,
|
||||
const std::vector<OperationPerTensorQuantizationRestriction>& quantizationRestrictions,
|
||||
const std::vector<PrecisionsRestriction>& precisionRestrictions,
|
||||
const std::vector<QuantizationGranularityRestriction>& quantizationRestrictions,
|
||||
const AttributeParameters& params) :
|
||||
precisionRestrictions(precisionRestrictions),
|
||||
quantizationRestrictions(quantizationRestrictions),
|
||||
@@ -174,7 +174,7 @@ bool ngraph::pass::low_precision::MarkupOptimizations::run_on_model(const std::s
|
||||
markup.register_pass<low_precision::MarkupPrecisions>(precisionRestrictions, params.defaultPrecisions);
|
||||
}
|
||||
if (!quantizationRestrictions.empty()) {
|
||||
markup.register_pass<low_precision::MarkupPerTensorQuantization>(quantizationRestrictions);
|
||||
markup.register_pass<low_precision::MarkupQuantizationGranularity>(quantizationRestrictions);
|
||||
}
|
||||
if (ngraph::op::util::has_op_with_type<ngraph::opset1::AvgPool>(f)) {
|
||||
markup.register_pass<low_precision::MarkupAvgPoolPrecisionPreserved>(params.defaultPrecisions);
|
||||
@@ -249,7 +249,7 @@ bool ngraph::pass::low_precision::LowPrecision::run_on_model(const std::shared_p
|
||||
// WA: precision restrictions for groupConv must be propagated to MultiplyToGroupConvolution transformation
|
||||
cleanup->add_matcher<ngraph::pass::low_precision::MultiplyToGroupConvolutionTransformation>(
|
||||
params,
|
||||
OperationPrecisionRestriction::getPrecisionsByOperationType<opset1::GroupConvolution>(precisionRestrictions));
|
||||
PrecisionsRestriction::getPrecisionsByOperationType<opset1::GroupConvolution>(precisionRestrictions));
|
||||
manager.register_pass<ngraph::pass::low_precision::FoldFakeQuantizeTransformation>(params);
|
||||
manager.register_pass<ngraph::pass::ConstantFolding>();
|
||||
|
||||
|
||||
@@ -20,7 +20,8 @@
|
||||
|
||||
using namespace ngraph;
|
||||
|
||||
ngraph::pass::low_precision::MarkupPrecisions::MarkupPrecisions(const std::vector<OperationPrecisionRestriction>& restrictions,
|
||||
ngraph::pass::low_precision::MarkupPrecisions::MarkupPrecisions(
|
||||
const std::vector<PrecisionsRestriction>& restrictions,
|
||||
const std::vector<ngraph::element::Type>& defaultPrecisions) : defaultPrecisions(defaultPrecisions) {
|
||||
for (const auto& restriction : restrictions) {
|
||||
const auto it = restrictionsByOperation.find(restriction.operationType.name);
|
||||
|
||||
+18
-20
@@ -2,54 +2,52 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "low_precision/markup_per_tensor_quantization.hpp"
|
||||
#include "low_precision/markup_quantization_granularity.hpp"
|
||||
|
||||
#include <cassert>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <ngraph/node.hpp>
|
||||
#include "low_precision/rt_info/per_tensor_quantization_attribute.hpp"
|
||||
#include "itt.hpp"
|
||||
#include "low_precision/rt_info/quantization_granularity_attribute.hpp"
|
||||
|
||||
using namespace ngraph;
|
||||
|
||||
ngraph::pass::low_precision::MarkupPerTensorQuantization::MarkupPerTensorQuantization(
|
||||
const std::vector<OperationPerTensorQuantizationRestriction>& restrictions) {
|
||||
for (const OperationPerTensorQuantizationRestriction& restriction : restrictions) {
|
||||
ngraph::pass::low_precision::MarkupQuantizationGranularity::MarkupQuantizationGranularity(
|
||||
const std::vector<QuantizationGranularityRestriction>& restrictions) {
|
||||
for (const auto& restriction : restrictions) {
|
||||
const auto it = restrictionsByOperation.find(restriction.operationType.name);
|
||||
OPENVINO_SUPPRESS_DEPRECATED_START
|
||||
if (it == restrictionsByOperation.end()) {
|
||||
PerTensorQuantization r(restriction.specifyVersion);
|
||||
r.portsByVersion.emplace(restriction.operationType.version, restriction.restrictedPorts);
|
||||
r.portsByVersion.emplace(restriction.operationType.version, restriction.restrictions);
|
||||
restrictionsByOperation.emplace(restriction.operationType.name, r);
|
||||
} else {
|
||||
it->second.add(restriction.operationType.version, restriction.restrictedPorts);
|
||||
it->second.add(restriction.operationType.version, restriction.restrictions);
|
||||
}
|
||||
OPENVINO_SUPPRESS_DEPRECATED_END
|
||||
}
|
||||
}
|
||||
|
||||
bool ngraph::pass::low_precision::MarkupPerTensorQuantization::run_on_model(const std::shared_ptr<ngraph::Function>& f) {
|
||||
bool ngraph::pass::low_precision::MarkupQuantizationGranularity::run_on_model(const std::shared_ptr<ngraph::Function>& f) {
|
||||
RUN_ON_FUNCTION_SCOPE(MarkupPerTensorQuantization);
|
||||
auto setRestriction = [](const std::shared_ptr<Node>& node, const std::vector<size_t>& restrictedPorts) {
|
||||
auto createAttribute = [](Input<Node>& input){
|
||||
auto setRestriction = [](const std::shared_ptr<Node>& node, const std::vector<PortQuantizationGranularityRestriction>& restrictedPorts) {
|
||||
auto createAttribute = [](Input<Node>& input, const QuantizationGranularityAttribute::Granularity granularity){
|
||||
auto &rt = input.get_rt_info();
|
||||
rt.emplace(
|
||||
PerTensorQuantizationAttribute::get_type_info_static(),
|
||||
PerTensorQuantizationAttribute());
|
||||
rt.emplace(QuantizationGranularityAttribute::get_type_info_static(), QuantizationGranularityAttribute(granularity));
|
||||
};
|
||||
|
||||
if (restrictedPorts.empty()) {
|
||||
// markup all ports
|
||||
// markup all ports with default granularity value
|
||||
for (size_t item = 0ul; item < node->get_input_size(); item++) {
|
||||
Input<Node> input = node->input(item);
|
||||
createAttribute(input);
|
||||
createAttribute(input, QuantizationGranularityAttribute::Granularity::PerTensor);
|
||||
}
|
||||
} else {
|
||||
// markup specific ports
|
||||
for (const size_t item : restrictedPorts) {
|
||||
Input<Node> input = node->input(item);
|
||||
createAttribute(input);
|
||||
for (const auto item : restrictedPorts) {
|
||||
Input<Node> input = node->input(item.port);
|
||||
createAttribute(input, item.granularity);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -77,11 +75,11 @@ bool ngraph::pass::low_precision::MarkupPerTensorQuantization::run_on_model(cons
|
||||
continue;
|
||||
}
|
||||
|
||||
const std::vector<size_t>& restrictedPorts = it2->second;
|
||||
const std::vector<PortQuantizationGranularityRestriction>& restrictedPorts = it2->second;
|
||||
setRestriction(node, restrictedPorts);
|
||||
} else {
|
||||
assert(restriction.portsByVersion.size() == 1ul);
|
||||
const std::vector<size_t>& restrictedPorts = restriction.portsByVersion.begin()->second;
|
||||
const std::vector<PortQuantizationGranularityRestriction>& restrictedPorts = restriction.portsByVersion.begin()->second;
|
||||
setRestriction(node, restrictedPorts);
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,7 @@ namespace low_precision {
|
||||
|
||||
MultiplyToGroupConvolutionTransformation::MultiplyToGroupConvolutionTransformation(
|
||||
const Params& params,
|
||||
const OperationPrecisionRestriction::PrecisionsByPort& restrictions) : LayerTransformation(params), restrictions(restrictions), groupSize(1ul) {
|
||||
const PrecisionsRestriction::PrecisionsByPort& restrictions) : LayerTransformation(params), restrictions(restrictions), groupSize(1ul) {
|
||||
MATCHER_SCOPE(MultiplyToGroupConvolutionTransformation);
|
||||
auto matcher = pattern::wrap_type<opset1::Multiply>();
|
||||
|
||||
|
||||
+4
@@ -15,6 +15,10 @@ using namespace ov;
|
||||
void AvgPoolPrecisionPreservedAttribute::merge(std::vector<ov::Any>& attributes) {
|
||||
}
|
||||
|
||||
bool AvgPoolPrecisionPreservedAttribute::is_skipped() const {
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string AvgPoolPrecisionPreservedAttribute::to_string() const {
|
||||
std::stringstream ss;
|
||||
ss << attribute->get_string();
|
||||
|
||||
-10
@@ -1,10 +0,0 @@
|
||||
// Copyright (C) 2018-2022 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "low_precision/rt_info/per_tensor_quantization_attribute.hpp"
|
||||
|
||||
using namespace ngraph;
|
||||
using namespace ov;
|
||||
|
||||
PerTensorQuantizationAttribute::~PerTensorQuantizationAttribute() = default;
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
// Copyright (C) 2022 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "low_precision/rt_info/quantization_granularity_attribute.hpp"
|
||||
|
||||
using namespace ngraph;
|
||||
using namespace ov;
|
||||
|
||||
bool QuantizationGranularityAttribute::is_skipped() const {
|
||||
assert((granularity == Granularity::PerChannel) || (granularity == Granularity::PerTensor));
|
||||
return granularity != Granularity::PerTensor;
|
||||
}
|
||||
|
||||
std::string QuantizationGranularityAttribute::to_string() const {
|
||||
assert((granularity == Granularity::PerChannel) || (granularity == Granularity::PerTensor));
|
||||
|
||||
std::stringstream ss;
|
||||
switch (granularity) {
|
||||
case Granularity::PerChannel: {
|
||||
ss << "PerChannel";
|
||||
break;
|
||||
}
|
||||
case Granularity::PerTensor: {
|
||||
ss << "PerTensor";
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
ss << "UNKNOWN";
|
||||
break;
|
||||
}
|
||||
}
|
||||
return ss.str();
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
// Copyright (C) 2022 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "low_precision/rt_info/quantization_mode_attribute.hpp"
|
||||
#include <assert.h>
|
||||
|
||||
using namespace ngraph;
|
||||
using namespace ov;
|
||||
|
||||
std::string QuantizationModeAttribute::to_string() const {
|
||||
assert((mode == Mode::Asymmetric) || (mode == Mode::Symmetric));
|
||||
|
||||
std::stringstream ss;
|
||||
switch (mode) {
|
||||
case Mode::Asymmetric: {
|
||||
ss << "Asymmetric";
|
||||
break;
|
||||
}
|
||||
case Mode::Symmetric: {
|
||||
ss << "Symmetric";
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
ss << "UNKNOWN";
|
||||
break;
|
||||
}
|
||||
}
|
||||
return ss.str();
|
||||
}
|
||||
@@ -4,8 +4,6 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <transformations_visibility.hpp>
|
||||
|
||||
#include <ngraph/pass/graph_rewrite.hpp>
|
||||
|
||||
namespace ngraph {
|
||||
|
||||
@@ -9,7 +9,6 @@
|
||||
#include <type_traits>
|
||||
|
||||
#include "openvino/frontend/extension/decoder_transformation.hpp"
|
||||
#include "openvino/frontend/visibility.hpp"
|
||||
|
||||
namespace ov {
|
||||
namespace frontend {
|
||||
|
||||
@@ -34,14 +34,14 @@ add_cpplint_target(${TARGET_NAME}_cpplint FOR_TARGETS ${TARGET_NAME})
|
||||
|
||||
ie_mark_target_as_cc(${TARGET_NAME})
|
||||
|
||||
# LTO
|
||||
|
||||
set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO})
|
||||
|
||||
if(ENABLE_FUNCTIONAL_TESTS)
|
||||
add_subdirectory(tests)
|
||||
endif()
|
||||
|
||||
# LTO
|
||||
|
||||
set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO})
|
||||
|
||||
# install
|
||||
|
||||
ov_install_static_lib(${TARGET_NAME} core)
|
||||
+9
@@ -50,6 +50,15 @@ bool ov::pass::MarkPrecisionSensitiveDivides::run_on_model(const std::shared_ptr
|
||||
const auto& input_node = input_value.get_node();
|
||||
if (visited.count(input_node))
|
||||
continue;
|
||||
|
||||
if (auto sub_graph_node = ov::as_type<ngraph::op::util::MultiSubGraphOp>(input_node)) {
|
||||
size_t sub_graphs_num = sub_graph_node->get_internal_subgraphs_size();
|
||||
for (size_t sub_graph_ind = 0; sub_graph_ind < sub_graphs_num; ++sub_graph_ind) {
|
||||
auto sub_graph = sub_graph_node->get_function(sub_graph_ind);
|
||||
run_on_model(sub_graph);
|
||||
}
|
||||
}
|
||||
|
||||
nodes.push_front(input_node);
|
||||
visited.insert(input_node);
|
||||
}
|
||||
|
||||
+10
-1
@@ -20,6 +20,7 @@ using namespace std;
|
||||
bool ov::pass::MarkPrecisionSensitiveSubgraphs::run_on_model(const std::shared_ptr<ov::Model>& f) {
|
||||
deque<Node*> nodes;
|
||||
unordered_set<Node*> visited, precision_sensitive_visited;
|
||||
|
||||
for (const auto& r : f->get_results()) {
|
||||
nodes.push_back(r.get());
|
||||
visited.insert(r.get());
|
||||
@@ -52,9 +53,17 @@ bool ov::pass::MarkPrecisionSensitiveSubgraphs::run_on_model(const std::shared_p
|
||||
const auto& input_node = input_value.get_node();
|
||||
if (visited.count(input_node))
|
||||
continue;
|
||||
|
||||
if (auto sub_graph_node = ov::as_type<ngraph::op::util::MultiSubGraphOp>(input_node)) {
|
||||
size_t sub_graphs_num = sub_graph_node->get_internal_subgraphs_size();
|
||||
for (size_t sub_graph_ind = 0; sub_graph_ind < sub_graphs_num; ++sub_graph_ind) {
|
||||
auto sub_graph = sub_graph_node->get_function(sub_graph_ind);
|
||||
run_on_model(sub_graph);
|
||||
}
|
||||
}
|
||||
nodes.push_front(input_node);
|
||||
visited.insert(input_node);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
+15
@@ -5,6 +5,7 @@
|
||||
#include <memory>
|
||||
#include <ngraph/pass/constant_folding.hpp>
|
||||
#include <ngraph/pass/manager.hpp>
|
||||
#include <transformations/common_optimizations/add_fake_quantize_fusion.hpp>
|
||||
#include <transformations/common_optimizations/align_eltwise_input_ranks.hpp>
|
||||
#include <transformations/common_optimizations/batch_to_space_fusion.hpp>
|
||||
#include <transformations/common_optimizations/binarize_weights.hpp>
|
||||
@@ -20,6 +21,8 @@
|
||||
#include <transformations/common_optimizations/disable_shapeof_constant_folding.hpp>
|
||||
#include <transformations/common_optimizations/divide_fusion.hpp>
|
||||
#include <transformations/common_optimizations/eliminate_unsqueeze_gather.hpp>
|
||||
#include <transformations/common_optimizations/fq_mul_fusion.hpp>
|
||||
#include <transformations/common_optimizations/fq_reshape_fusion.hpp>
|
||||
#include <transformations/common_optimizations/gelu_fusion.hpp>
|
||||
#include <transformations/common_optimizations/hsigmoid_fusion.hpp>
|
||||
#include <transformations/common_optimizations/hswish_fusion.hpp>
|
||||
@@ -29,6 +32,7 @@
|
||||
#include <transformations/common_optimizations/matmul_multiply_fusion.hpp>
|
||||
#include <transformations/common_optimizations/moc_transformations.hpp>
|
||||
#include <transformations/common_optimizations/mul_conv_fusion.hpp>
|
||||
#include <transformations/common_optimizations/mul_fake_quantize_fusion.hpp>
|
||||
#include <transformations/common_optimizations/mvn_fusion.hpp>
|
||||
#include <transformations/common_optimizations/nearest_neighbor_upsampling_fusion.hpp>
|
||||
#include <transformations/common_optimizations/nop_elimination.hpp>
|
||||
@@ -36,7 +40,9 @@
|
||||
#include <transformations/common_optimizations/optimize_strided_slice.hpp>
|
||||
#include <transformations/common_optimizations/pad_fusion.hpp>
|
||||
#include <transformations/common_optimizations/prelu_fusion.hpp>
|
||||
#include <transformations/common_optimizations/pull_transpose_through_fq.hpp>
|
||||
#include <transformations/common_optimizations/random_uniform_fusion.hpp>
|
||||
#include <transformations/common_optimizations/relu_fake_quantize_fusion.hpp>
|
||||
#include <transformations/common_optimizations/remove_concat_zero_dim_input.hpp>
|
||||
#include <transformations/common_optimizations/remove_filtering_boxes_by_size.hpp>
|
||||
#include <transformations/common_optimizations/remove_multi_subgraph_op_dangling_params.hpp>
|
||||
@@ -179,6 +185,15 @@ bool ngraph::pass::MOCTransformations::run_on_model(const std::shared_ptr<ngraph
|
||||
multiply_fusions->set_name("ngraph::pass::MultiplyFusions");
|
||||
manager.register_pass<ngraph::pass::ConstantFolding>();
|
||||
|
||||
auto fq_fusions = manager.register_pass<ngraph::pass::GraphRewrite>();
|
||||
fq_fusions->add_matcher<ngraph::pass::FakeQuantizeMulFusion>();
|
||||
fq_fusions->add_matcher<ngraph::pass::FakeQuantizeReshapeFusion>();
|
||||
fq_fusions->add_matcher<ngraph::pass::PullTransposeThroughFQUp>();
|
||||
fq_fusions->add_matcher<ngraph::pass::ReluFakeQuantizeFusion>();
|
||||
fq_fusions->add_matcher<ngraph::pass::AddFakeQuantizeFusion>();
|
||||
fq_fusions->add_matcher<ngraph::pass::MulFakeQuantizeFusion>();
|
||||
fq_fusions->set_name("ngraph::pass::FakeQuantizeFusions");
|
||||
|
||||
manager.register_pass<ngraph::pass::ReverseInputChannelsFusion>();
|
||||
|
||||
manager.register_pass<ngraph::pass::AlignEltwiseInputRanks>();
|
||||
|
||||
+1
-1
@@ -61,7 +61,7 @@ ngraph::pass::WrapInterpolateIntoTransposes::WrapInterpolateIntoTransposes() {
|
||||
return false;
|
||||
|
||||
int64_t input_rank = interpolate->get_input_partial_shape(0).rank().get_length();
|
||||
// If the input rank is equal to 1 or 2, then such Interpolate is supported by MKLDNN.
|
||||
// If the input rank is equal to 1 or 2, then such Interpolate is supported by OneDNN.
|
||||
if (input_rank < 3)
|
||||
return false;
|
||||
|
||||
|
||||
@@ -37,13 +37,25 @@ set_property(SOURCE ${MIXED_SRC}
|
||||
source_group("src" FILES ${LIBRARY_SRC})
|
||||
source_group("include" FILES ${PUBLIC_HEADERS})
|
||||
|
||||
#
|
||||
# Create ov_core_dev library
|
||||
#
|
||||
|
||||
add_library(ov_core_dev INTERFACE)
|
||||
add_library(openvino::core::dev ALIAS ov_core_dev)
|
||||
|
||||
target_include_directories(ov_core_dev INTERFACE $<BUILD_INTERFACE:${OpenVINO_SOURCE_DIR}/src/core/dev_api>)
|
||||
|
||||
set_target_properties(ov_core_dev PROPERTIES EXPORT_NAME core::dev)
|
||||
openvino_developer_export_targets(COMPONENT core TARGETS openvino::core::dev)
|
||||
|
||||
# Install static libraries for case BUILD_SHARED_LIBS=OFF
|
||||
ov_install_static_lib(ov_core_dev core)
|
||||
|
||||
#
|
||||
# Create static or shared library depending on BUILD_SHARED_LIBS
|
||||
#
|
||||
|
||||
add_library(ngraph_obj OBJECT ${LIBRARY_SRC} ${PUBLIC_HEADERS})
|
||||
|
||||
target_compile_definitions(ngraph_obj PRIVATE IMPLEMENT_OPENVINO_API)
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
|
||||
#include "openvino/core/visibility.hpp"
|
||||
|
||||
#define OV_NEW_API 1
|
||||
// Now we use the generic helper definitions above to define OPENVINO_API
|
||||
// OPENVINO_API is used for the public API symbols. It either DLL imports or DLL exports
|
||||
// (or does nothing for static build)
|
||||
|
||||
File renamed without changes.
@@ -2,6 +2,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE OFF)
|
||||
|
||||
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND NOT ENABLE_SANITIZER)
|
||||
# gold linker on ubuntu20.04 may fail to link binaries build with sanitizer
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fuse-ld=gold")
|
||||
|
||||
@@ -3,12 +3,12 @@
|
||||
//
|
||||
|
||||
#include <iostream>
|
||||
#include <mkldnn.hpp>
|
||||
#include <dnnl.hpp>
|
||||
#include <vector>
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
|
||||
static int tensor_volume(const mkldnn::memory::dims& t)
|
||||
static int tensor_volume(const dnnl::memory::dims& t)
|
||||
{
|
||||
int x = 1;
|
||||
for (const auto i : t)
|
||||
@@ -18,7 +18,7 @@ static int tensor_volume(const mkldnn::memory::dims& t)
|
||||
|
||||
void test()
|
||||
{
|
||||
using namespace mkldnn;
|
||||
using namespace dnnl;
|
||||
|
||||
auto cpu_engine = engine(engine::cpu, 0);
|
||||
|
||||
@@ -72,7 +72,7 @@ void test()
|
||||
stream(stream::kind::eager).submit({c3}).wait();
|
||||
}
|
||||
|
||||
TEST(mkldnn, engine)
|
||||
TEST(dnnl, engine)
|
||||
{
|
||||
EXPECT_NO_THROW(test());
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
# Copyright (C) 2018-2022 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
if (ENABLE_OV_PADDLE_FRONTEND)
|
||||
add_subdirectory(paddle)
|
||||
endif()
|
||||
|
||||
@@ -9,7 +9,8 @@ list(FILTER SRC EXCLUDE REGEX standalone_build)
|
||||
|
||||
add_executable(${TARGET_NAME} ${SRC})
|
||||
|
||||
target_link_libraries(${TARGET_NAME} PRIVATE frontend_shared_test_classes openvino_paddle_frontend openvino::runtime)
|
||||
target_link_libraries(${TARGET_NAME} PRIVATE cnpy frontend_shared_test_classes
|
||||
openvino_paddle_frontend openvino::runtime)
|
||||
|
||||
add_clang_format_target(${TARGET_NAME}_clang FOR_TARGETS ${TARGET_NAME})
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# Copyright (C) 2018-2022 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
set(TARGET_NAME "paddle_fe_standalone_build_test")
|
||||
|
||||
add_library(${TARGET_NAME} STATIC standalone_build_test.cpp)
|
||||
|
||||
@@ -834,7 +834,7 @@ non_zero_all_1s
|
||||
non_zero_all_0s
|
||||
|
||||
# NormalizeL2 - output mismatch,
|
||||
# mkldnn_normalize_nchw applies eps after sqrt for across_spatial
|
||||
# dnnl_normalize_nchw applies eps after sqrt for across_spatial
|
||||
# Issue: 59586
|
||||
IE_CPU.normalize_l2_4D_axes_123_big_eps_max
|
||||
IE_CPU.normalize_l2_4D_axes_123_big_eps_add
|
||||
|
||||
@@ -25,7 +25,7 @@ OutputVector detection_output(const Node& node) {
|
||||
attrs.top_k = node.get_attribute_value<int64_t>("top_k", -1);
|
||||
attrs.variance_encoded_in_target = node.get_attribute_value<int64_t>("variance_encoded_in_target", 0);
|
||||
// spec says keep_top_k is an array of ints, but some models use a single int
|
||||
// also mkldnn expects single integer
|
||||
// also CPU plugin expects single integer
|
||||
attrs.keep_top_k = {static_cast<int>(node.get_attribute_value<int64_t>("keep_top_k", 1))};
|
||||
|
||||
auto code_type = node.get_attribute_value<std::string>("code_type", std::string{"caffe.PriorBoxParameter.CORNER"});
|
||||
|
||||
@@ -82,7 +82,6 @@ TEST(TransposeSinkingTest, PoolAdd1) {
|
||||
ngraph::Shape input_shape{1, 3, 3, 1}; // NHWC (N=1, H=3, W=3, C=1)
|
||||
|
||||
auto input_type = ngraph::element::f32;
|
||||
auto output_type = ngraph::element::f32;
|
||||
|
||||
auto X = make_shared<Parameter>(input_type, input_shape); // NHWC
|
||||
|
||||
@@ -123,7 +122,6 @@ TEST(TransposeSinkingTest, PoolAdd2) {
|
||||
ngraph::Shape input_shape{1, 3, 3, 1}; // NHWC (N=1, H=3, W=3, C=1)
|
||||
|
||||
auto input_type = ngraph::element::f32;
|
||||
auto output_type = ngraph::element::f32;
|
||||
|
||||
auto X = make_shared<Parameter>(input_type, input_shape); // NHWC
|
||||
|
||||
@@ -178,7 +176,6 @@ TEST(TransposeSinkingTest, PoolAdd3) {
|
||||
ngraph::Shape input_shape{1, 3, 3, 1}; // NHWC (N=1, H=3, W=3, C=1)
|
||||
|
||||
auto input_type = ngraph::element::f32;
|
||||
auto output_type = ngraph::element::f32;
|
||||
|
||||
auto X = make_shared<Parameter>(input_type, input_shape); // NHWC
|
||||
|
||||
@@ -371,7 +368,6 @@ TEST(TransposeSinkingTest, MultiOutput) {
|
||||
ngraph::Shape shape_nchw{1, 1, 4, 6};
|
||||
|
||||
auto input_type = ngraph::element::f32;
|
||||
auto output_type = ngraph::element::f32;
|
||||
|
||||
auto X = make_shared<Parameter>(input_type, shape_nchw); // NCHW
|
||||
|
||||
@@ -441,7 +437,6 @@ TEST(TransposeSinkingTest, AlexnetPattern) {
|
||||
ngraph::Shape shape_nchw{1, 96, 55, 55};
|
||||
|
||||
auto input_type = ngraph::element::f32;
|
||||
auto output_type = ngraph::element::f32;
|
||||
|
||||
// X
|
||||
auto X = make_shared<Parameter>(input_type, shape_nhwc); // NHWC
|
||||
|
||||
@@ -2,4 +2,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE OFF)
|
||||
|
||||
add_subdirectory(frontend/shared)
|
||||
@@ -15,9 +15,17 @@ add_subdirectory(test_builtin_extensions_2)
|
||||
target_include_directories(${TARGET_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
|
||||
target_include_directories(${TARGET_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../..)
|
||||
|
||||
target_link_libraries(${TARGET_NAME} PUBLIC engines_test_util openvino::runtime
|
||||
offline_transformations cnpy commonTestUtils ngraph_test_util openvino::util
|
||||
nlohmann_json::nlohmann_json test_builtin_extensions_1 test_builtin_extensions_2)
|
||||
target_link_libraries(${TARGET_NAME}
|
||||
PUBLIC
|
||||
engines_test_util offline_transformations
|
||||
commonTestUtils ngraph_test_util openvino::util
|
||||
openvino::runtime
|
||||
PRIVATE
|
||||
cnpy
|
||||
# TODO: replace with nlohmann_json::nlohmann_json
|
||||
# but LTO fails for some reason
|
||||
nlohmann_json_schema_validator
|
||||
test_builtin_extensions_1 test_builtin_extensions_2)
|
||||
|
||||
target_compile_definitions(${TARGET_NAME}
|
||||
PRIVATE
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
#include <common_test_utils/graph_comparator.hpp>
|
||||
#include <extension/json_config.hpp>
|
||||
#include <extension/json_transformation.hpp>
|
||||
#include <nlohmann/json-schema.hpp>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <ostream>
|
||||
|
||||
#include "utils.hpp"
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
#include "cpp_interfaces/interface/ie_iinfer_request_internal.hpp"
|
||||
|
||||
#include <ie_parallel.hpp>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <openvino/core/partial_shape.hpp>
|
||||
@@ -324,7 +325,7 @@ void IInferRequestInternal::convertBatchedInputBlob(const std::string& name, con
|
||||
auto ptr = mem_blob->wmap();
|
||||
|
||||
// Perform memory copy
|
||||
for (size_t i = 0; i < batched_blob->size(); i++) {
|
||||
InferenceEngine::parallel_for(batched_blob->size(), [&](size_t i) {
|
||||
const auto& blob = as<MemoryBlob>(batched_blob->getBlob(i));
|
||||
OPENVINO_ASSERT(mem_blob, "Internal error - can't cast blob ", i, " to MemoryBlob");
|
||||
const auto& blob_desc = blob->getTensorDesc().getBlockingDesc();
|
||||
@@ -346,7 +347,7 @@ void IInferRequestInternal::convertBatchedInputBlob(const std::string& name, con
|
||||
blob->rmap().as<uint8_t*>() +
|
||||
blob->getTensorDesc().getBlockingDesc().getOffsetPadding() * blob->element_size(),
|
||||
blob->byteSize());
|
||||
}
|
||||
});
|
||||
SetBlob(name, mem_blob);
|
||||
}
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ if(SELECTIVE_BUILD STREQUAL "ON")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
target_link_libraries(${TARGET_NAME} PRIVATE mkldnn
|
||||
target_link_libraries(${TARGET_NAME} PRIVATE dnnl
|
||||
ov_shape_inference
|
||||
inference_engine_snippets)
|
||||
|
||||
@@ -50,7 +50,7 @@ target_compile_definitions(${TARGET_NAME} PRIVATE IMPLEMENT_INFERENCE_EXTENSION_
|
||||
target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
|
||||
|
||||
target_include_directories(${TARGET_NAME} SYSTEM PRIVATE
|
||||
$<TARGET_PROPERTY:mkldnn,INCLUDE_DIRECTORIES>)
|
||||
$<TARGET_PROPERTY:dnnl,INCLUDE_DIRECTORIES>)
|
||||
|
||||
# Cross compiled function
|
||||
# TODO: The same for proposal, proposalONNX, topk
|
||||
@@ -68,7 +68,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME})
|
||||
|
||||
if(BUILD_SHARED_LIBS)
|
||||
add_library(${TARGET_NAME}_obj OBJECT ${SOURCES} ${HEADERS})
|
||||
link_system_libraries(${TARGET_NAME}_obj PUBLIC mkldnn)
|
||||
link_system_libraries(${TARGET_NAME}_obj PUBLIC dnnl)
|
||||
|
||||
target_include_directories(${TARGET_NAME}_obj
|
||||
PRIVATE
|
||||
@@ -81,7 +81,7 @@ if(BUILD_SHARED_LIBS)
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src
|
||||
$<TARGET_PROPERTY:openvino::conditional_compilation,INTERFACE_INCLUDE_DIRECTORIES>)
|
||||
|
||||
target_include_directories(${TARGET_NAME}_obj SYSTEM PUBLIC $<TARGET_PROPERTY:mkldnn,INCLUDE_DIRECTORIES>)
|
||||
target_include_directories(${TARGET_NAME}_obj SYSTEM PUBLIC $<TARGET_PROPERTY:dnnl,INCLUDE_DIRECTORIES>)
|
||||
|
||||
set_ie_threading_interface_for(${TARGET_NAME}_obj)
|
||||
|
||||
@@ -93,6 +93,8 @@ if(BUILD_SHARED_LIBS)
|
||||
set_target_properties(${TARGET_NAME}_obj PROPERTIES EXCLUDE_FROM_ALL ON)
|
||||
|
||||
# LTO
|
||||
set_target_properties(${TARGET_NAME} ${TARGET_NAME}_obj
|
||||
PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO})
|
||||
set_target_properties(${TARGET_NAME}_obj PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO})
|
||||
endif()
|
||||
|
||||
# LTO
|
||||
set_target_properties(${TARGET_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ${ENABLE_LTO})
|
||||
@@ -61,7 +61,7 @@ struct Config {
|
||||
|
||||
std::string execGraphPath;
|
||||
std::string verbose;
|
||||
std::string blobDumpDir = "mkldnn_dump";
|
||||
std::string blobDumpDir = "cpu_dump";
|
||||
FORMAT blobDumpFormat = FORMAT::TEXT;
|
||||
// std::hash<int> is necessary for Ubuntu-16.04 (gcc-5.4 and defect in C++11 standart)
|
||||
std::unordered_map<FILTER, std::string, std::hash<int>> blobDumpFilters;
|
||||
|
||||
Loaded 100 of 417 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user